int_loop: consolidate maketodo.py, tweaks

Signed-off-by: John McMaster <johndmcmaster@gmail.com>

fuzzers/050-intpips/Makefile

@@ -14,6 +14,7 @@ SPECIMENS := $(addprefix build/$(ITER)/specimen_,$(shell seq -f '%03.0f' $(N)))
 SPECIMENS_OK := $(addsuffix /OK,$(SPECIMENS))
 # Individual fuzzer directory, such as ~/prjxray/fuzzers/010-lutinit
 export FUZDIR=$(shell pwd)
+MAKETODO_FLAGS=--re ".*" --not-endswith ".VCC_WIRE"
 
 # Specimens from current run must complete, but previous iterations may exist
 database: $(SPECIMENS_OK)
@@ -23,7 +24,10 @@ database: $(SPECIMENS_OK)
 	${XRAY_MASKMERGE} build/mask_clbll_r.db $(shell find build -name segdata_int_r.txt)
 	${XRAY_MASKMERGE} build/mask_clblm_l.db $(shell find build -name segdata_int_l.txt)
 	${XRAY_MASKMERGE} build/mask_clblm_r.db $(shell find build -name segdata_int_r.txt)
+# May be undersolved
+ifneq ($(QUICK),Y)
 	${XRAY_DBFIXUP} --db-root build --clb-int
+endif
 
 pushdb:
 	${XRAY_MERGEDB} int_l build/segbits_int_l.db
@@ -41,10 +45,11 @@ $(SPECIMENS_OK): build/todo.txt
 
 build/pips_int_l.txt: $(XRAY_DIR)/fuzzers/piplist.tcl
 	mkdir -p build/$(ITER)
-	cd build/$(ITER) && vivado -mode batch -source $(XRAY_DIR)/fuzzers/piplist.tcl
+	cd build && vivado -mode batch -source $(XRAY_DIR)/fuzzers/piplist.tcl
 
-build/todo.txt: build/pips_int_l.txt maketodo.py
-	python3 maketodo.py --build-dir build/$(ITER) >build/todo_all.txt
+build/todo.txt: build/pips_int_l.txt $(XRAY_DIR)/fuzzers/int_maketodo.py
+# Doesn't pushdb until very end. Compare against db so far
+	python3 $(XRAY_DIR)/fuzzers/int_maketodo.py --db-dir build/$(ITER) $(MAKETODO_FLAGS) >build/todo_all.txt
 	cat build/todo_all.txt | sort -R > build/todo.txt.tmp
 	mv build/todo.txt.tmp build/todo.txt
 

fuzzers/050-intpips/maketodo.py

@@ -1,57 +0,0 @@
-#!/usr/bin/env python3
-
-import os, re, sys
-from prjxray import util
-
-
-def maketodo(pipfile, dbfile, verbose=False):
-    '''Print name of all pips in pipfile but not dbfile'''
-    todos = set()
-    with open(pipfile, "r") as f:
-        for line in f:
-            line = line.split()
-            todos.add(line[0])
-    verbose and print(
-        '%s: %u entries' % (pipfile, len(todos)), file=sys.stderr)
-    # Support generate without existing DB
-    if os.path.exists(dbfile):
-        with open(dbfile, "r") as f:
-            for line in f:
-                line = line.split()
-                pip = line[0]
-                todos.remove(pip)
-    verbose and print(
-        'Remove %s: %u entries' % (dbfile, len(todos)), file=sys.stderr)
-    drops = 0
-    lines = 0
-    for line in todos:
-        if line.endswith(".VCC_WIRE"):
-            drops += 1
-            continue
-        print(line)
-        lines += 1
-    verbose and print(
-        'Print %u entries w/ %u drops' % (lines, drops), file=sys.stderr)
-
-
-def run(build_dir):
-    maketodo(
-        "%s/pips_int_l.txt" % build_dir, "%s/segbits_int_l.db" % build_dir)
-    maketodo(
-        "%s/pips_int_r.txt" % build_dir, "%s/segbits_int_r.db" % build_dir)
-
-
-def main():
-    import argparse
-
-    parser = argparse.ArgumentParser(
-        description="Print list of known but unsolved PIPs")
-
-    parser.add_argument('--build-dir', default="build", help='')
-    args = parser.parse_args()
-
-    run(build_dir=args.build_dir)
-
-
-if __name__ == '__main__':
-    main()

fuzzers/051-imuxlout/Makefile

@@ -1,2 +1,3 @@
+MAKETODO_FLAGS=--re "^INT_[LR].IMUX(_L)?[0-9]+\.LOGIC_OUTS(_L)?[0-9]+$$"
 include ../int_loop.mk
 

fuzzers/051-imuxlout/generate.tcl

@@ -22,7 +22,7 @@ route_design
 # write_checkpoint -force design.dcp
 
 
-set fp [open "../todo.txt" r]
+set fp [open "../../todo.txt" r]
 set todo_lines {}
 for {gets $fp line} {$line != ""} {gets $fp line} {
     lappend todo_lines [split $line .]

fuzzers/051-imuxlout/maketodo.py

@@ -1,27 +0,0 @@
-#!/usr/bin/env python3
-
-import os, re
-
-
-def maketodo(pipfile, dbfile):
-    todos = set()
-    with open(pipfile, "r") as f:
-        for line in f:
-            line = line.split()
-            todos.add(line[0])
-    with open(dbfile, "r") as f:
-        for line in f:
-            line = line.split()
-            todos.remove(line[0])
-    for line in todos:
-        if re.match(r"^INT_[LR].IMUX(_L)?[0-9]+\.LOGIC_OUTS(_L)?[0-9]+$",
-                    line):
-            print(line)
-
-
-maketodo(
-    "build/pips_int_l.txt", "%s/%s/segbits_int_l.db" %
-    (os.getenv("XRAY_DATABASE_DIR"), os.getenv("XRAY_DATABASE")))
-maketodo(
-    "build/pips_int_r.txt", "%s/%s/segbits_int_r.db" %
-    (os.getenv("XRAY_DATABASE_DIR"), os.getenv("XRAY_DATABASE")))

fuzzers/052-clkin/Makefile

@@ -1,2 +1,3 @@
+MAKETODO_FLAGS=--re "^INT_[LR].CLK"
 include ../int_loop.mk
 

fuzzers/052-clkin/generate.tcl

@@ -22,7 +22,7 @@ route_design
 # write_checkpoint -force design.dcp
 
 
-set fp [open "../todo.txt" r]
+set fp [open "../../todo.txt" r]
 set todo_lines {}
 for {gets $fp line} {$line != ""} {gets $fp line} {
     lappend todo_lines [split $line .]

fuzzers/052-clkin/maketodo.py

@@ -1,26 +0,0 @@
-#!/usr/bin/env python3
-
-import os, re
-
-
-def maketodo(pipfile, dbfile):
-    todos = set()
-    with open(pipfile, "r") as f:
-        for line in f:
-            line = line.split()
-            todos.add(line[0])
-    with open(dbfile, "r") as f:
-        for line in f:
-            line = line.split()
-            todos.remove(line[0])
-    for line in todos:
-        if re.match(r"^INT_[LR].CLK", line):
-            print(line)
-
-
-maketodo(
-    "build/pips_int_l.txt", "%s/%s/segbits_int_l.db" %
-    (os.getenv("XRAY_DATABASE_DIR"), os.getenv("XRAY_DATABASE")))
-maketodo(
-    "build/pips_int_r.txt", "%s/%s/segbits_int_r.db" %
-    (os.getenv("XRAY_DATABASE_DIR"), os.getenv("XRAY_DATABASE")))

fuzzers/053-ctrlin/Makefile

@@ -1,2 +1,3 @@
+MAKETODO_FLAGS=--re "^INT_[LR].CTRL"
 include ../int_loop.mk
 

fuzzers/053-ctrlin/generate.tcl

@@ -22,7 +22,7 @@ route_design
 # write_checkpoint -force design.dcp
 
 
-set fp [open "../todo.txt" r]
+set fp [open "../../todo.txt" r]
 set todo_lines {}
 for {gets $fp line} {$line != ""} {gets $fp line} {
     lappend todo_lines [split $line .]

fuzzers/053-ctrlin/maketodo.py

@@ -1,26 +0,0 @@
-#!/usr/bin/env python3
-
-import os, re
-
-
-def maketodo(pipfile, dbfile):
-    todos = set()
-    with open(pipfile, "r") as f:
-        for line in f:
-            line = line.split()
-            todos.add(line[0])
-    with open(dbfile, "r") as f:
-        for line in f:
-            line = line.split()
-            todos.remove(line[0])
-    for line in todos:
-        if re.match(r"^INT_[LR].CTRL", line):
-            print(line)
-
-
-maketodo(
-    "build/pips_int_l.txt", "%s/%s/segbits_int_l.db" %
-    (os.getenv("XRAY_DATABASE_DIR"), os.getenv("XRAY_DATABASE")))
-maketodo(
-    "build/pips_int_r.txt", "%s/%s/segbits_int_r.db" %
-    (os.getenv("XRAY_DATABASE_DIR"), os.getenv("XRAY_DATABASE")))

fuzzers/054-gfan/Makefile

@@ -1,2 +1,3 @@
+MAKETODO_FLAGS=--re "^INT_[LR].GFAN" --not-endswith ".GND_WIRE"
 include ../int_loop.mk
 

fuzzers/054-gfan/generate.tcl

@@ -60,7 +60,7 @@ proc loop { line idx int_l_tile int_r_tile } {
 }
 
 proc load_todo_lines {} {
-    set fp [open "../todo.txt" r]
+    set fp [open "../../todo.txt" r]
     set todo_lines {}
     for {gets $fp line} {$line != ""} {gets $fp line} {
         lappend todo_lines [split $line .]

fuzzers/054-gfan/maketodo.py

@@ -1,27 +0,0 @@
-#!/usr/bin/env python3
-
-import os, re
-
-
-def maketodo(pipfile, dbfile):
-    todos = set()
-    with open(pipfile, "r") as f:
-        for line in f:
-            line = line.split()
-            todos.add(line[0])
-    with open(dbfile, "r") as f:
-        for line in f:
-            line = line.split()
-            todos.remove(line[0])
-    for line in todos:
-        if re.match(r"^INT_[LR].GFAN",
-                    line) and not line.endswith(".GND_WIRE"):
-            print(line)
-
-
-maketodo(
-    "build/pips_int_l.txt", "%s/%s/segbits_int_l.db" %
-    (os.getenv("XRAY_DATABASE_DIR"), os.getenv("XRAY_DATABASE")))
-maketodo(
-    "build/pips_int_r.txt", "%s/%s/segbits_int_r.db" %
-    (os.getenv("XRAY_DATABASE_DIR"), os.getenv("XRAY_DATABASE")))

fuzzers/056-rempips/Makefile

@@ -1,2 +1,3 @@
+MAKETODO_FLAGS=--re ".*" --not-endswith ".VCC_WIRE"
 include ../int_loop.mk
 

fuzzers/056-rempips/maketodo.py

@@ -1,58 +0,0 @@
-#!/usr/bin/env python3
-
-import os, re, sys
-from prjxray import util
-
-
-def maketodo(pipfile, dbfile, verbose=False):
-    '''Print name of all pips in pipfile but not dbfile'''
-    todos = set()
-    with open(pipfile, "r") as f:
-        for line in f:
-            line = line.split()
-            todos.add(line[0])
-    verbose and print(
-        '%s: %u entries' % (pipfile, len(todos)), file=sys.stderr)
-    # Support generate without existing DB
-    if os.path.exists(dbfile):
-        with open(dbfile, "r") as f:
-            for line in f:
-                line = line.split()
-                pip = line[0]
-                todos.remove(pip)
-    verbose and print(
-        'Remove %s: %u entries' % (dbfile, len(todos)), file=sys.stderr)
-    drops = 0
-    lines = 0
-    for line in todos:
-        if line.endswith(".VCC_WIRE"):
-            drops += 1
-            continue
-        print(line)
-        lines += 1
-    verbose and print(
-        'Print %u entries w/ %u drops' % (lines, drops), file=sys.stderr)
-
-
-def run(build_dir):
-    db_dir = "%s/%s" % (
-        os.getenv("XRAY_DATABASE_DIR"), os.getenv("XRAY_DATABASE"))
-
-    maketodo("%s/pips_int_l.txt" % build_dir, "%s/segbits_int_l.db" % db_dir)
-    maketodo("%s/pips_int_r.txt" % build_dir, "%s/segbits_int_r.db" % db_dir)
-
-
-def main():
-    import argparse
-
-    parser = argparse.ArgumentParser(
-        description="Print list of known but unsolved PIPs")
-
-    parser.add_argument('--build-dir', default="build", help='')
-    args = parser.parse_args()
-
-    run(build_dir=args.build_dir)
-
-
-if __name__ == '__main__':
-    main()

fuzzers/int_loop.mk

@@ -1,39 +1,53 @@
-# WARNING: N cannot be reduced or -m will always fail
-N := 10
+# Number of spcimens
+ifeq ($(QUICK),Y)
+N ?= 1
+SEGMATCH_FLAGS=
+else
+# Should be at least the -m value
+N ?= 20
+SEGMATCH_FLAGS=-m 10 -M 20
+endif
+# Iteration number (each containing N specimens)
+# Driven by int_loop.sh
+ITER ?= 1
+MAKETODO_RE ?=
+
 # See int_loop_check.py
 # rempips took 35 iters once, so set 50 as a good start point
 CHECK_ARGS := --zero-entries --timeout-iters 50
-SPECIMENS := $(addprefix build/specimen_,$(shell seq -f '%03.0f' $(N)))
+SPECIMENS := $(addprefix build/$(ITER)/specimen_,$(shell seq -f '%03.0f' $(N)))
 SPECIMENS_OK := $(addsuffix /OK,$(SPECIMENS))
 # Individual fuzzer directory, such as ~/prjxray/fuzzers/010-lutinit
 export FUZDIR=$(shell pwd)
 
+# Specimens from current run must complete, but previous iterations may exist
 database: $(SPECIMENS_OK)
-	${XRAY_SEGMATCH} -m 5 -M 15 -o build/segbits_int_l.db $(addsuffix /segdata_int_l.txt,$(SPECIMENS))
-	${XRAY_SEGMATCH} -m 5 -M 15 -o build/segbits_int_r.db $(addsuffix /segdata_int_r.txt,$(SPECIMENS))
+	${XRAY_SEGMATCH} $(SEGMATCH_FLAGS) -o build/segbits_int_l.db $(shell find build -name segdata_int_l.txt)
+	${XRAY_SEGMATCH} $(SEGMATCH_FLAGS) -o build/segbits_int_r.db $(shell find build -name segdata_int_r.txt)
+# May be undersolved
+ifneq ($(QUICK),Y)
+	${XRAY_DBFIXUP} --db-root build --clb-int
+endif
 
 pushdb:
-	${XRAY_DBFIXUP} --db-root build --clb-int
 	${XRAY_MERGEDB} int_l build/segbits_int_l.db
 	${XRAY_MERGEDB} int_r build/segbits_int_r.db
 
 $(SPECIMENS_OK): build/todo.txt
-	mkdir -p build
+	mkdir -p build/$(ITER)
 	bash ${XRAY_DIR}/utils/top_generate.sh $(subst /OK,,$@)
 	touch $@
 
-build/pips_int_l.txt: $(FUZDIR)/../piplist.tcl
-	mkdir -p build
-	cd build && vivado -mode batch -source $(FUZDIR)/../piplist.tcl
+build/pips_int_l.txt: $(XRAY_DIR)/fuzzers/piplist.tcl
+	mkdir -p build/$(ITER)
+	cd build && vivado -mode batch -source $(XRAY_DIR)/fuzzers/piplist.tcl
 
 # Used 1) to see if we are done 2) pips to try in generate.tcl
-build/todo.txt: build/pips_int_l.txt maketodo.py
-	#python3 maketodo.py --no-strict | sort -R | head -n10 > build/todo.txt.tmp
-	python3 maketodo.py >build/todo_all.txt
+build/todo.txt: build/pips_int_l.txt $(XRAY_DIR)/fuzzers/int_maketodo.py
+	python3 $(XRAY_DIR)/fuzzers/int_maketodo.py $(MAKETODO_FLAGS) >build/todo_all.txt
 	cat build/todo_all.txt | sort -R | head -n10 > build/todo.txt.tmp
 	mv build/todo.txt.tmp build/todo.txt
 
-
 # XXX: conider moving to script
 run:
 	$(MAKE) clean
@@ -45,7 +59,7 @@ clean:
 
 # Remove iteration specific files, but keep piplist.tcl output
 cleanprj:
-	rm -rf build/specimen_* build/todo.txt build/*.db
+	rm -rf build/$(ITER) build/todo.txt
 
 .PHONY: database pushdb run clean cleanprj
 

fuzzers/int_loop.sh

@@ -36,6 +36,13 @@ while [[ $# -gt 0 ]]; do
     esac
 done
 
+# Quick solves are sloppy
+# Never push them in as they may be under solved
+if [ "$QUICK" = "Y" ] ; then
+    iter_pushdb=false
+    end_pushdb=false
+fi
+
 set -ex
 MAKE=${MAKE:-make}
 MAKEFLAGS=${MAKEFLAGS:-}

fuzzers/int_maketodo.py

@@ -0,0 +1,77 @@
+#!/usr/bin/env python3
+
+import os, re
+import sys
+
+
+def maketodo(pipfile, dbfile, intre, not_endswith=None, verbose=False):
+    todos = set()
+    with open(pipfile, "r") as f:
+        for line in f:
+            line = line.split()
+            todos.add(line[0])
+    verbose and print(
+        '%s: %u entries' % (pipfile, len(todos)), file=sys.stderr)
+    # Allow against empty db
+    if os.path.exists(dbfile):
+        with open(dbfile, "r") as f:
+            for line in f:
+                line = line.split()
+                todos.remove(line[0])
+    verbose and print(
+        'Remove %s: %u entries' % (dbfile, len(todos)), file=sys.stderr)
+    drops = 0
+    lines = 0
+    for line in todos:
+        if re.match(intre, line) and (not_endswith is None
+                                      or not line.endswith(not_endswith)):
+            print(line)
+        else:
+            drops += 1
+        lines += 1
+    verbose and print(
+        'Print %u entries w/ %u drops' % (lines, drops), file=sys.stderr)
+
+
+def run(build_dir, db_dir, intre, not_endswith=None, verbose=False):
+    if db_dir is None:
+        db_dir = "%s/%s" % (
+            os.getenv("XRAY_DATABASE_DIR"), os.getenv("XRAY_DATABASE"))
+
+    assert intre, "RE is required"
+    maketodo(
+        "%s/pips_int_l.txt" % build_dir,
+        "%s/segbits_int_l.db" % db_dir,
+        intre,
+        not_endswith,
+        verbose=verbose)
+    maketodo(
+        "%s/pips_int_r.txt" % build_dir,
+        "%s/segbits_int_r.db" % db_dir,
+        intre,
+        not_endswith,
+        verbose=verbose)
+
+
+def main():
+    import argparse
+
+    parser = argparse.ArgumentParser(
+        description="Print list of known but unsolved PIPs")
+
+    parser.add_argument('--build-dir', default="build", help='')
+    parser.add_argument('--db-dir', default=None, help='')
+    parser.add_argument('--re', required=True, help='')
+    parser.add_argument(
+        '--not-endswith', help='Drop lines if they end with this')
+    args = parser.parse_args()
+
+    run(
+        build_dir=args.build_dir,
+        db_dir=args.db_dir,
+        intre=args.re,
+        not_endswith=args.not_endswith)
+
+
+if __name__ == '__main__':
+    main()

fuzzers/piplist.tcl

@@ -22,7 +22,7 @@ source "$::env(XRAY_DIR)/utils/utils.tcl"
 
 proc print_tile_pips {tile_type filename} {
     set tile [lindex [get_tiles -filter "TYPE == $tile_type"] 0]
-    puts "Dumping PIPs for tile $tile ($tile_type) to $filename."
+    puts "Dumping PIPs for tile $tile ($tile_type) to $filename"
     set fp [open $filename w]
     foreach pip [lsort [get_pips -filter {IS_DIRECTIONAL} -of_objects [get_tiles $tile]]] {
         set src [get_wires -uphill -of_objects $pip]
@@ -36,3 +36,4 @@ proc print_tile_pips {tile_type filename} {
 
 print_tile_pips INT_L pips_int_l.txt
 print_tile_pips INT_R pips_int_r.txt
+puts "Done"