Merge pull request #317 from mcmasterg/int_loop2

intpips iterative solver

fuzzers/050-intpips/.gitignore

@@ -1,4 +1,3 @@
-/specimen_[0-9][0-9][0-9]/
+build
-/seg_int_[lr].segbits
+run.ok
-/mask_clbl[lm]_[lr].segbits
+todo
-/run.ok

fuzzers/050-intpips/Makefile

@@ -1,19 +1,28 @@
 ifeq ($(QUICK),Y)
-N := 10
+N := 1
 else
-N := 200
+# Do relatively large batch to keep parallelism high
+# LCM between 12 (CPUs on my system) and 16, a common CPU count
+N := 48
 endif
-
+# Driven by int_loop.sh
-SPECIMENS := $(addprefix build/specimen_,$(shell seq -f '%03.0f' $(N)))
+ITER := 1
+# See int_loop_check.py
+# Original did 200 specimins open loop
+CHECK_ARGS := --max-iters 6 --stable-iters 3
+SPECIMENS := $(addprefix build/$(ITER)/specimen_,$(shell seq -f '%03.0f' $(N)))
 SPECIMENS_OK := $(addsuffix /OK,$(SPECIMENS))
+# Individual fuzzer directory, such as ~/prjxray/fuzzers/010-lutinit
+export FUZDIR=$(shell pwd)
 
+# Specimens from current run must complete, but previous iterations may exist
 database: $(SPECIMENS_OK)
-	${XRAY_SEGMATCH} -m 5 -M 15 -o build/segbits_int_l.db $(addsuffix /segdata_int_l.txt,$(SPECIMENS))
+	${XRAY_SEGMATCH} -m 15 -M 45 -o build/segbits_int_l.db $(shell find build -name segdata_int_l.txt)
-	${XRAY_SEGMATCH} -m 5 -M 15 -o build/segbits_int_r.db $(addsuffix /segdata_int_r.txt,$(SPECIMENS))
+	${XRAY_SEGMATCH} -m 15 -M 45 -o build/segbits_int_r.db $(shell find build -name segdata_int_r.txt)
-	${XRAY_MASKMERGE} build/mask_clbll_l.db $(addsuffix /segdata_int_l.txt,$(SPECIMENS))
+	${XRAY_MASKMERGE} build/mask_clbll_l.db $(shell find build -name segdata_int_l.txt)
-	${XRAY_MASKMERGE} build/mask_clbll_r.db $(addsuffix /segdata_int_r.txt,$(SPECIMENS))
+	${XRAY_MASKMERGE} build/mask_clbll_r.db $(shell find build -name segdata_int_r.txt)
-	${XRAY_MASKMERGE} build/mask_clblm_l.db $(addsuffix /segdata_int_l.txt,$(SPECIMENS))
+	${XRAY_MASKMERGE} build/mask_clblm_l.db $(shell find build -name segdata_int_l.txt)
-	${XRAY_MASKMERGE} build/mask_clblm_r.db $(addsuffix /segdata_int_r.txt,$(SPECIMENS))
+	${XRAY_MASKMERGE} build/mask_clblm_r.db $(shell find build -name segdata_int_r.txt)
 	${XRAY_DBFIXUP} --db-root build --clb-int
 
 pushdb:
@@ -24,19 +33,33 @@ pushdb:
 	${XRAY_MERGEDB} mask_clblm_l build/mask_clblm_l.db
 	${XRAY_MERGEDB} mask_clblm_r build/mask_clblm_r.db
 
-$(SPECIMENS_OK):
+# FIXME: move to iter dir
-	mkdir -p build
+$(SPECIMENS_OK): build/todo.txt
+	mkdir -p build/$(ITER)
 	bash generate.sh $(subst /OK,,$@)
 	touch $@
 
+build/pips_int_l.txt: $(XRAY_DIR)/fuzzers/piplist.tcl
+	mkdir -p build/$(ITER)
+	cd build/$(ITER) && vivado -mode batch -source $(XRAY_DIR)/fuzzers/piplist.tcl
+
+build/todo.txt: build/pips_int_l.txt maketodo.py
+	python3 maketodo.py --build-dir build/$(ITER) >build/todo_all.txt
+	cat build/todo_all.txt | sort -R > build/todo.txt.tmp
+	mv build/todo.txt.tmp build/todo.txt
+
+# XXX: conider moving to script
 run:
 	$(MAKE) clean
-	$(MAKE) database
+	XRAY_DIR=${XRAY_DIR} MAKE="$(MAKE)" MAKEFLAGS="$(MAKEFLAGS)" QUICK=$(QUICK) $(XRAY_DIR)/fuzzers/int_loop.sh --check-args "$(CHECK_ARGS)"
-	$(MAKE) pushdb
 	touch run.ok
 
 clean:
-	rm -rf build run.ok
+	rm -rf build run.ok todo
 
-.PHONY: database pushdb run clean
+# Remove iteration specific files, but keep piplist.tcl output
+cleanprj:
+	rm -rf build/$(ITER) build/todo.txt
+
+.PHONY: database pushdb run clean cleanprj
 

fuzzers/050-intpips/generate.sh

@@ -1,5 +1,7 @@
 #!/bin/bash
 
+set -ex
+
 FUZDIR=$PWD
 source ${XRAY_GENHEADER}
 

fuzzers/050-intpips/generate.tcl

@@ -1,7 +1,7 @@
 create_project -force -part $::env(XRAY_PART) design design
 
-read_verilog ../../top.v
+read_verilog $::env(FUZDIR)/top.v
-read_verilog ../../picorv32.v
+read_verilog $::env(FUZDIR)/picorv32.v
 synth_design -top top
 
 set_property -dict "PACKAGE_PIN $::env(XRAY_PIN_00) IOSTANDARD LVCMOS33" [get_ports clk]
@@ -32,8 +32,13 @@ proc write_txtdata {filename} {
     puts "Writing $filename."
     set fp [open $filename w]
     set all_pips [lsort -unique [get_pips -of_objects [get_nets -hierarchical]]]
-    foreach tile [get_tiles [regsub -all {CLBL[LM]} [get_tiles -of_objects [get_sites -of_objects [get_pblocks roi]]] INT]] {
+    # FIXME: getting IOB. Don't think this works correctly
-        puts "Dumping pips from tile $tile"
+    set tiles [get_tiles [regsub -all {CLBL[LM]} [get_tiles -of_objects [get_sites -of_objects [get_pblocks roi]]] INT]]
+    set ntiles [llength $tiles]
+    set tilei 0
+    foreach tile $tiles {
+        incr tilei
+        puts "Dumping pips from tile $tile ($tilei / $ntiles)"
         foreach pip [filter $all_pips "TILE == $tile"] {
             set src_wire [get_wires -uphill -of_objects $pip]
             set dst_wire [get_wires -downhill -of_objects $pip]

fuzzers/050-intpips/maketodo.py

@@ -0,0 +1,57 @@
+#!/usr/bin/env python3
+
+import os, re, sys
+from prjxray import util
+
+
+def maketodo(pipfile, dbfile, verbose=False):
+    '''Print name of all pips in pipfile but not dbfile'''
+    todos = set()
+    with open(pipfile, "r") as f:
+        for line in f:
+            line = line.split()
+            todos.add(line[0])
+    verbose and print(
+        '%s: %u entries' % (pipfile, len(todos)), file=sys.stderr)
+    # Support generate without existing DB
+    if os.path.exists(dbfile):
+        with open(dbfile, "r") as f:
+            for line in f:
+                line = line.split()
+                pip = line[0]
+                todos.remove(pip)
+    verbose and print(
+        'Remove %s: %u entries' % (dbfile, len(todos)), file=sys.stderr)
+    drops = 0
+    lines = 0
+    for line in todos:
+        if line.endswith(".VCC_WIRE"):
+            drops += 1
+            continue
+        print(line)
+        lines += 1
+    verbose and print(
+        'Print %u entries w/ %u drops' % (lines, drops), file=sys.stderr)
+
+
+def run(build_dir):
+    maketodo(
+        "%s/pips_int_l.txt" % build_dir, "%s/segbits_int_l.db" % build_dir)
+    maketodo(
+        "%s/pips_int_r.txt" % build_dir, "%s/segbits_int_r.db" % build_dir)
+
+
+def main():
+    import argparse
+
+    parser = argparse.ArgumentParser(
+        description="Print list of known but unsolved PIPs")
+
+    parser.add_argument('--build-dir', default="build", help='')
+    args = parser.parse_args()
+
+    run(build_dir=args.build_dir)
+
+
+if __name__ == '__main__':
+    main()

fuzzers/056-rempips/maketodo.py

@@ -4,31 +4,24 @@ import os, re, sys
 from prjxray import util
 
 
-def maketodo(pipfile, dbfile, strict=True):
+def maketodo(pipfile, dbfile, verbose=False):
     '''Print name of all pips in pipfile but not dbfile'''
     todos = set()
     with open(pipfile, "r") as f:
         for line in f:
             line = line.split()
             todos.add(line[0])
-    print('%s: %u entries' % (pipfile, len(todos)), file=sys.stderr)
+    verbose and print(
+        '%s: %u entries' % (pipfile, len(todos)), file=sys.stderr)
     # Support generate without existing DB
-    if os.path.exists(dbfile) or strict:
+    if os.path.exists(dbfile):
         with open(dbfile, "r") as f:
             for line in f:
                 line = line.split()
                 pip = line[0]
-                try:
+                todos.remove(pip)
-                    todos.remove(pip)
+    verbose and print(
-                except KeyError:
+        'Remove %s: %u entries' % (dbfile, len(todos)), file=sys.stderr)
-                    # DB (incorrectly) had multiple entries
-                    # Workaround for testing on old DB revision
-                    if strict:
-                        raise
-                    print(
-                        'WARNING: failed to remove pip %s' % pip,
-                        file=sys.stderr)
-    print('Remove %s: %u entries' % (dbfile, len(todos)), file=sys.stderr)
     drops = 0
     lines = 0
     for line in todos:
@@ -37,20 +30,16 @@ def maketodo(pipfile, dbfile, strict=True):
             continue
         print(line)
         lines += 1
-    print('Print %u entries w/ %u drops' % (lines, drops), file=sys.stderr)
+    verbose and print(
+        'Print %u entries w/ %u drops' % (lines, drops), file=sys.stderr)
 
 
-def run(strict=True):
+def run(build_dir):
-    maketodo(
+    db_dir = "%s/%s" % (
-        "build/pips_int_l.txt",
+        os.getenv("XRAY_DATABASE_DIR"), os.getenv("XRAY_DATABASE"))
-        "%s/%s/segbits_int_l.db" %
+
-        (os.getenv("XRAY_DATABASE_DIR"), os.getenv("XRAY_DATABASE")),
+    maketodo("%s/pips_int_l.txt" % build_dir, "%s/segbits_int_l.db" % db_dir)
-        strict=strict)
+    maketodo("%s/pips_int_r.txt" % build_dir, "%s/segbits_int_r.db" % db_dir)
-    maketodo(
-        "build/pips_int_r.txt",
-        "%s/%s/segbits_int_r.db" %
-        (os.getenv("XRAY_DATABASE_DIR"), os.getenv("XRAY_DATABASE")),
-        strict=strict)
 
 
 def main():
@@ -59,11 +48,10 @@ def main():
     parser = argparse.ArgumentParser(
         description="Print list of known but unsolved PIPs")
 
-    # util.db_root_arg(parser)
+    parser.add_argument('--build-dir', default="build", help='')
-    parser.add_argument('--no-strict', action='store_true', help='')
     args = parser.parse_args()
 
-    run(strict=not args.no_strict)
+    run(build_dir=args.build_dir)
 
 
 if __name__ == '__main__':

fuzzers/int_loop.mk

@@ -37,7 +37,7 @@ build/todo.txt: build/pips_int_l.txt maketodo.py
 # XXX: conider moving to script
 run:
 	$(MAKE) clean
-	XRAY_DIR=${XRAY_DIR} MAKE="$(MAKE)" MAKEFLAGS="$(MAKEFLAGS)" QUICK=$(QUICK) $(XRAY_DIR)/fuzzers/int_loop.sh --check-args "$(CHECK_ARGS)"
+	XRAY_DIR=${XRAY_DIR} MAKE="$(MAKE)" MAKEFLAGS="$(MAKEFLAGS)" QUICK=$(QUICK) $(XRAY_DIR)/fuzzers/int_loop.sh --check-args "$(CHECK_ARGS)" --iter-pushdb
 	touch run.ok
 
 clean:

fuzzers/int_loop.sh

@@ -4,9 +4,14 @@ usage() {
     echo "Run makefile until termination condition"
     echo "usage: int_loop.sh [args]"
     echo "--check-args <args>         int_loop_check.py args"
+    # intpips ingests all segbits files at once and does a push at the end
+    # other loopers do a push every pass
+    echo "--iter-pushdb               make pushdb after successful make database as opposed to end"
 }
 
 check_args=
+iter_pushdb=false
+end_pushdb=true
 while [[ $# -gt 0 ]]; do
     case "$1" in
     --check-args)
@@ -14,6 +19,11 @@ while [[ $# -gt 0 ]]; do
         shift
         shift
         ;;
+    --iter-pushdb)
+        iter_pushdb=true
+        end_pushdb=false
+        shift
+        ;;
     -h|--help)
         usage
         exit 0
@@ -32,26 +42,33 @@ MAKEFLAGS=${MAKEFLAGS:-}
 echo "make: ${MAKE} ${MAKEFLAGS}"
 echo $MAKE
 mkdir -p todo;
+i=1
 while true; do
-    ${MAKE} ${MAKEFLAGS} cleanprj;
+    ${MAKE} ${MAKEFLAGS} ITER=$i cleanprj
-    ${MAKE} ${MAKEFLAGS} build/todo.txt || exit 1;
+    ${MAKE} ${MAKEFLAGS} ITER=$i build/todo.txt
     if python3 ${XRAY_DIR}/fuzzers/int_loop_check.py $check_args ; then
         break
     fi
-    if [ -f build/timeout ] ; then
+    if [ -f todo/timeout ] ; then
         echo "ERROR: timeout"
         exit 1
     fi
 
-    i=$((i+1));
     cp build/todo.txt todo/${i}.txt;
     cp build/todo_all.txt todo/${i}_all.txt;
-    if ${MAKE} ${MAKEFLAGS} database; then
+    if ${MAKE} ${MAKEFLAGS} ITER=$i database; then
-        ${MAKE} ${MAKEFLAGS} pushdb;
+        if $iter_pushdb ; then
+            ${MAKE} ${MAKEFLAGS} pushdb
+        fi
     fi;
     if [ "$QUICK" = "Y" ] ; then
         break;
     fi
+
+    i=$((i+1));
 done;
+if $end_pushdb ; then
+    ${MAKE} ${MAKEFLAGS} pushdb
+fi
 exit 0
 

fuzzers/int_loop_check.py

@@ -52,6 +52,7 @@ def run(
         min_iters=None,
         stable_iters=None,
         timeout_iters=None,
+        max_iters=None,
         zero_entries=None,
         verbose=False):
     timeout_fn = "%s/timeout" % todo_dir
@@ -82,6 +83,12 @@ def run(
         print("Incomplete: not enough iters")
         sys.exit(1)
 
+    if max_iters is not None and max_iter >= max_iters:
+        print(
+            "Complete: reached max iters (want %u, got %u)" %
+            (max_iters, max_iter))
+        sys.exit(0)
+
     if timeout_iters is not None and max_iter > timeout_iters:
         print("ERROR: timeout (max %u, got %u)" % (timeout_iters, max_iter))
         with open(timeout_fn, "w") as _f:
@@ -124,6 +131,10 @@ def main():
         '--timeout-iters',
         default=None,
         help='Max number of entries before creating todo/timeout')
+    parser.add_argument(
+        '--max-iters',
+        default=None,
+        help='Max number of entries before declaring success')
     parser.add_argument(
         '--zero-entries',
         action="store_true",
@@ -138,6 +149,7 @@ def main():
         zint(args.min_iters),
         zint(args.stable_iters),
         zint(args.timeout_iters),
+        zint(args.max_iters),
         args.zero_entries,
         verbose=args.verbose)
 

fuzzers/piplist.tcl

@@ -1,6 +1,6 @@
 create_project -force -part $::env(XRAY_PART) piplist piplist
 
-read_verilog $::env(FUZDIR)/top.v
+read_verilog $::env(XRAY_DIR)/fuzzers/piplist.v
 synth_design -top top
 
 set_property -dict "PACKAGE_PIN $::env(XRAY_PIN_00) IOSTANDARD LVCMOS33" [get_ports i]

fuzzers/piplist.v

@@ -0,0 +1,3 @@
+module top (input i, output o);
+	assign o = i;
+endmodule