Merge pull request #555 from natsfr/master

Adding modular arguments for parallel Fuzzer 072

fuzzers/072-ordered_wires/Makefile

@@ -2,6 +2,8 @@
 N := 1
 SPECIMENS := $(addprefix build/specimen_,$(shell seq -f '%03.0f' $(N)))
 SPECIMENS_OK := $(addsuffix /OK,$(SPECIMENS))
+MAX_VIVADO_PROCESS ?= 4
+MAX_PIPS_INSTANCE ?= 340000
 
 database: $(SPECIMENS_OK)
 	true
@@ -10,7 +12,7 @@ pushdb:
 	true
 
 $(SPECIMENS_OK):
-	bash generate.sh $(subst /OK,,$@)
+	bash generate.sh $(subst /OK,,$@) -p=$(MAX_VIVADO_PROCESS) -t=$(MAX_PIPS_INSTANCE)
 	touch $@
 
 run:

fuzzers/072-ordered_wires/generate.sh

@@ -2,5 +2,5 @@
 
 source ${XRAY_GENHEADER}
 
-python3 $FUZDIR/run_fuzzer.py
+python3 $FUZDIR/run_fuzzer.py $2 $3
 

fuzzers/072-ordered_wires/run_fuzzer.py

@@ -5,20 +5,16 @@ import subprocess
 import signal
 from multiprocessing import Pool
 from itertools import chain
+import argparse
 
 # Can be used to redirect vivado tons of output
 # stdout=DEVNULL in subprocess.check_call
-#try:
-#    from subprocess import DEVNULL
-#except ImportError:
-#    import os
-#    DEVNULL = open(os.devnull, 'wb')
 
 
 # Worker function called from threads
-def start_vivado(argList):
+def start_pips(argList):
-    blockID, start, stop = argList
+    blockID, start, stop, total = argList
-    print("Running instance :" + str(blockID))
+    print("Running instance :" + str(blockID) + " / " + str(total))
     subprocess.check_call(
         "${XRAY_VIVADO} -mode batch -source $FUZDIR/job.tcl -tclargs " +
         str(blockID) + " " + str(start) + " " + str(stop),
@@ -35,55 +31,101 @@ def get_nb_pips():
     return int(countfile.readline())
 
 
-def main(argv):
+def run_pool(itemcount, nbBlocks, blocksize, nbParBlock, workFunc):
-    nbBlocks = 64
-    nbParBlock = 4
-
-    pipscount = get_nb_pips()
-    blocksize = int(pipscount / nbBlocks)
-    intPipsCount = blocksize * nbBlocks
-
     # We handle the case of not integer multiple of pips
+    intitemcount = blocksize * nbBlocks
     lastRun = False
-    modBlocks = pipscount % nbBlocks
+    modBlocks = itemcount - intitemcount
     if modBlocks != 0:
         lastRun = True
         nbBlocks = nbBlocks + 1
 
-    if not os.path.exists("wires"):
-        os.mkdir("wires")
-
     print(
-        "Pips Count: " + str(pipscount) + " - Number of blocks: " +
+        "Items Count: " + str(itemcount) + " - Number of blocks: " +
         str(nbBlocks) + " - Parallel blocks: " + str(nbParBlock) +
         " - Blocksize: " + str(blocksize) + " - Modulo Blocks: " +
         str(modBlocks))
 
     blockId = range(0, nbBlocks)
-    startI = range(0, intPipsCount, blocksize)
+    startI = range(0, intitemcount, blocksize)
-    stopI = range(blocksize, intPipsCount + 1, blocksize)
+    stopI = range(blocksize, intitemcount + 1, blocksize)
+    totalBlock = [nbBlocks for _ in range(nbBlocks)]
 
     # In case we have a last incomplete block we add it as a last
     # element in the arguments list
     if lastRun == True:
-        startI = chain(startI, [intPipsCount])
+        startI = chain(startI, [intitemcount])
-        stopI = chain(stopI, [pipscount])
+        stopI = chain(stopI, [itemcount])
 
-    argList = zip(blockId, startI, stopI)
+    argList = zip(blockId, startI, stopI, totalBlock)
 
     with Pool(processes=nbParBlock) as pool:
-        pool.map(start_vivado, argList)
+        pool.map(workFunc, argList)
+
+    return nbBlocks
+
+
+# ==========================================================================
+# ===== FPGA Logic Items data ==============================================
+# For Artix 7 50T:
+#   - Total pips: 22002368
+#   - Total tiles: 18055
+#   - Total nodes: 1953452
+# For Kintex 7 70T:
+#   - Total pips: 29424910
+#   - Total tiles: 24453
+#   - Total nodes: 2663055
+# For Zynq 7 z010:
+#   - Total pips: 12462138
+#   - Total tiles: 13440
+#   - Total nodes: 1122477
+# =========================================================================
+# Dividing by about 64 over 4 core is not optimized but a default to run
+# on most computer
+# =========================================================================
+
+
+def main(argv):
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "-p",
+        "--nbPar",
+        help="Number of parallel instances of Vivado",
+        type=int,
+        default=4)
+    parser.add_argument(
+        "-t",
+        "--sizePipsBlock",
+        help="Define the number of pips to process per instance",
+        type=int,
+        default=340000)
+    args = parser.parse_args()
+
+    nbParBlock = args.nbPar
+    blockPipsSize = args.sizePipsBlock
+
+    pipscount = get_nb_pips()
+    nbPipsBlock = int(pipscount / blockPipsSize)
+
+    if not os.path.exists("wires"):
+        os.mkdir("wires")
+
+    print(
+        " nbPar: " + str(nbParBlock) + " blockPipsSize: " + str(blockPipsSize))
+
+    pipsFileCount = run_pool(
+        pipscount, nbPipsBlock, blockPipsSize, nbParBlock, start_pips)
 
     print("Generating final files")
 
     with open("uphill_wires.txt", "w") as wfd:
-        for j in range(0, nbBlocks):
+        for j in range(0, pipsFileCount):
             f = "wires/uphill_wires_" + str(j) + ".txt"
             with open(f, "r") as fd:
                 shutil.copyfileobj(fd, wfd)
 
     with open("downhill_wires.txt", "w") as wed:
-        for j in range(0, nbBlocks):
+        for j in range(0, pipsFileCount):
             e = "wires/downhill_wires_" + str(j) + ".txt"
             with open(e, "r") as ed:
                 shutil.copyfileobj(ed, wed)