From 3b4df2e88362d2a11027efadb323d2c16c10c4ff Mon Sep 17 00:00:00 2001 From: Mehdi Khairy Date: Fri, 18 Jan 2019 23:40:14 +0100 Subject: [PATCH] Adding modular arguments for parallel Fuzzer 072 Signed-off-by: Mehdi Khairy --- fuzzers/072-ordered_wires/Makefile | 4 +- fuzzers/072-ordered_wires/generate.sh | 2 +- fuzzers/072-ordered_wires/run_fuzzer.py | 100 +++++++++++++++++------- 3 files changed, 75 insertions(+), 31 deletions(-) diff --git a/fuzzers/072-ordered_wires/Makefile b/fuzzers/072-ordered_wires/Makefile index b372a55f..851bebc8 100644 --- a/fuzzers/072-ordered_wires/Makefile +++ b/fuzzers/072-ordered_wires/Makefile @@ -2,6 +2,8 @@ N := 1 SPECIMENS := $(addprefix build/specimen_,$(shell seq -f '%03.0f' $(N))) SPECIMENS_OK := $(addsuffix /OK,$(SPECIMENS)) +MAX_VIVADO_PROCESS ?= 4 +MAX_PIPS_INSTANCE ?= 340000 database: $(SPECIMENS_OK) true @@ -10,7 +12,7 @@ pushdb: true $(SPECIMENS_OK): - bash generate.sh $(subst /OK,,$@) + bash generate.sh $(subst /OK,,$@) -p=$(MAX_VIVADO_PROCESS) -t=$(MAX_PIPS_INSTANCE) touch $@ run: diff --git a/fuzzers/072-ordered_wires/generate.sh b/fuzzers/072-ordered_wires/generate.sh index afac1456..cfb3e65c 100644 --- a/fuzzers/072-ordered_wires/generate.sh +++ b/fuzzers/072-ordered_wires/generate.sh @@ -2,5 +2,5 @@ source ${XRAY_GENHEADER} -python3 $FUZDIR/run_fuzzer.py +python3 $FUZDIR/run_fuzzer.py $2 $3 diff --git a/fuzzers/072-ordered_wires/run_fuzzer.py b/fuzzers/072-ordered_wires/run_fuzzer.py index 5c5393af..f166c900 100644 --- a/fuzzers/072-ordered_wires/run_fuzzer.py +++ b/fuzzers/072-ordered_wires/run_fuzzer.py @@ -5,20 +5,16 @@ import subprocess import signal from multiprocessing import Pool from itertools import chain +import argparse # Can be used to redirect vivado tons of output # stdout=DEVNULL in subprocess.check_call -#try: -# from subprocess import DEVNULL -#except ImportError: -# import os -# DEVNULL = open(os.devnull, 'wb') # Worker function called from threads -def start_vivado(argList): - blockID, start, stop = argList - print("Running instance :" + str(blockID)) +def start_pips(argList): + blockID, start, stop, total = argList + print("Running instance :" + str(blockID) + " / " + str(total)) subprocess.check_call( "${XRAY_VIVADO} -mode batch -source $FUZDIR/job.tcl -tclargs " + str(blockID) + " " + str(start) + " " + str(stop), @@ -35,55 +31,101 @@ def get_nb_pips(): return int(countfile.readline()) -def main(argv): - nbBlocks = 64 - nbParBlock = 4 - - pipscount = get_nb_pips() - blocksize = int(pipscount / nbBlocks) - intPipsCount = blocksize * nbBlocks - +def run_pool(itemcount, nbBlocks, blocksize, nbParBlock, workFunc): # We handle the case of not integer multiple of pips + intitemcount = blocksize * nbBlocks lastRun = False - modBlocks = pipscount % nbBlocks + modBlocks = itemcount - intitemcount if modBlocks != 0: lastRun = True nbBlocks = nbBlocks + 1 - if not os.path.exists("wires"): - os.mkdir("wires") - print( - "Pips Count: " + str(pipscount) + " - Number of blocks: " + + "Items Count: " + str(itemcount) + " - Number of blocks: " + str(nbBlocks) + " - Parallel blocks: " + str(nbParBlock) + " - Blocksize: " + str(blocksize) + " - Modulo Blocks: " + str(modBlocks)) blockId = range(0, nbBlocks) - startI = range(0, intPipsCount, blocksize) - stopI = range(blocksize, intPipsCount + 1, blocksize) + startI = range(0, intitemcount, blocksize) + stopI = range(blocksize, intitemcount + 1, blocksize) + totalBlock = [nbBlocks for _ in range(nbBlocks)] # In case we have a last incomplete block we add it as a last # element in the arguments list if lastRun == True: - startI = chain(startI, [intPipsCount]) - stopI = chain(stopI, [pipscount]) + startI = chain(startI, [intitemcount]) + stopI = chain(stopI, [itemcount]) - argList = zip(blockId, startI, stopI) + argList = zip(blockId, startI, stopI, totalBlock) with Pool(processes=nbParBlock) as pool: - pool.map(start_vivado, argList) + pool.map(workFunc, argList) + + return nbBlocks + + +# ========================================================================== +# ===== FPGA Logic Items data ============================================== +# For Artix 7 50T: +# - Total pips: 22002368 +# - Total tiles: 18055 +# - Total nodes: 1953452 +# For Kintex 7 70T: +# - Total pips: 29424910 +# - Total tiles: 24453 +# - Total nodes: 2663055 +# For Zynq 7 z010: +# - Total pips: 12462138 +# - Total tiles: 13440 +# - Total nodes: 1122477 +# ========================================================================= +# Dividing by about 64 over 4 core is not optimized but a default to run +# on most computer +# ========================================================================= + + +def main(argv): + parser = argparse.ArgumentParser() + parser.add_argument( + "-p", + "--nbPar", + help="Number of parallel instances of Vivado", + type=int, + default=4) + parser.add_argument( + "-t", + "--sizePipsBlock", + help="Define the number of pips to process per instance", + type=int, + default=340000) + args = parser.parse_args() + + nbParBlock = args.nbPar + blockPipsSize = args.sizePipsBlock + + pipscount = get_nb_pips() + nbPipsBlock = int(pipscount / blockPipsSize) + + if not os.path.exists("wires"): + os.mkdir("wires") + + print( + " nbPar: " + str(nbParBlock) + " blockPipsSize: " + str(blockPipsSize)) + + pipsFileCount = run_pool( + pipscount, nbPipsBlock, blockPipsSize, nbParBlock, start_pips) print("Generating final files") with open("uphill_wires.txt", "w") as wfd: - for j in range(0, nbBlocks): + for j in range(0, pipsFileCount): f = "wires/uphill_wires_" + str(j) + ".txt" with open(f, "r") as fd: shutil.copyfileobj(fd, wfd) with open("downhill_wires.txt", "w") as wed: - for j in range(0, nbBlocks): + for j in range(0, pipsFileCount): e = "wires/downhill_wires_" + str(j) + ".txt" with open(e, "r") as ed: shutil.copyfileobj(ed, wed)