From 22cdae1536e3d45779c5aef2e5a88c9150da7639 Mon Sep 17 00:00:00 2001 From: Tomasz Michalak Date: Thu, 16 May 2019 15:56:58 +0200 Subject: [PATCH 1/3] Generate db files with fuzzer name of origin Signed-off-by: Tomasz Michalak --- fuzzers/int_maketodo.py | 2 +- prjxray/util.py | 22 ++++++++++++++++------ utils/dbfixup.py | 2 +- utils/mergedb.py | 19 +++++++++++++------ utils/mergedb.sh | 7 +++++++ utils/parsedb.py | 2 +- 6 files changed, 39 insertions(+), 15 deletions(-) diff --git a/fuzzers/int_maketodo.py b/fuzzers/int_maketodo.py index a2a70f54..0fd32096 100644 --- a/fuzzers/int_maketodo.py +++ b/fuzzers/int_maketodo.py @@ -58,7 +58,7 @@ def maketodo( with open(dbfile, "r") as f: # INT.BYP_ALT0.BYP_BOUNCE_N3_3 !22_07 !23_07 !25_07 21_07 24_07 for line in f: - tag, _bits, mode = util.parse_db_line(line.strip()) + tag, _bits, mode, _ = util.parse_db_line(line.strip()) # Only count resolved entries if mode: continue diff --git a/prjxray/util.py b/prjxray/util.py index 05750236..ffe6e717 100644 --- a/prjxray/util.py +++ b/prjxray/util.py @@ -126,18 +126,24 @@ def parse_db_line(line): assert re.match(r'[A-Z0-9_.]+', tag), "Invalid tag name: %s, line: %s" % (tag, line) orig_bits = line.replace(tag + " ", "") + if re.match(r'^origin:', orig_bits): + origin = parts[1][7:] + bits = frozenset(parts[2:]) + orig_bits = line.replace(tag + " " + origin + " ", "") + else: + origin = None + bits = frozenset(parts[1:]) # <0 candidates> etc # Ex: INT_L.BYP_BOUNCE5.BYP_ALT5 always if "<" in orig_bits or "always" == orig_bits: - return tag, None, orig_bits + return tag, None, orig_bits, origin - bits = frozenset(parts[1:]) # Ex: CLBLL_L.SLICEL_X0.AOUTMUX.A5Q !30_06 !30_08 !30_11 30_07 for bit in bits: # 19_39 # 100_319 assert re.match(r'[!]*[0-9]+_[0-9]+', bit), "Invalid bit: %s" % bit - return tag, bits, None + return tag, bits, None, origin def parse_db_lines(fn): @@ -146,10 +152,14 @@ def parse_db_lines(fn): yield line, parse_db_line(line) -def write_db_lines(fn, entries): +def write_db_lines(fn, entries, track_origin=False): new_lines = [] - for tag, bits in entries.items(): - new_line = " ".join([tag] + sorted(bits)) + for tag, (bits, origin) in entries.items(): + if track_origin: + assert origin is not None + new_line = " ".join([tag, "origin:" + origin] + sorted(bits)) + else: + new_line = " ".join([tag] + sorted(bits)) new_lines.append(new_line) with open(fn, "w") as f: diff --git a/utils/dbfixup.py b/utils/dbfixup.py index 48a5e797..3c949cd7 100755 --- a/utils/dbfixup.py +++ b/utils/dbfixup.py @@ -124,7 +124,7 @@ def add_zero_bits(fn_in, zero_db, clb_int=False, strict=True, verbose=False): if line == llast: continue - tag, bits, mode = util.parse_db_line(line) + tag, bits, mode, _ = util.parse_db_line(line) # an enum that needs masking # check below asserts that a mask was actually applied if mode and mode != "<0 candidates>" and not strict: diff --git a/utils/mergedb.py b/utils/mergedb.py index af9b5e4e..0f9af458 100755 --- a/utils/mergedb.py +++ b/utils/mergedb.py @@ -4,7 +4,7 @@ import os from prjxray import util -def run(fn_ins, fn_out, strict=False, verbose=False): +def run(fn_ins, fn_out, strict=False, track_origin=False, verbose=False): # tag to bits entries = {} # tag to (bits, line) @@ -13,18 +13,21 @@ def run(fn_ins, fn_out, strict=False, verbose=False): bitss = dict() for fn_in in fn_ins: - for line, (tag, bits, mode) in util.parse_db_lines(fn_in): + for line, (tag, bits, mode, origin) in util.parse_db_lines(fn_in): line = line.strip() assert mode is not None or mode != "always", "strict: got ill defined line: %s" % ( line, ) if tag in tags: - orig_bits, orig_line = tags[tag] + orig_bits, orig_line, orig_origin = tags[tag] if orig_bits != bits: print("WARNING: got duplicate tag %s" % (tag, )) print(" Orig line: %s" % orig_line) print(" New line : %s" % line) assert not strict, "strict: got duplicate tag" + origin = os.path.basename(os.getcwd()) + if track_origin and orig_origin != origin: + origin = orig_origin + "," + origin if bits in bitss: orig_tag, orig_line = bitss[bits] if orig_tag != tag: @@ -33,12 +36,14 @@ def run(fn_ins, fn_out, strict=False, verbose=False): print(" New line : %s" % line) assert not strict, "strict: got duplicate bits" - entries[tag] = bits - tags[tag] = (bits, line) + if track_origin and origin is None: + origin = os.path.basename(os.getcwd()) + entries[tag] = (bits, origin) + tags[tag] = (bits, line, origin) if bits != None: bitss[bits] = (tag, line) - util.write_db_lines(fn_out, entries) + util.write_db_lines(fn_out, entries, track_origin) def main(): @@ -48,6 +53,7 @@ def main(): util.db_root_arg(parser) parser.add_argument('--verbose', action='store_true', help='') + parser.add_argument('--track_origin', action='store_true', help='') parser.add_argument('--out', help='') parser.add_argument('ins', nargs='+', help='Last takes precedence') args = parser.parse_args() @@ -56,6 +62,7 @@ def main(): args.ins, args.out, strict=int(os.getenv("MERGEDB_STRICT", "1")), + track_origin=args.track_origin, verbose=args.verbose) diff --git a/utils/mergedb.sh b/utils/mergedb.sh index 78025412..0165ca7b 100755 --- a/utils/mergedb.sh +++ b/utils/mergedb.sh @@ -131,6 +131,13 @@ if $ismask ; then else # tmp1 must be placed second to take precedence over old bad entries python3 ${XRAY_DIR}/utils/mergedb.py --out "$tmp2" "$db" "$tmp1" + if ! $ismask ; then + db_origin=$XRAY_DATABASE_DIR/$XRAY_DATABASE/segbits_$1.origin_info.db + if [ ! -f $db_origin ] ; then + touch "$db_origin" + fi + python3 ${XRAY_DIR}/utils/mergedb.py --out "$db_origin" "$db_origin" "$tmp1" --track_origin + fi fi # Check aggregate db for consistency and make canonical ${XRAY_PARSEDB} --strict "$tmp2" "$db" diff --git a/utils/parsedb.py b/utils/parsedb.py index 888c0cb0..53ce7c46 100755 --- a/utils/parsedb.py +++ b/utils/parsedb.py @@ -15,7 +15,7 @@ def run(fnin, fnout=None, strict=False, verbose=False): # TODO: figure out what to do with masks if line.startswith("bit "): continue - tag, bits, mode = util.parse_db_line(line) + tag, bits, mode, _ = util.parse_db_line(line) if strict: if mode != "always": assert not mode, "strict: got ill defined line: %s" % (line, ) From d1ccb55a58c4af98b3ec9e7ea160170cd8b7eb8f Mon Sep 17 00:00:00 2001 From: Tomasz Michalak Date: Mon, 27 May 2019 08:36:53 +0200 Subject: [PATCH 2/3] Don't sort db files with fuzzer of origin information Signed-off-by: Tomasz Michalak --- utils/sort_db.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/utils/sort_db.py b/utils/sort_db.py index a1c3b120..c8698c27 100755 --- a/utils/sort_db.py +++ b/utils/sort_db.py @@ -339,6 +339,9 @@ def main(argv): for n in sorted(os.listdir()): if not os.path.isfile(n): continue + # Leave db files with fuzzer of origin untouched + if "origin_info" in n: + continue base, ext = os.path.splitext(n) From cb22f2f0fea3240a40f138b06dc92b60f6e2acb8 Mon Sep 17 00:00:00 2001 From: Tomasz Michalak Date: Mon, 27 May 2019 08:45:15 +0200 Subject: [PATCH 3/3] Don't parse db files with fuzzer of origin information during database validation Signed-off-by: Tomasz Michalak --- utils/checkdb.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/utils/checkdb.py b/utils/checkdb.py index 1a4741c4..9b939124 100755 --- a/utils/checkdb.py +++ b/utils/checkdb.py @@ -67,6 +67,9 @@ def parsedb_all(db_root, verbose=False): files = 0 for bit_fn in glob.glob('%s/segbits_*.db' % db_root): + # Don't parse db files with fuzzer origin information + if "origin_info" in bit_fn: + continue verbose and print("Checking %s" % bit_fn) parsedb.run(bit_fn, fnout=None, strict=True, verbose=verbose) files += 1