Merge pull request #842 from antmicro/bits_origin

Generate db files with fuzzer name of origin

fuzzers/int_maketodo.py

@@ -58,7 +58,7 @@ def maketodo(
         with open(dbfile, "r") as f:
             # INT.BYP_ALT0.BYP_BOUNCE_N3_3 !22_07 !23_07 !25_07 21_07 24_07
             for line in f:
-                tag, _bits, mode = util.parse_db_line(line.strip())
+                tag, _bits, mode, _ = util.parse_db_line(line.strip())
                 # Only count resolved entries
                 if mode:
                     continue

prjxray/util.py

@@ -126,18 +126,24 @@ def parse_db_line(line):
     assert re.match(r'[A-Z0-9_.]+',
                     tag), "Invalid tag name: %s, line: %s" % (tag, line)
     orig_bits = line.replace(tag + " ", "")
+    if re.match(r'^origin:', orig_bits):
+        origin = parts[1][7:]
+        bits = frozenset(parts[2:])
+        orig_bits = line.replace(tag + " " + origin + " ", "")
+    else:
+        origin = None
+        bits = frozenset(parts[1:])
     # <0 candidates> etc
     # Ex: INT_L.BYP_BOUNCE5.BYP_ALT5 always
     if "<" in orig_bits or "always" == orig_bits:
-        return tag, None, orig_bits
+        return tag, None, orig_bits, origin
 
-    bits = frozenset(parts[1:])
     # Ex: CLBLL_L.SLICEL_X0.AOUTMUX.A5Q !30_06 !30_08 !30_11 30_07
     for bit in bits:
         # 19_39
         # 100_319
         assert re.match(r'[!]*[0-9]+_[0-9]+', bit), "Invalid bit: %s" % bit
-    return tag, bits, None
+    return tag, bits, None, origin
 
 
 def parse_db_lines(fn):
@@ -146,10 +152,14 @@ def parse_db_lines(fn):
             yield line, parse_db_line(line)
 
 
-def write_db_lines(fn, entries):
+def write_db_lines(fn, entries, track_origin=False):
     new_lines = []
-    for tag, bits in entries.items():
-        new_line = " ".join([tag] + sorted(bits))
+    for tag, (bits, origin) in entries.items():
+        if track_origin:
+            assert origin is not None
+            new_line = " ".join([tag, "origin:" + origin] + sorted(bits))
+        else:
+            new_line = " ".join([tag] + sorted(bits))
         new_lines.append(new_line)
 
     with open(fn, "w") as f:

utils/checkdb.py

@@ -67,6 +67,9 @@ def parsedb_all(db_root, verbose=False):
 
     files = 0
     for bit_fn in glob.glob('%s/segbits_*.db' % db_root):
+        # Don't parse db files with fuzzer origin information
+        if "origin_info" in bit_fn:
+            continue
         verbose and print("Checking %s" % bit_fn)
         parsedb.run(bit_fn, fnout=None, strict=True, verbose=verbose)
         files += 1

utils/dbfixup.py

@@ -124,7 +124,7 @@ def add_zero_bits(fn_in, zero_db, clb_int=False, strict=True, verbose=False):
             if line == llast:
                 continue
 
-            tag, bits, mode = util.parse_db_line(line)
+            tag, bits, mode, _ = util.parse_db_line(line)
             # an enum that needs masking
             # check below asserts that a mask was actually applied
             if mode and mode != "<0 candidates>" and not strict:

utils/mergedb.py

@@ -4,7 +4,7 @@ import os
 from prjxray import util
 
 
-def run(fn_ins, fn_out, strict=False, verbose=False):
+def run(fn_ins, fn_out, strict=False, track_origin=False, verbose=False):
     # tag to bits
     entries = {}
     # tag to (bits, line)
@@ -13,18 +13,21 @@ def run(fn_ins, fn_out, strict=False, verbose=False):
     bitss = dict()
 
     for fn_in in fn_ins:
-        for line, (tag, bits, mode) in util.parse_db_lines(fn_in):
+        for line, (tag, bits, mode, origin) in util.parse_db_lines(fn_in):
             line = line.strip()
             assert mode is not None or mode != "always", "strict: got ill defined line: %s" % (
                 line, )
 
             if tag in tags:
-                orig_bits, orig_line = tags[tag]
+                orig_bits, orig_line, orig_origin = tags[tag]
                 if orig_bits != bits:
                     print("WARNING: got duplicate tag %s" % (tag, ))
                     print("  Orig line: %s" % orig_line)
                     print("  New line : %s" % line)
                     assert not strict, "strict: got duplicate tag"
+                origin = os.path.basename(os.getcwd())
+                if track_origin and orig_origin != origin:
+                    origin = orig_origin + "," + origin
             if bits in bitss:
                 orig_tag, orig_line = bitss[bits]
                 if orig_tag != tag:
@@ -33,12 +36,14 @@ def run(fn_ins, fn_out, strict=False, verbose=False):
                     print("  New line : %s" % line)
                     assert not strict, "strict: got duplicate bits"
 
-            entries[tag] = bits
-            tags[tag] = (bits, line)
+            if track_origin and origin is None:
+                origin = os.path.basename(os.getcwd())
+            entries[tag] = (bits, origin)
+            tags[tag] = (bits, line, origin)
             if bits != None:
                 bitss[bits] = (tag, line)
 
-    util.write_db_lines(fn_out, entries)
+    util.write_db_lines(fn_out, entries, track_origin)
 
 
 def main():
@@ -48,6 +53,7 @@ def main():
 
     util.db_root_arg(parser)
     parser.add_argument('--verbose', action='store_true', help='')
+    parser.add_argument('--track_origin', action='store_true', help='')
     parser.add_argument('--out', help='')
     parser.add_argument('ins', nargs='+', help='Last takes precedence')
     args = parser.parse_args()
@@ -56,6 +62,7 @@ def main():
         args.ins,
         args.out,
         strict=int(os.getenv("MERGEDB_STRICT", "1")),
+        track_origin=args.track_origin,
         verbose=args.verbose)
 
 

utils/mergedb.sh

@@ -131,6 +131,13 @@ if $ismask ; then
 else
     # tmp1 must be placed second to take precedence over old bad entries
     python3 ${XRAY_DIR}/utils/mergedb.py --out "$tmp2" "$db" "$tmp1"
+    if ! $ismask ; then
+	db_origin=$XRAY_DATABASE_DIR/$XRAY_DATABASE/segbits_$1.origin_info.db
+        if [ ! -f $db_origin ] ; then
+            touch "$db_origin"
+        fi
+        python3 ${XRAY_DIR}/utils/mergedb.py --out "$db_origin" "$db_origin" "$tmp1" --track_origin
+    fi
 fi
 # Check aggregate db for consistency and make canonical
 ${XRAY_PARSEDB} --strict "$tmp2" "$db"

utils/parsedb.py

@@ -15,7 +15,7 @@ def run(fnin, fnout=None, strict=False, verbose=False):
         # TODO: figure out what to do with masks
         if line.startswith("bit "):
             continue
-        tag, bits, mode = util.parse_db_line(line)
+        tag, bits, mode, _ = util.parse_db_line(line)
         if strict:
             if mode != "always":
                 assert not mode, "strict: got ill defined line: %s" % (line, )

utils/sort_db.py

@@ -339,6 +339,9 @@ def main(argv):
     for n in sorted(os.listdir()):
         if not os.path.isfile(n):
             continue
+        # Leave db files with fuzzer of origin untouched
+        if "origin_info" in n:
+            continue
 
         base, ext = os.path.splitext(n)