Merge pull request #305 from mcmasterg/dbfixup

Dbfixup

docs/format/db.rst

@@ -47,7 +47,8 @@ Related tools:
  * dbfixup.py: internal tool that expands multi-bit encodings (ex: one hot) into groups. For example:
    * .rdb file with one hot: BRAM.RAMB18_Y1.WRITE_WIDTH_A_18 27_267
    * .db: file expanded: BRAM.RAMB18_Y1.WRITE_WIDTH_A_18 !27_268 !27_269 27_267
- * dbcheck.py: valides that a database is fully and consistently solved
+ * parsedb.py: valides that a database is fully and consistently solved
+   * Optionally outputs to canonical form
    * Ex: complains if const0 entries exist
    * Ex: complains if symbols are duplicated (such as after a mergedb after rename)
  * mergedb.sh: adds new bit entries to an existing db

prjxray/db.py

@@ -33,6 +33,7 @@ class Database(object):
 
     """
         self.db_root = db_root
+        # tilegrid.json JSON object
         self.tilegrid = None
         self.tileconn = None
         self.tile_types = None

prjxray/util.py

@@ -96,3 +96,70 @@ def parse_db_line(line):
         # 100_319
         assert re.match(r'[!]*[0-9]+_[0-9]+', bit), "Invalid bit: %s" % bit
     return tag, bits, None
+
+
+def parse_tagbit(x):
+    # !30_07
+    if x[0] == '!':
+        isset = False
+        numstr = x[1:]
+    else:
+        isset = True
+        numstr = x
+    frame, word = numstr.split("_")
+    # second part forms a tuple refereced in sets
+    return (isset, (int(frame, 10), int(word, 10)))
+
+
+def addr_bit2word(bitaddr):
+    word = bitaddr // 32
+    bit = bitaddr % 32
+    return word, bit
+
+
+def addr2str(addr, word, bit):
+    # Make like .bits file: bit_00020b14_073_05
+    # also similar to .db file: CLBLL_L.SLICEL_X0.CEUSEDMUX 01_39
+    assert 0 <= bit <= 31
+    return "%08x_%03u_%02u" % (addr, word, bit)
+
+
+def gen_tile_bits(db_root, tilej, strict=False, verbose=False):
+    '''
+    For given tile yield
+    (absolute address, absolute FDRI bit offset, tag)
+
+    For each address space
+    Find applicable files
+    For each tag bit in those files, calculate absolute address and bit offsets
+
+    Sample file names:
+    segbits_clbll_l.db
+    segbits_int_l.db
+    segbits_bram_l.block_ram.db
+    '''
+    for block_type, blockj in tilej["bits"].items():
+        baseaddr = int(blockj["baseaddr"], 0)
+        bitbase = 32 * blockj["offset"]
+
+        if block_type == "CLB_IO_CLK":
+            fn = "%s/segbits_%s.db" % (db_root, tilej["type"].lower())
+        else:
+            fn = "%s/segbits_%s.db.%s" % (
+                db_root, tilej["type"].lower(), block_type.lower())
+        # tilegrid runs a lot earlier than fuzzers
+        # may not have been created yet
+        verbose and print("Check %s: %s" % (fn, os.path.exists(fn)))
+        if strict:
+            assert os.path.exists(fn)
+        elif not os.path.exists(fn):
+            continue
+
+        with open(fn, "r") as f:
+            for line in f:
+                tag, bits, mode = parse_db_line(line)
+                assert mode is None
+                for bitstr in bits:
+                    # 31_06
+                    _bit_inv, (bit_addroff, bit_bitoff) = parse_tagbit(bitstr)
+                    yield (baseaddr + bit_addroff, bitbase + bit_bitoff, tag)

utils/checkdb.py

@@ -0,0 +1,136 @@
+#!/usr/bin/env python3
+'''
+Check:
+-Individual files are valid
+-No overlap between any tile
+
+TODO:
+Can we use prjxray?
+Relies on 074, which is too far into the process
+'''
+
+from prjxray import util
+from prjxray import db as prjxraydb
+import os
+import parsedb
+#from prjxray import db as prjxraydb
+import glob
+
+
+def make_tile_mask(db_root, tile_name, tilej, strict=False, verbose=False):
+    '''
+    Return dict
+    key: (address, bit index)
+    val: sample description of where it came from (there may be multiple, only one)
+    '''
+
+    # FIXME: fix mask files https://github.com/SymbiFlow/prjxray/issues/301
+    # in the meantime build them on the fly
+    # We may want this to build them anyway
+
+    ret = dict()
+    for absaddr, bitaddr, tag in util.gen_tile_bits(
+            db_root, tilej, strict=strict, verbose=verbose):
+        name = "%s.%s" % (tile_name, tag)
+        ret.setdefault((absaddr, bitaddr), name)
+    return ret
+
+
+def parsedb_all(db_root, verbose=False):
+    '''Verify .db files are individually valid'''
+
+    files = 0
+    for bit_fn in glob.glob('%s/segbits_*.db' % db_root):
+        verbose and print("Checking %s" % bit_fn)
+        parsedb.run(bit_fn, fnout=None, strict=True, verbose=verbose)
+        files += 1
+    print("segbits_*.db: %d okay" % files)
+
+    files = 0
+    for bit_fn in glob.glob('%s/mask_*.db' % db_root):
+        verbose and print("Checking %s" % bit_fn)
+        parsedb.run(bit_fn, fnout=None, strict=True, verbose=verbose)
+        files += 1
+    print("mask_*.db: %d okay" % files)
+
+
+def check_tile_overlap(db, db_root, strict=False, verbose=False):
+    '''
+    Verifies that no two tiles use the same bit
+
+    Assume .db files are individually valid
+    Create a mask for all the bits the tile type uses
+    For each tile, create bitmasks over the entire bitstream for current part
+    Throw an exception if two tiles share an address
+    '''
+    mall = dict()
+
+    tiles_checked = 0
+
+    def subtiles(tile_names):
+        for tile_name in tile_names:
+            yield tile_name, db.tilegrid[tile_name]
+
+    for tile_name, tilej in db.tilegrid.items():
+        # for tile_name, tilej in subtiles(["CLBLL_L_X14Y112", "INT_L_X14Y112"]):
+
+        mtile = make_tile_mask(
+            db_root, tile_name, tilej, strict=strict, verbose=verbose)
+        verbose and print(
+            "Checking %s, type %s, bits: %s" %
+            (tile_name, tilej["type"], len(mtile)))
+        if len(mtile) == 0:
+            continue
+
+        collisions = set(mall.keys()).intersection(set(mtile.keys()))
+        if collisions:
+            print("ERROR: %s collisions" % len(collisions))
+            for ck in sorted(collisions):
+                addr, bitaddr = ck
+                word, bit = util.addr_bit2word(bitaddr)
+                print(
+                    "  %s: had %s, got %s" %
+                    (util.addr2str(addr, word, bit), mall[ck], mtile[ck]))
+            raise ValueError("%s collisions" % len(collisions))
+        mall.update(mtile)
+        tiles_checked += 1
+    print("Checked %s tiles, %s bits" % (tiles_checked, len(mall)))
+
+
+def run(db_root, strict=False, verbose=False):
+    # Start by running a basic check on db files
+    print("Checking individual .db...")
+    parsedb_all(db_root, verbose=verbose)
+
+    # Now load and verify tile consistency
+    db = prjxraydb.Database(db_root)
+    db._read_tilegrid()
+    '''
+    these don't load properly without .json files
+    See: https://github.com/SymbiFlow/prjxray/issues/303
+    db._read_tile_types()
+    print(db.tile_types.keys())
+    '''
+
+    verbose and print("")
+
+    print("Checking aggregate dir...")
+    check_tile_overlap(db, db_root, strict=strict, verbose=verbose)
+
+
+def main():
+    import argparse
+
+    parser = argparse.ArgumentParser(
+        description="Parse a db repository, checking for consistency")
+
+    util.db_root_arg(parser)
+    parser.add_argument('--strict', action='store_true', help='')
+    parser.add_argument('--verbose', action='store_true', help='')
+    args = parser.parse_args()
+
+    run(args.db_root, strict=args.strict, verbose=args.verbose)
+
+
+if __name__ == '__main__':
+    main()

utils/dbcheck.py

@@ -1,47 +0,0 @@
-#!/usr/bin/env python3
-
-import sys, re
-
-database = dict()
-database_r = dict()
-
-for arg in sys.argv[1:]:
-    with open(arg, "r") as f:
-        for line in f:
-            if "<" in line:
-                raise Exception("Found '<' in this line: %s" % line)
-
-            line = line.split()
-            key = line[0]
-            bits = tuple(sorted(set(line[1:])))
-
-            if key in database:
-                print("Warning: Duplicate key: %s %s" % (key, bits))
-
-            if bits in database_r:
-                print("Warning: Duplicate bits: %s %s" % (key, bits))
-
-            database[key] = bits
-            database_r[bits] = key
-
-
-def get_subsets(bits):
-    retval = list()
-    retval.append(bits)
-    for i in range(len(bits)):
-        for s in get_subsets(bits[i + 1:]):
-            retval.append(bits[0:i] + s)
-    return retval
-
-
-def check_subsets(bits):
-    for sub_bits in sorted(get_subsets(bits)):
-        if sub_bits != bits and sub_bits != ():
-            if sub_bits in database_r:
-                print(
-                    "Warning: Entry %s %s is a subset of entry %s %s." %
-                    (database_r[sub_bits], sub_bits, database_r[bits], bits))
-
-
-for key, bits in database.items():
-    check_subsets(bits)

utils/dbcheck.sh

@@ -1,7 +0,0 @@
-#!/bin/bash
-python3 ${XRAY_UTILS_DIR}/dbcheck.py ${XRAY_DATABASE_DIR}/${XRAY_DATABASE}/segbits_{clblm,int}_l.db
-python3 ${XRAY_UTILS_DIR}/dbcheck.py ${XRAY_DATABASE_DIR}/${XRAY_DATABASE}/segbits_{clblm,int}_r.db
-python3 ${XRAY_UTILS_DIR}/dbcheck.py ${XRAY_DATABASE_DIR}/${XRAY_DATABASE}/segbits_{clbll,int}_l.db
-python3 ${XRAY_UTILS_DIR}/dbcheck.py ${XRAY_DATABASE_DIR}/${XRAY_DATABASE}/segbits_{clbll,int}_r.db
-python3 ${XRAY_UTILS_DIR}/dbcheck.py ${XRAY_DATABASE_DIR}/${XRAY_DATABASE}/segbits_hclk_l.db
-python3 ${XRAY_UTILS_DIR}/dbcheck.py ${XRAY_DATABASE_DIR}/${XRAY_DATABASE}/segbits_hclk_r.db

utils/environment.sh

@@ -19,7 +19,6 @@ export XRAY_GENHEADER="${XRAY_UTILS_DIR}/genheader.sh"
 export XRAY_BITREAD="${XRAY_TOOLS_DIR}/bitread --part_file ${XRAY_PART_YAML}"
 export XRAY_MERGEDB="bash ${XRAY_UTILS_DIR}/mergedb.sh"
 export XRAY_DBFIXUP="python3 ${XRAY_UTILS_DIR}/dbfixup.py"
-export XRAY_DBCHECK="bash ${XRAY_UTILS_DIR}/dbcheck.sh"
 export XRAY_MASKMERGE="bash ${XRAY_UTILS_DIR}/maskmerge.sh"
 export XRAY_SEGMATCH="${XRAY_TOOLS_DIR}/segmatch"
 export XRAY_SEGPRINT="python3 ${XRAY_UTILS_DIR}/segprint.py"

utils/mergedb.sh

@@ -18,6 +18,12 @@ function finish {
 trap finish EXIT
 
 db=$XRAY_DATABASE_DIR/$XRAY_DATABASE/segbits_$1.db
+# Check existing DB
+if [ -f $db ] ; then
+    ${XRAY_PARSEDB} --strict "$db"
+fi
+# Check new DB
+${XRAY_PARSEDB} --strict "$2"
 
 # Fuzzers verify L/R data is equivilent
 # However, expand back to L/R to make downstream tools not depend on this
@@ -78,5 +84,6 @@ esac
 
 touch "$db"
 sort -u "$tmp1" "$db" | grep -v '<.*>' > "$tmp2" || true
+# Check aggregate db for consistency and make canonical
 ${XRAY_PARSEDB} --strict "$tmp2" "$db"
 

utils/parsedb.py

@@ -1,5 +1,6 @@
 #!/usr/bin/env python3
 
+import sys, re
 from prjxray import util
 
 
@@ -41,7 +42,7 @@ def main():
     import argparse
 
     parser = argparse.ArgumentParser(
-        description="Parse a db, check for consistency")
+        description="Parse a db file, checking for consistency")
 
     util.db_root_arg(parser)
     parser.add_argument('--verbose', action='store_true', help='')

utils/segprint.py

@@ -46,22 +46,10 @@ def get_database(db, tile_type, verbose=False):
         parts = l.split()
         name = parts[0]
 
-        def parsetag(x):
-            # !30_07
-            if x[0] == '!':
-                isset = False
-                numstr = x[1:]
-            else:
-                isset = True
-                numstr = x
-            frame, word = numstr.split("_")
-            # second part forms a tuple refereced in sets
-            return (isset, (int(frame, 10), int(word, 10)))
-
         if parts[1] == 'always' or parts[1] == 'hint':
             tagbits = []
         else:
-            tagbits = [parsetag(x) for x in parts[1:]]
+            tagbits = [util.parse_tagbit(x) for x in parts[1:]]
 
         tags.append(list([name] + tagbits))