From 2bb5677d44242a43e932f4dc710687c73113884a Mon Sep 17 00:00:00 2001 From: John McMaster Date: Sun, 9 Dec 2018 21:25:45 -0800 Subject: [PATCH 1/7] parsedb: merge in dbcheck Signed-off-by: John McMaster --- docs/format/db.rst | 3 ++- utils/dbcheck.py | 47 -------------------------------------------- utils/dbcheck.sh | 7 ------- utils/environment.sh | 1 - utils/parsedb.py | 1 + 5 files changed, 3 insertions(+), 56 deletions(-) delete mode 100755 utils/dbcheck.py delete mode 100755 utils/dbcheck.sh diff --git a/docs/format/db.rst b/docs/format/db.rst index 668e894a..6c5885f4 100644 --- a/docs/format/db.rst +++ b/docs/format/db.rst @@ -47,7 +47,8 @@ Related tools: * dbfixup.py: internal tool that expands multi-bit encodings (ex: one hot) into groups. For example: * .rdb file with one hot: BRAM.RAMB18_Y1.WRITE_WIDTH_A_18 27_267 * .db: file expanded: BRAM.RAMB18_Y1.WRITE_WIDTH_A_18 !27_268 !27_269 27_267 - * dbcheck.py: valides that a database is fully and consistently solved + * parsedb.py: valides that a database is fully and consistently solved + * Optionally outputs to canonical form * Ex: complains if const0 entries exist * Ex: complains if symbols are duplicated (such as after a mergedb after rename) * mergedb.sh: adds new bit entries to an existing db diff --git a/utils/dbcheck.py b/utils/dbcheck.py deleted file mode 100755 index fe62bf5c..00000000 --- a/utils/dbcheck.py +++ /dev/null @@ -1,47 +0,0 @@ -#!/usr/bin/env python3 - -import sys, re - -database = dict() -database_r = dict() - -for arg in sys.argv[1:]: - with open(arg, "r") as f: - for line in f: - if "<" in line: - raise Exception("Found '<' in this line: %s" % line) - - line = line.split() - key = line[0] - bits = tuple(sorted(set(line[1:]))) - - if key in database: - print("Warning: Duplicate key: %s %s" % (key, bits)) - - if bits in database_r: - print("Warning: Duplicate bits: %s %s" % (key, bits)) - - database[key] = bits - database_r[bits] = key - - -def get_subsets(bits): - retval = list() - retval.append(bits) - for i in range(len(bits)): - for s in get_subsets(bits[i + 1:]): - retval.append(bits[0:i] + s) - return retval - - -def check_subsets(bits): - for sub_bits in sorted(get_subsets(bits)): - if sub_bits != bits and sub_bits != (): - if sub_bits in database_r: - print( - "Warning: Entry %s %s is a subset of entry %s %s." % - (database_r[sub_bits], sub_bits, database_r[bits], bits)) - - -for key, bits in database.items(): - check_subsets(bits) diff --git a/utils/dbcheck.sh b/utils/dbcheck.sh deleted file mode 100755 index 3bf2550a..00000000 --- a/utils/dbcheck.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash -python3 ${XRAY_UTILS_DIR}/dbcheck.py ${XRAY_DATABASE_DIR}/${XRAY_DATABASE}/segbits_{clblm,int}_l.db -python3 ${XRAY_UTILS_DIR}/dbcheck.py ${XRAY_DATABASE_DIR}/${XRAY_DATABASE}/segbits_{clblm,int}_r.db -python3 ${XRAY_UTILS_DIR}/dbcheck.py ${XRAY_DATABASE_DIR}/${XRAY_DATABASE}/segbits_{clbll,int}_l.db -python3 ${XRAY_UTILS_DIR}/dbcheck.py ${XRAY_DATABASE_DIR}/${XRAY_DATABASE}/segbits_{clbll,int}_r.db -python3 ${XRAY_UTILS_DIR}/dbcheck.py ${XRAY_DATABASE_DIR}/${XRAY_DATABASE}/segbits_hclk_l.db -python3 ${XRAY_UTILS_DIR}/dbcheck.py ${XRAY_DATABASE_DIR}/${XRAY_DATABASE}/segbits_hclk_r.db diff --git a/utils/environment.sh b/utils/environment.sh index da87ff44..73cc2a0a 100644 --- a/utils/environment.sh +++ b/utils/environment.sh @@ -19,7 +19,6 @@ export XRAY_GENHEADER="${XRAY_UTILS_DIR}/genheader.sh" export XRAY_BITREAD="${XRAY_TOOLS_DIR}/bitread --part_file ${XRAY_PART_YAML}" export XRAY_MERGEDB="bash ${XRAY_UTILS_DIR}/mergedb.sh" export XRAY_DBFIXUP="python3 ${XRAY_UTILS_DIR}/dbfixup.py" -export XRAY_DBCHECK="bash ${XRAY_UTILS_DIR}/dbcheck.sh" export XRAY_MASKMERGE="bash ${XRAY_UTILS_DIR}/maskmerge.sh" export XRAY_SEGMATCH="${XRAY_TOOLS_DIR}/segmatch" export XRAY_SEGPRINT="python3 ${XRAY_UTILS_DIR}/segprint.py" diff --git a/utils/parsedb.py b/utils/parsedb.py index b24ad42f..7ff8b0bb 100755 --- a/utils/parsedb.py +++ b/utils/parsedb.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 +import sys, re from prjxray import util From 04053f4d31d435de651309090b2af1fca6f08c89 Mon Sep 17 00:00:00 2001 From: John McMaster Date: Sun, 9 Dec 2018 21:37:14 -0800 Subject: [PATCH 2/7] mergedb.sh: verify consistency earlier Signed-off-by: John McMaster --- utils/mergedb.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/utils/mergedb.sh b/utils/mergedb.sh index 90aca2b0..e653fb8f 100755 --- a/utils/mergedb.sh +++ b/utils/mergedb.sh @@ -18,6 +18,12 @@ function finish { trap finish EXIT db=$XRAY_DATABASE_DIR/$XRAY_DATABASE/segbits_$1.db +# Check existing DB +if [ -f $db ] ; then + ${XRAY_PARSEDB} --strict "$db" +fi +# Check new DB +${XRAY_PARSEDB} --strict "$2" # Fuzzers verify L/R data is equivilent # However, expand back to L/R to make downstream tools not depend on this @@ -78,5 +84,6 @@ esac touch "$db" sort -u "$tmp1" "$db" | grep -v '<.*>' > "$tmp2" || true +# Check aggregate db for consistency and make canonical ${XRAY_PARSEDB} --strict "$tmp2" "$db" From 34464f80f65f909794bbd98680ba75c234919ca8 Mon Sep 17 00:00:00 2001 From: John McMaster Date: Mon, 10 Dec 2018 11:21:56 -0800 Subject: [PATCH 3/7] parsedb: better help Signed-off-by: John McMaster --- utils/parsedb.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/parsedb.py b/utils/parsedb.py index 7ff8b0bb..5d6af833 100755 --- a/utils/parsedb.py +++ b/utils/parsedb.py @@ -42,7 +42,7 @@ def main(): import argparse parser = argparse.ArgumentParser( - description="Parse a db, check for consistency") + description="Parse a db file, checking for consistency") util.db_root_arg(parser) parser.add_argument('--verbose', action='store_true', help='') From ac139b8cb87a0fbbd86e256ed014e860bbd836f6 Mon Sep 17 00:00:00 2001 From: John McMaster Date: Mon, 10 Dec 2018 11:22:24 -0800 Subject: [PATCH 4/7] db.py: comment Signed-off-by: John McMaster --- prjxray/db.py | 1 + 1 file changed, 1 insertion(+) diff --git a/prjxray/db.py b/prjxray/db.py index 200bf0a0..4a886e17 100644 --- a/prjxray/db.py +++ b/prjxray/db.py @@ -33,6 +33,7 @@ class Database(object): """ self.db_root = db_root + # tilegrid.json JSON object self.tilegrid = None self.tileconn = None self.tile_types = None From 5060dae1be076723b37a6f5b2e419e4ce51fe6a5 Mon Sep 17 00:00:00 2001 From: John McMaster Date: Mon, 10 Dec 2018 11:22:40 -0800 Subject: [PATCH 5/7] checkdb: check all .db files Signed-off-by: John McMaster --- utils/checkdb.py | 107 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 107 insertions(+) create mode 100644 utils/checkdb.py diff --git a/utils/checkdb.py b/utils/checkdb.py new file mode 100644 index 00000000..29bae1b4 --- /dev/null +++ b/utils/checkdb.py @@ -0,0 +1,107 @@ +#!/usr/bin/env python3 + +''' +Check: +-Individual files are valid +-No overlap between any tile +''' + +import sys, re +from prjxray import util +import parsedb +#from prjxray import db as prjxraydb +import glob + + +def parsedb_all(db_root, verbose=False): + '''Verify .db files are individually valid''' + + for bit_fn in glob.glob('%s/segbits_*.db'): + parsedb.run(bit_fn, fnout=None, strict=True, verbose=verbose) + + for bit_fn in glob.glob('%s/mask_*.db'): + parsedb.run(bit_fn, fnout=None, strict=True, verbose=verbose) + +def process_db(tile_type, process, verbose): + #ttdb = db.get_tile_type(tile_type) + + fns = [ttdb.tile_dbs.segbits, ttdb.tile_dbs.ppips] + verbose and print("process_db(%s): %s" % (tile_type, fns)) + for fn in fns: + if fn: + with open(fn, "r") as f: + for line in f: + process(util.parse_db_line(line)) + + +def check_seg_overlap(db_root, verbose=False): + ''' + Assume .db files are individually valid + Create a mask for all the bits the tile type uses + For each tile, create bitmasks over the entire bitstream for current part + Throw an exception if two tiles share an address + ''' + # key: (address, bit index) + # val: sample description of where it came from (there may be multiple, only one) + used = dict() + + tiles_checked = 0 + for tile_name, tile in db.tilegrid.items(): + #ttdb = db.get_tile_type(tile["type"]) + # FIXME: check BRAM + bitj = tile["bits"].get("CLB_IO_CLK", None) + if not bitj: + continue + verbose and print("Checking %s, type %s" % (tile_name, tile["type"])) + baseaddr = int(bitj["baseaddr"], 0) + bitbase = 32 * bitj["offset"] + + # Create tile mask + tile_used = dict() + def process(lparse): + tag, bits, mode = lparse + assert mode is None + for (bit_addroff, bit_bitoff) in bits: + tile_used[(baseaddr + bit_addroff, bitbase + bit_bitoff)] = "%s.%s" % (tile_name, tag) + process_db(tile["type"], process, verbose=verbose) + + # See if tile mask intersects any existing bits + for (waddr, bitaddr), tile_desc in tile_used.items(): + used_desc = used.get((waddr, bitaddr), None) + if used_desc: + raise ValueError("Collision at %08X:%04X: had %s, got %s" % (waddr, bitaddr, used_desc, tile_desc)) + used[(waddr, bitaddr)] = tile_desc + tiles_checked += 1 + print("Checked %s tiles, %s bits" % tiles_checked, len(used)) + + +def run(db_root, verbose=False): + # Start by running a basic check on db files + parsedb_all(db_root, verbose=verbose) + + ''' + # Now load and verify tile consistency + db = prjxraydb.Database(db_root) + db._read_tilegrid() + db._read_tile_types() + print(db.tile_types.keys()) + ''' + + check_seg_overlap(db_root, verbose=verbose) + + +def main(): + import argparse + + parser = argparse.ArgumentParser( + description="Parse a db repository, checking for consistency") + + util.db_root_arg(parser) + parser.add_argument('--verbose', action='store_true', help='') + args = parser.parse_args() + + run(args.db_root, verbose=args.verbose) + + +if __name__ == '__main__': + main() From 96ee4ac44983837848494c490a599d0fb3deccee Mon Sep 17 00:00:00 2001 From: John McMaster Date: Mon, 10 Dec 2018 14:42:53 -0800 Subject: [PATCH 6/7] checkdb: check tile overlap Signed-off-by: John McMaster --- prjxray/util.py | 54 +++++++++++++++++++++ utils/checkdb.py | 116 +++++++++++++++++++++++++++------------------- utils/segprint.py | 14 +----- 3 files changed, 124 insertions(+), 60 deletions(-) diff --git a/prjxray/util.py b/prjxray/util.py index 09377c83..0e8ba60d 100644 --- a/prjxray/util.py +++ b/prjxray/util.py @@ -96,3 +96,57 @@ def parse_db_line(line): # 100_319 assert re.match(r'[!]*[0-9]+_[0-9]+', bit), "Invalid bit: %s" % bit return tag, bits, None + + +def parse_tagbit(x): + # !30_07 + if x[0] == '!': + isset = False + numstr = x[1:] + else: + isset = True + numstr = x + frame, word = numstr.split("_") + # second part forms a tuple refereced in sets + return (isset, (int(frame, 10), int(word, 10))) + + +def gen_tile_bits(db_root, tilej, strict=False, verbose=False): + ''' + For given tile yield + (absolute address, absolute FDRI bit offset, tag) + + For each address space + Find applicable files + For each tag bit in those files, calculate absolute address and bit offsets + + Sample file names: + segbits_clbll_l.db + segbits_int_l.db + segbits_bram_l.block_ram.db + ''' + for block_type, blockj in tilej["bits"].items(): + baseaddr = int(blockj["baseaddr"], 0) + bitbase = 32 * blockj["offset"] + + if block_type == "CLB_IO_CLK": + fn = "%s/segbits_%s.db" % (db_root, tilej["type"].lower()) + else: + fn = "%s/segbits_%s.db.%s" % ( + db_root, tilej["type"].lower(), block_type.lower()) + # tilegrid runs a lot earlier than fuzzers + # may not have been created yet + verbose and print("Check %s: %s" % (fn, os.path.exists(fn))) + if strict: + assert os.path.exists(fn) + elif not os.path.exists(fn): + continue + + with open(fn, "r") as f: + for line in f: + tag, bits, mode = parse_db_line(line) + assert mode is None + for bitstr in bits: + # 31_06 + _bit_inv, (bit_addroff, bit_bitoff) = parse_tagbit(bitstr) + yield (baseaddr + bit_addroff, bitbase + bit_bitoff, tag) diff --git a/utils/checkdb.py b/utils/checkdb.py index 29bae1b4..4a586957 100644 --- a/utils/checkdb.py +++ b/utils/checkdb.py @@ -1,93 +1,114 @@ #!/usr/bin/env python3 - ''' Check: -Individual files are valid -No overlap between any tile + +TODO: +Can we use prjxray? +Relies on 074, which is too far into the process ''' -import sys, re from prjxray import util +from prjxray import db as prjxraydb +import os import parsedb #from prjxray import db as prjxraydb import glob +def make_tile_mask(db_root, tile_name, tilej, strict=False, verbose=False): + ''' + Return dict + key: (address, bit index) + val: sample description of where it came from (there may be multiple, only one) + ''' + + # FIXME: fix mask files https://github.com/SymbiFlow/prjxray/issues/301 + # in the meantime build them on the fly + # We may want this to build them anyway + + ret = dict() + for absaddr, bitaddr, tag in util.gen_tile_bits( + db_root, tilej, strict=strict, verbose=verbose): + name = "%s.%s" % (tile_name, tag) + ret.setdefault((absaddr, bitaddr), name) + return ret + + def parsedb_all(db_root, verbose=False): '''Verify .db files are individually valid''' - for bit_fn in glob.glob('%s/segbits_*.db'): + files = 0 + for bit_fn in glob.glob('%s/segbits_*.db' % db_root): + verbose and print("Checking %s" % bit_fn) parsedb.run(bit_fn, fnout=None, strict=True, verbose=verbose) + files += 1 + print("segbits_*.db: %d okay" % files) - for bit_fn in glob.glob('%s/mask_*.db'): + files = 0 + for bit_fn in glob.glob('%s/mask_*.db' % db_root): + verbose and print("Checking %s" % bit_fn) parsedb.run(bit_fn, fnout=None, strict=True, verbose=verbose) - -def process_db(tile_type, process, verbose): - #ttdb = db.get_tile_type(tile_type) - - fns = [ttdb.tile_dbs.segbits, ttdb.tile_dbs.ppips] - verbose and print("process_db(%s): %s" % (tile_type, fns)) - for fn in fns: - if fn: - with open(fn, "r") as f: - for line in f: - process(util.parse_db_line(line)) + files += 1 + print("mask_*.db: %d okay" % files) -def check_seg_overlap(db_root, verbose=False): +def check_tile_overlap(db, db_root, strict=False, verbose=False): ''' + Verifies that no two tiles use the same bit + Assume .db files are individually valid Create a mask for all the bits the tile type uses For each tile, create bitmasks over the entire bitstream for current part Throw an exception if two tiles share an address ''' - # key: (address, bit index) - # val: sample description of where it came from (there may be multiple, only one) - used = dict() + mall = dict() tiles_checked = 0 - for tile_name, tile in db.tilegrid.items(): - #ttdb = db.get_tile_type(tile["type"]) - # FIXME: check BRAM - bitj = tile["bits"].get("CLB_IO_CLK", None) - if not bitj: + for tile_name, tilej in db.tilegrid.items(): + #for tile_name, tilej in [("CLBLL_L_X12Y138", db.tilegrid["CLBLL_L_X12Y138"])]: + mtile = make_tile_mask( + db_root, tile_name, tilej, strict=strict, verbose=verbose) + verbose and print( + "Checking %s, type %s, bits: %s" % + (tile_name, tilej["type"], len(mtile))) + if len(mtile) == 0: continue - verbose and print("Checking %s, type %s" % (tile_name, tile["type"])) - baseaddr = int(bitj["baseaddr"], 0) - bitbase = 32 * bitj["offset"] - # Create tile mask - tile_used = dict() - def process(lparse): - tag, bits, mode = lparse - assert mode is None - for (bit_addroff, bit_bitoff) in bits: - tile_used[(baseaddr + bit_addroff, bitbase + bit_bitoff)] = "%s.%s" % (tile_name, tag) - process_db(tile["type"], process, verbose=verbose) - - # See if tile mask intersects any existing bits - for (waddr, bitaddr), tile_desc in tile_used.items(): - used_desc = used.get((waddr, bitaddr), None) - if used_desc: - raise ValueError("Collision at %08X:%04X: had %s, got %s" % (waddr, bitaddr, used_desc, tile_desc)) - used[(waddr, bitaddr)] = tile_desc + collisions = set(mall.keys()).intersection(set(mtile.keys())) + if collisions: + print("ERROR: %s collisions" % len(collisions)) + for ck in sorted(collisions): + addr, bitaddr = ck + print( + " %08X_%04X: had %s, got %s" % + (addr, bitaddr, mall[ck], mtile[ck])) + raise ValueError("%s collisions" % len(collisions)) + mall.update(mtile) tiles_checked += 1 - print("Checked %s tiles, %s bits" % tiles_checked, len(used)) + print("Checked %s tiles, %s bits" % (tiles_checked, len(mall))) -def run(db_root, verbose=False): +def run(db_root, strict=False, verbose=False): # Start by running a basic check on db files + print("Checking individual .db...") parsedb_all(db_root, verbose=verbose) - ''' # Now load and verify tile consistency db = prjxraydb.Database(db_root) db._read_tilegrid() + ''' + these don't load properly without .json files + See: https://github.com/SymbiFlow/prjxray/issues/303 db._read_tile_types() print(db.tile_types.keys()) ''' - check_seg_overlap(db_root, verbose=verbose) + verbose and print("") + + print("Checking aggregate dir...") + check_tile_overlap(db, db_root, strict=strict, verbose=verbose) def main(): @@ -97,10 +118,11 @@ def main(): description="Parse a db repository, checking for consistency") util.db_root_arg(parser) + parser.add_argument('--strict', action='store_true', help='') parser.add_argument('--verbose', action='store_true', help='') args = parser.parse_args() - run(args.db_root, verbose=args.verbose) + run(args.db_root, strict=args.strict, verbose=args.verbose) if __name__ == '__main__': diff --git a/utils/segprint.py b/utils/segprint.py index fabad99b..18819b2c 100755 --- a/utils/segprint.py +++ b/utils/segprint.py @@ -46,22 +46,10 @@ def get_database(db, tile_type, verbose=False): parts = l.split() name = parts[0] - def parsetag(x): - # !30_07 - if x[0] == '!': - isset = False - numstr = x[1:] - else: - isset = True - numstr = x - frame, word = numstr.split("_") - # second part forms a tuple refereced in sets - return (isset, (int(frame, 10), int(word, 10))) - if parts[1] == 'always' or parts[1] == 'hint': tagbits = [] else: - tagbits = [parsetag(x) for x in parts[1:]] + tagbits = [util.parse_tagbit(x) for x in parts[1:]] tags.append(list([name] + tagbits)) From 77c52611306f6c728a0be709f752966dc6fcffd2 Mon Sep 17 00:00:00 2001 From: John McMaster Date: Mon, 10 Dec 2018 14:54:40 -0800 Subject: [PATCH 7/7] checkdb: print in standard address format Signed-off-by: John McMaster --- prjxray/util.py | 13 +++++++++++++ utils/checkdb.py | 13 ++++++++++--- 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/prjxray/util.py b/prjxray/util.py index 0e8ba60d..1c5621be 100644 --- a/prjxray/util.py +++ b/prjxray/util.py @@ -111,6 +111,19 @@ def parse_tagbit(x): return (isset, (int(frame, 10), int(word, 10))) +def addr_bit2word(bitaddr): + word = bitaddr // 32 + bit = bitaddr % 32 + return word, bit + + +def addr2str(addr, word, bit): + # Make like .bits file: bit_00020b14_073_05 + # also similar to .db file: CLBLL_L.SLICEL_X0.CEUSEDMUX 01_39 + assert 0 <= bit <= 31 + return "%08x_%03u_%02u" % (addr, word, bit) + + def gen_tile_bits(db_root, tilej, strict=False, verbose=False): ''' For given tile yield diff --git a/utils/checkdb.py b/utils/checkdb.py index 4a586957..425cf4d6 100644 --- a/utils/checkdb.py +++ b/utils/checkdb.py @@ -66,8 +66,14 @@ def check_tile_overlap(db, db_root, strict=False, verbose=False): mall = dict() tiles_checked = 0 + + def subtiles(tile_names): + for tile_name in tile_names: + yield tile_name, db.tilegrid[tile_name] + for tile_name, tilej in db.tilegrid.items(): - #for tile_name, tilej in [("CLBLL_L_X12Y138", db.tilegrid["CLBLL_L_X12Y138"])]: + # for tile_name, tilej in subtiles(["CLBLL_L_X14Y112", "INT_L_X14Y112"]): + mtile = make_tile_mask( db_root, tile_name, tilej, strict=strict, verbose=verbose) verbose and print( @@ -81,9 +87,10 @@ def check_tile_overlap(db, db_root, strict=False, verbose=False): print("ERROR: %s collisions" % len(collisions)) for ck in sorted(collisions): addr, bitaddr = ck + word, bit = util.addr_bit2word(bitaddr) print( - " %08X_%04X: had %s, got %s" % - (addr, bitaddr, mall[ck], mtile[ck])) + " %s: had %s, got %s" % + (util.addr2str(addr, word, bit), mall[ck], mtile[ck])) raise ValueError("%s collisions" % len(collisions)) mall.update(mtile) tiles_checked += 1