diff --git a/docs/format/db.rst b/docs/format/db.rst index 668e894a..6c5885f4 100644 --- a/docs/format/db.rst +++ b/docs/format/db.rst @@ -47,7 +47,8 @@ Related tools: * dbfixup.py: internal tool that expands multi-bit encodings (ex: one hot) into groups. For example: * .rdb file with one hot: BRAM.RAMB18_Y1.WRITE_WIDTH_A_18 27_267 * .db: file expanded: BRAM.RAMB18_Y1.WRITE_WIDTH_A_18 !27_268 !27_269 27_267 - * dbcheck.py: valides that a database is fully and consistently solved + * parsedb.py: valides that a database is fully and consistently solved + * Optionally outputs to canonical form * Ex: complains if const0 entries exist * Ex: complains if symbols are duplicated (such as after a mergedb after rename) * mergedb.sh: adds new bit entries to an existing db diff --git a/prjxray/db.py b/prjxray/db.py index 200bf0a0..4a886e17 100644 --- a/prjxray/db.py +++ b/prjxray/db.py @@ -33,6 +33,7 @@ class Database(object): """ self.db_root = db_root + # tilegrid.json JSON object self.tilegrid = None self.tileconn = None self.tile_types = None diff --git a/prjxray/util.py b/prjxray/util.py index 09377c83..1c5621be 100644 --- a/prjxray/util.py +++ b/prjxray/util.py @@ -96,3 +96,70 @@ def parse_db_line(line): # 100_319 assert re.match(r'[!]*[0-9]+_[0-9]+', bit), "Invalid bit: %s" % bit return tag, bits, None + + +def parse_tagbit(x): + # !30_07 + if x[0] == '!': + isset = False + numstr = x[1:] + else: + isset = True + numstr = x + frame, word = numstr.split("_") + # second part forms a tuple refereced in sets + return (isset, (int(frame, 10), int(word, 10))) + + +def addr_bit2word(bitaddr): + word = bitaddr // 32 + bit = bitaddr % 32 + return word, bit + + +def addr2str(addr, word, bit): + # Make like .bits file: bit_00020b14_073_05 + # also similar to .db file: CLBLL_L.SLICEL_X0.CEUSEDMUX 01_39 + assert 0 <= bit <= 31 + return "%08x_%03u_%02u" % (addr, word, bit) + + +def gen_tile_bits(db_root, tilej, strict=False, verbose=False): + ''' + For given tile yield + (absolute address, absolute FDRI bit offset, tag) + + For each address space + Find applicable files + For each tag bit in those files, calculate absolute address and bit offsets + + Sample file names: + segbits_clbll_l.db + segbits_int_l.db + segbits_bram_l.block_ram.db + ''' + for block_type, blockj in tilej["bits"].items(): + baseaddr = int(blockj["baseaddr"], 0) + bitbase = 32 * blockj["offset"] + + if block_type == "CLB_IO_CLK": + fn = "%s/segbits_%s.db" % (db_root, tilej["type"].lower()) + else: + fn = "%s/segbits_%s.db.%s" % ( + db_root, tilej["type"].lower(), block_type.lower()) + # tilegrid runs a lot earlier than fuzzers + # may not have been created yet + verbose and print("Check %s: %s" % (fn, os.path.exists(fn))) + if strict: + assert os.path.exists(fn) + elif not os.path.exists(fn): + continue + + with open(fn, "r") as f: + for line in f: + tag, bits, mode = parse_db_line(line) + assert mode is None + for bitstr in bits: + # 31_06 + _bit_inv, (bit_addroff, bit_bitoff) = parse_tagbit(bitstr) + yield (baseaddr + bit_addroff, bitbase + bit_bitoff, tag) diff --git a/utils/checkdb.py b/utils/checkdb.py new file mode 100644 index 00000000..425cf4d6 --- /dev/null +++ b/utils/checkdb.py @@ -0,0 +1,136 @@ +#!/usr/bin/env python3 +''' +Check: +-Individual files are valid +-No overlap between any tile + +TODO: +Can we use prjxray? +Relies on 074, which is too far into the process +''' + +from prjxray import util +from prjxray import db as prjxraydb +import os +import parsedb +#from prjxray import db as prjxraydb +import glob + + +def make_tile_mask(db_root, tile_name, tilej, strict=False, verbose=False): + ''' + Return dict + key: (address, bit index) + val: sample description of where it came from (there may be multiple, only one) + ''' + + # FIXME: fix mask files https://github.com/SymbiFlow/prjxray/issues/301 + # in the meantime build them on the fly + # We may want this to build them anyway + + ret = dict() + for absaddr, bitaddr, tag in util.gen_tile_bits( + db_root, tilej, strict=strict, verbose=verbose): + name = "%s.%s" % (tile_name, tag) + ret.setdefault((absaddr, bitaddr), name) + return ret + + +def parsedb_all(db_root, verbose=False): + '''Verify .db files are individually valid''' + + files = 0 + for bit_fn in glob.glob('%s/segbits_*.db' % db_root): + verbose and print("Checking %s" % bit_fn) + parsedb.run(bit_fn, fnout=None, strict=True, verbose=verbose) + files += 1 + print("segbits_*.db: %d okay" % files) + + files = 0 + for bit_fn in glob.glob('%s/mask_*.db' % db_root): + verbose and print("Checking %s" % bit_fn) + parsedb.run(bit_fn, fnout=None, strict=True, verbose=verbose) + files += 1 + print("mask_*.db: %d okay" % files) + + +def check_tile_overlap(db, db_root, strict=False, verbose=False): + ''' + Verifies that no two tiles use the same bit + + Assume .db files are individually valid + Create a mask for all the bits the tile type uses + For each tile, create bitmasks over the entire bitstream for current part + Throw an exception if two tiles share an address + ''' + mall = dict() + + tiles_checked = 0 + + def subtiles(tile_names): + for tile_name in tile_names: + yield tile_name, db.tilegrid[tile_name] + + for tile_name, tilej in db.tilegrid.items(): + # for tile_name, tilej in subtiles(["CLBLL_L_X14Y112", "INT_L_X14Y112"]): + + mtile = make_tile_mask( + db_root, tile_name, tilej, strict=strict, verbose=verbose) + verbose and print( + "Checking %s, type %s, bits: %s" % + (tile_name, tilej["type"], len(mtile))) + if len(mtile) == 0: + continue + + collisions = set(mall.keys()).intersection(set(mtile.keys())) + if collisions: + print("ERROR: %s collisions" % len(collisions)) + for ck in sorted(collisions): + addr, bitaddr = ck + word, bit = util.addr_bit2word(bitaddr) + print( + " %s: had %s, got %s" % + (util.addr2str(addr, word, bit), mall[ck], mtile[ck])) + raise ValueError("%s collisions" % len(collisions)) + mall.update(mtile) + tiles_checked += 1 + print("Checked %s tiles, %s bits" % (tiles_checked, len(mall))) + + +def run(db_root, strict=False, verbose=False): + # Start by running a basic check on db files + print("Checking individual .db...") + parsedb_all(db_root, verbose=verbose) + + # Now load and verify tile consistency + db = prjxraydb.Database(db_root) + db._read_tilegrid() + ''' + these don't load properly without .json files + See: https://github.com/SymbiFlow/prjxray/issues/303 + db._read_tile_types() + print(db.tile_types.keys()) + ''' + + verbose and print("") + + print("Checking aggregate dir...") + check_tile_overlap(db, db_root, strict=strict, verbose=verbose) + + +def main(): + import argparse + + parser = argparse.ArgumentParser( + description="Parse a db repository, checking for consistency") + + util.db_root_arg(parser) + parser.add_argument('--strict', action='store_true', help='') + parser.add_argument('--verbose', action='store_true', help='') + args = parser.parse_args() + + run(args.db_root, strict=args.strict, verbose=args.verbose) + + +if __name__ == '__main__': + main() diff --git a/utils/dbcheck.py b/utils/dbcheck.py deleted file mode 100755 index fe62bf5c..00000000 --- a/utils/dbcheck.py +++ /dev/null @@ -1,47 +0,0 @@ -#!/usr/bin/env python3 - -import sys, re - -database = dict() -database_r = dict() - -for arg in sys.argv[1:]: - with open(arg, "r") as f: - for line in f: - if "<" in line: - raise Exception("Found '<' in this line: %s" % line) - - line = line.split() - key = line[0] - bits = tuple(sorted(set(line[1:]))) - - if key in database: - print("Warning: Duplicate key: %s %s" % (key, bits)) - - if bits in database_r: - print("Warning: Duplicate bits: %s %s" % (key, bits)) - - database[key] = bits - database_r[bits] = key - - -def get_subsets(bits): - retval = list() - retval.append(bits) - for i in range(len(bits)): - for s in get_subsets(bits[i + 1:]): - retval.append(bits[0:i] + s) - return retval - - -def check_subsets(bits): - for sub_bits in sorted(get_subsets(bits)): - if sub_bits != bits and sub_bits != (): - if sub_bits in database_r: - print( - "Warning: Entry %s %s is a subset of entry %s %s." % - (database_r[sub_bits], sub_bits, database_r[bits], bits)) - - -for key, bits in database.items(): - check_subsets(bits) diff --git a/utils/dbcheck.sh b/utils/dbcheck.sh deleted file mode 100755 index 3bf2550a..00000000 --- a/utils/dbcheck.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash -python3 ${XRAY_UTILS_DIR}/dbcheck.py ${XRAY_DATABASE_DIR}/${XRAY_DATABASE}/segbits_{clblm,int}_l.db -python3 ${XRAY_UTILS_DIR}/dbcheck.py ${XRAY_DATABASE_DIR}/${XRAY_DATABASE}/segbits_{clblm,int}_r.db -python3 ${XRAY_UTILS_DIR}/dbcheck.py ${XRAY_DATABASE_DIR}/${XRAY_DATABASE}/segbits_{clbll,int}_l.db -python3 ${XRAY_UTILS_DIR}/dbcheck.py ${XRAY_DATABASE_DIR}/${XRAY_DATABASE}/segbits_{clbll,int}_r.db -python3 ${XRAY_UTILS_DIR}/dbcheck.py ${XRAY_DATABASE_DIR}/${XRAY_DATABASE}/segbits_hclk_l.db -python3 ${XRAY_UTILS_DIR}/dbcheck.py ${XRAY_DATABASE_DIR}/${XRAY_DATABASE}/segbits_hclk_r.db diff --git a/utils/environment.sh b/utils/environment.sh index da87ff44..73cc2a0a 100644 --- a/utils/environment.sh +++ b/utils/environment.sh @@ -19,7 +19,6 @@ export XRAY_GENHEADER="${XRAY_UTILS_DIR}/genheader.sh" export XRAY_BITREAD="${XRAY_TOOLS_DIR}/bitread --part_file ${XRAY_PART_YAML}" export XRAY_MERGEDB="bash ${XRAY_UTILS_DIR}/mergedb.sh" export XRAY_DBFIXUP="python3 ${XRAY_UTILS_DIR}/dbfixup.py" -export XRAY_DBCHECK="bash ${XRAY_UTILS_DIR}/dbcheck.sh" export XRAY_MASKMERGE="bash ${XRAY_UTILS_DIR}/maskmerge.sh" export XRAY_SEGMATCH="${XRAY_TOOLS_DIR}/segmatch" export XRAY_SEGPRINT="python3 ${XRAY_UTILS_DIR}/segprint.py" diff --git a/utils/mergedb.sh b/utils/mergedb.sh index 90aca2b0..e653fb8f 100755 --- a/utils/mergedb.sh +++ b/utils/mergedb.sh @@ -18,6 +18,12 @@ function finish { trap finish EXIT db=$XRAY_DATABASE_DIR/$XRAY_DATABASE/segbits_$1.db +# Check existing DB +if [ -f $db ] ; then + ${XRAY_PARSEDB} --strict "$db" +fi +# Check new DB +${XRAY_PARSEDB} --strict "$2" # Fuzzers verify L/R data is equivilent # However, expand back to L/R to make downstream tools not depend on this @@ -78,5 +84,6 @@ esac touch "$db" sort -u "$tmp1" "$db" | grep -v '<.*>' > "$tmp2" || true +# Check aggregate db for consistency and make canonical ${XRAY_PARSEDB} --strict "$tmp2" "$db" diff --git a/utils/parsedb.py b/utils/parsedb.py index b24ad42f..5d6af833 100755 --- a/utils/parsedb.py +++ b/utils/parsedb.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 +import sys, re from prjxray import util @@ -41,7 +42,7 @@ def main(): import argparse parser = argparse.ArgumentParser( - description="Parse a db, check for consistency") + description="Parse a db file, checking for consistency") util.db_root_arg(parser) parser.add_argument('--verbose', action='store_true', help='') diff --git a/utils/segprint.py b/utils/segprint.py index fabad99b..18819b2c 100755 --- a/utils/segprint.py +++ b/utils/segprint.py @@ -46,22 +46,10 @@ def get_database(db, tile_type, verbose=False): parts = l.split() name = parts[0] - def parsetag(x): - # !30_07 - if x[0] == '!': - isset = False - numstr = x[1:] - else: - isset = True - numstr = x - frame, word = numstr.split("_") - # second part forms a tuple refereced in sets - return (isset, (int(frame, 10), int(word, 10))) - if parts[1] == 'always' or parts[1] == 'hint': tagbits = [] else: - tagbits = [parsetag(x) for x in parts[1:]] + tagbits = [util.parse_tagbit(x) for x in parts[1:]] tags.append(list([name] + tagbits))