From 014462de26857497f5a689cde4e6899118331fc1 Mon Sep 17 00:00:00 2001 From: Maciej Kurc Date: Thu, 13 Feb 2020 13:47:32 +0100 Subject: [PATCH] Ported tag grouping to dbfixup.py Signed-off-by: Maciej Kurc --- fuzzers/030-iob/Makefile | 3 +- utils/dbfixup.py | 193 ++++++++++++++++++++++++++++++++++++--- 2 files changed, 181 insertions(+), 15 deletions(-) diff --git a/fuzzers/030-iob/Makefile b/fuzzers/030-iob/Makefile index a18dcbd6..72e26007 100644 --- a/fuzzers/030-iob/Makefile +++ b/fuzzers/030-iob/Makefile @@ -13,8 +13,7 @@ build/segbits_xiob33.rdb: $(SPECIMENS_OK) build/segbits_xiob33.db: build/segbits_xiob33.rdb process_rdb.py bits.dbf python3 process_rdb.py build/segbits_xiob33.rdb > build/segbits_xiob33_processed.rdb - python3 ${XRAY_DIR}/utils/group.py -g tag_groups.txt -i build/segbits_xiob33_processed.rdb -o build/segbits_xiob33_processed.rdb2 - ${XRAY_DBFIXUP} --db-root build --zero-db bits.dbf --seg-fn-in build/segbits_xiob33_processed.rdb2 --seg-fn-out $@ + ${XRAY_DBFIXUP} --db-root build --zero-db bits.dbf --groups tag_groups.txt --seg-fn-in build/segbits_xiob33_processed.rdb --seg-fn-out $@ ${XRAY_MASKMERGE} build/mask_xiob33.db $$(find -name segdata_liob33.txt) $$(find -name segdata_riob33.txt) build/segbits_hclk_ioi3.rdb: $(SPECIMENS_OK) diff --git a/utils/dbfixup.py b/utils/dbfixup.py index cac5e21d..3d74cf14 100755 --- a/utils/dbfixup.py +++ b/utils/dbfixup.py @@ -1,6 +1,7 @@ #/usr/bin/env python3 import sys, os, re +import itertools from prjxray import util clb_int_zero_db = [ @@ -211,6 +212,8 @@ def read_segbits(fn_in): # Hack: skip duplicate lines # This happens while merging a new multibit entry line = line.strip() + if len(line) == 0: + continue if line == llast: continue @@ -219,7 +222,8 @@ def read_segbits(fn_in): return lines -def add_zero_bits(fn_in, lines, zero_db, clb_int=False, strict=True, verbose=False): +def add_zero_bits( + fn_in, lines, zero_db, clb_int=False, strict=True, verbose=False): ''' Add multibit entries This requires adding some zero bits (ex: !31_09) @@ -376,7 +380,7 @@ def remove_ambiguous_solutions(fn_in, db_lines, strict=True, verbose=True): return 0, db_lines drops = 0 - output_lines = [] + output_lines = set() for l in db_lines: parts = l.split() @@ -384,7 +388,7 @@ def remove_ambiguous_solutions(fn_in, db_lines, strict=True, verbose=True): bits = frozenset(parts[1:]) if bits not in dropped_solutions: - output_lines.append(l) + output_lines.add(l) drops += 1 else: if verbose: @@ -397,8 +401,81 @@ def remove_ambiguous_solutions(fn_in, db_lines, strict=True, verbose=True): return drops, output_lines +def group_tags(lines, tag_groups, bit_groups): + """ + Implements tag grouping. If a tag belongs to a group then the common bits + of that group are added to is as zeros. + + >>> tg = [{"A", "B"}] + >>> bg = [{(1, 2), (3, 4)}] + >>> res = group_tags({"A 1_2", "B 3_4"}, tg, bg) + >>> (res[0], sorted(list(res[1]))) + (2, ['A 1_2 !3_4', 'B !1_2 3_4']) + + >>> tg = [{"A", "B"}] + >>> bg = [{(1, 2), (3, 4)}] + >>> res = group_tags({"A 1_2", "B 3_4", "C 1_2"}, tg, bg) + >>> (res[0], sorted(list(res[1]))) + (2, ['A 1_2 !3_4', 'B !1_2 3_4', 'C 1_2']) + """ + + changes = 0 + new_lines = set() + + # Process lines + for line in lines: + + line = line.strip() + if not len(line): + continue + + # Parse the line + tag, bits, mode, _ = util.parse_db_line(line) + if not bits: + bits = set() + else: + bits = set([util.parse_tagbit(b) for b in bits]) + + # Check if the tag belongs to a group + for tag_group, bit_group in zip(tag_groups, bit_groups): + if tag in tag_group: + + # Add zero bits to the tag if not already there + bit_coords = set([b[1] for b in bits]) + for zero_bit in bit_group: + if zero_bit not in bit_coords: + bits.add((False, zero_bit)) + + # Format the line + bit_strs = [] + for bit in sorted(list(bits), key=lambda b: b[1]): + s = "!" if not bit[0] else "" + s += "{}_{}".format(bit[1][0], bit[1][1]) + bit_strs.append(s) + + new_line = " ".join([tag] + bit_strs) + + # Add the line + new_lines.add(new_line) + changes += 1 + break + + # It does not, pass it through unchanged + else: + new_lines.add(line) + + return changes, new_lines + + def update_seg_fns( - fn_inouts, zero_db, clb_int, lazy=False, strict=True, verbose=False): + fn_inouts, + zero_db, + tag_groups, + clb_int, + lazy=False, + strict=True, + verbose=False): + seg_files = 0 seg_lines = 0 for fn_in, fn_out in fn_inouts: @@ -406,28 +483,34 @@ def update_seg_fns( if lazy and not os.path.exists(fn_in): continue - lines = read_segbits(fn_in) + org_lines = read_segbits(fn_in) - changes, new_lines = add_zero_bits( + changes, lines = add_zero_bits( fn_in, - lines, + org_lines, zero_db, clb_int=clb_int, strict=strict, - verbose=verbose - ) + verbose=verbose) - new_changes, final_lines = remove_ambiguous_solutions( + new_changes, new_lines = remove_ambiguous_solutions( fn_in, - new_lines, + lines, strict=strict, verbose=verbose, ) + changes += new_changes + lines |= new_lines + # Find common bits for tag groups + bit_groups = find_common_bits_for_tag_groups(lines, tag_groups) + + # Group tags + new_changes, lines = group_tags(lines, tag_groups, bit_groups) changes += new_changes with open(fn_out, "w") as f: - for line in sorted(final_lines): + for line in sorted(lines): print(line, file=f) if changes is not None: @@ -471,6 +554,7 @@ def update_segs( seg_fn_in, seg_fn_out, zero_db_fn, + tag_groups, strict=True, verbose=False): if clb_int: @@ -497,7 +581,74 @@ def update_segs( print("CLB INT mode: %s" % clb_int) print("Segbit groups: %s" % len(zero_db)) update_seg_fns( - fn_inouts, zero_db, clb_int, lazy=lazy, strict=strict, verbose=verbose) + fn_inouts, + zero_db, + tag_groups, + clb_int, + lazy=lazy, + strict=strict, + verbose=verbose) + + +def find_common_bits_for_tag_groups(lines, tag_groups): + """ + For each tag group finds a common set of bits that have value of one. + """ + + bit_groups = [] + + for tag_group in tag_groups: + bit_group = set() + + for line in lines: + tag, bits, mode, _ = util.parse_db_line(line) + if not bits: + continue + + bits = set([util.parse_tagbit(b) for b in bits]) + + if tag in tag_group and len(bits): + ones = set([b[1] for b in bits if b[0]]) + bit_group |= ones + + print(tag_group) + print(bit_group) + bit_groups.append(bit_group) + + return bit_groups + + +def load_tag_groups(file_name): + """ + Loads tag groups from a text file. + + A tag group is defined by specifying a space separated list of tags within + a single line. Lines that are empty or start with '#' are ignored. + """ + tag_groups = [] + + # Load tag group specifications + with open(file_name, "r") as fp: + for line in fp: + line = line.strip() + + if len(line) == 0 or line.startswith("#"): + continue + + group = set(line.split()) + if len(group): + tag_groups.append(group) + + # Check if all tag groups are exclusive + for tag_group_a, tag_group_b in itertools.combinations(tag_groups, 2): + + tags = tag_group_a & tag_group_b + if len(tags): + raise RuntimeError( + "Tag(s) {} are present in multiple groups".format( + " ".join(tags))) + + return tag_groups def run( @@ -506,12 +657,18 @@ def run( zero_db_fn=None, seg_fn_in=None, seg_fn_out=None, + groups_fn_in=None, strict=None, verbose=False): if strict is None: strict = not clb_int + # Load tag groups + tag_groups = [] + if groups_fn_in is not None: + tag_groups = load_tag_groups(groups_fn_in) + # Probably should split this into two programs update_segs( db_root, @@ -519,6 +676,7 @@ def run( seg_fn_in=seg_fn_in, seg_fn_out=seg_fn_out, zero_db_fn=zero_db_fn, + tag_groups=tag_groups, strict=strict, verbose=verbose) if clb_int: @@ -538,6 +696,14 @@ def main(): parser.add_argument('--seg-fn-in', help='') parser.add_argument('--seg-fn-out', help='') util.add_bool_arg(parser, "--strict", default=False) + + parser.add_argument( + "-g", + "--groups", + type=str, + default=None, + help="Input tag group definition file") + args = parser.parse_args() run( @@ -546,6 +712,7 @@ def main(): args.zero_db, args.seg_fn_in, args.seg_fn_out, + args.groups, strict=args.strict, verbose=args.verbose)