From b20bae5341f959bcb9bd899494453509e7a4fc8d Mon Sep 17 00:00:00 2001 From: Maciej Kurc Date: Fri, 20 Dec 2019 13:03:31 +0100 Subject: [PATCH 1/6] Added grouping of IN_TERM features so they can be decoded unambigosly. Signed-off-by: Maciej Kurc --- fuzzers/030-iob/Makefile | 3 +- fuzzers/030-iob/bits.dbf | 4 - fuzzers/030-iob/group.py | 220 +++++++++++++++++++++++++++++++++ fuzzers/030-iob/tag_groups.txt | 5 + 4 files changed, 227 insertions(+), 5 deletions(-) create mode 100755 fuzzers/030-iob/group.py create mode 100644 fuzzers/030-iob/tag_groups.txt diff --git a/fuzzers/030-iob/Makefile b/fuzzers/030-iob/Makefile index bb17e295..ab0a0168 100644 --- a/fuzzers/030-iob/Makefile +++ b/fuzzers/030-iob/Makefile @@ -13,7 +13,8 @@ build/segbits_xiob33.rdb: $(SPECIMENS_OK) build/segbits_xiob33.db: build/segbits_xiob33.rdb process_rdb.py bits.dbf python3 process_rdb.py build/segbits_xiob33.rdb > build/segbits_xiob33_processed.rdb - ${XRAY_DBFIXUP} --db-root build --zero-db bits.dbf --seg-fn-in build/segbits_xiob33_processed.rdb --seg-fn-out $@ + python3 group.py -g tag_groups.txt -i build/segbits_xiob33_processed.rdb -o build/segbits_xiob33_processed.rdb2 + ${XRAY_DBFIXUP} --db-root build --zero-db bits.dbf --seg-fn-in build/segbits_xiob33_processed.rdb2 --seg-fn-out $@ ${XRAY_MASKMERGE} build/mask_xiob33.db $$(find -name segdata_liob33.txt) $$(find -name segdata_riob33.txt) build/segbits_hclk_ioi3.rdb: $(SPECIMENS_OK) diff --git a/fuzzers/030-iob/bits.dbf b/fuzzers/030-iob/bits.dbf index 09d2e838..e69de29b 100644 --- a/fuzzers/030-iob/bits.dbf +++ b/fuzzers/030-iob/bits.dbf @@ -1,4 +0,0 @@ -38_92 39_93 38_94,IOB33.IOB_Y0.PULLTYPE.PULLDOWN -38_106 39_107 39_111 38_106 38_110 39_105 39_109,IOB33.IOB_Y0.SLEW.FAST -39_33 38_34 39_35,IOB33.IOB_Y1.PULLTYPE.PULLDOWN -39_21 38_16 38_20 38_18 38_22 39_17,IOB33.IOB_Y1.SLEW.FAST diff --git a/fuzzers/030-iob/group.py b/fuzzers/030-iob/group.py new file mode 100755 index 00000000..8a27cf18 --- /dev/null +++ b/fuzzers/030-iob/group.py @@ -0,0 +1,220 @@ +#!/usr/bin/env python3 +""" + +This script Reads tag group definition from a file and applies the tag grouping. +First a set of common bits for each group is found (logical OR among all tags +belonging to the group). Then in each tag belonging to the group those bits are +set to 0 but only if they are not already present there. + +The resulting data is written into a segbits file. +""" +import argparse +import re +import itertools + +# ============================================================================= + + +def load_tag_groups(file_name): + """ + Loads tag groups from a text file. + + A tag group is defined by specifying a space separated list of tags within + a single line. Lines that are empty or start with '#' are ignored. + """ + tag_groups = [] + + # Load tag group specifications + with open(file_name, "r") as fp: + for line in fp: + line = line.strip() + + if len(line) == 0 or line.startswith("#"): + continue + + group = set(line.split()) + if len(group): + tag_groups.append(group) + + # Check if all tag groups are exclusive + for tag_group_a, tag_group_b in itertools.combinations(tag_groups, 2): + + tags = tag_group_a & tag_group_b + if len(tags): + raise RuntimeError( + "Tag(s) {} are present in multiple groups".format( + " ".join(tags))) + + return tag_groups + + +# ============================================================================= + + +def parse_bit(bit): + """ + Decodes string describing a bit. Returns a tuple (frame, bit, value) + """ + match = re.match("^(!?)([0-9]+)_([0-9]+)$", bit) + assert match != None, bit + + val = int(match.group(1) != "!") + frm = int(match.group(2)) + bit = int(match.group(3)) + + return frm, bit, val + + +def bit_to_str(bit): + """ + Converts a tuple (frame, bit, value) to its string representation. + """ + s = "!" if not bit[2] else "" + return "{}{}_{:02d}".format(s, bit[0], bit[1]) + + +def load_segbits(file_name): + """ + Loads a segbits file. + """ + + segbits = {} + + with open(file_name, "r") as fp: + for line in fp: + line = line.strip() + fields = line.split() + + if len(fields) < 2: + raise RuntimeError("Malformed line: '%s'" % line) + + tag = fields[0] + + if "<" in line or ">" in line: + segbits[tag] = " ".join(fields[1:]) + + else: + bits = set([parse_bit(bit) for bit in fields[1:]]) + segbits[tag] = bits + + return segbits + + +def save_segbits(file_name, segbits): + """ + Save segbits to a .db or .rdb file + """ + + with open(file_name, "w") as fp: + for tag, bits in segbits.items(): + + if isinstance(bits, str): + line = tag + " " + bits + + elif isinstance(bits, set): + line = tag + " " + line += " ".join( + [bit_to_str(bit) for bit in sorted(list(bits))]) + + fp.write(line + "\n") + + +# ============================================================================= + + +def mask_out_bits(segbits, mask, tags_to_mask=None): + """ + Given a set of bits and a list of tags to affect (optional) removes all + the bits from each tag that are present (and equal) in the masking set. + """ + + if tags_to_mask is None: + tags_to_mask = segbits.keys() + + # Mask out matching bits + for tag in tags_to_mask: + bits = segbits[tag] + + bits = set(bits) - set(mask) + segbits[tag] = bits + + return segbits + + +def find_common_bits_for_tag_groups(segbits, tag_groups): + """ + For each tag group finds a common set of bits that have value of one. + """ + + bit_groups = [] + + for tag_group in tag_groups: + bit_group = set() + + for tag, bits in segbits.items(): + if tag in tag_group and isinstance(bits, set): + ones = set([b for b in bits if b[2]]) + bit_group |= ones + + bit_groups.append(bit_group) + + return bit_groups + + +def group_tags(segbits, tag_groups, bit_groups): + """ + Implements tag grouping. If a tag belongs to a group then the common bits + of that group are added to is as zeros. + """ + + for tag_group, bit_group in zip(tag_groups, bit_groups): + zeros = set([(b[0], b[1], 0) for b in bit_group]) + for tag in tag_group: + + # Insert zero bits + if tag in segbits.keys(): + bits = segbits[tag] + + if not isinstance(bits, set): + bits = set() + segbits[tag] = bits + + for z in zeros: + if not (z[0], z[1], 1) in bits: + bits.add(z) + + return segbits + + +# ============================================================================= + + +def main(): + + # Parse arguments + parser = argparse.ArgumentParser() + + parser.add_argument("-i", required=True, type=str, help="Input .rdb file") + parser.add_argument( + "-g", required=True, type=str, help="Input tag group definition file") + parser.add_argument("-o", required=True, type=str, help="Output .rdb file") + + args = parser.parse_args() + + # Load tag groups + tag_groups = load_tag_groups(args.g) + + # Load raw database file + segbits = load_segbits(args.i) + + # Find common bits + bit_groups = find_common_bits_for_tag_groups(segbits, tag_groups) + # Apply tag grouping + segbits = group_tags(segbits, tag_groups, bit_groups) + + # Save fixed database file + save_segbits(args.o, segbits) + + +if __name__ == "__main__": + main() diff --git a/fuzzers/030-iob/tag_groups.txt b/fuzzers/030-iob/tag_groups.txt new file mode 100644 index 00000000..388934fe --- /dev/null +++ b/fuzzers/030-iob/tag_groups.txt @@ -0,0 +1,5 @@ +IOB33.IOB_Y0.IN_TERM.NONE IOB33.IOB_Y0.IN_TERM.UNTUNED_SPLIT_40 IOB33.IOB_Y0.IN_TERM.UNTUNED_SPLIT_50 IOB33.IOB_Y0.IN_TERM.UNTUNED_SPLIT_60 +IOB33.IOB_Y1.IN_TERM.NONE IOB33.IOB_Y1.IN_TERM.UNTUNED_SPLIT_40 IOB33.IOB_Y1.IN_TERM.UNTUNED_SPLIT_50 IOB33.IOB_Y1.IN_TERM.UNTUNED_SPLIT_60 + +IOB33.IOB_Y0.PULLTYPE.KEEPER IOB33.IOB_Y0.PULLTYPE.NONE IOB33.IOB_Y0.PULLTYPE.PULLDOWN IOB33.IOB_Y0.PULLTYPE.PULLUP +IOB33.IOB_Y1.PULLTYPE.KEEPER IOB33.IOB_Y1.PULLTYPE.NONE IOB33.IOB_Y1.PULLTYPE.PULLDOWN IOB33.IOB_Y1.PULLTYPE.PULLUP From 1196f67f7175d26052ae5baca10c0ea100af2509 Mon Sep 17 00:00:00 2001 From: Maciej Kurc Date: Tue, 28 Jan 2020 09:12:54 +0100 Subject: [PATCH 2/6] Moved the group.py script to the utils dir. Signed-off-by: Maciej Kurc --- fuzzers/030-iob/Makefile | 2 +- {fuzzers/030-iob => utils}/group.py | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename {fuzzers/030-iob => utils}/group.py (100%) diff --git a/fuzzers/030-iob/Makefile b/fuzzers/030-iob/Makefile index ab0a0168..a18dcbd6 100644 --- a/fuzzers/030-iob/Makefile +++ b/fuzzers/030-iob/Makefile @@ -13,7 +13,7 @@ build/segbits_xiob33.rdb: $(SPECIMENS_OK) build/segbits_xiob33.db: build/segbits_xiob33.rdb process_rdb.py bits.dbf python3 process_rdb.py build/segbits_xiob33.rdb > build/segbits_xiob33_processed.rdb - python3 group.py -g tag_groups.txt -i build/segbits_xiob33_processed.rdb -o build/segbits_xiob33_processed.rdb2 + python3 ${XRAY_DIR}/utils/group.py -g tag_groups.txt -i build/segbits_xiob33_processed.rdb -o build/segbits_xiob33_processed.rdb2 ${XRAY_DBFIXUP} --db-root build --zero-db bits.dbf --seg-fn-in build/segbits_xiob33_processed.rdb2 --seg-fn-out $@ ${XRAY_MASKMERGE} build/mask_xiob33.db $$(find -name segdata_liob33.txt) $$(find -name segdata_riob33.txt) diff --git a/fuzzers/030-iob/group.py b/utils/group.py similarity index 100% rename from fuzzers/030-iob/group.py rename to utils/group.py From cbe53a98e0b9641e9260dd148cd270534e4c8ca6 Mon Sep 17 00:00:00 2001 From: Maciej Kurc Date: Tue, 11 Feb 2020 12:37:31 +0100 Subject: [PATCH 3/6] Separated segbit file reading into a function. Signed-off-by: Maciej Kurc --- utils/dbfixup.py | 52 ++++++++++++++++++++++++++++++++---------------- 1 file changed, 35 insertions(+), 17 deletions(-) diff --git a/utils/dbfixup.py b/utils/dbfixup.py index 4e74a2e7..cac5e21d 100755 --- a/utils/dbfixup.py +++ b/utils/dbfixup.py @@ -199,21 +199,12 @@ class ZeroGroups(object): bits.add("!" + bit) -def add_zero_bits(fn_in, zero_db, clb_int=False, strict=True, verbose=False): - ''' - Add multibit entries - This requires adding some zero bits (ex: !31_09) - If an entry has any of the - ''' - - zero_groups = ZeroGroups(zero_db) - +def read_segbits(fn_in): + """ + Reads a segbits file. Removes duplcated lines. Returns a list of the lines. + """ lines = [] - new_lines = set() - changes = 0 - llast = None - drops = 0 with open(fn_in, "r") as f: for line in f: @@ -225,10 +216,29 @@ def add_zero_bits(fn_in, zero_db, clb_int=False, strict=True, verbose=False): lines.append(line) - tag, bits, mode, _ = util.parse_db_line(line) + return lines - if bits is not None and mode is None: - zero_groups.add_tag_bits(tag, bits) + +def add_zero_bits(fn_in, lines, zero_db, clb_int=False, strict=True, verbose=False): + ''' + Add multibit entries + This requires adding some zero bits (ex: !31_09) + If an entry has any of the + ''' + + zero_groups = ZeroGroups(zero_db) + + new_lines = set() + changes = 0 + + drops = 0 + + for line in lines: + + tag, bits, mode, _ = util.parse_db_line(line) + + if bits is not None and mode is None: + zero_groups.add_tag_bits(tag, bits) if verbose: zero_groups.print_groups() @@ -396,8 +406,16 @@ def update_seg_fns( if lazy and not os.path.exists(fn_in): continue + lines = read_segbits(fn_in) + changes, new_lines = add_zero_bits( - fn_in, zero_db, clb_int=clb_int, strict=strict, verbose=verbose) + fn_in, + lines, + zero_db, + clb_int=clb_int, + strict=strict, + verbose=verbose + ) new_changes, final_lines = remove_ambiguous_solutions( fn_in, From 014462de26857497f5a689cde4e6899118331fc1 Mon Sep 17 00:00:00 2001 From: Maciej Kurc Date: Thu, 13 Feb 2020 13:47:32 +0100 Subject: [PATCH 4/6] Ported tag grouping to dbfixup.py Signed-off-by: Maciej Kurc --- fuzzers/030-iob/Makefile | 3 +- utils/dbfixup.py | 193 ++++++++++++++++++++++++++++++++++++--- 2 files changed, 181 insertions(+), 15 deletions(-) diff --git a/fuzzers/030-iob/Makefile b/fuzzers/030-iob/Makefile index a18dcbd6..72e26007 100644 --- a/fuzzers/030-iob/Makefile +++ b/fuzzers/030-iob/Makefile @@ -13,8 +13,7 @@ build/segbits_xiob33.rdb: $(SPECIMENS_OK) build/segbits_xiob33.db: build/segbits_xiob33.rdb process_rdb.py bits.dbf python3 process_rdb.py build/segbits_xiob33.rdb > build/segbits_xiob33_processed.rdb - python3 ${XRAY_DIR}/utils/group.py -g tag_groups.txt -i build/segbits_xiob33_processed.rdb -o build/segbits_xiob33_processed.rdb2 - ${XRAY_DBFIXUP} --db-root build --zero-db bits.dbf --seg-fn-in build/segbits_xiob33_processed.rdb2 --seg-fn-out $@ + ${XRAY_DBFIXUP} --db-root build --zero-db bits.dbf --groups tag_groups.txt --seg-fn-in build/segbits_xiob33_processed.rdb --seg-fn-out $@ ${XRAY_MASKMERGE} build/mask_xiob33.db $$(find -name segdata_liob33.txt) $$(find -name segdata_riob33.txt) build/segbits_hclk_ioi3.rdb: $(SPECIMENS_OK) diff --git a/utils/dbfixup.py b/utils/dbfixup.py index cac5e21d..3d74cf14 100755 --- a/utils/dbfixup.py +++ b/utils/dbfixup.py @@ -1,6 +1,7 @@ #/usr/bin/env python3 import sys, os, re +import itertools from prjxray import util clb_int_zero_db = [ @@ -211,6 +212,8 @@ def read_segbits(fn_in): # Hack: skip duplicate lines # This happens while merging a new multibit entry line = line.strip() + if len(line) == 0: + continue if line == llast: continue @@ -219,7 +222,8 @@ def read_segbits(fn_in): return lines -def add_zero_bits(fn_in, lines, zero_db, clb_int=False, strict=True, verbose=False): +def add_zero_bits( + fn_in, lines, zero_db, clb_int=False, strict=True, verbose=False): ''' Add multibit entries This requires adding some zero bits (ex: !31_09) @@ -376,7 +380,7 @@ def remove_ambiguous_solutions(fn_in, db_lines, strict=True, verbose=True): return 0, db_lines drops = 0 - output_lines = [] + output_lines = set() for l in db_lines: parts = l.split() @@ -384,7 +388,7 @@ def remove_ambiguous_solutions(fn_in, db_lines, strict=True, verbose=True): bits = frozenset(parts[1:]) if bits not in dropped_solutions: - output_lines.append(l) + output_lines.add(l) drops += 1 else: if verbose: @@ -397,8 +401,81 @@ def remove_ambiguous_solutions(fn_in, db_lines, strict=True, verbose=True): return drops, output_lines +def group_tags(lines, tag_groups, bit_groups): + """ + Implements tag grouping. If a tag belongs to a group then the common bits + of that group are added to is as zeros. + + >>> tg = [{"A", "B"}] + >>> bg = [{(1, 2), (3, 4)}] + >>> res = group_tags({"A 1_2", "B 3_4"}, tg, bg) + >>> (res[0], sorted(list(res[1]))) + (2, ['A 1_2 !3_4', 'B !1_2 3_4']) + + >>> tg = [{"A", "B"}] + >>> bg = [{(1, 2), (3, 4)}] + >>> res = group_tags({"A 1_2", "B 3_4", "C 1_2"}, tg, bg) + >>> (res[0], sorted(list(res[1]))) + (2, ['A 1_2 !3_4', 'B !1_2 3_4', 'C 1_2']) + """ + + changes = 0 + new_lines = set() + + # Process lines + for line in lines: + + line = line.strip() + if not len(line): + continue + + # Parse the line + tag, bits, mode, _ = util.parse_db_line(line) + if not bits: + bits = set() + else: + bits = set([util.parse_tagbit(b) for b in bits]) + + # Check if the tag belongs to a group + for tag_group, bit_group in zip(tag_groups, bit_groups): + if tag in tag_group: + + # Add zero bits to the tag if not already there + bit_coords = set([b[1] for b in bits]) + for zero_bit in bit_group: + if zero_bit not in bit_coords: + bits.add((False, zero_bit)) + + # Format the line + bit_strs = [] + for bit in sorted(list(bits), key=lambda b: b[1]): + s = "!" if not bit[0] else "" + s += "{}_{}".format(bit[1][0], bit[1][1]) + bit_strs.append(s) + + new_line = " ".join([tag] + bit_strs) + + # Add the line + new_lines.add(new_line) + changes += 1 + break + + # It does not, pass it through unchanged + else: + new_lines.add(line) + + return changes, new_lines + + def update_seg_fns( - fn_inouts, zero_db, clb_int, lazy=False, strict=True, verbose=False): + fn_inouts, + zero_db, + tag_groups, + clb_int, + lazy=False, + strict=True, + verbose=False): + seg_files = 0 seg_lines = 0 for fn_in, fn_out in fn_inouts: @@ -406,28 +483,34 @@ def update_seg_fns( if lazy and not os.path.exists(fn_in): continue - lines = read_segbits(fn_in) + org_lines = read_segbits(fn_in) - changes, new_lines = add_zero_bits( + changes, lines = add_zero_bits( fn_in, - lines, + org_lines, zero_db, clb_int=clb_int, strict=strict, - verbose=verbose - ) + verbose=verbose) - new_changes, final_lines = remove_ambiguous_solutions( + new_changes, new_lines = remove_ambiguous_solutions( fn_in, - new_lines, + lines, strict=strict, verbose=verbose, ) + changes += new_changes + lines |= new_lines + # Find common bits for tag groups + bit_groups = find_common_bits_for_tag_groups(lines, tag_groups) + + # Group tags + new_changes, lines = group_tags(lines, tag_groups, bit_groups) changes += new_changes with open(fn_out, "w") as f: - for line in sorted(final_lines): + for line in sorted(lines): print(line, file=f) if changes is not None: @@ -471,6 +554,7 @@ def update_segs( seg_fn_in, seg_fn_out, zero_db_fn, + tag_groups, strict=True, verbose=False): if clb_int: @@ -497,7 +581,74 @@ def update_segs( print("CLB INT mode: %s" % clb_int) print("Segbit groups: %s" % len(zero_db)) update_seg_fns( - fn_inouts, zero_db, clb_int, lazy=lazy, strict=strict, verbose=verbose) + fn_inouts, + zero_db, + tag_groups, + clb_int, + lazy=lazy, + strict=strict, + verbose=verbose) + + +def find_common_bits_for_tag_groups(lines, tag_groups): + """ + For each tag group finds a common set of bits that have value of one. + """ + + bit_groups = [] + + for tag_group in tag_groups: + bit_group = set() + + for line in lines: + tag, bits, mode, _ = util.parse_db_line(line) + if not bits: + continue + + bits = set([util.parse_tagbit(b) for b in bits]) + + if tag in tag_group and len(bits): + ones = set([b[1] for b in bits if b[0]]) + bit_group |= ones + + print(tag_group) + print(bit_group) + bit_groups.append(bit_group) + + return bit_groups + + +def load_tag_groups(file_name): + """ + Loads tag groups from a text file. + + A tag group is defined by specifying a space separated list of tags within + a single line. Lines that are empty or start with '#' are ignored. + """ + tag_groups = [] + + # Load tag group specifications + with open(file_name, "r") as fp: + for line in fp: + line = line.strip() + + if len(line) == 0 or line.startswith("#"): + continue + + group = set(line.split()) + if len(group): + tag_groups.append(group) + + # Check if all tag groups are exclusive + for tag_group_a, tag_group_b in itertools.combinations(tag_groups, 2): + + tags = tag_group_a & tag_group_b + if len(tags): + raise RuntimeError( + "Tag(s) {} are present in multiple groups".format( + " ".join(tags))) + + return tag_groups def run( @@ -506,12 +657,18 @@ def run( zero_db_fn=None, seg_fn_in=None, seg_fn_out=None, + groups_fn_in=None, strict=None, verbose=False): if strict is None: strict = not clb_int + # Load tag groups + tag_groups = [] + if groups_fn_in is not None: + tag_groups = load_tag_groups(groups_fn_in) + # Probably should split this into two programs update_segs( db_root, @@ -519,6 +676,7 @@ def run( seg_fn_in=seg_fn_in, seg_fn_out=seg_fn_out, zero_db_fn=zero_db_fn, + tag_groups=tag_groups, strict=strict, verbose=verbose) if clb_int: @@ -538,6 +696,14 @@ def main(): parser.add_argument('--seg-fn-in', help='') parser.add_argument('--seg-fn-out', help='') util.add_bool_arg(parser, "--strict", default=False) + + parser.add_argument( + "-g", + "--groups", + type=str, + default=None, + help="Input tag group definition file") + args = parser.parse_args() run( @@ -546,6 +712,7 @@ def main(): args.zero_db, args.seg_fn_in, args.seg_fn_out, + args.groups, strict=args.strict, verbose=args.verbose) From e8a52f927df11ddeabc9a46205dba6730fb7d7e1 Mon Sep 17 00:00:00 2001 From: Maciej Kurc Date: Fri, 14 Feb 2020 09:39:58 +0100 Subject: [PATCH 5/6] Fixed bug that allowed duplicate solutions. Signed-off-by: Maciej Kurc --- utils/dbfixup.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/utils/dbfixup.py b/utils/dbfixup.py index 3d74cf14..d2270b32 100755 --- a/utils/dbfixup.py +++ b/utils/dbfixup.py @@ -483,24 +483,23 @@ def update_seg_fns( if lazy and not os.path.exists(fn_in): continue - org_lines = read_segbits(fn_in) + lines = read_segbits(fn_in) changes, lines = add_zero_bits( fn_in, - org_lines, + lines, zero_db, clb_int=clb_int, strict=strict, verbose=verbose) - new_changes, new_lines = remove_ambiguous_solutions( + new_changes, lines = remove_ambiguous_solutions( fn_in, lines, strict=strict, verbose=verbose, ) changes += new_changes - lines |= new_lines # Find common bits for tag groups bit_groups = find_common_bits_for_tag_groups(lines, tag_groups) From d0c79257638f7a4b4fe502160323e67898c7fb02 Mon Sep 17 00:00:00 2001 From: Maciej Kurc Date: Tue, 18 Feb 2020 11:17:58 +0100 Subject: [PATCH 6/6] Move feature grouping before zero bits add and ambiguous solution removal. Signed-off-by: Maciej Kurc --- utils/dbfixup.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/utils/dbfixup.py b/utils/dbfixup.py index d2270b32..58f936b6 100755 --- a/utils/dbfixup.py +++ b/utils/dbfixup.py @@ -484,14 +484,23 @@ def update_seg_fns( continue lines = read_segbits(fn_in) + changes = 0 - changes, lines = add_zero_bits( + # Find common bits for tag groups + bit_groups = find_common_bits_for_tag_groups(lines, tag_groups) + + # Group tags + new_changes, lines = group_tags(lines, tag_groups, bit_groups) + changes += new_changes + + new_changes, lines = add_zero_bits( fn_in, lines, zero_db, clb_int=clb_int, strict=strict, verbose=verbose) + changes += new_changes new_changes, lines = remove_ambiguous_solutions( fn_in, @@ -501,13 +510,6 @@ def update_seg_fns( ) changes += new_changes - # Find common bits for tag groups - bit_groups = find_common_bits_for_tag_groups(lines, tag_groups) - - # Group tags - new_changes, lines = group_tags(lines, tag_groups, bit_groups) - changes += new_changes - with open(fn_out, "w") as f: for line in sorted(lines): print(line, file=f) @@ -610,8 +612,6 @@ def find_common_bits_for_tag_groups(lines, tag_groups): ones = set([b[1] for b in bits if b[0]]) bit_group |= ones - print(tag_group) - print(bit_group) bit_groups.append(bit_group) return bit_groups