mergedb: proper merge script

Signed-off-by: John McMaster <johndmcmaster@gmail.com>

utils/mergedb.py

@@ -0,0 +1,64 @@
+#!/usr/bin/env python3
+
+import sys, re
+import os
+from prjxray import util
+
+
+def run(fn_ins, fn_out, strict=False, verbose=False):
+    # tag to bits
+    entries = {}
+    # tag to (bits, line)
+    tags = dict()
+    # bits to (tag, line)
+    bitss = dict()
+
+    for fn_in in fn_ins:
+        for line, (tag, bits, mode) in util.parse_db_lines(fn_in):
+            line = line.strip()
+            assert mode is not None or mode != "always", "strict: got ill defined line: %s" % (
+                line, )
+
+            if tag in tags:
+                orig_bits, orig_line = tags[tag]
+                if orig_bits != bits:
+                    print("WARNING: got duplicate tag %s" % (tag, ))
+                    print("  Orig line: %s" % orig_line)
+                    print("  New line : %s" % line)
+                    assert not strict, "strict: got duplicate tag"
+            if bits in bitss:
+                orig_tag, orig_line = bitss[bits]
+                if orig_tag != tag:
+                    print("WARNING: got duplicate bits %s" % (bits, ))
+                    print("  Orig line: %s" % orig_line)
+                    print("  New line : %s" % line)
+                    assert not strict, "strict: got duplicate bits"
+
+        entries[tag] = bits
+        tags[tag] = (bits, line)
+        if bits != None:
+            bitss[bits] = (tag, line)
+
+    util.write_db_lines(fn_out, entries)
+
+
+def main():
+    import argparse
+
+    parser = argparse.ArgumentParser(description="Combine multiple .db files")
+
+    util.db_root_arg(parser)
+    parser.add_argument('--verbose', action='store_true', help='')
+    parser.add_argument('--out', help='')
+    parser.add_argument('ins', nargs='+', help='Last takes precedence')
+    args = parser.parse_args()
+
+    run(
+        args.ins,
+        args.out,
+        strict=int(os.getenv("MERGEDB_STRICT", "1")),
+        verbose=args.verbose)
+
+
+if __name__ == '__main__':
+    main()

utils/mergedb.sh

@@ -29,6 +29,7 @@ ${XRAY_PARSEDB} --strict "$2"
 # However, expand back to L/R to make downstream tools not depend on this
 # in case we later find exceptions
 
+ismask=false
 case "$1" in
 	clbll_l)
 		sed < "$2" > "$tmp1" \
@@ -77,6 +78,7 @@ case "$1" in
 
 	mask_*)
 		db=$XRAY_DATABASE_DIR/$XRAY_DATABASE/$1.db
+		ismask=true
 		cp "$2" "$tmp1" ;;
 
 	*)
@@ -86,7 +88,12 @@ case "$1" in
 esac
 
 touch "$db"
-sort -u "$tmp1" "$db" | grep -v '<.*>' > "$tmp2" || true
+if $ismask ; then
+    sort -u "$tmp1" "$db" | grep -v '<.*>' > "$tmp2" || true
+else
+    # tmp1 must be placed second to take precedence over old bad entries
+    python3 ${XRAY_DIR}/utils/mergedb.py --out "$tmp2" "$db" "$tmp1"
+fi
 # Check aggregate db for consistency and make canonical
 ${XRAY_PARSEDB} --strict "$tmp2" "$db"