Added generation of bit correlation report.

Signed-off-by: Maciej Kurc <mkurc@antmicro.com>

prjxray/lms_solver.py

@@ -37,8 +37,10 @@ other tags. This allows to remove bits from a "IS_BLOCK_IN_USE" type tag from
 other tags responsible for enabling other features of that block.
 '''
 import sys
+import os
 import argparse
 import itertools
+import json
 
 import numpy as np
 import numpy.linalg as linalg
@@ -46,7 +48,7 @@ import numpy.linalg as linalg
 # =============================================================================
 
 
-def load_data(file_name, tagfilter=lambda tag: True):
+def load_data(file_name, tagfilter=lambda tag: True, address_map=None):
     """
     Loads data generated by the segmaker.
 
@@ -57,6 +59,9 @@ def load_data(file_name, tagfilter=lambda tag: True):
         Name of the text file with data.
     tagfilter:
         A function for filtering tags. Should reqturn True or False.
+    address_map:
+        A dict indexed by tuples (address, offset) containing a list
+        of tile names.
 
     Returns
     -------
@@ -82,7 +87,23 @@ def load_data(file_name, tagfilter=lambda tag: True):
                         all_segdata.append(segdata)
                     segdata = None
 
-                segdata = {"seg": fields[1], "bit": [], "tag": []}
+                segname = fields[1]
+
+                # Map segment address to tile name
+                if address_map is not None:
+                    address = segname.split("_")
+                    address = (
+                        int(address[0], base=16),
+                        int(address[1]),
+                    )
+                    if address in address_map:
+                        segname = "_or_".join(address_map[address])
+
+                # Append file name
+                segname = file_name + ":" + segname
+
+                # Append segdata
+                segdata = {"seg": segname, "bit": [], "tag": []}
 
             if segdata is None:
                 continue
@@ -228,6 +249,8 @@ def dump_results(fp, all_tags, all_bits, W, X, E, tag_stats=None):
 
         lines.append(line)
 
+    lines.append("")
+
     # Write
     for line in lines:
         fp.write(line + "\n")
@@ -264,6 +287,37 @@ def dump_solution_to_csv(fp, all_tags, all_bits, X):
         fp.write(line[:-1] + "\n")
 
 
+def dump_correlation_report(
+        fp, all_tags, all_bits, W, C, correlation_exceptions):
+
+    for i, tag in enumerate(all_tags):
+
+        # No exceptions (100% correlation)
+        if len(correlation_exceptions[tag]) == 0:
+            continue
+
+        fp.write(tag + "\n")
+
+        for j, bit in enumerate(all_bits):
+
+            if bit not in correlation_exceptions[tag]:
+                continue
+
+            c = C[i, j]
+            w = W[i, j]
+
+            # Dump bit correlation factor
+            sgn = "+" if w > 0 else "-"
+            fp.write(" bit %s: (%s) %.1f%%\n" % (bit.ljust(6), sgn, c * 100.0))
+
+            # Dump counter-factual cases
+            e = correlation_exceptions[tag][bit]
+            for x, y, ex in e:
+                fp.write("  is %d, should be %d - %s\n" % (x, y, ex))
+
+    fp.write("\n")
+
+
 # =============================================================================
 
 
@@ -501,6 +555,72 @@ def detect_candidates(X, th, norm=None):
 # =============================================================================
 
 
+def compute_bit_correlations(tags_to_solve, bits_to_solve, segdata, W):
+    """
+    Basing on solution given in the matrix W returns a matrix C with 
+    correlation coefficients of each bit.
+
+    Also returns a dict of dicts indexed by tag names and bit names with
+    correlation exceptions - concrete specimen names where the correlation
+    does not occur.
+    """
+
+    C = np.zeros_like(W, dtype=float)
+    exceptions = {}
+
+    for i, tag in enumerate(tags_to_solve):
+
+        # Filter data for this tag
+        tag_segdata = [
+            data for data in segdata if tag in [t[0] for t in data["tag"]]
+        ]
+        exceptions[tag] = {}
+
+        # Compute bit correlation
+        for j, bit in enumerate(bits_to_solve):
+            w = W[i, j]
+
+            # No correlation with that bit
+            if w == 0:
+                continue
+
+            corr_sum = 0
+            corr_count = 0
+
+            # Compute for one bit
+            for k, data in enumerate(tag_segdata):
+                bits = data["bit"]
+
+                vt = [v for t, v in data["tag"] if t == tag][0]
+                vb = 1 if bit in bits else 0
+
+                # Negative correlation
+                if w < 0:
+                    vt = int(1 - vt)
+                else:
+                    vt = int(vt)
+
+                # Correlates
+                if vt == vb:
+                    corr_sum += 1
+                # Does not correlate
+                else:
+                    if bit not in exceptions[tag]:
+                        exceptions[tag][bit] = []
+                    exceptions[tag][bit].append((
+                        vb,
+                        vt,
+                        data["seg"],
+                    ))
+
+                corr_count += 1
+
+            # Store correlation
+            C[i, j] = corr_sum / corr_count
+
+    return C, exceptions
+
+
 def compute_tag_stats(all_tags, segdata):
     """
     Counts occurrence of all considered tags
@@ -553,6 +673,85 @@ def sort_bits(bit_name):
     )
 
 
+def build_address_map(tilegrid_file):
+    """
+    Loads the tilegrid and generates a map (baseaddr, offset) -> tile name(s).
+
+    Parameters
+    ----------
+
+    tilegrid_file:
+        The tilegrid.json file/    
+
+    Returns
+    -------
+    
+    A dict with lists of tile names.
+
+    """
+
+    address_map = {}
+
+    # Load tilegrid
+    with open(tilegrid_file, "r") as fp:
+        tilegrid = json.load(fp)
+
+    # Loop over tiles
+    for tile_name, tile_data in tilegrid.items():
+
+        # No bits or bits empty
+        if "bits" not in tile_data:
+            continue
+        if not len(tile_data["bits"]):
+            continue
+
+        bits = tile_data["bits"]
+
+        # No bus
+        if "CLB_IO_CLK" not in bits:
+            continue
+
+        bus = bits["CLB_IO_CLK"]
+
+        # Make the address as integers
+        baseaddr = int(bus["baseaddr"], 16)
+        offset = int(bus["offset"])
+        address = (
+            baseaddr,
+            offset,
+        )
+
+        # Add tile to the map
+        if address not in address_map:
+            address_map[address] = []
+        address_map[address].append(tile_name)
+
+    return address_map
+
+
+# =============================================================================
+
+
+class FileOrStream(object):
+    def __init__(self, file_name, stream=sys.stdout):
+        self.file_name = file_name
+        self.stream = stream
+        self.fp = None
+
+    def __enter__(self):
+        if self.file_name is None:
+            return self.stream
+        if self.file_name == "-":
+            return self.stream
+
+        self.fp = open(self.file_name, "w")
+        return self.fp
+
+    def __exit__(self, exc_typ, exc_val, exc_tb):
+        if self.fp is not None:
+            self.fp.close()
+
+
 # =============================================================================
 
 
@@ -602,6 +801,13 @@ def main():
         type=str,
         default=None,
         help="A CSV file name to Write the numerical solution to")
+    parser.add_argument(
+        "-r",
+        type=str,
+        default=None,
+        help=
+        "A text file name to write bit correlation report to. Specify '-' for stdout"
+    )
 
     parser.add_argument(
         "-m",
@@ -616,6 +822,12 @@ def main():
 
     args = parser.parse_args()
 
+    # Build (baseaddr, offset) -> tile name map
+    database_dir = os.path.join(
+        os.getenv("XRAY_DATABASE_DIR"), os.getenv("XRAY_DATABASE"))
+    tilegrid_file = os.path.join(database_dir, "tilegrid.json")
+    address_map = build_address_map(tilegrid_file)
+
     # Compute threshold
     th = args.t
 
@@ -629,7 +841,7 @@ def main():
 
     for name in args.files:
         print(name)
-        segdata.extend(load_data(name, tagfilter))
+        segdata.extend(load_data(name, tagfilter, address_map))
 
     # Make list of all bits
     all_bits = set()
@@ -751,6 +963,10 @@ def main():
         if E[r] > args.e:
             W[r, :] = 0
 
+    # Compute correlation
+    C, correlation_exceptions = compute_bit_correlations(
+        tags_to_solve, bits_to_solve, segdata, W)
+
     # Write segbits
     write_segbits(args.o, tags_to_solve, bits_to_solve, W)
 
@@ -759,9 +975,17 @@ def main():
         with open(args.x, "w") as fp:
             dump_solution_to_csv(fp, tags_to_solve, bits_to_solve, X)
 
-    # Dump
+    # Dump results
     dump_results(sys.stdout, tags_to_solve, bits_to_solve, W, X, E, tag_stats)
 
+    # Dump correlation report
+    if args.r is not None:
+        if args.r != "-":
+            print("Dumping bit correlation report to '{}'".format(args.r))
+        with FileOrStream(args.r, sys.stdout) as fp:
+            dump_correlation_report(
+                fp, tags_to_solve, bits_to_solve, W, C, correlation_exceptions)
+
 
 # =============================================================================