From 4c23f10d4d3980ed6efa297a40406dd7ed480d8c Mon Sep 17 00:00:00 2001 From: Maciej Kurc Date: Thu, 25 Jul 2019 12:11:56 +0200 Subject: [PATCH] Added generation of bit correlation report. Signed-off-by: Maciej Kurc --- prjxray/lms_solver.py | 232 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 228 insertions(+), 4 deletions(-) diff --git a/prjxray/lms_solver.py b/prjxray/lms_solver.py index dec04b63..3fe2f2a7 100755 --- a/prjxray/lms_solver.py +++ b/prjxray/lms_solver.py @@ -37,8 +37,10 @@ other tags. This allows to remove bits from a "IS_BLOCK_IN_USE" type tag from other tags responsible for enabling other features of that block. ''' import sys +import os import argparse import itertools +import json import numpy as np import numpy.linalg as linalg @@ -46,7 +48,7 @@ import numpy.linalg as linalg # ============================================================================= -def load_data(file_name, tagfilter=lambda tag: True): +def load_data(file_name, tagfilter=lambda tag: True, address_map=None): """ Loads data generated by the segmaker. @@ -57,6 +59,9 @@ def load_data(file_name, tagfilter=lambda tag: True): Name of the text file with data. tagfilter: A function for filtering tags. Should reqturn True or False. + address_map: + A dict indexed by tuples (address, offset) containing a list + of tile names. Returns ------- @@ -82,7 +87,23 @@ def load_data(file_name, tagfilter=lambda tag: True): all_segdata.append(segdata) segdata = None - segdata = {"seg": fields[1], "bit": [], "tag": []} + segname = fields[1] + + # Map segment address to tile name + if address_map is not None: + address = segname.split("_") + address = ( + int(address[0], base=16), + int(address[1]), + ) + if address in address_map: + segname = "_or_".join(address_map[address]) + + # Append file name + segname = file_name + ":" + segname + + # Append segdata + segdata = {"seg": segname, "bit": [], "tag": []} if segdata is None: continue @@ -228,6 +249,8 @@ def dump_results(fp, all_tags, all_bits, W, X, E, tag_stats=None): lines.append(line) + lines.append("") + # Write for line in lines: fp.write(line + "\n") @@ -264,6 +287,37 @@ def dump_solution_to_csv(fp, all_tags, all_bits, X): fp.write(line[:-1] + "\n") +def dump_correlation_report( + fp, all_tags, all_bits, W, C, correlation_exceptions): + + for i, tag in enumerate(all_tags): + + # No exceptions (100% correlation) + if len(correlation_exceptions[tag]) == 0: + continue + + fp.write(tag + "\n") + + for j, bit in enumerate(all_bits): + + if bit not in correlation_exceptions[tag]: + continue + + c = C[i, j] + w = W[i, j] + + # Dump bit correlation factor + sgn = "+" if w > 0 else "-" + fp.write(" bit %s: (%s) %.1f%%\n" % (bit.ljust(6), sgn, c * 100.0)) + + # Dump counter-factual cases + e = correlation_exceptions[tag][bit] + for x, y, ex in e: + fp.write(" is %d, should be %d - %s\n" % (x, y, ex)) + + fp.write("\n") + + # ============================================================================= @@ -501,6 +555,72 @@ def detect_candidates(X, th, norm=None): # ============================================================================= +def compute_bit_correlations(tags_to_solve, bits_to_solve, segdata, W): + """ + Basing on solution given in the matrix W returns a matrix C with + correlation coefficients of each bit. + + Also returns a dict of dicts indexed by tag names and bit names with + correlation exceptions - concrete specimen names where the correlation + does not occur. + """ + + C = np.zeros_like(W, dtype=float) + exceptions = {} + + for i, tag in enumerate(tags_to_solve): + + # Filter data for this tag + tag_segdata = [ + data for data in segdata if tag in [t[0] for t in data["tag"]] + ] + exceptions[tag] = {} + + # Compute bit correlation + for j, bit in enumerate(bits_to_solve): + w = W[i, j] + + # No correlation with that bit + if w == 0: + continue + + corr_sum = 0 + corr_count = 0 + + # Compute for one bit + for k, data in enumerate(tag_segdata): + bits = data["bit"] + + vt = [v for t, v in data["tag"] if t == tag][0] + vb = 1 if bit in bits else 0 + + # Negative correlation + if w < 0: + vt = int(1 - vt) + else: + vt = int(vt) + + # Correlates + if vt == vb: + corr_sum += 1 + # Does not correlate + else: + if bit not in exceptions[tag]: + exceptions[tag][bit] = [] + exceptions[tag][bit].append(( + vb, + vt, + data["seg"], + )) + + corr_count += 1 + + # Store correlation + C[i, j] = corr_sum / corr_count + + return C, exceptions + + def compute_tag_stats(all_tags, segdata): """ Counts occurrence of all considered tags @@ -553,6 +673,85 @@ def sort_bits(bit_name): ) +def build_address_map(tilegrid_file): + """ + Loads the tilegrid and generates a map (baseaddr, offset) -> tile name(s). + + Parameters + ---------- + + tilegrid_file: + The tilegrid.json file/ + + Returns + ------- + + A dict with lists of tile names. + + """ + + address_map = {} + + # Load tilegrid + with open(tilegrid_file, "r") as fp: + tilegrid = json.load(fp) + + # Loop over tiles + for tile_name, tile_data in tilegrid.items(): + + # No bits or bits empty + if "bits" not in tile_data: + continue + if not len(tile_data["bits"]): + continue + + bits = tile_data["bits"] + + # No bus + if "CLB_IO_CLK" not in bits: + continue + + bus = bits["CLB_IO_CLK"] + + # Make the address as integers + baseaddr = int(bus["baseaddr"], 16) + offset = int(bus["offset"]) + address = ( + baseaddr, + offset, + ) + + # Add tile to the map + if address not in address_map: + address_map[address] = [] + address_map[address].append(tile_name) + + return address_map + + +# ============================================================================= + + +class FileOrStream(object): + def __init__(self, file_name, stream=sys.stdout): + self.file_name = file_name + self.stream = stream + self.fp = None + + def __enter__(self): + if self.file_name is None: + return self.stream + if self.file_name == "-": + return self.stream + + self.fp = open(self.file_name, "w") + return self.fp + + def __exit__(self, exc_typ, exc_val, exc_tb): + if self.fp is not None: + self.fp.close() + + # ============================================================================= @@ -602,6 +801,13 @@ def main(): type=str, default=None, help="A CSV file name to Write the numerical solution to") + parser.add_argument( + "-r", + type=str, + default=None, + help= + "A text file name to write bit correlation report to. Specify '-' for stdout" + ) parser.add_argument( "-m", @@ -616,6 +822,12 @@ def main(): args = parser.parse_args() + # Build (baseaddr, offset) -> tile name map + database_dir = os.path.join( + os.getenv("XRAY_DATABASE_DIR"), os.getenv("XRAY_DATABASE")) + tilegrid_file = os.path.join(database_dir, "tilegrid.json") + address_map = build_address_map(tilegrid_file) + # Compute threshold th = args.t @@ -629,7 +841,7 @@ def main(): for name in args.files: print(name) - segdata.extend(load_data(name, tagfilter)) + segdata.extend(load_data(name, tagfilter, address_map)) # Make list of all bits all_bits = set() @@ -751,6 +963,10 @@ def main(): if E[r] > args.e: W[r, :] = 0 + # Compute correlation + C, correlation_exceptions = compute_bit_correlations( + tags_to_solve, bits_to_solve, segdata, W) + # Write segbits write_segbits(args.o, tags_to_solve, bits_to_solve, W) @@ -759,9 +975,17 @@ def main(): with open(args.x, "w") as fp: dump_solution_to_csv(fp, tags_to_solve, bits_to_solve, X) - # Dump + # Dump results dump_results(sys.stdout, tags_to_solve, bits_to_solve, W, X, E, tag_stats) + # Dump correlation report + if args.r is not None: + if args.r != "-": + print("Dumping bit correlation report to '{}'".format(args.r)) + with FileOrStream(args.r, sys.stdout) as fp: + dump_correlation_report( + fp, tags_to_solve, bits_to_solve, W, C, correlation_exceptions) + # =============================================================================