From 3c91c98e03ae2013f563727d68694779e8760958 Mon Sep 17 00:00:00 2001 From: Tim 'mithro' Ansell Date: Sun, 9 Feb 2020 23:17:56 -0600 Subject: [PATCH 1/6] Fix the sorting stage. - Rework how the json files are sorted (numbers are treated as numerics). - Sort csv and txt files. - Sort segbits.*origin_info.db files. - Sort the grid file. Signed-off-by: Tim 'mithro' Ansell --- utils/sort_db.py | 171 +++++++++++++++++++++++++++++++++++++---------- utils/xjson.py | 57 +++++++++++----- 2 files changed, 177 insertions(+), 51 deletions(-) diff --git a/utils/sort_db.py b/utils/sort_db.py index c8698c27..c8c915da 100755 --- a/utils/sort_db.py +++ b/utils/sort_db.py @@ -47,6 +47,7 @@ sort sets (lists where the order doesn't matter). """ +import csv import os import random import re @@ -284,16 +285,35 @@ def sortable_line_from_segbits(l): return (tag, tuple(bits)), l -def sort_db(filename): +def sortable_line_from_origin_segbits(l): + tag, origin, sbit = l.split(' ', 2) + tag = sortable_tag(tag) + + bits = bit.parseline(sbit) + + return (tag, tuple(bits)), l + + +def sort_db(pathname): """Sort a XXX.db file.""" + filename = os.path.split(pathname)[-1] if filename.startswith('segbits_'): - sortable_line_from_dbfile = sortable_line_from_segbits + if 'origin_info' in filename: + sortable_line_from_dbfile = sortable_line_from_origin_segbits + else: + sortable_line_from_dbfile = sortable_line_from_segbits + elif 'origin_info' in filename: + return False elif filename.startswith('ppips_'): sortable_line_from_dbfile = sortable_line_from_ppips + elif filename.startswith('grid-'): + sortable_line_from_dbfile = sortable_line_from_ppips elif filename.startswith('mask_'): sortable_line_from_dbfile = sortable_line_from_mask + else: + return False - lines = open(filename).readlines() + lines = open(pathname).readlines() tosort = [] for l in lines: @@ -305,16 +325,16 @@ def sort_db(filename): tosort.sort(key=cmp.cmp_key) # Make sure the sort is stable - for i in range(0, 4): - copy = tosort.copy() - random.shuffle(copy) - copy.sort(key=cmp.cmp_key) - assert len(copy) == len(tosort) - for i in range(0, len(copy)): - assert copy[i] == tosort[i], "\n%r\n != \n%r\n" % ( - copy[i], tosort[i]) + #for i in range(0, 4): + # copy = tosort.copy() + # random.shuffle(copy) + # copy.sort(key=cmp.cmp_key) + # assert len(copy) == len(tosort) + # for i in range(0, len(copy)): + # assert copy[i] == tosort[i], "\n%r\n != \n%r\n" % ( + # copy[i], tosort[i]) - with open(filename, 'w') as f: + with open(pathname, 'w') as f: for _, l in tosort: f.write(l) f.write('\n') @@ -322,11 +342,45 @@ def sort_db(filename): return True +def sort_csv(pathname): + rows = [] + fields = [] + delimiter = None + with open(pathname, newline='') as f: + if pathname.endswith('.csv'): + delimiter = ',' + elif pathname.endswith('.txt'): + delimiter = ' ' + reader = csv.DictReader(f, delimiter=delimiter) + fields.extend(reader.fieldnames) + rows.extend(reader) + del reader + + fields.sort() + + def sort_key(r): + v = [] + for field in fields: + v.append(sortable_tag(r[field])) + return tuple(v) + + rows.sort(key=sort_key) + + with open(pathname, 'w', newline='') as f: + writer = csv.DictWriter( + f, fields, delimiter=delimiter, lineterminator='\n') + writer.writeheader() + writer.writerows(rows) + + return True + + def sort_json(filename): """Sort a XXX.json file.""" try: d = json.load(open(filename)) - except json.JSONDecodeError: + except json.JSONDecodeError as e: + print(e) return False with open(filename, 'w') as f: @@ -335,30 +389,75 @@ def sort_json(filename): return True +def sort_db_text(n): + rows = [] + with open(n) as f: + for l in f: + rows.append(([extract_num(s) for s in l.split()], l)) + + rows.sort(key=lambda i: i[0]) + + with open(n, 'w') as f: + for l in rows: + f.write(l[-1]) + + return True + + +def sort_file(n): + + assert os.path.exists(n) + + base, ext = os.path.splitext(n) + dirname, base = os.path.split(base) + + # Leave db files with fuzzer of origin untouched + if "origin_info" in n and not base.startswith('segbits'): + print("Ignoring file {:45s}".format(n), flush=True) + return + + if ext == '.db': + print("Sorting DB file {:45s}".format(n), end=" ", flush=True) + x = sort_db(n) + elif ext == '.json': + print("Sorting JSON file {:45s}".format(n), end=" ", flush=True) + x = sort_json(n) + elif ext in ('.csv', '.txt'): + if n.endswith('-db.txt'): + print("Sorting txt file {:45s}".format(n), end=" ", flush=True) + x = sort_db_text(n) + else: + print("Sorting CSV file {:45s}".format(n), end=" ", flush=True) + x = sort_csv(n) + else: + print("Ignoring file {:45s}".format(n), end=" ", flush=True) + x = True + if x: + print(".. success.") + else: + print(".. failed.") + + +def sort_dir(dirname): + for n in sorted(os.listdir(dirname)): + n = os.path.join(dirname, n) + if os.path.isdir(n): + print("Entering dir {:45s}".format(n), flush=True) + sort_dir(n) + continue + elif not os.path.isfile(n): + print("Ignoring non-file {:45s}".format(n), flush=True) + continue + + sort_file(n) + + def main(argv): - for n in sorted(os.listdir()): - if not os.path.isfile(n): - continue - # Leave db files with fuzzer of origin untouched - if "origin_info" in n: - continue - - base, ext = os.path.splitext(n) - - if ext == '.db': - print("Sorting DB file {:40s}".format(n), end=" ", flush=True) - x = sort_db(n) - elif ext == '.json': - print("Sorting JSON file {:40s}".format(n), end=" ", flush=True) - x = sort_json(n) - else: - print("Ignoring file {:40s}".format(n), end=" ", flush=True) - x = True - if x: - print(".. success.") - else: - print(".. failed.") - + if argv[1:]: + for n in argv[1:]: + sort_file(n) + else: + sort_dir('.') return 0 diff --git a/utils/xjson.py b/utils/xjson.py index 77eaf515..f216fb83 100755 --- a/utils/xjson.py +++ b/utils/xjson.py @@ -4,6 +4,8 @@ import json import re import sys +from collections import OrderedDict + def extract_numbers(s): """ @@ -31,23 +33,48 @@ def sort(data): data.sort(key=lambda o: (o['tile_types'], o['grid_deltas'])) else: - def walker(o, f): - if isinstance(o, dict): - for i in o.values(): - walker(i, f) + def key(o): + if o is None: + return None + elif isinstance(o, str): + return extract_numbers(o) + elif isinstance(o, int): + return o elif isinstance(o, list): + return [key(i) for i in o] + elif isinstance(o, dict): + return [(key(k), key(v)) for k, v in o.items()] + raise ValueError(repr(o)) + + def rsorter(o): + if isinstance(o, dict): + nitems = [] + for k, v in o.items(): + nitems.append((key(k), k, rsorter(v))) + nitems.sort(key=lambda n: n[0]) + + new_dict = OrderedDict() + for _, k, v in nitems: + new_dict[k] = v + return new_dict + + elif isinstance(o, list): + if len(o) == 2: + return o + + nlist = [] for i in o: - walker(i, f) - f(o) + nlist.append((key(i), rsorter(i))) + nlist.sort(key=lambda n: n[0]) - def f(o): - if isinstance(o, list): - if len(o) > 2: - strings = all(isinstance(x, str) for x in o) - if strings: - o.sort() + new_list = [] + for _, i in nlist: + new_list.append(i) + return new_list + else: + return o - walker(data, f) + return rsorter(data) def pprint(f, data): @@ -55,8 +82,8 @@ def pprint(f, data): if not isinstance(f, io.TextIOBase): detach = True f = io.TextIOWrapper(f) - sort(data) - json.dump(data, f, sort_keys=True, indent=4) + data = sort(data) + json.dump(data, f, indent=4) f.write('\n') f.flush() if detach: From 21710d336f7884a9964d4ab9929a996712e64aeb Mon Sep 17 00:00:00 2001 From: Tim 'mithro' Ansell Date: Mon, 10 Feb 2020 08:12:24 -0800 Subject: [PATCH 2/6] Ignore docs/env. Signed-off-by: Tim 'mithro' Ansell --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 0715285e..e70eb895 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ SHELL = bash -ALL_EXCLUDE = third_party .git env build +ALL_EXCLUDE = third_party .git env build docs/env # Check if root ifeq ($(shell id -u),0) From 8b560d0d024d3ae0f443b318da0fbb52565b0188 Mon Sep 17 00:00:00 2001 From: Tim 'mithro' Ansell Date: Mon, 10 Feb 2020 11:07:56 -0800 Subject: [PATCH 3/6] utils: Fix broken sorting of tileconn.json Signed-off-by: Tim 'mithro' Ansell --- utils/xjson.py | 1 + 1 file changed, 1 insertion(+) diff --git a/utils/xjson.py b/utils/xjson.py index f216fb83..a4d1b3fc 100755 --- a/utils/xjson.py +++ b/utils/xjson.py @@ -31,6 +31,7 @@ def sort(data): key=lambda o: (extract_numbers(o[0]), extract_numbers(o[1]))) data.sort(key=lambda o: (o['tile_types'], o['grid_deltas'])) + return data else: def key(o): From 0362854b05b567c887d8e96ffdfdec78020d27f9 Mon Sep 17 00:00:00 2001 From: Tim 'mithro' Ansell Date: Mon, 10 Feb 2020 12:04:09 -0800 Subject: [PATCH 4/6] Don't sort the CSV fields. Signed-off-by: Tim 'mithro' Ansell --- utils/sort_db.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/utils/sort_db.py b/utils/sort_db.py index c8c915da..90444ba6 100755 --- a/utils/sort_db.py +++ b/utils/sort_db.py @@ -356,8 +356,6 @@ def sort_csv(pathname): rows.extend(reader) del reader - fields.sort() - def sort_key(r): v = [] for field in fields: From 18cd6aff1ca2dc639674ce5ba1a92fa47c7b9f8b Mon Sep 17 00:00:00 2001 From: Tim 'mithro' Ansell Date: Tue, 11 Feb 2020 08:42:14 -0800 Subject: [PATCH 5/6] xjson: Support sets + add doctest. Signed-off-by: Tim 'mithro' Ansell --- requirements.txt | 1 + utils/xjson.py | 49 +++++++++++++++++++++++++++++++++++++++++++----- 2 files changed, 45 insertions(+), 5 deletions(-) diff --git a/requirements.txt b/requirements.txt index 4a9f9c7f..eb09ebd3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,6 +2,7 @@ intervaltree junit-xml numpy +ordered-set parse progressbar2 pyjson5 diff --git a/utils/xjson.py b/utils/xjson.py index a4d1b3fc..4e670be7 100755 --- a/utils/xjson.py +++ b/utils/xjson.py @@ -5,6 +5,7 @@ import re import sys from collections import OrderedDict +from ordered_set import OrderedSet def extract_numbers(s): @@ -24,6 +25,31 @@ def extract_numbers(s): def sort(data): + """Sort data types via "natural" numbers. + + Supports all the basic Python data types. + >>> o = sort({ + ... 't1': {'c','b'}, # Set + ... 't2': ('a2', 'a10', 'e'), # Tuple + ... 't3': [5, 3, 2], # List + ... 't4': { # Dictionary + ... 'a4': ('b2', 'b3'), + ... 'a2': ['c1', 'c2', 'c0', 'c10'], + ... }, + ... 't5': ['a1b5', 'a2b1', 'a1b1'], + ... }) + >>> for t in o: + ... print(t+':', o[t]) + t1: OrderedSet(['b', 'c']) + t2: ('a2', 'a10', 'e') + t3: (2, 3, 5) + t4: OrderedDict([('a2', ('c0', 'c1', 'c2', 'c10')), ('a4', ('b2', 'b3'))]) + t5: ('a1b1', 'a1b5', 'a2b1') + + Don't mangle "pairs" + >>> sort([('b', 'c'), ('2', '1')]) + [('b', 'c'), ('2', '1')] + """ # FIXME: We assume that a list is a tileconn.json format... if isinstance(data, list) and len(data) > 0 and 'wire_pairs' in data[0]: for o in data: @@ -41,10 +67,12 @@ def sort(data): return extract_numbers(o) elif isinstance(o, int): return o - elif isinstance(o, list): - return [key(i) for i in o] + elif isinstance(o, (list, tuple)): + return tuple(key(i) for i in o) elif isinstance(o, dict): - return [(key(k), key(v)) for k, v in o.items()] + return tuple((key(k), key(v)) for k, v in o.items()) + elif isinstance(o, set): + return tuple(key(k) for k in o) raise ValueError(repr(o)) def rsorter(o): @@ -59,7 +87,18 @@ def sort(data): new_dict[k] = v return new_dict - elif isinstance(o, list): + elif isinstance(o, set): + nitems = [] + for k in o: + nitems.append((key(k), k)) + nitems.sort(key=lambda n: n[0]) + + new_set = OrderedSet() + for _, k in nitems: + new_set.add(k) + return new_set + + elif isinstance(o, (tuple, list)): if len(o) == 2: return o @@ -71,7 +110,7 @@ def sort(data): new_list = [] for _, i in nlist: new_list.append(i) - return new_list + return tuple(new_list) else: return o From 7dfd4adaa87d02cf0758a764ac7564fb620916c4 Mon Sep 17 00:00:00 2001 From: Keith Rothman <537074+litghost@users.noreply.github.com> Date: Wed, 12 Feb 2020 07:36:31 -0800 Subject: [PATCH 6/6] Remove xjson from 074. Signed-off-by: Keith Rothman <537074+litghost@users.noreply.github.com> --- fuzzers/074-dump_all/cleanup_site_pins.py | 4 +--- fuzzers/074-dump_all/create_node_tree.py | 5 ++--- fuzzers/074-dump_all/generate_grid.py | 5 ++--- fuzzers/074-dump_all/reduce_site_types.py | 4 +--- fuzzers/074-dump_all/reduce_tile_types.py | 7 +++---- 5 files changed, 9 insertions(+), 16 deletions(-) diff --git a/fuzzers/074-dump_all/cleanup_site_pins.py b/fuzzers/074-dump_all/cleanup_site_pins.py index 0f4005aa..c528ad27 100644 --- a/fuzzers/074-dump_all/cleanup_site_pins.py +++ b/fuzzers/074-dump_all/cleanup_site_pins.py @@ -15,8 +15,6 @@ import re import sys import copy -from utils import xjson - # All site names appear to follow the pattern _XY. # Generally speaking, only the tile relatively coordinates are required to # assemble arch defs, so we re-origin the coordinates to be relative to the tile @@ -115,7 +113,7 @@ def main(): site_pin['name'] = site_pin['name'][len(orig_site_name) + 1:] - xjson.pprint(sys.stdout, output_site_pins) + json.dumps(output_site_pins, indent=2, sort_keys=True) if __name__ == "__main__": diff --git a/fuzzers/074-dump_all/create_node_tree.py b/fuzzers/074-dump_all/create_node_tree.py index 53efb58c..96c11cdb 100644 --- a/fuzzers/074-dump_all/create_node_tree.py +++ b/fuzzers/074-dump_all/create_node_tree.py @@ -5,8 +5,7 @@ import os.path import prjxray.lib import pickle import collections - -from utils import xjson +import json def build_node_index(fname): @@ -273,7 +272,7 @@ def main(): print('{} Writing node tree'.format(datetime.datetime.now())) with open(os.path.join(args.output_dir, 'node_tree.json'), 'w') as f: - xjson.pprint(f, nodes) + json.dump(nodes, f, indent=2, sort_keys=True) if __name__ == '__main__': diff --git a/fuzzers/074-dump_all/generate_grid.py b/fuzzers/074-dump_all/generate_grid.py index c3b9576e..32f78ec6 100644 --- a/fuzzers/074-dump_all/generate_grid.py +++ b/fuzzers/074-dump_all/generate_grid.py @@ -11,7 +11,6 @@ import datetime import pickle import sys -from utils import xjson from prjxray import util, lib @@ -608,7 +607,7 @@ def main(): print('{} Writing tileconn'.format(datetime.datetime.now())) with open(tileconn_file, 'w') as f: - xjson.pprint(f, tileconn) + json.dump(tileconn, f, indent=2, sort_keys=True) else: with open(wire_map_file, 'rb') as f: wire_map = pickle.load(f) @@ -653,7 +652,7 @@ def main(): if len(error_nodes) > 0: error_nodes_file = os.path.join(args.output_dir, 'error_nodes.json') with open(error_nodes_file, 'w') as f: - xjson.pprint(f, error_nodes) + json.dump(error_nodes, f, indent=2, sort_keys=True) ignored_wires = [] ignored_wires_file = args.ignored_wires diff --git a/fuzzers/074-dump_all/reduce_site_types.py b/fuzzers/074-dump_all/reduce_site_types.py index dbd0eba2..a8aba61a 100644 --- a/fuzzers/074-dump_all/reduce_site_types.py +++ b/fuzzers/074-dump_all/reduce_site_types.py @@ -13,8 +13,6 @@ import os.path import re import json -from utils import xjson - def main(): parser = argparse.ArgumentParser( @@ -57,7 +55,7 @@ def main(): with open(os.path.join(args.output_dir, 'site_type_{}.json'.format(site_type)), 'w') as f: - xjson.pprint(f, proto_site_type) + json.dump(proto_site_type, f) if __name__ == '__main__': diff --git a/fuzzers/074-dump_all/reduce_tile_types.py b/fuzzers/074-dump_all/reduce_tile_types.py index 1e4f1d97..f99b16bc 100644 --- a/fuzzers/074-dump_all/reduce_tile_types.py +++ b/fuzzers/074-dump_all/reduce_tile_types.py @@ -17,8 +17,7 @@ import progressbar import multiprocessing import os import functools - -from utils import xjson +import json def check_and_strip_prefix(name, prefix): @@ -364,10 +363,10 @@ def main(): with open(os.path.join( args.output_dir, 'tile_type_{}_site_type_{}.json'.format( tile_type, site_types[site_type]['type'])), 'w') as f: - xjson.pprint(f, site_types[site_type]) + json.dump(site_types[site_type], f, indent=2, sort_keys=True) with open(tile_type_file, 'w') as f: - xjson.pprint(f, reduced_tile) + json.dump(reduced_tile, f, indent=2, sort_keys=True) if __name__ == '__main__':