From 70294b332920d23f97c93c720e6078e6358b7272 Mon Sep 17 00:00:00 2001 From: Tim 'mithro' Ansell Date: Thu, 13 Dec 2018 14:08:10 -0800 Subject: [PATCH 1/9] Add wrapper to allow easy loading of yaml files. Fixes #327. Signed-off-by: Tim 'mithro' Ansell --- prjxray/xyaml.py | 65 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) create mode 100644 prjxray/xyaml.py diff --git a/prjxray/xyaml.py b/prjxray/xyaml.py new file mode 100644 index 00000000..40e3bc2f --- /dev/null +++ b/prjxray/xyaml.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python3 +import io +import re +import yaml +import json +import unittest + + +def load(f): + data = f.read() + # Strip out of ! + data = re.sub("!<[^>]*>", "", data) + return yaml.load(io.StringIO(data)) + + +def tojson(f): + d = load(f) + return json.dumps(d, sort_keys=True, indent=4) + + +class XYamlTest(unittest.TestCase): + def test(self): + s = io.StringIO("""\ +! +idcode: 0x362d093 +global_clock_regions: + top: ! + rows: + 0: ! + configuration_buses: + CLB_IO_CLK: ! + configuration_columns: + 0: ! + frame_count: 42 +""") + djson = tojson(s) + self.assertMultiLineEqual(djson, """\ +{ + "global_clock_regions": { + "top": { + "rows": { + "0": { + "configuration_buses": { + "CLB_IO_CLK": { + "configuration_columns": { + "0": { + "frame_count": 42 + } + } + } + } + } + } + } + }, + "idcode": 56807571 +}""") + +if __name__ == "__main__": + import sys + if len(sys.argv) == 1: + unittest.main() + else: + assert len(sys.argv) == 2 + print(tojson(open(sys.argv[1]))) From db7e2d7dd36feefa491b683c5034d6013eae5dc3 Mon Sep 17 00:00:00 2001 From: Tim 'mithro' Ansell Date: Thu, 13 Dec 2018 14:32:29 -0800 Subject: [PATCH 2/9] Adding tool for canonicalizing json files. Fixes #154. Signed-off-by: Tim 'mithro' Ansell --- prjxray/xjson.py | 63 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) create mode 100644 prjxray/xjson.py diff --git a/prjxray/xjson.py b/prjxray/xjson.py new file mode 100644 index 00000000..c45d7911 --- /dev/null +++ b/prjxray/xjson.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python3 +import re +import json +import sys + + +def extract_numbers(s): + """ + >>> extract_numbers("CLK_HROW_WR10END2_3") + ('CLK_HROW_WR', 10, 'END', 2, '_', 3) + >>> extract_numbers("VBRK_WR1END2") + ('VBRK_WR', 1, 'END', 2) + """ + bits = [] + for m in re.finditer("([^0-9]*)([0-9]*)",s): + if m.group(1): + bits.append(m.group(1)) + if m.group(2): + bits.append(int(m.group(2))) + return tuple(bits) + + +def sort(data): + # FIXME: We assume that a list is a tileconn.json format... + if isinstance(data, list): + for o in data: + o['wire_pairs'].sort(key=lambda o: (extract_numbers(o[0]), extract_numbers(o[1]))) + + data.sort(key=lambda o: (o['tile_types'], o['grid_deltas'])) + else: + def walker(o, f): + if isinstance(o, dict): + for i in o.values(): + walker(i, f) + elif isinstance(o, list): + for i in o: + walker(i, f) + f(o) + + def f(o): + if isinstance(o, list): + if len(o) > 2: + strings = all(isinstance(x, str) for x in o) + if strings: + o.sort() + + walker(data, f) + + +def pprint(f, data): + sort(data) + json.dump(d, f, sort_keys=True, indent=4) + f.write('\n') + + +if __name__ == "__main__": + if len(sys.argv) == 1: + import doctest + doctest.testmod() + else: + assert len(sys.argv) == 2 + d = json.load(open(sys.argv[1])) + pprint(sys.stdout, d) From 178002d348b55611d9927abbc2ebdd8787684720 Mon Sep 17 00:00:00 2001 From: Tim 'mithro' Ansell Date: Thu, 13 Dec 2018 14:36:18 -0800 Subject: [PATCH 3/9] Generate the part JSON file at same time as yaml file. Signed-off-by: Tim 'mithro' Ansell --- fuzzers/001-part-yaml/Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/fuzzers/001-part-yaml/Makefile b/fuzzers/001-part-yaml/Makefile index 3e049293..2de00c35 100644 --- a/fuzzers/001-part-yaml/Makefile +++ b/fuzzers/001-part-yaml/Makefile @@ -10,6 +10,7 @@ $(SPECIMENS): Makefile.specimen pushdb: cp ${XRAY_PART}.yaml ${XRAY_DATABASE_DIR}/$(XRAY_DATABASE) + python -m prjxray.xyaml ${XRAY_PART}.yaml > ${XRAY_DATABASE_DIR}/$(XRAY_DATABASE)/${XRAY_PART}.json run: $(MAKE) clean From fb2d809659454b242eb83b5e34b5b9517e7ed5d5 Mon Sep 17 00:00:00 2001 From: Tim 'mithro' Ansell Date: Thu, 13 Dec 2018 14:55:04 -0800 Subject: [PATCH 4/9] Use xjson.pprint instead of json.dump for output. Signed-off-by: Tim 'mithro' Ansell --- fuzzers/005-tilegrid/generate.py | 9 +++------ fuzzers/074-dump_all/cleanup_site_pins.py | 5 +++-- fuzzers/074-dump_all/create_node_tree.py | 5 +++-- fuzzers/074-dump_all/generate_grid.py | 6 ++++-- fuzzers/074-dump_all/reduce_site_types.py | 5 +++-- fuzzers/074-dump_all/reduce_tile_types.py | 5 +++-- 6 files changed, 19 insertions(+), 16 deletions(-) diff --git a/fuzzers/005-tilegrid/generate.py b/fuzzers/005-tilegrid/generate.py index 1f67174f..0bcd6d69 100644 --- a/fuzzers/005-tilegrid/generate.py +++ b/fuzzers/005-tilegrid/generate.py @@ -2,6 +2,8 @@ import os, sys, json, re +from prjxray import xjson + def load_tiles(tiles_fn): ''' @@ -57,12 +59,7 @@ def run(tiles_fn, json_fn, verbose=False): database = make_database(tiles) # Save - json.dump( - database, - open(json_fn, 'w'), - sort_keys=True, - indent=4, - separators=(',', ': ')) + xjson.pprint(open(json_fn, 'w'), database) def main(): diff --git a/fuzzers/074-dump_all/cleanup_site_pins.py b/fuzzers/074-dump_all/cleanup_site_pins.py index bf22869b..84db9300 100644 --- a/fuzzers/074-dump_all/cleanup_site_pins.py +++ b/fuzzers/074-dump_all/cleanup_site_pins.py @@ -15,6 +15,8 @@ import re import sys import copy +from prjxray import xjson + # All site names appear to follow the pattern _XY. # Generally speaking, only the tile relatively coordinates are required to # assemble arch defs, so we re-origin the coordinates to be relative to the tile @@ -113,8 +115,7 @@ def main(): site_pin['name'] = site_pin['name'][len(orig_site_name) + 1:] - json.dump(output_site_pins, sys.stdout, indent=2) - sys.stdout.write('\n') + xjson.pprint(sys.stdout, output_site_pins) if __name__ == "__main__": diff --git a/fuzzers/074-dump_all/create_node_tree.py b/fuzzers/074-dump_all/create_node_tree.py index 0721d6b7..20e4e8d6 100644 --- a/fuzzers/074-dump_all/create_node_tree.py +++ b/fuzzers/074-dump_all/create_node_tree.py @@ -1,12 +1,13 @@ import argparse import datetime import progressbar -import json import os.path import prjxray.lib import pickle import collections +from prjxray import xjson + def build_node_index(fname): node_index = {} @@ -271,7 +272,7 @@ def main(): print('{} Writing node tree'.format(datetime.datetime.now())) with open(os.path.join(args.output_dir, 'node_tree.json'), 'w') as f: - json.dump(nodes, f, indent=2) + xjson.pprint(f, nodes) if __name__ == '__main__': diff --git a/fuzzers/074-dump_all/generate_grid.py b/fuzzers/074-dump_all/generate_grid.py index 7e4c5ec0..da1c1950 100644 --- a/fuzzers/074-dump_all/generate_grid.py +++ b/fuzzers/074-dump_all/generate_grid.py @@ -13,6 +13,8 @@ import datetime import pickle import sys +from prjxray import xjson + def get_tile_grid_info(fname): with open(fname, 'r') as f: @@ -606,7 +608,7 @@ def main(): print('{} Writing tileconn'.format(datetime.datetime.now())) with open(tileconn_file, 'w') as f: - json.dump(tileconn, f, indent=2) + xjson.pprint(f, tileconn) else: with open(wire_map_file, 'rb') as f: wire_map = pickle.load(f) @@ -651,7 +653,7 @@ def main(): if len(error_nodes) > 0: error_nodes_file = os.path.join(args.output_dir, 'error_nodes.json') with open(error_nodes_file, 'w') as f: - json.dump(error_nodes, f, indent=2) + xjson.pprint(f, error_nodes) ignored_wires = [] ignored_wires_file = args.ignored_wires diff --git a/fuzzers/074-dump_all/reduce_site_types.py b/fuzzers/074-dump_all/reduce_site_types.py index 8041f945..b102d10d 100644 --- a/fuzzers/074-dump_all/reduce_site_types.py +++ b/fuzzers/074-dump_all/reduce_site_types.py @@ -11,7 +11,8 @@ import prjxray.lib import os import os.path import re -import json + +from prxray import xjson def main(): @@ -55,7 +56,7 @@ def main(): with open(os.path.join(args.output_dir, 'site_type_{}.json'.format(site_type)), 'w') as f: - json.dump(proto_site_type, f, indent=2) + xjson.pprint(f, proto_site_type) if __name__ == '__main__': diff --git a/fuzzers/074-dump_all/reduce_tile_types.py b/fuzzers/074-dump_all/reduce_tile_types.py index a065132a..7e8bdcf9 100644 --- a/fuzzers/074-dump_all/reduce_tile_types.py +++ b/fuzzers/074-dump_all/reduce_tile_types.py @@ -18,6 +18,7 @@ import multiprocessing import os import functools +from prxray import xjson def check_and_strip_prefix(name, prefix): assert name.startswith(prefix), repr((name, prefix)) @@ -323,10 +324,10 @@ def main(): with open(os.path.join( args.output_dir, 'tile_type_{}_site_type_{}.json'.format( tile_type, site_types[site_type]['type'])), 'w') as f: - json.dump(site_types[site_type], f, indent=2) + xjson.pprint(f, site_types[site_type]) with open(tile_type_file, 'w') as f: - json.dump(reduced_tile, f, indent=2) + xjson.pprint(f, reduced_tile) if __name__ == '__main__': From 51ed9fee03c0acedfeb62e96635fb18d2e9b8cab Mon Sep 17 00:00:00 2001 From: Tim 'mithro' Ansell Date: Thu, 13 Dec 2018 15:12:33 -0800 Subject: [PATCH 5/9] Add pyyaml as a requirement. Signed-off-by: Tim 'mithro' Ansell --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 6788c6c2..85e6ac6c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,6 +3,7 @@ intervaltree numpy progressbar2 pyjson5 +pyyaml scipy sympy yapf==0.24.0 From a09ded06244a60267c993354eef8ffb659680a46 Mon Sep 17 00:00:00 2001 From: Tim 'mithro' Ansell Date: Thu, 13 Dec 2018 15:15:11 -0800 Subject: [PATCH 6/9] Fix unicode issues. Signed-off-by: Tim 'mithro' Ansell --- prjxray/xjson.py | 12 ++++++++++-- prjxray/xyaml.py | 8 +++++++- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/prjxray/xjson.py b/prjxray/xjson.py index c45d7911..a512dc99 100644 --- a/prjxray/xjson.py +++ b/prjxray/xjson.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 -import re +import io import json +import re import sys @@ -48,9 +49,16 @@ def sort(data): def pprint(f, data): + detach = False + if not isinstance(f, io.TextIOBase): + detach = True + f = io.TextIOWrapper(f) sort(data) - json.dump(d, f, sort_keys=True, indent=4) + json.dump(data, f, sort_keys=True, indent=4) f.write('\n') + f.flush() + if detach: + f.detach() if __name__ == "__main__": diff --git a/prjxray/xyaml.py b/prjxray/xyaml.py index 40e3bc2f..a7a8754d 100644 --- a/prjxray/xyaml.py +++ b/prjxray/xyaml.py @@ -5,9 +5,13 @@ import yaml import json import unittest +from prjxray import xjson + def load(f): data = f.read() + if isinstance(data, bytes): + data = data.decode('utf-8') # Strip out of ! data = re.sub("!<[^>]*>", "", data) return yaml.load(io.StringIO(data)) @@ -15,7 +19,9 @@ def load(f): def tojson(f): d = load(f) - return json.dumps(d, sort_keys=True, indent=4) + o = io.StringIO() + xjson.pprint(o, d) + return o.getvalue() class XYamlTest(unittest.TestCase): From 8d61ee5ac07a91e441b079f49859d0ce2618dab6 Mon Sep 17 00:00:00 2001 From: Tim 'mithro' Ansell Date: Thu, 13 Dec 2018 18:23:28 -0800 Subject: [PATCH 7/9] `make format` on new files. Signed-off-by: Tim 'mithro' Ansell --- fuzzers/074-dump_all/reduce_tile_types.py | 1 + prjxray/xjson.py | 6 ++++-- prjxray/xyaml.py | 7 +++++-- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/fuzzers/074-dump_all/reduce_tile_types.py b/fuzzers/074-dump_all/reduce_tile_types.py index 7e8bdcf9..bbe81a93 100644 --- a/fuzzers/074-dump_all/reduce_tile_types.py +++ b/fuzzers/074-dump_all/reduce_tile_types.py @@ -20,6 +20,7 @@ import functools from prxray import xjson + def check_and_strip_prefix(name, prefix): assert name.startswith(prefix), repr((name, prefix)) return name[len(prefix):] diff --git a/prjxray/xjson.py b/prjxray/xjson.py index a512dc99..ba675dc1 100644 --- a/prjxray/xjson.py +++ b/prjxray/xjson.py @@ -13,7 +13,7 @@ def extract_numbers(s): ('VBRK_WR', 1, 'END', 2) """ bits = [] - for m in re.finditer("([^0-9]*)([0-9]*)",s): + for m in re.finditer("([^0-9]*)([0-9]*)", s): if m.group(1): bits.append(m.group(1)) if m.group(2): @@ -25,10 +25,12 @@ def sort(data): # FIXME: We assume that a list is a tileconn.json format... if isinstance(data, list): for o in data: - o['wire_pairs'].sort(key=lambda o: (extract_numbers(o[0]), extract_numbers(o[1]))) + o['wire_pairs'].sort( + key=lambda o: (extract_numbers(o[0]), extract_numbers(o[1]))) data.sort(key=lambda o: (o['tile_types'], o['grid_deltas'])) else: + def walker(o, f): if isinstance(o, dict): for i in o.values(): diff --git a/prjxray/xyaml.py b/prjxray/xyaml.py index a7a8754d..5b9d8a7f 100644 --- a/prjxray/xyaml.py +++ b/prjxray/xyaml.py @@ -26,7 +26,8 @@ def tojson(f): class XYamlTest(unittest.TestCase): def test(self): - s = io.StringIO("""\ + s = io.StringIO( + """\ ! idcode: 0x362d093 global_clock_regions: @@ -40,7 +41,8 @@ global_clock_regions: frame_count: 42 """) djson = tojson(s) - self.assertMultiLineEqual(djson, """\ + self.assertMultiLineEqual( + djson, """\ { "global_clock_regions": { "top": { @@ -62,6 +64,7 @@ global_clock_regions: "idcode": 56807571 }""") + if __name__ == "__main__": import sys if len(sys.argv) == 1: From 8d3e95cd1a54a4ad877aeaf6ed1dee2dea17428d Mon Sep 17 00:00:00 2001 From: Tim 'mithro' Ansell Date: Fri, 14 Dec 2018 09:58:15 -0800 Subject: [PATCH 8/9] Move xjson / xyaml to utils instead of prjxray. Signed-off-by: Tim 'mithro' Ansell --- fuzzers/001-part-yaml/Makefile | 2 +- fuzzers/005-tilegrid/generate.py | 2 +- fuzzers/074-dump_all/cleanup_site_pins.py | 2 +- fuzzers/074-dump_all/create_node_tree.py | 2 +- fuzzers/074-dump_all/generate_grid.py | 2 +- fuzzers/074-dump_all/reduce_site_types.py | 2 +- fuzzers/074-dump_all/reduce_tile_types.py | 2 +- {prjxray => utils}/xjson.py | 0 {prjxray => utils}/xyaml.py | 2 +- 9 files changed, 8 insertions(+), 8 deletions(-) rename {prjxray => utils}/xjson.py (100%) rename {prjxray => utils}/xyaml.py (98%) diff --git a/fuzzers/001-part-yaml/Makefile b/fuzzers/001-part-yaml/Makefile index 2de00c35..d4d84248 100644 --- a/fuzzers/001-part-yaml/Makefile +++ b/fuzzers/001-part-yaml/Makefile @@ -10,7 +10,7 @@ $(SPECIMENS): Makefile.specimen pushdb: cp ${XRAY_PART}.yaml ${XRAY_DATABASE_DIR}/$(XRAY_DATABASE) - python -m prjxray.xyaml ${XRAY_PART}.yaml > ${XRAY_DATABASE_DIR}/$(XRAY_DATABASE)/${XRAY_PART}.json + python -m utils.xyaml ${XRAY_PART}.yaml > ${XRAY_DATABASE_DIR}/$(XRAY_DATABASE)/${XRAY_PART}.json run: $(MAKE) clean diff --git a/fuzzers/005-tilegrid/generate.py b/fuzzers/005-tilegrid/generate.py index 0bcd6d69..6924b69b 100644 --- a/fuzzers/005-tilegrid/generate.py +++ b/fuzzers/005-tilegrid/generate.py @@ -2,7 +2,7 @@ import os, sys, json, re -from prjxray import xjson +from utils import xjson def load_tiles(tiles_fn): diff --git a/fuzzers/074-dump_all/cleanup_site_pins.py b/fuzzers/074-dump_all/cleanup_site_pins.py index 84db9300..5ae67494 100644 --- a/fuzzers/074-dump_all/cleanup_site_pins.py +++ b/fuzzers/074-dump_all/cleanup_site_pins.py @@ -15,7 +15,7 @@ import re import sys import copy -from prjxray import xjson +from utils import xjson # All site names appear to follow the pattern _XY. # Generally speaking, only the tile relatively coordinates are required to diff --git a/fuzzers/074-dump_all/create_node_tree.py b/fuzzers/074-dump_all/create_node_tree.py index 20e4e8d6..e7834ec8 100644 --- a/fuzzers/074-dump_all/create_node_tree.py +++ b/fuzzers/074-dump_all/create_node_tree.py @@ -6,7 +6,7 @@ import prjxray.lib import pickle import collections -from prjxray import xjson +from utils import xjson def build_node_index(fname): diff --git a/fuzzers/074-dump_all/generate_grid.py b/fuzzers/074-dump_all/generate_grid.py index da1c1950..fb55e8e6 100644 --- a/fuzzers/074-dump_all/generate_grid.py +++ b/fuzzers/074-dump_all/generate_grid.py @@ -13,7 +13,7 @@ import datetime import pickle import sys -from prjxray import xjson +from utils import xjson def get_tile_grid_info(fname): diff --git a/fuzzers/074-dump_all/reduce_site_types.py b/fuzzers/074-dump_all/reduce_site_types.py index b102d10d..517a8335 100644 --- a/fuzzers/074-dump_all/reduce_site_types.py +++ b/fuzzers/074-dump_all/reduce_site_types.py @@ -12,7 +12,7 @@ import os import os.path import re -from prxray import xjson +from utils import xjson def main(): diff --git a/fuzzers/074-dump_all/reduce_tile_types.py b/fuzzers/074-dump_all/reduce_tile_types.py index bbe81a93..58163699 100644 --- a/fuzzers/074-dump_all/reduce_tile_types.py +++ b/fuzzers/074-dump_all/reduce_tile_types.py @@ -18,7 +18,7 @@ import multiprocessing import os import functools -from prxray import xjson +from utils import xjson def check_and_strip_prefix(name, prefix): diff --git a/prjxray/xjson.py b/utils/xjson.py similarity index 100% rename from prjxray/xjson.py rename to utils/xjson.py diff --git a/prjxray/xyaml.py b/utils/xyaml.py similarity index 98% rename from prjxray/xyaml.py rename to utils/xyaml.py index 5b9d8a7f..08e9e357 100644 --- a/prjxray/xyaml.py +++ b/utils/xyaml.py @@ -5,7 +5,7 @@ import yaml import json import unittest -from prjxray import xjson +from utils import xjson def load(f): From 60e3f6667439d743534e64c24e66bcb649619a1c Mon Sep 17 00:00:00 2001 From: Tim 'mithro' Ansell Date: Mon, 17 Dec 2018 16:28:42 -0800 Subject: [PATCH 9/9] Make sure YAML is installed. Signed-off-by: Tim 'mithro' Ansell --- Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 517d6226..437336e4 100644 --- a/Makefile +++ b/Makefile @@ -6,9 +6,10 @@ TCL_FORMAT ?= utils//tcl-reformat.sh IN_ENV = if [ -e env/bin/activate ]; then . env/bin/activate; fi; env: - virtualenv --python=python3 env + virtualenv --python=python3 --system-site-packages env . env/bin/activate; pip install -r requirements.txt ln -sf $(PWD)/prjxray env/lib/python3.*/site-packages/ + python -c "import yaml" || (echo "Unable to find python-yaml" && exit 1) build: git submodule update --init --recursive