diff --git a/Makefile b/Makefile index 517d6226..437336e4 100644 --- a/Makefile +++ b/Makefile @@ -6,9 +6,10 @@ TCL_FORMAT ?= utils//tcl-reformat.sh IN_ENV = if [ -e env/bin/activate ]; then . env/bin/activate; fi; env: - virtualenv --python=python3 env + virtualenv --python=python3 --system-site-packages env . env/bin/activate; pip install -r requirements.txt ln -sf $(PWD)/prjxray env/lib/python3.*/site-packages/ + python -c "import yaml" || (echo "Unable to find python-yaml" && exit 1) build: git submodule update --init --recursive diff --git a/fuzzers/001-part-yaml/Makefile b/fuzzers/001-part-yaml/Makefile index 3e049293..d4d84248 100644 --- a/fuzzers/001-part-yaml/Makefile +++ b/fuzzers/001-part-yaml/Makefile @@ -10,6 +10,7 @@ $(SPECIMENS): Makefile.specimen pushdb: cp ${XRAY_PART}.yaml ${XRAY_DATABASE_DIR}/$(XRAY_DATABASE) + python -m utils.xyaml ${XRAY_PART}.yaml > ${XRAY_DATABASE_DIR}/$(XRAY_DATABASE)/${XRAY_PART}.json run: $(MAKE) clean diff --git a/fuzzers/005-tilegrid/generate.py b/fuzzers/005-tilegrid/generate.py index 1f67174f..6924b69b 100644 --- a/fuzzers/005-tilegrid/generate.py +++ b/fuzzers/005-tilegrid/generate.py @@ -2,6 +2,8 @@ import os, sys, json, re +from utils import xjson + def load_tiles(tiles_fn): ''' @@ -57,12 +59,7 @@ def run(tiles_fn, json_fn, verbose=False): database = make_database(tiles) # Save - json.dump( - database, - open(json_fn, 'w'), - sort_keys=True, - indent=4, - separators=(',', ': ')) + xjson.pprint(open(json_fn, 'w'), database) def main(): diff --git a/fuzzers/074-dump_all/cleanup_site_pins.py b/fuzzers/074-dump_all/cleanup_site_pins.py index bf22869b..5ae67494 100644 --- a/fuzzers/074-dump_all/cleanup_site_pins.py +++ b/fuzzers/074-dump_all/cleanup_site_pins.py @@ -15,6 +15,8 @@ import re import sys import copy +from utils import xjson + # All site names appear to follow the pattern _XY. # Generally speaking, only the tile relatively coordinates are required to # assemble arch defs, so we re-origin the coordinates to be relative to the tile @@ -113,8 +115,7 @@ def main(): site_pin['name'] = site_pin['name'][len(orig_site_name) + 1:] - json.dump(output_site_pins, sys.stdout, indent=2) - sys.stdout.write('\n') + xjson.pprint(sys.stdout, output_site_pins) if __name__ == "__main__": diff --git a/fuzzers/074-dump_all/create_node_tree.py b/fuzzers/074-dump_all/create_node_tree.py index 0721d6b7..e7834ec8 100644 --- a/fuzzers/074-dump_all/create_node_tree.py +++ b/fuzzers/074-dump_all/create_node_tree.py @@ -1,12 +1,13 @@ import argparse import datetime import progressbar -import json import os.path import prjxray.lib import pickle import collections +from utils import xjson + def build_node_index(fname): node_index = {} @@ -271,7 +272,7 @@ def main(): print('{} Writing node tree'.format(datetime.datetime.now())) with open(os.path.join(args.output_dir, 'node_tree.json'), 'w') as f: - json.dump(nodes, f, indent=2) + xjson.pprint(f, nodes) if __name__ == '__main__': diff --git a/fuzzers/074-dump_all/generate_grid.py b/fuzzers/074-dump_all/generate_grid.py index 7e4c5ec0..fb55e8e6 100644 --- a/fuzzers/074-dump_all/generate_grid.py +++ b/fuzzers/074-dump_all/generate_grid.py @@ -13,6 +13,8 @@ import datetime import pickle import sys +from utils import xjson + def get_tile_grid_info(fname): with open(fname, 'r') as f: @@ -606,7 +608,7 @@ def main(): print('{} Writing tileconn'.format(datetime.datetime.now())) with open(tileconn_file, 'w') as f: - json.dump(tileconn, f, indent=2) + xjson.pprint(f, tileconn) else: with open(wire_map_file, 'rb') as f: wire_map = pickle.load(f) @@ -651,7 +653,7 @@ def main(): if len(error_nodes) > 0: error_nodes_file = os.path.join(args.output_dir, 'error_nodes.json') with open(error_nodes_file, 'w') as f: - json.dump(error_nodes, f, indent=2) + xjson.pprint(f, error_nodes) ignored_wires = [] ignored_wires_file = args.ignored_wires diff --git a/fuzzers/074-dump_all/reduce_site_types.py b/fuzzers/074-dump_all/reduce_site_types.py index 8041f945..517a8335 100644 --- a/fuzzers/074-dump_all/reduce_site_types.py +++ b/fuzzers/074-dump_all/reduce_site_types.py @@ -11,7 +11,8 @@ import prjxray.lib import os import os.path import re -import json + +from utils import xjson def main(): @@ -55,7 +56,7 @@ def main(): with open(os.path.join(args.output_dir, 'site_type_{}.json'.format(site_type)), 'w') as f: - json.dump(proto_site_type, f, indent=2) + xjson.pprint(f, proto_site_type) if __name__ == '__main__': diff --git a/fuzzers/074-dump_all/reduce_tile_types.py b/fuzzers/074-dump_all/reduce_tile_types.py index a065132a..58163699 100644 --- a/fuzzers/074-dump_all/reduce_tile_types.py +++ b/fuzzers/074-dump_all/reduce_tile_types.py @@ -18,6 +18,8 @@ import multiprocessing import os import functools +from utils import xjson + def check_and_strip_prefix(name, prefix): assert name.startswith(prefix), repr((name, prefix)) @@ -323,10 +325,10 @@ def main(): with open(os.path.join( args.output_dir, 'tile_type_{}_site_type_{}.json'.format( tile_type, site_types[site_type]['type'])), 'w') as f: - json.dump(site_types[site_type], f, indent=2) + xjson.pprint(f, site_types[site_type]) with open(tile_type_file, 'w') as f: - json.dump(reduced_tile, f, indent=2) + xjson.pprint(f, reduced_tile) if __name__ == '__main__': diff --git a/requirements.txt b/requirements.txt index 6788c6c2..85e6ac6c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,6 +3,7 @@ intervaltree numpy progressbar2 pyjson5 +pyyaml scipy sympy yapf==0.24.0 diff --git a/utils/xjson.py b/utils/xjson.py new file mode 100644 index 00000000..ba675dc1 --- /dev/null +++ b/utils/xjson.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python3 +import io +import json +import re +import sys + + +def extract_numbers(s): + """ + >>> extract_numbers("CLK_HROW_WR10END2_3") + ('CLK_HROW_WR', 10, 'END', 2, '_', 3) + >>> extract_numbers("VBRK_WR1END2") + ('VBRK_WR', 1, 'END', 2) + """ + bits = [] + for m in re.finditer("([^0-9]*)([0-9]*)", s): + if m.group(1): + bits.append(m.group(1)) + if m.group(2): + bits.append(int(m.group(2))) + return tuple(bits) + + +def sort(data): + # FIXME: We assume that a list is a tileconn.json format... + if isinstance(data, list): + for o in data: + o['wire_pairs'].sort( + key=lambda o: (extract_numbers(o[0]), extract_numbers(o[1]))) + + data.sort(key=lambda o: (o['tile_types'], o['grid_deltas'])) + else: + + def walker(o, f): + if isinstance(o, dict): + for i in o.values(): + walker(i, f) + elif isinstance(o, list): + for i in o: + walker(i, f) + f(o) + + def f(o): + if isinstance(o, list): + if len(o) > 2: + strings = all(isinstance(x, str) for x in o) + if strings: + o.sort() + + walker(data, f) + + +def pprint(f, data): + detach = False + if not isinstance(f, io.TextIOBase): + detach = True + f = io.TextIOWrapper(f) + sort(data) + json.dump(data, f, sort_keys=True, indent=4) + f.write('\n') + f.flush() + if detach: + f.detach() + + +if __name__ == "__main__": + if len(sys.argv) == 1: + import doctest + doctest.testmod() + else: + assert len(sys.argv) == 2 + d = json.load(open(sys.argv[1])) + pprint(sys.stdout, d) diff --git a/utils/xyaml.py b/utils/xyaml.py new file mode 100644 index 00000000..08e9e357 --- /dev/null +++ b/utils/xyaml.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python3 +import io +import re +import yaml +import json +import unittest + +from utils import xjson + + +def load(f): + data = f.read() + if isinstance(data, bytes): + data = data.decode('utf-8') + # Strip out of ! + data = re.sub("!<[^>]*>", "", data) + return yaml.load(io.StringIO(data)) + + +def tojson(f): + d = load(f) + o = io.StringIO() + xjson.pprint(o, d) + return o.getvalue() + + +class XYamlTest(unittest.TestCase): + def test(self): + s = io.StringIO( + """\ +! +idcode: 0x362d093 +global_clock_regions: + top: ! + rows: + 0: ! + configuration_buses: + CLB_IO_CLK: ! + configuration_columns: + 0: ! + frame_count: 42 +""") + djson = tojson(s) + self.assertMultiLineEqual( + djson, """\ +{ + "global_clock_regions": { + "top": { + "rows": { + "0": { + "configuration_buses": { + "CLB_IO_CLK": { + "configuration_columns": { + "0": { + "frame_count": 42 + } + } + } + } + } + } + } + }, + "idcode": 56807571 +}""") + + +if __name__ == "__main__": + import sys + if len(sys.argv) == 1: + unittest.main() + else: + assert len(sys.argv) == 2 + print(tojson(open(sys.argv[1])))