Merge pull request #328 from mithro/master

JSON improvements

Makefile

@@ -6,9 +6,10 @@ TCL_FORMAT ?= utils//tcl-reformat.sh
 
 IN_ENV = if [ -e env/bin/activate ]; then . env/bin/activate; fi;
 env:
-	virtualenv --python=python3 env
+	virtualenv --python=python3 --system-site-packages env
 	. env/bin/activate; pip install -r requirements.txt
 	ln -sf $(PWD)/prjxray env/lib/python3.*/site-packages/
+	python -c "import yaml" || (echo "Unable to find python-yaml" && exit 1)
 
 build:
 	git submodule update --init --recursive

fuzzers/001-part-yaml/Makefile

@@ -10,6 +10,7 @@ $(SPECIMENS): Makefile.specimen
 
 pushdb:
 	cp ${XRAY_PART}.yaml ${XRAY_DATABASE_DIR}/$(XRAY_DATABASE)
+	python -m utils.xyaml ${XRAY_PART}.yaml > ${XRAY_DATABASE_DIR}/$(XRAY_DATABASE)/${XRAY_PART}.json
 
 run:
 	$(MAKE) clean

fuzzers/005-tilegrid/generate.py

@@ -2,6 +2,8 @@
 
 import os, sys, json, re
 
+from utils import xjson
+
 
 def load_tiles(tiles_fn):
     '''
@@ -57,12 +59,7 @@ def run(tiles_fn, json_fn, verbose=False):
     database = make_database(tiles)
 
     # Save
-    json.dump(
-        database,
-        open(json_fn, 'w'),
-        sort_keys=True,
-        indent=4,
-        separators=(',', ': '))
+    xjson.pprint(open(json_fn, 'w'), database)
 
 
 def main():

fuzzers/074-dump_all/cleanup_site_pins.py

@@ -15,6 +15,8 @@ import re
 import sys
 import copy
 
+from utils import xjson
+
 # All site names appear to follow the pattern <type>_X<abs coord>Y<abs coord>.
 # Generally speaking, only the tile relatively coordinates are required to
 # assemble arch defs, so we re-origin the coordinates to be relative to the tile
@@ -113,8 +115,7 @@ def main():
 
             site_pin['name'] = site_pin['name'][len(orig_site_name) + 1:]
 
-    json.dump(output_site_pins, sys.stdout, indent=2)
-    sys.stdout.write('\n')
+    xjson.pprint(sys.stdout, output_site_pins)
 
 
 if __name__ == "__main__":

fuzzers/074-dump_all/create_node_tree.py

@@ -1,12 +1,13 @@
 import argparse
 import datetime
 import progressbar
-import json
 import os.path
 import prjxray.lib
 import pickle
 import collections
 
+from utils import xjson
+
 
 def build_node_index(fname):
     node_index = {}
@@ -271,7 +272,7 @@ def main():
 
     print('{} Writing node tree'.format(datetime.datetime.now()))
     with open(os.path.join(args.output_dir, 'node_tree.json'), 'w') as f:
-        json.dump(nodes, f, indent=2)
+        xjson.pprint(f, nodes)
 
 
 if __name__ == '__main__':

fuzzers/074-dump_all/generate_grid.py

@@ -13,6 +13,8 @@ import datetime
 import pickle
 import sys
 
+from utils import xjson
+
 
 def get_tile_grid_info(fname):
     with open(fname, 'r') as f:
@@ -606,7 +608,7 @@ def main():
 
         print('{} Writing tileconn'.format(datetime.datetime.now()))
         with open(tileconn_file, 'w') as f:
-            json.dump(tileconn, f, indent=2)
+            xjson.pprint(f, tileconn)
     else:
         with open(wire_map_file, 'rb') as f:
             wire_map = pickle.load(f)
@@ -651,7 +653,7 @@ def main():
     if len(error_nodes) > 0:
         error_nodes_file = os.path.join(args.output_dir, 'error_nodes.json')
         with open(error_nodes_file, 'w') as f:
-            json.dump(error_nodes, f, indent=2)
+            xjson.pprint(f, error_nodes)
 
         ignored_wires = []
         ignored_wires_file = args.ignored_wires

fuzzers/074-dump_all/reduce_site_types.py

@@ -11,7 +11,8 @@ import prjxray.lib
 import os
 import os.path
 import re
-import json
+
+from utils import xjson
 
 
 def main():
@@ -55,7 +56,7 @@ def main():
         with open(os.path.join(args.output_dir,
                                'site_type_{}.json'.format(site_type)),
                   'w') as f:
-            json.dump(proto_site_type, f, indent=2)
+            xjson.pprint(f, proto_site_type)
 
 
 if __name__ == '__main__':

fuzzers/074-dump_all/reduce_tile_types.py

@@ -18,6 +18,8 @@ import multiprocessing
 import os
 import functools
 
+from utils import xjson
+
 
 def check_and_strip_prefix(name, prefix):
     assert name.startswith(prefix), repr((name, prefix))
@@ -323,10 +325,10 @@ def main():
             with open(os.path.join(
                     args.output_dir, 'tile_type_{}_site_type_{}.json'.format(
                         tile_type, site_types[site_type]['type'])), 'w') as f:
-                json.dump(site_types[site_type], f, indent=2)
+                xjson.pprint(f, site_types[site_type])
 
         with open(tile_type_file, 'w') as f:
-            json.dump(reduced_tile, f, indent=2)
+            xjson.pprint(f, reduced_tile)
 
 
 if __name__ == '__main__':

requirements.txt

@@ -3,6 +3,7 @@ intervaltree
 numpy
 progressbar2
 pyjson5
+pyyaml
 scipy
 sympy
 yapf==0.24.0

utils/xjson.py

@@ -0,0 +1,73 @@
+#!/usr/bin/env python3
+import io
+import json
+import re
+import sys
+
+
+def extract_numbers(s):
+    """
+    >>> extract_numbers("CLK_HROW_WR10END2_3")
+    ('CLK_HROW_WR', 10, 'END', 2, '_', 3)
+    >>> extract_numbers("VBRK_WR1END2")
+    ('VBRK_WR', 1, 'END', 2)
+    """
+    bits = []
+    for m in re.finditer("([^0-9]*)([0-9]*)", s):
+        if m.group(1):
+            bits.append(m.group(1))
+        if m.group(2):
+            bits.append(int(m.group(2)))
+    return tuple(bits)
+
+
+def sort(data):
+    # FIXME: We assume that a list is a tileconn.json format...
+    if isinstance(data, list):
+        for o in data:
+            o['wire_pairs'].sort(
+                key=lambda o: (extract_numbers(o[0]), extract_numbers(o[1])))
+
+        data.sort(key=lambda o: (o['tile_types'], o['grid_deltas']))
+    else:
+
+        def walker(o, f):
+            if isinstance(o, dict):
+                for i in o.values():
+                    walker(i, f)
+            elif isinstance(o, list):
+                for i in o:
+                    walker(i, f)
+            f(o)
+
+        def f(o):
+            if isinstance(o, list):
+                if len(o) > 2:
+                    strings = all(isinstance(x, str) for x in o)
+                    if strings:
+                        o.sort()
+
+        walker(data, f)
+
+
+def pprint(f, data):
+    detach = False
+    if not isinstance(f, io.TextIOBase):
+        detach = True
+        f = io.TextIOWrapper(f)
+    sort(data)
+    json.dump(data, f, sort_keys=True, indent=4)
+    f.write('\n')
+    f.flush()
+    if detach:
+        f.detach()
+
+
+if __name__ == "__main__":
+    if len(sys.argv) == 1:
+        import doctest
+        doctest.testmod()
+    else:
+        assert len(sys.argv) == 2
+        d = json.load(open(sys.argv[1]))
+        pprint(sys.stdout, d)

utils/xyaml.py

@@ -0,0 +1,74 @@
+#!/usr/bin/env python3
+import io
+import re
+import yaml
+import json
+import unittest
+
+from utils import xjson
+
+
+def load(f):
+    data = f.read()
+    if isinstance(data, bytes):
+        data = data.decode('utf-8')
+    # Strip out of !<tags>
+    data = re.sub("!<[^>]*>", "", data)
+    return yaml.load(io.StringIO(data))
+
+
+def tojson(f):
+    d = load(f)
+    o = io.StringIO()
+    xjson.pprint(o, d)
+    return o.getvalue()
+
+
+class XYamlTest(unittest.TestCase):
+    def test(self):
+        s = io.StringIO(
+            """\
+!<xilinx/xc7series/part>
+idcode: 0x362d093
+global_clock_regions:
+  top: !<xilinx/xc7series/global_clock_region>
+    rows:
+      0: !<xilinx/xc7series/row>
+        configuration_buses:
+          CLB_IO_CLK: !<xilinx/xc7series/configuration_bus>
+            configuration_columns:
+              0: !<xilinx/xc7series/configuration_column>
+                frame_count: 42
+""")
+        djson = tojson(s)
+        self.assertMultiLineEqual(
+            djson, """\
+{
+    "global_clock_regions": {
+        "top": {
+            "rows": {
+                "0": {
+                    "configuration_buses": {
+                        "CLB_IO_CLK": {
+                            "configuration_columns": {
+                                "0": {
+                                    "frame_count": 42
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+        }
+    },
+    "idcode": 56807571
+}""")
+
+
+if __name__ == "__main__":
+    import sys
+    if len(sys.argv) == 1:
+        unittest.main()
+    else:
+        assert len(sys.argv) == 2
+        print(tojson(open(sys.argv[1])))