diff --git a/fuzzers/074-dump_all/Makefile b/fuzzers/074-dump_all/Makefile index a4ac483c..ed9b8d52 100644 --- a/fuzzers/074-dump_all/Makefile +++ b/fuzzers/074-dump_all/Makefile @@ -22,6 +22,7 @@ pushdb: rm ${XRAY_FAMILY_DIR}/tile_type_*_site_type_*.json cp $(BUILD_DIR)/output/site_type_*.json ${XRAY_FAMILY_DIR}/ cp $(BUILD_DIR)/output/tileconn.json ${XRAY_FAMILY_DIR}/$(XRAY_PART)/ + cp $(BUILD_DIR)/output/node_wires.json ${XRAY_FAMILY_DIR}/$(XRAY_PART)/ $(SPECIMENS_OK): bash generate.sh $(subst /OK,,$@) -p=$(MAX_VIVADO_PROCESS) -t=$(MAX_TILES_INSTANCE) -n=$(MAX_NODES_INSTANCE) diff --git a/fuzzers/074-dump_all/check_nodes.py b/fuzzers/074-dump_all/check_nodes.py new file mode 100644 index 00000000..f4f4d391 --- /dev/null +++ b/fuzzers/074-dump_all/check_nodes.py @@ -0,0 +1,146 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# Copyright (C) 2020 The Project X-Ray Authors. +# +# Use of this source code is governed by a ISC-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/ISC +# +# SPDX-License-Identifier: ISC +""" Load tileconn.json and node_wires.json and verify both node names and +node <-> wire mapping against raw node data. """ + +import argparse +import datetime +import json +import multiprocessing +import os.path +import progressbar +import pyjson5 as json5 + +from prjxray import util, lib +from prjxray.grid import Grid +from prjxray.connections import Connections +from prjxray.node_model import NodeModel + + +def read_json5(fname): + with open(fname, 'r') as f: + return json5.load(f) + + +def read_raw_node_data(pool, root_dir): + """ Read raw node data from root dir. """ + _, nodes = lib.read_root_csv(root_dir) + + raw_node_data = [] + with progressbar.ProgressBar(max_value=len(nodes)) as bar: + for idx, node in enumerate(pool.imap_unordered( + read_json5, + nodes, + chunksize=20, + )): + bar.update(idx) + raw_node_data.append(node) + bar.update(idx + 1) + + return raw_node_data + + +def main(): + parser = argparse.ArgumentParser( + description="Verify tileconn and node_wires.") + parser.add_argument('--root_dir', required=True) + parser.add_argument('--output_dir', required=True) + parser.add_argument('--max_cpu', type=int, default=10) + parser.add_argument('--ignored_wires') + + args = parser.parse_args() + + print('{} Reading tilegrid'.format(datetime.datetime.now())) + with open(os.path.join(util.get_db_root(), util.get_part(), + 'tilegrid.json')) as f: + tilegrid = json.load(f) + grid = Grid(db=None, tilegrid=tilegrid) + + print('{} Reading tileconn'.format(datetime.datetime.now())) + with open(os.path.join(args.output_dir, 'tileconn.json')) as f: + tileconn = json.load(f) + + print( + '{} Reading tile wires from tile types'.format( + datetime.datetime.now())) + tile_wires = {} + for f in os.listdir(args.output_dir): + if f.endswith('.json') and f.startswith('tile_type_'): + if '_site_type_' in f: + continue + + tile_type = f[len('tile_type_'):-len('.json')] + with open(os.path.join(args.output_dir, f)) as fin: + tile_wires[tile_type] = json.load(fin)['wires'] + + connections = Connections( + tilegrid=tilegrid, + tileconn=tileconn, + tile_wires=tile_wires, + ) + + print('{} Reading node wires'.format(datetime.datetime.now())) + with open(os.path.join(args.output_dir, 'node_wires.json')) as f: + node_wires = json.load(f) + + print('{} Build initial node model'.format(datetime.datetime.now())) + node_model = NodeModel( + grid=grid, + connections=connections, + tile_wires=tile_wires, + node_wires=node_wires, + progressbar=progressbar.progressbar) + + print('{} Build node model'.format(datetime.datetime.now())) + nodes = set(node_model.get_nodes()) + + print('{} Read raw node data for testing'.format(datetime.datetime.now())) + + processes = min(multiprocessing.cpu_count(), args.max_cpu) + with multiprocessing.Pool(processes=processes) as pool: + raw_node_data = read_raw_node_data(pool, args.root_dir) + + print('{} Read ignored wires list'.format(datetime.datetime.now())) + ignored_wires = [] + ignored_wires_file = args.ignored_wires + if os.path.exists(ignored_wires_file): + with open(ignored_wires_file) as f: + ignored_wires = set(tuple(l.strip().split('/')) for l in f) + + print( + '{} Verify nodes against raw node data'.format( + datetime.datetime.now())) + for node in progressbar.progressbar(raw_node_data): + tile, wire = node['node'].split('/') + + assert (tile, wire) in nodes + wires_for_model = node_model.get_wires_for_node(tile, wire) + + wires = set() + for wire in node['wires']: + wire_tile, wire_name = wire['wire'].split('/') + wires.add((wire_tile, wire_name)) + + if len(wires) != len(wires_for_model): + wires2 = set(wires_for_model) + a_minus_b = wires - wires2 + b_minus_a = wires2 - wires + + assert len(b_minus_a) == 0 + + assert len(a_minus_b - ignored_wires) == 0, a_minus_b + + for tile, wire in wires_for_model: + assert (tile, wire) in wires + + +if __name__ == '__main__': + main() diff --git a/fuzzers/074-dump_all/generate_after_dump.sh b/fuzzers/074-dump_all/generate_after_dump.sh index fbb8f96b..a992ceed 100755 --- a/fuzzers/074-dump_all/generate_after_dump.sh +++ b/fuzzers/074-dump_all/generate_after_dump.sh @@ -26,3 +26,12 @@ python3 generate_grid.py \ --output_dir ${BUILD_DIR}/output \ --ignored_wires ignored_wires/${XRAY_DATABASE}/${XRAY_PART}_ignored_wires.txt \ --max_cpu=${MAX_GRID_CPU:-${DEFAULT_MAX_GRID_CPU}} +python3 node_names.py \ + --root_dir ${BUILD_DIR}/specimen_001/ \ + --output_dir ${BUILD_DIR}/output/ \ + --max_cpu=${MAX_GRID_CPU:-${DEFAULT_MAX_GRID_CPU}} +python3 check_nodes.py \ + --root_dir ${BUILD_DIR}/specimen_001/ \ + --output_dir ${BUILD_DIR}/output/ \ + --ignored_wires ignored_wires/${XRAY_DATABASE}/${XRAY_PART}_ignored_wires.txt \ + --max_cpu=${MAX_GRID_CPU:-${DEFAULT_MAX_GRID_CPU}} diff --git a/fuzzers/074-dump_all/node_names.py b/fuzzers/074-dump_all/node_names.py new file mode 100644 index 00000000..64bca5b8 --- /dev/null +++ b/fuzzers/074-dump_all/node_names.py @@ -0,0 +1,183 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# Copyright (C) 2020 The Project X-Ray Authors. +# +# Use of this source code is governed by a ISC-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/ISC +# +# SPDX-License-Identifier: ISC +""" This script creates node_wires.json, which describes how nodes are named. + +This script consumes the raw node data from root_dir and outputs +node_wires.json to the output_dir. + +The class prjxray.node_model.NodeModel can be used to reconstruct node names +and node <-> wire mapping. + +The contents of node_wires.json is: + - The set of tile type wires that are always nodes, key "node_pattern_wires" + - The set of tile wires that are nodes within the graph, key + "specific_node_wires". + +""" + +import argparse +import datetime +import json +import multiprocessing +import progressbar +import pyjson5 as json5 +import os.path + +from prjxray import util, lib +from prjxray.grid import Grid + + +def read_json5(fname): + with open(fname, 'r') as f: + return json5.load(f) + + +def main(): + parser = argparse.ArgumentParser( + description="Reduce node names for wire connections.") + parser.add_argument('--root_dir', required=True) + parser.add_argument('--output_dir', required=True) + parser.add_argument('--max_cpu', type=int, default=10) + + args = parser.parse_args() + + _, nodes = lib.read_root_csv(args.root_dir) + + processes = min(multiprocessing.cpu_count(), args.max_cpu) + pool = multiprocessing.Pool(processes=processes) + + # Read tile grid and raw node data. + print('{} Reading tilegrid'.format(datetime.datetime.now())) + with open(os.path.join(util.get_db_root(), util.get_part(), + 'tilegrid.json')) as f: + grid = Grid(db=None, tilegrid=json.load(f)) + + raw_node_data = [] + with progressbar.ProgressBar(max_value=len(nodes)) as bar: + for idx, node in enumerate(pool.imap_unordered( + read_json5, + nodes, + chunksize=20, + )): + bar.update(idx) + raw_node_data.append(node) + bar.update(idx + 1) + + node_wires = set() + remove_node_wires = set() + specific_node_wires = set() + + # Create initial node wire pattern + for node in progressbar.progressbar(raw_node_data): + if len(node['wires']) <= 1: + continue + + node_tile, node_wire = node['node'].split('/') + + for wire in node['wires']: + wire_tile, wire_name = wire['wire'].split('/') + + if node['node'] == wire['wire']: + assert node_tile == wire_tile + assert node_wire == wire_name + gridinfo = grid.gridinfo_at_tilename(node_tile) + node_wires.add((gridinfo.tile_type, wire_name)) + + print( + 'Initial number of wires that are node drivers: {}'.format( + len(node_wires))) + + # Remove exceptional node wire names, create specific_node_wires set, + # which is simply the list of wires that are nodes in the graph. + for node in progressbar.progressbar(raw_node_data): + if len(node['wires']) <= 1: + continue + + for wire in node['wires']: + wire_tile, wire_name = wire['wire'].split('/') + gridinfo = grid.gridinfo_at_tilename(wire_tile) + key = gridinfo.tile_type, wire_name + + if node['node'] == wire['wire']: + assert key in node_wires + else: + if key in node_wires: + specific_node_wires.add(node['node']) + remove_node_wires.add(key) + + # Complete the specific_node_wires list after the pruning of the + # node_pattern_wires sets. + for node in progressbar.progressbar(raw_node_data): + if len(node['wires']) <= 1: + continue + + for wire in node['wires']: + wire_tile, wire_name = wire['wire'].split('/') + gridinfo = grid.gridinfo_at_tilename(wire_tile) + key = gridinfo.tile_type, wire_name + + if key in remove_node_wires and node['node'] == wire['wire']: + specific_node_wires.add(node['node']) + + node_wires -= remove_node_wires + print( + 'Final number of wires that are node drivers: {}'.format( + len(node_wires))) + print( + 'Number of wires that are node drivers: {}'.format( + len(specific_node_wires))) + + # Verify the node wire data. + for node in progressbar.progressbar(raw_node_data): + if len(node['wires']) <= 1: + continue + + found_node_wire = False + for wire in node['wires']: + if wire['wire'] in specific_node_wires: + assert wire['wire'] == node['node'] + + found_node_wire = True + break + + if not found_node_wire: + for wire in node['wires']: + wire_tile, wire_name = wire['wire'].split('/') + gridinfo = grid.gridinfo_at_tilename(wire_tile) + key = gridinfo.tile_type, wire_name + + if key in node_wires: + assert node['node'] == wire['wire'] + else: + assert node['node'] != wire['wire'] + + # Normalize output. + tile_types = {} + for tile_type, tile_wire in node_wires: + if tile_type not in tile_types: + tile_types[tile_type] = [] + + tile_types[tile_type].append(tile_wire) + + for tile_type in tile_types: + tile_types[tile_type].sort() + + out = { + 'node_pattern_wires': tile_types, + 'specific_node_wires': sorted(specific_node_wires), + } + + with open(os.path.join(args.output_dir, 'node_wires.json'), 'w') as f: + json.dump(out, f, indent=2, sort_keys=True) + + +if __name__ == '__main__': + main() diff --git a/fuzzers/Makefile b/fuzzers/Makefile index 7c3fe0d3..b9c4dde3 100644 --- a/fuzzers/Makefile +++ b/fuzzers/Makefile @@ -172,5 +172,6 @@ roi_only: 000-init-db/run.${XRAY_PART}.ok 001-part-yaml/run.${XRAY_PART}.ok 075- # Copy tilegrid and tileconn cp ${XRAY_FAMILY_DIR}/${XRAY_EQUIV_PART}/tilegrid.json ${XRAY_FAMILY_DIR}/${XRAY_PART}/tilegrid.json cp ${XRAY_FAMILY_DIR}/${XRAY_EQUIV_PART}/tileconn.json ${XRAY_FAMILY_DIR}/${XRAY_PART}/tileconn.json + cp ${XRAY_FAMILY_DIR}/${XRAY_EQUIV_PART}/node_wires.json ${XRAY_FAMILY_DIR}/${XRAY_PART}/node_wires.json .PHONY: all clean clean_fuzzers clean_logs quick part_only roi_only diff --git a/prjxray/db.py b/prjxray/db.py index b84cacf4..98a3fde4 100644 --- a/prjxray/db.py +++ b/prjxray/db.py @@ -15,6 +15,7 @@ from prjxray import tile from prjxray import tile_segbits from prjxray import site_type from prjxray import connections +from prjxray.node_model import NodeModel def get_available_databases(prjxray_root): @@ -47,7 +48,8 @@ class Database(object): # tilegrid.json JSON object self.tilegrid = None self.tileconn = None - self.tile_types = None + self.tile_types_json = None + self.node_wires = None self.tile_types = {} self.tile_segbits = {} @@ -138,26 +140,64 @@ class Database(object): 'tileconn.json')) as f: self.tileconn = json.load(f) + def _read_node_wires(self): + """ Read node wires if not already read. """ + if self.node_wires is None: + with open(os.path.join(self.db_root, self.part, + 'node_wires.json')) as f: + self.node_wires = json.load(f) + def grid(self): """ Return Grid object for database. """ self._read_tilegrid() return grid.Grid(self, self.tilegrid) def _read_tile_types(self): - for tile_type, db in self.tile_types.items(): - with open(db.tile_type) as f: - self.tile_types[tile_type] = json.load(f) + if self.tile_types_json is None: + self.tile_types_json = {} + for tile_type, db in self.tile_types.items(): + with open(db.tile_type) as f: + self.tile_types_json[tile_type] = json.load(f) + + def _get_tile_wires(self): + self._read_tile_types() + tile_wires = dict( + (tile_type, db['wires']) + for tile_type, db in self.tile_types_json.items()) + + return tile_wires def connections(self): self._read_tilegrid() self._read_tileconn() - self._read_tile_types() - tile_wires = dict( - (tile_type, db['wires']) - for tile_type, db in self.tile_types.items()) return connections.Connections( - self.tilegrid, self.tileconn, tile_wires) + self.tilegrid, self.tileconn, self._get_tile_wires()) + + def node_model(self, progressbar=lambda x: x): + """ Get node module for specified part. + + progressbar - Should be a function that takes an iteraable, and + yields all elements from that iterable. + This can be used to generate a progressbar, for example + the module progressbar satifies this interface. + + Example: + + import progressbar + + db = Database(...) + node_model = db.node_model(progressbar.progressbar) + + """ + self._read_node_wires() + + return NodeModel( + grid=self.grid(), + connections=self.connections(), + tile_wires=self._get_tile_wires(), + node_wires=self.node_wires, + progressbar=progressbar) def get_site_types(self): return self.site_types.keys() diff --git a/prjxray/node_model.py b/prjxray/node_model.py new file mode 100644 index 00000000..879f1cb3 --- /dev/null +++ b/prjxray/node_model.py @@ -0,0 +1,156 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# Copyright (C) 2017-2020 The Project X-Ray Authors. +# +# Use of this source code is governed by a ISC-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/ISC +# +# SPDX-License-Identifier: ISC +class NodeModel(): + """ Node lookup model + + Terminology: + Wire - A segment of metal in a tile + Node - A connected set of wires + + This class can provide a list of nodes, the wires in a node and the node + that a wire belongs too. + + The name of node is always the name of one wire in the node. + + It is recommended that this class be constructed by calling + Database.node_model rather than constructing this class directly. + + """ + + def __init__( + self, grid, connections, tile_wires, node_wires, progressbar=None): + self.grid = grid + self.connections = connections + self.tile_wires = tile_wires + self.specific_node_wires = set(node_wires['specific_node_wires']) + + node_pattern_wires = node_wires['node_pattern_wires'] + self.node_pattern_wires = {} + for tile_type in node_pattern_wires: + assert tile_type not in self.node_pattern_wires + self.node_pattern_wires[tile_type] = set( + node_pattern_wires[tile_type]) + + for tile_type in self.tile_wires: + if tile_type not in self.node_pattern_wires: + self.node_pattern_wires[tile_type] = set() + + self.nodes = None + + self.wire_to_node_map = None + + if progressbar is None: + self.progressbar = lambda x: x + else: + self.progressbar = progressbar + + def _build_nodes(self): + tile_wire_map = {} + wires = {} + flat_wires = [] + + for tile in self.progressbar(self.grid.tiles()): + gridinfo = self.grid.gridinfo_at_tilename(tile) + tile_type = gridinfo.tile_type + + for wire in self.tile_wires[tile_type]: + wire_pkey = len(flat_wires) + tile_wire_map[(tile, wire)] = wire_pkey + flat_wires.append((tile, wire)) + wires[wire_pkey] = None + + for connection in self.progressbar(self.connections.get_connections()): + a_pkey = tile_wire_map[( + connection.wire_a.tile, connection.wire_a.wire)] + b_pkey = tile_wire_map[( + connection.wire_b.tile, connection.wire_b.wire)] + + a_node = wires[a_pkey] + b_node = wires[b_pkey] + + if a_node is None: + a_node = set((a_pkey, )) + + if b_node is None: + b_node = set((b_pkey, )) + + if a_node is not b_node: + a_node |= b_node + + for wire in a_node: + wires[wire] = a_node + + nodes = {} + for wire_pkey, node in self.progressbar(wires.items()): + if node is None: + node = set((wire_pkey, )) + + assert wire_pkey in node + + nodes[id(node)] = node + + def get_node_wire_for_wires(wire_pkeys): + if len(wire_pkeys) == 1: + for wire_pkey in wire_pkeys: + return flat_wires[wire_pkey] + + for wire_pkey in wire_pkeys: + tile, wire = flat_wires[wire_pkey] + + if '{}/{}'.format(tile, wire) in self.specific_node_wires: + return tile, wire + + for wire_pkey in wire_pkeys: + tile, wire = flat_wires[wire_pkey] + + gridinfo = self.grid.gridinfo_at_tilename(tile) + + if wire in self.node_pattern_wires[gridinfo.tile_type]: + return tile, wire + + return None + + self.nodes = {} + for node_wire_pkeys in self.progressbar(nodes.values()): + node_wire = get_node_wire_for_wires(node_wire_pkeys) + if node_wire is None: + continue + + self.nodes[node_wire] = [ + flat_wires[wire_pkey] for wire_pkey in node_wire_pkeys + ] + + def get_nodes(self): + """ Return a set of node names. """ + if self.nodes is None: + self._build_nodes() + + return self.nodes.keys() + + def get_wires_for_node(self, tile, wire): + """ Get wires in node named for specified tile and wire. """ + if self.nodes is None: + self._build_nodes() + + return self.nodes[tile, wire] + + def _build_wire_to_node_map(self): + for node, wires in self.nodes.items(): + for tile_wire in wires: + assert tile_wire not in self.wire_to_node_map + self.wire_to_node_map[tile_wire] = node + + def get_node_for_wire(self, tile, wire): + """ Get node for specified tile and wire. """ + if self.wire_to_node_map is None: + self._build_wire_to_node_map() + + return self.wire_to_node_map[tile, wire]