Merge pull request #1467 from litghost/add_node_names

Add node names to database.
This commit is contained in:
litghost 2020-10-27 08:55:27 -07:00 committed by GitHub
commit 91d91357b5
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 545 additions and 9 deletions

View File

@ -22,6 +22,7 @@ pushdb:
rm ${XRAY_FAMILY_DIR}/tile_type_*_site_type_*.json
cp $(BUILD_DIR)/output/site_type_*.json ${XRAY_FAMILY_DIR}/
cp $(BUILD_DIR)/output/tileconn.json ${XRAY_FAMILY_DIR}/$(XRAY_PART)/
cp $(BUILD_DIR)/output/node_wires.json ${XRAY_FAMILY_DIR}/$(XRAY_PART)/
$(SPECIMENS_OK):
bash generate.sh $(subst /OK,,$@) -p=$(MAX_VIVADO_PROCESS) -t=$(MAX_TILES_INSTANCE) -n=$(MAX_NODES_INSTANCE)

View File

@ -0,0 +1,146 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# Copyright (C) 2020 The Project X-Ray Authors.
#
# Use of this source code is governed by a ISC-style
# license that can be found in the LICENSE file or at
# https://opensource.org/licenses/ISC
#
# SPDX-License-Identifier: ISC
""" Load tileconn.json and node_wires.json and verify both node names and
node <-> wire mapping against raw node data. """
import argparse
import datetime
import json
import multiprocessing
import os.path
import progressbar
import pyjson5 as json5
from prjxray import util, lib
from prjxray.grid import Grid
from prjxray.connections import Connections
from prjxray.node_model import NodeModel
def read_json5(fname):
with open(fname, 'r') as f:
return json5.load(f)
def read_raw_node_data(pool, root_dir):
""" Read raw node data from root dir. """
_, nodes = lib.read_root_csv(root_dir)
raw_node_data = []
with progressbar.ProgressBar(max_value=len(nodes)) as bar:
for idx, node in enumerate(pool.imap_unordered(
read_json5,
nodes,
chunksize=20,
)):
bar.update(idx)
raw_node_data.append(node)
bar.update(idx + 1)
return raw_node_data
def main():
parser = argparse.ArgumentParser(
description="Verify tileconn and node_wires.")
parser.add_argument('--root_dir', required=True)
parser.add_argument('--output_dir', required=True)
parser.add_argument('--max_cpu', type=int, default=10)
parser.add_argument('--ignored_wires')
args = parser.parse_args()
print('{} Reading tilegrid'.format(datetime.datetime.now()))
with open(os.path.join(util.get_db_root(), util.get_part(),
'tilegrid.json')) as f:
tilegrid = json.load(f)
grid = Grid(db=None, tilegrid=tilegrid)
print('{} Reading tileconn'.format(datetime.datetime.now()))
with open(os.path.join(args.output_dir, 'tileconn.json')) as f:
tileconn = json.load(f)
print(
'{} Reading tile wires from tile types'.format(
datetime.datetime.now()))
tile_wires = {}
for f in os.listdir(args.output_dir):
if f.endswith('.json') and f.startswith('tile_type_'):
if '_site_type_' in f:
continue
tile_type = f[len('tile_type_'):-len('.json')]
with open(os.path.join(args.output_dir, f)) as fin:
tile_wires[tile_type] = json.load(fin)['wires']
connections = Connections(
tilegrid=tilegrid,
tileconn=tileconn,
tile_wires=tile_wires,
)
print('{} Reading node wires'.format(datetime.datetime.now()))
with open(os.path.join(args.output_dir, 'node_wires.json')) as f:
node_wires = json.load(f)
print('{} Build initial node model'.format(datetime.datetime.now()))
node_model = NodeModel(
grid=grid,
connections=connections,
tile_wires=tile_wires,
node_wires=node_wires,
progressbar=progressbar.progressbar)
print('{} Build node model'.format(datetime.datetime.now()))
nodes = set(node_model.get_nodes())
print('{} Read raw node data for testing'.format(datetime.datetime.now()))
processes = min(multiprocessing.cpu_count(), args.max_cpu)
with multiprocessing.Pool(processes=processes) as pool:
raw_node_data = read_raw_node_data(pool, args.root_dir)
print('{} Read ignored wires list'.format(datetime.datetime.now()))
ignored_wires = []
ignored_wires_file = args.ignored_wires
if os.path.exists(ignored_wires_file):
with open(ignored_wires_file) as f:
ignored_wires = set(tuple(l.strip().split('/')) for l in f)
print(
'{} Verify nodes against raw node data'.format(
datetime.datetime.now()))
for node in progressbar.progressbar(raw_node_data):
tile, wire = node['node'].split('/')
assert (tile, wire) in nodes
wires_for_model = node_model.get_wires_for_node(tile, wire)
wires = set()
for wire in node['wires']:
wire_tile, wire_name = wire['wire'].split('/')
wires.add((wire_tile, wire_name))
if len(wires) != len(wires_for_model):
wires2 = set(wires_for_model)
a_minus_b = wires - wires2
b_minus_a = wires2 - wires
assert len(b_minus_a) == 0
assert len(a_minus_b - ignored_wires) == 0, a_minus_b
for tile, wire in wires_for_model:
assert (tile, wire) in wires
if __name__ == '__main__':
main()

View File

@ -26,3 +26,12 @@ python3 generate_grid.py \
--output_dir ${BUILD_DIR}/output \
--ignored_wires ignored_wires/${XRAY_DATABASE}/${XRAY_PART}_ignored_wires.txt \
--max_cpu=${MAX_GRID_CPU:-${DEFAULT_MAX_GRID_CPU}}
python3 node_names.py \
--root_dir ${BUILD_DIR}/specimen_001/ \
--output_dir ${BUILD_DIR}/output/ \
--max_cpu=${MAX_GRID_CPU:-${DEFAULT_MAX_GRID_CPU}}
python3 check_nodes.py \
--root_dir ${BUILD_DIR}/specimen_001/ \
--output_dir ${BUILD_DIR}/output/ \
--ignored_wires ignored_wires/${XRAY_DATABASE}/${XRAY_PART}_ignored_wires.txt \
--max_cpu=${MAX_GRID_CPU:-${DEFAULT_MAX_GRID_CPU}}

View File

@ -0,0 +1,183 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# Copyright (C) 2020 The Project X-Ray Authors.
#
# Use of this source code is governed by a ISC-style
# license that can be found in the LICENSE file or at
# https://opensource.org/licenses/ISC
#
# SPDX-License-Identifier: ISC
""" This script creates node_wires.json, which describes how nodes are named.
This script consumes the raw node data from root_dir and outputs
node_wires.json to the output_dir.
The class prjxray.node_model.NodeModel can be used to reconstruct node names
and node <-> wire mapping.
The contents of node_wires.json is:
- The set of tile type wires that are always nodes, key "node_pattern_wires"
- The set of tile wires that are nodes within the graph, key
"specific_node_wires".
"""
import argparse
import datetime
import json
import multiprocessing
import progressbar
import pyjson5 as json5
import os.path
from prjxray import util, lib
from prjxray.grid import Grid
def read_json5(fname):
with open(fname, 'r') as f:
return json5.load(f)
def main():
parser = argparse.ArgumentParser(
description="Reduce node names for wire connections.")
parser.add_argument('--root_dir', required=True)
parser.add_argument('--output_dir', required=True)
parser.add_argument('--max_cpu', type=int, default=10)
args = parser.parse_args()
_, nodes = lib.read_root_csv(args.root_dir)
processes = min(multiprocessing.cpu_count(), args.max_cpu)
pool = multiprocessing.Pool(processes=processes)
# Read tile grid and raw node data.
print('{} Reading tilegrid'.format(datetime.datetime.now()))
with open(os.path.join(util.get_db_root(), util.get_part(),
'tilegrid.json')) as f:
grid = Grid(db=None, tilegrid=json.load(f))
raw_node_data = []
with progressbar.ProgressBar(max_value=len(nodes)) as bar:
for idx, node in enumerate(pool.imap_unordered(
read_json5,
nodes,
chunksize=20,
)):
bar.update(idx)
raw_node_data.append(node)
bar.update(idx + 1)
node_wires = set()
remove_node_wires = set()
specific_node_wires = set()
# Create initial node wire pattern
for node in progressbar.progressbar(raw_node_data):
if len(node['wires']) <= 1:
continue
node_tile, node_wire = node['node'].split('/')
for wire in node['wires']:
wire_tile, wire_name = wire['wire'].split('/')
if node['node'] == wire['wire']:
assert node_tile == wire_tile
assert node_wire == wire_name
gridinfo = grid.gridinfo_at_tilename(node_tile)
node_wires.add((gridinfo.tile_type, wire_name))
print(
'Initial number of wires that are node drivers: {}'.format(
len(node_wires)))
# Remove exceptional node wire names, create specific_node_wires set,
# which is simply the list of wires that are nodes in the graph.
for node in progressbar.progressbar(raw_node_data):
if len(node['wires']) <= 1:
continue
for wire in node['wires']:
wire_tile, wire_name = wire['wire'].split('/')
gridinfo = grid.gridinfo_at_tilename(wire_tile)
key = gridinfo.tile_type, wire_name
if node['node'] == wire['wire']:
assert key in node_wires
else:
if key in node_wires:
specific_node_wires.add(node['node'])
remove_node_wires.add(key)
# Complete the specific_node_wires list after the pruning of the
# node_pattern_wires sets.
for node in progressbar.progressbar(raw_node_data):
if len(node['wires']) <= 1:
continue
for wire in node['wires']:
wire_tile, wire_name = wire['wire'].split('/')
gridinfo = grid.gridinfo_at_tilename(wire_tile)
key = gridinfo.tile_type, wire_name
if key in remove_node_wires and node['node'] == wire['wire']:
specific_node_wires.add(node['node'])
node_wires -= remove_node_wires
print(
'Final number of wires that are node drivers: {}'.format(
len(node_wires)))
print(
'Number of wires that are node drivers: {}'.format(
len(specific_node_wires)))
# Verify the node wire data.
for node in progressbar.progressbar(raw_node_data):
if len(node['wires']) <= 1:
continue
found_node_wire = False
for wire in node['wires']:
if wire['wire'] in specific_node_wires:
assert wire['wire'] == node['node']
found_node_wire = True
break
if not found_node_wire:
for wire in node['wires']:
wire_tile, wire_name = wire['wire'].split('/')
gridinfo = grid.gridinfo_at_tilename(wire_tile)
key = gridinfo.tile_type, wire_name
if key in node_wires:
assert node['node'] == wire['wire']
else:
assert node['node'] != wire['wire']
# Normalize output.
tile_types = {}
for tile_type, tile_wire in node_wires:
if tile_type not in tile_types:
tile_types[tile_type] = []
tile_types[tile_type].append(tile_wire)
for tile_type in tile_types:
tile_types[tile_type].sort()
out = {
'node_pattern_wires': tile_types,
'specific_node_wires': sorted(specific_node_wires),
}
with open(os.path.join(args.output_dir, 'node_wires.json'), 'w') as f:
json.dump(out, f, indent=2, sort_keys=True)
if __name__ == '__main__':
main()

View File

@ -172,5 +172,6 @@ roi_only: 000-init-db/run.${XRAY_PART}.ok 001-part-yaml/run.${XRAY_PART}.ok 075-
# Copy tilegrid and tileconn
cp ${XRAY_FAMILY_DIR}/${XRAY_EQUIV_PART}/tilegrid.json ${XRAY_FAMILY_DIR}/${XRAY_PART}/tilegrid.json
cp ${XRAY_FAMILY_DIR}/${XRAY_EQUIV_PART}/tileconn.json ${XRAY_FAMILY_DIR}/${XRAY_PART}/tileconn.json
cp ${XRAY_FAMILY_DIR}/${XRAY_EQUIV_PART}/node_wires.json ${XRAY_FAMILY_DIR}/${XRAY_PART}/node_wires.json
.PHONY: all clean clean_fuzzers clean_logs quick part_only roi_only

View File

@ -15,6 +15,7 @@ from prjxray import tile
from prjxray import tile_segbits
from prjxray import site_type
from prjxray import connections
from prjxray.node_model import NodeModel
def get_available_databases(prjxray_root):
@ -47,7 +48,8 @@ class Database(object):
# tilegrid.json JSON object
self.tilegrid = None
self.tileconn = None
self.tile_types = None
self.tile_types_json = None
self.node_wires = None
self.tile_types = {}
self.tile_segbits = {}
@ -138,26 +140,64 @@ class Database(object):
'tileconn.json')) as f:
self.tileconn = json.load(f)
def _read_node_wires(self):
""" Read node wires if not already read. """
if self.node_wires is None:
with open(os.path.join(self.db_root, self.part,
'node_wires.json')) as f:
self.node_wires = json.load(f)
def grid(self):
""" Return Grid object for database. """
self._read_tilegrid()
return grid.Grid(self, self.tilegrid)
def _read_tile_types(self):
for tile_type, db in self.tile_types.items():
with open(db.tile_type) as f:
self.tile_types[tile_type] = json.load(f)
if self.tile_types_json is None:
self.tile_types_json = {}
for tile_type, db in self.tile_types.items():
with open(db.tile_type) as f:
self.tile_types_json[tile_type] = json.load(f)
def _get_tile_wires(self):
self._read_tile_types()
tile_wires = dict(
(tile_type, db['wires'])
for tile_type, db in self.tile_types_json.items())
return tile_wires
def connections(self):
self._read_tilegrid()
self._read_tileconn()
self._read_tile_types()
tile_wires = dict(
(tile_type, db['wires'])
for tile_type, db in self.tile_types.items())
return connections.Connections(
self.tilegrid, self.tileconn, tile_wires)
self.tilegrid, self.tileconn, self._get_tile_wires())
def node_model(self, progressbar=lambda x: x):
""" Get node module for specified part.
progressbar - Should be a function that takes an iteraable, and
yields all elements from that iterable.
This can be used to generate a progressbar, for example
the module progressbar satifies this interface.
Example:
import progressbar
db = Database(...)
node_model = db.node_model(progressbar.progressbar)
"""
self._read_node_wires()
return NodeModel(
grid=self.grid(),
connections=self.connections(),
tile_wires=self._get_tile_wires(),
node_wires=self.node_wires,
progressbar=progressbar)
def get_site_types(self):
return self.site_types.keys()

156
prjxray/node_model.py Normal file
View File

@ -0,0 +1,156 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# Copyright (C) 2017-2020 The Project X-Ray Authors.
#
# Use of this source code is governed by a ISC-style
# license that can be found in the LICENSE file or at
# https://opensource.org/licenses/ISC
#
# SPDX-License-Identifier: ISC
class NodeModel():
""" Node lookup model
Terminology:
Wire - A segment of metal in a tile
Node - A connected set of wires
This class can provide a list of nodes, the wires in a node and the node
that a wire belongs too.
The name of node is always the name of one wire in the node.
It is recommended that this class be constructed by calling
Database.node_model rather than constructing this class directly.
"""
def __init__(
self, grid, connections, tile_wires, node_wires, progressbar=None):
self.grid = grid
self.connections = connections
self.tile_wires = tile_wires
self.specific_node_wires = set(node_wires['specific_node_wires'])
node_pattern_wires = node_wires['node_pattern_wires']
self.node_pattern_wires = {}
for tile_type in node_pattern_wires:
assert tile_type not in self.node_pattern_wires
self.node_pattern_wires[tile_type] = set(
node_pattern_wires[tile_type])
for tile_type in self.tile_wires:
if tile_type not in self.node_pattern_wires:
self.node_pattern_wires[tile_type] = set()
self.nodes = None
self.wire_to_node_map = None
if progressbar is None:
self.progressbar = lambda x: x
else:
self.progressbar = progressbar
def _build_nodes(self):
tile_wire_map = {}
wires = {}
flat_wires = []
for tile in self.progressbar(self.grid.tiles()):
gridinfo = self.grid.gridinfo_at_tilename(tile)
tile_type = gridinfo.tile_type
for wire in self.tile_wires[tile_type]:
wire_pkey = len(flat_wires)
tile_wire_map[(tile, wire)] = wire_pkey
flat_wires.append((tile, wire))
wires[wire_pkey] = None
for connection in self.progressbar(self.connections.get_connections()):
a_pkey = tile_wire_map[(
connection.wire_a.tile, connection.wire_a.wire)]
b_pkey = tile_wire_map[(
connection.wire_b.tile, connection.wire_b.wire)]
a_node = wires[a_pkey]
b_node = wires[b_pkey]
if a_node is None:
a_node = set((a_pkey, ))
if b_node is None:
b_node = set((b_pkey, ))
if a_node is not b_node:
a_node |= b_node
for wire in a_node:
wires[wire] = a_node
nodes = {}
for wire_pkey, node in self.progressbar(wires.items()):
if node is None:
node = set((wire_pkey, ))
assert wire_pkey in node
nodes[id(node)] = node
def get_node_wire_for_wires(wire_pkeys):
if len(wire_pkeys) == 1:
for wire_pkey in wire_pkeys:
return flat_wires[wire_pkey]
for wire_pkey in wire_pkeys:
tile, wire = flat_wires[wire_pkey]
if '{}/{}'.format(tile, wire) in self.specific_node_wires:
return tile, wire
for wire_pkey in wire_pkeys:
tile, wire = flat_wires[wire_pkey]
gridinfo = self.grid.gridinfo_at_tilename(tile)
if wire in self.node_pattern_wires[gridinfo.tile_type]:
return tile, wire
return None
self.nodes = {}
for node_wire_pkeys in self.progressbar(nodes.values()):
node_wire = get_node_wire_for_wires(node_wire_pkeys)
if node_wire is None:
continue
self.nodes[node_wire] = [
flat_wires[wire_pkey] for wire_pkey in node_wire_pkeys
]
def get_nodes(self):
""" Return a set of node names. """
if self.nodes is None:
self._build_nodes()
return self.nodes.keys()
def get_wires_for_node(self, tile, wire):
""" Get wires in node named for specified tile and wire. """
if self.nodes is None:
self._build_nodes()
return self.nodes[tile, wire]
def _build_wire_to_node_map(self):
for node, wires in self.nodes.items():
for tile_wire in wires:
assert tile_wire not in self.wire_to_node_map
self.wire_to_node_map[tile_wire] = node
def get_node_for_wire(self, tile, wire):
""" Get node for specified tile and wire. """
if self.wire_to_node_map is None:
self._build_wire_to_node_map()
return self.wire_to_node_map[tile, wire]