luke
/
prjxray
mirror of https://github.com/openXC7/prjxray.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add node names to database. 

Signed-off-by: Keith Rothman <537074+litghost@users.noreply.github.com>
This commit is contained in:
Keith Rothman 2020-10-23 14:39:04 -07:00
parent 1b17af7d5c
commit d117b73b18
7 changed files with 517 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
fuzzers/074-dump_all/Makefile
View File

@ -22,6 +22,7 @@ pushdb:
rm ${XRAY_FAMILY_DIR}/tile_type_*_site_type_*.json
cp $(BUILD_DIR)/output/site_type_*.json ${XRAY_FAMILY_DIR}/
cp $(BUILD_DIR)/output/tileconn.json ${XRAY_FAMILY_DIR}/$(XRAY_PART)/
cp $(BUILD_DIR)/output/node_wires.json ${XRAY_FAMILY_DIR}/$(XRAY_PART)/
$(SPECIMENS_OK):
bash generate.sh $(subst /OK,,$@) -p=$(MAX_VIVADO_PROCESS) -t=$(MAX_TILES_INSTANCE) -n=$(MAX_NODES_INSTANCE)

138
fuzzers/074-dump_all/check_nodes.py Normal file
View File

@ -0,0 +1,138 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# Copyright (C) 2020 The Project X-Ray Authors.
#
# Use of this source code is governed by a ISC-style
# license that can be found in the LICENSE file or at
# https://opensource.org/licenses/ISC
#
# SPDX-License-Identifier: ISC
import argparse
import datetime
import json
import multiprocessing
import os.path
import progressbar
import pyjson5 as json5
from prjxray import util, lib
from prjxray.grid import Grid
from prjxray.connections import Connections
from prjxray.node_model import NodeModel
def read_json5(fname):
with open(fname, 'r') as f:
return json5.load(f)
def read_raw_node_data(pool, root_dir):
_, nodes = lib.read_root_csv(root_dir)
raw_node_data = []
with progressbar.ProgressBar(max_value=len(nodes)) as bar:
for idx, node in enumerate(pool.imap_unordered(
read_json5,
nodes,
chunksize=20,
)):
bar.update(idx)
raw_node_data.append(node)
bar.update(idx + 1)
return raw_node_data
def main():
parser = argparse.ArgumentParser(description="")
parser.add_argument('--root_dir', required=True)
parser.add_argument('--output_dir', required=True)
parser.add_argument('--max_cpu', type=int, default=10)
parser.add_argument('--ignored_wires')
args = parser.parse_args()
print('{} Reading tilegrid'.format(datetime.datetime.now()))
with open(os.path.join(util.get_db_root(), util.get_part(),
'tilegrid.json')) as f:
tilegrid = json.load(f)
grid = Grid(db=None, tilegrid=tilegrid)
print('{} Reading tileconn'.format(datetime.datetime.now()))
with open(os.path.join(args.output_dir, 'tileconn.json')) as f:
tileconn = json.load(f)
print(
'{} Reading tile wires from tile types'.format(
datetime.datetime.now()))
tile_wires = {}
for f in os.listdir(args.output_dir):
if f.endswith('.json') and f.startswith('tile_type_'):
if '_site_type_' in f:
continue
tile_type = f[len('tile_type_'):-len('.json')]
with open(os.path.join(args.output_dir, f)) as fin:
tile_wires[tile_type] = json.load(fin)['wires']
connections = Connections(
tilegrid=tilegrid,
tileconn=tileconn,
tile_wires=tile_wires,
)
print('{} Reading node wires'.format(datetime.datetime.now()))
with open(os.path.join(args.output_dir, 'node_wires.json')) as f:
node_wires = json.load(f)
print('{} Build initial node model'.format(datetime.datetime.now()))
node_model = NodeModel(
grid=grid,
connections=connections,
tile_wires=tile_wires,
node_wires=node_wires,
progressbar=progressbar.progressbar)
print('{} Build node model'.format(datetime.datetime.now()))
nodes = set(node_model.get_nodes())
print('{} Read raw node data for testing'.format(datetime.datetime.now()))
processes = min(multiprocessing.cpu_count(), args.max_cpu)
with multiprocessing.Pool(processes=processes) as pool:
raw_node_data = read_raw_node_data(pool, args.root_dir)
ignored_wires = []
ignored_wires_file = args.ignored_wires
if os.path.exists(ignored_wires_file):
with open(ignored_wires_file) as f:
ignored_wires = set(tuple(l.strip().split('/')) for l in f)
for node in progressbar.progressbar(raw_node_data):
tile, wire = node['node'].split('/')
assert (tile, wire) in nodes
wires_for_model = node_model.get_wires_for_node(tile, wire)
wires = set()
for wire in node['wires']:
wire_tile, wire_name = wire['wire'].split('/')
wires.add((wire_tile, wire_name))
if len(wires) != len(wires_for_model):
wires2 = set(wires_for_model)
a_minus_b = wires - wires2
b_minus_a = wires2 - wires
assert len(b_minus_a) == 0
assert len(a_minus_b - ignored_wires) == 0, a_minus_b
for tile, wire in wires_for_model:
assert (tile, wire) in wires
if __name__ == '__main__':
main()

9
fuzzers/074-dump_all/generate_after_dump.sh
View File

@ -26,3 +26,12 @@ python3 generate_grid.py \
--output_dir ${BUILD_DIR}/output \
--ignored_wires ignored_wires/${XRAY_DATABASE}/${XRAY_PART}_ignored_wires.txt \
--max_cpu=${MAX_GRID_CPU:-${DEFAULT_MAX_GRID_CPU}}
python3 node_names.py \
--root_dir ${BUILD_DIR}/specimen_001/ \
--output_dir ${BUILD_DIR}/output/ \
--max_cpu=${MAX_GRID_CPU:-${DEFAULT_MAX_GRID_CPU}}
python3 check_nodes.py \
--root_dir ${BUILD_DIR}/specimen_001/ \
--output_dir ${BUILD_DIR}/output/ \
--ignored_wires ignored_wires/${XRAY_DATABASE}/${XRAY_PART}_ignored_wires.txt \
--max_cpu=${MAX_GRID_CPU:-${DEFAULT_MAX_GRID_CPU}}

163
fuzzers/074-dump_all/node_names.py Normal file
View File

@ -0,0 +1,163 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# Copyright (C) 2020 The Project X-Ray Authors.
#
# Use of this source code is governed by a ISC-style
# license that can be found in the LICENSE file or at
# https://opensource.org/licenses/ISC
#
# SPDX-License-Identifier: ISC
import argparse
import datetime
import json
import multiprocessing
import progressbar
import pyjson5 as json5
import os.path
from prjxray import util, lib
from prjxray.grid import Grid
def read_json5(fname):
with open(fname, 'r') as f:
return json5.load(f)
def main():
parser = argparse.ArgumentParser(
description=
"Reduces raw database dump into prototype tiles, grid, and connections."
)
parser.add_argument('--root_dir', required=True)
parser.add_argument('--output_dir', required=True)
parser.add_argument('--max_cpu', type=int, default=10)
args = parser.parse_args()
_, nodes = lib.read_root_csv(args.root_dir)
processes = min(multiprocessing.cpu_count(), args.max_cpu)
pool = multiprocessing.Pool(processes=processes)
print('{} Reading tilegrid'.format(datetime.datetime.now()))
with open(os.path.join(util.get_db_root(), util.get_part(),
'tilegrid.json')) as f:
grid = Grid(db=None, tilegrid=json.load(f))
raw_node_data = []
with progressbar.ProgressBar(max_value=len(nodes)) as bar:
for idx, node in enumerate(pool.imap_unordered(
read_json5,
nodes,
chunksize=20,
)):
bar.update(idx)
raw_node_data.append(node)
bar.update(idx + 1)
node_wires = set()
remove_node_wires = set()
specific_node_wires = set()
for node in progressbar.progressbar(raw_node_data):
if len(node['wires']) <= 1:
continue
node_tile, node_wire = node['node'].split('/')
for wire in node['wires']:
wire_tile, wire_name = wire['wire'].split('/')
if node['node'] == wire['wire']:
assert node_tile == wire_tile
assert node_wire == wire_name
gridinfo = grid.gridinfo_at_tilename(node_tile)
node_wires.add((gridinfo.tile_type, wire_name))
print(
'Initial number of wires that are node drivers: {}'.format(
len(node_wires)))
for node in progressbar.progressbar(raw_node_data):
if len(node['wires']) <= 1:
continue
for wire in node['wires']:
wire_tile, wire_name = wire['wire'].split('/')
gridinfo = grid.gridinfo_at_tilename(wire_tile)
key = gridinfo.tile_type, wire_name
if node['node'] == wire['wire']:
assert key in node_wires
else:
if key in node_wires:
specific_node_wires.add(node['node'])
remove_node_wires.add(key)
for node in progressbar.progressbar(raw_node_data):
if len(node['wires']) <= 1:
continue
for wire in node['wires']:
wire_tile, wire_name = wire['wire'].split('/')
gridinfo = grid.gridinfo_at_tilename(wire_tile)
key = gridinfo.tile_type, wire_name
if key in remove_node_wires and node['node'] == wire['wire']:
specific_node_wires.add(node['node'])
node_wires -= remove_node_wires
print(
'Final number of wires that are node drivers: {}'.format(
len(node_wires)))
print(
'Number of wires that are node drivers: {}'.format(
len(specific_node_wires)))
for node in progressbar.progressbar(raw_node_data):
if len(node['wires']) <= 1:
continue
found_node_wire = False
for wire in node['wires']:
if wire['wire'] in specific_node_wires:
assert wire['wire'] == node['node']
found_node_wire = True
break
if not found_node_wire:
for wire in node['wires']:
wire_tile, wire_name = wire['wire'].split('/')
gridinfo = grid.gridinfo_at_tilename(wire_tile)
key = gridinfo.tile_type, wire_name
if key in node_wires:
assert node['node'] == wire['wire']
else:
assert node['node'] != wire['wire']
tile_types = {}
for tile_type, tile_wire in node_wires:
if tile_type not in tile_types:
tile_types[tile_type] = []
tile_types[tile_type].append(tile_wire)
for tile_type in tile_types:
tile_types[tile_type].sort()
out = {
'node_pattern_wires': tile_types,
'specific_node_wires': sorted(specific_node_wires),
}
with open(os.path.join(args.output_dir, 'node_wires.json'), 'w') as f:
json.dump(out, f, indent=2, sort_keys=True)
if __name__ == '__main__':
main()

1
fuzzers/Makefile
View File

@ -172,5 +172,6 @@ roi_only: 000-init-db/run.${XRAY_PART}.ok 001-part-yaml/run.${XRAY_PART}.ok 075-
# Copy tilegrid and tileconn
cp ${XRAY_FAMILY_DIR}/${XRAY_EQUIV_PART}/tilegrid.json ${XRAY_FAMILY_DIR}/${XRAY_PART}/tilegrid.json
cp ${XRAY_FAMILY_DIR}/${XRAY_EQUIV_PART}/tileconn.json ${XRAY_FAMILY_DIR}/${XRAY_PART}/tileconn.json
cp ${XRAY_FAMILY_DIR}/${XRAY_EQUIV_PART}/node_wires.json ${XRAY_FAMILY_DIR}/${XRAY_PART}/node_wires.json
.PHONY: all clean clean_fuzzers clean_logs quick part_only roi_only

58
prjxray/db.py
View File

@ -15,6 +15,7 @@ from prjxray import tile
from prjxray import tile_segbits
from prjxray import site_type
from prjxray import connections
from prjxray.node_model import NodeModel
def get_available_databases(prjxray_root):
@ -47,7 +48,8 @@ class Database(object):
# tilegrid.json JSON object
self.tilegrid = None
self.tileconn = None
self.tile_types = None
self.tile_types_json = None
self.node_wires = None
self.tile_types = {}
self.tile_segbits = {}
@ -138,26 +140,64 @@ class Database(object):
'tileconn.json')) as f:
self.tileconn = json.load(f)
def _read_node_wires(self):
""" Read node wires if not already read. """
if self.node_wires is None:
with open(os.path.join(self.db_root, self.part,
'node_wires.json')) as f:
self.node_wires = json.load(f)
def grid(self):
""" Return Grid object for database. """
self._read_tilegrid()
return grid.Grid(self, self.tilegrid)
def _read_tile_types(self):
for tile_type, db in self.tile_types.items():
with open(db.tile_type) as f:
self.tile_types[tile_type] = json.load(f)
if self.tile_types_json is None:
self.tile_types_json = {}
for tile_type, db in self.tile_types.items():
with open(db.tile_type) as f:
self.tile_types_json[tile_type] = json.load(f)
def _get_tile_wires(self):
self._read_tile_types()
tile_wires = dict(
(tile_type, db['wires'])
for tile_type, db in self.tile_types_json.items())
return tile_wires
def connections(self):
self._read_tilegrid()
self._read_tileconn()
self._read_tile_types()
tile_wires = dict(
(tile_type, db['wires'])
for tile_type, db in self.tile_types.items())
return connections.Connections(
self.tilegrid, self.tileconn, tile_wires)
self.tilegrid, self.tileconn, self._get_tile_wires())
def node_model(self, progressbar=lambda x: x):
""" Get node module for specified part.
progressbar - Should be a function that takes an iteraable, and
yields all elements from that iterable.
This can be used to generate a progressbar, for example
the module progressbar satifies this interface.
Example:
import progressbar
db = Database(...)
node_model = db.node_model(progressbar.progressbar)
"""
self._read_node_wires()
return NodeModel(
grid=self.grid(),
connections=self.connections(),
tile_wires=self._get_tile_wires(),
node_wires=self.node_wires,
progressbar=progressbar)
def get_site_types(self):
return self.site_types.keys()

156
prjxray/node_model.py Normal file
View File

@ -0,0 +1,156 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# Copyright (C) 2017-2020 The Project X-Ray Authors.
#
# Use of this source code is governed by a ISC-style
# license that can be found in the LICENSE file or at
# https://opensource.org/licenses/ISC
#
# SPDX-License-Identifier: ISC
class NodeModel():
""" Node lookup model
Terminology:
Wire - A segment of metal in a tile
Node - A connected set of wires
This class can provide a list of nodes, the wires in a node and the node
that a wire belongs too.
The name of node is always the name of one wire in the node.
It is recommended that this class be constructed by calling
Database.node_model rather than constructing this class directly.
"""
def __init__(
self, grid, connections, tile_wires, node_wires, progressbar=None):
self.grid = grid
self.connections = connections
self.tile_wires = tile_wires
self.specific_node_wires = set(node_wires['specific_node_wires'])
node_pattern_wires = node_wires['node_pattern_wires']
self.node_pattern_wires = {}
for tile_type in node_pattern_wires:
assert tile_type not in self.node_pattern_wires
self.node_pattern_wires[tile_type] = set(
node_pattern_wires[tile_type])
for tile_type in self.tile_wires:
if tile_type not in self.node_pattern_wires:
self.node_pattern_wires[tile_type] = set()
self.nodes = None
self.wire_to_node_map = None
if progressbar is None:
self.progressbar = lambda x: x
else:
self.progressbar = progressbar
def _build_nodes(self):
tile_wire_map = {}
wires = {}
flat_wires = []
for tile in self.progressbar(self.grid.tiles()):
gridinfo = self.grid.gridinfo_at_tilename(tile)
tile_type = gridinfo.tile_type
for wire in self.tile_wires[tile_type]:
wire_pkey = len(flat_wires)
tile_wire_map[(tile, wire)] = wire_pkey
flat_wires.append((tile, wire))
wires[wire_pkey] = None
for connection in self.progressbar(self.connections.get_connections()):
a_pkey = tile_wire_map[(
connection.wire_a.tile, connection.wire_a.wire)]
b_pkey = tile_wire_map[(
connection.wire_b.tile, connection.wire_b.wire)]
a_node = wires[a_pkey]
b_node = wires[b_pkey]
if a_node is None:
a_node = set((a_pkey, ))
if b_node is None:
b_node = set((b_pkey, ))
if a_node is not b_node:
a_node |= b_node
for wire in a_node:
wires[wire] = a_node
nodes = {}
for wire_pkey, node in self.progressbar(wires.items()):
if node is None:
node = set((wire_pkey, ))
assert wire_pkey in node
nodes[id(node)] = node
def get_node_wire_for_wires(wire_pkeys):
if len(wire_pkeys) == 1:
for wire_pkey in wire_pkeys:
return flat_wires[wire_pkey]
for wire_pkey in wire_pkeys:
tile, wire = flat_wires[wire_pkey]
if '{}/{}'.format(tile, wire) in self.specific_node_wires:
return tile, wire
for wire_pkey in wire_pkeys:
tile, wire = flat_wires[wire_pkey]
gridinfo = self.grid.gridinfo_at_tilename(tile)
if wire in self.node_pattern_wires[gridinfo.tile_type]:
return tile, wire
return None
self.nodes = {}
for node_wire_pkeys in self.progressbar(nodes.values()):
node_wire = get_node_wire_for_wires(node_wire_pkeys)
if node_wire is None:
continue
self.nodes[node_wire] = [
flat_wires[wire_pkey] for wire_pkey in node_wire_pkeys
]
def get_nodes(self):
""" Return a set of node names. """
if self.nodes is None:
self._build_nodes()
return self.nodes.keys()
def get_wires_for_node(self, tile, wire):
""" Get wires in node named for specified tile and wire. """
if self.nodes is None:
self._build_nodes()
return self.nodes[tile, wire]
def _build_wire_to_node_map(self):
for node, wires in self.nodes.items():
for tile_wire in wires:
assert tile_wire not in self.wire_to_node_map
self.wire_to_node_map[tile_wire] = node
def get_node_for_wire(self, tile, wire):
""" Get node for specified tile and wire. """
if self.wire_to_node_map is None:
self._build_wire_to_node_map()
return self.wire_to_node_map[tile, wire]