luke
/
prjxray
mirror of https://github.com/openXC7/prjxray.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1846 from antmicro/use-gh-runners 

gh: initial use of gh action custom runners
This commit is contained in:
Alessandro Comodi 2022-03-08 09:41:41 +01:00 committed by GitHub
parent d02c90b1cd 847f97ab17
commit 7866deed02
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
17 changed files with 456 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
.github/kokoro/steps/hostsetup.sh vendored
View File

@ -87,14 +87,14 @@ sudo apt-get install -y \
lsb \
nodejs \
psmisc \
python3.8 \
python3.8-dev \
python3.8-venv
python3 \
python3-dev \
python3-venv
echo "========================================"
echo "Enter virtual env for python 3.8"
echo "----------------------------------------"
python3.8 -mvenv startup_python
python3 -mvenv startup_python
source startup_python/bin/activate
which python
python --version

100
.github/workflows/ci.yml vendored Normal file
View File

@ -0,0 +1,100 @@
on:
pull_request:
push:
schedule:
- cron: '0 0 * * *'
name: CI Build
jobs:
Tests:
container: ubuntu:bionic
runs-on: [self-hosted, Linux, X64]
env:
ALLOW_ROOT: true
steps:
- uses: actions/checkout@v2
with:
submodules: recursive
- name: Install
run: |
apt update
apt install -y \
bash bison build-essential ca-certificates clang-format cmake psmisc \
colordiff coreutils git flex python3 python3-dev python3-venv xsltproc
- name: Build
run: make build --output-sync=target --warn-undefined-variables -j$(nproc)
- name: Environment
run: make env --output-sync=target --warn-undefined-variables
- name: Run Test
run: make test --output-sync=target --warn-undefined-variables
- uses: actions/upload-artifact@v2
if: ${{ always() }}
with:
path: |
**/results*.gz
**/plot_*.svg
BuildDatabase:
container: ubuntu:bionic
runs-on: [self-hosted, Linux, X64]
strategy:
fail-fast: false
matrix:
family: ['artix7', 'zynq7', 'kintex7', 'spartan7']
env:
ALLOW_ROOT: true
GHA_EXTERNAL_DISK: "tools"
XILINX_LOCAL_USER_DATA: "no"
steps:
- uses: actions/checkout@v2
with:
submodules: recursive
- name: Install
run: |
apt update
apt install -y \
bash bison build-essential ca-certificates clang-format cmake psmisc \
colordiff coreutils git flex python3 python3-dev python3-venv xsltproc
- name: Build
run: make build --output-sync=target --warn-undefined-variables -j$(nproc)
- name: Environment
run: make env --output-sync=target --warn-undefined-variables
- name: Run Test
run: .github/workflows/scripts/db.sh
env:
XRAY_SETTINGS: ${{ matrix.family }}
- uses: actions/upload-artifact@v2
if: ${{ always() }}
with:
name: ${{ matrix.family }}
path: |
**/results*.gz
**/plot_*.svg
**/diff.html
**/diff.json
**/diff.patch
**/*sponge_log.xml
**/fuzzers/*.tgz
**/database/${{ matrix.family }}/**"

219
.github/workflows/scripts/db.sh vendored Executable file
View File

@ -0,0 +1,219 @@
#!/bin/bash
# Copyright (C) 2017-2020 The Project X-Ray Authors.
#
# Use of this source code is governed by a ISC-style
# license that can be found in the LICENSE file or at
# https://opensource.org/licenses/ISC
#
# SPDX-License-Identifier: ISC
set -e
source $(dirname "$0")/hostinfo.sh
echo
echo "======================================="
echo "Creating Vivado Symbolic Link"
echo "---------------------------------------"
ln -s /mnt/aux/Xilinx /opt/Xilinx
ls /opt/Xilinx/Vivado
source /opt/Xilinx/Vivado/2017.2/settings64.sh
vivado -version
echo
echo "========================================"
echo "Downloading current database"
echo "----------------------------------------"
(
script --return --flush --command "./download-latest-db.sh" -
)
echo "----------------------------------------"
echo
echo "========================================"
echo "Preparing database"
echo "----------------------------------------"
(
make db-prepare-${XRAY_SETTINGS}
)
echo "----------------------------------------"
source settings/$XRAY_SETTINGS.sh
echo
echo "========================================"
echo "Cleaning out current database"
echo "----------------------------------------"
(
cd database
make clean-${XRAY_SETTINGS}-db
)
echo "----------------------------------------"
echo
echo "========================================"
echo "Running Database build"
echo "----------------------------------------"
(
# Output which fuzzers we are going to run
echo "make --dry-run"
make --dry-run db-${XRAY_SETTINGS}-all
echo "----------------------------------------"
# Run the fuzzers
set -x +e
tmp=$(mktemp)
script --return --flush --command "make -j $CORES MAX_VIVADO_PROCESS=$MAX_VIVADO_PROCESS db-${XRAY_SETTINGS}-all" $tmp
DATABASE_RET=$?
set +x -e
if [[ $DATABASE_RET != 0 ]] ; then
# Collect the Vivado logs into one tgz archive
echo "Packing failing test cases"
# Looking for the failing directories and packing them
# example of line from which the failing fuzzer directory gets extracted:
# - Makefile:87: recipe for target '000-db-init/000-init-db/run.xc7a100tfgg676-1.ok' failed --> fuzzers/000-db-init
grep -Po "recipe for target '\K(.*)(?=\/run\..*\.ok')" $tmp | sed -e 's/^/fuzzers\//' | xargs tar -zcf fuzzers/fails.tgz
echo "----------------------------------------"
echo "A failure occurred during Database build."
echo "----------------------------------------"
rm $tmp
echo "========================================"
echo " Disk space in failure path"
echo "----------------------------------------"
du -sh
exit $DATABASE_RET
fi
)
echo "----------------------------------------"
# Format the database
make db-format-${XRAY_SETTINGS}
# Update the database/Info.md file
make db-info
# Output if the database has differences
echo
echo "========================================"
echo " Database Differences"
echo "----------------------------------------"
(
cd database
# Update the index with any new files
git add \
--verbose \
--all \
--ignore-errors \
.
# Output what git status
echo
echo "----------------------------------------"
echo " Database Status"
echo "----------------------------------------"
git status
echo "----------------------------------------"
# Output a summary of how the files have changed
echo
echo "----------------------------------------"
echo " Database Diff Summary"
echo "----------------------------------------"
git diff --stat --irreversible-delete --find-renames --find-copies --ignore-all-space origin/master
# Save the diff to be uploaded as an artifact
echo
echo "----------------------------------------"
echo " Saving diff output"
echo "----------------------------------------"
# Patch file
git diff \
--patch-with-stat --no-color --irreversible-delete --find-renames --find-copies origin/master \
> diff.patch
MAX_DIFF_LINES=50000
DIFF_LINES="$(wc -l diff.patch | sed -e's/ .*$//')"
if [ $DIFF_LINES -gt $MAX_DIFF_LINES ]; then
echo
echo "----------------------------------------"
echo " Database Diff"
echo "----------------------------------------"
echo "diff has $DIFF_LINES lines which is too large to display!"
echo
echo "----------------------------------------"
echo " Generating pretty diff output"
echo "----------------------------------------"
echo "diff has $DIFF_LINES lines which is too large for HTML output!"
else
# Output the actually diff
echo
echo "----------------------------------------"
echo " Database Diff"
echo "----------------------------------------"
git diff --color --irreversible-delete --find-renames --find-copies --ignore-all-space origin/master
echo
echo "----------------------------------------"
echo " Generating pretty diff output"
echo "----------------------------------------"
(
# Allow the diff2html to fail.
set +e
# Pretty HTML file version
diff2html --summary=open --file diff.html --format html \
-- \
--irreversible-delete --find-renames --find-copies \
--ignore-all-space origin/master || true
# Programmatic JSON version
diff2html --file diff.json --format json \
-- \
--irreversible-delete --find-renames --find-copies \
--ignore-all-space origin/master || true
) || true
fi
)
echo "----------------------------------------"
# Check the database and fail if it is broken.
set -x +e
make db-check-${XRAY_SETTINGS}
CHECK_RET=$?
set +x -e
echo
echo "========================================"
echo " Testing HTML generation"
echo "----------------------------------------"
(
cd htmlgen
source htmlgen.sh $XRAY_SETTINGS
)
# If we get here, then all the fuzzers completed fine. Hence we are
# going to assume we don't want to keep all the build / logs / etc (as
# they are quite large). Thus do a clean to get rid of them.
echo
echo "========================================"
echo " Cleaning up after success"
echo "----------------------------------------"
(
cd fuzzers
echo
echo "Cleaning up so CI doesn't save all the excess data."
make clean_fuzzers
make clean_piplists
)
echo "----------------------------------------"
echo "========================================"
echo " Final disk space after cleanup"
echo "----------------------------------------"
du -sh
exit $CHECK_RET

36
.github/workflows/scripts/hostinfo.sh vendored Executable file
View File

@ -0,0 +1,36 @@
#!/bin/bash
# Copyright (C) 2017-2020 The Project X-Ray Authors.
#
# Use of this source code is governed by a ISC-style
# license that can be found in the LICENSE file or at
# https://opensource.org/licenses/ISC
#
# SPDX-License-Identifier: ISC
set -e
echo
echo "========================================"
echo "Host Environment"
echo "----------------------------------------"
export
echo "----------------------------------------"
echo
echo "========================================"
echo "Host CPU"
echo "----------------------------------------"
export CORES=$(nproc --all)
echo "Cores: $CORES"
echo
echo "Memory"
echo "----------------------------------------"
cat /proc/meminfo
echo "----------------------------------------"
export MEM_GB=$(($(awk '/MemTotal/ {print $2}' /proc/meminfo)/(1024*1024)))
echo "Total Memory (GB): $MEM_GB"
# Approx memory per grid process
export MEM_PER_RUN=8
export MAX_GRID_CPU=$(($MEM_GB/$MEM_PER_RUN))
export MAX_VIVADO_PROCESS=$(($MEM_GB/$MEM_PER_RUN))

6
Makefile
View File

@ -10,10 +10,14 @@ ALL_EXCLUDE = third_party .git env build docs/env
INSTALL_DIR ?=
# Skip this check if the ALLOW_ROOT var is defined
# E.g. when running in GH action custom runners CI
ifndef ALLOW_ROOT
# Check if root
ifeq ($(shell id -u),0)
$(error ERROR: Running as ID 0)
endif
endif
# Tools + Environment
IN_ENV = if [ -e env/bin/activate ]; then . env/bin/activate; fi; source utils/environment.python.sh;
@ -275,4 +279,4 @@ clean:
$(MAKE) -C fuzzers clean
rm -rf build
.PHONY: clean
.PHONY: clean

2
fuzzers/065-gtp-common-pips/Makefile
View File

@ -54,7 +54,7 @@ $(BUILD_DIR)/cmt_regions.csv: output_cmt.tcl
cd $(BUILD_DIR)/ && ${XRAY_VIVADO} -mode batch -source ${FUZDIR}/output_cmt.tcl
clean:
rm -rf build_* run.*.ok
rm -rf ${BUILD_DIR} ${RUN_OK}
pushdb: database
${XRAY_MERGEDB} gtp_common_mid_left $(BUILD_DIR)/segbits_gtp_common.db

5
fuzzers/065b-gtp-common-pips/Makefile
View File

@ -5,6 +5,7 @@
# https://opensource.org/licenses/ISC
#
# SPDX-License-Identifier: ISC
export FUZDIR=$(shell pwd)
PIP_TYPE?=gtp_common_mid_${XRAY_PART}
PIP_FILE?=gtp_common_mid_ck_mux
@ -24,7 +25,7 @@ SEGMATCH_FLAGS=-c 3
SPECIMENS_DEPS=$(BUILD_DIR)/cmt_regions.csv
A_PIPLIST=gtp_common_mid_ck_mux.txt
CHECK_ARGS= --zero-entries --timeout-iters 5 --todo-dir $(BUILD_DIR)/todo
CHECK_ARGS= --zero-entries --timeout-iters 10 --todo-dir $(BUILD_DIR)/todo
include ../pip_loop.mk
@ -53,7 +54,7 @@ $(BUILD_DIR)/cmt_regions.csv: output_cmt.tcl
cd $(BUILD_DIR)/ && ${XRAY_VIVADO} -mode batch -source ${FUZDIR}/output_cmt.tcl
clean:
rm -rf build_* run.*.ok
rm -rf ${BUILD_DIR} ${RUN_OK}
pushdb: database
${XRAY_MERGEDB} gtp_common_mid_left $(BUILD_DIR)/segbits_gtp_common.db

9
fuzzers/072-ordered_wires/Makefile
View File

@ -7,7 +7,10 @@
# SPDX-License-Identifier: ISC
N := 1
BUILD_DIR = build_${XRAY_PART}
RUN_OK = run.${XRAY_PART}.ok
SPECIMENS := $(addprefix $(BUILD_DIR)/specimen_,$(shell seq -f '%03.0f' $(N)))
SPECIMENS_OK := $(addsuffix /OK,$(SPECIMENS))
MAX_VIVADO_PROCESS ?= 4
@ -24,13 +27,13 @@ $(SPECIMENS_OK):
touch $@
run:
rm -rf $(BUILD_DIR) run.${XRAY_PART}.ok
rm -rf $(BUILD_DIR) $(RUN_OK)
$(MAKE) database
$(MAKE) pushdb
touch run.${XRAY_PART}.ok
touch $(RUN_OK)
clean:
rm -rf build_* run.*.ok
rm -rf ${BUILD_DIR} ${RUN_OK}
.PHONY: database pushdb run clean

9
fuzzers/073-get_counts/Makefile
View File

@ -7,7 +7,10 @@
# SPDX-License-Identifier: ISC
N := 1
BUILD_DIR = build_${XRAY_PART}
RUN_OK = run.${XRAY_PART}.ok
SPECIMENS := $(addprefix $(BUILD_DIR)/specimen_,$(shell seq -f '%03.0f' $(N)))
SPECIMENS_OK := $(addsuffix /OK,$(SPECIMENS))
@ -22,13 +25,13 @@ $(SPECIMENS_OK):
touch $@
run:
rm -rf $(BUILD_DIR) run.${XRAY_PART}.ok
rm -rf $(BUILD_DIR) $(RUN_OK)
$(MAKE) database
$(MAKE) pushdb
touch run.${XRAY_PART}.ok
touch $(RUN_OK)
clean:
rm -rf build_* run.*.ok
rm -rf ${BUILD_DIR} ${RUN_OK}
.PHONY: database pushdb run clean

7
fuzzers/074-dump_all/Makefile
View File

@ -7,7 +7,10 @@
# SPDX-License-Identifier: ISC
N := 1
BUILD_DIR = build_${XRAY_PART}
RUN_OK = run.${XRAY_PART}.ok
SPECIMENS := $(addprefix $(BUILD_DIR)/specimen_,$(shell seq -f '%03.0f' $(N)))
SPECIMENS_OK := $(addsuffix /OK,$(SPECIMENS))
MAX_VIVADO_PROCESS ?= 4
@ -34,10 +37,10 @@ run:
$(MAKE) pushdb
# Clean up intermediate files after successful pushdb.
find $(BUILD_DIR) -name "*.json5" -delete
touch run.${XRAY_PART}.ok
touch ${RUN_OK}
clean:
rm -rf build_* run.*.ok
rm -rf ${BUILD_DIR} ${RUN_OK}
.PHONY: database pushdb run clean

8
fuzzers/075-pins/Makefile
View File

@ -7,6 +7,8 @@
# SPDX-License-Identifier: ISC
N := 1
BUILD_DIR = build_${XRAY_PART}
RUN_OK = run.${XRAY_PART}.ok
SPECIMENS := $(addprefix $(BUILD_DIR)/specimen_,$(shell seq -f '%03.0f' $(N)))
SPECIMENS_OK := $(addsuffix /OK,$(SPECIMENS))
@ -21,12 +23,12 @@ $(SPECIMENS_OK):
touch $@
run:
rm -rf $(BUILD_DIR) run.${XRAY_PART}.ok
rm -rf $(BUILD_DIR) $(RUN_OK)
$(MAKE) database
$(MAKE) pushdb
touch run.$(XRAY_PART).ok
touch $(RUN_OK)
clean:
rm -rf build_* run.*.ok
rm -rf ${BUILD_DIR} ${RUN_OK}
.PHONY: database pushdb run clean

4
fuzzers/Makefile
View File

@ -116,7 +116,11 @@ $(eval $(call fuzzer,030-iob,005-tilegrid,all))
$(eval $(call fuzzer,031-cmt-mmcm,005-tilegrid,all))
$(eval $(call fuzzer,032-cmt-pll,005-tilegrid,all))
$(eval $(call fuzzer,034-cmt-pll-pips,005-tilegrid 071-ppips,all))
ifneq ($(XRAY_DATABASE),kintex7)
# FIXME: 034b fuzzer is generating conflicting bits around the FREQ_BB[N] bits.
# The fuzzer can be re-enabled once the conflicting bits are not generated anymore
$(eval $(call fuzzer,034b-cmt-mmcm-pips,005-tilegrid 071-ppips,all))
endif
$(eval $(call fuzzer,035-iob-ilogic,005-tilegrid,all))
$(eval $(call fuzzer,035a-iob-idelay,005-tilegrid,all))
$(eval $(call fuzzer,035b-iob-iserdes,005-tilegrid,all))

2
fuzzers/run_fuzzer.py
View File

@ -512,7 +512,7 @@ def run_fuzzer(fuzzer_name, fuzzer_dir, fuzzer_logdir, logger, will_retry):
)
log(running_msg)
log_suffix = ".{}.log".format(time_start.isoformat())
log_suffix = ".{}.log".format(time_start.isoformat()).replace(":", "-")
fuzzer_stdout = os.path.join(fuzzer_logdir, "stdout" + log_suffix)
fuzzer_stderr = os.path.join(fuzzer_logdir, "stderr" + log_suffix)

7
prjxray/tile_segbits.py
View File

@ -10,6 +10,7 @@
# SPDX-License-Identifier: ISC
from collections import namedtuple
from prjxray import bitstream
from prjxray import util
from prjxray.grid_types import BlockType
import enum
@ -84,15 +85,21 @@ class TileSegbits(object):
if tile_db.ppips is not None:
with open(tile_db.ppips) as f:
util.lock_file(f, 10)
self.ppips = read_ppips(f)
util.unlock_file(f)
if tile_db.segbits is not None:
with open(tile_db.segbits) as f:
util.lock_file(f, 10)
self.segbits[BlockType.CLB_IO_CLK] = read_segbits(f)
util.unlock_file(f)
if tile_db.block_ram_segbits is not None:
with open(tile_db.block_ram_segbits) as f:
util.lock_file(f, 10)
self.segbits[BlockType.BLOCK_RAM] = read_segbits(f)
util.unlock_file(f)
for block_type in self.segbits:
for feature in self.segbits[block_type]:

25
prjxray/util.py
View File

@ -8,10 +8,12 @@
# https://opensource.org/licenses/ISC
#
# SPDX-License-Identifier: ISC
import fcntl
import math
import os
import random
import re
import signal
import yaml
from .roi import Roi
@ -254,8 +256,10 @@ def parse_db_line(line):
def parse_db_lines(fn):
with open(fn, "r") as f:
lock_file(f, 10)
for line in f:
yield line, parse_db_line(line)
unlock_file(f)
def write_db_lines(fn, entries, track_origin=False):
@ -269,8 +273,10 @@ def write_db_lines(fn, entries, track_origin=False):
new_lines.append(new_line)
with open(fn, "w") as f:
lock_file(f, 10)
for line in sorted(new_lines):
print(line, file=f)
unlock_file(f)
def parse_tagbit(x):
@ -402,3 +408,22 @@ def add_bool_arg(parser, yes_arg, default=False, **kwargs):
yes_arg, dest=dest, action='store_true', default=default, **kwargs)
parser.add_argument(
'--no-' + dashed, dest=dest, action='store_false', **kwargs)
def timeout_handler(signum, frame):
raise Exception("ERROR: could not lock file!")
def lock_file(fd, timeout):
try:
signal.signal(signal.SIGALRM, timeout_handler)
signal.alarm(timeout)
fcntl.flock(fd.fileno(), fcntl.LOCK_EX)
signal.alarm(0)
except Exception as e:
print(e)
exit(1)
def unlock_file(fd):
fcntl.flock(fd.fileno(), fcntl.LOCK_UN)

9
requirements.txt
View File

@ -1,6 +1,3 @@
-e third_party/fasm
-e third_party/python-sdf-timing
-e .
intervaltree
junit-xml
numpy
@ -10,9 +7,15 @@ parse
progressbar2
pyjson5
pytest
pytest-runner
pyyaml
scipy>=1.2.1
simplejson
sympy
textx
yapf==0.24.0
# Third party
-e third_party/fasm
-e third_party/python-sdf-timing
-e .

28
utils/roi_all.py
View File

@ -9,13 +9,26 @@
#
# SPDX-License-Identifier: ISC
import argparse
import yaml
import subprocess
import multiprocessing as mp
import os
import re
import subprocess
import yaml
from prjxray import util
def worker(arglist):
part, cwd = arglist
cmd = "make roi_only"
env = os.environ.copy()
env['XRAY_PART'] = part
print("running subprocess")
subprocess.run(cmd.split(' '), check=True, env=env, cwd=cwd)
def main():
"""Rois all parts for a family by calling "make roi_only" over all parts
with the same device as XRAY_PART.
@ -36,12 +49,17 @@ def main():
if device['fabric'] == information['device']:
valid_devices.append(name)
tasks = []
for part, data in util.get_parts(db_root).items():
if data['device'] in valid_devices:
command = "make roi_only"
env['XRAY_PART'] = part
cwd = os.getenv('XRAY_FUZZERS_DIR')
subprocess.run(command.split(' '), check=True, env=env, cwd=cwd)
tasks.append((part, cwd))
with mp.Pool() as pool:
for _ in pool.imap_unordered(worker, tasks):
pass
if __name__ == '__main__':