diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000..c9ca13bd --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,91 @@ +on: + pull_request: + push: + schedule: + - cron: '0 0 * * *' + +name: CI Build + +jobs: + + Tests: + container: ubuntu:bionic + + runs-on: [self-hosted, Linux, X64] + + env: + ALLOW_ROOT: true + + steps: + + - uses: actions/checkout@v2 + with: + submodules: recursive + + - name: Install + run: | + apt update + apt install -y \ + bash bison build-essential ca-certificates clang-format cmake psmisc \ + colordiff coreutils git flex python3 python3-dev python3-venv xsltproc + + - name: Build + run: make build --output-sync=target --warn-undefined-variables -j$(nproc) + + - name: Environment + run: make env --output-sync=target --warn-undefined-variables + + - name: Run Test + run: make test --output-sync=target --warn-undefined-variables + + - uses: actions/upload-artifact@v2 + if: ${{ always() }} + with: + path: | + **/results*.gz + **/plot_*.svg + + BuildDatabase: + container: ubuntu:bionic + + runs-on: [self-hosted, Linux, X64] + + strategy: + fail-fast: false + matrix: + family: ['artix7', 'zynq7', 'kintex7', 'spartan7'] + + env: + ALLOW_ROOT: true + GHA_EXTERNAL_DISK: "tools" + + steps: + + - uses: actions/checkout@v2 + with: + submodules: recursive + + - name: Install + run: | + apt update + apt install -y \ + bash bison build-essential ca-certificates clang-format cmake psmisc \ + colordiff coreutils git flex python3 python3-dev python3-venv xsltproc + + - name: Build + run: make build --output-sync=target --warn-undefined-variables -j$(nproc) + + - name: Environment + run: make env --output-sync=target --warn-undefined-variables + + - name: Run Test + run: .github/workflows/scripts/db.sh + env: + XRAY_SETTINGS: ${{ matrix.family }} + + - uses: actions/upload-artifact@v2 + if: ${{ always() }} + with: + path: | + **/results*.gz + **/plot_*.svg diff --git a/.github/workflows/scripts/db.sh b/.github/workflows/scripts/db.sh new file mode 100755 index 00000000..084afb28 --- /dev/null +++ b/.github/workflows/scripts/db.sh @@ -0,0 +1,216 @@ +#!/bin/bash +# Copyright (C) 2017-2020 The Project X-Ray Authors. +# +# Use of this source code is governed by a ISC-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/ISC +# +# SPDX-License-Identifier: ISC + +set -e + +source $(dirname "$0")/hostinfo.sh + +echo +echo "=======================================" +echo "Creating Vivado Symbolic Link" +echo "---------------------------------------" +ln -s /mnt/aux/Xilinx /opt/Xilinx +ls /opt/Xilinx/Vivado +source /opt/Xilinx/Vivado/2017.2/settings64.sh +vivado -version + +echo +echo "========================================" +echo "Downloading current database" +echo "----------------------------------------" +( + script --return --flush --command "./download-latest-db.sh" - +) +echo "----------------------------------------" + +echo +echo "========================================" +echo "Preparing database" +echo "----------------------------------------" +( + make db-prepare-${XRAY_SETTINGS} +) +echo "----------------------------------------" + +source settings/$XRAY_SETTINGS.sh + +echo +echo "========================================" +echo "Cleaning out current database" +echo "----------------------------------------" +( + cd database + make clean-${XRAY_SETTINGS}-db +) +echo "----------------------------------------" + +echo +echo "========================================" +echo "Running Database build" +echo "----------------------------------------" +( + # Output which fuzzers we are going to run + echo "make --dry-run" + make --dry-run db-${XRAY_SETTINGS}-all + echo "----------------------------------------" + + # Run the fuzzers + set -x +e + tmp=`mktemp` + script --return --flush --command "make -j $CORES MAX_VIVADO_PROCESS=$MAX_VIVADO_PROCESS db-${XRAY_SETTINGS}-all" $tmp + DATABASE_RET=$? + set +x -e + + if [[ $DATABASE_RET != 0 ]] ; then + # Collect the Vivado logs into one tgz archive + echo "Packing failing test cases" + grep "recipe for target" $tmp | awk 'match($0,/recipe for target.*'\''(.*)\/run\..*ok'\''/,res) {print "fuzzers/" res[1]}' | xargs tar -zcf fuzzers/fails.tgz + echo "----------------------------------------" + echo "A failure occurred during Database build." + echo "----------------------------------------" + rm $tmp + + echo "========================================" + echo " Disk space in failure path" + echo "----------------------------------------" + du -sh + + exit $DATABASE_RET + fi +) +echo "----------------------------------------" + +# Format the database +make db-format-${XRAY_SETTINGS} +# Update the database/Info.md file +make db-info + +# Output if the database has differences +echo +echo "========================================" +echo " Database Differences" +echo "----------------------------------------" +( + cd database + # Update the index with any new files + git add \ + --verbose \ + --all \ + --ignore-errors \ + . + + # Output what git status + echo + echo "----------------------------------------" + echo " Database Status" + echo "----------------------------------------" + git status + echo "----------------------------------------" + + + # Output a summary of how the files have changed + echo + echo "----------------------------------------" + echo " Database Diff Summary" + echo "----------------------------------------" + git diff --stat --irreversible-delete --find-renames --find-copies --ignore-all-space origin/master + + # Save the diff to be uploaded as an artifact + echo + echo "----------------------------------------" + echo " Saving diff output" + echo "----------------------------------------" + # Patch file + git diff \ + --patch-with-stat --no-color --irreversible-delete --find-renames --find-copies origin/master \ + > diff.patch + + MAX_DIFF_LINES=50000 + DIFF_LINES="$(wc -l diff.patch | sed -e's/ .*$//')" + if [ $DIFF_LINES -gt $MAX_DIFF_LINES ]; then + echo + echo "----------------------------------------" + echo " Database Diff" + echo "----------------------------------------" + echo "diff has $DIFF_LINES lines which is too large to display!" + + echo + echo "----------------------------------------" + echo " Generating pretty diff output" + echo "----------------------------------------" + echo "diff has $DIFF_LINES lines which is too large for HTML output!" + else + # Output the actually diff + echo + echo "----------------------------------------" + echo " Database Diff" + echo "----------------------------------------" + git diff --color --irreversible-delete --find-renames --find-copies --ignore-all-space origin/master + + echo + echo "----------------------------------------" + echo " Generating pretty diff output" + echo "----------------------------------------" + ( + # Allow the diff2html to fail. + set +e + + # Pretty HTML file version + diff2html --summary=open --file diff.html --format html \ + -- \ + --irreversible-delete --find-renames --find-copies \ + --ignore-all-space origin/master || true + + # Programmatic JSON version + diff2html --file diff.json --format json \ + -- \ + --irreversible-delete --find-renames --find-copies \ + --ignore-all-space origin/master || true + ) || true + fi +) +echo "----------------------------------------" + +# Check the database and fail if it is broken. +set -x +e +make db-check-${XRAY_SETTINGS} +CHECK_RET=$? +set +x -e + +echo +echo "========================================" +echo " Testing HTML generation" +echo "----------------------------------------" +( + cd htmlgen + source htmlgen.sh $XRAY_SETTINGS +) + +# If we get here, then all the fuzzers completed fine. Hence we are +# going to assume we don't want to keep all the build / logs / etc (as +# they are quite large). Thus do a clean to get rid of them. +echo +echo "========================================" +echo " Cleaning up after success" +echo "----------------------------------------" +( + cd fuzzers + echo + echo "Cleaning up so CI doesn't save all the excess data." + make clean_fuzzers + make clean_piplists +) +echo "----------------------------------------" + +echo "========================================" +echo " Final disk space after cleanup" +echo "----------------------------------------" +du -sh + +exit $CHECK_RET diff --git a/.github/workflows/scripts/hostinfo.sh b/.github/workflows/scripts/hostinfo.sh new file mode 100755 index 00000000..8e69c993 --- /dev/null +++ b/.github/workflows/scripts/hostinfo.sh @@ -0,0 +1,45 @@ +#!/bin/bash +# Copyright (C) 2017-2020 The Project X-Ray Authors. +# +# Use of this source code is governed by a ISC-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/ISC +# +# SPDX-License-Identifier: ISC + +set -e + +echo +echo "========================================" +echo "Host Environment" +echo "----------------------------------------" +export +echo "----------------------------------------" + +echo +echo "========================================" +echo "Host CPU" +echo "----------------------------------------" +export CORES=$(nproc --all) +echo "Cores: $CORES" +echo +echo "Memory" +echo "----------------------------------------" +cat /proc/meminfo +echo "----------------------------------------" +export MEM_GB=$(($(awk '/MemTotal/ {print $2}' /proc/meminfo)/(1024*1024))) +echo "Total Memory (GB): $MEM_GB" + +# Approx memory per grid process +export MEM_PER_RUN=8 +export MAX_GRID_CPU=$(($MEM_GB/$MEM_PER_RUN)) +export MAX_VIVADO_PROCESS=$(($MEM_GB/$MEM_PER_RUN)) + +echo +echo "========================================" +echo "Host files" +echo "----------------------------------------" +echo $PWD +echo "----------------------------------------" +find . | sort +echo "----------------------------------------" diff --git a/Makefile b/Makefile index 53c38789..acb7f426 100644 --- a/Makefile +++ b/Makefile @@ -10,10 +10,14 @@ ALL_EXCLUDE = third_party .git env build docs/env INSTALL_DIR ?= +# Skip this check if the ALLOW_ROOT var is defined +# E.g. when running in GH action custom runners CI +ifndef ALLOW_ROOT # Check if root ifeq ($(shell id -u),0) $(error ERROR: Running as ID 0) endif +endif # Tools + Environment IN_ENV = if [ -e env/bin/activate ]; then . env/bin/activate; fi; source utils/environment.python.sh; @@ -275,4 +279,4 @@ clean: $(MAKE) -C fuzzers clean rm -rf build -.PHONY: clean \ No newline at end of file +.PHONY: clean