From 1991669e8c7549f8f8cd7a1724edf319a83c7809 Mon Sep 17 00:00:00 2001 From: Tim 'mithro' Ansell Date: Sun, 3 Feb 2019 14:44:35 +1100 Subject: [PATCH] infra: Reworking kokoro scripts. Signed-off-by: Tim 'mithro' Ansell --- .github/kokoro/db-full.sh | 111 +++++++++++----------------- .github/kokoro/db-quick.sh | 85 ++++++--------------- .github/kokoro/steps/git.sh | 27 +++++++ .github/kokoro/steps/hostinfo.sh | 31 ++++++++ .github/kokoro/steps/hostsetup.sh | 36 +++++++++ .github/kokoro/steps/prjxray-env.sh | 16 ++++ .github/kokoro/steps/xilinx.sh | 8 ++ .github/kokoro/tests.sh | 95 +++++------------------- 8 files changed, 207 insertions(+), 202 deletions(-) create mode 100755 .github/kokoro/steps/git.sh create mode 100755 .github/kokoro/steps/hostinfo.sh create mode 100755 .github/kokoro/steps/hostsetup.sh create mode 100755 .github/kokoro/steps/prjxray-env.sh create mode 100755 .github/kokoro/steps/xilinx.sh diff --git a/.github/kokoro/db-full.sh b/.github/kokoro/db-full.sh index 0b10f39f..0e3627ea 100755 --- a/.github/kokoro/db-full.sh +++ b/.github/kokoro/db-full.sh @@ -1,82 +1,61 @@ #!/bin/bash -set -x set -e -sudo apt-get update -sudo apt-get install -y \ - bison \ - build-essential \ - ca-certificates \ - clang-format \ - cmake \ - curl \ - flex \ - fontconfig \ - git \ - jq \ - psmisc \ - python \ - python3 \ - python3-dev \ - python3-virtualenv \ - python3-yaml \ - virtualenv \ - -ls -l ~/.Xilinx -sudo chown -R $USER ~/.Xilinx - -CORES=$(nproc --all) - -echo "----------------------------------------" -echo "----------------------------------------" -echo "----------------------------------------" - -export - -echo "----------------------------------------" -echo "----------------------------------------" -echo "----------------------------------------" - -cat /proc/cpuinfo - -echo "----------------------------------------" -echo "----------------------------------------" -echo "----------------------------------------" - -find . - -echo "----------------------------------------" -echo "----------------------------------------" -echo "----------------------------------------" - -echo $PWD - -echo "----------------------------------------" - cd github/$KOKORO_DIR/ -git fetch --tags || true -git describe --tags || true +source ./.github/kokoro/steps/hostsetup.sh +source ./.github/kokoro/steps/hostinfo.sh +source ./.github/kokoro/steps/git.sh -# Build the C++ tools -make build --output-sync=target --warn-undefined-variables -j$CORES +source ./.github/kokoro/steps/xilinx.sh -# Setup the Python environment -make env --output-sync=target --warn-undefined-variables +source ./.github/kokoro/steps/prjxray-env.sh +echo +echo "========================================" +echo "Downloading current database" echo "----------------------------------------" -echo "----------------------------------------" +( + ./download-latest-db.sh +) echo "----------------------------------------" source settings/$XRAY_SETTINGS.sh -( - export XILINX_LOCAL_USER_DATA=no - cd fuzzers - make --warn-undefined-variables -j$CORES MAX_VIVADO_PROCESS=$CORES -) -# Output how the database differs +echo +echo "========================================" +echo "Cleaning out current database" +echo "----------------------------------------" +( + cd database + make clean-${XRAY_SETTINGS}-db +) +echo "----------------------------------------" + +echo +echo "========================================" +echo "Running Database build" +echo "----------------------------------------" +( + cd fuzzers + echo "make --dry-run" + make --dry-run + echo "----------------------------------------" + export MAX_VIVADO_PROCESS=$CORES + set -x + make -j $CORES MAX_VIVADO_PROCESS=$CORES + set +x + echo "----------------------------------------" + echo "make --dry-run" + make --dry-run +) +echo "----------------------------------------" + +echo +echo "========================================" +echo " Database Differences" +echo "----------------------------------------" ( make formatdb cd database @@ -92,5 +71,5 @@ source settings/$XRAY_SETTINGS.sh echo " Database Diff" echo "----------------------------------------" git diff - echo "----------------------------------------" ) +echo "----------------------------------------" diff --git a/.github/kokoro/db-quick.sh b/.github/kokoro/db-quick.sh index a839eeb1..58e0f69b 100755 --- a/.github/kokoro/db-quick.sh +++ b/.github/kokoro/db-quick.sh @@ -1,77 +1,40 @@ #!/bin/bash -set -x set -e -sudo apt-get update -sudo apt-get install -y \ - bison \ - build-essential \ - ca-certificates \ - clang-format \ - cmake \ - curl \ - flex \ - fontconfig \ - git \ - jq \ - psmisc \ - python \ - python3 \ - python3-dev \ - python3-virtualenv \ - python3-yaml \ - virtualenv \ - -ls -l ~/.Xilinx -sudo chown -R $USER ~/.Xilinx - -CORES=$(nproc --all) - -echo "----------------------------------------" -echo "----------------------------------------" -echo "----------------------------------------" - -export - -echo "----------------------------------------" -echo "----------------------------------------" -echo "----------------------------------------" - -cat /proc/cpuinfo - -echo "----------------------------------------" -echo "----------------------------------------" -echo "----------------------------------------" - -find . - -echo "----------------------------------------" -echo "----------------------------------------" -echo "----------------------------------------" - -echo $PWD - -echo "----------------------------------------" - cd github/$KOKORO_DIR/ -git fetch --tags || true -git describe --tags || true +source ./.github/kokoro/steps/hostsetup.sh +source ./.github/kokoro/steps/hostinfo.sh +source ./.github/kokoro/steps/git.sh -# Build the C++ tools -make build --output-sync=target --warn-undefined-variables -j$CORES +source ./.github/kokoro/steps/xilinx.sh -# Setup the Python environment -make env --output-sync=target --warn-undefined-variables +source ./.github/kokoro/steps/prjxray-env.sh +echo +echo "========================================" +echo "Downloading current database" echo "----------------------------------------" -echo "----------------------------------------" +( + ./download-latest-db.sh +) echo "----------------------------------------" source settings/$XRAY_SETTINGS.sh + +echo +echo "========================================" +echo "Running quick Database build" +echo "----------------------------------------" ( - export XILINX_LOCAL_USER_DATA=no cd fuzzers - make --warn-undefined-variables QUICK=y -j$CORES + echo "make --dry-run" + make --dry-run + echo "----------------------------------------" + export MAX_VIVADO_PROCESS=$CORES + set -x + make -j $CORES MAX_VIVADO_PROCESS=$CORES QUICK=y + set +x ) +echo "----------------------------------------" diff --git a/.github/kokoro/steps/git.sh b/.github/kokoro/steps/git.sh new file mode 100755 index 00000000..1837389c --- /dev/null +++ b/.github/kokoro/steps/git.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +set -e + +echo +echo "========================================" +echo "Git log" +echo "----------------------------------------" +echo "----------------------------------------" + +echo +echo "========================================" +echo "Git fetching tags" +echo "----------------------------------------" +git fetch --tags || true +echo "----------------------------------------" + +echo +echo "========================================" +echo "Git version info" +echo "----------------------------------------" +git log -n1 +echo "----------------------------------------" +git describe --tags || true +echo "----------------------------------------" +git describe --tags --always || true +echo "----------------------------------------" diff --git a/.github/kokoro/steps/hostinfo.sh b/.github/kokoro/steps/hostinfo.sh new file mode 100755 index 00000000..6a00c37f --- /dev/null +++ b/.github/kokoro/steps/hostinfo.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +set -e + +echo +echo "========================================" +echo "Host Environment" +echo "----------------------------------------" +export +echo "----------------------------------------" + +echo +echo "========================================" +echo "Host CPU" +echo "----------------------------------------" +export CORES=$(nproc --all) +echo "Cores: $CORES" +echo +echo "Memory" +echo "----------------------------------------" +cat /proc/meminfo +echo "----------------------------------------" + +echo +echo "========================================" +echo "Host files" +echo "----------------------------------------" +echo $PWD +echo "----------------------------------------" +find . | sort +echo "----------------------------------------" diff --git a/.github/kokoro/steps/hostsetup.sh b/.github/kokoro/steps/hostsetup.sh new file mode 100755 index 00000000..16a423b9 --- /dev/null +++ b/.github/kokoro/steps/hostsetup.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +set -e + +echo +echo "========================================" +echo "Host updating packages" +echo "----------------------------------------" +sudo apt-get update +echo "----------------------------------------" + +echo +echo "========================================" +echo "Host install packages" +echo "----------------------------------------" +sudo apt-get install -y \ + bison \ + build-essential \ + ca-certificates \ + clang-format \ + cmake \ + curl \ + flex \ + fontconfig \ + git \ + jq \ + psmisc \ + python \ + python3 \ + python3-dev \ + python3-virtualenv \ + python3-yaml \ + virtualenv \ + +echo "----------------------------------------" + diff --git a/.github/kokoro/steps/prjxray-env.sh b/.github/kokoro/steps/prjxray-env.sh new file mode 100755 index 00000000..42a54f9d --- /dev/null +++ b/.github/kokoro/steps/prjxray-env.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +set -e + +echo +echo "========================================" +echo "Build the C++ tools" +echo "----------------------------------------" +make build --output-sync=target --warn-undefined-variables -j $CORES +echo "----------------------------------------" + +echo +echo "========================================" +echo "Setup the Python environment" +echo "----------------------------------------" +make env --output-sync=target --warn-undefined-variables diff --git a/.github/kokoro/steps/xilinx.sh b/.github/kokoro/steps/xilinx.sh new file mode 100755 index 00000000..d874c37f --- /dev/null +++ b/.github/kokoro/steps/xilinx.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +# Fix up things related to Xilinx tool chain. + +ls -l ~/.Xilinx +sudo chown -R $USER ~/.Xilinx + +export XILINX_LOCAL_USER_DATA=no diff --git a/.github/kokoro/tests.sh b/.github/kokoro/tests.sh index ba87d3ed..91eb01e5 100755 --- a/.github/kokoro/tests.sh +++ b/.github/kokoro/tests.sh @@ -1,88 +1,33 @@ #!/bin/bash -set -x set -e -sudo apt-get update -sudo apt-get install -y \ - bison \ - build-essential \ - ca-certificates \ - clang-format \ - cmake \ - curl \ - flex \ - fontconfig \ - git \ - jq \ - psmisc \ - python \ - python3 \ - python3-dev \ - python3-virtualenv \ - python3-yaml \ - virtualenv \ - -echo "----------------------------------------" -echo "----------------------------------------" -echo "----------------------------------------" - -export - -echo "----------------------------------------" -echo "----------------------------------------" -echo "----------------------------------------" - -cat /proc/cpuinfo - -echo "----------------------------------------" -echo "----------------------------------------" -echo "----------------------------------------" - -find . - -echo "----------------------------------------" -echo "----------------------------------------" -echo "----------------------------------------" - -echo $PWD - -echo "----------------------------------------" - cd github/$KOKORO_DIR/ -git fetch --tags || true -git describe --tags || true +source ./.github/kokoro/steps/hostsetup.sh +source ./.github/kokoro/steps/hostinfo.sh +source ./.github/kokoro/steps/git.sh -# Build the C++ tools -make build --output-sync=target --warn-undefined-variables - -# Setup the Python environment -make env --output-sync=target --warn-undefined-variables +source ./.github/kokoro/steps/prjxray-env.sh +echo +echo "========================================" +echo "Running tests" echo "----------------------------------------" -echo "----------------------------------------" +( + make test --output-sync=target --warn-undefined-variables +) echo "----------------------------------------" -find . - -echo "----------------------------------------" -echo "----------------------------------------" -echo "----------------------------------------" - -# Run the tests -make test --output-sync=target --warn-undefined-variables - -echo "----------------------------------------" -echo "----------------------------------------" -echo "----------------------------------------" - -cat build/*test_results.xml -mkdir build/py -cp build/py_test_results.xml build/py/sponge_log.xml -mkdir build/cpp -cp build/cpp_test_results.xml build/cpp/sponge_log.xml - -echo "----------------------------------------" +echo +echo "========================================" +echo "Copying tests logs" echo "----------------------------------------" +( + cat build/*test_results.xml + mkdir build/py + cp build/py_test_results.xml build/py/sponge_log.xml + mkdir build/cpp + cp build/cpp_test_results.xml build/cpp/sponge_log.xml +) echo "----------------------------------------"