From fc1f36f3c03ebfe006c19b54e28a4c7dfb096b8e Mon Sep 17 00:00:00 2001 From: Tim 'mithro' Ansell Date: Fri, 11 Jan 2019 20:44:19 -0800 Subject: [PATCH] infra: Restructure. Signed-off-by: Tim 'mithro' Ansell --- .github/kokoro/continuous-db-artix7-full.cfg | 33 +++++++ .github/kokoro/continuous-db-artix7-quick.cfg | 32 +++++++ .github/kokoro/continuous-db-kintex7-full.cfg | 33 +++++++ .../kokoro/continuous-db-kintex7-quick.cfg | 32 +++++++ .github/kokoro/continuous-db-zynq7-full.cfg | 33 +++++++ .github/kokoro/continuous-db-zynq7-quick.cfg | 32 +++++++ .github/kokoro/continuous.cfg | 11 --- .../kokoro/{presubmit.cfg => database.cfg} | 4 +- .github/kokoro/{full.sh => db-full.sh} | 28 +++++- .github/kokoro/db-quick.sh | 76 ++++++++++++++++ .github/kokoro/kokoro-cfg.py | 91 +++++++++++++++++++ .github/kokoro/presubmit-db-artix7.cfg | 32 +++++++ .github/kokoro/presubmit-db-kintex7.cfg | 32 +++++++ .github/kokoro/presubmit-db-zynq7.cfg | 32 +++++++ .github/kokoro/quick.sh | 1 - .github/kokoro/tests.cfg | 19 ++++ .github/kokoro/tests.sh | 81 +++++++++++++++++ Makefile | 2 +- 18 files changed, 585 insertions(+), 19 deletions(-) create mode 100644 .github/kokoro/continuous-db-artix7-full.cfg create mode 100644 .github/kokoro/continuous-db-artix7-quick.cfg create mode 100644 .github/kokoro/continuous-db-kintex7-full.cfg create mode 100644 .github/kokoro/continuous-db-kintex7-quick.cfg create mode 100644 .github/kokoro/continuous-db-zynq7-full.cfg create mode 100644 .github/kokoro/continuous-db-zynq7-quick.cfg delete mode 100644 .github/kokoro/continuous.cfg rename .github/kokoro/{presubmit.cfg => database.cfg} (52%) rename .github/kokoro/{full.sh => db-full.sh} (53%) create mode 100755 .github/kokoro/db-quick.sh create mode 100755 .github/kokoro/kokoro-cfg.py create mode 100644 .github/kokoro/presubmit-db-artix7.cfg create mode 100644 .github/kokoro/presubmit-db-kintex7.cfg create mode 100644 .github/kokoro/presubmit-db-zynq7.cfg delete mode 120000 .github/kokoro/quick.sh create mode 100644 .github/kokoro/tests.cfg create mode 100755 .github/kokoro/tests.sh diff --git a/.github/kokoro/continuous-db-artix7-full.cfg b/.github/kokoro/continuous-db-artix7-full.cfg new file mode 100644 index 00000000..1048062e --- /dev/null +++ b/.github/kokoro/continuous-db-artix7-full.cfg @@ -0,0 +1,33 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +build_file: "symbiflow-prjxray-continuous-db-artix7-full/.github/kokoro/db-full.sh" + +timeout_mins: 4320 + +action { + define_artifacts { + regex: "**/*result*.xml" + regex: "**/*.log" + regex: "database/artix7/**" + } +} + +env_vars { + key: "KOKORO_TYPE" + value: "continuous" +} + +env_vars { + key: "KOKORO_DIR" + value: "symbiflow-prjxray-continuous-db-artix7-full" +} + +env_vars { + key: "XRAY_SETTINGS" + value: "artix7" +} + +env_vars { + key: "XRAY_BUILD_TYPE" + value: "full" +} diff --git a/.github/kokoro/continuous-db-artix7-quick.cfg b/.github/kokoro/continuous-db-artix7-quick.cfg new file mode 100644 index 00000000..42d2b777 --- /dev/null +++ b/.github/kokoro/continuous-db-artix7-quick.cfg @@ -0,0 +1,32 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +build_file: "symbiflow-prjxray-continuous-db-artix7-quick/.github/kokoro/db-quick.sh" + +timeout_mins: 180 + +action { + define_artifacts { + regex: "**/*result*.xml" + regex: "**/*.log" + } +} + +env_vars { + key: "KOKORO_TYPE" + value: "continuous" +} + +env_vars { + key: "KOKORO_DIR" + value: "symbiflow-prjxray-continuous-db-artix7-quick" +} + +env_vars { + key: "XRAY_SETTINGS" + value: "artix7" +} + +env_vars { + key: "XRAY_BUILD_TYPE" + value: "quick" +} diff --git a/.github/kokoro/continuous-db-kintex7-full.cfg b/.github/kokoro/continuous-db-kintex7-full.cfg new file mode 100644 index 00000000..bceb827f --- /dev/null +++ b/.github/kokoro/continuous-db-kintex7-full.cfg @@ -0,0 +1,33 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +build_file: "symbiflow-prjxray-continuous-db-kintex7-full/.github/kokoro/db-full.sh" + +timeout_mins: 4320 + +action { + define_artifacts { + regex: "**/*result*.xml" + regex: "**/*.log" + regex: "database/kintex7/**" + } +} + +env_vars { + key: "KOKORO_TYPE" + value: "continuous" +} + +env_vars { + key: "KOKORO_DIR" + value: "symbiflow-prjxray-continuous-db-kintex7-full" +} + +env_vars { + key: "XRAY_SETTINGS" + value: "kintex7" +} + +env_vars { + key: "XRAY_BUILD_TYPE" + value: "full" +} diff --git a/.github/kokoro/continuous-db-kintex7-quick.cfg b/.github/kokoro/continuous-db-kintex7-quick.cfg new file mode 100644 index 00000000..3b67b337 --- /dev/null +++ b/.github/kokoro/continuous-db-kintex7-quick.cfg @@ -0,0 +1,32 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +build_file: "symbiflow-prjxray-continuous-db-kintex7-quick/.github/kokoro/db-quick.sh" + +timeout_mins: 180 + +action { + define_artifacts { + regex: "**/*result*.xml" + regex: "**/*.log" + } +} + +env_vars { + key: "KOKORO_TYPE" + value: "continuous" +} + +env_vars { + key: "KOKORO_DIR" + value: "symbiflow-prjxray-continuous-db-kintex7-quick" +} + +env_vars { + key: "XRAY_SETTINGS" + value: "kintex7" +} + +env_vars { + key: "XRAY_BUILD_TYPE" + value: "quick" +} diff --git a/.github/kokoro/continuous-db-zynq7-full.cfg b/.github/kokoro/continuous-db-zynq7-full.cfg new file mode 100644 index 00000000..3e581974 --- /dev/null +++ b/.github/kokoro/continuous-db-zynq7-full.cfg @@ -0,0 +1,33 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +build_file: "symbiflow-prjxray-continuous-db-zynq7-full/.github/kokoro/db-full.sh" + +timeout_mins: 4320 + +action { + define_artifacts { + regex: "**/*result*.xml" + regex: "**/*.log" + regex: "database/zynq7/**" + } +} + +env_vars { + key: "KOKORO_TYPE" + value: "continuous" +} + +env_vars { + key: "KOKORO_DIR" + value: "symbiflow-prjxray-continuous-db-zynq7-full" +} + +env_vars { + key: "XRAY_SETTINGS" + value: "zynq7" +} + +env_vars { + key: "XRAY_BUILD_TYPE" + value: "full" +} diff --git a/.github/kokoro/continuous-db-zynq7-quick.cfg b/.github/kokoro/continuous-db-zynq7-quick.cfg new file mode 100644 index 00000000..24cfb1bd --- /dev/null +++ b/.github/kokoro/continuous-db-zynq7-quick.cfg @@ -0,0 +1,32 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +build_file: "symbiflow-prjxray-continuous-db-zynq7-quick/.github/kokoro/db-quick.sh" + +timeout_mins: 180 + +action { + define_artifacts { + regex: "**/*result*.xml" + regex: "**/*.log" + } +} + +env_vars { + key: "KOKORO_TYPE" + value: "continuous" +} + +env_vars { + key: "KOKORO_DIR" + value: "symbiflow-prjxray-continuous-db-zynq7-quick" +} + +env_vars { + key: "XRAY_SETTINGS" + value: "zynq7" +} + +env_vars { + key: "XRAY_BUILD_TYPE" + value: "quick" +} diff --git a/.github/kokoro/continuous.cfg b/.github/kokoro/continuous.cfg deleted file mode 100644 index 116fccbc..00000000 --- a/.github/kokoro/continuous.cfg +++ /dev/null @@ -1,11 +0,0 @@ -build_file: "symbiflow-prjxray-artix7/.github/kokoro/full.sh" - -timeout_mins: 4320 - -action { - define_artifacts { - regex: "**/*result*.xml" - regex: "**/*.log" - regex: "database/**" - } -} diff --git a/.github/kokoro/presubmit.cfg b/.github/kokoro/database.cfg similarity index 52% rename from .github/kokoro/presubmit.cfg rename to .github/kokoro/database.cfg index 9832de76..762600a4 100644 --- a/.github/kokoro/presubmit.cfg +++ b/.github/kokoro/database.cfg @@ -1,6 +1,6 @@ -build_file: "symbiflow-prjxray-artix7/.github/kokoro/quick.sh" +build_file: "symbiflow-prjxray-db/.github/kokoro/database.sh" -timeout_mins: 240 +timeout_mins: 10 action { define_artifacts { diff --git a/.github/kokoro/full.sh b/.github/kokoro/db-full.sh similarity index 53% rename from .github/kokoro/full.sh rename to .github/kokoro/db-full.sh index 4e60a9a5..3c623bd2 100755 --- a/.github/kokoro/full.sh +++ b/.github/kokoro/db-full.sh @@ -1,7 +1,7 @@ #!/bin/bash -set -e set -x +set -e sudo apt-get update sudo apt-get install -y \ @@ -25,7 +25,27 @@ sudo apt-get install -y \ ls -l ~/.Xilinx sudo chown -R $USER ~/.Xilinx -cd github/symbiflow-prjxray-artix7/ +echo "----------------------------------------" +echo "----------------------------------------" +echo "----------------------------------------" + +export + +echo "----------------------------------------" +echo "----------------------------------------" +echo "----------------------------------------" + +find . + +echo "----------------------------------------" +echo "----------------------------------------" +echo "----------------------------------------" + +echo $PWD + +echo "----------------------------------------" + +cd github/$KOKORO_DIR/ git fetch --tags || true git describe --tags || true @@ -36,9 +56,9 @@ make build --output-sync=target --warn-undefined-variables # Setup the Python environment make env --output-sync=target --warn-undefined-variables -source settings/artix7.sh +source settings/$XRAY_SETTINGS.sh ( export XILINX_LOCAL_USER_DATA=no cd fuzzers - make --output-sync=target --warn-undefined-variables QUICK=y + make --output-sync=target --warn-undefined-variables ) diff --git a/.github/kokoro/db-quick.sh b/.github/kokoro/db-quick.sh new file mode 100755 index 00000000..08e19752 --- /dev/null +++ b/.github/kokoro/db-quick.sh @@ -0,0 +1,76 @@ +#!/bin/bash + +set -x +set -e + +sudo apt-get update +sudo apt-get install -y \ + bison \ + build-essential \ + ca-certificates \ + clang-format \ + cmake \ + curl \ + flex \ + fontconfig \ + git \ + jq \ + python \ + python3 \ + python3-dev \ + python3-virtualenv \ + python3-yaml \ + virtualenv \ + +ls -l ~/.Xilinx +sudo chown -R $USER ~/.Xilinx + +CORES=$(nproc --all) + +echo "----------------------------------------" +echo "----------------------------------------" +echo "----------------------------------------" + +export + +echo "----------------------------------------" +echo "----------------------------------------" +echo "----------------------------------------" + +find . + +echo "----------------------------------------" +echo "----------------------------------------" +echo "----------------------------------------" + +echo $PWD + +echo "----------------------------------------" + +cd github/$KOKORO_DIR/ + +git fetch --tags || true +git describe --tags || true + +# Build the C++ tools +make build --output-sync=target --warn-undefined-variables + +# Setup the Python environment +make env --output-sync=target --warn-undefined-variables + +echo "----------------------------------------" +echo "----------------------------------------" +echo "----------------------------------------" + +find . + +echo "----------------------------------------" +echo "----------------------------------------" +echo "----------------------------------------" + +source settings/$XRAY_SETTINGS.sh +( + export XILINX_LOCAL_USER_DATA=no + cd fuzzers + make --output-sync=target --warn-undefined-variables QUICK=y -j$CORES +) diff --git a/.github/kokoro/kokoro-cfg.py b/.github/kokoro/kokoro-cfg.py new file mode 100755 index 00000000..2c0b5627 --- /dev/null +++ b/.github/kokoro/kokoro-cfg.py @@ -0,0 +1,91 @@ +#!/usr/bin/env python3 + +db_quick = """\ +# Format: //devtools/kokoro/config/proto/build.proto + +build_file: "symbiflow-prjxray-%(kokoro_type)s-db-%(part)s%(ext)s/.github/kokoro/db-quick.sh" + +timeout_mins: 180 + +action { + define_artifacts { + regex: "**/*result*.xml" + regex: "**/*.log" + } +} + +env_vars { + key: "KOKORO_TYPE" + value: "%(kokoro_type)s" +} + +env_vars { + key: "KOKORO_DIR" + value: "symbiflow-prjxray-%(kokoro_type)s-db-%(part)s%(ext)s" +} + +env_vars { + key: "XRAY_SETTINGS" + value: "%(part)s" +} + +env_vars { + key: "XRAY_BUILD_TYPE" + value: "quick" +} +""" + +db_full = """\ +# Format: //devtools/kokoro/config/proto/build.proto + +build_file: "symbiflow-prjxray-%(kokoro_type)s-db-%(part)s-full/.github/kokoro/db-full.sh" + +timeout_mins: 4320 + +action { + define_artifacts { + regex: "**/*result*.xml" + regex: "**/*.log" + regex: "database/%(part)s/**" + } +} + +env_vars { + key: "KOKORO_TYPE" + value: "%(kokoro_type)s" +} + +env_vars { + key: "KOKORO_DIR" + value: "symbiflow-prjxray-%(kokoro_type)s-db-%(part)s-full" +} + +env_vars { + key: "XRAY_SETTINGS" + value: "%(part)s" +} + +env_vars { + key: "XRAY_BUILD_TYPE" + value: "full" +} +""" + +for part in ['artix7', 'kintex7', 'zynq7']: + with open("continuous-db-%s-quick.cfg" % part, "w") as f: + f.write( + db_quick % { + 'part': part, + 'kokoro_type': 'continuous', + 'ext': '-quick' + }) + with open("continuous-db-%s-full.cfg" % part, "w") as f: + f.write(db_full % {'part': part, 'kokoro_type': 'continuous'}) + + with open("presubmit-db-%s.cfg" % part, "w") as f: + f.write( + db_quick % { + 'part': part, + 'kokoro_type': 'presubmit', + 'ext': '' + }) diff --git a/.github/kokoro/presubmit-db-artix7.cfg b/.github/kokoro/presubmit-db-artix7.cfg new file mode 100644 index 00000000..c7ca90ff --- /dev/null +++ b/.github/kokoro/presubmit-db-artix7.cfg @@ -0,0 +1,32 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +build_file: "symbiflow-prjxray-presubmit-db-artix7/.github/kokoro/db-quick.sh" + +timeout_mins: 180 + +action { + define_artifacts { + regex: "**/*result*.xml" + regex: "**/*.log" + } +} + +env_vars { + key: "KOKORO_TYPE" + value: "presubmit" +} + +env_vars { + key: "KOKORO_DIR" + value: "symbiflow-prjxray-presubmit-db-artix7" +} + +env_vars { + key: "XRAY_SETTINGS" + value: "artix7" +} + +env_vars { + key: "XRAY_BUILD_TYPE" + value: "quick" +} diff --git a/.github/kokoro/presubmit-db-kintex7.cfg b/.github/kokoro/presubmit-db-kintex7.cfg new file mode 100644 index 00000000..3a501168 --- /dev/null +++ b/.github/kokoro/presubmit-db-kintex7.cfg @@ -0,0 +1,32 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +build_file: "symbiflow-prjxray-presubmit-db-kintex7/.github/kokoro/db-quick.sh" + +timeout_mins: 180 + +action { + define_artifacts { + regex: "**/*result*.xml" + regex: "**/*.log" + } +} + +env_vars { + key: "KOKORO_TYPE" + value: "presubmit" +} + +env_vars { + key: "KOKORO_DIR" + value: "symbiflow-prjxray-presubmit-db-kintex7" +} + +env_vars { + key: "XRAY_SETTINGS" + value: "kintex7" +} + +env_vars { + key: "XRAY_BUILD_TYPE" + value: "quick" +} diff --git a/.github/kokoro/presubmit-db-zynq7.cfg b/.github/kokoro/presubmit-db-zynq7.cfg new file mode 100644 index 00000000..6026b61e --- /dev/null +++ b/.github/kokoro/presubmit-db-zynq7.cfg @@ -0,0 +1,32 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +build_file: "symbiflow-prjxray-presubmit-db-zynq7/.github/kokoro/db-quick.sh" + +timeout_mins: 180 + +action { + define_artifacts { + regex: "**/*result*.xml" + regex: "**/*.log" + } +} + +env_vars { + key: "KOKORO_TYPE" + value: "presubmit" +} + +env_vars { + key: "KOKORO_DIR" + value: "symbiflow-prjxray-presubmit-db-zynq7" +} + +env_vars { + key: "XRAY_SETTINGS" + value: "zynq7" +} + +env_vars { + key: "XRAY_BUILD_TYPE" + value: "quick" +} diff --git a/.github/kokoro/quick.sh b/.github/kokoro/quick.sh deleted file mode 120000 index ee2000b0..00000000 --- a/.github/kokoro/quick.sh +++ /dev/null @@ -1 +0,0 @@ -full.sh \ No newline at end of file diff --git a/.github/kokoro/tests.cfg b/.github/kokoro/tests.cfg new file mode 100644 index 00000000..93b85287 --- /dev/null +++ b/.github/kokoro/tests.cfg @@ -0,0 +1,19 @@ +build_file: "symbiflow-prjxray-tests/.github/kokoro/tests.sh" + +timeout_mins: 10 + +action { + define_artifacts { + regex: "**/*.log" + regex: "**/*sponge_log.xml" + regex: "**/build/**/*.xml" + regex: "**/build/*.xml" + regex: "*result*.xml" + strip_prefix: "github/symbiflow-prjxray-tests/" + } +} + +env_vars { + key: "KOKORO_DIR" + value: "symbiflow-prjxray-tests" +} diff --git a/.github/kokoro/tests.sh b/.github/kokoro/tests.sh new file mode 100755 index 00000000..93f94de1 --- /dev/null +++ b/.github/kokoro/tests.sh @@ -0,0 +1,81 @@ +#!/bin/bash + +set -x +set -e + +sudo apt-get update +sudo apt-get install -y \ + bison \ + build-essential \ + ca-certificates \ + clang-format \ + cmake \ + curl \ + flex \ + fontconfig \ + git \ + jq \ + python \ + python3 \ + python3-dev \ + python3-virtualenv \ + python3-yaml \ + virtualenv \ + +echo "----------------------------------------" +echo "----------------------------------------" +echo "----------------------------------------" + +export + +echo "----------------------------------------" +echo "----------------------------------------" +echo "----------------------------------------" + +find . + +echo "----------------------------------------" +echo "----------------------------------------" +echo "----------------------------------------" + +echo $PWD + +echo "----------------------------------------" + +cd github/$KOKORO_DIR/ + +git fetch --tags || true +git describe --tags || true + +# Build the C++ tools +make build --output-sync=target --warn-undefined-variables + +# Setup the Python environment +make env --output-sync=target --warn-undefined-variables + +echo "----------------------------------------" +echo "----------------------------------------" +echo "----------------------------------------" + +find . + +echo "----------------------------------------" +echo "----------------------------------------" +echo "----------------------------------------" + +# Run the tests +make test --output-sync=target --warn-undefined-variables + +echo "----------------------------------------" +echo "----------------------------------------" +echo "----------------------------------------" + +cat build/*test_results.xml +mkdir build/py +cp build/py_test_results.xml build/py/sponge_log.xml +mkdir build/cpp +cp build/cpp_test_results.xml build/cpp/sponge_log.xml + +echo "----------------------------------------" +echo "----------------------------------------" +echo "----------------------------------------" diff --git a/Makefile b/Makefile index 276e76ca..ad8ffd87 100644 --- a/Makefile +++ b/Makefile @@ -34,7 +34,7 @@ test: test-py test-cpp @true test-py: - $(IN_ENV) PYTHONPATH="$(PWD):$(PWD)/third_party/fasm:$PYTHONPATH" py.test $(TEST_EXCLUDE) --doctest-modules --junitxml=build/py_test_results.xml + $(IN_ENV) which py.test; py.test $(TEST_EXCLUDE) --doctest-modules --junitxml=build/py_test_results.xml test-cpp: mkdir -p build