From 1145e9a2707d6af43f95de7f9ea1f5bc23d3ac92 Mon Sep 17 00:00:00 2001 From: Keith Rothman <537074+litghost@users.noreply.github.com> Date: Fri, 8 Feb 2019 10:27:15 -0800 Subject: [PATCH 1/4] Dup jobserver fd's for retry. Signed-off-by: Keith Rothman <537074+litghost@users.noreply.github.com> --- fuzzers/run_fuzzer.py | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/fuzzers/run_fuzzer.py b/fuzzers/run_fuzzer.py index 7d2f2e21..172589e4 100755 --- a/fuzzers/run_fuzzer.py +++ b/fuzzers/run_fuzzer.py @@ -372,6 +372,7 @@ def main(argv): fuzzer_dir, fuzzer_logdir, logger, + will_retry=retry_count < (args.retries-1), ) if exit_code <= 0: break @@ -380,7 +381,7 @@ def main(argv): return exit_code -def run_fuzzer(fuzzer_name, fuzzer_dir, fuzzer_logdir, logger): +def run_fuzzer(fuzzer_name, fuzzer_dir, fuzzer_logdir, logger, will_retry): def log(msg, *a, **k): logger.log(msg, a, k, flush=True) @@ -436,17 +437,22 @@ def run_fuzzer(fuzzer_name, fuzzer_dir, fuzzer_logdir, logger): assert job_re, make_flags job_rd, job_wr = job_re.groups() + # Make copies of jobserver FDs in case a retry is needed. + job_rd = int(job_rd) job_wr = int(job_wr) assert job_rd > 2, (job_rd, job_wr, make_flags) assert job_wr > 2, (job_rd, job_wr, make_flags) # Make sure the file descriptors exist.. - job_rd_fd = os.fdopen(int(job_rd), 'rb', 0) + job_rd_fd = os.fdopen(job_rd, 'rb', 0) assert job_rd_fd - job_wr_fd = os.fdopen(int(job_wr), 'rb', 0) + job_wr_fd = os.fdopen(job_wr, 'rb', 0) assert job_wr_fd + job_rd_copy = os.dup(job_rd) + job_wr_copy = os.dup(job_wr) + p = None try: p = subprocess.Popen( @@ -554,6 +560,13 @@ Failed @ {time_end} with exit code: {retcode} error_log=error_log, time_end=time_end.isoformat()) + if will_retry: + # Restore jobserver FD's + os.dup2(job_rd_copy, job_rd) + os.dup2(job_wr_copy, job_wr) + else: + os.close(job_rd_copy) + os.close(job_wr_copy) else: # Log the last 100 lines of a successful run @@ -566,6 +579,9 @@ Succeeded! @ {} Succeeded! @ {} """, time_end.isoformat(), success_log, time_end.isoformat()) + os.close(job_rd_copy) + os.close(job_wr_copy) + logger.flush() signal.signal(signal.SIGINT, old_sigint_handler) return retcode From c47746476d02ea5a2211f922222232afcee0ae0a Mon Sep 17 00:00:00 2001 From: Keith Rothman <537074+litghost@users.noreply.github.com> Date: Fri, 8 Feb 2019 10:50:03 -0800 Subject: [PATCH 2/4] Make format. Signed-off-by: Keith Rothman <537074+litghost@users.noreply.github.com> --- fuzzers/run_fuzzer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fuzzers/run_fuzzer.py b/fuzzers/run_fuzzer.py index 172589e4..c1369cd8 100755 --- a/fuzzers/run_fuzzer.py +++ b/fuzzers/run_fuzzer.py @@ -372,7 +372,7 @@ def main(argv): fuzzer_dir, fuzzer_logdir, logger, - will_retry=retry_count < (args.retries-1), + will_retry=retry_count < (args.retries - 1), ) if exit_code <= 0: break From 820ea7703f2a781181131da84df1539046cb53a8 Mon Sep 17 00:00:00 2001 From: Keith Rothman <537074+litghost@users.noreply.github.com> Date: Fri, 8 Feb 2019 10:55:35 -0800 Subject: [PATCH 3/4] Update CI configuration: - Log output of run_fuzzer.py - Decrease number of processes running for 072 and 074 to lower memory load. Signed-off-by: Keith Rothman <537074+litghost@users.noreply.github.com> --- .github/kokoro/database.cfg | 2 ++ .github/kokoro/db-full.sh | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/kokoro/database.cfg b/.github/kokoro/database.cfg index c904160c..ebc11d87 100644 --- a/.github/kokoro/database.cfg +++ b/.github/kokoro/database.cfg @@ -6,6 +6,8 @@ action { define_artifacts { regex: "**/*result*.xml" regex: "**/build/*.log" + regex: "**/logs/*.log" + regex: "**/logs/*.xml" regex: "**/build/**" regex: "**/diff.*" regex: "**/database/**" diff --git a/.github/kokoro/db-full.sh b/.github/kokoro/db-full.sh index 905aa201..03c31b37 100755 --- a/.github/kokoro/db-full.sh +++ b/.github/kokoro/db-full.sh @@ -46,9 +46,9 @@ echo "----------------------------------------" echo "----------------------------------------" # Run the fuzzers - export MAX_VIVADO_PROCESS=$CORES + export MAX_VIVADO_PROCESS=$((CORES/2 < 20 ? CORES/2 : 20)) set -x - script --return --flush --command "make -j $CORES MAX_VIVADO_PROCESS=$CORES" - + script --return --flush --command "make -j $CORES MAX_VIVADO_PROCESS=$MAX_VIVADO_PROCESS" - set +x echo "----------------------------------------" From 24f34c7372851c6729b8f2a5f5e60ceda405599a Mon Sep 17 00:00:00 2001 From: Keith Rothman <537074+litghost@users.noreply.github.com> Date: Fri, 8 Feb 2019 12:47:31 -0800 Subject: [PATCH 4/4] Add comment about limiting MAX_VIVADO_INSTANCE to 20. Signed-off-by: Keith Rothman <537074+litghost@users.noreply.github.com> --- .github/kokoro/db-full.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/kokoro/db-full.sh b/.github/kokoro/db-full.sh index 03c31b37..3f3a1cd0 100755 --- a/.github/kokoro/db-full.sh +++ b/.github/kokoro/db-full.sh @@ -46,6 +46,12 @@ echo "----------------------------------------" echo "----------------------------------------" # Run the fuzzers + # + # Cap MAX_VIVADO_PROCESS at 20 to limit memory usage of 074 fuzzer. + # At MAX_VIVADO_PROCESS=20: + # - 072 completes in ~35 minutes + # - 074 completes in ~60 minutes + # which is well before the 05x INT fuzzers complete. export MAX_VIVADO_PROCESS=$((CORES/2 < 20 ? CORES/2 : 20)) set -x script --return --flush --command "make -j $CORES MAX_VIVADO_PROCESS=$MAX_VIVADO_PROCESS" -