diff --git a/.github/kokoro/database.cfg b/.github/kokoro/database.cfg index c904160c..ebc11d87 100644 --- a/.github/kokoro/database.cfg +++ b/.github/kokoro/database.cfg @@ -6,6 +6,8 @@ action { define_artifacts { regex: "**/*result*.xml" regex: "**/build/*.log" + regex: "**/logs/*.log" + regex: "**/logs/*.xml" regex: "**/build/**" regex: "**/diff.*" regex: "**/database/**" diff --git a/.github/kokoro/db-full.sh b/.github/kokoro/db-full.sh index 905aa201..3f3a1cd0 100755 --- a/.github/kokoro/db-full.sh +++ b/.github/kokoro/db-full.sh @@ -46,9 +46,15 @@ echo "----------------------------------------" echo "----------------------------------------" # Run the fuzzers - export MAX_VIVADO_PROCESS=$CORES + # + # Cap MAX_VIVADO_PROCESS at 20 to limit memory usage of 074 fuzzer. + # At MAX_VIVADO_PROCESS=20: + # - 072 completes in ~35 minutes + # - 074 completes in ~60 minutes + # which is well before the 05x INT fuzzers complete. + export MAX_VIVADO_PROCESS=$((CORES/2 < 20 ? CORES/2 : 20)) set -x - script --return --flush --command "make -j $CORES MAX_VIVADO_PROCESS=$CORES" - + script --return --flush --command "make -j $CORES MAX_VIVADO_PROCESS=$MAX_VIVADO_PROCESS" - set +x echo "----------------------------------------" diff --git a/fuzzers/run_fuzzer.py b/fuzzers/run_fuzzer.py index 7d2f2e21..c1369cd8 100755 --- a/fuzzers/run_fuzzer.py +++ b/fuzzers/run_fuzzer.py @@ -372,6 +372,7 @@ def main(argv): fuzzer_dir, fuzzer_logdir, logger, + will_retry=retry_count < (args.retries - 1), ) if exit_code <= 0: break @@ -380,7 +381,7 @@ def main(argv): return exit_code -def run_fuzzer(fuzzer_name, fuzzer_dir, fuzzer_logdir, logger): +def run_fuzzer(fuzzer_name, fuzzer_dir, fuzzer_logdir, logger, will_retry): def log(msg, *a, **k): logger.log(msg, a, k, flush=True) @@ -436,17 +437,22 @@ def run_fuzzer(fuzzer_name, fuzzer_dir, fuzzer_logdir, logger): assert job_re, make_flags job_rd, job_wr = job_re.groups() + # Make copies of jobserver FDs in case a retry is needed. + job_rd = int(job_rd) job_wr = int(job_wr) assert job_rd > 2, (job_rd, job_wr, make_flags) assert job_wr > 2, (job_rd, job_wr, make_flags) # Make sure the file descriptors exist.. - job_rd_fd = os.fdopen(int(job_rd), 'rb', 0) + job_rd_fd = os.fdopen(job_rd, 'rb', 0) assert job_rd_fd - job_wr_fd = os.fdopen(int(job_wr), 'rb', 0) + job_wr_fd = os.fdopen(job_wr, 'rb', 0) assert job_wr_fd + job_rd_copy = os.dup(job_rd) + job_wr_copy = os.dup(job_wr) + p = None try: p = subprocess.Popen( @@ -554,6 +560,13 @@ Failed @ {time_end} with exit code: {retcode} error_log=error_log, time_end=time_end.isoformat()) + if will_retry: + # Restore jobserver FD's + os.dup2(job_rd_copy, job_rd) + os.dup2(job_wr_copy, job_wr) + else: + os.close(job_rd_copy) + os.close(job_wr_copy) else: # Log the last 100 lines of a successful run @@ -566,6 +579,9 @@ Succeeded! @ {} Succeeded! @ {} """, time_end.isoformat(), success_log, time_end.isoformat()) + os.close(job_rd_copy) + os.close(job_wr_copy) + logger.flush() signal.signal(signal.SIGINT, old_sigint_handler) return retcode