Merge pull request #634 from litghost/update_ci

Update CI configuration:

.github/kokoro/database.cfg

@@ -6,6 +6,8 @@ action {
   define_artifacts {
     regex: "**/*result*.xml"
     regex: "**/build/*.log"
+    regex: "**/logs/*.log"
+    regex: "**/logs/*.xml"
     regex: "**/build/**"
     regex: "**/diff.*"
     regex: "**/database/**"

.github/kokoro/db-full.sh

@@ -46,9 +46,15 @@ echo "----------------------------------------"
 	echo "----------------------------------------"
 
 	# Run the fuzzers
-	export MAX_VIVADO_PROCESS=$CORES
+	#
+	# Cap MAX_VIVADO_PROCESS at 20 to limit memory usage of 074 fuzzer.
+	# At MAX_VIVADO_PROCESS=20:
+	# - 072 completes in ~35 minutes
+	# - 074 completes in ~60 minutes
+	# which is well before the 05x INT fuzzers complete.
+	export MAX_VIVADO_PROCESS=$((CORES/2 < 20 ? CORES/2 : 20))
 	set -x
-	script --return --flush --command "make -j $CORES MAX_VIVADO_PROCESS=$CORES" -
+	script --return --flush --command "make -j $CORES MAX_VIVADO_PROCESS=$MAX_VIVADO_PROCESS" -
 	set +x
 	echo "----------------------------------------"
 

fuzzers/run_fuzzer.py

@@ -372,6 +372,7 @@ def main(argv):
             fuzzer_dir,
             fuzzer_logdir,
             logger,
+            will_retry=retry_count < (args.retries - 1),
         )
         if exit_code <= 0:
             break
@@ -380,7 +381,7 @@ def main(argv):
     return exit_code
 
 
-def run_fuzzer(fuzzer_name, fuzzer_dir, fuzzer_logdir, logger):
+def run_fuzzer(fuzzer_name, fuzzer_dir, fuzzer_logdir, logger, will_retry):
     def log(msg, *a, **k):
         logger.log(msg, a, k, flush=True)
 
@@ -436,17 +437,22 @@ def run_fuzzer(fuzzer_name, fuzzer_dir, fuzzer_logdir, logger):
         assert job_re, make_flags
         job_rd, job_wr = job_re.groups()
 
+        # Make copies of jobserver FDs in case a retry is needed.
+
         job_rd = int(job_rd)
         job_wr = int(job_wr)
         assert job_rd > 2, (job_rd, job_wr, make_flags)
         assert job_wr > 2, (job_rd, job_wr, make_flags)
 
         # Make sure the file descriptors exist..
-        job_rd_fd = os.fdopen(int(job_rd), 'rb', 0)
+        job_rd_fd = os.fdopen(job_rd, 'rb', 0)
         assert job_rd_fd
-        job_wr_fd = os.fdopen(int(job_wr), 'rb', 0)
+        job_wr_fd = os.fdopen(job_wr, 'rb', 0)
         assert job_wr_fd
 
+        job_rd_copy = os.dup(job_rd)
+        job_wr_copy = os.dup(job_wr)
+
     p = None
     try:
         p = subprocess.Popen(
@@ -554,6 +560,13 @@ Failed @ {time_end} with exit code: {retcode}
             error_log=error_log,
             time_end=time_end.isoformat())
 
+        if will_retry:
+            # Restore jobserver FD's
+            os.dup2(job_rd_copy, job_rd)
+            os.dup2(job_wr_copy, job_wr)
+        else:
+            os.close(job_rd_copy)
+            os.close(job_wr_copy)
     else:
 
         # Log the last 100 lines of a successful run
@@ -566,6 +579,9 @@ Succeeded! @ {}
 Succeeded! @ {}
 """, time_end.isoformat(), success_log, time_end.isoformat())
 
+        os.close(job_rd_copy)
+        os.close(job_wr_copy)
+
     logger.flush()
     signal.signal(signal.SIGINT, old_sigint_handler)
     return retcode