From 2bc72b5cce1df6d11bad4e814d77b106f8d695e9 Mon Sep 17 00:00:00 2001
From: Keith Rothman <537074+litghost@users.noreply.github.com>
Date: Wed, 12 Feb 2020 15:47:36 -0800
Subject: [PATCH] Increase number of processes allowed to run during final grid
 reduction.

Signed-off-by: Keith Rothman <537074+litghost@users.noreply.github.com>
---
 .github/kokoro/db-full.sh                   | 4 ++++
 .github/kokoro/steps/hostinfo.sh            | 6 ++++++
 fuzzers/074-dump_all/generate_after_dump.sh | 6 +++++-
 fuzzers/074-dump_all/generate_grid.py       | 3 ++-
 4 files changed, 17 insertions(+), 2 deletions(-)

diff --git a/.github/kokoro/db-full.sh b/.github/kokoro/db-full.sh
index 23f8b780..696bc5c1 100755
--- a/.github/kokoro/db-full.sh
+++ b/.github/kokoro/db-full.sh
@@ -97,6 +97,10 @@ set -e
 set +e
 (
 	export MAX_VIVADO_PROCESS=$((CORES/2 < 20 ? CORES/2 : 20))
+
+	# Use all the memory available on the system for generating the final
+	# grid and tileconn.
+	export MAX_GRID_CPU=${MAX_CPU_PER_GRID}
 	make db-extras-${XRAY_SETTINGS}-parts -j $CORES
 )
 EXTRAS_PARTS_RET=$?
diff --git a/.github/kokoro/steps/hostinfo.sh b/.github/kokoro/steps/hostinfo.sh
index 6a00c37f..b0042f8a 100755
--- a/.github/kokoro/steps/hostinfo.sh
+++ b/.github/kokoro/steps/hostinfo.sh
@@ -20,6 +20,12 @@ echo "Memory"
 echo "----------------------------------------"
 cat /proc/meminfo
 echo "----------------------------------------"
+export MEM_GB=$(($(awk '/MemTotal/ {print $2}' /proc/meminfo)/(1024*1024)))
+echo "Total Memory (GB): $MEM_GB"
+
+# Approx memory per grid process
+export MEM_PER_GRID=8
+export MAX_CPU_PER_GRID=$(($MEM_GB/$MEM_PER_RUN))
 
 echo
 echo "========================================"
diff --git a/fuzzers/074-dump_all/generate_after_dump.sh b/fuzzers/074-dump_all/generate_after_dump.sh
index c1320779..5c5bc9d9 100755
--- a/fuzzers/074-dump_all/generate_after_dump.sh
+++ b/fuzzers/074-dump_all/generate_after_dump.sh
@@ -1,5 +1,8 @@
 #!/bin/bash -xe
 
+# By default use ~50 GiB for generate_grid.py, but allow override.
+export DEFAULT_MAX_GRID_CPU=10
+
 rm -rf build/output
 mkdir -p build/output
 python3 reduce_tile_types.py \
@@ -11,4 +14,5 @@ python3 create_node_tree.py \
   --output_dir build/output
 python3 reduce_site_types.py --output_dir build/output
 python3 generate_grid.py --root_dir build/specimen_001/ --output_dir build/output \
-  --ignored_wires ignored_wires/${XRAY_DATABASE}/${XRAY_PART}_ignored_wires.txt
+  --ignored_wires ignored_wires/${XRAY_DATABASE}/${XRAY_PART}_ignored_wires.txt \
+  --max_cpu=${MAX_GRID_CPU:-${DEFAULT_MAX_GRID_CPU}}
diff --git a/fuzzers/074-dump_all/generate_grid.py b/fuzzers/074-dump_all/generate_grid.py
index 4743d262..80cb5f20 100644
--- a/fuzzers/074-dump_all/generate_grid.py
+++ b/fuzzers/074-dump_all/generate_grid.py
@@ -559,12 +559,13 @@ def main():
     parser.add_argument('--output_dir', required=True)
     parser.add_argument('--verify_only', action='store_true')
     parser.add_argument('--ignored_wires')
+    parser.add_argument('--max_cpu', type=int, default=10)
 
     args = parser.parse_args()
 
     tiles, nodes = lib.read_root_csv(args.root_dir)
 
-    processes = min(multiprocessing.cpu_count(), 10)
+    processes = min(multiprocessing.cpu_count(), args.max_cpu)
     print('{} Running {} processes'.format(datetime.datetime.now(), processes))
     pool = multiprocessing.Pool(processes=processes)