Increase number of processes allowed to run during final grid reduction.

Signed-off-by: Keith Rothman <537074+litghost@users.noreply.github.com>

.github/kokoro/db-full.sh

@@ -97,6 +97,10 @@ set -e
 set +e
 (
 	export MAX_VIVADO_PROCESS=$((CORES/2 < 20 ? CORES/2 : 20))
+
+	# Use all the memory available on the system for generating the final
+	# grid and tileconn.
+	export MAX_GRID_CPU=${MAX_CPU_PER_GRID}
 	make db-extras-${XRAY_SETTINGS}-parts -j $CORES
 )
 EXTRAS_PARTS_RET=$?

.github/kokoro/steps/hostinfo.sh

@@ -20,6 +20,12 @@ echo "Memory"
 echo "----------------------------------------"
 cat /proc/meminfo
 echo "----------------------------------------"
+export MEM_GB=$(($(awk '/MemTotal/ {print $2}' /proc/meminfo)/(1024*1024)))
+echo "Total Memory (GB): $MEM_GB"
+
+# Approx memory per grid process
+export MEM_PER_GRID=8
+export MAX_CPU_PER_GRID=$(($MEM_GB/$MEM_PER_RUN))
 
 echo
 echo "========================================"

fuzzers/074-dump_all/generate_after_dump.sh

@@ -1,5 +1,8 @@
 #!/bin/bash -xe
 
+# By default use ~50 GiB for generate_grid.py, but allow override.
+export DEFAULT_MAX_GRID_CPU=10
+
 rm -rf build/output
 mkdir -p build/output
 python3 reduce_tile_types.py \
@@ -11,4 +14,5 @@ python3 create_node_tree.py \
   --output_dir build/output
 python3 reduce_site_types.py --output_dir build/output
 python3 generate_grid.py --root_dir build/specimen_001/ --output_dir build/output \
-  --ignored_wires ignored_wires/${XRAY_DATABASE}/${XRAY_PART}_ignored_wires.txt
+  --ignored_wires ignored_wires/${XRAY_DATABASE}/${XRAY_PART}_ignored_wires.txt \
+  --max_cpu=${MAX_GRID_CPU:-${DEFAULT_MAX_GRID_CPU}}

fuzzers/074-dump_all/generate_grid.py

@@ -559,12 +559,13 @@ def main():
     parser.add_argument('--output_dir', required=True)
     parser.add_argument('--verify_only', action='store_true')
     parser.add_argument('--ignored_wires')
+    parser.add_argument('--max_cpu', type=int, default=10)
 
     args = parser.parse_args()
 
     tiles, nodes = lib.read_root_csv(args.root_dir)
 
-    processes = min(multiprocessing.cpu_count(), 10)
+    processes = min(multiprocessing.cpu_count(), args.max_cpu)
     print('{} Running {} processes'.format(datetime.datetime.now(), processes))
     pool = multiprocessing.Pool(processes=processes)