From 4bb8ff9ebf9ab18a0cf83159ae25ff22a2fd329a Mon Sep 17 00:00:00 2001
From: Tim 'mithro' Ansell <me@mith.ro>
Date: Tue, 5 Feb 2019 13:08:58 +1100
Subject: [PATCH 01/28] make: Split the formatdb / checkdb into part specific
 targets.

Signed-off-by: Tim 'mithro' Ansell <me@mith.ro>
---
 Makefile | 51 ++++++++++++++++++++++++++++++++++++---------------
 1 file changed, 36 insertions(+), 15 deletions(-)

diff --git a/Makefile b/Makefile
index 9c49b52f..98365c45 100644
--- a/Makefile
+++ b/Makefile
@@ -69,28 +69,49 @@ format: format-cpp format-docs format-py format-tcl
 
 .PHONY: format format-cpp format-py format-tcl
 
-# Project X-Ray database
+# Targets related to Project X-Ray databases
 # ------------------------
 
+DATABASES=artix7 kintex7 zynq7
+
+define database
+
+# $(1) - Database name
+
+checkdb-$(1):
+	@echo
+	@echo "Checking $(1) database"
+	@echo "============================"
+	@$(IN_ENV) python3 utils/checkdb.py --db-root database/$(1)
+
+formatdb-$(1):
+	@echo
+	@echo "Formatting $(1) database"
+	@echo "============================"
+	@$(IN_ENV) cd database/$(1); python3 ../../utils/sort_db.py
+	@if [ -e database/Info.md ]; then $(IN_ENV) ./utils/info_md.py --keep; fi
+
+.PHONY: checkdb-$(1) formatdb-$(1)
+.NOTPARALLEL: checkdb-$(1) formatdb-$(1)
+
+checkdb: checkdb-$(1)
+formatdb: formatdb-$(1)
+
+endef
+
+$(foreach DB,$(DATABASES),$(eval $(call database,$(DB))))
+
 checkdb:
-	@for DB in database/*; do if [ -d $$DB ]; then \
-		echo ; \
-		echo "Checking $$DB"; \
-		echo "============================"; \
-		$(IN_ENV) python3 utils/checkdb.py --db-root $$DB; \
-	fi; done
+	@true
 
 formatdb:
-	@for DB in database/*; do if [ -d $$DB ]; then \
-		echo ; \
-		echo "Formatting $$DB"; \
-		echo "============================"; \
-		($(IN_ENV) cd $$DB; python3 ../../utils/sort_db.py || exit 1) || exit 1; \
-	fi; done
-	@make checkdb
-	$(IN_ENV) ./utils/info_md.py --keep
+	@true
+
+.PHONY: checkdb formatdb
 
 clean:
 	$(MAKE) -C database clean
 	$(MAKE) -C fuzzers clean
 	rm -rf build
+
+.PHONY: clean

From 39098dc79e5bab58410f36cfae40ef3349cc4907 Mon Sep 17 00:00:00 2001
From: Tim 'mithro' Ansell <me@mith.ro>
Date: Tue, 5 Feb 2019 13:09:29 +1100
Subject: [PATCH 02/28] utils: Output path when assert fails.

Signed-off-by: Tim 'mithro' Ansell <me@mith.ro>
---
 utils/info_md.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/utils/info_md.py b/utils/info_md.py
index aac36360..cd9d292e 100755
--- a/utils/info_md.py
+++ b/utils/info_md.py
@@ -70,7 +70,7 @@ Keep the existing commit information.
     args = parser.parse_args()
 
     info_md_filename = os.path.join('database', 'Info.md')
-    assert os.path.exists(info_md_filename)
+    assert os.path.exists(info_md_filename), info_md_filename
 
     info_md = []
 

From f7724d91573dae3465a1ed52392134a5d01bf89c Mon Sep 17 00:00:00 2001
From: Tim 'mithro' Ansell <me@mith.ro>
Date: Tue, 8 Jan 2019 23:04:24 -0800
Subject: [PATCH 03/28] infra: Adding kokoro configs.

Signed-off-by: Tim 'mithro' Ansell <me@mith.ro>
---
 .github/kokoro/continuous.cfg |  1 +
 .github/kokoro/full.sh        | 44 +++++++++++++++++++++++++++++++++++
 .github/kokoro/presubmit.cfg  |  1 +
 .github/kokoro/quick.sh       |  1 +
 4 files changed, 47 insertions(+)
 create mode 100644 .github/kokoro/continuous.cfg
 create mode 100755 .github/kokoro/full.sh
 create mode 100644 .github/kokoro/presubmit.cfg
 create mode 120000 .github/kokoro/quick.sh

diff --git a/.github/kokoro/continuous.cfg b/.github/kokoro/continuous.cfg
new file mode 100644
index 00000000..ff44c4cf
--- /dev/null
+++ b/.github/kokoro/continuous.cfg
@@ -0,0 +1 @@
+build_file: "symbiflow-prjxray-artix7/.github/kokoro/full.sh"
diff --git a/.github/kokoro/full.sh b/.github/kokoro/full.sh
new file mode 100755
index 00000000..4e60a9a5
--- /dev/null
+++ b/.github/kokoro/full.sh
@@ -0,0 +1,44 @@
+#!/bin/bash
+
+set -e
+set -x
+
+sudo apt-get update
+sudo apt-get install -y \
+        bison \
+        build-essential \
+        ca-certificates \
+        clang-format \
+        cmake \
+        curl \
+        flex \
+        fontconfig \
+        git \
+        jq \
+        python \
+        python3 \
+        python3-dev \
+        python3-virtualenv \
+        python3-yaml \
+        virtualenv \
+
+ls -l ~/.Xilinx
+sudo chown -R $USER ~/.Xilinx
+
+cd github/symbiflow-prjxray-artix7/
+
+git fetch --tags || true
+git describe --tags || true
+
+# Build the C++ tools
+make build --output-sync=target --warn-undefined-variables
+
+# Setup the Python environment
+make env --output-sync=target --warn-undefined-variables
+
+source settings/artix7.sh
+(
+	export XILINX_LOCAL_USER_DATA=no
+	cd fuzzers
+	make --output-sync=target --warn-undefined-variables QUICK=y
+)
diff --git a/.github/kokoro/presubmit.cfg b/.github/kokoro/presubmit.cfg
new file mode 100644
index 00000000..117c4189
--- /dev/null
+++ b/.github/kokoro/presubmit.cfg
@@ -0,0 +1 @@
+build_file: "symbiflow-prjxray-artix7/.github/kokoro/quick.sh"
diff --git a/.github/kokoro/quick.sh b/.github/kokoro/quick.sh
new file mode 120000
index 00000000..ee2000b0
--- /dev/null
+++ b/.github/kokoro/quick.sh
@@ -0,0 +1 @@
+full.sh
\ No newline at end of file

From 745965d4de4931cd807a107bf99ddcf1e066c3a9 Mon Sep 17 00:00:00 2001
From: Tim 'mithro' Ansell <me@mith.ro>
Date: Thu, 10 Jan 2019 18:47:50 -0800
Subject: [PATCH 04/28] infra: Capture test results in Kokoro.

Signed-off-by: Tim 'mithro' Ansell <me@mith.ro>
---
 .github/kokoro/continuous.cfg  |   8 +++
 .github/kokoro/ctest2junit.xsl | 120 +++++++++++++++++++++++++++++++++
 .github/kokoro/presubmit.cfg   |   7 ++
 Makefile                       |   1 +
 4 files changed, 136 insertions(+)
 create mode 100644 .github/kokoro/ctest2junit.xsl

diff --git a/.github/kokoro/continuous.cfg b/.github/kokoro/continuous.cfg
index ff44c4cf..87086ebb 100644
--- a/.github/kokoro/continuous.cfg
+++ b/.github/kokoro/continuous.cfg
@@ -1 +1,9 @@
 build_file: "symbiflow-prjxray-artix7/.github/kokoro/full.sh"
+
+action {
+  define_artifacts {
+    regex: "**/*result*.xml"
+    regex: "**/*.log"
+    regex: "database/**"
+  }
+}
diff --git a/.github/kokoro/ctest2junit.xsl b/.github/kokoro/ctest2junit.xsl
new file mode 100644
index 00000000..3ea29e50
--- /dev/null
+++ b/.github/kokoro/ctest2junit.xsl
@@ -0,0 +1,120 @@
+<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform" version="1.0">
+<xsl:output method="xml" indent="yes"/>
+    <xsl:template match="/Site">
+	<xsl:variable name="Name"><xsl:value-of select="@Name"/></xsl:variable>
+	<xsl:variable name="Hostname"><xsl:value-of select="@Hostname"/></xsl:variable>
+	<xsl:variable name="TestCount"><xsl:value-of select="count(//TestList/Test)"/> </xsl:variable>
+	<xsl:variable name="ErrorCount"><xsl:value-of select="count(//TestList/Test[@Status='error'])"/> </xsl:variable>
+	<xsl:variable name="FailureCount"><xsl:value-of select="count(//Testing/Test[@Status='failed'])"/> </xsl:variable>
+	<testsuite name="{$Name}" hostname="{$Hostname}" errors="0" failures="{$FailureCount}" tests="{$TestCount}">
+	    <xsl:variable name="BuildName"><xsl:value-of select="@BuildName"/></xsl:variable>
+	    <xsl:variable name="BuildStamp"><xsl:value-of select="@BuildStamp"/></xsl:variable>
+	    <xsl:variable name="Generator"><xsl:value-of select="@Generator"/></xsl:variable>
+	    <xsl:variable name="CompilerName"><xsl:value-of select="@CompilerName"/></xsl:variable>
+	    <xsl:variable name="OSName"><xsl:value-of select="@OSName"/></xsl:variable>
+	    <xsl:variable name="OSRelease"><xsl:value-of select="@OSRelease"/></xsl:variable>
+	    <xsl:variable name="OSVersion"><xsl:value-of select="@OSVersion"/></xsl:variable>
+	    <xsl:variable name="OSPlatform"><xsl:value-of select="@OSPlatform"/></xsl:variable>
+	    <xsl:variable name="Is64Bits"><xsl:value-of select="@Is64Bits"/></xsl:variable>
+	    <xsl:variable name="VendorString"><xsl:value-of select="@VendorString"/></xsl:variable>
+	    <xsl:variable name="VendorID"><xsl:value-of select="@VendorID"/></xsl:variable>
+	    <xsl:variable name="FamilyID"><xsl:value-of select="@FamilyID"/></xsl:variable>
+	    <xsl:variable name="ModelID"><xsl:value-of select="@ModelID"/></xsl:variable>
+	    <xsl:variable name="ProcessorCacheSize"><xsl:value-of select="@ProcessorCacheSize"/></xsl:variable>
+	    <xsl:variable name="NumberOfLogicalCPU"><xsl:value-of select="@NumberOfLogicalCPU"/></xsl:variable>
+	    <xsl:variable name="NumberOfPhysicalCPU"><xsl:value-of select="@NumberOfPhysicalCPU"/></xsl:variable>
+	    <xsl:variable name="TotalVirtualMemory"><xsl:value-of select="@TotalVirtualMemory"/></xsl:variable>
+	    <xsl:variable name="TotalPhysicalMemory"><xsl:value-of select="@TotalPhysicalMemory"/></xsl:variable>
+	    <xsl:variable name="LogicalProcessorsPerPhysical"><xsl:value-of select="@LogicalProcessorsPerPhysical"/></xsl:variable>
+	    <xsl:variable name="ProcessorClockFrequency"><xsl:value-of select="@ProcessorClockFrequency"/></xsl:variable>
+	    <properties>
+		<property name="BuildName" value="{$BuildName}" />
+		<property name="BuildStamp" value="{$BuildStamp}" />
+		<property name="Name" value="{$Name}" />
+		<property name="Generator" value="{$Generator}" />
+		<property name="CompilerName" value="{$CompilerName}" />
+		<property name="OSName" value="{$OSName}" />
+		<property name="Hostname" value="{$Hostname}" />
+		<property name="OSRelease" value="{$OSRelease}" />
+		<property name="OSVersion" value="{$OSVersion}" />
+		<property name="OSPlatform" value="{$OSPlatform}" />
+		<property name="Is64Bits" value="{$Is64Bits}" />
+		<property name="VendorString" value="{$VendorString}" />
+		<property name="VendorID" value="{$VendorID}" />
+		<property name="FamilyID" value="{$FamilyID}" />
+		<property name="ModelID" value="{$ModelID}" />
+		<property name="ProcessorCacheSize" value="{$ProcessorCacheSize}" />
+		<property name="NumberOfLogicalCPU" value="{$NumberOfLogicalCPU}" />
+		<property name="NumberOfPhysicalCPU" value="{$NumberOfPhysicalCPU}" />
+		<property name="TotalVirtualMemory" value="{$TotalVirtualMemory}" />
+		<property name="TotalPhysicalMemory" value="{$TotalPhysicalMemory}" />
+		<property name="LogicalProcessorsPerPhysical" value="{$LogicalProcessorsPerPhysical}" />
+		<property name="ProcessorClockFrequency" value="{$ProcessorClockFrequency}" />
+	    </properties>
+	    <xsl:apply-templates select="Testing/Test"/>
+
+	    <system-out>
+		BuildName: <xsl:value-of select="$BuildName" />
+		BuildStamp: <xsl:value-of select="$BuildStamp" />
+		Name: <xsl:value-of select="$Name" />
+		Generator: <xsl:value-of select="$Generator" />
+		CompilerName: <xsl:value-of select="$CompilerName" />
+		OSName: <xsl:value-of select="$OSName" />
+		Hostname: <xsl:value-of select="$Hostname" />
+		OSRelease: <xsl:value-of select="$OSRelease" />
+		OSVersion: <xsl:value-of select="$OSVersion" />
+		OSPlatform: <xsl:value-of select="$OSPlatform" />
+		Is64Bits: <xsl:value-of select="$Is64Bits" />
+		VendorString: <xsl:value-of select="$VendorString" />
+		VendorID: <xsl:value-of select="$VendorID" />
+		FamilyID: <xsl:value-of select="$FamilyID" />
+		ModelID: <xsl:value-of select="$ModelID" />
+		ProcessorCacheSize: <xsl:value-of select="$ProcessorCacheSize" />
+		NumberOfLogicalCPU: <xsl:value-of select="$NumberOfLogicalCPU" />
+		NumberOfPhysicalCPU: <xsl:value-of select="$NumberOfPhysicalCPU" />
+		TotalVirtualMemory: <xsl:value-of select="$TotalVirtualMemory" />
+		TotalPhysicalMemory: <xsl:value-of select="$TotalPhysicalMemory" />
+		LogicalProcessorsPerPhysical: <xsl:value-of select="$LogicalProcessorsPerPhysical" />
+		ProcessorClockFrequency: <xsl:value-of select="$ProcessorClockFrequency" />
+	    </system-out>
+	</testsuite>
+    </xsl:template>
+
+    <xsl:template match="Testing/Test">
+	<xsl:variable name="testcasename"><xsl:value-of select= "Name"/></xsl:variable>
+	<xsl:variable name="testclassname"><xsl:value-of select= " concat('this', substring(Path,2))"/></xsl:variable>
+	<xsl:variable name="exectime">
+	    <xsl:for-each select="Results/NamedMeasurement">
+		<xsl:if test="@name = 'Execution Time'">
+		    <xsl:value-of select="."/>
+		</xsl:if>
+	    </xsl:for-each>
+	</xsl:variable>
+
+	<testcase name="{$testcasename}" classname="{$testclassname}" time="{$exectime}">
+	    <xsl:if test="@Status = 'passed'">
+	    </xsl:if>
+	    <xsl:if test="@Status = 'failed'">
+		<xsl:variable name="failtype">
+		    <xsl:for-each select="Results/NamedMeasurement">
+			<xsl:if test="@name = 'Exit Code'">
+			    <xsl:value-of select="."/>
+			</xsl:if>
+		    </xsl:for-each>
+		</xsl:variable>
+		<xsl:variable name="failcode">
+		    <xsl:for-each select="Results/NamedMeasurement">
+			<xsl:if test="@name = 'Exit Value'">
+			    <xsl:value-of select="."/>
+			</xsl:if>
+		    </xsl:for-each>
+		</xsl:variable>
+		<failure message="{$failtype} ({$failcode})"><xsl:value-of select="Results/Measurement/Value/text()" /></failure>
+	    </xsl:if>
+	    <xsl:if test="@Status = 'notrun'">
+		<skipped><xsl:value-of select="Results/Measurement/Value/text()" /></skipped>
+	    </xsl:if>
+	</testcase>
+    </xsl:template>
+
+</xsl:stylesheet>
diff --git a/.github/kokoro/presubmit.cfg b/.github/kokoro/presubmit.cfg
index 117c4189..85be999d 100644
--- a/.github/kokoro/presubmit.cfg
+++ b/.github/kokoro/presubmit.cfg
@@ -1 +1,8 @@
 build_file: "symbiflow-prjxray-artix7/.github/kokoro/quick.sh"
+
+action {
+  define_artifacts {
+    regex: "**/*result*.xml"
+    regex: "**/*.log"
+  }
+}
diff --git a/Makefile b/Makefile
index 98365c45..276e76ca 100644
--- a/Makefile
+++ b/Makefile
@@ -41,6 +41,7 @@ test-cpp:
 	cd build && cmake -DPRJXRAY_BUILD_TESTING=ON ..
 	cd build && $(MAKE) -s
 	cd build && ctest --no-compress-output -T Test -C RelWithDebInfo --output-on-failure
+	xsltproc .github/kokoro/ctest2junit.xsl build/Testing/*/Test.xml > build/cpp_test_results.xml
 
 .PHONY: test test-py test-cpp
 

From 656bbd55c52926c8b7b50fb1449cbad4a6e7b7b7 Mon Sep 17 00:00:00 2001
From: Tim 'mithro' Ansell <me@mith.ro>
Date: Thu, 10 Jan 2019 21:13:40 -0800
Subject: [PATCH 05/28] infra: Increase timeouts.

Signed-off-by: Tim 'mithro' Ansell <me@mith.ro>
---
 .github/kokoro/continuous.cfg | 2 ++
 .github/kokoro/presubmit.cfg  | 2 ++
 2 files changed, 4 insertions(+)

diff --git a/.github/kokoro/continuous.cfg b/.github/kokoro/continuous.cfg
index 87086ebb..116fccbc 100644
--- a/.github/kokoro/continuous.cfg
+++ b/.github/kokoro/continuous.cfg
@@ -1,5 +1,7 @@
 build_file: "symbiflow-prjxray-artix7/.github/kokoro/full.sh"
 
+timeout_mins: 4320
+
 action {
   define_artifacts {
     regex: "**/*result*.xml"
diff --git a/.github/kokoro/presubmit.cfg b/.github/kokoro/presubmit.cfg
index 85be999d..9832de76 100644
--- a/.github/kokoro/presubmit.cfg
+++ b/.github/kokoro/presubmit.cfg
@@ -1,5 +1,7 @@
 build_file: "symbiflow-prjxray-artix7/.github/kokoro/quick.sh"
 
+timeout_mins: 240
+
 action {
   define_artifacts {
     regex: "**/*result*.xml"

From f61695585bedbee632c0bf9be7efa95a7c04219a Mon Sep 17 00:00:00 2001
From: Tim 'mithro' Ansell <me@mith.ro>
Date: Wed, 16 Jan 2019 22:02:24 +1300
Subject: [PATCH 06/28] infra: Install xsltproc

Signed-off-by: Tim 'mithro' Ansell <me@mith.ro>
---
 .travis.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.travis.yml b/.travis.yml
index db1a2588..75ba367a 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -10,6 +10,7 @@ addons:
     packages:
       - clang-format-5.0
       - g++-6
+      - xsltproc
 
 install:
   - export CC=gcc-6

From fc1f36f3c03ebfe006c19b54e28a4c7dfb096b8e Mon Sep 17 00:00:00 2001
From: Tim 'mithro' Ansell <me@mith.ro>
Date: Fri, 11 Jan 2019 20:44:19 -0800
Subject: [PATCH 07/28] infra: Restructure.

Signed-off-by: Tim 'mithro' Ansell <me@mith.ro>
---
 .github/kokoro/continuous-db-artix7-full.cfg  | 33 +++++++
 .github/kokoro/continuous-db-artix7-quick.cfg | 32 +++++++
 .github/kokoro/continuous-db-kintex7-full.cfg | 33 +++++++
 .../kokoro/continuous-db-kintex7-quick.cfg    | 32 +++++++
 .github/kokoro/continuous-db-zynq7-full.cfg   | 33 +++++++
 .github/kokoro/continuous-db-zynq7-quick.cfg  | 32 +++++++
 .github/kokoro/continuous.cfg                 | 11 ---
 .../kokoro/{presubmit.cfg => database.cfg}    |  4 +-
 .github/kokoro/{full.sh => db-full.sh}        | 28 +++++-
 .github/kokoro/db-quick.sh                    | 76 ++++++++++++++++
 .github/kokoro/kokoro-cfg.py                  | 91 +++++++++++++++++++
 .github/kokoro/presubmit-db-artix7.cfg        | 32 +++++++
 .github/kokoro/presubmit-db-kintex7.cfg       | 32 +++++++
 .github/kokoro/presubmit-db-zynq7.cfg         | 32 +++++++
 .github/kokoro/quick.sh                       |  1 -
 .github/kokoro/tests.cfg                      | 19 ++++
 .github/kokoro/tests.sh                       | 81 +++++++++++++++++
 Makefile                                      |  2 +-
 18 files changed, 585 insertions(+), 19 deletions(-)
 create mode 100644 .github/kokoro/continuous-db-artix7-full.cfg
 create mode 100644 .github/kokoro/continuous-db-artix7-quick.cfg
 create mode 100644 .github/kokoro/continuous-db-kintex7-full.cfg
 create mode 100644 .github/kokoro/continuous-db-kintex7-quick.cfg
 create mode 100644 .github/kokoro/continuous-db-zynq7-full.cfg
 create mode 100644 .github/kokoro/continuous-db-zynq7-quick.cfg
 delete mode 100644 .github/kokoro/continuous.cfg
 rename .github/kokoro/{presubmit.cfg => database.cfg} (52%)
 rename .github/kokoro/{full.sh => db-full.sh} (53%)
 create mode 100755 .github/kokoro/db-quick.sh
 create mode 100755 .github/kokoro/kokoro-cfg.py
 create mode 100644 .github/kokoro/presubmit-db-artix7.cfg
 create mode 100644 .github/kokoro/presubmit-db-kintex7.cfg
 create mode 100644 .github/kokoro/presubmit-db-zynq7.cfg
 delete mode 120000 .github/kokoro/quick.sh
 create mode 100644 .github/kokoro/tests.cfg
 create mode 100755 .github/kokoro/tests.sh

diff --git a/.github/kokoro/continuous-db-artix7-full.cfg b/.github/kokoro/continuous-db-artix7-full.cfg
new file mode 100644
index 00000000..1048062e
--- /dev/null
+++ b/.github/kokoro/continuous-db-artix7-full.cfg
@@ -0,0 +1,33 @@
+# Format: //devtools/kokoro/config/proto/build.proto
+
+build_file: "symbiflow-prjxray-continuous-db-artix7-full/.github/kokoro/db-full.sh"
+
+timeout_mins: 4320
+
+action {
+  define_artifacts {
+    regex: "**/*result*.xml"
+    regex: "**/*.log"
+    regex: "database/artix7/**"
+  }
+}
+
+env_vars {
+  key: "KOKORO_TYPE"
+  value: "continuous"
+}
+
+env_vars {
+  key: "KOKORO_DIR"
+  value: "symbiflow-prjxray-continuous-db-artix7-full"
+}
+
+env_vars {
+  key: "XRAY_SETTINGS"
+  value: "artix7"
+}
+
+env_vars {
+  key: "XRAY_BUILD_TYPE"
+  value: "full"
+}
diff --git a/.github/kokoro/continuous-db-artix7-quick.cfg b/.github/kokoro/continuous-db-artix7-quick.cfg
new file mode 100644
index 00000000..42d2b777
--- /dev/null
+++ b/.github/kokoro/continuous-db-artix7-quick.cfg
@@ -0,0 +1,32 @@
+# Format: //devtools/kokoro/config/proto/build.proto
+
+build_file: "symbiflow-prjxray-continuous-db-artix7-quick/.github/kokoro/db-quick.sh"
+
+timeout_mins: 180
+
+action {
+  define_artifacts {
+    regex: "**/*result*.xml"
+    regex: "**/*.log"
+  }
+}
+
+env_vars {
+  key: "KOKORO_TYPE"
+  value: "continuous"
+}
+
+env_vars {
+  key: "KOKORO_DIR"
+  value: "symbiflow-prjxray-continuous-db-artix7-quick"
+}
+
+env_vars {
+  key: "XRAY_SETTINGS"
+  value: "artix7"
+}
+
+env_vars {
+  key: "XRAY_BUILD_TYPE"
+  value: "quick"
+}
diff --git a/.github/kokoro/continuous-db-kintex7-full.cfg b/.github/kokoro/continuous-db-kintex7-full.cfg
new file mode 100644
index 00000000..bceb827f
--- /dev/null
+++ b/.github/kokoro/continuous-db-kintex7-full.cfg
@@ -0,0 +1,33 @@
+# Format: //devtools/kokoro/config/proto/build.proto
+
+build_file: "symbiflow-prjxray-continuous-db-kintex7-full/.github/kokoro/db-full.sh"
+
+timeout_mins: 4320
+
+action {
+  define_artifacts {
+    regex: "**/*result*.xml"
+    regex: "**/*.log"
+    regex: "database/kintex7/**"
+  }
+}
+
+env_vars {
+  key: "KOKORO_TYPE"
+  value: "continuous"
+}
+
+env_vars {
+  key: "KOKORO_DIR"
+  value: "symbiflow-prjxray-continuous-db-kintex7-full"
+}
+
+env_vars {
+  key: "XRAY_SETTINGS"
+  value: "kintex7"
+}
+
+env_vars {
+  key: "XRAY_BUILD_TYPE"
+  value: "full"
+}
diff --git a/.github/kokoro/continuous-db-kintex7-quick.cfg b/.github/kokoro/continuous-db-kintex7-quick.cfg
new file mode 100644
index 00000000..3b67b337
--- /dev/null
+++ b/.github/kokoro/continuous-db-kintex7-quick.cfg
@@ -0,0 +1,32 @@
+# Format: //devtools/kokoro/config/proto/build.proto
+
+build_file: "symbiflow-prjxray-continuous-db-kintex7-quick/.github/kokoro/db-quick.sh"
+
+timeout_mins: 180
+
+action {
+  define_artifacts {
+    regex: "**/*result*.xml"
+    regex: "**/*.log"
+  }
+}
+
+env_vars {
+  key: "KOKORO_TYPE"
+  value: "continuous"
+}
+
+env_vars {
+  key: "KOKORO_DIR"
+  value: "symbiflow-prjxray-continuous-db-kintex7-quick"
+}
+
+env_vars {
+  key: "XRAY_SETTINGS"
+  value: "kintex7"
+}
+
+env_vars {
+  key: "XRAY_BUILD_TYPE"
+  value: "quick"
+}
diff --git a/.github/kokoro/continuous-db-zynq7-full.cfg b/.github/kokoro/continuous-db-zynq7-full.cfg
new file mode 100644
index 00000000..3e581974
--- /dev/null
+++ b/.github/kokoro/continuous-db-zynq7-full.cfg
@@ -0,0 +1,33 @@
+# Format: //devtools/kokoro/config/proto/build.proto
+
+build_file: "symbiflow-prjxray-continuous-db-zynq7-full/.github/kokoro/db-full.sh"
+
+timeout_mins: 4320
+
+action {
+  define_artifacts {
+    regex: "**/*result*.xml"
+    regex: "**/*.log"
+    regex: "database/zynq7/**"
+  }
+}
+
+env_vars {
+  key: "KOKORO_TYPE"
+  value: "continuous"
+}
+
+env_vars {
+  key: "KOKORO_DIR"
+  value: "symbiflow-prjxray-continuous-db-zynq7-full"
+}
+
+env_vars {
+  key: "XRAY_SETTINGS"
+  value: "zynq7"
+}
+
+env_vars {
+  key: "XRAY_BUILD_TYPE"
+  value: "full"
+}
diff --git a/.github/kokoro/continuous-db-zynq7-quick.cfg b/.github/kokoro/continuous-db-zynq7-quick.cfg
new file mode 100644
index 00000000..24cfb1bd
--- /dev/null
+++ b/.github/kokoro/continuous-db-zynq7-quick.cfg
@@ -0,0 +1,32 @@
+# Format: //devtools/kokoro/config/proto/build.proto
+
+build_file: "symbiflow-prjxray-continuous-db-zynq7-quick/.github/kokoro/db-quick.sh"
+
+timeout_mins: 180
+
+action {
+  define_artifacts {
+    regex: "**/*result*.xml"
+    regex: "**/*.log"
+  }
+}
+
+env_vars {
+  key: "KOKORO_TYPE"
+  value: "continuous"
+}
+
+env_vars {
+  key: "KOKORO_DIR"
+  value: "symbiflow-prjxray-continuous-db-zynq7-quick"
+}
+
+env_vars {
+  key: "XRAY_SETTINGS"
+  value: "zynq7"
+}
+
+env_vars {
+  key: "XRAY_BUILD_TYPE"
+  value: "quick"
+}
diff --git a/.github/kokoro/continuous.cfg b/.github/kokoro/continuous.cfg
deleted file mode 100644
index 116fccbc..00000000
--- a/.github/kokoro/continuous.cfg
+++ /dev/null
@@ -1,11 +0,0 @@
-build_file: "symbiflow-prjxray-artix7/.github/kokoro/full.sh"
-
-timeout_mins: 4320
-
-action {
-  define_artifacts {
-    regex: "**/*result*.xml"
-    regex: "**/*.log"
-    regex: "database/**"
-  }
-}
diff --git a/.github/kokoro/presubmit.cfg b/.github/kokoro/database.cfg
similarity index 52%
rename from .github/kokoro/presubmit.cfg
rename to .github/kokoro/database.cfg
index 9832de76..762600a4 100644
--- a/.github/kokoro/presubmit.cfg
+++ b/.github/kokoro/database.cfg
@@ -1,6 +1,6 @@
-build_file: "symbiflow-prjxray-artix7/.github/kokoro/quick.sh"
+build_file: "symbiflow-prjxray-db/.github/kokoro/database.sh"
 
-timeout_mins: 240
+timeout_mins: 10
 
 action {
   define_artifacts {
diff --git a/.github/kokoro/full.sh b/.github/kokoro/db-full.sh
similarity index 53%
rename from .github/kokoro/full.sh
rename to .github/kokoro/db-full.sh
index 4e60a9a5..3c623bd2 100755
--- a/.github/kokoro/full.sh
+++ b/.github/kokoro/db-full.sh
@@ -1,7 +1,7 @@
 #!/bin/bash
 
-set -e
 set -x
+set -e
 
 sudo apt-get update
 sudo apt-get install -y \
@@ -25,7 +25,27 @@ sudo apt-get install -y \
 ls -l ~/.Xilinx
 sudo chown -R $USER ~/.Xilinx
 
-cd github/symbiflow-prjxray-artix7/
+echo "----------------------------------------"
+echo "----------------------------------------"
+echo "----------------------------------------"
+
+export
+
+echo "----------------------------------------"
+echo "----------------------------------------"
+echo "----------------------------------------"
+
+find .
+
+echo "----------------------------------------"
+echo "----------------------------------------"
+echo "----------------------------------------"
+
+echo $PWD
+
+echo "----------------------------------------"
+
+cd github/$KOKORO_DIR/
 
 git fetch --tags || true
 git describe --tags || true
@@ -36,9 +56,9 @@ make build --output-sync=target --warn-undefined-variables
 # Setup the Python environment
 make env --output-sync=target --warn-undefined-variables
 
-source settings/artix7.sh
+source settings/$XRAY_SETTINGS.sh
 (
 	export XILINX_LOCAL_USER_DATA=no
 	cd fuzzers
-	make --output-sync=target --warn-undefined-variables QUICK=y
+	make --output-sync=target --warn-undefined-variables
 )
diff --git a/.github/kokoro/db-quick.sh b/.github/kokoro/db-quick.sh
new file mode 100755
index 00000000..08e19752
--- /dev/null
+++ b/.github/kokoro/db-quick.sh
@@ -0,0 +1,76 @@
+#!/bin/bash
+
+set -x
+set -e
+
+sudo apt-get update
+sudo apt-get install -y \
+        bison \
+        build-essential \
+        ca-certificates \
+        clang-format \
+        cmake \
+        curl \
+        flex \
+        fontconfig \
+        git \
+        jq \
+        python \
+        python3 \
+        python3-dev \
+        python3-virtualenv \
+        python3-yaml \
+        virtualenv \
+
+ls -l ~/.Xilinx
+sudo chown -R $USER ~/.Xilinx
+
+CORES=$(nproc --all)
+
+echo "----------------------------------------"
+echo "----------------------------------------"
+echo "----------------------------------------"
+
+export
+
+echo "----------------------------------------"
+echo "----------------------------------------"
+echo "----------------------------------------"
+
+find .
+
+echo "----------------------------------------"
+echo "----------------------------------------"
+echo "----------------------------------------"
+
+echo $PWD
+
+echo "----------------------------------------"
+
+cd github/$KOKORO_DIR/
+
+git fetch --tags || true
+git describe --tags || true
+
+# Build the C++ tools
+make build --output-sync=target --warn-undefined-variables
+
+# Setup the Python environment
+make env --output-sync=target --warn-undefined-variables
+
+echo "----------------------------------------"
+echo "----------------------------------------"
+echo "----------------------------------------"
+
+find .
+
+echo "----------------------------------------"
+echo "----------------------------------------"
+echo "----------------------------------------"
+
+source settings/$XRAY_SETTINGS.sh
+(
+	export XILINX_LOCAL_USER_DATA=no
+	cd fuzzers
+	make --output-sync=target --warn-undefined-variables QUICK=y -j$CORES
+)
diff --git a/.github/kokoro/kokoro-cfg.py b/.github/kokoro/kokoro-cfg.py
new file mode 100755
index 00000000..2c0b5627
--- /dev/null
+++ b/.github/kokoro/kokoro-cfg.py
@@ -0,0 +1,91 @@
+#!/usr/bin/env python3
+
+db_quick = """\
+# Format: //devtools/kokoro/config/proto/build.proto
+
+build_file: "symbiflow-prjxray-%(kokoro_type)s-db-%(part)s%(ext)s/.github/kokoro/db-quick.sh"
+
+timeout_mins: 180
+
+action {
+  define_artifacts {
+    regex: "**/*result*.xml"
+    regex: "**/*.log"
+  }
+}
+
+env_vars {
+  key: "KOKORO_TYPE"
+  value: "%(kokoro_type)s"
+}
+
+env_vars {
+  key: "KOKORO_DIR"
+  value: "symbiflow-prjxray-%(kokoro_type)s-db-%(part)s%(ext)s"
+}
+
+env_vars {
+  key: "XRAY_SETTINGS"
+  value: "%(part)s"
+}
+
+env_vars {
+  key: "XRAY_BUILD_TYPE"
+  value: "quick"
+}
+"""
+
+db_full = """\
+# Format: //devtools/kokoro/config/proto/build.proto
+
+build_file: "symbiflow-prjxray-%(kokoro_type)s-db-%(part)s-full/.github/kokoro/db-full.sh"
+
+timeout_mins: 4320
+
+action {
+  define_artifacts {
+    regex: "**/*result*.xml"
+    regex: "**/*.log"
+    regex: "database/%(part)s/**"
+  }
+}
+
+env_vars {
+  key: "KOKORO_TYPE"
+  value: "%(kokoro_type)s"
+}
+
+env_vars {
+  key: "KOKORO_DIR"
+  value: "symbiflow-prjxray-%(kokoro_type)s-db-%(part)s-full"
+}
+
+env_vars {
+  key: "XRAY_SETTINGS"
+  value: "%(part)s"
+}
+
+env_vars {
+  key: "XRAY_BUILD_TYPE"
+  value: "full"
+}
+"""
+
+for part in ['artix7', 'kintex7', 'zynq7']:
+    with open("continuous-db-%s-quick.cfg" % part, "w") as f:
+        f.write(
+            db_quick % {
+                'part': part,
+                'kokoro_type': 'continuous',
+                'ext': '-quick'
+            })
+    with open("continuous-db-%s-full.cfg" % part, "w") as f:
+        f.write(db_full % {'part': part, 'kokoro_type': 'continuous'})
+
+    with open("presubmit-db-%s.cfg" % part, "w") as f:
+        f.write(
+            db_quick % {
+                'part': part,
+                'kokoro_type': 'presubmit',
+                'ext': ''
+            })
diff --git a/.github/kokoro/presubmit-db-artix7.cfg b/.github/kokoro/presubmit-db-artix7.cfg
new file mode 100644
index 00000000..c7ca90ff
--- /dev/null
+++ b/.github/kokoro/presubmit-db-artix7.cfg
@@ -0,0 +1,32 @@
+# Format: //devtools/kokoro/config/proto/build.proto
+
+build_file: "symbiflow-prjxray-presubmit-db-artix7/.github/kokoro/db-quick.sh"
+
+timeout_mins: 180
+
+action {
+  define_artifacts {
+    regex: "**/*result*.xml"
+    regex: "**/*.log"
+  }
+}
+
+env_vars {
+  key: "KOKORO_TYPE"
+  value: "presubmit"
+}
+
+env_vars {
+  key: "KOKORO_DIR"
+  value: "symbiflow-prjxray-presubmit-db-artix7"
+}
+
+env_vars {
+  key: "XRAY_SETTINGS"
+  value: "artix7"
+}
+
+env_vars {
+  key: "XRAY_BUILD_TYPE"
+  value: "quick"
+}
diff --git a/.github/kokoro/presubmit-db-kintex7.cfg b/.github/kokoro/presubmit-db-kintex7.cfg
new file mode 100644
index 00000000..3a501168
--- /dev/null
+++ b/.github/kokoro/presubmit-db-kintex7.cfg
@@ -0,0 +1,32 @@
+# Format: //devtools/kokoro/config/proto/build.proto
+
+build_file: "symbiflow-prjxray-presubmit-db-kintex7/.github/kokoro/db-quick.sh"
+
+timeout_mins: 180
+
+action {
+  define_artifacts {
+    regex: "**/*result*.xml"
+    regex: "**/*.log"
+  }
+}
+
+env_vars {
+  key: "KOKORO_TYPE"
+  value: "presubmit"
+}
+
+env_vars {
+  key: "KOKORO_DIR"
+  value: "symbiflow-prjxray-presubmit-db-kintex7"
+}
+
+env_vars {
+  key: "XRAY_SETTINGS"
+  value: "kintex7"
+}
+
+env_vars {
+  key: "XRAY_BUILD_TYPE"
+  value: "quick"
+}
diff --git a/.github/kokoro/presubmit-db-zynq7.cfg b/.github/kokoro/presubmit-db-zynq7.cfg
new file mode 100644
index 00000000..6026b61e
--- /dev/null
+++ b/.github/kokoro/presubmit-db-zynq7.cfg
@@ -0,0 +1,32 @@
+# Format: //devtools/kokoro/config/proto/build.proto
+
+build_file: "symbiflow-prjxray-presubmit-db-zynq7/.github/kokoro/db-quick.sh"
+
+timeout_mins: 180
+
+action {
+  define_artifacts {
+    regex: "**/*result*.xml"
+    regex: "**/*.log"
+  }
+}
+
+env_vars {
+  key: "KOKORO_TYPE"
+  value: "presubmit"
+}
+
+env_vars {
+  key: "KOKORO_DIR"
+  value: "symbiflow-prjxray-presubmit-db-zynq7"
+}
+
+env_vars {
+  key: "XRAY_SETTINGS"
+  value: "zynq7"
+}
+
+env_vars {
+  key: "XRAY_BUILD_TYPE"
+  value: "quick"
+}
diff --git a/.github/kokoro/quick.sh b/.github/kokoro/quick.sh
deleted file mode 120000
index ee2000b0..00000000
--- a/.github/kokoro/quick.sh
+++ /dev/null
@@ -1 +0,0 @@
-full.sh
\ No newline at end of file
diff --git a/.github/kokoro/tests.cfg b/.github/kokoro/tests.cfg
new file mode 100644
index 00000000..93b85287
--- /dev/null
+++ b/.github/kokoro/tests.cfg
@@ -0,0 +1,19 @@
+build_file: "symbiflow-prjxray-tests/.github/kokoro/tests.sh"
+
+timeout_mins: 10
+
+action {
+  define_artifacts {
+    regex: "**/*.log"
+    regex: "**/*sponge_log.xml"
+    regex: "**/build/**/*.xml"
+    regex: "**/build/*.xml"
+    regex: "*result*.xml"
+    strip_prefix: "github/symbiflow-prjxray-tests/"
+  }
+}
+
+env_vars {
+  key: "KOKORO_DIR"
+  value: "symbiflow-prjxray-tests"
+}
diff --git a/.github/kokoro/tests.sh b/.github/kokoro/tests.sh
new file mode 100755
index 00000000..93f94de1
--- /dev/null
+++ b/.github/kokoro/tests.sh
@@ -0,0 +1,81 @@
+#!/bin/bash
+
+set -x
+set -e
+
+sudo apt-get update
+sudo apt-get install -y \
+        bison \
+        build-essential \
+        ca-certificates \
+        clang-format \
+        cmake \
+        curl \
+        flex \
+        fontconfig \
+        git \
+        jq \
+        python \
+        python3 \
+        python3-dev \
+        python3-virtualenv \
+        python3-yaml \
+        virtualenv \
+
+echo "----------------------------------------"
+echo "----------------------------------------"
+echo "----------------------------------------"
+
+export
+
+echo "----------------------------------------"
+echo "----------------------------------------"
+echo "----------------------------------------"
+
+find .
+
+echo "----------------------------------------"
+echo "----------------------------------------"
+echo "----------------------------------------"
+
+echo $PWD
+
+echo "----------------------------------------"
+
+cd github/$KOKORO_DIR/
+
+git fetch --tags || true
+git describe --tags || true
+
+# Build the C++ tools
+make build --output-sync=target --warn-undefined-variables
+
+# Setup the Python environment
+make env --output-sync=target --warn-undefined-variables
+
+echo "----------------------------------------"
+echo "----------------------------------------"
+echo "----------------------------------------"
+
+find .
+
+echo "----------------------------------------"
+echo "----------------------------------------"
+echo "----------------------------------------"
+
+# Run the tests
+make test --output-sync=target --warn-undefined-variables
+
+echo "----------------------------------------"
+echo "----------------------------------------"
+echo "----------------------------------------"
+
+cat build/*test_results.xml
+mkdir build/py
+cp build/py_test_results.xml build/py/sponge_log.xml
+mkdir build/cpp
+cp build/cpp_test_results.xml build/cpp/sponge_log.xml
+
+echo "----------------------------------------"
+echo "----------------------------------------"
+echo "----------------------------------------"
diff --git a/Makefile b/Makefile
index 276e76ca..ad8ffd87 100644
--- a/Makefile
+++ b/Makefile
@@ -34,7 +34,7 @@ test: test-py test-cpp
 	@true
 
 test-py:
-	$(IN_ENV) PYTHONPATH="$(PWD):$(PWD)/third_party/fasm:$PYTHONPATH" py.test $(TEST_EXCLUDE) --doctest-modules --junitxml=build/py_test_results.xml
+	$(IN_ENV) which py.test; py.test $(TEST_EXCLUDE) --doctest-modules --junitxml=build/py_test_results.xml
 
 test-cpp:
 	mkdir -p build

From d27dfd541a661df8752f3b9348ca921b0ccb1ba5 Mon Sep 17 00:00:00 2001
From: Tim 'mithro' Ansell <me@mith.ro>
Date: Thu, 31 Jan 2019 19:39:27 +1100
Subject: [PATCH 08/28] infra: More aggressive compile.

Signed-off-by: Tim 'mithro' Ansell <me@mith.ro>
---
 .github/kokoro/db-full.sh  | 29 +++++++++++++++++++++++++++--
 .github/kokoro/db-quick.sh |  8 +-------
 2 files changed, 28 insertions(+), 9 deletions(-)

diff --git a/.github/kokoro/db-full.sh b/.github/kokoro/db-full.sh
index 3c623bd2..b2127e87 100755
--- a/.github/kokoro/db-full.sh
+++ b/.github/kokoro/db-full.sh
@@ -25,6 +25,8 @@ sudo apt-get install -y \
 ls -l ~/.Xilinx
 sudo chown -R $USER ~/.Xilinx
 
+CORES=$(nproc --all)
+
 echo "----------------------------------------"
 echo "----------------------------------------"
 echo "----------------------------------------"
@@ -51,14 +53,37 @@ git fetch --tags || true
 git describe --tags || true
 
 # Build the C++ tools
-make build --output-sync=target --warn-undefined-variables
+make build --output-sync=target --warn-undefined-variables -j$CORES
 
 # Setup the Python environment
 make env --output-sync=target --warn-undefined-variables
 
+echo "----------------------------------------"
+echo "----------------------------------------"
+echo "----------------------------------------"
+
 source settings/$XRAY_SETTINGS.sh
 (
 	export XILINX_LOCAL_USER_DATA=no
 	cd fuzzers
-	make --output-sync=target --warn-undefined-variables
+	make --output-sync=target --warn-undefined-variables -j$CORES MAX_VIVADO_PROCESS=$CORES
+)
+
+# Output how the database differs
+(
+	make formatdb
+	cd database
+	echo "----------------------------------------"
+	echo " Database Status"
+	echo "----------------------------------------"
+	git status
+	echo "----------------------------------------"
+	echo
+	echo
+	echo
+	echo "----------------------------------------"
+	echo " Database Diff"
+	echo "----------------------------------------"
+	git diff
+	echo "----------------------------------------"
 )
diff --git a/.github/kokoro/db-quick.sh b/.github/kokoro/db-quick.sh
index 08e19752..49431d31 100755
--- a/.github/kokoro/db-quick.sh
+++ b/.github/kokoro/db-quick.sh
@@ -53,7 +53,7 @@ git fetch --tags || true
 git describe --tags || true
 
 # Build the C++ tools
-make build --output-sync=target --warn-undefined-variables
+make build --output-sync=target --warn-undefined-variables -j$CORES
 
 # Setup the Python environment
 make env --output-sync=target --warn-undefined-variables
@@ -62,12 +62,6 @@ echo "----------------------------------------"
 echo "----------------------------------------"
 echo "----------------------------------------"
 
-find .
-
-echo "----------------------------------------"
-echo "----------------------------------------"
-echo "----------------------------------------"
-
 source settings/$XRAY_SETTINGS.sh
 (
 	export XILINX_LOCAL_USER_DATA=no

From 2d8d4832f1a56e786ffccbab5086b7b1c041058e Mon Sep 17 00:00:00 2001
From: Tim 'mithro' Ansell <me@mith.ro>
Date: Thu, 31 Jan 2019 20:23:55 +1100
Subject: [PATCH 09/28] infra: Remove quick/full split.

Full should be fast enough now.

Signed-off-by: Tim 'mithro' Ansell <me@mith.ro>
---
 .github/kokoro/continuous-db-artix7-quick.cfg | 32 ----------
 ...tix7-full.cfg => continuous-db-artix7.cfg} |  4 +-
 .../kokoro/continuous-db-kintex7-quick.cfg    | 32 ----------
 ...ex7-full.cfg => continuous-db-kintex7.cfg} |  4 +-
 .github/kokoro/continuous-db-zynq7-quick.cfg  | 32 ----------
 ...zynq7-full.cfg => continuous-db-zynq7.cfg} |  4 +-
 .github/kokoro/kokoro-cfg.py                  | 63 ++++---------------
 .github/kokoro/presubmit-db-artix7.cfg        |  7 ++-
 .github/kokoro/presubmit-db-kintex7.cfg       |  7 ++-
 .github/kokoro/presubmit-db-zynq7.cfg         |  7 ++-
 10 files changed, 29 insertions(+), 163 deletions(-)
 delete mode 100644 .github/kokoro/continuous-db-artix7-quick.cfg
 rename .github/kokoro/{continuous-db-artix7-full.cfg => continuous-db-artix7.cfg} (73%)
 delete mode 100644 .github/kokoro/continuous-db-kintex7-quick.cfg
 rename .github/kokoro/{continuous-db-kintex7-full.cfg => continuous-db-kintex7.cfg} (73%)
 delete mode 100644 .github/kokoro/continuous-db-zynq7-quick.cfg
 rename .github/kokoro/{continuous-db-zynq7-full.cfg => continuous-db-zynq7.cfg} (74%)

diff --git a/.github/kokoro/continuous-db-artix7-quick.cfg b/.github/kokoro/continuous-db-artix7-quick.cfg
deleted file mode 100644
index 42d2b777..00000000
--- a/.github/kokoro/continuous-db-artix7-quick.cfg
+++ /dev/null
@@ -1,32 +0,0 @@
-# Format: //devtools/kokoro/config/proto/build.proto
-
-build_file: "symbiflow-prjxray-continuous-db-artix7-quick/.github/kokoro/db-quick.sh"
-
-timeout_mins: 180
-
-action {
-  define_artifacts {
-    regex: "**/*result*.xml"
-    regex: "**/*.log"
-  }
-}
-
-env_vars {
-  key: "KOKORO_TYPE"
-  value: "continuous"
-}
-
-env_vars {
-  key: "KOKORO_DIR"
-  value: "symbiflow-prjxray-continuous-db-artix7-quick"
-}
-
-env_vars {
-  key: "XRAY_SETTINGS"
-  value: "artix7"
-}
-
-env_vars {
-  key: "XRAY_BUILD_TYPE"
-  value: "quick"
-}
diff --git a/.github/kokoro/continuous-db-artix7-full.cfg b/.github/kokoro/continuous-db-artix7.cfg
similarity index 73%
rename from .github/kokoro/continuous-db-artix7-full.cfg
rename to .github/kokoro/continuous-db-artix7.cfg
index 1048062e..a48eb6b7 100644
--- a/.github/kokoro/continuous-db-artix7-full.cfg
+++ b/.github/kokoro/continuous-db-artix7.cfg
@@ -1,6 +1,6 @@
 # Format: //devtools/kokoro/config/proto/build.proto
 
-build_file: "symbiflow-prjxray-continuous-db-artix7-full/.github/kokoro/db-full.sh"
+build_file: "symbiflow-prjxray-continuous-db-artix7/.github/kokoro/db-full.sh"
 
 timeout_mins: 4320
 
@@ -19,7 +19,7 @@ env_vars {
 
 env_vars {
   key: "KOKORO_DIR"
-  value: "symbiflow-prjxray-continuous-db-artix7-full"
+  value: "symbiflow-prjxray-continuous-db-artix7"
 }
 
 env_vars {
diff --git a/.github/kokoro/continuous-db-kintex7-quick.cfg b/.github/kokoro/continuous-db-kintex7-quick.cfg
deleted file mode 100644
index 3b67b337..00000000
--- a/.github/kokoro/continuous-db-kintex7-quick.cfg
+++ /dev/null
@@ -1,32 +0,0 @@
-# Format: //devtools/kokoro/config/proto/build.proto
-
-build_file: "symbiflow-prjxray-continuous-db-kintex7-quick/.github/kokoro/db-quick.sh"
-
-timeout_mins: 180
-
-action {
-  define_artifacts {
-    regex: "**/*result*.xml"
-    regex: "**/*.log"
-  }
-}
-
-env_vars {
-  key: "KOKORO_TYPE"
-  value: "continuous"
-}
-
-env_vars {
-  key: "KOKORO_DIR"
-  value: "symbiflow-prjxray-continuous-db-kintex7-quick"
-}
-
-env_vars {
-  key: "XRAY_SETTINGS"
-  value: "kintex7"
-}
-
-env_vars {
-  key: "XRAY_BUILD_TYPE"
-  value: "quick"
-}
diff --git a/.github/kokoro/continuous-db-kintex7-full.cfg b/.github/kokoro/continuous-db-kintex7.cfg
similarity index 73%
rename from .github/kokoro/continuous-db-kintex7-full.cfg
rename to .github/kokoro/continuous-db-kintex7.cfg
index bceb827f..fca573bf 100644
--- a/.github/kokoro/continuous-db-kintex7-full.cfg
+++ b/.github/kokoro/continuous-db-kintex7.cfg
@@ -1,6 +1,6 @@
 # Format: //devtools/kokoro/config/proto/build.proto
 
-build_file: "symbiflow-prjxray-continuous-db-kintex7-full/.github/kokoro/db-full.sh"
+build_file: "symbiflow-prjxray-continuous-db-kintex7/.github/kokoro/db-full.sh"
 
 timeout_mins: 4320
 
@@ -19,7 +19,7 @@ env_vars {
 
 env_vars {
   key: "KOKORO_DIR"
-  value: "symbiflow-prjxray-continuous-db-kintex7-full"
+  value: "symbiflow-prjxray-continuous-db-kintex7"
 }
 
 env_vars {
diff --git a/.github/kokoro/continuous-db-zynq7-quick.cfg b/.github/kokoro/continuous-db-zynq7-quick.cfg
deleted file mode 100644
index 24cfb1bd..00000000
--- a/.github/kokoro/continuous-db-zynq7-quick.cfg
+++ /dev/null
@@ -1,32 +0,0 @@
-# Format: //devtools/kokoro/config/proto/build.proto
-
-build_file: "symbiflow-prjxray-continuous-db-zynq7-quick/.github/kokoro/db-quick.sh"
-
-timeout_mins: 180
-
-action {
-  define_artifacts {
-    regex: "**/*result*.xml"
-    regex: "**/*.log"
-  }
-}
-
-env_vars {
-  key: "KOKORO_TYPE"
-  value: "continuous"
-}
-
-env_vars {
-  key: "KOKORO_DIR"
-  value: "symbiflow-prjxray-continuous-db-zynq7-quick"
-}
-
-env_vars {
-  key: "XRAY_SETTINGS"
-  value: "zynq7"
-}
-
-env_vars {
-  key: "XRAY_BUILD_TYPE"
-  value: "quick"
-}
diff --git a/.github/kokoro/continuous-db-zynq7-full.cfg b/.github/kokoro/continuous-db-zynq7.cfg
similarity index 74%
rename from .github/kokoro/continuous-db-zynq7-full.cfg
rename to .github/kokoro/continuous-db-zynq7.cfg
index 3e581974..4f38df77 100644
--- a/.github/kokoro/continuous-db-zynq7-full.cfg
+++ b/.github/kokoro/continuous-db-zynq7.cfg
@@ -1,6 +1,6 @@
 # Format: //devtools/kokoro/config/proto/build.proto
 
-build_file: "symbiflow-prjxray-continuous-db-zynq7-full/.github/kokoro/db-full.sh"
+build_file: "symbiflow-prjxray-continuous-db-zynq7/.github/kokoro/db-full.sh"
 
 timeout_mins: 4320
 
@@ -19,7 +19,7 @@ env_vars {
 
 env_vars {
   key: "KOKORO_DIR"
-  value: "symbiflow-prjxray-continuous-db-zynq7-full"
+  value: "symbiflow-prjxray-continuous-db-zynq7"
 }
 
 env_vars {
diff --git a/.github/kokoro/kokoro-cfg.py b/.github/kokoro/kokoro-cfg.py
index 2c0b5627..a89b1b53 100755
--- a/.github/kokoro/kokoro-cfg.py
+++ b/.github/kokoro/kokoro-cfg.py
@@ -1,44 +1,9 @@
 #!/usr/bin/env python3
 
-db_quick = """\
-# Format: //devtools/kokoro/config/proto/build.proto
-
-build_file: "symbiflow-prjxray-%(kokoro_type)s-db-%(part)s%(ext)s/.github/kokoro/db-quick.sh"
-
-timeout_mins: 180
-
-action {
-  define_artifacts {
-    regex: "**/*result*.xml"
-    regex: "**/*.log"
-  }
-}
-
-env_vars {
-  key: "KOKORO_TYPE"
-  value: "%(kokoro_type)s"
-}
-
-env_vars {
-  key: "KOKORO_DIR"
-  value: "symbiflow-prjxray-%(kokoro_type)s-db-%(part)s%(ext)s"
-}
-
-env_vars {
-  key: "XRAY_SETTINGS"
-  value: "%(part)s"
-}
-
-env_vars {
-  key: "XRAY_BUILD_TYPE"
-  value: "quick"
-}
-"""
-
 db_full = """\
 # Format: //devtools/kokoro/config/proto/build.proto
 
-build_file: "symbiflow-prjxray-%(kokoro_type)s-db-%(part)s-full/.github/kokoro/db-full.sh"
+build_file: "symbiflow-prjxray-%(kokoro_type)s-db-%(part)s/.github/kokoro/db-full.sh"
 
 timeout_mins: 4320
 
@@ -57,7 +22,7 @@ env_vars {
 
 env_vars {
   key: "KOKORO_DIR"
-  value: "symbiflow-prjxray-%(kokoro_type)s-db-%(part)s-full"
+  value: "symbiflow-prjxray-%(kokoro_type)s-db-%(part)s"
 }
 
 env_vars {
@@ -72,20 +37,14 @@ env_vars {
 """
 
 for part in ['artix7', 'kintex7', 'zynq7']:
-    with open("continuous-db-%s-quick.cfg" % part, "w") as f:
-        f.write(
-            db_quick % {
-                'part': part,
-                'kokoro_type': 'continuous',
-                'ext': '-quick'
-            })
-    with open("continuous-db-%s-full.cfg" % part, "w") as f:
-        f.write(db_full % {'part': part, 'kokoro_type': 'continuous'})
+    with open("continuous-db-%s.cfg" % part, "w") as f:
+        f.write(db_full % {
+            'part': part,
+            'kokoro_type': 'continuous',
+        })
 
     with open("presubmit-db-%s.cfg" % part, "w") as f:
-        f.write(
-            db_quick % {
-                'part': part,
-                'kokoro_type': 'presubmit',
-                'ext': ''
-            })
+        f.write(db_full % {
+            'part': part,
+            'kokoro_type': 'presubmit',
+        })
diff --git a/.github/kokoro/presubmit-db-artix7.cfg b/.github/kokoro/presubmit-db-artix7.cfg
index c7ca90ff..27d76a16 100644
--- a/.github/kokoro/presubmit-db-artix7.cfg
+++ b/.github/kokoro/presubmit-db-artix7.cfg
@@ -1,13 +1,14 @@
 # Format: //devtools/kokoro/config/proto/build.proto
 
-build_file: "symbiflow-prjxray-presubmit-db-artix7/.github/kokoro/db-quick.sh"
+build_file: "symbiflow-prjxray-presubmit-db-artix7/.github/kokoro/db-full.sh"
 
-timeout_mins: 180
+timeout_mins: 4320
 
 action {
   define_artifacts {
     regex: "**/*result*.xml"
     regex: "**/*.log"
+    regex: "database/artix7/**"
   }
 }
 
@@ -28,5 +29,5 @@ env_vars {
 
 env_vars {
   key: "XRAY_BUILD_TYPE"
-  value: "quick"
+  value: "full"
 }
diff --git a/.github/kokoro/presubmit-db-kintex7.cfg b/.github/kokoro/presubmit-db-kintex7.cfg
index 3a501168..f5957068 100644
--- a/.github/kokoro/presubmit-db-kintex7.cfg
+++ b/.github/kokoro/presubmit-db-kintex7.cfg
@@ -1,13 +1,14 @@
 # Format: //devtools/kokoro/config/proto/build.proto
 
-build_file: "symbiflow-prjxray-presubmit-db-kintex7/.github/kokoro/db-quick.sh"
+build_file: "symbiflow-prjxray-presubmit-db-kintex7/.github/kokoro/db-full.sh"
 
-timeout_mins: 180
+timeout_mins: 4320
 
 action {
   define_artifacts {
     regex: "**/*result*.xml"
     regex: "**/*.log"
+    regex: "database/kintex7/**"
   }
 }
 
@@ -28,5 +29,5 @@ env_vars {
 
 env_vars {
   key: "XRAY_BUILD_TYPE"
-  value: "quick"
+  value: "full"
 }
diff --git a/.github/kokoro/presubmit-db-zynq7.cfg b/.github/kokoro/presubmit-db-zynq7.cfg
index 6026b61e..11f748ac 100644
--- a/.github/kokoro/presubmit-db-zynq7.cfg
+++ b/.github/kokoro/presubmit-db-zynq7.cfg
@@ -1,13 +1,14 @@
 # Format: //devtools/kokoro/config/proto/build.proto
 
-build_file: "symbiflow-prjxray-presubmit-db-zynq7/.github/kokoro/db-quick.sh"
+build_file: "symbiflow-prjxray-presubmit-db-zynq7/.github/kokoro/db-full.sh"
 
-timeout_mins: 180
+timeout_mins: 4320
 
 action {
   define_artifacts {
     regex: "**/*result*.xml"
     regex: "**/*.log"
+    regex: "database/zynq7/**"
   }
 }
 
@@ -28,5 +29,5 @@ env_vars {
 
 env_vars {
   key: "XRAY_BUILD_TYPE"
-  value: "quick"
+  value: "full"
 }

From c8f7480cb8893c6d47cba3a82dbda82de9baf8fa Mon Sep 17 00:00:00 2001
From: Tim 'mithro' Ansell <me@mith.ro>
Date: Fri, 1 Feb 2019 19:50:49 +1100
Subject: [PATCH 10/28] infra: Remove --output-sync=master

Signed-off-by: Tim 'mithro' Ansell <me@mith.ro>
---
 .github/kokoro/db-full.sh  | 2 +-
 .github/kokoro/db-quick.sh | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/kokoro/db-full.sh b/.github/kokoro/db-full.sh
index b2127e87..2fedc74f 100755
--- a/.github/kokoro/db-full.sh
+++ b/.github/kokoro/db-full.sh
@@ -66,7 +66,7 @@ source settings/$XRAY_SETTINGS.sh
 (
 	export XILINX_LOCAL_USER_DATA=no
 	cd fuzzers
-	make --output-sync=target --warn-undefined-variables -j$CORES MAX_VIVADO_PROCESS=$CORES
+	make --warn-undefined-variables -j$CORES MAX_VIVADO_PROCESS=$CORES
 )
 
 # Output how the database differs
diff --git a/.github/kokoro/db-quick.sh b/.github/kokoro/db-quick.sh
index 49431d31..f2a70dc2 100755
--- a/.github/kokoro/db-quick.sh
+++ b/.github/kokoro/db-quick.sh
@@ -66,5 +66,5 @@ source settings/$XRAY_SETTINGS.sh
 (
 	export XILINX_LOCAL_USER_DATA=no
 	cd fuzzers
-	make --output-sync=target --warn-undefined-variables QUICK=y -j$CORES
+	make --warn-undefined-variables QUICK=y -j$CORES
 )

From bea995b5eaf8f717a9bedbe8c8c04879312cae5e Mon Sep 17 00:00:00 2001
From: Tim 'mithro' Ansell <me@mith.ro>
Date: Fri, 1 Feb 2019 22:32:19 +1100
Subject: [PATCH 11/28] infra: Output cpu info

Signed-off-by: Tim 'mithro' Ansell <me@mith.ro>
---
 .github/kokoro/db-full.sh  | 6 ++++++
 .github/kokoro/db-quick.sh | 6 ++++++
 .github/kokoro/tests.sh    | 6 ++++++
 3 files changed, 18 insertions(+)

diff --git a/.github/kokoro/db-full.sh b/.github/kokoro/db-full.sh
index 2fedc74f..32a2d463 100755
--- a/.github/kokoro/db-full.sh
+++ b/.github/kokoro/db-full.sh
@@ -37,6 +37,12 @@ echo "----------------------------------------"
 echo "----------------------------------------"
 echo "----------------------------------------"
 
+cat /proc/cpuinfo
+
+echo "----------------------------------------"
+echo "----------------------------------------"
+echo "----------------------------------------"
+
 find .
 
 echo "----------------------------------------"
diff --git a/.github/kokoro/db-quick.sh b/.github/kokoro/db-quick.sh
index f2a70dc2..24ef2654 100755
--- a/.github/kokoro/db-quick.sh
+++ b/.github/kokoro/db-quick.sh
@@ -37,6 +37,12 @@ echo "----------------------------------------"
 echo "----------------------------------------"
 echo "----------------------------------------"
 
+cat /proc/cpuinfo
+
+echo "----------------------------------------"
+echo "----------------------------------------"
+echo "----------------------------------------"
+
 find .
 
 echo "----------------------------------------"
diff --git a/.github/kokoro/tests.sh b/.github/kokoro/tests.sh
index 93f94de1..52592fc3 100755
--- a/.github/kokoro/tests.sh
+++ b/.github/kokoro/tests.sh
@@ -32,6 +32,12 @@ echo "----------------------------------------"
 echo "----------------------------------------"
 echo "----------------------------------------"
 
+cat /proc/cpuinfo
+
+echo "----------------------------------------"
+echo "----------------------------------------"
+echo "----------------------------------------"
+
 find .
 
 echo "----------------------------------------"

From e6d737896f146bab541262fe69575be3d202c628 Mon Sep 17 00:00:00 2001
From: Tim 'mithro' Ansell <me@mith.ro>
Date: Fri, 1 Feb 2019 23:26:23 +1100
Subject: [PATCH 12/28] infra: Install pstree

Signed-off-by: Tim 'mithro' Ansell <me@mith.ro>
---
 .github/kokoro/db-full.sh  | 1 +
 .github/kokoro/db-quick.sh | 1 +
 .github/kokoro/tests.sh    | 1 +
 3 files changed, 3 insertions(+)

diff --git a/.github/kokoro/db-full.sh b/.github/kokoro/db-full.sh
index 32a2d463..0b10f39f 100755
--- a/.github/kokoro/db-full.sh
+++ b/.github/kokoro/db-full.sh
@@ -15,6 +15,7 @@ sudo apt-get install -y \
         fontconfig \
         git \
         jq \
+        psmisc \
         python \
         python3 \
         python3-dev \
diff --git a/.github/kokoro/db-quick.sh b/.github/kokoro/db-quick.sh
index 24ef2654..a839eeb1 100755
--- a/.github/kokoro/db-quick.sh
+++ b/.github/kokoro/db-quick.sh
@@ -15,6 +15,7 @@ sudo apt-get install -y \
         fontconfig \
         git \
         jq \
+        psmisc \
         python \
         python3 \
         python3-dev \
diff --git a/.github/kokoro/tests.sh b/.github/kokoro/tests.sh
index 52592fc3..ba87d3ed 100755
--- a/.github/kokoro/tests.sh
+++ b/.github/kokoro/tests.sh
@@ -15,6 +15,7 @@ sudo apt-get install -y \
         fontconfig \
         git \
         jq \
+        psmisc \
         python \
         python3 \
         python3-dev \

From 1991669e8c7549f8f8cd7a1724edf319a83c7809 Mon Sep 17 00:00:00 2001
From: Tim 'mithro' Ansell <me@mith.ro>
Date: Sun, 3 Feb 2019 14:44:35 +1100
Subject: [PATCH 13/28] infra: Reworking kokoro scripts.

Signed-off-by: Tim 'mithro' Ansell <me@mith.ro>
---
 .github/kokoro/db-full.sh           | 111 +++++++++++-----------------
 .github/kokoro/db-quick.sh          |  85 ++++++---------------
 .github/kokoro/steps/git.sh         |  27 +++++++
 .github/kokoro/steps/hostinfo.sh    |  31 ++++++++
 .github/kokoro/steps/hostsetup.sh   |  36 +++++++++
 .github/kokoro/steps/prjxray-env.sh |  16 ++++
 .github/kokoro/steps/xilinx.sh      |   8 ++
 .github/kokoro/tests.sh             |  95 +++++-------------------
 8 files changed, 207 insertions(+), 202 deletions(-)
 create mode 100755 .github/kokoro/steps/git.sh
 create mode 100755 .github/kokoro/steps/hostinfo.sh
 create mode 100755 .github/kokoro/steps/hostsetup.sh
 create mode 100755 .github/kokoro/steps/prjxray-env.sh
 create mode 100755 .github/kokoro/steps/xilinx.sh

diff --git a/.github/kokoro/db-full.sh b/.github/kokoro/db-full.sh
index 0b10f39f..0e3627ea 100755
--- a/.github/kokoro/db-full.sh
+++ b/.github/kokoro/db-full.sh
@@ -1,82 +1,61 @@
 #!/bin/bash
 
-set -x
 set -e
 
-sudo apt-get update
-sudo apt-get install -y \
-        bison \
-        build-essential \
-        ca-certificates \
-        clang-format \
-        cmake \
-        curl \
-        flex \
-        fontconfig \
-        git \
-        jq \
-        psmisc \
-        python \
-        python3 \
-        python3-dev \
-        python3-virtualenv \
-        python3-yaml \
-        virtualenv \
-
-ls -l ~/.Xilinx
-sudo chown -R $USER ~/.Xilinx
-
-CORES=$(nproc --all)
-
-echo "----------------------------------------"
-echo "----------------------------------------"
-echo "----------------------------------------"
-
-export
-
-echo "----------------------------------------"
-echo "----------------------------------------"
-echo "----------------------------------------"
-
-cat /proc/cpuinfo
-
-echo "----------------------------------------"
-echo "----------------------------------------"
-echo "----------------------------------------"
-
-find .
-
-echo "----------------------------------------"
-echo "----------------------------------------"
-echo "----------------------------------------"
-
-echo $PWD
-
-echo "----------------------------------------"
-
 cd github/$KOKORO_DIR/
 
-git fetch --tags || true
-git describe --tags || true
+source ./.github/kokoro/steps/hostsetup.sh
+source ./.github/kokoro/steps/hostinfo.sh
+source ./.github/kokoro/steps/git.sh
 
-# Build the C++ tools
-make build --output-sync=target --warn-undefined-variables -j$CORES
+source ./.github/kokoro/steps/xilinx.sh
 
-# Setup the Python environment
-make env --output-sync=target --warn-undefined-variables
+source ./.github/kokoro/steps/prjxray-env.sh
 
+echo
+echo "========================================"
+echo "Downloading current database"
 echo "----------------------------------------"
-echo "----------------------------------------"
+(
+	./download-latest-db.sh
+)
 echo "----------------------------------------"
 
 source settings/$XRAY_SETTINGS.sh
-(
-	export XILINX_LOCAL_USER_DATA=no
-	cd fuzzers
-	make --warn-undefined-variables -j$CORES MAX_VIVADO_PROCESS=$CORES
-)
 
-# Output how the database differs
+echo
+echo "========================================"
+echo "Cleaning out current database"
+echo "----------------------------------------"
+(
+	cd database
+	make clean-${XRAY_SETTINGS}-db
+)
+echo "----------------------------------------"
+
+echo
+echo "========================================"
+echo "Running Database build"
+echo "----------------------------------------"
+(
+	cd fuzzers
+	echo "make --dry-run"
+	make --dry-run
+	echo "----------------------------------------"
+	export MAX_VIVADO_PROCESS=$CORES
+	set -x
+	make -j $CORES MAX_VIVADO_PROCESS=$CORES
+	set +x
+	echo "----------------------------------------"
+	echo "make --dry-run"
+	make --dry-run
+)
+echo "----------------------------------------"
+
+echo
+echo "========================================"
+echo " Database Differences"
+echo "----------------------------------------"
 (
 	make formatdb
 	cd database
@@ -92,5 +71,5 @@ source settings/$XRAY_SETTINGS.sh
 	echo " Database Diff"
 	echo "----------------------------------------"
 	git diff
-	echo "----------------------------------------"
 )
+echo "----------------------------------------"
diff --git a/.github/kokoro/db-quick.sh b/.github/kokoro/db-quick.sh
index a839eeb1..58e0f69b 100755
--- a/.github/kokoro/db-quick.sh
+++ b/.github/kokoro/db-quick.sh
@@ -1,77 +1,40 @@
 #!/bin/bash
 
-set -x
 set -e
 
-sudo apt-get update
-sudo apt-get install -y \
-        bison \
-        build-essential \
-        ca-certificates \
-        clang-format \
-        cmake \
-        curl \
-        flex \
-        fontconfig \
-        git \
-        jq \
-        psmisc \
-        python \
-        python3 \
-        python3-dev \
-        python3-virtualenv \
-        python3-yaml \
-        virtualenv \
-
-ls -l ~/.Xilinx
-sudo chown -R $USER ~/.Xilinx
-
-CORES=$(nproc --all)
-
-echo "----------------------------------------"
-echo "----------------------------------------"
-echo "----------------------------------------"
-
-export
-
-echo "----------------------------------------"
-echo "----------------------------------------"
-echo "----------------------------------------"
-
-cat /proc/cpuinfo
-
-echo "----------------------------------------"
-echo "----------------------------------------"
-echo "----------------------------------------"
-
-find .
-
-echo "----------------------------------------"
-echo "----------------------------------------"
-echo "----------------------------------------"
-
-echo $PWD
-
-echo "----------------------------------------"
-
 cd github/$KOKORO_DIR/
 
-git fetch --tags || true
-git describe --tags || true
+source ./.github/kokoro/steps/hostsetup.sh
+source ./.github/kokoro/steps/hostinfo.sh
+source ./.github/kokoro/steps/git.sh
 
-# Build the C++ tools
-make build --output-sync=target --warn-undefined-variables -j$CORES
+source ./.github/kokoro/steps/xilinx.sh
 
-# Setup the Python environment
-make env --output-sync=target --warn-undefined-variables
+source ./.github/kokoro/steps/prjxray-env.sh
 
+echo
+echo "========================================"
+echo "Downloading current database"
 echo "----------------------------------------"
-echo "----------------------------------------"
+(
+	./download-latest-db.sh
+)
 echo "----------------------------------------"
 
 source settings/$XRAY_SETTINGS.sh
+
+echo
+echo "========================================"
+echo "Running quick Database build"
+echo "----------------------------------------"
 (
-	export XILINX_LOCAL_USER_DATA=no
 	cd fuzzers
-	make --warn-undefined-variables QUICK=y -j$CORES
+	echo "make --dry-run"
+	make --dry-run
+	echo "----------------------------------------"
+	export MAX_VIVADO_PROCESS=$CORES
+	set -x
+	make -j $CORES MAX_VIVADO_PROCESS=$CORES QUICK=y
+	set +x
 )
+echo "----------------------------------------"
diff --git a/.github/kokoro/steps/git.sh b/.github/kokoro/steps/git.sh
new file mode 100755
index 00000000..1837389c
--- /dev/null
+++ b/.github/kokoro/steps/git.sh
@@ -0,0 +1,27 @@
+#!/bin/bash
+
+set -e
+
+echo
+echo "========================================"
+echo "Git log"
+echo "----------------------------------------"
+echo "----------------------------------------"
+
+echo
+echo "========================================"
+echo "Git fetching tags"
+echo "----------------------------------------"
+git fetch --tags || true
+echo "----------------------------------------"
+
+echo
+echo "========================================"
+echo "Git version info"
+echo "----------------------------------------"
+git log -n1
+echo "----------------------------------------"
+git describe --tags || true
+echo "----------------------------------------"
+git describe --tags --always || true
+echo "----------------------------------------"
diff --git a/.github/kokoro/steps/hostinfo.sh b/.github/kokoro/steps/hostinfo.sh
new file mode 100755
index 00000000..6a00c37f
--- /dev/null
+++ b/.github/kokoro/steps/hostinfo.sh
@@ -0,0 +1,31 @@
+#!/bin/bash
+
+set -e
+
+echo
+echo "========================================"
+echo "Host Environment"
+echo "----------------------------------------"
+export
+echo "----------------------------------------"
+
+echo
+echo "========================================"
+echo "Host CPU"
+echo "----------------------------------------"
+export CORES=$(nproc --all)
+echo "Cores: $CORES"
+echo
+echo "Memory"
+echo "----------------------------------------"
+cat /proc/meminfo
+echo "----------------------------------------"
+
+echo
+echo "========================================"
+echo "Host files"
+echo "----------------------------------------"
+echo $PWD
+echo "----------------------------------------"
+find . | sort
+echo "----------------------------------------"
diff --git a/.github/kokoro/steps/hostsetup.sh b/.github/kokoro/steps/hostsetup.sh
new file mode 100755
index 00000000..16a423b9
--- /dev/null
+++ b/.github/kokoro/steps/hostsetup.sh
@@ -0,0 +1,36 @@
+#!/bin/bash
+
+set -e
+
+echo
+echo "========================================"
+echo "Host updating packages"
+echo "----------------------------------------"
+sudo apt-get update
+echo "----------------------------------------"
+
+echo
+echo "========================================"
+echo "Host install packages"
+echo "----------------------------------------"
+sudo apt-get install -y \
+        bison \
+        build-essential \
+        ca-certificates \
+        clang-format \
+        cmake \
+        curl \
+        flex \
+        fontconfig \
+        git \
+        jq \
+        psmisc \
+        python \
+        python3 \
+        python3-dev \
+        python3-virtualenv \
+        python3-yaml \
+        virtualenv \
+
+echo "----------------------------------------"
+
diff --git a/.github/kokoro/steps/prjxray-env.sh b/.github/kokoro/steps/prjxray-env.sh
new file mode 100755
index 00000000..42a54f9d
--- /dev/null
+++ b/.github/kokoro/steps/prjxray-env.sh
@@ -0,0 +1,16 @@
+#!/bin/bash
+
+set -e
+
+echo
+echo "========================================"
+echo "Build the C++ tools"
+echo "----------------------------------------"
+make build --output-sync=target --warn-undefined-variables -j $CORES
+echo "----------------------------------------"
+
+echo
+echo "========================================"
+echo "Setup the Python environment"
+echo "----------------------------------------"
+make env --output-sync=target --warn-undefined-variables
diff --git a/.github/kokoro/steps/xilinx.sh b/.github/kokoro/steps/xilinx.sh
new file mode 100755
index 00000000..d874c37f
--- /dev/null
+++ b/.github/kokoro/steps/xilinx.sh
@@ -0,0 +1,8 @@
+#!/bin/bash
+
+# Fix up things related to Xilinx tool chain.
+
+ls -l ~/.Xilinx
+sudo chown -R $USER ~/.Xilinx
+
+export XILINX_LOCAL_USER_DATA=no
diff --git a/.github/kokoro/tests.sh b/.github/kokoro/tests.sh
index ba87d3ed..91eb01e5 100755
--- a/.github/kokoro/tests.sh
+++ b/.github/kokoro/tests.sh
@@ -1,88 +1,33 @@
 #!/bin/bash
 
-set -x
 set -e
 
-sudo apt-get update
-sudo apt-get install -y \
-        bison \
-        build-essential \
-        ca-certificates \
-        clang-format \
-        cmake \
-        curl \
-        flex \
-        fontconfig \
-        git \
-        jq \
-        psmisc \
-        python \
-        python3 \
-        python3-dev \
-        python3-virtualenv \
-        python3-yaml \
-        virtualenv \
-
-echo "----------------------------------------"
-echo "----------------------------------------"
-echo "----------------------------------------"
-
-export
-
-echo "----------------------------------------"
-echo "----------------------------------------"
-echo "----------------------------------------"
-
-cat /proc/cpuinfo
-
-echo "----------------------------------------"
-echo "----------------------------------------"
-echo "----------------------------------------"
-
-find .
-
-echo "----------------------------------------"
-echo "----------------------------------------"
-echo "----------------------------------------"
-
-echo $PWD
-
-echo "----------------------------------------"
-
 cd github/$KOKORO_DIR/
 
-git fetch --tags || true
-git describe --tags || true
+source ./.github/kokoro/steps/hostsetup.sh
+source ./.github/kokoro/steps/hostinfo.sh
+source ./.github/kokoro/steps/git.sh
 
-# Build the C++ tools
-make build --output-sync=target --warn-undefined-variables
-
-# Setup the Python environment
-make env --output-sync=target --warn-undefined-variables
+source ./.github/kokoro/steps/prjxray-env.sh
 
+echo
+echo "========================================"
+echo "Running tests"
 echo "----------------------------------------"
-echo "----------------------------------------"
+(
+	make test --output-sync=target --warn-undefined-variables
+)
 echo "----------------------------------------"
 
-find .
-
-echo "----------------------------------------"
-echo "----------------------------------------"
-echo "----------------------------------------"
-
-# Run the tests
-make test --output-sync=target --warn-undefined-variables
-
-echo "----------------------------------------"
-echo "----------------------------------------"
-echo "----------------------------------------"
-
-cat build/*test_results.xml
-mkdir build/py
-cp build/py_test_results.xml build/py/sponge_log.xml
-mkdir build/cpp
-cp build/cpp_test_results.xml build/cpp/sponge_log.xml
-
-echo "----------------------------------------"
+echo
+echo "========================================"
+echo "Copying tests logs"
 echo "----------------------------------------"
+(
+	cat build/*test_results.xml
+	mkdir build/py
+	cp build/py_test_results.xml build/py/sponge_log.xml
+	mkdir build/cpp
+	cp build/cpp_test_results.xml build/cpp/sponge_log.xml
+)
 echo "----------------------------------------"

From fd4083983008e534dbf9ca08416f36d080109f35 Mon Sep 17 00:00:00 2001
From: Tim 'mithro' Ansell <me@mith.ro>
Date: Mon, 4 Feb 2019 00:21:32 +1100
Subject: [PATCH 14/28] infra: Install some custom deb packages.

Signed-off-by: Tim 'mithro' Ansell <me@mith.ro>
---
 .github/kokoro/steps/hostsetup.sh | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/.github/kokoro/steps/hostsetup.sh b/.github/kokoro/steps/hostsetup.sh
index 16a423b9..30d1a374 100755
--- a/.github/kokoro/steps/hostsetup.sh
+++ b/.github/kokoro/steps/hostsetup.sh
@@ -34,3 +34,19 @@ sudo apt-get install -y \
 
 echo "----------------------------------------"
 
+(
+	cd /tmp
+	# Download ptyget to enable ptybandage.
+	wget https://storage.googleapis.com/prjxray-deps-debs/ptyget_6_amd64.deb
+	sudo dpkg --install ptyget_6_amd64.deb
+	which ptybandage
+	ptybandage --help || true
+
+	# Upgrade pstree to support the -T flag.
+	wget https://storage.googleapis.com/prjxray-deps-debs/psmisc_23.2-1_amd64.deb
+	sudo dpkg --install psmisc_23.2-1_amd64.deb
+	which pstree
+	pstree --help || true
+)
+
+echo "----------------------------------------"

From 655af515398051f486bf1cbe2ee07f722083374c Mon Sep 17 00:00:00 2001
From: Tim 'mithro' Ansell <me@mith.ro>
Date: Tue, 5 Feb 2019 00:43:10 +1100
Subject: [PATCH 15/28] infra: Run kokoro build inside script.

This fixes the issue with make not output status as things run (instead
waiting until a whole command has finished).

Signed-off-by: Tim 'mithro' Ansell <me@mith.ro>
---
 .github/kokoro/db-full.sh  | 2 +-
 .github/kokoro/db-quick.sh | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/kokoro/db-full.sh b/.github/kokoro/db-full.sh
index 0e3627ea..1b8d91f1 100755
--- a/.github/kokoro/db-full.sh
+++ b/.github/kokoro/db-full.sh
@@ -44,7 +44,7 @@ echo "----------------------------------------"
 	echo "----------------------------------------"
 	export MAX_VIVADO_PROCESS=$CORES
 	set -x
-	make -j $CORES MAX_VIVADO_PROCESS=$CORES
+	script --return --flush --command "make -j $CORES MAX_VIVADO_PROCESS=$CORES" -
 	set +x
 	echo "----------------------------------------"
 	echo "make --dry-run"
diff --git a/.github/kokoro/db-quick.sh b/.github/kokoro/db-quick.sh
index 58e0f69b..13c91e10 100755
--- a/.github/kokoro/db-quick.sh
+++ b/.github/kokoro/db-quick.sh
@@ -34,7 +34,7 @@ echo "----------------------------------------"
 	echo "----------------------------------------"
 	export MAX_VIVADO_PROCESS=$CORES
 	set -x
-	make -j $CORES MAX_VIVADO_PROCESS=$CORES QUICK=y
+	script --return --flush --command "make -j $CORES MAX_VIVADO_PROCESS=$CORES QUICK=y" -
 	set +x
 )
 echo "----------------------------------------"

From c038539f4c0919350d4b28dce84060f53340bc73 Mon Sep 17 00:00:00 2001
From: Tim 'mithro' Ansell <me@mith.ro>
Date: Tue, 5 Feb 2019 13:09:46 +1100
Subject: [PATCH 16/28] infra: Only formatdb / checkdb for part currently
 running.

Signed-off-by: Tim 'mithro' Ansell <me@mith.ro>
---
 .github/kokoro/db-full.sh  | 10 +++++++++-
 .github/kokoro/db-quick.sh |  2 +-
 2 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/.github/kokoro/db-full.sh b/.github/kokoro/db-full.sh
index 1b8d91f1..a1bf4926 100755
--- a/.github/kokoro/db-full.sh
+++ b/.github/kokoro/db-full.sh
@@ -52,12 +52,17 @@ echo "----------------------------------------"
 )
 echo "----------------------------------------"
 
+# Check the database
+make checkdb-${XRAY_SETTINGS} || true
+# Format the database
+make formatdb-${XRAY_SETTINGS}
+
+# Output if the database has differences
 echo
 echo "========================================"
 echo " Database Differences"
 echo "----------------------------------------"
 (
-	make formatdb
 	cd database
 	echo "----------------------------------------"
 	echo " Database Status"
@@ -73,3 +78,6 @@ echo "----------------------------------------"
 	git diff
 )
 echo "----------------------------------------"
+
+# Check the database and fail if it is broken.
+make checkdb-${XRAY_SETTINGS}
diff --git a/.github/kokoro/db-quick.sh b/.github/kokoro/db-quick.sh
index 13c91e10..35c2aa42 100755
--- a/.github/kokoro/db-quick.sh
+++ b/.github/kokoro/db-quick.sh
@@ -25,7 +25,7 @@ source settings/$XRAY_SETTINGS.sh
 
 echo
 echo "========================================"
-echo "Running quick Database build"
+echo "Running quick fuzzer sanity check"
 echo "----------------------------------------"
 (
 	cd fuzzers

From 5543fc579c225a6996f78ea11678d09c23c22133 Mon Sep 17 00:00:00 2001
From: Tim 'mithro' Ansell <me@mith.ro>
Date: Tue, 5 Feb 2019 13:22:27 +1100
Subject: [PATCH 17/28] infra: Make sure all fuzzers complete.

If they don't output debugging information.

Signed-off-by: Tim 'mithro' Ansell <me@mith.ro>
---
 .github/kokoro/db-full.sh | 18 ++++++++++++++++--
 1 file changed, 16 insertions(+), 2 deletions(-)

diff --git a/.github/kokoro/db-full.sh b/.github/kokoro/db-full.sh
index a1bf4926..d8168bda 100755
--- a/.github/kokoro/db-full.sh
+++ b/.github/kokoro/db-full.sh
@@ -39,16 +39,30 @@ echo "Running Database build"
 echo "----------------------------------------"
 (
 	cd fuzzers
+
+	# Output which fuzzers we are going to run
 	echo "make --dry-run"
 	make --dry-run
 	echo "----------------------------------------"
+
+	# Run the fuzzers
 	export MAX_VIVADO_PROCESS=$CORES
 	set -x
 	script --return --flush --command "make -j $CORES MAX_VIVADO_PROCESS=$CORES" -
 	set +x
 	echo "----------------------------------------"
-	echo "make --dry-run"
-	make --dry-run
+
+	# Check there is nothing to do after running...
+	if [ $(make --dry-run | wc -l) -gt 0 ]; then
+		echo "The following targets need to still run!"
+		make --dry-run
+		echo "----------------------------------------"
+		echo "Debug output on why they still need to run"
+		make --dry-run --debug
+		echo "----------------------------------------"
+	else
+		echo "All good, nothing more to do!"
+	fi
 )
 echo "----------------------------------------"
 

From d144c9d2c8c1450a329202656385a1fa507b24a6 Mon Sep 17 00:00:00 2001
From: Tim 'mithro' Ansell <me@mith.ro>
Date: Tue, 5 Feb 2019 17:21:00 +1100
Subject: [PATCH 18/28] fuzzers: Disable the DSP fuzzer for the moment.

Signed-off-by: Tim 'mithro' Ansell <me@mith.ro>
---
 fuzzers/Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/fuzzers/Makefile b/fuzzers/Makefile
index 20754216..78dd8a1c 100644
--- a/fuzzers/Makefile
+++ b/fuzzers/Makefile
@@ -87,7 +87,7 @@ $(eval $(call fuzzer,073-get_counts,))
 $(eval $(call fuzzer,074-dump_all,005-tilegrid 072-ordered_wires))
 endif
 endif
-$(eval $(call fuzzer,100-dsp-mskpat,005-tilegrid))
+#$(eval $(call fuzzer,100-dsp-mskpat,005-tilegrid))
 
 quick:
 	$(MAKE) QUICK=Y

From c3f6ba5c7ca7404d11aa17749f96119d021f99a7 Mon Sep 17 00:00:00 2001
From: Tim 'mithro' Ansell <me@mith.ro>
Date: Tue, 5 Feb 2019 22:19:59 +1100
Subject: [PATCH 19/28] infra: Detect no changes.

Signed-off-by: Tim 'mithro' Ansell <me@mith.ro>
---
 .github/kokoro/db-full.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/kokoro/db-full.sh b/.github/kokoro/db-full.sh
index d8168bda..7b0b0a79 100755
--- a/.github/kokoro/db-full.sh
+++ b/.github/kokoro/db-full.sh
@@ -53,7 +53,7 @@ echo "----------------------------------------"
 	echo "----------------------------------------"
 
 	# Check there is nothing to do after running...
-	if [ $(make --dry-run | wc -l) -gt 0 ]; then
+	if [ $(make --dry-run | grep -v 'Nothing to be done' | wc -l) -gt 0 ]; then
 		echo "The following targets need to still run!"
 		make --dry-run
 		echo "----------------------------------------"

From 863621ca1485d996e9b05d2811323a2e5d948d5f Mon Sep 17 00:00:00 2001
From: Tim 'mithro' Ansell <me@mith.ro>
Date: Tue, 5 Feb 2019 22:25:23 +1100
Subject: [PATCH 20/28] infra: Disable checkdb.

At the moment `make checkdb` seems to get stuck and never finish
sometimes. See issue #583.

Signed-off-by: Tim 'mithro' Ansell <me@mith.ro>
---
 .github/kokoro/db-full.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/kokoro/db-full.sh b/.github/kokoro/db-full.sh
index 7b0b0a79..65af5677 100755
--- a/.github/kokoro/db-full.sh
+++ b/.github/kokoro/db-full.sh
@@ -67,7 +67,7 @@ echo "----------------------------------------"
 echo "----------------------------------------"
 
 # Check the database
-make checkdb-${XRAY_SETTINGS} || true
+#make checkdb-${XRAY_SETTINGS} || true
 # Format the database
 make formatdb-${XRAY_SETTINGS}
 

From 8d65b6921df3be598ac404acc3076afc38306285 Mon Sep 17 00:00:00 2001
From: Tim 'mithro' Ansell <me@mith.ro>
Date: Wed, 6 Feb 2019 11:46:44 +1100
Subject: [PATCH 21/28] infra: Rework the artifact capture.

 * Should capture logs / build directory output on failure.
 * Should capture the database output on success.

Signed-off-by: Tim 'mithro' Ansell <me@mith.ro>
---
 .github/kokoro/continuous-db-artix7.cfg  |  6 ++++--
 .github/kokoro/continuous-db-kintex7.cfg |  6 ++++--
 .github/kokoro/continuous-db-zynq7.cfg   |  6 ++++--
 .github/kokoro/database.cfg              |  5 ++++-
 .github/kokoro/db-full.sh                | 19 ++++++++++++++++++-
 .github/kokoro/presubmit-db-artix7.cfg   |  6 ++++--
 .github/kokoro/presubmit-db-kintex7.cfg  |  6 ++++--
 .github/kokoro/presubmit-db-zynq7.cfg    |  6 ++++--
 8 files changed, 46 insertions(+), 14 deletions(-)

diff --git a/.github/kokoro/continuous-db-artix7.cfg b/.github/kokoro/continuous-db-artix7.cfg
index a48eb6b7..aa641dea 100644
--- a/.github/kokoro/continuous-db-artix7.cfg
+++ b/.github/kokoro/continuous-db-artix7.cfg
@@ -7,8 +7,10 @@ timeout_mins: 4320
 action {
   define_artifacts {
     regex: "**/*result*.xml"
-    regex: "**/*.log"
-    regex: "database/artix7/**"
+    regex: "**/build/**"
+    regex: "**/logs/**"
+    regex: "**/database/zynq7/**"
+    strip_prefix: "github/symbiflow-prjxray-continuous-db-artix7/"
   }
 }
 
diff --git a/.github/kokoro/continuous-db-kintex7.cfg b/.github/kokoro/continuous-db-kintex7.cfg
index fca573bf..b6f7f24e 100644
--- a/.github/kokoro/continuous-db-kintex7.cfg
+++ b/.github/kokoro/continuous-db-kintex7.cfg
@@ -7,8 +7,10 @@ timeout_mins: 4320
 action {
   define_artifacts {
     regex: "**/*result*.xml"
-    regex: "**/*.log"
-    regex: "database/kintex7/**"
+    regex: "**/build/**"
+    regex: "**/logs/**"
+    regex: "**/database/zynq7/**"
+    strip_prefix: "github/symbiflow-prjxray-continuous-db-kintex7/"
   }
 }
 
diff --git a/.github/kokoro/continuous-db-zynq7.cfg b/.github/kokoro/continuous-db-zynq7.cfg
index 4f38df77..0e2d5a91 100644
--- a/.github/kokoro/continuous-db-zynq7.cfg
+++ b/.github/kokoro/continuous-db-zynq7.cfg
@@ -7,8 +7,10 @@ timeout_mins: 4320
 action {
   define_artifacts {
     regex: "**/*result*.xml"
-    regex: "**/*.log"
-    regex: "database/zynq7/**"
+    regex: "**/build/**"
+    regex: "**/logs/**"
+    regex: "**/database/zynq7/**"
+    strip_prefix: "github/symbiflow-prjxray-continuous-db-zynq7/"
   }
 }
 
diff --git a/.github/kokoro/database.cfg b/.github/kokoro/database.cfg
index 762600a4..3097fda1 100644
--- a/.github/kokoro/database.cfg
+++ b/.github/kokoro/database.cfg
@@ -5,6 +5,9 @@ timeout_mins: 10
 action {
   define_artifacts {
     regex: "**/*result*.xml"
-    regex: "**/*.log"
+    regex: "**/logs/**"
+    regex: "**/build/**"
+    regex: "**/database/**"
+    strip_prefix: "github/symbiflow-prjxray-db/"
   }
 }
diff --git a/.github/kokoro/db-full.sh b/.github/kokoro/db-full.sh
index 65af5677..c64628b6 100755
--- a/.github/kokoro/db-full.sh
+++ b/.github/kokoro/db-full.sh
@@ -53,6 +53,7 @@ echo "----------------------------------------"
 	echo "----------------------------------------"
 
 	# Check there is nothing to do after running...
+	echo
 	if [ $(make --dry-run | grep -v 'Nothing to be done' | wc -l) -gt 0 ]; then
 		echo "The following targets need to still run!"
 		make --dry-run
@@ -60,6 +61,7 @@ echo "----------------------------------------"
 		echo "Debug output on why they still need to run"
 		make --dry-run --debug
 		echo "----------------------------------------"
+		exit 1
 	else
 		echo "All good, nothing more to do!"
 	fi
@@ -94,4 +96,19 @@ echo "----------------------------------------"
 echo "----------------------------------------"
 
 # Check the database and fail if it is broken.
-make checkdb-${XRAY_SETTINGS}
+#make checkdb-${XRAY_SETTINGS}
+
+# If we get here, then all the fuzzers completed fine. Hence we are
+# going to assume we don't want to keep all the build / logs / etc (as
+# they are quite large). Thus do a clean to get rid of them.
+echo
+echo "========================================"
+echo " Cleaning up after success"
+echo "----------------------------------------"
+(
+	cd fuzzers
+	echo
+	echo "Cleaning up so CI doesn't save all the excess data."
+	make clean
+)
+echo "----------------------------------------"
diff --git a/.github/kokoro/presubmit-db-artix7.cfg b/.github/kokoro/presubmit-db-artix7.cfg
index 27d76a16..a026e6e6 100644
--- a/.github/kokoro/presubmit-db-artix7.cfg
+++ b/.github/kokoro/presubmit-db-artix7.cfg
@@ -7,8 +7,10 @@ timeout_mins: 4320
 action {
   define_artifacts {
     regex: "**/*result*.xml"
-    regex: "**/*.log"
-    regex: "database/artix7/**"
+    regex: "**/logs/**"
+    regex: "**/build/**"
+    regex: "**/database/artix7/**"
+    strip_prefix: "github/symbiflow-prjxray-presubmit-db-artix7/"
   }
 }
 
diff --git a/.github/kokoro/presubmit-db-kintex7.cfg b/.github/kokoro/presubmit-db-kintex7.cfg
index f5957068..c9f90bc6 100644
--- a/.github/kokoro/presubmit-db-kintex7.cfg
+++ b/.github/kokoro/presubmit-db-kintex7.cfg
@@ -7,8 +7,10 @@ timeout_mins: 4320
 action {
   define_artifacts {
     regex: "**/*result*.xml"
-    regex: "**/*.log"
-    regex: "database/kintex7/**"
+    regex: "**/logs/**"
+    regex: "**/build/**"
+    regex: "**/database/kintex7/**"
+    strip_prefix: "github/symbiflow-prjxray-presubmit-db-kintex7/"
   }
 }
 
diff --git a/.github/kokoro/presubmit-db-zynq7.cfg b/.github/kokoro/presubmit-db-zynq7.cfg
index 11f748ac..176dec40 100644
--- a/.github/kokoro/presubmit-db-zynq7.cfg
+++ b/.github/kokoro/presubmit-db-zynq7.cfg
@@ -7,8 +7,10 @@ timeout_mins: 4320
 action {
   define_artifacts {
     regex: "**/*result*.xml"
-    regex: "**/*.log"
-    regex: "database/zynq7/**"
+    regex: "**/logs/**"
+    regex: "**/build/**"
+    regex: "**/database/zynq7/**"
+    strip_prefix: "github/symbiflow-prjxray-presubmit-db-zynq7/"
   }
 }
 

From c12d3a02b181e7bee92826ffb6636195d44ada2e Mon Sep 17 00:00:00 2001
From: Tim 'mithro' Ansell <me@mith.ro>
Date: Wed, 6 Feb 2019 11:51:34 +1100
Subject: [PATCH 22/28] infra: ptybandage wasn't needed in the end.

Signed-off-by: Tim 'mithro' Ansell <me@mith.ro>
---
 .github/kokoro/steps/hostsetup.sh | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/.github/kokoro/steps/hostsetup.sh b/.github/kokoro/steps/hostsetup.sh
index 30d1a374..b959cd21 100755
--- a/.github/kokoro/steps/hostsetup.sh
+++ b/.github/kokoro/steps/hostsetup.sh
@@ -36,12 +36,6 @@ echo "----------------------------------------"
 
 (
 	cd /tmp
-	# Download ptyget to enable ptybandage.
-	wget https://storage.googleapis.com/prjxray-deps-debs/ptyget_6_amd64.deb
-	sudo dpkg --install ptyget_6_amd64.deb
-	which ptybandage
-	ptybandage --help || true
-
 	# Upgrade pstree to support the -T flag.
 	wget https://storage.googleapis.com/prjxray-deps-debs/psmisc_23.2-1_amd64.deb
 	sudo dpkg --install psmisc_23.2-1_amd64.deb

From 7412a1b695c02c75668c12d8d5f25c7ac1c690d3 Mon Sep 17 00:00:00 2001
From: Tim 'mithro' Ansell <me@mith.ro>
Date: Wed, 6 Feb 2019 17:13:06 +1100
Subject: [PATCH 23/28] infra: Only collect log files for now.

Signed-off-by: Tim 'mithro' Ansell <me@mith.ro>
---
 .github/kokoro/continuous-db-artix7.cfg  | 2 +-
 .github/kokoro/continuous-db-kintex7.cfg | 2 +-
 .github/kokoro/continuous-db-zynq7.cfg   | 2 +-
 .github/kokoro/database.cfg              | 2 +-
 .github/kokoro/presubmit-db-artix7.cfg   | 2 +-
 .github/kokoro/presubmit-db-kintex7.cfg  | 2 +-
 .github/kokoro/presubmit-db-zynq7.cfg    | 2 +-
 7 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/.github/kokoro/continuous-db-artix7.cfg b/.github/kokoro/continuous-db-artix7.cfg
index aa641dea..1c947f2a 100644
--- a/.github/kokoro/continuous-db-artix7.cfg
+++ b/.github/kokoro/continuous-db-artix7.cfg
@@ -7,7 +7,7 @@ timeout_mins: 4320
 action {
   define_artifacts {
     regex: "**/*result*.xml"
-    regex: "**/build/**"
+    regex: "**/build/*.log"
     regex: "**/logs/**"
     regex: "**/database/zynq7/**"
     strip_prefix: "github/symbiflow-prjxray-continuous-db-artix7/"
diff --git a/.github/kokoro/continuous-db-kintex7.cfg b/.github/kokoro/continuous-db-kintex7.cfg
index b6f7f24e..6beeda62 100644
--- a/.github/kokoro/continuous-db-kintex7.cfg
+++ b/.github/kokoro/continuous-db-kintex7.cfg
@@ -7,7 +7,7 @@ timeout_mins: 4320
 action {
   define_artifacts {
     regex: "**/*result*.xml"
-    regex: "**/build/**"
+    regex: "**/build/*.log"
     regex: "**/logs/**"
     regex: "**/database/zynq7/**"
     strip_prefix: "github/symbiflow-prjxray-continuous-db-kintex7/"
diff --git a/.github/kokoro/continuous-db-zynq7.cfg b/.github/kokoro/continuous-db-zynq7.cfg
index 0e2d5a91..d744cc5f 100644
--- a/.github/kokoro/continuous-db-zynq7.cfg
+++ b/.github/kokoro/continuous-db-zynq7.cfg
@@ -7,7 +7,7 @@ timeout_mins: 4320
 action {
   define_artifacts {
     regex: "**/*result*.xml"
-    regex: "**/build/**"
+    regex: "**/build/*.log"
     regex: "**/logs/**"
     regex: "**/database/zynq7/**"
     strip_prefix: "github/symbiflow-prjxray-continuous-db-zynq7/"
diff --git a/.github/kokoro/database.cfg b/.github/kokoro/database.cfg
index 3097fda1..4dabb33f 100644
--- a/.github/kokoro/database.cfg
+++ b/.github/kokoro/database.cfg
@@ -5,7 +5,7 @@ timeout_mins: 10
 action {
   define_artifacts {
     regex: "**/*result*.xml"
-    regex: "**/logs/**"
+    regex: "**/build/*.log"
     regex: "**/build/**"
     regex: "**/database/**"
     strip_prefix: "github/symbiflow-prjxray-db/"
diff --git a/.github/kokoro/presubmit-db-artix7.cfg b/.github/kokoro/presubmit-db-artix7.cfg
index a026e6e6..9c339d85 100644
--- a/.github/kokoro/presubmit-db-artix7.cfg
+++ b/.github/kokoro/presubmit-db-artix7.cfg
@@ -8,7 +8,7 @@ action {
   define_artifacts {
     regex: "**/*result*.xml"
     regex: "**/logs/**"
-    regex: "**/build/**"
+    regex: "**/build/*.log"
     regex: "**/database/artix7/**"
     strip_prefix: "github/symbiflow-prjxray-presubmit-db-artix7/"
   }
diff --git a/.github/kokoro/presubmit-db-kintex7.cfg b/.github/kokoro/presubmit-db-kintex7.cfg
index c9f90bc6..91d5444e 100644
--- a/.github/kokoro/presubmit-db-kintex7.cfg
+++ b/.github/kokoro/presubmit-db-kintex7.cfg
@@ -8,7 +8,7 @@ action {
   define_artifacts {
     regex: "**/*result*.xml"
     regex: "**/logs/**"
-    regex: "**/build/**"
+    regex: "**/build/*.log"
     regex: "**/database/kintex7/**"
     strip_prefix: "github/symbiflow-prjxray-presubmit-db-kintex7/"
   }
diff --git a/.github/kokoro/presubmit-db-zynq7.cfg b/.github/kokoro/presubmit-db-zynq7.cfg
index 176dec40..6a2c4355 100644
--- a/.github/kokoro/presubmit-db-zynq7.cfg
+++ b/.github/kokoro/presubmit-db-zynq7.cfg
@@ -8,7 +8,7 @@ action {
   define_artifacts {
     regex: "**/*result*.xml"
     regex: "**/logs/**"
-    regex: "**/build/**"
+    regex: "**/build/*.log"
     regex: "**/database/zynq7/**"
     strip_prefix: "github/symbiflow-prjxray-presubmit-db-zynq7/"
   }

From 2927f1c572923a0f53775af360b66b748be4163f Mon Sep 17 00:00:00 2001
From: Tim 'mithro' Ansell <me@mith.ro>
Date: Wed, 6 Feb 2019 18:27:34 +1100
Subject: [PATCH 24/28] Don't remove the database.

Why was the download-latest-db.sh doing a clone and then removing all
the stuff it just cloned?

Signed-off-by: Tim 'mithro' Ansell <me@mith.ro>
---
 download-latest-db.sh | 8 +-------
 1 file changed, 1 insertion(+), 7 deletions(-)

diff --git a/download-latest-db.sh b/download-latest-db.sh
index a01c44d1..8d0ce8aa 100755
--- a/download-latest-db.sh
+++ b/download-latest-db.sh
@@ -4,11 +4,5 @@ GITHUB_PROTO=${1:-https}
 GITHUB_URL=$GITHUB_PROTO://github.com/SymbiFlow/prjxray-db.git
 rm -rf database
 git clone $GITHUB_URL database
-# Causes confusion if you try to commit in DB dir
-# But doesn't effect most people, probably leave as is
-# rm -rf database/.git
-# travis, .gitignore, etc
-rm -f database/* 2>/dev/null
-rm -f database/.* 2>/dev/null
-# Restore settings files so fuzzers run correctly
+# Update files in the database from our version so fuzzers run correctly.
 git checkout HEAD database

From d34c210c1df2ba28ff0415d1595f3d9c147f0223 Mon Sep 17 00:00:00 2001
From: Tim 'mithro' Ansell <me@mith.ro>
Date: Wed, 6 Feb 2019 18:30:21 +1100
Subject: [PATCH 25/28] infra: Run database download in script.

Means the details about the download are captured.

Signed-off-by: Tim 'mithro' Ansell <me@mith.ro>
---
 .github/kokoro/db-full.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/kokoro/db-full.sh b/.github/kokoro/db-full.sh
index c64628b6..8f37db47 100755
--- a/.github/kokoro/db-full.sh
+++ b/.github/kokoro/db-full.sh
@@ -17,7 +17,7 @@ echo "========================================"
 echo "Downloading current database"
 echo "----------------------------------------"
 (
-	./download-latest-db.sh
+	script --return --flush --command "./download-latest-db.sh" -
 )
 echo "----------------------------------------"
 

From 910bc688c3470406ddb05d08484e5691fc88a0aa Mon Sep 17 00:00:00 2001
From: Tim 'mithro' Ansell <me@mith.ro>
Date: Wed, 6 Feb 2019 18:39:56 +1100
Subject: [PATCH 26/28] infra: Working on better diff output.

Signed-off-by: Tim 'mithro' Ansell <me@mith.ro>
---
 .github/kokoro/continuous-db-artix7.cfg  |  9 ++++++++-
 .github/kokoro/continuous-db-kintex7.cfg |  9 ++++++++-
 .github/kokoro/continuous-db-zynq7.cfg   |  7 +++++++
 .github/kokoro/database.cfg              |  1 +
 .github/kokoro/db-full.sh                | 20 ++++++++++++++++++--
 .github/kokoro/kokoro-cfg.py             | 13 +++++++++++--
 .github/kokoro/presubmit-db-artix7.cfg   |  9 ++++++++-
 .github/kokoro/presubmit-db-kintex7.cfg  |  9 ++++++++-
 .github/kokoro/presubmit-db-zynq7.cfg    |  9 ++++++++-
 .github/kokoro/steps/hostsetup.sh        |  9 +++++++++
 10 files changed, 86 insertions(+), 9 deletions(-)

diff --git a/.github/kokoro/continuous-db-artix7.cfg b/.github/kokoro/continuous-db-artix7.cfg
index 1c947f2a..6870b866 100644
--- a/.github/kokoro/continuous-db-artix7.cfg
+++ b/.github/kokoro/continuous-db-artix7.cfg
@@ -6,10 +6,17 @@ timeout_mins: 4320
 
 action {
   define_artifacts {
+    # File types
+    regex: "**/diff.html"
+    regex: "**/diff.json"
     regex: "**/*result*.xml"
+    regex: "**/*sponge_log.xml"
+    # Whole directories
+    # regex: "**/build/**" - Currently kokoro dies on number of artifacts.
     regex: "**/build/*.log"
     regex: "**/logs/**"
-    regex: "**/database/zynq7/**"
+    # The database
+    regex: "**/database/artix7/**"
     strip_prefix: "github/symbiflow-prjxray-continuous-db-artix7/"
   }
 }
diff --git a/.github/kokoro/continuous-db-kintex7.cfg b/.github/kokoro/continuous-db-kintex7.cfg
index 6beeda62..cad4fc27 100644
--- a/.github/kokoro/continuous-db-kintex7.cfg
+++ b/.github/kokoro/continuous-db-kintex7.cfg
@@ -6,10 +6,17 @@ timeout_mins: 4320
 
 action {
   define_artifacts {
+    # File types
+    regex: "**/diff.html"
+    regex: "**/diff.json"
     regex: "**/*result*.xml"
+    regex: "**/*sponge_log.xml"
+    # Whole directories
+    # regex: "**/build/**" - Currently kokoro dies on number of artifacts.
     regex: "**/build/*.log"
     regex: "**/logs/**"
-    regex: "**/database/zynq7/**"
+    # The database
+    regex: "**/database/kintex7/**"
     strip_prefix: "github/symbiflow-prjxray-continuous-db-kintex7/"
   }
 }
diff --git a/.github/kokoro/continuous-db-zynq7.cfg b/.github/kokoro/continuous-db-zynq7.cfg
index d744cc5f..ba23dabd 100644
--- a/.github/kokoro/continuous-db-zynq7.cfg
+++ b/.github/kokoro/continuous-db-zynq7.cfg
@@ -6,9 +6,16 @@ timeout_mins: 4320
 
 action {
   define_artifacts {
+    # File types
+    regex: "**/diff.html"
+    regex: "**/diff.json"
     regex: "**/*result*.xml"
+    regex: "**/*sponge_log.xml"
+    # Whole directories
+    # regex: "**/build/**" - Currently kokoro dies on number of artifacts.
     regex: "**/build/*.log"
     regex: "**/logs/**"
+    # The database
     regex: "**/database/zynq7/**"
     strip_prefix: "github/symbiflow-prjxray-continuous-db-zynq7/"
   }
diff --git a/.github/kokoro/database.cfg b/.github/kokoro/database.cfg
index 4dabb33f..c904160c 100644
--- a/.github/kokoro/database.cfg
+++ b/.github/kokoro/database.cfg
@@ -7,6 +7,7 @@ action {
     regex: "**/*result*.xml"
     regex: "**/build/*.log"
     regex: "**/build/**"
+    regex: "**/diff.*"
     regex: "**/database/**"
     strip_prefix: "github/symbiflow-prjxray-db/"
   }
diff --git a/.github/kokoro/db-full.sh b/.github/kokoro/db-full.sh
index 8f37db47..a2ee522d 100755
--- a/.github/kokoro/db-full.sh
+++ b/.github/kokoro/db-full.sh
@@ -84,14 +84,30 @@ echo "----------------------------------------"
 	echo " Database Status"
 	echo "----------------------------------------"
 	git status
+	git add *
 	echo "----------------------------------------"
 	echo
-	echo
+	echo "----------------------------------------"
+	echo " Database Diff Summary"
+	echo "----------------------------------------"
+	git diff --stat --irreversible-delete --find-renames --find-copies --ignore-all-space
 	echo
 	echo "----------------------------------------"
 	echo " Database Diff"
 	echo "----------------------------------------"
-	git diff
+	git diff --color --irreversible-delete --find-renames --find-copies --ignore-all-space
+	echo
+	echo "----------------------------------------"
+	echo " Generating pretty diff"
+	echo "----------------------------------------"
+	diff2html --summary=open --file diff.html --format html \
+		-- \
+		--irreversible-delete --find-renames --find-copies --ignore-all-space \
+		|| true
+	diff2html --file diff.json --format json \
+		-- \
+		--irreversible-delete --find-renames --find-copies --ignore-all-space \
+		|| true
 )
 echo "----------------------------------------"
 
diff --git a/.github/kokoro/kokoro-cfg.py b/.github/kokoro/kokoro-cfg.py
index a89b1b53..aa8a1a7e 100755
--- a/.github/kokoro/kokoro-cfg.py
+++ b/.github/kokoro/kokoro-cfg.py
@@ -9,9 +9,18 @@ timeout_mins: 4320
 
 action {
   define_artifacts {
+    # File types
+    regex: "**/diff.html"
+    regex: "**/diff.json"
     regex: "**/*result*.xml"
-    regex: "**/*.log"
-    regex: "database/%(part)s/**"
+    regex: "**/*sponge_log.xml"
+    # Whole directories
+    # regex: "**/build/**" - Currently kokoro dies on number of artifacts.
+    regex: "**/build/*.log"
+    regex: "**/logs/**"
+    # The database
+    regex: "**/database/%(part)s/**"
+    strip_prefix: "github/symbiflow-prjxray-%(kokoro_type)s-db-%(part)s/"
   }
 }
 
diff --git a/.github/kokoro/presubmit-db-artix7.cfg b/.github/kokoro/presubmit-db-artix7.cfg
index 9c339d85..acbb99ef 100644
--- a/.github/kokoro/presubmit-db-artix7.cfg
+++ b/.github/kokoro/presubmit-db-artix7.cfg
@@ -6,9 +6,16 @@ timeout_mins: 4320
 
 action {
   define_artifacts {
+    # File types
+    regex: "**/diff.html"
+    regex: "**/diff.json"
     regex: "**/*result*.xml"
-    regex: "**/logs/**"
+    regex: "**/*sponge_log.xml"
+    # Whole directories
+    # regex: "**/build/**" - Currently kokoro dies on number of artifacts.
     regex: "**/build/*.log"
+    regex: "**/logs/**"
+    # The database
     regex: "**/database/artix7/**"
     strip_prefix: "github/symbiflow-prjxray-presubmit-db-artix7/"
   }
diff --git a/.github/kokoro/presubmit-db-kintex7.cfg b/.github/kokoro/presubmit-db-kintex7.cfg
index 91d5444e..21b6b471 100644
--- a/.github/kokoro/presubmit-db-kintex7.cfg
+++ b/.github/kokoro/presubmit-db-kintex7.cfg
@@ -6,9 +6,16 @@ timeout_mins: 4320
 
 action {
   define_artifacts {
+    # File types
+    regex: "**/diff.html"
+    regex: "**/diff.json"
     regex: "**/*result*.xml"
-    regex: "**/logs/**"
+    regex: "**/*sponge_log.xml"
+    # Whole directories
+    # regex: "**/build/**" - Currently kokoro dies on number of artifacts.
     regex: "**/build/*.log"
+    regex: "**/logs/**"
+    # The database
     regex: "**/database/kintex7/**"
     strip_prefix: "github/symbiflow-prjxray-presubmit-db-kintex7/"
   }
diff --git a/.github/kokoro/presubmit-db-zynq7.cfg b/.github/kokoro/presubmit-db-zynq7.cfg
index 6a2c4355..1413a966 100644
--- a/.github/kokoro/presubmit-db-zynq7.cfg
+++ b/.github/kokoro/presubmit-db-zynq7.cfg
@@ -6,9 +6,16 @@ timeout_mins: 4320
 
 action {
   define_artifacts {
+    # File types
+    regex: "**/diff.html"
+    regex: "**/diff.json"
     regex: "**/*result*.xml"
-    regex: "**/logs/**"
+    regex: "**/*sponge_log.xml"
+    # Whole directories
+    # regex: "**/build/**" - Currently kokoro dies on number of artifacts.
     regex: "**/build/*.log"
+    regex: "**/logs/**"
+    # The database
     regex: "**/database/zynq7/**"
     strip_prefix: "github/symbiflow-prjxray-presubmit-db-zynq7/"
   }
diff --git a/.github/kokoro/steps/hostsetup.sh b/.github/kokoro/steps/hostsetup.sh
index b959cd21..26cf26bf 100755
--- a/.github/kokoro/steps/hostsetup.sh
+++ b/.github/kokoro/steps/hostsetup.sh
@@ -24,6 +24,7 @@ sudo apt-get install -y \
         fontconfig \
         git \
         jq \
+        nodejs \
         psmisc \
         python \
         python3 \
@@ -44,3 +45,11 @@ echo "----------------------------------------"
 )
 
 echo "----------------------------------------"
+
+echo
+echo "========================================"
+echo "Getting diff2html to produce pretty database diffs"
+echo "----------------------------------------"
+(
+	sudo npm install -g diff2html-cli
+)

From 20ca1ee3aaf16082a8a336ff53a95fee18eb5515 Mon Sep 17 00:00:00 2001
From: Tim 'mithro' Ansell <me@mith.ro>
Date: Thu, 7 Feb 2019 21:38:02 +1100
Subject: [PATCH 27/28] infra: Remove some stuff in the host setup.

Signed-off-by: Tim 'mithro' Ansell <me@mith.ro>
---
 .github/kokoro/steps/hostsetup.sh | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/.github/kokoro/steps/hostsetup.sh b/.github/kokoro/steps/hostsetup.sh
index 26cf26bf..35395950 100755
--- a/.github/kokoro/steps/hostsetup.sh
+++ b/.github/kokoro/steps/hostsetup.sh
@@ -9,6 +9,14 @@ echo "----------------------------------------"
 sudo apt-get update
 echo "----------------------------------------"
 
+echo
+echo "========================================"
+echo "Host remove packages"
+echo "----------------------------------------"
+sudo apt-get remove -y \
+	python-pytest \
+
+echo "----------------------------------------"
 echo
 echo "========================================"
 echo "Host install packages"

From 14642f64f80df4a9c7cf929ffac4ba1a55b1e7f8 Mon Sep 17 00:00:00 2001
From: Tim 'mithro' Ansell <me@mith.ro>
Date: Thu, 7 Feb 2019 21:41:13 +1100
Subject: [PATCH 28/28] Don't use system's site-packages.

Signed-off-by: Tim 'mithro' Ansell <me@mith.ro>
---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index ad8ffd87..875132ab 100644
--- a/Makefile
+++ b/Makefile
@@ -3,7 +3,7 @@ ALL_EXCLUDE = third_party .git env build
 # Tools + Environment
 IN_ENV = if [ -e env/bin/activate ]; then . env/bin/activate; fi;
 env:
-	virtualenv --python=python3 --system-site-packages env
+	virtualenv --python=python3 env
 	# Install prjxray
 	ln -sf $(PWD)/prjxray env/lib/python3.*/site-packages/
 	$(IN_ENV) python -c "import prjxray"