infra: Working on better diff output.

Signed-off-by: Tim 'mithro' Ansell <me@mith.ro>

.github/kokoro/continuous-db-artix7.cfg

@@ -6,10 +6,17 @@ timeout_mins: 4320
 
 action {
   define_artifacts {
+    # File types
+    regex: "**/diff.html"
+    regex: "**/diff.json"
     regex: "**/*result*.xml"
+    regex: "**/*sponge_log.xml"
+    # Whole directories
+    # regex: "**/build/**" - Currently kokoro dies on number of artifacts.
     regex: "**/build/*.log"
     regex: "**/logs/**"
-    regex: "**/database/zynq7/**"
+    # The database
+    regex: "**/database/artix7/**"
     strip_prefix: "github/symbiflow-prjxray-continuous-db-artix7/"
   }
 }

.github/kokoro/continuous-db-kintex7.cfg

@@ -6,10 +6,17 @@ timeout_mins: 4320
 
 action {
   define_artifacts {
+    # File types
+    regex: "**/diff.html"
+    regex: "**/diff.json"
     regex: "**/*result*.xml"
+    regex: "**/*sponge_log.xml"
+    # Whole directories
+    # regex: "**/build/**" - Currently kokoro dies on number of artifacts.
     regex: "**/build/*.log"
     regex: "**/logs/**"
-    regex: "**/database/zynq7/**"
+    # The database
+    regex: "**/database/kintex7/**"
     strip_prefix: "github/symbiflow-prjxray-continuous-db-kintex7/"
   }
 }

.github/kokoro/continuous-db-zynq7.cfg

@@ -6,9 +6,16 @@ timeout_mins: 4320
 
 action {
   define_artifacts {
+    # File types
+    regex: "**/diff.html"
+    regex: "**/diff.json"
     regex: "**/*result*.xml"
+    regex: "**/*sponge_log.xml"
+    # Whole directories
+    # regex: "**/build/**" - Currently kokoro dies on number of artifacts.
     regex: "**/build/*.log"
     regex: "**/logs/**"
+    # The database
     regex: "**/database/zynq7/**"
     strip_prefix: "github/symbiflow-prjxray-continuous-db-zynq7/"
   }

.github/kokoro/database.cfg

@@ -7,6 +7,7 @@ action {
     regex: "**/*result*.xml"
     regex: "**/build/*.log"
     regex: "**/build/**"
+    regex: "**/diff.*"
     regex: "**/database/**"
     strip_prefix: "github/symbiflow-prjxray-db/"
   }

.github/kokoro/db-full.sh

@@ -84,14 +84,30 @@ echo "----------------------------------------"
 	echo " Database Status"
 	echo "----------------------------------------"
 	git status
+	git add *
 	echo "----------------------------------------"
 	echo
-	echo
+	echo "----------------------------------------"
+	echo " Database Diff Summary"
+	echo "----------------------------------------"
+	git diff --stat --irreversible-delete --find-renames --find-copies --ignore-all-space
 	echo
 	echo "----------------------------------------"
 	echo " Database Diff"
 	echo "----------------------------------------"
-	git diff
+	git diff --color --irreversible-delete --find-renames --find-copies --ignore-all-space
+	echo
+	echo "----------------------------------------"
+	echo " Generating pretty diff"
+	echo "----------------------------------------"
+	diff2html --summary=open --file diff.html --format html \
+		-- \
+		--irreversible-delete --find-renames --find-copies --ignore-all-space \
+		|| true
+	diff2html --file diff.json --format json \
+		-- \
+		--irreversible-delete --find-renames --find-copies --ignore-all-space \
+		|| true
 )
 echo "----------------------------------------"
 

.github/kokoro/kokoro-cfg.py

@@ -9,9 +9,18 @@ timeout_mins: 4320
 
 action {
   define_artifacts {
+    # File types
+    regex: "**/diff.html"
+    regex: "**/diff.json"
     regex: "**/*result*.xml"
-    regex: "**/*.log"
-    regex: "database/%(part)s/**"
+    regex: "**/*sponge_log.xml"
+    # Whole directories
+    # regex: "**/build/**" - Currently kokoro dies on number of artifacts.
+    regex: "**/build/*.log"
+    regex: "**/logs/**"
+    # The database
+    regex: "**/database/%(part)s/**"
+    strip_prefix: "github/symbiflow-prjxray-%(kokoro_type)s-db-%(part)s/"
   }
 }
 

.github/kokoro/presubmit-db-artix7.cfg

@@ -6,9 +6,16 @@ timeout_mins: 4320
 
 action {
   define_artifacts {
+    # File types
+    regex: "**/diff.html"
+    regex: "**/diff.json"
     regex: "**/*result*.xml"
-    regex: "**/logs/**"
+    regex: "**/*sponge_log.xml"
+    # Whole directories
+    # regex: "**/build/**" - Currently kokoro dies on number of artifacts.
     regex: "**/build/*.log"
+    regex: "**/logs/**"
+    # The database
     regex: "**/database/artix7/**"
     strip_prefix: "github/symbiflow-prjxray-presubmit-db-artix7/"
   }

.github/kokoro/presubmit-db-kintex7.cfg

@@ -6,9 +6,16 @@ timeout_mins: 4320
 
 action {
   define_artifacts {
+    # File types
+    regex: "**/diff.html"
+    regex: "**/diff.json"
     regex: "**/*result*.xml"
-    regex: "**/logs/**"
+    regex: "**/*sponge_log.xml"
+    # Whole directories
+    # regex: "**/build/**" - Currently kokoro dies on number of artifacts.
     regex: "**/build/*.log"
+    regex: "**/logs/**"
+    # The database
     regex: "**/database/kintex7/**"
     strip_prefix: "github/symbiflow-prjxray-presubmit-db-kintex7/"
   }

.github/kokoro/presubmit-db-zynq7.cfg

@@ -6,9 +6,16 @@ timeout_mins: 4320
 
 action {
   define_artifacts {
+    # File types
+    regex: "**/diff.html"
+    regex: "**/diff.json"
     regex: "**/*result*.xml"
-    regex: "**/logs/**"
+    regex: "**/*sponge_log.xml"
+    # Whole directories
+    # regex: "**/build/**" - Currently kokoro dies on number of artifacts.
     regex: "**/build/*.log"
+    regex: "**/logs/**"
+    # The database
     regex: "**/database/zynq7/**"
     strip_prefix: "github/symbiflow-prjxray-presubmit-db-zynq7/"
   }

.github/kokoro/steps/hostsetup.sh

@@ -24,6 +24,7 @@ sudo apt-get install -y \
         fontconfig \
         git \
         jq \
+        nodejs \
         psmisc \
         python \
         python3 \
@@ -44,3 +45,11 @@ echo "----------------------------------------"
 )
 
 echo "----------------------------------------"
+
+echo
+echo "========================================"
+echo "Getting diff2html to produce pretty database diffs"
+echo "----------------------------------------"
+(
+	sudo npm install -g diff2html-cli
+)