Merge pull request #78 from kc8apf/docker_db_builder

Docker-based db builder

.gcloudignore

@@ -0,0 +1,15 @@
+# This file specifies files that are *not* uploaded to Google Cloud Platform
+# using gcloud. It follows the same syntax as .gitignore, with the addition of
+# "#!include" directives (which insert the entries of the given .gitignore-style
+# file at that point).
+#
+# For more information, run:
+#   $ gcloud topic gcloudignore
+#
+.gcloudignore
+#!include:.gitignore
+
+# Don't bother uploading files not needed to generate a database.
+experiments
+minitests
+vagrant

.gitignore

@@ -1,3 +1,5 @@
 /build/
 .Xil
-database/*
+# Ignore database directories _except_ for their settings
+database/*/*
+!database/*/settings.sh

Dockerfile

@@ -0,0 +1,12 @@
+ARG DEV_ENV_IMAGE
+
+FROM ${DEV_ENV_IMAGE} AS db_builder
+ARG NUM_PARALLEL_JOBS=1
+
+COPY . /source
+RUN cd /source && make -j${NUM_PARALLEL_JOBS} database
+RUN find /source/database -mindepth 1 -maxdepth 1 -type d -exec /source/htmlgen/htmlgen.py --settings={}/settings.sh --output=/output/html \;
+RUN mkdir -p /output/raw && find /source/database -mindepth 1 -maxdepth 1 -type d -exec cp -R {} /output/raw \;
+
+FROM nginx:alpine
+COPY --from=db_builder /output /usr/share/nginx/html

Makefile

@@ -1,13 +1,21 @@
-JOBS ?= $(shell nproc)
-JOBS ?= 2
 CLANG_FORMAT ?= clang-format
 
-go:
+.PHONY: database format clean
+
+build:
 	git submodule update --init --recursive
 	mkdir -p build
-	cd build; cmake ..; make -j$(JOBS)
+	cd build; cmake ..; $(MAKE)
+
+database: build
+	$(MAKE) -C $@
 
 format:
 	find . -name \*.cc -and -not -path './third_party/*' -and -not -path './.git/*' -exec $(CLANG_FORMAT) -style=file -i {} \;
 	find . -name \*.h -and -not -path './third_party/*' -and -not -path './.git/*' -exec $(CLANG_FORMAT) -style=file -i {} \;
 	find . -name \*.py -and -not -path './third_party/*' -and -not -path './.git/*' -exec yapf -p -i {} \;
+
+clean:
+	$(MAKE) -C database clean
+	$(MAKE) -C fuzzers clean
+	rm -rf build

cloudbuild.yaml

@@ -0,0 +1,21 @@
+steps:
+- name: 'gcr.io/cloud-builders/docker'
+  args:
+    - 'build'
+    - '--build-arg'
+    - 'DEV_ENV_IMAGE=${_DEV_ENV_IMAGE}'
+    - '--build-arg'
+    - 'NUM_PARALLEL_JOBS=${_NUM_CPUS}'
+    - '-t'
+    - '${_GCR_ZONE}/$PROJECT_ID/${_IMAGE_NAME}:${TAG_NAME}'
+    - '.'
+options:
+  disk_size_gb: 1000
+  machine_type: N1_HIGHCPU_32
+timeout: '21600s'
+substitutions:
+  _GCR_ZONE: 'gcr.io'
+  _IMAGE_NAME: 'prjxray-db'
+  _NUM_CPUS: '16'
+images:
+  - '${_GCR_ZONE}/$PROJECT_ID/${_IMAGE_NAME}:${TAG_NAME}'

database/Makefile

@@ -0,0 +1,10 @@
+.NOTPARALLEL:
+
+SUBDIRS := $(patsubst %/,%, $(wildcard */))
+
+.PHONY: $(SUBDIRS)
+
+$(MAKECMDGOALS): $(SUBDIRS)
+
+$(SUBDIRS):
+	$(MAKE) -C $@ -f ../Makefile.database $(MAKECMDGOALS)

database/Makefile.database

@@ -0,0 +1,15 @@
+.PHONY: all clean
+
+all: tilegrid.json
+
+# Small dance to say that there is a single recipe that is run once to generate
+# multiple files.
+tileconn.json tilegrid.json: fuzzers
+.INTERMEDIATE: fuzzers
+fuzzers: SHELL:=/bin/bash
+fuzzers: settings.sh
+	source settings.sh && $(MAKE) -C ../../fuzzers all
+
+clean:
+	rm -f *.db tileconn.json tilegrid.json *.yaml
+	rm -rf html