From d1ea210dba4f609ff5eec66640194cd69f8e06e3 Mon Sep 17 00:00:00 2001 From: Rick Altherr Date: Mon, 5 Feb 2018 17:51:20 -0800 Subject: [PATCH 1/6] Tweak .gitignore so database settings.sh is tracked Signed-off-by: Rick Altherr --- .gitignore | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 05e207ba..28f10758 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ /build/ .Xil -database/* +# Ignore database directories _except_ for their settings +database/*/* +!database/*/settings.sh From 7980e069dd18b8abc75de2959c475ad94a2dc2c0 Mon Sep 17 00:00:00 2001 From: Rick Altherr Date: Mon, 5 Feb 2018 17:51:32 -0800 Subject: [PATCH 2/6] Rename 'make go' to 'make build' Signed-off-by: Rick Altherr --- Makefile | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 297b666e..088c7b50 100644 --- a/Makefile +++ b/Makefile @@ -1,11 +1,9 @@ -JOBS ?= $(shell nproc) -JOBS ?= 2 CLANG_FORMAT ?= clang-format -go: +build: git submodule update --init --recursive mkdir -p build - cd build; cmake ..; make -j$(JOBS) + cd build; cmake ..; $(MAKE) format: find . -name \*.cc -and -not -path './third_party/*' -and -not -path './.git/*' -exec $(CLANG_FORMAT) -style=file -i {} \; From 47811acccdf3d74faa15fda4b095b466bfed25cd Mon Sep 17 00:00:00 2001 From: Rick Altherr Date: Mon, 5 Feb 2018 17:51:33 -0800 Subject: [PATCH 3/6] Top-level clean target Signed-off-by: Rick Altherr --- Makefile | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Makefile b/Makefile index 088c7b50..acc911e2 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,7 @@ CLANG_FORMAT ?= clang-format +.PHONY: format clean + build: git submodule update --init --recursive mkdir -p build @@ -9,3 +11,7 @@ format: find . -name \*.cc -and -not -path './third_party/*' -and -not -path './.git/*' -exec $(CLANG_FORMAT) -style=file -i {} \; find . -name \*.h -and -not -path './third_party/*' -and -not -path './.git/*' -exec $(CLANG_FORMAT) -style=file -i {} \; find . -name \*.py -and -not -path './third_party/*' -and -not -path './.git/*' -exec yapf -p -i {} \; + +clean: + $(MAKE) -C fuzzers clean + rm -rf build From 5d103c0498233e35773fdd2714ba706094a632f6 Mon Sep 17 00:00:00 2001 From: Rick Altherr Date: Mon, 5 Feb 2018 17:51:35 -0800 Subject: [PATCH 4/6] Makefile rules for generating complete databases Signed-off-by: Rick Altherr --- Makefile | 6 +++++- database/Makefile | 10 ++++++++++ database/Makefile.database | 15 +++++++++++++++ 3 files changed, 30 insertions(+), 1 deletion(-) create mode 100644 database/Makefile create mode 100644 database/Makefile.database diff --git a/Makefile b/Makefile index acc911e2..2028b5d9 100644 --- a/Makefile +++ b/Makefile @@ -1,17 +1,21 @@ CLANG_FORMAT ?= clang-format -.PHONY: format clean +.PHONY: database format clean build: git submodule update --init --recursive mkdir -p build cd build; cmake ..; $(MAKE) +database: build + $(MAKE) -C $@ + format: find . -name \*.cc -and -not -path './third_party/*' -and -not -path './.git/*' -exec $(CLANG_FORMAT) -style=file -i {} \; find . -name \*.h -and -not -path './third_party/*' -and -not -path './.git/*' -exec $(CLANG_FORMAT) -style=file -i {} \; find . -name \*.py -and -not -path './third_party/*' -and -not -path './.git/*' -exec yapf -p -i {} \; clean: + $(MAKE) -C database clean $(MAKE) -C fuzzers clean rm -rf build diff --git a/database/Makefile b/database/Makefile new file mode 100644 index 00000000..99fab407 --- /dev/null +++ b/database/Makefile @@ -0,0 +1,10 @@ +.NOTPARALLEL: + +SUBDIRS := $(patsubst %/,%, $(wildcard */)) + +.PHONY: $(SUBDIRS) + +$(MAKECMDGOALS): $(SUBDIRS) + +$(SUBDIRS): + $(MAKE) -C $@ -f ../Makefile.database $(MAKECMDGOALS) diff --git a/database/Makefile.database b/database/Makefile.database new file mode 100644 index 00000000..2e407968 --- /dev/null +++ b/database/Makefile.database @@ -0,0 +1,15 @@ +.PHONY: all clean + +all: tilegrid.json + +# Small dance to say that there is a single recipe that is run once to generate +# multiple files. +tileconn.json tilegrid.json: fuzzers +.INTERMEDIATE: fuzzers +fuzzers: SHELL:=/bin/bash +fuzzers: settings.sh + source settings.sh && $(MAKE) -C ../../fuzzers all + +clean: + rm -f *.db tileconn.json tilegrid.json *.yaml + rm -rf html From f4ddb607cb26cca034826669a05e2a6a12aa2b9a Mon Sep 17 00:00:00 2001 From: Rick Altherr Date: Mon, 5 Feb 2018 17:51:36 -0800 Subject: [PATCH 5/6] Dockerfile for building database from scratch Runs all fuzzers to construct a database from scratch. Raw and HTML versions of database are packaged into a docker image that serves the files via nginx. Signed-off-by: Rick Altherr --- Dockerfile | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 Dockerfile diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..a2544951 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,12 @@ +ARG DEV_ENV_IMAGE + +FROM ${DEV_ENV_IMAGE} AS db_builder +ARG NUM_PARALLEL_JOBS=1 + +COPY . /source +RUN cd /source && make -j${NUM_PARALLEL_JOBS} database +RUN find /source/database -mindepth 1 -maxdepth 1 -type d -exec /source/htmlgen/htmlgen.py --settings={}/settings.sh --output=/output/html \; +RUN mkdir -p /output/raw && find /source/database -mindepth 1 -maxdepth 1 -type d -exec cp -R {} /output/raw \; + +FROM nginx:alpine +COPY --from=db_builder /output /usr/share/nginx/html From 7b81b6734be5c16dd77bc772eb975409fa07b9b4 Mon Sep 17 00:00:00 2001 From: Rick Altherr Date: Mon, 5 Feb 2018 17:51:37 -0800 Subject: [PATCH 6/6] Google Cloud Builder configuration for database generation Signed-off-by: Rick Altherr --- .gcloudignore | 15 +++++++++++++++ cloudbuild.yaml | 21 +++++++++++++++++++++ 2 files changed, 36 insertions(+) create mode 100644 .gcloudignore create mode 100644 cloudbuild.yaml diff --git a/.gcloudignore b/.gcloudignore new file mode 100644 index 00000000..ef2ea2e4 --- /dev/null +++ b/.gcloudignore @@ -0,0 +1,15 @@ +# This file specifies files that are *not* uploaded to Google Cloud Platform +# using gcloud. It follows the same syntax as .gitignore, with the addition of +# "#!include" directives (which insert the entries of the given .gitignore-style +# file at that point). +# +# For more information, run: +# $ gcloud topic gcloudignore +# +.gcloudignore +#!include:.gitignore + +# Don't bother uploading files not needed to generate a database. +experiments +minitests +vagrant diff --git a/cloudbuild.yaml b/cloudbuild.yaml new file mode 100644 index 00000000..c3f12bcc --- /dev/null +++ b/cloudbuild.yaml @@ -0,0 +1,21 @@ +steps: +- name: 'gcr.io/cloud-builders/docker' + args: + - 'build' + - '--build-arg' + - 'DEV_ENV_IMAGE=${_DEV_ENV_IMAGE}' + - '--build-arg' + - 'NUM_PARALLEL_JOBS=${_NUM_CPUS}' + - '-t' + - '${_GCR_ZONE}/$PROJECT_ID/${_IMAGE_NAME}:${TAG_NAME}' + - '.' +options: + disk_size_gb: 1000 + machine_type: N1_HIGHCPU_32 +timeout: '21600s' +substitutions: + _GCR_ZONE: 'gcr.io' + _IMAGE_NAME: 'prjxray-db' + _NUM_CPUS: '16' +images: + - '${_GCR_ZONE}/$PROJECT_ID/${_IMAGE_NAME}:${TAG_NAME}'