Docker reorganization + Continuous Integration (#84)

Continuous integration will run on every PR (and it should work)!

Pre-requisites for adding dependencies

* Docker
* Gcloud
* Gcloud setup with o1labs

How to add a new dependency:

1. Add the dependency to base-Dockerfile (like opam install whatever)
2. Run make update-deps
3. Commit all the changes and push!
4. Changes that don't require new dependencies, don't require any extra work on the programmer.

.circleci/config.yml

@@ -0,0 +1,25 @@
+version: 2
+jobs:
+  build:
+    docker:
+      - image: o1labs/ci-base:latest
+    steps:
+      - checkout
+
+      - setup_remote_docker
+
+      - run: echo $GCLOUD_SERVICE_KEY | base64 --decode --ignore-garbage > ${HOME}/gcloud-service-key.json
+
+      - run: /root/google-cloud-sdk/bin/gcloud --quiet components update
+      - run: /root/google-cloud-sdk/bin/gcloud auth activate-service-account --key-file=${HOME}/gcloud-service-key.json
+      - run: /root/google-cloud-sdk/bin/gcloud config set project o1labs-192920
+
+      - run: /root/google-cloud-sdk/bin/gcloud docker -- pull gcr.io/o1labs-192920/ocaml-base:c239fa619a9a3bcba0cfd3b83b6256410bdb3afa
+
+      - run:
+          name: Nanobit docker
+          command: make nanobit-docker
+
+      - run:
+          name: Testbridge docker
+          command: make testbridge-docker

.gitmodules

@@ -1,3 +0,0 @@
-[submodule "lib/camlsnark/src/camlsnark_c/libsnark-caml"]
-	path = lib/camlsnark/src/camlsnark_c/libsnark-caml
-	url = git@github.com:o1-labs/libsnark.git

Dockerfile

@@ -1,7 +1,4 @@
-FROM ocaml-camlsnark:latest
-
-RUN opam install merlin
-RUN opam install ocp-indent
+FROM gcr.io/o1labs-192920/ocaml-base:d6eecbf879e7badc4faa83f1b3c8421abd146621
 
 ENV PATH "/home/opam/.opam/4.05.0/bin:$PATH"
 ENV CAML_LD_LIBRARY_PATH "/home/opam/.opam/4.05.0/lib/stublibs"

Makefile

@@ -12,7 +12,6 @@ dev : docker
 	@echo "** Talk to bkase to help you setup your vimrc"
 	@echo "****"
 
-
 nanobit-docker :
 	./rebuild-docker.sh nanobit nanobit-Dockerfile
 
@@ -38,4 +37,13 @@ base-minikube :
 	./rebuild-minikube.sh ocaml-base base-Dockerfile
 
 base-googlecloud :
-	./rebuild-googlecloud.sh ocaml-base base-Dockerfile
+	./rebuild-googlecloud.sh ocaml-base base-Dockerfile $(shell git rev-parse HEAD)
+
+ocaml405-googlecloud:
+	./rebuild-googlecloud.sh ocaml405 ocaml405-Dockerfile
+
+ci-base-docker:
+	./rebuild-docker.sh o1labs/ci-base ci-base-Dockerfile
+
+update-deps: base-googlecloud
+	./rewrite-from-dockerfile.sh ocaml-base $(shell git rev-parse HEAD)

base-Dockerfile

@@ -1,39 +1,22 @@
-# OPAM for debian-stable with local switch of OCaml 4.05.0
-# Autogenerated by OCaml-Dockerfile scripts
-FROM ocaml/ocaml:debian-stable
-LABEL distro_style="apt" distro="debian" distro_long="debian-stable" arch="x86_64" ocaml_version="4.05.0" opam_version="1.2" operatingsystem="linux"
-RUN apt-get -y update && \
-  DEBIAN_FRONTEND=noninteractive apt-get -y upgrade && \
-  DEBIAN_FRONTEND=noninteractive apt-get -y install aspcud && \
-  git clone -b 1.2 git://github.com/ocaml/opam /tmp/opam && \
-  sh -c "cd /tmp/opam && make cold && make install && echo Not installing OPAM2 wrappers && rm -rf /tmp/opam" && \
-  echo 'opam ALL=(ALL:ALL) NOPASSWD:ALL' > /etc/sudoers.d/opam && \
-  chmod 440 /etc/sudoers.d/opam && \
-  chown root:root /etc/sudoers.d/opam && \
-  adduser --disabled-password --gecos '' opam && \
-  passwd -l opam && \
-  chown -R opam:opam /home/opam
-USER opam
-ENV HOME /home/opam
-WORKDIR /home/opam
-RUN mkdir .ssh && \
-  chmod 700 .ssh && \
-  git config --global user.email "docker@example.com" && \
-  git config --global user.name "Docker CI" && \
-  sudo -u opam sh -c "git clone -b master git://github.com/ocaml/opam-repository" && \
-  sudo -u opam sh -c "opam init -a -y --comp 4.05.0 /home/opam/opam-repository" && \
-  sudo -u opam sh -c "opam install -y depext travis-opam"
+FROM gcr.io/o1labs-192920/ocaml405:latest
+
+RUN sudo apt-get install --yes cmake
+RUN sudo apt-get install --yes libssl-dev
+RUN sudo apt-get install --yes libffi-dev
+RUN sudo apt-get install --yes libprocps-dev
+RUN sudo apt-get install --yes libgmp-dev
+RUN sudo apt-get install --yes libgmp3-dev
+RUN sudo apt-get install --yes libboost-dev
+RUN sudo apt-get install --yes libboost-program-options-dev
 
 RUN opam depext -i jbuilder
 RUN opam install core async
 
-RUN DEBIAN_FRONTEND=noninteractive sudo apt-get -y install libssl-dev
-RUN DEBIAN_FRONTEND=noninteractive sudo apt-get -y install libffi-dev
-
-RUN opam install yojson menhir 
+RUN opam install merlin ocp-indent
+RUN opam install ctypes-foreign
+RUN opam install ctypes
+RUN opam install jbuilder
+RUN opam install yojson menhir
 RUN opam install async_ssl cohttp cohttp-async
-
-RUN DEBIAN_FRONTEND=noninteractive sudo apt-get -y install libgmp3-dev
-
 RUN opam install bignum
 

ci-base-Dockerfile

@@ -0,0 +1,6 @@
+FROM python:3.6.4-stretch
+
+RUN curl -L -o /tmp/docker-17.03.0-ce.tgz https://download.docker.com/linux/static/stable/x86_64/docker-17.03.0-ce.tgz && tar -xz -C /tmp -f /tmp/docker-17.03.0-ce.tgz && mv /tmp/docker/* /usr/bin
+
+RUN CLOUDSDK_CORE_DISABLE_PROMPTS=1 curl https://sdk.cloud.google.com | bash
+

lib/camlsnark/Dockerfile

@@ -1,9 +1,12 @@
 FROM ocaml-base:latest
 
+
 RUN sudo apt-get install libffi-dev
 RUN opam install ctypes-foreign
 RUN opam install ctypes
 RUN opam install jbuilder
+RUN opam install merlin
+RUN opam install ocp-indent
 RUN sudo apt-get install --yes cmake
 RUN sudo apt-get install --yes libssl-dev
 RUN sudo apt-get install --yes libprocps-dev

lib/camlsnark/src/camlsnark_c/libsnark-caml/.gitignore

@@ -0,0 +1,51 @@
+*.o
+*.a
+*.so
+*.d
+depinst/
+depsrc/
+README.html
+doxygen/
+libsnark/gadgetlib2/examples/tutorial
+libsnark/gadgetlib2/tests/gadgetlib2_test
+
+libsnark/algebra/curves/tests/test_bilinearity
+libsnark/algebra/curves/tests/test_groups
+libsnark/algebra/fields/tests/test_fields
+libsnark/common/routing_algorithms/profiling/profile_routing_algorithms
+libsnark/common/routing_algorithms/tests/test_routing_algorithms
+libsnark/gadgetlib1/gadgets/cpu_checkers/fooram/examples/test_fooram
+libsnark/gadgetlib1/gadgets/hashes/knapsack/tests/test_knapsack_gadget
+libsnark/gadgetlib1/gadgets/hashes/sha256/tests/test_sha256_gadget
+libsnark/gadgetlib1/gadgets/merkle_tree/tests/test_merkle_tree_gadgets
+libsnark/gadgetlib1/gadgets/routing/profiling/profile_routing_gadgets
+libsnark/gadgetlib1/gadgets/set_commitment/tests/test_set_commitment_gadget
+libsnark/gadgetlib1/gadgets/verifiers/tests/test_r1cs_ppzksnark_verifier_gadget
+libsnark/reductions/ram_to_r1cs/examples/demo_arithmetization
+libsnark/relations/arithmetic_programs/qap/tests/test_qap
+libsnark/relations/arithmetic_programs/ssp/tests/test_ssp
+libsnark/zk_proof_systems/pcd/r1cs_pcd/r1cs_mp_ppzkpcd/profiling/profile_r1cs_mp_ppzkpcd
+libsnark/zk_proof_systems/pcd/r1cs_pcd/r1cs_mp_ppzkpcd/tests/test_r1cs_mp_ppzkpcd
+libsnark/zk_proof_systems/pcd/r1cs_pcd/r1cs_sp_ppzkpcd/profiling/profile_r1cs_sp_ppzkpcd
+libsnark/zk_proof_systems/pcd/r1cs_pcd/r1cs_sp_ppzkpcd/tests/test_r1cs_sp_ppzkpcd
+libsnark/zk_proof_systems/ppzkadsnark/r1cs_ppzkadsnark/examples/demo_r1cs_ppzkadsnark
+libsnark/zk_proof_systems/ppzksnark/bacs_ppzksnark/profiling/profile_bacs_ppzksnark
+libsnark/zk_proof_systems/ppzksnark/bacs_ppzksnark/tests/test_bacs_ppzksnark
+libsnark/zk_proof_systems/ppzksnark/r1cs_gg_ppzksnark/profiling/profile_r1cs_gg_ppzksnark
+libsnark/zk_proof_systems/ppzksnark/r1cs_gg_ppzksnark/tests/test_r1cs_gg_ppzksnark
+libsnark/zk_proof_systems/ppzksnark/r1cs_ppzksnark/profiling/profile_r1cs_ppzksnark
+libsnark/zk_proof_systems/ppzksnark/r1cs_ppzksnark/tests/test_r1cs_ppzksnark
+libsnark/zk_proof_systems/ppzksnark/ram_ppzksnark/examples/demo_ram_ppzksnark
+libsnark/zk_proof_systems/ppzksnark/ram_ppzksnark/examples/demo_ram_ppzksnark_generator
+libsnark/zk_proof_systems/ppzksnark/ram_ppzksnark/examples/demo_ram_ppzksnark_prover
+libsnark/zk_proof_systems/ppzksnark/ram_ppzksnark/examples/demo_ram_ppzksnark_verifier
+libsnark/zk_proof_systems/ppzksnark/ram_ppzksnark/profiling/profile_ram_ppzksnark
+libsnark/zk_proof_systems/ppzksnark/ram_ppzksnark/tests/test_ram_ppzksnark
+libsnark/zk_proof_systems/ppzksnark/tbcs_ppzksnark/profiling/profile_tbcs_ppzksnark
+libsnark/zk_proof_systems/ppzksnark/tbcs_ppzksnark/tests/test_tbcs_ppzksnark
+libsnark/zk_proof_systems/ppzksnark/uscs_ppzksnark/profiling/profile_uscs_ppzksnark
+libsnark/zk_proof_systems/ppzksnark/uscs_ppzksnark/tests/test_uscs_ppzksnark
+libsnark/zk_proof_systems/zksnark/ram_zksnark/profiling/profile_ram_zksnark
+libsnark/zk_proof_systems/zksnark/ram_zksnark/tests/test_ram_zksnark
+
+build

lib/camlsnark/src/camlsnark_c/libsnark-caml/.travis.yml

@@ -0,0 +1,33 @@
+language: cpp
+sudo: required
+dist: trusty
+compiler:
+  - gcc
+os:
+  - linux
+
+addons:
+  apt:
+    sources:
+      - ubuntu-toolchain-r-test
+    packages:
+      - gcc-5
+      - g++-5
+
+before_install:
+  - sudo apt-get install build-essential cmake git libboost-all-dev cmake libgmp3-dev libssl-dev libprocps3-dev pkg-config gnuplot-x11 python-markdown
+
+before_script:
+  - git submodule init && git submodule update
+  - mkdir build
+  - cd build
+  - cmake ..
+
+script:
+  - make
+  # TODO (howardwu): Resolve runtime on targets:
+  # gadgetlib1_fooram_test, zk_proof_systems_r1cs_mp_ppzkpcd_test, zk_proof_systems_r1cs_sp_ppzkpcd_test, zk_proof_systems_ram_zksnark_test
+  - make check
+
+notifications:
+  email: false

lib/camlsnark/src/camlsnark_c/libsnark-caml/AUTHORS

@@ -0,0 +1,20 @@
+SCIPR Lab:
+    Eli Ben-Sasson
+    Alessandro Chiesa
+    Daniel Genkin
+    Shaul Kfir
+    Eran Tromer
+    Madars Virza
+    Howard Wu
+
+External contributors:
+    Michael Backes
+    Manuel Barbosa
+    Alexander Chernyakhovsky (Google Inc.)
+    Dario Fiore
+    Jens Groth
+    Joshua A. Kroll
+    Shigeo MITSUNARI
+    Aleksejs Popovs
+    Raphael Reischuk
+    Tadanori TERUYA