Merge pull request #62 from EGA-archive/feature/tls-client-verification

TLS client verification

.gitignore

@@ -10,6 +10,7 @@ tests/**/*.debug
 tests/**/*.d
 tests/_common/users.json
 tests/_common/backup
+tests/_common/mq/*.pem
 
 # =====================================
 # Byte-compiled / optimized / DLL files

.travis.yml

@@ -30,13 +30,12 @@ jobs:
       if: type = pull_request
       services: docker
       before_script:
-        - pip3 install -r requirements.txt
         - git clone https://github.com/bats-core/bats-core.git
         - pushd bats-core && git checkout v1.1.0 && sudo ./install.sh /usr/local && popd
         - cd deploy
         - make -C images
-        - make prepare
-        - make bootstrap ARGS="--keyserver ega"
+        - make install-bootstrap-packages prepare
+        - make bootstrap
         - make up
         - make preflight-check
       script:
@@ -47,13 +46,12 @@ jobs:
       if: type = pull_request
       services: docker
       before_script:
-        - pip3 install -r requirements.txt
         - git clone https://github.com/bats-core/bats-core.git
         - pushd bats-core && git checkout v1.1.0 && sudo ./install.sh /usr/local && popd
         - cd deploy
         - make -C images
-        - make prepare
-        - make bootstrap ARGS="--keyserver ega --archive-backend posix"
+        - make install-bootstrap-packages prepare
+        - make bootstrap ARGS='--archive-backend posix'
         - make up
         - make preflight-check
       script:
@@ -64,13 +62,12 @@ jobs:
       if: type = pull_request
       services: docker
       before_script:
-        - pip3 install -r requirements.txt
         - git clone https://github.com/bats-core/bats-core.git
         - pushd bats-core && git checkout v1.1.0 && sudo ./install.sh /usr/local && popd
         - cd deploy
         - make -C images
-        - make prepare
-        - make bootstrap ARGS="--keyserver ega"
+        - make install-bootstrap-packages prepare
+        - make bootstrap
         - make up
         - make preflight-check
       script:
@@ -81,49 +78,48 @@ jobs:
       if: type = pull_request
       services: docker
       before_script:
-        - pip3 install -r requirements.txt
         - git clone https://github.com/bats-core/bats-core.git
         - pushd bats-core && git checkout v1.1.0 && sudo ./install.sh /usr/local && popd
         - cd deploy
         - make -C images
-        - make prepare
-        - make bootstrap ARGS="--keyserver ega"
+        - make install-bootstrap-packages prepare
+        - make bootstrap
         - make up
         - make preflight-check
       script:
         - cd ../tests
         - bats security
-    #
-    # Cucumber Tests
-    #
-    - stage: tests
-      name: "Cucumber Ingestion Tests"
-      if: type = pull_request
-      before_script:
-        - pip3 install -r requirements.txt
-        - cd deploy
-        - make -C images
-        - make prepare
-        - make bootstrap ARGS='--inbox mina --keyserver ega'
-        - make up
-        - make preflight-check
-      script:
-        - cd tests
-        - mvn test -Dtest=IngestionTests
-    - stage: tests
-      name: "Cucumber Robustness Tests"
-      if: type = pull_request
-      before_script:
-        - pip3 install -r requirements.txt
-        - cd deploy
-        - make -C images
-        - make prepare
-        - make bootstrap ARGS='--inbox mina --keyserver ega'
-        - make up
-        - make preflight-check
-      script:
-        - cd tests
-        - mvn test -Dtest=RobustnessTests
+    # #
+    # # Cucumber Tests
+    # #
+    # - stage: tests
+    #   name: "Cucumber Ingestion Tests"
+    #   if: type = pull_request
+    #   before_script:
+    #     - pip3 install -r requirements.txt
+    #     - cd deploy
+    #     - make -C images
+    #     - make prepare
+    #     - make bootstrap ARGS='--inbox mina'
+    #     - make up
+    #     - make preflight-check
+    #   script:
+    #     - cd tests
+    #     - mvn test -Dtest=IngestionTests
+    # - stage: tests
+    #   name: "Cucumber Robustness Tests"
+    #   if: type = pull_request
+    #   before_script:
+    #     - pip3 install -r requirements.txt
+    #     - cd deploy
+    #     - make -C images
+    #     - make prepare
+    #     - make bootstrap ARGS='--inbox mina'
+    #     - make up
+    #     - make preflight-check
+    #   script:
+    #     - cd tests
+    #     - mvn test -Dtest=RobustnessTests
 
 
 notifications:

Dockerfile

@@ -35,12 +35,15 @@ COPY --from=BUILD /usr/local/bin/lega-cryptor /usr/local/bin/
 
 COPY --from=BUILD /usr/local/bin/ega-* /usr/local/bin/
 
-VOLUME /etc/ega
-
 RUN mkdir -p /ega/archive && \
     chgrp lega /ega/archive && \
     chmod 2770 /ega/archive
-
 VOLUME /ega/archive
 
+RUN mkdir -p /etc/ega && \
+    chgrp lega /etc/ega && \
+    chmod 2770 /etc/ega
+VOLUME /etc/ega
+
 USER lega
+

deploy/Makefile

@@ -2,14 +2,14 @@ SHELL := /bin/bash
 
 ARGS=
 
-.PHONY: help bootstrap private network up down clean ps clean-volumes clean-all prepare
+.PHONY: help bootstrap private network up down clean ps clean-volumes clean-all prepare images install-bootstrap-packages
 
 help:
 	@echo "Usage: make <target>\n"
 	@echo "where <target> is: 'bootstrap', 'up' 'ps', 'down', 'network' or 'clean'\n"
 
 # If DEPLOY_DEV is yes, we use dummy passwords
-bootstrap-dev: DEPLOY_DEV=yes
+bootstrap-dev: export DEPLOY_DEV=yes
 .env private/lega.yml private bootstrap bootstrap-dev:
 	@bootstrap/run.sh ${ARGS} || { cat private/.err; exit 1; }
 
@@ -29,23 +29,23 @@ down: #.env
 
 clean:
 	rm -rf .env private
+	make -C bootstrap/certs clean
 
 clean-all: clean clean-volumes
 
 preflight-check:
 	@echo "Ensure the system is up"
-	sleep 120
+	sleep 80
 
-# Note: "sleep 120" will be replaced with a script that checks for open port, mq settings, etc.
+# Note: "sleep 80" will be replaced with a script that checks for open port, mq settings, etc.
 # It might be a good idea to run it in the same network as the containers.
 # docker run --rm -it --network=lega_lega \
-#        -v ${PWD}/integration/ensure_system_up.sh:/ensure_system_up.sh \
-#        nbisweden/ega-base /ensure_system_up.sh
+#        -v ${PWD}/bootstrap/ensure_system_up.sh:/ensure_system_up.sh \
+#        egarchive/lega-base /ensure_system_up.sh
 
 # For the moment, we simply do sleep 60, because we need
-# - the inbox notification server up
 # - the rabbitmq shovel to CentralEGA (the federated queue can be late, it doesn't hurt)
-# - the key server (the java spring needs 97 seconds, 10 are enough for the other)
+# - the key server (slow setup with java spring)
 
 
 # Decrypt the encoded data from the repository
@@ -59,3 +59,10 @@ prepare: ../tests/_common/dummy.sec ../tests/_common/john.sec ../tests/_common/j
 %.json: %.json.enc
 	openssl enc -aes-256-cbc -d -in $< -out $@ -k ${REPO_PASSWORD}
 
+images:
+	make -C images
+
+# That's for Travis
+install-bootstrap-packages:
+	pip3 install git+https://github.com/NBISweden/LocalEGA-cryptor.git
+	pip3 install pgpy pika==1.0.1

deploy/bootstrap/certs/.gitignore

@@ -0,0 +1,2 @@
+data
+test

deploy/bootstrap/certs/LocalEGA.cnf

@@ -0,0 +1,120 @@
+# OpenSSL root LocalEGA configuration file.
+
+[ ca ]
+# `man ca`
+default_ca = CA_default
+
+[ CA_default ]
+# Directory and file locations.
+dir               = ./data
+certs             = $dir
+crl_dir           = $dir
+new_certs_dir     = $dir
+database          = $dir/index.txt
+serial            = $dir/serial
+RANDFILE          = $dir/rand
+
+# The root key and root certificate.
+private_key       = $dir/CA.sec.pem
+certificate       = $dir/CA.cert.pem
+
+# For certificate revocation lists.
+crlnumber         = $dir/crlnumber
+crl               = $dir/CA.crl.pem
+crl_extensions    = crl_ext
+default_crl_days  = 30
+
+# SHA-1 is deprecated, so use SHA-2 instead.
+default_md        = sha256
+
+name_opt          = ca_default
+cert_opt          = ca_default
+default_days      = 7300
+preserve          = no
+policy            = policy_strict
+
+[ policy_strict ]
+# The root CA should only sign intermediate certificates that match.
+# See the POLICY FORMAT section of `man ca`.
+countryName             = match
+stateOrProvinceName     = match
+organizationName        = match
+organizationalUnitName  = optional
+commonName              = supplied
+emailAddress            = optional
+
+[ policy_loose ]
+# Allow the intermediate CA to sign a more diverse range of certificates.
+# See the POLICY FORMAT section of the `ca` man page.
+countryName             = optional
+stateOrProvinceName     = optional
+localityName            = optional
+organizationName        = optional
+organizationalUnitName  = optional
+commonName              = supplied
+emailAddress            = optional
+
+[ req ]
+# Options for the `req` tool (`man req`).
+default_bits        = 2048
+distinguished_name  = req_distinguished_name
+string_mask         = utf8only
+
+# SHA-1 is deprecated, so use SHA-2 instead.
+default_md          = sha256
+
+# Extension to add when the -x509 option is used.
+x509_extensions     = v3_ca
+
+[ req_distinguished_name ]
+# See <https://en.wikipedia.org/wiki/Certificate_signing_request>.
+countryName                     = ES
+stateOrProvinceName             = Spain
+localityName                    = Barcelona
+0.organizationName              = CRG
+organizationalUnitName          = EGA
+commonName                      = LocalEGA root CA
+emailAddress                    = dev.ega@crg.eu
+
+[ v3_ca ]
+# Extensions for a typical CA (`man x509v3_config`).
+subjectKeyIdentifier = hash
+authorityKeyIdentifier = keyid:always,issuer
+basicConstraints = critical, CA:true
+keyUsage = critical, digitalSignature, cRLSign, keyCertSign
+#nsCertType = sslCA
+nsComment = "LocalEGA Root CA"
+
+[ server_client_cert ]
+# Extensions for server+client certificates (`man x509v3_config`).
+basicConstraints = CA:FALSE
+nsCertType = server,client
+nsComment = "LocalEGA Server+Client Certificate"
+subjectKeyIdentifier = hash
+authorityKeyIdentifier = keyid,issuer:always
+keyUsage = critical, nonRepudiation, digitalSignature, keyEncipherment
+extendedKeyUsage = critical, clientAuth, serverAuth
+
+[ client_cert ]
+# Extensions for client certificates (`man x509v3_config`).
+basicConstraints = CA:FALSE
+nsCertType = client
+nsComment = "LocalEGA Client Certificate"
+subjectKeyIdentifier = hash
+authorityKeyIdentifier = keyid,issuer:always
+keyUsage = critical, nonRepudiation, digitalSignature, keyEncipherment
+extendedKeyUsage = critical, clientAuth
+
+[ server_cert ]
+# Extensions for server certificates (`man x509v3_config`).
+basicConstraints = CA:FALSE
+nsCertType = server
+nsComment = "LocalEGA Server Certificate"
+subjectKeyIdentifier = hash
+authorityKeyIdentifier = keyid,issuer:always
+keyUsage = critical, nonRepudiation, digitalSignature, keyEncipherment
+extendedKeyUsage = critical, serverAuth
+
+[ crl_ext ]
+# Extension for CRLs (`man x509v3_config`).
+authorityKeyIdentifier=keyid:always