Merge pull request #100 from EGA-archive/pipeline-components

Refactoring pipeline components

.github/workflows/testsuite.yml

@@ -27,9 +27,9 @@ jobs:
     - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@master
       with:
-        python-version: 3.6
+        python-version: 3.7
     - name: (re)Build docker image (with Runner id $(id -u))
-      run: make -C deploy image
+      run: make -C deploy -j 4 images
     - name: Install required tools
       run: |
         git clone https://github.com/bats-core/bats-core.git

README.md

@@ -5,7 +5,7 @@
 [![Testsuite](https://github.com/EGA-archive/LocalEGA/workflows/Testsuite/badge.svg)](https://github.com/EGA-archive/LocalEGA/actions)
 
 
-The [code](lega) is written in Python (3.6+).
+The [code](lega) is written in Python (3.7+).
 
 You can provision and deploy the different components, locally, using [docker-compose](deploy).
 

deploy/Makefile

@@ -4,9 +4,9 @@ SELECTED=
 HOSTNAME_DOMAIN=
 #HOSTNAME_DOMAIN=.localega
 
-.PHONY: up down clean ps clean-volumes clean-all image erase purge preflight-check
+.PHONY: up down clean ps clean-volumes clean-all images erase purge preflight-check
 
-up: .env
+up: .env private/lega.yml
 	@docker-compose up -d ${SELECTED}
 
 private/lega.yml .env:
@@ -20,7 +20,8 @@ ps:
 	@docker-compose ps
 
 down:
-	@[[ -f private/lega.yml ]] && docker-compose down -v || echo -e "No recipe to bring containers down\nHave you bootstrapped? (ie make bootstrap)"
+	@[[ -f private/lega.yml ]] && docker-compose down -v \
+	 || echo -e "No recipe to bring containers down\nHave you bootstrapped? (ie make bootstrap)"
 
 preflight-check:
 	@echo "Ensure the system is up"
@@ -31,7 +32,7 @@ preflight-check:
 # docker run --rm -it --network=lega_lega \
 #        -v ${PWD}/bootstrap/ensure_system_up.sh:/ensure_system_up.sh \
 #        egarchive/lega-base /ensure_system_up.sh
-
+#
 # For the moment, we simply do sleep 20, because we need
 # the rabbitmq shovel to CentralEGA (the federated queue can be late, it doesn't hurt)
 
@@ -40,21 +41,39 @@ logs:
 # 	@docker-compose logs -f logs
 
 ####################################################
-## Base Image
+## Docker Images
 ####################################################
 
 IMAGE_ARGS=
-# eg --no-cache
-# eg --build-arg LEGA_GID=1000
+MQ_ARGS=
+DB_ARGS=
+# eg --no-cache, or --build-arg LEGA_GID=1000
+
+.PHONY: images image mq-image db-image
+images: image mq-image db-image
 
 image:
-	cd .. && \
+	cd ../ingestion && \
 	docker build -f Dockerfile \
 		     --build-arg LEGA_UID=$(shell id -u) \
 		     $(IMAGE_ARGS) \
 		     --tag egarchive/lega-base:latest \
 		     .
 
+mq-image:
+	cd ../ingestion/mq && \
+	docker build -f Dockerfile \
+		     $(MQ_ARGS) \
+		     --tag egarchive/lega-mq:latest \
+		     .
+
+db-image:
+	cd ../ingestion/db && \
+	docker build -f Dockerfile \
+		     $(DB_ARGS) \
+		     --tag egarchive/lega-db:latest \
+		     .
+
 
 ####################################################
 # Cleaning docker images
@@ -64,7 +83,7 @@ define remove_dangling
 endef
 
 erase:
-	@$(call remove_dangling,egarchive/lega-base)
+	@$(call remove_dangling,egarchive/lega-*)
 
 purge:
 	@$(call remove_dangling,)

deploy/bootstrap/Makefile

@@ -36,9 +36,8 @@ endif
 .PHONY: all users certs clean clean-all
 .DELETE_ON_ERROR: ../private/*
 
-SECRETS:= master.key.passphrase \
-	  db.lega.in db.lega.out \
-	  mq
+SECRETS:= master.key.passphrase db.lega mq.admin
+
 ifdef S3
 SECRETS+= s3.access s3.secret
 endif

deploy/bootstrap/run/lega/conf.py

@@ -50,7 +50,7 @@ def get_secret(s):
     #### Local Message Broker
     #################################################
     mq_user = 'admin'
-    mq_password = get_secret('mq')
+    mq_password = get_secret('mq.admin')
 
     # Pika is not parsing the URL the way RabbitMQ likes.
     # So we add the parameters on the configuration file and
@@ -78,19 +78,18 @@ def get_secret(s):
     #### Local Database
     #################################################
 
-    db_lega_in_pwd = get_secret('db.lega.in')
-    db_lega_out_pwd = get_secret('db.lega.out')
+    db_lega_pwd = get_secret('db.lega')
     db_connection_params = urlencode({ 'application_name': 'LocalEGA',
 				       'sslmode': 'verify-full',
 				       'sslcert': '/etc/ega/ssl.cert',
 				       'sslkey': '/etc/ega/ssl.key.lega',
 				       'sslrootcert': '/etc/ega/CA.cert',
     }, safe='/-_.')
-    db_connection=f"postgres://lega_in:{db_lega_in_pwd}@db{HOSTNAME_DOMAIN}:5432/lega"
+    db_connection=f"postgres://lega:{db_lega_pwd}@db{HOSTNAME_DOMAIN}:5432/lega"
 
     config['db'] = {
-        'lega_in': db_lega_in_pwd,
-        'lega_out': db_lega_out_pwd,
+        'user': 'lega',
+        'password': db_lega_pwd,
         'connection': db_connection,
         'connection_params': db_connection_params,
     }

deploy/bootstrap/run/lega/services.py

@@ -54,6 +54,7 @@ def main(cega_conf, conf, args):
 
     lega['services'] = {
         'mq': {
+            'build': '../../ingestion/mq',
             'environment': [
                 'CEGA_CONNECTION='+cega_conf.get('mq', 'connection'),
                 'MQ_USER=admin',
@@ -81,9 +82,9 @@ def main(cega_conf, conf, args):
             ],
         },
         'db': {
+            'build': '../../ingestion/db',
             'environment': [
-                'DB_LEGA_IN_PASSWORD='+conf.get('db', 'lega_in'),
-                'DB_LEGA_OUT_PASSWORD='+conf.get('db', 'lega_out'),
+                'DB_PASSWORD='+conf.get('db', 'password'),
                 'PGDATA=/ega/data',
                 'PG_SERVER_CERT=/etc/ega/pg.cert',
                 'PG_SERVER_KEY=/etc/ega/pg.key',
@@ -151,9 +152,9 @@ def main(cega_conf, conf, args):
                 'S3_SECRET_KEY='+conf.get('s3','secret_key'),
             ] if with_s3 else []),
             'hostname': f'ingest{HOSTNAME_DOMAIN}',
-            'build': '../..',  # Just in case we docker-compose up before building the image locally
-                               # This might be useless since the image from the master branch is built on docker hub.
-                               # so it will get downloaded
+            'build': '../../ingestion',  # Just in case we docker-compose up before building the image locally
+                                         # This might be useless since the image from the master branch is built on docker hub.
+                                         # so it will get downloaded
             'image': 'egarchive/lega-base:latest',
             'container_name': f'ingest{HOSTNAME_DOMAIN}',
             'volumes': [
@@ -314,7 +315,7 @@ def main(cega_conf, conf, args):
         for s in ['ingest', 'verify', 'finalize']:
             service = lega['services'][s]
             volumes = service['volumes']
-            volumes.append('../../lega:/home/lega/.local/lib/python3.8/site-packages/lega')
+            volumes.append('../../ingestion/lega:/home/lega/.local/lib/python3.8/site-packages/lega')
             del service['command']
             service['entrypoint'] = ["/bin/sleep", "1000000000000"]
 

ingestion/.dockerignore

@@ -0,0 +1,2 @@
+db
+mq

Dockerfile → ingestion/Dockerfile

@@ -11,13 +11,13 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
 	&& rm -rf /var/lib/apt/lists/*
 
 # This will pin the package versions
-COPY deploy/requirements.txt /root/LocalEGA/requirements.txt
-RUN pip install --upgrade pip && \
-    pip install -r /root/LocalEGA/requirements.txt
-
+COPY requirements.txt /root/LocalEGA/requirements.txt
 COPY setup.py /root/LocalEGA/setup.py
 COPY lega /root/LocalEGA/lega
-RUN pip install /root/LocalEGA
+
+RUN pip install --upgrade pip && \
+    pip install -r /root/LocalEGA/requirements.txt && \
+    pip install /root/LocalEGA
 
 
 ##########################

ingestion/db/Dockerfile

@@ -0,0 +1,30 @@
+FROM postgres:12.1-alpine
+
+LABEL maintainer "EGA System Developers"
+LABEL org.label-schema.schema-version="2.1"
+LABEL org.label-schema.vcs-url="https://github.com/EGA-archive/LocalEGA"
+
+ENV SSL_SUBJ             /C=ES/ST=Spain/L=Barcelona/O=CRG/OU=SysDevs/CN=LocalEGA/emailAddress=dev.ega@crg.eu
+ENV TZ                   Europe/Madrid
+ENV PGDATA               /ega/data
+
+EXPOSE 5432
+VOLUME /ega/data
+
+RUN apk add --no-cache openssl
+
+
+RUN mkdir -p /etc/ega/initdb.d            && \
+    mkdir -p /var/run/postgresql          && \
+    chmod 2775 /var/run/postgresql
+
+COPY pg.conf       /etc/ega/pg.conf.default
+COPY initdb.d      /etc/ega/initdb.d
+
+RUN chown -R postgres /etc/ega            && \
+    chown -R postgres /var/run/postgresql
+
+COPY entrypoint.sh /usr/local/bin/entrypoint.sh
+RUN chmod 755 /usr/local/bin/entrypoint.sh
+
+ENTRYPOINT ["/usr/local/bin/entrypoint.sh"]

ingestion/db/README.md

@@ -0,0 +1,43 @@
+# LocalEGA database definitions and docker image
+
+We use
+[Postgres 12.1](https://github.com/docker-library/postgres/tree/0d0485cb02e526f5a240b7740b46c35404aaf13f/12/alpine)
+and Alpine 3.10.
+
+Security is hardened:
+- We do not use 'trust' even for local connections
+- Requiring password authentication for all
+- Using scram-sha-256 is stronger than md5
+- Enforcing TLS communication
+- Enforcing client-certificate verification
+
+## Configuration
+
+There are 2 users (`lega_in` and `lega_out`), and 2 schemas
+(`local_ega` and `local_ega_download`).
+
+The following environment variables can be used to configure the database:
+
+| Variable                | Description                      | Default value |
+|------------------------:|:---------------------------------|:--------------|
+| PGDATA                  | The data directory               | `/ega/data`   |
+| DB\_LEGA\_IN\_PASSWORD  | `lega_in`'s password             | -             |
+| TZ                      | Timezone for the Postgres server | Europe/Madrid |
+
+
+<a title="See Initialization scripts" href="https://hub.docker.com/_/postgres">As usual</a>, include your own `.sh`, `.sql` or `.sql.gz` files in `/docker-entrypoint-initdb.d/` in order to have them included at initialization time.
+
+## TLS support
+
+| Variable         | Description                                      | Default value      |
+|-----------------:|:-------------------------------------------------|:-------------------|
+| PG\_SERVER\_CERT | Public Certificate in PEM format                 | `/etc/ega/pg.cert` |
+| PG\_SERVER\_KEY  | Private Key in PEM format                        | `/etc/ega/pg.key`  |
+| PG\_CA           | Public CA Certificate in PEM format              | `/etc/ega/CA.cert` |
+| PG\_VERIFY\_PEER | Enforce client verification                      | 0                  |
+| SSL\_SUBJ        | Subject for the self-signed certificate creation | `/C=ES/ST=Spain/L=Barcelona/O=CRG/OU=SysDevs/CN=LocalEGA/emailAddress=all.ega@crg.eu` |
+
+If not already injected, the files located at `PG_SERVER_CERT` and `PG_SERVER_KEY` will be generated, as a self-signed public/private certificate pair, using `SSL_SUBJ`.
+
+Client verification is enforced if and only if `PG_CA` exists and `PG_VERIFY_PEER` is set to `1`.
+