Fixing package version and removing S3 software.

you'd better use s3fs, mount the s3 backend and present
a POSIX file system rather than including if-then-else all over the place
and the boto3 package in the software stack.

deploy/bootstrap/Makefile

@@ -13,7 +13,8 @@
 # * The certificates
 # * the secrets
 #
-#
+#   Use make PYTHON=python3 PIP=pip3 if you don't have 
+#   `python` and `pip` pointing to python 3
 #########################################################
 
 
@@ -23,15 +24,14 @@ OPENSSL=openssl
 HOSTNAME_DOMAIN=
 #HOSTNAME_DOMAIN=.localega
 
+PYTHON=python
+PIP=pip
+
 .PHONY: all users certs clean clean-all
 .DELETE_ON_ERROR: ../private/*
 
 SECRETS:= master.key.passphrase db.lega mq.admin archive-db.lega
 
-ifdef S3
-SECRETS+= s3.access s3.secret
-endif
-
 ALL_FILES=$(addprefix ../private/, lega-entrypoint.sh cega-entrypoint.sh \
 		                   cega-mq-defs.json cega-mq-rabbitmq.config cega-accession.ini \
                                    master.key.sec \
@@ -41,7 +41,7 @@ ALL_FILES=$(addprefix ../private/, lega-entrypoint.sh cega-entrypoint.sh \
 all: requirements ../.env $(ALL_FILES) certs users
 
 requirements:
-	pip install -r requirements.txt
+	$(PIP) install -r requirements.txt
 
 dev: export DEPLOY_DEV=yesplease
 dev: all
@@ -57,11 +57,11 @@ dev: all
 
 ../private/cega.conf: run/cega/conf.py | ../private
 	@echo "Creating CentralEGA configuration trace: $(@F)"
-	@python -m run.cega.conf > $@
+	@$(PYTHON) -m run.cega.conf > $@
 
 ../private/lega.conf: run/lega/conf.py $(addprefix ../private/secrets/,$(SECRETS))
 	@echo "Creating LocalEGA configuration trace: $(@F)"
-	@python -m run.lega.conf --secrets $(CURDIR)/../private/secrets > $@
+	@$(PYTHON) -m run.lega.conf --secrets $(CURDIR)/../private/secrets > $@
 
 ../.env: | ../private
 	@echo "COMPOSE_PROJECT_NAME=lega" > $@
@@ -72,49 +72,49 @@ dev: all
 
 ../private/cega-mq-defs.json: ../private/cega.conf run/cega/mq_defs.py | ../private
 	@echo "Creating CentralEGA MQ defs: $(@F)"
-	@python -m run.cega.mq_defs < $< > $@
+	@$(PYTHON) -m run.cega.mq_defs < $< > $@
 
 ../private/cega-mq-rabbitmq.config: run/cega/mq-rabbitmq.config | ../private
 	@echo "Creating CentralEGA MQ conf: $(@F)"
 	@cp $< $@
 
 ../private/cega.yml: run/cega/services.py | ../private
 	@echo "Creating CentralEGA docker-compose file: $(@F)"
-	@python -m run.cega.services > $@
+	@$(PYTHON) -m run.cega.services > $@
 
 ../private/cega-accession.ini: ../private/cega.conf run/cega/accession.py | ../private
 	@echo "Creating CentralEGA docker-compose file: $(@F)"
-	@python -m run.cega.accession $< > $@
+	@$(PYTHON) -m run.cega.accession $< > $@
 
 ############### Generate LocalEGA settings
 
 ../private/lega.yml: run/lega/services.py | ../private
 ../private/lega.yml: ../private/cega.conf ../private/lega.conf
 	@echo "Creating LocalEGA docker-compose file: $(@F)"
-	@python -m run.lega.services ../private/cega.conf ../private/lega.conf 2>../private/.err > $@
+	@$(PYTHON) -m run.lega.services ../private/cega.conf ../private/lega.conf 2>../private/.err > $@
 
 ../private/pipeline.yml: run/lega/pipeline.py | ../private
 ../private/pipeline.yml: ../private/cega.conf ../private/lega.conf
 	@echo "Creating LocalEGA docker-compose file: $(@F)"
-	@python -m run.lega.pipeline ../private/cega.conf ../private/lega.conf 2>../private/.err > $@
+	@$(PYTHON) -m run.lega.pipeline ../private/cega.conf ../private/lega.conf 2>../private/.err > $@
 
 ../private/backup1.ini: ../private/lega.conf run/lega/backup.py | ../private
 	@echo "Creating LocalEGA service configuration: $(@F)"
-	@python -m run.lega.backup --queue accession \
+	@$(PYTHON) -m run.lega.backup --queue accession \
 	                           --destination /ega/vault \
 			           --routing_key backup1 \
 	                      $< 2>../private/.err > $@
 
 ../private/backup2.ini: ../private/lega.conf run/lega/backup.py | ../private
 	@echo "Creating LocalEGA service configuration: $(@F)"
-	@python -m run.lega.backup --queue backup1 \
+	@$(PYTHON) -m run.lega.backup --queue backup1 \
 	                           --destination /ega/vault.bkp \
 	                           --routing_key backup2 \
 	                      $< 2>../private/.err > $@
 
 ../private/%.ini: ../private/lega.conf run/lega/%.py | ../private
 	@echo "Creating LocalEGA service configuration: $(@F)"
-	@python -m run.lega.$* $< 2>../private/.err > $@
+	@$(PYTHON) -m run.lega.$* $< 2>../private/.err > $@
 
 ../private/master.key.sec: ../private/secrets/master.key.passphrase
 	@echo "Creating master key: $(@F)"
@@ -125,17 +125,11 @@ dev: all
 # Stronger passphrase
 ../private/secrets/master.key.passphrase: | ../private/secrets
 	@echo "Creating master key passphrase: $(@F)"
-	@python -m run.pwd_gen 32 > $@
-
-ifdef S3
-../private/secrets/s3.access ../private/secrets/s3.secret: | ../private/secrets
-	@echo "Creating S3 credentials: $(@F)"
-	@python -m run.pwd_gen 32 > $@
-endif
+	@$(PYTHON) -m run.pwd_gen 32 > $@
 
 ../private/secrets/%: | ../private/secrets
 	@echo "Creating secret: $(@F)"
-	@python -m run.pwd_gen 16 > $@
+	@$(PYTHON) -m run.pwd_gen 16 > $@
 
 ############### Generate CentralEGA users
 
@@ -161,10 +155,6 @@ COMPONENTS=dispatcher \
            inbox \
 	   archive-db
 
-ifdef S3
-COMPONENTS+=archive inbox-s3-backend
-endif
-
 COMPONENTS+=cega-mq cega-users cega-accession testsuite
 
 CERTS=$(COMPONENTS:%=../private/certs/%.sec.pem) \

deploy/bootstrap/cega-users/Dockerfile

@@ -1,7 +1,7 @@
 ##########################
 ## Build env
 ##########################
-FROM python:3.8-alpine3.11 as BUILD
+FROM python:3.10-alpine3.14 as BUILD
 
 RUN apk add gcc musl-dev
 
@@ -11,7 +11,7 @@ RUN pip install --upgrade pip && \
 ##########################
 ## Final image
 ##########################
-FROM python:3.8-alpine3.11
+FROM python:3.10-alpine3.14
 
 LABEL maintainer "EGA System Developers"
 LABEL org.label-schema.schema-version="1.0"

deploy/bootstrap/requirements.txt

@@ -1,7 +1,4 @@
 bcrypt==3.2.0
-cffi==1.14.5
 docopt==0.6.2
-pycparser==2.20
-PyYAML==5.4.1
-ruamel.yaml==0.17.4
-six==1.16.0
+PyYAML==6.0
+ruamel.yaml==0.17.17

ingestion/db/Dockerfile

@@ -1,4 +1,4 @@
-FROM postgres:14.1-alpine
+FROM postgres:13.4-alpine
 
 LABEL maintainer "EGA System Developers"
 LABEL org.label-schema.schema-version="2.1"

ingestion/requirements.txt

@@ -1,23 +1,12 @@
-AMQPStorm==2.7.2
-bcrypt==3.1.7
-boto3==1.11.7
-botocore==1.14.7
-certifi==2020.4.5.1
-cffi==1.14.5
-chardet==3.0.4
+AMQPStorm==2.10.4
+bcrypt==3.2.0
+cffi==1.15.0
 crypt4gh==1.5
-cryptography==3.3.2
+cryptography==36.0.0
 docopt==0.6.2
-#docutils==0.15.2
-idna==2.9
-jmespath==0.10.0
 pamqp==2.3.0
 psycopg2-binary==2.8.5
-pycparser==2.20
-PyNaCl==1.3.0
-python-dateutil==2.8.1
-PyYAML==5.4
-requests==2.23.0
-s3transfer==0.3.1
-six==1.14.0
-urllib3==1.26.5
+pycparser==2.21
+PyNaCl==1.4.0
+PyYAML==6.0
+six==1.16.0

ingestion/setup.py

@@ -31,9 +31,7 @@
       platforms='any',
       install_requires=[
           'amqpstorm',
-          'requests',
           'psycopg2-binary>=2.8.5',
           'PyYaml',
-          'boto3',
           'crypt4gh>=1.3',
       ])