Merge pull request #245 from NBISweden/feature/user-cache

Updating the inbox so that it uses a local cache

.travis.yml

@@ -6,15 +6,15 @@ services:
 before_install:
   - |
         cd deployments/docker/images
-        make pull common
-        make -j 4 images
-        make bootstrap
+        # make pull
+        make images
         cd ..
         make bootstrap
 
 install:
   - docker-compose up -d
   - docker-compose ps
+  - sleep 120
 
 script:
   - cd ../../tests

deployments/docker/bootstrap/instance.sh

@@ -53,8 +53,8 @@ ${GPG_CONF} --kill gpg-agent
 #########################################################################
 
 echomsg "\t* the RSA public and private key"
-${OPENSSL} genrsa -out ${PRIVATE}/${INSTANCE}/rsa/ega.sec -passout pass:${RSA_PASSPHRASE} 2048
-${OPENSSL} rsa -in ${PRIVATE}/${INSTANCE}/rsa/ega.sec -passin pass:${RSA_PASSPHRASE} -pubout -out ${PRIVATE}/${INSTANCE}/rsa/ega.pub
+${OPENSSL} genpkey -algorithm RSA -out ${PRIVATE}/${INSTANCE}/rsa/ega.sec -pkeyopt rsa_keygen_bits:2048
+${OPENSSL} rsa -pubout -in ${PRIVATE}/${INSTANCE}/rsa/ega.sec -out ${PRIVATE}/${INSTANCE}/rsa/ega.pub
 
 #########################################################################
 
@@ -71,7 +71,6 @@ active_master_key = 1
 [master.key.1]
 seckey = /etc/ega/rsa/sec.pem
 pubkey = /etc/ega/rsa/pub.pem
-passphrase = ${RSA_PASSPHRASE}
 EOF
 
 echomsg "\t* ega.conf"
@@ -258,11 +257,10 @@ cat > ${PRIVATE}/${INSTANCE}/cega.env <<EOF
 #
 LEGA_GREETINGS=${LEGA_GREETINGS}
 #
-CEGA_ENDPOINT=http://cega-users/user/%s
-CEGA_ENDPOINT_USER=${INSTANCE}
-CEGA_ENDPOINT_PASSWORD=${CEGA_REST_PASSWORD}
-CEGA_ENDPOINT_RESP_PASSWD=.password_hash
-CEGA_ENDPOINT_RESP_PUBKEY=.pubkey
+CEGA_ENDPOINT=http://cega-users/user/
+CEGA_ENDPOINT_CREDS=${INSTANCE}:${CEGA_REST_PASSWORD}
+CEGA_ENDPOINT_JSON_PASSWD=.password_hash
+CEGA_ENDPOINT_JSON_PUBKEY=.pubkey
 EOF
 
 echomsg "\t* Elasticsearch configuration file"
@@ -342,7 +340,6 @@ GPG_PASSPHRASE            = ${GPG_PASSPHRASE}
 GPG_NAME                  = ${GPG_NAME}
 GPG_COMMENT               = ${GPG_COMMENT}
 GPG_EMAIL                 = ${GPG_EMAIL}
-RSA_PASSPHRASE            = ${RSA_PASSPHRASE}
 SSL_SUBJ                  = ${SSL_SUBJ}
 #
 DB_USER                   = ${DB_USER}

deployments/docker/bootstrap/settings/fin1

@@ -17,8 +17,6 @@ DB_TRY=30
 GPG_NAME="EGA Finland"
 GPG_COMMENT="@CSC"
 GPG_EMAIL="ega@csc.fi"
-
 GPG_PASSPHRASE=$(generate_password 16)
-RSA_PASSPHRASE=$(generate_password 16)
 
 LOG_LEVEL=INFO

deployments/docker/bootstrap/settings/swe1

@@ -17,8 +17,6 @@ DB_TRY=30
 GPG_NAME="EGA Sweden"
 GPG_COMMENT="@NBIS"
 GPG_EMAIL="ega@nbis.se"
-
 GPG_PASSPHRASE=$(generate_password 16)
-RSA_PASSPHRASE=$(generate_password 16)
 
 LOG_LEVEL=DEBUG

deployments/docker/ega.yml

@@ -29,7 +29,6 @@ services:
     hostname: ega-frontend
     depends_on:
       - db-swe1
-      - logstash-swe1
     ports:
       - "9000:80"
     expose:
@@ -45,10 +44,7 @@ services:
   inbox-swe1:
     hostname: ega-inbox
     depends_on:
-      - db-swe1
-      - logstash-swe1
-      - elasticsearch-swe1
-      - kibana-swe1
+      - mq-swe1
       - cega-users
     env_file:
       - private/swe1/db.env
@@ -75,7 +71,6 @@ services:
       - db-swe1
       - mq-swe1
       - inbox-swe1
-      - logstash-swe1
     hostname: ega-vault
     container_name: ega-vault-swe1
     image: nbisweden/ega-vault
@@ -95,8 +90,6 @@ services:
       - db-swe1
       - mq-swe1
       - keys-swe1
-      - inbox-swe1
-      - logstash-swe1
     image: nbisweden/ega-worker
     environment:
       - GPG_TTY=/dev/console
@@ -117,8 +110,6 @@ services:
   # Key server
   keys-swe1:
     env_file: private/swe1/gpg.env
-    depends_on:
-      - logstash-swe1
     environment:
       - GPG_TTY=/dev/console
       - KEYSERVER_PORT=9010
@@ -202,7 +193,6 @@ services:
     hostname: ega-frontend
     depends_on:
       - db-fin1
-      - logstash-fin1
     ports:
       - "9001:80"
     expose:
@@ -218,10 +208,7 @@ services:
   inbox-fin1:
     hostname: ega-inbox
     depends_on:
-      - db-fin1
-      - logstash-fin1
-      - elasticsearch-fin1
-      - kibana-fin1
+      - mq-fin1
       - cega-users
     env_file:
       - private/fin1/db.env
@@ -247,7 +234,6 @@ services:
       - db-fin1
       - mq-fin1
       - inbox-fin1
-      - logstash-fin1
     hostname: ega-vault
     container_name: ega-vault-fin1
     image: nbisweden/ega-vault
@@ -267,8 +253,6 @@ services:
       - db-fin1
       - mq-fin1
       - keys-fin1
-      - inbox-fin1
-      - logstash-fin1
     image: nbisweden/ega-worker
     environment:
       - GPG_TTY=/dev/console
@@ -289,8 +273,6 @@ services:
   # Key server
   keys-fin1:
     env_file: private/fin1/gpg.env
-    depends_on:
-      - logstash-fin1
     environment:
       - GPG_TTY=/dev/console
       - KEYSERVER_PORT=9010

deployments/docker/images/README.md

@@ -21,7 +21,7 @@ A typical build goes as follows:
 
 # Results
 
-`rabbitmq:management`, `postgres:latest`, `centos:latest` are pulled from the main docker hub.
+`rabbitmq:management`, `postgres:latest`, `centos:7.4.1708` are pulled from the main docker hub.
 
 The following images are created locally:
 

deployments/docker/images/common/Dockerfile

@@ -1,4 +1,4 @@
-FROM centos:latest
+FROM centos:7.4.1708
 LABEL maintainer "Frédéric Haziza, NBIS"
 
 RUN yum -y install https://centos7.iuscommunity.org/ius-release.rpm && \
@@ -8,7 +8,8 @@ RUN yum -y install https://centos7.iuscommunity.org/ius-release.rpm && \
     	   	   openssl \
 		   nss-tools nc nmap tcpdump lsof strace \
 	       	   bash-completion bash-completion-extras \
-		   python36u python36u-pip
+		   python36u python36u-pip && \
+    yum clean all
 
 RUN [[ -e /lib64/libpython3.6m.so ]] || ln -s /lib64/libpython3.6m.so.1.0 /lib64/libpython3.6m.so
 

deployments/docker/images/inbox/Dockerfile

@@ -1,10 +1,11 @@
 FROM nbisweden/ega-common:latest
 LABEL maintainer "Frédéric Haziza, NBIS"
 
-RUN yum -y install openssh-server postgresql-devel pam-devel libcurl-devel jq-devel fuse fuse-libs cronie
+RUN yum -y install openssh-server pam-devel libcurl-devel jq-devel fuse fuse-libs cronie && \
+    yum clean all
 
 ##################################
-EXPOSE 22
+EXPOSE 9000
 VOLUME /ega/inbox
 ENV DB_INSTANCE=
 
@@ -20,9 +21,9 @@ RUN ssh-keygen -t rsa     -N '' -f /etc/ssh/ssh_host_rsa_key && \
     echo 'Welcome to Local EGA' > /ega/banner && \
     cp /etc/nsswitch.conf /etc/nsswitch.conf.bak && \
     sed -i -e 's/^passwd:\(.*\)files/passwd:\1files ega/' /etc/nsswitch.conf && \
-    git clone -b fuse https://github.com/NBISweden/LocalEGA-auth /root/ega-auth && \
+    git clone -b no-db https://github.com/NBISweden/LocalEGA-auth /root/ega-auth && \
     cd /root/ega-auth/src && \
-    make debug clean && \
+    make install clean && \
     ldconfig -v && \
     chown root:ega /ega/inbox && \
     chmod 750 /ega/inbox && \

deployments/docker/images/inbox/entrypoint.sh

@@ -10,34 +10,23 @@ EGA_UID=$(id -u ega)
 EGA_GID=$(id -g ega)
 
 cat > /etc/ega/auth.conf <<EOF
-debug = ok_why_not
-
-##################
-# Databases
-##################
-db_connection = host=${EGA_DB_IP} port=5432 dbname=lega user=${POSTGRES_USER} password=${POSTGRES_PASSWORD} connect_timeout=1 sslmode=disable
-
 enable_cega = yes
 cega_endpoint = ${CEGA_ENDPOINT}
-cega_user = ${CEGA_ENDPOINT_USER}
-cega_password = ${CEGA_ENDPOINT_PASSWORD}
-cega_resp_passwd = ${CEGA_ENDPOINT_RESP_PASSWD}
-cega_resp_pubkey = ${CEGA_ENDPOINT_RESP_PUBKEY}
+cega_creds = ${CEGA_ENDPOINT_CREDS}
+cega_json_passwd = ${CEGA_ENDPOINT_JSON_PASSWD}
+cega_json_pubkey = ${CEGA_ENDPOINT_JSON_PUBKEY}
 
 ##################
-# NSS & PAM Queries
+# NSS & PAM
 ##################
-get_ent = SELECT elixir_id FROM users WHERE elixir_id = \$1 LIMIT 1
-add_user = SELECT insert_user(\$1,\$2,\$3)
-get_password = SELECT password_hash FROM users WHERE elixir_id = \$1 LIMIT 1
-get_account = SELECT elixir_id FROM users WHERE elixir_id = \$1 and current_timestamp < last_accessed + expiration
-
-#prompt = Knock Knock:
-
+# prompt = Knock Knock:
 ega_uid = ${EGA_UID}
 ega_gid = ${EGA_GID}
-ega_gecos = EGA User
-ega_shell = /sbin/nologin
+# ega_gecos = EGA User
+# ega_shell = /sbin/nologin
+
+ega_dir = /ega/inbox
+ega_dir_attrs = 2750 # rwxr-s---
 
 ##################
 # FUSE mount
@@ -46,21 +35,13 @@ ega_fuse_dir = /lega
 ega_fuse_exec = /usr/bin/ega-fs
 ega_fuse_flags = nodev,noexec,uid=${EGA_UID},gid=${EGA_GID},suid
 
-ega_dir = /ega/inbox
-ega_dir_attrs = 2750 # rwxr-s---
 EOF
 
-cat > /usr/local/bin/ega_ssh_keys.sh <<EOF
-#!/bin/bash
-
-eid=\${1%%@*} # strip what's after the @ symbol
-
-query="SELECT pubkey from users where elixir_id = '\${eid}' LIMIT 1"
-
-PGPASSWORD=${POSTGRES_PASSWORD} psql -tqA -U ${POSTGRES_USER} -h ${DB_INSTANCE} -d lega -c "\${query}"
-EOF
-chmod 750 /usr/local/bin/ega_ssh_keys.sh
-chgrp ega /usr/local/bin/ega_ssh_keys.sh
+# for the ramfs cache
+mkdir -p /ega/cache
+sed -i -e '/ega/ d' /etc/fstab
+echo "ramfs /ega/cache ramfs   size=200m 0 0" >> /etc/fstab
+mount /ega/cache
 
 # Greetings per site
 [[ -z "${LEGA_GREETINGS}" ]] || echo ${LEGA_GREETING} > /ega/banner

deployments/docker/images/inbox/sshd_config

@@ -29,5 +29,5 @@ AcceptEnv LC_IDENTIFICATION LC_ALL LANGUAGE
 AcceptEnv XMODIFIERS
 Subsystem sftp internal-sftp
 Banner /ega/banner
-AuthorizedKeysCommand /usr/local/bin/ega_ssh_keys.sh
-AuthorizedKeysCommandUser ega
+AuthorizedKeysCommand /usr/local/bin/ega_ssh_keys
+AuthorizedKeysCommandUser root

deployments/docker/images/keys/Dockerfile

@@ -1,7 +1,8 @@
 FROM nbisweden/ega-common:latest
 LABEL maintainer "Frédéric Haziza, NBIS"
 
-RUN yum -y install vim-common zlib-devel bzip2-devel
+RUN yum -y install vim-common zlib-devel bzip2-devel && \
+    yum clean all
 
 # Copy the RPMS from git
 RUN for f in libgpg-error-1.27 libgcrypt-1.8.1 libassuan-2.4.3 libksba-1.3.5 npth-1.5 ncurses-6.0 pinentry-1.0.0 gnupg-2.2.2; \

deployments/docker/images/worker/Dockerfile

@@ -1,7 +1,8 @@
 FROM nbisweden/ega-common:latest
 LABEL maintainer "Frédéric Haziza, NBIS"
 
-RUN yum -y install vim-common zlib-devel bzip2-devel
+RUN yum -y install vim-common zlib-devel bzip2-devel && \
+    yum clean all
 
 # Copy the RPMS from git
 RUN for f in libgpg-error-1.27 libgcrypt-1.8.1 libassuan-2.4.3 libksba-1.3.5 npth-1.5 ncurses-6.0 pinentry-1.0.0 gnupg-2.2.2; \

deployments/docker/images/worker/Dockerfile.bootstrap

@@ -1,7 +1,8 @@
 FROM nbisweden/ega-common:latest
 LABEL maintainer "Frédéric Haziza, NBIS"
 
-RUN yum -y install vim-common zlib-devel bzip2-devel
+RUN yum -y install vim-common zlib-devel bzip2-devel && \
+    yum clean all
 
 # Copy the RPMS from git
 RUN for f in libgpg-error-1.27 libgcrypt-1.8.1 libassuan-2.4.3 libksba-1.3.5 npth-1.5 ncurses-6.0 pinentry-1.0.0 gnupg-2.2.2; \