Skip to content

Commit

Permalink
[WIP] S3 cache
Browse files Browse the repository at this point in the history
  • Loading branch information
eins78 committed Jul 8, 2020
1 parent 7d5395d commit 2221033
Show file tree
Hide file tree
Showing 14 changed files with 302 additions and 127 deletions.
6 changes: 0 additions & 6 deletions .dockerignore
Original file line number Diff line number Diff line change
@@ -1,9 +1,3 @@
# ignore large files (for performance)
# * leihs source code, not needed to create the builder/controller image,
/leihs/
# …except for the ansible version definition, because ansible is part of the image.
!/leihs/deploy/ansible-requirements.txt

# ignore irrelant files to not break the cache when they change
/Dockerfile
/.virtualenv/
Expand Down
50 changes: 22 additions & 28 deletions all.yml
Original file line number Diff line number Diff line change
@@ -1,26 +1,23 @@
leihs_send_mails: No

LEIHS_ROOT_DIR: '/leihs'

LEIHS_ROOT_DIR: "/leihs"

# ✨ # ✨ # ✨ # ✨ # ✨ # ✨ # ✨ # ✨ # ✨ # ✨ # ✨ # ✨ # ✨ # ✨ # ✨ # ✨ # ✨ # ✨
LEIHS_DEPLOY_NEW_BORROW_APP: false # this is still beta! ✨ # ✨ # ✨ # ✨ # ✨ # ✨ # ✨ #
# ✨ # ✨ # ✨ # ✨ # ✨ # ✨ # ✨ # ✨ # ✨ # ✨ # ✨ # ✨ # ✨ # ✨ # ✨ # ✨ # ✨ # ✨


### reverse proxy configuration ###############################################

leihs_virtual_hosts:
- hostname: "localhost"
ip: "localhost"
logfile_infix: "localhost"
force_redirect_to_https: no
- hostname: NULL
ip: '*'
ssl_certificate_file: /etc/ssl/certs/ssl-cert-snakeoil.pem
ssl_certificate_key_file: /etc/ssl/private/ssl-cert-snakeoil.key
force_redirect_to_https: no

- hostname: "localhost"
ip: "localhost"
logfile_infix: "localhost"
force_redirect_to_https: no
- hostname: NULL
ip: "*"
ssl_certificate_file: /etc/ssl/certs/ssl-cert-snakeoil.pem
ssl_certificate_key_file: /etc/ssl/private/ssl-cert-snakeoil.key
force_redirect_to_https: no

### database backup ############################################################

Expand All @@ -36,23 +33,22 @@ leihs_database_user: leihs-database
leihs_database_path: /leihs/database
leihs_database_clean_slate: false


### legacy service configuration ###############################################

leihs_legacy_ruby_version: 2.6.6
leihs_legacy_clean_slate: false
leihs_legacy_user: leihs-legacy
leihs_legacy_max_threads_per_worker: 2
leihs_leagcy_workers: '{{ansible_processor_vcpus}}'
leihs_leagcy_workers: "{{ansible_processor_vcpus}}"

### micro services configuration ###############################################

LEIHS_LEGACY_HTTP_PORT: '3210'
LEIHS_ADMIN_HTTP_PORT: '3220'
LEIHS_PROCURE_HTTP_PORT: '3230'
LEIHS_PROCURE_CLIENT_HTTP_PORT: '3231'
LEIHS_MY_HTTP_PORT: '3240'
LEIHS_BORROW_HTTP_PORT: '3250'
LEIHS_LEGACY_HTTP_PORT: "3210"
LEIHS_ADMIN_HTTP_PORT: "3220"
LEIHS_PROCURE_HTTP_PORT: "3230"
LEIHS_PROCURE_CLIENT_HTTP_PORT: "3231"
LEIHS_MY_HTTP_PORT: "3240"
LEIHS_BORROW_HTTP_PORT: "3250"

################################################################################

Expand All @@ -72,10 +68,8 @@ manipulate_data_revoke_all_admins: false

### building/compiling #########################################################

build_host: 'localhost'
build_host_java: '{{ build_host }}'
build_host_nodejs: '{{ build_host }}'

# setting this to `true` will disable all build artefact caching
force_rebuild: false
debug_build_caching: false
# build artefact caching will only be active when `s3_cache_endpoint` is set
s3_cache_endpoint: "{{lookup('env', 'S3_CACHE_ENDPOINT') | default('')}}"
s3_cache_bucket: "{{lookup('env', 'S3_CACHE_BUCKET') | default('')}}"
s3_cache_access_key: "{{lookup('env', 'S3_ACCESS_KEY_ID') | default('')}}"
s3_cache_secret_key: "{{lookup('env', 'S3_SECRET_ACCESS_KEY') | default('')}}"
3 changes: 2 additions & 1 deletion ansible.cfg
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,8 @@ forks=20
#fact_caching_connection = tmp/facts-cache
#fact_caching_timeout = 86400

stdout_callback = yaml
stdout_callback = debug
# display_args_to_stdout = True

[ssh_connection]
ssh_args = -o ForwardAgent=yes -o ControlMaster=auto -o ControlPersist=60m
85 changes: 85 additions & 0 deletions helpers/build-artefact-with-s3-cache.bash
Original file line number Diff line number Diff line change
@@ -0,0 +1,85 @@
#!/bin/bash

# NOTE: this snippet is used via templating, everything below
# works exactly as if written directly in a `shell` task!

S3_CACHE_ENDPOINT='{{s3_cache_endpoint}}'
S3_CACHE_BUCKET='{{s3_cache_bucket}}'
export AWS_ACCESS_KEY_ID='{{s3_cache_access_key}}'
export AWS_SECRET_ACCESS_KEY='{{s3_cache_secret_key}}'
USE_S3_CACHE=$(test ! -z ${S3_CACHE_ENDPOINT:+x} && echo 1)

if [ -z $ARTEFACT_PATH ] ; then
echo "ERROR: \$ARTEFACT_PATH is empty!"
exit 1
fi

if [ -z $ARTEFACT_DIGEST ] ; then
echo "ERROR: \$ARTEFACT_DIGEST is empty!"
exit 1
fi

ARTEFACT_FILENAME="${ARTEFACT_S3_CACHE_FILE_NAME}_${ARTEFACT_DIGEST}"
ARTEFACT_S3_URL="s3://${S3_CACHE_BUCKET}/${ARTEFACT_FILENAME}"

function check_cache {
aws --endpoint "$S3_CACHE_ENDPOINT" s3 ls "s3://${S3_CACHE_BUCKET}" > /dev/null
}

function upload_to_cache {
aws --no-progress --endpoint "$S3_CACHE_ENDPOINT" \
s3 cp "$ARTEFACT_PATH" "$ARTEFACT_S3_URL"
}

function download_from_cache {
aws --quiet --only-show-errors --no-progress --endpoint "$S3_CACHE_ENDPOINT" \
s3 cp "$ARTEFACT_S3_URL" "$ARTEFACT_PATH"
}

cd "$PROJECT_DIR"
rm -rf "$ARTEFACT_PATH"

if [ $USE_S3_CACHE ]; then
if check_cache; then
echo "INFO: using build cache ${ARTEFACT_S3_URL}"
else
echo "ERROR: 's3_cache' is enabled, but the storage bucket could not be connected to."
exit 1
fi
fi

if download_from_cache; then
echo "INFO: Rerrieved artefact from s3 cache ${ARTEFACT_S3_URL}"
if declare -F restore_artefact >/dev/null; then
if restore_artefact; then
echo "INFO: Restored artefact from file ${ARTEFACT_PATH}"
else
echo "ERROR: Restored artefact from file ${ARTEFACT_PATH}"
exit 1
fi
fi
else
echo "INFO: No S3 cached artefact found ${ARTEFACT_S3_URL}"
fi

if [ ! $USE_S3_CACHE ] || [ ! -f "${ARTEFACT_PATH}" ]; then
echo "INFO: building artefact ${ARTEFACT_PATH}"
if build_artefact ; then
echo "INFO: built artefact ${ARTEFACT_PATH}"
else
echo "ERROR: could not build artefact ${ARTEFACT_PATH}"
exit 1
fi
fi

if [ $USE_S3_CACHE ]; then
if ! download_from_cache; then
echo "INFO: No S3 cached artefact found; uploading ours now"
if upload_to_cache; then
echo "OK"
else
echo "ERROR: could not upload artefact to ${ARTEFACT_S3_URL}"
exit 1
fi
fi
fi
29 changes: 19 additions & 10 deletions roles/leihs-admin-install/tasks/jar.yml
Original file line number Diff line number Diff line change
@@ -1,23 +1,32 @@
- name: create jar
delegate_to: '{{ build_host_java }}'
tags: [builds_artefact, builds_artefact_with_java]
delegate_to: localhost
args:
executable: /bin/bash
shell: |
set -eux
DEPLOY_DIR='{{playbook_dir}}'
cd $DEPLOY_DIR/../admin
export LEIN_SNAPSHOTS_IN_RELEASE=1
$DEPLOY_DIR/bin/lein uberjar
PROJECT_DIR="${DEPLOY_DIR}/../admin"
ARTEFACT_DIGEST="$(cd "$PROJECT_DIR" && git log -n 1 HEAD --pretty=%T)"
ARTEFACT_PATH="${PROJECT_DIR}/target/leihs-admin.jar"
ARTEFACT_S3_CACHE_FILE_NAME="leihs-admin.jar"
function build_artefact() {
export LEIN_SNAPSHOTS_IN_RELEASE=1
$DEPLOY_DIR/bin/lein clean
$DEPLOY_DIR/bin/lein uberjar
}
{{ lookup('template', "{{playbook_dir}}/helpers/build-artefact-with-s3-cache.bash") }}
- file:
path: '{{leihs_admin_dir}}'
path: "{{leihs_admin_dir}}"
state: directory
owner: '{{leihs_admin_user}}'
owner: "{{leihs_admin_user}}"
recurse: yes
name: create empty {{leihs_admin_dir}}

- copy:
src: '{{playbook_dir}}/../admin/target/leihs-admin.jar'
dest: '{{leihs_admin_dir}}/leihs-admin.jar'
owner: '{{leihs_admin_user}}'
src: "{{playbook_dir}}/../admin/target/leihs-admin.jar"
dest: "{{leihs_admin_dir}}/leihs-admin.jar"
owner: "{{leihs_admin_user}}"
name: copy jar over to server
39 changes: 24 additions & 15 deletions roles/leihs-admin-install/tasks/shared-ui.yml
Original file line number Diff line number Diff line change
@@ -1,29 +1,38 @@
- name: create shared-ui
delegate_to: '{{ build_host_nodejs }}'
- name: build shared-ui
tags: [builds_artefact, builds_artefact_with_nodejs]
delegate_to: localhost
args:
executable: /bin/bash
shell: |
set -eux
DEPLOY_DIR='{{playbook_dir}}'
cd "${DEPLOY_DIR}/../admin"
sh scripts/prepare-shared-ui.sh
PROJECT_DIR="${DEPLOY_DIR}/../admin"
- name: make shared-ui tarball
delegate_to: '{{ build_host_nodejs }}'
command: 'tar -cvz -f "{{playbook_dir}}/../admin/leihs-ui.tgz" .'
args:
chdir: '{{playbook_dir}}/../admin/leihs-ui'
creates: '{{playbook_dir}}/../admin/leihs-ui.tgz'
ARTEFACT_DIGEST="$(cd "$PROJECT_DIR/leihs-ui" && git log -n 1 HEAD --pretty=%T)"
ARTEFACT_PATH="${PROJECT_DIR}/leihs-ui.tgz"
ARTEFACT_S3_CACHE_FILE_NAME="leihs-shared-ui.tgz"
function build_artefact {
cd "$PROJECT_DIR"
sh scripts/prepare-shared-ui.sh || { echo "build error!"; exit 1; }
tar -cz --exclude-vcs --exclude 'leihs-ui/node_modules' --exclude 'leihs-ui/bootstrap-theme-leihs/node_modules' \
-f "$ARTEFACT_PATH" leihs-ui
}
function restore_artefact {
cd "$PROJECT_DIR"
tar -xz -f "${ARTEFACT_PATH}"
}
{{ lookup('template', "{{playbook_dir}}/helpers/build-artefact-with-s3-cache.bash") }}
- name: create empty {{leihs_admin_dir}}/leihs-ui
file:
path: '{{leihs_admin_dir}}/leihs-ui'
path: "{{leihs_admin_dir}}/leihs-ui"
state: directory
owner: '{{leihs_admin_user}}'
owner: "{{leihs_admin_user}}"
recurse: yes

- name: copy shared-ui over to server
unarchive:
src: '{{playbook_dir}}/../admin/leihs-ui.tgz'
dest: '{{leihs_admin_dir}}/leihs-ui'
owner: '{{leihs_admin_user}}'
src: "{{playbook_dir}}/../admin/leihs-ui.tgz"
dest: "{{leihs_admin_dir}}/leihs-ui"
owner: "{{leihs_admin_user}}"
18 changes: 14 additions & 4 deletions roles/leihs-borrow-install/tasks/jar.yml
Original file line number Diff line number Diff line change
@@ -1,12 +1,22 @@
- name: create jar
delegate_to: "{{ build_host_java }}"
tags: [builds_artefact, builds_artefact_with_java]
delegate_to: localhost
args:
executable: /bin/bash
shell: |
set -eux
DEPLOY_DIR='{{playbook_dir}}'
cd $DEPLOY_DIR/../borrow
./scripts/build-uberjar-prod.sh
PROJECT_DIR="${DEPLOY_DIR}/../borrow"
ARTEFACT_DIGEST="$(cd "$PROJECT_DIR" && git log -n 1 HEAD --pretty=%T)"
ARTEFACT_PATH="${PROJECT_DIR}/target/leihs-borrow.jar"
ARTEFACT_S3_CACHE_FILE_NAME="leihs-borrow.jar"
function build_artefact() {
cd "$PROJECT_DIR"
mkdir -p $PROJECT_DIR/target
./scripts/build-uberjar-prod.sh
}
{{ lookup('template', "{{playbook_dir}}/helpers/build-artefact-with-s3-cache.bash") }}
- file:
path: "{{leihs_borrow_dir}}"
Expand Down
30 changes: 19 additions & 11 deletions roles/leihs-borrow-install/tasks/shared-ui.yml
Original file line number Diff line number Diff line change
@@ -1,19 +1,27 @@
- name: create shared-ui
delegate_to: "{{ build_host_nodejs }}"
- name: build shared-ui
tags: [builds_artefact, builds_artefact_with_nodejs]
delegate_to: localhost
args:
executable: /bin/bash
shell: |
set -eux
DEPLOY_DIR='{{playbook_dir}}'
cd "${DEPLOY_DIR}/../borrow"
sh scripts/prepare-shared-ui.sh
PROJECT_DIR="${DEPLOY_DIR}/../borrow"
- name: make shared-ui tarball
delegate_to: localhost
command: 'tar -cvz -f "{{playbook_dir}}/../borrow/leihs-ui.tgz" .'
args:
chdir: "{{playbook_dir}}/../borrow/leihs-ui"
creates: "{{playbook_dir}}/../borrow/leihs-ui.tgz"
ARTEFACT_DIGEST="$(cd "$PROJECT_DIR/leihs-ui" && git log -n 1 HEAD --pretty=%T)"
ARTEFACT_PATH="${PROJECT_DIR}/leihs-ui.tgz"
ARTEFACT_S3_CACHE_FILE_NAME="leihs-shared-ui.tgz"
function build_artefact {
cd "$PROJECT_DIR"
sh scripts/prepare-shared-ui.sh || { echo "build error!"; exit 1; }
tar -cz --exclude-vcs --exclude 'leihs-ui/node_modules' --exclude 'leihs-ui/bootstrap-theme-leihs/node_modules' \
-f "$ARTEFACT_PATH" leihs-ui
}
function restore_artefact {
cd "$PROJECT_DIR"
tar -xz -f "${ARTEFACT_PATH}"
}
{{ lookup('template', "{{playbook_dir}}/helpers/build-artefact-with-s3-cache.bash") }}
- name: create empty {{leihs_borrow_dir}}/leihs-ui
file:
Expand Down
27 changes: 18 additions & 9 deletions roles/leihs-mail-install/tasks/jar.yml
Original file line number Diff line number Diff line change
@@ -1,22 +1,31 @@
- name: create jar
delegate_to: '{{ build_host_java }}'
tags: [builds_artefact, builds_artefact_with_java]
delegate_to: localhost
args:
executable: /bin/bash
shell: |
set -eux
DEPLOY_DIR='{{playbook_dir}}'
cd $DEPLOY_DIR/../mail
$DEPLOY_DIR/bin/boot uberjar
PROJECT_DIR="${DEPLOY_DIR}/../mail"
ARTEFACT_DIGEST="$(cd "$PROJECT_DIR" && git log -n 1 HEAD --pretty=%T)"
ARTEFACT_PATH="${PROJECT_DIR}/target/leihs-mail.jar"
ARTEFACT_S3_CACHE_FILE_NAME="leihs-mail.jar"
function build_artefact() {
cd $PROJECT_DIR
$DEPLOY_DIR/bin/boot --no-colors uberjar
}
{{ lookup('template', "{{playbook_dir}}/helpers/build-artefact-with-s3-cache.bash") }}
- file:
path: '{{leihs_mail_dir}}'
path: "{{leihs_mail_dir}}"
state: directory
owner: '{{leihs_mail_user}}'
owner: "{{leihs_mail_user}}"
recurse: yes
name: create empty {{leihs_mail_dir}}

- copy:
src: '{{playbook_dir}}/../mail/target/leihs-mail.jar'
dest: '{{leihs_mail_dir}}/leihs-mail.jar'
owner: '{{leihs_mail_user}}'
src: "{{playbook_dir}}/../mail/target/leihs-mail.jar"
dest: "{{leihs_mail_dir}}/leihs-mail.jar"
owner: "{{leihs_mail_user}}"
name: copy jar over to server
Loading

0 comments on commit 2221033

Please sign in to comment.