diff --git a/README.md b/README.md index 7400c70a4..d2f3bf041 100644 --- a/README.md +++ b/README.md @@ -6,5 +6,6 @@ for a full-fledged production platform. Power Analytics and Visualization for Climate Science - Powered by Birdhouse and other ESGF software How to deploy the entire PAVICS platform, see the -[README](birdhouse/README.md) and the +[README](birdhouse/README.md) and the various extra components +[README for extra core components](birdhouse/components/README.rst), [README for optional components](birdhouse/optional-components/README.md). diff --git a/birdhouse/components/monitoring/default.env b/birdhouse/components/monitoring/default.env new file mode 100644 index 000000000..18b7856b5 --- /dev/null +++ b/birdhouse/components/monitoring/default.env @@ -0,0 +1,28 @@ +# All env in this default.env can be overridden by env.local. +# All env in this default.env must not depend on any env in env.local. + +export GRAFANA_ADMIN_PASSWORD="changeme!" +export ALERTMANAGER_ADMIN_EMAIL_RECEIVER="" # "user1@example.com,user2@example.com" +export SMTP_SERVER="" # "smtp.example.com:25" +export ALERTMANAGER_EXTRA_GLOBAL="" +export ALERTMANAGER_EXTRA_ROUTES="" +export ALERTMANAGER_EXTRA_INHIBITION="" +export ALERTMANAGER_EXTRA_RECEIVERS="" + + + +# add vars only needed to be substituted in templates + +VARS=" + $VARS + \$ALERTMANAGER_ADMIN_EMAIL_RECEIVER + \$SMTP_SERVER +" + +OPTIONAL_VARS=" + $OPTIONAL_VARS + \$ALERTMANAGER_EXTRA_GLOBAL + \$ALERTMANAGER_EXTRA_ROUTES + \$ALERTMANAGER_EXTRA_INHIBITION + \$ALERTMANAGER_EXTRA_RECEIVERS +" diff --git a/birdhouse/components/monitoring/pre-docker-compose-up b/birdhouse/components/monitoring/pre-docker-compose-up new file mode 100755 index 000000000..71b2ce0b5 --- /dev/null +++ b/birdhouse/components/monitoring/pre-docker-compose-up @@ -0,0 +1,5 @@ +#!/bin/sh -x + +docker volume create prometheus_persistence # metrics db +docker volume create grafana_persistence # dashboard and config db +docker volume create alertmanager_persistence # storage diff --git a/birdhouse/config/thredds/entrypointwrapper b/birdhouse/config/thredds/entrypointwrapper index 62dcdb0c7..a1d617823 100755 --- a/birdhouse/config/thredds/entrypointwrapper +++ b/birdhouse/config/thredds/entrypointwrapper @@ -20,8 +20,12 @@ fi # https://www.unidata.ucar.edu/software/tds/v4.6/reference/TomcatBehindProxyServer.html WEBAPPS_ROOT="/usr/local/tomcat/webapps" EXISTING_CONTEXT_ROOT="thredds" -WANTED_CONTEXT_ROOT="twitcher/ows/proxy/thredds" -WANTED_CONTEXT_ROOT_WARFILE_NAME="twitcher#ows#proxy#thredds" +if [ -z "$WANTED_CONTEXT_ROOT" ]; then + WANTED_CONTEXT_ROOT="twitcher/ows/proxy/thredds" +fi +if [ -z "$WANTED_CONTEXT_ROOT_WARFILE_NAME" ]; then + WANTED_CONTEXT_ROOT_WARFILE_NAME="twitcher#ows#proxy#thredds" +fi if [ -d "$WEBAPPS_ROOT/$EXISTING_CONTEXT_ROOT" ]; then mv "$WEBAPPS_ROOT/$EXISTING_CONTEXT_ROOT" "$WEBAPPS_ROOT/$WANTED_CONTEXT_ROOT_WARFILE_NAME" diff --git a/birdhouse/common.env b/birdhouse/default.env similarity index 82% rename from birdhouse/common.env rename to birdhouse/default.env index 122eff96c..344f2183d 100644 --- a/birdhouse/common.env +++ b/birdhouse/default.env @@ -1,12 +1,12 @@ -# All env this common.env can be overridden by env.local. +# All env in this default.env can be overridden by env.local. +# All env in this default.env must not depend on any env in env.local. # Jupyter single-user server image export DOCKER_NOTEBOOK_IMAGE="pavics/workflow-tests:200803" export FINCH_IMAGE="birdhouse/finch:version-0.5.2" -export GENERIC_BIRD_IMAGE="$FINCH_IMAGE" -export GENERIC_BIRD_PORT="8010" -export GENERIC_BIRD_NAME="generic_bird" + +export THREDDS_IMAGE="unidata/thredds-docker:4.6.14" # Folder on the host to persist Jupyter user data (noteboooks, HOME settings) export JUPYTERHUB_USER_DATA_DIR="/data/jupyterhub_user_data" @@ -25,6 +25,12 @@ export JUPYTER_LOGIN_BANNER_BOTTOM_SECTION="" # Folder inside "proxy" container to drop extra monitoring config export CANARIE_MONITORING_EXTRA_CONF_DIR="/conf.d" +export THREDDS_ORGANIZATION="Birdhouse" + +export MAGPIE_DB_NAME="magpiedb" + +export VERIFY_SSL="true" + # Folder containing ssh deploy keys for all extra git repos # # Note when overriding this variable in env.local, do not use HOME environment diff --git a/birdhouse/deployment/certbotwrapper b/birdhouse/deployment/certbotwrapper index e76f54c6a..98ad655a1 100755 --- a/birdhouse/deployment/certbotwrapper +++ b/birdhouse/deployment/certbotwrapper @@ -18,7 +18,7 @@ THIS_FILE="`realpath "$0"`" THIS_DIR="`dirname "$THIS_FILE"`" # Default values -. $THIS_DIR/../common.env +. $THIS_DIR/../default.env if [ -e "$THIS_DIR/../env.local" ]; then # Override default values diff --git a/birdhouse/deployment/deploy.sh b/birdhouse/deployment/deploy.sh index 3e2084dca..f1648b208 100755 --- a/birdhouse/deployment/deploy.sh +++ b/birdhouse/deployment/deploy.sh @@ -121,10 +121,10 @@ done cd $COMPOSE_DIR -. ./common.env +. ./default.env set +x # hide passwd in env.local in logs -# reload again after common.env since env.local can override common.env +# reload again after default.env since env.local can override default.env . $ENV_LOCAL_FILE set -x @@ -162,10 +162,10 @@ done cd $COMPOSE_DIR # reload again after git pull because this file could be changed by the pull -. ./common.env +. ./default.env set +x # hide passwd in env.local in logs -# reload again after common.env since env.local can override common.env +# reload again after default.env since env.local can override default.env # (ex: JUPYTERHUB_USER_DATA_DIR) . $ENV_LOCAL_FILE set -x diff --git a/birdhouse/deployment/fix-write-perm b/birdhouse/deployment/fix-write-perm index 80a9f6a90..f115a995c 100755 --- a/birdhouse/deployment/fix-write-perm +++ b/birdhouse/deployment/fix-write-perm @@ -39,7 +39,7 @@ THIS_DIR="`dirname "$THIS_FILE"`" cd $THIS_DIR/../.. # Default values -. birdhouse/common.env +. birdhouse/default.env if [ -e "birdhouse/env.local" ]; then # Override default values diff --git a/birdhouse/deployment/install-deploy-notebook b/birdhouse/deployment/install-deploy-notebook index b6c956881..177e470d1 100755 --- a/birdhouse/deployment/install-deploy-notebook +++ b/birdhouse/deployment/install-deploy-notebook @@ -29,7 +29,7 @@ if [ ! -e "$REPO_ROOT/birdhouse/deployment/trigger-deploy-notebook" ]; then exit 2 fi -. "$REPO_ROOT/birdhouse/common.env" +. "$REPO_ROOT/birdhouse/default.env" if [ -f "$REPO_ROOT/birdhouse/env.local" ]; then # allow override of JUPYTERHUB_USER_DATA_DIR diff --git a/birdhouse/deployment/trigger-deploy-notebook b/birdhouse/deployment/trigger-deploy-notebook index 44fec249f..dd883672f 100755 --- a/birdhouse/deployment/trigger-deploy-notebook +++ b/birdhouse/deployment/trigger-deploy-notebook @@ -15,7 +15,7 @@ if [ -z "$JUPYTERHUB_USER_DATA_DIR" ]; then # running script manually (not with cron) source env.local file. COMPOSE_DIR="$(dirname -- "$(dirname -- "$(realpath "$0")")")" - . "$COMPOSE_DIR/common.env" # default JUPYTERHUB_USER_DATA_DIR + . "$COMPOSE_DIR/default.env" # default JUPYTERHUB_USER_DATA_DIR if [ -e "$COMPOSE_DIR/env.local" ]; then . "$COMPOSE_DIR/env.local" # optional override JUPYTERHUB_USER_DATA_DIR fi diff --git a/birdhouse/docker-compose.yml b/birdhouse/docker-compose.yml index 05f4a56ee..f56c48a4d 100644 --- a/birdhouse/docker-compose.yml +++ b/birdhouse/docker-compose.yml @@ -228,7 +228,7 @@ services: restart: always thredds: - image: unidata/thredds-docker:4.6.14 + image: ${THREDDS_IMAGE} container_name: thredds ports: - "8083:8080" diff --git a/birdhouse/env.local.example b/birdhouse/env.local.example index 216f8d4fc..67988cd60 100644 --- a/birdhouse/env.local.example +++ b/birdhouse/env.local.example @@ -31,10 +31,11 @@ export POSTGRES_MAGPIE_PASSWORD=postgres-qwerty # Optional vars ############################################################################# -# Extra dirs containing docker-compose-extra.yml file to override the default -# docker-compose.yml file. Inside each extra dir, there must be a -# docker-compose-extra.yml file. If this file is not found, the dir is -# ignored. +# Extra dirs possibly containing: +# * `docker-compose-extra.yml` file to override the default docker-compose.yml file +# * `default.env` file to provide extra defaults for each component +# * `pre-docker-compose-up` script to execute before `docker-compose up` +# * `post-docker-compose-up` script to execute after `docker-compose up` # # Useful to split configs into different dirs leveraging docker-compose # override capabilities, see https://docs.docker.com/compose/extends/. @@ -51,9 +52,19 @@ export POSTGRES_MAGPIE_PASSWORD=postgres-qwerty # * Assemble different combinations of components/functionalities by including # only the config/docker-compose fragment necessary. # -# Last dir/docker-compose fragment in the list have highest override precedence. -# Ex: last docker-compose volume mount to same destination win over the -# previous docker-compose volume mount. +# Last dir/component in the EXTRA_CONF_DIRS list have highest override +# precedence, example: +# +# * Last docker-compose volume mount to same destination win over the +# previous docker-compose volume mount. +# +# * Last default.env can change the values of all previous default.env. +# +# * Last pre/post docker-compose-up script can potentially undo actions +# from previous scripts. +# +# Suggested to keep the private-config-repo last in the list to be able to +# override anything. # # Format: space separated list of dirs # @@ -107,7 +118,7 @@ export POSTGRES_MAGPIE_PASSWORD=postgres-qwerty #export AUTODEPLOY_DEPLOY_KEY_ROOT_DIR="/path/to/ssh-deploy-keys-for-all-repos" # Frequency to trigger the various autodeploy tasks. -# See common.env for default. +# See default.env for default. # # For all possible syntax, see implementation at # https://github.com/Ouranosinc/docker-crontab/blob/3ac8cfa363b3f2ffdd0ead6089d355ff84521dc9/docker-entrypoint#L137-L184 @@ -207,7 +218,7 @@ export POSTGRES_MAGPIE_PASSWORD=postgres-qwerty ############################################################################# # # Below are Mandatory if monitoring component is enabled: -export GRAFANA_ADMIN_PASSWORD=changeme! +#export GRAFANA_ADMIN_PASSWORD=changeme! #export ALERTMANAGER_ADMIN_EMAIL_RECEIVER="user1@example.com,user2@example.com" #export SMTP_SERVER="smtp.example.com:25" diff --git a/birdhouse/optional-components/README.md b/birdhouse/optional-components/README.md index e1ab5b1af..135097644 100644 --- a/birdhouse/optional-components/README.md +++ b/birdhouse/optional-components/README.md @@ -17,15 +17,32 @@ How to enable this config in `env.local` (a copy from ## Emu WPS service for testing +Preconfigured for Emu but can also be used to quickly deploy any birds +temporarily without changing code. Good to preview new birds or test +alternative configuration of existing birds. + +No Postgres DB configured. If need Postgres DB, use generic_bird component +instead. + How to enable Emu in `env.local` (a copy from [`env.local.example`](../env.local.example)): * Add `./optional-components/emu` to `EXTRA_CONF_DIRS`. -* Set `EMU_IMAGE`. +* Optionally set `EMU_IMAGE`, `EMU_PORT`, + `EMU_NAME`, `EMU_INTERNAL_PORT`, + `EMU_WPS_OUTPUTS_VOL` in `env.local` for further customizations. + Default values are in [`emu/default.env`](emu/default.env). + +Emu service will be available at `http://PAVICS_FQDN:EMU_PORT/wps` or +`https://PAVICS_FQDN_PUBLIC/TWITCHER_PROTECTED_PATH/EMU_NAME` where +`PAVICS_FQDN`, `PAVICS_FQDN_PUBLIC` and `TWITCHER_PROTECTED_PATH` are defined +in your `env.local`. -Emu service will be available at `http://PAVICS_FQDN:8888/wps` or -`https://PAVICS_FQDN_PUBLIC/twitcher/ows/proxy/emu` where `PAVICS_FQDN` -and `PAVICS_FQDN_PUBLIC` are defined in your `env.local`. +Magpie will be automatically configured to give complete public anonymous +access for this Emu WPS service. + +Canarie monitoring will also be automatically configured for this Emu WPS +service. ## A second Thredds server for testing @@ -35,10 +52,19 @@ How to enable in `env.local` (a copy from * Add `./optional-components/testthredds` to `EXTRA_CONF_DIRS`. -Test Thredds service will be available at `http://PAVICS_FQDN:8084/testthredds` -or `https://PAVICS_FQDN_PUBLIC/testthredds` where `PAVICS_FQDN` and +* Optionally set `TESTTHREDDS_IMAGE`, `TESTTHREDDS_PORT`, + `TESTTHREDDS_CONTEXT_ROOT`, `TESTTHREDDS_WARFILE_NAME`, + `TESTTHREDDS_INTERNAL_PORT`, `TESTTHREDDS_NAME`, in `env.local` for further + customizations. Default values are in + [`testthredds/default.env`](testthredds/default.env). + +Test Thredds service will be available at +`http://PAVICS_FQDN:TESTTHREDDS_PORT/TESTTHREDDS_CONTEXT_ROOT` or +`https://PAVICS_FQDN_PUBLIC/TESTTHREDDS_CONTEXT_ROOT` where `PAVICS_FQDN` and `PAVICS_FQDN_PUBLIC` are defined in your `env.local`. +Use same docker image as regular Thredds by default but can be customized. + New container have new `TestDatasets` with volume-mount to `/data/testdatasets` on the host. So your testing `.nc` and `.ncml` files should be added to `/data/testdatasets` on the host for them to show up on this Test Thredds @@ -52,6 +78,9 @@ server. No Twitcher/Magpie access control, this Test Thredds is directly behind the Nginx proxy. +Canarie monitoring will also be automatically configured for this second +Thredds server. + ## A generic bird WPS service @@ -63,8 +92,10 @@ How to enable in `env.local` (a copy from * Add `./optional-components/generic_bird` to `EXTRA_CONF_DIRS`. -* Optionally set `GENERIC_BIRD_IMAGE`, `GENERIC_BIRD_PORT`, `GENERIC_BIRD_NAME` in `env.local` - for further customizations. Default values are in [`common.env`](../common.env). +* Optionally set `GENERIC_BIRD_IMAGE`, `GENERIC_BIRD_PORT`, + `GENERIC_BIRD_NAME`, `GENERIC_BIRD_INTERNAL_PORT`, + `GENERIC_BIRD_POSTGRES_IMAGE` in `env.local` for further customizations. + Default values are in [`generic_bird/default.env`](generic_bird/default.env). The WPS service will be available at `http://PAVICS_FQDN:GENERIC_BIRD_PORT/wps` or `https://PAVICS_FQDN_PUBLIC/TWITCHER_PROTECTED_PATH/GENERIC_BIRD_NAME` where diff --git a/birdhouse/optional-components/emu/default.env b/birdhouse/optional-components/emu/default.env new file mode 100644 index 000000000..9c0b17b24 --- /dev/null +++ b/birdhouse/optional-components/emu/default.env @@ -0,0 +1,23 @@ +# All env in this default.env can be overridden by env.local. +# All env in this default.env must not depend on any env in env.local. + +# Should have been "birdhouse/emu" but at the moment the current config only +# works with the "watchdog/jobqueue" branch so have to default to an image that +# works by default. +export EMU_IMAGE="tlvu/emu:watchdog" +export EMU_PORT="8888" +export EMU_INTERNAL_PORT="5000" +# name in Twitcher/Magpie and Canarie monitoring +export EMU_NAME="emu" +# set to 'testwps_outputs' when used together with testthredds component +export EMU_WPS_OUTPUTS_VOL="wps_outputs" + + + +# add vars only needed to be substituted in templates + +OPTIONAL_VARS=" + $OPTIONAL_VARS + \$EMU_PORT + \$EMU_NAME +" diff --git a/birdhouse/optional-components/emu/docker-compose-extra.yml b/birdhouse/optional-components/emu/docker-compose-extra.yml index 37bf37a3c..efdf03210 100644 --- a/birdhouse/optional-components/emu/docker-compose-extra.yml +++ b/birdhouse/optional-components/emu/docker-compose-extra.yml @@ -6,10 +6,10 @@ services: environment: - PYWPS_CFG=/wps.cfg ports: - - "8888:5000" + - "${EMU_PORT}:${EMU_INTERNAL_PORT}" volumes: - ./optional-components/emu/wps.cfg:/wps.cfg - - ${EMU_WPS_OUTPUTS_VOL:-wps_outputs}:/data/wpsoutputs + - ${EMU_WPS_OUTPUTS_VOL}:/data/wpsoutputs - /tmp restart: always diff --git a/birdhouse/optional-components/emu/emu-magpie-permission.cfg.template b/birdhouse/optional-components/emu/emu-magpie-permission.cfg.template index 7d3f1fbd4..8c61c79c9 100644 --- a/birdhouse/optional-components/emu/emu-magpie-permission.cfg.template +++ b/birdhouse/optional-components/emu/emu-magpie-permission.cfg.template @@ -1,15 +1,15 @@ permissions: - - service: emu + - service: ${EMU_NAME} permission: getcapabilities group: anonymous action: create - - service: emu + - service: ${EMU_NAME} permission: describeprocess group: anonymous action: create - - service: emu + - service: ${EMU_NAME} permission: execute group: anonymous action: create diff --git a/birdhouse/optional-components/emu/emu-magpie-provider.cfg.template b/birdhouse/optional-components/emu/emu-magpie-provider.cfg.template index 555fa32f9..737734723 100644 --- a/birdhouse/optional-components/emu/emu-magpie-provider.cfg.template +++ b/birdhouse/optional-components/emu/emu-magpie-provider.cfg.template @@ -1,7 +1,7 @@ providers: - emu: - url: http://${PAVICS_FQDN}:8888/wps - title: Emu + ${EMU_NAME}: + url: http://${PAVICS_FQDN}:${EMU_PORT}/wps + title: ${EMU_NAME} public: true c4i: false type: wps diff --git a/birdhouse/optional-components/emu/emu_canarie_api_monitoring.py.template b/birdhouse/optional-components/emu/emu_canarie_api_monitoring.py.template index 63b13e763..5b2180b71 100644 --- a/birdhouse/optional-components/emu/emu_canarie_api_monitoring.py.template +++ b/birdhouse/optional-components/emu/emu_canarie_api_monitoring.py.template @@ -1,12 +1,12 @@ SERVICES['node']['monitoring'].update({ - 'Emu-public': { + '${EMU_NAME}-public': { 'request': { - 'url': 'https://${PAVICS_FQDN_PUBLIC}/${TWITCHER_PROTECTED_PATH}/emu?service=WPS&version=1.0.0&request=GetCapabilities' + 'url': 'https://${PAVICS_FQDN_PUBLIC}/${TWITCHER_PROTECTED_PATH}/${EMU_NAME}?service=WPS&version=1.0.0&request=GetCapabilities' }, }, - 'Emu': { + '${EMU_NAME}': { 'request': { - 'url': 'http://${PAVICS_FQDN}:8888/wps?service=WPS&version=1.0.0&request=GetCapabilities' + 'url': 'http://${PAVICS_FQDN}:${EMU_PORT}/wps?service=WPS&version=1.0.0&request=GetCapabilities' } }, }) diff --git a/birdhouse/optional-components/generic_bird/default.env b/birdhouse/optional-components/generic_bird/default.env new file mode 100644 index 000000000..3771bd8d8 --- /dev/null +++ b/birdhouse/optional-components/generic_bird/default.env @@ -0,0 +1,19 @@ +# All env in this default.env can be overridden by env.local. +# All env in this default.env must not depend on any env in env.local. + +export GENERIC_BIRD_IMAGE="$FINCH_IMAGE" +export GENERIC_BIRD_PORT="8010" +export GENERIC_BIRD_INTERNAL_PORT="5000" +# name in Twitcher/Magpie and Canarie monitoring +export GENERIC_BIRD_NAME="generic_bird" +export GENERIC_BIRD_POSTGRES_IMAGE="postgres:10.12" + + + +# add vars only needed to be substituted in templates + +OPTIONAL_VARS=" + $OPTIONAL_VARS + \$GENERIC_BIRD_PORT + \$GENERIC_BIRD_NAME +" diff --git a/birdhouse/optional-components/generic_bird/docker-compose-extra.yml b/birdhouse/optional-components/generic_bird/docker-compose-extra.yml index 0db309244..0bc19f12c 100644 --- a/birdhouse/optional-components/generic_bird/docker-compose-extra.yml +++ b/birdhouse/optional-components/generic_bird/docker-compose-extra.yml @@ -6,7 +6,7 @@ services: environment: PYWPS_CFG: /wps.cfg ports: - - "${GENERIC_BIRD_PORT}:${GENERIC_BIRD_INTERNAL_PORT:-5000}" + - "${GENERIC_BIRD_PORT}:${GENERIC_BIRD_INTERNAL_PORT}" volumes: - ./optional-components/generic_bird/wps.cfg:/wps.cfg:ro - wps_outputs:/data/wpsoutputs @@ -16,7 +16,7 @@ services: restart: always postgres_generic_bird: - image: ${GENERIC_BIRD_POSTGRES_IMAGE:-postgres:10.12} + image: ${GENERIC_BIRD_POSTGRES_IMAGE} container_name: postgres_generic_bird volumes: - postgres_generic_bird:/var/lib/postgresql/data/pgdata diff --git a/birdhouse/optional-components/testthredds/.gitignore b/birdhouse/optional-components/testthredds/.gitignore index 369ba21e4..1059eec6e 100644 --- a/birdhouse/optional-components/testthredds/.gitignore +++ b/birdhouse/optional-components/testthredds/.gitignore @@ -1 +1,2 @@ catalog.xml +testthredds_canarie_api_monitoring.py diff --git a/birdhouse/optional-components/testthredds/catalog.xml.template b/birdhouse/optional-components/testthredds/catalog.xml.template index d60a10de7..0e6764091 100644 --- a/birdhouse/optional-components/testthredds/catalog.xml.template +++ b/birdhouse/optional-components/testthredds/catalog.xml.template @@ -4,13 +4,13 @@ xmlns:xlink="http://www.w3.org/1999/xlink" > - - - - - - - + + + + + + + diff --git a/birdhouse/optional-components/testthredds/conf.extra-service.d/testthredds-service.conf.template b/birdhouse/optional-components/testthredds/conf.extra-service.d/testthredds-service.conf.template index 7d231e9da..bd787ecfa 100644 --- a/birdhouse/optional-components/testthredds/conf.extra-service.d/testthredds-service.conf.template +++ b/birdhouse/optional-components/testthredds/conf.extra-service.d/testthredds-service.conf.template @@ -1,6 +1,6 @@ - location /testthredds/ { + location /${TESTTHREDDS_CONTEXT_ROOT}/ { # direct hit Thredds, bypassing Twitcher - proxy_pass http://testthredds:8080/testthredds/; + proxy_pass http://testthredds:${TESTTHREDDS_INTERNAL_PORT}/${TESTTHREDDS_CONTEXT_ROOT}/; proxy_set_header Host $host; proxy_set_header X-Forwarded-Proto $real_scheme; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; diff --git a/birdhouse/optional-components/testthredds/default.env b/birdhouse/optional-components/testthredds/default.env new file mode 100644 index 000000000..79cdccb15 --- /dev/null +++ b/birdhouse/optional-components/testthredds/default.env @@ -0,0 +1,24 @@ +# All env in this default.env can be overridden by env.local. +# All env in this default.env must not depend on any env in env.local. + +export TESTTHREDDS_IMAGE="$THREDDS_IMAGE" +export TESTTHREDDS_PORT="8084" +export TESTTHREDDS_INTERNAL_PORT="8080" +# context root for Nginx proxy and Thredds catalog +export TESTTHREDDS_CONTEXT_ROOT="testthredds" +# match TESTTHREDDS_CONTEXT_ROOT but replace '/' to '#' +export TESTTHREDDS_WARFILE_NAME="testthredds" +# name in Canarie monitoring +export TESTTHREDDS_NAME="testthredds" + + + +# add vars only needed to be substituted in templates + +OPTIONAL_VARS=" + $OPTIONAL_VARS + \$TESTTHREDDS_INTERNAL_PORT + \$TESTTHREDDS_CONTEXT_ROOT + \$TESTTHREDDS_PORT + \$TESTTHREDDS_NAME +" diff --git a/birdhouse/optional-components/testthredds/docker-compose-extra.yml b/birdhouse/optional-components/testthredds/docker-compose-extra.yml index 948ab958d..f7bd423fb 100644 --- a/birdhouse/optional-components/testthredds/docker-compose-extra.yml +++ b/birdhouse/optional-components/testthredds/docker-compose-extra.yml @@ -3,20 +3,23 @@ services: proxy: volumes: - ./optional-components/testthredds/conf.extra-service.d:/etc/nginx/conf.extra-service.d/testthredds:ro + - ./optional-components/testthredds/testthredds_canarie_api_monitoring.py:${CANARIE_MONITORING_EXTRA_CONF_DIR}/testthredds_canarie_api_monitoring.py:ro links: - testthredds testthredds: - image: unidata/thredds-docker:4.6.14 + image: ${TESTTHREDDS_IMAGE} container_name: testthredds ports: - - "8084:8080" + - "${TESTTHREDDS_PORT}:${TESTTHREDDS_INTERNAL_PORT}" env_file: - ./config/thredds/thredds.env environment: # for reconstructing proper URL back to user when Thredds behind proxy # because Twitcher eats the "Host" http header set by Nginx PAVICS_FQDN_PUBLIC: $PAVICS_FQDN_PUBLIC + WANTED_CONTEXT_ROOT: $TESTTHREDDS_CONTEXT_ROOT + WANTED_CONTEXT_ROOT_WARFILE_NAME: $TESTTHREDDS_WARFILE_NAME volumes: - testthredds_persistence:/usr/local/tomcat/content/thredds - /data/testdatasets:/pavics-testdata:ro @@ -27,7 +30,7 @@ services: - ./optional-components/testthredds/catalog.xml:/usr/local/tomcat/content/thredds/catalog.xml:ro - ./config/thredds/threddsConfig.xml:/usr/local/tomcat/content/thredds/threddsConfig.xml:ro - ./config/thredds/wmsConfig.xml:/usr/local/tomcat/content/thredds/wmsConfig.xml:ro - - ./optional-components/testthredds/entrypointwrapper:/entrypointwrapper:ro + - ./config/thredds/entrypointwrapper:/entrypointwrapper:ro entrypoint: /entrypointwrapper restart: always healthcheck: @@ -36,7 +39,7 @@ services: "CMD", "curl", "--fail", - "http://localhost:8080/testthredds/catalog.html", + "http://localhost:${TESTTHREDDS_INTERNAL_PORT}/${TESTTHREDDS_CONTEXT_ROOT}/catalog.html", ] volumes: diff --git a/birdhouse/optional-components/testthredds/entrypointwrapper b/birdhouse/optional-components/testthredds/entrypointwrapper deleted file mode 100755 index 357bbbad8..000000000 --- a/birdhouse/optional-components/testthredds/entrypointwrapper +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/sh -x - -CONF_FILE="/usr/local/tomcat/conf/server.xml" - -if ! grep ' relaxedQueryChars=' $CONF_FILE; then - # allow angle bracket in query params, ex: - # https://boreas.ouranos.ca/twitcher/ows/proxy/thredds/dodsC/birdhouse/testdata/tasmax_day_BCCAQv2+ANUSPLIN300_BNU-ESM_historical+rcp85_r1i1p1_19500101-21001231.nc.ascii?tasmax[0:1:0][0:1:0][0:1:0] - cp -v $CONF_FILE ${CONF_FILE}.bak.relaxedQueryChars.$$ - sed -i 's/$EXISTING_CONTEXT_ROOT@$WANTED_CONTEXT_ROOT@g" "$WEBAPPS_ROOT/$WANTED_CONTEXT_ROOT_WARFILE_NAME/WEB-INF/web.xml" -fi - - -# chain existing entrypoint -/entrypoint.sh catalina.sh run diff --git a/birdhouse/optional-components/testthredds/testthredds_canarie_api_monitoring.py.template b/birdhouse/optional-components/testthredds/testthredds_canarie_api_monitoring.py.template new file mode 100644 index 000000000..4978ab07f --- /dev/null +++ b/birdhouse/optional-components/testthredds/testthredds_canarie_api_monitoring.py.template @@ -0,0 +1,14 @@ +SERVICES['node']['monitoring'].update({ + '${TESTTHREDDS_NAME}-public': { + 'request': { + 'url': 'https://${PAVICS_FQDN_PUBLIC}/${TESTTHREDDS_CONTEXT_ROOT}/catalog.html' + }, + }, + '${TESTTHREDDS_NAME}': { + 'request': { + 'url': 'http://${PAVICS_FQDN}:${TESTTHREDDS_PORT}/${TESTTHREDDS_CONTEXT_ROOT}/catalog.html' + } + }, +}) + +# vi: tabstop=8 expandtab shiftwidth=4 softtabstop=4 syntax=python diff --git a/birdhouse/pavics-compose.sh b/birdhouse/pavics-compose.sh index 87289ecf8..100aa598d 100755 --- a/birdhouse/pavics-compose.sh +++ b/birdhouse/pavics-compose.sh @@ -25,8 +25,6 @@ VARS=' $POSTGRES_PAVICS_PASSWORD $POSTGRES_MAGPIE_USERNAME $POSTGRES_MAGPIE_PASSWORD - $ALERTMANAGER_ADMIN_EMAIL_RECEIVER - $SMTP_SERVER ' # list of vars to be substituted in template but they do not have to be set in @@ -48,24 +46,30 @@ OPTIONAL_VARS=' $AUTODEPLOY_PLATFORM_FREQUENCY $AUTODEPLOY_NOTEBOOK_FREQUENCY $AUTODEPLOY_EXTRA_SCHEDULER_JOBS - $GENERIC_BIRD_PORT - $GENERIC_BIRD_NAME - $ALERTMANAGER_EXTRA_GLOBAL - $ALERTMANAGER_EXTRA_ROUTES - $ALERTMANAGER_EXTRA_INHIBITION - $ALERTMANAGER_EXTRA_RECEIVERS ' # we switch to the real directory of the script, so it still works when used from $PATH # tip: ln -s /path/to/pavics-compose.sh ~/bin/ cd $(dirname $(readlink -f $0 || realpath $0)) -. ./common.env +. ./default.env # we source local configs, if present # we don't use usual .env filename, because docker-compose uses it [ -f env.local ] && . ./env.local +for adir in ${EXTRA_CONF_DIRS}; do + COMPONENT_DEFAULT_ENV="$adir/default.env" + if [ -f "$COMPONENT_DEFAULT_ENV" ]; then + echo "reading '$COMPONENT_DEFAULT_ENV'" + . "$COMPONENT_DEFAULT_ENV" + fi +done + +# Re-read env.local to make sure it can override ALL defaults from all +# components. +[ -f env.local ] && . ./env.local + for i in ${VARS} do v="${i}" @@ -108,21 +112,6 @@ else export INCLUDE_FOR_PORT_80="include /etc/nginx/conf.d/redirect-to-https.include;" fi -if [ -z "$THREDDS_ORGANIZATION" ]; then - # default value before instantiating template configs - export THREDDS_ORGANIZATION="Birdhouse" -fi - -if [ -z "$MAGPIE_DB_NAME" ]; then - # default value before instantiating template configs - export MAGPIE_DB_NAME="magpiedb" -fi - -if [ -z "$VERIFY_SSL" ]; then - # default value before instantiating template configs - export VERIFY_SSL="true" -fi - export AUTODEPLOY_EXTRA_REPOS_AS_DOCKER_VOLUMES="" for adir in $AUTODEPLOY_EXTRA_REPOS; do # 4 spaces in front of '--volume' is important @@ -148,9 +137,15 @@ if [ x"$1" = x"up" ]; then # create externally so nothing will delete these data volume automatically docker volume create jupyterhub_data_persistence # jupyterhub db and cookie secret docker volume create thredds_persistence # logs, cache - docker volume create prometheus_persistence # metrics db - docker volume create grafana_persistence # dashboard and config db - docker volume create alertmanager_persistence # storage + + for adir in ${EXTRA_CONF_DIRS}; do + COMPONENT_PRE_COMPOSE_UP="$adir/pre-docker-compose-up" + if [ -x "$COMPONENT_PRE_COMPOSE_UP" ]; then + echo "executing '$COMPONENT_PRE_COMPOSE_UP'" + sh -x "$COMPONENT_PRE_COMPOSE_UP" + fi + done + fi COMPOSE_CONF_LIST="-f docker-compose.yml" @@ -186,6 +181,14 @@ do docker exec ${postgres_id} /postgres-setup.sh fi + for adir in ${EXTRA_CONF_DIRS}; do + COMPONENT_POST_COMPOSE_UP="$adir/post-docker-compose-up" + if [ -x "$COMPONENT_POST_COMPOSE_UP" ]; then + echo "executing '$COMPONENT_POST_COMPOSE_UP'" + sh -x "$COMPONENT_POST_COMPOSE_UP" + fi + done + # Note: This command should stay last, as it can take a while depending on network and drive speeds # immediately cache the new notebook image for faster startup by JupyterHub docker pull ${DOCKER_NOTEBOOK_IMAGE} diff --git a/birdhouse/scripts/detect-user-install-in-jupyter-env b/birdhouse/scripts/detect-user-install-in-jupyter-env index dbd3e5a57..03ed00680 100755 --- a/birdhouse/scripts/detect-user-install-in-jupyter-env +++ b/birdhouse/scripts/detect-user-install-in-jupyter-env @@ -11,7 +11,7 @@ THIS_FILE="`realpath "$0"`" THIS_DIR="`dirname "$THIS_FILE"`" # Default values -. $THIS_DIR/../common.env +. $THIS_DIR/../default.env if [ -e "$THIS_DIR/../env.local" ]; then # Override default values diff --git a/birdhouse/scripts/migrate-jupyterhub-user-persistence b/birdhouse/scripts/migrate-jupyterhub-user-persistence index 50bfc6edc..56e921f2a 100755 --- a/birdhouse/scripts/migrate-jupyterhub-user-persistence +++ b/birdhouse/scripts/migrate-jupyterhub-user-persistence @@ -9,7 +9,7 @@ THIS_DIR="`dirname "$THIS_FILE"`" COMPOSE_DIR="`dirname "$THIS_DIR"`" # Default JUPYTERHUB_USER_DATA_DIR -. $COMPOSE_DIR/common.env +. $COMPOSE_DIR/default.env if [ -e "$COMPOSE_DIR/env.local" ]; then # Optional override JUPYTERHUB_USER_DATA_DIR