diff --git a/README.md b/README.md
index 7400c70a4..d2f3bf041 100644
--- a/README.md
+++ b/README.md
@@ -6,5 +6,6 @@ for a full-fledged production platform.
Power Analytics and Visualization for Climate Science - Powered by Birdhouse and other ESGF software
How to deploy the entire PAVICS platform, see the
-[README](birdhouse/README.md) and the
+[README](birdhouse/README.md) and the various extra components
+[README for extra core components](birdhouse/components/README.rst),
[README for optional components](birdhouse/optional-components/README.md).
diff --git a/birdhouse/components/monitoring/default.env b/birdhouse/components/monitoring/default.env
new file mode 100644
index 000000000..18b7856b5
--- /dev/null
+++ b/birdhouse/components/monitoring/default.env
@@ -0,0 +1,28 @@
+# All env in this default.env can be overridden by env.local.
+# All env in this default.env must not depend on any env in env.local.
+
+export GRAFANA_ADMIN_PASSWORD="changeme!"
+export ALERTMANAGER_ADMIN_EMAIL_RECEIVER="" # "user1@example.com,user2@example.com"
+export SMTP_SERVER="" # "smtp.example.com:25"
+export ALERTMANAGER_EXTRA_GLOBAL=""
+export ALERTMANAGER_EXTRA_ROUTES=""
+export ALERTMANAGER_EXTRA_INHIBITION=""
+export ALERTMANAGER_EXTRA_RECEIVERS=""
+
+
+
+# add vars only needed to be substituted in templates
+
+VARS="
+ $VARS
+ \$ALERTMANAGER_ADMIN_EMAIL_RECEIVER
+ \$SMTP_SERVER
+"
+
+OPTIONAL_VARS="
+ $OPTIONAL_VARS
+ \$ALERTMANAGER_EXTRA_GLOBAL
+ \$ALERTMANAGER_EXTRA_ROUTES
+ \$ALERTMANAGER_EXTRA_INHIBITION
+ \$ALERTMANAGER_EXTRA_RECEIVERS
+"
diff --git a/birdhouse/components/monitoring/pre-docker-compose-up b/birdhouse/components/monitoring/pre-docker-compose-up
new file mode 100755
index 000000000..71b2ce0b5
--- /dev/null
+++ b/birdhouse/components/monitoring/pre-docker-compose-up
@@ -0,0 +1,5 @@
+#!/bin/sh -x
+
+docker volume create prometheus_persistence # metrics db
+docker volume create grafana_persistence # dashboard and config db
+docker volume create alertmanager_persistence # storage
diff --git a/birdhouse/config/thredds/entrypointwrapper b/birdhouse/config/thredds/entrypointwrapper
index 62dcdb0c7..a1d617823 100755
--- a/birdhouse/config/thredds/entrypointwrapper
+++ b/birdhouse/config/thredds/entrypointwrapper
@@ -20,8 +20,12 @@ fi
# https://www.unidata.ucar.edu/software/tds/v4.6/reference/TomcatBehindProxyServer.html
WEBAPPS_ROOT="/usr/local/tomcat/webapps"
EXISTING_CONTEXT_ROOT="thredds"
-WANTED_CONTEXT_ROOT="twitcher/ows/proxy/thredds"
-WANTED_CONTEXT_ROOT_WARFILE_NAME="twitcher#ows#proxy#thredds"
+if [ -z "$WANTED_CONTEXT_ROOT" ]; then
+ WANTED_CONTEXT_ROOT="twitcher/ows/proxy/thredds"
+fi
+if [ -z "$WANTED_CONTEXT_ROOT_WARFILE_NAME" ]; then
+ WANTED_CONTEXT_ROOT_WARFILE_NAME="twitcher#ows#proxy#thredds"
+fi
if [ -d "$WEBAPPS_ROOT/$EXISTING_CONTEXT_ROOT" ]; then
mv "$WEBAPPS_ROOT/$EXISTING_CONTEXT_ROOT" "$WEBAPPS_ROOT/$WANTED_CONTEXT_ROOT_WARFILE_NAME"
diff --git a/birdhouse/common.env b/birdhouse/default.env
similarity index 82%
rename from birdhouse/common.env
rename to birdhouse/default.env
index 122eff96c..344f2183d 100644
--- a/birdhouse/common.env
+++ b/birdhouse/default.env
@@ -1,12 +1,12 @@
-# All env this common.env can be overridden by env.local.
+# All env in this default.env can be overridden by env.local.
+# All env in this default.env must not depend on any env in env.local.
# Jupyter single-user server image
export DOCKER_NOTEBOOK_IMAGE="pavics/workflow-tests:200803"
export FINCH_IMAGE="birdhouse/finch:version-0.5.2"
-export GENERIC_BIRD_IMAGE="$FINCH_IMAGE"
-export GENERIC_BIRD_PORT="8010"
-export GENERIC_BIRD_NAME="generic_bird"
+
+export THREDDS_IMAGE="unidata/thredds-docker:4.6.14"
# Folder on the host to persist Jupyter user data (noteboooks, HOME settings)
export JUPYTERHUB_USER_DATA_DIR="/data/jupyterhub_user_data"
@@ -25,6 +25,12 @@ export JUPYTER_LOGIN_BANNER_BOTTOM_SECTION=""
# Folder inside "proxy" container to drop extra monitoring config
export CANARIE_MONITORING_EXTRA_CONF_DIR="/conf.d"
+export THREDDS_ORGANIZATION="Birdhouse"
+
+export MAGPIE_DB_NAME="magpiedb"
+
+export VERIFY_SSL="true"
+
# Folder containing ssh deploy keys for all extra git repos
#
# Note when overriding this variable in env.local, do not use HOME environment
diff --git a/birdhouse/deployment/certbotwrapper b/birdhouse/deployment/certbotwrapper
index e76f54c6a..98ad655a1 100755
--- a/birdhouse/deployment/certbotwrapper
+++ b/birdhouse/deployment/certbotwrapper
@@ -18,7 +18,7 @@ THIS_FILE="`realpath "$0"`"
THIS_DIR="`dirname "$THIS_FILE"`"
# Default values
-. $THIS_DIR/../common.env
+. $THIS_DIR/../default.env
if [ -e "$THIS_DIR/../env.local" ]; then
# Override default values
diff --git a/birdhouse/deployment/deploy.sh b/birdhouse/deployment/deploy.sh
index 3e2084dca..f1648b208 100755
--- a/birdhouse/deployment/deploy.sh
+++ b/birdhouse/deployment/deploy.sh
@@ -121,10 +121,10 @@ done
cd $COMPOSE_DIR
-. ./common.env
+. ./default.env
set +x # hide passwd in env.local in logs
-# reload again after common.env since env.local can override common.env
+# reload again after default.env since env.local can override default.env
. $ENV_LOCAL_FILE
set -x
@@ -162,10 +162,10 @@ done
cd $COMPOSE_DIR
# reload again after git pull because this file could be changed by the pull
-. ./common.env
+. ./default.env
set +x # hide passwd in env.local in logs
-# reload again after common.env since env.local can override common.env
+# reload again after default.env since env.local can override default.env
# (ex: JUPYTERHUB_USER_DATA_DIR)
. $ENV_LOCAL_FILE
set -x
diff --git a/birdhouse/deployment/fix-write-perm b/birdhouse/deployment/fix-write-perm
index 80a9f6a90..f115a995c 100755
--- a/birdhouse/deployment/fix-write-perm
+++ b/birdhouse/deployment/fix-write-perm
@@ -39,7 +39,7 @@ THIS_DIR="`dirname "$THIS_FILE"`"
cd $THIS_DIR/../..
# Default values
-. birdhouse/common.env
+. birdhouse/default.env
if [ -e "birdhouse/env.local" ]; then
# Override default values
diff --git a/birdhouse/deployment/install-deploy-notebook b/birdhouse/deployment/install-deploy-notebook
index b6c956881..177e470d1 100755
--- a/birdhouse/deployment/install-deploy-notebook
+++ b/birdhouse/deployment/install-deploy-notebook
@@ -29,7 +29,7 @@ if [ ! -e "$REPO_ROOT/birdhouse/deployment/trigger-deploy-notebook" ]; then
exit 2
fi
-. "$REPO_ROOT/birdhouse/common.env"
+. "$REPO_ROOT/birdhouse/default.env"
if [ -f "$REPO_ROOT/birdhouse/env.local" ]; then
# allow override of JUPYTERHUB_USER_DATA_DIR
diff --git a/birdhouse/deployment/trigger-deploy-notebook b/birdhouse/deployment/trigger-deploy-notebook
index 44fec249f..dd883672f 100755
--- a/birdhouse/deployment/trigger-deploy-notebook
+++ b/birdhouse/deployment/trigger-deploy-notebook
@@ -15,7 +15,7 @@
if [ -z "$JUPYTERHUB_USER_DATA_DIR" ]; then
# running script manually (not with cron) source env.local file.
COMPOSE_DIR="$(dirname -- "$(dirname -- "$(realpath "$0")")")"
- . "$COMPOSE_DIR/common.env" # default JUPYTERHUB_USER_DATA_DIR
+ . "$COMPOSE_DIR/default.env" # default JUPYTERHUB_USER_DATA_DIR
if [ -e "$COMPOSE_DIR/env.local" ]; then
. "$COMPOSE_DIR/env.local" # optional override JUPYTERHUB_USER_DATA_DIR
fi
diff --git a/birdhouse/docker-compose.yml b/birdhouse/docker-compose.yml
index 05f4a56ee..f56c48a4d 100644
--- a/birdhouse/docker-compose.yml
+++ b/birdhouse/docker-compose.yml
@@ -228,7 +228,7 @@ services:
restart: always
thredds:
- image: unidata/thredds-docker:4.6.14
+ image: ${THREDDS_IMAGE}
container_name: thredds
ports:
- "8083:8080"
diff --git a/birdhouse/env.local.example b/birdhouse/env.local.example
index 216f8d4fc..67988cd60 100644
--- a/birdhouse/env.local.example
+++ b/birdhouse/env.local.example
@@ -31,10 +31,11 @@ export POSTGRES_MAGPIE_PASSWORD=postgres-qwerty
# Optional vars
#############################################################################
-# Extra dirs containing docker-compose-extra.yml file to override the default
-# docker-compose.yml file. Inside each extra dir, there must be a
-# docker-compose-extra.yml file. If this file is not found, the dir is
-# ignored.
+# Extra dirs possibly containing:
+# * `docker-compose-extra.yml` file to override the default docker-compose.yml file
+# * `default.env` file to provide extra defaults for each component
+# * `pre-docker-compose-up` script to execute before `docker-compose up`
+# * `post-docker-compose-up` script to execute after `docker-compose up`
#
# Useful to split configs into different dirs leveraging docker-compose
# override capabilities, see https://docs.docker.com/compose/extends/.
@@ -51,9 +52,19 @@ export POSTGRES_MAGPIE_PASSWORD=postgres-qwerty
# * Assemble different combinations of components/functionalities by including
# only the config/docker-compose fragment necessary.
#
-# Last dir/docker-compose fragment in the list have highest override precedence.
-# Ex: last docker-compose volume mount to same destination win over the
-# previous docker-compose volume mount.
+# Last dir/component in the EXTRA_CONF_DIRS list have highest override
+# precedence, example:
+#
+# * Last docker-compose volume mount to same destination win over the
+# previous docker-compose volume mount.
+#
+# * Last default.env can change the values of all previous default.env.
+#
+# * Last pre/post docker-compose-up script can potentially undo actions
+# from previous scripts.
+#
+# Suggested to keep the private-config-repo last in the list to be able to
+# override anything.
#
# Format: space separated list of dirs
#
@@ -107,7 +118,7 @@ export POSTGRES_MAGPIE_PASSWORD=postgres-qwerty
#export AUTODEPLOY_DEPLOY_KEY_ROOT_DIR="/path/to/ssh-deploy-keys-for-all-repos"
# Frequency to trigger the various autodeploy tasks.
-# See common.env for default.
+# See default.env for default.
#
# For all possible syntax, see implementation at
# https://github.com/Ouranosinc/docker-crontab/blob/3ac8cfa363b3f2ffdd0ead6089d355ff84521dc9/docker-entrypoint#L137-L184
@@ -207,7 +218,7 @@ export POSTGRES_MAGPIE_PASSWORD=postgres-qwerty
#############################################################################
#
# Below are Mandatory if monitoring component is enabled:
-export GRAFANA_ADMIN_PASSWORD=changeme!
+#export GRAFANA_ADMIN_PASSWORD=changeme!
#export ALERTMANAGER_ADMIN_EMAIL_RECEIVER="user1@example.com,user2@example.com"
#export SMTP_SERVER="smtp.example.com:25"
diff --git a/birdhouse/optional-components/README.md b/birdhouse/optional-components/README.md
index e1ab5b1af..135097644 100644
--- a/birdhouse/optional-components/README.md
+++ b/birdhouse/optional-components/README.md
@@ -17,15 +17,32 @@ How to enable this config in `env.local` (a copy from
## Emu WPS service for testing
+Preconfigured for Emu but can also be used to quickly deploy any birds
+temporarily without changing code. Good to preview new birds or test
+alternative configuration of existing birds.
+
+No Postgres DB configured. If need Postgres DB, use generic_bird component
+instead.
+
How to enable Emu in `env.local` (a copy from
[`env.local.example`](../env.local.example)):
* Add `./optional-components/emu` to `EXTRA_CONF_DIRS`.
-* Set `EMU_IMAGE`.
+* Optionally set `EMU_IMAGE`, `EMU_PORT`,
+ `EMU_NAME`, `EMU_INTERNAL_PORT`,
+ `EMU_WPS_OUTPUTS_VOL` in `env.local` for further customizations.
+ Default values are in [`emu/default.env`](emu/default.env).
+
+Emu service will be available at `http://PAVICS_FQDN:EMU_PORT/wps` or
+`https://PAVICS_FQDN_PUBLIC/TWITCHER_PROTECTED_PATH/EMU_NAME` where
+`PAVICS_FQDN`, `PAVICS_FQDN_PUBLIC` and `TWITCHER_PROTECTED_PATH` are defined
+in your `env.local`.
-Emu service will be available at `http://PAVICS_FQDN:8888/wps` or
-`https://PAVICS_FQDN_PUBLIC/twitcher/ows/proxy/emu` where `PAVICS_FQDN`
-and `PAVICS_FQDN_PUBLIC` are defined in your `env.local`.
+Magpie will be automatically configured to give complete public anonymous
+access for this Emu WPS service.
+
+Canarie monitoring will also be automatically configured for this Emu WPS
+service.
## A second Thredds server for testing
@@ -35,10 +52,19 @@ How to enable in `env.local` (a copy from
* Add `./optional-components/testthredds` to `EXTRA_CONF_DIRS`.
-Test Thredds service will be available at `http://PAVICS_FQDN:8084/testthredds`
-or `https://PAVICS_FQDN_PUBLIC/testthredds` where `PAVICS_FQDN` and
+* Optionally set `TESTTHREDDS_IMAGE`, `TESTTHREDDS_PORT`,
+ `TESTTHREDDS_CONTEXT_ROOT`, `TESTTHREDDS_WARFILE_NAME`,
+ `TESTTHREDDS_INTERNAL_PORT`, `TESTTHREDDS_NAME`, in `env.local` for further
+ customizations. Default values are in
+ [`testthredds/default.env`](testthredds/default.env).
+
+Test Thredds service will be available at
+`http://PAVICS_FQDN:TESTTHREDDS_PORT/TESTTHREDDS_CONTEXT_ROOT` or
+`https://PAVICS_FQDN_PUBLIC/TESTTHREDDS_CONTEXT_ROOT` where `PAVICS_FQDN` and
`PAVICS_FQDN_PUBLIC` are defined in your `env.local`.
+Use same docker image as regular Thredds by default but can be customized.
+
New container have new `TestDatasets` with volume-mount to `/data/testdatasets`
on the host. So your testing `.nc` and `.ncml` files should be added to
`/data/testdatasets` on the host for them to show up on this Test Thredds
@@ -52,6 +78,9 @@ server.
No Twitcher/Magpie access control, this Test Thredds is directly behind the
Nginx proxy.
+Canarie monitoring will also be automatically configured for this second
+Thredds server.
+
## A generic bird WPS service
@@ -63,8 +92,10 @@ How to enable in `env.local` (a copy from
* Add `./optional-components/generic_bird` to `EXTRA_CONF_DIRS`.
-* Optionally set `GENERIC_BIRD_IMAGE`, `GENERIC_BIRD_PORT`, `GENERIC_BIRD_NAME` in `env.local`
- for further customizations. Default values are in [`common.env`](../common.env).
+* Optionally set `GENERIC_BIRD_IMAGE`, `GENERIC_BIRD_PORT`,
+ `GENERIC_BIRD_NAME`, `GENERIC_BIRD_INTERNAL_PORT`,
+ `GENERIC_BIRD_POSTGRES_IMAGE` in `env.local` for further customizations.
+ Default values are in [`generic_bird/default.env`](generic_bird/default.env).
The WPS service will be available at `http://PAVICS_FQDN:GENERIC_BIRD_PORT/wps`
or `https://PAVICS_FQDN_PUBLIC/TWITCHER_PROTECTED_PATH/GENERIC_BIRD_NAME` where
diff --git a/birdhouse/optional-components/emu/default.env b/birdhouse/optional-components/emu/default.env
new file mode 100644
index 000000000..9c0b17b24
--- /dev/null
+++ b/birdhouse/optional-components/emu/default.env
@@ -0,0 +1,23 @@
+# All env in this default.env can be overridden by env.local.
+# All env in this default.env must not depend on any env in env.local.
+
+# Should have been "birdhouse/emu" but at the moment the current config only
+# works with the "watchdog/jobqueue" branch so have to default to an image that
+# works by default.
+export EMU_IMAGE="tlvu/emu:watchdog"
+export EMU_PORT="8888"
+export EMU_INTERNAL_PORT="5000"
+# name in Twitcher/Magpie and Canarie monitoring
+export EMU_NAME="emu"
+# set to 'testwps_outputs' when used together with testthredds component
+export EMU_WPS_OUTPUTS_VOL="wps_outputs"
+
+
+
+# add vars only needed to be substituted in templates
+
+OPTIONAL_VARS="
+ $OPTIONAL_VARS
+ \$EMU_PORT
+ \$EMU_NAME
+"
diff --git a/birdhouse/optional-components/emu/docker-compose-extra.yml b/birdhouse/optional-components/emu/docker-compose-extra.yml
index 37bf37a3c..efdf03210 100644
--- a/birdhouse/optional-components/emu/docker-compose-extra.yml
+++ b/birdhouse/optional-components/emu/docker-compose-extra.yml
@@ -6,10 +6,10 @@ services:
environment:
- PYWPS_CFG=/wps.cfg
ports:
- - "8888:5000"
+ - "${EMU_PORT}:${EMU_INTERNAL_PORT}"
volumes:
- ./optional-components/emu/wps.cfg:/wps.cfg
- - ${EMU_WPS_OUTPUTS_VOL:-wps_outputs}:/data/wpsoutputs
+ - ${EMU_WPS_OUTPUTS_VOL}:/data/wpsoutputs
- /tmp
restart: always
diff --git a/birdhouse/optional-components/emu/emu-magpie-permission.cfg.template b/birdhouse/optional-components/emu/emu-magpie-permission.cfg.template
index 7d3f1fbd4..8c61c79c9 100644
--- a/birdhouse/optional-components/emu/emu-magpie-permission.cfg.template
+++ b/birdhouse/optional-components/emu/emu-magpie-permission.cfg.template
@@ -1,15 +1,15 @@
permissions:
- - service: emu
+ - service: ${EMU_NAME}
permission: getcapabilities
group: anonymous
action: create
- - service: emu
+ - service: ${EMU_NAME}
permission: describeprocess
group: anonymous
action: create
- - service: emu
+ - service: ${EMU_NAME}
permission: execute
group: anonymous
action: create
diff --git a/birdhouse/optional-components/emu/emu-magpie-provider.cfg.template b/birdhouse/optional-components/emu/emu-magpie-provider.cfg.template
index 555fa32f9..737734723 100644
--- a/birdhouse/optional-components/emu/emu-magpie-provider.cfg.template
+++ b/birdhouse/optional-components/emu/emu-magpie-provider.cfg.template
@@ -1,7 +1,7 @@
providers:
- emu:
- url: http://${PAVICS_FQDN}:8888/wps
- title: Emu
+ ${EMU_NAME}:
+ url: http://${PAVICS_FQDN}:${EMU_PORT}/wps
+ title: ${EMU_NAME}
public: true
c4i: false
type: wps
diff --git a/birdhouse/optional-components/emu/emu_canarie_api_monitoring.py.template b/birdhouse/optional-components/emu/emu_canarie_api_monitoring.py.template
index 63b13e763..5b2180b71 100644
--- a/birdhouse/optional-components/emu/emu_canarie_api_monitoring.py.template
+++ b/birdhouse/optional-components/emu/emu_canarie_api_monitoring.py.template
@@ -1,12 +1,12 @@
SERVICES['node']['monitoring'].update({
- 'Emu-public': {
+ '${EMU_NAME}-public': {
'request': {
- 'url': 'https://${PAVICS_FQDN_PUBLIC}/${TWITCHER_PROTECTED_PATH}/emu?service=WPS&version=1.0.0&request=GetCapabilities'
+ 'url': 'https://${PAVICS_FQDN_PUBLIC}/${TWITCHER_PROTECTED_PATH}/${EMU_NAME}?service=WPS&version=1.0.0&request=GetCapabilities'
},
},
- 'Emu': {
+ '${EMU_NAME}': {
'request': {
- 'url': 'http://${PAVICS_FQDN}:8888/wps?service=WPS&version=1.0.0&request=GetCapabilities'
+ 'url': 'http://${PAVICS_FQDN}:${EMU_PORT}/wps?service=WPS&version=1.0.0&request=GetCapabilities'
}
},
})
diff --git a/birdhouse/optional-components/generic_bird/default.env b/birdhouse/optional-components/generic_bird/default.env
new file mode 100644
index 000000000..3771bd8d8
--- /dev/null
+++ b/birdhouse/optional-components/generic_bird/default.env
@@ -0,0 +1,19 @@
+# All env in this default.env can be overridden by env.local.
+# All env in this default.env must not depend on any env in env.local.
+
+export GENERIC_BIRD_IMAGE="$FINCH_IMAGE"
+export GENERIC_BIRD_PORT="8010"
+export GENERIC_BIRD_INTERNAL_PORT="5000"
+# name in Twitcher/Magpie and Canarie monitoring
+export GENERIC_BIRD_NAME="generic_bird"
+export GENERIC_BIRD_POSTGRES_IMAGE="postgres:10.12"
+
+
+
+# add vars only needed to be substituted in templates
+
+OPTIONAL_VARS="
+ $OPTIONAL_VARS
+ \$GENERIC_BIRD_PORT
+ \$GENERIC_BIRD_NAME
+"
diff --git a/birdhouse/optional-components/generic_bird/docker-compose-extra.yml b/birdhouse/optional-components/generic_bird/docker-compose-extra.yml
index 0db309244..0bc19f12c 100644
--- a/birdhouse/optional-components/generic_bird/docker-compose-extra.yml
+++ b/birdhouse/optional-components/generic_bird/docker-compose-extra.yml
@@ -6,7 +6,7 @@ services:
environment:
PYWPS_CFG: /wps.cfg
ports:
- - "${GENERIC_BIRD_PORT}:${GENERIC_BIRD_INTERNAL_PORT:-5000}"
+ - "${GENERIC_BIRD_PORT}:${GENERIC_BIRD_INTERNAL_PORT}"
volumes:
- ./optional-components/generic_bird/wps.cfg:/wps.cfg:ro
- wps_outputs:/data/wpsoutputs
@@ -16,7 +16,7 @@ services:
restart: always
postgres_generic_bird:
- image: ${GENERIC_BIRD_POSTGRES_IMAGE:-postgres:10.12}
+ image: ${GENERIC_BIRD_POSTGRES_IMAGE}
container_name: postgres_generic_bird
volumes:
- postgres_generic_bird:/var/lib/postgresql/data/pgdata
diff --git a/birdhouse/optional-components/testthredds/.gitignore b/birdhouse/optional-components/testthredds/.gitignore
index 369ba21e4..1059eec6e 100644
--- a/birdhouse/optional-components/testthredds/.gitignore
+++ b/birdhouse/optional-components/testthredds/.gitignore
@@ -1 +1,2 @@
catalog.xml
+testthredds_canarie_api_monitoring.py
diff --git a/birdhouse/optional-components/testthredds/catalog.xml.template b/birdhouse/optional-components/testthredds/catalog.xml.template
index d60a10de7..0e6764091 100644
--- a/birdhouse/optional-components/testthredds/catalog.xml.template
+++ b/birdhouse/optional-components/testthredds/catalog.xml.template
@@ -4,13 +4,13 @@
xmlns:xlink="http://www.w3.org/1999/xlink" >
-
-
-
-
-
-
-
+
+
+
+
+
+
+
diff --git a/birdhouse/optional-components/testthredds/conf.extra-service.d/testthredds-service.conf.template b/birdhouse/optional-components/testthredds/conf.extra-service.d/testthredds-service.conf.template
index 7d231e9da..bd787ecfa 100644
--- a/birdhouse/optional-components/testthredds/conf.extra-service.d/testthredds-service.conf.template
+++ b/birdhouse/optional-components/testthredds/conf.extra-service.d/testthredds-service.conf.template
@@ -1,6 +1,6 @@
- location /testthredds/ {
+ location /${TESTTHREDDS_CONTEXT_ROOT}/ {
# direct hit Thredds, bypassing Twitcher
- proxy_pass http://testthredds:8080/testthredds/;
+ proxy_pass http://testthredds:${TESTTHREDDS_INTERNAL_PORT}/${TESTTHREDDS_CONTEXT_ROOT}/;
proxy_set_header Host $host;
proxy_set_header X-Forwarded-Proto $real_scheme;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
diff --git a/birdhouse/optional-components/testthredds/default.env b/birdhouse/optional-components/testthredds/default.env
new file mode 100644
index 000000000..79cdccb15
--- /dev/null
+++ b/birdhouse/optional-components/testthredds/default.env
@@ -0,0 +1,24 @@
+# All env in this default.env can be overridden by env.local.
+# All env in this default.env must not depend on any env in env.local.
+
+export TESTTHREDDS_IMAGE="$THREDDS_IMAGE"
+export TESTTHREDDS_PORT="8084"
+export TESTTHREDDS_INTERNAL_PORT="8080"
+# context root for Nginx proxy and Thredds catalog
+export TESTTHREDDS_CONTEXT_ROOT="testthredds"
+# match TESTTHREDDS_CONTEXT_ROOT but replace '/' to '#'
+export TESTTHREDDS_WARFILE_NAME="testthredds"
+# name in Canarie monitoring
+export TESTTHREDDS_NAME="testthredds"
+
+
+
+# add vars only needed to be substituted in templates
+
+OPTIONAL_VARS="
+ $OPTIONAL_VARS
+ \$TESTTHREDDS_INTERNAL_PORT
+ \$TESTTHREDDS_CONTEXT_ROOT
+ \$TESTTHREDDS_PORT
+ \$TESTTHREDDS_NAME
+"
diff --git a/birdhouse/optional-components/testthredds/docker-compose-extra.yml b/birdhouse/optional-components/testthredds/docker-compose-extra.yml
index 948ab958d..f7bd423fb 100644
--- a/birdhouse/optional-components/testthredds/docker-compose-extra.yml
+++ b/birdhouse/optional-components/testthredds/docker-compose-extra.yml
@@ -3,20 +3,23 @@ services:
proxy:
volumes:
- ./optional-components/testthredds/conf.extra-service.d:/etc/nginx/conf.extra-service.d/testthredds:ro
+ - ./optional-components/testthredds/testthredds_canarie_api_monitoring.py:${CANARIE_MONITORING_EXTRA_CONF_DIR}/testthredds_canarie_api_monitoring.py:ro
links:
- testthredds
testthredds:
- image: unidata/thredds-docker:4.6.14
+ image: ${TESTTHREDDS_IMAGE}
container_name: testthredds
ports:
- - "8084:8080"
+ - "${TESTTHREDDS_PORT}:${TESTTHREDDS_INTERNAL_PORT}"
env_file:
- ./config/thredds/thredds.env
environment:
# for reconstructing proper URL back to user when Thredds behind proxy
# because Twitcher eats the "Host" http header set by Nginx
PAVICS_FQDN_PUBLIC: $PAVICS_FQDN_PUBLIC
+ WANTED_CONTEXT_ROOT: $TESTTHREDDS_CONTEXT_ROOT
+ WANTED_CONTEXT_ROOT_WARFILE_NAME: $TESTTHREDDS_WARFILE_NAME
volumes:
- testthredds_persistence:/usr/local/tomcat/content/thredds
- /data/testdatasets:/pavics-testdata:ro
@@ -27,7 +30,7 @@ services:
- ./optional-components/testthredds/catalog.xml:/usr/local/tomcat/content/thredds/catalog.xml:ro
- ./config/thredds/threddsConfig.xml:/usr/local/tomcat/content/thredds/threddsConfig.xml:ro
- ./config/thredds/wmsConfig.xml:/usr/local/tomcat/content/thredds/wmsConfig.xml:ro
- - ./optional-components/testthredds/entrypointwrapper:/entrypointwrapper:ro
+ - ./config/thredds/entrypointwrapper:/entrypointwrapper:ro
entrypoint: /entrypointwrapper
restart: always
healthcheck:
@@ -36,7 +39,7 @@ services:
"CMD",
"curl",
"--fail",
- "http://localhost:8080/testthredds/catalog.html",
+ "http://localhost:${TESTTHREDDS_INTERNAL_PORT}/${TESTTHREDDS_CONTEXT_ROOT}/catalog.html",
]
volumes:
diff --git a/birdhouse/optional-components/testthredds/entrypointwrapper b/birdhouse/optional-components/testthredds/entrypointwrapper
deleted file mode 100755
index 357bbbad8..000000000
--- a/birdhouse/optional-components/testthredds/entrypointwrapper
+++ /dev/null
@@ -1,33 +0,0 @@
-#!/bin/sh -x
-
-CONF_FILE="/usr/local/tomcat/conf/server.xml"
-
-if ! grep ' relaxedQueryChars=' $CONF_FILE; then
- # allow angle bracket in query params, ex:
- # https://boreas.ouranos.ca/twitcher/ows/proxy/thredds/dodsC/birdhouse/testdata/tasmax_day_BCCAQv2+ANUSPLIN300_BNU-ESM_historical+rcp85_r1i1p1_19500101-21001231.nc.ascii?tasmax[0:1:0][0:1:0][0:1:0]
- cp -v $CONF_FILE ${CONF_FILE}.bak.relaxedQueryChars.$$
- sed -i 's/$EXISTING_CONTEXT_ROOT@$WANTED_CONTEXT_ROOT@g" "$WEBAPPS_ROOT/$WANTED_CONTEXT_ROOT_WARFILE_NAME/WEB-INF/web.xml"
-fi
-
-
-# chain existing entrypoint
-/entrypoint.sh catalina.sh run
diff --git a/birdhouse/optional-components/testthredds/testthredds_canarie_api_monitoring.py.template b/birdhouse/optional-components/testthredds/testthredds_canarie_api_monitoring.py.template
new file mode 100644
index 000000000..4978ab07f
--- /dev/null
+++ b/birdhouse/optional-components/testthredds/testthredds_canarie_api_monitoring.py.template
@@ -0,0 +1,14 @@
+SERVICES['node']['monitoring'].update({
+ '${TESTTHREDDS_NAME}-public': {
+ 'request': {
+ 'url': 'https://${PAVICS_FQDN_PUBLIC}/${TESTTHREDDS_CONTEXT_ROOT}/catalog.html'
+ },
+ },
+ '${TESTTHREDDS_NAME}': {
+ 'request': {
+ 'url': 'http://${PAVICS_FQDN}:${TESTTHREDDS_PORT}/${TESTTHREDDS_CONTEXT_ROOT}/catalog.html'
+ }
+ },
+})
+
+# vi: tabstop=8 expandtab shiftwidth=4 softtabstop=4 syntax=python
diff --git a/birdhouse/pavics-compose.sh b/birdhouse/pavics-compose.sh
index 87289ecf8..100aa598d 100755
--- a/birdhouse/pavics-compose.sh
+++ b/birdhouse/pavics-compose.sh
@@ -25,8 +25,6 @@ VARS='
$POSTGRES_PAVICS_PASSWORD
$POSTGRES_MAGPIE_USERNAME
$POSTGRES_MAGPIE_PASSWORD
- $ALERTMANAGER_ADMIN_EMAIL_RECEIVER
- $SMTP_SERVER
'
# list of vars to be substituted in template but they do not have to be set in
@@ -48,24 +46,30 @@ OPTIONAL_VARS='
$AUTODEPLOY_PLATFORM_FREQUENCY
$AUTODEPLOY_NOTEBOOK_FREQUENCY
$AUTODEPLOY_EXTRA_SCHEDULER_JOBS
- $GENERIC_BIRD_PORT
- $GENERIC_BIRD_NAME
- $ALERTMANAGER_EXTRA_GLOBAL
- $ALERTMANAGER_EXTRA_ROUTES
- $ALERTMANAGER_EXTRA_INHIBITION
- $ALERTMANAGER_EXTRA_RECEIVERS
'
# we switch to the real directory of the script, so it still works when used from $PATH
# tip: ln -s /path/to/pavics-compose.sh ~/bin/
cd $(dirname $(readlink -f $0 || realpath $0))
-. ./common.env
+. ./default.env
# we source local configs, if present
# we don't use usual .env filename, because docker-compose uses it
[ -f env.local ] && . ./env.local
+for adir in ${EXTRA_CONF_DIRS}; do
+ COMPONENT_DEFAULT_ENV="$adir/default.env"
+ if [ -f "$COMPONENT_DEFAULT_ENV" ]; then
+ echo "reading '$COMPONENT_DEFAULT_ENV'"
+ . "$COMPONENT_DEFAULT_ENV"
+ fi
+done
+
+# Re-read env.local to make sure it can override ALL defaults from all
+# components.
+[ -f env.local ] && . ./env.local
+
for i in ${VARS}
do
v="${i}"
@@ -108,21 +112,6 @@ else
export INCLUDE_FOR_PORT_80="include /etc/nginx/conf.d/redirect-to-https.include;"
fi
-if [ -z "$THREDDS_ORGANIZATION" ]; then
- # default value before instantiating template configs
- export THREDDS_ORGANIZATION="Birdhouse"
-fi
-
-if [ -z "$MAGPIE_DB_NAME" ]; then
- # default value before instantiating template configs
- export MAGPIE_DB_NAME="magpiedb"
-fi
-
-if [ -z "$VERIFY_SSL" ]; then
- # default value before instantiating template configs
- export VERIFY_SSL="true"
-fi
-
export AUTODEPLOY_EXTRA_REPOS_AS_DOCKER_VOLUMES=""
for adir in $AUTODEPLOY_EXTRA_REPOS; do
# 4 spaces in front of '--volume' is important
@@ -148,9 +137,15 @@ if [ x"$1" = x"up" ]; then
# create externally so nothing will delete these data volume automatically
docker volume create jupyterhub_data_persistence # jupyterhub db and cookie secret
docker volume create thredds_persistence # logs, cache
- docker volume create prometheus_persistence # metrics db
- docker volume create grafana_persistence # dashboard and config db
- docker volume create alertmanager_persistence # storage
+
+ for adir in ${EXTRA_CONF_DIRS}; do
+ COMPONENT_PRE_COMPOSE_UP="$adir/pre-docker-compose-up"
+ if [ -x "$COMPONENT_PRE_COMPOSE_UP" ]; then
+ echo "executing '$COMPONENT_PRE_COMPOSE_UP'"
+ sh -x "$COMPONENT_PRE_COMPOSE_UP"
+ fi
+ done
+
fi
COMPOSE_CONF_LIST="-f docker-compose.yml"
@@ -186,6 +181,14 @@ do
docker exec ${postgres_id} /postgres-setup.sh
fi
+ for adir in ${EXTRA_CONF_DIRS}; do
+ COMPONENT_POST_COMPOSE_UP="$adir/post-docker-compose-up"
+ if [ -x "$COMPONENT_POST_COMPOSE_UP" ]; then
+ echo "executing '$COMPONENT_POST_COMPOSE_UP'"
+ sh -x "$COMPONENT_POST_COMPOSE_UP"
+ fi
+ done
+
# Note: This command should stay last, as it can take a while depending on network and drive speeds
# immediately cache the new notebook image for faster startup by JupyterHub
docker pull ${DOCKER_NOTEBOOK_IMAGE}
diff --git a/birdhouse/scripts/detect-user-install-in-jupyter-env b/birdhouse/scripts/detect-user-install-in-jupyter-env
index dbd3e5a57..03ed00680 100755
--- a/birdhouse/scripts/detect-user-install-in-jupyter-env
+++ b/birdhouse/scripts/detect-user-install-in-jupyter-env
@@ -11,7 +11,7 @@ THIS_FILE="`realpath "$0"`"
THIS_DIR="`dirname "$THIS_FILE"`"
# Default values
-. $THIS_DIR/../common.env
+. $THIS_DIR/../default.env
if [ -e "$THIS_DIR/../env.local" ]; then
# Override default values
diff --git a/birdhouse/scripts/migrate-jupyterhub-user-persistence b/birdhouse/scripts/migrate-jupyterhub-user-persistence
index 50bfc6edc..56e921f2a 100755
--- a/birdhouse/scripts/migrate-jupyterhub-user-persistence
+++ b/birdhouse/scripts/migrate-jupyterhub-user-persistence
@@ -9,7 +9,7 @@ THIS_DIR="`dirname "$THIS_FILE"`"
COMPOSE_DIR="`dirname "$THIS_DIR"`"
# Default JUPYTERHUB_USER_DATA_DIR
-. $COMPOSE_DIR/common.env
+. $COMPOSE_DIR/default.env
if [ -e "$COMPOSE_DIR/env.local" ]; then
# Optional override JUPYTERHUB_USER_DATA_DIR