From 88de526b916ecc95a03160348cc5b21d87683b8f Mon Sep 17 00:00:00 2001 From: Pete Meyer Date: Mon, 11 Mar 2019 17:27:29 -0400 Subject: [PATCH 01/22] only use dcm version 1x --- conf/docker-dcm/0prep.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/conf/docker-dcm/0prep.sh b/conf/docker-dcm/0prep.sh index 98cff3c805a..425a6cd204a 100755 --- a/conf/docker-dcm/0prep.sh +++ b/conf/docker-dcm/0prep.sh @@ -1,3 +1,4 @@ #!/bin/sh +DCM_VERSION=0.5 -wget https://github.com/sbgrid/data-capture-module/releases/download/0.5/dcm-0.5-0.noarch.rpm +wget https://github.com/sbgrid/data-capture-module/releases/download/${DCM_VERSION}/dcm-${DCM_VERSION}-0.noarch.rpm From f8daf61cc72dc84276f77b4735ebc669ce748b08 Mon Sep 17 00:00:00 2001 From: Pete Meyer Date: Mon, 11 Mar 2019 17:32:31 -0400 Subject: [PATCH 02/22] config file org --- conf/docker-dcm/{ => cfg/dcm}/bashrc | 0 conf/docker-dcm/{ => cfg/dcm}/entrypoint-dcm.sh | 0 conf/docker-dcm/{ => cfg/dcm}/healthcheck-dcm.sh | 0 conf/docker-dcm/{ => cfg/dcm}/rq-init-d | 0 conf/docker-dcm/{ => cfg/dcm}/test_install.sh | 0 conf/docker-dcm/dcmsrv.dockerfile | 10 +++++----- 6 files changed, 5 insertions(+), 5 deletions(-) rename conf/docker-dcm/{ => cfg/dcm}/bashrc (100%) rename conf/docker-dcm/{ => cfg/dcm}/entrypoint-dcm.sh (100%) rename conf/docker-dcm/{ => cfg/dcm}/healthcheck-dcm.sh (100%) rename conf/docker-dcm/{ => cfg/dcm}/rq-init-d (100%) rename conf/docker-dcm/{ => cfg/dcm}/test_install.sh (100%) diff --git a/conf/docker-dcm/bashrc b/conf/docker-dcm/cfg/dcm/bashrc similarity index 100% rename from conf/docker-dcm/bashrc rename to conf/docker-dcm/cfg/dcm/bashrc diff --git a/conf/docker-dcm/entrypoint-dcm.sh b/conf/docker-dcm/cfg/dcm/entrypoint-dcm.sh similarity index 100% rename from conf/docker-dcm/entrypoint-dcm.sh rename to conf/docker-dcm/cfg/dcm/entrypoint-dcm.sh diff --git a/conf/docker-dcm/healthcheck-dcm.sh b/conf/docker-dcm/cfg/dcm/healthcheck-dcm.sh similarity index 100% rename from conf/docker-dcm/healthcheck-dcm.sh rename to conf/docker-dcm/cfg/dcm/healthcheck-dcm.sh diff --git a/conf/docker-dcm/rq-init-d b/conf/docker-dcm/cfg/dcm/rq-init-d similarity index 100% rename from conf/docker-dcm/rq-init-d rename to conf/docker-dcm/cfg/dcm/rq-init-d diff --git a/conf/docker-dcm/test_install.sh b/conf/docker-dcm/cfg/dcm/test_install.sh similarity index 100% rename from conf/docker-dcm/test_install.sh rename to conf/docker-dcm/cfg/dcm/test_install.sh diff --git a/conf/docker-dcm/dcmsrv.dockerfile b/conf/docker-dcm/dcmsrv.dockerfile index 4ec6fb86c06..9989fa3a89d 100644 --- a/conf/docker-dcm/dcmsrv.dockerfile +++ b/conf/docker-dcm/dcmsrv.dockerfile @@ -3,17 +3,17 @@ FROM centos:6 RUN yum install -y epel-release ARG RPMFILE=dcm-0.5-0.noarch.rpm COPY ${RPMFILE} /tmp/ -COPY bashrc /root/.bashrc -COPY test_install.sh /root/ +COPY cfg/dcm/bashrc /root/.bashrc +COPY cfg/dcm/test_install.sh /root/ RUN yum localinstall -y /tmp/${RPMFILE} RUN pip install -r /opt/dcm/requirements.txt RUN pip install awscli==1.15.75 run export PATH=~/.local/bin:$PATH RUN /root/test_install.sh -COPY rq-init-d /etc/init.d/rq +COPY cfg/dcm/rq-init-d /etc/init.d/rq RUN useradd glassfish -COPY entrypoint-dcm.sh / -COPY healthcheck-dcm.sh / +COPY cfg/dcm/entrypoint-dcm.sh / +COPY cfg/dcm/healthcheck-dcm.sh / EXPOSE 80 EXPOSE 22 VOLUME /hold From ceff696de4935b2f68b31cb91dbe82682012c285 Mon Sep 17 00:00:00 2001 From: Pete Meyer Date: Mon, 11 Mar 2019 17:35:54 -0400 Subject: [PATCH 03/22] try out import of dockerfile / integration test configs from rsal repo --- conf/docker-dcm/cfg/rsal/entrypoint-rsal.sh | 5 + .../docker-dcm/cfg/rsal/lighttpd-modules.conf | 174 ++++++++++++++++++ conf/docker-dcm/cfg/rsal/lighttpd.conf | 43 +++++ conf/docker-dcm/cfg/rsal/rsyncd.conf | 8 + conf/docker-dcm/rsalsrv.dockerfile | 20 ++ 5 files changed, 250 insertions(+) create mode 100755 conf/docker-dcm/cfg/rsal/entrypoint-rsal.sh create mode 100644 conf/docker-dcm/cfg/rsal/lighttpd-modules.conf create mode 100644 conf/docker-dcm/cfg/rsal/lighttpd.conf create mode 100644 conf/docker-dcm/cfg/rsal/rsyncd.conf create mode 100644 conf/docker-dcm/rsalsrv.dockerfile diff --git a/conf/docker-dcm/cfg/rsal/entrypoint-rsal.sh b/conf/docker-dcm/cfg/rsal/entrypoint-rsal.sh new file mode 100755 index 00000000000..92466c3bd4b --- /dev/null +++ b/conf/docker-dcm/cfg/rsal/entrypoint-rsal.sh @@ -0,0 +1,5 @@ +#!/bin/sh + +#/usr/bin/rsync --no-detach --daemon --config /etc/rsyncd.conf +/usr/bin/rsync --daemon --config /etc/rsyncd.conf +lighttpd -D -f /etc/lighttpd/lighttpd.conf diff --git a/conf/docker-dcm/cfg/rsal/lighttpd-modules.conf b/conf/docker-dcm/cfg/rsal/lighttpd-modules.conf new file mode 100644 index 00000000000..cdb1438af82 --- /dev/null +++ b/conf/docker-dcm/cfg/rsal/lighttpd-modules.conf @@ -0,0 +1,174 @@ +####################################################################### +## +## ansible managed +# +## Modules to load +## ----------------- +## +## at least mod_access and mod_accesslog should be loaded +## all other module should only be loaded if really neccesary +## +## - saves some time +## - saves memory +## +## the default module set contains: +## +## "mod_indexfile", "mod_dirlisting", "mod_staticfile" +## +## you dont have to include those modules in your list +## +## Modules, which are pulled in via conf.d/*.conf +## +## NOTE: the order of modules is important. +## +## - mod_accesslog -> conf.d/access_log.conf +## - mod_compress -> conf.d/compress.conf +## - mod_status -> conf.d/status.conf +## - mod_webdav -> conf.d/webdav.conf +## - mod_cml -> conf.d/cml.conf +## - mod_evhost -> conf.d/evhost.conf +## - mod_simple_vhost -> conf.d/simple_vhost.conf +## - mod_mysql_vhost -> conf.d/mysql_vhost.conf +## - mod_trigger_b4_dl -> conf.d/trigger_b4_dl.conf +## - mod_userdir -> conf.d/userdir.conf +## - mod_rrdtool -> conf.d/rrdtool.conf +## - mod_ssi -> conf.d/ssi.conf +## - mod_cgi -> conf.d/cgi.conf +## - mod_scgi -> conf.d/scgi.conf +## - mod_fastcgi -> conf.d/fastcgi.conf +## - mod_proxy -> conf.d/proxy.conf +## - mod_secdownload -> conf.d/secdownload.conf +## - mod_expire -> conf.d/expire.conf +## + +server.modules = ( + "mod_access", +# "mod_alias", +# "mod_auth", +# "mod_evasive", +# "mod_redirect", +# "mod_rewrite", +# "mod_setenv", +# "mod_usertrack", +) + +## +####################################################################### + +####################################################################### +## +## Config for various Modules +## + +## +## mod_ssi +## +#include "conf.d/ssi.conf" + +## +## mod_status +## +#include "conf.d/status.conf" + +## +## mod_webdav +## +#include "conf.d/webdav.conf" + +## +## mod_compress +## +#include "conf.d/compress.conf" + +## +## mod_userdir +## +#include "conf.d/userdir.conf" + +## +## mod_magnet +## +#include "conf.d/magnet.conf" + +## +## mod_cml +## +#include "conf.d/cml.conf" + +## +## mod_rrdtool +## +#include "conf.d/rrdtool.conf" + +## +## mod_proxy +## +#include "conf.d/proxy.conf" + +## +## mod_expire +## +#include "conf.d/expire.conf" + +## +## mod_secdownload +## +#include "conf.d/secdownload.conf" + +## +####################################################################### + +####################################################################### +## +## CGI modules +## + +## +## SCGI (mod_scgi) +## +#include "conf.d/scgi.conf" + +## +## FastCGI (mod_fastcgi) +## +#include "conf.d/fastcgi.conf" + +## +## plain old CGI (mod_cgi) +## +include "conf.d/cgi.conf" + +## +####################################################################### + +####################################################################### +## +## VHost Modules +## +## Only load ONE of them! +## ======================== +## + +## +## You can use conditionals for vhosts aswell. +## +## see http://www.lighttpd.net/documentation/configuration.html +## + +## +## mod_evhost +## +#include "conf.d/evhost.conf" + +## +## mod_simple_vhost +## +#include "conf.d/simple_vhost.conf" + +## +## mod_mysql_vhost +## +#include "conf.d/mysql_vhost.conf" + +## +####################################################################### diff --git a/conf/docker-dcm/cfg/rsal/lighttpd.conf b/conf/docker-dcm/cfg/rsal/lighttpd.conf new file mode 100644 index 00000000000..5874d60eb48 --- /dev/null +++ b/conf/docker-dcm/cfg/rsal/lighttpd.conf @@ -0,0 +1,43 @@ +## lighttpd configuration customized for RSAL; centos7 + +# refuse connections not from frontend or localhost +# DO NOT HAVE THIS OPEN TO THE WORLD!!! +#$HTTP["remoteip"] !~ "192.168.2.2|127.0.0.1" { +#url.access-deny = ("") +#} +server.breakagelog = "/var/log/lighttpd/breakage.log" + +####################################################################### +## +## Some Variable definition which will make chrooting easier. +## +## if you add a variable here. Add the corresponding variable in the +## chroot example aswell. +## +var.log_root = "/var/log/lighttpd" +var.server_root = "/opt/rsal/api" +var.state_dir = "/var/run" +var.home_dir = "/var/lib/lighttpd" +var.conf_dir = "/etc/lighttpd" + +var.cache_dir = "/var/cache/lighttpd" +var.socket_dir = home_dir + "/sockets" +include "modules.conf" +server.port = 80 +server.use-ipv6 = "disable" +server.username = "lighttpd" +server.groupname = "lighttpd" +server.document-root = server_root +server.pid-file = state_dir + "/lighttpd.pid" +server.errorlog = log_root + "/error.log" +include "conf.d/access_log.conf" +include "conf.d/debug.conf" +server.event-handler = "linux-sysepoll" +server.network-backend = "linux-sendfile" +server.stat-cache-engine = "simple" +server.max-connections = 1024 +static-file.exclude-extensions = ( ".php", ".pl", ".fcgi", ".scgi" ) +include "conf.d/mime.conf" +include "conf.d/dirlisting.conf" +server.follow-symlink = "enable" +server.upload-dirs = ( "/var/tmp" ) diff --git a/conf/docker-dcm/cfg/rsal/rsyncd.conf b/conf/docker-dcm/cfg/rsal/rsyncd.conf new file mode 100644 index 00000000000..5a15ab28a12 --- /dev/null +++ b/conf/docker-dcm/cfg/rsal/rsyncd.conf @@ -0,0 +1,8 @@ +lock file=/var/run/rsync.lock +log file=/var/log/rsyncd.log +pid file=/var/log/rsyncd.pid + +[10.5072] + path=/public/ + read only=yes + diff --git a/conf/docker-dcm/rsalsrv.dockerfile b/conf/docker-dcm/rsalsrv.dockerfile new file mode 100644 index 00000000000..1276c193b33 --- /dev/null +++ b/conf/docker-dcm/rsalsrv.dockerfile @@ -0,0 +1,20 @@ +FROM centos:7 +ARG RPMFILE=rsal-0.1-0.noarch.rpm +RUN yum update; yum install -y epel-release +COPY dist/${RPMFILE} /tmp/ +RUN yum localinstall -y /tmp/${RPMFILE} +COPY cfg/rsal/rsyncd.conf /etc/rsyncd.conf +COPY cfg/rsal/entrypoint.sh / +COPY cfg/rsal/lighttpd-modules.conf /etc/lighttpd/modules.conf +COPY cfg/rsal/lighttpd.conf /etc/lighttpd/lighttpd.conf +RUN mkdir -p /public/FK2 +RUN pip2 install -r /opt/rsal/scn/requirements.txt +#COPY doc/testdata/ /hold/ +ARG DV_HOST=http://dv_srv:8080 +ARG DV_API_KEY=burrito +ENV DV_HOST ${DV_HOST} +ENV DV_API_KEY ${DV_API_KEY} +EXPOSE 873 +EXPOSE 80 +HEALTHCHECK CMD curl --fail http://localhost/hw.py || exit 1 +CMD ["/entrypoint.sh"] From deb07a1a0e7d8fb7597a9e2d6c68cfa9f3233ff2 Mon Sep 17 00:00:00 2001 From: Pete Meyer Date: Mon, 11 Mar 2019 17:38:44 -0400 Subject: [PATCH 04/22] path typo --- conf/docker-dcm/rsalsrv.dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conf/docker-dcm/rsalsrv.dockerfile b/conf/docker-dcm/rsalsrv.dockerfile index 1276c193b33..2385076dcdc 100644 --- a/conf/docker-dcm/rsalsrv.dockerfile +++ b/conf/docker-dcm/rsalsrv.dockerfile @@ -1,10 +1,10 @@ FROM centos:7 ARG RPMFILE=rsal-0.1-0.noarch.rpm RUN yum update; yum install -y epel-release -COPY dist/${RPMFILE} /tmp/ +COPY ${RPMFILE} /tmp/ RUN yum localinstall -y /tmp/${RPMFILE} COPY cfg/rsal/rsyncd.conf /etc/rsyncd.conf -COPY cfg/rsal/entrypoint.sh / +COPY cfg/rsal/entrypoint-rsal.sh / COPY cfg/rsal/lighttpd-modules.conf /etc/lighttpd/modules.conf COPY cfg/rsal/lighttpd.conf /etc/lighttpd/lighttpd.conf RUN mkdir -p /public/FK2 From 3fafc5b9d75977f6a9f317b275f210ab5c0d573c Mon Sep 17 00:00:00 2001 From: Pete Meyer Date: Mon, 11 Mar 2019 17:39:29 -0400 Subject: [PATCH 05/22] check for dcm rpm before downloading --- conf/docker-dcm/0prep.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/conf/docker-dcm/0prep.sh b/conf/docker-dcm/0prep.sh index 425a6cd204a..4eb79709052 100755 --- a/conf/docker-dcm/0prep.sh +++ b/conf/docker-dcm/0prep.sh @@ -1,4 +1,9 @@ #!/bin/sh DCM_VERSION=0.5 +RSAL_VERSION=0.1 + +if [ ! -e dcm-${DCM_VERSION}-0.noarch.rpm ]; then + wget https://github.com/sbgrid/data-capture-module/releases/download/${DCM_VERSION}/dcm-${DCM_VERSION}-0.noarch.rpm +fi + -wget https://github.com/sbgrid/data-capture-module/releases/download/${DCM_VERSION}/dcm-${DCM_VERSION}-0.noarch.rpm From 99d662ce20aaa704eafb0a9a2baa44954338f5ca Mon Sep 17 00:00:00 2001 From: Pete Meyer Date: Mon, 11 Mar 2019 17:40:31 -0400 Subject: [PATCH 06/22] todo for rsal release rpm download --- conf/docker-dcm/0prep.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/conf/docker-dcm/0prep.sh b/conf/docker-dcm/0prep.sh index 4eb79709052..ca5c84826b1 100755 --- a/conf/docker-dcm/0prep.sh +++ b/conf/docker-dcm/0prep.sh @@ -6,4 +6,6 @@ if [ ! -e dcm-${DCM_VERSION}-0.noarch.rpm ]; then wget https://github.com/sbgrid/data-capture-module/releases/download/${DCM_VERSION}/dcm-${DCM_VERSION}-0.noarch.rpm fi - +if [ ! -e rsal-${RSAL_VERSION}-0.noarch.rpm ] ;then + echo "TODO - rsal release url not configured yet" +fi From 148c9e68a102408db3e146f0315f43ee02a9132f Mon Sep 17 00:00:00 2001 From: Pete Meyer Date: Mon, 11 Mar 2019 18:59:05 -0400 Subject: [PATCH 07/22] fix entrypoint typo --- conf/docker-dcm/rsalsrv.dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/docker-dcm/rsalsrv.dockerfile b/conf/docker-dcm/rsalsrv.dockerfile index 2385076dcdc..844432afe6b 100644 --- a/conf/docker-dcm/rsalsrv.dockerfile +++ b/conf/docker-dcm/rsalsrv.dockerfile @@ -4,7 +4,7 @@ RUN yum update; yum install -y epel-release COPY ${RPMFILE} /tmp/ RUN yum localinstall -y /tmp/${RPMFILE} COPY cfg/rsal/rsyncd.conf /etc/rsyncd.conf -COPY cfg/rsal/entrypoint-rsal.sh / +COPY cfg/rsal/entrypoint-rsal.sh /entrypoint.sh COPY cfg/rsal/lighttpd-modules.conf /etc/lighttpd/modules.conf COPY cfg/rsal/lighttpd.conf /etc/lighttpd/lighttpd.conf RUN mkdir -p /public/FK2 From 1292ca853d9bf648e289549a3f0659d7619c7855 Mon Sep 17 00:00:00 2001 From: Pete Meyer Date: Mon, 11 Mar 2019 18:59:23 -0400 Subject: [PATCH 08/22] iterating on compose file --- conf/docker-dcm/docker-compose.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/conf/docker-dcm/docker-compose.yml b/conf/docker-dcm/docker-compose.yml index 36c951a832a..49d4467d349 100644 --- a/conf/docker-dcm/docker-compose.yml +++ b/conf/docker-dcm/docker-compose.yml @@ -10,6 +10,21 @@ services: container_name: dcmsrv volumes: - hold:/hold + rsalsrv: + build: + context: . + dockerfile: rsalsrv.dockerfile + container_name: rsalsrv +# image: rsalrepo_rsal + volumes: + - hold:/hold + - ./:/mnt + environment: + DV_HOST: http://dvsrv:8080 + DV_APIKEY: burrito + ports: + - "8889:80" + - "873:873" dvsrv: build: context: . @@ -17,6 +32,7 @@ services: container_name: dvsrv volumes: - hold:/hold + - ./:/mnt ports: - "8083:8080" - "8084:80" From 5f7b08752170fac0a1a33208d22ad2da74796889 Mon Sep 17 00:00:00 2001 From: Pete Meyer Date: Mon, 11 Mar 2019 19:00:00 -0400 Subject: [PATCH 09/22] minor doc updates --- conf/docker-dcm/readme.txt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/conf/docker-dcm/readme.txt b/conf/docker-dcm/readme.txt index 36f56370dbb..c0c792a968f 100644 --- a/conf/docker-dcm/readme.txt +++ b/conf/docker-dcm/readme.txt @@ -7,8 +7,10 @@ Setup: - download/prepare dependencies: `./0prep.sh` - build dcm/dv0dcm images with docker-compose: `docker-compose -f docker-compose.yml build` - start containers: `docker-compose -f docker-compose.yml up -d` -- wait for container to show "healthy" (aka - `docker ps`), then wait another 4-5 minutes (even though it shows healthy, glassfish is still standing itself up), then run dataverse app installation: `docker exec -it dvsrv /opt/dv/install.bash` +- wait for container to show "healthy" (aka - `docker ps`), then run dataverse app installation: `docker exec -it dvsrv /opt/dv/install.bash` +- for development, you probably want to use the `FAKE` DOI provider: `docker exec -it dvsrv /opt/dv/configure_doi.bash` - configure dataverse application to use DCM: `docker exec -it dvsrv /opt/dv/configure_dcm.sh` +- TODO configure dataverse application to use RSAL (if desired) Operation: The dataverse installation is accessible at `http://localhost:8084`. From cde0e671e11f71addec91561d853de5b69fc3ae4 Mon Sep 17 00:00:00 2001 From: Pete Meyer Date: Tue, 12 Mar 2019 11:43:58 -0400 Subject: [PATCH 10/22] initial rsal config script --- conf/docker-dcm/configure_rsal.sh | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100755 conf/docker-dcm/configure_rsal.sh diff --git a/conf/docker-dcm/configure_rsal.sh b/conf/docker-dcm/configure_rsal.sh new file mode 100755 index 00000000000..5b55beb501a --- /dev/null +++ b/conf/docker-dcm/configure_rsal.sh @@ -0,0 +1,15 @@ +#!/bin/sh + +fn=rsal-workflow2.json +# needs an actual IP (vs a hostname) for whitelist +rsalip=`dig +short rsalsrv` + +# create workflow +curl -s -X POST -H "Content-type: application/json" -d @${fn} "http://localhost:8080/api/admin/workflows" + +# put rsal on the whitelist +curl -X PUT -d "127.0.0.1;${rsalip}" "http://localhost:8080/api/admin/workflows/ip-whitelist" + +# set workflow as default +curl -X PUT -d "1" "http://localhost:8080/api/admin/workflows/default/PrePublishDataset" + From 73fa08fbfab8e3c554f7ac1825cac0fb10758408 Mon Sep 17 00:00:00 2001 From: Pete Meyer Date: Tue, 12 Mar 2019 11:47:03 -0400 Subject: [PATCH 11/22] workflow file --- conf/docker-dcm/rsal-workflow2.json | 31 +++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 conf/docker-dcm/rsal-workflow2.json diff --git a/conf/docker-dcm/rsal-workflow2.json b/conf/docker-dcm/rsal-workflow2.json new file mode 100644 index 00000000000..322d3ecbcf7 --- /dev/null +++ b/conf/docker-dcm/rsal-workflow2.json @@ -0,0 +1,31 @@ +{ + "name": "RSAL file move for publication", + "steps": [ + { + "provider":":internal", + "stepType":"log", + "parameters": { + "message": "Pre-http request" + } + }, + { + "provider":":internal", + "stepType":"http/sr", + "parameters": { + "url":"http://rsalsrv/rr.py", + "method":"POST", + "contentType":"text/plain", + "body":"${invocationId}\ndataset.id=${dataset.id}\ndataset.identifier=${dataset.identifier}\ndataset.globalId=${dataset.globalId}", + "expectedResponse":"OK.*", + "rollbackMethod":"DELETE" + } + }, + { + "provider":":internal", + "stepType":"log", + "parameters": { + "message": "Post-http request" + } + } + ] +} From 09e7e3309e120812558e0c023d4257e7c8a30b0f Mon Sep 17 00:00:00 2001 From: Pete Meyer Date: Tue, 12 Mar 2019 11:47:19 -0400 Subject: [PATCH 12/22] update dv dockerfile --- conf/docker-dcm/dv0dcm.dockerfile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/conf/docker-dcm/dv0dcm.dockerfile b/conf/docker-dcm/dv0dcm.dockerfile index 25981496d2a..a1c97c5a99b 100644 --- a/conf/docker-dcm/dv0dcm.dockerfile +++ b/conf/docker-dcm/dv0dcm.dockerfile @@ -1,4 +1,7 @@ # dv0 assumed to be image name for docker-aio FROM dv0 +RUN yum install -y bind-utils COPY configure_dcm.sh /opt/dv/ +COPY configure_rsal.sh /opt/dv/ +COPY rsal-workflow2.json /opt/dv/ VOLUME /hold From 1e1e4375f5b5e930426f2528b2caf968cca5a0bd Mon Sep 17 00:00:00 2001 From: Pete Meyer Date: Tue, 12 Mar 2019 11:54:51 -0400 Subject: [PATCH 13/22] iterate on docs --- conf/docker-dcm/readme.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/conf/docker-dcm/readme.txt b/conf/docker-dcm/readme.txt index c0c792a968f..7f0d4f42e0d 100644 --- a/conf/docker-dcm/readme.txt +++ b/conf/docker-dcm/readme.txt @@ -10,12 +10,13 @@ Setup: - wait for container to show "healthy" (aka - `docker ps`), then run dataverse app installation: `docker exec -it dvsrv /opt/dv/install.bash` - for development, you probably want to use the `FAKE` DOI provider: `docker exec -it dvsrv /opt/dv/configure_doi.bash` - configure dataverse application to use DCM: `docker exec -it dvsrv /opt/dv/configure_dcm.sh` -- TODO configure dataverse application to use RSAL (if desired) +- configure dataverse application to use RSAL (if desired): `docker exec -it dvsrv /opt/dv/configure_rsal.sh` Operation: The dataverse installation is accessible at `http://localhost:8084`. The `dcm_client` container is intended to be used for executing transfer scripts, and `conf/docker-dcm` is available at `/mnt` inside the container; this container can be accessed with `docker exec -it dcm_client bash`. The DCM cron job is NOT configured here; for development purposes the DCM checks can be run manually with `docker exec -it dcmsrv /opt/dcm/scn/post_upload.bash`. +The RSAL cron job is similarly NOT configured; for development purposes `docker exec -it rsalsrv /opt/rsal/scn/pub.py` can be run manually. Cleanup: From 1af4f0a913e0b1ec573f7aa54ca38171c5b2f092 Mon Sep 17 00:00:00 2001 From: Pete Meyer Date: Tue, 12 Mar 2019 14:04:50 -0400 Subject: [PATCH 14/22] local access / storage sites --- conf/docker-dcm/configure_rsal.sh | 6 ++++++ conf/docker-dcm/dv0dcm.dockerfile | 2 +- conf/docker-dcm/publish_major.bash | 17 +++++++++++++++++ conf/docker-dcm/site-primary.json | 6 ++++++ conf/docker-dcm/site-remote.json | 6 ++++++ 5 files changed, 36 insertions(+), 1 deletion(-) create mode 100755 conf/docker-dcm/publish_major.bash create mode 100644 conf/docker-dcm/site-primary.json create mode 100644 conf/docker-dcm/site-remote.json diff --git a/conf/docker-dcm/configure_rsal.sh b/conf/docker-dcm/configure_rsal.sh index 5b55beb501a..5db43a34381 100755 --- a/conf/docker-dcm/configure_rsal.sh +++ b/conf/docker-dcm/configure_rsal.sh @@ -13,3 +13,9 @@ curl -X PUT -d "127.0.0.1;${rsalip}" "http://localhost:8080/api/admin/workflows/ # set workflow as default curl -X PUT -d "1" "http://localhost:8080/api/admin/workflows/default/PrePublishDataset" +# local access path +curl -X PUT -d "/hpc/storage" "http://localhost:8080/api/admin/settings/:LocalDataAccessPath" + +# storage sites +curl -X POST -H "Content-type: application/json" --upload-file site-primary.json "http://localhost:8080/api/admin/storageSites" +curl -X POST -H "Content-type: application/json" --upload-file site-remote.json "http://localhost:8080/api/admin/storageSites" diff --git a/conf/docker-dcm/dv0dcm.dockerfile b/conf/docker-dcm/dv0dcm.dockerfile index a1c97c5a99b..021534c8978 100644 --- a/conf/docker-dcm/dv0dcm.dockerfile +++ b/conf/docker-dcm/dv0dcm.dockerfile @@ -3,5 +3,5 @@ FROM dv0 RUN yum install -y bind-utils COPY configure_dcm.sh /opt/dv/ COPY configure_rsal.sh /opt/dv/ -COPY rsal-workflow2.json /opt/dv/ +COPY rsal-workflow2.json site-primary.json site-remote.json /opt/dv/ VOLUME /hold diff --git a/conf/docker-dcm/publish_major.bash b/conf/docker-dcm/publish_major.bash new file mode 100755 index 00000000000..6a3fd1288ca --- /dev/null +++ b/conf/docker-dcm/publish_major.bash @@ -0,0 +1,17 @@ +#!/usr/bin/env bash + +# publish dataset based on database id + +dset_id=$1 +if [ -z "$dset_id" ]; then + echo "no dataset id specified, bailing out" + exit 1 +fi + +k_d=burrito + +h=http://dvsrv + +curl -X POST -H "X-Dataverse-key: ${k_d}" "${h}/api/datasets/${dset_id}/actions/:publish?type=major" + + diff --git a/conf/docker-dcm/site-primary.json b/conf/docker-dcm/site-primary.json new file mode 100644 index 00000000000..0b96d94560d --- /dev/null +++ b/conf/docker-dcm/site-primary.json @@ -0,0 +1,6 @@ +{ + "hostname": "rsal", + "name": "LibraScholar University", + "primaryStorage": true, + "transferProtocols": "rsync,posix" +} diff --git a/conf/docker-dcm/site-remote.json b/conf/docker-dcm/site-remote.json new file mode 100644 index 00000000000..d47c3ef4dda --- /dev/null +++ b/conf/docker-dcm/site-remote.json @@ -0,0 +1,6 @@ +{ + "hostname": "remote.libra.research", + "name": "LibraResearch Institute", + "primaryStorage": false, + "transferProtocols": "rsync" +} From 6fbcd7ac079f41db4ea4781e3d64c6279a32af5f Mon Sep 17 00:00:00 2001 From: Pete Meyer Date: Tue, 12 Mar 2019 14:09:00 -0400 Subject: [PATCH 15/22] make primary site url work in example --- conf/docker-dcm/site-primary.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/docker-dcm/site-primary.json b/conf/docker-dcm/site-primary.json index 0b96d94560d..35b217edffd 100644 --- a/conf/docker-dcm/site-primary.json +++ b/conf/docker-dcm/site-primary.json @@ -1,5 +1,5 @@ { - "hostname": "rsal", + "hostname": "rsalsrv", "name": "LibraScholar University", "primaryStorage": true, "transferProtocols": "rsync,posix" From 95de39d1e86ed5afbce6501dec6625619953a60a Mon Sep 17 00:00:00 2001 From: Pete Meyer Date: Tue, 12 Mar 2019 17:35:48 -0400 Subject: [PATCH 16/22] update docs --- conf/docker-dcm/readme.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/docker-dcm/readme.txt b/conf/docker-dcm/readme.txt index 7f0d4f42e0d..a70cf1be397 100644 --- a/conf/docker-dcm/readme.txt +++ b/conf/docker-dcm/readme.txt @@ -7,7 +7,7 @@ Setup: - download/prepare dependencies: `./0prep.sh` - build dcm/dv0dcm images with docker-compose: `docker-compose -f docker-compose.yml build` - start containers: `docker-compose -f docker-compose.yml up -d` -- wait for container to show "healthy" (aka - `docker ps`), then run dataverse app installation: `docker exec -it dvsrv /opt/dv/install.bash` +- wait for container to show "healthy" (aka - `docker ps`), then run dataverse app installation: `docker exec -it dvsrv /opt/dv/install.bash` (until https://github.com/IQSS/dataverse/issues/5374 is sorted, it may be preferable to connect to the container and run configuration scripts inside - e.g. `docker exec -it dvsrv bash ; ./install.bash`) - for development, you probably want to use the `FAKE` DOI provider: `docker exec -it dvsrv /opt/dv/configure_doi.bash` - configure dataverse application to use DCM: `docker exec -it dvsrv /opt/dv/configure_dcm.sh` - configure dataverse application to use RSAL (if desired): `docker exec -it dvsrv /opt/dv/configure_rsal.sh` From b13363f8907b5077301deb63af89aecf8732604a Mon Sep 17 00:00:00 2001 From: Pete Meyer Date: Tue, 12 Mar 2019 17:36:38 -0400 Subject: [PATCH 17/22] switch readme to markdown --- conf/docker-dcm/{readme.txt => readme.md} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename conf/docker-dcm/{readme.txt => readme.md} (100%) diff --git a/conf/docker-dcm/readme.txt b/conf/docker-dcm/readme.md similarity index 100% rename from conf/docker-dcm/readme.txt rename to conf/docker-dcm/readme.md From a2d37fd2c373e5da462482d9c98797b78ed2590f Mon Sep 17 00:00:00 2001 From: Pete Meyer Date: Tue, 12 Mar 2019 17:36:56 -0400 Subject: [PATCH 18/22] typo fix --- conf/docker-dcm/readme.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/docker-dcm/readme.md b/conf/docker-dcm/readme.md index a70cf1be397..b6fca1451a3 100644 --- a/conf/docker-dcm/readme.md +++ b/conf/docker-dcm/readme.md @@ -2,7 +2,7 @@ This docker-compose setup is intended for use in development, small scale evalua Setup: -- build docker-aio image with name dv0 as described in ../docker-aio` (don't start up the docker image or run setupIT.bash) +- build docker-aio image with name dv0 as described in `../docker-aio` (don't start up the docker image or run setupIT.bash) - work in the `conf/docker-dcm` directory for below commands - download/prepare dependencies: `./0prep.sh` - build dcm/dv0dcm images with docker-compose: `docker-compose -f docker-compose.yml build` From a7fc30def590ea18fa06dec7a9f6ab84c3f9e002 Mon Sep 17 00:00:00 2001 From: Pete Meyer Date: Tue, 12 Mar 2019 17:42:54 -0400 Subject: [PATCH 19/22] initial guides update --- doc/sphinx-guides/source/developers/big-data-support.rst | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/developers/big-data-support.rst b/doc/sphinx-guides/source/developers/big-data-support.rst index 06917073b32..ee7d4ff3c65 100644 --- a/doc/sphinx-guides/source/developers/big-data-support.rst +++ b/doc/sphinx-guides/source/developers/big-data-support.rst @@ -114,7 +114,7 @@ Docker Image Set-up ^^^^^^^^^^^^^^^^^^^ - Install docker if you do not have it -- Follow these steps (extracted from ``docker-aio/readme.md`` & ``docker-dcm/readme.txt``) : +- Follow these steps (extracted from ``docker-aio/readme.md`` & ``docker-dcm/readme.md``) : - ``cd conf/docker-aio`` and run ``./0prep_deps.sh`` to create Glassfish and Solr tarballs in conf/docker-aio/dv/deps. - Run ``./1prep.sh`` @@ -225,6 +225,12 @@ Additional DCM docker development tips Repository Storage Abstraction Layer (RSAL) ------------------------------------------- + +Steps to set up a RSAL via Docker for Development +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +- See ``docker-dcm/readme.md`` + Configuring the RSAL Mock ~~~~~~~~~~~~~~~~~~~~~~~~~ From fd7482016de380539ff3923d4223951082d2e17e Mon Sep 17 00:00:00 2001 From: Pete Meyer Date: Tue, 12 Mar 2019 18:02:27 -0400 Subject: [PATCH 20/22] un-mess guides from bad merge --- .../source/developers/big-data-support.rst | 91 ++++--------------- 1 file changed, 18 insertions(+), 73 deletions(-) diff --git a/doc/sphinx-guides/source/developers/big-data-support.rst b/doc/sphinx-guides/source/developers/big-data-support.rst index ee7d4ff3c65..b9159bf0f45 100644 --- a/doc/sphinx-guides/source/developers/big-data-support.rst +++ b/doc/sphinx-guides/source/developers/big-data-support.rst @@ -16,7 +16,7 @@ Data Capture Module (DCM) is an experimental component that allows users to uplo Install a DCM ~~~~~~~~~~~~~ -Installation instructions can be found at https://github.com/sbgrid/data-capture-module . Note that a shared filesystem (posix or AWS S3) between Dataverse and your DCM is required. You cannot use a DCM with Swift at this point in time. +Installation instructions can be found at https://github.com/sbgrid/data-capture-module/blob/master/doc/installation.md. Note that shared storage (posix or AWS S3) between Dataverse and your DCM is required. You cannot use a DCM with Swift at this point in time. .. FIXME: Explain what ``dataverse.files.dcm-s3-bucket-name`` is for and what it has to do with ``dataverse.files.s3-bucket-name``. @@ -57,31 +57,8 @@ Here's the syntax for sending the JSON. Steps to set up a DCM mock for Development ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Install Flask. +See instructions at https://github.com/sbgrid/data-capture-module/blob/master/doc/mock.md -Download and run the mock. You will be cloning the https://github.com/sbgrid/data-capture-module repo. - -- ``git clone git://github.com/sbgrid/data-capture-module.git`` -- ``cd data-capture-module/api`` -- ``./dev_mock.sh`` - -If you see an error about not having Flask installed, install it as explained below. - -On Mac, you can install Flask with: - -- ``mkvirtualenv mockdcm`` -- ``pip install -r requirements-mock.txt`` - -On Ubuntu/Debian, you can install Flask with: - -- ``sudo apt install python-pip`` (will install python as well) -- ``pip install flask`` - -Once you have Flask installed, try running the dev mock script again: - -- ``./dev_mock.sh`` - -This should spin up the DCM mock on port 5000. Add Dataverse settings to use mock (same as using DCM, noted above): @@ -96,6 +73,8 @@ At this point you should be able to download a placeholder rsync script. Dataver - ``echo "hello" > file1.txt`` - ``shasum file1.txt > files.sha`` + + Now the files are in place and you need to send JSON to Dataverse with a success or failure message as described above. Make a copy of ``doc/sphinx-guides/source/_static/installation/files/root/big-data-support/checksumValidationSuccess.json`` and put the identifier in place such as "X1METO" under "uploadFolder"). Then use curl as described above to send the JSON. Troubleshooting @@ -113,36 +92,17 @@ If you need a fully operating DCM client for development purposes, these steps w Docker Image Set-up ^^^^^^^^^^^^^^^^^^^ +See https://github.com/IQSS/dataverse/blob/develop/conf/docker-dcm/readme.txt + - Install docker if you do not have it -- Follow these steps (extracted from ``docker-aio/readme.md`` & ``docker-dcm/readme.md``) : - - - ``cd conf/docker-aio`` and run ``./0prep_deps.sh`` to create Glassfish and Solr tarballs in conf/docker-aio/dv/deps. - - Run ``./1prep.sh`` - - Build the docker image: ``docker build -t dv0 -f c7.dockerfile .`` - - ``cd ../docker-dcm`` and run ``./0prep.sh`` - - Build dcm/dv0dcm images with docker-compose: ``docker-compose -f docker-compose.yml build`` - - Start containers: ``docker-compose -f docker-compose.yml up -d`` - - Wait for container to show "healthy" (aka - ``docker ps``), then wait another 5 minutes (even though it shows healthy, glassfish is still standing itself up). Then run Dataverse app installation: ``docker exec -it dvsrv /opt/dv/install.bash`` - - Configure Dataverse application to use DCM (run from outside the container): ``docker exec -it dvsrv /opt/dv/configure_dcm.sh`` - - The Dataverse installation is accessible at ``http://localhost:8084``. - - You may need to change the DoiProvider inside dvsrv (ezid does not work): - - - ``curl -X DELETE -d EZID "localhost:8080/api/admin/settings/:DoiProvider"`` - - ``curl -X PUT -d DataCite "localhost:8080/api/admin/settings/:DoiProvider"`` - - Also change the doi.baseUrlString, doi.username, doi.password Optional steps for setting up the S3 Docker DCM Variant ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - Before: the default bucket for DCM to hold files in S3 is named test-dcm. It is coded into `post_upload_s3.bash` (line 30). Change to a different bucket if needed. -- Add AWS bucket info to dcmsrv - - - You need a credentials files in ~/.aws - - - ``mkdir ~/.aws`` - - ``yum install nano`` (or use a different editor below) - - ``nano ~/.aws/credentials`` and add these contents with your keys: + - Add AWS bucket info to dcmsrv + - Add AWS credentials to ``~/.aws/credentials`` - ``[default]`` - ``aws_access_key_id =`` - ``aws_secret_access_key =`` @@ -156,16 +116,12 @@ Optional steps for setting up the S3 Docker DCM Variant - ``./asadmin create-jvm-options "\-Ddataverse.files.storage-driver-id=s3"`` - Add AWS bucket info to Dataverse - - - ``mkdir ~/.aws`` - - ``yum install nano`` (or use a different editor below) - - ``nano ~/.aws/credentials`` and add these contents with your keys: - + - Add AWS credentials to ``~/.aws/credentials`` - ``[default]`` - ``aws_access_key_id =`` - ``aws_secret_access_key =`` - - Also: ``nano ~/.aws/config`` to create a region file. Add these contents: + - Also: set region in ``~/.aws/config`` to create a region file. Add these contents: - ``[default]`` - ``region = us-east-1`` @@ -188,30 +144,26 @@ Using the DCM Docker Containers For using these commands, you will need to connect to the shell prompt inside various containers (e.g. ``docker exec -it dvsrv /bin/bash``) - Create a dataset and download rsync upload script -- Upload script to dcm_client (if needed, you can probably do all the actions for create/download inside dcm_client) - - ``docker cp ~/Downloads/upload-FK2_NN49YM.bash dcm_client:/tmp`` + - connect to client container: ``docker exec -it dcm_client bash`` + - create dataset: ``cd /mnt ; ./create.bash`` ; this will echo the database ID to stdout + - download transfer script: ``./get_transfer.bash $database_id_from_create_script`` + - execute the transfer script: ``bash ./upload-${database_id_from-create_script}.bash`` , and follow instructions from script. -- Create a folder of files to upload (files can be empty) - Run script - - e.g. ``bash ./upload-FK2_NN49YM.bash`` + - e.g. ``bash ./upload-3.bash`` (``3`` being the database id from earlier commands in this example). - Manually run post upload script on dcmsrv - - for posix implementation: ``bash ./opt/dcm/scn/post_upload.bash`` - - for S3 implementation: ``bash ./opt/dcm/scn/post_upload_s3.bash`` + - for posix implementation: ``docker exec -it dcmsrv /opt/dcm/scn/post_upload.bash`` + - for S3 implementation: ``docker exec -it dcmsrv /opt/dcm/scn/post_upload_s3.bash`` Additional DCM docker development tips ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - You can completely blow away all the docker images with these commands (including non DCM ones!) - - - ``docker stop dvsrv`` - - ``docker stop dcm_client`` - - ``docker stop dcmsrv`` - - ``docker rm $(docker ps -a -q)`` - - ``docker rmi $(docker images -q)`` + - ``docker-compose -f docmer-compose.yml down -v`` - There are a few logs to tail @@ -219,18 +171,11 @@ Additional DCM docker development tips - dcmsrv : ``tail -n 2000 -f /var/log/lighttpd/breakage.log`` - dcmsrv : ``tail -n 2000 -f /var/log/lighttpd/access.log`` -- Note that by default the docker container will stop running if the process it is following is turned off. For example flask with dcmsrv. You can get around this by having the script being followed never close (e.g. sleep infinity) https://stackoverflow.com/questions/31870222/how-can-i-keep-container-running-on-kubernetes - You may have to restart the glassfish domain occasionally to deal with memory filling up. If deployment is getting reallllllly slow, its a good time. Repository Storage Abstraction Layer (RSAL) ------------------------------------------- - -Steps to set up a RSAL via Docker for Development -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -- See ``docker-dcm/readme.md`` - Configuring the RSAL Mock ~~~~~~~~~~~~~~~~~~~~~~~~~ From 8a8d726bccb418963b0ff6415d5a8ba03891a6bc Mon Sep 17 00:00:00 2001 From: Pete Meyer Date: Tue, 12 Mar 2019 18:08:56 -0400 Subject: [PATCH 21/22] update guides for rsal / data access --- .../source/developers/big-data-support.rst | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/developers/big-data-support.rst b/doc/sphinx-guides/source/developers/big-data-support.rst index b9159bf0f45..18737066852 100644 --- a/doc/sphinx-guides/source/developers/big-data-support.rst +++ b/doc/sphinx-guides/source/developers/big-data-support.rst @@ -92,7 +92,7 @@ If you need a fully operating DCM client for development purposes, these steps w Docker Image Set-up ^^^^^^^^^^^^^^^^^^^ -See https://github.com/IQSS/dataverse/blob/develop/conf/docker-dcm/readme.txt +See https://github.com/IQSS/dataverse/blob/develop/conf/docker-dcm/readme.md - Install docker if you do not have it @@ -176,6 +176,19 @@ Additional DCM docker development tips Repository Storage Abstraction Layer (RSAL) ------------------------------------------- +Steps to set up a DCM via Docker for Development +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +See https://github.com/IQSS/dataverse/blob/develop/conf/docker-dcm/readme.md + +Using the RSAL Docker Containers +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +- Create a dataset (either with the procedure mentioned in DCM Docker Containers, or another process) +- Publish the dataset (from the client container): ``cd /mnt; ./publish_major.bash ${database_id}`` +- Run the RSAL component of the workflow (from the host): ``docker exec -it rsalsrv /opt/rsal/scn/pub.py`` +- If desired, from the client container you can download the dataset following the instructions in the dataset access section of the dataset page. + Configuring the RSAL Mock ~~~~~~~~~~~~~~~~~~~~~~~~~ From 4b0a0956f48cca0dda7c2f63960eefe13fd98077 Mon Sep 17 00:00:00 2001 From: Pete Meyer Date: Thu, 14 Mar 2019 13:25:27 -0400 Subject: [PATCH 22/22] update rsal rpm url to release --- conf/docker-dcm/0prep.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/docker-dcm/0prep.sh b/conf/docker-dcm/0prep.sh index ca5c84826b1..300aa39d567 100755 --- a/conf/docker-dcm/0prep.sh +++ b/conf/docker-dcm/0prep.sh @@ -7,5 +7,5 @@ if [ ! -e dcm-${DCM_VERSION}-0.noarch.rpm ]; then fi if [ ! -e rsal-${RSAL_VERSION}-0.noarch.rpm ] ;then - echo "TODO - rsal release url not configured yet" + wget https://github.com/sbgrid/rsal/releases/download/${RSAL_VERSION}/rsal-${RSAL_VERSION}-0.noarch.rpm fi