From af79c2857d54d55a8ec0e4f8ab8f17c0864ef413 Mon Sep 17 00:00:00 2001
From: 7048730 <7048730@users.noreply.github.com>
Date: Fri, 10 Nov 2017 14:26:45 +0100
Subject: [PATCH 01/53] upd readme-win
---
README-WIN.md | 17 +++++++++--------
1 file changed, 9 insertions(+), 8 deletions(-)
diff --git a/README-WIN.md b/README-WIN.md
index d9a0c06..bc6ec08 100644
--- a/README-WIN.md
+++ b/README-WIN.md
@@ -12,12 +12,12 @@ Take a look at the "Troubleshooting" sections in this file if you run into probl
The environmental variables must be passed separately on Windows, followed by the docker-compose commands:
-```powershell
-$env:OAUTH_CLIENT_ID = "<... required ...>"
-$env:OAUTH_CLIENT_SECRET = "<... required ...>"
-$env:OAUTH_URL_CALLBACK = "http://localhost/api/v1/auth/login"
-$env:ZENODO_TOKEN = "<... optional ...>"
-docker-compose.exe up
+```shell
+setx OAUTH_CLIENT_ID = "<... required ...>"
+setx OAUTH_CLIENT_SECRET = "<... required ...>"
+setx OAUTH_URL_CALLBACK = "http://localhost/api/v1/auth/login"
+setx ZENODO_TOKEN = "<... optional ...>"
+docker-compose up
```
The services are available at `http://localhost`.
@@ -28,8 +28,9 @@ The services are available at `http://localhost`.
- Reconsider using a Linux virtual machine.
- Install the latest _edge_ channel of Docker, see https://docs.docker.com/docker-for-windows/faqs/#questions-about-stable-and-edge-channels
- Make sure that mounting files into containers works. This thread contains many potentially useful hints and ideas: https://forums.docker.com/t/volume-mounts-in-windows-does-not-work/10693/4
+- If you want to set environment variables in PowerShell, use this syntax `$env:VARIABLENAME = "VARIABLEVALUE"`
- Inspect your environment variables in PowerShell with `Get-ChildItem Env:`
-
+- If you encounter problems with docker-compose, you might want to install python 2.7, especially when a newer python distribution is installed on your machine
## Windows with Docker Toolbox
@@ -43,4 +44,4 @@ Also, the client's defaults (i.e. using `localhost`) does not work. We must moun
COMPOSE_CONVERT_WINDOWS_PATHS=1 OAUTH_CLIENT_ID=<...> OAUTH_CLIENT_SECRET=<...> OAUTH_URL_CALLBACK=<...> ZENODO_TOKEN=<...> docker-compose up
```
-The services are available at `http://<machine-ip>`.
\ No newline at end of file
+The services are available at `http://<machine-ip>`.
From 467544666041096fe3f339f29a2000c0911393a7 Mon Sep 17 00:00:00 2001
From: nuest <daniel.nuest@uni-muenster.de>
Date: Thu, 21 Dec 2017 17:30:28 +0100
Subject: [PATCH 02/53] Update microservices
- update docs for ORCID sandbox
- add inspecter
- update finder
---
.gitmodules | 3 ++
Makefile | 5 ++++
README.md | 11 +++++--
docker-compose-local.yml | 53 +++++++++++++++++++++++++++-----
docker-compose.yml | 65 ++++++++++++++++++++++++++++++++--------
nginx.conf | 12 ++++----
o2r-inspecter | 1 +
7 files changed, 121 insertions(+), 29 deletions(-)
create mode 160000 o2r-inspecter
diff --git a/.gitmodules b/.gitmodules
index ba79547..3764a3c 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -43,3 +43,6 @@
[submodule "erc-examples"]
path = erc-examples
url = https://github.com/o2r-project/erc-examples
+[submodule "o2r-inspecter"]
+ path = o2r-inspecter
+ url = https://github.com/o2r-project/o2r-inspecter
diff --git a/Makefile b/Makefile
index 33cf46a..b91b4b2 100644
--- a/Makefile
+++ b/Makefile
@@ -8,6 +8,7 @@ init:
git submodule add https://github.com/o2r-project/o2r-bouncer
git submodule add https://github.com/o2r-project/o2r-finder
git submodule add https://github.com/o2r-project/o2r-informer
+ git submodule add https://github.com/o2r-project/o2r-inspecter
git submodule add https://github.com/o2r-project/o2r-loader
git submodule add https://github.com/o2r-project/o2r-meta
git submodule add https://github.com/o2r-project/o2r-muncher
@@ -25,6 +26,7 @@ build_images:
cd o2r-bouncer; docker build --tag o2r_refimpl_bouncer .; cd ..;
cd o2r-finder; docker build --tag o2r_refimpl_finder .; cd ..;
cd o2r-informer; docker build --tag o2r_refimpl_informer .; cd ..;
+ cd o2r-inspecter; docker build --tag o2r_refimpl_insepcter .; cd ..;
cd o2r-loader; docker build --tag o2r_refimpl_loader .; cd ..;
cd o2r-muncher; docker build --tag o2r_refimpl_muncher .; cd ..;
cd o2r-platform; docker build --tag o2r_refimpl_platform .; cd ..;
@@ -42,6 +44,7 @@ show_versions_local:
@docker inspect --format '{{index .Config.Labels "org.label-schema.name"}}: {{index .Config.Labels "org.label-schema.version"}}' o2r_refimpl_bouncer ;
@docker inspect --format '{{index .Config.Labels "org.label-schema.name"}}: {{index .Config.Labels "org.label-schema.version"}}' o2r_refimpl_finder ;
@docker inspect --format '{{index .Config.Labels "org.label-schema.name"}}: {{index .Config.Labels "org.label-schema.version"}}' o2r_refimpl_informer ;
+ @docker inspect --format '{{index .Config.Labels "org.label-schema.name"}}: {{index .Config.Labels "org.label-schema.version"}}' o2r_refimpl_inspecter ;
@docker inspect --format '{{index .Config.Labels "org.label-schema.name"}}: {{index .Config.Labels "org.label-schema.version"}}' o2r_refimpl_loader ;
@docker inspect --format '{{index .Config.Labels "org.label-schema.name"}}: {{index .Config.Labels "org.label-schema.version"}}' o2r_refimpl_muncher ;
@docker inspect --format '{{index .Config.Labels "org.label-schema.name"}}: {{index .Config.Labels "org.label-schema.version"}}' o2r_refimpl_platform ;
@@ -58,6 +61,7 @@ pull_hub_images:
docker pull o2rproject/o2r-bouncer;
docker pull o2rproject/o2r-finder;
docker pull o2rproject/o2r-informer;
+ docker pull o2rproject/o2r-inspecter;
docker pull o2rproject/o2r-loader;
docker pull o2rproject/o2r-muncher;
docker pull o2rproject/o2r-platform;
@@ -72,6 +76,7 @@ show_versions_hub:
@docker inspect --format '{{index .Config.Labels "org.label-schema.name"}}: {{index .Config.Labels "org.label-schema.version"}}' o2rproject/o2r-bouncer;
@docker inspect --format '{{index .Config.Labels "org.label-schema.name"}}: {{index .Config.Labels "org.label-schema.version"}}' o2rproject/o2r-finder;
@docker inspect --format '{{index .Config.Labels "org.label-schema.name"}}: {{index .Config.Labels "org.label-schema.version"}}' o2rproject/o2r-informer;
+ @docker inspect --format '{{index .Config.Labels "org.label-schema.name"}}: {{index .Config.Labels "org.label-schema.version"}}' o2rproject/o2r-inspecter;
@docker inspect --format '{{index .Config.Labels "org.label-schema.name"}}: {{index .Config.Labels "org.label-schema.version"}}' o2rproject/o2r-loader;
@docker inspect --format '{{index .Config.Labels "org.label-schema.name"}}: {{index .Config.Labels "org.label-schema.version"}}' o2rproject/o2r-muncher;
@docker inspect --format '{{index .Config.Labels "org.label-schema.name"}}: {{index .Config.Labels "org.label-schema.version"}}' o2rproject/o2r-platform;
diff --git a/README.md b/README.md
index 1c0c29c..015d714 100644
--- a/README.md
+++ b/README.md
@@ -74,7 +74,9 @@ The reference implementation relies on [ORCID](https://orcid.org/) for authentic
There is no other way to log in on the platform, therefore you must enable your installation of the reference implementation to connect with the ORCID API.
The client credentials used by the o2r team cannot be shared publicly for security reasons.
-You _must_ register a public API client application with ORCID as [described here](https://support.orcid.org/knowledgebase/articles/343182-register-a-public-api-client-application).
+You _must_ register an account and get a authentication tokens for public API client application with **[ORCID Sandbox](https://sandbox.orcid.org/signin)**.
+Alternatively, use your existing ORCID account and register an application as [described here](https://support.orcid.org/knowledgebase/articles/343182-register-a-public-api-client-application).
+Note that in the latter case you must adjust the URLs in the example below: remove the `sandbox.` part.
In the developer tools, use any name, website URL, and description.
Important is the `Redirect URIs` list, which must include `http://localhost` for your local installation.
@@ -109,7 +111,9 @@ To download all o2r source code at once, navigate to the `reference-implementati
Once all repositories have been pulled successfully, build docker images of the microservices and run them in containers by executing:
```bash
-O2R_ORCID_ID=<your orcid id> O2R_ORCID_SECRET=<your orcid secret> O2R_ORCID_CALLBACK=http://localhost/api/v1/auth/login O2R_ZENODO_TOKEN=<your token> \
+OAUTH_CLIENT_ID=<your orcid id> OAUTH_CLIENT_SECRET=<your orcid secret> \
+ OAUTH_URL_AUTHORIZATION=https://sandbox.orcid.org/oauth/authorize OAUTH_URL_TOKEN=https://sandbox.orcid.org/oauth/token OAUTH_URL_CALLBACK=http://localhost/api/v1/auth/login \
+ SHIPPER_REPO_TOKENS="{\"zenodo\": \"<your Zenodo token>\", \"zenodo_sandbox\": \"<your Zenodo Sandbox token>\", "download": \"\" }" \
make build_images run_local
```
@@ -122,7 +126,8 @@ All o2r software projects have automatic builds [available on Docker Hub](https:
The following command executes a `docker-compose` command to pull and run these images.
```bash
-O2R_ORCID_ID=<your orcid id> O2R_ORCID_SECRET=<your orcid secret> O2R_ORCID_CALLBACK=http://localhost/api/v1/auth/login O2R_ZENODO_TOKEN=<your token> \
+O2R_ORCID_ID=<your orcid id> O2R_ORCID_SECRET=<your orcid secret> O2R_ORCID_CALLBACK=http://localhost/api/v1/auth/login \
+ SHIPPER_REPO_TOKENS="{\"zenodo\": \"<your Zenodo token>\", \"zenodo_sandbox\": \"<your Zenodo Sandbox token>\", "download": \"\" }" \
make run_hub
```
diff --git a/docker-compose-local.yml b/docker-compose-local.yml
index 3d8d795..863f88d 100644
--- a/docker-compose-local.yml
+++ b/docker-compose-local.yml
@@ -14,7 +14,7 @@
#
#
---
-version: '3'
+version: '2'
volumes:
o2rvol: {}
@@ -41,10 +41,30 @@ services:
- "1234:1234"
elasticsearch:
- image: docker.elastic.co/elasticsearch/elasticsearch:5.6.3
+ image: docker.elastic.co/elasticsearch/elasticsearch:5.6.5
environment:
- - ES_JAVA_OPTS=-Xms512m -Xmx512m
+ - cluster.name=o2rplatform-es-cluster
+ - bootstrap.memory_lock=true
+ - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
- "xpack.security.enabled=false"
+ ulimits:
+ memlock:
+ soft: -1
+ hard: -1
+ mem_limit: 1g
+ elasticsearch2:
+ image: docker.elastic.co/elasticsearch/elasticsearch:5.6.5
+ environment:
+ - cluster.name=o2rplatform-es-cluster
+ - bootstrap.memory_lock=true
+ - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
+ - "discovery.zen.ping.unicast.hosts=elasticsearch"
+ - "xpack.security.enabled=false"
+ ulimits:
+ memlock:
+ soft: -1
+ hard: -1
+ mem_limit: 1g
muncher:
image: o2r_refimpl_muncher
@@ -95,6 +115,8 @@ services:
- OAUTH_URL_CALLBACK=${OAUTH_URL_CALLBACK}
- OAUTH_CLIENT_ID=${OAUTH_CLIENT_ID}
- OAUTH_CLIENT_SECRET=${OAUTH_CLIENT_SECRET}
+ - OAUTH_URL_AUTHORIZATION=${OAUTH_URL_AUTHORIZATION}
+ - OAUTH_URL_TOKEN=${OAUTH_URL_TOKEN}
- SLACK_VERIFICATION_TOKEN=${SLACK_VERIFICATION_TOKEN}
- SLACK_BOT_TOKEN=${SLACK_BOT_TOKEN}
@@ -107,9 +129,9 @@ services:
volumes:
- o2rvol:/tmp/o2r
environment:
- - "FINDER_MONGODB_USER_DATABASE=mongodb://mongodb/muncher"
+ - "FINDER_MONGODB=mongodb://mongodb"
- FINDER_PORT=8084
- - DEBUG=finder
+ - DEBUG=finder,finder:*
- ELASTIC_SEARCH_URL=elasticsearch:9200
- "MONGO_DATA_URL=mongodb://mongodb/muncher"
- "MONGO_OPLOG_URL=mongodb://mongodb/muncher"
@@ -137,9 +159,10 @@ services:
- o2rvol:/tmp/o2r
environment:
SHIPPER_MONGODB: "mongodb://mongodb"
- SHIPPER_REPO_TOKENS: '{"zenodo_sandbox": "${ZENODO_TOKEN}", "download": "" }'
+ SHIPPER_REPO_TOKENS: "${SHIPPER_REPO_TOKENS}"
SHIPPER_BOTTLE_HOST: "0.0.0.0"
SHIPPER_BASE_PATH: "/tmp/o2r"
+ PYTHONUNBUFFERED: "0"
substituter:
image: o2r_refimpl_substituter
@@ -153,6 +176,15 @@ services:
- SUBSTITUTER_PORT=8090
- DEBUG=substituter,substituter:*
+ inspecter:
+ image: o2rproject/o2r-inspecter:0.0.1
+ restart: unless-stopped
+ volumes:
+ - o2rvol:/tmp/o2r
+ environment:
+ - INSPECTER_PORT=8091
+ - DEBUGME=inspecter
+
platform:
image: o2r_refimpl_platform
restart: unless-stopped
@@ -161,10 +193,15 @@ services:
image: nginx:latest
restart: unless-stopped
depends_on:
- - muncher
- bouncer
+ - finder
+ - muncher
+ - informer
+ - inspecter
+ - loader
- transporter
- - platform
+ - shipper
+ - substituter
volumes:
- "./nginx.conf:/etc/nginx/nginx.conf:ro"
ports:
diff --git a/docker-compose.yml b/docker-compose.yml
index 447b0b5..0608b6a 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -14,7 +14,7 @@
#
#
---
-version: '3'
+version: '2'
volumes:
o2rvol: {}
@@ -41,13 +41,35 @@ services:
- "1234:1234"
elasticsearch:
- image: docker.elastic.co/elasticsearch/elasticsearch:5.6.3
+ image: docker.elastic.co/elasticsearch/elasticsearch:5.6.5
environment:
- - ES_JAVA_OPTS=-Xms512m -Xmx512m
+ - cluster.name=o2rplatform-es-cluster
+ - bootstrap.memory_lock=true
+ - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
- "xpack.security.enabled=false"
+ ulimits:
+ memlock:
+ soft: -1
+ hard: -1
+ mem_limit: 1g
+# ports:
+# - "9200:9200"
+ elasticsearch2:
+ image: docker.elastic.co/elasticsearch/elasticsearch:5.6.5
+ environment:
+ - cluster.name=o2rplatform-es-cluster
+ - bootstrap.memory_lock=true
+ - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
+ - "discovery.zen.ping.unicast.hosts=elasticsearch"
+ - "xpack.security.enabled=false"
+ ulimits:
+ memlock:
+ soft: -1
+ hard: -1
+ mem_limit: 1g
muncher:
- image: o2rproject/o2r-muncher:0.12.3
+ image: o2rproject/o2r-muncher:0.13.5
restart: unless-stopped
depends_on:
- configmongodb
@@ -57,10 +79,13 @@ services:
environment:
- "MUNCHER_MONGODB=mongodb://mongodb"
- MUNCHER_PORT=8080
+ - MUNCHER_VOLUME=referenceimplementation_o2rvol
+ - MUNCHER_CONTAINER_USER=root
+ - MUNCHER_CONTAINERIT_USER=root
- DEBUG=*,-mquery,-express:*,-express-session,-body-parser:*
loader:
- image: o2rproject/o2r-loader:0.7.3
+ image: o2rproject/o2r-loader:0.8.1
restart: unless-stopped
depends_on:
- configmongodb
@@ -71,6 +96,7 @@ services:
- "LOADER_MONGODB=mongodb://mongodb"
- LOADER_PORT=8088
- DEBUG=*,-mquery,-express:*,-express-session,-body-parser:*
+ - LOADER_VOLUME=referenceimplementation_o2rvol
informer:
image: o2rproject/o2r-informer:0.3.0
@@ -95,11 +121,13 @@ services:
- OAUTH_URL_CALLBACK=${OAUTH_URL_CALLBACK}
- OAUTH_CLIENT_ID=${OAUTH_CLIENT_ID}
- OAUTH_CLIENT_SECRET=${OAUTH_CLIENT_SECRET}
+ - OAUTH_URL_AUTHORIZATION=${OAUTH_URL_AUTHORIZATION}
+ - OAUTH_URL_TOKEN=${OAUTH_URL_TOKEN}
- SLACK_VERIFICATION_TOKEN=${SLACK_VERIFICATION_TOKEN}
- SLACK_BOT_TOKEN=${SLACK_BOT_TOKEN}
finder:
- image: o2rproject/o2r-finder:0.3.2
+ image: o2rproject/o2r-finder:0.5.1
restart: unless-stopped
depends_on:
- configmongodb
@@ -107,16 +135,16 @@ services:
volumes:
- o2rvol:/tmp/o2r
environment:
- - "FINDER_MONGODB_USER_DATABASE=mongodb://mongodb/muncher"
+ - "FINDER_MONGODB=mongodb://mongodb"
- FINDER_PORT=8084
- - DEBUG=finder
+ - DEBUG=finder,finder:*
- ELASTIC_SEARCH_URL=elasticsearch:9200
- "MONGO_DATA_URL=mongodb://mongodb/muncher"
- "MONGO_OPLOG_URL=mongodb://mongodb/muncher"
- BATCH_COUNT=20
transporter:
- image: o2rproject/o2r-transporter:0.4.0
+ image: o2rproject/o2r-transporter:0.4.1
restart: unless-stopped
depends_on:
- configmongodb
@@ -129,7 +157,7 @@ services:
- DEBUG=transporter,transporter:*
shipper:
- image: o2rproject/o2r-shipper:e3a837c
+ image: o2rproject/o2r-shipper:c4b2d27
restart: unless-stopped
depends_on:
- configmongodb
@@ -137,12 +165,13 @@ services:
- o2rvol:/tmp/o2r
environment:
SHIPPER_MONGODB: "mongodb://mongodb"
- SHIPPER_REPO_TOKENS: '{"zenodo_sandbox": "${ZENODO_TOKEN}", "download": "" }'
+ SHIPPER_REPO_TOKENS: "${SHIPPER_REPO_TOKENS}"
SHIPPER_BOTTLE_HOST: "0.0.0.0"
SHIPPER_BASE_PATH: "/tmp/o2r"
+ PYTHONUNBUFFERED: "0"
substituter:
- image: o2rproject/o2r-substituter:0.3.0
+ image: o2rproject/o2r-substituter:0.4.0
restart: unless-stopped
depends_on:
- configmongodb
@@ -153,8 +182,17 @@ services:
- SUBSTITUTER_PORT=8090
- DEBUG=substituter,substituter:*
+ inspecter:
+ image: o2rproject/o2r-inspecter:0.0.1
+ restart: unless-stopped
+ volumes:
+ - o2rvol:/tmp/o2r
+ environment:
+ - INSPECTER_PORT=8091
+ - DEBUGME=inspecter
+
platform:
- image: o2rproject/o2r-platform:0.9.3
+ image: o2rproject/o2r-platform:1.0.0
restart: unless-stopped
webserver:
@@ -165,6 +203,7 @@ services:
- finder
- muncher
- informer
+ - inspecter
- loader
- transporter
- shipper
diff --git a/nginx.conf b/nginx.conf
index a328d97..22cfa19 100644
--- a/nginx.conf
+++ b/nginx.conf
@@ -87,11 +87,7 @@ http {
location /api/v1/search {
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
- proxy_pass http://elasticsearch:9200/o2r/_search;
-
- limit_except GET {
- deny all;
- }
+ proxy_pass http://finder:8084;
}
location /api/v1/shipment {
@@ -111,5 +107,11 @@ http {
proxy_set_header X-Real-IP $remote_addr;
proxy_pass http://substituter:8090;
}
+
+ location /api/v1/inspection {
+ proxy_set_header Host $host;
+ proxy_set_header X-Real-IP $remote_addr;
+ proxy_pass http://inspecter:8091;
+ }
}
}
diff --git a/o2r-inspecter b/o2r-inspecter
new file mode 160000
index 0000000..0a92d37
--- /dev/null
+++ b/o2r-inspecter
@@ -0,0 +1 @@
+Subproject commit 0a92d37646202bce184be1e94c05164b24877bc9
From b3504055841e99236d7bfd5ec045d92666bded14 Mon Sep 17 00:00:00 2001
From: nuest <daniel.nuest@uni-muenster.de>
Date: Thu, 21 Dec 2017 17:30:38 +0100
Subject: [PATCH 03/53] Update submodules
---
architecture | 2 +-
erc-checker | 2 +-
erc-examples | 2 +-
erc-spec | 2 +-
o2r-finder | 2 +-
o2r-loader | 2 +-
o2r-meta | 2 +-
o2r-muncher | 2 +-
o2r-platform | 2 +-
o2r-shipper | 2 +-
o2r-substituter | 2 +-
o2r-transporter | 2 +-
o2r-web-api | 2 +-
13 files changed, 13 insertions(+), 13 deletions(-)
diff --git a/architecture b/architecture
index 8f82796..f671430 160000
--- a/architecture
+++ b/architecture
@@ -1 +1 @@
-Subproject commit 8f8279609c40fd72c4df55a84995704c0b6100a0
+Subproject commit f671430d9be9dc3e974a29fda3a6df65e887a86f
diff --git a/erc-checker b/erc-checker
index fd3eeb6..91bd93d 160000
--- a/erc-checker
+++ b/erc-checker
@@ -1 +1 @@
-Subproject commit fd3eeb6959ef7694496a0ba1c2b177d64d8495e0
+Subproject commit 91bd93d3d9b366709c83e1b97c91b41645a9f1da
diff --git a/erc-examples b/erc-examples
index ea44b31..b646d85 160000
--- a/erc-examples
+++ b/erc-examples
@@ -1 +1 @@
-Subproject commit ea44b3199c20d2dce9e51a7de115cc6a2ff41a30
+Subproject commit b646d8577aa4f6df97e31c9af3f16cd31a5503e4
diff --git a/erc-spec b/erc-spec
index 03888d6..2056534 160000
--- a/erc-spec
+++ b/erc-spec
@@ -1 +1 @@
-Subproject commit 03888d6bb642e044c45b00b37768a590a3d4f7b3
+Subproject commit 20565342ea4b34fcdc950b1c78848126523e7a17
diff --git a/o2r-finder b/o2r-finder
index 59e0fe3..bcc63c0 160000
--- a/o2r-finder
+++ b/o2r-finder
@@ -1 +1 @@
-Subproject commit 59e0fe39451e47f4f03d4543f78c7216f3eb8569
+Subproject commit bcc63c0538d5ba32a0c0cf338f12019a3d2d4723
diff --git a/o2r-loader b/o2r-loader
index e3d226b..fe0a485 160000
--- a/o2r-loader
+++ b/o2r-loader
@@ -1 +1 @@
-Subproject commit e3d226b2794019aee1bded37cc6c71cfbf5577a6
+Subproject commit fe0a48595525494b54f586c4a33e8cf876fd5223
diff --git a/o2r-meta b/o2r-meta
index 2c79485..cdaea30 160000
--- a/o2r-meta
+++ b/o2r-meta
@@ -1 +1 @@
-Subproject commit 2c79485104b14642a9e610a32f012cecbb0b23a3
+Subproject commit cdaea3053301a0c1398df552965f244c3f588620
diff --git a/o2r-muncher b/o2r-muncher
index 869a7d7..8c11f19 160000
--- a/o2r-muncher
+++ b/o2r-muncher
@@ -1 +1 @@
-Subproject commit 869a7d7e48f8de044f2481d323538f004039ba43
+Subproject commit 8c11f19ee811b15a052a4cd558e4ee1b25b0b273
diff --git a/o2r-platform b/o2r-platform
index 04969ab..16c0a9d 160000
--- a/o2r-platform
+++ b/o2r-platform
@@ -1 +1 @@
-Subproject commit 04969ab21abb1964eb2c66b97db82fd96206bd4b
+Subproject commit 16c0a9df5d59420616ce16b73f620df9fb2db631
diff --git a/o2r-shipper b/o2r-shipper
index e3a837c..c4b2d27 160000
--- a/o2r-shipper
+++ b/o2r-shipper
@@ -1 +1 @@
-Subproject commit e3a837c2bfa399845348a94b88637f048cc1b423
+Subproject commit c4b2d27750492bb484beb8d4722716ab98533518
diff --git a/o2r-substituter b/o2r-substituter
index bf0dcd0..bd6c5d8 160000
--- a/o2r-substituter
+++ b/o2r-substituter
@@ -1 +1 @@
-Subproject commit bf0dcd0524abcd5739b83298f0d4acc70f8479e4
+Subproject commit bd6c5d822c6aec12ad4f6cb7b98a2dd55db5d4ca
diff --git a/o2r-transporter b/o2r-transporter
index 9210939..e36b4a0 160000
--- a/o2r-transporter
+++ b/o2r-transporter
@@ -1 +1 @@
-Subproject commit 92109399d0b72718a83ebae73afe41502240fd69
+Subproject commit e36b4a05591c4f3f0a42e8559e1299f3d3d16d72
diff --git a/o2r-web-api b/o2r-web-api
index cd5554a..72b311d 160000
--- a/o2r-web-api
+++ b/o2r-web-api
@@ -1 +1 @@
-Subproject commit cd5554a212595772aaa3eb684b07554282de38f0
+Subproject commit 72b311ddba061d7b8d140fcc35d52df1507e0cfc
From 0e1beacf72f39556d3bf65ea5d837bf57c4247db Mon Sep 17 00:00:00 2001
From: nuest <daniel.nuest@uni-muenster.de>
Date: Mon, 22 Jan 2018 15:31:53 +0100
Subject: [PATCH 04/53] update remote microservice versions
---
docker-compose.yml | 10 +++++-----
1 file changed, 5 insertions(+), 5 deletions(-)
diff --git a/docker-compose.yml b/docker-compose.yml
index 0608b6a..00ac36f 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -69,7 +69,7 @@ services:
mem_limit: 1g
muncher:
- image: o2rproject/o2r-muncher:0.13.5
+ image: o2rproject/o2r-muncher:0.13.8
restart: unless-stopped
depends_on:
- configmongodb
@@ -85,7 +85,7 @@ services:
- DEBUG=*,-mquery,-express:*,-express-session,-body-parser:*
loader:
- image: o2rproject/o2r-loader:0.8.1
+ image: o2rproject/o2r-loader:0.8.3
restart: unless-stopped
depends_on:
- configmongodb
@@ -127,7 +127,7 @@ services:
- SLACK_BOT_TOKEN=${SLACK_BOT_TOKEN}
finder:
- image: o2rproject/o2r-finder:0.5.1
+ image: o2rproject/o2r-finder:0.6.0
restart: unless-stopped
depends_on:
- configmongodb
@@ -157,7 +157,7 @@ services:
- DEBUG=transporter,transporter:*
shipper:
- image: o2rproject/o2r-shipper:c4b2d27
+ image: o2rproject/o2r-shipper:168ae80
restart: unless-stopped
depends_on:
- configmongodb
@@ -171,7 +171,7 @@ services:
PYTHONUNBUFFERED: "0"
substituter:
- image: o2rproject/o2r-substituter:0.4.0
+ image: o2rproject/o2r-substituter:0.4.1
restart: unless-stopped
depends_on:
- configmongodb
From cea1350928498a055127fda01b90432010aaec65 Mon Sep 17 00:00:00 2001
From: nuest <daniel.nuest@uni-muenster.de>
Date: Mon, 22 Jan 2018 15:40:46 +0100
Subject: [PATCH 05/53] update nested repositories
---
erc-examples | 2 +-
erc-spec | 2 +-
o2r-finder | 2 +-
o2r-inspecter | 2 +-
o2r-loader | 2 +-
o2r-meta | 2 +-
o2r-muncher | 2 +-
o2r-platform | 2 +-
o2r-shipper | 2 +-
o2r-substituter | 2 +-
o2r-web-api | 2 +-
11 files changed, 11 insertions(+), 11 deletions(-)
diff --git a/erc-examples b/erc-examples
index b646d85..c0a7f53 160000
--- a/erc-examples
+++ b/erc-examples
@@ -1 +1 @@
-Subproject commit b646d8577aa4f6df97e31c9af3f16cd31a5503e4
+Subproject commit c0a7f538d90745380fc1350d6162de96d651f70b
diff --git a/erc-spec b/erc-spec
index 2056534..556df4c 160000
--- a/erc-spec
+++ b/erc-spec
@@ -1 +1 @@
-Subproject commit 20565342ea4b34fcdc950b1c78848126523e7a17
+Subproject commit 556df4c7ba143b5e5735ca3bd3191b9e475dc37c
diff --git a/o2r-finder b/o2r-finder
index bcc63c0..502995e 160000
--- a/o2r-finder
+++ b/o2r-finder
@@ -1 +1 @@
-Subproject commit bcc63c0538d5ba32a0c0cf338f12019a3d2d4723
+Subproject commit 502995e1bdfd2c9b1fa0c88d2718f4046ad7456b
diff --git a/o2r-inspecter b/o2r-inspecter
index 0a92d37..ee5e86a 160000
--- a/o2r-inspecter
+++ b/o2r-inspecter
@@ -1 +1 @@
-Subproject commit 0a92d37646202bce184be1e94c05164b24877bc9
+Subproject commit ee5e86ada517972b246f8297ddb8b43580e3eac1
diff --git a/o2r-loader b/o2r-loader
index fe0a485..49c0fb4 160000
--- a/o2r-loader
+++ b/o2r-loader
@@ -1 +1 @@
-Subproject commit fe0a48595525494b54f586c4a33e8cf876fd5223
+Subproject commit 49c0fb4345394549cf6a576cd5a47cce1d5caf53
diff --git a/o2r-meta b/o2r-meta
index cdaea30..1ab8868 160000
--- a/o2r-meta
+++ b/o2r-meta
@@ -1 +1 @@
-Subproject commit cdaea3053301a0c1398df552965f244c3f588620
+Subproject commit 1ab886850b3d589480187b42d6e4ce90a8fdc0d5
diff --git a/o2r-muncher b/o2r-muncher
index 8c11f19..776a9f5 160000
--- a/o2r-muncher
+++ b/o2r-muncher
@@ -1 +1 @@
-Subproject commit 8c11f19ee811b15a052a4cd558e4ee1b25b0b273
+Subproject commit 776a9f5768bf8e32ac6f75e6a7dd39392f5f886e
diff --git a/o2r-platform b/o2r-platform
index 16c0a9d..fcf8e54 160000
--- a/o2r-platform
+++ b/o2r-platform
@@ -1 +1 @@
-Subproject commit 16c0a9df5d59420616ce16b73f620df9fb2db631
+Subproject commit fcf8e544ae687b822b011dc0d7387dc0eede5a11
diff --git a/o2r-shipper b/o2r-shipper
index c4b2d27..168ae80 160000
--- a/o2r-shipper
+++ b/o2r-shipper
@@ -1 +1 @@
-Subproject commit c4b2d27750492bb484beb8d4722716ab98533518
+Subproject commit 168ae80b0f196a25c68f9ce693fb8309aa798878
diff --git a/o2r-substituter b/o2r-substituter
index bd6c5d8..0863e61 160000
--- a/o2r-substituter
+++ b/o2r-substituter
@@ -1 +1 @@
-Subproject commit bd6c5d822c6aec12ad4f6cb7b98a2dd55db5d4ca
+Subproject commit 0863e61330f0209e839f707788a291ba99737ed0
diff --git a/o2r-web-api b/o2r-web-api
index 72b311d..5a5d65c 160000
--- a/o2r-web-api
+++ b/o2r-web-api
@@ -1 +1 @@
-Subproject commit 72b311ddba061d7b8d140fcc35d52df1507e0cfc
+Subproject commit 5a5d65cbe97811cd3268b84a5c3862e4ce70f417
From 5b5d5a8342048b857561e779e32b0769b3ebd7df Mon Sep 17 00:00:00 2001
From: nuest <daniel.nuest@uni-muenster.de>
Date: Mon, 22 Jan 2018 18:20:44 +0100
Subject: [PATCH 06/53] closes #1 and #4
---
Makefile | 6 +++++-
docker-compose-local.yml | 2 ++
2 files changed, 7 insertions(+), 1 deletion(-)
diff --git a/Makefile b/Makefile
index b91b4b2..3697cb8 100644
--- a/Makefile
+++ b/Makefile
@@ -26,8 +26,9 @@ build_images:
cd o2r-bouncer; docker build --tag o2r_refimpl_bouncer .; cd ..;
cd o2r-finder; docker build --tag o2r_refimpl_finder .; cd ..;
cd o2r-informer; docker build --tag o2r_refimpl_informer .; cd ..;
- cd o2r-inspecter; docker build --tag o2r_refimpl_insepcter .; cd ..;
+ cd o2r-inspecter; docker build --tag o2r_refimpl_inspecter .; cd ..;
cd o2r-loader; docker build --tag o2r_refimpl_loader .; cd ..;
+ cd o2r-meta; docker build --tag o2r_refimpl_meta .; cd ..;
cd o2r-muncher; docker build --tag o2r_refimpl_muncher .; cd ..;
cd o2r-platform; docker build --tag o2r_refimpl_platform .; cd ..;
cd o2r-shipper; docker build --tag o2r_refimpl_shipper .; cd ..;
@@ -46,6 +47,7 @@ show_versions_local:
@docker inspect --format '{{index .Config.Labels "org.label-schema.name"}}: {{index .Config.Labels "org.label-schema.version"}}' o2r_refimpl_informer ;
@docker inspect --format '{{index .Config.Labels "org.label-schema.name"}}: {{index .Config.Labels "org.label-schema.version"}}' o2r_refimpl_inspecter ;
@docker inspect --format '{{index .Config.Labels "org.label-schema.name"}}: {{index .Config.Labels "org.label-schema.version"}}' o2r_refimpl_loader ;
+ @docker inspect --format '{{index .Config.Labels "org.label-schema.name"}}: {{index .Config.Labels "org.label-schema.version"}}' o2r_refimpl_meta ;
@docker inspect --format '{{index .Config.Labels "org.label-schema.name"}}: {{index .Config.Labels "org.label-schema.version"}}' o2r_refimpl_muncher ;
@docker inspect --format '{{index .Config.Labels "org.label-schema.name"}}: {{index .Config.Labels "org.label-schema.version"}}' o2r_refimpl_platform ;
@docker inspect --format '{{index .Config.Labels "org.label-schema.name"}}: {{index .Config.Labels "org.label-schema.vcs-ref"}}' o2r_refimpl_shipper ;
@@ -63,6 +65,7 @@ pull_hub_images:
docker pull o2rproject/o2r-informer;
docker pull o2rproject/o2r-inspecter;
docker pull o2rproject/o2r-loader;
+ docker pull o2rproject/o2r-meta;
docker pull o2rproject/o2r-muncher;
docker pull o2rproject/o2r-platform;
docker pull o2rproject/o2r-shipper;
@@ -78,6 +81,7 @@ show_versions_hub:
@docker inspect --format '{{index .Config.Labels "org.label-schema.name"}}: {{index .Config.Labels "org.label-schema.version"}}' o2rproject/o2r-informer;
@docker inspect --format '{{index .Config.Labels "org.label-schema.name"}}: {{index .Config.Labels "org.label-schema.version"}}' o2rproject/o2r-inspecter;
@docker inspect --format '{{index .Config.Labels "org.label-schema.name"}}: {{index .Config.Labels "org.label-schema.version"}}' o2rproject/o2r-loader;
+ @docker inspect --format '{{index .Config.Labels "org.label-schema.name"}}: {{index .Config.Labels "org.label-schema.version"}}' o2rproject/o2r-meta;
@docker inspect --format '{{index .Config.Labels "org.label-schema.name"}}: {{index .Config.Labels "org.label-schema.version"}}' o2rproject/o2r-muncher;
@docker inspect --format '{{index .Config.Labels "org.label-schema.name"}}: {{index .Config.Labels "org.label-schema.version"}}' o2rproject/o2r-platform;
@docker inspect --format '{{index .Config.Labels "org.label-schema.name"}}: {{index .Config.Labels "org.label-schema.vcs-ref"}}' o2rproject/o2r-shipper;
diff --git a/docker-compose-local.yml b/docker-compose-local.yml
index 863f88d..a2985f0 100644
--- a/docker-compose-local.yml
+++ b/docker-compose-local.yml
@@ -78,6 +78,7 @@ services:
- "MUNCHER_MONGODB=mongodb://mongodb"
- MUNCHER_PORT=8080
- DEBUG=*,-mquery,-express:*,-express-session,-body-parser:*
+ - "MUNCHER_META_TOOL_CONTAINER=o2r_refimpl_meta"
loader:
image: o2r_refimpl_loader
@@ -91,6 +92,7 @@ services:
- "LOADER_MONGODB=mongodb://mongodb"
- LOADER_PORT=8088
- DEBUG=*,-mquery,-express:*,-express-session,-body-parser:*
+ - "MUNCHER_META_TOOL_CONTAINER=o2r_refimpl_meta"
informer:
image: o2r_refimpl_informer
From 58e33b9861423120cf3e3bf2a5abc92f9359cebf Mon Sep 17 00:00:00 2001
From: nuest <daniel.nuest@uni-muenster.de>
Date: Mon, 5 Feb 2018 18:12:19 +0100
Subject: [PATCH 07/53] add target for building the documentation locally
---
.dockerignore | 1 +
.gitignore | 1 +
Makefile | 13 ++++++++++-
README.md | 7 ++++++
etc/Dockerfile.documentations | 44 +++++++++++++++++++++++++++++++++++
5 files changed, 65 insertions(+), 1 deletion(-)
create mode 100644 .dockerignore
create mode 100644 .gitignore
create mode 100644 etc/Dockerfile.documentations
diff --git a/.dockerignore b/.dockerignore
new file mode 100644
index 0000000..f59ec20
--- /dev/null
+++ b/.dockerignore
@@ -0,0 +1 @@
+*
\ No newline at end of file
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..a136337
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1 @@
+*.pdf
diff --git a/Makefile b/Makefile
index 3697cb8..315296b 100644
--- a/Makefile
+++ b/Makefile
@@ -92,7 +92,18 @@ clean:
docker ps -a | grep o2r | awk '{print $1}' | xargs docker rm -f
docker images | grep o2r | awk '{print $3}' | xargs docker rmi --force
-release:
+build_documentation:
+ rm *.pdf
+ docker build --tag docbuilder --file etc/Dockerfile.documentations .
+ docker run -it -v $(CURDIR)/architecture:/doc:rw docbuilder make build pdf
+ docker run -it -v $(CURDIR)/o2r-web-api:/doc:rw docbuilder make build pdf
+ docker run -it -v $(CURDIR)/erc-spec:/doc:rw docbuilder make build pdf_tinytex
+ mv architecture/site/*.pdf .
+ mv erc-spec/*.pdf .
+ mv o2r-web-api/*.pdf .
+ echo "ERC, architecture, and web API documentation created, see files PDF files in the project root directory"
+
+release: build_documentation
git clone --recursive https://github.com/o2r-project/reference-implementation
# TODO build all images, export them to files
diff --git a/README.md b/README.md
index 015d714..24cdac8 100644
--- a/README.md
+++ b/README.md
@@ -166,6 +166,13 @@ docker volume ls
docker volume inspect reference-implementation_o2r_test_storage
```
+## Read documentation
+
+This repository also contains specification and documentation projects.
+These documentations are created in Markdown format and can be rendered to HTML and PDF documents using the make target `build_documentation`, which relies on a local `Dockerfile` for the rendering runtime environment.
+The PDF files are moved to the project root directory.
+The HTML files can be found in the respective projects in the `site` directory.
+
## Reproducibility
This repository serves the goal to make the developments of the o2r project reproducible, not only by running the reference implementation (see above) but also by creating an archivable package of the software.
diff --git a/etc/Dockerfile.documentations b/etc/Dockerfile.documentations
new file mode 100644
index 0000000..595b21c
--- /dev/null
+++ b/etc/Dockerfile.documentations
@@ -0,0 +1,44 @@
+#
+# A Dockerfile for an environment to build the documentation contained in this repository
+#
+
+FROM rocker/verse
+
+# System dependencies
+RUN apt-get update && apt-get install -y --no-install-recommends \
+ python3-pip \
+ python3-setuptools \
+ # next one needed by one of the tex packages
+ xzdec \
+ # next one needed for untaring wkhtmltox
+ xz-utils \
+ libssl1.0-dev
+
+# ERC spec, see erc-spec/.travis.yml
+
+# need newer pandoc
+RUN wget https://github.com/jgm/pandoc/releases/download/2.1.1/pandoc-2.1.1-linux.tar.gz \
+ && tar -xvzf pandoc*.tar.gz --strip-components 1 -C /usr/local/
+
+RUN tlmgr init-usertree \
+ && tlmgr install xcolor mdframed needspace
+RUN pip3 install --upgrade pip \
+ && pip3 install \
+ mkdocs \
+ pymdown-extensions \
+ mkdocs-pandoc \
+ pandoc-latex-admonition \
+ git+https://github.com/o2r-project/mkdocs-combine.git
+
+# architecture, see architecture/.travis.yml
+RUN pip3 install mkdocs mkdocs-cinder markdown-include
+RUN apt-get update && apt-get install -y --no-install-recommends \
+ xfonts-base xfonts-75dpi
+RUN wget https://github.com/wkhtmltopdf/wkhtmltopdf/releases/download/0.12.4/wkhtmltox-0.12.4_linux-generic-amd64.tar.xz \
+ && tar -xvf wkhtmltox-0.12.4_linux-generic-amd64.tar.xz \
+ && mv wkhtmltox/bin/wkhtmltopdf /usr/bin/wkhtmltopdf
+
+VOLUME /doc
+WORKDIR /doc
+
+USER 1000
From 7c14c2ed8773d41428aaa8d2ead6c6f6c55294bb Mon Sep 17 00:00:00 2001
From: nuest <daniel.nuest@uni-muenster.de>
Date: Mon, 5 Feb 2018 18:32:14 +0100
Subject: [PATCH 08/53] update submodules
---
Makefile | 3 ++-
architecture | 2 +-
erc-spec | 2 +-
o2r-loader | 2 +-
o2r-meta | 2 +-
o2r-muncher | 2 +-
o2r-platform | 2 +-
o2r-shipper | 2 +-
o2r-substituter | 2 +-
o2r-transporter | 2 +-
o2r-web-api | 2 +-
11 files changed, 12 insertions(+), 11 deletions(-)
diff --git a/Makefile b/Makefile
index 315296b..bec3c53 100644
--- a/Makefile
+++ b/Makefile
@@ -19,7 +19,8 @@ init:
git submodule add https://github.com/o2r-project/o2r-web-api
update:
- git submodule update --init --recursive
+ git pull --recurse-submodules
+ git submodule update --init --recursive --remote
git submodule foreach git pull origin master
build_images:
diff --git a/architecture b/architecture
index f671430..5df1ea9 160000
--- a/architecture
+++ b/architecture
@@ -1 +1 @@
-Subproject commit f671430d9be9dc3e974a29fda3a6df65e887a86f
+Subproject commit 5df1ea9d1288e931746fbe13fd70e7d4c068cb3e
diff --git a/erc-spec b/erc-spec
index 556df4c..50da6d0 160000
--- a/erc-spec
+++ b/erc-spec
@@ -1 +1 @@
-Subproject commit 556df4c7ba143b5e5735ca3bd3191b9e475dc37c
+Subproject commit 50da6d0d7ccedc9b02d9ec7ccbaf136c58bb6bc4
diff --git a/o2r-loader b/o2r-loader
index 49c0fb4..adaf578 160000
--- a/o2r-loader
+++ b/o2r-loader
@@ -1 +1 @@
-Subproject commit 49c0fb4345394549cf6a576cd5a47cce1d5caf53
+Subproject commit adaf57810a344aa66c868f6e8bac3cfc7ea1b007
diff --git a/o2r-meta b/o2r-meta
index 1ab8868..e0025d0 160000
--- a/o2r-meta
+++ b/o2r-meta
@@ -1 +1 @@
-Subproject commit 1ab886850b3d589480187b42d6e4ce90a8fdc0d5
+Subproject commit e0025d0b6a26ce33b6e21ab53aba89768a02efc3
diff --git a/o2r-muncher b/o2r-muncher
index 776a9f5..914c040 160000
--- a/o2r-muncher
+++ b/o2r-muncher
@@ -1 +1 @@
-Subproject commit 776a9f5768bf8e32ac6f75e6a7dd39392f5f886e
+Subproject commit 914c0409f3d82a7a4b9d9b53752327bed54f5884
diff --git a/o2r-platform b/o2r-platform
index fcf8e54..8ca3454 160000
--- a/o2r-platform
+++ b/o2r-platform
@@ -1 +1 @@
-Subproject commit fcf8e544ae687b822b011dc0d7387dc0eede5a11
+Subproject commit 8ca3454247c7fb900c09f26ca5646ff4da8f0376
diff --git a/o2r-shipper b/o2r-shipper
index 168ae80..e7398ab 160000
--- a/o2r-shipper
+++ b/o2r-shipper
@@ -1 +1 @@
-Subproject commit 168ae80b0f196a25c68f9ce693fb8309aa798878
+Subproject commit e7398ab2aa05e30f422396ef7ed12d0d5c161d3e
diff --git a/o2r-substituter b/o2r-substituter
index 0863e61..e6a2e49 160000
--- a/o2r-substituter
+++ b/o2r-substituter
@@ -1 +1 @@
-Subproject commit 0863e61330f0209e839f707788a291ba99737ed0
+Subproject commit e6a2e49847a3f2d528531d729a24958e03a32ef7
diff --git a/o2r-transporter b/o2r-transporter
index e36b4a0..4c4babe 160000
--- a/o2r-transporter
+++ b/o2r-transporter
@@ -1 +1 @@
-Subproject commit e36b4a05591c4f3f0a42e8559e1299f3d3d16d72
+Subproject commit 4c4babecc73bda41db0984afcef5a5534d74a432
diff --git a/o2r-web-api b/o2r-web-api
index 5a5d65c..085396d 160000
--- a/o2r-web-api
+++ b/o2r-web-api
@@ -1 +1 @@
-Subproject commit 5a5d65cbe97811cd3268b84a5c3862e4ce70f417
+Subproject commit 085396dbc1d359443210cdb033be5e484ffeb9d4
From ce618d4f5a5858466e39d27cfcad5609b961e1aa Mon Sep 17 00:00:00 2001
From: nuest <daniel.nuest@uni-muenster.de>
Date: Wed, 14 Feb 2018 16:01:07 +0100
Subject: [PATCH 09/53] update submodules
---
architecture | 2 +-
erc-spec | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/architecture b/architecture
index 5df1ea9..e2447f2 160000
--- a/architecture
+++ b/architecture
@@ -1 +1 @@
-Subproject commit 5df1ea9d1288e931746fbe13fd70e7d4c068cb3e
+Subproject commit e2447f29df7f9f6f84067430a53f90a277306d1a
diff --git a/erc-spec b/erc-spec
index 50da6d0..77978a7 160000
--- a/erc-spec
+++ b/erc-spec
@@ -1 +1 @@
-Subproject commit 50da6d0d7ccedc9b02d9ec7ccbaf136c58bb6bc4
+Subproject commit 77978a757e5c426f6ee09e8c21e0d34e16deac9f
From 66d14c3a3c1d347c68e154f9fddf9308c8caabff Mon Sep 17 00:00:00 2001
From: nuest <daniel.nuest@uni-muenster.de>
Date: Wed, 14 Feb 2018 16:03:18 +0100
Subject: [PATCH 10/53] fix build documentation make target
---
Makefile | 7 +++----
1 file changed, 3 insertions(+), 4 deletions(-)
diff --git a/Makefile b/Makefile
index bec3c53..6190658 100644
--- a/Makefile
+++ b/Makefile
@@ -89,12 +89,12 @@ show_versions_hub:
@docker inspect --format '{{index .Config.Labels "org.label-schema.name"}}: {{index .Config.Labels "org.label-schema.version"}}' o2rproject/o2r-substituter;
@docker inspect --format '{{index .Config.Labels "org.label-schema.name"}}: {{index .Config.Labels "org.label-schema.version"}}' o2rproject/o2r-transporter;
-clean:
+clean_containers_and_images:
docker ps -a | grep o2r | awk '{print $1}' | xargs docker rm -f
docker images | grep o2r | awk '{print $3}' | xargs docker rmi --force
build_documentation:
- rm *.pdf
+ rm -f *.pdf
docker build --tag docbuilder --file etc/Dockerfile.documentations .
docker run -it -v $(CURDIR)/architecture:/doc:rw docbuilder make build pdf
docker run -it -v $(CURDIR)/o2r-web-api:/doc:rw docbuilder make build pdf
@@ -104,8 +104,7 @@ build_documentation:
mv o2r-web-api/*.pdf .
echo "ERC, architecture, and web API documentation created, see files PDF files in the project root directory"
-release: build_documentation
- git clone --recursive https://github.com/o2r-project/reference-implementation
+release: update build_documentation
# TODO build all images, export them to files
reproduce:
From 1a94c30b3a16d929709bf46dabff759e0bfbf8f3 Mon Sep 17 00:00:00 2001
From: LukasLohoff <lukas.lohoff@googlemail.com>
Date: Mon, 19 Feb 2018 09:53:08 +0100
Subject: [PATCH 11/53] Add guestlister submodule
---
.gitmodules | 3 +++
o2r-guestlister | 1 +
2 files changed, 4 insertions(+)
create mode 160000 o2r-guestlister
diff --git a/.gitmodules b/.gitmodules
index 3764a3c..5a438ae 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -46,3 +46,6 @@
[submodule "o2r-inspecter"]
path = o2r-inspecter
url = https://github.com/o2r-project/o2r-inspecter
+[submodule "o2r-guestlister"]
+ path = o2r-guestlister
+ url = https://github.com/o2r-project/o2r-guestlister
diff --git a/o2r-guestlister b/o2r-guestlister
new file mode 160000
index 0000000..1b6b9d5
--- /dev/null
+++ b/o2r-guestlister
@@ -0,0 +1 @@
+Subproject commit 1b6b9d549b159cd2d591061f7b195f0233413aa2
From 1ef6ea932c621169e639e810e2041eec532b6467 Mon Sep 17 00:00:00 2001
From: LukasLohoff <lukas.lohoff@googlemail.com>
Date: Tue, 20 Feb 2018 16:24:53 +0100
Subject: [PATCH 12/53] Add guestlister to docker-compose
---
docker-compose-local.yml | 15 +++++++++++++++
docker-compose.yml | 18 +++++++++++++++++-
o2r-guestlister | 2 +-
3 files changed, 33 insertions(+), 2 deletions(-)
diff --git a/docker-compose-local.yml b/docker-compose-local.yml
index a2985f0..ad7ce26 100644
--- a/docker-compose-local.yml
+++ b/docker-compose-local.yml
@@ -187,6 +187,21 @@ services:
- INSPECTER_PORT=8091
- DEBUGME=inspecter
+ guestlister:
+ image: o2rproject/o2r-guestlister:0.1.0
+ restart: unless-stopped
+ depends_on:
+ - configmongodb
+ environment:
+ - "GUESTLISTER_MONGODB=mongodb://mongodb"
+ - GUESTLISTER_PORT=8383
+ - DEBUG=guestlister
+ - OAUTH_URL_CALLBACK=${OAUTH_URL_CALLBACK}
+ - OAUTH_CLIENT_ID=${OAUTH_CLIENT_ID}
+ - OAUTH_CLIENT_SECRET=${OAUTH_CLIENT_SECRET}
+ - OAUTH_URL_AUTHORIZATION=${OAUTH_URL_AUTHORIZATION}
+ - OAUTH_URL_TOKEN=${OAUTH_URL_TOKEN}
+
platform:
image: o2r_refimpl_platform
restart: unless-stopped
diff --git a/docker-compose.yml b/docker-compose.yml
index 00ac36f..324c36d 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -110,7 +110,7 @@ services:
- DEBUG=informer
bouncer:
- image: o2rproject/o2r-bouncer:0.8.0
+ image: o2rproject/o2r-bouncer:0.10.1
restart: unless-stopped
depends_on:
- configmongodb
@@ -191,6 +191,21 @@ services:
- INSPECTER_PORT=8091
- DEBUGME=inspecter
+ guestlister:
+ image: o2rproject/o2r-guestlister:0.1.0
+ restart: unless-stopped
+ depends_on:
+ - configmongodb
+ environment:
+ - "GUESTLISTER_MONGODB=mongodb://mongodb"
+ - GUESTLISTER_PORT=8383
+ - DEBUG=guestlister
+ - OAUTH_URL_CALLBACK=${OAUTH_URL_CALLBACK}
+ - OAUTH_CLIENT_ID=${OAUTH_CLIENT_ID}
+ - OAUTH_CLIENT_SECRET=${OAUTH_CLIENT_SECRET}
+ - OAUTH_URL_AUTHORIZATION=${OAUTH_URL_AUTHORIZATION}
+ - OAUTH_URL_TOKEN=${OAUTH_URL_TOKEN}
+
platform:
image: o2rproject/o2r-platform:1.0.0
restart: unless-stopped
@@ -208,6 +223,7 @@ services:
- transporter
- shipper
- substituter
+ - guestlister
volumes:
- "./nginx.conf:/etc/nginx/nginx.conf:ro"
ports:
diff --git a/o2r-guestlister b/o2r-guestlister
index 1b6b9d5..c56ea09 160000
--- a/o2r-guestlister
+++ b/o2r-guestlister
@@ -1 +1 @@
-Subproject commit 1b6b9d549b159cd2d591061f7b195f0233413aa2
+Subproject commit c56ea095ad138d2194ad30577bf41fb9f0cb206d
From 417f302affcb65da03f5b6026d7adac8ab97e161 Mon Sep 17 00:00:00 2001
From: LukasLohoff <lukas.lohoff@googlemail.com>
Date: Tue, 20 Feb 2018 16:25:41 +0100
Subject: [PATCH 13/53] Fix environment vars and nginx.conf
---
.env | 8 ++++++++
nginx.conf | 8 +++++++-
2 files changed, 15 insertions(+), 1 deletion(-)
create mode 100644 .env
diff --git a/.env b/.env
new file mode 100644
index 0000000..23a67dd
--- /dev/null
+++ b/.env
@@ -0,0 +1,8 @@
+OAUTH_URL_AUTHORIZATION=http://localhost/oauth/authorize
+OAUTH_URL_TOKEN=http://webserver/oauth/token
+OAUTH_CLIENT_ID=testClient
+OAUTH_CLIENT_SECRET=testSecret
+OAUTH_URL_CALLBACK=http://localhost/api/v1/auth/login
+OAUTH_STARTUP_TEST=true
+OAUTH_STARTUP_FAIL_ON_ERROR=true
+SHIPPER_REPO_TOKENS={"download": "" }
\ No newline at end of file
diff --git a/nginx.conf b/nginx.conf
index 22cfa19..8db841b 100644
--- a/nginx.conf
+++ b/nginx.conf
@@ -113,5 +113,11 @@ http {
proxy_set_header X-Real-IP $remote_addr;
proxy_pass http://inspecter:8091;
}
+
+ location /oauth {
+ proxy_set_header Host $host;
+ proxy_set_header X-Real-IP $remote_addr;
+ proxy_pass http://guestlister:8383;
+ }
}
-}
+}
\ No newline at end of file
From 2c107dbdfde41c77f149bea241def00998ea6c84 Mon Sep 17 00:00:00 2001
From: LukasLohoff <lukas.lohoff@googlemail.com>
Date: Wed, 21 Feb 2018 14:04:52 +0100
Subject: [PATCH 14/53] update makefile for use with guestlister
---
Makefile | 9 +++++++--
docker-compose-local.yml | 2 +-
2 files changed, 8 insertions(+), 3 deletions(-)
diff --git a/Makefile b/Makefile
index bec3c53..f3e6f34 100644
--- a/Makefile
+++ b/Makefile
@@ -17,6 +17,7 @@ init:
git submodule add https://github.com/o2r-project/o2r-substituter
git submodule add https://github.com/o2r-project/o2r-transporter
git submodule add https://github.com/o2r-project/o2r-web-api
+ git submodule add https://github.com/o2r-project/o2r-guestlister
update:
git pull --recurse-submodules
@@ -35,9 +36,10 @@ build_images:
cd o2r-shipper; docker build --tag o2r_refimpl_shipper .; cd ..;
cd o2r-substituter; docker build --tag o2r_refimpl_substituter .; cd ..;
cd o2r-transporter; docker build --tag o2r_refimpl_transporter .; cd ..;
+ cd o2r-guestlister; docker build --tag o2r_refimpl_guestlister .; cd ..;
run_local:
- OAUTH_CLIENT_ID=$(value O2R_ORCID_ID) OAUTH_CLIENT_SECRET=$(value O2R_ORCID_SECRET) OAUTH_URL_CALLBACK=$(value O2R_ORCID_CALLBACK) ZENODO_TOKEN=$(value O2R_ZENODO_TOKEN) docker-compose --file docker-compose-local.yml up;
+ docker-compose --file docker-compose-local.yml up;
stop_local:
docker-compose --file docker-compose-local.yml down;
@@ -54,6 +56,7 @@ show_versions_local:
@docker inspect --format '{{index .Config.Labels "org.label-schema.name"}}: {{index .Config.Labels "org.label-schema.vcs-ref"}}' o2r_refimpl_shipper ;
@docker inspect --format '{{index .Config.Labels "org.label-schema.name"}}: {{index .Config.Labels "org.label-schema.version"}}' o2r_refimpl_substituter ;
@docker inspect --format '{{index .Config.Labels "org.label-schema.name"}}: {{index .Config.Labels "org.label-schema.version"}}' o2r_refimpl_transporter ;
+ @docker inspect --format '{{index .Config.Labels "org.label-schema.name"}}: {{index .Config.Labels "org.label-schema.version"}}' o2r_refimpl_guestlister ;
run_hub: pull_hub_images run_hub_images
@@ -72,9 +75,10 @@ pull_hub_images:
docker pull o2rproject/o2r-shipper;
docker pull o2rproject/o2r-substituter;
docker pull o2rproject/o2r-transporter;
+ docker pull o2rproject/o2r-guestlister;
run_hub_images:
- OAUTH_CLIENT_ID=$(value O2R_ORCID_ID) OAUTH_CLIENT_SECRET=$(value O2R_ORCID_SECRET) OAUTH_URL_CALLBACK=$(value O2R_ORCID_CALLBACK) ZENODO_TOKEN=$(value O2R_ZENODO_TOKEN) docker-compose up;
+ docker-compose up;
show_versions_hub:
@docker inspect --format '{{index .Config.Labels "org.label-schema.name"}}: {{index .Config.Labels "org.label-schema.version"}}' o2rproject/o2r-bouncer;
@@ -88,6 +92,7 @@ show_versions_hub:
@docker inspect --format '{{index .Config.Labels "org.label-schema.name"}}: {{index .Config.Labels "org.label-schema.vcs-ref"}}' o2rproject/o2r-shipper;
@docker inspect --format '{{index .Config.Labels "org.label-schema.name"}}: {{index .Config.Labels "org.label-schema.version"}}' o2rproject/o2r-substituter;
@docker inspect --format '{{index .Config.Labels "org.label-schema.name"}}: {{index .Config.Labels "org.label-schema.version"}}' o2rproject/o2r-transporter;
+ @docker inspect --format '{{index .Config.Labels "org.label-schema.name"}}: {{index .Config.Labels "org.label-schema.version"}}' o2rproject/o2r-guestlister;
clean:
docker ps -a | grep o2r | awk '{print $1}' | xargs docker rm -f
diff --git a/docker-compose-local.yml b/docker-compose-local.yml
index ad7ce26..845972b 100644
--- a/docker-compose-local.yml
+++ b/docker-compose-local.yml
@@ -188,7 +188,7 @@ services:
- DEBUGME=inspecter
guestlister:
- image: o2rproject/o2r-guestlister:0.1.0
+ image: o2r_refimpl_guestlister
restart: unless-stopped
depends_on:
- configmongodb
From d18f66b8dadb5d6e78bb1b9b9e3581aaabeeada7 Mon Sep 17 00:00:00 2001
From: LukasLohoff <lukas.lohoff@googlemail.com>
Date: Wed, 21 Feb 2018 14:40:49 +0100
Subject: [PATCH 15/53] Update readme
---
README.md | 23 +++++++++++------------
1 file changed, 11 insertions(+), 12 deletions(-)
diff --git a/README.md b/README.md
index 24cdac8..1a526e6 100644
--- a/README.md
+++ b/README.md
@@ -68,20 +68,22 @@ Run `make update` or the respective commands on your operating system to initial
#### Accounts and tokens
-##### ORCID
+##### o2r-guestlister
-The reference implementation relies on [ORCID](https://orcid.org/) for authentication and authorisation.
-There is no other way to log in on the platform, therefore you must enable your installation of the reference implementation to connect with the ORCID API.
-The client credentials used by the o2r team cannot be shared publicly for security reasons.
+By default, the reference implementation uses the offline OAuth2 implementation provided by the [o2r-guestlister](https://github.com/o2r-project/o2r-guestlister).
+
+##### ORCID (optional)
+
+The reference implementation can be configured to use [ORCID](https://orcid.org/) for authentication and authorisation, replacing the offline login provided by o2r-guestlister.
+
+This requires an ORCID account which provides authentication tokens for public API client application with **[ORCID Sandbox](https://sandbox.orcid.org/signin)**.
-You _must_ register an account and get a authentication tokens for public API client application with **[ORCID Sandbox](https://sandbox.orcid.org/signin)**.
Alternatively, use your existing ORCID account and register an application as [described here](https://support.orcid.org/knowledgebase/articles/343182-register-a-public-api-client-application).
-Note that in the latter case you must adjust the URLs in the example below: remove the `sandbox.` part.
In the developer tools, use any name, website URL, and description.
Important is the `Redirect URIs` list, which must include `http://localhost` for your local installation.
-The client ID, client secret, and redirect URI must then be provided to the docker-compose configurations via environment variables as shown below.
+The client ID, client secret, redirect URI and the OAuth URLs have to be provided by modifying the `.env` file in the base directory. Note that environment variables provided throught the shell have priority over the `.env` file configuration. For more information on how the `.env` file works, see the docker-compose [documentation](https://docs.docker.com/compose/env-file/).
##### Repositories (optional)
@@ -90,6 +92,8 @@ These repositories also require an authentication token.
- [Create access token](https://zenodo.org/login/?next=%2Faccount%2Fsettings%2Fapplications%2Ftokens%2Fnew%2F) for [Zenodo](https://zenodo.org/)
+These tokens can be provided to the docker-compose configurations by setting them as environment variables in the `.env` file, similar to the ORCID configuration.
+
#### Elasticsearch host preparation
The implementation uses an [Elasticsearch](http://elastic.co) document search engine.
@@ -111,9 +115,6 @@ To download all o2r source code at once, navigate to the `reference-implementati
Once all repositories have been pulled successfully, build docker images of the microservices and run them in containers by executing:
```bash
-OAUTH_CLIENT_ID=<your orcid id> OAUTH_CLIENT_SECRET=<your orcid secret> \
- OAUTH_URL_AUTHORIZATION=https://sandbox.orcid.org/oauth/authorize OAUTH_URL_TOKEN=https://sandbox.orcid.org/oauth/token OAUTH_URL_CALLBACK=http://localhost/api/v1/auth/login \
- SHIPPER_REPO_TOKENS="{\"zenodo\": \"<your Zenodo token>\", \"zenodo_sandbox\": \"<your Zenodo Sandbox token>\", "download": \"\" }" \
make build_images run_local
```
@@ -126,8 +127,6 @@ All o2r software projects have automatic builds [available on Docker Hub](https:
The following command executes a `docker-compose` command to pull and run these images.
```bash
-O2R_ORCID_ID=<your orcid id> O2R_ORCID_SECRET=<your orcid secret> O2R_ORCID_CALLBACK=http://localhost/api/v1/auth/login \
- SHIPPER_REPO_TOKENS="{\"zenodo\": \"<your Zenodo token>\", \"zenodo_sandbox\": \"<your Zenodo Sandbox token>\", "download": \"\" }" \
make run_hub
```
From b7886ab0b6c781e78ce02a69cc2b235d731843aa Mon Sep 17 00:00:00 2001
From: LukasLohoff <lukas.lohoff@googlemail.com>
Date: Wed, 21 Feb 2018 15:28:40 +0100
Subject: [PATCH 16/53] Update windows-readme and readme
---
README-WIN.md | 16 +++++++---------
README.md | 13 +++++++++++--
2 files changed, 18 insertions(+), 11 deletions(-)
diff --git a/README-WIN.md b/README-WIN.md
index bc6ec08..5fb52a2 100644
--- a/README-WIN.md
+++ b/README-WIN.md
@@ -3,25 +3,23 @@
This document contains Windows-specific configuration and steps.
**Please carefully read the sections "Basics" and "Prerequisites" in the file `README.md` for general information.
-In the sections "Build images from source and run" and "Build images from source and run", please be aware of the general remarks but use the commands from this file instead.
-Take a look at the "Troubleshooting" sections in this file if you run into problems.
+In the sections "Build images from source and run" and "Build images from source and run", please be aware of the general remarks but use the `docker-compose` command from this file instead of the commands using `make`.
+Take a look at the "Troubleshooting" sections in this file if you run into problems.**
## Windows with Docker for Windows
[Docker for Windows](https://docs.docker.com/docker-for-windows/) is available for 64bit Windows 10 Pro, Enterprise and Education (with Hyper-V available) and on Windows Server 2016 (see [Docker Docs](https://docs.docker.com/docker-for-windows/install/#what-to-know-before-you-install)).
-The environmental variables must be passed separately on Windows, followed by the docker-compose commands:
+Simply run the following command to start the services:
```shell
-setx OAUTH_CLIENT_ID = "<... required ...>"
-setx OAUTH_CLIENT_SECRET = "<... required ...>"
-setx OAUTH_URL_CALLBACK = "http://localhost/api/v1/auth/login"
-setx ZENODO_TOKEN = "<... optional ...>"
docker-compose up
```
The services are available at `http://localhost`.
+Advanced configuration can be applied by editing the `.env` file or by specifying the settings as environment variables before executing the `docker-compose up` command.
+
### Troubleshooting
- Make sure that the drive where you cloned this repository to is shared (Docker > Shared Drives), because a configuration file (`nginx.conf`) must be mounted into the service `webserver`.
@@ -38,10 +36,10 @@ If your system does not meet the requirements to run Docker for Windows, you can
When using Compose with Docker Toolbox/Machine on Windows, [volume paths are no longer converted from by default](https://github.com/docker/compose/releases/tag/1.9.0), but we need this conversion to be able to mount the docker volume to the o2r microservices. To re-enable this conversion for `docker-compose >= 1.9.0` set the environment variable `COMPOSE_CONVERT_WINDOWS_PATHS=1`.
-Also, the client's defaults (i.e. using `localhost`) does not work. We must mount a config file to point the API to the correct location, see `win/config-toolbox.js`, and use the prepared configuration file `win/docker-compose-toolbox.yml`.
+Add `COMPOSE_CONVERT_WINDOWS_PATHS=1` to the `.env` file and run:
```bash
-COMPOSE_CONVERT_WINDOWS_PATHS=1 OAUTH_CLIENT_ID=<...> OAUTH_CLIENT_SECRET=<...> OAUTH_URL_CALLBACK=<...> ZENODO_TOKEN=<...> docker-compose up
+docker-compose up
```
The services are available at `http://<machine-ip>`.
diff --git a/README.md b/README.md
index 1a526e6..802fdd6 100644
--- a/README.md
+++ b/README.md
@@ -31,6 +31,7 @@ The documentation is also available online for reading, though availability may
| platform (web UI) | `o2r-platform` | https://github.com/o2r-project/o2r-platform|
| shipper (microservice) | `o2r-shipper` | https://github.com/o2r-project/o2r-shipper |
| substituter (microservice) | `o2r-substituter` | https://github.com/o2r-project/o2r-substituter |
+| guestlister (microservice) | `o2r-guestlister` | https://github.com/o2r-project/o2r-guestlister |
| transporter (microservice) | `o2r-transporter` | https://github.com/o2r-project/o2r-transporter |
### Supported operating systems
@@ -70,11 +71,13 @@ Run `make update` or the respective commands on your operating system to initial
##### o2r-guestlister
-By default, the reference implementation uses the offline OAuth2 implementation provided by the [o2r-guestlister](https://github.com/o2r-project/o2r-guestlister).
+By default, the reference implementation uses the offline OAuth2 implementation provided by the [o2r-guestlister](https://github.com/o2r-project/o2r-guestlister).
+
+This allows access to the o2r platform by selecting one of three demo users. The users represent different user roles with different levels, i.e. an admin (level 1000), an editor (level 500) and a basic author (level 100).
##### ORCID (optional)
-The reference implementation can be configured to use [ORCID](https://orcid.org/) for authentication and authorisation, replacing the offline login provided by o2r-guestlister.
+The reference implementation can be alternatively configured to use [ORCID](https://orcid.org/) for authentication and authorisation, replacing the offline login provided by o2r-guestlister.
This requires an ORCID account which provides authentication tokens for public API client application with **[ORCID Sandbox](https://sandbox.orcid.org/signin)**.
@@ -94,6 +97,12 @@ These repositories also require an authentication token.
These tokens can be provided to the docker-compose configurations by setting them as environment variables in the `.env` file, similar to the ORCID configuration.
+Modify the `SHIPPER_REPO_TOKENS` entry in the `.env` file to include the tokens:
+
+```
+SHIPPER_REPO_TOKENS={"zenodo": "<your Zenodo token>", "zenodo_sandbox": "<your Zenodo Sandbox token>", "download": "" }
+```
+
#### Elasticsearch host preparation
The implementation uses an [Elasticsearch](http://elastic.co) document search engine.
From 727894f450f5007e71bb28cf4c483820b33a7786 Mon Sep 17 00:00:00 2001
From: LukasLohoff <lukas.lohoff@googlemail.com>
Date: Wed, 21 Feb 2018 16:10:03 +0100
Subject: [PATCH 17/53] fix makefile
---
Makefile | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/Makefile b/Makefile
index f3e6f34..cde661c 100644
--- a/Makefile
+++ b/Makefile
@@ -75,7 +75,7 @@ pull_hub_images:
docker pull o2rproject/o2r-shipper;
docker pull o2rproject/o2r-substituter;
docker pull o2rproject/o2r-transporter;
- docker pull o2rproject/o2r-guestlister;
+ docker pull o2rproject/o2r-guestlister;
run_hub_images:
docker-compose up;
From bb8cba2b4c026a1602fdabe3894a156d338695dc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20N=C3=BCst?= <daniel.nuest@wwu.de>
Date: Wed, 21 Feb 2018 16:51:48 +0100
Subject: [PATCH 18/53] Update README.md
---
README.md | 5 ++---
1 file changed, 2 insertions(+), 3 deletions(-)
diff --git a/README.md b/README.md
index 802fdd6..d812cd9 100644
--- a/README.md
+++ b/README.md
@@ -79,9 +79,7 @@ This allows access to the o2r platform by selecting one of three demo users. The
The reference implementation can be alternatively configured to use [ORCID](https://orcid.org/) for authentication and authorisation, replacing the offline login provided by o2r-guestlister.
-This requires an ORCID account which provides authentication tokens for public API client application with **[ORCID Sandbox](https://sandbox.orcid.org/signin)**.
-
-Alternatively, use your existing ORCID account and register an application as [described here](https://support.orcid.org/knowledgebase/articles/343182-register-a-public-api-client-application).
+This requires an ORCID account which provides authentication tokens for public API client application with **[ORCID Sandbox](https://sandbox.orcid.org/signin)** (preferred for testing) or the registering an application as [described here](https://support.orcid.org/knowledgebase/articles/343182-register-a-public-api-client-application) with your regular ORCID account.
In the developer tools, use any name, website URL, and description.
Important is the `Redirect URIs` list, which must include `http://localhost` for your local installation.
@@ -91,6 +89,7 @@ The client ID, client secret, redirect URI and the OAuth URLs have to be provide
##### Repositories (optional)
The reference implementation can deliver the created ERC to different repositories.
+By default only the "Download" repository is supported, i.e. users may download a complete ERC as an archive file.
These repositories also require an authentication token.
- [Create access token](https://zenodo.org/login/?next=%2Faccount%2Fsettings%2Fapplications%2Ftokens%2Fnew%2F) for [Zenodo](https://zenodo.org/)
From e8dadebf5af01d73b12cec523ea47c1851285c14 Mon Sep 17 00:00:00 2001
From: nuest <daniel.nuest@uni-muenster.de>
Date: Fri, 23 Feb 2018 14:42:15 +0100
Subject: [PATCH 19/53] update microservices
---
docker-compose.yml | 10 ++++++----
1 file changed, 6 insertions(+), 4 deletions(-)
diff --git a/docker-compose.yml b/docker-compose.yml
index 00ac36f..58572e4 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -69,7 +69,7 @@ services:
mem_limit: 1g
muncher:
- image: o2rproject/o2r-muncher:0.13.8
+ image: o2rproject/o2r-muncher:0.14.3
restart: unless-stopped
depends_on:
- configmongodb
@@ -83,6 +83,7 @@ services:
- MUNCHER_CONTAINER_USER=root
- MUNCHER_CONTAINERIT_USER=root
- DEBUG=*,-mquery,-express:*,-express-session,-body-parser:*
+ - "MUNCHER_META_TOOL_CONTAINER=o2rproject/o2r-meta:5c12559"
loader:
image: o2rproject/o2r-loader:0.8.3
@@ -97,6 +98,7 @@ services:
- LOADER_PORT=8088
- DEBUG=*,-mquery,-express:*,-express-session,-body-parser:*
- LOADER_VOLUME=referenceimplementation_o2rvol
+ - "LOADER_META_TOOL_CONTAINER=o2rproject/o2r-meta:5c12559"
informer:
image: o2rproject/o2r-informer:0.3.0
@@ -144,7 +146,7 @@ services:
- BATCH_COUNT=20
transporter:
- image: o2rproject/o2r-transporter:0.4.1
+ image: o2rproject/o2r-transporter:0.5.0
restart: unless-stopped
depends_on:
- configmongodb
@@ -157,7 +159,7 @@ services:
- DEBUG=transporter,transporter:*
shipper:
- image: o2rproject/o2r-shipper:168ae80
+ image: o2rproject/o2r-shipper:e7398ab
restart: unless-stopped
depends_on:
- configmongodb
@@ -171,7 +173,7 @@ services:
PYTHONUNBUFFERED: "0"
substituter:
- image: o2rproject/o2r-substituter:0.4.1
+ image: o2rproject/o2r-substituter:0.4.3
restart: unless-stopped
depends_on:
- configmongodb
From 95287c81fdc780d306313362306f8a66cfe834fc Mon Sep 17 00:00:00 2001
From: nuest <daniel.nuest@uni-muenster.de>
Date: Fri, 23 Feb 2018 16:23:19 +0100
Subject: [PATCH 20/53] update microservice, make target names, and README
---
Makefile | 20 +++++++++++---------
README.md | 20 ++++++++++++++------
docker-compose.yml | 2 +-
3 files changed, 26 insertions(+), 16 deletions(-)
diff --git a/Makefile b/Makefile
index 79884ca..87b9863 100644
--- a/Makefile
+++ b/Makefile
@@ -24,7 +24,7 @@ update:
git submodule update --init --recursive --remote
git submodule foreach git pull origin master
-build_images:
+local_images:
cd o2r-bouncer; docker build --tag o2r_refimpl_bouncer .; cd ..;
cd o2r-finder; docker build --tag o2r_refimpl_finder .; cd ..;
cd o2r-informer; docker build --tag o2r_refimpl_informer .; cd ..;
@@ -38,13 +38,15 @@ build_images:
cd o2r-transporter; docker build --tag o2r_refimpl_transporter .; cd ..;
cd o2r-guestlister; docker build --tag o2r_refimpl_guestlister .; cd ..;
-run_local:
+local_up:
docker-compose --file docker-compose-local.yml up;
-stop_local:
+local_down:
docker-compose --file docker-compose-local.yml down;
-show_versions_local:
+local: local_images local_versions local_up
+
+local_versions:
@docker inspect --format '{{index .Config.Labels "org.label-schema.name"}}: {{index .Config.Labels "org.label-schema.version"}}' o2r_refimpl_bouncer ;
@docker inspect --format '{{index .Config.Labels "org.label-schema.name"}}: {{index .Config.Labels "org.label-schema.version"}}' o2r_refimpl_finder ;
@docker inspect --format '{{index .Config.Labels "org.label-schema.name"}}: {{index .Config.Labels "org.label-schema.version"}}' o2r_refimpl_informer ;
@@ -58,12 +60,12 @@ show_versions_local:
@docker inspect --format '{{index .Config.Labels "org.label-schema.name"}}: {{index .Config.Labels "org.label-schema.version"}}' o2r_refimpl_transporter ;
@docker inspect --format '{{index .Config.Labels "org.label-schema.name"}}: {{index .Config.Labels "org.label-schema.version"}}' o2r_refimpl_guestlister ;
-run_hub: pull_hub_images run_hub_images
+hub: hub_images hub_versions hub_up
-stop_hub:
+hub_down:
docker-compose down;
-pull_hub_images:
+hub_images:
docker pull o2rproject/o2r-bouncer;
docker pull o2rproject/o2r-finder;
docker pull o2rproject/o2r-informer;
@@ -77,10 +79,10 @@ pull_hub_images:
docker pull o2rproject/o2r-transporter;
docker pull o2rproject/o2r-guestlister;
-run_hub_images:
+hub_up:
docker-compose up;
-show_versions_hub:
+hub_versions:
@docker inspect --format '{{index .Config.Labels "org.label-schema.name"}}: {{index .Config.Labels "org.label-schema.version"}}' o2rproject/o2r-bouncer;
@docker inspect --format '{{index .Config.Labels "org.label-schema.name"}}: {{index .Config.Labels "org.label-schema.version"}}' o2rproject/o2r-finder;
@docker inspect --format '{{index .Config.Labels "org.label-schema.name"}}: {{index .Config.Labels "org.label-schema.version"}}' o2rproject/o2r-informer;
diff --git a/README.md b/README.md
index d812cd9..3052d3a 100644
--- a/README.md
+++ b/README.md
@@ -120,13 +120,13 @@ To download all o2r source code at once, navigate to the `reference-implementati
make update
```
-Once all repositories have been pulled successfully, build docker images of the microservices and run them in containers by executing:
+Once all repositories have been pulled successfully, build Docker images of the microservices and run them in containers by executing:
```bash
- make build_images run_local
+ make local
```
-Wait for a while, then open **http://localhost**.
+Wait until the log shows no new messages, then open **http://localhost** and continue in section ["Use platform"](#use-platform).
### Download images and run
@@ -135,10 +135,10 @@ All o2r software projects have automatic builds [available on Docker Hub](https:
The following command executes a `docker-compose` command to pull and run these images.
```bash
- make run_hub
+ make hub
```
-Wait a bit, then open **http://localhost**.
+Wait until the log shows no new messages, then open **http://localhost** and continue in section ["Use platform"](#use-platform).
### Load data
@@ -146,7 +146,15 @@ The o2r API supports two way to load scientific workflows: direct upload as a ZI
Ready to use **Direct upload** examples are available in the directory `erc-examples`.
-Examples for loading from a public share are available here: https://uni-muenster.sciebo.de/index.php/s/G8vxQ1h50V4HpuA (see file `README.txt`).
+Examples for loading from a public share are available [in this online share](https://uni-muenster.sciebo.de/index.php/s/G8vxQ1h50V4HpuA) (see file `README.txt`) and via the "EXAMPLES" menu in the "Create" section of the platform website.
+
+### Use platform
+
+1. Click on "LOGIN" in the upper right hand corner
+1. Select one of the available users, e.g. "Author"
+1. In the "Create" section of the platform website, select a workspace from the "EXAMPLES" menu
+1. Fill in required metadata
+1. Finish the upload and open the ERC page: explore the running job and all interaction possiblities
### Explore back-end
diff --git a/docker-compose.yml b/docker-compose.yml
index 63aed05..fe103a9 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -86,7 +86,7 @@ services:
- "MUNCHER_META_TOOL_CONTAINER=o2rproject/o2r-meta:5c12559"
loader:
- image: o2rproject/o2r-loader:0.8.3
+ image: o2rproject/o2r-loader:0.10.0
restart: unless-stopped
depends_on:
- configmongodb
From b3be4166221653240e3e3c11980b0ff61d58479e Mon Sep 17 00:00:00 2001
From: nuest <daniel.nuest@uni-muenster.de>
Date: Fri, 23 Feb 2018 16:23:27 +0100
Subject: [PATCH 21/53] update submodules
---
architecture | 2 +-
erc-spec | 2 +-
o2r-bouncer | 2 +-
o2r-guestlister | 2 +-
o2r-meta | 2 +-
o2r-muncher | 2 +-
6 files changed, 6 insertions(+), 6 deletions(-)
diff --git a/architecture b/architecture
index e2447f2..3049592 160000
--- a/architecture
+++ b/architecture
@@ -1 +1 @@
-Subproject commit e2447f29df7f9f6f84067430a53f90a277306d1a
+Subproject commit 3049592d79509dd832535467cc752cc38d0dc499
diff --git a/erc-spec b/erc-spec
index 77978a7..7d5d3bc 160000
--- a/erc-spec
+++ b/erc-spec
@@ -1 +1 @@
-Subproject commit 77978a757e5c426f6ee09e8c21e0d34e16deac9f
+Subproject commit 7d5d3bc3130116d1050cae5ee486e31bae92b0ff
diff --git a/o2r-bouncer b/o2r-bouncer
index ad8e931..4d6b59a 160000
--- a/o2r-bouncer
+++ b/o2r-bouncer
@@ -1 +1 @@
-Subproject commit ad8e931e158c19029fbde9238ebe9783931df01e
+Subproject commit 4d6b59aaa5933b794f3f9fb333ecb05eb6185149
diff --git a/o2r-guestlister b/o2r-guestlister
index c56ea09..7109e4a 160000
--- a/o2r-guestlister
+++ b/o2r-guestlister
@@ -1 +1 @@
-Subproject commit c56ea095ad138d2194ad30577bf41fb9f0cb206d
+Subproject commit 7109e4ade2b10c74f1597b019dde80f6479976e0
diff --git a/o2r-meta b/o2r-meta
index e0025d0..5c12559 160000
--- a/o2r-meta
+++ b/o2r-meta
@@ -1 +1 @@
-Subproject commit e0025d0b6a26ce33b6e21ab53aba89768a02efc3
+Subproject commit 5c12559803106db3ded76de661696a9414728f7d
diff --git a/o2r-muncher b/o2r-muncher
index 914c040..0a2e90f 160000
--- a/o2r-muncher
+++ b/o2r-muncher
@@ -1 +1 @@
-Subproject commit 914c0409f3d82a7a4b9d9b53752327bed54f5884
+Subproject commit 0a2e90fc254e93358f77485d5d89f82744862da6
From e3bbefb66c9e7cfad7fe711a14871f7c5c9cd4e5 Mon Sep 17 00:00:00 2001
From: nuest <daniel.nuest@uni-muenster.de>
Date: Fri, 23 Feb 2018 17:25:09 +0100
Subject: [PATCH 22/53] update microservices
---
docker-compose.yml | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/docker-compose.yml b/docker-compose.yml
index fe103a9..f12b363 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -69,7 +69,7 @@ services:
mem_limit: 1g
muncher:
- image: o2rproject/o2r-muncher:0.14.3
+ image: o2rproject/o2r-muncher:0.15.0
restart: unless-stopped
depends_on:
- configmongodb
@@ -129,7 +129,7 @@ services:
- SLACK_BOT_TOKEN=${SLACK_BOT_TOKEN}
finder:
- image: o2rproject/o2r-finder:0.6.0
+ image: o2rproject/o2r-finder:0.6.1
restart: unless-stopped
depends_on:
- configmongodb
From 4d27c744c09c5187affaa51fc66f976747097e70 Mon Sep 17 00:00:00 2001
From: nuest <daniel.nuest@uni-muenster.de>
Date: Thu, 8 Mar 2018 12:18:04 +0100
Subject: [PATCH 23/53] update bouncer and guestlister (simplified defaults)
---
.env | 4 ----
docker-compose-local.yml | 6 ------
docker-compose.yml | 10 ++--------
3 files changed, 2 insertions(+), 18 deletions(-)
diff --git a/.env b/.env
index 23a67dd..cf71683 100644
--- a/.env
+++ b/.env
@@ -1,8 +1,4 @@
OAUTH_URL_AUTHORIZATION=http://localhost/oauth/authorize
OAUTH_URL_TOKEN=http://webserver/oauth/token
-OAUTH_CLIENT_ID=testClient
-OAUTH_CLIENT_SECRET=testSecret
OAUTH_URL_CALLBACK=http://localhost/api/v1/auth/login
-OAUTH_STARTUP_TEST=true
-OAUTH_STARTUP_FAIL_ON_ERROR=true
SHIPPER_REPO_TOKENS={"download": "" }
\ No newline at end of file
diff --git a/docker-compose-local.yml b/docker-compose-local.yml
index 845972b..796fcca 100644
--- a/docker-compose-local.yml
+++ b/docker-compose-local.yml
@@ -115,12 +115,8 @@ services:
- BOUNCER_PORT=8083
- DEBUG=bouncer
- OAUTH_URL_CALLBACK=${OAUTH_URL_CALLBACK}
- - OAUTH_CLIENT_ID=${OAUTH_CLIENT_ID}
- - OAUTH_CLIENT_SECRET=${OAUTH_CLIENT_SECRET}
- OAUTH_URL_AUTHORIZATION=${OAUTH_URL_AUTHORIZATION}
- OAUTH_URL_TOKEN=${OAUTH_URL_TOKEN}
- - SLACK_VERIFICATION_TOKEN=${SLACK_VERIFICATION_TOKEN}
- - SLACK_BOT_TOKEN=${SLACK_BOT_TOKEN}
finder:
image: o2r_refimpl_finder
@@ -197,8 +193,6 @@ services:
- GUESTLISTER_PORT=8383
- DEBUG=guestlister
- OAUTH_URL_CALLBACK=${OAUTH_URL_CALLBACK}
- - OAUTH_CLIENT_ID=${OAUTH_CLIENT_ID}
- - OAUTH_CLIENT_SECRET=${OAUTH_CLIENT_SECRET}
- OAUTH_URL_AUTHORIZATION=${OAUTH_URL_AUTHORIZATION}
- OAUTH_URL_TOKEN=${OAUTH_URL_TOKEN}
diff --git a/docker-compose.yml b/docker-compose.yml
index f12b363..27bf708 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -112,7 +112,7 @@ services:
- DEBUG=informer
bouncer:
- image: o2rproject/o2r-bouncer:0.10.1
+ image: o2rproject/o2r-bouncer:0.11.0
restart: unless-stopped
depends_on:
- configmongodb
@@ -121,12 +121,8 @@ services:
- BOUNCER_PORT=8083
- DEBUG=bouncer
- OAUTH_URL_CALLBACK=${OAUTH_URL_CALLBACK}
- - OAUTH_CLIENT_ID=${OAUTH_CLIENT_ID}
- - OAUTH_CLIENT_SECRET=${OAUTH_CLIENT_SECRET}
- OAUTH_URL_AUTHORIZATION=${OAUTH_URL_AUTHORIZATION}
- OAUTH_URL_TOKEN=${OAUTH_URL_TOKEN}
- - SLACK_VERIFICATION_TOKEN=${SLACK_VERIFICATION_TOKEN}
- - SLACK_BOT_TOKEN=${SLACK_BOT_TOKEN}
finder:
image: o2rproject/o2r-finder:0.6.1
@@ -194,7 +190,7 @@ services:
- DEBUGME=inspecter
guestlister:
- image: o2rproject/o2r-guestlister:0.1.0
+ image: o2rproject/o2r-guestlister:0.2.0
restart: unless-stopped
depends_on:
- configmongodb
@@ -203,8 +199,6 @@ services:
- GUESTLISTER_PORT=8383
- DEBUG=guestlister
- OAUTH_URL_CALLBACK=${OAUTH_URL_CALLBACK}
- - OAUTH_CLIENT_ID=${OAUTH_CLIENT_ID}
- - OAUTH_CLIENT_SECRET=${OAUTH_CLIENT_SECRET}
- OAUTH_URL_AUTHORIZATION=${OAUTH_URL_AUTHORIZATION}
- OAUTH_URL_TOKEN=${OAUTH_URL_TOKEN}
From ae04a6cb45ab23ca5b23f66ab89d21945c8141d4 Mon Sep 17 00:00:00 2001
From: nuest <daniel.nuest@uni-muenster.de>
Date: Thu, 8 Mar 2018 12:18:20 +0100
Subject: [PATCH 24/53] add make target for down with volume
---
Makefile | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/Makefile b/Makefile
index 87b9863..e925ca4 100644
--- a/Makefile
+++ b/Makefile
@@ -62,8 +62,8 @@ local_versions:
hub: hub_images hub_versions hub_up
-hub_down:
- docker-compose down;
+hub_down_volume:
+ docker-compose down --volume;
hub_images:
docker pull o2rproject/o2r-bouncer;
@@ -96,7 +96,7 @@ hub_versions:
@docker inspect --format '{{index .Config.Labels "org.label-schema.name"}}: {{index .Config.Labels "org.label-schema.version"}}' o2rproject/o2r-transporter;
@docker inspect --format '{{index .Config.Labels "org.label-schema.name"}}: {{index .Config.Labels "org.label-schema.version"}}' o2rproject/o2r-guestlister;
-clean_containers_and_images:
+clean: hub_down_volume
docker ps -a | grep o2r | awk '{print $1}' | xargs docker rm -f
docker images | grep o2r | awk '{print $3}' | xargs docker rmi --force
From e46030d1a6a3470ed9aec166b6450b3b8fc0d517 Mon Sep 17 00:00:00 2001
From: nuest <daniel.nuest@uni-muenster.de>
Date: Thu, 8 Mar 2018 12:18:42 +0100
Subject: [PATCH 25/53] update muncher microservice and reduce finder batch
size
---
docker-compose-local.yml | 2 +-
docker-compose.yml | 4 ++--
2 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/docker-compose-local.yml b/docker-compose-local.yml
index 796fcca..a97ae3d 100644
--- a/docker-compose-local.yml
+++ b/docker-compose-local.yml
@@ -133,7 +133,7 @@ services:
- ELASTIC_SEARCH_URL=elasticsearch:9200
- "MONGO_DATA_URL=mongodb://mongodb/muncher"
- "MONGO_OPLOG_URL=mongodb://mongodb/muncher"
- - BATCH_COUNT=20
+ - BATCH_COUNT=5
transporter:
image: o2r_refimpl_transporter
diff --git a/docker-compose.yml b/docker-compose.yml
index 27bf708..6cf60a5 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -69,7 +69,7 @@ services:
mem_limit: 1g
muncher:
- image: o2rproject/o2r-muncher:0.15.0
+ image: o2rproject/o2r-muncher:0.17.0
restart: unless-stopped
depends_on:
- configmongodb
@@ -139,7 +139,7 @@ services:
- ELASTIC_SEARCH_URL=elasticsearch:9200
- "MONGO_DATA_URL=mongodb://mongodb/muncher"
- "MONGO_OPLOG_URL=mongodb://mongodb/muncher"
- - BATCH_COUNT=20
+ - BATCH_COUNT=5
transporter:
image: o2rproject/o2r-transporter:0.5.0
From e8d64430958d59e9a051e0b7186f89f265a57589 Mon Sep 17 00:00:00 2001
From: nuest <daniel.nuest@uni-muenster.de>
Date: Thu, 8 Mar 2018 16:31:24 +0100
Subject: [PATCH 26/53] Update subrepos
---
architecture | 2 +-
o2r-bouncer | 2 +-
o2r-finder | 2 +-
o2r-guestlister | 2 +-
o2r-muncher | 2 +-
o2r-platform | 2 +-
o2r-web-api | 2 +-
7 files changed, 7 insertions(+), 7 deletions(-)
diff --git a/architecture b/architecture
index 3049592..90d5a33 160000
--- a/architecture
+++ b/architecture
@@ -1 +1 @@
-Subproject commit 3049592d79509dd832535467cc752cc38d0dc499
+Subproject commit 90d5a33ad2aa44a77446025aa0c2551d14a6789d
diff --git a/o2r-bouncer b/o2r-bouncer
index 4d6b59a..851f45f 160000
--- a/o2r-bouncer
+++ b/o2r-bouncer
@@ -1 +1 @@
-Subproject commit 4d6b59aaa5933b794f3f9fb333ecb05eb6185149
+Subproject commit 851f45f7e4f92d2cf01b968bdd2f6f2a9ea4aca3
diff --git a/o2r-finder b/o2r-finder
index 502995e..2ffbbf0 160000
--- a/o2r-finder
+++ b/o2r-finder
@@ -1 +1 @@
-Subproject commit 502995e1bdfd2c9b1fa0c88d2718f4046ad7456b
+Subproject commit 2ffbbf0d41fb20e8078c45bd21bb2a97b90fe1b1
diff --git a/o2r-guestlister b/o2r-guestlister
index 7109e4a..dac99d2 160000
--- a/o2r-guestlister
+++ b/o2r-guestlister
@@ -1 +1 @@
-Subproject commit 7109e4ade2b10c74f1597b019dde80f6479976e0
+Subproject commit dac99d24cd4b4c6b3bbf185fccf5a495458f101e
diff --git a/o2r-muncher b/o2r-muncher
index 0a2e90f..0e2a5cf 160000
--- a/o2r-muncher
+++ b/o2r-muncher
@@ -1 +1 @@
-Subproject commit 0a2e90fc254e93358f77485d5d89f82744862da6
+Subproject commit 0e2a5cfff68a6adeddcf43d2df42fde52849e6fc
diff --git a/o2r-platform b/o2r-platform
index 8ca3454..681b674 160000
--- a/o2r-platform
+++ b/o2r-platform
@@ -1 +1 @@
-Subproject commit 8ca3454247c7fb900c09f26ca5646ff4da8f0376
+Subproject commit 681b67490d828ba635d2e4c2b869a3e963133282
diff --git a/o2r-web-api b/o2r-web-api
index 085396d..c466789 160000
--- a/o2r-web-api
+++ b/o2r-web-api
@@ -1 +1 @@
-Subproject commit 085396dbc1d359443210cdb033be5e484ffeb9d4
+Subproject commit c4667890745b84ba7b95f93466cbc617b828d790
From 23ebbd9fbf81be95eefc5bf296df3bd81880b98c Mon Sep 17 00:00:00 2001
From: nuest <daniel.nuest@uni-muenster.de>
Date: Thu, 16 Aug 2018 08:57:15 +0200
Subject: [PATCH 27/53] update
---
.gitmodules | 5 ++++-
Makefile | 14 ++++++++++----
README.md | 12 ++++++------
api | 1 +
architecture | 2 +-
docker-compose-local.yml | 2 +-
docker-compose.yml | 6 +++---
erc-checker | 2 +-
erc-examples | 2 +-
erc-spec | 2 +-
nginx.conf => etc/nginx.conf | 0
o2r-bindings | 1 +
o2r-bouncer | 2 +-
o2r-finder | 2 +-
o2r-guestlister | 2 +-
o2r-informer | 2 +-
o2r-loader | 2 +-
o2r-meta | 2 +-
o2r-muncher | 2 +-
o2r-platform | 2 +-
o2r-shipper | 2 +-
o2r-substituter | 2 +-
o2r-transporter | 2 +-
o2r-web-api | 1 -
24 files changed, 41 insertions(+), 31 deletions(-)
create mode 160000 api
rename nginx.conf => etc/nginx.conf (100%)
create mode 160000 o2r-bindings
delete mode 160000 o2r-web-api
diff --git a/.gitmodules b/.gitmodules
index 5a438ae..93335d4 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -23,7 +23,7 @@
path = o2r-platform
url = https://github.com/o2r-project/o2r-platform
[submodule "o2r-web-api"]
- path = o2r-web-api
+ path = api
url = https://github.com/o2r-project/o2r-web-api
[submodule "erc-spec"]
path = erc-spec
@@ -49,3 +49,6 @@
[submodule "o2r-guestlister"]
path = o2r-guestlister
url = https://github.com/o2r-project/o2r-guestlister
+[submodule "o2r-bindings"]
+ path = o2r-bindings
+ url = https://github.com/o2r-project/o2r-bindings
diff --git a/Makefile b/Makefile
index e925ca4..cae03b8 100644
--- a/Makefile
+++ b/Makefile
@@ -1,6 +1,7 @@
init:
git clone https://github.com/o2r-project/reference-implementation
cd reference-implementation
+ git submodule add https://github.com/o2r-project/api
git submodule add https://github.com/o2r-project/architecture
git submodule add https://github.com/o2r-project/erc-spec
git submodule add https://github.com/o2r-project/erc-checker
@@ -16,8 +17,8 @@ init:
git submodule add https://github.com/o2r-project/o2r-shipper
git submodule add https://github.com/o2r-project/o2r-substituter
git submodule add https://github.com/o2r-project/o2r-transporter
- git submodule add https://github.com/o2r-project/o2r-web-api
git submodule add https://github.com/o2r-project/o2r-guestlister
+ git submodule add https://github.com/o2r-project/o2r-bindings
update:
git pull --recurse-submodules
@@ -37,6 +38,7 @@ local_images:
cd o2r-substituter; docker build --tag o2r_refimpl_substituter .; cd ..;
cd o2r-transporter; docker build --tag o2r_refimpl_transporter .; cd ..;
cd o2r-guestlister; docker build --tag o2r_refimpl_guestlister .; cd ..;
+ cd o2r-bindings; docker build --tag o2r_refimpl_bindins .; cd ..;
local_up:
docker-compose --file docker-compose-local.yml up;
@@ -78,6 +80,7 @@ hub_images:
docker pull o2rproject/o2r-substituter;
docker pull o2rproject/o2r-transporter;
docker pull o2rproject/o2r-guestlister;
+ docker pull o2rproject/o2r-bindings;
hub_up:
docker-compose up;
@@ -95,6 +98,7 @@ hub_versions:
@docker inspect --format '{{index .Config.Labels "org.label-schema.name"}}: {{index .Config.Labels "org.label-schema.version"}}' o2rproject/o2r-substituter;
@docker inspect --format '{{index .Config.Labels "org.label-schema.name"}}: {{index .Config.Labels "org.label-schema.version"}}' o2rproject/o2r-transporter;
@docker inspect --format '{{index .Config.Labels "org.label-schema.name"}}: {{index .Config.Labels "org.label-schema.version"}}' o2rproject/o2r-guestlister;
+ @docker inspect --format '{{index .Config.Labels "org.label-schema.name"}}: {{index .Config.Labels "org.label-schema.version"}}' o2rproject/o2r-bindings;
clean: hub_down_volume
docker ps -a | grep o2r | awk '{print $1}' | xargs docker rm -f
@@ -104,11 +108,11 @@ build_documentation:
rm -f *.pdf
docker build --tag docbuilder --file etc/Dockerfile.documentations .
docker run -it -v $(CURDIR)/architecture:/doc:rw docbuilder make build pdf
- docker run -it -v $(CURDIR)/o2r-web-api:/doc:rw docbuilder make build pdf
- docker run -it -v $(CURDIR)/erc-spec:/doc:rw docbuilder make build pdf_tinytex
+ docker run -it -v $(CURDIR)/api-spec:/doc:rw docbuilder make build pdf
+ docker run -it -v $(CURDIR)/erc-spec:/doc:rw docbuilder make build pdf_tinytex
mv architecture/site/*.pdf .
mv erc-spec/*.pdf .
- mv o2r-web-api/*.pdf .
+ mv api-spec/*.pdf .
echo "ERC, architecture, and web API documentation created, see files PDF files in the project root directory"
release: update build_documentation
@@ -116,3 +120,5 @@ release: update build_documentation
reproduce:
# TODO import images from files
+ # run make local
+ # deploy some examples
diff --git a/README.md b/README.md
index 3052d3a..9cdc1c7 100644
--- a/README.md
+++ b/README.md
@@ -17,8 +17,8 @@ The documentation is also available online for reading, though availability may
| Component | directory | online |
| ------ | ------ | ------ |
-| ERC specification (document) | `erc-spec` | http://o2r.info/erc-spec/ |
-| Web API specification (document) | `o2r-web-api` | https://github.com/o2r-project/o2r-web-api |
+| ERC specification (document) | `erc-spec` | https://o2r.info/erc-spec/ |
+| Web API specification (document) | `api` | https://github.com/o2r-project/api |
| Architecture specification (document) | `architecture` | https://github.com/o2r-project/architecture |
| ERC checker (library/tool) | `erc-checker` | https://github.com/o2r-project/erc-checker/ |
| ERC and workspace examples (misc) | `erc-examples` | https://github.com/o2r-project/erc-examples |
@@ -54,7 +54,7 @@ If you are familiar with virtual machines (VMs) we strongly consider you run the
#### Software
The o2r reference implementation is build on [Docker](http://docker.com/).
-You must [install Docker](https://www.docker.com/get-docker) (i.e. Docker Community Edition, _edge_ channel, tested with version `17.09.0-ce` but later version should work) and [docker-compose](https://docs.docker.com/compose/) (version `1.13.0` or higher) first to continue.
+You must [install Docker](https://www.docker.com/get-docker) (i.e. Docker Community Edition, tested with version `17.09.0-ce` but later versions should work) and [docker-compose](https://docs.docker.com/compose/) (version `1.13.0` or higher) first to continue.
The tasks below are automated using [Make](https://en.wikipedia.org/wiki/Make_(software)).
Make sure that you have GNU Make (tested) or one of the derivatives (not tested) running on your system.
@@ -117,13 +117,13 @@ This repository already includes a `.gitmodules` file, which lists all required
To download all o2r source code at once, navigate to the `reference-implementation` base directory and use
```bash
- make update
+make update
```
Once all repositories have been pulled successfully, build Docker images of the microservices and run them in containers by executing:
```bash
- make local
+make local
```
Wait until the log shows no new messages, then open **http://localhost** and continue in section ["Use platform"](#use-platform).
@@ -135,7 +135,7 @@ All o2r software projects have automatic builds [available on Docker Hub](https:
The following command executes a `docker-compose` command to pull and run these images.
```bash
- make hub
+make hub
```
Wait until the log shows no new messages, then open **http://localhost** and continue in section ["Use platform"](#use-platform).
diff --git a/api b/api
new file mode 160000
index 0000000..b400ca7
--- /dev/null
+++ b/api
@@ -0,0 +1 @@
+Subproject commit b400ca7a73735c959e8fbd177021e4d0d5ec37bb
diff --git a/architecture b/architecture
index 90d5a33..eaf33f2 160000
--- a/architecture
+++ b/architecture
@@ -1 +1 @@
-Subproject commit 90d5a33ad2aa44a77446025aa0c2551d14a6789d
+Subproject commit eaf33f2aa4c5af7dbcdd3de95a9d4196f64da64b
diff --git a/docker-compose-local.yml b/docker-compose-local.yml
index a97ae3d..39c970f 100644
--- a/docker-compose-local.yml
+++ b/docker-compose-local.yml
@@ -214,7 +214,7 @@ services:
- shipper
- substituter
volumes:
- - "./nginx.conf:/etc/nginx/nginx.conf:ro"
+ - "./etc/nginx.conf:/etc/nginx/nginx.conf:ro"
ports:
- "80:80"
diff --git a/docker-compose.yml b/docker-compose.yml
index 6cf60a5..0254fce 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -106,7 +106,7 @@ services:
depends_on:
- configmongodb
environment:
- - "INFORMER_MONGODB=mongodb://mongodb"
+ - "INFORMER_MONGODB=mongodb://mongodb:27017"
- INFORMER_MONGODB_HOST=mongodb
- INFORMER_PORT=8082
- DEBUG=informer
@@ -190,7 +190,7 @@ services:
- DEBUGME=inspecter
guestlister:
- image: o2rproject/o2r-guestlister:0.2.0
+ image: o2rproject/o2r-guestlister:0.2.1
restart: unless-stopped
depends_on:
- configmongodb
@@ -221,7 +221,7 @@ services:
- substituter
- guestlister
volumes:
- - "./nginx.conf:/etc/nginx/nginx.conf:ro"
+ - "./etc/nginx.conf:/etc/nginx/nginx.conf:ro"
ports:
- "80:80"
diff --git a/erc-checker b/erc-checker
index 91bd93d..43fba4a 160000
--- a/erc-checker
+++ b/erc-checker
@@ -1 +1 @@
-Subproject commit 91bd93d3d9b366709c83e1b97c91b41645a9f1da
+Subproject commit 43fba4a9a237956642e72d654c3d84a1465fb546
diff --git a/erc-examples b/erc-examples
index c0a7f53..0b79d48 160000
--- a/erc-examples
+++ b/erc-examples
@@ -1 +1 @@
-Subproject commit c0a7f538d90745380fc1350d6162de96d651f70b
+Subproject commit 0b79d4858a2d0a45024b93cb140e217f144ad6a8
diff --git a/erc-spec b/erc-spec
index 7d5d3bc..c2081e4 160000
--- a/erc-spec
+++ b/erc-spec
@@ -1 +1 @@
-Subproject commit 7d5d3bc3130116d1050cae5ee486e31bae92b0ff
+Subproject commit c2081e45a5c8352aa6b98acbb0434f10f8e032ea
diff --git a/nginx.conf b/etc/nginx.conf
similarity index 100%
rename from nginx.conf
rename to etc/nginx.conf
diff --git a/o2r-bindings b/o2r-bindings
new file mode 160000
index 0000000..4e998c8
--- /dev/null
+++ b/o2r-bindings
@@ -0,0 +1 @@
+Subproject commit 4e998c8407fb21a8d2d7e67ad55bc7f17822971a
diff --git a/o2r-bouncer b/o2r-bouncer
index 851f45f..6bc703d 160000
--- a/o2r-bouncer
+++ b/o2r-bouncer
@@ -1 +1 @@
-Subproject commit 851f45f7e4f92d2cf01b968bdd2f6f2a9ea4aca3
+Subproject commit 6bc703decd3d5dfc49e7935a37da625dd0921c94
diff --git a/o2r-finder b/o2r-finder
index 2ffbbf0..97029b6 160000
--- a/o2r-finder
+++ b/o2r-finder
@@ -1 +1 @@
-Subproject commit 2ffbbf0d41fb20e8078c45bd21bb2a97b90fe1b1
+Subproject commit 97029b6322223e5e4677e03f41d2b1b375634f5c
diff --git a/o2r-guestlister b/o2r-guestlister
index dac99d2..fe35183 160000
--- a/o2r-guestlister
+++ b/o2r-guestlister
@@ -1 +1 @@
-Subproject commit dac99d24cd4b4c6b3bbf185fccf5a495458f101e
+Subproject commit fe351833fa05455b99b5644d26c944ae6500309f
diff --git a/o2r-informer b/o2r-informer
index 749a12d..cd507af 160000
--- a/o2r-informer
+++ b/o2r-informer
@@ -1 +1 @@
-Subproject commit 749a12d67cca0a4bed63657dffcbd4849b95747a
+Subproject commit cd507af98a12c2544148145c2cffe07853d909f7
diff --git a/o2r-loader b/o2r-loader
index adaf578..d83498b 160000
--- a/o2r-loader
+++ b/o2r-loader
@@ -1 +1 @@
-Subproject commit adaf57810a344aa66c868f6e8bac3cfc7ea1b007
+Subproject commit d83498b8d71be6999e8ff2e3a8c13c6574de11a9
diff --git a/o2r-meta b/o2r-meta
index 5c12559..9198450 160000
--- a/o2r-meta
+++ b/o2r-meta
@@ -1 +1 @@
-Subproject commit 5c12559803106db3ded76de661696a9414728f7d
+Subproject commit 9198450c57e4969bbce6c9e737e59b18401107f4
diff --git a/o2r-muncher b/o2r-muncher
index 0e2a5cf..1471014 160000
--- a/o2r-muncher
+++ b/o2r-muncher
@@ -1 +1 @@
-Subproject commit 0e2a5cfff68a6adeddcf43d2df42fde52849e6fc
+Subproject commit 14710140647d1e489aedecf7f0d6ca949efef818
diff --git a/o2r-platform b/o2r-platform
index 681b674..4ad78d6 160000
--- a/o2r-platform
+++ b/o2r-platform
@@ -1 +1 @@
-Subproject commit 681b67490d828ba635d2e4c2b869a3e963133282
+Subproject commit 4ad78d6ccb055ec6c6bad294ae71f6933d644c8e
diff --git a/o2r-shipper b/o2r-shipper
index e7398ab..9ee605f 160000
--- a/o2r-shipper
+++ b/o2r-shipper
@@ -1 +1 @@
-Subproject commit e7398ab2aa05e30f422396ef7ed12d0d5c161d3e
+Subproject commit 9ee605f2bf4e378014b89ffec3415a3b7a78ccff
diff --git a/o2r-substituter b/o2r-substituter
index e6a2e49..1d5be50 160000
--- a/o2r-substituter
+++ b/o2r-substituter
@@ -1 +1 @@
-Subproject commit e6a2e49847a3f2d528531d729a24958e03a32ef7
+Subproject commit 1d5be501c9c2c2a6821ab1caff0c14e4320952dd
diff --git a/o2r-transporter b/o2r-transporter
index 4c4babe..ed9f8be 160000
--- a/o2r-transporter
+++ b/o2r-transporter
@@ -1 +1 @@
-Subproject commit 4c4babecc73bda41db0984afcef5a5534d74a432
+Subproject commit ed9f8be91d7195c3fe14b565d1b51505f963c17c
diff --git a/o2r-web-api b/o2r-web-api
deleted file mode 160000
index c466789..0000000
--- a/o2r-web-api
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit c4667890745b84ba7b95f93466cbc617b828d790
From 1ef15d7c45493d17c8b33abcf5a5c44f61b4c37d Mon Sep 17 00:00:00 2001
From: nuest <daniel.nuest@uni-muenster.de>
Date: Fri, 17 Aug 2018 14:13:49 +0200
Subject: [PATCH 28/53] add containerit repo
---
.gitmodules | 3 +++
Makefile | 28 +++++++++++-----------------
containerit | 1 +
3 files changed, 15 insertions(+), 17 deletions(-)
create mode 160000 containerit
diff --git a/.gitmodules b/.gitmodules
index 93335d4..a8eddf4 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -52,3 +52,6 @@
[submodule "o2r-bindings"]
path = o2r-bindings
url = https://github.com/o2r-project/o2r-bindings
+[submodule "containerit"]
+ path = containerit
+ url = https://github.com/o2r-project/containerit
diff --git a/Makefile b/Makefile
index cae03b8..e999940 100644
--- a/Makefile
+++ b/Makefile
@@ -3,6 +3,7 @@ init:
cd reference-implementation
git submodule add https://github.com/o2r-project/api
git submodule add https://github.com/o2r-project/architecture
+ git submodule add https://github.com/o2r-project/containerit
git submodule add https://github.com/o2r-project/erc-spec
git submodule add https://github.com/o2r-project/erc-checker
git submodule add https://github.com/o2r-project/erc-examples
@@ -19,34 +20,23 @@ init:
git submodule add https://github.com/o2r-project/o2r-transporter
git submodule add https://github.com/o2r-project/o2r-guestlister
git submodule add https://github.com/o2r-project/o2r-bindings
+ git submodule add https://github.com/o2r-project/containerit
update:
git pull --recurse-submodules
git submodule update --init --recursive --remote
git submodule foreach git pull origin master
-local_images:
- cd o2r-bouncer; docker build --tag o2r_refimpl_bouncer .; cd ..;
- cd o2r-finder; docker build --tag o2r_refimpl_finder .; cd ..;
- cd o2r-informer; docker build --tag o2r_refimpl_informer .; cd ..;
- cd o2r-inspecter; docker build --tag o2r_refimpl_inspecter .; cd ..;
- cd o2r-loader; docker build --tag o2r_refimpl_loader .; cd ..;
- cd o2r-meta; docker build --tag o2r_refimpl_meta .; cd ..;
- cd o2r-muncher; docker build --tag o2r_refimpl_muncher .; cd ..;
- cd o2r-platform; docker build --tag o2r_refimpl_platform .; cd ..;
- cd o2r-shipper; docker build --tag o2r_refimpl_shipper .; cd ..;
- cd o2r-substituter; docker build --tag o2r_refimpl_substituter .; cd ..;
- cd o2r-transporter; docker build --tag o2r_refimpl_transporter .; cd ..;
- cd o2r-guestlister; docker build --tag o2r_refimpl_guestlister .; cd ..;
- cd o2r-bindings; docker build --tag o2r_refimpl_bindins .; cd ..;
-
local_up:
docker-compose --file docker-compose-local.yml up;
local_down:
docker-compose --file docker-compose-local.yml down;
-local: local_images local_versions local_up
+local_down_volume:
+ docker-compose --file docker-compose-local.yml down --volume;
+
+local: local_up
local_versions:
@docker inspect --format '{{index .Config.Labels "org.label-schema.name"}}: {{index .Config.Labels "org.label-schema.version"}}' o2r_refimpl_bouncer ;
@@ -61,6 +51,8 @@ local_versions:
@docker inspect --format '{{index .Config.Labels "org.label-schema.name"}}: {{index .Config.Labels "org.label-schema.version"}}' o2r_refimpl_substituter ;
@docker inspect --format '{{index .Config.Labels "org.label-schema.name"}}: {{index .Config.Labels "org.label-schema.version"}}' o2r_refimpl_transporter ;
@docker inspect --format '{{index .Config.Labels "org.label-schema.name"}}: {{index .Config.Labels "org.label-schema.version"}}' o2r_refimpl_guestlister ;
+ @docker inspect --format '{{index .Config.Labels "org.label-schema.name"}}: {{index .Config.Labels "org.label-schema.version"}}' o2r_refimpl_bindings ;
+ @docker inspect --format '{{index .Config.Labels "org.label-schema.name"}}: {{index .Config.Labels "org.label-schema.version"}}' o2r_refimpl_containerit ;
hub: hub_images hub_versions hub_up
@@ -68,6 +60,7 @@ hub_down_volume:
docker-compose down --volume;
hub_images:
+ # pull ":latest" images so that we don't need to update versions here as well
docker pull o2rproject/o2r-bouncer;
docker pull o2rproject/o2r-finder;
docker pull o2rproject/o2r-informer;
@@ -99,8 +92,9 @@ hub_versions:
@docker inspect --format '{{index .Config.Labels "org.label-schema.name"}}: {{index .Config.Labels "org.label-schema.version"}}' o2rproject/o2r-transporter;
@docker inspect --format '{{index .Config.Labels "org.label-schema.name"}}: {{index .Config.Labels "org.label-schema.version"}}' o2rproject/o2r-guestlister;
@docker inspect --format '{{index .Config.Labels "org.label-schema.name"}}: {{index .Config.Labels "org.label-schema.version"}}' o2rproject/o2r-bindings;
+ @docker inspect --format '{{index .Config.Labels "org.label-schema.name"}}: {{index .Config.Labels "org.label-schema.version"}}' o2rproject/containerit;
-clean: hub_down_volume
+clean: hub_down_volume local_down_volume
docker ps -a | grep o2r | awk '{print $1}' | xargs docker rm -f
docker images | grep o2r | awk '{print $3}' | xargs docker rmi --force
diff --git a/containerit b/containerit
new file mode 160000
index 0000000..51d7b26
--- /dev/null
+++ b/containerit
@@ -0,0 +1 @@
+Subproject commit 51d7b2679623492dc4e09a4baeb31affdf96b686
From ceb4e68804bf7b895f16231aa1fa45f6e2bb7a65 Mon Sep 17 00:00:00 2001
From: nuest <daniel.nuest@uni-muenster.de>
Date: Fri, 17 Aug 2018 14:14:26 +0200
Subject: [PATCH 29/53] update service versions and build meta + containerit
locally as well
also add o2r-bindings to compose files
---
docker-compose-local.yml | 88 +++++++++++++++++++++++++++++-----------
docker-compose.yml | 76 +++++++++++++++++-----------------
etc/nginx.conf | 6 +++
3 files changed, 108 insertions(+), 62 deletions(-)
diff --git a/docker-compose-local.yml b/docker-compose-local.yml
index 39c970f..9734c7c 100644
--- a/docker-compose-local.yml
+++ b/docker-compose-local.yml
@@ -21,11 +21,11 @@ volumes:
services:
mongodb:
- image: mongo:3.4
+ image: mongo:3.6
command: "--replSet rso2r"
configmongodb:
- image: mongo:3.4
+ image: mongo:3.6
depends_on:
- mongodb
command: bash -c "sleep 5; mongo --verbose --host mongodb --eval 'printjson(rs.initiate()); printjson(rs.conf()); printjson(rs.status()); printjson(rs.slaveOk());'"
@@ -41,7 +41,7 @@ services:
- "1234:1234"
elasticsearch:
- image: docker.elastic.co/elasticsearch/elasticsearch:5.6.5
+ image: docker.elastic.co/elasticsearch/elasticsearch:5.6.10
environment:
- cluster.name=o2rplatform-es-cluster
- bootstrap.memory_lock=true
@@ -53,7 +53,7 @@ services:
hard: -1
mem_limit: 1g
elasticsearch2:
- image: docker.elastic.co/elasticsearch/elasticsearch:5.6.5
+ image: docker.elastic.co/elasticsearch/elasticsearch:5.6.10
environment:
- cluster.name=o2rplatform-es-cluster
- bootstrap.memory_lock=true
@@ -66,68 +66,90 @@ services:
hard: -1
mem_limit: 1g
+ meta:
+ image: o2r_refimpl_meta
+ build:
+ context: o2r-meta
+
+ containerit:
+ image: o2r_refimpl_containerit
+ build:
+ context: containerit
+ dockerfile: inst/docker/geospatial/Dockerfile.local
+
muncher:
image: o2r_refimpl_muncher
- restart: unless-stopped
+ build:
+ context: ./o2r-muncher
depends_on:
- configmongodb
volumes:
- o2rvol:/tmp/o2r
- /var/run/docker.sock:/var/run/docker.sock
environment:
- - "MUNCHER_MONGODB=mongodb://mongodb"
+ - "MUNCHER_MONGODB=mongodb://mongodb:27017/"
- MUNCHER_PORT=8080
+ - MUNCHER_VOLUME=referenceimplementation_o2rvol
+ - MUNCHER_CONTAINER_USER=root
+ - MUNCHER_CONTAINERIT_USER=root
+ - MUNCHER_CONTAINERIT_IMAGE=o2r_refimpl_containerit
- DEBUG=*,-mquery,-express:*,-express-session,-body-parser:*
- "MUNCHER_META_TOOL_CONTAINER=o2r_refimpl_meta"
loader:
image: o2r_refimpl_loader
- restart: unless-stopped
+ build:
+ context: ./o2r-loader
depends_on:
- configmongodb
volumes:
- o2rvol:/tmp/o2r
- /var/run/docker.sock:/var/run/docker.sock
environment:
- - "LOADER_MONGODB=mongodb://mongodb"
+ - "LOADER_MONGODB=mongodb://mongodb:27017/"
- LOADER_PORT=8088
- DEBUG=*,-mquery,-express:*,-express-session,-body-parser:*
- - "MUNCHER_META_TOOL_CONTAINER=o2r_refimpl_meta"
+ - LOADER_VOLUME=referenceimplementation_o2rvol
+ - "LOADER_META_TOOL_CONTAINER=o2r_refimpl_meta"
informer:
image: o2r_refimpl_informer
- restart: unless-stopped
+ build:
+ context: ./o2r-informer
depends_on:
- configmongodb
environment:
- - "INFORMER_MONGODB=mongodb://mongodb"
+ - "INFORMER_MONGODB=mongodb://mongodb:27017/"
- INFORMER_MONGODB_HOST=mongodb
- INFORMER_PORT=8082
- DEBUG=informer
bouncer:
image: o2r_refimpl_bouncer
- restart: unless-stopped
+ build:
+ context: ./o2r-bouncer
depends_on:
- configmongodb
environment:
- - "BOUNCER_MONGODB=mongodb://mongodb"
+ - "BOUNCER_MONGODB=mongodb://mongodb:27017/"
- BOUNCER_PORT=8083
- DEBUG=bouncer
- OAUTH_URL_CALLBACK=${OAUTH_URL_CALLBACK}
- OAUTH_URL_AUTHORIZATION=${OAUTH_URL_AUTHORIZATION}
- OAUTH_URL_TOKEN=${OAUTH_URL_TOKEN}
+ - OAUTH_STARTUP_FAIL_ON_ERROR=false
finder:
image: o2r_refimpl_finder
- restart: unless-stopped
+ build:
+ context: ./o2r-finder
depends_on:
- configmongodb
- elasticsearch
volumes:
- o2rvol:/tmp/o2r
environment:
- - "FINDER_MONGODB=mongodb://mongodb"
+ - "FINDER_MONGODB=mongodb://mongodb:27017/"
- FINDER_PORT=8084
- DEBUG=finder,finder:*
- ELASTIC_SEARCH_URL=elasticsearch:9200
@@ -137,20 +159,22 @@ services:
transporter:
image: o2r_refimpl_transporter
- restart: unless-stopped
+ build:
+ context: ./o2r-transporter
depends_on:
- configmongodb
volumes:
- o2rvol:/tmp/o2r
- "/var/run/docker.sock:/var/run/docker.sock"
environment:
- - "TRANSPORTER_MONGODB=mongodb://mongodb"
+ - "TRANSPORTER_MONGODB=mongodb://mongodb:27017/"
- TRANSPORTER_PORT=8081
- DEBUG=transporter,transporter:*
shipper:
image: o2r_refimpl_shipper
- restart: unless-stopped
+ build:
+ context: ./o2r-shipper
depends_on:
- configmongodb
volumes:
@@ -164,32 +188,46 @@ services:
substituter:
image: o2r_refimpl_substituter
- restart: unless-stopped
+ build:
+ context: ./o2r-substituter
depends_on:
- configmongodb
volumes:
- o2rvol:/tmp/o2r
environment:
- - "SUBSTITUTER_MONGODB=mongodb://mongodb"
+ - "SUBSTITUTER_MONGODB=mongodb://mongodb:27017/"
- SUBSTITUTER_PORT=8090
- DEBUG=substituter,substituter:*
inspecter:
- image: o2rproject/o2r-inspecter:0.0.1
- restart: unless-stopped
+ image: o2r_refimpl_inspecter
+ build:
+ context: ./o2r-inspecter
volumes:
- o2rvol:/tmp/o2r
environment:
- INSPECTER_PORT=8091
- DEBUGME=inspecter
+ bindings:
+ image: o2r_refimple_bindings
+ build:
+ context: ./o2r-bindings
+ ports:
+ - "5000-5499:5000-5499"
+ volumes:
+ - o2rvol:/tmp/o2r
+ environment:
+ - DEBUG=bindings
+
guestlister:
image: o2r_refimpl_guestlister
- restart: unless-stopped
+ build:
+ context: ./o2r-guestlister
depends_on:
- configmongodb
environment:
- - "GUESTLISTER_MONGODB=mongodb://mongodb"
+ - "GUESTLISTER_MONGODB=mongodb://mongodb:27017/"
- GUESTLISTER_PORT=8383
- DEBUG=guestlister
- OAUTH_URL_CALLBACK=${OAUTH_URL_CALLBACK}
@@ -198,6 +236,8 @@ services:
platform:
image: o2r_refimpl_platform
+ build:
+ context: ./o2r-platform
restart: unless-stopped
webserver:
diff --git a/docker-compose.yml b/docker-compose.yml
index 0254fce..cc43b72 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -21,11 +21,11 @@ volumes:
services:
mongodb:
- image: mongo:3.4
+ image: mongo:3.6
command: "--replSet rso2r"
configmongodb:
- image: mongo:3.4
+ image: mongo:3.6
depends_on:
- mongodb
command: bash -c "sleep 5; mongo --verbose --host mongodb --eval 'printjson(rs.initiate()); printjson(rs.conf()); printjson(rs.status()); printjson(rs.slaveOk());'"
@@ -41,7 +41,7 @@ services:
- "1234:1234"
elasticsearch:
- image: docker.elastic.co/elasticsearch/elasticsearch:5.6.5
+ image: docker.elastic.co/elasticsearch/elasticsearch:5.6.10
environment:
- cluster.name=o2rplatform-es-cluster
- bootstrap.memory_lock=true
@@ -55,7 +55,7 @@ services:
# ports:
# - "9200:9200"
elasticsearch2:
- image: docker.elastic.co/elasticsearch/elasticsearch:5.6.5
+ image: docker.elastic.co/elasticsearch/elasticsearch:5.6.10
environment:
- cluster.name=o2rplatform-es-cluster
- bootstrap.memory_lock=true
@@ -69,71 +69,68 @@ services:
mem_limit: 1g
muncher:
- image: o2rproject/o2r-muncher:0.17.0
- restart: unless-stopped
+ image: o2rproject/o2r-muncher:0.21.0
depends_on:
- configmongodb
volumes:
- o2rvol:/tmp/o2r
- /var/run/docker.sock:/var/run/docker.sock
environment:
- - "MUNCHER_MONGODB=mongodb://mongodb"
+ - "MUNCHER_MONGODB=mongodb://mongodb:27017/"
- MUNCHER_PORT=8080
- MUNCHER_VOLUME=referenceimplementation_o2rvol
- MUNCHER_CONTAINER_USER=root
- MUNCHER_CONTAINERIT_USER=root
- DEBUG=*,-mquery,-express:*,-express-session,-body-parser:*
- - "MUNCHER_META_TOOL_CONTAINER=o2rproject/o2r-meta:5c12559"
+ - "MUNCHER_META_TOOL_CONTAINER=o2rproject/o2r-meta:9198450"
loader:
- image: o2rproject/o2r-loader:0.10.0
- restart: unless-stopped
+ image: o2rproject/o2r-loader:0.11.0
depends_on:
- configmongodb
volumes:
- o2rvol:/tmp/o2r
- /var/run/docker.sock:/var/run/docker.sock
environment:
- - "LOADER_MONGODB=mongodb://mongodb"
+ - "LOADER_MONGODB=mongodb://mongodb:27017/"
- LOADER_PORT=8088
- DEBUG=*,-mquery,-express:*,-express-session,-body-parser:*
- LOADER_VOLUME=referenceimplementation_o2rvol
- - "LOADER_META_TOOL_CONTAINER=o2rproject/o2r-meta:5c12559"
+ - "LOADER_META_TOOL_CONTAINER=o2rproject/o2r-meta:9198450"
informer:
- image: o2rproject/o2r-informer:0.3.0
- restart: unless-stopped
+ image: o2rproject/o2r-informer:0.5.0
depends_on:
- configmongodb
environment:
- - "INFORMER_MONGODB=mongodb://mongodb:27017"
+ - "INFORMER_MONGODB=mongodb://mongodb:27017/"
- INFORMER_MONGODB_HOST=mongodb
- INFORMER_PORT=8082
- - DEBUG=informer
+ - DEBUG=informer,informer:*
bouncer:
- image: o2rproject/o2r-bouncer:0.11.0
- restart: unless-stopped
+ image: o2rproject/o2r-bouncer:0.12.1
depends_on:
- configmongodb
+ - guestlister
environment:
- - "BOUNCER_MONGODB=mongodb://mongodb"
+ - "BOUNCER_MONGODB=mongodb://mongodb:27017/"
- BOUNCER_PORT=8083
- - DEBUG=bouncer
+ - DEBUG=bouncer,bouncer:*
- OAUTH_URL_CALLBACK=${OAUTH_URL_CALLBACK}
- OAUTH_URL_AUTHORIZATION=${OAUTH_URL_AUTHORIZATION}
- OAUTH_URL_TOKEN=${OAUTH_URL_TOKEN}
+ - OAUTH_STARTUP_FAIL_ON_ERROR=false
finder:
- image: o2rproject/o2r-finder:0.6.1
- restart: unless-stopped
+ image: o2rproject/o2r-finder:0.6.2
depends_on:
- configmongodb
- elasticsearch
volumes:
- o2rvol:/tmp/o2r
environment:
- - "FINDER_MONGODB=mongodb://mongodb"
+ - "FINDER_MONGODB=mongodb://mongodb:27017/"
- FINDER_PORT=8084
- DEBUG=finder,finder:*
- ELASTIC_SEARCH_URL=elasticsearch:9200
@@ -142,21 +139,19 @@ services:
- BATCH_COUNT=5
transporter:
- image: o2rproject/o2r-transporter:0.5.0
- restart: unless-stopped
+ image: o2rproject/o2r-transporter:0.6.0
depends_on:
- configmongodb
volumes:
- o2rvol:/tmp/o2r
- "/var/run/docker.sock:/var/run/docker.sock"
environment:
- - "TRANSPORTER_MONGODB=mongodb://mongodb"
+ - "TRANSPORTER_MONGODB=mongodb://mongodb:27017/"
- TRANSPORTER_PORT=8081
- DEBUG=transporter,transporter:*
shipper:
- image: o2rproject/o2r-shipper:e7398ab
- restart: unless-stopped
+ image: o2rproject/o2r-shipper:8f95fd6
depends_on:
- configmongodb
volumes:
@@ -169,46 +164,50 @@ services:
PYTHONUNBUFFERED: "0"
substituter:
- image: o2rproject/o2r-substituter:0.4.3
- restart: unless-stopped
+ image: o2rproject/o2r-substituter:0.5.0
depends_on:
- configmongodb
volumes:
- o2rvol:/tmp/o2r
environment:
- - "SUBSTITUTER_MONGODB=mongodb://mongodb"
+ - "SUBSTITUTER_MONGODB=mongodb://mongodb:27017/"
- SUBSTITUTER_PORT=8090
- DEBUG=substituter,substituter:*
inspecter:
image: o2rproject/o2r-inspecter:0.0.1
- restart: unless-stopped
volumes:
- o2rvol:/tmp/o2r
environment:
- INSPECTER_PORT=8091
- DEBUGME=inspecter
+ bindings:
+ image: o2rproject/o2r-bindings:0.0.1
+ ports:
+ - "5000-5499:5000-5499"
+ volumes:
+ - o2rvol:/tmp/o2r
+ environment:
+ - DEBUG=bindings
+
guestlister:
image: o2rproject/o2r-guestlister:0.2.1
- restart: unless-stopped
depends_on:
- configmongodb
environment:
- - "GUESTLISTER_MONGODB=mongodb://mongodb"
+ - "GUESTLISTER_MONGODB=mongodb://mongodb:27017/"
- GUESTLISTER_PORT=8383
- - DEBUG=guestlister
+ - DEBUG=guestlister,guestlister:*
- OAUTH_URL_CALLBACK=${OAUTH_URL_CALLBACK}
- OAUTH_URL_AUTHORIZATION=${OAUTH_URL_AUTHORIZATION}
- OAUTH_URL_TOKEN=${OAUTH_URL_TOKEN}
platform:
- image: o2rproject/o2r-platform:1.0.0
- restart: unless-stopped
+ image: o2rproject/o2r-platform:1.0.2
webserver:
image: nginx:latest
- restart: unless-stopped
depends_on:
- bouncer
- finder
@@ -220,6 +219,7 @@ services:
- shipper
- substituter
- guestlister
+ - bindings
volumes:
- "./etc/nginx.conf:/etc/nginx/nginx.conf:ro"
ports:
diff --git a/etc/nginx.conf b/etc/nginx.conf
index 8db841b..c2e34be 100644
--- a/etc/nginx.conf
+++ b/etc/nginx.conf
@@ -114,6 +114,12 @@ http {
proxy_pass http://inspecter:8091;
}
+ location /api/v1/bindings {
+ proxy_set_header Host $host;
+ proxy_set_header X-Real-IP $remote_addr;
+ proxy_pass http://bindings:8092;
+ }
+
location /oauth {
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
From ae10dc673d4b1236de5eff5d22cc987bc333b8ac Mon Sep 17 00:00:00 2001
From: nuest <daniel.nuest@uni-muenster.de>
Date: Fri, 17 Aug 2018 14:24:40 +0200
Subject: [PATCH 30/53] update submodules
---
containerit | 2 +-
erc-spec | 2 +-
o2r-bindings | 2 +-
o2r-shipper | 2 +-
4 files changed, 4 insertions(+), 4 deletions(-)
diff --git a/containerit b/containerit
index 51d7b26..6ba7d1d 160000
--- a/containerit
+++ b/containerit
@@ -1 +1 @@
-Subproject commit 51d7b2679623492dc4e09a4baeb31affdf96b686
+Subproject commit 6ba7d1de6e75aeec935bd742ba40add835bd4ffc
diff --git a/erc-spec b/erc-spec
index c2081e4..64b70f6 160000
--- a/erc-spec
+++ b/erc-spec
@@ -1 +1 @@
-Subproject commit c2081e45a5c8352aa6b98acbb0434f10f8e032ea
+Subproject commit 64b70f69f97b0d2783714e9509166ff355f226fd
diff --git a/o2r-bindings b/o2r-bindings
index 4e998c8..aa77f5a 160000
--- a/o2r-bindings
+++ b/o2r-bindings
@@ -1 +1 @@
-Subproject commit 4e998c8407fb21a8d2d7e67ad55bc7f17822971a
+Subproject commit aa77f5afe189b78398324ea699daa74010b313ce
diff --git a/o2r-shipper b/o2r-shipper
index 9ee605f..8f95fd6 160000
--- a/o2r-shipper
+++ b/o2r-shipper
@@ -1 +1 @@
-Subproject commit 9ee605f2bf4e378014b89ffec3415a3b7a78ccff
+Subproject commit 8f95fd6cc2c2071658ddcefa3457390207ff5ffc
From ed49ef49094f75be4718ea2a154840d221eb9b22 Mon Sep 17 00:00:00 2001
From: nuest <daniel.nuest@uni-muenster.de>
Date: Wed, 12 Dec 2018 10:30:42 +0100
Subject: [PATCH 31/53] fix documentation building environment
see https://github.com/pypa/pip/issues/5599
---
etc/Dockerfile.documentations | 9 +++++----
1 file changed, 5 insertions(+), 4 deletions(-)
diff --git a/etc/Dockerfile.documentations b/etc/Dockerfile.documentations
index 595b21c..c5eca48 100644
--- a/etc/Dockerfile.documentations
+++ b/etc/Dockerfile.documentations
@@ -2,7 +2,7 @@
# A Dockerfile for an environment to build the documentation contained in this repository
#
-FROM rocker/verse
+FROM rocker/verse:3.4.4
# System dependencies
RUN apt-get update && apt-get install -y --no-install-recommends \
@@ -20,10 +20,11 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
RUN wget https://github.com/jgm/pandoc/releases/download/2.1.1/pandoc-2.1.1-linux.tar.gz \
&& tar -xvzf pandoc*.tar.gz --strip-components 1 -C /usr/local/
-RUN tlmgr init-usertree \
+RUN tlmgr update --self \
+ && tlmgr init-usertree \
&& tlmgr install xcolor mdframed needspace
RUN pip3 install --upgrade pip \
- && pip3 install \
+ && python3 -m pip install \
mkdocs \
pymdown-extensions \
mkdocs-pandoc \
@@ -31,7 +32,7 @@ RUN pip3 install --upgrade pip \
git+https://github.com/o2r-project/mkdocs-combine.git
# architecture, see architecture/.travis.yml
-RUN pip3 install mkdocs mkdocs-cinder markdown-include
+RUN python3 -m pip install mkdocs mkdocs-cinder markdown-include
RUN apt-get update && apt-get install -y --no-install-recommends \
xfonts-base xfonts-75dpi
RUN wget https://github.com/wkhtmltopdf/wkhtmltopdf/releases/download/0.12.4/wkhtmltox-0.12.4_linux-generic-amd64.tar.xz \
From 6746158a6ec23b65002a15a6787a1e2cbbd4ea4a Mon Sep 17 00:00:00 2001
From: nuest <daniel.nuest@uni-muenster.de>
Date: Wed, 12 Dec 2018 10:43:08 +0100
Subject: [PATCH 32/53] add local_build target
---
Makefile | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
diff --git a/Makefile b/Makefile
index e999940..50db917 100644
--- a/Makefile
+++ b/Makefile
@@ -27,6 +27,9 @@ update:
git submodule update --init --recursive --remote
git submodule foreach git pull origin master
+local_build:
+ docker-compose --file docker-compose-local.yml build;
+
local_up:
docker-compose --file docker-compose-local.yml up;
@@ -36,7 +39,7 @@ local_down:
local_down_volume:
docker-compose --file docker-compose-local.yml down --volume;
-local: local_up
+local: update local_build local_up
local_versions:
@docker inspect --format '{{index .Config.Labels "org.label-schema.name"}}: {{index .Config.Labels "org.label-schema.version"}}' o2r_refimpl_bouncer ;
From e7f6c782231cb968be6dc34b2bac8837a23c960e Mon Sep 17 00:00:00 2001
From: nuest <daniel.nuest@uni-muenster.de>
Date: Wed, 12 Dec 2018 10:45:05 +0100
Subject: [PATCH 33/53] add targets for saving image and files
---
.gitignore | 6 ++++++
Makefile | 29 +++++++++++++++++++++++++++--
2 files changed, 33 insertions(+), 2 deletions(-)
diff --git a/.gitignore b/.gitignore
index a136337..f27dafb 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1 +1,7 @@
*.pdf
+
+o2r-reference-implementation-modules\.zip
+
+o2r-reference-implementation-images\.tar\.gz
+
+o2r-docs\.zip
diff --git a/Makefile b/Makefile
index 50db917..b59dd1c 100644
--- a/Makefile
+++ b/Makefile
@@ -112,8 +112,33 @@ build_documentation:
mv api-spec/*.pdf .
echo "ERC, architecture, and web API documentation created, see files PDF files in the project root directory"
-release: update build_documentation
- # TODO build all images, export them to files
+local_save_images:
+ docker save \
+ mongo:3.6 \
+ adicom/admin-mongo:latest \
+ docker.elastic.co/elasticsearch/elasticsearch:5.6.10 \
+ o2r_refimpl_containerit \
+ o2r_refimpl_muncher \
+ o2r_refimpl_loader \
+ o2r_refimpl_informer \
+ o2r_refimpl_bouncer \
+ o2r_refimpl_finder \
+ o2r_refimpl_transporter \
+ o2r_refimpl_shipper \
+ o2r_refimpl_substituter \
+ o2r_refimpl_inspecter \
+ o2r_refimpl_bindings \
+ o2r_refimpl_guestlister \
+ o2r_refimpl_platform \
+ nginx:latest | gzip -c > o2r-reference-implementation-images.tar.gz;
+ tar -tvf o2r-reference-implementation-images.tar.gz;
+ ls -sh o2r-reference-implementation-images.tar.gz;
+ @echo "Inspecting tarball manifest:";
+ tar -xf o2r-reference-implementation-images.tar.gz manifest.json -O | python -m json.tool;
+
+create_archive:
+ zip -r -q o2r-reference-implementation-modules.zip software;
+ zip -r -q o2r-docs.zip api/ architecture/ erc-examples/ erc-spec/;
reproduce:
# TODO import images from files
From 9afd6e0a06e0e03d72f05c69c198ec3066e35f22 Mon Sep 17 00:00:00 2001
From: nuest <daniel.nuest@uni-muenster.de>
Date: Wed, 12 Dec 2018 10:45:58 +0100
Subject: [PATCH 34/53] update version logging
---
Makefile | 27 ++++++++-------------------
etc/local_versions.sh | 25 +++++++++++++++++++++++++
2 files changed, 33 insertions(+), 19 deletions(-)
create mode 100755 etc/local_versions.sh
diff --git a/Makefile b/Makefile
index b59dd1c..26f1907 100644
--- a/Makefile
+++ b/Makefile
@@ -42,25 +42,10 @@ local_down_volume:
local: update local_build local_up
local_versions:
- @docker inspect --format '{{index .Config.Labels "org.label-schema.name"}}: {{index .Config.Labels "org.label-schema.version"}}' o2r_refimpl_bouncer ;
- @docker inspect --format '{{index .Config.Labels "org.label-schema.name"}}: {{index .Config.Labels "org.label-schema.version"}}' o2r_refimpl_finder ;
- @docker inspect --format '{{index .Config.Labels "org.label-schema.name"}}: {{index .Config.Labels "org.label-schema.version"}}' o2r_refimpl_informer ;
- @docker inspect --format '{{index .Config.Labels "org.label-schema.name"}}: {{index .Config.Labels "org.label-schema.version"}}' o2r_refimpl_inspecter ;
- @docker inspect --format '{{index .Config.Labels "org.label-schema.name"}}: {{index .Config.Labels "org.label-schema.version"}}' o2r_refimpl_loader ;
- @docker inspect --format '{{index .Config.Labels "org.label-schema.name"}}: {{index .Config.Labels "org.label-schema.version"}}' o2r_refimpl_meta ;
- @docker inspect --format '{{index .Config.Labels "org.label-schema.name"}}: {{index .Config.Labels "org.label-schema.version"}}' o2r_refimpl_muncher ;
- @docker inspect --format '{{index .Config.Labels "org.label-schema.name"}}: {{index .Config.Labels "org.label-schema.version"}}' o2r_refimpl_platform ;
- @docker inspect --format '{{index .Config.Labels "org.label-schema.name"}}: {{index .Config.Labels "org.label-schema.vcs-ref"}}' o2r_refimpl_shipper ;
- @docker inspect --format '{{index .Config.Labels "org.label-schema.name"}}: {{index .Config.Labels "org.label-schema.version"}}' o2r_refimpl_substituter ;
- @docker inspect --format '{{index .Config.Labels "org.label-schema.name"}}: {{index .Config.Labels "org.label-schema.version"}}' o2r_refimpl_transporter ;
- @docker inspect --format '{{index .Config.Labels "org.label-schema.name"}}: {{index .Config.Labels "org.label-schema.version"}}' o2r_refimpl_guestlister ;
- @docker inspect --format '{{index .Config.Labels "org.label-schema.name"}}: {{index .Config.Labels "org.label-schema.version"}}' o2r_refimpl_bindings ;
- @docker inspect --format '{{index .Config.Labels "org.label-schema.name"}}: {{index .Config.Labels "org.label-schema.version"}}' o2r_refimpl_containerit ;
-
-hub: hub_images hub_versions hub_up
-
-hub_down_volume:
- docker-compose down --volume;
+ etc/local_versions.sh
+
+local_versions_save: local_versions
+ make local_versions >> versions.txt
hub_images:
# pull ":latest" images so that we don't need to update versions here as well
@@ -140,6 +125,10 @@ create_archive:
zip -r -q o2r-reference-implementation-modules.zip software;
zip -r -q o2r-docs.zip api/ architecture/ erc-examples/ erc-spec/;
+versions:
+ git --version;
+ docker --version;
+
reproduce:
# TODO import images from files
# run make local
diff --git a/etc/local_versions.sh b/etc/local_versions.sh
new file mode 100755
index 0000000..1241463
--- /dev/null
+++ b/etc/local_versions.sh
@@ -0,0 +1,25 @@
+#!/bin/bash
+
+function versioninfo {
+ cd $1
+ if [ -e bower.json ]
+ then
+ echo $1 $(cat README.md | grep -i --after-context 2 "platform version" | tail -1 ) $(git rev-parse HEAD)
+ elif [ -e package.json ]
+ then
+ echo $1 $(sed -nE 's/^\s*"version": "(.*?)",$/\1/p' package.json) $(git rev-parse HEAD)
+ elif [ -e DESCRIPTION ]
+ then
+ echo $1 $(sed -n '/Version:/p' DESCRIPTION | sed 's/[^0-9,.]*//g') $(git rev-parse HEAD)
+ else
+ echo $1 $(git rev-parse HEAD)
+ fi
+ cd ..
+}
+
+for dir in $(ls -1d */ | grep -v 'etc')
+do
+ versioninfo ${dir}
+done
+
+echo ". (reference-implementation)" $(git rev-parse HEAD)
From 6dfd3f8ce930c373f1b1b291ed086e1b009f1d94 Mon Sep 17 00:00:00 2001
From: nuest <daniel.nuest@uni-muenster.de>
Date: Wed, 12 Dec 2018 10:46:26 +0100
Subject: [PATCH 35/53] checkout master in submodules before pulling
---
Makefile | 3 ++-
versions.txt | 23 +++++++++++++++++++++++
2 files changed, 25 insertions(+), 1 deletion(-)
create mode 100644 versions.txt
diff --git a/Makefile b/Makefile
index 26f1907..1129ba8 100644
--- a/Makefile
+++ b/Makefile
@@ -25,7 +25,8 @@ init:
update:
git pull --recurse-submodules
git submodule update --init --recursive --remote
- git submodule foreach git pull origin master
+ git submodule foreach --recursive git checkout master
+ git submodule foreach --recursive git pull origin master
local_build:
docker-compose --file docker-compose-local.yml build;
diff --git a/versions.txt b/versions.txt
new file mode 100644
index 0000000..bb4fb85
--- /dev/null
+++ b/versions.txt
@@ -0,0 +1,23 @@
+make[1]: Entering directory '/home/daniel/git/o2r/reference-implementation'
+etc/local_versions.sh
+api/ e9bb2b86ecd8fa23dde9a109284dd3506d243184
+architecture/ eaf33f2aa4c5af7dbcdd3de95a9d4196f64da64b
+containerit/ 0.5.0.9003 fb107e2979c48b69336f7019f991f9e6f9900757
+erc-checker/ 1.2.0 fb1ed1617464b862404dc18c3641678543bca470
+erc-examples/ d5892550b2d68e858f006e4867de4fae5cc76a11
+erc-spec/ a112fd85b26a4659997ac2463ec0aff7315ebd22
+o2r-bindings/ 0.0.1 f68e17d12dc7573bd7c6b927d444ef35b09f14fc
+o2r-bouncer/ 0.12.2 6bc703decd3d5dfc49e7935a37da625dd0921c94
+o2r-finder/ 0.6.3 97029b6322223e5e4677e03f41d2b1b375634f5c
+o2r-guestlister/ 0.2.2 fe351833fa05455b99b5644d26c944ae6500309f
+o2r-informer/ 0.5.0 cd507af98a12c2544148145c2cffe07853d909f7
+o2r-inspecter/ 0.0.1 ee5e86ada517972b246f8297ddb8b43580e3eac1
+o2r-loader/ 0.11.0 d83498b8d71be6999e8ff2e3a8c13c6574de11a9
+o2r-meta/ f79cc1f838ce8a6151cd10967e497b4c9040e79d
+o2r-muncher/ 0.21.1 c68b4d369a52082ef764b0d576c7964258e0fd63
+o2r-platform/ 1.0.2 f7075b9be074dee841ab9cced667af6c66eadc3e
+o2r-shipper/ 8f95fd6cc2c2071658ddcefa3457390207ff5ffc
+o2r-substituter/ 0.5.0 1d5be501c9c2c2a6821ab1caff0c14e4320952dd
+o2r-transporter/ 0.6.0 ed9f8be91d7195c3fe14b565d1b51505f963c17c
+. (reference-implementation) 71c5a8184bf0c0cbb232de563eefd7c3b27fb2c3
+make[1]: Leaving directory '/home/daniel/git/o2r/reference-implementation'
From f03c7d160981becfb733a71912fe63308526d733 Mon Sep 17 00:00:00 2001
From: nuest <daniel.nuest@uni-muenster.de>
Date: Wed, 12 Dec 2018 13:39:09 +0100
Subject: [PATCH 36/53] fix doc building and expose via localhost/docs/
---
Makefile | 3 ++-
README.md | 6 +++--
docker-compose-local.yml | 4 +++
etc/Dockerfile.documentations | 5 ++--
etc/index.html | 49 +++++++++++++++++++++++++++++++++++
etc/nginx.conf | 13 +++++++---
6 files changed, 71 insertions(+), 9 deletions(-)
create mode 100644 etc/index.html
diff --git a/Makefile b/Makefile
index 1129ba8..94e69e4 100644
--- a/Makefile
+++ b/Makefile
@@ -90,8 +90,9 @@ clean: hub_down_volume local_down_volume
build_documentation:
rm -f *.pdf
docker build --tag docbuilder --file etc/Dockerfile.documentations .
+ @echo $(CURDIR)
docker run -it -v $(CURDIR)/architecture:/doc:rw docbuilder make build pdf
- docker run -it -v $(CURDIR)/api-spec:/doc:rw docbuilder make build pdf
+ docker run -it -v $(CURDIR)/api:/doc:rw docbuilder make build pdf
docker run -it -v $(CURDIR)/erc-spec:/doc:rw docbuilder make build pdf_tinytex
mv architecture/site/*.pdf .
mv erc-spec/*.pdf .
diff --git a/README.md b/README.md
index 9cdc1c7..c4bd889 100644
--- a/README.md
+++ b/README.md
@@ -184,10 +184,12 @@ docker volume inspect reference-implementation_o2r_test_storage
## Read documentation
This repository also contains specification and documentation projects.
-These documentations are created in Markdown format and can be rendered to HTML and PDF documents using the make target `build_documentation`, which relies on a local `Dockerfile` for the rendering runtime environment.
-The PDF files are moved to the project root directory.
+These documentations are created in Markdown format and can be rendered to HTML and PDF documents using the make target `build_documentation`, which relies on a local `Dockerfile` for the rendering environment.
+The PDF files are available to the project root directory.
The HTML files can be found in the respective projects in the `site` directory.
+If the `local` configuration is used and the documentation was build, or if you [reproduce](#reproduce) you can also browse it at http://localhost/docs/.
+
## Reproducibility
This repository serves the goal to make the developments of the o2r project reproducible, not only by running the reference implementation (see above) but also by creating an archivable package of the software.
diff --git a/docker-compose-local.yml b/docker-compose-local.yml
index 9734c7c..f1403e3 100644
--- a/docker-compose-local.yml
+++ b/docker-compose-local.yml
@@ -255,6 +255,10 @@ services:
- substituter
volumes:
- "./etc/nginx.conf:/etc/nginx/nginx.conf:ro"
+ - "./api/site/:/www/docs/api:ro"
+ - "./architecture/site/:/www/docs/architecture:ro"
+ - "./erc-spec/site/:/www/docs/erc-spec:ro"
+ - "./etc/index.html:/www/docs/index.html:ro"
ports:
- "80:80"
diff --git a/etc/Dockerfile.documentations b/etc/Dockerfile.documentations
index c5eca48..41b60cb 100644
--- a/etc/Dockerfile.documentations
+++ b/etc/Dockerfile.documentations
@@ -35,9 +35,8 @@ RUN pip3 install --upgrade pip \
RUN python3 -m pip install mkdocs mkdocs-cinder markdown-include
RUN apt-get update && apt-get install -y --no-install-recommends \
xfonts-base xfonts-75dpi
-RUN wget https://github.com/wkhtmltopdf/wkhtmltopdf/releases/download/0.12.4/wkhtmltox-0.12.4_linux-generic-amd64.tar.xz \
- && tar -xvf wkhtmltox-0.12.4_linux-generic-amd64.tar.xz \
- && mv wkhtmltox/bin/wkhtmltopdf /usr/bin/wkhtmltopdf
+RUN wget https://downloads.wkhtmltopdf.org/0.12/0.12.5/wkhtmltox_0.12.5-1.stretch_amd64.deb \
+ && dpkg -i wkhtmltox_0.12.5-1.stretch_amd64.deb
VOLUME /doc
WORKDIR /doc
diff --git a/etc/index.html b/etc/index.html
new file mode 100644
index 0000000..e35cb2a
--- /dev/null
+++ b/etc/index.html
@@ -0,0 +1,49 @@
+<!DOCTYPE html>
+
+<head>
+ <meta charset="utf-8">
+ <meta http-equiv="X-UA-Compatible" content="IE=edge">
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+ <meta name="author" content="Daniel Nüst">
+ <link rel="shortcut icon" href="https://o2r.info/favicon.ico">
+ <title>o2r Reference Implementation - Documentation</title>
+ <link href='https://fonts.googleapis.com/css?family=Lato:400,700|Roboto+Slab:400,700|Inconsolata:400,700' rel='stylesheet'
+ type='text/css'>
+
+ <link rel="stylesheet" href="./api/css/theme.css" type="text/css" />
+ <link rel="stylesheet" href="./api/css/theme_extra.css" type="text/css" />
+ <!--<link href="./erc.css" rel="stylesheet">-->
+
+</head>
+
+<body role="document">
+ <section>
+ <div class="wy-nav-content">
+ <div role="main">
+ <div class="section">
+
+ <h1>o2r Reference Implementation - Documentation</h1>
+
+ <ul>
+ <li><a href="./api/">API documentation</a></li>
+ <li><a href="./architecture">o2r System architecture</a></li>
+ <li><a href="./erc-spec">ERC specification</a></li>
+ </ul>
+
+ Go back to the <a href="../">demo server</a>.
+
+ <hr />
+
+ <p><strong>Credits</strong></p>
+ <p>These documents are developed by the members of the DFG-funded project Opening Reproducible Research.
+ Learn more at <strong><a href="https://o2r.info">https://o2r.info</a></strong>.</p>
+ <p><a href="http://o2r.info"><img alt="Opening Reproducible Research" src="./api/logo.png" /></a></p>
+
+ </div>
+ </div>
+ </div>
+ </section>
+
+</body>
+
+</html>
\ No newline at end of file
diff --git a/etc/nginx.conf b/etc/nginx.conf
index c2e34be..02d7542 100644
--- a/etc/nginx.conf
+++ b/etc/nginx.conf
@@ -68,7 +68,7 @@ http {
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection $connection_upgrade;
-
+
proxy_pass http://informer:8082;
}
@@ -118,12 +118,19 @@ http {
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_pass http://bindings:8092;
- }
+ }
location /oauth {
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_pass http://guestlister:8383;
- }
+ }
+
+ location /docs {
+ include /etc/nginx/mime.types;
+ index index.html;
+ root /www;
+ }
+
}
}
\ No newline at end of file
From 9d2496cf0854627f243888abbf6e20238b598f8c Mon Sep 17 00:00:00 2001
From: nuest <daniel.nuest@uni-muenster.de>
Date: Wed, 12 Dec 2018 13:40:58 +0100
Subject: [PATCH 37/53] update submodules
---
.gitmodules | 4 ++--
api | 2 +-
architecture | 2 +-
containerit | 2 +-
erc-checker | 2 +-
erc-examples | 2 +-
erc-spec | 2 +-
o2r-bindings | 2 +-
o2r-meta | 2 +-
o2r-muncher | 2 +-
o2r-platform | 2 +-
11 files changed, 12 insertions(+), 12 deletions(-)
diff --git a/.gitmodules b/.gitmodules
index a8eddf4..cdc6edc 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -22,9 +22,9 @@
[submodule "o2r-platform"]
path = o2r-platform
url = https://github.com/o2r-project/o2r-platform
-[submodule "o2r-web-api"]
+[submodule "api"]
path = api
- url = https://github.com/o2r-project/o2r-web-api
+ url = https://github.com/o2r-project/api
[submodule "erc-spec"]
path = erc-spec
url = https://github.com/o2r-project/erc-spec
diff --git a/api b/api
index b400ca7..e9bb2b8 160000
--- a/api
+++ b/api
@@ -1 +1 @@
-Subproject commit b400ca7a73735c959e8fbd177021e4d0d5ec37bb
+Subproject commit e9bb2b86ecd8fa23dde9a109284dd3506d243184
diff --git a/architecture b/architecture
index eaf33f2..00756bc 160000
--- a/architecture
+++ b/architecture
@@ -1 +1 @@
-Subproject commit eaf33f2aa4c5af7dbcdd3de95a9d4196f64da64b
+Subproject commit 00756bca5b3a69ce4267149dead6f57d08a4bfce
diff --git a/containerit b/containerit
index 6ba7d1d..fb107e2 160000
--- a/containerit
+++ b/containerit
@@ -1 +1 @@
-Subproject commit 6ba7d1de6e75aeec935bd742ba40add835bd4ffc
+Subproject commit fb107e2979c48b69336f7019f991f9e6f9900757
diff --git a/erc-checker b/erc-checker
index 43fba4a..fb1ed16 160000
--- a/erc-checker
+++ b/erc-checker
@@ -1 +1 @@
-Subproject commit 43fba4a9a237956642e72d654c3d84a1465fb546
+Subproject commit fb1ed1617464b862404dc18c3641678543bca470
diff --git a/erc-examples b/erc-examples
index 0b79d48..d589255 160000
--- a/erc-examples
+++ b/erc-examples
@@ -1 +1 @@
-Subproject commit 0b79d4858a2d0a45024b93cb140e217f144ad6a8
+Subproject commit d5892550b2d68e858f006e4867de4fae5cc76a11
diff --git a/erc-spec b/erc-spec
index 64b70f6..a112fd8 160000
--- a/erc-spec
+++ b/erc-spec
@@ -1 +1 @@
-Subproject commit 64b70f69f97b0d2783714e9509166ff355f226fd
+Subproject commit a112fd85b26a4659997ac2463ec0aff7315ebd22
diff --git a/o2r-bindings b/o2r-bindings
index aa77f5a..f68e17d 160000
--- a/o2r-bindings
+++ b/o2r-bindings
@@ -1 +1 @@
-Subproject commit aa77f5afe189b78398324ea699daa74010b313ce
+Subproject commit f68e17d12dc7573bd7c6b927d444ef35b09f14fc
diff --git a/o2r-meta b/o2r-meta
index 9198450..f79cc1f 160000
--- a/o2r-meta
+++ b/o2r-meta
@@ -1 +1 @@
-Subproject commit 9198450c57e4969bbce6c9e737e59b18401107f4
+Subproject commit f79cc1f838ce8a6151cd10967e497b4c9040e79d
diff --git a/o2r-muncher b/o2r-muncher
index 1471014..c68b4d3 160000
--- a/o2r-muncher
+++ b/o2r-muncher
@@ -1 +1 @@
-Subproject commit 14710140647d1e489aedecf7f0d6ca949efef818
+Subproject commit c68b4d369a52082ef764b0d576c7964258e0fd63
diff --git a/o2r-platform b/o2r-platform
index 4ad78d6..f7075b9 160000
--- a/o2r-platform
+++ b/o2r-platform
@@ -1 +1 @@
-Subproject commit 4ad78d6ccb055ec6c6bad294ae71f6933d644c8e
+Subproject commit f7075b9be074dee841ab9cced667af6c66eadc3e
From ee9c503497f079da4973c455f0271b9ec689a653 Mon Sep 17 00:00:00 2001
From: nuest <daniel.nuest@uni-muenster.de>
Date: Wed, 12 Dec 2018 13:41:33 +0100
Subject: [PATCH 38/53] update hub-related make targets
---
Makefile | 16 +++++++++++-----
1 file changed, 11 insertions(+), 5 deletions(-)
diff --git a/Makefile b/Makefile
index 94e69e4..25dd6a0 100644
--- a/Makefile
+++ b/Makefile
@@ -46,6 +46,7 @@ local_versions:
etc/local_versions.sh
local_versions_save: local_versions
+ rm -f versions.txt
make local_versions >> versions.txt
hub_images:
@@ -64,9 +65,6 @@ hub_images:
docker pull o2rproject/o2r-guestlister;
docker pull o2rproject/o2r-bindings;
-hub_up:
- docker-compose up;
-
hub_versions:
@docker inspect --format '{{index .Config.Labels "org.label-schema.name"}}: {{index .Config.Labels "org.label-schema.version"}}' o2rproject/o2r-bouncer;
@docker inspect --format '{{index .Config.Labels "org.label-schema.name"}}: {{index .Config.Labels "org.label-schema.version"}}' o2rproject/o2r-finder;
@@ -83,8 +81,16 @@ hub_versions:
@docker inspect --format '{{index .Config.Labels "org.label-schema.name"}}: {{index .Config.Labels "org.label-schema.version"}}' o2rproject/o2r-bindings;
@docker inspect --format '{{index .Config.Labels "org.label-schema.name"}}: {{index .Config.Labels "org.label-schema.version"}}' o2rproject/containerit;
-clean: hub_down_volume local_down_volume
- docker ps -a | grep o2r | awk '{print $1}' | xargs docker rm -f
+hub_up:
+ docker-compose up;
+
+hub: hub_images hub_versions hub_up
+
+hub_down_volume:
+ docker-compose down --volume;
+
+hub_clean: hub_down_volume
+ docker ps -a | grep o2r | awk '{print $1}' | xargs docker rm --force
docker images | grep o2r | awk '{print $3}' | xargs docker rmi --force
build_documentation:
From 0681f7a79039e3e9d6957fe3a49e29f23c4e0b62 Mon Sep 17 00:00:00 2001
From: nuest <daniel.nuest@uni-muenster.de>
Date: Wed, 12 Dec 2018 13:41:42 +0100
Subject: [PATCH 39/53] fix image name typo
---
docker-compose-local.yml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/docker-compose-local.yml b/docker-compose-local.yml
index f1403e3..592d478 100644
--- a/docker-compose-local.yml
+++ b/docker-compose-local.yml
@@ -210,7 +210,7 @@ services:
- DEBUGME=inspecter
bindings:
- image: o2r_refimple_bindings
+ image: o2r_refimpl_bindings
build:
context: ./o2r-bindings
ports:
From 38a46ddbf6a776fe214b891def307cb09a13dcb5 Mon Sep 17 00:00:00 2001
From: nuest <daniel.nuest@uni-muenster.de>
Date: Wed, 12 Dec 2018 13:42:35 +0100
Subject: [PATCH 40/53] add release target and extend README
---
Makefile | 7 ++++
README.md | 117 ++++++++++++++++++++++++++++++++++++++----------------
2 files changed, 90 insertions(+), 34 deletions(-)
diff --git a/Makefile b/Makefile
index 25dd6a0..998e56a 100644
--- a/Makefile
+++ b/Makefile
@@ -93,6 +93,11 @@ hub_clean: hub_down_volume
docker ps -a | grep o2r | awk '{print $1}' | xargs docker rm --force
docker images | grep o2r | awk '{print $3}' | xargs docker rmi --force
+local_clean: local_down_volume
+ rm -f *.tar;
+ docker ps -a | grep o2r | awk '{print $1}' | xargs docker rm --force
+ docker images | grep o2r_refimpl | awk '{print $3}' | xargs docker rmi --force
+
build_documentation:
rm -f *.pdf
docker build --tag docbuilder --file etc/Dockerfile.documentations .
@@ -137,6 +142,8 @@ versions:
git --version;
docker --version;
+release: versions update build_documentation local_versions local_build local_save_images create_archive
+
reproduce:
# TODO import images from files
# run make local
diff --git a/README.md b/README.md
index c4bd889..686082a 100644
--- a/README.md
+++ b/README.md
@@ -1,23 +1,33 @@
# o2r reference implementation
-A catch-all repository to run all [o2r](http://o2r.info) microservices and the user interface platform.
+A catch-all repository to run all [o2r](https://o2r.info) microservices and the user interface platform which together form the [o2r prototype for a reproducibility service and publishing system](https://o2r.info/results).
+
+In the remainder of this document, specifications or projects are referenced with their code repository, even if a user-friendly online HTML rending exists, e.g. https://github.com/o2r-project/erc-spec/ instead of https://o2r.info/erc-spec/.
+Please check the code repository metadata or respective README files for documentation that is easier to read or browse.
## Run o2r platform and reproducibility service
+### tl;dr
+
+To quickly get the reference implementation running locally, you can use the ready-to-use images from Docker Hub:
+
+```bash
+make hub
+```
+
### Basics
The o2r reference implementation realises a workflow for publishing reproducible articles using the [Executable Research Compendium](https://github.com/o2r-project/erc-spec) (ERC) and has two broad components: a user interface for demonstrating novel interactions with ERC, a.k.a. the "platform", and a side collection of microservices implementing the [o2r Web API](https://github.com/o2r-project/o2r-web-api), a.k.a. the "reproducibility service".
-For more information on the architecture and the microservices see [o2r Software Architecture](https://github.com/o2r-project/architecture).
+For more information on the architecture and the microservices see [o2r System Architecture](https://github.com/o2r-project/architecture).
### Contents
-All of the mentioned documentation and software is available in this project.
-The documentation is also available online for reading, though availability may change.
+All of the software, specification and documentation for the o2r system are available nested in this project.
-| Component | directory | online |
+| **Component** | **Directory | **Online** |
| ------ | ------ | ------ |
-| ERC specification (document) | `erc-spec` | https://o2r.info/erc-spec/ |
+| ERC specification (document) | `erc-spec` | https://github.com/o2r-project/erc-spec/ |
| Web API specification (document) | `api` | https://github.com/o2r-project/api |
| Architecture specification (document) | `architecture` | https://github.com/o2r-project/architecture |
| ERC checker (library/tool) | `erc-checker` | https://github.com/o2r-project/erc-checker/ |
@@ -38,8 +48,9 @@ The documentation is also available online for reading, though availability may
This project contains configurations and scripts to make running the o2r reference implementation as easy as possible.
-The project uses [`make`](https://www.gnu.org/software/make/), which helps simplifying the execution of the various steps necessary to run your own o2r-platform into a simple set of commands.
+The project uses [`make`](https://www.gnu.org/software/make/), which helps simplifying the execution of the various steps necessary to run the reference implementation into a set of console commands.
These commands are formulated in the `Makefile` included in this project.
+If `make` is not available, you can execute the respective commands manually.
The commands in _this file_ (`README.md`) assume a Unix shell and related common tools, so they work on a recent installation of Linux or MacOS X.
@@ -73,7 +84,7 @@ Run `make update` or the respective commands on your operating system to initial
By default, the reference implementation uses the offline OAuth2 implementation provided by the [o2r-guestlister](https://github.com/o2r-project/o2r-guestlister).
-This allows access to the o2r platform by selecting one of three demo users. The users represent different user roles with different levels, i.e. an admin (level 1000), an editor (level 500) and a basic author (level 100).
+This allows access to the o2r platform by selecting one of three demo users. The users represent different user roles with different levels, i.e. an admin (level `1000`), an editor (level `500`) and a basic author (level `100`).
##### ORCID (optional)
@@ -113,32 +124,46 @@ You can find instructions for all hosts (including Docker Toolbox) in the [Elast
### Build images from source and run
-This repository already includes a `.gitmodules` file, which lists all required o2r microservices as git submodules.
+This repository already includes a `.gitmodules` file, which lists all required o2r microservices and tools as git submodules.
To download all o2r source code at once, navigate to the `reference-implementation` base directory and use
```bash
make update
```
-Once all repositories have been pulled successfully, build Docker images of the microservices and run them in containers by executing:
+Once all repositories have been pulled successfully, build Docker images of the microservices and tools:
```bash
-make local
+make local_build
```
-Wait until the log shows no new messages, then open **http://localhost** and continue in section ["Use platform"](#use-platform).
+Then run the microservices and platform in containers:
+
+```bash
+make local_up
+```
+
+Wait until the log shows no new messages for a few seconds, then open **http://localhost** and continue in section ["Load data"](#load-data).
+
+The above three steps can also be executed with a single target:
+
+```bash
+make local
+```
### Download images and run
-[Docker Hub](https://hub.docker.com/) is a repository for Docker images.
-All o2r software projects have automatic builds [available on Docker Hub](https://hub.docker.com/r/o2rproject/).
-The following command executes a `docker-compose` command to pull and run these images.
+[Docker Hub](https://hub.docker.com/) is a [public registry](https://en.wikipedia.org/wiki/Docker_(software)#Components) for Docker images.
+All o2r software projects are automatically built [on Docker Hub](https://hub.docker.com/r/o2rproject/) when there is a new version uploaded to the code repository.
+The images have tags corresponding to the software version (as specific e.g. in a `package.json` for a Node.js-based microservice).
+
+The following target pulls the latest images, prints the versions to the terminal, and executes a `docker-compose` command to run the reference implementation:
```bash
make hub
```
-Wait until the log shows no new messages, then open **http://localhost** and continue in section ["Use platform"](#use-platform).
+Wait until the log shows no new messages for a few seconds, then open **http://localhost** and continue in section ["Load data"](#load-data).
### Load data
@@ -192,42 +217,66 @@ If the `local` configuration is used and the documentation was build, or if you
## Reproducibility
-This repository serves the goal to make the developments of the o2r project reproducible, not only by running the reference implementation (see above) but also by creating an archivable package of the software.
-As the implementations component are spread over multiple git repositories, we cannot rely on the direct export of a GitHub release to Zenodo.
-This repository imports all relevant software projects as [git submodules](https://git-scm.com/docs/git-submodule) and manually creates a release package.
-
-**TBD**
+### Rationale
-- clone repo
-- build all images
-- export all image
-- create make target to load all images and run these
-- create huge zip
-- publish to Zenodo (with make target)
+This repository serves the goal to make the developments of the o2r project reproducible, not only by running the reference implementation (see above) but also by creating an archivable package of the software in a reproducible (i.e. scripted) way.
+As the implementations component are spread over multiple git repositories, we cannot rely on the direct export of a GitHub release to Zenodo.
+This repository captures all software projects to create a release package, which can be uploaded to a data repository manually.
### Known limitations
-- Nested code projects are currently installed from GitHub during the building of Docker images ((see also [#1]()).
+- Nested code projects are currently installed from GitHub during the building of Docker images ((see also [#1](#1)).
- `o2r-muncher`'s Docker image contains `o2r-meta` and `erc-checker`
- `o2r-loaders`'s Docker image contains `o2r-meta`
- The used Docker base images and dependencies for the services, such as npm or pip packages, must be available online in the required version to build the images locally. See project files such as `package.json` or `requirements.txt` for a full list of dependencies.
+- The locally built images do not have a proper version tag but instead are `latest`, since they are build as part of the `docker-compose` configuration. Note that you can easily distinguish images by this, too: The hub images will always have a specific version tag.
### Create package
-**TODO**
+The creation of the reproducibility package consists of the following steps and is relies where possible on Makefile targets.
+The package uses the remote images from Docker Hub at the time of creation
+1. Clone the `reference-implementation` repository to an empty directory: `git clone https://github.com/o2r-project/reference-implementation.git`
+1. Create the package with `make release`, which...
+ - cleans up potentially existing artifacts (`make local_clean`),
+ - prints software versions (`make versions`),
+ - updates the nested projects (`make update`),
+ - builds all documentation locally (`make build_documentation`),
+ - save local version information of software and repositories to single file `versions.txt` (`make local_versions_save`),
+ - builds all images locally (see [limitations](#known-limitations), `make local_build`),
+ - saves the just built images into the file `o2r-reference-implementation-images.tar` (`make local_save_images`),
+ - saves all nested code repositories with their histories, to `o2r-reference-implementation-modules.zip`
+ - saves all nested documentation repositories with their histories, to `o2r-docs.zip`
+1. Create a new deposit or a new version of the existing deposit on Zenodo
+1. Upload the files to Zenodo deposit: `ZENODO_DEPOSIT_ID=2203844 ZENODO_ACCESS_TOKEN=xxxxxx make upload_deposit`
+1. Fill out metadata form on Zenodo and publish using the "Publish" button
+
+### Reproduce
+
+The reproduction consists of three steps, assuming the local images do not exist on the machine used:
+
+1. Download the saved images from Zenodo
+1. Load the saved images from the tarball
+1. Run the `docker-compose` configuration without building the images (to ensure the loaded ones are used)
+
+The last two steps can be executed with
```bash
-make show_microservices_versions >> dist/versions.txt
+make reproduce
```
-### Archival to Zenodo
+Wait until the log shows no new messages for a few seconds, then open **http://localhost** and continue see section ["Load data"](#load-data) for creating ERCs.
-**TODO**
+You can stop all containers and clean up local images with
+```bash
+make local_down
+# to also delete local storage use make local_down_volumes
+make local_clean
+```
## License
-This project is licensed under Apache License, Version 2.0, see file LICENSE. Copyright © 2017 - o2r project.
+This project is licensed under Apache License, Version 2.0, see file LICENSE. Copyright © 2018 - o2r project.
-All included software projects have their own LICENSE files, see `o2r-<component name>/LICENSE`.
+All included software projects have their own LICENSE files, see `<component name>/LICENSE`.
From 762c142822d8e4c004a45f903e8cd51ddcc71025 Mon Sep 17 00:00:00 2001
From: nuest <daniel.nuest@uni-muenster.de>
Date: Thu, 13 Dec 2018 17:29:12 +0100
Subject: [PATCH 41/53] add upload script for Zenodo
---
.gitignore | 6 +--
Makefile | 24 ++++++----
README.md | 3 +-
etc/zenodo_upload.py | 103 +++++++++++++++++++++++++++++++++++++++++++
4 files changed, 123 insertions(+), 13 deletions(-)
create mode 100644 etc/zenodo_upload.py
diff --git a/.gitignore b/.gitignore
index f27dafb..03987b7 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,7 +1,5 @@
*.pdf
-o2r-reference-implementation-modules\.zip
-
-o2r-reference-implementation-images\.tar\.gz
-
+o2r-reference-implementation*\.tar\.gz
+o2r-reference-implementation*\.zip
o2r-docs\.zip
diff --git a/Makefile b/Makefile
index 998e56a..31c1298 100644
--- a/Makefile
+++ b/Makefile
@@ -94,7 +94,6 @@ hub_clean: hub_down_volume
docker images | grep o2r | awk '{print $3}' | xargs docker rmi --force
local_clean: local_down_volume
- rm -f *.tar;
docker ps -a | grep o2r | awk '{print $1}' | xargs docker rm --force
docker images | grep o2r_refimpl | awk '{print $3}' | xargs docker rmi --force
@@ -108,7 +107,7 @@ build_documentation:
mv architecture/site/*.pdf .
mv erc-spec/*.pdf .
mv api-spec/*.pdf .
- echo "ERC, architecture, and web API documentation created, see files PDF files in the project root directory"
+ @echo "ERC, architecture, and web API documentation created, see files PDF files in the project root directory"
local_save_images:
docker save \
@@ -116,6 +115,7 @@ local_save_images:
adicom/admin-mongo:latest \
docker.elastic.co/elasticsearch/elasticsearch:5.6.10 \
o2r_refimpl_containerit \
+ o2r_refimpl_meta \
o2r_refimpl_muncher \
o2r_refimpl_loader \
o2r_refimpl_informer \
@@ -128,22 +128,30 @@ local_save_images:
o2r_refimpl_bindings \
o2r_refimpl_guestlister \
o2r_refimpl_platform \
- nginx:latest | gzip -c > o2r-reference-implementation-images.tar.gz;
- tar -tvf o2r-reference-implementation-images.tar.gz;
- ls -sh o2r-reference-implementation-images.tar.gz;
- @echo "Inspecting tarball manifest:";
- tar -xf o2r-reference-implementation-images.tar.gz manifest.json -O | python -m json.tool;
+ nginx:latest | pigz --stdout --fast > o2r-reference-implementation-images.tar.gz;
+ ls -1sh o2r-reference-implementation-images*.tar.gz;
+
+#@echo "Inspect tarball manifests with";
+#tar -xf o2r-reference-implementation-images.tar.gz manifest.json -O | python -m json.tool;
create_archive:
- zip -r -q o2r-reference-implementation-modules.zip software;
+ zip -r -q o2r-reference-implementation-modules.zip containerit/ erc-checker/ o2r-*/;
zip -r -q o2r-docs.zip api/ architecture/ erc-examples/ erc-spec/;
+ zip -r -q o2r-reference-implementation-files.zip etc/ README*.md docker-compose*.yml versions.txt;
versions:
git --version;
docker --version;
+release_clean:
+ rm -f *.zip
+ rm -f *.tar;
+
release: versions update build_documentation local_versions local_build local_save_images create_archive
+upload_files_to_zenodo:
+ python etc/zenodo_upload.py
+
reproduce:
# TODO import images from files
# run make local
diff --git a/README.md b/README.md
index 686082a..76313dc 100644
--- a/README.md
+++ b/README.md
@@ -248,7 +248,8 @@ The package uses the remote images from Docker Hub at the time of creation
- saves all nested code repositories with their histories, to `o2r-reference-implementation-modules.zip`
- saves all nested documentation repositories with their histories, to `o2r-docs.zip`
1. Create a new deposit or a new version of the existing deposit on Zenodo
-1. Upload the files to Zenodo deposit: `ZENODO_DEPOSIT_ID=2203844 ZENODO_ACCESS_TOKEN=xxxxxx make upload_deposit`
+1. Upload the _small files_ to [Zenodo deposit](https://zenodo.org/api/deposit/depositions/2203844): `ZENODO_DEPOSIT_ID=2203844 ZENODO_ACCESS_TOKEN=xxxxxx make upload_files_to_zenodo` (the used Python scripts needs the module `humanfriendly`)
+1. Manually upload the _big file_ to Zenodo deposit ([Zenodo API currently has a 100 MB file limit](http://developers.zenodo.org/#deposition-files))
1. Fill out metadata form on Zenodo and publish using the "Publish" button
### Reproduce
diff --git a/etc/zenodo_upload.py b/etc/zenodo_upload.py
new file mode 100644
index 0000000..c8df9a1
--- /dev/null
+++ b/etc/zenodo_upload.py
@@ -0,0 +1,103 @@
+"""
+Copyright (c) 2018 o2r project
+
+Helper functions to clear existing Zenodo deposit and upload files.
+Creation of deposit, setting the version/metadata, and publishing the deposit are intentionally not included here.
+"""
+
+import os
+import sys
+import logging
+import requests
+import humanfriendly
+
+# (you must install the required Python modules before the first run: `pip install clint requests_toolbelt`)
+#from clint.textui.progress import Bar
+#from requests_toolbelt import MultipartEncoder, MultipartEncoderMonitor
+
+logging.basicConfig(level=logging.INFO) # logging for requests
+
+token = os.getenv('ZENODO_ACCESS_TOKEN')
+deposit = os.getenv('ZENODO_DEPOSIT_ID')
+ZENODO_URL = 'https://zenodo.org/api/deposit/depositions'
+
+def upload_file(file, deposit_id):
+ print('Uploading %s' % (file))
+ file_name = os.path.basename(file)
+
+ # streaming upload, https://toolbelt.readthedocs.io/en/latest/uploading-data.html#streaming-multipart-data-encoder
+ #encoder = MultipartEncoder({
+ # 'filename': file_name,
+ # 'file': open(file, 'rb')
+ # })
+ #bar = Bar(expected_size = encoder.len, filled_char = '=')
+ #def callback(monitor):
+ # bar.show(monitor.bytes_read)
+ #monitor = MultipartEncoderMonitor(encoder, callback)
+ #print(monitor.content_type)
+ # r = requests.post('%s/%s/files?access_token=%s' % (ZENODO_URL, deposit_id, token),
+ # data = m,
+ # headers = {'Content-Type': m.content_type})
+
+ r = requests.post('%s/%s/files?access_token=%s' % (ZENODO_URL, deposit_id, token),
+ data = {'filename': file_name},
+ files = {'file': open(file, 'rb')})
+ if r.status_code != 201:
+ print('Error uploading file: %s' % (r.text))
+ sys.exit()
+ else:
+ print('Uploaded file (HTTP %s): %s' % (r.status_code, r.json()['links']['self']))
+ return r.json()['links']['self']
+
+def clear_files(deposit_id):
+ print('Clearing files from deposit %s ...' % (deposit_id))
+ r = requests.get('%s/%s' % (ZENODO_URL, deposit_id), params = {'access_token': token})
+ if r.status_code != 200:
+ print('Error getting list of files: %s' % (r.text))
+ sys.exit()
+ else:
+ for file in r.json()['files']:
+ print('Deleting %s | %s' % (file['id'], file['links']['download']))
+ r_delete = requests.delete('%s/%s/files/%s' % (ZENODO_URL, deposit_id, file['id']), params = {'access_token': token})
+ if r_delete.status_code != 204:
+ print('Error deleting existing file (HTTP %s): %s', (r_delete.text, r_delete.status_code))
+ sys.exit()
+ else:
+ print('Deleted %s' % (file['id']))
+
+def new_version(deposit_id):
+ print('Creating new version for %s' % (deposit_id))
+ r = requests.post('%s/%s/actions/newversion' % (ZENODO_URL, deposit_id), params = {'access_token': token})
+ if r.status_code != 201:
+ print('Error creating new version: %s' % (r.text))
+ sys.exit()
+ else:
+ print('Created new version %s' % (r.json()['links']['latest_draft']))
+ return os.path.basename(r.json()['links']['latest_draft'])
+
+
+if __name__ == '__main__':
+ #deposit = new_version(deposit)
+ clear_files(deposit)
+
+ files = ['README.md',
+ 'README-WIN.md',
+ 'Makefile',
+ 'o2r-docs.zip',
+ 'o2r-reference-implementation-files.zip',
+ #'o2r-reference-implementation-images.tar.gz',
+ 'o2r-reference-implementation-modules.zip',
+ 'versions.txt'
+ ]
+ for file in files:
+ upload_file(file, deposit)
+
+ print('Uploaded small files to %s/%s: %s' % (ZENODO_URL, deposit, str(files)))
+ raw_input('Go to https://zenodo.org/deposit/%s and upload the file %s. Done?' % (deposit, 'o2r-reference-implementation-images.tar.gz'))
+ print('Deposition files:')
+ r = requests.get('%s/%s' % (ZENODO_URL, deposit), params = {'access_token': token})
+ for file in r.json()['files']:
+ print(' %s (%s)' % (file['filename'], humanfriendly.format_size(file['filesize'])))
+
+ #publish_deposit(deposit)
+ #print('Published new deposit at: %s/%s' % (ZENODO_URL, deposit))
From 75d09862d6dc49125523d27a80081bc2b78865de Mon Sep 17 00:00:00 2001
From: nuest <daniel.nuest@uni-muenster.de>
Date: Fri, 14 Dec 2018 14:51:56 +0100
Subject: [PATCH 42/53] fix documentation building
---
Makefile | 15 +++++++--------
api | 2 +-
etc/Dockerfile.documentations | 25 ++++++++++++-------------
3 files changed, 20 insertions(+), 22 deletions(-)
diff --git a/Makefile b/Makefile
index 31c1298..cfd9c24 100644
--- a/Makefile
+++ b/Makefile
@@ -99,14 +99,13 @@ local_clean: local_down_volume
build_documentation:
rm -f *.pdf
- docker build --tag docbuilder --file etc/Dockerfile.documentations .
- @echo $(CURDIR)
- docker run -it -v $(CURDIR)/architecture:/doc:rw docbuilder make build pdf
- docker run -it -v $(CURDIR)/api:/doc:rw docbuilder make build pdf
- docker run -it -v $(CURDIR)/erc-spec:/doc:rw docbuilder make build pdf_tinytex
- mv architecture/site/*.pdf .
- mv erc-spec/*.pdf .
- mv api-spec/*.pdf .
+ docker build --tag o2r_refimpl_docbuilder --file etc/Dockerfile.documentations .
+ docker run -it -v $(CURDIR)/architecture:/doc:rw o2r_refimpl_docbuilder make build pdf
+ docker run -it -v $(CURDIR)/api:/doc:rw o2r_refimpl_docbuilder make build pdf
+ docker run -it -v $(CURDIR)/erc-spec:/doc:rw o2r_refimpl_docbuilder make build travis_pdf
+ mv architecture/site/o2r-architecture.pdf .
+ mv erc-spec/site/erc-spec.pdf .
+ mv api/o2r-web-api.pdf .
@echo "ERC, architecture, and web API documentation created, see files PDF files in the project root directory"
local_save_images:
diff --git a/api b/api
index e9bb2b8..8d75e2f 160000
--- a/api
+++ b/api
@@ -1 +1 @@
-Subproject commit e9bb2b86ecd8fa23dde9a109284dd3506d243184
+Subproject commit 8d75e2ff1847073b8bc9025bc901e9a2e2cb9b3f
diff --git a/etc/Dockerfile.documentations b/etc/Dockerfile.documentations
index 41b60cb..e37daa3 100644
--- a/etc/Dockerfile.documentations
+++ b/etc/Dockerfile.documentations
@@ -20,23 +20,22 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
RUN wget https://github.com/jgm/pandoc/releases/download/2.1.1/pandoc-2.1.1-linux.tar.gz \
&& tar -xvzf pandoc*.tar.gz --strip-components 1 -C /usr/local/
-RUN tlmgr update --self \
- && tlmgr init-usertree \
- && tlmgr install xcolor mdframed needspace
-RUN pip3 install --upgrade pip \
- && python3 -m pip install \
- mkdocs \
- pymdown-extensions \
- mkdocs-pandoc \
- pandoc-latex-admonition \
- git+https://github.com/o2r-project/mkdocs-combine.git
-
-# architecture, see architecture/.travis.yml
-RUN python3 -m pip install mkdocs mkdocs-cinder markdown-include
+# install required tools, see architecture/.travis.yml and erc-spec/.travis.yml
+RUN pip3 install --upgrade pip
+RUN python3 -m pip install \
+ mkdocs \
+ pygments \
+ mkdocs-cinder \
+ mkdocs-pandoc \
+ markdown-include \
+ pymdown-extensions \
+ pandoc-latex-admonition \
+ git+https://github.com/o2r-project/mkdocs-combine.git
RUN apt-get update && apt-get install -y --no-install-recommends \
xfonts-base xfonts-75dpi
RUN wget https://downloads.wkhtmltopdf.org/0.12/0.12.5/wkhtmltox_0.12.5-1.stretch_amd64.deb \
&& dpkg -i wkhtmltox_0.12.5-1.stretch_amd64.deb
+RUN R --quiet -e 'tinytex::tlmgr_install(pkgs=c("xcolor", "mdframed", "mdwtools", "needspace", "unicode-math", "filehook", "lm-math"))'
VOLUME /doc
WORKDIR /doc
From bb6618591e8b361384089ea10ccd801ff78f5339 Mon Sep 17 00:00:00 2001
From: nuest <daniel.nuest@uni-muenster.de>
Date: Fri, 14 Dec 2018 14:52:42 +0100
Subject: [PATCH 43/53] do not fail cleaning of images and containers if they
do not exists
---
Makefile | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/Makefile b/Makefile
index cfd9c24..b35a289 100644
--- a/Makefile
+++ b/Makefile
@@ -90,12 +90,12 @@ hub_down_volume:
docker-compose down --volume;
hub_clean: hub_down_volume
- docker ps -a | grep o2r | awk '{print $1}' | xargs docker rm --force
- docker images | grep o2r | awk '{print $3}' | xargs docker rmi --force
+ docker ps -a | grep o2r | awk '{print $$1}' | xargs sh -c "docker rm --force || true"
+ docker images | grep o2r | awk '{print $$3}' | xargs sh -c "docker rmi --force || true"
local_clean: local_down_volume
- docker ps -a | grep o2r | awk '{print $1}' | xargs docker rm --force
- docker images | grep o2r_refimpl | awk '{print $3}' | xargs docker rmi --force
+ docker ps -a | grep o2r | awk '{print $$1}' | xargs sh -c "docker rm --force || true"
+ docker images | grep o2r_refimpl | awk '{print $$3}' | xargs sh -c "docker rmi --force || true"
build_documentation:
rm -f *.pdf
From bb39b9ad807c21672430164546567a90e7f7a68b Mon Sep 17 00:00:00 2001
From: nuest <daniel.nuest@uni-muenster.de>
Date: Fri, 14 Dec 2018 14:53:06 +0100
Subject: [PATCH 44/53] log more versions
---
Makefile | 6 ++++++
1 file changed, 6 insertions(+)
diff --git a/Makefile b/Makefile
index b35a289..390f74d 100644
--- a/Makefile
+++ b/Makefile
@@ -139,8 +139,14 @@ create_archive:
zip -r -q o2r-reference-implementation-files.zip etc/ README*.md docker-compose*.yml versions.txt;
versions:
+ make --version;
git --version;
docker --version;
+ docker-compose --version;
+ pigz --version;
+ python --version;
+ unzip --help | head -1;
+ zip --help | head -2 | tail -1;
release_clean:
rm -f *.zip
From 6fbf7fab949d2fd70c1cbcdf4dd85c4a206e4808 Mon Sep 17 00:00:00 2001
From: nuest <daniel.nuest@uni-muenster.de>
Date: Fri, 14 Dec 2018 14:54:46 +0100
Subject: [PATCH 45/53] rename release task to "package", reformat README
remove trailing whitespace
---
Makefile | 12 ++++++++++--
README.md | 21 ++++++++++++++-------
2 files changed, 24 insertions(+), 9 deletions(-)
diff --git a/Makefile b/Makefile
index 390f74d..dcf4480 100644
--- a/Makefile
+++ b/Makefile
@@ -148,11 +148,19 @@ versions:
unzip --help | head -1;
zip --help | head -2 | tail -1;
-release_clean:
+package_clean:
rm -f *.zip
rm -f *.tar;
-release: versions update build_documentation local_versions local_build local_save_images create_archive
+package: package_clean \
+ local_clean \
+ versions \
+ update \
+ build_documentation \
+ local_versions_save \
+ local_build \
+ local_save_images \
+ create_archives
upload_files_to_zenodo:
python etc/zenodo_upload.py
diff --git a/README.md b/README.md
index 76313dc..9c5d89a 100644
--- a/README.md
+++ b/README.md
@@ -9,12 +9,16 @@ Please check the code repository metadata or respective README files for documen
### tl;dr
-To quickly get the reference implementation running locally, you can use the ready-to-use images from Docker Hub:
+To quickly get the reference implementation running locally, you can use the ready-to-use images from Docker Hub with
```bash
+git clone https://github.com/o2r-project/reference-implementation.git
+cd reference-implemenation
make hub
```
+or jump ahead the the section [Reproduce](#reproduce) to use the archived reproduction package.
+
### Basics
The o2r reference implementation realises a workflow for publishing reproducible articles using the [Executable Research Compendium](https://github.com/o2r-project/erc-spec) (ERC) and has two broad components: a user interface for demonstrating novel interactions with ERC, a.k.a. the "platform", and a side collection of microservices implementing the [o2r Web API](https://github.com/o2r-project/o2r-web-api), a.k.a. the "reproducibility service".
@@ -48,7 +52,7 @@ All of the software, specification and documentation for the o2r system are avai
This project contains configurations and scripts to make running the o2r reference implementation as easy as possible.
-The project uses [`make`](https://www.gnu.org/software/make/), which helps simplifying the execution of the various steps necessary to run the reference implementation into a set of console commands.
+The project uses [`make`](https://www.gnu.org/software/make/), which helps simplifying the execution of the various steps necessary to run the reference implementation into a set of console commands.
These commands are formulated in the `Makefile` included in this project.
If `make` is not available, you can execute the respective commands manually.
@@ -84,7 +88,8 @@ Run `make update` or the respective commands on your operating system to initial
By default, the reference implementation uses the offline OAuth2 implementation provided by the [o2r-guestlister](https://github.com/o2r-project/o2r-guestlister).
-This allows access to the o2r platform by selecting one of three demo users. The users represent different user roles with different levels, i.e. an admin (level `1000`), an editor (level `500`) and a basic author (level `100`).
+This allows access to the o2r platform by selecting one of three demo users.
+The users represent different user roles with different levels, i.e. an admin (level `1000`), an editor (level `500`) and a regular author (level `100`).
##### ORCID (optional)
@@ -95,7 +100,9 @@ This requires an ORCID account which provides authentication tokens for public A
In the developer tools, use any name, website URL, and description.
Important is the `Redirect URIs` list, which must include `http://localhost` for your local installation.
-The client ID, client secret, redirect URI and the OAuth URLs have to be provided by modifying the `.env` file in the base directory. Note that environment variables provided throught the shell have priority over the `.env` file configuration. For more information on how the `.env` file works, see the docker-compose [documentation](https://docs.docker.com/compose/env-file/).
+The client ID, client secret, redirect URI and the OAuth URLs have to be provided by modifying the `.env` file in the base directory.
+Note that environment variables provided through the shell have priority over the `.env` file configuration.
+For more information on how the `.env` file works, see the [`docker-compose` documentation](https://docs.docker.com/compose/env-file/).
##### Repositories (optional)
@@ -124,7 +131,7 @@ You can find instructions for all hosts (including Docker Toolbox) in the [Elast
### Build images from source and run
-This repository already includes a `.gitmodules` file, which lists all required o2r microservices and tools as git submodules.
+This repository already includes a `.gitmodules` file, which lists all required o2r microservices and tools as git submodules.
To download all o2r source code at once, navigate to the `reference-implementation` base directory and use
```bash
@@ -220,8 +227,8 @@ If the `local` configuration is used and the documentation was build, or if you
### Rationale
This repository serves the goal to make the developments of the o2r project reproducible, not only by running the reference implementation (see above) but also by creating an archivable package of the software in a reproducible (i.e. scripted) way.
-As the implementations component are spread over multiple git repositories, we cannot rely on the direct export of a GitHub release to Zenodo.
-This repository captures all software projects to create a release package, which can be uploaded to a data repository manually.
+As the implementation components are spread over multiple git repositories and rely on container images for reproduction, we cannot use the direct export of a GitHub release to Zenodo.
+This repository captures all software projects to create a reproduction package, which can be uploaded to a data repository manually.
### Known limitations
From 86a484cf10af7bcc00bfc456d98bbf744defc9b1 Mon Sep 17 00:00:00 2001
From: nuest <daniel.nuest@uni-muenster.de>
Date: Fri, 14 Dec 2018 14:55:24 +0100
Subject: [PATCH 46/53] package .env file as well
---
Makefile | 6 +++---
README.md | 20 +++++++++++---------
2 files changed, 14 insertions(+), 12 deletions(-)
diff --git a/Makefile b/Makefile
index dcf4480..52ecff1 100644
--- a/Makefile
+++ b/Makefile
@@ -132,11 +132,11 @@ local_save_images:
#@echo "Inspect tarball manifests with";
#tar -xf o2r-reference-implementation-images.tar.gz manifest.json -O | python -m json.tool;
-
-create_archive:
+
+create_archives:
zip -r -q o2r-reference-implementation-modules.zip containerit/ erc-checker/ o2r-*/;
zip -r -q o2r-docs.zip api/ architecture/ erc-examples/ erc-spec/;
- zip -r -q o2r-reference-implementation-files.zip etc/ README*.md docker-compose*.yml versions.txt;
+ zip -r -q o2r-reference-implementation-files.zip etc/ .env docker-compose*.yml LICENSE versions.txt;
versions:
make --version;
diff --git a/README.md b/README.md
index 9c5d89a..e684c62 100644
--- a/README.md
+++ b/README.md
@@ -240,22 +240,24 @@ This repository captures all software projects to create a reproduction package,
### Create package
-The creation of the reproducibility package consists of the following steps and is relies where possible on Makefile targets.
-The package uses the remote images from Docker Hub at the time of creation
+The creation of the reproducibility package consists of the following steps and relies on `Makefile` targets where possible (see brackets `(make ...)`).
+The package uses the locally built images.
1. Clone the `reference-implementation` repository to an empty directory: `git clone https://github.com/o2r-project/reference-implementation.git`
-1. Create the package with `make release`, which...
- - cleans up potentially existing artifacts (`make local_clean`),
- - prints software versions (`make versions`),
+1. Create the package with `make package`, which...
+ - cleans up existing artifacts (`make package_clean local_clean`),
+ - prints _used_ software versions (git, Docker etc., `make versions`),
- updates the nested projects (`make update`),
- builds all documentation locally (`make build_documentation`),
- - save local version information of software and repositories to single file `versions.txt` (`make local_versions_save`),
+ - saves local version information of software and repositories to single file `versions.txt` (`make local_versions_save`),
- builds all images locally (see [limitations](#known-limitations), `make local_build`),
- saves the just built images into the file `o2r-reference-implementation-images.tar` (`make local_save_images`),
- - saves all nested code repositories with their histories, to `o2r-reference-implementation-modules.zip`
- - saves all nested documentation repositories with their histories, to `o2r-docs.zip`
+ - saves all nested repositories (including their histories)
+ - reference implementation software to `o2r-reference-implementation-modules.zip`
+ - documentation to `o2r-docs.zip`
+ - reproduction package files (everything except `README` files and `Makefile`) to `o2r-reference-implementation-files.zip`
1. Create a new deposit or a new version of the existing deposit on Zenodo
-1. Upload the _small files_ to [Zenodo deposit](https://zenodo.org/api/deposit/depositions/2203844): `ZENODO_DEPOSIT_ID=2203844 ZENODO_ACCESS_TOKEN=xxxxxx make upload_files_to_zenodo` (the used Python scripts needs the module `humanfriendly`)
+1. Upload the _small files_ to the just created Zenodo deposit ([first version](https://zenodo.org/api/deposit/depositions/2203844), the used Python scripts needs the module `humanfriendly`; `ZENODO_DEPOSIT_ID=xxxxxx ZENODO_ACCESS_TOKEN=xxxxxx make upload_files_to_zenodo`)
1. Manually upload the _big file_ to Zenodo deposit ([Zenodo API currently has a 100 MB file limit](http://developers.zenodo.org/#deposition-files))
1. Fill out metadata form on Zenodo and publish using the "Publish" button
From 47f6736dbbcb467a90d1ab4de69a7c6c83f558e7 Mon Sep 17 00:00:00 2001
From: nuest <daniel.nuest@uni-muenster.de>
Date: Fri, 14 Dec 2018 15:39:11 +0100
Subject: [PATCH 47/53] update submodules
---
containerit | 2 +-
versions.txt | 8 ++++----
2 files changed, 5 insertions(+), 5 deletions(-)
diff --git a/containerit b/containerit
index fb107e2..0745231 160000
--- a/containerit
+++ b/containerit
@@ -1 +1 @@
-Subproject commit fb107e2979c48b69336f7019f991f9e6f9900757
+Subproject commit 07452318e0768cea858b74dad2f1e5594dcb2926
diff --git a/versions.txt b/versions.txt
index bb4fb85..2c9cb01 100644
--- a/versions.txt
+++ b/versions.txt
@@ -1,8 +1,8 @@
make[1]: Entering directory '/home/daniel/git/o2r/reference-implementation'
etc/local_versions.sh
-api/ e9bb2b86ecd8fa23dde9a109284dd3506d243184
-architecture/ eaf33f2aa4c5af7dbcdd3de95a9d4196f64da64b
-containerit/ 0.5.0.9003 fb107e2979c48b69336f7019f991f9e6f9900757
+api/ 8d75e2ff1847073b8bc9025bc901e9a2e2cb9b3f
+architecture/ 00756bca5b3a69ce4267149dead6f57d08a4bfce
+containerit/ 0.5.0.9004 07452318e0768cea858b74dad2f1e5594dcb2926
erc-checker/ 1.2.0 fb1ed1617464b862404dc18c3641678543bca470
erc-examples/ d5892550b2d68e858f006e4867de4fae5cc76a11
erc-spec/ a112fd85b26a4659997ac2463ec0aff7315ebd22
@@ -19,5 +19,5 @@ o2r-platform/ 1.0.2 f7075b9be074dee841ab9cced667af6c66eadc3e
o2r-shipper/ 8f95fd6cc2c2071658ddcefa3457390207ff5ffc
o2r-substituter/ 0.5.0 1d5be501c9c2c2a6821ab1caff0c14e4320952dd
o2r-transporter/ 0.6.0 ed9f8be91d7195c3fe14b565d1b51505f963c17c
-. (reference-implementation) 71c5a8184bf0c0cbb232de563eefd7c3b27fb2c3
+. (reference-implementation) 06898083b0e443c55926077aa2dbd1d4c146b197
make[1]: Leaving directory '/home/daniel/git/o2r/reference-implementation'
From 8b94f9b3397a859b8a03e8aa7948461878731aa0 Mon Sep 17 00:00:00 2001
From: nuest <daniel.nuest@uni-muenster.de>
Date: Fri, 14 Dec 2018 19:12:29 +0100
Subject: [PATCH 48/53] update reproduce target and documentation
---
Makefile | 13 +++++++-----
README.md | 63 +++++++++++++++++++++++++++++++------------------------
2 files changed, 44 insertions(+), 32 deletions(-)
diff --git a/Makefile b/Makefile
index 52ecff1..796cb3c 100644
--- a/Makefile
+++ b/Makefile
@@ -23,7 +23,7 @@ init:
git submodule add https://github.com/o2r-project/containerit
update:
- git pull --recurse-submodules
+ git pull
git submodule update --init --recursive --remote
git submodule foreach --recursive git checkout master
git submodule foreach --recursive git pull origin master
@@ -149,8 +149,9 @@ versions:
zip --help | head -2 | tail -1;
package_clean:
- rm -f *.zip
+ rm -f *.zip;
rm -f *.tar;
+ rm -f *.tar.gz;
package: package_clean \
local_clean \
@@ -166,6 +167,8 @@ upload_files_to_zenodo:
python etc/zenodo_upload.py
reproduce:
- # TODO import images from files
- # run make local
- # deploy some examples
+ unzip o2r-docs.zip;
+ o2r-reference-implementation-modules.zip;
+ unzip o2r-reference-implementation-files.zip;
+ docker load --input o2r-reference-implementation-images.tar.gz;
+ docker-compose --file docker-compose-local.yml --project-name reference-implementation up --no-build;
diff --git a/README.md b/README.md
index e684c62..1f5eaed 100644
--- a/README.md
+++ b/README.md
@@ -174,39 +174,40 @@ Wait until the log shows no new messages for a few seconds, then open **http://l
### Load data
-The o2r API supports two way to load scientific workflows: direct upload as a ZIP archive, or import from a publish share.
+You have to **log in** to upload workspaces:
-Ready to use **Direct upload** examples are available in the directory `erc-examples`.
+1. Click on "LOGIN" in the upper right hand corner
+1. You are taken to a local login server: select one of the available users, e.g. "User"
+1. After selecting the account type you are redirected to the o2r platform landing page
-Examples for loading from a public share are available [in this online share](https://uni-muenster.sciebo.de/index.php/s/G8vxQ1h50V4HpuA) (see file `README.txt`) and via the "EXAMPLES" menu in the "Create" section of the platform website.
+The o2r API supports two way to load scientific workflows: direct upload as a ZIP archive, or loading from a publish share.
-### Use platform
+- Ready to use **direct upload examples** are available in the directory `erc-examples`.
+- Examples for **loading from a public share** are available [in this online share](https://uni-muenster.sciebo.de/index.php/s/G8vxQ1h50V4HpuA) (see file `README.txt`) and via the "EXAMPLES" menu in the "Create" section of the platform website.
-1. Click on "LOGIN" in the upper right hand corner
-1. Select one of the available users, e.g. "Author"
-1. In the "Create" section of the platform website, select a workspace from the "EXAMPLES" menu
-1. Fill in required metadata
-1. Finish the upload and open the ERC page: explore the running job and all interaction possiblities
+After filling in required metadata you can publish the ERC.
+Then open the ERC page and start a new job.
+Check the job results and explore the interaction possibilities
### Explore back-end
#### Database administration
-An [adminMongo](https://adminmongo.markmoffat.com/) instance is included in the reference implementation. Open it at http://localhost:1234. In mongoAdmin please manually create a connection to host db, i.e. mongodb://db:27017 to edit the database (click "Update" first if you edit the existing connection, then "Connect").
+An [adminMongo](https://adminmongo.markmoffat.com/) instance is included in the reference implementation.
+Open it at http://localhost:1234.
+In mongoAdmin manually create a connection to host database:
-The docker compose configuration includes an adminMongo instance. Open it at http://localhost:1234, create a new connection with the following settings:
-
-- Connection name: any name
+- Connection name: any name, e.g. `o2r`
- Connection string: `mongodb://mongodb:27017` (which is the default port and the host `mongodb`, no password)
-- Connection options: `{}` (empty/default)
+- Connection options: leave to `{}` (empty/default)
-Click on "Add connection", then on "Connections" at the top right corner, and then "Connect" using the just created connection.
+Wait a few moments for the interface to refresh, then click "Connect" in the line of the just created connection to explore the main database.
#### File storage
The configurations all use a common [Docker volume](https://docs.docker.com/engine/admin/volumes/volumes/) `o2rvol` with the global name `referenceimplementation_o2rvol`.
-The volume and network can be inspected for development purposes:
+The volume can be inspected for development purposes:
```bash
docker volume ls
@@ -235,8 +236,10 @@ This repository captures all software projects to create a reproduction package,
- Nested code projects are currently installed from GitHub during the building of Docker images ((see also [#1](#1)).
- `o2r-muncher`'s Docker image contains `o2r-meta` and `erc-checker`
- `o2r-loaders`'s Docker image contains `o2r-meta`
-- The used Docker base images and dependencies for the services, such as npm or pip packages, must be available online in the required version to build the images locally. See project files such as `package.json` or `requirements.txt` for a full list of dependencies.
-- The locally built images do not have a proper version tag but instead are `latest`, since they are build as part of the `docker-compose` configuration. Note that you can easily distinguish images by this, too: The hub images will always have a specific version tag.
+- The used Docker base images and dependencies for the services, such as npm or pip packages, must be available online in the required version to build the images locally.
+ See project files such as `package.json` or `requirements.txt` for a full list of dependencies.
+- The locally built images do not have a proper version tag but instead are `latest`, since they are build as part of the `docker-compose` configuration.
+ Note that you can easily distinguish images by this, too: The hub images will always have a specific version tag.
### Create package
@@ -263,13 +266,16 @@ The package uses the locally built images.
### Reproduce
-The reproduction consists of three steps, assuming the local images do not exist on the machine used:
+The reproduction consists of the following steps:
-1. Download the saved images from Zenodo
-1. Load the saved images from the tarball
-1. Run the `docker-compose` configuration without building the images (to ensure the loaded ones are used)
+1. Install [Docker CE](https://docs.docker.com/install/) and [`docker-compose`](https://docs.docker.com/compose/install/)
+1. Download all files of the reproduction package from Zenodo: https://10.5281/zenodo.2203844
+1. Unzip the file `o2r-docs.zip` and `o2r-reference-implementation-files.zip` - they contain documentation you might want to explore, and required configuration files and examples respectively.
+1. Unzip the file `o2r-reference-implementation-modules.zip` - it contains the source code of all software and [the directories are needed for `docker-compose`](https://github.com/docker/compose/issues/3391) even though they are not used
+1. Load the saved images from the tarball using `docker load`.
+1. Run the reference implementation with `docker-compose up` using the configuration file `docker-compose-local.yml` and making sure the loaded images are used with the option `--no-build` (i.e. not building the images)
-The last two steps can be executed with
+All except the first two steps can be executed with
```bash
make reproduce
@@ -277,16 +283,19 @@ make reproduce
Wait until the log shows no new messages for a few seconds, then open **http://localhost** and continue see section ["Load data"](#load-data) for creating ERCs.
-You can stop all containers and clean up local images with
+You can stop all containers with `Ctrl+C` and clean up local images and files with
```bash
make local_down
-# to also delete local storage use make local_down_volumes
+# to also delete local storage use: make local_down_volumes
make local_clean
```
+Please check the [Docker documentation](https://docs.docker.com/) if any problems arise and feel free to contact the [o2r team](https://o2r.info/about/#team).
+
## License
-This project is licensed under Apache License, Version 2.0, see file LICENSE. Copyright © 2018 - o2r project.
+This project is licensed under Apache License, Version 2.0, see file LICENSE.
+Copyright © 2018 - o2r project.
-All included software projects have their own LICENSE files, see `<component name>/LICENSE`.
+All included software projects and data files have their respective and potentially differing licenses, see e.g. `<component name>/LICENSE`.
From dec19235039e553c9564aeb3aebce96106bf957e Mon Sep 17 00:00:00 2001
From: nuest <daniel.nuest@uni-muenster.de>
Date: Fri, 14 Dec 2018 19:13:09 +0100
Subject: [PATCH 49/53] update microservice submodules
---
docker-compose.yml | 4 ++--
o2r-bouncer | 2 +-
o2r-loader | 2 +-
o2r-platform | 2 +-
4 files changed, 5 insertions(+), 5 deletions(-)
diff --git a/docker-compose.yml b/docker-compose.yml
index cc43b72..27ded56 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -85,7 +85,7 @@ services:
- "MUNCHER_META_TOOL_CONTAINER=o2rproject/o2r-meta:9198450"
loader:
- image: o2rproject/o2r-loader:0.11.0
+ image: o2rproject/o2r-loader:0.12.0
depends_on:
- configmongodb
volumes:
@@ -109,7 +109,7 @@ services:
- DEBUG=informer,informer:*
bouncer:
- image: o2rproject/o2r-bouncer:0.12.1
+ image: o2rproject/o2r-bouncer:0.13.0
depends_on:
- configmongodb
- guestlister
diff --git a/o2r-bouncer b/o2r-bouncer
index 6bc703d..0c99901 160000
--- a/o2r-bouncer
+++ b/o2r-bouncer
@@ -1 +1 @@
-Subproject commit 6bc703decd3d5dfc49e7935a37da625dd0921c94
+Subproject commit 0c99901a7f0f6e633866d1eaca4f55c95cce6aa3
diff --git a/o2r-loader b/o2r-loader
index d83498b..a12115b 160000
--- a/o2r-loader
+++ b/o2r-loader
@@ -1 +1 @@
-Subproject commit d83498b8d71be6999e8ff2e3a8c13c6574de11a9
+Subproject commit a12115b47ab3a4c931efc4ed8d54a1deeb42606a
diff --git a/o2r-platform b/o2r-platform
index f7075b9..9673a91 160000
--- a/o2r-platform
+++ b/o2r-platform
@@ -1 +1 @@
-Subproject commit f7075b9be074dee841ab9cced667af6c66eadc3e
+Subproject commit 9673a91bda45ce3257b2c8595c20fcbcdc246e94
From fd3068c2e0c7096077a99f8a50801e0dcea16ad1 Mon Sep 17 00:00:00 2001
From: nuest <daniel.nuest@uni-muenster.de>
Date: Fri, 14 Dec 2018 20:08:31 +0100
Subject: [PATCH 50/53] update submodules
---
o2r-platform | 2 +-
versions.txt | 8 ++++----
2 files changed, 5 insertions(+), 5 deletions(-)
diff --git a/o2r-platform b/o2r-platform
index 9673a91..806c795 160000
--- a/o2r-platform
+++ b/o2r-platform
@@ -1 +1 @@
-Subproject commit 9673a91bda45ce3257b2c8595c20fcbcdc246e94
+Subproject commit 806c7952b86b74fae20e631f8a29bf1e1a086e1c
diff --git a/versions.txt b/versions.txt
index 2c9cb01..0333e6b 100644
--- a/versions.txt
+++ b/versions.txt
@@ -7,17 +7,17 @@ erc-checker/ 1.2.0 fb1ed1617464b862404dc18c3641678543bca470
erc-examples/ d5892550b2d68e858f006e4867de4fae5cc76a11
erc-spec/ a112fd85b26a4659997ac2463ec0aff7315ebd22
o2r-bindings/ 0.0.1 f68e17d12dc7573bd7c6b927d444ef35b09f14fc
-o2r-bouncer/ 0.12.2 6bc703decd3d5dfc49e7935a37da625dd0921c94
+o2r-bouncer/ 0.13.0 0c99901a7f0f6e633866d1eaca4f55c95cce6aa3
o2r-finder/ 0.6.3 97029b6322223e5e4677e03f41d2b1b375634f5c
o2r-guestlister/ 0.2.2 fe351833fa05455b99b5644d26c944ae6500309f
o2r-informer/ 0.5.0 cd507af98a12c2544148145c2cffe07853d909f7
o2r-inspecter/ 0.0.1 ee5e86ada517972b246f8297ddb8b43580e3eac1
-o2r-loader/ 0.11.0 d83498b8d71be6999e8ff2e3a8c13c6574de11a9
+o2r-loader/ 0.12.0 a12115b47ab3a4c931efc4ed8d54a1deeb42606a
o2r-meta/ f79cc1f838ce8a6151cd10967e497b4c9040e79d
o2r-muncher/ 0.21.1 c68b4d369a52082ef764b0d576c7964258e0fd63
-o2r-platform/ 1.0.2 f7075b9be074dee841ab9cced667af6c66eadc3e
+o2r-platform/ 1.0.2 806c7952b86b74fae20e631f8a29bf1e1a086e1c
o2r-shipper/ 8f95fd6cc2c2071658ddcefa3457390207ff5ffc
o2r-substituter/ 0.5.0 1d5be501c9c2c2a6821ab1caff0c14e4320952dd
o2r-transporter/ 0.6.0 ed9f8be91d7195c3fe14b565d1b51505f963c17c
-. (reference-implementation) 06898083b0e443c55926077aa2dbd1d4c146b197
+. (reference-implementation) 48efaca2b61c756153dc1fcdfb7fe27d349a8781
make[1]: Leaving directory '/home/daniel/git/o2r/reference-implementation'
From 44ba5c41891777e6507171a56bdabb6acb1c9f61 Mon Sep 17 00:00:00 2001
From: nuest <daniel.nuest@uni-muenster.de>
Date: Fri, 14 Dec 2018 22:49:34 +0100
Subject: [PATCH 51/53] upload specs as pdfs
---
etc/zenodo_upload.py | 3 +++
1 file changed, 3 insertions(+)
diff --git a/etc/zenodo_upload.py b/etc/zenodo_upload.py
index c8df9a1..6e4d716 100644
--- a/etc/zenodo_upload.py
+++ b/etc/zenodo_upload.py
@@ -83,6 +83,9 @@ def new_version(deposit_id):
files = ['README.md',
'README-WIN.md',
'Makefile',
+ 'erc-spec.pdf',
+ 'o2r-architecture.pdf',
+ 'o2r-web-api.pdf'
'o2r-docs.zip',
'o2r-reference-implementation-files.zip',
#'o2r-reference-implementation-images.tar.gz',
From 7225de22641fca6d5c763687eba80aa2657ac097 Mon Sep 17 00:00:00 2001
From: nuest <daniel.nuest@uni-muenster.de>
Date: Mon, 17 Dec 2018 14:30:46 +0100
Subject: [PATCH 52/53] add missing unzip command
---
Makefile | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/Makefile b/Makefile
index 796cb3c..480da62 100644
--- a/Makefile
+++ b/Makefile
@@ -168,7 +168,7 @@ upload_files_to_zenodo:
reproduce:
unzip o2r-docs.zip;
- o2r-reference-implementation-modules.zip;
+ unzip o2r-reference-implementation-modules.zip;
unzip o2r-reference-implementation-files.zip;
docker load --input o2r-reference-implementation-images.tar.gz;
docker-compose --file docker-compose-local.yml --project-name reference-implementation up --no-build;
From 3b9d3bc56cda36265b79bba77c9462c70cbc491c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20N=C3=BCst?= <daniel.nuest@wwu.de>
Date: Tue, 27 Jul 2021 12:22:17 +0200
Subject: [PATCH 53/53] add Zenodo badge to README
---
README.md | 2 ++
1 file changed, 2 insertions(+)
diff --git a/README.md b/README.md
index 1f5eaed..17a1066 100644
--- a/README.md
+++ b/README.md
@@ -1,5 +1,7 @@
# o2r reference implementation
+[](https://doi.org/10.5281/zenodo.2203843)
+
A catch-all repository to run all [o2r](https://o2r.info) microservices and the user interface platform which together form the [o2r prototype for a reproducibility service and publishing system](https://o2r.info/results).
In the remainder of this document, specifications or projects are referenced with their code repository, even if a user-friendly online HTML rending exists, e.g. https://github.com/o2r-project/erc-spec/ instead of https://o2r.info/erc-spec/.