diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..f59ec20 --- /dev/null +++ b/.dockerignore @@ -0,0 +1 @@ +* \ No newline at end of file diff --git a/.env b/.env new file mode 100644 index 0000000..cf71683 --- /dev/null +++ b/.env @@ -0,0 +1,4 @@ +OAUTH_URL_AUTHORIZATION=http://localhost/oauth/authorize +OAUTH_URL_TOKEN=http://webserver/oauth/token +OAUTH_URL_CALLBACK=http://localhost/api/v1/auth/login +SHIPPER_REPO_TOKENS={"download": "" } \ No newline at end of file diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..03987b7 --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +*.pdf + +o2r-reference-implementation*\.tar\.gz +o2r-reference-implementation*\.zip +o2r-docs\.zip diff --git a/.gitmodules b/.gitmodules index ba79547..cdc6edc 100644 --- a/.gitmodules +++ b/.gitmodules @@ -22,9 +22,9 @@ [submodule "o2r-platform"] path = o2r-platform url = https://github.com/o2r-project/o2r-platform -[submodule "o2r-web-api"] - path = o2r-web-api - url = https://github.com/o2r-project/o2r-web-api +[submodule "api"] + path = api + url = https://github.com/o2r-project/api [submodule "erc-spec"] path = erc-spec url = https://github.com/o2r-project/erc-spec @@ -43,3 +43,15 @@ [submodule "erc-examples"] path = erc-examples url = https://github.com/o2r-project/erc-examples +[submodule "o2r-inspecter"] + path = o2r-inspecter + url = https://github.com/o2r-project/o2r-inspecter +[submodule "o2r-guestlister"] + path = o2r-guestlister + url = https://github.com/o2r-project/o2r-guestlister +[submodule "o2r-bindings"] + path = o2r-bindings + url = https://github.com/o2r-project/o2r-bindings +[submodule "containerit"] + path = containerit + url = https://github.com/o2r-project/containerit diff --git a/Makefile b/Makefile index 33cf46a..480da62 100644 --- a/Makefile +++ b/Makefile @@ -1,13 +1,16 @@ init: git clone https://github.com/o2r-project/reference-implementation cd reference-implementation + git submodule add https://github.com/o2r-project/api git submodule add https://github.com/o2r-project/architecture + git submodule add https://github.com/o2r-project/containerit git submodule add https://github.com/o2r-project/erc-spec git submodule add https://github.com/o2r-project/erc-checker git submodule add https://github.com/o2r-project/erc-examples git submodule add https://github.com/o2r-project/o2r-bouncer git submodule add https://github.com/o2r-project/o2r-finder git submodule add https://github.com/o2r-project/o2r-informer + git submodule add https://github.com/o2r-project/o2r-inspecter git submodule add https://github.com/o2r-project/o2r-loader git submodule add https://github.com/o2r-project/o2r-meta git submodule add https://github.com/o2r-project/o2r-muncher @@ -15,77 +18,157 @@ init: git submodule add https://github.com/o2r-project/o2r-shipper git submodule add https://github.com/o2r-project/o2r-substituter git submodule add https://github.com/o2r-project/o2r-transporter - git submodule add https://github.com/o2r-project/o2r-web-api + git submodule add https://github.com/o2r-project/o2r-guestlister + git submodule add https://github.com/o2r-project/o2r-bindings + git submodule add https://github.com/o2r-project/containerit update: - git submodule update --init --recursive - git submodule foreach git pull origin master - -build_images: - cd o2r-bouncer; docker build --tag o2r_refimpl_bouncer .; cd ..; - cd o2r-finder; docker build --tag o2r_refimpl_finder .; cd ..; - cd o2r-informer; docker build --tag o2r_refimpl_informer .; cd ..; - cd o2r-loader; docker build --tag o2r_refimpl_loader .; cd ..; - cd o2r-muncher; docker build --tag o2r_refimpl_muncher .; cd ..; - cd o2r-platform; docker build --tag o2r_refimpl_platform .; cd ..; - cd o2r-shipper; docker build --tag o2r_refimpl_shipper .; cd ..; - cd o2r-substituter; docker build --tag o2r_refimpl_substituter .; cd ..; - cd o2r-transporter; docker build --tag o2r_refimpl_transporter .; cd ..; - -run_local: - OAUTH_CLIENT_ID=$(value O2R_ORCID_ID) OAUTH_CLIENT_SECRET=$(value O2R_ORCID_SECRET) OAUTH_URL_CALLBACK=$(value O2R_ORCID_CALLBACK) ZENODO_TOKEN=$(value O2R_ZENODO_TOKEN) docker-compose --file docker-compose-local.yml up; - -stop_local: + git pull + git submodule update --init --recursive --remote + git submodule foreach --recursive git checkout master + git submodule foreach --recursive git pull origin master + +local_build: + docker-compose --file docker-compose-local.yml build; + +local_up: + docker-compose --file docker-compose-local.yml up; + +local_down: docker-compose --file docker-compose-local.yml down; -show_versions_local: - @docker inspect --format '{{index .Config.Labels "org.label-schema.name"}}: {{index .Config.Labels "org.label-schema.version"}}' o2r_refimpl_bouncer ; - @docker inspect --format '{{index .Config.Labels "org.label-schema.name"}}: {{index .Config.Labels "org.label-schema.version"}}' o2r_refimpl_finder ; - @docker inspect --format '{{index .Config.Labels "org.label-schema.name"}}: {{index .Config.Labels "org.label-schema.version"}}' o2r_refimpl_informer ; - @docker inspect --format '{{index .Config.Labels "org.label-schema.name"}}: {{index .Config.Labels "org.label-schema.version"}}' o2r_refimpl_loader ; - @docker inspect --format '{{index .Config.Labels "org.label-schema.name"}}: {{index .Config.Labels "org.label-schema.version"}}' o2r_refimpl_muncher ; - @docker inspect --format '{{index .Config.Labels "org.label-schema.name"}}: {{index .Config.Labels "org.label-schema.version"}}' o2r_refimpl_platform ; - @docker inspect --format '{{index .Config.Labels "org.label-schema.name"}}: {{index .Config.Labels "org.label-schema.vcs-ref"}}' o2r_refimpl_shipper ; - @docker inspect --format '{{index .Config.Labels "org.label-schema.name"}}: {{index .Config.Labels "org.label-schema.version"}}' o2r_refimpl_substituter ; - @docker inspect --format '{{index .Config.Labels "org.label-schema.name"}}: {{index .Config.Labels "org.label-schema.version"}}' o2r_refimpl_transporter ; +local_down_volume: + docker-compose --file docker-compose-local.yml down --volume; + +local: update local_build local_up -run_hub: pull_hub_images run_hub_images +local_versions: + etc/local_versions.sh -stop_hub: - docker-compose down; +local_versions_save: local_versions + rm -f versions.txt + make local_versions >> versions.txt -pull_hub_images: +hub_images: + # pull ":latest" images so that we don't need to update versions here as well docker pull o2rproject/o2r-bouncer; docker pull o2rproject/o2r-finder; docker pull o2rproject/o2r-informer; + docker pull o2rproject/o2r-inspecter; docker pull o2rproject/o2r-loader; + docker pull o2rproject/o2r-meta; docker pull o2rproject/o2r-muncher; docker pull o2rproject/o2r-platform; docker pull o2rproject/o2r-shipper; docker pull o2rproject/o2r-substituter; docker pull o2rproject/o2r-transporter; + docker pull o2rproject/o2r-guestlister; + docker pull o2rproject/o2r-bindings; -run_hub_images: - OAUTH_CLIENT_ID=$(value O2R_ORCID_ID) OAUTH_CLIENT_SECRET=$(value O2R_ORCID_SECRET) OAUTH_URL_CALLBACK=$(value O2R_ORCID_CALLBACK) ZENODO_TOKEN=$(value O2R_ZENODO_TOKEN) docker-compose up; - -show_versions_hub: +hub_versions: @docker inspect --format '{{index .Config.Labels "org.label-schema.name"}}: {{index .Config.Labels "org.label-schema.version"}}' o2rproject/o2r-bouncer; @docker inspect --format '{{index .Config.Labels "org.label-schema.name"}}: {{index .Config.Labels "org.label-schema.version"}}' o2rproject/o2r-finder; @docker inspect --format '{{index .Config.Labels "org.label-schema.name"}}: {{index .Config.Labels "org.label-schema.version"}}' o2rproject/o2r-informer; + @docker inspect --format '{{index .Config.Labels "org.label-schema.name"}}: {{index .Config.Labels "org.label-schema.version"}}' o2rproject/o2r-inspecter; @docker inspect --format '{{index .Config.Labels "org.label-schema.name"}}: {{index .Config.Labels "org.label-schema.version"}}' o2rproject/o2r-loader; + @docker inspect --format '{{index .Config.Labels "org.label-schema.name"}}: {{index .Config.Labels "org.label-schema.version"}}' o2rproject/o2r-meta; @docker inspect --format '{{index .Config.Labels "org.label-schema.name"}}: {{index .Config.Labels "org.label-schema.version"}}' o2rproject/o2r-muncher; @docker inspect --format '{{index .Config.Labels "org.label-schema.name"}}: {{index .Config.Labels "org.label-schema.version"}}' o2rproject/o2r-platform; @docker inspect --format '{{index .Config.Labels "org.label-schema.name"}}: {{index .Config.Labels "org.label-schema.vcs-ref"}}' o2rproject/o2r-shipper; @docker inspect --format '{{index .Config.Labels "org.label-schema.name"}}: {{index .Config.Labels "org.label-schema.version"}}' o2rproject/o2r-substituter; @docker inspect --format '{{index .Config.Labels "org.label-schema.name"}}: {{index .Config.Labels "org.label-schema.version"}}' o2rproject/o2r-transporter; + @docker inspect --format '{{index .Config.Labels "org.label-schema.name"}}: {{index .Config.Labels "org.label-schema.version"}}' o2rproject/o2r-guestlister; + @docker inspect --format '{{index .Config.Labels "org.label-schema.name"}}: {{index .Config.Labels "org.label-schema.version"}}' o2rproject/o2r-bindings; + @docker inspect --format '{{index .Config.Labels "org.label-schema.name"}}: {{index .Config.Labels "org.label-schema.version"}}' o2rproject/containerit; + +hub_up: + docker-compose up; + +hub: hub_images hub_versions hub_up + +hub_down_volume: + docker-compose down --volume; + +hub_clean: hub_down_volume + docker ps -a | grep o2r | awk '{print $$1}' | xargs sh -c "docker rm --force || true" + docker images | grep o2r | awk '{print $$3}' | xargs sh -c "docker rmi --force || true" + +local_clean: local_down_volume + docker ps -a | grep o2r | awk '{print $$1}' | xargs sh -c "docker rm --force || true" + docker images | grep o2r_refimpl | awk '{print $$3}' | xargs sh -c "docker rmi --force || true" + +build_documentation: + rm -f *.pdf + docker build --tag o2r_refimpl_docbuilder --file etc/Dockerfile.documentations . + docker run -it -v $(CURDIR)/architecture:/doc:rw o2r_refimpl_docbuilder make build pdf + docker run -it -v $(CURDIR)/api:/doc:rw o2r_refimpl_docbuilder make build pdf + docker run -it -v $(CURDIR)/erc-spec:/doc:rw o2r_refimpl_docbuilder make build travis_pdf + mv architecture/site/o2r-architecture.pdf . + mv erc-spec/site/erc-spec.pdf . + mv api/o2r-web-api.pdf . + @echo "ERC, architecture, and web API documentation created, see files PDF files in the project root directory" + +local_save_images: + docker save \ + mongo:3.6 \ + adicom/admin-mongo:latest \ + docker.elastic.co/elasticsearch/elasticsearch:5.6.10 \ + o2r_refimpl_containerit \ + o2r_refimpl_meta \ + o2r_refimpl_muncher \ + o2r_refimpl_loader \ + o2r_refimpl_informer \ + o2r_refimpl_bouncer \ + o2r_refimpl_finder \ + o2r_refimpl_transporter \ + o2r_refimpl_shipper \ + o2r_refimpl_substituter \ + o2r_refimpl_inspecter \ + o2r_refimpl_bindings \ + o2r_refimpl_guestlister \ + o2r_refimpl_platform \ + nginx:latest | pigz --stdout --fast > o2r-reference-implementation-images.tar.gz; + ls -1sh o2r-reference-implementation-images*.tar.gz; + +#@echo "Inspect tarball manifests with"; +#tar -xf o2r-reference-implementation-images.tar.gz manifest.json -O | python -m json.tool; + +create_archives: + zip -r -q o2r-reference-implementation-modules.zip containerit/ erc-checker/ o2r-*/; + zip -r -q o2r-docs.zip api/ architecture/ erc-examples/ erc-spec/; + zip -r -q o2r-reference-implementation-files.zip etc/ .env docker-compose*.yml LICENSE versions.txt; + +versions: + make --version; + git --version; + docker --version; + docker-compose --version; + pigz --version; + python --version; + unzip --help | head -1; + zip --help | head -2 | tail -1; -clean: - docker ps -a | grep o2r | awk '{print $1}' | xargs docker rm -f - docker images | grep o2r | awk '{print $3}' | xargs docker rmi --force +package_clean: + rm -f *.zip; + rm -f *.tar; + rm -f *.tar.gz; -release: - git clone --recursive https://github.com/o2r-project/reference-implementation - # TODO build all images, export them to files +package: package_clean \ + local_clean \ + versions \ + update \ + build_documentation \ + local_versions_save \ + local_build \ + local_save_images \ + create_archives +upload_files_to_zenodo: + python etc/zenodo_upload.py + reproduce: - # TODO import images from files + unzip o2r-docs.zip; + unzip o2r-reference-implementation-modules.zip; + unzip o2r-reference-implementation-files.zip; + docker load --input o2r-reference-implementation-images.tar.gz; + docker-compose --file docker-compose-local.yml --project-name reference-implementation up --no-build; diff --git a/README-WIN.md b/README-WIN.md index d9a0c06..5fb52a2 100644 --- a/README-WIN.md +++ b/README-WIN.md @@ -3,33 +3,32 @@ This document contains Windows-specific configuration and steps. **Please carefully read the sections "Basics" and "Prerequisites" in the file `README.md` for general information. -In the sections "Build images from source and run" and "Build images from source and run", please be aware of the general remarks but use the commands from this file instead. -Take a look at the "Troubleshooting" sections in this file if you run into problems. +In the sections "Build images from source and run" and "Build images from source and run", please be aware of the general remarks but use the `docker-compose` command from this file instead of the commands using `make`. +Take a look at the "Troubleshooting" sections in this file if you run into problems.** ## Windows with Docker for Windows [Docker for Windows](https://docs.docker.com/docker-for-windows/) is available for 64bit Windows 10 Pro, Enterprise and Education (with Hyper-V available) and on Windows Server 2016 (see [Docker Docs](https://docs.docker.com/docker-for-windows/install/#what-to-know-before-you-install)). -The environmental variables must be passed separately on Windows, followed by the docker-compose commands: +Simply run the following command to start the services: -```powershell -$env:OAUTH_CLIENT_ID = "<... required ...>" -$env:OAUTH_CLIENT_SECRET = "<... required ...>" -$env:OAUTH_URL_CALLBACK = "http://localhost/api/v1/auth/login" -$env:ZENODO_TOKEN = "<... optional ...>" -docker-compose.exe up +```shell +docker-compose up ``` The services are available at `http://localhost`. +Advanced configuration can be applied by editing the `.env` file or by specifying the settings as environment variables before executing the `docker-compose up` command. + ### Troubleshooting - Make sure that the drive where you cloned this repository to is shared (Docker > Shared Drives), because a configuration file (`nginx.conf`) must be mounted into the service `webserver`. - Reconsider using a Linux virtual machine. - Install the latest _edge_ channel of Docker, see https://docs.docker.com/docker-for-windows/faqs/#questions-about-stable-and-edge-channels - Make sure that mounting files into containers works. This thread contains many potentially useful hints and ideas: https://forums.docker.com/t/volume-mounts-in-windows-does-not-work/10693/4 +- If you want to set environment variables in PowerShell, use this syntax `$env:VARIABLENAME = "VARIABLEVALUE"` - Inspect your environment variables in PowerShell with `Get-ChildItem Env:` - +- If you encounter problems with docker-compose, you might want to install python 2.7, especially when a newer python distribution is installed on your machine ## Windows with Docker Toolbox @@ -37,10 +36,10 @@ If your system does not meet the requirements to run Docker for Windows, you can When using Compose with Docker Toolbox/Machine on Windows, [volume paths are no longer converted from by default](https://github.com/docker/compose/releases/tag/1.9.0), but we need this conversion to be able to mount the docker volume to the o2r microservices. To re-enable this conversion for `docker-compose >= 1.9.0` set the environment variable `COMPOSE_CONVERT_WINDOWS_PATHS=1`. -Also, the client's defaults (i.e. using `localhost`) does not work. We must mount a config file to point the API to the correct location, see `win/config-toolbox.js`, and use the prepared configuration file `win/docker-compose-toolbox.yml`. +Add `COMPOSE_CONVERT_WINDOWS_PATHS=1` to the `.env` file and run: ```bash -COMPOSE_CONVERT_WINDOWS_PATHS=1 OAUTH_CLIENT_ID=<...> OAUTH_CLIENT_SECRET=<...> OAUTH_URL_CALLBACK=<...> ZENODO_TOKEN=<...> docker-compose up +docker-compose up ``` -The services are available at `http://<machine-ip>`. \ No newline at end of file +The services are available at `http://<machine-ip>`. diff --git a/README.md b/README.md index 1c0c29c..17a1066 100644 --- a/README.md +++ b/README.md @@ -1,24 +1,40 @@ # o2r reference implementation -A catch-all repository to run all [o2r](http://o2r.info) microservices and the user interface platform. +[](https://doi.org/10.5281/zenodo.2203843) + +A catch-all repository to run all [o2r](https://o2r.info) microservices and the user interface platform which together form the [o2r prototype for a reproducibility service and publishing system](https://o2r.info/results). + +In the remainder of this document, specifications or projects are referenced with their code repository, even if a user-friendly online HTML rending exists, e.g. https://github.com/o2r-project/erc-spec/ instead of https://o2r.info/erc-spec/. +Please check the code repository metadata or respective README files for documentation that is easier to read or browse. ## Run o2r platform and reproducibility service +### tl;dr + +To quickly get the reference implementation running locally, you can use the ready-to-use images from Docker Hub with + +```bash +git clone https://github.com/o2r-project/reference-implementation.git +cd reference-implemenation +make hub +``` + +or jump ahead the the section [Reproduce](#reproduce) to use the archived reproduction package. + ### Basics The o2r reference implementation realises a workflow for publishing reproducible articles using the [Executable Research Compendium](https://github.com/o2r-project/erc-spec) (ERC) and has two broad components: a user interface for demonstrating novel interactions with ERC, a.k.a. the "platform", and a side collection of microservices implementing the [o2r Web API](https://github.com/o2r-project/o2r-web-api), a.k.a. the "reproducibility service". -For more information on the architecture and the microservices see [o2r Software Architecture](https://github.com/o2r-project/architecture). +For more information on the architecture and the microservices see [o2r System Architecture](https://github.com/o2r-project/architecture). ### Contents -All of the mentioned documentation and software is available in this project. -The documentation is also available online for reading, though availability may change. +All of the software, specification and documentation for the o2r system are available nested in this project. -| Component | directory | online | +| **Component** | **Directory | **Online** | | ------ | ------ | ------ | -| ERC specification (document) | `erc-spec` | http://o2r.info/erc-spec/ | -| Web API specification (document) | `o2r-web-api` | https://github.com/o2r-project/o2r-web-api | +| ERC specification (document) | `erc-spec` | https://github.com/o2r-project/erc-spec/ | +| Web API specification (document) | `api` | https://github.com/o2r-project/api | | Architecture specification (document) | `architecture` | https://github.com/o2r-project/architecture | | ERC checker (library/tool) | `erc-checker` | https://github.com/o2r-project/erc-checker/ | | ERC and workspace examples (misc) | `erc-examples` | https://github.com/o2r-project/erc-examples | @@ -31,14 +47,16 @@ The documentation is also available online for reading, though availability may | platform (web UI) | `o2r-platform` | https://github.com/o2r-project/o2r-platform| | shipper (microservice) | `o2r-shipper` | https://github.com/o2r-project/o2r-shipper | | substituter (microservice) | `o2r-substituter` | https://github.com/o2r-project/o2r-substituter | +| guestlister (microservice) | `o2r-guestlister` | https://github.com/o2r-project/o2r-guestlister | | transporter (microservice) | `o2r-transporter` | https://github.com/o2r-project/o2r-transporter | ### Supported operating systems This project contains configurations and scripts to make running the o2r reference implementation as easy as possible. -The project uses [`make`](https://www.gnu.org/software/make/), which helps simplifying the execution of the various steps necessary to run your own o2r-platform into a simple set of commands. +The project uses [`make`](https://www.gnu.org/software/make/), which helps simplifying the execution of the various steps necessary to run the reference implementation into a set of console commands. These commands are formulated in the `Makefile` included in this project. +If `make` is not available, you can execute the respective commands manually. The commands in _this file_ (`README.md`) assume a Unix shell and related common tools, so they work on a recent installation of Linux or MacOS X. @@ -53,7 +71,7 @@ If you are familiar with virtual machines (VMs) we strongly consider you run the #### Software The o2r reference implementation is build on [Docker](http://docker.com/). -You must [install Docker](https://www.docker.com/get-docker) (i.e. Docker Community Edition, _edge_ channel, tested with version `17.09.0-ce` but later version should work) and [docker-compose](https://docs.docker.com/compose/) (version `1.13.0` or higher) first to continue. +You must [install Docker](https://www.docker.com/get-docker) (i.e. Docker Community Edition, tested with version `17.09.0-ce` but later versions should work) and [docker-compose](https://docs.docker.com/compose/) (version `1.13.0` or higher) first to continue. The tasks below are automated using [Make](https://en.wikipedia.org/wiki/Make_(software)). Make sure that you have GNU Make (tested) or one of the derivatives (not tested) running on your system. @@ -68,26 +86,42 @@ Run `make update` or the respective commands on your operating system to initial #### Accounts and tokens -##### ORCID +##### o2r-guestlister + +By default, the reference implementation uses the offline OAuth2 implementation provided by the [o2r-guestlister](https://github.com/o2r-project/o2r-guestlister). -The reference implementation relies on [ORCID](https://orcid.org/) for authentication and authorisation. -There is no other way to log in on the platform, therefore you must enable your installation of the reference implementation to connect with the ORCID API. -The client credentials used by the o2r team cannot be shared publicly for security reasons. +This allows access to the o2r platform by selecting one of three demo users. +The users represent different user roles with different levels, i.e. an admin (level `1000`), an editor (level `500`) and a regular author (level `100`). -You _must_ register a public API client application with ORCID as [described here](https://support.orcid.org/knowledgebase/articles/343182-register-a-public-api-client-application). +##### ORCID (optional) + +The reference implementation can be alternatively configured to use [ORCID](https://orcid.org/) for authentication and authorisation, replacing the offline login provided by o2r-guestlister. + +This requires an ORCID account which provides authentication tokens for public API client application with **[ORCID Sandbox](https://sandbox.orcid.org/signin)** (preferred for testing) or the registering an application as [described here](https://support.orcid.org/knowledgebase/articles/343182-register-a-public-api-client-application) with your regular ORCID account. In the developer tools, use any name, website URL, and description. Important is the `Redirect URIs` list, which must include `http://localhost` for your local installation. -The client ID, client secret, and redirect URI must then be provided to the docker-compose configurations via environment variables as shown below. +The client ID, client secret, redirect URI and the OAuth URLs have to be provided by modifying the `.env` file in the base directory. +Note that environment variables provided through the shell have priority over the `.env` file configuration. +For more information on how the `.env` file works, see the [`docker-compose` documentation](https://docs.docker.com/compose/env-file/). ##### Repositories (optional) The reference implementation can deliver the created ERC to different repositories. +By default only the "Download" repository is supported, i.e. users may download a complete ERC as an archive file. These repositories also require an authentication token. - [Create access token](https://zenodo.org/login/?next=%2Faccount%2Fsettings%2Fapplications%2Ftokens%2Fnew%2F) for [Zenodo](https://zenodo.org/) +These tokens can be provided to the docker-compose configurations by setting them as environment variables in the `.env` file, similar to the ORCID configuration. + +Modify the `SHIPPER_REPO_TOKENS` entry in the `.env` file to include the tokens: + +``` +SHIPPER_REPO_TOKENS={"zenodo": "<your Zenodo token>", "zenodo_sandbox": "<your Zenodo Sandbox token>", "download": "" } +``` + #### Elasticsearch host preparation The implementation uses an [Elasticsearch](http://elastic.co) document search engine. @@ -99,106 +133,171 @@ You can find instructions for all hosts (including Docker Toolbox) in the [Elast ### Build images from source and run -This repository already includes a `.gitmodules` file, which lists all required o2r microservices as git submodules. +This repository already includes a `.gitmodules` file, which lists all required o2r microservices and tools as git submodules. To download all o2r source code at once, navigate to the `reference-implementation` base directory and use ```bash - make update +make update +``` + +Once all repositories have been pulled successfully, build Docker images of the microservices and tools: + +```bash +make local_build ``` -Once all repositories have been pulled successfully, build docker images of the microservices and run them in containers by executing: +Then run the microservices and platform in containers: ```bash -O2R_ORCID_ID=<your orcid id> O2R_ORCID_SECRET=<your orcid secret> O2R_ORCID_CALLBACK=http://localhost/api/v1/auth/login O2R_ZENODO_TOKEN=<your token> \ - make build_images run_local +make local_up ``` -Wait for a while, then open **http://localhost**. +Wait until the log shows no new messages for a few seconds, then open **http://localhost** and continue in section ["Load data"](#load-data). + +The above three steps can also be executed with a single target: + +```bash +make local +``` ### Download images and run -[Docker Hub](https://hub.docker.com/) is a repository for Docker images. -All o2r software projects have automatic builds [available on Docker Hub](https://hub.docker.com/r/o2rproject/). -The following command executes a `docker-compose` command to pull and run these images. +[Docker Hub](https://hub.docker.com/) is a [public registry](https://en.wikipedia.org/wiki/Docker_(software)#Components) for Docker images. +All o2r software projects are automatically built [on Docker Hub](https://hub.docker.com/r/o2rproject/) when there is a new version uploaded to the code repository. +The images have tags corresponding to the software version (as specific e.g. in a `package.json` for a Node.js-based microservice). + +The following target pulls the latest images, prints the versions to the terminal, and executes a `docker-compose` command to run the reference implementation: ```bash -O2R_ORCID_ID=<your orcid id> O2R_ORCID_SECRET=<your orcid secret> O2R_ORCID_CALLBACK=http://localhost/api/v1/auth/login O2R_ZENODO_TOKEN=<your token> \ - make run_hub +make hub ``` -Wait a bit, then open **http://localhost**. +Wait until the log shows no new messages for a few seconds, then open **http://localhost** and continue in section ["Load data"](#load-data). ### Load data -The o2r API supports two way to load scientific workflows: direct upload as a ZIP archive, or import from a publish share. +You have to **log in** to upload workspaces: + +1. Click on "LOGIN" in the upper right hand corner +1. You are taken to a local login server: select one of the available users, e.g. "User" +1. After selecting the account type you are redirected to the o2r platform landing page + +The o2r API supports two way to load scientific workflows: direct upload as a ZIP archive, or loading from a publish share. -Ready to use **Direct upload** examples are available in the directory `erc-examples`. +- Ready to use **direct upload examples** are available in the directory `erc-examples`. +- Examples for **loading from a public share** are available [in this online share](https://uni-muenster.sciebo.de/index.php/s/G8vxQ1h50V4HpuA) (see file `README.txt`) and via the "EXAMPLES" menu in the "Create" section of the platform website. -Examples for loading from a public share are available here: https://uni-muenster.sciebo.de/index.php/s/G8vxQ1h50V4HpuA (see file `README.txt`). +After filling in required metadata you can publish the ERC. +Then open the ERC page and start a new job. +Check the job results and explore the interaction possibilities ### Explore back-end #### Database administration -An [adminMongo](https://adminmongo.markmoffat.com/) instance is included in the reference implementation. Open it at http://localhost:1234. In mongoAdmin please manually create a connection to host db, i.e. mongodb://db:27017 to edit the database (click "Update" first if you edit the existing connection, then "Connect"). +An [adminMongo](https://adminmongo.markmoffat.com/) instance is included in the reference implementation. +Open it at http://localhost:1234. +In mongoAdmin manually create a connection to host database: -The docker compose configuration includes an adminMongo instance. Open it at http://localhost:1234, create a new connection with the following settings: - -- Connection name: any name +- Connection name: any name, e.g. `o2r` - Connection string: `mongodb://mongodb:27017` (which is the default port and the host `mongodb`, no password) -- Connection options: `{}` (empty/default) +- Connection options: leave to `{}` (empty/default) -Click on "Add connection", then on "Connections" at the top right corner, and then "Connect" using the just created connection. +Wait a few moments for the interface to refresh, then click "Connect" in the line of the just created connection to explore the main database. #### File storage The configurations all use a common [Docker volume](https://docs.docker.com/engine/admin/volumes/volumes/) `o2rvol` with the global name `referenceimplementation_o2rvol`. -The volume and network can be inspected for development purposes: +The volume can be inspected for development purposes: ```bash docker volume ls docker volume inspect reference-implementation_o2r_test_storage ``` -## Reproducibility +## Read documentation + +This repository also contains specification and documentation projects. +These documentations are created in Markdown format and can be rendered to HTML and PDF documents using the make target `build_documentation`, which relies on a local `Dockerfile` for the rendering environment. +The PDF files are available to the project root directory. +The HTML files can be found in the respective projects in the `site` directory. -This repository serves the goal to make the developments of the o2r project reproducible, not only by running the reference implementation (see above) but also by creating an archivable package of the software. -As the implementations component are spread over multiple git repositories, we cannot rely on the direct export of a GitHub release to Zenodo. -This repository imports all relevant software projects as [git submodules](https://git-scm.com/docs/git-submodule) and manually creates a release package. +If the `local` configuration is used and the documentation was build, or if you [reproduce](#reproduce) you can also browse it at http://localhost/docs/. + +## Reproducibility -**TBD** +### Rationale -- clone repo -- build all images -- export all image -- create make target to load all images and run these -- create huge zip -- publish to Zenodo (with make target) +This repository serves the goal to make the developments of the o2r project reproducible, not only by running the reference implementation (see above) but also by creating an archivable package of the software in a reproducible (i.e. scripted) way. +As the implementation components are spread over multiple git repositories and rely on container images for reproduction, we cannot use the direct export of a GitHub release to Zenodo. +This repository captures all software projects to create a reproduction package, which can be uploaded to a data repository manually. ### Known limitations -- Nested code projects are currently installed from GitHub during the building of Docker images ((see also [#1]()). +- Nested code projects are currently installed from GitHub during the building of Docker images ((see also [#1](#1)). - `o2r-muncher`'s Docker image contains `o2r-meta` and `erc-checker` - `o2r-loaders`'s Docker image contains `o2r-meta` -- The used Docker base images and dependencies for the services, such as npm or pip packages, must be available online in the required version to build the images locally. See project files such as `package.json` or `requirements.txt` for a full list of dependencies. +- The used Docker base images and dependencies for the services, such as npm or pip packages, must be available online in the required version to build the images locally. + See project files such as `package.json` or `requirements.txt` for a full list of dependencies. +- The locally built images do not have a proper version tag but instead are `latest`, since they are build as part of the `docker-compose` configuration. + Note that you can easily distinguish images by this, too: The hub images will always have a specific version tag. ### Create package -**TODO** - +The creation of the reproducibility package consists of the following steps and relies on `Makefile` targets where possible (see brackets `(make ...)`). +The package uses the locally built images. + +1. Clone the `reference-implementation` repository to an empty directory: `git clone https://github.com/o2r-project/reference-implementation.git` +1. Create the package with `make package`, which... + - cleans up existing artifacts (`make package_clean local_clean`), + - prints _used_ software versions (git, Docker etc., `make versions`), + - updates the nested projects (`make update`), + - builds all documentation locally (`make build_documentation`), + - saves local version information of software and repositories to single file `versions.txt` (`make local_versions_save`), + - builds all images locally (see [limitations](#known-limitations), `make local_build`), + - saves the just built images into the file `o2r-reference-implementation-images.tar` (`make local_save_images`), + - saves all nested repositories (including their histories) + - reference implementation software to `o2r-reference-implementation-modules.zip` + - documentation to `o2r-docs.zip` + - reproduction package files (everything except `README` files and `Makefile`) to `o2r-reference-implementation-files.zip` +1. Create a new deposit or a new version of the existing deposit on Zenodo +1. Upload the _small files_ to the just created Zenodo deposit ([first version](https://zenodo.org/api/deposit/depositions/2203844), the used Python scripts needs the module `humanfriendly`; `ZENODO_DEPOSIT_ID=xxxxxx ZENODO_ACCESS_TOKEN=xxxxxx make upload_files_to_zenodo`) +1. Manually upload the _big file_ to Zenodo deposit ([Zenodo API currently has a 100 MB file limit](http://developers.zenodo.org/#deposition-files)) +1. Fill out metadata form on Zenodo and publish using the "Publish" button + +### Reproduce + +The reproduction consists of the following steps: + +1. Install [Docker CE](https://docs.docker.com/install/) and [`docker-compose`](https://docs.docker.com/compose/install/) +1. Download all files of the reproduction package from Zenodo: https://10.5281/zenodo.2203844 +1. Unzip the file `o2r-docs.zip` and `o2r-reference-implementation-files.zip` - they contain documentation you might want to explore, and required configuration files and examples respectively. +1. Unzip the file `o2r-reference-implementation-modules.zip` - it contains the source code of all software and [the directories are needed for `docker-compose`](https://github.com/docker/compose/issues/3391) even though they are not used +1. Load the saved images from the tarball using `docker load`. +1. Run the reference implementation with `docker-compose up` using the configuration file `docker-compose-local.yml` and making sure the loaded images are used with the option `--no-build` (i.e. not building the images) + +All except the first two steps can be executed with ```bash -make show_microservices_versions >> dist/versions.txt +make reproduce ``` -### Archival to Zenodo +Wait until the log shows no new messages for a few seconds, then open **http://localhost** and continue see section ["Load data"](#load-data) for creating ERCs. + +You can stop all containers with `Ctrl+C` and clean up local images and files with -**TODO** +```bash +make local_down +# to also delete local storage use: make local_down_volumes +make local_clean +``` +Please check the [Docker documentation](https://docs.docker.com/) if any problems arise and feel free to contact the [o2r team](https://o2r.info/about/#team). ## License -This project is licensed under Apache License, Version 2.0, see file LICENSE. Copyright © 2017 - o2r project. +This project is licensed under Apache License, Version 2.0, see file LICENSE. +Copyright © 2018 - o2r project. -All included software projects have their own LICENSE files, see `o2r-<component name>/LICENSE`. +All included software projects and data files have their respective and potentially differing licenses, see e.g. `<component name>/LICENSE`. diff --git a/api b/api new file mode 160000 index 0000000..8d75e2f --- /dev/null +++ b/api @@ -0,0 +1 @@ +Subproject commit 8d75e2ff1847073b8bc9025bc901e9a2e2cb9b3f diff --git a/architecture b/architecture index 8f82796..00756bc 160000 --- a/architecture +++ b/architecture @@ -1 +1 @@ -Subproject commit 8f8279609c40fd72c4df55a84995704c0b6100a0 +Subproject commit 00756bca5b3a69ce4267149dead6f57d08a4bfce diff --git a/containerit b/containerit new file mode 160000 index 0000000..0745231 --- /dev/null +++ b/containerit @@ -0,0 +1 @@ +Subproject commit 07452318e0768cea858b74dad2f1e5594dcb2926 diff --git a/docker-compose-local.yml b/docker-compose-local.yml index 3d8d795..592d478 100644 --- a/docker-compose-local.yml +++ b/docker-compose-local.yml @@ -14,18 +14,18 @@ # # --- -version: '3' +version: '2' volumes: o2rvol: {} services: mongodb: - image: mongo:3.4 + image: mongo:3.6 command: "--replSet rso2r" configmongodb: - image: mongo:3.4 + image: mongo:3.6 depends_on: - mongodb command: bash -c "sleep 5; mongo --verbose --host mongodb --eval 'printjson(rs.initiate()); printjson(rs.conf()); printjson(rs.status()); printjson(rs.slaveOk());'" @@ -41,132 +41,224 @@ services: - "1234:1234" elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:5.6.3 + image: docker.elastic.co/elasticsearch/elasticsearch:5.6.10 environment: - - ES_JAVA_OPTS=-Xms512m -Xmx512m + - cluster.name=o2rplatform-es-cluster + - bootstrap.memory_lock=true + - "ES_JAVA_OPTS=-Xms512m -Xmx512m" - "xpack.security.enabled=false" + ulimits: + memlock: + soft: -1 + hard: -1 + mem_limit: 1g + elasticsearch2: + image: docker.elastic.co/elasticsearch/elasticsearch:5.6.10 + environment: + - cluster.name=o2rplatform-es-cluster + - bootstrap.memory_lock=true + - "ES_JAVA_OPTS=-Xms512m -Xmx512m" + - "discovery.zen.ping.unicast.hosts=elasticsearch" + - "xpack.security.enabled=false" + ulimits: + memlock: + soft: -1 + hard: -1 + mem_limit: 1g + + meta: + image: o2r_refimpl_meta + build: + context: o2r-meta + + containerit: + image: o2r_refimpl_containerit + build: + context: containerit + dockerfile: inst/docker/geospatial/Dockerfile.local muncher: image: o2r_refimpl_muncher - restart: unless-stopped + build: + context: ./o2r-muncher depends_on: - configmongodb volumes: - o2rvol:/tmp/o2r - /var/run/docker.sock:/var/run/docker.sock environment: - - "MUNCHER_MONGODB=mongodb://mongodb" + - "MUNCHER_MONGODB=mongodb://mongodb:27017/" - MUNCHER_PORT=8080 + - MUNCHER_VOLUME=referenceimplementation_o2rvol + - MUNCHER_CONTAINER_USER=root + - MUNCHER_CONTAINERIT_USER=root + - MUNCHER_CONTAINERIT_IMAGE=o2r_refimpl_containerit - DEBUG=*,-mquery,-express:*,-express-session,-body-parser:* + - "MUNCHER_META_TOOL_CONTAINER=o2r_refimpl_meta" loader: image: o2r_refimpl_loader - restart: unless-stopped + build: + context: ./o2r-loader depends_on: - configmongodb volumes: - o2rvol:/tmp/o2r - /var/run/docker.sock:/var/run/docker.sock environment: - - "LOADER_MONGODB=mongodb://mongodb" + - "LOADER_MONGODB=mongodb://mongodb:27017/" - LOADER_PORT=8088 - DEBUG=*,-mquery,-express:*,-express-session,-body-parser:* + - LOADER_VOLUME=referenceimplementation_o2rvol + - "LOADER_META_TOOL_CONTAINER=o2r_refimpl_meta" informer: image: o2r_refimpl_informer - restart: unless-stopped + build: + context: ./o2r-informer depends_on: - configmongodb environment: - - "INFORMER_MONGODB=mongodb://mongodb" + - "INFORMER_MONGODB=mongodb://mongodb:27017/" - INFORMER_MONGODB_HOST=mongodb - INFORMER_PORT=8082 - DEBUG=informer bouncer: image: o2r_refimpl_bouncer - restart: unless-stopped + build: + context: ./o2r-bouncer depends_on: - configmongodb environment: - - "BOUNCER_MONGODB=mongodb://mongodb" + - "BOUNCER_MONGODB=mongodb://mongodb:27017/" - BOUNCER_PORT=8083 - DEBUG=bouncer - OAUTH_URL_CALLBACK=${OAUTH_URL_CALLBACK} - - OAUTH_CLIENT_ID=${OAUTH_CLIENT_ID} - - OAUTH_CLIENT_SECRET=${OAUTH_CLIENT_SECRET} - - SLACK_VERIFICATION_TOKEN=${SLACK_VERIFICATION_TOKEN} - - SLACK_BOT_TOKEN=${SLACK_BOT_TOKEN} + - OAUTH_URL_AUTHORIZATION=${OAUTH_URL_AUTHORIZATION} + - OAUTH_URL_TOKEN=${OAUTH_URL_TOKEN} + - OAUTH_STARTUP_FAIL_ON_ERROR=false finder: image: o2r_refimpl_finder - restart: unless-stopped + build: + context: ./o2r-finder depends_on: - configmongodb - elasticsearch volumes: - o2rvol:/tmp/o2r environment: - - "FINDER_MONGODB_USER_DATABASE=mongodb://mongodb/muncher" + - "FINDER_MONGODB=mongodb://mongodb:27017/" - FINDER_PORT=8084 - - DEBUG=finder + - DEBUG=finder,finder:* - ELASTIC_SEARCH_URL=elasticsearch:9200 - "MONGO_DATA_URL=mongodb://mongodb/muncher" - "MONGO_OPLOG_URL=mongodb://mongodb/muncher" - - BATCH_COUNT=20 + - BATCH_COUNT=5 transporter: image: o2r_refimpl_transporter - restart: unless-stopped + build: + context: ./o2r-transporter depends_on: - configmongodb volumes: - o2rvol:/tmp/o2r - "/var/run/docker.sock:/var/run/docker.sock" environment: - - "TRANSPORTER_MONGODB=mongodb://mongodb" + - "TRANSPORTER_MONGODB=mongodb://mongodb:27017/" - TRANSPORTER_PORT=8081 - DEBUG=transporter,transporter:* shipper: image: o2r_refimpl_shipper - restart: unless-stopped + build: + context: ./o2r-shipper depends_on: - configmongodb volumes: - o2rvol:/tmp/o2r environment: SHIPPER_MONGODB: "mongodb://mongodb" - SHIPPER_REPO_TOKENS: '{"zenodo_sandbox": "${ZENODO_TOKEN}", "download": "" }' + SHIPPER_REPO_TOKENS: "${SHIPPER_REPO_TOKENS}" SHIPPER_BOTTLE_HOST: "0.0.0.0" SHIPPER_BASE_PATH: "/tmp/o2r" + PYTHONUNBUFFERED: "0" substituter: image: o2r_refimpl_substituter - restart: unless-stopped + build: + context: ./o2r-substituter depends_on: - configmongodb volumes: - o2rvol:/tmp/o2r environment: - - "SUBSTITUTER_MONGODB=mongodb://mongodb" + - "SUBSTITUTER_MONGODB=mongodb://mongodb:27017/" - SUBSTITUTER_PORT=8090 - DEBUG=substituter,substituter:* + inspecter: + image: o2r_refimpl_inspecter + build: + context: ./o2r-inspecter + volumes: + - o2rvol:/tmp/o2r + environment: + - INSPECTER_PORT=8091 + - DEBUGME=inspecter + + bindings: + image: o2r_refimpl_bindings + build: + context: ./o2r-bindings + ports: + - "5000-5499:5000-5499" + volumes: + - o2rvol:/tmp/o2r + environment: + - DEBUG=bindings + + guestlister: + image: o2r_refimpl_guestlister + build: + context: ./o2r-guestlister + depends_on: + - configmongodb + environment: + - "GUESTLISTER_MONGODB=mongodb://mongodb:27017/" + - GUESTLISTER_PORT=8383 + - DEBUG=guestlister + - OAUTH_URL_CALLBACK=${OAUTH_URL_CALLBACK} + - OAUTH_URL_AUTHORIZATION=${OAUTH_URL_AUTHORIZATION} + - OAUTH_URL_TOKEN=${OAUTH_URL_TOKEN} + platform: image: o2r_refimpl_platform + build: + context: ./o2r-platform restart: unless-stopped webserver: image: nginx:latest restart: unless-stopped depends_on: - - muncher - bouncer + - finder + - muncher + - informer + - inspecter + - loader - transporter - - platform + - shipper + - substituter volumes: - - "./nginx.conf:/etc/nginx/nginx.conf:ro" + - "./etc/nginx.conf:/etc/nginx/nginx.conf:ro" + - "./api/site/:/www/docs/api:ro" + - "./architecture/site/:/www/docs/architecture:ro" + - "./erc-spec/site/:/www/docs/erc-spec:ro" + - "./etc/index.html:/www/docs/index.html:ro" ports: - "80:80" diff --git a/docker-compose.yml b/docker-compose.yml index 864e2f4..27ded56 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -14,18 +14,18 @@ # # --- -version: '3' +version: '2' volumes: o2rvol: {} services: mongodb: - image: mongo:3.4 + image: mongo:3.6 command: "--replSet rso2r" configmongodb: - image: mongo:3.4 + image: mongo:3.6 depends_on: - mongodb command: bash -c "sleep 5; mongo --verbose --host mongodb --eval 'printjson(rs.initiate()); printjson(rs.conf()); printjson(rs.status()); printjson(rs.slaveOk());'" @@ -41,137 +41,187 @@ services: - "1234:1234" elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:5.6.3 + image: docker.elastic.co/elasticsearch/elasticsearch:5.6.10 environment: - - ES_JAVA_OPTS=-Xms512m -Xmx512m + - cluster.name=o2rplatform-es-cluster + - bootstrap.memory_lock=true + - "ES_JAVA_OPTS=-Xms512m -Xmx512m" - "xpack.security.enabled=false" + ulimits: + memlock: + soft: -1 + hard: -1 + mem_limit: 1g +# ports: +# - "9200:9200" + elasticsearch2: + image: docker.elastic.co/elasticsearch/elasticsearch:5.6.10 + environment: + - cluster.name=o2rplatform-es-cluster + - bootstrap.memory_lock=true + - "ES_JAVA_OPTS=-Xms512m -Xmx512m" + - "discovery.zen.ping.unicast.hosts=elasticsearch" + - "xpack.security.enabled=false" + ulimits: + memlock: + soft: -1 + hard: -1 + mem_limit: 1g muncher: - image: o2rproject/o2r-muncher:0.12.3 - restart: unless-stopped + image: o2rproject/o2r-muncher:0.21.0 depends_on: - configmongodb volumes: - o2rvol:/tmp/o2r - /var/run/docker.sock:/var/run/docker.sock environment: - - "MUNCHER_MONGODB=mongodb://mongodb" + - "MUNCHER_MONGODB=mongodb://mongodb:27017/" - MUNCHER_PORT=8080 + - MUNCHER_VOLUME=referenceimplementation_o2rvol + - MUNCHER_CONTAINER_USER=root + - MUNCHER_CONTAINERIT_USER=root - DEBUG=*,-mquery,-express:*,-express-session,-body-parser:* + - "MUNCHER_META_TOOL_CONTAINER=o2rproject/o2r-meta:9198450" loader: - image: o2rproject/o2r-loader:0.7.3 - restart: unless-stopped + image: o2rproject/o2r-loader:0.12.0 depends_on: - configmongodb volumes: - o2rvol:/tmp/o2r - /var/run/docker.sock:/var/run/docker.sock environment: - - "LOADER_MONGODB=mongodb://mongodb" + - "LOADER_MONGODB=mongodb://mongodb:27017/" - LOADER_PORT=8088 - DEBUG=*,-mquery,-express:*,-express-session,-body-parser:* + - LOADER_VOLUME=referenceimplementation_o2rvol + - "LOADER_META_TOOL_CONTAINER=o2rproject/o2r-meta:9198450" informer: - image: o2rproject/o2r-informer:0.3.0 - restart: unless-stopped + image: o2rproject/o2r-informer:0.5.0 depends_on: - configmongodb environment: - - "INFORMER_MONGODB=mongodb://mongodb" + - "INFORMER_MONGODB=mongodb://mongodb:27017/" - INFORMER_MONGODB_HOST=mongodb - INFORMER_PORT=8082 - - DEBUG=informer + - DEBUG=informer,informer:* bouncer: - image: o2rproject/o2r-bouncer:0.8.0 - restart: unless-stopped + image: o2rproject/o2r-bouncer:0.13.0 depends_on: - configmongodb + - guestlister environment: - - "BOUNCER_MONGODB=mongodb://mongodb" + - "BOUNCER_MONGODB=mongodb://mongodb:27017/" - BOUNCER_PORT=8083 - - DEBUG=bouncer + - DEBUG=bouncer,bouncer:* - OAUTH_URL_CALLBACK=${OAUTH_URL_CALLBACK} - - OAUTH_CLIENT_ID=${OAUTH_CLIENT_ID} - - OAUTH_CLIENT_SECRET=${OAUTH_CLIENT_SECRET} - - SLACK_VERIFICATION_TOKEN=${SLACK_VERIFICATION_TOKEN} - - SLACK_BOT_TOKEN=${SLACK_BOT_TOKEN} + - OAUTH_URL_AUTHORIZATION=${OAUTH_URL_AUTHORIZATION} + - OAUTH_URL_TOKEN=${OAUTH_URL_TOKEN} + - OAUTH_STARTUP_FAIL_ON_ERROR=false finder: - image: o2rproject/o2r-finder:0.3.2 - restart: unless-stopped + image: o2rproject/o2r-finder:0.6.2 depends_on: - configmongodb - elasticsearch volumes: - o2rvol:/tmp/o2r environment: - - "FINDER_MONGODB_USER_DATABASE=mongodb://mongodb/muncher" + - "FINDER_MONGODB=mongodb://mongodb:27017/" - FINDER_PORT=8084 - - DEBUG=finder + - DEBUG=finder,finder:* - ELASTIC_SEARCH_URL=elasticsearch:9200 - "MONGO_DATA_URL=mongodb://mongodb/muncher" - "MONGO_OPLOG_URL=mongodb://mongodb/muncher" - - BATCH_COUNT=20 + - BATCH_COUNT=5 transporter: - image: o2rproject/o2r-transporter:0.4.0 - restart: unless-stopped + image: o2rproject/o2r-transporter:0.6.0 depends_on: - configmongodb volumes: - o2rvol:/tmp/o2r - "/var/run/docker.sock:/var/run/docker.sock" environment: - - "TRANSPORTER_MONGODB=mongodb://mongodb" + - "TRANSPORTER_MONGODB=mongodb://mongodb:27017/" - TRANSPORTER_PORT=8081 - DEBUG=transporter,transporter:* shipper: - image: o2rproject/o2r-shipper:e3a837c - restart: unless-stopped + image: o2rproject/o2r-shipper:8f95fd6 depends_on: - configmongodb volumes: - o2rvol:/tmp/o2r environment: SHIPPER_MONGODB: "mongodb://mongodb" - SHIPPER_REPO_TOKENS: '{"zenodo_sandbox": "${ZENODO_TOKEN}", "download": "" }' + SHIPPER_REPO_TOKENS: "${SHIPPER_REPO_TOKENS}" SHIPPER_BOTTLE_HOST: "0.0.0.0" SHIPPER_BASE_PATH: "/tmp/o2r" PYTHONUNBUFFERED: "0" substituter: - image: o2rproject/o2r-substituter:0.3.0 - restart: unless-stopped + image: o2rproject/o2r-substituter:0.5.0 depends_on: - configmongodb volumes: - o2rvol:/tmp/o2r environment: - - "SUBSTITUTER_MONGODB=mongodb://mongodb" + - "SUBSTITUTER_MONGODB=mongodb://mongodb:27017/" - SUBSTITUTER_PORT=8090 - DEBUG=substituter,substituter:* + inspecter: + image: o2rproject/o2r-inspecter:0.0.1 + volumes: + - o2rvol:/tmp/o2r + environment: + - INSPECTER_PORT=8091 + - DEBUGME=inspecter + + bindings: + image: o2rproject/o2r-bindings:0.0.1 + ports: + - "5000-5499:5000-5499" + volumes: + - o2rvol:/tmp/o2r + environment: + - DEBUG=bindings + + guestlister: + image: o2rproject/o2r-guestlister:0.2.1 + depends_on: + - configmongodb + environment: + - "GUESTLISTER_MONGODB=mongodb://mongodb:27017/" + - GUESTLISTER_PORT=8383 + - DEBUG=guestlister,guestlister:* + - OAUTH_URL_CALLBACK=${OAUTH_URL_CALLBACK} + - OAUTH_URL_AUTHORIZATION=${OAUTH_URL_AUTHORIZATION} + - OAUTH_URL_TOKEN=${OAUTH_URL_TOKEN} + platform: - image: o2rproject/o2r-platform:0.9.3 - restart: unless-stopped + image: o2rproject/o2r-platform:1.0.2 webserver: image: nginx:latest - restart: unless-stopped depends_on: - bouncer - finder - muncher - informer + - inspecter - loader - transporter - shipper - substituter + - guestlister + - bindings volumes: - - "./nginx.conf:/etc/nginx/nginx.conf:ro" + - "./etc/nginx.conf:/etc/nginx/nginx.conf:ro" ports: - "80:80" diff --git a/erc-checker b/erc-checker index fd3eeb6..fb1ed16 160000 --- a/erc-checker +++ b/erc-checker @@ -1 +1 @@ -Subproject commit fd3eeb6959ef7694496a0ba1c2b177d64d8495e0 +Subproject commit fb1ed1617464b862404dc18c3641678543bca470 diff --git a/erc-examples b/erc-examples index ea44b31..d589255 160000 --- a/erc-examples +++ b/erc-examples @@ -1 +1 @@ -Subproject commit ea44b3199c20d2dce9e51a7de115cc6a2ff41a30 +Subproject commit d5892550b2d68e858f006e4867de4fae5cc76a11 diff --git a/erc-spec b/erc-spec index 03888d6..a112fd8 160000 --- a/erc-spec +++ b/erc-spec @@ -1 +1 @@ -Subproject commit 03888d6bb642e044c45b00b37768a590a3d4f7b3 +Subproject commit a112fd85b26a4659997ac2463ec0aff7315ebd22 diff --git a/etc/Dockerfile.documentations b/etc/Dockerfile.documentations new file mode 100644 index 0000000..e37daa3 --- /dev/null +++ b/etc/Dockerfile.documentations @@ -0,0 +1,43 @@ +# +# A Dockerfile for an environment to build the documentation contained in this repository +# + +FROM rocker/verse:3.4.4 + +# System dependencies +RUN apt-get update && apt-get install -y --no-install-recommends \ + python3-pip \ + python3-setuptools \ + # next one needed by one of the tex packages + xzdec \ + # next one needed for untaring wkhtmltox + xz-utils \ + libssl1.0-dev + +# ERC spec, see erc-spec/.travis.yml + +# need newer pandoc +RUN wget https://github.com/jgm/pandoc/releases/download/2.1.1/pandoc-2.1.1-linux.tar.gz \ + && tar -xvzf pandoc*.tar.gz --strip-components 1 -C /usr/local/ + +# install required tools, see architecture/.travis.yml and erc-spec/.travis.yml +RUN pip3 install --upgrade pip +RUN python3 -m pip install \ + mkdocs \ + pygments \ + mkdocs-cinder \ + mkdocs-pandoc \ + markdown-include \ + pymdown-extensions \ + pandoc-latex-admonition \ + git+https://github.com/o2r-project/mkdocs-combine.git +RUN apt-get update && apt-get install -y --no-install-recommends \ + xfonts-base xfonts-75dpi +RUN wget https://downloads.wkhtmltopdf.org/0.12/0.12.5/wkhtmltox_0.12.5-1.stretch_amd64.deb \ + && dpkg -i wkhtmltox_0.12.5-1.stretch_amd64.deb +RUN R --quiet -e 'tinytex::tlmgr_install(pkgs=c("xcolor", "mdframed", "mdwtools", "needspace", "unicode-math", "filehook", "lm-math"))' + +VOLUME /doc +WORKDIR /doc + +USER 1000 diff --git a/etc/index.html b/etc/index.html new file mode 100644 index 0000000..e35cb2a --- /dev/null +++ b/etc/index.html @@ -0,0 +1,49 @@ +<!DOCTYPE html> + +<head> + <meta charset="utf-8"> + <meta http-equiv="X-UA-Compatible" content="IE=edge"> + <meta name="viewport" content="width=device-width, initial-scale=1.0"> + <meta name="author" content="Daniel Nüst"> + <link rel="shortcut icon" href="https://o2r.info/favicon.ico"> + <title>o2r Reference Implementation - Documentation</title> + <link href='https://fonts.googleapis.com/css?family=Lato:400,700|Roboto+Slab:400,700|Inconsolata:400,700' rel='stylesheet' + type='text/css'> + + <link rel="stylesheet" href="./api/css/theme.css" type="text/css" /> + <link rel="stylesheet" href="./api/css/theme_extra.css" type="text/css" /> + <!--<link href="./erc.css" rel="stylesheet">--> + +</head> + +<body role="document"> + <section> + <div class="wy-nav-content"> + <div role="main"> + <div class="section"> + + <h1>o2r Reference Implementation - Documentation</h1> + + <ul> + <li><a href="./api/">API documentation</a></li> + <li><a href="./architecture">o2r System architecture</a></li> + <li><a href="./erc-spec">ERC specification</a></li> + </ul> + + Go back to the <a href="../">demo server</a>. + + <hr /> + + <p><strong>Credits</strong></p> + <p>These documents are developed by the members of the DFG-funded project Opening Reproducible Research. + Learn more at <strong><a href="https://o2r.info">https://o2r.info</a></strong>.</p> + <p><a href="http://o2r.info"><img alt="Opening Reproducible Research" src="./api/logo.png" /></a></p> + + </div> + </div> + </div> + </section> + +</body> + +</html> \ No newline at end of file diff --git a/etc/local_versions.sh b/etc/local_versions.sh new file mode 100755 index 0000000..1241463 --- /dev/null +++ b/etc/local_versions.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +function versioninfo { + cd $1 + if [ -e bower.json ] + then + echo $1 $(cat README.md | grep -i --after-context 2 "platform version" | tail -1 ) $(git rev-parse HEAD) + elif [ -e package.json ] + then + echo $1 $(sed -nE 's/^\s*"version": "(.*?)",$/\1/p' package.json) $(git rev-parse HEAD) + elif [ -e DESCRIPTION ] + then + echo $1 $(sed -n '/Version:/p' DESCRIPTION | sed 's/[^0-9,.]*//g') $(git rev-parse HEAD) + else + echo $1 $(git rev-parse HEAD) + fi + cd .. +} + +for dir in $(ls -1d */ | grep -v 'etc') +do + versioninfo ${dir} +done + +echo ". (reference-implementation)" $(git rev-parse HEAD) diff --git a/nginx.conf b/etc/nginx.conf similarity index 81% rename from nginx.conf rename to etc/nginx.conf index a328d97..02d7542 100644 --- a/nginx.conf +++ b/etc/nginx.conf @@ -68,7 +68,7 @@ http { proxy_set_header X-Real-IP $remote_addr; proxy_set_header Upgrade $http_upgrade; proxy_set_header Connection $connection_upgrade; - + proxy_pass http://informer:8082; } @@ -87,11 +87,7 @@ http { location /api/v1/search { proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; - proxy_pass http://elasticsearch:9200/o2r/_search; - - limit_except GET { - deny all; - } + proxy_pass http://finder:8084; } location /api/v1/shipment { @@ -111,5 +107,30 @@ http { proxy_set_header X-Real-IP $remote_addr; proxy_pass http://substituter:8090; } + + location /api/v1/inspection { + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_pass http://inspecter:8091; + } + + location /api/v1/bindings { + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_pass http://bindings:8092; + } + + location /oauth { + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_pass http://guestlister:8383; + } + + location /docs { + include /etc/nginx/mime.types; + index index.html; + root /www; + } + } -} +} \ No newline at end of file diff --git a/etc/zenodo_upload.py b/etc/zenodo_upload.py new file mode 100644 index 0000000..6e4d716 --- /dev/null +++ b/etc/zenodo_upload.py @@ -0,0 +1,106 @@ +""" +Copyright (c) 2018 o2r project + +Helper functions to clear existing Zenodo deposit and upload files. +Creation of deposit, setting the version/metadata, and publishing the deposit are intentionally not included here. +""" + +import os +import sys +import logging +import requests +import humanfriendly + +# (you must install the required Python modules before the first run: `pip install clint requests_toolbelt`) +#from clint.textui.progress import Bar +#from requests_toolbelt import MultipartEncoder, MultipartEncoderMonitor + +logging.basicConfig(level=logging.INFO) # logging for requests + +token = os.getenv('ZENODO_ACCESS_TOKEN') +deposit = os.getenv('ZENODO_DEPOSIT_ID') +ZENODO_URL = 'https://zenodo.org/api/deposit/depositions' + +def upload_file(file, deposit_id): + print('Uploading %s' % (file)) + file_name = os.path.basename(file) + + # streaming upload, https://toolbelt.readthedocs.io/en/latest/uploading-data.html#streaming-multipart-data-encoder + #encoder = MultipartEncoder({ + # 'filename': file_name, + # 'file': open(file, 'rb') + # }) + #bar = Bar(expected_size = encoder.len, filled_char = '=') + #def callback(monitor): + # bar.show(monitor.bytes_read) + #monitor = MultipartEncoderMonitor(encoder, callback) + #print(monitor.content_type) + # r = requests.post('%s/%s/files?access_token=%s' % (ZENODO_URL, deposit_id, token), + # data = m, + # headers = {'Content-Type': m.content_type}) + + r = requests.post('%s/%s/files?access_token=%s' % (ZENODO_URL, deposit_id, token), + data = {'filename': file_name}, + files = {'file': open(file, 'rb')}) + if r.status_code != 201: + print('Error uploading file: %s' % (r.text)) + sys.exit() + else: + print('Uploaded file (HTTP %s): %s' % (r.status_code, r.json()['links']['self'])) + return r.json()['links']['self'] + +def clear_files(deposit_id): + print('Clearing files from deposit %s ...' % (deposit_id)) + r = requests.get('%s/%s' % (ZENODO_URL, deposit_id), params = {'access_token': token}) + if r.status_code != 200: + print('Error getting list of files: %s' % (r.text)) + sys.exit() + else: + for file in r.json()['files']: + print('Deleting %s | %s' % (file['id'], file['links']['download'])) + r_delete = requests.delete('%s/%s/files/%s' % (ZENODO_URL, deposit_id, file['id']), params = {'access_token': token}) + if r_delete.status_code != 204: + print('Error deleting existing file (HTTP %s): %s', (r_delete.text, r_delete.status_code)) + sys.exit() + else: + print('Deleted %s' % (file['id'])) + +def new_version(deposit_id): + print('Creating new version for %s' % (deposit_id)) + r = requests.post('%s/%s/actions/newversion' % (ZENODO_URL, deposit_id), params = {'access_token': token}) + if r.status_code != 201: + print('Error creating new version: %s' % (r.text)) + sys.exit() + else: + print('Created new version %s' % (r.json()['links']['latest_draft'])) + return os.path.basename(r.json()['links']['latest_draft']) + + +if __name__ == '__main__': + #deposit = new_version(deposit) + clear_files(deposit) + + files = ['README.md', + 'README-WIN.md', + 'Makefile', + 'erc-spec.pdf', + 'o2r-architecture.pdf', + 'o2r-web-api.pdf' + 'o2r-docs.zip', + 'o2r-reference-implementation-files.zip', + #'o2r-reference-implementation-images.tar.gz', + 'o2r-reference-implementation-modules.zip', + 'versions.txt' + ] + for file in files: + upload_file(file, deposit) + + print('Uploaded small files to %s/%s: %s' % (ZENODO_URL, deposit, str(files))) + raw_input('Go to https://zenodo.org/deposit/%s and upload the file %s. Done?' % (deposit, 'o2r-reference-implementation-images.tar.gz')) + print('Deposition files:') + r = requests.get('%s/%s' % (ZENODO_URL, deposit), params = {'access_token': token}) + for file in r.json()['files']: + print(' %s (%s)' % (file['filename'], humanfriendly.format_size(file['filesize']))) + + #publish_deposit(deposit) + #print('Published new deposit at: %s/%s' % (ZENODO_URL, deposit)) diff --git a/o2r-bindings b/o2r-bindings new file mode 160000 index 0000000..f68e17d --- /dev/null +++ b/o2r-bindings @@ -0,0 +1 @@ +Subproject commit f68e17d12dc7573bd7c6b927d444ef35b09f14fc diff --git a/o2r-bouncer b/o2r-bouncer index ad8e931..0c99901 160000 --- a/o2r-bouncer +++ b/o2r-bouncer @@ -1 +1 @@ -Subproject commit ad8e931e158c19029fbde9238ebe9783931df01e +Subproject commit 0c99901a7f0f6e633866d1eaca4f55c95cce6aa3 diff --git a/o2r-finder b/o2r-finder index 59e0fe3..97029b6 160000 --- a/o2r-finder +++ b/o2r-finder @@ -1 +1 @@ -Subproject commit 59e0fe39451e47f4f03d4543f78c7216f3eb8569 +Subproject commit 97029b6322223e5e4677e03f41d2b1b375634f5c diff --git a/o2r-guestlister b/o2r-guestlister new file mode 160000 index 0000000..fe35183 --- /dev/null +++ b/o2r-guestlister @@ -0,0 +1 @@ +Subproject commit fe351833fa05455b99b5644d26c944ae6500309f diff --git a/o2r-informer b/o2r-informer index 749a12d..cd507af 160000 --- a/o2r-informer +++ b/o2r-informer @@ -1 +1 @@ -Subproject commit 749a12d67cca0a4bed63657dffcbd4849b95747a +Subproject commit cd507af98a12c2544148145c2cffe07853d909f7 diff --git a/o2r-inspecter b/o2r-inspecter new file mode 160000 index 0000000..ee5e86a --- /dev/null +++ b/o2r-inspecter @@ -0,0 +1 @@ +Subproject commit ee5e86ada517972b246f8297ddb8b43580e3eac1 diff --git a/o2r-loader b/o2r-loader index e3d226b..a12115b 160000 --- a/o2r-loader +++ b/o2r-loader @@ -1 +1 @@ -Subproject commit e3d226b2794019aee1bded37cc6c71cfbf5577a6 +Subproject commit a12115b47ab3a4c931efc4ed8d54a1deeb42606a diff --git a/o2r-meta b/o2r-meta index 2c79485..f79cc1f 160000 --- a/o2r-meta +++ b/o2r-meta @@ -1 +1 @@ -Subproject commit 2c79485104b14642a9e610a32f012cecbb0b23a3 +Subproject commit f79cc1f838ce8a6151cd10967e497b4c9040e79d diff --git a/o2r-muncher b/o2r-muncher index 869a7d7..c68b4d3 160000 --- a/o2r-muncher +++ b/o2r-muncher @@ -1 +1 @@ -Subproject commit 869a7d7e48f8de044f2481d323538f004039ba43 +Subproject commit c68b4d369a52082ef764b0d576c7964258e0fd63 diff --git a/o2r-platform b/o2r-platform index 04969ab..806c795 160000 --- a/o2r-platform +++ b/o2r-platform @@ -1 +1 @@ -Subproject commit 04969ab21abb1964eb2c66b97db82fd96206bd4b +Subproject commit 806c7952b86b74fae20e631f8a29bf1e1a086e1c diff --git a/o2r-shipper b/o2r-shipper index e3a837c..8f95fd6 160000 --- a/o2r-shipper +++ b/o2r-shipper @@ -1 +1 @@ -Subproject commit e3a837c2bfa399845348a94b88637f048cc1b423 +Subproject commit 8f95fd6cc2c2071658ddcefa3457390207ff5ffc diff --git a/o2r-substituter b/o2r-substituter index bf0dcd0..1d5be50 160000 --- a/o2r-substituter +++ b/o2r-substituter @@ -1 +1 @@ -Subproject commit bf0dcd0524abcd5739b83298f0d4acc70f8479e4 +Subproject commit 1d5be501c9c2c2a6821ab1caff0c14e4320952dd diff --git a/o2r-transporter b/o2r-transporter index 9210939..ed9f8be 160000 --- a/o2r-transporter +++ b/o2r-transporter @@ -1 +1 @@ -Subproject commit 92109399d0b72718a83ebae73afe41502240fd69 +Subproject commit ed9f8be91d7195c3fe14b565d1b51505f963c17c diff --git a/o2r-web-api b/o2r-web-api deleted file mode 160000 index cd5554a..0000000 --- a/o2r-web-api +++ /dev/null @@ -1 +0,0 @@ -Subproject commit cd5554a212595772aaa3eb684b07554282de38f0 diff --git a/versions.txt b/versions.txt new file mode 100644 index 0000000..0333e6b --- /dev/null +++ b/versions.txt @@ -0,0 +1,23 @@ +make[1]: Entering directory '/home/daniel/git/o2r/reference-implementation' +etc/local_versions.sh +api/ 8d75e2ff1847073b8bc9025bc901e9a2e2cb9b3f +architecture/ 00756bca5b3a69ce4267149dead6f57d08a4bfce +containerit/ 0.5.0.9004 07452318e0768cea858b74dad2f1e5594dcb2926 +erc-checker/ 1.2.0 fb1ed1617464b862404dc18c3641678543bca470 +erc-examples/ d5892550b2d68e858f006e4867de4fae5cc76a11 +erc-spec/ a112fd85b26a4659997ac2463ec0aff7315ebd22 +o2r-bindings/ 0.0.1 f68e17d12dc7573bd7c6b927d444ef35b09f14fc +o2r-bouncer/ 0.13.0 0c99901a7f0f6e633866d1eaca4f55c95cce6aa3 +o2r-finder/ 0.6.3 97029b6322223e5e4677e03f41d2b1b375634f5c +o2r-guestlister/ 0.2.2 fe351833fa05455b99b5644d26c944ae6500309f +o2r-informer/ 0.5.0 cd507af98a12c2544148145c2cffe07853d909f7 +o2r-inspecter/ 0.0.1 ee5e86ada517972b246f8297ddb8b43580e3eac1 +o2r-loader/ 0.12.0 a12115b47ab3a4c931efc4ed8d54a1deeb42606a +o2r-meta/ f79cc1f838ce8a6151cd10967e497b4c9040e79d +o2r-muncher/ 0.21.1 c68b4d369a52082ef764b0d576c7964258e0fd63 +o2r-platform/ 1.0.2 806c7952b86b74fae20e631f8a29bf1e1a086e1c +o2r-shipper/ 8f95fd6cc2c2071658ddcefa3457390207ff5ffc +o2r-substituter/ 0.5.0 1d5be501c9c2c2a6821ab1caff0c14e4320952dd +o2r-transporter/ 0.6.0 ed9f8be91d7195c3fe14b565d1b51505f963c17c +. (reference-implementation) 48efaca2b61c756153dc1fcdfb7fe27d349a8781 +make[1]: Leaving directory '/home/daniel/git/o2r/reference-implementation'