global: directory reorganisation

.gitignore

@@ -1,2 +1,2 @@
-outputs/
+results/
 code/.ipynb_checkpoints

.travis.yml

@@ -19,5 +19,13 @@ before_install:
 
 script:
   - yadage-validate workflow/yadage/workflow.yaml | grep -q 'workflow validates'
-  - docker run -i -t --rm -v `pwd`/code:/code -v `pwd`/inputs:/inputs -v `pwd`/outputs:/outputs reanahub/reana-env-jupyter papermill /code/worldpopulation.ipynb /dev/null
-  - ls -l `pwd`/outputs/plot.png
+  - docker run -i -t --rm -v `pwd`:/workdir reanahub/reana-env-jupyter bash -c '
+      cd /workdir &&
+      mkdir -p ./results &&
+      papermill ./code/worldpopulation.ipynb /dev/null
+        -p input_file ./data/World_historical_and_predicted_populations_in_percentage.csv
+        -p output_file ./results/plot.png
+        -p region Europe
+        -p year_min 1600
+        -p year_max 2010'
+  - ls -l `pwd`/results/plot.png

README.rst

@@ -33,7 +33,7 @@ cloud and run the analysis to obtain (5) output results.
 
 We shall use the following input dataset:
 
-- `World_historical_and_predicted_populations_in_percentage.csv <inputs/World_historical_and_predicted_populations_in_percentage.csv>`_
+- `World_historical_and_predicted_populations_in_percentage.csv <data/World_historical_and_predicted_populations_in_percentage.csv>`_
 
 It contains historical and predicted world population numbers in CSV format and
 was compiled from `Wikipedia <https://en.wikipedia.org/wiki/World_population>`_.
@@ -78,7 +78,7 @@ notebook inputs.
 
 The input parameters are located in a tagged cell and define:
 
-- ``input_file`` - the location of the input CVS data file (see above)
+- ``input_file`` - the location of the input CSV data file (see above)
 - ``region`` - the region of teh world to analyse (e.g. Africa)
 - ``year_min`` - starting year
 - ``year_max`` - ending year
@@ -107,14 +107,14 @@ For example:
 .. code-block:: console
 
     $ papermill ./code/worldpopulation.ipynb /dev/null \
-         -p input_file ./inputs/World_historical_and_predicted_populations_in_percentage.csv \
-         -p output_file ./outputs/plot.png \
+         -p input_file ./data/World_historical_and_predicted_populations_in_percentage.csv \
+         -p output_file ./results/plot.png \
          -p region Europe \
          -p year_min 1600 \
          -p year_max 2010
-    $ ls -l outputs/plot.png
+    $ ls -l results/plot.png
 
-Note that you can also use `CWL <http://www.commonwl.org/v1.0/>`_ or `Yadage
+Note that we can also use `CWL <http://www.commonwl.org/v1.0/>`_ or `Yadage
 <https://github.com/diana-hep/yadage>`_ workflow specifications:
 
 - `workflow definition using CWL <workflow/cwl/worldpopulation.cwl>`_
@@ -131,186 +131,79 @@ region relative to the total world population as a function of time:
    :alt: plot.png
    :align: center
 
-Local testing
-=============
-
-*Optional*
-
-If you would like to test the analysis locally (i.e. outside of the REANA
-platform), you can proceed as follows.
-
-Using pure Docker:
-
-.. code-block:: console
-
-    $ rm -rf outputs && mkdir outputs
-    $ docker run -i -t  --rm \
-                  -v `pwd`/code:/code \
-                  -v `pwd`/inputs:/inputs \
-                  -v `pwd`/outputs:/outputs \
-                  reanahub/reana-env-jupyter \
-              papermill /code/worldpopulation.ipynb /dev/null
-    $ firefox outputs/plot.png
-
-In case you are using CWL workflow specification:
-
-.. code-block:: console
-
-    $ mkdir cwl-local-run
-    $ cd cwl-local-run
-    $ cp -a ../code ../inputs ../workflow/cwl/worldpopulation_job.yml .
-    $ cwltool --quiet --outdir="../outputs" ../workflow/cwl/worldpopulation.cwl worldpopulation_job.yml
-    $ firefox ../outputs/plot.png
-
-In case you are using Yadage workflow specification:
-
-.. code-block:: console
-
-    $ mkdir -p yadage-local-run/yadage-inputs
-    $ cd yadage-local-run
-    $ cp -a ../code ../inputs yadage-inputs
-    $ yadage-run . ../workflow/yadage/workflow.yaml \
-         -p notebook=code/worldpopulation.ipynb \
-         -p input_file=inputs/World_historical_and_predicted_populations_in_percentage.csv \
-         -p region=Africa \
-         -p year_min=1500 \
-         -p year_max=2012 \
-         -d initdir=`pwd`/yadage-inputs
-    $ firefox worldpopulation/plot.png
-
 Running the example on REANA cloud
 ==================================
 
-We are now ready to run this example and on the `REANA <http://www.reana.io/>`_
-cloud.
-
-First we need to create a `reana.yaml <reana.yaml>`_ file describing the
-structure of our analysis with its inputs, the code, the runtime environment,
-the computational workflow steps and the expected outputs:
+We start by creating a `reana.yaml <reana.yaml>`_ file describing the above
+analysis structure with its inputs, code, runtime environment, computational
+workflow steps and expected outputs:
 
 .. code-block:: yaml
 
     version: 0.3.0
     inputs:
       files:
         - code/worldpopulation.ipynb
-        - inputs/World_historical_and_predicted_populations_in_percentage.csv
+        - data/World_historical_and_predicted_populations_in_percentage.csv
       parameters:
         notebook: code/worldpopulation.ipynb
-        input_file: inputs/World_historical_and_predicted_populations_in_percentage.csv
-        output_file: outputs/plot.png
+        input_file: data/World_historical_and_predicted_populations_in_percentage.csv
+        output_file: results/plot.png
         region: Africa
         year_min: 1500
         year_max: 2012
-    outputs:
-      files:
-       - outputs/plot.png
     workflow:
       type: serial
       specification:
         steps:
           - environment: 'reanahub/reana-env-jupyter'
             commands:
-              - mkdir -p outputs && papermill $notebook /dev/null -p input_file ${input_file} -p output_file ${output_file} -p region $region -p year_min ${year_min} -p year_max ${year_max}
-
-In case you are using CWL or Yadage workflow specifications:
+              - mkdir -p results && papermill ${notebook} /dev/null -p input_file ${input_file} -p output_file ${output_file} -p region ${region} -p year_min ${year_min} -p year_max ${year_max}
+    outputs:
+      files:
+        - results/plot.png
 
-- `reana.yaml using CWL <reana-cwl.yaml>`_
-- `reana.yaml using Yadage <reana-yadage.yaml>`_
+In this example we are using a simple Serial workflow engine to represent our
+sequential computational workflow steps. Note that we can also use the CWL
+workflow specification (see `reana-cwl.yaml <reana-cwl.yaml>`_) or the Yadage
+workflow specification (see `reana-yadage.yaml <reana-yadage.yaml>`_).
 
-We proceed by installing the REANA command-line client:
+We can now install the REANA command-line client, run the analysis and download
+the resulting plots:
 
 .. code-block:: console
 
+    $ # install REANA client:
     $ mkvirtualenv reana-client
     $ pip install reana-client
-
-We should now connect the client to the remote REANA cloud where the analysis
-will run. We do this by setting the ``REANA_SERVER_URL`` environment variable
-and ``REANA_ACCESS_TOKEN`` with a valid access token:
-
-.. code-block:: console
-
+    $ # connect to some REANA cloud instance:
     $ export REANA_SERVER_URL=https://reana.cern.ch/
-    $ export REANA_ACCESS_TOKEN=<ACCESS_TOKEN>
-
-Note that if you `run REANA cluster locally
-<http://reana-cluster.readthedocs.io/en/latest/gettingstarted.html#deploy-reana-cluster-locally>`_
-on your laptop, you would do:
-
-.. code-block:: console
-
-    $ eval $(reana-cluster env --all)
-
-Let us test the client-to-server connection:
-
-.. code-block:: console
-
-    $ reana-client ping
-    Connected to https://reana.cern.ch - Server is running.
-
-We proceed to create a new workflow instance:
-
-.. code-block:: console
-
-    $ reana-client create
-    workflow.1
-    $ export REANA_WORKON=workflow.1
-
-We can now seed the analysis workspace with our input CSV data file and our
-Jupyter notebook:
-
-.. code-block:: console
-
-    $ reana-client upload ./inputs ./code
-    File inputs/World_historical_and_predicted_populations_in_percentage.csv was successfully uploaded.
-    File code/worldpopulation.ipynb was successfully uploaded.
-
-    $ reana-client list
-    NAME                                                                  SIZE    LAST-MODIFIED
-    code/worldpopulation.ipynb                                            19223   2018-08-29 08:11:57.575697+00:00
-    inputs/World_historical_and_predicted_populations_in_percentage.csv   574     2018-08-29 08:11:57.542697+00:00
-
-We can now start the workflow execution:
-
-.. code-block:: console
-
+    $ export REANA_ACCESS_TOKEN=XXXXXXX
+    $ # create new workflow:
+    $ reana-client create -n my-analysis
+    $ export REANA_WORKON=my-analysis
+    $ # upload input code and data to the workspace:
+    $ reana-client upload ./code ./data
+    $ # start computational workflow:
     $ reana-client start
-    workflow.1 has been started.
-
-After several minutes the workflow should be successfully finished. Let us query
-its status:
-
-.. code-block:: console
-
+    $ # ... should be finished in about a minute:
     $ reana-client status
-    NAME       RUN_NUMBER   CREATED               STATUS     PROGRESS
-    workflow   1            2018-08-29T08:11:35   finished   2/2
-
-We can list the output files:
-
-.. code-block:: console
-
+    $ # list workspace files:
     $ reana-client list
-    NAME                                                                  SIZE    LAST-MODIFIED
-    outputs/plot.png                                                      15879   2018-08-29 08:12:54.547782+00:00
-    code/worldpopulation.ipynb                                            19223   2018-08-29 08:11:57.575697+00:00
-    inputs/World_historical_and_predicted_populations_in_percentage.csv   574     2018-08-29 08:11:57.542697+00:00
-
-We finish by downloading the generated plot:
-
-.. code-block:: console
+    $ # download output results:
+    $ reana-client download results/plot.png
 
-    $ reana-client download outputs/plot.png
-    File outputs/plot.png downloaded to /home/simko/private/project/reana/src/reana-demo-worldpopulation.
+Please see the `REANA-Client <https://reana-client.readthedocs.io/>`_
+documentation for more detailed explanation of typical ``reana-client`` usage
+scenarios.
 
 Contributors
 ============
 
 The list of contributors in alphabetical order:
 
-- Alizee Pace <alizee.pace@gmail.com>
-- `Anton Khodak <https://orcid.org/0000-0003-3263-4553>`_ <anton.khodak@ukr.net>
-- `Diego Rodriguez <https://orcid.org/0000-0003-0649-2002>`_ <diego.rodriguez@cern.ch>
-- `Dinos Kousidis <https://orcid.org/0000-0002-4914-4289>`_ <dinos.kousidis@cern.ch>
-- `Tibor Simko <https://orcid.org/0000-0001-7202-5803>`_ <tibor.simko@cern.ch>
+- `Alizee Pace <https://www.linkedin.com/in/aliz%C3%A9e-pace-516b4314b/>`_
+- `Anton Khodak <https://orcid.org/0000-0003-3263-4553>`_
+- `Diego Rodriguez <https://orcid.org/0000-0003-0649-2002>`_
+- `Dinos Kousidis <https://orcid.org/0000-0002-4914-4289>`_
+- `Tibor Simko <https://orcid.org/0000-0001-7202-5803>`_

code/worldpopulation.ipynb

@@ -22,8 +22,8 @@
    "outputs": [],
    "source": [
     "# define input parameters\n",
-    "input_file = \"../inputs/World_historical_and_predicted_populations_in_percentage.csv\"\n",
-    "output_file = \"../outputs/plot.png\"\n",
+    "input_file = \"../data/World_historical_and_predicted_populations_in_percentage.csv\"\n",
+    "output_file = \"../results/plot.png\"\n",
     "region = 'Africa'\n",
     "year_min = 1500\n",
     "year_max = 2012"

reana-cwl.yaml

@@ -2,12 +2,12 @@ version: 0.3.0
 inputs:
   files:
     - code/worldpopulation.ipynb
-    - inputs/World_historical_and_predicted_populations_in_percentage.csv
+    - data/World_historical_and_predicted_populations_in_percentage.csv
   parameters:
     input: workflow/cwl/worldpopulation_job.yml
-outputs:
-  files:
-   - outputs/worldpopulation.html
 workflow:
   type: cwl
   file: workflow/cwl/worldpopulation.cwl
+outputs:
+  files:
+   - results/plot.png

reana-yadage.yaml

@@ -2,17 +2,17 @@ version: 0.3.0
 inputs:
   files:
     - code/worldpopulation.ipynb
-    - inputs/World_historical_and_predicted_populations_in_percentage.csv
+    - data/World_historical_and_predicted_populations_in_percentage.csv
   parameters:
     notebook: code/worldpopulation.ipynb
-    input_file: inputs/World_historical_and_predicted_populations_in_percentage.csv
-    output_file: outputs/plot.png
+    input_file: data/World_historical_and_predicted_populations_in_percentage.csv
+    output_file: results/plot.png
     region: Africa
     year_min: 1500
     year_max: 2012
-outputs:
-  files:
-   - outputs/plot.png
 workflow:
   type: yadage
   file: workflow/yadage/workflow.yaml
+outputs:
+  files:
+   - results/plot.png

reana.yaml

@@ -2,21 +2,21 @@ version: 0.3.0
 inputs:
   files:
     - code/worldpopulation.ipynb
-    - inputs/World_historical_and_predicted_populations_in_percentage.csv
+    - data/World_historical_and_predicted_populations_in_percentage.csv
   parameters:
     notebook: code/worldpopulation.ipynb
-    input_file: inputs/World_historical_and_predicted_populations_in_percentage.csv
-    output_file: outputs/plot.png
+    input_file: data/World_historical_and_predicted_populations_in_percentage.csv
+    output_file: results/plot.png
     region: Africa
     year_min: 1500
     year_max: 2012
-outputs:
-  files:
-   - outputs/plot.png
 workflow:
   type: serial
   specification:
     steps:
       - environment: 'reanahub/reana-env-jupyter'
         commands:
-          - mkdir -p outputs && papermill $notebook /dev/null -p input_file ${input_file} -p output_file ${output_file} -p region $region -p year_min ${year_min} -p year_max ${year_max}
+          - mkdir -p results && papermill ${notebook} /dev/null -p input_file ${input_file} -p output_file ${output_file} -p region ${region} -p year_min ${year_min} -p year_max ${year_max}
+outputs:
+  files:
+    - results/plot.png

workflow/cwl/worldpopulation.cwl

@@ -1,5 +1,15 @@
 #!/usr/bin/env cwl-runner
 
+# Note that if you are working on the analysis development locally, i.e. outside
+# of the REANA platform, you can proceed as follows:
+#
+#   $ cd reana-demo-worldpopulation
+#   $ mkdir cwl-local-run
+#   $ cd cwl-local-run
+#   $ cp -a ../code ../data ../workflow/cwl/worldpopulation_job.yml .
+#   $ cwltool --quiet --outdir="../results" ../workflow/cwl/worldpopulation.cwl worldpopulation_job.yml
+#   $ firefox ../results/plot.png
+
 cwlVersion: v1.0
 class: Workflow
 

workflow/cwl/worldpopulation_job.yml

@@ -3,7 +3,7 @@ notebook:
   path: code/worldpopulation.ipynb
 input_file:
   class: File
-  path: inputs/World_historical_and_predicted_populations_in_percentage.csv
+  path: data/World_historical_and_predicted_populations_in_percentage.csv
 output_file: plot.png
 region: Africa
 year_min: 1500