Merge branch 'publish' into 'master'

version 1.6.3

Closes #31 and #30

See merge request atgc/waves/waves-core!2

.gitignore

@@ -62,18 +62,25 @@ celerybeat-schedule
 .venv
 venv/
 
-/logs/*
-!/logs/.gitkeep
-/data/*
+# default dirs
+!/data/logs/.gitkeep
 !/data/.gitkeep
-/bin/*
-!/bin/.gitkeep
-/media/*
-!/media/.gitkeep
-*.sqlite3
-/tests/data/jobs/
+/data/bin/*
+!/data/bin/.gitkeep
+/data/jobs/*
+!/data/jobs/.gitkeep
+/data/media/*
+!/data/media/.gitkeep
+/data/sample/*
+!/data/sample/.gitkeep
+# tests dirs
+/tests/settings.ini
+/tests/data/jobs/*/*
 !/tests/data/jobs/.gitkeep
+!/tests/settings.ini.sample
+*.sqlite3
 /staticfiles/
-/tests/settings.ini
 /tests/data/*.json
-local.env
+/data/tests/*.json
+local.env
+*.pid

.gitlab-ci.yml

@@ -0,0 +1,48 @@
+# This file is a template, and might need editing before it works on your project.
+# Official framework image. Look for the different tagged releases at:
+# https://hub.docker.com/r/library/python
+image: python:2.7
+
+# Pick zero or more services to be used on all builds.
+# Only needed when using a docker container to run your tests in.
+# Check out: http://docs.gitlab.com/ce/ci/docker/using_docker_images.html#what-is-a-service
+# services:
+
+# This folder is cached between builds
+# http://docs.gitlab.com/ce/ci/yaml/README.html#cache
+cache:
+  paths:
+  - ~/.cache/pip/
+
+# This is a basic example for a gem or script which doesn't use
+# services such as redis or postgres
+before_script:
+  - python -V                                   # Print out python version for debugging
+  # Uncomment next line if your Django app needs a JS runtime:
+  # - apt-get update -q && apt-get install nodejs -yqq
+  - pip install -r requirements.txt
+  - pip install coverage==4.5.1
+
+# To get Django tests to work you may need to create a settings file using
+# the following DATABASES:
+# 
+# DATABASES = {
+#     'default': {
+#        'ENGINE': 'django.db.backends.postgresql_psycopg2',
+#        'NAME': 'ci',
+#        'USER': 'postgres',
+#        'PASSWORD': 'postgres',
+#        'HOST': 'postgres',
+#        'PORT': '5432',
+#    },
+# }
+#
+# and then adding `--settings app.settings.ci` (or similar) to the test command
+
+test:
+  variables:
+    WAVES_SSH_TEST_SGE_CELL: $WAVES_SSH_TEST_SGE_CELL
+    WAVES_TEST_SGE_BASE_DIR: $WAVES_TEST_SGE_BASE_DIR
+  script:
+  - python manage.py test
+  - coverage run --source='.' manage.py test waves

CHANGES.md

@@ -4,6 +4,9 @@ CHANGELOG
 Version 1.6.x - 2018-03-10
 ----------------------------
 
+- [Layout] - Corrected missing directories
+- [ignore] - Added ignored files
+- [Bug] - Corrected display in BO for passwords
 - [USERS] - Added API USER class, unifying authentication, url redirection dedicated to REST Api users 
     - Added multi-site association allow returning site url prefix for jobs urls
     - Test if job.client user has a 'site' property, in such case, retrieve domain name to generate Job / JobOutputs link

MANIFEST.in

@@ -1,8 +1,9 @@
 include LICENSE.md
 include README.md
+include requirements.txt
 recursive-include waves/wcore/static *
 recursive-include waves/wcore/templates *
 recursive-include waves/front/templates *
 recursive-include docs *
 recursive-exclude waves_core *
-recursive-exclude waves/wcore/migrations *
+# recursive-exclude waves/wcore/migrations *.

ROADMAP.md

@@ -13,6 +13,7 @@ Near-term (early 2018 - v1.1.7)
 
 - Integrate WAVES into a `SINGULARITY <http://singularity.lbl.gov/>`_ container
 - Reactivate Job Input dynamic validators
+- Add OpenApi Support
 - Adapter to run containers (SINGULARITY first)
 - Adapter / Service : Import / Export json serialized data
 - Introduce Form Component classes for Service inputs

docs/dev_doc/dev_doc.rst

@@ -45,4 +45,5 @@ Documentation
 
     waves_dev_doc
     api_user_doc
+    sample_code
 

docs/dev_doc/sample_code.rst

@@ -0,0 +1,89 @@
+WAVES python samples code
+=========================
+
+
+.. note::
+These examples use coreapi package functionalities
+    see: http://www.coreapi.org/
+
+
+Interact with services
+----------------------
+
+.. code-block:: python
+
+    from coreapi import Client, auth
+    from coreapi.codecs import CoreJSONCodec, JSONCodec
+
+    decoders = [JSONCodec(), CoreJSONCodec()]
+    # Create a client client codecs
+    client = Client(decoders=decoders)
+    document = client.get('http://waves.demo.atgc-montpellier.fr/waves/api/schema')
+    # get service list - replace with actual waves-api urls
+    serviceList = client.action(document, ['services', 'list'])
+    print(serviceList)
+    # get service details
+    serviceDetails = client.action(document, ["services", "read"], params={'service_app_name': 'sample_service'})
+    print(serviceDetails['name'])
+    # get service submissions
+    submissions = client.action(document, ["services", "submissions_list"],
+                                params={"service_app_name": 'sample_service'})
+    # get first submission details
+    sub_details = client.action(document, ["services", "submission"],
+                                params={"service_app_name": "sample_service",
+                                        "submission_app_name": "default"})
+    # get inputs / outputs
+    expected_inputs = sub_details['inputs']
+    print(expected_inputs)
+    expected_outputs = sub_details['outputs']
+    print(expected_outputs)
+
+
+Authenticate with token:
+------------------------
+
+Some WAVES API entries required to be authenticated (jobs list, job details, job submission)
+
+.. code-block:: python
+
+    client = Client(decoders=decoders,
+                auth=auth.TokenAuthentication(
+                    token="6241961ef45e4bbe7bb01a05f938ed9f0f2a3926",
+                    scheme="Token"))
+    document = client.get('http://waves.demo.atgc-montpellier.fr/waves/api/schema')
+    # list jobs
+    # get job list
+    job_list = client.action(document, ['jobs', 'list'])
+    print("job_list", job_list)
+    if (len(job_list) > 0):
+        job_details = client.action(document, ['jobs', 'read'], params={'unique_id': job_list[0]['slug']})
+        print(job_details['title'])
+        print(job_details['created'])
+        print(job_details['updated'])
+
+
+Create a job:
+-------------
+
+.. code-block:: python
+
+    # submit a job
+    from coreapi.utils import File
+    from os.path import join, dirname
+
+    with open(join(dirname(__file__), "test.fasta"), 'r') as f:
+        inputs = {
+            "text_input": "This is text input",
+            "input_file": File("test.fasta", f)
+        }
+        client.action(document, ["services", "submissions", "jobs", "create"],
+                      params={
+                          **inputs,
+                          "title": "Job Name",
+                          "service_app_name": "sample_service",
+                          "submission_app_name": "default"
+                      }, validate=False, encoding='multipart/form-data')
+    job_list = client.action(document, ['jobs', 'list'])
+    print(job_list)
+
+

docs/django_sphinx.py

@@ -23,14 +23,16 @@ def process_docstring(app, what, name, obj, options, lines):
                 # Decode and strip any html out of the field's help text
             try:
                 help_text = strip_tags(force_unicode(field.help_text))
-            except:
+            except StandardError:
                 help_text = ''
 
             # Decode and capitalize the verbose name, for use if there isn't
             # any help text
             try:
                 verbose_name = force_unicode(field.verbose_name).capitalize()
-            except:
+            except UnicodeError:
+                verbose_name = ''
+            except AttributeError:
                 verbose_name = ''
 
             if help_text:
@@ -57,8 +59,8 @@ def process_docstring(app, what, name, obj, options, lines):
                 latelines.append('')
                 latelines.append(u'   %s to :class:`%s.%s`' % (type(field).__name__, to.__module__, to.__name__))
                 latelines.append('')
-        else:
-            lines.append(u':type %s: %s' % (field.attname, type(field).__name__))
+            else:
+                lines.append(u':type %s: %s' % (field.attname, type(field).__name__))
         lines.append('')
         lines += latelines
     # Return the extended docstring

docs/installation.rst

@@ -20,6 +20,17 @@ Installation
 
 You can install WAVES-core either as a stand alone application, or inside any existing Django project
 
+0. Prerequisites
+----------------
+    .. note::
+        In order to install WAVES you will need:
+            - python 2.7.X (WAVES is not yet compatible with python 3.5)
+            - pip package manager
+            - A web server: `Apache <https://httpd.apache.org/>`_ or `NGINX <https://nginx.org/>`_
+            - A database backend (Mysql or Postgres) but by default WAVES runs with sqlite
+
+    In case you need to use MySQL with WAVES, you may install following dependencies:
+
 
 1. Install WAVES-core as stand alone application
 ------------------------------------------------
@@ -32,7 +43,11 @@ You can install WAVES-core either as a stand alone application, or inside any ex
 
         user@host:$ git clone https://github.com/lirmm/waves-core.git [your_app]
         user@host:$ cd [your_app]
-        user@host:~your_app$ git checkout tags/v[VERSION]
+
+    .. note::
+        To checkout a particular version:
+
+        user@host:~your_app$ git checkout tags/[VERSION]
 
     1.2. Install dependencies:
 
@@ -49,25 +64,34 @@ You can install WAVES-core either as a stand alone application, or inside any ex
     .. code-block:: bash
 
         (.venv) user@host:~your_app$ ./manage.py check
-        (.venv) user@host:~your_app$ ./manage.py makemigrations wcore
+        (.venv) user@host:~your_app$ ./manage.py makemigrations wcore (may only display "No changes detected in app 'wcore'"
         (.venv) user@host:~your_app$ ./manage.py migrate
-        (.venv) user@host:~your_app$ ./manage.py createsuperuser
+        (.venv) user@host:~your_app$ ./manage.py createsuperuser (then follow instructions)
 
 
     1.4. If everything is ok:
 
-    You can start your test server like this:
+    You can start your test server and job queue like this:
 
     .. code-block:: bash
 
-        (.venv) user@host:~your_app$ ./manage.py waves queue start
+        (.venv) user@host:~your_app$ ./manage.py wqueue start
         (.venv) user@host:~your_app$ ./manage.py runserver
 
+    Go to http://127.0.0.1:8000:admin to setup your services
+    WAVES comes with default front pages visible at http://127.0.0.1:8000
+
+
 2. Install WAVES-core inside existing Django project
 ----------------------------------------------------
 
     To create a Django project, have a look at `Django tutorial <https://docs.djangoproject.com/en/2.11/intro/tutorial01/>`_
 
+    .. seealso::
+
+        WAVES is a reusable app see: https://docs.djangoproject.com/en/1.11/intro/reusable-apps/#your-project-and-your-reusable-app
+
+
     2.0. Setup a virtualenv for your project:
 
         ``virtualenv ~/.venv/[waves_env]``
@@ -102,6 +126,7 @@ You can install WAVES-core either as a stand alone application, or inside any ex
             'polymorphic', # mandatory
             ...
             'waves.wcore', # mandatory
+            'waves.authentication', # mandatory if API token access needed
             'crispy_forms', # mandatory
             'rest_framework', # mandatory
             ...
@@ -143,7 +168,21 @@ You can install WAVES-core either as a stand alone application, or inside any ex
 
         To use this service with apache in mod_wsgi: please mind to enable "WSGIPassAuthorization On" parameter in conf
 
-3. Go for production:
----------------------
+3. Use other than SqlLite default DB layer:
+-------------------------------------------
+
+    You may need to install the Python and MySQL development headers and libraries like so:
+
+        - sudo apt-get install python-dev default-libmysqlclient-dev # Debian / Ubuntu
+        - sudo yum install python-devel mysql-devel # Red Hat / CentOS
+        - brew install mysql-connector-c # macOS (Homebrew) (Currently, it has bug. See below)
+
+    On Windows, there are binary wheels you can install without MySQLConnector/C or MSVC.
+
+    Then install pip mysql package in your virtualenv:
+
+        ``pip install mysqlclient``
+
+    .. seealso::
 
-    Please refer to `Django Official documentation <https://docs.djangoproject.com/fr/1.11/howto/deployment/>`_
+        https://docs.djangoproject.com/fr/1.11/ref/databases/

requirements.txt

@@ -1,19 +1,17 @@
-coreapi==2.3.3
 Django~=1.11
+coreapi==2.3.3
 daemons==1.3.0
 django-admin-sortable2==0.6.19
-django-environ==0.4.4
 django-cors-headers==2.2.0
 django-crispy-forms==1.7.2
 django-polymorphic==2.0.2
-djangorestframework==3.7.7
-backports.ssl-match-hostname==3.5.0.1
-git+https://github.com/marcoooo/django-crontab#egg=django_crontab
+djangorestframework==3.8.2
+backports.ssl-match-hostname
 inflection==0.3.1
-psutil==5.4.0
+psutil==5.4.5
 pycrypto==2.6.1
-python-magic==0.4.13
-saga-python==0.47
+python-magic==0.4.15
+saga-python==0.47.6
 setproctitle==1.1.10
 six==1.11.0
 swapper==1.1.0