Merge branch 'izi-dev' into 'master'

Merge izi-dev into master

Closes #40

See merge request oneButton/uap!107

.gitignore

@@ -1,5 +1,4 @@
 # uap specific
-uap
 python_env
 python3_venv
 *.pyc
@@ -76,4 +75,4 @@ example-configurations/example-out/
 example-configurations/genomes/
 *-out
 
-.idea/
+.idea/

.gitlab-ci.yml

@@ -1,39 +1,60 @@
 stages:
   - build
   - test
-
-
-.dummy:
-  stage: build
-  script:
-    - pwd
-    - ls ~/.ssh/*
-    - git clone git@ribogit.izi.fraunhofer.de:oneButton/uap_test.git
-
+
+variables:
+  PIP_CACHE_DIR: "$CI_PROJECT_DIR/pip"
+  GIT_SUBMODULE_STRATEGY: recursive
 
 prepare:
   stage: build
   script:
-    - pwd
+    - module load Python/2.7.15-foss-2018b
     - ./bootstrap.sh
-
+    - source python_env/bin/activate
+    - pip install sphinx sphinx_rtd_theme
+    - deactivate
+    - module load Python/3.6.6-foss-2018b
+    - python3 -m venv venv
+    - source venv/bin/activate
+    - pip install --upgrade pip
+    - pip install pyaml pyyaml
+  cache:
+    key: python-envs
+    paths:
+      - python_env
+      - venv
+      - pip
   artifacts:
     paths:
       - python_env
+      - venv
     expire_in: 24 hours
 
 
 steptests:
   stage: test
   script:
-    - git clone git@ribogit.izi.fraunhofer.de:oneButton/uap_test.git 
-    - pwd
-    - cd  uap_test
-    - make clean
     - module load Python/3.6.6-foss-2018b
-    - python3 -m venv venv
     - source venv/bin/activate
-    - pip install pyaml
-    - python3 scripts/uap_test.py run-tests --uap-test-dir . --uap-path ../uap 
+    - cd uap_test
+    - python3 scripts/uap_test.py run-tests --uap-test-dir . --uap-path ../uap
+  dependencies:
+    - prepare
+
+pages:
+  stage: test
+  script:
+    - module load Python/2.7.15-foss-2018b
+    - source python_env/bin/activate
+    - sphinx-build -b html doc/source public/$CI_COMMIT_REF_SLUG
+    - doc/make_pages_index.sh public
+  cache:
+    key: pages-cache
+    paths:
+      - public
   dependencies:
-    - prepare
+    - prepare
+  artifacts:
+    paths:
+    - public

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "uap_test"]
+	path = uap_test
+	url = ../uap_test.git

CHANGELOG.md

@@ -0,0 +1,22 @@
+## 1.1 (20.01.2020)
+
+**Fixed**
+ * tests in uap_test repo (#113)
+ * CI pipeline (#110)
+ * automatic volatilization (#98)
+ * fastqscreen: move html output files (#95)
+ * removed option --optional in patched fastq_screen version (#94)
+ * display correct uap version (#93)
+ * deprecated warning from python package PyYAML (#91)
+ * _cluster_job_quota is not read on slurm (#40)
+ * fastq_screen: forgot to modify nohits option (#30)
+ * fixed fastqscreen and rseqc file path issues (#120)
+
+**Features**
+ * tools sections defaults (#103)
+
+**additional stuff**
+ * fastq_screen is not running on ribnode018 (#97)
+ * slurm cluster gives finished for failed runs (#60)
+ * released documentation with gitlab pages (#117)
+ * added uap_test as git submodule and modify gitlab ci process (#108)

README.md

@@ -0,0 +1,33 @@
+# uap -- Universal Analysis Pipeline
+
+The **uap** package is a framework to configure, run, and control
+large data multi-step analyses.
+Its main focus is on the analysis of high-throughput sequencing data.
+
+The aim of this data processing pipeline is to enable robust and straightforward
+bioinformatics data evaluation.
+It is implemented in Python, runs under GNU/Linux and can be controlled from the
+command-line interface.
+Although the primary focus is the evaluation of sequencing data, its design
+allows for a variety of other applications.
+
+
+# Documentation
+
+The documentation of **uap** is available as [Giltab Page](https://onebutton.ribogitpages.izi.fraunhofer.de/uap/).
+
+# Local Testing
+
+In order to use the testing repo [uap_test](https://ribogit.izi.fraunhofer.de/oneButton/uap_test)
+locally you have to change the file .gitmodules
+by replacing `url = ../uap_test.git` with `url = git@ribogit.izi.fraunhofer.de:oneButton/uap_test.git`.
+The entry for `uap_test` looks like this:
+```
+[submodule "uap_test"]
+        path = uap_test
+        url = git@ribogit.izi.fraunhofer.de:oneButton/uap_test.git
+```
+
+Then you can run `git submodule sync && git checkout -- .gitmodules && git submodule update`
+to have the testing repo in `uap_test`. Please consult tutorials for submodules for further info.
+E.g., https://git-scm.com/book/en/v2/Git-Tools-Submodules.

README.rst

@@ -1,21 +1,6 @@
 uap -- Universal Analysis Pipeline
 ==================================
 
-|docs| |travis|
-
-.. |docs| image:: https://readthedocs.org/projects/izi-uap/badge/?version=latest
-    :alt: Documentation Status
-    :scale: 100%
-    :target: http://izi-uap.readthedocs.io/en/latest/?badge=latest
-
-.. |travis| image:: https://travis-ci.org/alexanderscholz/uap.svg?branch=master
-    :alt: Travis CI Build Status
-    :scale: 100%
-    :target: https://travis-ci.org/alexanderscholz/uap
-
-Introduction
-============
-
 The **uap** package is a framework to configure, run, and control
 large data multi-step analyses.
 Its main focus is on the analysis of high-throughput sequencing data.
@@ -31,4 +16,4 @@ allows for a variety of other applications.
 Documentation
 =============
 
-The documentation of **uap** is available at `Read the Docs <http://izi-uap.readthedocs.io/en/latest/?badge=latest>`_.
+The documentation of **uap** is available as `Giltab Page <https://onebutton.ribogitpages.izi.fraunhofer.de/uap/>`_.

doc/make_pages_index.sh

@@ -0,0 +1,21 @@
+#! /bin/bash -
+
+cd "$1"
+
+printf '
+<html>
+<body>
+<h2>UAP Sphinx Docu per branch</h2>
+<p>
+' > index.html
+
+for file in $(ls -1); do
+    [[ "$file" == "index.html" ]] && continue
+    printf '<li><a href=%s>%s</a></li>' "$file" "$file" >> index.html
+done
+
+printf '
+</p>
+</body>
+</html>
+' >> index.html

doc/source/conf.py

@@ -42,7 +42,7 @@
     'sphinx.ext.doctest',
     'sphinx.ext.intersphinx',
     'sphinx.ext.graphviz',
-    'sphinx.ext.pngmath',
+    'sphinx.ext.imgmath',
     'sphinx.ext.todo',
     'sphinx.ext.viewcode'
 ]
@@ -64,16 +64,16 @@
 # General information about the project.
 project = u'uap'
 copyright = u'2015, Christoph Kämpf, Michael Specht'
-author = u'Christoph Kämpf, Michael Specht'
+author = u'Christoph Kämpf, Michael Specht, Alexander Scholz, Dominik Otto'
 
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the
 # built documents.
 #
 # The short X.Y version.
-version = '0.1'
+version = '1.1'
 # The full version, including alpha/beta/rc tags.
-release = '0.1.1'
+release = '1.1.1'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

include/abstract_step.py

@@ -907,31 +907,27 @@ def run(self, run_id):
                 attachment['data'] = open(annotation_path + '.png').read()
             self.get_pipeline().notify(message, attachment)
 
-            # and now... check whether we have any volatile parents. If we find
-            # one, determine for each of its output files A whether all output
-            # files B which depend on A are already in place and whether the
-            # task which produced the output file B is finished. In that case,
-            # we can truncate output file A and rename it to act as a 'volatile
-            # placeholder'.
-            task_id = '%s/%s' % (self, run_id)
-            input_files = set()
-            if task_id in self.get_pipeline().input_files_for_task_id:
-                input_files = self.get_pipeline().input_files_for_task_id[task_id]
-            candidate_tasks = set()
-            # Only source steps do have empty strings in the input files list
-            # so we can savely exclude them here
-            for inpath in [x for x in input_files if x != '']:
-                task_id = self.get_pipeline().task_id_for_output_file[inpath]
-                if task_id in self.get_pipeline().task_for_task_id:
-                    task = self.get_pipeline().task_for_task_id[task_id]
-                    if task.step._options['_volatile'] == True:
-                        candidate_tasks.add(task)
-
-            for task in candidate_tasks:
-                task.volatilize_if_possible(srsly = True)
-
             self._reset()
 
+    def get_parents(self, run_id):
+        '''
+        Returns all tasks that produce input for this task.
+        '''
+        task_id = '%s/%s' % (self, run_id)
+        input_files = set()
+        if task_id in self.get_pipeline().input_files_for_task_id:
+            input_files = self.get_pipeline().input_files_for_task_id[task_id]
+        parents = set()
+        # Only source steps do have empty strings in the input files list
+        # so we can safely exclude them here
+        for inpath in [x for x in input_files if x != '']:
+            task_id = self.get_pipeline().task_id_for_output_file[inpath]
+            if task_id in self.get_pipeline().task_for_task_id:
+                task = self.get_pipeline().task_for_task_id[task_id]
+                if task.step._options['_volatile'] == True:
+                    parents.add(task)
+        return parents
+
     def reports(self, run_id, out_connection_output_files):
         '''
         Abstract method this must be implemented by actual step.

include/fscache.py

@@ -33,7 +33,7 @@ def load_yaml_from_file(self, path):
             return self.cache['load_yaml_from_file'][path]
 
         f = open(path, 'r')
-        data = yaml.load(f)
+        data = yaml.load(f, Loader=yaml.FullLoader)
         f.close()
         self.cache['load_yaml_from_file'][path] = data
         return data