Merge pull request #144 from CDCgov/integration

Integration

CHANGELOG.md

@@ -1,6 +1,13 @@
 # Changelog
 
 
+## 2020/04/15 - v1.13.7: bug fix release
+
+1. Fix bug: Run pending jobs CLI does not change status of pending jobs fast enough.
+
+2. Fix bug: App version numbers with invalid characters cause Agave registration to fail.
+
+
 ## 2020/03/16 - v1.13.6: bug fix release
 
 1. Fix bug: Non-required inputs to Agave jobs cannot be passed as empty strings.

README.md

@@ -1,6 +1,6 @@
 # GeneFlow
 
-Version: 1.13.6
+Version: 1.13.7
 
 GeneFlow (GF) is a light-weight platform-agnostic workflow engine for scientific computing.
 

VERSION

@@ -1 +1 @@
-1.13.6
+1.13.7

docs/conf.py

@@ -62,7 +62,7 @@
 # The short X.Y version.
 version = '1.13'
 # The full version, including alpha/beta/rc tags.
-release = '1.13.6'
+release = '1.13.7'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

docs/index.rst

@@ -3,7 +3,7 @@
 GeneFlow Documentation
 ======================
 
-Version: 1.13.6
+Version: 1.13.7
 
 GeneFlow (GF) is a light-weight platform-agnostic workflow engine for scientific computing.
 

src/geneflow/__init__.py

@@ -12,5 +12,5 @@
 GF_PACKAGE_PATH = Path.resolve(Path(__file__)).parents[0]
 
 # version info
-__version_info__ = ('1', '13', '6')
+__version_info__ = ('1', '13', '7')
 __version__ = '.'.join(__version_info__)

src/geneflow/app_installer.py

@@ -8,6 +8,7 @@
 from git import Repo
 from git.exc import GitError
 import yaml
+from slugify import slugify
 
 from geneflow.data_manager import DataManager
 from geneflow.log import Log
@@ -218,6 +219,15 @@ def load_config(self):
             CONFIG_SCHEMA[GF_VERSION]
         )
 
+        # check formatting of version
+        self._config['agave_version'] = slugify(self._config['version'].lower()).replace('-','.')
+        if self._config['agave_version'].islower():
+            # contains letters, invalid version
+            Log.an().error(
+                'app config validation error: app version cannot contain letters'
+            )
+            return False
+
         if not valid_def:
             Log.an().error(
                 'app config validation error:\n%s',

src/geneflow/cli/run_pending.py

@@ -87,6 +87,15 @@ def run_pending(args):
         'pending jobs found:\n%s', pprint.pformat(pending_jobs)
     )
 
+    # set job status to QUEUED to minimize the chance that another
+    # process will try to run it
+    for job in pending_jobs:
+        if not data_source.update_job_status(job['id'], 'QUEUED', ''):
+            Log.a().warning('cannot update job status in data source')
+            data_source.rollback()
+        data_source.commit()
+
+    # create a thread pool to run at most 5 jobs concurrently
     pool = Pool(min(5, len(pending_jobs)))
     jobs = [
         {

src/geneflow/data/templates/agave-app-def.json.j2.j2

@@ -1,6 +1,6 @@
 {
   "name": "{{ '{{' }} agave.appsPrefix {{ '}}' }}-{{ name }}",
-  "version": "{{ version }}",
+  "version": "{{ agave_version }}",
   "label": "{{ description }}",
   "defaultNodeCount": 1,
   "defaultMemoryPerNode": 1,

src/geneflow/data/templates/app.yaml.j2.j2

@@ -30,7 +30,7 @@ parameters:
 definition:
 {{ '{%' }} if agave {{ '%}' }}
   agave:
-    agave_app_id: {{ '{{' }} agave.appsPrefix {{ '}}' }}-{{ name }}-{{ version }}{{ '{{' }} revision {{ '}}' }}
+    agave_app_id: {{ '{{' }} agave.appsPrefix {{ '}}' }}-{{ name }}-{{ agave_version }}{{ '{{' }} revision {{ '}}' }}
 {{ '{%' }} endif {{ '%}' }}
   local:
     script: {{ name }}.sh