Merge pull request #141 from CDCgov/2.0-integration

2.0 integration

CHANGELOG.md

@@ -1,12 +1,19 @@
 # Changelog
 
 
-## 2020/03/23 - v2.0.0-alpha.1: grid engine support
+## 2020/04/08 - v2.0.0-alpha.2: simplify CLI arguments
+
+1. Simplify and standardize CLI arguments
+
+
+## 2020/04/01 - v2.0.0-alpha.1: grid engine support
 
 1. Major enhancement: add support for Grid Engine (UGE, SGE, etc) execution.
 
 2. Add execution parameters options (in addition to context and method).
 
+3. Remove apps-repo.yaml file and add a required "apps" section to the workflow definition.
+
 
 ## 2020/03/16 - v1.13.6: bug fix release
 

README.md

@@ -1,6 +1,6 @@
 # GeneFlow
 
-Version: 2.0.0-alpha.1
+Version: 2.0.0-alpha.2
 
 GeneFlow (GF) is a light-weight platform-agnostic workflow engine for scientific computing.
 

VERSION

@@ -1 +1 @@
-2.0.0-alpha.1
+2.0.0-alpha.2

docs/conf.py

@@ -62,7 +62,7 @@
 # The short X.Y version.
 version = '2.0'
 # The full version, including alpha/beta/rc tags.
-release = '2.0.0-alpha.1'
+release = '2.0.0-alpha.2'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

docs/index.rst

@@ -3,7 +3,7 @@
 GeneFlow Documentation
 ======================
 
-Version: 2.0.0-alpha.1
+Version: 2.0.0-alpha.2
 
 GeneFlow (GF) is a light-weight platform-agnostic workflow engine for scientific computing.
 

setup.py

@@ -63,6 +63,7 @@ def run(self):
                        'data/migrations/20180821-01.py',
                        'data/migrations/20180907-01.py',
                        'data/migrations/20200228-01.py',
+                       'data/migrations/20200401-01.py',
                        'data/templates/app.yaml.j2.j2',
                        'data/templates/agave-app-def.json.j2.j2',
                        'data/templates/wrapper-script.sh.j2',

src/geneflow/__init__.py

@@ -12,5 +12,5 @@
 GF_PACKAGE_PATH = Path.resolve(Path(__file__)).parents[0]
 
 # version info
-__version_info__ = ('2', '0', '0-alpha', '1')
-__version__ = '.'.join(__version_info__)
+__version_info__ = ('2', '0', '0-alpha', '2')
+__version__ = '.'.join(__version_info__)

src/geneflow/__main__.py

@@ -44,39 +44,42 @@ def parse_args():
 
     # shared arguments
     parser.add_argument(
-        '--log_level',
+        '--log-level',
         type=str,
         default='info',
+        dest='log_level',
         help='logging level'
     )
     parser.add_argument(
-        '--log_file',
+        '--log-file',
         type=str,
         default=None,
+        dest='log_file',
         help='log file'
     )
 
     parser.set_defaults(func=None)
-    subparsers = parser.add_subparsers(help='Functions')
+    subparsers = parser.add_subparsers(help='Functions', dest='command')
+    subparser_dict = {}
 
     # configure arguments for sub-commands
-    geneflow.cli.add_apps.init_subparser(subparsers)
-    geneflow.cli.add_workflows.init_subparser(subparsers)
-    geneflow.cli.help.init_subparser(subparsers)
-    geneflow.cli.init_db.init_subparser(subparsers)
-    geneflow.cli.install_workflow.init_subparser(subparsers)
-    geneflow.cli.make_app.init_subparser(subparsers)
-    geneflow.cli.migrate_db.init_subparser(subparsers)
-    geneflow.cli.run.init_subparser(subparsers)
-    geneflow.cli.run_pending.init_subparser(subparsers)
+    subparser_dict['add-apps'] = geneflow.cli.add_apps.init_subparser(subparsers)
+    subparser_dict['add-workflows'] = geneflow.cli.add_workflows.init_subparser(subparsers)
+    subparser_dict['help'] = geneflow.cli.help.init_subparser(subparsers)
+    subparser_dict['init-db'] = geneflow.cli.init_db.init_subparser(subparsers)
+    subparser_dict['install-workflow'] = geneflow.cli.install_workflow.init_subparser(subparsers)
+    subparser_dict['make-app'] = geneflow.cli.make_app.init_subparser(subparsers)
+    subparser_dict['migrate-db'] = geneflow.cli.migrate_db.init_subparser(subparsers)
+    subparser_dict['run'] = geneflow.cli.run.init_subparser(subparsers)
+    subparser_dict['run-pending'] = geneflow.cli.run_pending.init_subparser(subparsers)
 
     # parse arguments
-    args = parser.parse_args()
-    if not args.func:
+    args = parser.parse_known_args()
+    if not args[0].func:
         parser.print_help()
         return False
 
-    return args
+    return args, subparser_dict[args[0].command]
 
 
 def main():
@@ -90,18 +93,22 @@ def main():
         Nothing.
 
     """
-    args = parse_args()
+    args, subparser = parse_args()
     if not args:
         sys.exit(1)
 
     # configure logging
-    Log.config(args.log_level, args.log_file)
+    Log.config(args[0].log_level, args[0].log_file)
 
     # display GeneFlow version
     Log.some().info('GeneFlow %s', __version__)
 
     # call the appropriate command
-    if not args.func(args):
+    if not args[0].func(
+            args=args[0],
+            other_args=args[1],
+            subparser=subparser
+    ):
         sys.exit(1)
 
     sys.exit(0)

src/geneflow/app_installer.py

@@ -7,6 +7,9 @@
 import shutil
 from git import Repo
 from git.exc import GitError
+import os
+from slugify import slugify
+import stat
 import yaml
 
 from geneflow.data_manager import DataManager
@@ -111,7 +114,7 @@ def __init__(
         Args:
             self: class instance
             path: local path to the app package
-            app: app information (git repo, tag, folder)
+            app: app information (name, git repo, version)
 
         Returns:
             None
@@ -168,9 +171,9 @@ def clone_git_repo(self):
 
         # clone app's git repo into target location
         try:
-            if self._app['tag']:
+            if self._app['version']:
                 Repo.clone_from(
-                    self._app['repo'], str(self._path), branch=self._app['tag'],
+                    self._app['repo'], str(self._path), branch=self._app['version'],
                     config='http.sslVerify=false'
                 )
             else:
@@ -270,6 +273,7 @@ def make_def(self):
                 None,
                 'app.yaml.j2.j2',
                 str(self._path / 'app.yaml.j2'),
+                slugify_name=slugify(self._config['name']),
                 **self._config
         ):
             Log.an().error('cannot compile GeneFlow app definition template')
@@ -296,6 +300,7 @@ def make_agave(self):
                 None,
                 'agave-app-def.json.j2.j2',
                 str(self._path / 'agave-app-def.json.j2'),
+                slugify_name=slugify(self._config['name']),
                 **self._config
         ):
             Log.an().error('cannot compile GeneFlow Agave app definition template')
@@ -320,21 +325,22 @@ def make_wrapper(self):
         asset_path = Path(self._path / 'assets')
         asset_path.mkdir(exist_ok=True)
 
-        Log.some().info(
-            'compiling %s',
-            str(asset_path / '{}.sh'.format(self._config['name']))
-        )
+        script_path = str(asset_path / '{}.sh'.format(slugify(self._config['name'])))
+        Log.some().info('compiling %s', script_path)
 
         # compile jinja2 template
         if not TemplateCompiler.compile_template(
                 None,
                 'wrapper-script.sh.j2',
-                str(asset_path / '{}.sh'.format(self._config['name'])),
+                script_path,
                 **self._config
         ):
             Log.an().error('cannot compile GeneFlow app wrapper script')
             return False
 
+        # make script executable by owner
+        os.chmod(script_path, stat.S_IRWXU)
+
         return True
 
 
@@ -354,18 +360,22 @@ def make_test(self):
         test_path = Path(self._path / 'test')
         test_path.mkdir(exist_ok=True)
 
-        Log.some().info('compiling %s', str(test_path / 'test.sh'))
+        script_path = str(test_path / 'test.sh')
+        Log.some().info('compiling %s', script_path)
 
         # compile jinja2 template
         if not TemplateCompiler.compile_template(
                 None,
                 'test.sh.j2',
-                str(test_path / 'test.sh'),
+                script_path,
                 **self._config
         ):
             Log.an().error('cannot compile GeneFlow app test script')
             return False
 
+        # make script executable by owner
+        os.chmod(script_path, stat.S_IRWXU)
+
         return True
 
 
@@ -659,10 +669,11 @@ def register_agave_app(self, agave_wrapper, agave_params, agave_publish):
 
         # delete app uri if it exists
         parsed_app_uri = URIParser.parse(
-            'agave://{}/{}/{}'.format(
+            'agave://{}/{}/{}-{}'.format(
                 agave_params['agave']['deploymentSystem'],
                 agave_params['agave']['appsDir'],
-                self._app['folder']
+                slugify(self._config['name']),
+                self._config['version']
             )
         )
         Log.some().info(

src/geneflow/cli/add_apps.py

@@ -30,8 +30,10 @@ def init_subparser(subparsers):
     )
     parser.set_defaults(func=add_apps)
 
+    return parser
 
-def add_apps(args):
+
+def add_apps(args, other_args, subparser=None):
     """
     Add GeneFlow apps to database.
 

src/geneflow/cli/add_workflows.py

@@ -17,7 +17,7 @@ def init_subparser(subparsers):
         help='geneflow definition yaml with workflows'
     )
     parser.add_argument(
-        '-c', '--config_file',
+        '-c', '--config',
         type=str,
         required=True,
         help='geneflow config file path'
@@ -30,14 +30,16 @@ def init_subparser(subparsers):
     )
     parser.set_defaults(func=add_workflows)
 
+    return parser
 
-def add_workflows(args):
+
+def add_workflows(args, other_args, subparser=None):
     """
     Add GeneFlow workflows to database.
 
     Args:
         args.workflow_yaml: GeneFlow definition with workflows.
-        args.config_file: GeneFlow config file path.
+        args.config: GeneFlow config file path.
         args.environment: Config environment.
 
     Returns:
@@ -46,13 +48,13 @@ def add_workflows(args):
 
     """
     workflow_yaml = args.workflow_yaml
-    config_file = args.config_file
+    config = args.config
     environment = args.environment
 
     # load config file
     cfg = Config()
-    if not cfg.load(config_file):
-        Log.an().error('cannot load config file: %s', config_file)
+    if not cfg.load(config):
+        Log.an().error('cannot load config file: %s', config)
         return False
 
     config_dict = cfg.config(environment)

src/geneflow/cli/help.py

@@ -18,6 +18,8 @@ def init_subparser(subparsers):
     )
     parser.set_defaults(func=help_func)
 
+    return parser
+
 
 def resolve_workflow_path(workflow_identifier):
     """
@@ -60,7 +62,7 @@ def resolve_workflow_path(workflow_identifier):
     return False
 
 
-def help_func(args):
+def help_func(args, other_args, subparser=None):
     """
     GeneFlow workflow help.
 

src/geneflow/cli/init_db.py

@@ -12,7 +12,7 @@ def init_subparser(subparsers):
         'init-db', help='initialize database'
     )
     parser.add_argument(
-        '-c', '--config_file',
+        '-c', '--config',
         type=str,
         required=True,
         help='geneflow config file path'
@@ -25,26 +25,28 @@ def init_subparser(subparsers):
     )
     parser.set_defaults(func=init_db)
 
+    return parser
 
-def init_db(args):
+
+def init_db(args, other_args, subparser=None):
     """
     Initialize SQLite DB schema.
 
     Args:
-        args.config_file: GeneFlow config file path.
+        args.config: GeneFlow config file path.
         args.environment: Config environment.
 
     Returns:
         On success: True.
         On failure: False.
 
     """
-    config_file = args.config_file
+    config = args.config
     environment = args.environment
 
     cfg = Config()
-    if not cfg.load(config_file):
-        Log.an().error('cannot load config file: %s', config_file)
+    if not cfg.load(config):
+        Log.an().error('cannot load config file: %s', config)
         return False
 
     config_dict = cfg.config(environment)