Merge pull request #31 from BeamIO-Inc/integration

Merge of updates to support new project
BeamIO-Inc · Feb 10, 2021 · 35bb4ae · 35bb4ae
2 parents 707ca09 + 1e4ff6e
commit 35bb4ae
Show file tree

Hide file tree

Showing 20 changed files with 609 additions and 221 deletions.
diff --git a/README.md b/README.md
@@ -1,4 +1,4 @@
-# Welcome to the Algorithm Toolkit (BETA)!
+# Welcome to the Algorithm Toolkit!
 
 We built the Algorithm Toolkit (ATK) to provide researchers and data anaysts the tools they need to process data quickly and efficiently, so they can focus on actually doing scientific or other analysis work.
 
@@ -169,6 +169,17 @@ sudo yum install python3-devel
 sudo yum install python-devel
 ```
 
+# Docker
+
+This repository contains 2 Docker files
+
+* [atk](/docker/atk/Dockerfile)
+* [atk-dev](/docker/atk-dev/Dockerfile)
+
+Once built and run, Docker containers are generated, which gives users access to easy to use development/deployment tools.
+
+Check out our official [Docker documentation](https://algorithm-toolkit.readthedocs.io/en/latest/docker.html) to learn more
+
 # Contributing
 
 Thanks for your interest in contributing to the Algorithm Toolkit codebase! You should know a few things.

diff --git a/algorithm_toolkit/__init__.py b/algorithm_toolkit/__init__.py
@@ -48,7 +48,7 @@
 app.config['CORS_ORIGIN_WHITELIST'].append('https://tdprocess.com')
 app.config['TILEDRIVER_URL'] = 'https://app.tiledriver.com/'
 
-from .views.home import home, main, chain_run_status
+from .views.home import home, main, run_chain, chain_run_status
 from .views.manage import manage
 
 app.register_blueprint(home)
@@ -58,6 +58,7 @@
 
 csrf = CSRFProtect(app)
 csrf.exempt(main)
+csrf.exempt(run_chain)
 csrf.exempt(chain_run_status)
 
 

diff --git a/algorithm_toolkit/templates/base.html b/algorithm_toolkit/templates/base.html
@@ -13,7 +13,7 @@
     </head>
     <body>
         <nav class="navbar navbar-expand-md navbar-dark fixed-top bg-dark">
-          <a class="navbar-brand" data-badge="BETA" href="/">Algorithm Toolkit</a>
+          <a class="navbar-brand" href="/">Algorithm Toolkit</a>
           <button class="navbar-toggler" type="button" data-toggle="collapse" data-target="#atkNavbar" aria-controls="navbarsExampleDefault" aria-expanded="false" aria-label="Toggle navigation">
             <span class="navbar-toggler-icon"></span>
           </button>

diff --git a/algorithm_toolkit/templates/test_run.html b/algorithm_toolkit/templates/test_run.html
@@ -270,7 +270,7 @@ <h4 class="modal-title" id="error-modal-title"></h4>
         }
 
         var xhr = new XMLHttpRequest();
-        xhr.open('POST', thisProtocol + '//' + thisHost + '/main/', true);
+        xhr.open('POST', thisProtocol + '//' + thisHost + '/' + 'chains' + '/' + '{{ chain_name }}' + '/', true);
         xhr.setRequestHeader("X-CSRFToken", csrf_token);
         xhr.onload = function(evt) {
             var error = false;

diff --git a/algorithm_toolkit/utils/file_utils.py b/algorithm_toolkit/utils/file_utils.py
@@ -42,15 +42,25 @@ def get_algorithm(path):
 
 
 def get_chain_def(path, chain_name=None):
+    chain_path = os.path.join(path, 'chains')
+    if not os.path.exists(chain_path):
+        os.makedirs(chain_path)
+
     chain_defs = {}
-    try:
-        with open(os.path.join(path, 'chains.json'), 'r') as chain_file:
-            try:
-                chain_defs = json.load(chain_file)
-            except ValueError:
-                return {}
-    except IOError:
-        return {}
+
+    for root, dirs, files in os.walk(chain_path):
+        for f in files:
+            if '.json' in f:
+                temp_key = f.replace('.json', '')
+                try:
+                    with open(os.path.join(chain_path, f)) as the_file:
+                        try:
+                            temp_obj = json.load(the_file)
+                            chain_defs[temp_key] = temp_obj[temp_key]
+                        except ValueError:
+                            return {}
+                except IOError:
+                    return {}
 
     if chain_name:
         try:
@@ -61,6 +71,27 @@ def get_chain_def(path, chain_name=None):
         return chain_defs
 
 
+def clear_chains(path):
+    chain_path = os.path.join(path, 'chains')
+
+    for root, dirs, files in os.walk(chain_path):
+        for f in files:
+            try:
+                os.unlink(os.path.join(chain_path, f))
+            except IOError:
+                pass
+
+
+def save_chain_files(path, chains):
+    for k, v in chains.items():
+        try:
+            with open(os.path.join(path, 'chains', k + '.json'), 'w') as f:
+                temp_json = {k: v}
+                json.dump(temp_json, f, indent=4, separators=(',', ': '))
+        except IOError:
+            pass
+
+
 def get_json_path(path, a):
     a = a.replace('\\', os.sep).replace('/', os.sep)
     return os.path.join(path, 'algorithms', a, 'algorithm.json')
diff --git a/algorithm_toolkit/utils/home_utils.py b/algorithm_toolkit/utils/home_utils.py
@@ -0,0 +1,140 @@
+import requests
+import json
+import os
+
+from flask import (
+    make_response,
+    jsonify
+)
+from .. import (
+    AlgorithmChain,
+    app
+)
+from .data_utils import create_random_string
+from .file_utils import (
+    get_chain_def,
+    make_dir_if_not_exists
+)
+
+
+def check_chain_request_form(request, chain_name, path):
+
+    chain = None
+    status_key = None
+    run_mode = None
+    iter_param = None
+    iter_type = None
+    iter_value = None
+
+    if request.method == 'POST':
+        try:
+            chain = request.form['chain']
+        except KeyError:
+            chain = None
+
+        try:
+            status_key = request.form['status_key']
+        except KeyError:
+            status_key = None
+
+        if 'run_mode' in request.form:
+            if request.form['run_mode'] == 'batch':
+                try:
+                    run_mode = 'batch'
+                    iter_param = request.form['iter_param']
+                    iter_type = request.form['iter_type']
+                    iter_value = request.form['iter_value']
+                except KeyError:
+                    return make_response('Batch mode misconfigured', 400)
+            else:
+                run_mode = 'single'
+        else:
+            run_mode = 'single'
+    elif request.method == 'GET':  # pragma: no branch
+        chain = request.args.get('chain', None)
+        status_key = request.args.get('status_key', None)
+        run_mode = request.args.get('run_mode', 'single')
+        iter_param = request.args.get('iter_param', None)
+        iter_type = request.args.get('iter_type', None)
+        iter_value = request.args.get('iter_value', None)
+
+    if status_key is None:
+        status_key = create_random_string(http_safe=True)
+
+    if chain is None:
+        return make_response('Missing chain parameter in request', 400)
+
+    if chain.lower() == 'from_global':
+        try:
+            chain = app.config['CHAIN_DATA'].pop(status_key)
+        except ValueError:
+            return make_response(
+                'Chain parameter not present in app configuration', 400)
+    else:
+        try:
+            chain = json.loads(chain)
+        except ValueError:
+            return make_response('Chain parameter not properly formatted', 400)
+
+    if chain_name not in get_chain_def(path):
+        return make_response('Chain name does not exist', 400)
+
+    if 'algorithms' not in chain:
+        return make_response('Algorithms not defined', 400)
+
+    return ({
+        'chain': chain,
+        'status_key': status_key,
+        'run_mode': run_mode,
+        'iter_param': iter_param,
+        'iter_type': iter_type,
+        'iter_value': iter_value
+    })
+
+
+def process_chain_request(checked_response, path):
+
+    chain = checked_response['chain']
+    status_key = checked_response['status_key']
+    run_mode = checked_response['run_mode']
+    iter_param = checked_response['iter_param']
+    iter_type = checked_response['iter_type']
+    iter_value = checked_response['iter_value']
+
+    c_obj = AlgorithmChain(path, chain)
+    if c_obj.chain_definition == {}:
+        return make_response('Chain name not found', 404)
+
+    cl = c_obj.create_ledger(status_key)
+    cl.make_working_folders()
+
+    if run_mode == 'single':
+        response = c_obj.call_chain_algorithms()
+        save_fname = status_key + '.json'
+
+        if 'CHAIN_LEDGER_HISTORY_PATH' in app.config:
+            save_path = os.path.join(
+                app.config['CHAIN_LEDGER_HISTORY_PATH'], save_fname)
+        else:
+            make_dir_if_not_exists(os.path.join(path, 'history'))
+            save_path = os.path.join(path, 'history', save_fname)
+        c_obj.chain_ledger.save_history_to_json(save_path, pretty=True)
+
+        if 'CHAIN_HISTORY' in app.config:
+
+            if app.config['CHAIN_HISTORY_LENGTH'] > 0:
+                ch = app.config['CHAIN_HISTORY']
+
+                if len(ch) > app.config['CHAIN_HISTORY_LENGTH']:
+                    ch.popitem(last=False)
+
+                ch[status_key] = c_obj.chain_ledger
+    else:
+        response = c_obj.call_batch(iter_param, iter_type, iter_value)
+
+    cl.remove_working_folders()
+
+    if response['output_type'] == 'error':
+        return make_response(jsonify(response), 400)
+
+    return jsonify(response)