Merge pull request #119 from hed-standard/develop

Updated with the new interface

deploy_hed/Dockerfile

@@ -10,8 +10,9 @@ libxml2-dev \
 apache2 \
 apache2-dev && \
 pip3 install --upgrade pip && \
-pip3 install --no-cache-dir -r requirements.txt && \
-pip3 install git+https://github.com/hed-standard/hed-python/@master && \
+pip3 install -r requirements.txt
+ARG CACHE_BUST
+RUN pip3 install git+https://github.com/hed-standard/hed-python/@master && \
 mkdir -p /var/www/localhost/htdocs && \
 cp /etc/mime.types /var/www/mime.types && \
 mkdir -p /var/log/hedtools && \

deploy_hed/deploy.sh

@@ -65,7 +65,7 @@ cd "${DEPLOY_DIR}" || error_exit "Cannot access $DEPLOY_DIR"
 build_new_container()
 {
 echo "Building new container ${IMAGE_NAME} ..."
-docker build -t $IMAGE_NAME .
+docker build -t $IMAGE_NAME . --build-arg CACHE_BUST=$(date +%s)
 }
 
 delete_old_container()

deploy_hed/requirements.txt

@@ -6,7 +6,6 @@ defusedxml==0.7.1
 et-xmlfile==1.1.0
 Flask==2.1.2
 Flask-WTF==1.0.1
-hedtools==0.1.0
 inflect>=5.5.2
 itsdangerous==2.1.2
 jdcal==1.4.1

deploy_hed_dev/Dockerfile

@@ -10,8 +10,9 @@ libxml2-dev \
 apache2 \
 apache2-dev && \
 pip3 install --upgrade pip && \
-pip3 install --no-cache-dir -r requirements.txt && \
-pip3 install git+https://github.com/hed-standard/hed-python/@develop && \
+pip3 install -r requirements.txt
+ARG CACHE_BUST
+RUN pip3 install git+https://github.com/hed-standard/hed-python/@develop && \
 mkdir -p /var/www/localhost/htdocs && \
 cp /etc/mime.types /var/www/mime.types && \
 mkdir -p /var/log/hedtools && \

deploy_hed_dev/deploy.sh

@@ -65,7 +65,7 @@ cd "${DEPLOY_DIR}" || error_exit "Cannot access $DEPLOY_DIR"
 build_new_container()
 {
 echo "Building new container ${IMAGE_NAME} ..."
-docker build -t $IMAGE_NAME .
+docker build -t $IMAGE_NAME . --build-arg CACHE_BUST=$(date +%s)
 }
 
 delete_old_container()

hedweb/columns.py

@@ -148,23 +148,25 @@ def get_prefix_dict(form_dict):
         form_dict (dict): The dictionary returned from a form that contains a column prefix table.
 
     Returns:
-        dict: A dictionary whose keys names (or COLUMN_XX) and values are tag prefixes to prepend.
+        list: List of selected columns
+        dict: 
 
+    Note: The form counts columns starting from 1 but prefix dictionary starts with index 0.
     """
     tag_columns = []
     prefix_dict = {}
     keys = form_dict.keys()
     for key in keys:
-        if not key.startswith('column') or key.endswith('check'):
+        index_check = key.rfind('_check')
+        if index_check == -1 or form_dict[key] != 'on':
             continue
-        pieces = key.split('_')
-        check = 'column_' + pieces[1] + '_check'
-        if form_dict.get(check, None) != 'on':
-            continue
-        if form_dict[key]:
-            prefix_dict[int(pieces[1])] = form_dict[key]
+        pieces = key.split("_")
+        column_number = int(pieces[1])
+        info_key = key[0: index_check] + "_input"
+        if form_dict.get(info_key, None):
+            prefix_dict[column_number] = form_dict[info_key]
         else:
-            tag_columns.append(int(pieces[1]))
+            tag_columns.append(column_number)
     return tag_columns, prefix_dict
 
 

hedweb/routes.py

@@ -7,7 +7,12 @@
 from hedweb.constants import base_constants, page_constants
 from hedweb.constants import route_constants, file_constants
 from hedweb.web_util import handle_http_error, handle_error, package_results
-from hedweb import sidecar, events, spreadsheet, services, strings, schema
+from hedweb import sidecar as sidecar
+from hedweb import events as events
+from hedweb import spreadsheet as spreadsheet
+from hedweb import services as services
+from hedweb import strings as strings
+from hedweb import schema as schema
 from hedweb.columns import get_columns_request
 
 app_config = current_app.config
@@ -152,7 +157,9 @@ def sidecar_results():
     try:
         input_arguments = sidecar.get_input_from_form(request)
         a = sidecar.process(input_arguments)
-        return package_results(a)
+        b = package_results(a)
+        return b
+        # return package_results(a)
     except Exception as ex:
         return handle_http_error(ex)
 

hedweb/schema.py

@@ -7,8 +7,7 @@
 from hed import schema as hedschema
 from hed.errors import get_printable_issue_string
 from hed.errors import HedFileError
-from hed.tools.util.io_util import generate_filename
-from hedweb.web_util import form_has_file, form_has_option, form_has_url
+from hedweb.web_util import form_has_file, form_has_option, form_has_url, generate_filename
 from hedweb.constants import base_constants, file_constants
 
 app_config = current_app.config
@@ -38,7 +37,8 @@ def get_schema(arguments):
         else:
             file_found = False
     except HedFileError as e:
-        issues.append({'code': e.args[0], 'message': e.args[1]})
+        # issues.append({'code': e.args[0], 'message': e.args[1]})
+        issues = issues + e.issues
     if not file_found:
         raise HedFileError("SCHEMA_NOT_FOUND", "Must provide a loadable schema", "")
     return hed_schema, issues
@@ -96,9 +96,9 @@ def process(arguments):
     display_name = arguments.get('schema_display_name', 'unknown_source')
     hed_schema, issues = get_schema(arguments)
     if issues:
-        issue_str = get_issue_string(issues, f"Schema for {display_name} had these errors")
+        issue_str = get_issue_string(issues, f"Schema for {display_name} had these issues")
         file_name = generate_filename(arguments[base_constants.SCHEMA_DISPLAY_NAME],
-                                      name_suffix='schema__errors', extension='.txt')
+                                      name_suffix='schema_issues', extension='.txt')
         return {'command': arguments[base_constants.COMMAND],
                 base_constants.COMMAND_TARGET: 'schema',
                 'data': issue_str, 'output_display_name': file_name,
@@ -159,21 +159,21 @@ def schema_validate(hed_schema, display_name):
 
     issues = hed_schema.check_compliance()
     if issues:
-        issue_str = get_printable_issue_string(issues, f"Schema HED 3G compliance errors for {display_name}:")
-        file_name = generate_filename(display_name, name_suffix='schema_3G_compliance_errors', extension='.txt')
+        issue_str = get_printable_issue_string(issues, f"Schema issues for {display_name}:")
+        file_name = generate_filename(display_name, name_suffix='schema_issues', extension='.txt')
         return {'command': base_constants.COMMAND_VALIDATE,
                 base_constants.COMMAND_TARGET: 'schema',
                 'data': issue_str, 'output_display_name': file_name,
                 'schema_version': hed_schema.get_formatted_version(as_string=True),
                 'msg_category': 'warning',
-                'msg': 'Schema is not HED 3G compliant'}
+                'msg': 'Schema has validation issues'}
     else:
         return {'command': base_constants.COMMAND_VALIDATE,
                 base_constants.COMMAND_TARGET: 'schema',
                 'data': '', 'output_display_name': display_name,
                 'schema_version': hed_schema.get_formatted_version(as_string=True),
                 'msg_category': 'success',
-                'msg': 'Schema had no HED-3G validation errors'}
+                'msg': 'Schema had no validation issues'}
 
 
 def get_issue_string(issues, title=None):

hedweb/services.py

@@ -2,11 +2,17 @@
 import io
 import json
 from flask import current_app
-from hed.models import HedString, Sidecar, SpreadsheetInput, TabularInput
+from hed.models.hed_string import HedString
+from hed.models.sidecar import Sidecar
+from hed.models.spreadsheet_input import SpreadsheetInput
+from hed.models.tabular_input import TabularInput
 from hed.errors import HedFileError
 from hed import schema as hedschema
 from hedweb.constants import base_constants
-from hedweb import events, spreadsheet, sidecar, strings
+from hedweb import events as events
+from hedweb import spreadsheet as spreadsheet
+from hedweb import sidecar as sidecar
+from hedweb import strings as strings
 
 
 app_config = current_app.config
@@ -81,8 +87,7 @@ def get_sidecar(arguments, params):
         file_list = []
         for s_string in sidecar_list:
             file_list.append(io.StringIO(s_string))
-        schema = arguments.get('schema', None)
-        arguments[base_constants.SIDECAR] = Sidecar(files=file_list, name="Merged_Sidecar", hed_schema=schema)
+        arguments[base_constants.SIDECAR] = Sidecar(files=file_list, name="Merged_Sidecar")
     else:
         arguments[base_constants.SIDECAR] = None
 
@@ -102,14 +107,14 @@ def get_input_objects(arguments, params):
     if base_constants.EVENTS_STRING in params and params[base_constants.EVENTS_STRING]:
         arguments[base_constants.EVENTS] = \
             TabularInput(file=io.StringIO(params[base_constants.EVENTS_STRING]),
-                         sidecar=arguments.get(base_constants.SIDECAR, None), name='Events', hed_schema=schema)
+                         sidecar=arguments.get(base_constants.SIDECAR, None), name='Events')
     if base_constants.SPREADSHEET_STRING in params and params[base_constants.SPREADSHEET_STRING]:
         tag_columns, prefix_dict = spreadsheet.get_prefix_dict(params)
         has_column_names = arguments.get(base_constants.HAS_COLUMN_NAMES, None)
         arguments[base_constants.SPREADSHEET] = \
             SpreadsheetInput(file=io.StringIO(params[base_constants.SPREADSHEET_STRING]), file_type=".tsv",
                              tag_columns=tag_columns, has_column_names=has_column_names,
-                             column_prefix_dictionary=prefix_dict, name='spreadsheet.tsv', hed_schema=schema)
+                             column_prefix_dictionary=prefix_dict, name='spreadsheet.tsv')
     if base_constants.STRING_LIST in params and params[base_constants.STRING_LIST]:
         s_list = []
         for s in params[base_constants.STRING_LIST]:
@@ -241,7 +246,7 @@ def package_spreadsheet(results):
 
 
     """
-    if results['msg_category'] == 'success' and base_constants.SPREADSHEET in results:
+    if results['msg_category'] == 'success' and results.get(base_constants.SPREADSHEET, ''):
         results[base_constants.SPREADSHEET] = results[base_constants.SPREADSHEET].to_csv(file=None)
     elif base_constants.SPREADSHEET in results:
         del results[base_constants.SPREADSHEET]