Merge pull request #1098 from sandialabs/project-data-toggle

Project data toggle and HDF5

packages/slycat/web/server/engine.py

@@ -113,7 +113,6 @@ def abspath(path):
   dispatcher.connect("put-project-csv-data", "/projects/:pid/data/:file_key/parser/:parser/mid/:mid/aids/:aids", slycat.web.server.handlers.put_project_csv_data, conditions={"method": ["PUT"]})
   dispatcher.connect("get-project-data", "/projects/data/:did", slycat.web.server.handlers.get_project_data, conditions={"method": ["GET"]})
   dispatcher.connect("put-project-data-parameter", "/data/:did/aids/:aid", slycat.web.server.handlers.put_project_data_parameter, conditions={"method": ["PUT"]})
-  dispatcher.connect("post-project-data", "/projects/data/:pid", slycat.web.server.handlers.create_project_data_from_pid, conditions={"method": ["POST"]})
   dispatcher.connect("get-project-data-in-model", "/projects/data/model/:mid", slycat.web.server.handlers.get_project_data_in_model, conditions={"method": ["GET"]})
   dispatcher.connect("get-project-file-names", "/projects/:pid/name", slycat.web.server.handlers.get_project_file_names, conditions={"method": ["GET"]})
   dispatcher.connect("get-project-data-parameter", "/projects/data/:did/parameters/:param", slycat.web.server.handlers.get_project_data_parameter, conditions={"method": ["GET"]})
@@ -174,7 +173,7 @@ def abspath(path):
 
   dispatcher.connect("post-uploads", "/uploads", slycat.web.server.handlers.post_uploads, conditions={"method" : ["POST"]})
   dispatcher.connect("put-upload-file-part", "/uploads/:uid/files/:fid/parts/:pid", slycat.web.server.handlers.put_upload_file_part, conditions={"method" : ["PUT"]})
-  dispatcher.connect("post-upload-finshed", "/uploads/:uid/finished", slycat.web.server.handlers.post_upload_finished, conditions={"method" : ["POST"]})
+  dispatcher.connect("post-upload-finished", "/uploads/:uid/finished", slycat.web.server.handlers.post_upload_finished, conditions={"method" : ["POST"]})
 
   dispatcher.connect("clean-project-data", "/clean/project-data", slycat.web.server.handlers.clean_project_data, conditions={"method" : ["GET"]})
   dispatcher.connect("clear-ssh-sessions", "/clear/ssh-sessions", slycat.web.server.handlers.clear_ssh_sessions, conditions={"method" : ["GET"]})

packages/slycat/web/server/handlers.py

@@ -36,6 +36,9 @@
 import uuid
 import functools
 import datetime
+import h5py
+import csv
+import tempfile
 from urllib.parse import urlparse, urlencode, parse_qs
 
 # decode base64 byte streams for file upload
@@ -387,11 +390,38 @@ def put_project_csv_data(pid, file_key, parser, mid, aids):
         for item in project_datas:
             if item["project"] == pid and item["file_name"] == file_key:
                 fid = item["_id"]
-                http_response = database.get_attachment(item, "content")
-                file = http_response.read()
-                # Must convert from bytes to string
-                file = str(file, 'utf-8')
-                attachment.append(file)
+                hdf5_name = item["hdf5_name"]
+                hdf5_path = cherrypy.request.app.config["slycat-web-server"]["data-store"] + "/" + hdf5_name
+
+                # Read HDF5 file
+                decoded_data = []
+                decoded_col = []
+                with h5py.File(hdf5_path, "r") as f:
+                    data = list(f['data_table'])
+                    for col in data:
+                        for item in col:
+                            decoded_item = item.decode('utf-8')
+                            decoded_col.append(decoded_item)
+                        decoded_data.append(decoded_col)
+                        decoded_col = []
+                decoded_rows = numpy.array(decoded_data).T
+
+                giant_csv_list = []
+                temp_row = []
+                for row in decoded_rows:
+                    for i, entry in enumerate(row):
+                        if i == 0:
+                            temp_row.append(str(entry))
+                        elif i == (len(row) - 1):
+                            temp_row.append(str(entry) + '\n')
+                        else:
+                            temp_row.append(str(entry) + ',')
+                    temp_row_string = ''.join(temp_row)
+                    giant_csv_list.append(temp_row_string)
+                    temp_row = []
+                giant_csv_string = ''.join(giant_csv_list)
+                attachment.append(giant_csv_string)
+
     # if we didnt fined the file repspond with not found
     if fid is None:
         raise cherrypy.HTTPError("404 There was no file with name %s found." % file_key)
@@ -401,19 +431,8 @@ def put_project_csv_data(pid, file_key, parser, mid, aids):
         database.save(project_data)
     except Exception as e:
         cherrypy.log.error(str(e))
-    # clean up the attachment by removing white space
-    try:
-        attachment[0] = attachment[0].replace('\\n', '\n')
-        attachment[0] = attachment[0].replace('["', '')
-        attachment[0] = attachment[0].replace('"]', '')
-    except Exception as e:
-        cherrypy.log.error(str(e))
 
     model = database.get("model", mid)
-    if "project_data" not in model:
-        model["project_data"] = []
-    model["project_data"].append(project_data["_id"])
-    database.save(model)
     slycat.web.server.parse_existing_file(database, parser, True, attachment, model, aids)
     return {"Status": "Success"}
 
@@ -509,53 +528,74 @@ def post_project_models(pid):
     cherrypy.response.status = "201 Model created."
     return {"id": mid}
 
-# @cherrypy.tools.json_in(on=True)
-# @cherrypy.tools.json_out(on=True)
-def create_project_data_from_pid(pid, file=None, file_name=None):
+def create_project_data(mid, aid, file):
     """
-    creates a project level data object from a project id 
-    that can be used to create new
+    creates a project level data object that can be used to create new
     models in the current project
-    :param file_name: artifact ID
+    :param mid: model ID
+    :param aid: artifact ID
     :param file: file attachment
     :return: not used
     """
 
-    database = slycat.web.server.database.couchdb.connect()
-    project = database.get("project", pid)
-    slycat.web.server.authentication.require_project_writer(project)
+    def isNumeric(some_thing):
+        """
+        Check if input is numeric
 
-    csv_data = str(file.file.read(), 'utf-8')
+        :param some_thing: object
+        :return: boolean
+        """
+        try:
+            x = float(some_thing)
+        except ValueError:
+            return False
+        return True
 
-    content_type = "text/csv"
-    timestamp = time.time()
-    formatted_timestamp = datetime.datetime.fromtimestamp(timestamp).strftime('%Y-%m-%d %H:%M:%S')
-    did = uuid.uuid4().hex
-
-    data = {
-        "_id": did,
-        "type": "project_data",
-        "file_name": formatted_timestamp + "_" + file_name,
-        "data_table": "data-table",
-        "project": pid,
-        "mid": [""],
-        "created": datetime.datetime.utcnow().isoformat(),
-        "creator": cherrypy.request.login,
-    }
+    rows = []
 
-    database.save(data)
-    database.put_attachment(data, filename="content", content_type=content_type, content=csv_data)
-    cherrypy.log.error("[MICROSERVICE] Added project data %s." % data["file_name"])
+    split_file = file[0].split('\n')
 
-def create_project_data(mid, aid, file):
-    """
-    creates a project level data object that can be used to create new
-    models in the current project
-    :param mid: model ID
-    :param aid: artifact ID
-    :param file: file attachment
-    :return: not used
-    """
+    for row in split_file:
+        row_list = [row]
+        split_row = row_list[0].split(',')
+        rows.append(split_row)
+
+    columns = numpy.array(rows).T
+
+    for i, column in enumerate(columns):
+        for j, entry in enumerate(column):
+            columns[i,j] = entry.encode("utf-8")
+
+    column_names = [name.strip() for name in rows[0]]
+    column_names = ["%eval_id"] + column_names
+    column_types = ["string" for name in column_names]
+
+    column_types[0] = "int64"
+
+    for index in range(1, len(columns)):  # repack data cols as numpy arrays
+        try:
+            if isNumeric(columns[index][0]):
+                columns[index] = numpy.array(columns[index], dtype="float64")
+                column_types[index] = "float64"
+            else:
+                stringType = "S" + str(len(columns[index][0]))  # using length of first string for whole column
+                columns[index] = numpy.array(columns[index], dtype=stringType)
+                column_types[index] = "string"
+        except:
+            pass
+
+    # Edit with path to store HDF5
+    hdf5_path = cherrypy.request.app.config["slycat-web-server"]["data-store"] + "/"
+    # Edit with name for HDF5 file
+    unique_name = uuid.uuid4().hex
+    hdf5_name = f"{unique_name}.hdf5"
+    hdf5_file_path = os.path.join(hdf5_path, hdf5_name)
+
+    h5f = h5py.File((hdf5_file_path), "w")
+    h5f.create_dataset('data_table', data=numpy.array(columns, dtype='S'))
+    h5f.close()
+
+    # Make the entry in couch for project data
     content_type = "text/csv"
     database = slycat.web.server.database.couchdb.connect()
     model = database.get("model", mid)
@@ -584,6 +624,7 @@ def create_project_data(mid, aid, file):
         data = {
             "_id": did,
             "type": "project_data",
+            "hdf5_name": hdf5_name,
             "file_name": formatted_timestamp + "_" + aid[1],
             "data_table": aid[0],
             "project": pid,
@@ -973,7 +1014,7 @@ def post_model_files(mid, input=None, files=None, sids=None, paths=None, aids=No
 
     try:
         slycat.web.server.plugin.manager.parsers[parser]["parse"](database, model, input, files, aids, **kwargs)
-        create_project_data(mid, aids, files)
+        # create_project_data(mid, aids, files)
     except Exception as e:
         cherrypy.log.error("handles Exception parsing posted files: %s" % e)
         cherrypy.log.error("slycat.web.server.handlers.py post_model_files",
@@ -1049,8 +1090,9 @@ def post_upload_finished(uid):
     :return: status of upload
     """
     uploaded = require_integer_array_json_parameter("uploaded")
+    useProjectData = require_boolean_json_parameter("useProjectData")
     with slycat.web.server.upload.get_session(uid) as session:
-        return session.post_upload_finished(uploaded)
+        return session.post_upload_finished(uploaded, useProjectData)
 
 
 def delete_upload(uid):

packages/slycat/web/server/upload.py

@@ -146,7 +146,7 @@ def put_upload_file_part(self, fid, pid, data):
       file.write(data)
     self._received.add((fid, pid))
 
-  def post_upload_finished(self, uploaded):
+  def post_upload_finished(self, uploaded, useProjectData):
     """
     checks for missing and excess files, if neither are found moves on to
     finishing the upload and parsing the uploaded item.
@@ -169,6 +169,7 @@ def post_upload_finished(self, uploaded):
     uploaded = {(fid, pid) for fid in range(len(uploaded)) for pid in range(uploaded[fid])}
     missing = [part for part in uploaded if part not in self._received]
     excess = [part for part in self._received if part not in uploaded]
+    self.useProjectData = useProjectData
 
     if missing:
       cherrypy.response.status = "400 Upload incomplete."
@@ -250,7 +251,8 @@ def numeric_order(x):
         else:
           slycat.web.server.plugin.manager.parsers[self._parser]["parse"](database, model, self._input,
                                                                           files, self._aids, **self._kwargs)
-        if model["model-type"] == "parameter-image":
+        if model["model-type"] == "parameter-image" and self.useProjectData == True:
+          # Use project data
           slycat.web.server.handlers.create_project_data(self._mid, self._aids, files)
       except Exception as e:
         cherrypy.log.error("Exception parsing posted files: %s" % e)

web-server/js/slycat-file-uploader-factory.js

@@ -24,7 +24,7 @@ module.MEGABYTE = 10000000;//the number of bytes we will split large files on
  *      success: function called if upload is successful
  *    }
  */
-module.uploadFile = function (fileObject)
+module.uploadFile = function (fileObject, useProjectData)
 {
   if(fileObject.hostname && fileObject.paths){
     // console.log("creating remote file upload session");
@@ -47,7 +47,7 @@ module.uploadFile = function (fileObject)
       aids: fileObject.aids,
       success: function (uid) {
         // console.log("Upload session created.");
-        uploadFile(fileObject.pid, fileObject.mid, uid, fileObject.file, fileObject);
+        uploadFile(fileObject.pid, fileObject.mid, uid, fileObject.file, fileObject, useProjectData);
       }
     });
   }
@@ -421,7 +421,7 @@ function uploadFileSlice(uid, sliceNumber, file){
  * @param fileObject
  *  json object that contains the file info
  */
-function uploadFile(pid, mid, uid, file, fileObject)
+function uploadFile(pid, mid, uid, file, fileObject, useProjectData)
 {
   // Make sure we actually have a file first
   if(file == undefined)
@@ -487,7 +487,7 @@ function uploadFile(pid, mid, uid, file, fileObject)
         {
           fileObject.progress(fileObject.progress() + progressIncreasePerSlice);
         }
-        finishUpload(pid, mid, uid, file, 1, fileObject);
+        finishUpload(pid, mid, uid, file, 1, fileObject, useProjectData);
       },
       error: function(){
         if(fileObject.error) {
@@ -514,11 +514,12 @@ function uploadFile(pid, mid, uid, file, fileObject)
  * @param fileObject
  *  json object that contains the file info
  */
-function finishUpload(pid, mid, uid, file, numberOfUploadedSlices, fileObject)
+function finishUpload(pid, mid, uid, file, numberOfUploadedSlices, fileObject, useProjectData)
 {
   client.post_upload_finished({
     uid: uid,
     uploaded: [numberOfUploadedSlices],
+    useProjectData: useProjectData,
     success: function()
     {
       // console.log("Upload session finished.");
@@ -544,11 +545,11 @@ function finishUpload(pid, mid, uid, file, numberOfUploadedSlices, fileObject)
         missingElements.forEach(function(missingElement){
           console.log("missing element " + missingElement[1] + " for file" + missingElement[0]);
           uploadFileSlice(uid, missingElement[1], file);
-          finishUpload(pid, mid, uid, file, numberOfUploadedSlices, fileObject);
+          finishUpload(pid, mid, uid, file, numberOfUploadedSlices, fileObject, useProjectData);
         });
       }else if(request.status === 423){
         setTimeout(function(){
-          finishUpload(pid, mid, uid, file, numberOfUploadedSlices, fileObject)
+          finishUpload(pid, mid, uid, file, numberOfUploadedSlices, fileObject, useProjectData)
         }, 5000);
       }
       else{

web-server/js/slycat-web-client.js

@@ -229,6 +229,30 @@ module.put_project_csv_data = function(params)
       });
 };
 
+module.post_project_data = function(params)
+{
+  var formData = new FormData();
+  formData.append('file', params.file);
+  $.ajax(
+    {
+      type: "POST",
+      data: formData,
+      url: URI(api_root + "projects/data/" + params.pid),
+      contentType: false,
+      processData: false,
+      success: function(result)
+      {
+        if(params.success)
+          params.success(result);
+      },
+      error: function(request, status, reason_phrase)
+      {
+        if(params.error)
+          params.error(request, status, reason_phrase);
+      },
+    });
+}
+
 module.get_project_data = function(params)
 {
   $.ajax(
@@ -1652,7 +1676,8 @@ module.post_upload_finished = function(params)
     contentType: "application/json",
     data: JSON.stringify(
     {
-      "uploaded": params.uploaded
+      "uploaded": params.uploaded,
+      "useProjectData": params.useProjectData ?? false
     }),
     type: "POST",
     url: api_root + "uploads/" + params.uid + "/finished",

web-server/plugins/slycat-parameter-image/js/wizard-ui.js

@@ -30,6 +30,7 @@ function constructor(params) {
   component.current_aids = ko.observable("");
   component.csv_data = ko.observableArray();
   component.error_messages = ko.observable("");
+  component.useProjectData = ko.observable(false);
   // Alex removing default model name per team meeting discussion
   // component.model = mapping.fromJS({_id: null, name: "New Parameter Space Model", description: "", marking: markings.preselected()});
   component.model = mapping.fromJS({
@@ -326,7 +327,18 @@ function constructor(params) {
         component.browser.progress_status("");
       },
     };
-    fileUploader.uploadFile(fileObject);
+
+    client.post_project_data({
+      pid: component.project._id(),
+      file: file,
+      success: function(result) {
+        console.log("Success!");
+      },
+      error: function(status) {
+        console.log("Failure...");
+      }
+    })
+    fileUploader.uploadFile(fileObject, component.useProjectData());
   };
   component.connectSMB = function () {
     component.remote.enable(false);