Merge 4a507ca into b495d90

.travis.yml

@@ -18,6 +18,7 @@ matrix:
       - docker pull xgaia/corese:latest
       - docker pull xgaia/isql-api:2.1.0
       - sudo docker run -d --name virtuoso -p 8891:8890 -p 1112:1111 -e DBA_PASSWORD=dba -e SPARQL_UPDATE=true -e DEFAULT_GRAPH=http://localhost:8891/DAV -t askomics/virtuoso:7.2.5.1 /bin/sh -c "netstat -nr | grep '^0\.0\.0\.0' | grep -oE '((1?[0-9][0-9]?|2[0-4][0-9]|25[0-5])\.){3}(1?[0-9][0-9]?|2[0-4][0-9]|25[0-5])' | grep -v '^0\.0\.0\.0' | sed 's/$/ askomics-host/' >> /etc/hosts && /virtuoso/virtuoso.sh"
+      - sleep 1m
       - sudo docker run -d --name redis -p 6380:6379 -t redis:4.0
       - sudo docker run -d --name galaxy -p 8081:80 -t bgruening/galaxy-stable:19.01
       - sudo docker run -d --name corese -p 8082:8080 -t xgaia/corese:latest /bin/sh -c "netstat -nr | grep '^0\.0\.0\.0' | grep -oE '((1?[0-9][0-9]?|2[0-4][0-9]|25[0-5])\.){3}(1?[0-9][0-9]?|2[0-4][0-9]|25[0-5])' | grep -v '^0\.0\.0\.0' | sed 's/$/ askomics-host/' >> /etc/hosts && /corese/start.sh"

askomics/libaskomics/BedFile.py

@@ -1,4 +1,6 @@
 import rdflib
+import sys
+import traceback
 
 from rdflib import BNode
 from pybedtools import BedTool
@@ -39,7 +41,13 @@ def __init__(self, app, session, file_info, host_url=None, external_endpoint=Non
 
     def set_preview(self):
         """Set entity name preview"""
-        self.entity_name = self.human_name
+        try:
+            BedTool(self.path).count()
+            self.entity_name = self.human_name
+        except Exception as e:
+            self.error = True
+            self.error_message = "Malformated BED ({})".format(str(e))
+            traceback.print_exc(file=sys.stdout)
 
     def get_preview(self):
         """Get file preview
@@ -53,6 +61,8 @@ def get_preview(self):
             'type': self.type,
             'id': self.id,
             'name': self.human_name,
+            'error': self.error,
+            'error_message': self.error_message,
             "entity_name": self.entity_name
         }
 

askomics/libaskomics/CsvFile.py

@@ -1,6 +1,8 @@
 import csv
 import re
 import rdflib
+import sys
+import traceback
 
 from rdflib import BNode
 
@@ -67,6 +69,8 @@ def get_preview(self):
             'type': self.type,
             'id': self.id,
             'name': self.human_name,
+            'error': self.error,
+            'error_message': self.error_message,
             'data': {
                 'header': self.header,
                 'content_preview': self.preview,
@@ -96,28 +100,34 @@ def force_header_names(self, forced_header_names):
 
     def set_preview_and_header(self):
         """Set the preview and header by looking in the fists lines of the file"""
-        with open(self.path, 'r', encoding='utf-8') as csv_file:
-            reader = csv.reader(csv_file, dialect=self.dialect)
-            count = 0
-            # Store header
-            header = next(reader)
-            self.header = [h.strip() for h in header]
-
-            # Loop on lines
-            preview = []
-            for row in reader:
-                res_row = {}
-                for i, cell in enumerate(row):
-                    res_row[self.header[i]] = cell
-                preview.append(res_row)
-
-                # Stop after x lines
-                if self.preview_limit:
-                    count += 1
-                    if count > self.preview_limit:
-                        break
-
-        self.preview = preview
+        try:
+            with open(self.path, 'r', encoding='utf-8') as csv_file:
+                reader = csv.reader(csv_file, dialect=self.dialect)
+                count = 0
+                # Store header
+                header = next(reader)
+                self.header = [h.strip() for h in header]
+
+                # Loop on lines
+                preview = []
+                for row in reader:
+                    res_row = {}
+                    res_row = dict.fromkeys(self.header, "")
+                    for i, cell in enumerate(row):
+                        res_row[self.header[i]] = cell
+                    preview.append(res_row)
+
+                    # Stop after x lines
+                    if self.preview_limit:
+                        count += 1
+                        if count > self.preview_limit:
+                            break
+            self.preview = preview
+
+        except Exception as e:
+            self.error = True
+            self.error_message = "Malformated CSV/TSV ({})".format(str(e))
+            traceback.print_exc(file=sys.stdout)
 
     def set_columns_type(self):
         """Set the columns type by guessing them"""

askomics/libaskomics/Dataset.py

@@ -77,7 +77,7 @@ def save_in_db(self):
             ?,
             ?,
             ?,
-            ?,
+            NULL,
             ?,
             "queued",
             strftime('%s', 'now'),
@@ -94,7 +94,6 @@ def save_in_db(self):
             self.celery_id,
             self.file_id,
             self.name,
-            self.graph_name,
             self.public,
             0
         ), get_id=True)
@@ -139,7 +138,7 @@ def update_celery(self, celery_id):
 
         database.execute_sql_query(query, (celery_id, self.session['user']['id'], self.id))
 
-    def update_in_db(self, status, update_celery=False, error=False, error_message=None, ntriples=0, traceback=None):
+    def update_in_db(self, status, update_celery=False, update_date=False, update_graph=False, error=False, error_message=None, ntriples=0, traceback=None):
         """Update the dataset when integration is done
 
         Parameters
@@ -152,25 +151,35 @@ def update_in_db(self, status, update_celery=False, error=False, error_message=N
             Number of triples integrated
         """
         message = error_message if error else ""
+
         update_celery_id_substr = "celery_id=?," if update_celery else ""
+        update_date_substr = "start=strftime('%s', 'now')," if update_date else ""
+        update_graph_substr = "graph_name=?," if update_graph else ""
 
         database = Database(self.app, self.session)
 
         query = '''
         UPDATE datasets SET
         {}
+        {}
+        {}
         status=?,
         end=strftime('%s', 'now'),
         ntriples=?,
         error_message=?,
         traceback=?
         WHERE user_id = ? AND id=?
-        '''.format(update_celery_id_substr)
+        '''.format(update_celery_id_substr, update_date_substr, update_graph_substr)
+
+        variables = [status, ntriples, message, traceback, self.session['user']['id'], self.id]
+
+        if update_graph:
+            variables.insert(0, self.graph_name)
 
         if update_celery:
-            database.execute_sql_query(query, (self.celery_id, status, ntriples, message, traceback, self.session['user']['id'], self.id))
-        else:
-            database.execute_sql_query(query, (status, ntriples, message, traceback, self.session['user']['id'], self.id))
+            variables.insert(0, self.celery_id)
+
+        database.execute_sql_query(query, tuple(variables))
 
     def delete_from_db(self):
         """Delete a dataset from the database"""

askomics/libaskomics/DatasetsHandler.py

@@ -59,13 +59,19 @@ def get_datasets(self):
 
         datasets = []
         for row in rows:
+
+            exec_time = 0
+            if row[5] is not None and row[4] is not None:
+                exec_time = row[5] - row[4]
+
             dataset = {
                 'id': row[0],
                 'name': row[1],
                 'public': True if int(row[2] == 1) else False,
                 'status': row[3],
                 'start': row[4],
                 'end': row[5],
+                'exec_time': exec_time,
                 'ntriples': row[6],
                 'error_message': row[7],
                 'traceback': row[8],

askomics/libaskomics/File.py

@@ -150,6 +150,9 @@ def __init__(self, app, session, file_info, host_url=None, external_endpoint=Non
         self.graph_chunk = RdfGraph(self.app, self.session)
         self.graph_abstraction_dk = RdfGraph(self.app, self.session)
 
+        self.error = False
+        self.error_message = ""
+
     def edit_name_in_db(self, new_name):
         """Edit file name
 

askomics/libaskomics/GffFile.py

@@ -1,4 +1,6 @@
 import rdflib
+import sys
+import traceback
 
 from rdflib import BNode
 from BCBio.GFF import GFFExaminer
@@ -45,13 +47,18 @@ def __init__(self, app, session, file_info, host_url=None, external_endpoint=Non
 
     def set_preview(self):
         """Summary"""
-        exam = GFFExaminer()
-        handle = open(self.path, encoding="utf-8", errors="ignore")
-        gff_type = exam.available_limits(handle)['gff_type']
-        for entity in gff_type:
-            self.entities.append(entity[0])
-
-        handle.close()
+        try:
+            exam = GFFExaminer()
+            handle = open(self.path, encoding="utf-8", errors="ignore")
+            gff_type = exam.available_limits(handle)['gff_type']
+            for entity in gff_type:
+                self.entities.append(entity[0])
+
+            handle.close()
+        except Exception as e:
+            self.error = True
+            self.error_message = "Malformated GFF ({})".format(str(e))
+            traceback.print_exc(file=sys.stdout)
 
     def get_preview(self):
         """Get gff file preview (list of entities)
@@ -65,6 +72,8 @@ def get_preview(self):
             'type': self.type,
             'id': self.id,
             'name': self.human_name,
+            'error': self.error,
+            'error_message': self.error_message,
             'data': {
                 'entities': self.entities
             }

askomics/libaskomics/RdfFile.py

@@ -53,6 +53,8 @@ def get_preview(self):
             'type': self.type,
             'id': self.id,
             'name': self.human_name,
+            'error': self.error,
+            'error_message': self.error_message,
             'data': {
                 'preview': head
             }

askomics/libaskomics/Result.py

@@ -307,15 +307,11 @@ def save_result_in_file(self, headers, results):
             File size
         """
         with open(self.file_path, 'w') as file:
-            writer = csv.writer(file, delimiter="\t")
-            writer.writerow(headers)
-            if len(results) > 0:
-                for i in results:
-                    row = []
-                    self.nrows += 1
-                    for header, value in i.items():
-                        row.append(value)
-                    writer.writerow(row)
+            writer = csv.DictWriter(file, delimiter="\t", fieldnames=headers)
+            writer.writeheader()
+            for row in results:
+                self.nrows += 1
+                writer.writerow(row)
 
         return os.path.getsize(self.file_path)
 
@@ -383,7 +379,7 @@ def update_public_status(self, public):
             self.id
         ))
 
-    def update_db_status(self, status, size=None, update_celery=False, error=False, error_message=None, traceback=None):
+    def update_db_status(self, status, size=None, update_celery=False, update_date=False, error=False, error_message=None, traceback=None):
         """Update status of results in db
 
         Parameters
@@ -398,6 +394,8 @@ def update_db_status(self, status, size=None, update_celery=False, error=False,
         if update_celery:
             update_celery_substr = "celery_id=?,"
 
+        update_date_substr = "start=strftime('%s', 'now')," if update_date else ""
+
         size_string = ""
         if size:
             size_string = "size=?,"
@@ -410,14 +408,15 @@ def update_db_status(self, status, size=None, update_celery=False, error=False,
         UPDATE results SET
         {celery}
         {size}
+        {date}
         status=?,
         end=?,
         path=?,
         nrows=?,
         error=?,
         traceback=?
         WHERE user_id=? AND id=?
-        '''.format(celery=update_celery_substr, size=size_string)
+        '''.format(celery=update_celery_substr, size=size_string, date=update_date_substr)
 
         variables = [
             status,

askomics/libaskomics/ResultsHandler.py

@@ -59,13 +59,18 @@ def get_files_info(self):
         files = []
 
         for row in rows:
+
+            exec_time = 0
+            if row[4] is not None and row[3] is not None:
+                exec_time = row[4] - row[3]
+
             files.append({
                 'id': row[0],
                 'status': row[1],
                 'path': row[2],
                 'start': row[3],
                 'end': row[4],
-                'execTime': row[4] - row[3],
+                'execTime': exec_time,
                 'graphState': row[5],
                 'nrows': row[6],
                 'errorMessage': row[7],

askomics/libaskomics/SparqlQueryLauncher.py

@@ -245,7 +245,7 @@ def drop_dataset(self, graph):
         '''.format(graph)
         self.execute_query(query, disable_log=True)
 
-    def process_query(self, query):
+    def process_query(self, query, no_isql=False):
         """Execute a query and return parsed results
 
         Parameters
@@ -258,7 +258,7 @@ def process_query(self, query):
         list
             Parsed results
         """
-        return self.parse_results(self.execute_query(query))
+        return self.parse_results(self.execute_query(query, no_isql=False))
 
     def execute_query(self, query, disable_log=False, no_isql=False):
         """Execute a sparql query
@@ -282,17 +282,17 @@ def execute_query(self, query, disable_log=False, no_isql=False):
                 isqlapi = self.settings.get("triplestore", "isqlapi")
             except Exception:
                 pass
-            use_isl = True if triplestore == "virtuoso" and isqlapi and self.local_query and not no_isql else False
+            use_isql = True if triplestore == "virtuoso" and isqlapi and self.local_query and not no_isql else False
 
             start_time = time.time()
             self.endpoint.setQuery(query)
 
             # Debug
             if self.settings.getboolean('askomics', 'debug'):
-                self.log.debug("Launch {} query on {} ({})".format("ISQL" if use_isl else "SPARQL", self.triplestore, self.url_endpoint))
+                self.log.debug("Launch {} query on {} ({})".format("ISQL" if use_isql else "SPARQL", self.triplestore, self.url_endpoint))
                 self.log.debug(query)
 
-            if use_isl:
+            if use_isql:
                 formatted_query = "SPARQL {}".format(query)
                 json = {"command": formatted_query, "disable_log": disable_log, "sparql_select": not self.endpoint.isSparqlUpdateRequest()}
                 response = requests.post(url=isqlapi, json=json)

askomics/react/src/routes/datasets/datasetstable.jsx

@@ -8,6 +8,7 @@ import PropTypes from 'prop-types'
 import Utils from '../../classes/utils'
 import SyntaxHighlighter from 'react-syntax-highlighter'
 import { monokai } from 'react-syntax-highlighter/dist/esm/styles/hljs'
+import pretty from 'pretty-time'
 
 export default class DatasetsTable extends Component {
   constructor (props) {
@@ -103,6 +104,12 @@ export default class DatasetsTable extends Component {
       sort: true,
       formatter: (cell, row) => { return this.utils.humanDate(cell) }
     }, {
+      dataField: 'exec_time',
+      text: 'Exec time',
+      sort: true,
+      formatter: (cell, row) => { return ["started", "queued"].indexOf(row.status) == -1 ? cell == 0 ? '<1s' : pretty([cell, 0], 's') : ""},
+      editable: false
+    }, {
 
       dataField: 'public',
       text: 'Public',