Merge pull request #74 from ofilangi/master

Fix Bug and Import of RDF Files
askomics · Oct 13, 2016 · decd8ff · decd8ff
2 parents d1d30df + 6c4f0ef
commit decd8ff
Show file tree

Hide file tree

Showing 15 changed files with 364 additions and 42 deletions.
diff --git a/Dockerfile b/Dockerfile
@@ -53,4 +53,4 @@ RUN gulp
 #-------------------------------------------------------------------------------------------
 EXPOSE 6543
 ENTRYPOINT ["./startAskomics.sh"]
-CMD ["fuseki", "prod"]
+CMD ["virtuoso", "prod"]
diff --git a/askomics/__init__.py b/askomics/__init__.py
@@ -23,6 +23,7 @@ def main(global_config, **settings):
     config.add_route('sparqlquery', '/sparqlquery')
     config.add_route('getSparqlQueryInTextFormat', '/getSparqlQueryInTextFormat')
     config.add_route('source_files_overview', '/source_files_overview')
+    config.add_route('insert_files_rdf', '/insert_files_rdf')
     config.add_route('load_data_into_graph', '/load_data_into_graph')
     config.add_route('preview_ttl', '/preview_ttl')
     config.add_route('check_existing_data', '/check_existing_data')

diff --git a/askomics/ask_view.py b/askomics/ask_view.py
@@ -1,8 +1,8 @@
 #! /usr/bin/env python3
 # -*- coding: utf-8 -*-
 
-import os,sys, traceback
-import re
+import os,sys,traceback
+import re,shutil
 
 from pyramid.view import view_config, view_defaults
 from pyramid.response import FileResponse
@@ -255,6 +255,40 @@ def source_files_overview(self):
 
         return data
 
+    @view_config(route_name='insert_files_rdf', request_method='GET')
+    def source_files_overview_rdf(self):
+        """
+        Get preview data for all the available files
+        """
+        self.log.debug(" ========= Askview:source_files_overview_rdf =============")
+        sfc = SourceFileConvertor(self.settings, self.request.session)
+
+        files = sfc.get_rdf_files()
+
+        data = {}
+        data['files'] = []
+
+        urlbase = re.search(r'(http:\/\/.*)\/.*', self.request.current_route_url())
+        urlbase = urlbase.group(1)
+
+        for src_f in files:
+            pathttl = src_f.get_ttl_directory()
+            infos = {}
+            infos['filename'] = os.path.basename(src_f.path)
+            try:
+                shutil.copy(src_f.path,pathttl);
+                fo = open(pathttl+"/"+infos['filename'],'r');
+                src_f.load_data_from_file(fo,urlbase);
+            except Exception as e:
+                traceback.print_exc(file=sys.stdout)
+                data['error'] = str(e)
+                self.log.error(str(e))
+                return data
+
+            data['files'].append(infos)
+
+        return data
+
     @view_config(route_name='preview_ttl', request_method='POST')
     def preview_ttl(self):
         """

diff --git a/askomics/libaskomics/SourceFileConvertor.py b/askomics/libaskomics/SourceFileConvertor.py
@@ -36,6 +36,20 @@ def get_source_files(self):
 
         return files
 
+    def get_rdf_files(self):
+        """
+        :return: List of the file to convert paths
+        :rtype: List
+        """
+        src_dir = self.get_source_file_directory()
+        paths = glob(src_dir + '/*[.ttl,.rdf]')
+
+        files = []
+        for p in paths:
+            files.append(SourceFile(self.settings, self.session, p, int(self.settings["askomics.overview_lines_limit"])))
+
+        return files
+
     def get_source_file(self, name):
         """
         Return an object representing a source file
@@ -48,7 +62,6 @@ def get_source_file(self, name):
         files = self.get_source_files()
 
         for f in files:
-            print(f.name)
             if f.name == name:
                 return f
 

diff --git a/askomics/libaskomics/source_file/SourceFile.py b/askomics/libaskomics/source_file/SourceFile.py
@@ -687,6 +687,7 @@ def load_data_from_file(self, fp, urlbase):
         ql.insert_data(ttlNamedGraph, self.get_param("askomics.graph"), sparqlHeader)
 
         url = urlbase+"/ttl/"+os.path.basename(fp.name)
+        self.log.debug(url)
         data = {}
         try:
             if self.is_defined("askomics.file_upload_url"):

diff --git a/askomics/static/js/AskomicsMenuFile.js b/askomics/static/js/AskomicsMenuFile.js
@@ -26,7 +26,7 @@ class AskomicsMenuFile {
       service.postsync(jdata,function(data) {
         query = data.query;
       });
-      $(this).attr("href", "data:application/sparql-query," + encodeURIComponent(query)).attr("download", "query-" + date + ".sparql");
+      $(this).attr("href", "data:application/sparql-query," + encodeURIComponent(query)).attr("download", "query-" + date + ".rq");
     });
   }
 

diff --git a/askomics/static/js/MainAskomics.js b/askomics/static/js/MainAskomics.js
@@ -80,6 +80,8 @@ function loadStartPoints() {
 
 function loadNamedGraphs() {
 
+    new AskomicsUserAbstraction().loadUserAbstraction();
+
     var select = $('#dropNamedGraphSelected');
     select.empty();
     manageDelGraphButton();
@@ -138,10 +140,7 @@ function loadStatistics() {
 
   displayModal('Please Wait', '', 'Close');
 
-
-  abstraction = new AskomicsUserAbstraction();
-  abstraction.loadUserAbstraction();
-
+  new AskomicsUserAbstraction().loadUserAbstraction();
 
   var service = new RestServiceJs("statistics");
   service.getAll(function(stats) {
@@ -191,7 +190,7 @@ function loadStatistics() {
     });
     $('#statistics_div').append(table);
 
-    var entities = abstraction.getEntities() ;
+    var entities = new AskomicsUserAbstraction().getEntities() ;
 
     table=$("<table></table>").addClass('table').addClass('table-bordered');
     th = $("<tr></tr>").addClass("table-bordered").attr("style", "text-align:center;");
@@ -201,9 +200,9 @@ function loadStatistics() {
 
     for (let ent1 in entities ) {
       let tr = $("<tr></tr>")
-            .append($("<td></td>").text(abstraction.getAttrib(entities[ent1],'rdfs:label')));
+            .append($("<td></td>").text(new AskomicsUserAbstraction().getAttrib(entities[ent1],'rdfs:label')));
             let rels = "";
-            var t = abstraction.getRelationsObjectsAndSubjectsWithURI(entities[ent1]);
+            var t = new AskomicsUserAbstraction().getRelationsObjectsAndSubjectsWithURI(entities[ent1]);
             var subjectTarget = t[0];
             for ( var ent2 in subjectTarget) {
               for (var rel of subjectTarget[ent2]) {
@@ -229,10 +228,10 @@ function loadStatistics() {
     for (let ent1 in entities ) {
     //$.each(stats['class'], function(key, value) {
       let tr = $("<tr></tr>")
-            .append($("<td></td>").text(abstraction.getAttrib(entities[ent1],'rdfs:label')));
+            .append($("<td></td>").text(new AskomicsUserAbstraction().getAttrib(entities[ent1],'rdfs:label')));
             let attrs = "";
             let cats = "";
-            var listAtt = abstraction.getAttributesEntity(entities[ent1]);
+            var listAtt = new AskomicsUserAbstraction().getAttributesEntity(entities[ent1]);
             for (var att of listAtt) {
                 attrs += '- '+att.label +"</br>";
             }
@@ -341,6 +340,59 @@ function downloadTextAsFile(filename, text) {
 }
 
 
+function setUploadForm(content,titleForm,route_overview,callback) {
+  var service = new RestServiceJs("up/");
+  service.getAll(function(formHtmlforUploadFiles) {
+    formHtmlforUploadFiles.html = formHtmlforUploadFiles.html.replace("___TITLE_UPLOAD___",titleForm);
+
+    $(content).html(formHtmlforUploadFiles.html);
+    /*
+
+          MAIN UPLOAD Third party => copy from /static/js/third-party/upload/main.js (thi file is disbale in askomics)
+
+    */
+    // Initialize the jQuery File Upload widget
+    $(content).find('#fileupload').fileupload({
+        // Uncomment the following to send cross-domain cookies:
+        //xhrFields: {withCredentials: true},
+        url: '/up/file/'
+    });
+
+    // Enable iframe cross-domain access via redirect option
+    $(content).find('#fileupload').fileupload(
+        'option',
+        'redirect',
+        window.location.href.replace(
+            /\/[^\/]*$/,
+            '/cors/result.html?%s'
+        )
+    );
+
+    // Load existing files
+    $(content).find('#fileupload').addClass('fileupload-processing');
+    $.ajax({
+        // Uncomment the following to send cross-domain cookies:
+        //xhrFields: {withCredentials: true},
+        url: $(content).find('#fileupload').fileupload('option', 'url'),
+        dataType: 'json',
+        context: $(content).find('#fileupload')[0]
+    }).always(function () {
+        $(this).removeClass('fileupload-processing');
+    }).done(function (result) {
+        $(this).fileupload('option', 'done')
+            .call(this, $.Event('done'), {result: result});
+    });
+
+    // Integrate button
+    $('.integrate-button').click(function() {
+        var service = new RestServiceJs(route_overview);
+        service.getAll(function(data) {
+            callback(data);
+        });
+    });
+  });
+}
+
 $(function () {
   // TODO: move inside AskomicsMenuFile
     // Startpoints definition
@@ -360,12 +412,21 @@ $(function () {
       }
     });
 
+    /*
+
+      Click general GU Interface of Askomics :
+      - manage navbar
+      - show/hide content_{section}
+
+    */
+
     // Get the overview of files to integrate
     $("#integration").click(function() {
-        var service = new RestServiceJs("up/");
-        service.getAll(function(formHtmlforUploadFiles) {
-          $('div#content_integration').html(formHtmlforUploadFiles.html);
-        });
+        setUploadForm('div#content_integration',"Upload User CSV/TSV Files","source_files_overview",displayTableTabularFile);
+    });
+
+    $("#integration_ttl").click(function() {
+        setUploadForm('div#content_integration_ttl',"Upload User TTL Files","insert_files_rdf",displayTableRDF);
     });
 
     // Visual effect on active tab (Ask! / Integrate / Credits)
@@ -378,8 +439,7 @@ $(function () {
         }
 
 
-
-        if (  ! ( $(this).attr('id') in { 'help' : '','admin':'' }) ) {
+        if (  ! ( $(this).attr('id') in { 'help' : '','admin':'','userdata':'' }) ) {
           $('.container').hide();
           $('.container#navbar_content').show();
           $('.container#content_' + $(this).attr('id')).show();

diff --git a/askomics/static/js/integration.js b/askomics/static/js/integration.js
@@ -56,7 +56,7 @@ function cols2rows(items) {
 /**
  * Show preview data on the page
  */
-function displayTable(data) {
+function displayTableTabularFile(data) {
     // Transform columns to rows
     for(let i=0;i<data.files.length;i++) {
         if ('preview_data' in data.files[i]) {
@@ -114,6 +114,18 @@ function displayTable(data) {
     }
 }
 
+/**
+ *
+ */
+function displayTableRDF(data) {
+  let info = "";//$('<div></div>');
+  for(let i=0;i<data.files.length;i++) {
+    info+="Insertion of "+ data.files[i].filename+".\n";
+  }
+  displayModal(info, '', 'Close');
+  if (data.error !== undefined ) alert(JSON.stringify(data.error));
+}
+
 /**
  * Get ttl representation of preview data
  */

diff --git a/...mics/static/js/third-party/upload/main.js → ...atic/js/third-party/upload/main_unused.js b/...mics/static/js/third-party/upload/main.js → ...atic/js/third-party/upload/main_unused.js
@@ -48,6 +48,8 @@ $(function () {
 
     // Integrate button
     $('.integrate-button').click(function() {
+        console.log("-----------------------------------");
+        console.log($('[id="^content_"]').length);
         var service = new RestServiceJs("source_files_overview");
         service.getAll(function(data) {
             displayTable(data);

diff --git a/askomics/static/js/view/AskomicsNodeView.js b/askomics/static/js/view/AskomicsNodeView.js
@@ -385,17 +385,17 @@ class AskomicsNodeView extends AskomicsObjectView {
         let node = new AskomicsGraphBuilder().getInstanciedNode(nodeid);
 
         if (icon.hasClass('glyphicon-eye-close')) {
-            icon.removeClass('glyphicon-eye-close');
-            icon.addClass('glyphicon-search');
-            node.setActiveAttribute(sparqlid,true,true);
-          } else if (icon.hasClass('glyphicon-search')) {
-            icon.removeClass('glyphicon-search');
-            icon.addClass('glyphicon-eye-open');
-            node.setActiveAttribute(sparqlid,true,false);
-          } else {
-            icon.removeClass('glyphicon-eye-open');
-            icon.addClass('glyphicon-eye-close');
-            node.setActiveAttribute(sparqlid,false,false);
+          icon.removeClass('glyphicon-eye-close');
+          icon.addClass('glyphicon-eye-open');
+          node.setActiveAttribute(sparqlid,true,false);
+        } else if (icon.hasClass('glyphicon-eye-open')) {
+          icon.removeClass('glyphicon-eye-open');
+          icon.addClass('glyphicon-search');
+          node.setActiveAttribute(sparqlid,true,true);
+        } else {
+          icon.removeClass('glyphicon-search');
+          icon.addClass('glyphicon-eye-close');
+          node.setActiveAttribute(sparqlid,false,false);
           }
       });
       return icon;

diff --git a/askomics/templates/index.pt b/askomics/templates/index.pt
@@ -17,7 +17,7 @@
 
         <!--<script src="https://cdnjs.cloudflare.com/ajax/libs/jstree/3.2.1/jstree.min.js"></script>-->
         <script src="static/js/third-party/jstree.min.js"></script>
-        
+
         <link href="static/css/jquery-ui.min.css" rel="stylesheet" >
         <link href="static/css/bootstrap.min.css" rel="stylesheet" type="text/css">
         <link href="static/css/style.css" rel="stylesheet" type="text/css">
@@ -41,13 +41,21 @@
                 <div id="collapse-menu-main" class="collapse navbar-collapse">
                     <ul class="nav navbar-nav navbar-right">
                         <li id="interrogation" class="active"><a href="#" onclick="loadStartPoints();"><span class="glyphicon glyphicon-play"></span></a></li>
-                        <li id="integration"><a href="#"><span class="glyphicon glyphicon-cloud-upload"></span></a></li>
+                        <li id=""><a href="#"><span class=""></span></a></li>
+
+                        <li id="userdata" class="dropdown">
+                          <a class="dropdown-toggle" data-toggle="dropdown" href="#"><span class="glyphicon glyphicon-cloud-upload"></span><span class="caret"></span></a>
+                            <ul class="dropdown-menu">
+                              <li id="integration"><a href="#">Tabular Data Files(CSV,TSV)</a></li>
+                              <li id="integration_ttl"><a href="#">RDF Files</a></li>
+                            </ul>
+                          </li>
+
                         <li id="admin" class="dropdown">
-                          <a class="dropdown-toggle" data-toggle="dropdown" href="#"><span class="glyphicon glyphicon-cog"></span>
-                            <span class="caret"></span></a>
+                          <a class="dropdown-toggle" data-toggle="dropdown" href="#"><span class="glyphicon glyphicon-cog"></span><span class="caret"></span></a>
                             <ul class="dropdown-menu">
-                              <li id="datasets"><a href="#">Datasets</a></li>
-                              <li id="statistics" onclick="loadNamedGraphs()"><a href="#">Statistics</a></li>
+                              <li id="datasets" onclick="loadNamedGraphs()"><a href="#">Datasets</a></li>
+                              <li id="statistics" onclick="loadStatistics()"><a href="#">Statistics</a></li>
                               <li class="divider"></li>
                               <li id="triplestore_parameters"><a onclick="new TriplestoreParametersView().configurationView();" href="#"><span class="glyphicon glyphicon-cloud"></span>&nbsp;Private Triplestore</a></li>
                               <li id="gene_ontology"><a onclick="new GOParametersView().configurationView();" href="#"><span class="glyphicon glyphicon-cloud"></span>&nbsp;Gene Ontology</a></li>
@@ -149,6 +157,9 @@
             <div id="overview" class="row"></div>
         </div>
 
+        <div id="content_integration_ttl" class="container" hidden>
+        </div>
+
         <div id="content_datasets" class="container" hidden>
             <hr/>
             <h3 class="header-div">Manage User Datasets</h3>
@@ -169,10 +180,11 @@
             <hr/>
             <h3 class="header-div">Statistics</h3>
             <hr/>
+            <!--
             <p id='btn-st'>
                 <button id='btn-stats' onclick='loadStatistics()' class='btn btn-default'>Get stats</button>
             </p>
-
+            -->
             <div id="statistics_div" class="row"></div>
         </div>