Skip to content
This repository has been archived by the owner on Jan 16, 2020. It is now read-only.

Commit

Permalink
Merge pull request #102 from ofilangi/master
Browse files Browse the repository at this point in the history 
Improve GFF loader
  • Loading branch information
Olivier Filangi committed Feb 3, 2017
2 parents 290f6b7 + 817aea7 commit 89ba95c
Show file tree
Hide file tree
Showing 9 changed files with 225 additions and 153 deletions.
8 changes: 8 additions & 0 deletions askomics/libaskomics/ParamManager.py
View file
Expand Up @@ -131,3 +131,11 @@ def encodeToRDFURI(toencode):
obj = obj.replace(".", "_dot_")
obj = obj.replace("-", "_sep_")
return obj

@staticmethod
def decodeToRDFURI(toencode):
import urllib.parse
obj = toencode.replace("_dot_",".")
obj = obj.replace("_sep_","-")
obj = urllib.parse.unquote(obj)
return obj
2 changes: 1 addition & 1 deletion askomics/libaskomics/SourceFileConvertor.py
View file
Expand Up @@ -108,7 +108,7 @@ def get_source_files_gff(self, tax='', ent=[]):
:rtype: List
"""
src_dir = self.get_source_file_directory()
paths = glob(src_dir + '/*.gff3')
paths = glob(src_dir + '/*')

files = []
for p in paths:
Expand Down
18 changes: 8 additions & 10 deletions askomics/libaskomics/source_file/SourceFile.py
View file
Expand Up @@ -28,14 +28,19 @@ def __init__(self, settings, session, path):

self.timestamp = datetime.datetime.now().isoformat()

self.graph = ''

self.path = path

self.metadatas = {}

# The name should not contain extension as dots are not allowed in rdf names
self.name = os.path.splitext(os.path.basename(path))[0]

self.graph = 'askomics:unkown:uri:graph'
if 'graph' in self.session:
self.graph = self.session['graph']

self.graph = self.graph + ':' + self.name + '_' + self.timestamp

# FIXME check name uniqueness as we remove extension (collision if uploading example.tsv and example.txt)

self.log = logging.getLogger(__name__)
Expand Down Expand Up @@ -93,15 +98,8 @@ def persist(self, urlbase, method, public):
:return: a dictionnary with information on the success or failure of the operation
:rtype: Dict
"""

g = 'askomics:unkown:uri:graph'
if 'graph' in self.session:
g = self.session['graph']

self.graph = g + ':' + self.name + '_' + self.timestamp

self.insert_metadatas(public)

content_ttl = self.get_turtle()

ql = QueryLauncher(self.settings, self.session)
Expand Down
204 changes: 144 additions & 60 deletions askomics/libaskomics/source_file/SourceFileGff.py
View file

Large diffs are not rendered by default.

6 changes: 3 additions & 3 deletions askomics/libaskomics/source_file/SourceFileTsv.py
View file
Expand Up @@ -281,7 +281,7 @@ def get_domain_knowledge(self):

if all(types in self.forced_column_types for types in ('start', 'end')): # a positionable entity have to have a start and a end
ttl += ":" + self.encodeToRDFURI(self.headers[0]) + ' displaySetting:is_positionable "true"^^xsd:boolean .\n'
ttl += ":is_positionable rdfs:label 'is_positionable' .\n"
ttl += ":is_positionable rdfs:label 'is_positionable'^^xsd:string .\n"
ttl += ":is_positionable rdf:type owl:ObjectProperty .\n"

for header, categories in self.category_values.items():
Expand All @@ -290,7 +290,7 @@ def get_domain_knowledge(self):
ttl += (" , \n" + indent + ":").join(map(self.encodeToRDFURI,categories)) + " .\n"

for item in categories:
ttl += ":" + self.encodeToRDFURI(item) + " rdf:type :" + self.encodeToRDFURI(header) + " ;\n" + len(item) * " " + " rdfs:label \"" + item + "\" .\n"
ttl += ":" + self.encodeToRDFURI(item) + " rdf:type :" + self.encodeToRDFURI(header) + " ;\n" + len(item) * " " + " rdfs:label \"" + item + "\"^^xsd:string .\n"

return ttl

Expand Down Expand Up @@ -362,7 +362,7 @@ def get_turtle(self, preview_only=False):
entity_id = self.key_id(row)
indent = (len(entity_id) + 1) * " "
ttl += ":" + self.encodeToRDFURI(entity_id) + " rdf:type :" + self.encodeToRDFURI(self.headers[0]) + " ;\n"
ttl += indent + " rdfs:label \"" + entity_label + "\" ;\n"
ttl += indent + " rdfs:label \"" + entity_label + "\"^^xsd:string ;\n"

# Add data from other columns
for i, header in enumerate(self.headers): # Skip the first column
Expand Down
9 changes: 3 additions & 6 deletions askomics/libaskomics/source_file/SourceFileTtl.py
View file
Expand Up @@ -40,17 +40,14 @@ def get_preview_ttl(self):
formatter = HtmlFormatter(cssclass='preview_field', nowrap=True, nobackground=True)
return highlight(ttl, TurtleLexer(), formatter) # Formated html

def persist(self, urlbase):
def persist(self, urlbase, public):
"""
insert the ttl sourcefile in the TS
"""

pathttl = self.get_ttl_directory()
shutil.copy(self.path, pathttl)
fo = open(pathttl + '/' + os.path.basename(self.path))

self.insert_metadatas(public)
self.load_data_from_file(fo, urlbase)
self.get_metadatas()


8 changes: 0 additions & 8 deletions askomics/static/js/MainAskomics.js
View file
Expand Up @@ -704,14 +704,6 @@ function displayNavbar(loged, username, admin, blocked) {
setUploadForm('div#content_integration',"Upload User Files","source_files_overview",displayIntegrationForm);
});

// $("#integration_ttl").click(function() {
// setUploadForm('div#content_integration_ttl',"Upload User TTL Files","insert_files_rdf",displayTableRDF);
// });

// $("#integration_gff").click(function() {
// setUploadForm('div#content_integration_gff', 'Upload User GFF3 Files', "source_files_overview_gff", displayGffForm);
// });

// Visual effect on active tab (Ask! / Integrate / Credits)
$('.nav li').click(function(e) {

Expand Down
95 changes: 44 additions & 51 deletions askomics/static/js/integration.js
View file
Expand Up @@ -40,23 +40,23 @@ $(function () {
});

$("#content_integration").on('click', '.load_data_gff', function() {
let filename = $(this).attr('id');
loadSourceFileGff(filename, false);
let idfile = $(this).attr('id');
loadSourceFileGff(idfile, false);
});

$("#content_integration").on('click', '.load_data_gff_public', function() {
let filename = $(this).attr('id');
loadSourceFileGff(filename, true);
let idfile = $(this).attr('id');
loadSourceFileGff(idfile, true);
});

$("#content_integration").on('click', '.load_data_ttl', function() {
let filename = $(this).attr('id');
loadSourceFileTtl(filename, false);
let idfile = $(this).attr('id');
loadSourceFileTtl(idfile, false);
});

$("#content_integration").on('click', '.load_data_ttl_public', function() {
let filename = $(this).attr('id');
loadSourceFileTtl(filename, true);
let idfile = $(this).attr('id');
loadSourceFileTtl(idfile, true);
});

// $('.taxon-selector').change(function() {
Expand Down Expand Up @@ -101,6 +101,10 @@ function displayIntegrationForm(data) {
}
}

function getIdFile(file) {
return file.name.split('.').join('_');
}

function displayTSVForm(file) {
console.log('-+-+- displayTSVForm -+-+-');
// tranform columns to rows
Expand All @@ -117,7 +121,7 @@ function displayTSVForm(file) {
let source = $('#template-csv-form').html();
let template = Handlebars.compile(source);

let context = {file: file, admin: admin};
let context = {idfile: getIdFile(file),file: file, admin: admin};
let html = template(context);

$("#content_integration").append(html);
Expand All @@ -127,6 +131,7 @@ function displayTSVForm(file) {
function displayGffForm(file, taxons) {
console.log('-+-+- displayGffForm -+-+-');
let source = $('#template-gff-form').html();

let template = Handlebars.compile(source);

// User is admin if administration element is present in navbar
Expand All @@ -135,7 +140,7 @@ function displayGffForm(file, taxons) {
admin = true;
}

let context = {file: file, taxons: taxons, admin: admin};
let context = {idfile: getIdFile(file),file: file, taxons: taxons, admin: admin};
let html = template(context);

$("#content_integration").append(html);
Expand All @@ -152,13 +157,13 @@ function displayTtlForm(file) {
admin = true;
}

let context = {file: file, admin: admin};
let context = {idfile: getIdFile(file),file: file, admin: admin};
let html = template(context);

$('#content_integration').append(html);

$('#source-file-ttl-' + file.name).find(".preview_field").html(file.preview);
$('#source-file-ttl-' + file.name).find(".preview_field").show();
$('#source-file-ttl-' + getIdFile(file)).find(".preview_field").html(file.preview);
$('#source-file-ttl-' + getIdFile(file)).find(".preview_field").show();
}

function setCorrectType(file) {
Expand All @@ -175,7 +180,7 @@ function setCorrectType(file) {
if ('column_types' in file) {
var cols = file.column_types;
for(let i=0; i<cols.length; i++) {
var selectbox = $('div#content_integration form#source-file-tsv-' + file.name + ' select.column_type:eq(' + i + ')');
var selectbox = $('div#content_integration form#source-file-tsv-' + getIdFile(file) + ' select.column_type:eq(' + i + ')');
var values = selectbox.find("option").map(mapCallback);

if ($.inArray(cols[i], ['start', 'end', 'numeric']) == -1) {
Expand All @@ -191,33 +196,19 @@ function setCorrectType(file) {
}

// Check what is in the db
checkExistingData($('div#content_integration form#source-file-tsv-' + file.name));
checkExistingData($('div#content_integration form#source-file-tsv-' + getIdFile(file)));
}
}


}


/**
* FIXME: DEAD CODE - FUNCTION NEVER CALLED
*/
function displayTableRDF(data) {
let info = "";//$('<div></div>');
for(let i=0;i<data.files.length;i++) {
info+="Insertion of "+ data.files[i].filename+".\n";
}
displayModal(info, '', 'Close');
if (data.error !== undefined ) alert(JSON.stringify(data.error));
}


/**
* Get ttl representation of preview data
*/
function previewTtl(file_elem) {

var file_name = file_elem.find('.file_name').attr('id');
var idfile = file_elem.find('.file_name').attr('id');

// Get column types
var col_types = file_elem.find('.column_type').map(function() {
Expand Down Expand Up @@ -245,7 +236,7 @@ function previewTtl(file_elem) {
}

var service = new RestServiceJs("preview_ttl");
var model = { 'file_name': file_name,
var model = { 'file_name': $("#"+idfile).attr("filename"),
'col_types': col_types,
'key_columns' : key_columns,
'disabled_columns': disabled_columns };
Expand All @@ -265,7 +256,6 @@ function previewTtl(file_elem) {
}

function hidePreview(file_elem) {
var file_name = file_elem.find('.file_name').text();
file_elem.find(".preview_field").hide();
}

Expand Down Expand Up @@ -296,7 +286,8 @@ function containAll(Array1,Array2){
*/
function checkExistingData(file_elem) {

var file_name = file_elem.find('.file_name').attr('id');
let idfile = file_elem.find('.file_name').attr('id');
let file_name = $("#"+idfile).attr("filename");

// Get column types
var col_types = file_elem.find('.column_type').map(function() {
Expand Down Expand Up @@ -339,7 +330,7 @@ function checkExistingData(file_elem) {
}

var service = new RestServiceJs("check_existing_data");
var model = { 'file_name': file_name,
var model = { 'file_name': $("#"+idfile).attr("filename"),
'col_types': col_types,
'disabled_columns': disabled_columns,
'key_columns':key_columns
Expand Down Expand Up @@ -381,7 +372,7 @@ function checkExistingData(file_elem) {
function loadSourceFile(file_elem, pub) {
console.log('---> loadSourceFile');

let file_name = file_elem.find('.file_name').attr('id');
let idfile = file_elem.find('.file_name').attr('id');

// Get column types
let col_types = file_elem.find('.column_type').map(function() {
Expand Down Expand Up @@ -411,7 +402,7 @@ function loadSourceFile(file_elem, pub) {
displayModal('Please wait', '', 'Close');

var service = new RestServiceJs("load_data_into_graph");
var model = { 'file_name': file_name,
var model = { 'file_name': $("#"+idfile).attr("filename"),
'col_types': col_types,
'disabled_columns': disabled_columns,
'key_columns':key_columns,
Expand Down Expand Up @@ -487,24 +478,23 @@ function loadSourceFile(file_elem, pub) {
/**
* Load a GFF source_file into the triplestore
*/
function loadSourceFileGff(filename, pub) {
console.log('-----> loadSourceFileGff <-----');
function loadSourceFileGff(idfile, pub) {
console.log('-----> loadSourceFileGff <----- :');
// get taxon
let taxon = '';

let file_elem = $("#source-file-gff-" + filename);
let file_elem = $("#source-file-gff-" + idfile);

// get the taxon in the selector or in the input field
if ($('#' + filename + '-selector').val() === null || $('#' + filename + '-selector').val() === undefined) {
taxon = $('#tax-'+filename).val();
}else{
taxon = $('#' + filename + '-selector').val();
taxon = $('#' + idfile + '-selector').val();
if ( taxon === null || taxon === undefined) {
taxon = $('#tax-'+idfile).val();
}

// get entities
let entities = [];

$("#"+filename+" > label > input").each(function() {
$("#"+idfile+" > label > input").each(function() {
if ($(this).is(":checked")) {
entities.push($(this).attr('id'));
}
Expand All @@ -513,7 +503,8 @@ function loadSourceFileGff(filename, pub) {
displayModal('Please wait', '', 'Close');

let service = new RestServiceJs("load_gff_into_graph");
let model = { 'file_name': filename,

let model = { 'file_name': $("#"+idfile).attr("filename"),
'taxon': taxon,
'entities': entities,
'public': pub };
Expand All @@ -528,7 +519,7 @@ function loadSourceFileGff(filename, pub) {
return;
}
// Show a success message isertion is OK
let div_entity = $("#" + filename);
let div_entity = $("#" + idfile);
let entities_string = '';
for (var i = 0; i < entities.length; i++) {
entities_string += '<b>' + entities[i] + '</b>';
Expand Down Expand Up @@ -561,15 +552,17 @@ function loadSourceFileGff(filename, pub) {
});
}

function loadSourceFileTtl(filename, pub) {
function loadSourceFileTtl(idfile, pub) {
console.log('--- loadSourceFileTtl ---');
displayModal('Please wait', '', 'Close');

let file_elem = $("#source-file-ttl-" + filename);
let file_elem = $("#source-file-ttl-" + idfile);

let service = new RestServiceJs('load_ttl_into_graph');
let model = {'file_name' : filename,
'public': pub};
let model = {
'file_name' : $("#"+idfile).attr("filename"),
'public': pub
};

service.post(model, function(data) {
console.log('---> ttl insert');
Expand All @@ -591,7 +584,7 @@ function loadSourceFileTtl(filename, pub) {
.removeClass('hidden alert-warning')
.addClass('show alert-danger');
}else{
insert_status_elem.html('<span class="glyphicon glyphicon-ok"></span> ' + filename + ' inserted with success.')
insert_status_elem.html('<span class="glyphicon glyphicon-ok"></span> ' + idfile + ' inserted with success.')
.removeClass('hidden alert-danger')
.removeClass('hidden alert-warning')
.addClass('show alert-success');
Expand Down

0 comments on commit 89ba95c

Please sign in to comment.