From 46657cbe64071d26d1434c49cd837a1c1ee62b2c Mon Sep 17 00:00:00 2001 From: Miled Rousset Date: Wed, 19 Jun 2019 15:56:50 +0200 Subject: [PATCH] =?UTF-8?q?Import=20d'un=20th=C3=A9saurus=20complet=20en?= =?UTF-8?q?=20CSV?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pom.xml | 4 +- .../SelectedBeans/HelpFileDownload.java | 54 + .../opentheso/SelectedBeans/NewTreeBean.java | 36 +- .../SelectedBeans/rdf4jFileBean.java | 230 +- .../trd/opentheso/bdd/helper/TermHelper.java | 4 +- .../core/imports/csv/CsvImportHelper.java | 696 +++++- .../core/imports/csv/CsvReadHelper.java | 843 ++++++- .../core/imports/rdf4j/ReadRdf4j.java | 1 - .../resources/samples/SkosPourMaquetteCSV.xml | 164 ++ src/main/resources/samples/sampleCSV.csv | 11 + src/main/webapp/WEB-INF/web.xml | 2 +- src/main/webapp/css/style.css | 13 + .../webapp/dialogs/help/csvImportHelp.xhtml | 69 + .../webapp/dialogs/import/importDlg.xhtml | 297 ++- .../webapp/dialogs/nt/ntListImportDlg.xhtml | 15 + src/main/webapp/edition.xhtml | 14 +- src/main/webapp/images/download-2-16.png | Bin 0 -> 285 bytes src/main/webapp/images/download-2-24.png | Bin 0 -> 595 bytes src/main/webapp/includes.xhtml | 3 + src/main/webapp/newjavascript.js | 7 - src/main/webapp/test-resbeans.css | 319 --- src/main/webapp/test-resbeans.js | 2114 ----------------- src/main/webapp/test-services.html | 57 - src/test/java/csv/ImportCsv.java | 2 +- 24 files changed, 2159 insertions(+), 2796 deletions(-) create mode 100644 src/main/java/mom/trd/opentheso/SelectedBeans/HelpFileDownload.java create mode 100644 src/main/resources/samples/SkosPourMaquetteCSV.xml create mode 100644 src/main/resources/samples/sampleCSV.csv create mode 100644 src/main/webapp/dialogs/help/csvImportHelp.xhtml create mode 100644 src/main/webapp/images/download-2-16.png create mode 100644 src/main/webapp/images/download-2-24.png delete mode 100644 src/main/webapp/newjavascript.js delete mode 100644 src/main/webapp/test-resbeans.css delete mode 100644 src/main/webapp/test-resbeans.js delete mode 100644 src/main/webapp/test-services.html diff --git a/pom.xml b/pom.xml index f426baf..74dd4ca 100644 --- a/pom.xml +++ b/pom.xml @@ -5,7 +5,7 @@ trd opentheso - 4.4.8 + 4.4.9 war opentheso @@ -20,7 +20,7 @@ org.apache.commons commons-csv - 1.6 + 1.7 diff --git a/src/main/java/mom/trd/opentheso/SelectedBeans/HelpFileDownload.java b/src/main/java/mom/trd/opentheso/SelectedBeans/HelpFileDownload.java new file mode 100644 index 0000000..cc60909 --- /dev/null +++ b/src/main/java/mom/trd/opentheso/SelectedBeans/HelpFileDownload.java @@ -0,0 +1,54 @@ +package mom.trd.opentheso.SelectedBeans; + +import java.io.InputStream; +import javax.faces.bean.ManagedBean; +import javax.faces.bean.ViewScoped; +import javax.faces.context.FacesContext; +import org.primefaces.model.DefaultStreamedContent; +import org.primefaces.model.StreamedContent; + + +@ManagedBean(name = "helpfileDownload", eager = true) +@ViewScoped +public class HelpFileDownload { + + private StreamedContent file; + + private String resourcePath; + private String resourceType; + + public HelpFileDownload() { +/* InputStream stream = FacesContext.getCurrentInstance().getExternalContext().getResourceAsStream(resourcePath);//"/resources/demo/images/boromir.jpg"); + file = new DefaultStreamedContent(stream, "image/jpg", resourcePath.substring(resourcePath.lastIndexOf("/"), resourcePath.length()));//"downloaded_boromir.jpg"); + */ } + + public StreamedContent downloadCSVSample() { + resourcePath = "/samples/sampleCSV.csv"; + // this.getClass().getResourceAsStream("/samples/maquetteCSV.csv"); + InputStream stream = this.getClass().getResourceAsStream(resourcePath);//"/resources/demo/images/boromir.jpg"); + file = new DefaultStreamedContent(stream, "txt/CSV", resourcePath.substring(resourcePath.lastIndexOf("/"), resourcePath.length()));//"downloaded_boromir.jpg"); + return file; + } + + + public StreamedContent getFile() { + return file; + } + + public String getResourcePath() { + return resourcePath; + } + + public void setResourcePath(String resourcePath) { + this.resourcePath = resourcePath; + } + + public String getResourceType() { + return resourceType; + } + + public void setResourceType(String resourceType) { + this.resourceType = resourceType; + } + +} diff --git a/src/main/java/mom/trd/opentheso/SelectedBeans/NewTreeBean.java b/src/main/java/mom/trd/opentheso/SelectedBeans/NewTreeBean.java index 690dd5c..15adad0 100644 --- a/src/main/java/mom/trd/opentheso/SelectedBeans/NewTreeBean.java +++ b/src/main/java/mom/trd/opentheso/SelectedBeans/NewTreeBean.java @@ -96,6 +96,9 @@ public class NewTreeBean implements Serializable { private boolean isSortByNotation = false; private String iconSort = "ui-icon-sort-16"; private String sortTitle = "tree.sortByNotation"; + + private char delimiter = ','; + private int choiceDelimiter = 0; private ArrayList conceptObjects; @@ -2337,8 +2340,16 @@ public void addMultipleNT2() { reloadTree(); } + public void actionChoice() { + if(choiceDelimiter == 0) + delimiter = ','; + if(choiceDelimiter == 1) + delimiter = ';'; + if(choiceDelimiter == 2) + delimiter = '\t'; + } public void handleFileUpload(FileUploadEvent event) { - this.multipleNT = event.getFile().getContents(); + // this.multipleNT = event.getFile().getContents(); if (!readCsvFile(event)) { FacesContext.getCurrentInstance().addMessage(null, new FacesMessage(FacesMessage.SEVERITY_ERROR, @@ -2354,8 +2365,9 @@ public void handleFileUpload(FileUploadEvent event) { private boolean readCsvFile(FileUploadEvent event) { try { + //char delimiter = ','; Reader in1 = new InputStreamReader(event.getFile().getInputstream()); - CsvReadHelper csvHelper = new CsvReadHelper(); + CsvReadHelper csvHelper = new CsvReadHelper(delimiter); if(! csvHelper.setLangs(in1)){ return false; } @@ -2671,4 +2683,24 @@ public void setExternalResources(ExternalResources externalResources) { this.externalResources = externalResources; } + public char getDelimiter() { + return delimiter; + } + + public void setDelimiter(char delimiter) { + this.delimiter = delimiter; + } + + public int getChoiceDelimiter() { + return choiceDelimiter; + } + + public void setChoiceDelimiter(int choiceDelimiter) { + this.choiceDelimiter = choiceDelimiter; + } + + + + + } diff --git a/src/main/java/mom/trd/opentheso/SelectedBeans/rdf4jFileBean.java b/src/main/java/mom/trd/opentheso/SelectedBeans/rdf4jFileBean.java index db72132..78c0981 100644 --- a/src/main/java/mom/trd/opentheso/SelectedBeans/rdf4jFileBean.java +++ b/src/main/java/mom/trd/opentheso/SelectedBeans/rdf4jFileBean.java @@ -8,10 +8,13 @@ import mom.trd.opentheso.bdd.helper.nodes.MyTreeNode; import java.io.IOException; import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.Reader; import java.io.Serializable; import java.sql.Connection; import java.sql.SQLException; import java.text.ParseException; +import java.util.ArrayList; import java.util.logging.Level; import javax.faces.application.FacesMessage; import javax.faces.bean.ManagedBean; @@ -21,6 +24,8 @@ import javax.faces.event.PhaseId; import mom.trd.opentheso.bdd.helper.CandidateHelper; import mom.trd.opentheso.bdd.helper.ConceptHelper; +import mom.trd.opentheso.core.imports.csv.CsvImportHelper; +import mom.trd.opentheso.core.imports.csv.CsvReadHelper; import mom.trd.opentheso.core.imports.rdf4j.ReadRdf4j; import mom.trd.opentheso.core.imports.rdf4j.helper.ImportRdf4jHelper; import mom.trd.opentheso.skosapi.SKOSDocumentation; @@ -56,6 +61,15 @@ public class rdf4jFileBean implements Serializable { private int typeImport; private String selectedIdentifier ="sans"; private String prefixHandle; + + + // import CSV + private char delimiterCsv = ','; + private int choiceDelimiter = 0; + private String thesaurusName; + private ArrayList conceptObjects; + private ArrayList langs; + /* @ManagedProperty(value = "#{langueBean}") private LanguageBean langueBean; @@ -85,6 +99,21 @@ public void init() { uploadEnable = true; BDDinsertEnable = false; selectedIdentifier = "sans"; + sKOSXmlDocument = null; + if(conceptObjects != null) + conceptObjects.clear(); + if(langs != null) + langs.clear(); + + } + + public void actionChoice() { + if(choiceDelimiter == 0) + delimiterCsv = ','; + if(choiceDelimiter == 1) + delimiterCsv = ';'; + if(choiceDelimiter == 2) + delimiterCsv = '\t'; } public void chargeSkos(FileUploadEvent event) { @@ -163,7 +192,9 @@ public void chargeFile(FileUploadEvent event) { case 2: chargeTurtle(event); break; - + case 3: + chargeCsv(event); + break; } } @@ -209,9 +240,17 @@ public void chargeJsonLd(FileUploadEvent event) { error.append(System.getProperty("line.separator")); error.append(ex.getMessage()); } - ReadRdf4j readRdf4j = null; + ReadRdf4j readRdf4j; try { readRdf4j = new ReadRdf4j(is, 1); + warning = readRdf4j.getMessage(); + progress = 100; + sKOSXmlDocument = readRdf4j.getsKOSXmlDocument(); + total = sKOSXmlDocument.getConceptList().size() + sKOSXmlDocument.getGroupList().size() + 1; + uri = sKOSXmlDocument.getTitle(); + uploadEnable = false; + BDDinsertEnable = true; + info = "File correctly loaded"; } catch (IOException ex) { error.append(System.getProperty("line.separator")); error.append(ex.getMessage()); @@ -219,14 +258,6 @@ public void chargeJsonLd(FileUploadEvent event) { error.append(System.getProperty("line.separator")); error.append(ex.getMessage()); } - warning = readRdf4j.getMessage(); - progress = 100; - sKOSXmlDocument = readRdf4j.getsKOSXmlDocument(); - total = sKOSXmlDocument.getConceptList().size() + sKOSXmlDocument.getGroupList().size() + 1; - uri = sKOSXmlDocument.getTitle(); - uploadEnable = false; - BDDinsertEnable = true; - info = "File correctly loaded"; } catch (Exception e) { error.append(System.getProperty("line.separator")); error.append(e.toString()); @@ -259,9 +290,17 @@ public void chargeJson(FileUploadEvent event) { error.append(System.getProperty("line.separator")); error.append(ex.getMessage()); } - ReadRdf4j readRdf4j = null; + ReadRdf4j readRdf4j; try { readRdf4j = new ReadRdf4j(is, 3); + warning = readRdf4j.getMessage(); + progress = 100; + sKOSXmlDocument = readRdf4j.getsKOSXmlDocument(); + total = sKOSXmlDocument.getConceptList().size() + sKOSXmlDocument.getGroupList().size() + 1; + uri = sKOSXmlDocument.getTitle(); + uploadEnable = false; + BDDinsertEnable = true; + info = "File correctly loaded"; } catch (IOException ex) { error.append(System.getProperty("line.separator")); error.append(ex.getMessage()); @@ -269,15 +308,7 @@ public void chargeJson(FileUploadEvent event) { error.append(System.getProperty("line.separator")); error.append(ex.getMessage()); } - if(readRdf4j == null) return; - warning = readRdf4j.getMessage(); - progress = 100; - sKOSXmlDocument = readRdf4j.getsKOSXmlDocument(); - total = sKOSXmlDocument.getConceptList().size() + sKOSXmlDocument.getGroupList().size() + 1; - uri = sKOSXmlDocument.getTitle(); - uploadEnable = false; - BDDinsertEnable = true; - info = "File correctly loaded"; + } catch (Exception e) { error.append(System.getProperty("line.separator")); error.append(e.toString()); @@ -288,6 +319,57 @@ public void chargeJson(FileUploadEvent event) { } } + /** + * + * @param event + */ + public void chargeCsv(FileUploadEvent event) { + progress = 0; + + if (!PhaseId.INVOKE_APPLICATION.equals(event.getPhaseId())) { + event.setPhaseId(PhaseId.INVOKE_APPLICATION); + event.queue(); + } else { + try { + CsvReadHelper csvReadHelper; + try { + csvReadHelper = new CsvReadHelper(delimiterCsv); + Reader reader1 = new InputStreamReader(event.getFile().getInputstream()); + if(! csvReadHelper.setLangs(reader1)){ + error.append(csvReadHelper.getMessage()); + } + Reader reader2 = new InputStreamReader(event.getFile().getInputstream()); + if (!csvReadHelper.readFile(reader2)) { + error.append(csvReadHelper.getMessage()); + } + + warning = csvReadHelper.getMessage(); + progress = 100; + + conceptObjects = csvReadHelper.getConceptObjects(); + if(conceptObjects != null) { + langs = csvReadHelper.getLangs(); + total = conceptObjects.size(); + uri = "";//csvReadHelper.getUri(); + uploadEnable = false; + BDDinsertEnable = true; + info = "File correctly loaded"; + } + } catch (Exception ex) { + error.append(System.getProperty("line.separator")); + error.append(ex.getMessage()); + } + + } catch (Exception e) { + error.append(System.getProperty("line.separator")); + error.append(e.toString()); + } finally { + showError(); + } + + } + } + /** * * @param event @@ -310,9 +392,17 @@ public void chargeTurtle(FileUploadEvent event) { error.append(System.getProperty("line.separator")); error.append(ex.getMessage()); } - ReadRdf4j readRdf4j = null; + ReadRdf4j readRdf4j; try { readRdf4j = new ReadRdf4j(is, 2); + warning = readRdf4j.getMessage(); + progress = 100; + sKOSXmlDocument = readRdf4j.getsKOSXmlDocument(); + total = sKOSXmlDocument.getConceptList().size() + sKOSXmlDocument.getGroupList().size() + 1; + uri = sKOSXmlDocument.getTitle(); + uploadEnable = false; + BDDinsertEnable = true; + info = "File correctly loaded"; } catch (IOException ex) { error.append(System.getProperty("line.separator")); error.append(ex.getMessage()); @@ -320,15 +410,6 @@ public void chargeTurtle(FileUploadEvent event) { error.append(System.getProperty("line.separator")); error.append(ex.getMessage()); } - warning = readRdf4j.getMessage(); - progress = 100; - sKOSXmlDocument = readRdf4j.getsKOSXmlDocument(); - total = sKOSXmlDocument.getConceptList().size() + sKOSXmlDocument.getGroupList().size() + 1; - uri = sKOSXmlDocument.getTitle(); - uploadEnable = false; - BDDinsertEnable = true; - - info = "File correctly loaded"; } catch (Exception e) { error.append(System.getProperty("line.separator")); error.append(e.toString()); @@ -437,9 +518,71 @@ private String getNotes(SKOSResource sKOSResource) { return notes; } + + + /** + * insert un thésaurus dans la BDD (CSV) + * + * @param selectedUserGroup + */ + public void insertCsvBDD(String selectedUserGroup) { + if(conceptObjects == null) return; + if(conceptObjects.isEmpty()) return; + + error = new StringBuffer(); + info = ""; + warning = ""; + + int idGroup; + + if(currentUser.getUser().isIsSuperAdmin()) + idGroup = -1; + else + idGroup = Integer.parseInt(selectedUserGroup); + + try { + progress = 0; + progress_abs = 0; + CsvImportHelper csvImportHelper = new CsvImportHelper(roleOnTheso.getNodePreference()); + csvImportHelper.setInfos( + formatDate, + currentUser.getUser().getIdUser(), idGroup, + connect.getWorkLanguage()); + + + progress_abs++; + progress = progress_abs / total * 100; + + if(!csvImportHelper.addTheso( + connect.getPoolConnexion(), + this, + thesaurusName, conceptObjects, + langs)) { + error.append(csvImportHelper.getMessage()); + } + + //new UserHelper().addRole(connect.getPoolConnexion().getConnection(), idUser,idRole, ImportRdf4jHelper.getIdFromUri(uri) , ""); + + uploadEnable = true; + BDDinsertEnable = false; + uri = null; + total = 0; + + info = "Thesaurus correctly insert into data base"; + info = info + "\n" + csvImportHelper.getMessage().toString(); +// showError(); + + } catch (Exception e) { + error.append(System.getProperty("line.separator")); + error.append(e.toString()); + } finally { + showError(); + } + + } /** - * insrt un thésaurus dans la BDD + * insert un thésaurus dans la BDD (Skos) * * @param selectedUserGroup */ @@ -782,4 +925,29 @@ public void setRoleOnTheso(RoleOnThesoBean roleOnTheso) { this.roleOnTheso = roleOnTheso; } + public char getDelimiterCsv() { + return delimiterCsv; + } + + public void setDelimiterCsv(char delimiterCsv) { + this.delimiterCsv = delimiterCsv; + } + + + public String getThesaurusName() { + return thesaurusName; + } + + public void setThesaurusName(String thesaurusName) { + this.thesaurusName = thesaurusName; + } + + public int getChoiceDelimiter() { + return choiceDelimiter; + } + + public void setChoiceDelimiter(int choiceDelimiter) { + this.choiceDelimiter = choiceDelimiter; + } + } diff --git a/src/main/java/mom/trd/opentheso/bdd/helper/TermHelper.java b/src/main/java/mom/trd/opentheso/bdd/helper/TermHelper.java index b722001..c7d13bd 100644 --- a/src/main/java/mom/trd/opentheso/bdd/helper/TermHelper.java +++ b/src/main/java/mom/trd/opentheso/bdd/helper/TermHelper.java @@ -584,8 +584,8 @@ private boolean addUSE(Connection conn, } } catch (SQLException sqle) { // Log exception - if (!sqle.getSQLState().equalsIgnoreCase("23505")) { - status = false; + if (sqle.getSQLState().equalsIgnoreCase("23505")) { + status = true; } } return status; diff --git a/src/main/java/mom/trd/opentheso/core/imports/csv/CsvImportHelper.java b/src/main/java/mom/trd/opentheso/core/imports/csv/CsvImportHelper.java index 6a4c75a..95da074 100644 --- a/src/main/java/mom/trd/opentheso/core/imports/csv/CsvImportHelper.java +++ b/src/main/java/mom/trd/opentheso/core/imports/csv/CsvImportHelper.java @@ -6,14 +6,29 @@ package mom.trd.opentheso.core.imports.csv; import com.zaxxer.hikari.HikariDataSource; +import java.sql.Connection; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.logging.Level; +import java.util.logging.Logger; +import mom.trd.opentheso.SelectedBeans.rdf4jFileBean; import mom.trd.opentheso.bdd.datas.Concept; +import mom.trd.opentheso.bdd.datas.ConceptGroupLabel; import mom.trd.opentheso.bdd.datas.Term; +import mom.trd.opentheso.bdd.datas.Thesaurus; +import mom.trd.opentheso.bdd.helper.AlignmentHelper; import mom.trd.opentheso.bdd.helper.ConceptHelper; +import mom.trd.opentheso.bdd.helper.GpsHelper; +import mom.trd.opentheso.bdd.helper.GroupHelper; import mom.trd.opentheso.bdd.helper.NoteHelper; +import mom.trd.opentheso.bdd.helper.RelationsHelper; import mom.trd.opentheso.bdd.helper.TermHelper; +import mom.trd.opentheso.bdd.helper.ThesaurusHelper; +import mom.trd.opentheso.bdd.helper.ToolsHelper; +import mom.trd.opentheso.bdd.helper.UserHelper2; +import mom.trd.opentheso.bdd.helper.nodes.NodeAlignment; import mom.trd.opentheso.bdd.helper.nodes.NodePreference; - /** * * @author miled.rousset @@ -22,15 +37,43 @@ public class CsvImportHelper { private String message = ""; private NodePreference nodePreference; - + + private String langueSource; + private String formatDate; + private String adressSite; + private int idUser; + private int idGroupUser; + + private String idDefaultGroup; + public CsvImportHelper(NodePreference nodePreference) { this.nodePreference = nodePreference; } + /** + * initialisation des paramètres d'import + * + * @param formatDate + * @param idGroupUser + * @param idUser + * @param langueSource + * @return + */ + public boolean setInfos( + String formatDate, int idUser, + int idGroupUser, + String langueSource) { + this.formatDate = formatDate; + this.idUser = idUser; + this.idGroupUser = idGroupUser; + this.langueSource = langueSource; + return true; + } + public String getMessage() { return message; } - + public void addSingleConcept( HikariDataSource ds, String idTheso, @@ -42,43 +85,44 @@ public void addSingleConcept( boolean first = true; String idConcept = null; String idTerm = null; - + // ajout du concept ConceptHelper conceptHelper = new ConceptHelper(); conceptHelper.setNodePreference(nodePreference); Concept concept = new Concept(); TermHelper termHelper = new TermHelper(); - + // On vérifie si le conceptPere est un Groupe, alors il faut ajouter un TopTerm, sinon, c'est un concept avec des reraltions - if(idConceptPere == null) + if (idConceptPere == null) { concept.setTopConcept(true); - else + } else { concept.setTopConcept(false); - + } + concept.setIdGroup(idGroup); concept.setIdThesaurus(idTheso); concept.setStatus(""); concept.setNotation(""); - concept.setIdConcept(conceptObject.getId()); + concept.setIdConcept(conceptObject.getId()); Term term = new Term(); term.setId_thesaurus(idTheso); - + // ajout des PrefLabel for (CsvReadHelper.Label prefLabel : conceptObject.getPrefLabels()) { - if(first) { + if (first) { term.setLang(prefLabel.getLang()); term.setLexical_value(prefLabel.getLabel()); term.setSource(""); - term.setStatus(""); - idConcept = conceptHelper.addConcept(ds, idConceptPere, "NT", concept, term, idUser); - if(idConcept == null) { + term.setStatus(""); + idConcept = conceptHelper.addConcept(ds, idConceptPere, "NT", concept, term, idUser); + if (idConcept == null) { message = message + "\n" + "erreur dans l'intégration du concept " + prefLabel.getLabel(); } idTerm = termHelper.getIdTermOfConcept(ds, idConcept, idTheso); - if(idTerm == null) { + if (idTerm == null) { message = message + "\n" + "erreur dans l'intégration du concept " + prefLabel.getLabel(); - } + } first = false; } // ajout des traductions else { @@ -97,20 +141,18 @@ public void addSingleConcept( } } } - + NoteHelper noteHelper = new NoteHelper(); // add définition if (idConcept != null) { - for (CsvReadHelper.Label definition : conceptObject.getDefinition()) { - noteHelper.addTermNote(ds,idTerm, - definition.getLang(), - idTheso, - definition.getLabel(), - "definition", idUser); + for (CsvReadHelper.Label definition : conceptObject.getDefinitions()) { + noteHelper.addTermNote(ds, idTerm, + definition.getLang(), + idTheso, + definition.getLabel(), + "definition", idUser); } } - - // add altLabel if (idConcept != null) { for (CsvReadHelper.Label altLabel : conceptObject.getAltLabels()) { @@ -120,14 +162,612 @@ public void addSingleConcept( term.setLexical_value(altLabel.getLabel()); term.setHidden(false); term.setStatus("USE"); - term.setSource(""); - + term.setSource(""); + if (!termHelper.addNonPreferredTerm(ds, term, idUser)) { message = message + "\n" + "erreur dans l'intégration du synonyme : " + altLabel.getLabel(); } } } - } + + /** + * permet d'intégrer le thésaurs dans la base de données (d'après un objet + * lu d'un fichier CSV) + * + * @param ds + * @param fileBean + * @param thesoName + * @param conceptObject + * @param langs + * @return + */ + public boolean addTheso(HikariDataSource ds, + rdf4jFileBean fileBean, + String thesoName, + ArrayList conceptObject, + ArrayList langs) { + // création du thésaurus + String idTheso = createTheso(ds, thesoName); + if (idTheso == null) { + return false; + } + addLangsToThesaurus(ds, idTheso, thesoName, langs); + if (!addGroupDefault(ds, idTheso)) { + return false; + } + for (CsvReadHelper.ConceptObject conceptObject1 : conceptObject) { + fileBean.setAbs_progress(fileBean.getAbs_progress() + 1); + fileBean.setProgress(fileBean.getAbs_progress() / fileBean.getTotal() * 100); + + switch (conceptObject1.getType().trim().toLowerCase()) { + case "skos:concept": + // ajout de concept + if (!addConcept(ds, idTheso, conceptObject1)) { + return false; + } + break; + case "": + // ajout de concept + if (!addConcept(ds, idTheso, conceptObject1)) { + return false; + } + break; + default: + // ajout de groupe + if (!addGroup(ds, idTheso, conceptObject1)) { + return false; + } + break; + } + } + return true; + } + + /** + * Cette fonction permet de créer un thésaurus avec ses traductions (Import) + * elle retourne l'identifiant du thésaurus, sinon Null + * + * @return + */ + private String createTheso(HikariDataSource ds, String thesoName) { + try { + Thesaurus thesaurus = new Thesaurus(); + + thesaurus.setCreator(""); + thesaurus.setContributor(""); + + ThesaurusHelper thesaurusHelper = new ThesaurusHelper(); + thesaurusHelper.setIdentifierType("2"); + Connection conn = ds.getConnection(); + conn.setAutoCommit(false); + String idTheso1; + if (thesaurus.getLanguage() == null) { + thesaurus.setLanguage(langueSource); + } + if ((idTheso1 = thesaurusHelper.addThesaurusRollBack(conn, "", false)) == null) { + conn.rollback(); + conn.close(); + return null; + } + + thesaurus.setId_thesaurus(idTheso1); + + if (thesoName.isEmpty()) { + thesoName = "theso_" + idTheso1; + } + thesaurus.setTitle(thesoName); + + if (!thesaurusHelper.addThesaurusTraductionRollBack(conn, thesaurus)) { + conn.rollback(); + conn.close(); + return null; + } + + // ajouter le thésaurus dans le group de l'utilisateur + if (idGroupUser != -1) { // si le groupeUser = - 1, c'est le cas d'un SuperAdmin, alors on n'intègre pas le thésaurus dans un groupUser + UserHelper2 userHelper = new UserHelper2(); + if (!userHelper.addThesoToGroup(conn, thesaurus.getId_thesaurus(), + idGroupUser)) { + conn.rollback(); + conn.close(); + return null; + } + } + conn.commit(); + conn.close(); + + return idTheso1; + } catch (SQLException ex) { + Logger.getLogger(CsvImportHelper.class.getName()).log(Level.SEVERE, null, ex); + } + return null; + } + + /** + * permet d'ajouter les langues détectées au thésaurus + * + * @param ds + * @param idThesaurus + * @param langs + */ + private void addLangsToThesaurus(HikariDataSource ds, + String idThesaurus, + String name, + ArrayList langs) { + + ThesaurusHelper thesaurusHelper = new ThesaurusHelper(); + if (name.isEmpty()) { + name = "theso_" + idThesaurus; + } + + for (int i = 0; i < langs.size(); i++) { + if (!thesaurusHelper.isLanguageExistOfThesaurus(ds, idThesaurus, langs.get(i).trim())) { + Thesaurus thesaurus1 = new Thesaurus(); + thesaurus1.setId_thesaurus(idThesaurus); + thesaurus1.setContributor(""); + thesaurus1.setCoverage(""); + thesaurus1.setCreator(""); + thesaurus1.setDescription(""); + thesaurus1.setFormat(""); + thesaurus1.setLanguage(langs.get(i)); + thesaurus1.setPublisher(""); + thesaurus1.setRelation(""); + thesaurus1.setRights(""); + thesaurus1.setSource(""); + thesaurus1.setSubject(""); + thesaurus1.setTitle(name); + thesaurus1.setType(""); + thesaurusHelper.addThesaurusTraduction(ds, thesaurus1); + } + } + } + + /** + * Permet d'ajouter les groupes/collections... + * + * @param ds + * @param idTheso + * @param conceptObject + * @return + */ + private boolean addGroup(HikariDataSource ds, + String idTheso, + CsvReadHelper.ConceptObject conceptObject) { + // récupération des groups ou domaine + GroupHelper groupHelper = new GroupHelper(); + + String idGroup = conceptObject.getId(); + if (idGroup == null || idGroup.isEmpty()) { + message = message + "\n" + "Identifiant Groupe manquant"; + return false; + } + groupHelper.insertGroup(ds, idGroup, + idTheso, "C", + conceptObject.getNotation(), + "", false, idUser); + + ConceptGroupLabel conceptGroupLabel = new ConceptGroupLabel(); + for (CsvReadHelper.Label label : conceptObject.getPrefLabels()) { + // ajouter les traductions des Groupes + conceptGroupLabel.setIdgroup(idGroup); + conceptGroupLabel.setIdthesaurus(idTheso); + conceptGroupLabel.setLang(label.getLang()); + conceptGroupLabel.setLexicalvalue(label.getLabel()); + groupHelper.addGroupTraduction(ds, conceptGroupLabel, idUser); + } + + return true; + } + + private boolean addConcept(HikariDataSource ds, + String idTheso, + CsvReadHelper.ConceptObject conceptObject) { + + if (!addPrefLabel(ds, idTheso, conceptObject)) { + return false; + } + // synonymes et cachés + if (!addAltLabels(ds, idTheso, conceptObject)) { + return false; + } + // notes + if (!addNotes(ds, idTheso, conceptObject)) { + return false; + } + // relations + if (!addRelations(ds, idTheso, conceptObject)) { + return false; + } + // alignements + if (!addAlignments(ds, idTheso, conceptObject)) { + return false; + } + // géolocalisation + if (!addGeoLocalisation(ds, idTheso, conceptObject)) { + return false; + } + // Membres ou appartenance aux groupes + if (!addMembers(ds, idTheso, conceptObject)) { + return false; + } + + return true; + } + + private boolean addPrefLabel( + HikariDataSource ds, + String idTheso, + CsvReadHelper.ConceptObject conceptObject) { + + if (conceptObject.getId() == null || conceptObject.getId().isEmpty()) { + message = message + "\n" + "concept sans identifiant"; + return false; + } + + // ajout du concept + ConceptHelper conceptHelper = new ConceptHelper(); + TermHelper termHelper = new TermHelper(); + + conceptHelper.setNodePreference(nodePreference); + Concept concept = new Concept(); + + // On vérifie si le concept a des BT (termes génériques), alors il faut ajouter un TopTerm, sinon, c'est un concept avec des rerlations + if (conceptObject.getBroaders().isEmpty()) { + concept.setTopConcept(true); + } else { + concept.setTopConcept(false); + } + + concept.setIdThesaurus(idTheso); + concept.setStatus(""); + concept.setNotation(conceptObject.getNotation()); + concept.setIdConcept(conceptObject.getId()); + + // ajout du concept + if (!conceptHelper.insertConceptInTable(ds, concept, idUser)) { + message = message + "\n" + "erreur dans l'intégration du concept " + conceptObject.getId(); + } + + Term term = new Term(); + term.setId_thesaurus(idTheso); + term.setId_term(conceptObject.getId()); + term.setContributor(idUser); + term.setCreator(idUser); + term.setSource(""); + term.setStatus(""); + + try { + // ajout de la relation entre le concept et le terme + if (!termHelper.addLinkTerm(ds.getConnection(), term, conceptObject.getId(), idUser)) { + message = message + "\n" + "erreur dans l'intégration du concept " + conceptObject.getId(); + return false; + } + } catch (SQLException ex) { + Logger.getLogger(CsvImportHelper.class.getName()).log(Level.SEVERE, null, ex); + return false; + } + + // ajout des PrefLabel + for (CsvReadHelper.Label prefLabel : conceptObject.getPrefLabels()) { + // ajout des traductions + term.setId_thesaurus(idTheso); + term.setLang(prefLabel.getLang()); + term.setLexical_value(prefLabel.getLabel()); + term.setId_term(conceptObject.getId()); + term.setContributor(idUser); + term.setCreator(idUser); + term.setSource(""); + term.setStatus(""); + if (!conceptHelper.addConceptTraduction(ds, term, idUser)) { + message = message + "\n" + "erreur dans l'intégration du terme " + prefLabel.getLabel(); + } + } + return true; + } + + /** + * Intègre les synonymes et synonymes cachés + * + * @param ds + * @param idTheso + * @param conceptObject + * @return + */ + private boolean addAltLabels( + HikariDataSource ds, + String idTheso, + CsvReadHelper.ConceptObject conceptObject) { + Term term = new Term(); + TermHelper termHelper = new TermHelper(); + for (CsvReadHelper.Label altLabel : conceptObject.getAltLabels()) { + term.setId_term(conceptObject.getId()); + term.setId_thesaurus(idTheso); + term.setLang(altLabel.getLang()); + term.setLexical_value(altLabel.getLabel()); + term.setHidden(false); + term.setStatus("USE"); + term.setSource(""); + + if (!termHelper.addNonPreferredTerm(ds, + term, idUser)) { + message = message + "\n" + "erreur dans l'intégration du synonyme : " + altLabel.getLabel(); + } + } + for (CsvReadHelper.Label altLabel : conceptObject.getHiddenLabels()) { + term.setId_term(conceptObject.getId()); + term.setId_thesaurus(idTheso); + term.setLang(altLabel.getLang()); + term.setLexical_value(altLabel.getLabel()); + term.setHidden(true); + term.setStatus("Hiddden"); + term.setSource(""); + + if (!termHelper.addNonPreferredTerm(ds, + term, idUser)) { + message = message + "\n" + "erreur dans l'intégration du synonyme : " + altLabel.getLabel(); + } + } + return true; + } + + /** + * Intègre les notes + * + * @param ds + * @param idTheso + * @param conceptObject + * @return + */ + private boolean addNotes( + HikariDataSource ds, + String idTheso, + CsvReadHelper.ConceptObject conceptObject) { + NoteHelper noteHelper = new NoteHelper(); + for (CsvReadHelper.Label note : conceptObject.getDefinitions()) { + noteHelper.addTermNote(ds, conceptObject.getId(), + note.getLang(), + idTheso, + note.getLabel(), + "definition", idUser); + } + for (CsvReadHelper.Label note : conceptObject.getChangeNotes()) { + noteHelper.addTermNote(ds, conceptObject.getId(), + note.getLang(), + idTheso, + note.getLabel(), + "changeNote", idUser); + } + for (CsvReadHelper.Label note : conceptObject.getEditorialNotes()) { + noteHelper.addTermNote(ds, conceptObject.getId(), + note.getLang(), + idTheso, + note.getLabel(), + "editorialNote", idUser); + } + for (CsvReadHelper.Label note : conceptObject.getHistoryNotes()) { + noteHelper.addTermNote(ds, conceptObject.getId(), + note.getLang(), + idTheso, + note.getLabel(), + "historyNote", idUser); + } + for (CsvReadHelper.Label note : conceptObject.getScopeNotes()) { + noteHelper.addTermNote(ds, conceptObject.getId(), + note.getLang(), + idTheso, + note.getLabel(), + "scopeNote", idUser); + } + for (CsvReadHelper.Label note : conceptObject.getExamples()) { + noteHelper.addTermNote(ds, conceptObject.getId(), + note.getLang(), + idTheso, + note.getLabel(), + "example", idUser); + } + return true; + } + + private boolean addRelations( + HikariDataSource ds, + String idTheso, + CsvReadHelper.ConceptObject conceptObject) { + + RelationsHelper relationsHelper = new RelationsHelper(); + + for (String idConcept2 : conceptObject.getBroaders()) { + if (!relationsHelper.insertHierarchicalRelation(ds, + conceptObject.getId(), + idTheso, + "BT", + idConcept2)) { + message = message + "\n" + "erreur dans de la relation BT: " + conceptObject.getId(); + } + // pour créer la relation réciproque si elle n'existe pas + if (!relationsHelper.insertHierarchicalRelation(ds, + idConcept2, + idTheso, + "NT", + conceptObject.getId())) { + message = message + "\n" + "erreur dans de la relation BT: " + conceptObject.getId(); + } + } + for (String idConcept2 : conceptObject.getNarrowers()) { + if (!relationsHelper.insertHierarchicalRelation(ds, + conceptObject.getId(), + idTheso, + "NT", + idConcept2)) { + message = message + "\n" + "erreur dans de la relation NT: " + conceptObject.getId(); + } + // pour créer la relation réciproque si elle n'existe pas + if (!relationsHelper.insertHierarchicalRelation(ds, + idConcept2, + idTheso, + "BT", + conceptObject.getId())) { + message = message + "\n" + "erreur dans de la relation NT: " + conceptObject.getId(); + } + } + for (String idConcept2 : conceptObject.getRelateds()) { + if (!relationsHelper.insertHierarchicalRelation(ds, + conceptObject.getId(), + idTheso, + "RT", + idConcept2)) { + message = message + "\n" + "erreur dans de la relation RT: " + conceptObject.getId(); + } +// // pour créer la relation réciproque si elle n'existe pas +// if (!relationsHelper.insertHierarchicalRelation(ds, +// idConcept2, +// idTheso, +// "RT", +// conceptObject.getId())) { +// message = message + "\n" + "erreur dans de la relation RT: " + conceptObject.getId(); +// } + } + return true; + } + + private boolean addAlignments( + HikariDataSource ds, + String idTheso, + CsvReadHelper.ConceptObject conceptObject) { + + AlignmentHelper alignmentHelper = new AlignmentHelper(); + NodeAlignment nodeAlignment = new NodeAlignment(); + nodeAlignment.setId_author(idUser); + nodeAlignment.setConcept_target(""); + nodeAlignment.setThesaurus_target(""); + nodeAlignment.setInternal_id_concept(conceptObject.getId()); + nodeAlignment.setInternal_id_thesaurus(idTheso); + +// exactMatch = 1; +// closeMatch = 2; +// broadMatch = 3; +// relatedMatch = 4; +// narrowMatch = 5; + for (String uri : conceptObject.getExactMatchs()) { + nodeAlignment.setUri_target(uri); + nodeAlignment.setAlignement_id_type(1); + if (!alignmentHelper.addNewAlignment(ds, nodeAlignment)) { + message = message + "\n" + "erreur dans l'ajout de l'alignement : " + conceptObject.getId(); + } + } + for (String uri : conceptObject.getCloseMatchs()) { + nodeAlignment.setUri_target(uri); + nodeAlignment.setAlignement_id_type(2); + if (!alignmentHelper.addNewAlignment(ds, nodeAlignment)) { + message = message + "\n" + "erreur dans l'ajout de l'alignement : " + conceptObject.getId(); + } + } + for (String uri : conceptObject.getBroadMatchs()) { + nodeAlignment.setUri_target(uri); + nodeAlignment.setAlignement_id_type(3); + if (!alignmentHelper.addNewAlignment(ds, nodeAlignment)) { + message = message + "\n" + "erreur dans l'ajout de l'alignement : " + conceptObject.getId(); + } + } + for (String uri : conceptObject.getRelatedMatchs()) { + nodeAlignment.setUri_target(uri); + nodeAlignment.setAlignement_id_type(4); + if (!alignmentHelper.addNewAlignment(ds, nodeAlignment)) { + message = message + "\n" + "erreur dans l'ajout de l'alignement : " + conceptObject.getId(); + } + } + for (String uri : conceptObject.getNarrowMatchs()) { + nodeAlignment.setUri_target(uri); + nodeAlignment.setAlignement_id_type(5); + if (!alignmentHelper.addNewAlignment(ds, nodeAlignment)) { + message = message + "\n" + "erreur dans l'ajout de l'alignement : " + conceptObject.getId(); + } + } + + return true; + } + + private boolean addGeoLocalisation( + HikariDataSource ds, + String idTheso, + CsvReadHelper.ConceptObject conceptObject) { + + Double latitude; + Double longitude; + + if (conceptObject.getLatitude() == null || conceptObject.getLatitude().isEmpty()) { + return true; + } + if (conceptObject.getLongitude() == null || conceptObject.getLongitude().isEmpty()) { + return true; + } + try { + latitude = Double.parseDouble(conceptObject.getLatitude()); + longitude = Double.parseDouble(conceptObject.getLongitude()); + } catch (Exception e) { + return true; + } + GpsHelper gpsHelper = new GpsHelper(); + gpsHelper.insertCoordonees(ds, conceptObject.getId(), + idTheso, + latitude, longitude); + return true; + } + + private boolean addMembers( + HikariDataSource ds, + String idTheso, + CsvReadHelper.ConceptObject conceptObject) { + + GroupHelper groupHelper = new GroupHelper(); + if (conceptObject.getMembers().isEmpty()) { + // ajout dans le groupe par defaut (NoGroup) + groupHelper.addConceptGroupConcept(ds, idDefaultGroup, conceptObject.getId(), idTheso); + } else { + for (String member : conceptObject.getMembers()) { + groupHelper.addConceptGroupConcept(ds, member.trim(), conceptObject.getId(), idTheso); + } + } + return true; + } + + /** + * Permet d'ajouter le Groupe par défaut pour les concepts qui sont + * orphelins ou sans groupes + * + * @return + */ + private boolean addGroupDefault( + HikariDataSource ds, + String idTheso) { + GroupHelper groupHelper = new GroupHelper(); + ToolsHelper toolsHelper = new ToolsHelper(); + String idGroup = toolsHelper.getNewId(10); + while (groupHelper.isIdOfGroup(ds, idGroup, idTheso)) { + idGroup = toolsHelper.getNewId(10); + } + + groupHelper.insertGroup(ds, + idGroup, + idTheso, + "C", + "", //notation + "", + false, + idUser); + + // Création du domaine par défaut + // ajouter les traductions des Groupes + ConceptGroupLabel conceptGroupLabel = new ConceptGroupLabel(); + conceptGroupLabel.setIdgroup(idGroup); + conceptGroupLabel.setIdthesaurus(idTheso); + + conceptGroupLabel.setLang(langueSource); + conceptGroupLabel.setLexicalvalue("noGroup"); + groupHelper.addGroupTraduction(ds, conceptGroupLabel, idUser); + idDefaultGroup = idGroup; + return true; + } + } diff --git a/src/main/java/mom/trd/opentheso/core/imports/csv/CsvReadHelper.java b/src/main/java/mom/trd/opentheso/core/imports/csv/CsvReadHelper.java index 1b34da4..4920387 100644 --- a/src/main/java/mom/trd/opentheso/core/imports/csv/CsvReadHelper.java +++ b/src/main/java/mom/trd/opentheso/core/imports/csv/CsvReadHelper.java @@ -10,6 +10,7 @@ import java.util.ArrayList; import java.util.Map; import java.util.logging.Level; +import mom.trd.opentheso.bdd.tools.StringPlus; import org.apache.commons.csv.CSVFormat; import org.apache.commons.csv.CSVRecord; @@ -24,18 +25,22 @@ public class CsvReadHelper { private Logger log = LoggerFactory.getLogger(this.getClass().getName()); private String message = ""; - ArrayList langs; + private char delimiter = ','; + private String uri; + + private ArrayList langs; private final ArrayList conceptObjects; - public CsvReadHelper() { + public CsvReadHelper(char delimiter) { + this.delimiter = delimiter; conceptObjects = new ArrayList<>(); } public boolean setLangs(Reader in){ langs = new ArrayList<>(); try { - Map headers = CSVFormat.RFC4180.withFirstRecordAsHeader().parse(in).getHeaderMap(); + Map headers = CSVFormat.RFC4180.withFirstRecordAsHeader().withDelimiter(delimiter).parse(in).getHeaderMap(); String values[]; for (String columnName : headers.keySet()) { if(columnName.contains("@")) { @@ -51,55 +56,63 @@ public boolean setLangs(Reader in){ return !langs.isEmpty(); } - public boolean readFile(Reader in){ //String path) { + public boolean readFile(Reader in){ try { - //Reader in = new FileReader("/Users/Miled/Desktop/sample.csv"); - // Reader in = new FileReader(path); - - Iterable records = CSVFormat.RFC4180.withFirstRecordAsHeader().parse(in); - - Label label; - ArrayList