Skip to content

Commit

Permalink
preparing 1.2.3
Browse files Browse the repository at this point in the history
  • Loading branch information
@maximelefrancois86
maximelefrancois86 committed Oct 3, 2018
2 parents 9652be1 + 907bfe2 commit 43a0d56
Show file tree
Hide file tree
Showing 44 changed files with 1,060 additions and 412 deletions.
2 changes: 1 addition & 1 deletion .gitignore
View file
Expand Up @@ -9,6 +9,7 @@ notes.txt
nbactions.xml
nb-configuration.xml
/nbproject/
/sparql-generate-jena/nbproject/

# Eclipse
.classpath
Expand All @@ -33,4 +34,3 @@ buildNumber.properties
# deploy website
.sh
.bat
/sparql-generate-jena/nbproject/
2 changes: 1 addition & 1 deletion pom.xml
View file
Expand Up @@ -376,4 +376,4 @@
</profile>
</profiles>

</project>
</project>
13 changes: 9 additions & 4 deletions sparql-generate-jena/pom.xml
View file
Expand Up @@ -76,15 +76,20 @@
<version>2.25.1</version>
</dependency>
<dependency>
<groupId>com.univocity</groupId>
<artifactId>univocity-parsers</artifactId>
<version>2.7.6</version>
<groupId>org.eclipse.paho</groupId>
<artifactId>org.eclipse.paho.client.mqttv3</artifactId>
<version>1.2.0</version>
</dependency>
<dependency>
<groupId>org.java-websocket</groupId>
<artifactId>Java-WebSocket</artifactId>
<version>1.3.9</version>
</dependency>
<dependency>
<groupId>com.univocity</groupId>
<artifactId>univocity-parsers</artifactId>
<version>2.7.6</version>
</dependency>
</dependencies>

<build>
Expand Down Expand Up @@ -433,4 +438,4 @@
</plugins>
</build>

</project>
</project>
8 changes: 4 additions & 4 deletions ...ate-jena/src/main/java/com/github/thesmartenergy/sparql/generate/jena/SPARQLGenerate.java
View file
Expand Up @@ -133,7 +133,6 @@ public static void init() {
fnreg.put(FUN_dateTime.URI, FUN_dateTime.class);
fnreg.put(FUN_GeoJSONGeometry.URI, FUN_GeoJSONGeometry.class);


IteratorFunctionRegistry itereg = IteratorFunctionRegistry.get();
itereg.put(ITER_JSONPath.URI, ITER_JSONPath.class);
itereg.put(ITER_JSONListKeys.URI, ITER_JSONListKeys.class);
Expand All @@ -152,10 +151,11 @@ public static void init() {
itereg.put(ITER_for.URI, ITER_for.class);
itereg.put(ITER_CSVMultipleOutput.URI, ITER_CSVMultipleOutput.class);
itereg.put(ITER_GeoJSONFeatures.URI, ITER_GeoJSONFeatures.class);
itereg.put(ITER_GeoJSON.URI, ITER_GeoJSON.class);
itereg.put(ITER_regexgroups.URI, ITER_regexgroups.class);
itereg.put(ITER_PeriodicHTTP.URI, ITER_PeriodicHTTP.class);
itereg.put(ITER_StreamWebSocket.URI, ITER_StreamWebSocket.class);
itereg.put(ITER_GeoJSON.URI, ITER_GeoJSON.class);
itereg.put(ITER_HTTPGet.URI, ITER_HTTPGet.class);
itereg.put(ITER_MQTTSubscribe.URI, ITER_MQTTSubscribe.class);
itereg.put(ITER_WebSocket.URI, ITER_WebSocket.class);


SPARQLParserRegistry.get()
Expand Down
42 changes: 17 additions & 25 deletions ...-jena/src/main/java/com/github/thesmartenergy/sparql/generate/jena/SPARQLGenerateCli.java
View file
Expand Up @@ -24,16 +24,7 @@
import com.github.thesmartenergy.sparql.generate.jena.stream.SPARQLGenerateStreamManager;
import com.github.thesmartenergy.sparql.generate.jena.utils.SPARQLGenerateUtils;
import com.google.gson.Gson;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.PrintStream;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.DefaultParser;
import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
import org.apache.commons.cli.*;
import org.apache.commons.io.IOUtils;
import org.apache.jena.graph.Triple;
import org.apache.jena.query.Dataset;
Expand All @@ -45,19 +36,18 @@
import org.apache.jena.sparql.core.Quad;
import org.apache.jena.sparql.serializer.SerializationContext;
import org.apache.jena.sparql.util.FmtUtils;
import org.apache.log4j.Appender;
import org.apache.log4j.ConsoleAppender;
import org.apache.log4j.Layout;
import org.slf4j.LoggerFactory;
import org.apache.log4j.*;
import org.slf4j.Logger;
import org.apache.log4j.Level;
import org.apache.log4j.PatternLayout;
import org.slf4j.LoggerFactory;

import java.io.*;
import java.time.Duration;
import java.time.Instant;
import java.util.concurrent.TimeUnit;

/**
*
* @author Noorani Bakerally <noorani.bakerally at emse.fr>, Maxime Lefrançois
* <maxime.lefrancois at emse.fr>
*
*/
public class SPARQLGenerateCli {

Expand All @@ -69,7 +59,7 @@ public class SPARQLGenerateCli {
private static final org.apache.log4j.Logger rootLogger = org.apache.log4j.Logger.getRootLogger();

public static void main(String[] args) throws ParseException {

Instant start = Instant.now();
CommandLine cl = CMDConfigurations.parseArguments(args);

if (cl.getOptions().length == 0) {
Expand All @@ -89,11 +79,11 @@ public static void main(String[] args) throws ParseException {
try {
String conf = IOUtils.toString(new FileInputStream(new File(dir, "sparql-generate-conf.json")), "utf-8");
request = gson.fromJson(conf, Request.class);
} catch(Exception ex) {
} catch (Exception ex) {
LOG.warn("Error while loading the location mapping model for the queryset. No named queries will be used");
request = Request.DEFAULT;
}

// initialize stream manager
SPARQLGenerateStreamManager sm = SPARQLGenerateStreamManager.makeStreamManager(new LocatorFileAccept(dir.toURI().getPath()));
sm.setLocationMapper(request.asLocationMapper());
Expand Down Expand Up @@ -173,6 +163,8 @@ public static void main(String[] args) throws ParseException {
}

}
long millis = Duration.between(start, Instant.now()).toMillis();
System.out.println("Program finished in " + String.format("%d min, %d sec", TimeUnit.MILLISECONDS.toMinutes(millis), TimeUnit.MILLISECONDS.toSeconds(millis) - TimeUnit.MINUTES.toSeconds(TimeUnit.MILLISECONDS.toMinutes(millis))));
}

private static class ConsoleStreamRDF implements StreamRDF {
Expand All @@ -181,8 +173,8 @@ private static class ConsoleStreamRDF implements StreamRDF {
private final SerializationContext context;

private PrintStream out;
int i = 0;

int i = 0;

public ConsoleStreamRDF(PrintStream out, PrefixMapping pm) {
this.out = out;
Expand Down Expand Up @@ -211,8 +203,8 @@ public void prefix(String prefix, String uri) {
public void triple(Triple triple) {
out.append(FmtUtils.stringForTriple(triple, context)).append(" .\n");
i++;
if(i>1000) {
i=0;
if (i > 1000) {
i = 0;
out.flush();
}
}
Expand Down
11 changes: 8 additions & 3 deletions ...ain/java/com/github/thesmartenergy/sparql/generate/jena/engine/impl/IteratorPlanImpl.java
View file
Expand Up @@ -96,9 +96,9 @@ final public void exec(
sb = null;
}
values.forEach((binding) -> {
List<BindingHashMapOverwrite> newValues = new ArrayList();
try {
iterator.exec(binding, exprList, null, (nodeValues) -> {
List<BindingHashMapOverwrite> newValues = new ArrayList<>();
if (nodeValues == null || nodeValues.isEmpty()) {
for (Var v : vars) {
newValues.add(new BindingHashMapOverwrite(binding, v, null));
Expand All @@ -109,8 +109,13 @@ final public void exec(
BindingHashMapOverwrite bindingHashMapOverwrite = new BindingHashMapOverwrite(binding, null, null);
for (int i = 0; i < vars.size(); i++) {
Var v = vars.get(i);
Node n = nodeValues.get(i).get(j).asNode();
bindingHashMapOverwrite.add(v, n);
try {
Node n = nodeValues.get(i).get(j).asNode();
bindingHashMapOverwrite.add(v, n);
} catch (IndexOutOfBoundsException ex) {
LOG.warn("The number of variables does not match the number of names provided to the iterator arguments");
break;
}
}
newValues.add(bindingHashMapOverwrite);
}
Expand Down
7 changes: 4 additions & 3 deletions ...ain/java/com/github/thesmartenergy/sparql/generate/jena/function/library/FUN_HTMLTag.java
View file
Expand Up @@ -20,11 +20,11 @@
import org.apache.jena.sparql.expr.NodeValue;
import org.apache.jena.sparql.expr.nodevalue.NodeValueString;
import org.apache.jena.sparql.function.FunctionBase2;
import org.jsoup.parser.Parser;
import org.slf4j.LoggerFactory;
import org.slf4j.Logger;
import org.jsoup.Jsoup;
import org.jsoup.parser.Parser;
import org.jsoup.select.Elements;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
* Binding function
Expand Down Expand Up @@ -67,6 +67,7 @@ public NodeValue exec(NodeValue html, NodeValue v2) {

String selectPath = String.valueOf(v2.asNode().getLiteralLexicalForm());
Elements elements = htmldoc.select(selectPath);
String x = elements.text();
return new NodeValueString(elements.text());
} catch (Exception ex) {
LOG.debug("No evaluation of " + html + ", " + v2, ex);
Expand Down
1 change: 0 additions & 1 deletion ...in/java/com/github/thesmartenergy/sparql/generate/jena/function/library/FUN_JSONPath.java
View file
Expand Up @@ -93,7 +93,6 @@ public NodeValue exec(NodeValue json, NodeValue jsonpath) {
try {
Object value = JsonPath.parse(json.asNode().getLiteralLexicalForm())
.limit(1).read(jsonpath.getString());

if (value instanceof String) {
return new NodeValueString((String) value);
} else if (value instanceof Float) {
Expand Down
4 changes: 2 additions & 2 deletions ...in/java/com/github/thesmartenergy/sparql/generate/jena/function/library/FUN_dateTime.java
View file
Expand Up @@ -80,9 +80,9 @@ public final class FUN_dateTime extends FunctionBase {
public NodeValue exec(List<NodeValue> args) {

NodeValue dt = args.get(0);
String dateTimeString = dt.getString();
String dateTimeString = dt.asString();

if (dt == null || dt.getString().isEmpty()) {
if (dt == null || dateTimeString.isEmpty()) {
LOG.debug("The NodeValue is Null");
throw new ExprEvalException("The NodeValue is Null");
}
Expand Down
4 changes: 2 additions & 2 deletions .../main/java/com/github/thesmartenergy/sparql/generate/jena/function/library/FUN_regex.java
View file
Expand Up @@ -80,8 +80,8 @@ public final class FUN_regex extends FunctionBase3 {
@Override
public NodeValue exec(NodeValue stringValue, NodeValue regex, NodeValue locationV) {

String string = stringValue.getString();
String regexString = regex.getString();
String string = stringValue.asString();
String regexString = regex.asString();

int location = locationV.getInteger().intValue();

Expand Down
8 changes: 4 additions & 4 deletions .../com/github/thesmartenergy/sparql/generate/jena/iterator/IteratorStreamFunctionBase4.java
View file
Expand Up @@ -43,10 +43,10 @@ public final void exec(List<NodeValue> args, Consumer<List<List<NodeValue>>> nod
+ ": Null args list");
}
int size = args.size();
NodeValue v1 = args.size() > 1 ? args.get(0) : null;
NodeValue v2 = args.size() > 2 ? args.get(1) : null;
NodeValue v3 = args.size() > 3 ? args.get(2) : null;
NodeValue v4 = args.size() > 4 ? args.get(3) : null;
NodeValue v1 = args.size() >= 1 ? args.get(0) : null;
NodeValue v2 = args.size() >= 2 ? args.get(1) : null;
NodeValue v3 = args.size() >= 3 ? args.get(2) : null;
NodeValue v4 = args.size() >= 4 ? args.get(3) : null;
exec(v1, v2, v3, v4, nodeValuesStream);
}

Expand Down
7 changes: 4 additions & 3 deletions ...in/java/com/github/thesmartenergy/sparql/generate/jena/iterator/library/ITER_CSSPath.java
View file
Expand Up @@ -41,13 +41,14 @@
* Iterator function
* <a href="http://w3id.org/sparql-generate/iter/CSSPath">iter:CSSPath</a>
* extracts a list of HTML elements from a HTML document, according to a
* Selector CSS-like query.
* CSS-Selector-like query.
*
* <ul>
* <li>Param 1: (html) is a literal that contains a HTML element;</li>
* <li>Param 2: (cssSelector) is the CSS-like query. See
* https://jsoup.org/apidocs/org/jsoup/select/Selector.html for the syntax
* specification;</li>
* <a href="https://jsoup.org/apidocs/org/jsoup/select/Selector.html">
* https://jsoup.org/apidocs/org/jsoup/select/Selector.html</a> for the syntax
* specification.</li>
* </ul>
*
* @author Noorani Bakerally <noorani.bakerally at emse.fr>
Expand Down
7 changes: 5 additions & 2 deletions ...c/main/java/com/github/thesmartenergy/sparql/generate/jena/iterator/library/ITER_CSV.java
View file
Expand Up @@ -42,19 +42,22 @@
/**
* Iterator function
* <a href="http://w3id.org/sparql-generate/iter/CSV">iter:CSV</a>
* returns a row of a CSV document, together with the header.
* returns CSV documents with two rows: the header and the current row.
*
* <ul>
* <li>Param 1: (csv) is the CSV document with a header line.</li>
* </ul>
*
* <p>
* For very large CSV files (typically above 100.000 lines), prefer <a href="http://w3id.org/sparql-generate/iter/CSVStream">CSVStream</a>.
* For very large CSV files (typically above 100.000 lines), prefer
* <a href="http://w3id.org/sparql-generate/iter/CSVStream">CSVStream</a>.
* </p>
*
* @author Noorani Bakerally <noorani.bakerally at emse.fr>
* @see com.github.thesmartenergy.sparql.generate.jena.function.library.FN_CustomCSV
* for CSV document with different dialects
* @see com.github.thesmartenergy.sparql.generate.jena.function.library.ITER_CSVStream
* to process very large CSV files
*/
public class ITER_CSV extends IteratorFunctionBase1 {

Expand Down
34 changes: 19 additions & 15 deletions ...m/github/thesmartenergy/sparql/generate/jena/iterator/library/ITER_CSVMultipleOutput.java
View file
Expand Up @@ -40,8 +40,8 @@
/**
* Iterator function
* <a href="http://w3id.org/sparql-generate/iter/CSVMultipleOutput">iter:CSVMultipleOutput</a>
* iterates over the rows of a CSV document.
* For each iteration, it binds the content of some cells to the given variables.
* processes CSV documents and iteratively binds the content of one or multiple
* cells to variables.
*
* <ul>
* <li>Param 1: (a String) is the CSV document with a header line;</li>
Expand All @@ -54,25 +54,29 @@
* </p>
*
* <b>Examples: </b>
* <p>
* Iterating over this CSV document<br>
* <p>Iterating over this CSV document</p>
* <pre>
* id,stop,latitude,longitude,date<br>
* 6523,25,50.901389,4.484444,01/01/01<br>
* 7000,40,56.901389,4.584444,02/02/02<br>
* 7001,41,57.901389,5.584444,03/03/03<br>
* 7002,42,58.901389,6.584444,23/12/80<br>
* id,stop,latitude,longitude,date
* 6523,25,50.901389,4.484444,01/01/01
* 7000,40,56.901389,4.584444,02/02/02
* 7001,41,57.901389,5.584444,03/03/03
* 7002,42,58.901389,6.584444,23/12/80
* </pre>
* with <tt>ITERATOR ite:CSVMultipleOutput(?source, ",", "id", "stop") AS ?id ?stop</tt> returns:<br>
* <p>with</p>
* <code>ITERATOR ite:CSVMultipleOutput(?source, ",", "id", "stop") AS
* ?id ?stop</code>
* <p>returns:</p>
* <pre>
* ?id => "6523"^^xsd#string, ?stop => "25"^^xsd#string<br>
* ?id => "7000"^^xsd#string, ?stop => "40"^^xsd#string<br>
* ?id => "7001"^^xsd#string, ?stop => "41"^^xsd#string<br>
* ?id => "7002"^^xsd#string, ?stop => "42"^^xsd#string<br>
* ?id => "6523"^^xsd#string, ?stop => "25"^^xsd#string
* ?id => "7000"^^xsd#string, ?stop => "40"^^xsd#string
* ?id => "7001"^^xsd#string, ?stop => "41"^^xsd#string
* ?id => "7002"^^xsd#string, ?stop => "42"^^xsd#string
* </pre>
*
* @author El Mehdi Khalfi <el-mehdi.khalfi at emse.fr>
* @since 2018-09-04
* @see com.github.thesmartenergy.sparql.generate.jena.function.library.ITER_CSVStream
* to process very large CSV files
*/
public class ITER_CSVMultipleOutput extends IteratorFunctionBase {

Expand Down Expand Up @@ -148,4 +152,4 @@ public void checkBuild(ExprList args) {
+ this.getClass().getName() + "' takes at least three arguments: (1) the CSV document, (2) the separator character, and (3) at least one column to iterate over.");
}
}
}
}

0 comments on commit 43a0d56

Please sign in to comment.