Merge branch 'feature/hdt' into 'development'

Feature/hdt Closes #85 See merge request rml/proc/rmlmapper-java!33
RMLio · Nov 6, 2018 · 020094b · 020094b
2 parents 845052b + 58faa80
commit 020094b
Show file tree

Hide file tree

Showing 6 changed files with 116 additions and 9 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 ## Unreleased
 
+### Added
+
+- output format: hdt
+
 ### Fixed
 
 - local build on Windows 7 works

diff --git a/pom.xml b/pom.xml
@@ -13,6 +13,13 @@
         <url>https://github.com/RMLio/rmlmapper-java</url>
     </scm>
 
+    <repositories>
+        <repository>
+            <id>jitpack.io</id>
+            <url>https://jitpack.io</url>
+        </repository>
+    </repositories>
+
     <dependencies>
         <dependency>
             <groupId>ch.qos.logback</groupId>
@@ -104,6 +111,21 @@
             <artifactId>jackson-core</artifactId>
             <version>2.9.6</version>
         </dependency>
+        <dependency>
+            <groupId>com.github.rdfhdt</groupId>
+            <artifactId>hdt-java</artifactId>
+            <version>hdt-2.1-SNAPSHOT</version>
+        </dependency>
+        <!--<dependency>-->
+            <!--<groupId>org.rdfhdt</groupId>-->
+            <!--<artifactId>hdt-java-core</artifactId>-->
+            <!--<version>2.1-SNAPSHOT</version>-->
+        <!--</dependency>-->
+        <!--<dependency>-->
+            <!--<groupId>org.rdfhdt</groupId>-->
+            <!--<artifactId>hdt-api</artifactId>-->
+            <!--<version>2.1-SNAPSHOT</version>-->
+        <!--</dependency>-->
     </dependencies>
 
     <build>

diff --git a/src/main/java/be/ugent/rml/Utils.java b/src/main/java/be/ugent/rml/Utils.java
@@ -12,6 +12,11 @@
 import com.google.common.escape.Escaper;
 import com.google.common.net.UrlEscapers;
 import org.eclipse.rdf4j.rio.RDFParseException;
+import org.rdfhdt.hdt.enums.RDFNotation;
+import org.rdfhdt.hdt.exceptions.ParserException;
+import org.rdfhdt.hdt.hdt.HDT;
+import org.rdfhdt.hdt.hdt.HDTManager;
+import org.rdfhdt.hdt.options.HDTSpecification;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.eclipse.rdf4j.model.Model;
@@ -400,4 +405,21 @@ public static String hashCode(String s) {
         }
         return Integer.toString(Math.abs(hash));
     }
+
+    public static void ntriples2hdt(String rdfInputPath, String hdtOutputPath) {
+        // Configuration variables
+        String baseURI = "http://example.com/mydataset";
+        String inputType = "ntriples";
+
+        try {
+            // Create HDT from RDF file
+            HDT hdt = HDTManager.generateHDT(rdfInputPath, baseURI, RDFNotation.parse(inputType), new HDTSpecification(), null);
+            // Save generated HDT to a file
+            hdt.saveToHDT(hdtOutputPath, null);
+            // IMPORTANT: Free resources
+            hdt.close();
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+    }
 }
diff --git a/src/main/java/be/ugent/rml/cli/Main.java b/src/main/java/be/ugent/rml/cli/Main.java
@@ -129,7 +129,7 @@ public static void main(String[] args) {
                 String outputFormat = getPriorityOptionValue(serializationFormatOption, lineArgs, configFile);
                 QuadStore outputStore;
 
-                if (outputFormat == null || outputFormat.equals("nquads")) {
+                if (outputFormat == null || outputFormat.equals("nquads") || outputFormat.equals("hdt")) {
                     outputStore = new SimpleQuadStore();
                 } else {
                     outputStore = new RDF4JStore();
@@ -261,12 +261,34 @@ private static void setLoggerLevel(Level level) {
     }
 
     private static void writeOutput(QuadStore store, String outputFile, String format) {
-
-        if (format != null) {
-            format = format.toLowerCase();
+        boolean hdt = format != null && format.equals("hdt");
+
+        if (hdt) {
+            try {
+                format = "nquads";
+                File tmpFile = File.createTempFile("file", ".nt");
+                tmpFile.deleteOnExit();
+                String uncompressedOutputFile = tmpFile.getAbsolutePath();
+
+                File nquadsFile = writeOutputUncompressed(store, uncompressedOutputFile, format);
+                Utils.ntriples2hdt(uncompressedOutputFile, outputFile);
+                nquadsFile.deleteOnExit();
+            } catch (IOException e) {
+                e.printStackTrace();
+            }
         } else {
-            format = "nquads";
+            if (format != null) {
+                format = format.toLowerCase();
+            } else {
+                format = "nquads";
+            }
+
+            writeOutputUncompressed(store, outputFile, format);
         }
+    }
+
+    private static File writeOutputUncompressed(QuadStore store, String outputFile, String format) {
+        File targetFile = null;
 
         if (store.size() > 1) {
             logger.info(store.size() + " quads were generated");
@@ -275,13 +297,12 @@ private static void writeOutput(QuadStore store, String outputFile, String forma
         }
 
         try {
-
             BufferedWriter out;
             String doneMessage = null;
 
             //if output file provided, write to triples output file
             if (outputFile != null) {
-                File targetFile = new File(outputFile);
+                targetFile = new File(outputFile);
                 logger.info("Writing quads to " + targetFile.getPath() + "...");
 
                 if (!targetFile.isAbsolute()) {
@@ -305,5 +326,7 @@ private static void writeOutput(QuadStore store, String outputFile, String forma
         } catch(IOException e) {
             System.err.println( "Writing output failed. Reason: " + e.getMessage() );
         }
+
+        return targetFile;
     }
 }
diff --git a/src/test/java/be/ugent/rml/Arguments_Test.java b/src/test/java/be/ugent/rml/Arguments_Test.java
@@ -2,13 +2,16 @@
 
 import be.ugent.rml.cli.Main;
 import org.junit.Test;
+import org.rdfhdt.hdt.hdt.HDT;
+import org.rdfhdt.hdt.hdt.HDTManager;
+import org.rdfhdt.hdt.triples.*;
 
-import java.io.File;
-import java.io.IOException;
+import java.io.*;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Paths;
 
+import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
 
 public class Arguments_Test extends TestCore {
@@ -107,6 +110,39 @@ public void outputJSON() {
         }
     }
 
+    @Test
+    public void outputHDT() throws IOException {
+        Main.main("-m ./argument/mapping.ttl -o ./generated_output.hdt -s hdt".split(" "));
+
+        File file1 = new File("./src/test/resources/argument/output-hdt/target_output.hdt");
+        File file2 = new File("./generated_output.hdt");
+
+        // Load HDT file.
+        HDT hdt1 = HDTManager.loadHDT(file1.getAbsolutePath(), null);
+        HDT hdt2 = HDTManager.loadHDT(file2.getAbsolutePath(), null);
+
+        try {
+            Triples triples1 = hdt1.getTriples();
+            Triples triples2 = hdt2.getTriples();
+
+            assertEquals(triples1.size(), triples2.size());
+
+            IteratorTripleID iteratorTripleID1 = triples1.searchAll();
+            IteratorTripleID iteratorTripleID2 = triples2.searchAll();
+
+            while(iteratorTripleID1.hasNext()) {
+                TripleID tripleID1 = iteratorTripleID1.next();
+                TripleID tripleID2 = iteratorTripleID2.next();
+
+                assertTrue(tripleID1.equals(tripleID2));
+            }
+        } finally {
+            hdt1.close();
+            hdt2.close();
+            assertTrue(file2.delete());
+        }
+    }
+
 
     @Test
     public void quoteInLiteral() {

diff --git a/src/test/resources/argument/output-hdt/target_output.hdt b/src/test/resources/argument/output-hdt/target_output.hdt