1.0.2: libreoffice compatibility

pom.xml

@@ -6,7 +6,7 @@
 
     <groupId>com.github.rzymek</groupId>
     <artifactId>opczip</artifactId>
-    <version>1.0.1</version>
+    <version>1.0.2</version>
     <packaging>jar</packaging>
 
     <name>OPC Output Stream</name>

src/main/java/com/github/rzymek/opczip/Zip64Impl.java

@@ -12,9 +12,12 @@ class Zip64Impl {
     private static final long PK0506 = 0x06054b50L;
     private static final long PK0708 = 0x08074b50L;
 
+    private static final int VERSION_20 = 20;
     private static final int VERSION_45 = 45;
     private static final int DATA_DESCRIPTOR_USED = 0x08;
     private static final int ZIP64_FIELD = 0x0001;
+    private static final long MAX32 = 0xffffffffL;
+
     private final OutputStream out;
     private int written = 0;
 
@@ -71,16 +74,16 @@ int writeDAT(Entry entry) throws IOException {
      */
     int writeCEN(Entry entry) throws IOException {
         written = 0;
-        boolean useZip64 = entry.size > 0xffffffffL;
+        boolean useZip64 = entry.size > MAX32;
         writeInt(PK0102);                              // "PK\001\002"
         writeShort(VERSION_45);                        // version made by: 4.5
-        writeShort(VERSION_45);                        // version required: 4.5
+        writeShort(useZip64 ? VERSION_45 : VERSION_20);// version required: 4.5
         writeShort(DATA_DESCRIPTOR_USED);              // flags: 8 = data descriptor used
         writeShort(ZipEntry.DEFLATED);                 // compression method: 8 = deflate
         writeInt(0);                                // file modification time & date
         writeInt(entry.crc);                           // CRC-32
         writeInt(entry.compressedSize);                // compressed size
-        writeInt(useZip64 ? 0xffffffffL : entry.size); // uncompressed size
+        writeInt(useZip64 ? MAX32 : entry.size); // uncompressed size
         writeShort(entry.filename.length());           // filename length
         writeShort(useZip64
                 ? (2 + 2 + 8)  /* short + short + long*/

src/test/java/com/github/rzymek/opczip/LibreOfficeCompatIT.java

@@ -0,0 +1,121 @@
+package com.github.rzymek.opczip;
+
+import com.github.rzymek.opczip.utils.StreamGobbler;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Enumeration;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipFile;
+import java.util.zip.ZipOutputStream;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static java.nio.file.Files.readAllBytes;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+
+public class LibreOfficeCompatIT {
+    static File input = new File("target/test-classes/libre.xlsx");
+    static File xlsxDir = new File("target/libre/");
+
+    @BeforeAll
+    public static void unzip() throws IOException {
+        unzip(input, xlsxDir);
+    }
+
+    @Test
+    public void base() throws Exception {
+        libreValidate(input);
+    }
+
+    @Test
+    public void jdk() throws Exception {
+        File xlsx = new File("target/jdk.xlsx");
+        xlsx.delete();
+        zip(xlsxDir, xlsx);
+        libreValidate(xlsx);
+    }
+    @Test
+    public void opczip() throws Exception {
+        File xlsx = new File("target/opczip.xlsx");
+        xlsx.delete();
+        opczip(xlsxDir, xlsx);
+        libreValidate(xlsx);
+    }
+
+    protected static void unzip(File input, File target) throws IOException {
+        target.mkdirs();
+        ZipFile zip = new ZipFile(input);
+        for (Enumeration<? extends ZipEntry> entries = zip.entries(); entries.hasMoreElements(); ) {
+            ZipEntry entry = entries.nextElement();
+            Path to = target.toPath().resolve(entry.getName());
+            to.getParent().toFile().mkdirs();
+            Files.copy(zip.getInputStream(entry), to);
+        }
+    }
+
+    protected void libreValidate(File input) throws IOException, InterruptedException {
+        File targetDir = new File("xlsxDir");
+        File output = new File(targetDir, input.getName().replace(".xlsx", ".csv"));
+        output.delete();
+        assertFalse(output.exists(), output + " exists");
+        Process soffice = new ProcessBuilder()
+                .command("soffice", "--headless",
+                        "--convert-to", "csv",
+                        "--outdir", targetDir.getPath(),
+                        input.getPath()
+                ).start();
+        ExecutorService executorService = Executors.newFixedThreadPool(2);
+        executorService.submit(new StreamGobbler(soffice.getErrorStream(), System.err::println));
+        executorService.submit(new StreamGobbler(soffice.getInputStream(), System.out::println));
+        soffice.waitFor();
+        assertEquals(
+                "ok\n",
+                new String(readAllBytes(output.toPath()), UTF_8)
+        );
+    }
+
+    public static void zip(File sourceDirPath, File zipFilePath) throws IOException {
+        Path p = Files.createFile(zipFilePath.toPath());
+        try (ZipOutputStream zs = new ZipOutputStream(Files.newOutputStream(p))) {
+            Path pp = sourceDirPath.toPath();
+            Files.walk(pp)
+                    .filter(path -> !Files.isDirectory(path))
+                    .forEach(path -> {
+                        ZipEntry zipEntry = new ZipEntry(pp.relativize(path).toString());
+                        try {
+                            zs.putNextEntry(zipEntry);
+                            Files.copy(path, zs);
+                            zs.closeEntry();
+                        } catch (IOException e) {
+                            throw new UncheckedIOException(e);
+                        }
+                    });
+        }
+    }
+    public static void opczip(File sourceDirPath, File zipFilePath) throws IOException {
+        Path p = Files.createFile(zipFilePath.toPath());
+        try (OpcOutputStream zs = new OpcOutputStream(Files.newOutputStream(p))) {
+            Path pp = sourceDirPath.toPath();
+            Files.walk(pp)
+                    .filter(path -> !Files.isDirectory(path))
+                    .forEach(path -> {
+                        ZipEntry zipEntry = new ZipEntry(pp.relativize(path).toString());
+                        try {
+                            zs.putNextEntry(zipEntry);
+                            Files.copy(path, zs);
+                            zs.closeEntry();
+                        } catch (IOException e) {
+                            throw new UncheckedIOException(e);
+                        }
+                    });
+        }
+    }
+}

src/test/java/com/github/rzymek/opczip/ZipCompatibilityTests.java

@@ -16,6 +16,7 @@
 import java.util.stream.Collectors;
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipFile;
+import java.util.zip.ZipInputStream;
 import java.util.zip.ZipOutputStream;
 
 import static org.hamcrest.MatcherAssert.assertThat;
@@ -41,6 +42,7 @@ private void validate(byte[] zipBytes) throws IOException {
         assertThat(zipBytes.length, greaterThan(0));
         validateCommonsCompress(new ByteArrayInputStream(zipBytes));
         validateUsingZipFile(zipBytes);
+//        validateJdk(new ByteArrayInputStream(zipBytes));
     }
 
     private byte[] generate(Consumer<OutputStream> consumer) throws IOException {
@@ -74,6 +76,20 @@ private void validateCommonsCompress(InputStream in) throws IOException {
         assertThat(leftToRead, empty());
     }
 
+    private void validateJdk(InputStream in) throws IOException {
+        ZipInputStream zip = new ZipInputStream(in);
+        Set<String> leftToRead = new HashSet<>(contents.keySet());
+        for (; ; ) {
+            ZipEntry entry = zip.getNextEntry();
+            if (entry == null) {
+                break;
+            }
+            assertEquals(contents.get(entry.getName()), readFully(zip));
+            assertTrue(leftToRead.remove(entry.getName()));
+        }
+        assertThat(leftToRead, empty());
+    }
+
     private void validateZipFile(File file) throws IOException {
         ZipFile zipFile = new ZipFile(file);
         Set<String> leftToRead = new HashSet<>(contents.keySet());

src/test/java/com/github/rzymek/opczip/utils/StreamGobbler.java

@@ -0,0 +1,22 @@
+package com.github.rzymek.opczip.utils;
+
+import java.io.BufferedReader;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.util.function.Consumer;
+
+public class StreamGobbler implements Runnable {
+    private InputStream inputStream;
+    private Consumer<String> consumer;
+
+    public StreamGobbler(InputStream inputStream, Consumer<String> consumer) {
+        this.inputStream = inputStream;
+        this.consumer = consumer;
+    }
+
+    @Override
+    public void run() {
+        new BufferedReader(new InputStreamReader(inputStream)).lines()
+                .forEach(consumer);
+    }
+}