Merge pull request #2910 from entur/otp2_ds6_Google_cloud_storage_sup…

…port

Otp2 ds6 google cloud storage support

docs/sandbox/GoogleCloudStorage.md

@@ -0,0 +1,31 @@
+# Google Cloud Storage - Using GCS Bucket as a OTP Data Source 
+
+## Contact Info
+
+- Thomas Gran, Entur, Norway
+
+
+## Changelog
+
+### OTP 2.0
+- Initial implementation of Google Cloud Storage
+
+## Documentation
+To enable this turn on `OTPFeature`. Each artifact to load or save to the cloud must be 
+configured in build-config.json. See `StorageParameters` on how to configure artifacts.
+
+Example (build-config.json):
+```json
+{
+    :
+    storage : {
+        gcsCredentials: "/Users/alf/secret/otp-test-1234567890.json",
+        osm : [ "gs://otp-test-bucket/a/b/northpole.pbf" ], 
+        dem : [ "gs://otp-test-bucket/a/b/northpole.dem.tif" ],
+        gtfs: [ "gs://otp-test-bucket/a/b/gtfs.zip" ],
+        graph: "gs://otp-test-bucket/a/b/graph.obj"
+        buildReportDir: "gs://otp-test-bucket/a/b/np-report"
+    }
+}
+```
+

pom.xml

@@ -526,6 +526,12 @@
             <artifactId>jul-to-slf4j</artifactId>
             <version>1.7.6</version>
         </dependency>
+        <dependency>
+            <groupId>com.google.cloud</groupId>
+            <artifactId>google-cloud-storage</artifactId>
+            <version>1.101.0</version>
+        </dependency>
+
         <!-- GEOTOOLS AND JTS TOPOLOGY: geometry, rasters and projections. -->
         <!-- GEOTOOLS includes JTS as a transitive dependency. -->
         <dependency>

src/ext/java/org/opentripplanner/ext/datastore/gs/AbstractGsDataSource.java

@@ -0,0 +1,43 @@
+package org.opentripplanner.ext.datastore.gs;
+
+import com.google.cloud.storage.BlobId;
+import org.opentripplanner.datastore.DataSource;
+import org.opentripplanner.datastore.FileType;
+
+abstract class AbstractGsDataSource implements DataSource {
+    private final BlobId blobId;
+    private final FileType type;
+
+     AbstractGsDataSource(BlobId blobId, FileType type) {
+        this.blobId = blobId;
+        this.type = type;
+    }
+
+    BlobId blobId() {
+        return blobId;
+    }
+
+    String bucketName() {
+        return blobId.getBucket();
+    }
+
+    @Override
+    public final String name() {
+        return blobId.getName();
+    }
+
+    @Override
+    public final String path() {
+        return GsHelper.toUriString(blobId);
+    }
+
+    @Override
+    public final FileType type() {
+        return type;
+    }
+
+    @Override
+    public final String toString() {
+        return type + " " + path();
+    }
+}

src/ext/java/org/opentripplanner/ext/datastore/gs/GsDataSourceRepository.java

@@ -0,0 +1,107 @@
+package org.opentripplanner.ext.datastore.gs;
+
+
+import com.google.auth.oauth2.GoogleCredentials;
+import com.google.cloud.storage.Blob;
+import com.google.cloud.storage.BlobId;
+import com.google.cloud.storage.Storage;
+import com.google.cloud.storage.StorageOptions;
+import org.opentripplanner.datastore.CompositeDataSource;
+import org.opentripplanner.datastore.DataSource;
+import org.opentripplanner.datastore.FileType;
+import org.opentripplanner.datastore.base.DataSourceRepository;
+import org.opentripplanner.datastore.base.ZipStreamDataSourceDecorator;
+
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.net.URI;
+import java.util.Collections;
+
+/**
+ * This data store uses the local file system to access in-/out- data files.
+ */
+public class GsDataSourceRepository implements DataSourceRepository {
+    private final String credentialsFilename;
+    private Storage storage;
+
+    public GsDataSourceRepository(String credentialsFilename) {
+        this.credentialsFilename = credentialsFilename;
+    }
+
+    @Override
+    public void open() {
+        this.storage = connectToStorage();
+    }
+
+    @Override
+    public String description() {
+        return "Google Cloud Storage";
+    }
+
+    @Override
+    public DataSource findSource(URI uri, FileType type) {
+        if(skipUri(uri)) { return null; }
+        BlobId blobId = GsHelper.toBlobId(uri);
+        return createSource(blobId, type);
+    }
+
+    @Override
+    public CompositeDataSource findCompositeSource(URI uri, FileType type) {
+        if(skipUri(uri)) { return null; }
+        return createCompositeSource(GsHelper.toBlobId(uri), type);
+    }
+
+    /* private methods */
+
+    private static boolean skipUri(URI uri) {
+        return !"gs".equals(uri.getScheme());
+    }
+
+    private DataSource createSource(BlobId blobId, FileType type) {
+        Blob blob = storage.get(blobId);
+
+        if(blob != null) {
+            return new GsFileDataSource(blob, type);
+        }
+        else {
+            return new GsOutFileDataSource(storage, blobId, type);
+        }
+    }
+
+    private CompositeDataSource createCompositeSource(BlobId blobId, FileType type) {
+        if(GsHelper.isRoot(blobId)) {
+            return new GsDirectoryDataSource(storage, blobId, type);
+        }
+
+        if(blobId.getName().endsWith(".zip")) {
+            Blob blob = storage.get(blobId);
+
+            if(blob == null) {
+                throw new IllegalArgumentException(
+                        type.text() +  " not found: " + GsHelper.toUriString(blobId)
+                );
+            }
+            DataSource gsSource = new GsFileDataSource(blob, type);
+            return new ZipStreamDataSourceDecorator(gsSource);
+        }
+        return new GsDirectoryDataSource(storage, blobId, type);
+    }
+
+    private Storage connectToStorage() {
+        try {
+            StorageOptions.Builder builder = StorageOptions.getDefaultInstance().toBuilder();
+
+            if(credentialsFilename != null) {
+                GoogleCredentials credentials = GoogleCredentials
+                        .fromStream(new FileInputStream(credentialsFilename))
+                        .createScoped(Collections.singletonList(
+                                "https://www.googleapis.com/auth/cloud-platform"));
+                builder.setCredentials(credentials);
+            }
+            return builder.build().getService();
+        }
+        catch (IOException e) {
+            throw new RuntimeException(e.getLocalizedMessage(), e);
+        }
+    }
+}

src/ext/java/org/opentripplanner/ext/datastore/gs/GsDirectoryDataSource.java

@@ -0,0 +1,104 @@
+package org.opentripplanner.ext.datastore.gs;
+
+import com.google.api.gax.paging.Page;
+import com.google.cloud.storage.Blob;
+import com.google.cloud.storage.BlobId;
+import com.google.cloud.storage.Bucket;
+import com.google.cloud.storage.Storage;
+import org.opentripplanner.datastore.CompositeDataSource;
+import org.opentripplanner.datastore.DataSource;
+import org.opentripplanner.datastore.FileType;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.function.Consumer;
+
+
+/**
+ * This is a an adapter to to simulate a file directory on a GCS. Files created using an instance of this
+ * class wil have a common namespace. It does only support creating new output sources, it can not
+ * be used to list files with the common namespace (directory path).
+ */
+public class GsDirectoryDataSource extends AbstractGsDataSource implements CompositeDataSource {
+
+    private final Storage storage;
+
+    GsDirectoryDataSource(Storage storage, BlobId blobId, FileType type) {
+        super(blobId, type);
+        this.storage = storage;
+    }
+
+    @Override
+    public boolean exists() {
+        return getBucket().list(
+                Storage.BlobListOption.prefix(name()),
+                Storage.BlobListOption.pageSize(1),
+                Storage.BlobListOption.currentDirectory()
+        ).getValues().iterator().hasNext();
+    }
+
+    @Override
+    public DataSource entry(String name) {
+        Blob blob = childBlob(name);
+        // If file exist
+        if(blob != null) {
+            return new GsFileDataSource(blob, type());
+        }
+        // New file
+        BlobId childBlobId = BlobId.of(bucketName(), childPath(name));
+        return new GsOutFileDataSource(storage, childBlobId, type());
+    }
+
+    @Override
+    public Collection<DataSource> content() {
+        Collection<DataSource> content = new ArrayList<>();
+        forEachChildBlob(blob -> content.add(new GsFileDataSource(blob, type())));
+        return content;
+    }
+
+    @Override
+    public void delete() {
+        forEachChildBlob(Blob::delete);
+    }
+
+    @Override
+    public void close() { }
+
+
+    /* private methods */
+
+    private Bucket getBucket() {
+        Bucket bucket = storage.get(bucketName());
+        if(bucket == null) {
+            throw new IllegalArgumentException("Bucket not found: " + bucketName());
+        }
+        return bucket;
+    }
+
+    private Blob childBlob(String name) {
+        return getBucket().get(childPath(name));
+    }
+
+    private String childPrefix() {
+        return GsHelper.isRoot(blobId()) ? "" : name() + "/";
+    }
+
+    private String childPath(String name) {
+        return childPrefix() + name;
+    }
+
+    private void forEachChildBlob(Consumer<Blob>  consumer) {
+        int pathIndex = childPrefix().length();
+        for (Blob blob : listBlobs().iterateAll()) {
+            String name = blob.getName().substring(pathIndex);
+            // Skip nested content
+            if(!name.contains("/")) {
+                consumer.accept(blob);
+            }
+        }
+    }
+
+    private Page<Blob> listBlobs() {
+        return getBucket().list(Storage.BlobListOption.prefix(childPrefix()));
+    }
+}

src/ext/java/org/opentripplanner/ext/datastore/gs/GsFileDataSource.java

@@ -0,0 +1,79 @@
+package org.opentripplanner.ext.datastore.gs;
+
+import com.google.cloud.storage.Blob;
+import org.opentripplanner.datastore.DataSource;
+import org.opentripplanner.datastore.FileType;
+import org.opentripplanner.datastore.file.DirectoryDataSource;
+import org.opentripplanner.datastore.file.ZipFileDataSource;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.zip.GZIPInputStream;
+
+import static java.nio.channels.Channels.newInputStream;
+import static java.nio.channels.Channels.newOutputStream;
+
+/**
+ * This class is a wrapper around and EXISTING Google Cloud Store bucket blob. It can
+ * be read and overwritten.
+ * <p>
+ * Reading compressed blobs is supported. The only format supported is gzip (extension .gz).
+ */
+class GsFileDataSource extends AbstractGsDataSource implements DataSource {
+    private final Blob blob;
+
+
+    /**
+     * Create a data source wrapper around a file. This wrapper handles GZIP(.gz) compressed files
+     * as well as normal files. It does not handle directories({@link DirectoryDataSource}) or
+     * zip-files {@link ZipFileDataSource} witch contain multiple files.
+     */
+    GsFileDataSource(Blob blob, FileType type) {
+        super(blob.getBlobId(), type);
+        this.blob = blob;
+    }
+
+    @Override
+    public long size() {
+        return blob.getSize();
+    }
+
+    @Override
+    public long lastModified() {
+        return blob.getUpdateTime();
+    }
+
+    @Override
+    public boolean exists() {
+        return blob.exists();
+    }
+
+    @Override
+    public boolean isWritable() {
+        return true;
+    }
+
+    @Override
+    public InputStream asInputStream() {
+        // We support both gzip and unzipped files when reading.
+        InputStream in = newInputStream(blob.reader());
+
+        if (blob.getName().endsWith(".gz")) {
+            try {
+                return new GZIPInputStream(in);
+            }
+            catch (IOException e) {
+                throw new IllegalStateException(e.getLocalizedMessage(), e);
+            }
+        }
+        else {
+            return in;
+        }
+    }
+
+    @Override
+    public OutputStream asOutputStream() {
+        return newOutputStream(blob.writer());
+    }
+}