From 7c1124c22c07a3ac3b58c494b2adc3945d20d8b7 Mon Sep 17 00:00:00 2001 From: Pavel Pscheidl Date: Sat, 7 Jul 2018 09:21:13 +0200 Subject: [PATCH] Pavel pubdev 5688 gcs env check master (#2609) * PUBDEV-5688- GCS started in unauthorized regime when there is no connection available. (cherry picked from commit b481e14) * GCS Storage instance lazily initialized in PersistGcs class (cherry picked from commit df771bc) * GCS Storage lazy initialization moved to a provider class (cherry picked from commit eea73ba) --- .../water/persist/GcsStorageProvider.java | 28 ++++++++++++ .../main/java/water/persist/PersistGcs.java | 43 ++++++++++--------- 2 files changed, 51 insertions(+), 20 deletions(-) create mode 100644 h2o-persist-gcs/src/main/java/water/persist/GcsStorageProvider.java diff --git a/h2o-persist-gcs/src/main/java/water/persist/GcsStorageProvider.java b/h2o-persist-gcs/src/main/java/water/persist/GcsStorageProvider.java new file mode 100644 index 000000000000..aed7ae610b90 --- /dev/null +++ b/h2o-persist-gcs/src/main/java/water/persist/GcsStorageProvider.java @@ -0,0 +1,28 @@ +package water.persist; + +import com.google.cloud.storage.Storage; +import com.google.cloud.storage.StorageOptions; + +/** + * A class wrapping {@link Storage}, enabling safe lazy initialization by only providing getStorage method, not risking for + * developers to access storage field directly. + */ +final class GcsStorageProvider { + + private Storage storage; + + /** + * Returns an existing instance of {@link Storage} or creates a new one, if not initialized. + * Lazy-initialization of storage does not slow down startup of H2O (attempts are made to connect to GCS). + * The connection status and {@link com.google.auth.Credentials} are checked at actual request-time. + * + * @return An instance of {@link Storage}, if initialized + */ + protected Storage getStorage() { + if (storage == null) { + storage = StorageOptions.getDefaultInstance().getService(); + } + + return storage; + } +} diff --git a/h2o-persist-gcs/src/main/java/water/persist/PersistGcs.java b/h2o-persist-gcs/src/main/java/water/persist/PersistGcs.java index 9710bb23dd6d..a0c5fc6e8da5 100644 --- a/h2o-persist-gcs/src/main/java/water/persist/PersistGcs.java +++ b/h2o-persist-gcs/src/main/java/water/persist/PersistGcs.java @@ -29,9 +29,11 @@ /** * Persistence backend for GCS */ +@SuppressWarnings("unused") public final class PersistGcs extends Persist { - private final Storage storage = StorageOptions.getDefaultInstance().getService(); + private GcsStorageProvider storageProvider = new GcsStorageProvider(); + @Override public byte[] load(final Value v) throws IOException { @@ -46,7 +48,7 @@ public byte[] load(final Value v) throws IOException { offset = FileVec.chunkOffset(k); // The offset } - final ReadChannel reader = storage.reader(blobId); + final ReadChannel reader = storageProvider.getStorage().reader(blobId); reader.seek(offset); reader.read(wrappingBuffer); @@ -56,7 +58,7 @@ public byte[] load(final Value v) throws IOException { @Override public Key uriToKey(URI uri) throws IOException { final GcsBlob blob = GcsBlob.of(uri); - final Long contentSize = storage.get(blob.getBlobId()).getSize(); + final Long contentSize = storageProvider.getStorage().get(blob.getBlobId()).getSize(); return GcsFileVec.make(blob.getCanonical(), contentSize); } @@ -67,14 +69,14 @@ public void store(Value v) throws IOException { final GcsBlob blob = GcsBlob.of(v._key); Log.debug("Storing: " + blob.toString()); final ByteBuffer buffer = ByteBuffer.wrap(payload); - storage.create(blob.getBlobInfo()).writer().write(buffer); + storageProvider.getStorage().create(blob.getBlobInfo()).writer().write(buffer); } @Override public void delete(Value v) { final BlobId blobId = GcsBlob.of(v._key).getBlobId(); Log.debug("Deleting: " + blobId.toString()); - storage.get(blobId).delete(); + storageProvider.getStorage().get(blobId).delete(); } @Override @@ -89,7 +91,7 @@ public void cleanUp() { @Override public List load(String key) { final List blobs = new ArrayList<>(); - for (Blob b : storage.get(key).list().iterateAll()) { + for (Blob b : storageProvider.getStorage().get(key).list().iterateAll()) { blobs.add(b.getName()); } return blobs; @@ -103,7 +105,7 @@ public List load(String key) { @Override public List load(Object key) { final List fileNames = new ArrayList<>(); - for (Bucket b : storage.list().iterateAll()) { + for (Bucket b : storageProvider.getStorage().list().iterateAll()) { fileNames.add(b.getName()); } return fileNames; @@ -155,7 +157,7 @@ public void importFiles(String path, parseBucket(bk[0], files, keys, fails); } else { try { - Blob blob = storage.get(bk[0], bk[1]); + Blob blob = storageProvider.getStorage().get(bk[0], bk[1]); final GcsBlob gcsBlob = GcsBlob.of(blob.getBlobId()); final Key k = GcsFileVec.make(path, blob.getSize()); keys.add(k.toString()); @@ -172,7 +174,7 @@ private void parseBucket(String bucketId, ArrayList files, ArrayList keys, ArrayList fails) { - final Bucket bucket = storage.get(bucketId); + final Bucket bucket = storageProvider.getStorage().get(bucketId); for (Blob blob : bucket.list().iterateAll()) { final GcsBlob gcsBlob = GcsBlob.of(blob.getBlobId()); Log.debug("Importing: " + gcsBlob.toString()); @@ -191,7 +193,7 @@ private void parseBucket(String bucketId, public InputStream open(final String path) { final GcsBlob gcsBlob = GcsBlob.of(path); Log.debug("Opening: " + gcsBlob.toString()); - final Blob blob = storage.get(gcsBlob.getBlobId()); + final Blob blob = storageProvider.getStorage().get(gcsBlob.getBlobId()); return new InputStream() { final ReadChannel reader = blob.reader(); @@ -238,7 +240,7 @@ public void close() throws IOException { public OutputStream create(String path, boolean overwrite) { final GcsBlob gcsBlob = GcsBlob.of(path); Log.debug("Creating: " + gcsBlob.getCanonical()); - final WriteChannel writer = storage.create(gcsBlob.getBlobInfo()).writer(); + final WriteChannel writer = storageProvider.getStorage().create(gcsBlob.getBlobInfo()).writer(); return new OutputStream() { @Override public void write(int b) throws IOException { @@ -270,19 +272,19 @@ public boolean rename(String fromPath, String toPath) { final BlobId fromBlob = GcsBlob.of(fromPath).getBlobId(); final BlobId toBlob = GcsBlob.of(toPath).getBlobId(); - storage.get(fromBlob).copyTo(toBlob); + storageProvider.getStorage().get(fromBlob).copyTo(toBlob); keyCache.invalidate(fromBlob.getBucket()); keyCache.invalidate(toBlob.getBucket()); - return storage.delete(fromBlob); + return storageProvider.getStorage().delete(fromBlob); } @Override public boolean exists(String path) { final String bk[] = GcsBlob.removePrefix(path).split("/", 2); if (bk.length == 1) { - return storage.get(bk[0]).exists(); + return storageProvider.getStorage().get(bk[0]).exists(); } else if (bk.length == 2) { - Blob blob = storage.get(bk[0], bk[1]); + Blob blob = storageProvider.getStorage().get(bk[0], bk[1]); return blob != null && blob.exists(); } else { return false; @@ -293,13 +295,13 @@ public boolean exists(String path) { public boolean delete(String path) { final BlobId blob = GcsBlob.of(path).getBlobId(); keyCache.invalidate(blob.getBucket()); - return storage.get(blob).delete(); + return storageProvider.getStorage().get(blob).delete(); } @Override public long length(String path) { final BlobId blob = GcsBlob.of(path).getBlobId(); - return storage.get(blob).getSize(); + return storageProvider.getStorage().get(blob).getSize(); } /** @@ -316,7 +318,7 @@ public PersistEntry[] list(String path) { int substrLen = bk.length == 2 ? bk[1].length() : 0; List results = new ArrayList<>(); try { - for (Blob b : storage.list(bk[0]).iterateAll()) { + for (Blob b : storageProvider.getStorage().list(bk[0]).iterateAll()) { if (bk.length == 1 || (bk.length == 2 && b.getName().startsWith(bk[1]))) { String relativeName = b.getName().substring(substrLen); if (relativeName.startsWith("/")) { @@ -337,9 +339,9 @@ public boolean mkdirs(String path) { final String input = GcsBlob.removePrefix(path); final String[] bk = input.split("/", 2); if (bk.length > 0) { - Bucket b = storage.get(bk[0]); + Bucket b = storageProvider.getStorage().get(bk[0]); if (b == null || !b.exists()) { - storage.create(BucketInfo.of(bk[0])); + storageProvider.getStorage().create(BucketInfo.of(bk[0])); } return true; } else { @@ -350,4 +352,5 @@ public boolean mkdirs(String path) { return false; } } + }