From 72e0cb95c2ab1394723adb3a13d05dd53af97f9c Mon Sep 17 00:00:00 2001 From: Eugene Cheah Date: Tue, 19 Jul 2022 01:44:36 +0000 Subject: [PATCH 01/42] Tweak to moongodb index priority --- .../dstack/mongodb/MongoDB_DataObjectMap.java | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/main/java/picoded/dstack/mongodb/MongoDB_DataObjectMap.java b/src/main/java/picoded/dstack/mongodb/MongoDB_DataObjectMap.java index b08fbe0a..b956000a 100644 --- a/src/main/java/picoded/dstack/mongodb/MongoDB_DataObjectMap.java +++ b/src/main/java/picoded/dstack/mongodb/MongoDB_DataObjectMap.java @@ -91,13 +91,20 @@ public void systemSetup() { IndexOptions opt = new IndexOptions(); opt = opt.unique(true); opt = opt.name("_oid"); - opt = opt.background(true); + + // Due to the need for _oid to ensure consistency, we would not be creating it in the background + // opt = opt.background(true); + + // Lets create the index collection.createIndex(Indexes.ascending("_oid"), opt); - + + // // Wildcard indexing // // This helps improve general performance for arbitary data - // at a huge cost of write performance + // at a huge cost of write performance. Useful for general purpose DataObjectMap + // but not useful when fine hand-tunning is requried for some use cases + // if (configMap.getBoolean("setupWildcardIndex", true)) { opt = new IndexOptions(); opt = opt.name("wildcard"); From c090ebdd2731974530ffb2dcc0b56c6947443efa Mon Sep 17 00:00:00 2001 From: Eugene Cheah Date: Tue, 19 Jul 2022 02:59:08 +0000 Subject: [PATCH 02/42] Added read and write stream support polyfill's --- .../dstack/core/Core_FileWorkspace.java | 30 ++++++++- .../dstack/core/Core_FileWorkspaceMap.java | 63 ++++++++++++++++++- 2 files changed, 91 insertions(+), 2 deletions(-) diff --git a/src/main/java/picoded/dstack/core/Core_FileWorkspace.java b/src/main/java/picoded/dstack/core/Core_FileWorkspace.java index 95ac8832..ebf88418 100755 --- a/src/main/java/picoded/dstack/core/Core_FileWorkspace.java +++ b/src/main/java/picoded/dstack/core/Core_FileWorkspace.java @@ -2,6 +2,8 @@ // Java imports import java.io.File; +import java.io.InputStream; +import java.io.OutputStream; import java.util.*; // Picoded imports @@ -160,6 +162,32 @@ public void removeFile(final String filepath) { main.backend_removeFile(_oid, normalizeFilePathString(filepath)); } + // Read / write input/output stream + //-------------------------------------------------------------------------- + + /** + * Reads the contents of a file into a byte array. + * + * @param filepath in the workspace to extract + * + * @return the file contents, null if file does not exists + */ + public InputStream readByteStream(final String filepath) { + return main.backend_fileReadStream(_oid, normalizeFilePathString(filepath)); + } + + /** + * Writes an output array to a file creating the file if it does not exist. + * + * the parent directories of the file will be created if they do not exist. + * + * @param filepath in the workspace to extract + * @param data the content to write to the file + **/ + public void writeByteStream(final String filepath, final OutputStream data) { + main.backend_fileWriteStream(_oid, normalizeFilePathString(filepath), data); + } + // Read / write byteArray information //-------------------------------------------------------------------------- @@ -190,7 +218,7 @@ public void writeByteArray(final String filepath, final byte[] data) { * Appends a byte array to a file creating the file if it does not exist. * * NOTE that by default this DOES NOT perform any file locks. As such, - * if used in a concurrent access situation. Segmentys may get out of sync. + * if used in a concurrent access situation. Segments may get out of sync. * * @param file the file to write to * @param data the content to write to the file diff --git a/src/main/java/picoded/dstack/core/Core_FileWorkspaceMap.java b/src/main/java/picoded/dstack/core/Core_FileWorkspaceMap.java index 28bfb886..33556aed 100755 --- a/src/main/java/picoded/dstack/core/Core_FileWorkspaceMap.java +++ b/src/main/java/picoded/dstack/core/Core_FileWorkspaceMap.java @@ -6,8 +6,11 @@ import picoded.dstack.*; import java.util.HashSet; -import java.util.List; import java.util.Set; +import java.io.InputStream; +import java.io.OutputStream; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; /** * Common base utility class of FileWorkspaceMap @@ -160,6 +163,64 @@ public void setupWorkspace(String oid) { **/ abstract public void backend_fileWrite(final String oid, final String filepath, final byte[] data); + // File read and write using byte stream + //-------------------------------------------------------------------------- + + /** + * [Internal use, to be extended in future implementation] + * + * Get and return the stored data as a byte stream. + * + * This overwrite is useful for backends which supports this flow. + * Else it would simply be a wrapper over the non-stream version. + * + * @param ObjectID of workspace + * @param filepath to use for the workspace + * + * @return the stored byte array of the file + **/ + public InputStream backend_fileReadStream(final String oid, final String filepath) { + // Get the byte data + byte[] rawBytes = backend_fileRead(oid, filepath); + if( rawBytes == null ) { + return null; + } + return new ByteArrayInputStream( rawBytes ); + } + + /** + * [Internal use, to be extended in future implementation] + * + * Writes the full byte array of a file in the backend + * + * This overwrite is useful for backends which supports this flow. + * Else it would simply be a wrapper over the non-stream version. + * + * @param ObjectID of workspace + * @param filepath to use for the workspace + * @param data to write the file with + **/ + abstract public void backend_fileWriteStream(final String oid, final String filepath, final OutputStream data) { + + // forward the null, and let the error handling below settle it + if( data == null ) { + backend_fileWrite(oid, filepath, null); + } + + // Converts it to bytearray respectively + byte[] rawBytes = null; + if( data instanceof ByteArrayOutputStream ) { + rawBytes = data.toByteArray(); + } else { + ByteArrayOutputStream buffer = new ByteArrayOutputStream(); + buffer.writeTo(data); + rawBytes = buffer.toByteArray(); + } + + // Does the bytearray writes + backend_fileWrite(oid, filepath, rawBytes); + } + // Folder Pathing support //-------------------------------------------------------------------------- From 61277e88ed4bff32802345fed3e9ebf689ee903f Mon Sep 17 00:00:00 2001 From: Eugene Cheah Date: Tue, 19 Jul 2022 03:10:30 +0000 Subject: [PATCH 03/42] Reorganize Stack_FileWorkspaceMap code ordering --- .../dstack/stack/Stack_FileWorkspaceMap.java | 74 +++++++++++-------- 1 file changed, 42 insertions(+), 32 deletions(-) diff --git a/src/main/java/picoded/dstack/stack/Stack_FileWorkspaceMap.java b/src/main/java/picoded/dstack/stack/Stack_FileWorkspaceMap.java index 61cd347a..568b0af1 100755 --- a/src/main/java/picoded/dstack/stack/Stack_FileWorkspaceMap.java +++ b/src/main/java/picoded/dstack/stack/Stack_FileWorkspaceMap.java @@ -74,6 +74,10 @@ public CommonStructure[] commonStructureStack() { // [Internal use, to be extended in future implementation] // //-------------------------------------------------------------------------- + + // Workspace operations + //-------------------------------------------------------------------------- + /** * [Internal use, to be extended in future implementation] * @@ -110,6 +114,24 @@ public boolean backend_workspaceExist(String oid) { return false; } + /** + * Setup the current fileWorkspace within the fileWorkspaceMap, + * + * This ensures the workspace _oid is registered within the map, + * even if there is 0 files. + * + * Does not throw any error if workspace was previously setup + */ + @Override + public void backend_setupWorkspace(String oid) { + for (int i = dataLayers.length - 1; i >= 0; --i) { + dataLayers[i].backend_setupWorkspace(oid); + } + } + + // File read and write using byte array + //-------------------------------------------------------------------------- + /** * [Internal use, to be extended in future implementation] * @@ -141,6 +163,26 @@ public byte[] backend_fileRead(String oid, String filepath) { } + /** + * [Internal use, to be extended in future implementation] + * + * Writes the full byte array of a file in the backend + * + * @param ObjectID of workspace + * @param filepath to use for the workspace + * @param data to write the file with + **/ + @Override + public void backend_fileWrite(String oid, String filepath, byte[] data) { + // Write the data starting from the lowest layer + for (int i = dataLayers.length - 1; i >= 0; --i) { + dataLayers[i].backend_fileWrite(oid, filepath, data); + } + } + + // File exist / removal + //-------------------------------------------------------------------------- + /** * [Internal use, to be extended in future implementation] * @@ -167,23 +209,6 @@ public boolean backend_fileExist(final String oid, final String filepath) { return false; } - /** - * [Internal use, to be extended in future implementation] - * - * Writes the full byte array of a file in the backend - * - * @param ObjectID of workspace - * @param filepath to use for the workspace - * @param data to write the file with - **/ - @Override - public void backend_fileWrite(String oid, String filepath, byte[] data) { - // Write the data starting from the lowest layer - for (int i = dataLayers.length - 1; i >= 0; --i) { - dataLayers[i].backend_fileWrite(oid, filepath, data); - } - } - /** * [Internal use, to be extended in future implementation] * @@ -200,21 +225,6 @@ public void backend_removeFile(String oid, String filepath) { } } - /** - * Setup the current fileWorkspace within the fileWorkspaceMap, - * - * This ensures the workspace _oid is registered within the map, - * even if there is 0 files. - * - * Does not throw any error if workspace was previously setup - */ - @Override - public void backend_setupWorkspace(String oid) { - for (int i = dataLayers.length - 1; i >= 0; --i) { - dataLayers[i].backend_setupWorkspace(oid); - } - } - //-------------------------------------------------------------------------- // // Folder Pathing support From e62c4dedaac6d70a2a91bea0f452dcc7523518f4 Mon Sep 17 00:00:00 2001 From: Eugene Cheah Date: Tue, 19 Jul 2022 04:17:00 +0000 Subject: [PATCH 04/42] Adding fileRead/WriteStream stack optimization for single layer stack --- .../dstack/stack/Stack_FileWorkspaceMap.java | 80 ++++++++++++++++++- 1 file changed, 76 insertions(+), 4 deletions(-) diff --git a/src/main/java/picoded/dstack/stack/Stack_FileWorkspaceMap.java b/src/main/java/picoded/dstack/stack/Stack_FileWorkspaceMap.java index 568b0af1..dbfa42ce 100755 --- a/src/main/java/picoded/dstack/stack/Stack_FileWorkspaceMap.java +++ b/src/main/java/picoded/dstack/stack/Stack_FileWorkspaceMap.java @@ -3,6 +3,10 @@ import picoded.dstack.CommonStructure; import picoded.dstack.core.Core_FileWorkspaceMap; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.OutputStream; import java.util.List; import java.util.Set; @@ -133,8 +137,6 @@ public void backend_setupWorkspace(String oid) { //-------------------------------------------------------------------------- /** - * [Internal use, to be extended in future implementation] - * * Get and return the stored data as a byte[] * * @param ObjectID of workspace @@ -164,8 +166,6 @@ public byte[] backend_fileRead(String oid, String filepath) { } /** - * [Internal use, to be extended in future implementation] - * * Writes the full byte array of a file in the backend * * @param ObjectID of workspace @@ -180,6 +180,78 @@ public void backend_fileWrite(String oid, String filepath, byte[] data) { } } + // File read and write using byte stream + //-------------------------------------------------------------------------- + + /** + * Get and return the stored data as a InputStream + * + * @param ObjectID of workspace + * @param filepath to use for the workspace + * + * @return the stored byte array of the file + **/ + @Override + public InputStream backend_fileReadStream(final String oid, final String filepath) { + + // Due to the behaviour of how the file data needs to be handled across multiple layers + // we only use an optimized "readStream" call if the filesystem is a single stack layer + if( dataLayers.length == 1 ) { + return dataLayers[0].backend_fileReadStream(oid, filepath); + } + + // Fallback behaviour, polyfill the byte[] implementation + //------------------------------------------------------------ + byte[] rawBytes = backend_fileRead(oid, filepath); + if( rawBytes == null ) { + return null; + } + return new ByteArrayInputStream( rawBytes ); + } + + /** + * Writes the full by of a file in the backend + * + * @param ObjectID of workspace + * @param filepath to use for the workspace + * @param data to write the file with + **/ + @Override + public void backend_fileWriteStream(final String oid, final String filepath, final OutputStream data) { + + // Due to the behaviour of how the file data needs to be handled across multiple layers + // we only use an optimized "readStream" call if the filesystem is a single stack layer + if( dataLayers.length == 1 ) { + dataLayers[0].backend_fileWriteStream(oid, filepath, data); + return; + } + + // Fallback behaviour, polyfill the byte[] implementation + //------------------------------------------------------------ + + // forward the null, and let the error handling below settle it + if( data == null ) { + backend_fileWrite(oid, filepath, null); + } + + // Converts it to bytearray respectively + try { + byte[] rawBytes = null; + if( data instanceof ByteArrayOutputStream ) { + rawBytes = ((ByteArrayOutputStream)data).toByteArray(); + } else { + ByteArrayOutputStream buffer = new ByteArrayOutputStream(); + buffer.writeTo(data); + rawBytes = buffer.toByteArray(); + } + } catch(IOException e) { + throw new RuntimeException(e); + } + + // Does the bytearray writes + backend_fileWrite(oid, filepath, rawBytes); + } + // File exist / removal //-------------------------------------------------------------------------- From dbd2b0f93533835ddf73b0599813390f5e5d7679 Mon Sep 17 00:00:00 2001 From: Eugene Cheah Date: Tue, 19 Jul 2022 04:17:38 +0000 Subject: [PATCH 05/42] Update the Core_FileWorkspaceMap error handling --- .../dstack/core/Core_FileWorkspaceMap.java | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/src/main/java/picoded/dstack/core/Core_FileWorkspaceMap.java b/src/main/java/picoded/dstack/core/Core_FileWorkspaceMap.java index 33556aed..6fdf2e46 100755 --- a/src/main/java/picoded/dstack/core/Core_FileWorkspaceMap.java +++ b/src/main/java/picoded/dstack/core/Core_FileWorkspaceMap.java @@ -208,13 +208,17 @@ abstract public void backend_fileWriteStream(final String oid, final String file } // Converts it to bytearray respectively - byte[] rawBytes = null; - if( data instanceof ByteArrayOutputStream ) { - rawBytes = data.toByteArray(); - } else { - ByteArrayOutputStream buffer = new ByteArrayOutputStream(); - buffer.writeTo(data); - rawBytes = buffer.toByteArray(); + try { + byte[] rawBytes = null; + if( data instanceof ByteArrayOutputStream ) { + rawBytes = ((ByteArrayOutputStream)data).toByteArray(); + } else { + ByteArrayOutputStream buffer = new ByteArrayOutputStream(); + buffer.writeTo(data); + rawBytes = buffer.toByteArray(); + } + } catch(IOException e) { + throw new RuntimeException(e); } // Does the bytearray writes From 28a45660044957fb2887886dc63c6d46d6ef1bf1 Mon Sep 17 00:00:00 2001 From: Eugene Cheah Date: Tue, 19 Jul 2022 04:43:04 +0000 Subject: [PATCH 06/42] Fix build issues --- src/main/java/picoded/dstack/core/Core_FileWorkspaceMap.java | 5 +++-- .../java/picoded/dstack/stack/Stack_FileWorkspaceMap.java | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/main/java/picoded/dstack/core/Core_FileWorkspaceMap.java b/src/main/java/picoded/dstack/core/Core_FileWorkspaceMap.java index 6fdf2e46..3ca86599 100755 --- a/src/main/java/picoded/dstack/core/Core_FileWorkspaceMap.java +++ b/src/main/java/picoded/dstack/core/Core_FileWorkspaceMap.java @@ -11,6 +11,7 @@ import java.io.OutputStream; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; +import java.io.IOException; /** * Common base utility class of FileWorkspaceMap @@ -200,7 +201,7 @@ public InputStream backend_fileReadStream(final String oid, final String filepat * @param filepath to use for the workspace * @param data to write the file with **/ - abstract public void backend_fileWriteStream(final String oid, final String filepath, final OutputStream data) { + public void backend_fileWriteStream(final String oid, final String filepath, final OutputStream data) { // forward the null, and let the error handling below settle it if( data == null ) { @@ -208,8 +209,8 @@ abstract public void backend_fileWriteStream(final String oid, final String file } // Converts it to bytearray respectively + byte[] rawBytes = null; try { - byte[] rawBytes = null; if( data instanceof ByteArrayOutputStream ) { rawBytes = ((ByteArrayOutputStream)data).toByteArray(); } else { diff --git a/src/main/java/picoded/dstack/stack/Stack_FileWorkspaceMap.java b/src/main/java/picoded/dstack/stack/Stack_FileWorkspaceMap.java index dbfa42ce..02e3e9d4 100755 --- a/src/main/java/picoded/dstack/stack/Stack_FileWorkspaceMap.java +++ b/src/main/java/picoded/dstack/stack/Stack_FileWorkspaceMap.java @@ -6,6 +6,7 @@ import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.io.InputStream; import java.io.OutputStream; import java.util.List; import java.util.Set; @@ -218,7 +219,7 @@ public InputStream backend_fileReadStream(final String oid, final String filepat **/ @Override public void backend_fileWriteStream(final String oid, final String filepath, final OutputStream data) { - + // Due to the behaviour of how the file data needs to be handled across multiple layers // we only use an optimized "readStream" call if the filesystem is a single stack layer if( dataLayers.length == 1 ) { @@ -235,8 +236,8 @@ public void backend_fileWriteStream(final String oid, final String filepath, fin } // Converts it to bytearray respectively + byte[] rawBytes = null; try { - byte[] rawBytes = null; if( data instanceof ByteArrayOutputStream ) { rawBytes = ((ByteArrayOutputStream)data).toByteArray(); } else { From a1b3f0aaaf778c20c20f0f62eeab50abcbc7cdb1 Mon Sep 17 00:00:00 2001 From: Eugene Cheah Date: Mon, 1 Aug 2022 14:04:51 +0000 Subject: [PATCH 07/42] Added polyfill, and unit test for Input/output stream support --- .../java/picoded/dstack/FileWorkspace.java | 32 +++++++++++++++++++ .../dstack/core/Core_FileWorkspace.java | 10 +++--- .../dstack/core/Core_FileWorkspaceMap.java | 4 +-- .../dstack/stack/Stack_FileWorkspaceMap.java | 8 ++--- .../StructSimple_FileWorkspaceMap_test.java | 18 +++++++++++ 5 files changed, 61 insertions(+), 11 deletions(-) diff --git a/src/main/java/picoded/dstack/FileWorkspace.java b/src/main/java/picoded/dstack/FileWorkspace.java index d72af525..7b6d7de4 100755 --- a/src/main/java/picoded/dstack/FileWorkspace.java +++ b/src/main/java/picoded/dstack/FileWorkspace.java @@ -4,8 +4,11 @@ import picoded.core.conv.StringConv; import java.io.File; +import java.io.IOException; import java.io.InputStream; +import java.io.OutputStream; import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; import java.util.List; import java.util.Set; @@ -143,6 +146,35 @@ default InputStream readInputStream(final String filePath) { return new ByteArrayInputStream(byteArr); } + /** + * Writes an output array to a file creating the file if it does not exist. + * + * the parent directories of the file will be created if they do not exist. + * + * @param filepath in the workspace to extract + * @param data the content to write to the file + **/ + default void writeOutputStream(final String filepath, final OutputStream data) { + + // Converts it to bytearray respectively + byte[] rawBytes = null; + try { + if( data instanceof ByteArrayOutputStream ) { + rawBytes = ((ByteArrayOutputStream)data).toByteArray(); + } else { + ByteArrayOutputStream buffer = new ByteArrayOutputStream(); + buffer.writeTo(data); + rawBytes = buffer.toByteArray(); + } + } catch(IOException e) { + throw new RuntimeException(e); + } + + // Does the bytearray writes + writeByteArray(filepath, rawBytes); + } + + // // Folder Pathing support //-------------------------------------------------------------------------- diff --git a/src/main/java/picoded/dstack/core/Core_FileWorkspace.java b/src/main/java/picoded/dstack/core/Core_FileWorkspace.java index ebf88418..022da45c 100755 --- a/src/main/java/picoded/dstack/core/Core_FileWorkspace.java +++ b/src/main/java/picoded/dstack/core/Core_FileWorkspace.java @@ -162,7 +162,7 @@ public void removeFile(final String filepath) { main.backend_removeFile(_oid, normalizeFilePathString(filepath)); } - // Read / write input/output stream + // Read/write input/output stream //-------------------------------------------------------------------------- /** @@ -172,8 +172,8 @@ public void removeFile(final String filepath) { * * @return the file contents, null if file does not exists */ - public InputStream readByteStream(final String filepath) { - return main.backend_fileReadStream(_oid, normalizeFilePathString(filepath)); + public InputStream readInputStream(final String filepath) { + return main.backend_fileReadInputStream(_oid, normalizeFilePathString(filepath)); } /** @@ -184,8 +184,8 @@ public InputStream readByteStream(final String filepath) { * @param filepath in the workspace to extract * @param data the content to write to the file **/ - public void writeByteStream(final String filepath, final OutputStream data) { - main.backend_fileWriteStream(_oid, normalizeFilePathString(filepath), data); + public void writeOutputStream(final String filepath, final OutputStream data) { + main.backend_fileWriteOutputStream(_oid, normalizeFilePathString(filepath), data); } // Read / write byteArray information diff --git a/src/main/java/picoded/dstack/core/Core_FileWorkspaceMap.java b/src/main/java/picoded/dstack/core/Core_FileWorkspaceMap.java index 3ca86599..d83113b6 100755 --- a/src/main/java/picoded/dstack/core/Core_FileWorkspaceMap.java +++ b/src/main/java/picoded/dstack/core/Core_FileWorkspaceMap.java @@ -180,7 +180,7 @@ public void setupWorkspace(String oid) { * * @return the stored byte array of the file **/ - public InputStream backend_fileReadStream(final String oid, final String filepath) { + public InputStream backend_fileReadInputStream(final String oid, final String filepath) { // Get the byte data byte[] rawBytes = backend_fileRead(oid, filepath); if( rawBytes == null ) { @@ -201,7 +201,7 @@ public InputStream backend_fileReadStream(final String oid, final String filepat * @param filepath to use for the workspace * @param data to write the file with **/ - public void backend_fileWriteStream(final String oid, final String filepath, final OutputStream data) { + public void backend_fileWriteOutputStream(final String oid, final String filepath, final OutputStream data) { // forward the null, and let the error handling below settle it if( data == null ) { diff --git a/src/main/java/picoded/dstack/stack/Stack_FileWorkspaceMap.java b/src/main/java/picoded/dstack/stack/Stack_FileWorkspaceMap.java index 02e3e9d4..46502247 100755 --- a/src/main/java/picoded/dstack/stack/Stack_FileWorkspaceMap.java +++ b/src/main/java/picoded/dstack/stack/Stack_FileWorkspaceMap.java @@ -193,12 +193,12 @@ public void backend_fileWrite(String oid, String filepath, byte[] data) { * @return the stored byte array of the file **/ @Override - public InputStream backend_fileReadStream(final String oid, final String filepath) { + public InputStream backend_fileReadInputStream(final String oid, final String filepath) { // Due to the behaviour of how the file data needs to be handled across multiple layers // we only use an optimized "readStream" call if the filesystem is a single stack layer if( dataLayers.length == 1 ) { - return dataLayers[0].backend_fileReadStream(oid, filepath); + return dataLayers[0].backend_fileReadInputStream(oid, filepath); } // Fallback behaviour, polyfill the byte[] implementation @@ -218,12 +218,12 @@ public InputStream backend_fileReadStream(final String oid, final String filepat * @param data to write the file with **/ @Override - public void backend_fileWriteStream(final String oid, final String filepath, final OutputStream data) { + public void backend_fileWriteOutputStream(final String oid, final String filepath, final OutputStream data) { // Due to the behaviour of how the file data needs to be handled across multiple layers // we only use an optimized "readStream" call if the filesystem is a single stack layer if( dataLayers.length == 1 ) { - dataLayers[0].backend_fileWriteStream(oid, filepath, data); + dataLayers[0].backend_fileWriteOutputStream(oid, filepath, data); return; } diff --git a/src/test/java/picoded/dstack/struct/simple/StructSimple_FileWorkspaceMap_test.java b/src/test/java/picoded/dstack/struct/simple/StructSimple_FileWorkspaceMap_test.java index 32da49c2..93fdb838 100755 --- a/src/test/java/picoded/dstack/struct/simple/StructSimple_FileWorkspaceMap_test.java +++ b/src/test/java/picoded/dstack/struct/simple/StructSimple_FileWorkspaceMap_test.java @@ -7,9 +7,12 @@ import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; +import java.io.ByteArrayOutputStream; +import java.io.InputStream; import java.util.Arrays; import java.util.HashSet; +import org.apache.commons.io.IOUtils; // Test Case include import org.junit.After; import org.junit.Before; @@ -329,6 +332,21 @@ public void readNonExistenceFile() { } + @Test + public void writeAndReadToFile_stream() throws Exception { + // Output stream to use for content + ByteArrayOutputStream buffer = new ByteArrayOutputStream(); + buffer.write( "data to write".getBytes() ); + + FileWorkspace fileWorkspace = testObj.newEntry(); + fileWorkspace.writeOutputStream("testPath", buffer ); + assertNotNull(testObj.get(fileWorkspace._oid()).readByteArray("testPath")); + + InputStream readData = testObj.get(fileWorkspace._oid()).readInputStream("testPath"); + byte[] readArray = IOUtils.toByteArray(readData); + assertEquals(new String(readArray), "data to write"); + } + @Test public void deleteExistingFile() { FileWorkspace fileWorkspace = testObj.newEntry(); From 5848061b17d0f3bde2534fbec1e33cf945c2edb5 Mon Sep 17 00:00:00 2001 From: Eugene Cheah Date: Mon, 1 Aug 2022 14:07:42 +0000 Subject: [PATCH 08/42] Commentry tweak --- src/main/java/picoded/dstack/FileWorkspace.java | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/main/java/picoded/dstack/FileWorkspace.java b/src/main/java/picoded/dstack/FileWorkspace.java index 7b6d7de4..28b5e2cb 100755 --- a/src/main/java/picoded/dstack/FileWorkspace.java +++ b/src/main/java/picoded/dstack/FileWorkspace.java @@ -136,7 +136,9 @@ default void writeString(final String filepath, String content, String encoding) //-------------------------------------------------------------------------- /** - * Get the input stream representation of a given filepath + * Get the input stream representation of a given filepath. + * + * You are expected to close, the stream on your own, to avoid memory leaks * * @param filePath in the workspace to extract * @return the file contents, null if file does not exists @@ -147,10 +149,12 @@ default InputStream readInputStream(final String filePath) { } /** - * Writes an output array to a file creating the file if it does not exist. - * + * Writes an output stream to a file creating the file if it does not exist. * the parent directories of the file will be created if they do not exist. * + * Note that depending on the implementaiton, this may not be optimized, + * and may only return after the OutputStream is fully processedd. + * * @param filepath in the workspace to extract * @param data the content to write to the file **/ From 009ebd6979b3037f8a2515d90e2fea36fdc59d45 Mon Sep 17 00:00:00 2001 From: Eugene Cheah Date: Tue, 2 Aug 2022 02:02:16 +0000 Subject: [PATCH 09/42] WIP MongoDB FileWorkspaceMap --- .../mongodb/MongoDB_FileWorkspaceMap.java | 687 ++++++++++++++++++ 1 file changed, 687 insertions(+) create mode 100755 src/main/java/picoded/dstack/mongodb/MongoDB_FileWorkspaceMap.java diff --git a/src/main/java/picoded/dstack/mongodb/MongoDB_FileWorkspaceMap.java b/src/main/java/picoded/dstack/mongodb/MongoDB_FileWorkspaceMap.java new file mode 100755 index 00000000..42b55014 --- /dev/null +++ b/src/main/java/picoded/dstack/mongodb/MongoDB_FileWorkspaceMap.java @@ -0,0 +1,687 @@ +package picoded.dstack.mongodb; + +import java.io.ByteArrayInputStream; +// Java imports +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +// JavaCommons imports +import picoded.core.common.EmptyArray; +import picoded.core.file.FileUtil; +import picoded.dstack.FileWorkspace; +import picoded.dstack.core.Core_FileWorkspaceMap; + +// MongoDB imports +import org.bson.Document; +import org.bson.types.Binary; +import org.bson.conversions.Bson; +import com.mongodb.client.*; +import com.mongodb.client.gridfs.*; +import com.mongodb.client.gridfs.GridFSBuckets; +import com.mongodb.client.gridfs.model.GridFSUploadOptions; + +/** + * ## Purpose + * Support MongoDB implementation of FileWorkspaceMap + * + * Built ontop of the Core_FileWorkspaceMap implementation. + * + * ## Dev Notes + * Developers of this class would need to reference the following in MongoDB + * + * - GridFS : https://www.mongodb.com/docs/drivers/java/sync/current/fundamentals/gridfs/ + * - API: https://mongodb.github.io/mongo-java-driver/4.7/apidocs/mongodb-driver-sync/com/mongodb/client/gridfs/GridFSBucket.html + **/ +public class MongoDB_FileWorkspaceMap /* extends Core_FileWorkspaceMap */ { + + // -------------------------------------------------------------------------- + // + // Constructor + // + // -------------------------------------------------------------------------- + + /** MongoDB instance representing the backend connection */ + GridFSBucket gridFSBucket = null; + + /** + * Constructor, with name constructor + * + * @param inStack hazelcast stack to use + * @param name of data object map to use + */ + public MongoDB_FileWorkspaceMap(MongoDBStack inStack, String name) { + super(); + + // Initialize the gridfs bucket, + // with the relevent DB, name, and config + gridFSBucket = GridFSBuckets.create(inStack.db_conn, name) // + .withChunkSizeBytes( 8 * 1000 * 1000 ); + + // + // Note that we intentionally chose 8*1000*1000 chunk sizes + // As this will give about 1-4kb space for chunk headers to + // help ensure overall efficent chunk storage usage. + // + // This is due to the underlying storage rounding up to power + // of 2 : https://jira.mongodb.org/browse/SERVER-13331 + // + // Meaning a full "8 * 1000 * 1000" chunk would use "8 * 1024 * 1024" + // worth of space, after adding the unknown headers (<=2kb) + // + } + + //-------------------------------------------------------------------------- + // + // Backend system setup / teardown (DStackCommon) + // + //-------------------------------------------------------------------------- + + /** + * Setsup the backend storage table, etc. If needed + **/ + @Override + public void systemSetup() { + } + + /** + * Teardown and delete the backend storage table, etc. If needed + **/ + public void systemDestroy() { + gridFSBucket.drop(); + } + + /** + * Removes all data, without tearing down setup + **/ + @Override + public void clear() { + gridFSBucket.drop(); + } + + //-------------------------------------------------------------------------- + // + // Workspace setup / exist funcitons + // [Internal use, to be extended in future implementation] + // + //-------------------------------------------------------------------------- + + /** + * [Internal use, to be extended in future implementation] + * + * Checks and return of a workspace exists + * + * @param Object ID of workspace to get + * + * @return boolean to check if workspace exists + **/ + @Override + public boolean backend_workspaceExist(String oid) { + // Lets build the query for the "root file" + Bson query = Filters.eq("filename", oid); + + // Lets prepare the search + FindIterable search = gridFSBucket.find(query).limit(1); + + // Lets iterate the search result, and return true on an item + try (MongoCursor cursor = search.iterator()) { + while (cursor.hasNext()) { + // ret.add(cursor.next().getString("_oid")); + return true; + } + } + + // Fail, as the search found no iterations + return false; + } + + /** + * Setup the current fileWorkspace within the fileWorkspaceMap, + * + * This ensures the workspace _oid is registered within the map, + * even if there is 0 files. + * + * Does not throw any error if workspace was previously setup + */ + @Override + public void backend_setupWorkspace(String oid) { + // In general we will upload a blank file + // with the relevent _oid, that can be easily lookedup + // + // This is done using a closable input stream, with an empty byte array + try ( ByteArrayInputStream emptyStream = new ByteArrayInputStream(EmptyArray.BYTE) ) { + // Setup the metadata for the file + Document metadata = new Document(); + metadata.append("_oid", oid); + metadata.append("type", "root"); + + // Prepare the upload options + GridFSUploadOptions opt = (new GridFSUploadOptions()).metadata(metadata); + gridFSBucket.uploadFromStream(oid, emptyStream, opt); + } + } + + // /** + // * [Internal use, to be extended in future implementation] + // * + // * Removes the FileWorkspace, used to nuke an entire workspace + // * + // * @param ObjectID of workspace to remove + // **/ + // @Override + // public void backend_workspaceRemove(String oid) { + // try { + // accessLock.writeLock().lock(); + // fileContentMap.remove(oid); + // } finally { + // accessLock.writeLock().unlock(); + // } + // } + + // //-------------------------------------------------------------------------- + // // + // // File read / write + // // [Internal use, to be extended in future implementation] + // // + // //-------------------------------------------------------------------------- + + // /** + // * [Internal use, to be extended in future implementation] + // * + // * Get and return the stored data as a byte[] + // * + // * @param ObjectID of workspace + // * @param filepath to use for the workspace + // * + // * @return the stored byte array of the file + // **/ + // @Override + // public byte[] backend_fileRead(String oid, String filepath) { + // try { + // accessLock.readLock().lock(); + + // ConcurrentHashMap workspace = fileContentMap.get(oid); + // if (workspace != null && filepath != null) { + // return workspace.get(filepath); + // } + // return null; + // } finally { + // accessLock.readLock().unlock(); + // } + + // } + + // /** + // * [Internal use, to be extended in future implementation] + // * + // * Get and return if the file exists, due to the potentially + // * large size nature of files stored in FileWorkspace. + // * + // * Its highly recommended to optimize this function, + // * instead of leaving it as default + // * + // * @param ObjectID of workspace + // * @param filepath to use for the workspace + // * + // * @return boolean true, if file eixst + // **/ + // public boolean backend_fileExist(final String oid, final String filepath) { + // try { + // accessLock.readLock().lock(); + + // ConcurrentHashMap workspace = fileContentMap.get(oid); + // if (workspace != null && filepath != null) { + // return workspace.get(filepath) != null; + // } + // } finally { + // accessLock.readLock().unlock(); + // } + // return false; + // } + + // /** + // * [Internal use, to be extended in future implementation] + // * + // * Writes the full byte array of a file in the backend + // * + // * @param ObjectID of workspace + // * @param filepath to use for the workspace + // * @param data to write the file with + // **/ + // @Override + // public void backend_fileWrite(String oid, String filepath, byte[] data) { + // try { + // accessLock.writeLock().lock(); + + // // Get workspace, with normalized parent path + // ConcurrentHashMap workspace = noLock_setupWorkspaceFolderPath(oid, + // FileUtil.getParentPath(filepath)); + + // // And put in the filepth data + // workspace.put(filepath, data); + + // } finally { + // accessLock.writeLock().unlock(); + // } + // } + + // /** + // * [Internal use, to be extended in future implementation] + // * + // * Removes the specified file path from the workspace in the backend + // * + // * @param oid identifier to the workspace + // * @param filepath the file to be removed + // */ + // @Override + // public void backend_removeFile(String oid, String filepath) { + // try { + // accessLock.writeLock().lock(); + + // ConcurrentHashMap workspace = fileContentMap.get(oid); + + // // workspace exist, remove the file in the workspace + // if (workspace != null) { + // workspace.remove(filepath); + // } + + // } finally { + // accessLock.writeLock().unlock(); + // } + // } + + // //-------------------------------------------------------------------------- + // // + // // Folder pathing support + // // + // //-------------------------------------------------------------------------- + + // /** + // * [Internal use, to be extended in future implementation] + // * + // * Delete an existing path from the workspace. + // * This recursively removes all file content under the given path prefix + // * + // * @param ObjectID of workspace + // * @param folderPath in the workspace (note, folderPath is normalized to end with "/") + // * + // * @return the stored byte array of the file + // **/ + // public void backend_removeFolderPath(final String oid, final String folderPath) { + // try { + // accessLock.writeLock().lock(); + + // // Get the workspace, and abort if null + // ConcurrentHashMap workspace = fileContentMap.get(oid); + // if (workspace == null) { + // return; + // } + + // // Get the keyset - in a new hashset + // // (so it wouldnt crash when we do modification) + // Set allKeys = new HashSet<>(workspace.keySet()); + // for (String key : allKeys) { + // // If folder path match - remove it + // if (key.startsWith(folderPath)) { + // workspace.remove(key); + // } + // } + + // } finally { + // accessLock.writeLock().unlock(); + // } + // } + + // /** + // * [Internal use, to be extended in future implementation] + // * + // * Validate the given folder path exists. + // * + // * @param ObjectID of workspace + // * @param folderPath in the workspace (note, folderPath is normalized to end with "/") + // * + // * @return the stored byte array of the file + // **/ + // public boolean backend_folderPathExist(final String oid, final String folderPath) { + // try { + // accessLock.readLock().lock(); + // ConcurrentHashMap workspace = fileContentMap.get(oid); + // return workspace != null && workspace.get(folderPath) != null; + // } finally { + // accessLock.readLock().unlock(); + // } + // } + + // /** + // * [Internal use, to be extended in future implementation] + // * + // * Automatically generate a given folder path if it does not exist + // * + // * @param ObjectID of workspace + // * @param folderPath in the workspace (note, folderPath is normalized to end with "/") + // * + // * @return the stored byte array of the file + // **/ + // public void backend_ensureFolderPath(final String oid, final String folderPath) { + // try { + // accessLock.writeLock().lock(); + // noLock_setupWorkspaceFolderPath(oid, folderPath); + // } finally { + // accessLock.writeLock().unlock(); + // } + // } + + // //-------------------------------------------------------------------------- + // // + // // Move support + // // + // //-------------------------------------------------------------------------- + + // /** + // * @return if the current configured implementation supports atomic move operations. + // */ + // public boolean atomicMoveSupported() { + // // True due to StructSimple use of a globle write lock + // return true; + // } + + // /** + // * [Internal use, to be extended in future implementation] + // * + // * Move a given file within the system + // * + // * WARNING: Move operations are typically not "atomic" in nature, and can be unsafe where + // * missing files / corrupted data can occur when executed concurrently with other operations. + // * + // * In general "S3-like" object storage will not safely support atomic move operations. + // * Please use the `atomicMoveSupported()` function to validate if such operations are supported. + // * + // * This operation may in effect function as a rename + // * If the destionation file exists, it will be overwritten + // * + // * @param ObjectID of workspace + // * @param sourceFile + // * @param destinationFile + // */ + // public void backend_moveFile(final String oid, final String sourceFile, + // final String destinationFile) { + // try { + // accessLock.writeLock().lock(); + + // // Get the workspace, and abort if null + // ConcurrentHashMap workspace = fileContentMap.get(oid); + // if (workspace == null) { + // throw new RuntimeException("FileWorkspace does not exist : " + oid); + // } + + // // Check if sourceFolder exist + // if (workspace.get(sourceFile) == null) { + // throw new RuntimeException("sourceFile does not exist (oid=" + oid + ") : " + // + sourceFile); + // } + + // // Initialize the destionation folder + // noLock_setupWorkspaceFolderPath(oid, FileUtil.getParentPath(destinationFile)); + + // // Copy the file + // workspace.put(destinationFile, workspace.get(sourceFile)); + + // // And remove the old copy + // workspace.remove(sourceFile); + // } finally { + // accessLock.writeLock().unlock(); + // } + // } + + // /** + // * [Internal use, to be extended in future implementation] + // * + // * Move a given file within the system + // * + // * WARNING: Move operations are typically not "atomic" in nature, and can be unsafe where + // * missing files / corrupted data can occur when executed concurrently with other operations. + // * + // * In general "S3-like" object storage will not safely support atomic move operations. + // * Please use the `atomicMoveSupported()` function to validate if such operations are supported. + // * + // * Note that both source, and destionation folder will be normalized to include the "/" path. + // * This operation may in effect function as a rename + // * If the destionation folder exists with content, the result will be merged. With the sourceFolder files, overwriting on conflicts. + // * + // * @param ObjectID of workspace + // * @param sourceFolder + // * @param destinationFolder + // * + // */ + // public void backend_moveFolderPath(final String oid, final String sourceFolder, + // final String destinationFolder) { + // try { + // accessLock.writeLock().lock(); + + // // Get the workspace, and abort if null + // ConcurrentHashMap workspace = fileContentMap.get(oid); + // if (workspace == null) { + // throw new RuntimeException("FileWorkspace does not exist : " + oid); + // } + + // // Check if sourceFolder exist + // if (workspace.get(sourceFolder) == null) { + // throw new RuntimeException("sourceFolder does not exist (oid=" + oid + ") : " + // + sourceFolder); + // } + + // // Get the keyset - in a new hashset + // // (so it wouldnt crash when we do modification) + // Set allKeys = new HashSet<>(workspace.keySet()); + // for (String key : allKeys) { + // // If folder path match - migrate it + // if (key.startsWith(sourceFolder)) { + // // Copy it over + // workspace.put(destinationFolder + key.substring(sourceFolder.length()), + // workspace.get(key)); + // // Remove it + // workspace.remove(key); + // } + // } + + // } finally { + // accessLock.writeLock().unlock(); + // } + // } + + // //-------------------------------------------------------------------------- + // // + // // Copy support + // // + // //-------------------------------------------------------------------------- + + // /** + // * @return if the current configured implementation supports atomic Copy operations. + // */ + // public boolean atomicCopySupported() { + // // True due to StructSimple use of a globle write lock + // return true; + // } + + // /** + // * [Internal use, to be extended in future implementation] + // * + // * Copy a given file within the system + // * + // * WARNING: Copy operations are typically not "atomic" in nature, and can be unsafe where + // * missing files / corrupted data can occur when executed concurrently with other operations. + // * + // * In general "S3-like" object storage will not safely support atomic Copy operations. + // * Please use the `atomicCopySupported()` function to validate if such operations are supported. + // * + // * This operation may in effect function as a rename + // * If the destionation file exists, it will be overwritten + // * + // * @param ObjectID of workspace + // * @param sourceFile + // * @param destinationFile + // */ + // public void backend_copyFile(final String oid, final String sourceFile, + // final String destinationFile) { + // try { + // accessLock.writeLock().lock(); + + // // Get the workspace, and abort if null + // ConcurrentHashMap workspace = fileContentMap.get(oid); + // if (workspace == null) { + // throw new RuntimeException("FileWorkspace does not exist : " + oid); + // } + + // // Check if sourceFolder exist + // if (workspace.get(sourceFile) == null) { + // throw new RuntimeException("sourceFile does not exist (oid=" + oid + ") : " + // + sourceFile); + // } + + // // Initialize the destionation folder + // noLock_setupWorkspaceFolderPath(oid, FileUtil.getParentPath(destinationFile)); + + // // Copy the file + // workspace.put(destinationFile, workspace.get(sourceFile)); + // } finally { + // accessLock.writeLock().unlock(); + // } + // } + + // /** + // * [Internal use, to be extended in future implementation] + // * + // * Copy a given file within the system + // * + // * WARNING: Copy operations are typically not "atomic" in nature, and can be unsafe where + // * missing files / corrupted data can occur when executed concurrently with other operations. + // * + // * In general "S3-like" object storage will not safely support atomic Copy operations. + // * Please use the `atomicCopySupported()` function to validate if such operations are supported. + // * + // * Note that both source, and destionation folder will be normalized to include the "/" path. + // * This operation may in effect function as a rename + // * If the destionation folder exists with content, the result will be merged. With the sourceFolder files, overwriting on conflicts. + // * + // * @param ObjectID of workspace + // * @param sourceFolder + // * @param destinationFolder + // * + // */ + // public void backend_copyFolderPath(final String oid, final String sourceFolder, + // final String destinationFolder) { + // try { + // accessLock.writeLock().lock(); + + // // Get the workspace, and abort if null + // ConcurrentHashMap workspace = fileContentMap.get(oid); + // if (workspace == null) { + // throw new RuntimeException("FileWorkspace does not exist : " + oid); + // } + + // // Check if sourceFolder exist + // if (workspace.get(sourceFolder) == null) { + // throw new RuntimeException("sourceFolder does not exist (oid=" + oid + ") : " + // + sourceFolder); + // } + + // // Get the keyset - in a new hashset + // // (so it wouldnt crash when we do modification) + // Set allKeys = new HashSet<>(workspace.keySet()); + // for (String key : allKeys) { + // // If folder path match - migrate it + // if (key.startsWith(sourceFolder)) { + // // Copy it over + // workspace.put(destinationFolder + key.substring(sourceFolder.length()), + // workspace.get(key)); + // } + // } + + // } finally { + // accessLock.writeLock().unlock(); + // } + // } + + // //-------------------------------------------------------------------------- + // // + // // Listing support + // // + // //-------------------------------------------------------------------------- + + // /** + // * List all the various files and folders found in the given folderPath + // * + // * @param ObjectID of workspace + // * @param folderPath in the workspace (note, folderPath is normalized to end with "/") + // * @param minDepth minimum depth count, before outputing the listing (uses a <= match) + // * @param maxDepth maximum depth count, to stop the listing (-1 for infinite, uses a >= match) + // * + // * @return list of path strings - relative to the given folderPath (folders end with "/") + // */ + // public Set backend_getFileAndFolderPathSet(final String oid, final String folderPath, + // final int minDepth, final int maxDepth) { + // try { + // accessLock.readLock().lock(); + + // // Get the workspace, and abort if null + // ConcurrentHashMap workspace = fileContentMap.get(oid); + // if (workspace == null) { + // throw new RuntimeException("FileWorkspace does not exist : " + oid); + // } + + // // Check if folderPath exist + // String searchPath = folderPath; + // if (searchPath.equals("/")) { + // searchPath = ""; + // } + // if (searchPath.length() > 0 && workspace.get(searchPath) == null) { + // throw new RuntimeException("folderPath does not exist (oid=" + oid + ") : " + // + searchPath); + // } + + // // Return a filtered set + // return backend_filtterPathSet(workspace.keySet(), searchPath, minDepth, maxDepth, 0); + // } finally { + // accessLock.readLock().unlock(); + // } + // } + + // //-------------------------------------------------------------------------- + // // + // // Constructor and maintenance + // // + // //-------------------------------------------------------------------------- + + // @Override + // public void systemSetup() { + + // } + + // @Override + // public void systemDestroy() { + // clear(); + // } + + // /** + // * Maintenance step call, however due to the nature of most implementation not + // * having any form of time "expiry", this call does nothing in most implementation. + // * + // * As such im making that the default =) + // **/ + // @Override + // public void maintenance() { + // // Do nothing + // } + + // @Override + // public void clear() { + // try { + // accessLock.writeLock().lock(); + // fileContentMap.clear(); + // } finally { + // accessLock.writeLock().unlock(); + // } + // } +} From b29ee2c2da380cc0d91d22c1ea58631f8fafe59c Mon Sep 17 00:00:00 2001 From: Eugene Cheah Date: Tue, 2 Aug 2022 02:02:40 +0000 Subject: [PATCH 10/42] Tweaks with FileWorkspaceMap --- .../file/simple/FileSimple_FileWorkspaceMap.java | 8 -------- .../picoded/dstack/jsql/JSql_FileWorkspaceMap.java | 10 ---------- .../picoded/dstack/mongodb/MongoDB_DataObjectMap.java | 5 ++--- 3 files changed, 2 insertions(+), 21 deletions(-) diff --git a/src/main/java/picoded/dstack/file/simple/FileSimple_FileWorkspaceMap.java b/src/main/java/picoded/dstack/file/simple/FileSimple_FileWorkspaceMap.java index d6d52409..3318544c 100755 --- a/src/main/java/picoded/dstack/file/simple/FileSimple_FileWorkspaceMap.java +++ b/src/main/java/picoded/dstack/file/simple/FileSimple_FileWorkspaceMap.java @@ -6,17 +6,9 @@ import java.io.File; import java.io.IOException; import java.nio.file.attribute.BasicFileAttributes; -import java.nio.file.FileVisitOption; import java.nio.file.Files; -import java.nio.file.Path; import java.util.HashSet; -import java.util.List; import java.util.Set; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import javax.management.RuntimeErrorException; - /** * Reference class for Core_FileWorkspaceMap * Provide Crud operation backed by actual files diff --git a/src/main/java/picoded/dstack/jsql/JSql_FileWorkspaceMap.java b/src/main/java/picoded/dstack/jsql/JSql_FileWorkspaceMap.java index 34a96c8b..0916a376 100755 --- a/src/main/java/picoded/dstack/jsql/JSql_FileWorkspaceMap.java +++ b/src/main/java/picoded/dstack/jsql/JSql_FileWorkspaceMap.java @@ -6,18 +6,8 @@ import picoded.dstack.connector.jsql.JSql; import picoded.dstack.connector.jsql.JSqlResult; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Paths; -import java.util.ArrayList; import java.util.HashSet; import java.util.Set; -import java.util.List; -import java.util.concurrent.ConcurrentHashMap; - -import javax.management.RuntimeErrorException; - -import org.apache.commons.io.FileUtils; public class JSql_FileWorkspaceMap extends Core_FileWorkspaceMap { diff --git a/src/main/java/picoded/dstack/mongodb/MongoDB_DataObjectMap.java b/src/main/java/picoded/dstack/mongodb/MongoDB_DataObjectMap.java index b956000a..57bf600d 100644 --- a/src/main/java/picoded/dstack/mongodb/MongoDB_DataObjectMap.java +++ b/src/main/java/picoded/dstack/mongodb/MongoDB_DataObjectMap.java @@ -40,10 +40,10 @@ * ## Purpose * Support MongoDB implementation of DataObjectMap data structure. * - * Built ontop of the Core_DataObjectMap_struct implementation. + * Built ontop of the Core_DataObjectMap implementation. * * ## Dev Notes - * Developers of this class would need to reference the following + * Developers of this class would need to reference the following in MongoDB * * - Collection API : https://mongodb.github.io/mongo-java-driver/4.6/apidocs/mongodb-driver-sync/com/mongodb/client/MongoCollection.html * - Filter API: https://mongodb.github.io/mongo-java-driver/3.6/javadoc/com/mongodb/client/model/Filters.html#where-java.lang.String- @@ -57,7 +57,6 @@ public class MongoDB_DataObjectMap extends Core_DataObjectMap { //-------------------------------------------------------------------------- /** MongoDB instance representing the backend connection */ - MongoDBStack hazelcastStack = null; MongoCollection collection = null; /** From 143c90b2752ba7e02c495ff272602188ac5796f1 Mon Sep 17 00:00:00 2001 From: Eugene Cheah Date: Wed, 3 Aug 2022 08:36:11 +0000 Subject: [PATCH 11/42] Expeerimental MongoDB_KeyValueMap support --- .../dstack/mongodb/MongoDB_KeyValueMap.java | 358 ++++++++++++++++++ 1 file changed, 358 insertions(+) create mode 100644 src/main/java/picoded/dstack/mongodb/MongoDB_KeyValueMap.java diff --git a/src/main/java/picoded/dstack/mongodb/MongoDB_KeyValueMap.java b/src/main/java/picoded/dstack/mongodb/MongoDB_KeyValueMap.java new file mode 100644 index 00000000..6436c331 --- /dev/null +++ b/src/main/java/picoded/dstack/mongodb/MongoDB_KeyValueMap.java @@ -0,0 +1,358 @@ +package picoded.dstack.mongodb; + +// Java imports +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Date; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.ReentrantReadWriteLock; + +// JavaCommons imports +import picoded.core.conv.ConvertJSON; +import picoded.core.conv.GenericConvert; +import picoded.core.conv.NestedObjectFetch; +import picoded.core.conv.NestedObjectUtil; +import picoded.core.conv.StringEscape; +import picoded.core.struct.MutablePair; +import picoded.core.struct.query.OrderBy; +import picoded.core.struct.query.Query; +import picoded.core.struct.query.QueryType; +import picoded.core.common.ObjectToken; +import picoded.dstack.*; +import picoded.dstack.core.*; + +// MongoDB imports +import org.bson.Document; +import org.bson.types.Binary; +import org.bson.conversions.Bson; +import com.mongodb.client.*; +import com.mongodb.client.model.Projections; +import com.mongodb.client.model.Filters; +import com.mongodb.client.model.IndexOptions; +import com.mongodb.client.model.Indexes; +import com.mongodb.client.model.FindOneAndUpdateOptions; +import com.mongodb.client.model.Aggregates; + +/** + * ## Purpose Support MongoDB implementation of KeyValueMap + * + * Built ontop of the Core_KeyValueMap implementation. + **/ +public class MongoDB_KeyValueMap extends Core_KeyValueMap { + + // -------------------------------------------------------------------------- + // + // Constructor + // + // -------------------------------------------------------------------------- + + /** MongoDB instance representing the backend connection */ + MongoCollection collection = null; + + /** + * Constructor, with name constructor + * + * @param inStack hazelcast stack to use + * @param name of data object map to use + */ + public MongoDB_KeyValueMap(MongoDBStack inStack, String name) { + super(); + collection = inStack.db_conn.getCollection(name); + } + + @Override + public void systemSetup() { + // + // By mongodb default we use its native _id implementation + // and handle our _oid seperately. + // + // We intentionally DO NOT use mongodb _id, allowing it retain optimal performance. + // + + // Lets create the unique key index + IndexOptions opt = new IndexOptions().unique(true).name("key"); + collection.createIndex(Indexes.ascending("key"), opt); + + // Expirary key support + opt = new IndexOptions().expireAfter(0L, TimeUnit.SECONDS); + collection.createIndex(Indexes.ascending("expireAt"), opt); + } + + /** + * Teardown and delete the backend storage table, etc. If needed + **/ + public void systemDestroy() { + collection.drop(); + } + + /** + * Removes all data, without tearing down setup + **/ + @Override + public void clear() { + // Delete all items + // + // Due to the lack of an all * wildcard + // we are using a exists OR condition, which is true + // for all objects + collection.deleteMany( // + Filters.or( // + Filters.exists("key", true), // + Filters.exists("key", false) // + ) // + ); // + } + + //-------------------------------------------------------------------------- + // + // Internal functions, used by DataObject + // + //-------------------------------------------------------------------------- + + /** + * Generate a BSON filter set, for unexpired items + * this should be used in combination with an AND clause filter + **/ + protected Bson filterForUnexpired() { + // Current timestamp + Date now = new Date(); + + // the or array to join + return Filters.or( // + Filters.exists("expireAt", false), // + Filters.gt("expireAt", now), // + Filters.lte("expireAt", 0) // + ); // + } + + /** + * Search using the value, all the relevent key mappings + * + * Handles re-entrant lock where applicable + * + * @param key, note that null matches ALL + * + * @return array of keys + **/ + @Override + public Set keySet(String value) { + // The return hashset + HashSet ret = new HashSet(); + + // Search result + FindIterable search = null; + + // Lets either fetch with a value, or everything + if (value == null) { + // Lets fetch everything ... D= + search = collection.find(filterForUnexpired()); + } else { + search = collection.find(Filters.and( // + filterForUnexpired(), // + Filters.eq("val", value) // + )); // + } + + // Get all the various keys + search = search.projection(Projections.include("key")); + + // Lets iterate the search + try (MongoCursor cursor = search.iterator()) { + while (cursor.hasNext()) { + ret.add(cursor.next().getString("key")); + } + } + + // Return the full keyset + return ret; + } + + //-------------------------------------------------------------------------- + // + // Fundemental set/get value (core) + // + //-------------------------------------------------------------------------- + + /** + * [Internal use, to be extended in future implementation] + * Sets the value, with validation + * + * @param key + * @param value, null means removal + * @param expire timestamp, 0 means not timestamp + * + * @return null + **/ + @Override + public String setValueRaw(String key, String value, long expire) { + // Configure this to be an "upsert" query + FindOneAndUpdateOptions opt = new FindOneAndUpdateOptions(); + opt.upsert(true); + + // Generate the document of changes + // See: https://www.mongodb.com/docs/manual/reference/operator/update/setOnInsert/ + Document set_doc = new Document(); + set_doc.append("val", value); + + // Expire timestamp if its configured, else it should be ignored + if (expire > 0) { + set_doc.append("expireAt", new Date(expire)); + } + + // Set the key on insert + Document setOnInsert_doc = new Document(); + setOnInsert_doc.append("key", key); + + // Generate the "update" doc + Document updateDoc = new Document(); + updateDoc.append("$set", set_doc); + updateDoc.append("$setOnInsert", setOnInsert_doc); + + // Upsert the document + collection.findOneAndUpdate(Filters.eq("key", key), updateDoc, opt); + return null; + } + + /** + * [Internal use, to be extended in future implementation] + * + * Returns the value and expiry, with validation against the current timestamp + * + * @param key as String + * @param now timestamp + * + * @return String value + **/ + @Override + public MutablePair getValueExpiryRaw(String key, long now) { + // Get the find result + FindIterable res = collection.find(Filters.eq("key", key)); + + // Get the Document object + Document resObj = res.first(); + if (resObj == null) { + return null; + } + + // Lets get all the key values + String val = GenericConvert.toString(resObj.get("val"), null); + long expireAt = GenericConvert.toLong(resObj.get("expireAt"), 0); + + // Check for null objects + if (val == null || val.isEmpty()) { + return null; + } + + // No valid value found, return null + if (expireAt < 0) { + return null; + } + + // Expired value, return null + if (expireAt != 0 && expireAt < now) { + return null; + } + + // Get the value, and return the pair + return new MutablePair(val, expireAt); + } + + /** + * [Internal use, to be extended in future implementation] + * Sets the expire time stamp value, raw without validation + * + * @param key as String + * @param expireAt timestamp in seconds, 0 means NO expire + **/ + @Override + public void setExpiryRaw(String key, long expireAt) { + // Configure this to be an "update" query + FindOneAndUpdateOptions opt = new FindOneAndUpdateOptions(); + + // Generate the document of changes + // See: https://www.mongodb.com/docs/manual/reference/operator/update/setOnInsert/ + + // Generate the "update" doc + Document updateDoc = new Document(); + + // Expire timestamp if its configured, else it should be ignored + if (expireAt > 0) { + Document set_doc = new Document(); + set_doc.append("expireAt", new Date(expireAt)); + updateDoc.append("$set", set_doc); + } else { + Document unset_doc = new Document(); + unset_doc.append("expireAt", ""); + updateDoc.append("$unset", unset_doc); + } + + // Upsert the document + collection.findOneAndUpdate(Filters.eq("key", key), updateDoc, opt); + } + + //-------------------------------------------------------------------------- + // + // Remove call + // + //-------------------------------------------------------------------------- + + /** + * Remove the value, given the key + * + * @param key param find the thae meta key + * + * @return null + **/ + @Override + public KeyValue remove(Object key) { + removeValue(key); + return null; + } + + /** + * Remove the value, given the key + * + * Important note: It does not return the previously stored value + * Its return String type is to maintain consistency with Map interfaces + * + * @param key param find the thae meta key + * + * @return null + **/ + @Override + public String removeValue(Object key) { + if (key == null) { + throw new IllegalArgumentException("delete 'key' cannot be null"); + } + + // Delete the data + collection.deleteOne(Filters.eq("key", key)); + return null; + } + + //-------------------------------------------------------------------------- + // + // Maintenance calls + // + //-------------------------------------------------------------------------- + + /** + * Incremental maintainance should not trigger maintenance. + * As its potentially blocking with a very long call + **/ + public void incrementalMaintenance() { + // does nothing + } + + @Override + public void maintenance() { + // @TODO : something? (not sure what needs to be done) + } + +} \ No newline at end of file From f051a3c768674b5b2d588e479753137108c861d7 Mon Sep 17 00:00:00 2001 From: Eugene Cheah Date: Wed, 3 Aug 2022 08:39:13 +0000 Subject: [PATCH 12/42] Abstract out WIP FileWorkspaceMap --- .../mongodb/MongoDB_FileWorkspaceMap.java | 110 +++++++++--------- 1 file changed, 57 insertions(+), 53 deletions(-) diff --git a/src/main/java/picoded/dstack/mongodb/MongoDB_FileWorkspaceMap.java b/src/main/java/picoded/dstack/mongodb/MongoDB_FileWorkspaceMap.java index 42b55014..afe1b7ae 100755 --- a/src/main/java/picoded/dstack/mongodb/MongoDB_FileWorkspaceMap.java +++ b/src/main/java/picoded/dstack/mongodb/MongoDB_FileWorkspaceMap.java @@ -1,6 +1,7 @@ package picoded.dstack.mongodb; import java.io.ByteArrayInputStream; +import java.io.IOException; // Java imports import java.util.ArrayList; import java.util.HashSet; @@ -21,6 +22,7 @@ import com.mongodb.client.*; import com.mongodb.client.gridfs.*; import com.mongodb.client.gridfs.GridFSBuckets; +import com.mongodb.client.gridfs.model.GridFSFile; import com.mongodb.client.gridfs.model.GridFSUploadOptions; /** @@ -35,7 +37,7 @@ * - GridFS : https://www.mongodb.com/docs/drivers/java/sync/current/fundamentals/gridfs/ * - API: https://mongodb.github.io/mongo-java-driver/4.7/apidocs/mongodb-driver-sync/com/mongodb/client/gridfs/GridFSBucket.html **/ -public class MongoDB_FileWorkspaceMap /* extends Core_FileWorkspaceMap */ { +abstract public class MongoDB_FileWorkspaceMap extends Core_FileWorkspaceMap { // -------------------------------------------------------------------------- // @@ -54,12 +56,12 @@ public class MongoDB_FileWorkspaceMap /* extends Core_FileWorkspaceMap */ { */ public MongoDB_FileWorkspaceMap(MongoDBStack inStack, String name) { super(); - + // Initialize the gridfs bucket, // with the relevent DB, name, and config gridFSBucket = GridFSBuckets.create(inStack.db_conn, name) // - .withChunkSizeBytes( 8 * 1000 * 1000 ); - + .withChunkSizeBytes(8 * 1000 * 1000); + // // Note that we intentionally chose 8*1000*1000 chunk sizes // As this will give about 1-4kb space for chunk headers to @@ -119,20 +121,20 @@ public void clear() { **/ @Override public boolean backend_workspaceExist(String oid) { - // Lets build the query for the "root file" - Bson query = Filters.eq("filename", oid); - - // Lets prepare the search - FindIterable search = gridFSBucket.find(query).limit(1); - - // Lets iterate the search result, and return true on an item - try (MongoCursor cursor = search.iterator()) { - while (cursor.hasNext()) { - // ret.add(cursor.next().getString("_oid")); - return true; - } - } - + // // Lets build the query for the "root file" + // Bson query = Filters.eq("filename", oid); + + // // Lets prepare the search + // FindIterable search = gridFSBucket.find(query).limit(1); + + // // Lets iterate the search result, and return true on an item + // try (MongoCursor cursor = search.iterator()) { + // while (cursor.hasNext()) { + // // ret.add(cursor.next().getString("_oid")); + // return true; + // } + // } + // Fail, as the search found no iterations return false; } @@ -151,15 +153,17 @@ public void backend_setupWorkspace(String oid) { // with the relevent _oid, that can be easily lookedup // // This is done using a closable input stream, with an empty byte array - try ( ByteArrayInputStream emptyStream = new ByteArrayInputStream(EmptyArray.BYTE) ) { + try (ByteArrayInputStream emptyStream = new ByteArrayInputStream(EmptyArray.BYTE)) { // Setup the metadata for the file Document metadata = new Document(); metadata.append("_oid", oid); metadata.append("type", "root"); - + // Prepare the upload options GridFSUploadOptions opt = (new GridFSUploadOptions()).metadata(metadata); gridFSBucket.uploadFromStream(oid, emptyStream, opt); + } catch (IOException e) { + throw new RuntimeException(e); } } @@ -201,7 +205,7 @@ public void backend_setupWorkspace(String oid) { // public byte[] backend_fileRead(String oid, String filepath) { // try { // accessLock.readLock().lock(); - + // ConcurrentHashMap workspace = fileContentMap.get(oid); // if (workspace != null && filepath != null) { // return workspace.get(filepath); @@ -210,7 +214,7 @@ public void backend_setupWorkspace(String oid) { // } finally { // accessLock.readLock().unlock(); // } - + // } // /** @@ -230,7 +234,7 @@ public void backend_setupWorkspace(String oid) { // public boolean backend_fileExist(final String oid, final String filepath) { // try { // accessLock.readLock().lock(); - + // ConcurrentHashMap workspace = fileContentMap.get(oid); // if (workspace != null && filepath != null) { // return workspace.get(filepath) != null; @@ -254,14 +258,14 @@ public void backend_setupWorkspace(String oid) { // public void backend_fileWrite(String oid, String filepath, byte[] data) { // try { // accessLock.writeLock().lock(); - + // // Get workspace, with normalized parent path // ConcurrentHashMap workspace = noLock_setupWorkspaceFolderPath(oid, // FileUtil.getParentPath(filepath)); - + // // And put in the filepth data // workspace.put(filepath, data); - + // } finally { // accessLock.writeLock().unlock(); // } @@ -279,14 +283,14 @@ public void backend_setupWorkspace(String oid) { // public void backend_removeFile(String oid, String filepath) { // try { // accessLock.writeLock().lock(); - + // ConcurrentHashMap workspace = fileContentMap.get(oid); - + // // workspace exist, remove the file in the workspace // if (workspace != null) { // workspace.remove(filepath); // } - + // } finally { // accessLock.writeLock().unlock(); // } @@ -312,13 +316,13 @@ public void backend_setupWorkspace(String oid) { // public void backend_removeFolderPath(final String oid, final String folderPath) { // try { // accessLock.writeLock().lock(); - + // // Get the workspace, and abort if null // ConcurrentHashMap workspace = fileContentMap.get(oid); // if (workspace == null) { // return; // } - + // // Get the keyset - in a new hashset // // (so it wouldnt crash when we do modification) // Set allKeys = new HashSet<>(workspace.keySet()); @@ -328,7 +332,7 @@ public void backend_setupWorkspace(String oid) { // workspace.remove(key); // } // } - + // } finally { // accessLock.writeLock().unlock(); // } @@ -409,25 +413,25 @@ public void backend_setupWorkspace(String oid) { // final String destinationFile) { // try { // accessLock.writeLock().lock(); - + // // Get the workspace, and abort if null // ConcurrentHashMap workspace = fileContentMap.get(oid); // if (workspace == null) { // throw new RuntimeException("FileWorkspace does not exist : " + oid); // } - + // // Check if sourceFolder exist // if (workspace.get(sourceFile) == null) { // throw new RuntimeException("sourceFile does not exist (oid=" + oid + ") : " // + sourceFile); // } - + // // Initialize the destionation folder // noLock_setupWorkspaceFolderPath(oid, FileUtil.getParentPath(destinationFile)); - + // // Copy the file // workspace.put(destinationFile, workspace.get(sourceFile)); - + // // And remove the old copy // workspace.remove(sourceFile); // } finally { @@ -459,19 +463,19 @@ public void backend_setupWorkspace(String oid) { // final String destinationFolder) { // try { // accessLock.writeLock().lock(); - + // // Get the workspace, and abort if null // ConcurrentHashMap workspace = fileContentMap.get(oid); // if (workspace == null) { // throw new RuntimeException("FileWorkspace does not exist : " + oid); // } - + // // Check if sourceFolder exist // if (workspace.get(sourceFolder) == null) { // throw new RuntimeException("sourceFolder does not exist (oid=" + oid + ") : " // + sourceFolder); // } - + // // Get the keyset - in a new hashset // // (so it wouldnt crash when we do modification) // Set allKeys = new HashSet<>(workspace.keySet()); @@ -485,7 +489,7 @@ public void backend_setupWorkspace(String oid) { // workspace.remove(key); // } // } - + // } finally { // accessLock.writeLock().unlock(); // } @@ -527,22 +531,22 @@ public void backend_setupWorkspace(String oid) { // final String destinationFile) { // try { // accessLock.writeLock().lock(); - + // // Get the workspace, and abort if null // ConcurrentHashMap workspace = fileContentMap.get(oid); // if (workspace == null) { // throw new RuntimeException("FileWorkspace does not exist : " + oid); // } - + // // Check if sourceFolder exist // if (workspace.get(sourceFile) == null) { // throw new RuntimeException("sourceFile does not exist (oid=" + oid + ") : " // + sourceFile); // } - + // // Initialize the destionation folder // noLock_setupWorkspaceFolderPath(oid, FileUtil.getParentPath(destinationFile)); - + // // Copy the file // workspace.put(destinationFile, workspace.get(sourceFile)); // } finally { @@ -574,19 +578,19 @@ public void backend_setupWorkspace(String oid) { // final String destinationFolder) { // try { // accessLock.writeLock().lock(); - + // // Get the workspace, and abort if null // ConcurrentHashMap workspace = fileContentMap.get(oid); // if (workspace == null) { // throw new RuntimeException("FileWorkspace does not exist : " + oid); // } - + // // Check if sourceFolder exist // if (workspace.get(sourceFolder) == null) { // throw new RuntimeException("sourceFolder does not exist (oid=" + oid + ") : " // + sourceFolder); // } - + // // Get the keyset - in a new hashset // // (so it wouldnt crash when we do modification) // Set allKeys = new HashSet<>(workspace.keySet()); @@ -598,7 +602,7 @@ public void backend_setupWorkspace(String oid) { // workspace.get(key)); // } // } - + // } finally { // accessLock.writeLock().unlock(); // } @@ -624,13 +628,13 @@ public void backend_setupWorkspace(String oid) { // final int minDepth, final int maxDepth) { // try { // accessLock.readLock().lock(); - + // // Get the workspace, and abort if null // ConcurrentHashMap workspace = fileContentMap.get(oid); // if (workspace == null) { // throw new RuntimeException("FileWorkspace does not exist : " + oid); // } - + // // Check if folderPath exist // String searchPath = folderPath; // if (searchPath.equals("/")) { @@ -640,7 +644,7 @@ public void backend_setupWorkspace(String oid) { // throw new RuntimeException("folderPath does not exist (oid=" + oid + ") : " // + searchPath); // } - + // // Return a filtered set // return backend_filtterPathSet(workspace.keySet(), searchPath, minDepth, maxDepth, 0); // } finally { @@ -656,7 +660,7 @@ public void backend_setupWorkspace(String oid) { // @Override // public void systemSetup() { - + // } // @Override From 16cde49b8345a6309de34a033bdf73896c804195 Mon Sep 17 00:00:00 2001 From: Eugene Cheah Date: Wed, 3 Aug 2022 08:39:21 +0000 Subject: [PATCH 13/42] Code cleanup --- .../java/picoded/dstack/FileWorkspace.java | 9 +++-- .../dstack/core/Core_FileWorkspaceMap.java | 21 +++++------ .../simple/FileSimple_FileWorkspaceMap.java | 1 + .../dstack/mongodb/MongoDB_DataObjectMap.java | 6 ++-- .../dstack/stack/Stack_FileWorkspaceMap.java | 35 ++++++++++--------- .../StructSimple_FileWorkspaceMap_test.java | 6 ++-- 6 files changed, 40 insertions(+), 38 deletions(-) diff --git a/src/main/java/picoded/dstack/FileWorkspace.java b/src/main/java/picoded/dstack/FileWorkspace.java index 28b5e2cb..8a6e2cc1 100755 --- a/src/main/java/picoded/dstack/FileWorkspace.java +++ b/src/main/java/picoded/dstack/FileWorkspace.java @@ -163,22 +163,21 @@ default void writeOutputStream(final String filepath, final OutputStream data) { // Converts it to bytearray respectively byte[] rawBytes = null; try { - if( data instanceof ByteArrayOutputStream ) { - rawBytes = ((ByteArrayOutputStream)data).toByteArray(); + if (data instanceof ByteArrayOutputStream) { + rawBytes = ((ByteArrayOutputStream) data).toByteArray(); } else { ByteArrayOutputStream buffer = new ByteArrayOutputStream(); buffer.writeTo(data); rawBytes = buffer.toByteArray(); } - } catch(IOException e) { + } catch (IOException e) { throw new RuntimeException(e); } - + // Does the bytearray writes writeByteArray(filepath, rawBytes); } - // // Folder Pathing support //-------------------------------------------------------------------------- diff --git a/src/main/java/picoded/dstack/core/Core_FileWorkspaceMap.java b/src/main/java/picoded/dstack/core/Core_FileWorkspaceMap.java index d83113b6..e676682e 100755 --- a/src/main/java/picoded/dstack/core/Core_FileWorkspaceMap.java +++ b/src/main/java/picoded/dstack/core/Core_FileWorkspaceMap.java @@ -183,10 +183,10 @@ public void setupWorkspace(String oid) { public InputStream backend_fileReadInputStream(final String oid, final String filepath) { // Get the byte data byte[] rawBytes = backend_fileRead(oid, filepath); - if( rawBytes == null ) { + if (rawBytes == null) { return null; } - return new ByteArrayInputStream( rawBytes ); + return new ByteArrayInputStream(rawBytes); } /** @@ -201,27 +201,28 @@ public InputStream backend_fileReadInputStream(final String oid, final String fi * @param filepath to use for the workspace * @param data to write the file with **/ - public void backend_fileWriteOutputStream(final String oid, final String filepath, final OutputStream data) { - + public void backend_fileWriteOutputStream(final String oid, final String filepath, + final OutputStream data) { + // forward the null, and let the error handling below settle it - if( data == null ) { + if (data == null) { backend_fileWrite(oid, filepath, null); } - + // Converts it to bytearray respectively byte[] rawBytes = null; try { - if( data instanceof ByteArrayOutputStream ) { - rawBytes = ((ByteArrayOutputStream)data).toByteArray(); + if (data instanceof ByteArrayOutputStream) { + rawBytes = ((ByteArrayOutputStream) data).toByteArray(); } else { ByteArrayOutputStream buffer = new ByteArrayOutputStream(); buffer.writeTo(data); rawBytes = buffer.toByteArray(); } - } catch(IOException e) { + } catch (IOException e) { throw new RuntimeException(e); } - + // Does the bytearray writes backend_fileWrite(oid, filepath, rawBytes); } diff --git a/src/main/java/picoded/dstack/file/simple/FileSimple_FileWorkspaceMap.java b/src/main/java/picoded/dstack/file/simple/FileSimple_FileWorkspaceMap.java index 3318544c..e1961087 100755 --- a/src/main/java/picoded/dstack/file/simple/FileSimple_FileWorkspaceMap.java +++ b/src/main/java/picoded/dstack/file/simple/FileSimple_FileWorkspaceMap.java @@ -9,6 +9,7 @@ import java.nio.file.Files; import java.util.HashSet; import java.util.Set; + /** * Reference class for Core_FileWorkspaceMap * Provide Crud operation backed by actual files diff --git a/src/main/java/picoded/dstack/mongodb/MongoDB_DataObjectMap.java b/src/main/java/picoded/dstack/mongodb/MongoDB_DataObjectMap.java index 57bf600d..27d2a8ee 100644 --- a/src/main/java/picoded/dstack/mongodb/MongoDB_DataObjectMap.java +++ b/src/main/java/picoded/dstack/mongodb/MongoDB_DataObjectMap.java @@ -90,13 +90,13 @@ public void systemSetup() { IndexOptions opt = new IndexOptions(); opt = opt.unique(true); opt = opt.name("_oid"); - + // Due to the need for _oid to ensure consistency, we would not be creating it in the background // opt = opt.background(true); - + // Lets create the index collection.createIndex(Indexes.ascending("_oid"), opt); - + // // Wildcard indexing // diff --git a/src/main/java/picoded/dstack/stack/Stack_FileWorkspaceMap.java b/src/main/java/picoded/dstack/stack/Stack_FileWorkspaceMap.java index 46502247..24d31fae 100755 --- a/src/main/java/picoded/dstack/stack/Stack_FileWorkspaceMap.java +++ b/src/main/java/picoded/dstack/stack/Stack_FileWorkspaceMap.java @@ -79,7 +79,7 @@ public CommonStructure[] commonStructureStack() { // [Internal use, to be extended in future implementation] // //-------------------------------------------------------------------------- - + // Workspace operations //-------------------------------------------------------------------------- @@ -194,20 +194,20 @@ public void backend_fileWrite(String oid, String filepath, byte[] data) { **/ @Override public InputStream backend_fileReadInputStream(final String oid, final String filepath) { - + // Due to the behaviour of how the file data needs to be handled across multiple layers // we only use an optimized "readStream" call if the filesystem is a single stack layer - if( dataLayers.length == 1 ) { + if (dataLayers.length == 1) { return dataLayers[0].backend_fileReadInputStream(oid, filepath); } - + // Fallback behaviour, polyfill the byte[] implementation //------------------------------------------------------------ byte[] rawBytes = backend_fileRead(oid, filepath); - if( rawBytes == null ) { + if (rawBytes == null) { return null; } - return new ByteArrayInputStream( rawBytes ); + return new ByteArrayInputStream(rawBytes); } /** @@ -218,37 +218,38 @@ public InputStream backend_fileReadInputStream(final String oid, final String fi * @param data to write the file with **/ @Override - public void backend_fileWriteOutputStream(final String oid, final String filepath, final OutputStream data) { - + public void backend_fileWriteOutputStream(final String oid, final String filepath, + final OutputStream data) { + // Due to the behaviour of how the file data needs to be handled across multiple layers // we only use an optimized "readStream" call if the filesystem is a single stack layer - if( dataLayers.length == 1 ) { + if (dataLayers.length == 1) { dataLayers[0].backend_fileWriteOutputStream(oid, filepath, data); return; } - + // Fallback behaviour, polyfill the byte[] implementation //------------------------------------------------------------ - + // forward the null, and let the error handling below settle it - if( data == null ) { + if (data == null) { backend_fileWrite(oid, filepath, null); } - + // Converts it to bytearray respectively byte[] rawBytes = null; try { - if( data instanceof ByteArrayOutputStream ) { - rawBytes = ((ByteArrayOutputStream)data).toByteArray(); + if (data instanceof ByteArrayOutputStream) { + rawBytes = ((ByteArrayOutputStream) data).toByteArray(); } else { ByteArrayOutputStream buffer = new ByteArrayOutputStream(); buffer.writeTo(data); rawBytes = buffer.toByteArray(); } - } catch(IOException e) { + } catch (IOException e) { throw new RuntimeException(e); } - + // Does the bytearray writes backend_fileWrite(oid, filepath, rawBytes); } diff --git a/src/test/java/picoded/dstack/struct/simple/StructSimple_FileWorkspaceMap_test.java b/src/test/java/picoded/dstack/struct/simple/StructSimple_FileWorkspaceMap_test.java index 93fdb838..25f7bdca 100755 --- a/src/test/java/picoded/dstack/struct/simple/StructSimple_FileWorkspaceMap_test.java +++ b/src/test/java/picoded/dstack/struct/simple/StructSimple_FileWorkspaceMap_test.java @@ -336,10 +336,10 @@ public void readNonExistenceFile() { public void writeAndReadToFile_stream() throws Exception { // Output stream to use for content ByteArrayOutputStream buffer = new ByteArrayOutputStream(); - buffer.write( "data to write".getBytes() ); - + buffer.write("data to write".getBytes()); + FileWorkspace fileWorkspace = testObj.newEntry(); - fileWorkspace.writeOutputStream("testPath", buffer ); + fileWorkspace.writeOutputStream("testPath", buffer); assertNotNull(testObj.get(fileWorkspace._oid()).readByteArray("testPath")); InputStream readData = testObj.get(fileWorkspace._oid()).readInputStream("testPath"); From 370ea8e7aa7ca8815ce6f197feee2e6fc2780ae0 Mon Sep 17 00:00:00 2001 From: Eugene Cheah Date: Wed, 3 Aug 2022 08:44:28 +0000 Subject: [PATCH 14/42] MongoDB_KeyValueMap test --- .../picoded/dstack/mongodb/MongoDBStack.java | 2 + .../mongodb/MongoDB_KeyValueMap_test.java | 47 +++++++++++++++++++ 2 files changed, 49 insertions(+) create mode 100755 src/test/java/picoded/dstack/mongodb/MongoDB_KeyValueMap_test.java diff --git a/src/main/java/picoded/dstack/mongodb/MongoDBStack.java b/src/main/java/picoded/dstack/mongodb/MongoDBStack.java index e7dddda2..0cb83e2f 100644 --- a/src/main/java/picoded/dstack/mongodb/MongoDBStack.java +++ b/src/main/java/picoded/dstack/mongodb/MongoDBStack.java @@ -126,6 +126,8 @@ protected Core_DataStructure initDataStructure(String name, String type) { Core_DataStructure ret = null; if (type.equalsIgnoreCase("DataObjectMap")) { ret = new MongoDB_DataObjectMap(this, name); + } else if (type.equalsIgnoreCase("KeyValueMap")) { + ret = new MongoDB_KeyValueMap(this, name); } // If datastrucutre initialized, setup name diff --git a/src/test/java/picoded/dstack/mongodb/MongoDB_KeyValueMap_test.java b/src/test/java/picoded/dstack/mongodb/MongoDB_KeyValueMap_test.java new file mode 100755 index 00000000..5976b97d --- /dev/null +++ b/src/test/java/picoded/dstack/mongodb/MongoDB_KeyValueMap_test.java @@ -0,0 +1,47 @@ +package picoded.dstack.mongodb; + +import picoded.core.struct.*; +import picoded.dstack.*; +import picoded.dstack.struct.simple.*; + +/** + * ## Purpose + * This class is meant to test the MongoDB_KeyValueMap implementation, + * and ensure that it passes all the test layed out in StructSImple_KeyValueMap_test + * + */ +public class MongoDB_KeyValueMap_test extends StructSimple_KeyValueMap_test { + + // Hazelcast stack instance + protected static volatile MongoDBStack instance = null; + + // To override for implementation + //----------------------------------------------------- + + /// Impomentation constructor + public KeyValueMap implementationConstructor() { + + // Initialize server + synchronized (MongoDB_KeyValueMap_test.class) { + if (instance == null) { + // The default config uses localhost, 27017 + GenericConvertMap mongodbConfig = new GenericConvertHashMap<>(); + mongodbConfig.put("host", DStackTestConfig.MONGODB_HOST()); + mongodbConfig.put("port", DStackTestConfig.MONGODB_PORT()); + + // Use a random DB name + mongodbConfig.put("name", DStackTestConfig.randomTablePrefix()); + + GenericConvertMap stackConfig = new GenericConvertHashMap<>(); + stackConfig.put("name", "MongoDB_KeyValueMap_test"); + stackConfig.put("mongodb", mongodbConfig); + + instance = new MongoDBStack(stackConfig); + } + } + + // Load the KeyValueMap + return instance.keyValueMap(DStackTestConfig.randomTablePrefix()); + } + +} From a64f3567256ef959d9a0fe1ce3c03979b7c4a2b7 Mon Sep 17 00:00:00 2001 From: Eugene Cheah Date: Wed, 3 Aug 2022 08:53:23 +0000 Subject: [PATCH 15/42] Ensure proper reading of KeyValue expiry Date --- .../java/picoded/dstack/mongodb/MongoDB_KeyValueMap.java | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/main/java/picoded/dstack/mongodb/MongoDB_KeyValueMap.java b/src/main/java/picoded/dstack/mongodb/MongoDB_KeyValueMap.java index 6436c331..debfe840 100644 --- a/src/main/java/picoded/dstack/mongodb/MongoDB_KeyValueMap.java +++ b/src/main/java/picoded/dstack/mongodb/MongoDB_KeyValueMap.java @@ -242,8 +242,14 @@ public MutablePair getValueExpiryRaw(String key, long now) { // Lets get all the key values String val = GenericConvert.toString(resObj.get("val"), null); - long expireAt = GenericConvert.toLong(resObj.get("expireAt"), 0); + Date expireAt_date = resObj.get("expireAt"); + long expireAt_long = 0; + // Check if expireAt date is set + if( expireAt_date != null ) { + expireAt_long = expireAt_date.getTime(); + } + // Check for null objects if (val == null || val.isEmpty()) { return null; From 763fa38941a111b64dc964c0c00416d9f3cb2697 Mon Sep 17 00:00:00 2001 From: Eugene Cheah Date: Wed, 3 Aug 2022 09:10:27 +0000 Subject: [PATCH 16/42] Add setValueRaw, enforcing the expire timestamp --- .../dstack/mongodb/MongoDB_KeyValueMap.java | 24 ++++++++++++------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/src/main/java/picoded/dstack/mongodb/MongoDB_KeyValueMap.java b/src/main/java/picoded/dstack/mongodb/MongoDB_KeyValueMap.java index debfe840..f2fb3420 100644 --- a/src/main/java/picoded/dstack/mongodb/MongoDB_KeyValueMap.java +++ b/src/main/java/picoded/dstack/mongodb/MongoDB_KeyValueMap.java @@ -197,21 +197,27 @@ public String setValueRaw(String key, String value, long expire) { // Generate the document of changes // See: https://www.mongodb.com/docs/manual/reference/operator/update/setOnInsert/ - Document set_doc = new Document(); - set_doc.append("val", value); + + // Generate the "update" doc + Document updateDoc = new Document(); + Document set_doc = new Document(); - // Expire timestamp if its configured, else it should be ignored - if (expire > 0) { - set_doc.append("expireAt", new Date(expire)); + // Expire timestamp if its configured, else it should be removed + if (expireAt > 0) { + set_doc.append("expireAt", new Date(expireAt)); + } else { + Document unset_doc = new Document(); + unset_doc.append("expireAt", ""); + updateDoc.append("$unset", unset_doc); } + // Setup the value on update/insert/upsert + set_doc.append("val", value); + updateDoc.append("$set", set_doc); + // Set the key on insert Document setOnInsert_doc = new Document(); setOnInsert_doc.append("key", key); - - // Generate the "update" doc - Document updateDoc = new Document(); - updateDoc.append("$set", set_doc); updateDoc.append("$setOnInsert", setOnInsert_doc); // Upsert the document From 32384092eb1ce165efd1fe257aeb9636f04c56ee Mon Sep 17 00:00:00 2001 From: Eugene Cheah Date: Wed, 3 Aug 2022 09:14:55 +0000 Subject: [PATCH 17/42] Fixing expireAt_long --- .../java/picoded/dstack/mongodb/MongoDB_KeyValueMap.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/picoded/dstack/mongodb/MongoDB_KeyValueMap.java b/src/main/java/picoded/dstack/mongodb/MongoDB_KeyValueMap.java index f2fb3420..808d0c6f 100644 --- a/src/main/java/picoded/dstack/mongodb/MongoDB_KeyValueMap.java +++ b/src/main/java/picoded/dstack/mongodb/MongoDB_KeyValueMap.java @@ -262,17 +262,17 @@ public MutablePair getValueExpiryRaw(String key, long now) { } // No valid value found, return null - if (expireAt < 0) { + if (expireAt_long < 0) { return null; } // Expired value, return null - if (expireAt != 0 && expireAt < now) { + if (expireAt_long != 0 && expireAt_long < now) { return null; } // Get the value, and return the pair - return new MutablePair(val, expireAt); + return new MutablePair(val, expireAt_long); } /** From a3d52b3d883f987b171842dfe1cf652da8725f3a Mon Sep 17 00:00:00 2001 From: Eugene Cheah Date: Wed, 3 Aug 2022 09:25:54 +0000 Subject: [PATCH 18/42] Fixing KeyValueMap compliation issue --- .../dstack/mongodb/MongoDB_KeyValueMap.java | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/src/main/java/picoded/dstack/mongodb/MongoDB_KeyValueMap.java b/src/main/java/picoded/dstack/mongodb/MongoDB_KeyValueMap.java index 808d0c6f..3458d52f 100644 --- a/src/main/java/picoded/dstack/mongodb/MongoDB_KeyValueMap.java +++ b/src/main/java/picoded/dstack/mongodb/MongoDB_KeyValueMap.java @@ -190,17 +190,17 @@ public Set keySet(String value) { * @return null **/ @Override - public String setValueRaw(String key, String value, long expire) { + public String setValueRaw(String key, String value, long expireAt) { // Configure this to be an "upsert" query FindOneAndUpdateOptions opt = new FindOneAndUpdateOptions(); opt.upsert(true); // Generate the document of changes // See: https://www.mongodb.com/docs/manual/reference/operator/update/setOnInsert/ - + // Generate the "update" doc Document updateDoc = new Document(); - Document set_doc = new Document(); + Document set_doc = new Document(); // Expire timestamp if its configured, else it should be removed if (expireAt > 0) { @@ -211,9 +211,9 @@ public String setValueRaw(String key, String value, long expire) { updateDoc.append("$unset", unset_doc); } - // Setup the value on update/insert/upsert + // Setup the value on update/insert/upsert set_doc.append("val", value); - updateDoc.append("$set", set_doc); + updateDoc.append("$set", set_doc); // Set the key on insert Document setOnInsert_doc = new Document(); @@ -248,14 +248,14 @@ public MutablePair getValueExpiryRaw(String key, long now) { // Lets get all the key values String val = GenericConvert.toString(resObj.get("val"), null); - Date expireAt_date = resObj.get("expireAt"); - long expireAt_long = 0; + Date expireAt_date = resObj.getDate("expireAt"); + long expireAt_long = 0; + + // Check if expireAt date is set + if (expireAt_date != null) { + expireAt_long = expireAt_date.getTime(); + } - // Check if expireAt date is set - if( expireAt_date != null ) { - expireAt_long = expireAt_date.getTime(); - } - // Check for null objects if (val == null || val.isEmpty()) { return null; From ebb99328e8b67406da6c9c4db72cb3f2adefa68b Mon Sep 17 00:00:00 2001 From: Eugene Cheah Date: Wed, 3 Aug 2022 10:06:30 +0000 Subject: [PATCH 19/42] Prototype KeyLongMap implementation --- .../dstack/mongodb/MongoDB_KeyLongMap.java | 468 ++++++++++++++++++ 1 file changed, 468 insertions(+) create mode 100644 src/main/java/picoded/dstack/mongodb/MongoDB_KeyLongMap.java diff --git a/src/main/java/picoded/dstack/mongodb/MongoDB_KeyLongMap.java b/src/main/java/picoded/dstack/mongodb/MongoDB_KeyLongMap.java new file mode 100644 index 00000000..4869b423 --- /dev/null +++ b/src/main/java/picoded/dstack/mongodb/MongoDB_KeyLongMap.java @@ -0,0 +1,468 @@ +package picoded.dstack.mongodb; + +// Java imports +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Date; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.ReentrantReadWriteLock; + +// JavaCommons imports +import picoded.core.conv.ConvertJSON; +import picoded.core.conv.GenericConvert; +import picoded.core.conv.NestedObjectFetch; +import picoded.core.conv.NestedObjectUtil; +import picoded.core.conv.StringEscape; +import picoded.core.struct.MutablePair; +import picoded.core.struct.query.OrderBy; +import picoded.core.struct.query.Query; +import picoded.core.struct.query.QueryType; +import picoded.core.common.ObjectToken; +import picoded.dstack.*; +import picoded.dstack.core.*; + +// MongoDB imports +import org.bson.Document; +import org.bson.types.Binary; +import org.bson.conversions.Bson; +import com.mongodb.client.*; +import com.mongodb.client.model.Projections; +import com.mongodb.client.model.Filters; +import com.mongodb.client.model.IndexOptions; +import com.mongodb.client.model.Indexes; +import com.mongodb.client.model.FindOneAndUpdateOptions; +import com.mongodb.client.model.Aggregates; +import com.mongodb.client.result.InsertOneResult; + +/** + * ## Purpose Support MongoDB implementation of KeyLongMap + * + * Built ontop of the Core_KeyLongMap implementation. + **/ +public class MongoDB_KeyLongMap extends Core_KeyLongMap { + + // -------------------------------------------------------------------------- + // + // Constructor + // + // -------------------------------------------------------------------------- + + /** MongoDB instance representing the backend connection */ + MongoCollection collection = null; + + /** + * Constructor, with name constructor + * + * @param inStack hazelcast stack to use + * @param name of data object map to use + */ + public MongoDB_KeyLongMap(MongoDBStack inStack, String name) { + super(); + collection = inStack.db_conn.getCollection(name); + } + + @Override + public void systemSetup() { + // + // By mongodb default we use its native _id implementation + // and handle our _oid seperately. + // + // We intentionally DO NOT use mongodb _id, allowing it retain optimal performance. + // + + // Lets create the unique key index + IndexOptions opt = new IndexOptions().unique(true).name("key"); + collection.createIndex(Indexes.ascending("key"), opt); + + // Expirary key support + opt = new IndexOptions().expireAfter(0L, TimeUnit.SECONDS); + collection.createIndex(Indexes.ascending("expireAt"), opt); + } + + /** + * Teardown and delete the backend storage table, etc. If needed + **/ + public void systemDestroy() { + collection.drop(); + } + + /** + * Removes all data, without tearing down setup + **/ + @Override + public void clear() { + // Delete all items + // + // Due to the lack of an all * wildcard + // we are using a exists OR condition, which is true + // for all objects + collection.deleteMany( // + Filters.or( // + Filters.exists("key", true), // + Filters.exists("key", false) // + ) // + ); // + } + + //-------------------------------------------------------------------------- + // + // Internal functions, used by DataObject + // + //-------------------------------------------------------------------------- + + /** + * Generate a BSON filter set, for unexpired items + * this should be used in combination with an AND clause filter + **/ + protected Bson filterForUnexpired(Date now) { + // the or array to join + return Filters.or( // + Filters.exists("expireAt", false), // + Filters.gt("expireAt", now), // + Filters.lte("expireAt", 0) // + ); // + } + + /** + * Search using the value, all the relevent key mappings + * + * Handles re-entrant lock where applicable + * + * @param key, note that null matches ALL + * + * @return array of keys + **/ + @Override + public Set keySet(Long value) { + // The return hashset + HashSet ret = new HashSet(); + + // Search result + FindIterable search = null; + + // Lets either fetch with a value, or everything + if (value == null) { + // Lets fetch everything ... D= + search = collection.find(filterForUnexpired(new Date())); + } else { + search = collection.find(Filters.and( // + filterForUnexpired(new Date()), // + Filters.eq("val", value) // + )); // + } + + // Get all the various keys + search = search.projection(Projections.include("key")); + + // Lets iterate the search + try (MongoCursor cursor = search.iterator()) { + while (cursor.hasNext()) { + ret.add(cursor.next().getString("key")); + } + } + + // Return the full keyset + return ret; + } + + //-------------------------------------------------------------------------- + // + // Fundemental set/get value (core) + // + //-------------------------------------------------------------------------- + + /** + * [Internal use, to be extended in future implementation] + * Sets the value, with validation + * + * @param key + * @param value, null means removal + * @param expire timestamp, 0 means not timestamp + * + * @return null + **/ + @Override + public Long setValueRaw(String key, Long value, long expireAt) { + // Configure this to be an "upsert" query + FindOneAndUpdateOptions opt = new FindOneAndUpdateOptions(); + opt.upsert(true); + + // Generate the document of changes + // See: https://www.mongodb.com/docs/manual/reference/operator/update/setOnInsert/ + + // Generate the "update" doc + Document updateDoc = new Document(); + Document set_doc = new Document(); + + // Expire timestamp if its configured, else it should be removed + if (expireAt > 0) { + set_doc.append("expireAt", new Date(expireAt)); + } else { + Document unset_doc = new Document(); + unset_doc.append("expireAt", ""); + updateDoc.append("$unset", unset_doc); + } + + // Setup the value on update/insert/upsert + set_doc.append("val", value); + updateDoc.append("$set", set_doc); + + // Set the key on insert + Document setOnInsert_doc = new Document(); + setOnInsert_doc.append("key", key); + updateDoc.append("$setOnInsert", setOnInsert_doc); + + // Upsert the document + collection.findOneAndUpdate(Filters.eq("key", key), updateDoc, opt); + return null; + } + + /** + * [Internal use, to be extended in future implementation] + * + * Returns the value and expiry, with validation against the current timestamp + * + * @param key as String + * @param now timestamp + * + * @return String value + **/ + @Override + public MutablePair getValueExpiryRaw(String key, long now) { + // Get the find result + FindIterable res = collection.find(Filters.eq("key", key)); + + // Get the Document object + Document resObj = res.first(); + if (resObj == null) { + return null; + } + + // Lets get all the key values + Long val = resObj.getLong("val"); + Date expireAt_date = resObj.getDate("expireAt"); + long expireAt_long = 0; + + // Check if expireAt date is set + if (expireAt_date != null) { + expireAt_long = expireAt_date.getTime(); + } + + // Check for null objects + if (val == null) { + return null; + } + + // No valid value found, return null + if (expireAt_long < 0) { + return null; + } + + // Expired value, return null + if (expireAt_long != 0 && expireAt_long < now) { + return null; + } + + // Get the value, and return the pair + return new MutablePair(val, expireAt_long); + } + + /** + * [Internal use, to be extended in future implementation] + * Sets the expire time stamp value, raw without validation + * + * @param key as String + * @param expireAt timestamp in seconds, 0 means NO expire + **/ + @Override + public void setExpiryRaw(String key, long expireAt) { + // Configure this to be an "update" query + FindOneAndUpdateOptions opt = new FindOneAndUpdateOptions(); + + // Generate the document of changes + // See: https://www.mongodb.com/docs/manual/reference/operator/update/setOnInsert/ + + // Generate the "update" doc + Document updateDoc = new Document(); + + // Expire timestamp if its configured, else it should be ignored + if (expireAt > 0) { + Document set_doc = new Document(); + set_doc.append("expireAt", new Date(expireAt)); + updateDoc.append("$set", set_doc); + } else { + Document unset_doc = new Document(); + unset_doc.append("expireAt", ""); + updateDoc.append("$unset", unset_doc); + } + + // Upsert the document + collection.findOneAndUpdate(Filters.eq("key", key), updateDoc, opt); + } + + //-------------------------------------------------------------------------- + // + // Incremental operations + // + //-------------------------------------------------------------------------- + + /** + * Stores (and overwrites if needed) key, value pair + * + * Important note: It does not return the previously stored value + * + * @param key as String + * @param expect as Long + * @param update as Long + * + * @return true if successful + **/ + public boolean weakCompareAndSet(String key, Long expect, Long update) { + // now timestamp + Date now = new Date(); + + // Configure this to be an "update" query + FindOneAndUpdateOptions opt = new FindOneAndUpdateOptions(); + + // Lets generate the mongodb "update" document rule + Document updateDoc = new Document(); + + // Disable expire timestamp when using weakCompareAndSet + Document unset_doc = new Document(); + unset_doc.append("expireAt", ""); + updateDoc.append("$unset", unset_doc); + + // Setup the value on update/insert/upsert + Document set_doc = new Document(); + set_doc.append("val", update); + updateDoc.append("$set", set_doc); + + // + // In general there are the following compare and set scenerios to handle + // + // 1) expecting value is 0 + // a) existing record is expired + // b) existing record does not exist + // c) existing record is NOT expired, and is 0 + // 2) expecting value is non-zero + // a) existing record is NOT expired, and is expected value. + // + + // Potentially a new upsert, ensure there is something to "update" atleast + // initializing an empty row if it does not exist + if (expect == null || expect == 0l) { + // Expect is now atleast 0 + expect = 0l; + } + + // + // We update any existing values + // this handle scenerio 1a, 1c & 2a + // + // We can do this safely here, as mongodb handles the expire + // natively, so we do not need to worry about race conditions. + // + + // + // Upsert the document + // + Object ret = collection.findOneAndUpdate(Filters.and(Filters.eq("key", key), // + Filters.or( // + // Handles an expired record + Filters.and(Filters.gt("expireAt", new Date(0l)), Filters.lt("expireAt", now)), + // Handles an non-expired record + Filters.and(Filters.eq("val", expect), filterForUnexpired(now)))), updateDoc, opt); + + // Return true on succesful update + if (ret != null) { + return true; + } + + // + // We insert a record if possible, this handle sceneric 1b + // + if (expect == 0l) { + try { + InsertOneResult res = collection.insertOne( // + new Document().append("key", key).append("val", update) // + ); + + if (res.wasAcknowledged()) { + return true; + } + } catch (Exception e) { + // This is probably due to a conflicting index + return false; + } + } + + // All Failed + return false; + } + + //-------------------------------------------------------------------------- + // + // Remove call + // + //-------------------------------------------------------------------------- + + /** + * Remove the value, given the key + * + * @param key param find the thae meta key + * + * @return null + **/ + @Override + public KeyLong remove(Object key) { + removeValue(key); + return null; + } + + /** + * Remove the value, given the key + * + * Important note: It does not return the previously stored value + * Its return String type is to maintain consistency with Map interfaces + * + * @param key param find the thae meta key + * + * @return null + **/ + @Override + public Long removeValue(Object key) { + if (key == null) { + throw new IllegalArgumentException("delete 'key' cannot be null"); + } + + // Delete the data + collection.deleteOne(Filters.eq("key", key)); + return null; + } + + //-------------------------------------------------------------------------- + // + // Maintenance calls + // + //-------------------------------------------------------------------------- + + /** + * Incremental maintainance should not trigger maintenance. + * As its potentially blocking with a very long call + **/ + public void incrementalMaintenance() { + // does nothing + } + + @Override + public void maintenance() { + // @TODO : something? (not sure what needs to be done) + } + +} \ No newline at end of file From d9148a4856f1ccae0608d39eed0f459046d8d509 Mon Sep 17 00:00:00 2001 From: Eugene Cheah Date: Wed, 3 Aug 2022 10:09:42 +0000 Subject: [PATCH 20/42] KeyLongMap test --- .../picoded/dstack/mongodb/MongoDBStack.java | 2 + .../mongodb/MongoDB_KeyLongMap_test.java | 47 +++++++++++++++++++ 2 files changed, 49 insertions(+) create mode 100755 src/test/java/picoded/dstack/mongodb/MongoDB_KeyLongMap_test.java diff --git a/src/main/java/picoded/dstack/mongodb/MongoDBStack.java b/src/main/java/picoded/dstack/mongodb/MongoDBStack.java index 0cb83e2f..58b373bd 100644 --- a/src/main/java/picoded/dstack/mongodb/MongoDBStack.java +++ b/src/main/java/picoded/dstack/mongodb/MongoDBStack.java @@ -128,6 +128,8 @@ protected Core_DataStructure initDataStructure(String name, String type) { ret = new MongoDB_DataObjectMap(this, name); } else if (type.equalsIgnoreCase("KeyValueMap")) { ret = new MongoDB_KeyValueMap(this, name); + } else if (type.equalsIgnoreCase("KeyLongMap")) { + ret = new MongoDB_KeyLongMap(this, name); } // If datastrucutre initialized, setup name diff --git a/src/test/java/picoded/dstack/mongodb/MongoDB_KeyLongMap_test.java b/src/test/java/picoded/dstack/mongodb/MongoDB_KeyLongMap_test.java new file mode 100755 index 00000000..27a05252 --- /dev/null +++ b/src/test/java/picoded/dstack/mongodb/MongoDB_KeyLongMap_test.java @@ -0,0 +1,47 @@ +package picoded.dstack.mongodb; + +import picoded.core.struct.*; +import picoded.dstack.*; +import picoded.dstack.struct.simple.*; + +/** + * ## Purpose + * This class is meant to test the MongoDB_KeyLongMap implementation, + * and ensure that it passes all the test layed out in StructSImple_KeyLongMap_test + * + */ +public class MongoDB_KeyLongMap_test extends StructSimple_KeyLongMap_test { + + // Hazelcast stack instance + protected static volatile MongoDBStack instance = null; + + // To override for implementation + //----------------------------------------------------- + + /// Impomentation constructor + public KeyLongMap implementationConstructor() { + + // Initialize server + synchronized (MongoDB_KeyLongMap_test.class) { + if (instance == null) { + // The default config uses localhost, 27017 + GenericConvertMap mongodbConfig = new GenericConvertHashMap<>(); + mongodbConfig.put("host", DStackTestConfig.MONGODB_HOST()); + mongodbConfig.put("port", DStackTestConfig.MONGODB_PORT()); + + // Use a random DB name + mongodbConfig.put("name", DStackTestConfig.randomTablePrefix()); + + GenericConvertMap stackConfig = new GenericConvertHashMap<>(); + stackConfig.put("name", "MongoDB_KeyLongMap_test"); + stackConfig.put("mongodb", mongodbConfig); + + instance = new MongoDBStack(stackConfig); + } + } + + // Load the KeyLongMap + return instance.keyLongMap(DStackTestConfig.randomTablePrefix()); + } + +} From 8fa1e61b33ba26059b46b9e4dcc14a1162400530 Mon Sep 17 00:00:00 2001 From: Eugene Cheah Date: Wed, 3 Aug 2022 10:57:44 +0000 Subject: [PATCH 21/42] Standardising to fileWriteInputStream (it was confused with outputStream) --- .../java/picoded/dstack/FileWorkspace.java | 18 ++++++------------ .../dstack/core/Core_FileWorkspace.java | 4 ++-- .../dstack/core/Core_FileWorkspaceMap.java | 15 ++++++--------- .../dstack/stack/Stack_FileWorkspaceMap.java | 16 ++++++---------- .../StructSimple_FileWorkspaceMap_test.java | 6 +++--- 5 files changed, 23 insertions(+), 36 deletions(-) diff --git a/src/main/java/picoded/dstack/FileWorkspace.java b/src/main/java/picoded/dstack/FileWorkspace.java index 8a6e2cc1..7c017f46 100755 --- a/src/main/java/picoded/dstack/FileWorkspace.java +++ b/src/main/java/picoded/dstack/FileWorkspace.java @@ -12,6 +12,8 @@ import java.util.List; import java.util.Set; +import org.apache.commons.io.IOUtils; + /** * Represent a file storage backend for a workspace * @@ -141,7 +143,7 @@ default void writeString(final String filepath, String content, String encoding) * You are expected to close, the stream on your own, to avoid memory leaks * * @param filePath in the workspace to extract - * @return the file contents, null if file does not exists + * @return the file contents as an input stream, null if file does not exists */ default InputStream readInputStream(final String filePath) { byte[] byteArr = readByteArray(filePath); @@ -149,7 +151,7 @@ default InputStream readInputStream(final String filePath) { } /** - * Writes an output stream to a file creating the file if it does not exist. + * Reads an input stream, and writes it to a fil, creating the file if it does not exist. * the parent directories of the file will be created if they do not exist. * * Note that depending on the implementaiton, this may not be optimized, @@ -158,22 +160,14 @@ default InputStream readInputStream(final String filePath) { * @param filepath in the workspace to extract * @param data the content to write to the file **/ - default void writeOutputStream(final String filepath, final OutputStream data) { - + default void writeInputStream(final String filepath, final InputStream data) { // Converts it to bytearray respectively byte[] rawBytes = null; try { - if (data instanceof ByteArrayOutputStream) { - rawBytes = ((ByteArrayOutputStream) data).toByteArray(); - } else { - ByteArrayOutputStream buffer = new ByteArrayOutputStream(); - buffer.writeTo(data); - rawBytes = buffer.toByteArray(); - } + rawBytes = IOUtils.toByteArray(data); } catch (IOException e) { throw new RuntimeException(e); } - // Does the bytearray writes writeByteArray(filepath, rawBytes); } diff --git a/src/main/java/picoded/dstack/core/Core_FileWorkspace.java b/src/main/java/picoded/dstack/core/Core_FileWorkspace.java index 022da45c..9297d3a1 100755 --- a/src/main/java/picoded/dstack/core/Core_FileWorkspace.java +++ b/src/main/java/picoded/dstack/core/Core_FileWorkspace.java @@ -184,8 +184,8 @@ public InputStream readInputStream(final String filepath) { * @param filepath in the workspace to extract * @param data the content to write to the file **/ - public void writeOutputStream(final String filepath, final OutputStream data) { - main.backend_fileWriteOutputStream(_oid, normalizeFilePathString(filepath), data); + public void writeInputStream(final String filepath, final InputStream data) { + main.backend_fileWriteInputStream(_oid, normalizeFilePathString(filepath), data); } // Read / write byteArray information diff --git a/src/main/java/picoded/dstack/core/Core_FileWorkspaceMap.java b/src/main/java/picoded/dstack/core/Core_FileWorkspaceMap.java index e676682e..86b7db1c 100755 --- a/src/main/java/picoded/dstack/core/Core_FileWorkspaceMap.java +++ b/src/main/java/picoded/dstack/core/Core_FileWorkspaceMap.java @@ -7,6 +7,9 @@ import java.util.HashSet; import java.util.Set; + +import org.apache.commons.io.IOUtils; + import java.io.InputStream; import java.io.OutputStream; import java.io.ByteArrayInputStream; @@ -201,8 +204,8 @@ public InputStream backend_fileReadInputStream(final String oid, final String fi * @param filepath to use for the workspace * @param data to write the file with **/ - public void backend_fileWriteOutputStream(final String oid, final String filepath, - final OutputStream data) { + public void backend_fileWriteInputStream(final String oid, final String filepath, + final InputStream data) { // forward the null, and let the error handling below settle it if (data == null) { @@ -212,13 +215,7 @@ public void backend_fileWriteOutputStream(final String oid, final String filepat // Converts it to bytearray respectively byte[] rawBytes = null; try { - if (data instanceof ByteArrayOutputStream) { - rawBytes = ((ByteArrayOutputStream) data).toByteArray(); - } else { - ByteArrayOutputStream buffer = new ByteArrayOutputStream(); - buffer.writeTo(data); - rawBytes = buffer.toByteArray(); - } + rawBytes = IOUtils.toByteArray(data); } catch (IOException e) { throw new RuntimeException(e); } diff --git a/src/main/java/picoded/dstack/stack/Stack_FileWorkspaceMap.java b/src/main/java/picoded/dstack/stack/Stack_FileWorkspaceMap.java index 24d31fae..4c3437a0 100755 --- a/src/main/java/picoded/dstack/stack/Stack_FileWorkspaceMap.java +++ b/src/main/java/picoded/dstack/stack/Stack_FileWorkspaceMap.java @@ -11,6 +11,8 @@ import java.util.List; import java.util.Set; +import org.apache.commons.io.IOUtils; + /** * Stacked implementation of KeyValueMap data structure. * @@ -218,13 +220,13 @@ public InputStream backend_fileReadInputStream(final String oid, final String fi * @param data to write the file with **/ @Override - public void backend_fileWriteOutputStream(final String oid, final String filepath, - final OutputStream data) { + public void backend_fileWriteInputStream(final String oid, final String filepath, + final InputStream data) { // Due to the behaviour of how the file data needs to be handled across multiple layers // we only use an optimized "readStream" call if the filesystem is a single stack layer if (dataLayers.length == 1) { - dataLayers[0].backend_fileWriteOutputStream(oid, filepath, data); + dataLayers[0].backend_fileWriteInputStream(oid, filepath, data); return; } @@ -239,13 +241,7 @@ public void backend_fileWriteOutputStream(final String oid, final String filepat // Converts it to bytearray respectively byte[] rawBytes = null; try { - if (data instanceof ByteArrayOutputStream) { - rawBytes = ((ByteArrayOutputStream) data).toByteArray(); - } else { - ByteArrayOutputStream buffer = new ByteArrayOutputStream(); - buffer.writeTo(data); - rawBytes = buffer.toByteArray(); - } + rawBytes = IOUtils.toByteArray(data); } catch (IOException e) { throw new RuntimeException(e); } diff --git a/src/test/java/picoded/dstack/struct/simple/StructSimple_FileWorkspaceMap_test.java b/src/test/java/picoded/dstack/struct/simple/StructSimple_FileWorkspaceMap_test.java index 25f7bdca..87db2bc1 100755 --- a/src/test/java/picoded/dstack/struct/simple/StructSimple_FileWorkspaceMap_test.java +++ b/src/test/java/picoded/dstack/struct/simple/StructSimple_FileWorkspaceMap_test.java @@ -7,6 +7,7 @@ import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; +import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.InputStream; import java.util.Arrays; @@ -335,11 +336,10 @@ public void readNonExistenceFile() { @Test public void writeAndReadToFile_stream() throws Exception { // Output stream to use for content - ByteArrayOutputStream buffer = new ByteArrayOutputStream(); - buffer.write("data to write".getBytes()); + ByteArrayInputStream buffer = new ByteArrayInputStream("data to write".getBytes()); FileWorkspace fileWorkspace = testObj.newEntry(); - fileWorkspace.writeOutputStream("testPath", buffer); + fileWorkspace.writeInputStream("testPath", buffer); assertNotNull(testObj.get(fileWorkspace._oid()).readByteArray("testPath")); InputStream readData = testObj.get(fileWorkspace._oid()).readInputStream("testPath"); From 17747b37e9d715f2cdc59a84f20fdd887da42ef2 Mon Sep 17 00:00:00 2001 From: Eugene Cheah Date: Thu, 4 Aug 2022 07:46:23 +0000 Subject: [PATCH 22/42] WIP FileWorkspaceMap support --- .../java/picoded/dstack/FileWorkspace.java | 6 + .../dstack/core/Core_FileWorkspaceMap.java | 8 +- .../mongodb/MongoDB_FileWorkspaceMap.java | 1061 ++++++++--------- 3 files changed, 531 insertions(+), 544 deletions(-) diff --git a/src/main/java/picoded/dstack/FileWorkspace.java b/src/main/java/picoded/dstack/FileWorkspace.java index 7c017f46..d3b8b8c6 100755 --- a/src/main/java/picoded/dstack/FileWorkspace.java +++ b/src/main/java/picoded/dstack/FileWorkspace.java @@ -167,6 +167,12 @@ default void writeInputStream(final String filepath, final InputStream data) { rawBytes = IOUtils.toByteArray(data); } catch (IOException e) { throw new RuntimeException(e); + } finally { + try { + data.close(); + } catch (IOException e) { + throw new RuntimeException(e); + } } // Does the bytearray writes writeByteArray(filepath, rawBytes); diff --git a/src/main/java/picoded/dstack/core/Core_FileWorkspaceMap.java b/src/main/java/picoded/dstack/core/Core_FileWorkspaceMap.java index 86b7db1c..1c343bdc 100755 --- a/src/main/java/picoded/dstack/core/Core_FileWorkspaceMap.java +++ b/src/main/java/picoded/dstack/core/Core_FileWorkspaceMap.java @@ -181,7 +181,7 @@ public void setupWorkspace(String oid) { * @param ObjectID of workspace * @param filepath to use for the workspace * - * @return the stored byte array of the file + * @return the stored byte stream of the file **/ public InputStream backend_fileReadInputStream(final String oid, final String filepath) { // Get the byte data @@ -218,6 +218,12 @@ public void backend_fileWriteInputStream(final String oid, final String filepath rawBytes = IOUtils.toByteArray(data); } catch (IOException e) { throw new RuntimeException(e); + } finally { + try { + data.close(); + } catch (IOException e) { + throw new RuntimeException(e); + } } // Does the bytearray writes diff --git a/src/main/java/picoded/dstack/mongodb/MongoDB_FileWorkspaceMap.java b/src/main/java/picoded/dstack/mongodb/MongoDB_FileWorkspaceMap.java index afe1b7ae..5f05b3dc 100755 --- a/src/main/java/picoded/dstack/mongodb/MongoDB_FileWorkspaceMap.java +++ b/src/main/java/picoded/dstack/mongodb/MongoDB_FileWorkspaceMap.java @@ -1,13 +1,17 @@ package picoded.dstack.mongodb; -import java.io.ByteArrayInputStream; -import java.io.IOException; // Java imports import java.util.ArrayList; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.regex.Pattern; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; // JavaCommons imports import picoded.core.common.EmptyArray; @@ -15,6 +19,7 @@ import picoded.dstack.FileWorkspace; import picoded.dstack.core.Core_FileWorkspaceMap; +import org.apache.commons.io.IOUtils; // MongoDB imports import org.bson.Document; import org.bson.types.Binary; @@ -24,6 +29,7 @@ import com.mongodb.client.gridfs.GridFSBuckets; import com.mongodb.client.gridfs.model.GridFSFile; import com.mongodb.client.gridfs.model.GridFSUploadOptions; +import com.mongodb.client.model.Filters; /** * ## Purpose @@ -37,7 +43,7 @@ * - GridFS : https://www.mongodb.com/docs/drivers/java/sync/current/fundamentals/gridfs/ * - API: https://mongodb.github.io/mongo-java-driver/4.7/apidocs/mongodb-driver-sync/com/mongodb/client/gridfs/GridFSBucket.html **/ -abstract public class MongoDB_FileWorkspaceMap extends Core_FileWorkspaceMap { +public class MongoDB_FileWorkspaceMap extends Core_FileWorkspaceMap { // -------------------------------------------------------------------------- // @@ -71,7 +77,7 @@ public MongoDB_FileWorkspaceMap(MongoDBStack inStack, String name) { // of 2 : https://jira.mongodb.org/browse/SERVER-13331 // // Meaning a full "8 * 1000 * 1000" chunk would use "8 * 1024 * 1024" - // worth of space, after adding the unknown headers (<=2kb) + // worth of space, after adding the unknown headers (<=4kb of space : 8*24*24) // } @@ -121,22 +127,8 @@ public void clear() { **/ @Override public boolean backend_workspaceExist(String oid) { - // // Lets build the query for the "root file" - // Bson query = Filters.eq("filename", oid); - - // // Lets prepare the search - // FindIterable search = gridFSBucket.find(query).limit(1); - - // // Lets iterate the search result, and return true on an item - // try (MongoCursor cursor = search.iterator()) { - // while (cursor.hasNext()) { - // // ret.add(cursor.next().getString("_oid")); - // return true; - // } - // } - - // Fail, as the search found no iterations - return false; + // The folder root, will only contain the "oid" + return fullRawPathExist(oid); } /** @@ -149,6 +141,100 @@ public boolean backend_workspaceExist(String oid) { */ @Override public void backend_setupWorkspace(String oid) { + // We setup a blank file with type root + if(!fullRawPathExist(oid)) { + setupAnchorFile(oid, oid, "root"); + } + } + + /** + * [Internal use, to be extended in future implementation] + * + * Removes the FileWorkspace, used to nuke an entire workspace + * + * @param ObjectID of workspace to remove + **/ + @Override + public void backend_workspaceRemove(String oid) { + removeFilePathRecursively(oid, null); + } + + //-------------------------------------------------------------------------- + // + // Utility functions + // + //-------------------------------------------------------------------------- + + /** + * Given a filepath, ensure a clean filepath (without starting "/") + */ + protected static String cleanFilePath(final String filepath) { + // Note that the FileUtil.normalize step is not needed, as + // this is already done in the Core_FileWorkspaceMap + // --- + // String cleanFilePath = FileUtil.normalize(filepath); + + // Cleanup the file apth + String cleanFilePath = filepath; + while (cleanFilePath.startsWith("/")) { + cleanFilePath = cleanFilePath.substring(1); + } + return cleanFilePath; + } + + /** Utility function used, to check if a workspace, or file exists **/ + protected boolean fullRawPathExist(String fullpath) { + // Lets build the query for the "root file" + Bson query = Filters.eq("filename", fullpath); + + // Lets prepare the search + GridFSFindIterable search = gridFSBucket.find(query).limit(1); + + // Lets iterate the search result, and return true on an item + try (MongoCursor cursor = search.iterator()) { + if (cursor.hasNext()) { + // ret.add(cursor.next().getString("_oid")); + return true; + } + } + + // Fail, as the search found no iterations + return false; + } + + /** Utility function used, to check if a folder, or file with folder prefix exists **/ + protected boolean prefixPathExist(String oid, String path) { + // Lets build the query for the "root file" + Bson query = null; + + // Cleanup the path + path = cleanFilePath(path); + + // Remove matching path + query = Filters.and( + Filters.eq("metadata._oid", oid), + Filters.regex("filename", "^"+Pattern.quote(oid+"/"+path)+".*") + ); + + // Lets prepare the search + GridFSFindIterable search = gridFSBucket.find(query).limit(1); + + // Lets iterate the search result, and return true on an item + try (MongoCursor cursor = search.iterator()) { + if (cursor.hasNext()) { + return true; + } + } + + // No match found, fail + return false; + } + + /** + * Setup an empty file, used for various use cases + */ + @Override + public void setupAnchorFile(String oid, String fullPath, String type) { // In general we will upload a blank file // with the relevent _oid, that can be easily lookedup // @@ -157,535 +243,424 @@ public void backend_setupWorkspace(String oid) { // Setup the metadata for the file Document metadata = new Document(); metadata.append("_oid", oid); - metadata.append("type", "root"); + metadata.append("type", type); // Prepare the upload options GridFSUploadOptions opt = (new GridFSUploadOptions()).metadata(metadata); - gridFSBucket.uploadFromStream(oid, emptyStream, opt); + gridFSBucket.uploadFromStream(fullPath, emptyStream, opt); } catch (IOException e) { throw new RuntimeException(e); } } - // /** - // * [Internal use, to be extended in future implementation] - // * - // * Removes the FileWorkspace, used to nuke an entire workspace - // * - // * @param ObjectID of workspace to remove - // **/ - // @Override - // public void backend_workspaceRemove(String oid) { - // try { - // accessLock.writeLock().lock(); - // fileContentMap.remove(oid); - // } finally { - // accessLock.writeLock().unlock(); - // } - // } - - // //-------------------------------------------------------------------------- - // // - // // File read / write - // // [Internal use, to be extended in future implementation] - // // - // //-------------------------------------------------------------------------- - - // /** - // * [Internal use, to be extended in future implementation] - // * - // * Get and return the stored data as a byte[] - // * - // * @param ObjectID of workspace - // * @param filepath to use for the workspace - // * - // * @return the stored byte array of the file - // **/ - // @Override - // public byte[] backend_fileRead(String oid, String filepath) { - // try { - // accessLock.readLock().lock(); - - // ConcurrentHashMap workspace = fileContentMap.get(oid); - // if (workspace != null && filepath != null) { - // return workspace.get(filepath); - // } - // return null; - // } finally { - // accessLock.readLock().unlock(); - // } - - // } - - // /** - // * [Internal use, to be extended in future implementation] - // * - // * Get and return if the file exists, due to the potentially - // * large size nature of files stored in FileWorkspace. - // * - // * Its highly recommended to optimize this function, - // * instead of leaving it as default - // * - // * @param ObjectID of workspace - // * @param filepath to use for the workspace - // * - // * @return boolean true, if file eixst - // **/ - // public boolean backend_fileExist(final String oid, final String filepath) { - // try { - // accessLock.readLock().lock(); - - // ConcurrentHashMap workspace = fileContentMap.get(oid); - // if (workspace != null && filepath != null) { - // return workspace.get(filepath) != null; - // } - // } finally { - // accessLock.readLock().unlock(); - // } - // return false; - // } - - // /** - // * [Internal use, to be extended in future implementation] - // * - // * Writes the full byte array of a file in the backend - // * - // * @param ObjectID of workspace - // * @param filepath to use for the workspace - // * @param data to write the file with - // **/ - // @Override - // public void backend_fileWrite(String oid, String filepath, byte[] data) { - // try { - // accessLock.writeLock().lock(); - - // // Get workspace, with normalized parent path - // ConcurrentHashMap workspace = noLock_setupWorkspaceFolderPath(oid, - // FileUtil.getParentPath(filepath)); - - // // And put in the filepth data - // workspace.put(filepath, data); - - // } finally { - // accessLock.writeLock().unlock(); - // } - // } - - // /** - // * [Internal use, to be extended in future implementation] - // * - // * Removes the specified file path from the workspace in the backend - // * - // * @param oid identifier to the workspace - // * @param filepath the file to be removed - // */ - // @Override - // public void backend_removeFile(String oid, String filepath) { - // try { - // accessLock.writeLock().lock(); - - // ConcurrentHashMap workspace = fileContentMap.get(oid); - - // // workspace exist, remove the file in the workspace - // if (workspace != null) { - // workspace.remove(filepath); - // } - - // } finally { - // accessLock.writeLock().unlock(); - // } - // } - - // //-------------------------------------------------------------------------- - // // - // // Folder pathing support - // // - // //-------------------------------------------------------------------------- - - // /** - // * [Internal use, to be extended in future implementation] - // * - // * Delete an existing path from the workspace. - // * This recursively removes all file content under the given path prefix - // * - // * @param ObjectID of workspace - // * @param folderPath in the workspace (note, folderPath is normalized to end with "/") - // * - // * @return the stored byte array of the file - // **/ - // public void backend_removeFolderPath(final String oid, final String folderPath) { - // try { - // accessLock.writeLock().lock(); - - // // Get the workspace, and abort if null - // ConcurrentHashMap workspace = fileContentMap.get(oid); - // if (workspace == null) { - // return; - // } - - // // Get the keyset - in a new hashset - // // (so it wouldnt crash when we do modification) - // Set allKeys = new HashSet<>(workspace.keySet()); - // for (String key : allKeys) { - // // If folder path match - remove it - // if (key.startsWith(folderPath)) { - // workspace.remove(key); - // } - // } - - // } finally { - // accessLock.writeLock().unlock(); - // } - // } - - // /** - // * [Internal use, to be extended in future implementation] - // * - // * Validate the given folder path exists. - // * - // * @param ObjectID of workspace - // * @param folderPath in the workspace (note, folderPath is normalized to end with "/") - // * - // * @return the stored byte array of the file - // **/ - // public boolean backend_folderPathExist(final String oid, final String folderPath) { - // try { - // accessLock.readLock().lock(); - // ConcurrentHashMap workspace = fileContentMap.get(oid); - // return workspace != null && workspace.get(folderPath) != null; - // } finally { - // accessLock.readLock().unlock(); - // } - // } - - // /** - // * [Internal use, to be extended in future implementation] - // * - // * Automatically generate a given folder path if it does not exist - // * - // * @param ObjectID of workspace - // * @param folderPath in the workspace (note, folderPath is normalized to end with "/") - // * - // * @return the stored byte array of the file - // **/ - // public void backend_ensureFolderPath(final String oid, final String folderPath) { - // try { - // accessLock.writeLock().lock(); - // noLock_setupWorkspaceFolderPath(oid, folderPath); - // } finally { - // accessLock.writeLock().unlock(); - // } - // } - - // //-------------------------------------------------------------------------- - // // - // // Move support - // // - // //-------------------------------------------------------------------------- - - // /** - // * @return if the current configured implementation supports atomic move operations. - // */ - // public boolean atomicMoveSupported() { - // // True due to StructSimple use of a globle write lock - // return true; - // } - - // /** - // * [Internal use, to be extended in future implementation] - // * - // * Move a given file within the system - // * - // * WARNING: Move operations are typically not "atomic" in nature, and can be unsafe where - // * missing files / corrupted data can occur when executed concurrently with other operations. - // * - // * In general "S3-like" object storage will not safely support atomic move operations. - // * Please use the `atomicMoveSupported()` function to validate if such operations are supported. - // * - // * This operation may in effect function as a rename - // * If the destionation file exists, it will be overwritten - // * - // * @param ObjectID of workspace - // * @param sourceFile - // * @param destinationFile - // */ - // public void backend_moveFile(final String oid, final String sourceFile, - // final String destinationFile) { - // try { - // accessLock.writeLock().lock(); - - // // Get the workspace, and abort if null - // ConcurrentHashMap workspace = fileContentMap.get(oid); - // if (workspace == null) { - // throw new RuntimeException("FileWorkspace does not exist : " + oid); - // } - - // // Check if sourceFolder exist - // if (workspace.get(sourceFile) == null) { - // throw new RuntimeException("sourceFile does not exist (oid=" + oid + ") : " - // + sourceFile); - // } - - // // Initialize the destionation folder - // noLock_setupWorkspaceFolderPath(oid, FileUtil.getParentPath(destinationFile)); - - // // Copy the file - // workspace.put(destinationFile, workspace.get(sourceFile)); - - // // And remove the old copy - // workspace.remove(sourceFile); - // } finally { - // accessLock.writeLock().unlock(); - // } - // } - - // /** - // * [Internal use, to be extended in future implementation] - // * - // * Move a given file within the system - // * - // * WARNING: Move operations are typically not "atomic" in nature, and can be unsafe where - // * missing files / corrupted data can occur when executed concurrently with other operations. - // * - // * In general "S3-like" object storage will not safely support atomic move operations. - // * Please use the `atomicMoveSupported()` function to validate if such operations are supported. - // * - // * Note that both source, and destionation folder will be normalized to include the "/" path. - // * This operation may in effect function as a rename - // * If the destionation folder exists with content, the result will be merged. With the sourceFolder files, overwriting on conflicts. - // * - // * @param ObjectID of workspace - // * @param sourceFolder - // * @param destinationFolder - // * - // */ - // public void backend_moveFolderPath(final String oid, final String sourceFolder, - // final String destinationFolder) { - // try { - // accessLock.writeLock().lock(); - - // // Get the workspace, and abort if null - // ConcurrentHashMap workspace = fileContentMap.get(oid); - // if (workspace == null) { - // throw new RuntimeException("FileWorkspace does not exist : " + oid); - // } - - // // Check if sourceFolder exist - // if (workspace.get(sourceFolder) == null) { - // throw new RuntimeException("sourceFolder does not exist (oid=" + oid + ") : " - // + sourceFolder); - // } - - // // Get the keyset - in a new hashset - // // (so it wouldnt crash when we do modification) - // Set allKeys = new HashSet<>(workspace.keySet()); - // for (String key : allKeys) { - // // If folder path match - migrate it - // if (key.startsWith(sourceFolder)) { - // // Copy it over - // workspace.put(destinationFolder + key.substring(sourceFolder.length()), - // workspace.get(key)); - // // Remove it - // workspace.remove(key); - // } - // } - - // } finally { - // accessLock.writeLock().unlock(); - // } - // } - - // //-------------------------------------------------------------------------- - // // - // // Copy support - // // - // //-------------------------------------------------------------------------- - - // /** - // * @return if the current configured implementation supports atomic Copy operations. - // */ - // public boolean atomicCopySupported() { - // // True due to StructSimple use of a globle write lock - // return true; - // } - - // /** - // * [Internal use, to be extended in future implementation] - // * - // * Copy a given file within the system - // * - // * WARNING: Copy operations are typically not "atomic" in nature, and can be unsafe where - // * missing files / corrupted data can occur when executed concurrently with other operations. - // * - // * In general "S3-like" object storage will not safely support atomic Copy operations. - // * Please use the `atomicCopySupported()` function to validate if such operations are supported. - // * - // * This operation may in effect function as a rename - // * If the destionation file exists, it will be overwritten - // * - // * @param ObjectID of workspace - // * @param sourceFile - // * @param destinationFile - // */ - // public void backend_copyFile(final String oid, final String sourceFile, - // final String destinationFile) { - // try { - // accessLock.writeLock().lock(); - - // // Get the workspace, and abort if null - // ConcurrentHashMap workspace = fileContentMap.get(oid); - // if (workspace == null) { - // throw new RuntimeException("FileWorkspace does not exist : " + oid); - // } - - // // Check if sourceFolder exist - // if (workspace.get(sourceFile) == null) { - // throw new RuntimeException("sourceFile does not exist (oid=" + oid + ") : " - // + sourceFile); - // } - - // // Initialize the destionation folder - // noLock_setupWorkspaceFolderPath(oid, FileUtil.getParentPath(destinationFile)); - - // // Copy the file - // workspace.put(destinationFile, workspace.get(sourceFile)); - // } finally { - // accessLock.writeLock().unlock(); - // } - // } - - // /** - // * [Internal use, to be extended in future implementation] - // * - // * Copy a given file within the system - // * - // * WARNING: Copy operations are typically not "atomic" in nature, and can be unsafe where - // * missing files / corrupted data can occur when executed concurrently with other operations. - // * - // * In general "S3-like" object storage will not safely support atomic Copy operations. - // * Please use the `atomicCopySupported()` function to validate if such operations are supported. - // * - // * Note that both source, and destionation folder will be normalized to include the "/" path. - // * This operation may in effect function as a rename - // * If the destionation folder exists with content, the result will be merged. With the sourceFolder files, overwriting on conflicts. - // * - // * @param ObjectID of workspace - // * @param sourceFolder - // * @param destinationFolder - // * - // */ - // public void backend_copyFolderPath(final String oid, final String sourceFolder, - // final String destinationFolder) { - // try { - // accessLock.writeLock().lock(); - - // // Get the workspace, and abort if null - // ConcurrentHashMap workspace = fileContentMap.get(oid); - // if (workspace == null) { - // throw new RuntimeException("FileWorkspace does not exist : " + oid); - // } - - // // Check if sourceFolder exist - // if (workspace.get(sourceFolder) == null) { - // throw new RuntimeException("sourceFolder does not exist (oid=" + oid + ") : " - // + sourceFolder); - // } - - // // Get the keyset - in a new hashset - // // (so it wouldnt crash when we do modification) - // Set allKeys = new HashSet<>(workspace.keySet()); - // for (String key : allKeys) { - // // If folder path match - migrate it - // if (key.startsWith(sourceFolder)) { - // // Copy it over - // workspace.put(destinationFolder + key.substring(sourceFolder.length()), - // workspace.get(key)); - // } - // } - - // } finally { - // accessLock.writeLock().unlock(); - // } - // } - - // //-------------------------------------------------------------------------- - // // - // // Listing support - // // - // //-------------------------------------------------------------------------- - - // /** - // * List all the various files and folders found in the given folderPath - // * - // * @param ObjectID of workspace - // * @param folderPath in the workspace (note, folderPath is normalized to end with "/") - // * @param minDepth minimum depth count, before outputing the listing (uses a <= match) - // * @param maxDepth maximum depth count, to stop the listing (-1 for infinite, uses a >= match) - // * - // * @return list of path strings - relative to the given folderPath (folders end with "/") - // */ - // public Set backend_getFileAndFolderPathSet(final String oid, final String folderPath, - // final int minDepth, final int maxDepth) { - // try { - // accessLock.readLock().lock(); - - // // Get the workspace, and abort if null - // ConcurrentHashMap workspace = fileContentMap.get(oid); - // if (workspace == null) { - // throw new RuntimeException("FileWorkspace does not exist : " + oid); - // } - - // // Check if folderPath exist - // String searchPath = folderPath; - // if (searchPath.equals("/")) { - // searchPath = ""; - // } - // if (searchPath.length() > 0 && workspace.get(searchPath) == null) { - // throw new RuntimeException("folderPath does not exist (oid=" + oid + ") : " - // + searchPath); - // } - - // // Return a filtered set - // return backend_filtterPathSet(workspace.keySet(), searchPath, minDepth, maxDepth, 0); - // } finally { - // accessLock.readLock().unlock(); - // } - // } - - // //-------------------------------------------------------------------------- - // // - // // Constructor and maintenance - // // - // //-------------------------------------------------------------------------- - - // @Override - // public void systemSetup() { - - // } - - // @Override - // public void systemDestroy() { - // clear(); - // } - - // /** - // * Maintenance step call, however due to the nature of most implementation not - // * having any form of time "expiry", this call does nothing in most implementation. - // * - // * As such im making that the default =) - // **/ - // @Override - // public void maintenance() { - // // Do nothing - // } - - // @Override - // public void clear() { - // try { - // accessLock.writeLock().lock(); - // fileContentMap.clear(); - // } finally { - // accessLock.writeLock().unlock(); - // } - // } + /** + * Utility function used, to recursively delete all files within a specific path + **/ + protected void removeFilePathRecursively(String oid, String path) { + // Lets build the query for the "root file" + Bson query = null; + + if( path == null ) { + // Remove everything under the oid + query = Filters.eq("metadata._oid", oid); + } else { + // Cleanup the path + path = cleanFilePath(path); + + // Remove matching path + query = Filters.and( + Filters.eq("metadata._oid", oid), + Filters.regex("filename", "^"+Pattern.quote(oid+"/"+path)+".*") + ); + } + + // Lets prepare the search + GridFSFindIterable search = gridFSBucket.find(query); + + // Lets iterate the search result, and return true on an item + try (MongoCursor cursor = search.iterator()) { + while (cursor.hasNext()) { + GridFSFile fileObj = cursor.next(); + gridFSBucket.delete(fileObj.getId()); + } + } + } + + /** + * Utility function used, to remove a specific file + **/ + protected boolean removeFilePathOnce(String oid, String path) { + // Lets build the query for the "root file" + Bson query = null; + + // Cleanup the path + path = cleanFilePath(path); + + // Remove matching path + query = Filters.eq("filename", oid+"/"+path); + + // Lets prepare the search + GridFSFindIterable search = gridFSBucket.find(query).limit(1); + + // Lets iterate the search result, and return true on an item + try (MongoCursor cursor = search.iterator()) { + if (cursor.hasNext()) { + GridFSFile fileObj = cursor.next(); + gridFSBucket.delete(fileObj.getId()); + return true; + } + } + + // removal didn't occur + return false; + } + + //-------------------------------------------------------------------------- + // + // File write + // [Internal use, to be extended in future implementation] + // + //-------------------------------------------------------------------------- + + /** + * [Internal use, to be extended in future implementation] + * + * Writes the full byte array of a file in the backend + * + * @param ObjectID of workspace + * @param filepath to use for the workspace + * @param data to write the file with + **/ + @Override + public void backend_fileWrite(String oid, String filepath, byte[] data) { + // Build the input stream + ByteArrayInputStream buffer = null; + + // Only build if its not null + if (data != null) { + buffer = new ByteArrayInputStream(data); + } + + // Then pump it + backend_fileWriteInputStream(oid, filepath, buffer); + } + + /** + * [Internal use, to be extended in future implementation] + * + * Writes the full byte array of a file in the backend + * + * This overwrite is useful for backends which supports this flow. + * Else it would simply be a wrapper over the non-stream version. + * + * @param ObjectID of workspace + * @param filepath to use for the workspace + * @param data to write the file with + **/ + public void backend_fileWriteInputStream(String oid, String filepath, InputStream data) { + // Get the clean file path + String cleanPath = cleanFilePath(filepath); + + // Build the full path + String fullPath = oid + "/" + cleanPath; + + if (data == null) { + data = new ByteArrayInputStream(EmptyArray.BYTE); + } + + // Write the file + try { + // Setup the metadata for the file + Document metadata = new Document(); + metadata.append("_oid", oid); + metadata.append("type", "file"); + + // Prepare the upload options + GridFSUploadOptions opt = (new GridFSUploadOptions()).metadata(metadata); + gridFSBucket.uploadFromStream(oid, data, opt); + } catch (Exception e) { + throw new RuntimeException(e); + } finally { + try { + data.close(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + } + + //-------------------------------------------------------------------------- + // + // File read / exists + // [Internal use, to be extended in future implementation] + // + //-------------------------------------------------------------------------- + + /** + * [Internal use, to be extended in future implementation] + * + * Get and return the stored data as a byte[] + * + * @param ObjectID of workspace + * @param filepath to use for the workspace + * + * @return the stored byte array of the file + **/ + @Override + public byte[] backend_fileRead(String oid, String filepath) { + InputStream buffer = backend_fileReadInputStream(oid, filepath); + byte[] ret = null; + try { + ret = IOUtils.toByteArray(buffer); + } catch (IOException e) { + throw new RuntimeException(e); + } finally { + try { + buffer.close(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + return ret; + } + + /** + * [Internal use, to be extended in future implementation] + * + * Get and return the stored data as a byte stream. + * + * This overwrite is useful for backends which supports this flow. + * Else it would simply be a wrapper over the non-stream version. + * + * @param ObjectID of workspace + * @param filepath to use for the workspace + * + * @return the stored byte array of the file + **/ + public InputStream backend_fileReadInputStream(String oid, String filepath) { + return gridFSBucket.openDownloadStream(oid + "/" + cleanFilePath(filepath)); + } + + @Override + public boolean backend_fileExist(String oid, String filepath) { + // Check against the full file path + return fullRawPathExist(oid + "/" + cleanFilePath(filepath)); + } + + @Override + public void backend_removeFile(String oid, String filepath) { + removeFilePathOnce(oid, cleanFilePath(filepath)); + } + + // Folder Pathing support + //-------------------------------------------------------------------------- + + /** + * [Internal use, to be extended in future implementation] + * + * Delete an existing path from the workspace. + * This recursively removes all file content under the given path prefix + * + * @param ObjectID of workspace + * @param folderPath in the workspace (note, folderPath is normalized to end with "/") + * + * @return the stored byte array of the file + **/ + public void backend_removeFolderPath(final String oid, final String folderPath) { + removeFilePathRecursively(oid, cleanFilePath(folderPath)); + } + + /** + * [Internal use, to be extended in future implementation] + * + * Validate the given folder path exists. + * + * @param ObjectID of workspace + * @param folderPath in the workspace (note, folderPath is normalized to end with "/") + * + * @return the stored byte array of the file + **/ + public boolean backend_folderPathExist(final String oid, final String folderPath) { + // Note that this passes if any of the files were created directly without folders + return prefixPathExist(oid, cleanFilePath(folderPath)); + } + + /** + * [Internal use, to be extended in future implementation] + * + * Automatically generate a given folder path if it does not exist + * + * @param ObjectID of workspace + * @param folderPath in the workspace (note, folderPath is normalized to end with "/") + * + * @return the stored byte array of the file + **/ + public void backend_ensureFolderPath(final String oid, final String folderPath) { + // Cleanup folderPath + String path = cleanFilePath(folderPath); + + // We setup a blank file with type root, this checks only for the anchor file + // if it does not exists, we will make it + if(!fullRawPathExist(oid+"/"+path)) { + setupAnchorFile(oid, path, "dir"); + } + } + + // Move support + //-------------------------------------------------------------------------- + + /** + * [Internal use, to be extended in future implementation] + * + * Move a given file within the system + * + * WARNING: Move operations are typically not "atomic" in nature, and can be unsafe where + * missing files / corrupted data can occur when executed concurrently with other operations. + * + * In general "S3-like" object storage will not safely support atomic move operations. + * Please use the `atomicMoveSupported()` function to validate if such operations are supported. + * + * This operation may in effect function as a rename + * If the destionation file exists, it will be overwritten + * + * @param ObjectID of workspace + * @param sourceFile + * @param destinationFile + */ + public void backend_moveFile(final String oid, final String sourceFile, + final String destinationFile) { + throw new RuntimeException("Missing backend implementation"); + } + + /** + * [Internal use, to be extended in future implementation] + * + * Move a given file within the system + * + * WARNING: Move operations are typically not "atomic" in nature, and can be unsafe where + * missing files / corrupted data can occur when executed concurrently with other operations. + * + * In general "S3-like" object storage will not safely support atomic move operations. + * Please use the `atomicMoveSupported()` function to validate if such operations are supported. + * + * Note that both source, and destionation folder will be normalized to include the "/" path. + * This operation may in effect function as a rename + * If the destionation folder exists with content, the result will be merged. With the sourceFolder files, overwriting on conflicts. + * + * @param ObjectID of workspace + * @param sourceFolder + * @param destinationFolder + * + */ + public void backend_moveFolderPath(final String oid, final String sourceFolder, + final String destinationFolder) { + throw new RuntimeException("Missing backend implementation"); + } + + // Copy support + //-------------------------------------------------------------------------- + + /** + * [Internal use, to be extended in future implementation] + * + * Copy a given file within the system + * + * WARNING: Copy operations are typically not "atomic" in nature, and can be unsafe where + * missing files / corrupted data can occur when executed concurrently with other operations. + * + * In general "S3-like" object storage will not safely support atomic copy operations. + * Please use the `atomicCopySupported()` function to validate if such operations are supported. + * + * This operation may in effect function as a rename + * If the destionation file exists, it will be overwritten + * + * @param ObjectID of workspace + * @param sourceFile + * @param destinationFile + */ + public void backend_copyFile(final String oid, final String sourceFile, + final String destinationFile) { + throw new RuntimeException("Missing backend implementation"); + } + + /** + * [Internal use, to be extended in future implementation] + * + * Copy a given file within the system + * + * WARNING: Copy operations are typically not "atomic" in nature, and can be unsafe where + * missing files / corrupted data can occur when executed concurrently with other operations. + * + * In general "S3-like" object storage will not safely support atomic Copy operations. + * Please use the `atomicCopySupported()` function to validate if such operations are supported. + * + * Note that both source, and destionation folder will be normalized to include the "/" path. + * This operation may in effect function as a rename + * If the destionation folder exists with content, the result will be merged. With the sourceFolder files, overwriting on conflicts. + * + * @param ObjectID of workspace + * @param sourceFolder + * @param destinationFolder + * + */ + public void backend_copyFolderPath(final String oid, final String sourceFolder, + final String destinationFolder) { + throw new RuntimeException("Missing backend implementation"); + } + + // + // Create and updated timestamp support + // + // Note that this feature does not have "normalized" support across + // backend implementation, and is provided "as-it-is" for applicable + // backend implementations. + // + //-------------------------------------------------------------------------- + + /** + * [Internal use, to be extended in future implementation] + + * The created timestamp of the map in ms, + * note that -1 means the current backend does not support this feature + * + * @param ObjectID of workspace + * @param filepath in the workspace to check + * + * @return DataObject created timestamp in ms + */ + public long backend_createdTimestamp(final String oid, final String filepath) { + return -1; + } + + /** + * [Internal use, to be extended in future implementation] + + * The modified timestamp of the map in ms, + * note that -1 means the current backend does not support this feature + * + * @param ObjectID of workspace + * @param filepath in the workspace to check + * + * @return DataObject created timestamp in ms + */ + public long backend_modifiedTimestamp(final String oid, final String filepath) { + return -1; + } + + /** + * List all the various files and folders found in the given folderPath + * + * @param ObjectID of workspace + * @param folderPath in the workspace (note, folderPath is normalized to end with "/") + * @param minDepth minimum depth count, before outputing the listing (uses a <= match) + * @param maxDepth maximum depth count, to stop the listing (-1 for infinite, uses a >= match) + * + * @return list of path strings - relative to the given folderPath (folders end with "/") + */ + @Override + public Set backend_getFileAndFolderPathSet(final String oid, final String folderPath, + final int minDepth, final int maxDepth) { + throw new RuntimeException("Missing backend implementation"); + } + } From 5dffda39fc862f95c3149a2190376c0b99480df9 Mon Sep 17 00:00:00 2001 From: Eugene Cheah Date: Thu, 4 Aug 2022 07:48:22 +0000 Subject: [PATCH 23/42] FileWorkspaceMap test --- .../picoded/dstack/mongodb/MongoDBStack.java | 2 + .../MongoDB_FileWorkspaceMap_test.java | 47 +++++++++++++++++++ 2 files changed, 49 insertions(+) create mode 100755 src/test/java/picoded/dstack/mongodb/MongoDB_FileWorkspaceMap_test.java diff --git a/src/main/java/picoded/dstack/mongodb/MongoDBStack.java b/src/main/java/picoded/dstack/mongodb/MongoDBStack.java index 58b373bd..6e835143 100644 --- a/src/main/java/picoded/dstack/mongodb/MongoDBStack.java +++ b/src/main/java/picoded/dstack/mongodb/MongoDBStack.java @@ -130,6 +130,8 @@ protected Core_DataStructure initDataStructure(String name, String type) { ret = new MongoDB_KeyValueMap(this, name); } else if (type.equalsIgnoreCase("KeyLongMap")) { ret = new MongoDB_KeyLongMap(this, name); + } else if (type.equalsIgnoreCase("FileWorkspaceMap")) { + ret = new MongoDB_FileWorkspaceMap(this, name); } // If datastrucutre initialized, setup name diff --git a/src/test/java/picoded/dstack/mongodb/MongoDB_FileWorkspaceMap_test.java b/src/test/java/picoded/dstack/mongodb/MongoDB_FileWorkspaceMap_test.java new file mode 100755 index 00000000..00b034eb --- /dev/null +++ b/src/test/java/picoded/dstack/mongodb/MongoDB_FileWorkspaceMap_test.java @@ -0,0 +1,47 @@ +package picoded.dstack.mongodb; + +import picoded.core.struct.*; +import picoded.dstack.*; +import picoded.dstack.struct.simple.*; + +/** + * ## Purpose + * This class is meant to test the MongoDB_FileWorkspaceMap implementation, + * and ensure that it passes all the test layed out in StructSImple_FileWorkspaceMap_test + * + */ +public class MongoDB_FileWorkspaceMap_test extends StructSimple_FileWorkspaceMap_test { + + // Hazelcast stack instance + protected static volatile MongoDBStack instance = null; + + // To override for implementation + //----------------------------------------------------- + + /// Impomentation constructor + public FileWorkspaceMap implementationConstructor() { + + // Initialize server + synchronized (MongoDB_FileWorkspaceMap_test.class) { + if (instance == null) { + // The default config uses localhost, 27017 + GenericConvertMap mongodbConfig = new GenericConvertHashMap<>(); + mongodbConfig.put("host", DStackTestConfig.MONGODB_HOST()); + mongodbConfig.put("port", DStackTestConfig.MONGODB_PORT()); + + // Use a random DB name + mongodbConfig.put("name", DStackTestConfig.randomTablePrefix()); + + GenericConvertMap stackConfig = new GenericConvertHashMap<>(); + stackConfig.put("name", "MongoDB_FileWorkspaceMap_test"); + stackConfig.put("mongodb", mongodbConfig); + + instance = new MongoDBStack(stackConfig); + } + } + + // Load the FileWorkspaceMap + return instance.fileWorkspaceMap(DStackTestConfig.randomTablePrefix()); + } + +} From 6eb69a8954c42a28a9071e826cf7f882f2db3d3f Mon Sep 17 00:00:00 2001 From: Eugene Cheah Date: Thu, 4 Aug 2022 07:48:30 +0000 Subject: [PATCH 24/42] Fix invalid @Override --- .../java/picoded/dstack/mongodb/MongoDB_FileWorkspaceMap.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/picoded/dstack/mongodb/MongoDB_FileWorkspaceMap.java b/src/main/java/picoded/dstack/mongodb/MongoDB_FileWorkspaceMap.java index 5f05b3dc..b60954e8 100755 --- a/src/main/java/picoded/dstack/mongodb/MongoDB_FileWorkspaceMap.java +++ b/src/main/java/picoded/dstack/mongodb/MongoDB_FileWorkspaceMap.java @@ -233,7 +233,6 @@ protected boolean prefixPathExist(String oid, String path) { /** * Setup an empty file, used for various use cases */ - @Override public void setupAnchorFile(String oid, String fullPath, String type) { // In general we will upload a blank file // with the relevent _oid, that can be easily lookedup From 386875aa1188bb6dfad079418e7da54009181137 Mon Sep 17 00:00:00 2001 From: Eugene Cheah Date: Thu, 4 Aug 2022 08:51:21 +0000 Subject: [PATCH 25/42] Polyfill basic copy / move operations --- .../dstack/core/Core_FileWorkspaceMap.java | 44 ++++++++++++++++--- 1 file changed, 38 insertions(+), 6 deletions(-) diff --git a/src/main/java/picoded/dstack/core/Core_FileWorkspaceMap.java b/src/main/java/picoded/dstack/core/Core_FileWorkspaceMap.java index 1c343bdc..028fe1a7 100755 --- a/src/main/java/picoded/dstack/core/Core_FileWorkspaceMap.java +++ b/src/main/java/picoded/dstack/core/Core_FileWorkspaceMap.java @@ -299,7 +299,8 @@ public void backend_ensureFolderPath(final String oid, final String folderPath) */ public void backend_moveFile(final String oid, final String sourceFile, final String destinationFile) { - throw new RuntimeException("Missing backend implementation"); + backend_copyFile(oid, sourceFile, destinationFile); + backend_removeFile(oid, sourceFile); } /** @@ -324,7 +325,23 @@ public void backend_moveFile(final String oid, final String sourceFile, */ public void backend_moveFolderPath(final String oid, final String sourceFolder, final String destinationFolder) { - throw new RuntimeException("Missing backend implementation"); + // Get the list of valid sub paths in the sourceFolder + Set subPath = backend_getFileAndFolderPathSet(oid, sourceFolder, -1, -1); + + // Lets sync up all the folders first + for(String dir : subPath) { + if(dir.endsWith("/")) { + backend_ensureFolderPath(oid, destinationFolder+subPath); + } + } + // Lets sync up all the files next + for(String file : subPath) { + if(!file.endsWith("/")) { + backend_moveFile(oid, sourceFolder+subPath, destinationFolder+subPath); + } + } + // Lets remove the original folders + backend_removeFolderPath(oid, sourceFolder); } // Copy support @@ -350,7 +367,7 @@ public void backend_moveFolderPath(final String oid, final String sourceFolder, */ public void backend_copyFile(final String oid, final String sourceFile, final String destinationFile) { - throw new RuntimeException("Missing backend implementation"); + backend_fileWriteInputStream(oid, destinationFile, backend_fileReadInputStream(oid, sourceFile)); } /** @@ -375,7 +392,21 @@ public void backend_copyFile(final String oid, final String sourceFile, */ public void backend_copyFolderPath(final String oid, final String sourceFolder, final String destinationFolder) { - throw new RuntimeException("Missing backend implementation"); + // Get the list of valid sub paths in the sourceFolder + Set subPath = backend_getFileAndFolderPathSet(oid, sourceFolder, -1, -1); + + // Lets sync up all the folders first + for(String dir : subPath) { + if(dir.endsWith("/")) { + backend_ensureFolderPath(oid, destinationFolder+subPath); + } + } + // Lets sync up all the files next + for(String file : subPath) { + if(!file.endsWith("/")) { + backend_copyFile(oid, sourceFolder+subPath, destinationFolder+subPath); + } + } } // @@ -421,7 +452,8 @@ public long backend_modifiedTimestamp(final String oid, final String filepath) { //-------------------------------------------------------------------------- /** - * Internal utility function used to filter a path set, and remove items that does not match + * Internal utility function used to filter a path set, and remove items that does not match. + * This is used to help filter raw results, from existing implementation * * - its folderPath prefix * - min/max depth @@ -429,7 +461,7 @@ public long backend_modifiedTimestamp(final String oid, final String filepath) { * * @param rawSet * @param folderPath - * @param minDepth + * @param minDepth (0 = all items, 1 = must be in atleast a folder, 2 = folder, inside a folder) * @param maxDepth * @param pathType (0 = any, 1 = file, 2 = folder) * @return From aa19c456645f8af85577bfdcf47379acdac896c2 Mon Sep 17 00:00:00 2001 From: Eugene Cheah Date: Thu, 4 Aug 2022 08:51:46 +0000 Subject: [PATCH 26/42] MongoDb file listing support --- .../mongodb/MongoDB_FileWorkspaceMap.java | 166 +++++++----------- 1 file changed, 63 insertions(+), 103 deletions(-) diff --git a/src/main/java/picoded/dstack/mongodb/MongoDB_FileWorkspaceMap.java b/src/main/java/picoded/dstack/mongodb/MongoDB_FileWorkspaceMap.java index b60954e8..2619342b 100755 --- a/src/main/java/picoded/dstack/mongodb/MongoDB_FileWorkspaceMap.java +++ b/src/main/java/picoded/dstack/mongodb/MongoDB_FileWorkspaceMap.java @@ -2,6 +2,7 @@ // Java imports import java.util.ArrayList; +import java.util.Arrays; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -505,108 +506,7 @@ public void backend_ensureFolderPath(final String oid, final String folderPath) } } - // Move support //-------------------------------------------------------------------------- - - /** - * [Internal use, to be extended in future implementation] - * - * Move a given file within the system - * - * WARNING: Move operations are typically not "atomic" in nature, and can be unsafe where - * missing files / corrupted data can occur when executed concurrently with other operations. - * - * In general "S3-like" object storage will not safely support atomic move operations. - * Please use the `atomicMoveSupported()` function to validate if such operations are supported. - * - * This operation may in effect function as a rename - * If the destionation file exists, it will be overwritten - * - * @param ObjectID of workspace - * @param sourceFile - * @param destinationFile - */ - public void backend_moveFile(final String oid, final String sourceFile, - final String destinationFile) { - throw new RuntimeException("Missing backend implementation"); - } - - /** - * [Internal use, to be extended in future implementation] - * - * Move a given file within the system - * - * WARNING: Move operations are typically not "atomic" in nature, and can be unsafe where - * missing files / corrupted data can occur when executed concurrently with other operations. - * - * In general "S3-like" object storage will not safely support atomic move operations. - * Please use the `atomicMoveSupported()` function to validate if such operations are supported. - * - * Note that both source, and destionation folder will be normalized to include the "/" path. - * This operation may in effect function as a rename - * If the destionation folder exists with content, the result will be merged. With the sourceFolder files, overwriting on conflicts. - * - * @param ObjectID of workspace - * @param sourceFolder - * @param destinationFolder - * - */ - public void backend_moveFolderPath(final String oid, final String sourceFolder, - final String destinationFolder) { - throw new RuntimeException("Missing backend implementation"); - } - - // Copy support - //-------------------------------------------------------------------------- - - /** - * [Internal use, to be extended in future implementation] - * - * Copy a given file within the system - * - * WARNING: Copy operations are typically not "atomic" in nature, and can be unsafe where - * missing files / corrupted data can occur when executed concurrently with other operations. - * - * In general "S3-like" object storage will not safely support atomic copy operations. - * Please use the `atomicCopySupported()` function to validate if such operations are supported. - * - * This operation may in effect function as a rename - * If the destionation file exists, it will be overwritten - * - * @param ObjectID of workspace - * @param sourceFile - * @param destinationFile - */ - public void backend_copyFile(final String oid, final String sourceFile, - final String destinationFile) { - throw new RuntimeException("Missing backend implementation"); - } - - /** - * [Internal use, to be extended in future implementation] - * - * Copy a given file within the system - * - * WARNING: Copy operations are typically not "atomic" in nature, and can be unsafe where - * missing files / corrupted data can occur when executed concurrently with other operations. - * - * In general "S3-like" object storage will not safely support atomic Copy operations. - * Please use the `atomicCopySupported()` function to validate if such operations are supported. - * - * Note that both source, and destionation folder will be normalized to include the "/" path. - * This operation may in effect function as a rename - * If the destionation folder exists with content, the result will be merged. With the sourceFolder files, overwriting on conflicts. - * - * @param ObjectID of workspace - * @param sourceFolder - * @param destinationFolder - * - */ - public void backend_copyFolderPath(final String oid, final String sourceFolder, - final String destinationFolder) { - throw new RuntimeException("Missing backend implementation"); - } - // // Create and updated timestamp support // @@ -646,6 +546,12 @@ public long backend_modifiedTimestamp(final String oid, final String filepath) { return -1; } + //-------------------------------------------------------------------------- + // + // Query, and listing support + // + //-------------------------------------------------------------------------- + /** * List all the various files and folders found in the given folderPath * @@ -657,9 +563,63 @@ public long backend_modifiedTimestamp(final String oid, final String filepath) { * @return list of path strings - relative to the given folderPath (folders end with "/") */ @Override - public Set backend_getFileAndFolderPathSet(final String oid, final String folderPath, + public Set backend_getFileAndFolderPathSet(final String oid, String folderPath, final int minDepth, final int maxDepth) { - throw new RuntimeException("Missing backend implementation"); + + // Lets build the query for the "root file" + Bson query = null; + + // The fulle prefix path + String fullPrefixPath = oid+"/"; + + // Lets build the query, for fetchign the relevent items + if( folderPath == null || folderPath.equals("/") || folderPath.isEmpty() ) { + // Handles query for all folder paths + query = Filters.eq("metadata._oid", oid); + } else { + // Cleanup the path + folderPath = cleanFilePath(folderPath); + fullPrefixPath = fullPrefixPath+folderPath; + + // Remove matching path + query = Filters.and( + Filters.eq("metadata._oid", oid), + Filters.regex("filename", "^"+Pattern.quote(folderPath)+".*") + ); + } + + // The return set + Set ret = new HashSet<>(); + + // Lets prepare the search + GridFSFindIterable search = gridFSBucket.find(query); + + // Lets iterate the search result, and return true on an item + try (MongoCursor cursor = search.iterator()) { + while (cursor.hasNext()) { + // Get the fileobj and filename + GridFSFile fileObj = cursor.next(); + String fullFilename = fileObj.getFilename(); + + // Remove the oid prefix + String filepath = fullFilename.substring( fullPrefixPath.length() ); + + // Register the validpath + ret.add(filepath); + + // Lets split the filepath + String[] filepathArr = filepath.split("/"); + List filepathList = Arrays.asList(filepathArr); + + // Lets handle parent folders + for(int i=1+Math.max(minDepth,0); i<(filepathArr.length-1); ++i) { + ret.add( String.join("/", filepathList.subList(0,i)+"/" ) ); + } + } + } + + // Filter and return the final set + return backend_filtterPathSet( ret, folderPath, minDepth, maxDepth, 0); } } From 442157b11b3c68e8338b2ef20c3ce91296044730 Mon Sep 17 00:00:00 2001 From: Eugene Cheah Date: Thu, 4 Aug 2022 09:22:02 +0000 Subject: [PATCH 27/42] Modified timestamp support --- .../mongodb/MongoDB_FileWorkspaceMap.java | 21 ++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/src/main/java/picoded/dstack/mongodb/MongoDB_FileWorkspaceMap.java b/src/main/java/picoded/dstack/mongodb/MongoDB_FileWorkspaceMap.java index 2619342b..3259cb6c 100755 --- a/src/main/java/picoded/dstack/mongodb/MongoDB_FileWorkspaceMap.java +++ b/src/main/java/picoded/dstack/mongodb/MongoDB_FileWorkspaceMap.java @@ -13,6 +13,7 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.sql.Date; // JavaCommons imports import picoded.core.common.EmptyArray; @@ -528,7 +529,9 @@ public void backend_ensureFolderPath(final String oid, final String folderPath) * @return DataObject created timestamp in ms */ public long backend_createdTimestamp(final String oid, final String filepath) { - return -1; + + // Currently only modified timestamp is supported + return backend_modifiedTimestamp(oid, filepath); } /** @@ -543,6 +546,22 @@ public long backend_createdTimestamp(final String oid, final String filepath) { * @return DataObject created timestamp in ms */ public long backend_modifiedTimestamp(final String oid, final String filepath) { + // Lets build the query for the "root file" + Bson query = Filters.eq("filename", fullpath); + + // Lets prepare the search + GridFSFindIterable search = gridFSBucket.find(query).limit(1); + + // Lets iterate the search result, and return true on an item + try (MongoCursor cursor = search.iterator()) { + if (cursor.hasNext()) { + GridFSFile fileObj = cursor.next(); + Date uploadDate = fileObj.getUploadDate(); + return uploadDate.getTime(); + } + } + + // Fail, as the search found no iterations return -1; } From 1366b1bf3fea26d1c812baa0a1607e2e5968e486 Mon Sep 17 00:00:00 2001 From: Eugene Cheah Date: Thu, 4 Aug 2022 09:25:42 +0000 Subject: [PATCH 28/42] Fixing compile issues --- .../picoded/dstack/mongodb/MongoDB_FileWorkspaceMap.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/main/java/picoded/dstack/mongodb/MongoDB_FileWorkspaceMap.java b/src/main/java/picoded/dstack/mongodb/MongoDB_FileWorkspaceMap.java index 3259cb6c..22f0a803 100755 --- a/src/main/java/picoded/dstack/mongodb/MongoDB_FileWorkspaceMap.java +++ b/src/main/java/picoded/dstack/mongodb/MongoDB_FileWorkspaceMap.java @@ -547,7 +547,7 @@ public long backend_createdTimestamp(final String oid, final String filepath) { */ public long backend_modifiedTimestamp(final String oid, final String filepath) { // Lets build the query for the "root file" - Bson query = Filters.eq("filename", fullpath); + Bson query = Filters.eq("filename", cleanFilePath(filepath)); // Lets prepare the search GridFSFindIterable search = gridFSBucket.find(query).limit(1); @@ -556,8 +556,7 @@ public long backend_modifiedTimestamp(final String oid, final String filepath) { try (MongoCursor cursor = search.iterator()) { if (cursor.hasNext()) { GridFSFile fileObj = cursor.next(); - Date uploadDate = fileObj.getUploadDate(); - return uploadDate.getTime(); + return fileObj.getUploadDate().getTime(); } } From 424e96fbb5dd22462e718276753c85341ac1d1a0 Mon Sep 17 00:00:00 2001 From: Eugene Cheah Date: Thu, 4 Aug 2022 10:18:25 +0000 Subject: [PATCH 29/42] Fixing upload pathing --- .../dstack/mongodb/MongoDB_FileWorkspaceMap.java | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/main/java/picoded/dstack/mongodb/MongoDB_FileWorkspaceMap.java b/src/main/java/picoded/dstack/mongodb/MongoDB_FileWorkspaceMap.java index 22f0a803..638ca247 100755 --- a/src/main/java/picoded/dstack/mongodb/MongoDB_FileWorkspaceMap.java +++ b/src/main/java/picoded/dstack/mongodb/MongoDB_FileWorkspaceMap.java @@ -290,7 +290,7 @@ protected void removeFilePathRecursively(String oid, String path) { /** * Utility function used, to remove a specific file **/ - protected boolean removeFilePathOnce(String oid, String path) { + protected boolean removeFilePath(String oid, String path) { // Lets build the query for the "root file" Bson query = null; @@ -300,16 +300,16 @@ protected boolean removeFilePathOnce(String oid, String path) { // Remove matching path query = Filters.eq("filename", oid+"/"+path); - // Lets prepare the search - GridFSFindIterable search = gridFSBucket.find(query).limit(1); + // Lets prepare the search (removes all versions) + GridFSFindIterable search = gridFSBucket.find(query); // Lets iterate the search result, and return true on an item try (MongoCursor cursor = search.iterator()) { - if (cursor.hasNext()) { + while (cursor.hasNext()) { GridFSFile fileObj = cursor.next(); gridFSBucket.delete(fileObj.getId()); - return true; } + return true; } // removal didn't occur @@ -378,7 +378,7 @@ public void backend_fileWriteInputStream(String oid, String filepath, InputStrea // Prepare the upload options GridFSUploadOptions opt = (new GridFSUploadOptions()).metadata(metadata); - gridFSBucket.uploadFromStream(oid, data, opt); + gridFSBucket.uploadFromStream(fullPath, data, opt); } catch (Exception e) { throw new RuntimeException(e); } finally { @@ -450,7 +450,7 @@ public boolean backend_fileExist(String oid, String filepath) { @Override public void backend_removeFile(String oid, String filepath) { - removeFilePathOnce(oid, cleanFilePath(filepath)); + removeFilePath(oid, cleanFilePath(filepath)); } // Folder Pathing support From 717a377a01b7e98c4c9d9bcc548d98d0cc3a1363 Mon Sep 17 00:00:00 2001 From: Eugene Cheah Date: Thu, 4 Aug 2022 10:20:08 +0000 Subject: [PATCH 30/42] Fixing rm flag status --- .../picoded/dstack/mongodb/MongoDB_FileWorkspaceMap.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/main/java/picoded/dstack/mongodb/MongoDB_FileWorkspaceMap.java b/src/main/java/picoded/dstack/mongodb/MongoDB_FileWorkspaceMap.java index 638ca247..40a34bd0 100755 --- a/src/main/java/picoded/dstack/mongodb/MongoDB_FileWorkspaceMap.java +++ b/src/main/java/picoded/dstack/mongodb/MongoDB_FileWorkspaceMap.java @@ -304,16 +304,18 @@ protected boolean removeFilePath(String oid, String path) { GridFSFindIterable search = gridFSBucket.find(query); // Lets iterate the search result, and return true on an item + boolean rmFlag = false; try (MongoCursor cursor = search.iterator()) { while (cursor.hasNext()) { GridFSFile fileObj = cursor.next(); gridFSBucket.delete(fileObj.getId()); + + rmFlag = true; } - return true; } - // removal didn't occur - return false; + // Return the remove status + return rmFlag; } //-------------------------------------------------------------------------- From 1aa344cc5eff313d6947295ad46d9e1b6592a91e Mon Sep 17 00:00:00 2001 From: Eugene Cheah Date: Thu, 4 Aug 2022 11:42:50 +0000 Subject: [PATCH 31/42] Fixing file path search regex --- .../picoded/dstack/mongodb/MongoDB_FileWorkspaceMap.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/main/java/picoded/dstack/mongodb/MongoDB_FileWorkspaceMap.java b/src/main/java/picoded/dstack/mongodb/MongoDB_FileWorkspaceMap.java index 40a34bd0..66d923b8 100755 --- a/src/main/java/picoded/dstack/mongodb/MongoDB_FileWorkspaceMap.java +++ b/src/main/java/picoded/dstack/mongodb/MongoDB_FileWorkspaceMap.java @@ -318,6 +318,10 @@ protected boolean removeFilePath(String oid, String path) { return rmFlag; } + protected void performFileCleanup(String oid, String path) { + + } + //-------------------------------------------------------------------------- // // File write @@ -604,7 +608,7 @@ public Set backend_getFileAndFolderPathSet(final String oid, String fold // Remove matching path query = Filters.and( Filters.eq("metadata._oid", oid), - Filters.regex("filename", "^"+Pattern.quote(folderPath)+".*") + Filters.regex("filename", "^"+Pattern.quote(fullPrefixPath)+".*") ); } From 610efd08ecfc73c46d85eab8a5ea635fae5cbdfd Mon Sep 17 00:00:00 2001 From: Eugene Cheah Date: Thu, 4 Aug 2022 12:02:20 +0000 Subject: [PATCH 32/42] Tweak removeFilePathRecursively handling of empty path --- .../java/picoded/dstack/mongodb/MongoDB_FileWorkspaceMap.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/picoded/dstack/mongodb/MongoDB_FileWorkspaceMap.java b/src/main/java/picoded/dstack/mongodb/MongoDB_FileWorkspaceMap.java index 66d923b8..6aa7a219 100755 --- a/src/main/java/picoded/dstack/mongodb/MongoDB_FileWorkspaceMap.java +++ b/src/main/java/picoded/dstack/mongodb/MongoDB_FileWorkspaceMap.java @@ -261,7 +261,7 @@ protected void removeFilePathRecursively(String oid, String path) { // Lets build the query for the "root file" Bson query = null; - if( path == null ) { + if( path == null || path.equals("/") || path.isEmpty() ) { // Remove everything under the oid query = Filters.eq("metadata._oid", oid); } else { From 8bb20a7c3b51be0dc07ac8a80d14ee5d050e5713 Mon Sep 17 00:00:00 2001 From: Eugene Cheah Date: Fri, 5 Aug 2022 07:15:04 +0000 Subject: [PATCH 33/42] Core_FileWorkspaceMap enforcing system setup on write --- .../dstack/core/Core_FileWorkspace.java | 80 +++++++++++++++---- .../dstack/core/Core_FileWorkspaceMap.java | 33 ++++++++ 2 files changed, 99 insertions(+), 14 deletions(-) diff --git a/src/main/java/picoded/dstack/core/Core_FileWorkspace.java b/src/main/java/picoded/dstack/core/Core_FileWorkspace.java index 9297d3a1..420b3915 100755 --- a/src/main/java/picoded/dstack/core/Core_FileWorkspace.java +++ b/src/main/java/picoded/dstack/core/Core_FileWorkspace.java @@ -37,6 +37,12 @@ public class Core_FileWorkspace implements FileWorkspace { **/ protected String _oid = null; + /** + * Boolean flag, if true, indicates that the current FileWorkspace is uninitialized + * if so, we will setup the workspace if needed. + */ + protected boolean _isUninitialized = false; + // Constructor //---------------------------------------------- @@ -62,6 +68,7 @@ public Core_FileWorkspace(Core_FileWorkspaceMap inMain, String inOID) { // Issue a GUID if (_oid == null) { _oid = GUID.base58(); + _isUninitialized = true; } if (_oid.length() < 4) { @@ -97,6 +104,15 @@ public String _oid() { public void setupWorkspace() { main.setupWorkspace(_oid()); } + + /** + * Calls setupWorkspace if _isUninitialized is true + */ + protected void setupUninitializedWorkspace() { + if( _isUninitialized ) { + setupWorkspace(); + } + } // File / Folder string normalization //-------------------------------------------------------------------------- @@ -150,6 +166,9 @@ private static String normalizeFolderPathString(final String folderPath) { * @return true, if file exists (and writable), false if it does not. Possible a folder */ public boolean fileExist(final String filepath) { + if( _isUninitialized ) { + return false; + } return main.backend_fileExist(_oid, normalizeFilePathString(filepath)); } @@ -159,6 +178,9 @@ public boolean fileExist(final String filepath) { * @param filepath in the workspace to delete */ public void removeFile(final String filepath) { + if( _isUninitialized ) { + return; + } main.backend_removeFile(_oid, normalizeFilePathString(filepath)); } @@ -173,6 +195,9 @@ public void removeFile(final String filepath) { * @return the file contents, null if file does not exists */ public InputStream readInputStream(final String filepath) { + if( _isUninitialized ) { + return null; + } return main.backend_fileReadInputStream(_oid, normalizeFilePathString(filepath)); } @@ -185,6 +210,7 @@ public InputStream readInputStream(final String filepath) { * @param data the content to write to the file **/ public void writeInputStream(final String filepath, final InputStream data) { + setupUninitializedWorkspace(); main.backend_fileWriteInputStream(_oid, normalizeFilePathString(filepath), data); } @@ -199,6 +225,9 @@ public void writeInputStream(final String filepath, final InputStream data) { * @return the file contents, null if file does not exists */ public byte[] readByteArray(final String filepath) { + if( _isUninitialized ) { + return null; + } return main.backend_fileRead(_oid, normalizeFilePathString(filepath)); } @@ -211,6 +240,7 @@ public byte[] readByteArray(final String filepath) { * @param data the content to write to the file **/ public void writeByteArray(final String filepath, final byte[] data) { + setupUninitializedWorkspace(); main.backend_fileWrite(_oid, normalizeFilePathString(filepath), data); } @@ -224,20 +254,8 @@ public void writeByteArray(final String filepath, final byte[] data) { * @param data the content to write to the file **/ public void appendByteArray(final String filepath, final byte[] data) { - // Normalize the file path - String path = normalizeFilePathString(filepath); - - // Get existing data - byte[] read = readByteArray(path); - if (read == null) { - writeByteArray(path, data); - } - - // Append new data to existing data - byte[] jointData = ArrayConv.addAll(read, data); - - // Write the new joint data - writeByteArray(path, jointData); + setupUninitializedWorkspace(); + main.backend_fileAppendByteArray(_oid, normalizeFilePathString(filepath), data); } // Folder Pathing support @@ -250,6 +268,9 @@ public void appendByteArray(final String filepath, final byte[] data) { * @param folderPath in the workspace (note, folderPath is normalized to end with "/") */ public void removeFolderPath(final String folderPath) { + if( _isUninitialized ) { + return; + } main.backend_removeFolderPath(_oid, normalizeFolderPathString(folderPath)); } @@ -260,6 +281,9 @@ public void removeFolderPath(final String folderPath) { * @return true if folderPath is valid */ public boolean folderPathExist(final String folderPath) { + if( _isUninitialized ) { + return false; + } return main.backend_folderPathExist(_oid, normalizeFolderPathString(folderPath)); } @@ -269,6 +293,7 @@ public boolean folderPathExist(final String folderPath) { * @param folderPath in the workspace (note, folderPath is normalized to end with "/") */ public void ensureFolderPath(final String folderPath) { + setupUninitializedWorkspace(); main.backend_ensureFolderPath(_oid, normalizeFolderPathString(folderPath)); } @@ -290,6 +315,9 @@ public void ensureFolderPath(final String folderPath) { * @return DataObject created timestamp in ms */ public long createdTimestamp(final String filepath) { + if( _isUninitialized ) { + return -1; + } return main.backend_createdTimestamp(_oid, normalizeFilePathString(filepath)); } @@ -302,6 +330,9 @@ public long createdTimestamp(final String filepath) { * @return DataObject created timestamp in ms */ public long modifiedTimestamp(final String filepath) { + if( _isUninitialized ) { + return -1; + } return main.backend_modifiedTimestamp(_oid, normalizeFilePathString(filepath)); } @@ -324,6 +355,9 @@ public long modifiedTimestamp(final String filepath) { * @param destinationFile */ public void moveFile(final String sourceFile, final String destinationFile) { + if( _isUninitialized ) { + return; + } main.backend_moveFile(_oid, normalizeFilePathString(sourceFile), normalizeFilePathString(destinationFile)); } @@ -345,6 +379,9 @@ public void moveFile(final String sourceFile, final String destinationFile) { * @param destinationFolder */ public void moveFolderPath(final String sourceFolder, final String destinationFolder) { + if( _isUninitialized ) { + return; + } main.backend_moveFolderPath(_oid, normalizeFolderPathString(sourceFolder), normalizeFolderPathString(destinationFolder)); } @@ -368,6 +405,9 @@ public void moveFolderPath(final String sourceFolder, final String destinationFo * @param destinationFile */ public void copyFile(final String sourceFile, final String destinationFile) { + if( _isUninitialized ) { + return; + } main.backend_copyFile(_oid, normalizeFilePathString(sourceFile), normalizeFilePathString(destinationFile)); } @@ -389,6 +429,9 @@ public void copyFile(final String sourceFile, final String destinationFile) { * @param destinationFolder */ public void copyFolderPath(final String sourceFolder, final String destinationFolder) { + if( _isUninitialized ) { + return; + } main.backend_copyFolderPath(_oid, normalizeFolderPathString(sourceFolder), normalizeFolderPathString(destinationFolder)); } @@ -406,6 +449,9 @@ public void copyFolderPath(final String sourceFolder, final String destinationFo */ public Set getFileAndFolderPathSet(final String folderPath, final int minDepth, final int maxDepth) { + if( _isUninitialized ) { + return new HashSet<>(); + } return main.backend_getFileAndFolderPathSet(_oid, normalizeFolderPathString(folderPath), minDepth, maxDepth); } @@ -419,6 +465,9 @@ public Set getFileAndFolderPathSet(final String folderPath, final int mi * @return list of path strings - relative to the given folderPath */ public Set getFilePathSet(final String folderPath, final int minDepth, final int maxDepth) { + if( _isUninitialized ) { + return new HashSet<>(); + } return main.backend_getFilePathSet(_oid, normalizeFolderPathString(folderPath), minDepth, maxDepth); } @@ -433,6 +482,9 @@ public Set getFilePathSet(final String folderPath, final int minDepth, f */ public Set getFolderPathSet(final String folderPath, final int minDepth, final int maxDepth) { + if( _isUninitialized ) { + return new HashSet<>(); + } return main.backend_getFolderPathSet(_oid, normalizeFolderPathString(folderPath), minDepth, maxDepth); } diff --git a/src/main/java/picoded/dstack/core/Core_FileWorkspaceMap.java b/src/main/java/picoded/dstack/core/Core_FileWorkspaceMap.java index 028fe1a7..45fd3e8e 100755 --- a/src/main/java/picoded/dstack/core/Core_FileWorkspaceMap.java +++ b/src/main/java/picoded/dstack/core/Core_FileWorkspaceMap.java @@ -1,5 +1,7 @@ package picoded.dstack.core; +import picoded.core.conv.ArrayConv; + // Java imports // Picoded imports @@ -230,6 +232,37 @@ public void backend_fileWriteInputStream(final String oid, final String filepath backend_fileWrite(oid, filepath, rawBytes); } + /** + * [Internal use, to be extended in future implementation] + * + * Writes the full byte array of a file in the backend + * + * This overwrite is useful for backends which supports this flow. + * Else it would simply be a wrapper over the non-stream version. + * + * @param ObjectID of workspace + * @param filepath to use for the workspace + * @param data to write the file with + **/ + public void backend_fileAppendByteArray(final String oid, final String filepath, + final byte[] data) { + + // Get the existing byte array + byte[] read = backend_fileRead(oid, filepath); + + // Just write it as it is (read is null) + if (read == null) { + backend_fileWrite(oid, filepath, data); + return; + } + + // Append new data to existing data + byte[] jointData = ArrayConv.addAll(read, data); + + // Write the new joint data + backend_fileWrite(oid, filepath, jointData); + } + // Folder Pathing support //-------------------------------------------------------------------------- From 4b5a066903b9ecc11dc0e5e31192eb58fc2a8ac4 Mon Sep 17 00:00:00 2001 From: Eugene Cheah Date: Fri, 5 Aug 2022 07:28:16 +0000 Subject: [PATCH 34/42] Fix invalid _isUninitialized setup flag --- src/main/java/picoded/dstack/core/Core_FileWorkspace.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/picoded/dstack/core/Core_FileWorkspace.java b/src/main/java/picoded/dstack/core/Core_FileWorkspace.java index 420b3915..728e3810 100755 --- a/src/main/java/picoded/dstack/core/Core_FileWorkspace.java +++ b/src/main/java/picoded/dstack/core/Core_FileWorkspace.java @@ -103,6 +103,7 @@ public String _oid() { @Override public void setupWorkspace() { main.setupWorkspace(_oid()); + _isUninitialized = false; } /** From 5a3fdfc118af66f28fa7d15fd70bfed4e78126f7 Mon Sep 17 00:00:00 2001 From: Eugene Cheah Date: Fri, 5 Aug 2022 07:59:34 +0000 Subject: [PATCH 35/42] Fixing filter typo --- .../dstack/core/Core_FileWorkspaceMap.java | 2 +- .../simple/FileSimple_FileWorkspaceMap.java | 2 +- .../dstack/jsql/JSql_FileWorkspaceMap.java | 2 +- .../simple/StructSimple_FileWorkspaceMap.java | 2 +- src/test/README.md | 22 +++++++++++++++++++ 5 files changed, 26 insertions(+), 4 deletions(-) create mode 100644 src/test/README.md diff --git a/src/main/java/picoded/dstack/core/Core_FileWorkspaceMap.java b/src/main/java/picoded/dstack/core/Core_FileWorkspaceMap.java index 45fd3e8e..8c171e3c 100755 --- a/src/main/java/picoded/dstack/core/Core_FileWorkspaceMap.java +++ b/src/main/java/picoded/dstack/core/Core_FileWorkspaceMap.java @@ -499,7 +499,7 @@ public long backend_modifiedTimestamp(final String oid, final String filepath) { * @param pathType (0 = any, 1 = file, 2 = folder) * @return */ - protected Set backend_filtterPathSet(final Set rawSet, final String folderPath, + protected Set backend_filterPathSet(final Set rawSet, final String folderPath, final int minDepth, final int maxDepth, final int pathType) { // Normalize the folder path diff --git a/src/main/java/picoded/dstack/file/simple/FileSimple_FileWorkspaceMap.java b/src/main/java/picoded/dstack/file/simple/FileSimple_FileWorkspaceMap.java index e1961087..58d7af31 100755 --- a/src/main/java/picoded/dstack/file/simple/FileSimple_FileWorkspaceMap.java +++ b/src/main/java/picoded/dstack/file/simple/FileSimple_FileWorkspaceMap.java @@ -624,7 +624,7 @@ public Set backend_getFileAndFolderPathSet(final String oid, final Strin recusively_populatePathSet(retSet, folderObj, "", maxDepth); // Return with minDepth filtering - return backend_filtterPathSet(retSet, "", minDepth, maxDepth, 0); + return backend_filterPathSet(retSet, "", minDepth, maxDepth, 0); } } diff --git a/src/main/java/picoded/dstack/jsql/JSql_FileWorkspaceMap.java b/src/main/java/picoded/dstack/jsql/JSql_FileWorkspaceMap.java index 0916a376..c59d76d6 100755 --- a/src/main/java/picoded/dstack/jsql/JSql_FileWorkspaceMap.java +++ b/src/main/java/picoded/dstack/jsql/JSql_FileWorkspaceMap.java @@ -654,7 +654,7 @@ public Set backend_getFileAndFolderPathSet(final String oid, final Strin } // Filter and return it accordingly - return backend_filtterPathSet(rawSet, folderPath, minDepth, maxDepth, 0); + return backend_filterPathSet(rawSet, folderPath, minDepth, maxDepth, 0); } } diff --git a/src/main/java/picoded/dstack/struct/simple/StructSimple_FileWorkspaceMap.java b/src/main/java/picoded/dstack/struct/simple/StructSimple_FileWorkspaceMap.java index a5c4e6c0..67e742cd 100755 --- a/src/main/java/picoded/dstack/struct/simple/StructSimple_FileWorkspaceMap.java +++ b/src/main/java/picoded/dstack/struct/simple/StructSimple_FileWorkspaceMap.java @@ -614,7 +614,7 @@ public Set backend_getFileAndFolderPathSet(final String oid, final Strin } // Return a filtered set - return backend_filtterPathSet(workspace.keySet(), searchPath, minDepth, maxDepth, 0); + return backend_filterPathSet(workspace.keySet(), searchPath, minDepth, maxDepth, 0); } finally { accessLock.readLock().unlock(); } diff --git a/src/test/README.md b/src/test/README.md new file mode 100644 index 00000000..cff49382 --- /dev/null +++ b/src/test/README.md @@ -0,0 +1,22 @@ +# Docker Commands to setup "testing DB" locally + +## MongoDB + +DB Setup + +``` +sudo docker run --name dstack-mongodb -p 27017:27017 -d mongo:5 +``` + +Run Tests + +``` +./gradlew test -Ptest_all --tests picoded.dstack.mongodb.* +``` + +Cleanup DB + +``` +sudo docker stop dstack-mongodb; +sudo docker rm dstack-mongodb; +``` \ No newline at end of file From ec49c86f47ab996849787a2e660efb0b3ee983dd Mon Sep 17 00:00:00 2001 From: Eugene Cheah Date: Fri, 5 Aug 2022 07:59:45 +0000 Subject: [PATCH 36/42] WIP fixing mongodb --- .../mongodb/MongoDB_FileWorkspaceMap.java | 80 +++++++------------ 1 file changed, 27 insertions(+), 53 deletions(-) diff --git a/src/main/java/picoded/dstack/mongodb/MongoDB_FileWorkspaceMap.java b/src/main/java/picoded/dstack/mongodb/MongoDB_FileWorkspaceMap.java index 6aa7a219..3e16cfe2 100755 --- a/src/main/java/picoded/dstack/mongodb/MongoDB_FileWorkspaceMap.java +++ b/src/main/java/picoded/dstack/mongodb/MongoDB_FileWorkspaceMap.java @@ -8,6 +8,9 @@ import java.util.Map; import java.util.Set; import java.util.regex.Pattern; + +import javax.management.RuntimeErrorException; + import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -136,7 +139,7 @@ public boolean backend_workspaceExist(String oid) { /** * Setup the current fileWorkspace within the fileWorkspaceMap, * - * This ensures the workspace _oid is registered within the map, + * This ensures the workspace oid is registered within the map, * even if there is 0 files. * * Does not throw any error if workspace was previously setup @@ -167,23 +170,6 @@ public void backend_workspaceRemove(String oid) { // //-------------------------------------------------------------------------- - /** - * Given a filepath, ensure a clean filepath (without starting "/") - */ - protected static String cleanFilePath(final String filepath) { - // Note that the FileUtil.normalize step is not needed, as - // this is already done in the Core_FileWorkspaceMap - // --- - // String cleanFilePath = FileUtil.normalize(filepath); - - // Cleanup the file apth - String cleanFilePath = filepath; - while (cleanFilePath.startsWith("/")) { - cleanFilePath = cleanFilePath.substring(1); - } - return cleanFilePath; - } - /** Utility function used, to check if a workspace, or file exists **/ protected boolean fullRawPathExist(String fullpath) { // Lets build the query for the "root file" @@ -195,7 +181,7 @@ protected boolean fullRawPathExist(String fullpath) { // Lets iterate the search result, and return true on an item try (MongoCursor cursor = search.iterator()) { if (cursor.hasNext()) { - // ret.add(cursor.next().getString("_oid")); + // ret.add(cursor.next().getString("oid")); return true; } } @@ -209,12 +195,9 @@ protected boolean prefixPathExist(String oid, String path) { // Lets build the query for the "root file" Bson query = null; - // Cleanup the path - path = cleanFilePath(path); - // Remove matching path query = Filters.and( - Filters.eq("metadata._oid", oid), + Filters.eq("metadata.oid", oid), Filters.regex("filename", "^"+Pattern.quote(oid+"/"+path)+".*") ); @@ -237,13 +220,13 @@ protected boolean prefixPathExist(String oid, String path) { */ public void setupAnchorFile(String oid, String fullPath, String type) { // In general we will upload a blank file - // with the relevent _oid, that can be easily lookedup + // with the relevent oid, that can be easily lookedup // // This is done using a closable input stream, with an empty byte array try (ByteArrayInputStream emptyStream = new ByteArrayInputStream(EmptyArray.BYTE)) { // Setup the metadata for the file Document metadata = new Document(); - metadata.append("_oid", oid); + metadata.append("oid", oid); metadata.append("type", type); // Prepare the upload options @@ -263,14 +246,11 @@ protected void removeFilePathRecursively(String oid, String path) { if( path == null || path.equals("/") || path.isEmpty() ) { // Remove everything under the oid - query = Filters.eq("metadata._oid", oid); + query = Filters.eq("metadata.oid", oid); } else { - // Cleanup the path - path = cleanFilePath(path); - // Remove matching path query = Filters.and( - Filters.eq("metadata._oid", oid), + Filters.eq("metadata.oid", oid), Filters.regex("filename", "^"+Pattern.quote(oid+"/"+path)+".*") ); } @@ -294,9 +274,6 @@ protected boolean removeFilePath(String oid, String path) { // Lets build the query for the "root file" Bson query = null; - // Cleanup the path - path = cleanFilePath(path); - // Remove matching path query = Filters.eq("filename", oid+"/"+path); @@ -365,11 +342,8 @@ public void backend_fileWrite(String oid, String filepath, byte[] data) { * @param data to write the file with **/ public void backend_fileWriteInputStream(String oid, String filepath, InputStream data) { - // Get the clean file path - String cleanPath = cleanFilePath(filepath); - // Build the full path - String fullPath = oid + "/" + cleanPath; + String fullPath = oid + "/" + filepath; if (data == null) { data = new ByteArrayInputStream(EmptyArray.BYTE); @@ -379,7 +353,7 @@ public void backend_fileWriteInputStream(String oid, String filepath, InputStrea try { // Setup the metadata for the file Document metadata = new Document(); - metadata.append("_oid", oid); + metadata.append("oid", oid); metadata.append("type", "file"); // Prepare the upload options @@ -445,18 +419,18 @@ public byte[] backend_fileRead(String oid, String filepath) { * @return the stored byte array of the file **/ public InputStream backend_fileReadInputStream(String oid, String filepath) { - return gridFSBucket.openDownloadStream(oid + "/" + cleanFilePath(filepath)); + return gridFSBucket.openDownloadStream(oid + "/" + filepath); } @Override public boolean backend_fileExist(String oid, String filepath) { // Check against the full file path - return fullRawPathExist(oid + "/" + cleanFilePath(filepath)); + return fullRawPathExist(oid + "/" + filepath); } @Override public void backend_removeFile(String oid, String filepath) { - removeFilePath(oid, cleanFilePath(filepath)); + removeFilePath(oid, filepath); } // Folder Pathing support @@ -474,7 +448,7 @@ public void backend_removeFile(String oid, String filepath) { * @return the stored byte array of the file **/ public void backend_removeFolderPath(final String oid, final String folderPath) { - removeFilePathRecursively(oid, cleanFilePath(folderPath)); + removeFilePathRecursively(oid, folderPath); } /** @@ -489,7 +463,7 @@ public void backend_removeFolderPath(final String oid, final String folderPath) **/ public boolean backend_folderPathExist(final String oid, final String folderPath) { // Note that this passes if any of the files were created directly without folders - return prefixPathExist(oid, cleanFilePath(folderPath)); + return prefixPathExist(oid, folderPath); } /** @@ -503,13 +477,10 @@ public boolean backend_folderPathExist(final String oid, final String folderPath * @return the stored byte array of the file **/ public void backend_ensureFolderPath(final String oid, final String folderPath) { - // Cleanup folderPath - String path = cleanFilePath(folderPath); - // We setup a blank file with type root, this checks only for the anchor file // if it does not exists, we will make it - if(!fullRawPathExist(oid+"/"+path)) { - setupAnchorFile(oid, path, "dir"); + if(!fullRawPathExist(oid+"/"+folderPath)) { + setupAnchorFile(oid, folderPath, "dir"); } } @@ -553,7 +524,7 @@ public long backend_createdTimestamp(final String oid, final String filepath) { */ public long backend_modifiedTimestamp(final String oid, final String filepath) { // Lets build the query for the "root file" - Bson query = Filters.eq("filename", cleanFilePath(filepath)); + Bson query = Filters.eq("filename", filepath); // Lets prepare the search GridFSFindIterable search = gridFSBucket.find(query).limit(1); @@ -599,15 +570,18 @@ public Set backend_getFileAndFolderPathSet(final String oid, String fold // Lets build the query, for fetchign the relevent items if( folderPath == null || folderPath.equals("/") || folderPath.isEmpty() ) { // Handles query for all folder paths - query = Filters.eq("metadata._oid", oid); + query = Filters.and( + Filters.eq("metadata.oid", oid), + // Filters.ne("filename", oid) + Filters.regex("filename", "^"+Pattern.quote(fullPrefixPath)+".*") + ); } else { // Cleanup the path - folderPath = cleanFilePath(folderPath); fullPrefixPath = fullPrefixPath+folderPath; // Remove matching path query = Filters.and( - Filters.eq("metadata._oid", oid), + Filters.eq("metadata.oid", oid), Filters.regex("filename", "^"+Pattern.quote(fullPrefixPath)+".*") ); } @@ -643,7 +617,7 @@ public Set backend_getFileAndFolderPathSet(final String oid, String fold } // Filter and return the final set - return backend_filtterPathSet( ret, folderPath, minDepth, maxDepth, 0); + return backend_filterPathSet( ret, folderPath, minDepth, maxDepth, 0); } } From 5c2f9d4fad6eeeae507d656dc6294a8831a33afa Mon Sep 17 00:00:00 2001 From: Eugene Cheah Date: Fri, 5 Aug 2022 09:20:20 +0000 Subject: [PATCH 37/42] Working mongodb getFileAndFolderPathSet --- .../mongodb/MongoDB_FileWorkspaceMap.java | 69 ++++++++++++++----- 1 file changed, 51 insertions(+), 18 deletions(-) diff --git a/src/main/java/picoded/dstack/mongodb/MongoDB_FileWorkspaceMap.java b/src/main/java/picoded/dstack/mongodb/MongoDB_FileWorkspaceMap.java index 3e16cfe2..ff6dc7d8 100755 --- a/src/main/java/picoded/dstack/mongodb/MongoDB_FileWorkspaceMap.java +++ b/src/main/java/picoded/dstack/mongodb/MongoDB_FileWorkspaceMap.java @@ -35,6 +35,8 @@ import com.mongodb.client.gridfs.model.GridFSFile; import com.mongodb.client.gridfs.model.GridFSUploadOptions; import com.mongodb.client.model.Filters; +import com.mongodb.client.model.IndexOptions; +import com.mongodb.client.model.Indexes; /** * ## Purpose @@ -56,9 +58,13 @@ public class MongoDB_FileWorkspaceMap extends Core_FileWorkspaceMap { // // -------------------------------------------------------------------------- - /** MongoDB instance representing the backend connection */ + /** MongoDB instance representing gridFS */ GridFSBucket gridFSBucket = null; + /** MongoDB instance representing the files and chunks collection (internal to the gridFSBucket) */ + MongoCollection filesCollection = null; + MongoCollection chunksCollection = null; + /** * Constructor, with name constructor * @@ -84,6 +90,9 @@ public MongoDB_FileWorkspaceMap(MongoDBStack inStack, String name) { // Meaning a full "8 * 1000 * 1000" chunk would use "8 * 1024 * 1024" // worth of space, after adding the unknown headers (<=4kb of space : 8*24*24) // + + filesCollection = inStack.db_conn.getCollection(name+".files"); + chunksCollection = inStack.db_conn.getCollection(name+".chunks"); } //-------------------------------------------------------------------------- @@ -97,6 +106,21 @@ public MongoDB_FileWorkspaceMap(MongoDBStack inStack, String name) { **/ @Override public void systemSetup() { + + // We insert a "root" object, to ensure the tables are initialized + // --- + if(!fullRawPathExist("root")) { + setupAnchorFile("root", "root", "root"); + } + + // Lets setup the index for the metadata fields (which is not enabled by default) + // --- + + // Lets create the index for the oid + IndexOptions opt = new IndexOptions(); + opt = opt.name("metadata.oid"); + filesCollection.createIndex(Indexes.ascending("oid"), opt); + } /** @@ -148,7 +172,7 @@ public boolean backend_workspaceExist(String oid) { public void backend_setupWorkspace(String oid) { // We setup a blank file with type root if(!fullRawPathExist(oid)) { - setupAnchorFile(oid, oid, "root"); + setupAnchorFile(oid, oid, "space"); } } @@ -341,7 +365,8 @@ public void backend_fileWrite(String oid, String filepath, byte[] data) { * @param filepath to use for the workspace * @param data to write the file with **/ - public void backend_fileWriteInputStream(String oid, String filepath, InputStream data) { + @Override + public void backend_fileWriteInputStream(final String oid, final String filepath, InputStream data) { // Build the full path String fullPath = oid + "/" + filepath; @@ -567,18 +592,13 @@ public Set backend_getFileAndFolderPathSet(final String oid, String fold // The fulle prefix path String fullPrefixPath = oid+"/"; - // Lets build the query, for fetchign the relevent items if( folderPath == null || folderPath.equals("/") || folderPath.isEmpty() ) { - // Handles query for all folder paths - query = Filters.and( - Filters.eq("metadata.oid", oid), - // Filters.ne("filename", oid) - Filters.regex("filename", "^"+Pattern.quote(fullPrefixPath)+".*") - ); + // Query everything (using only the oid) + query = Filters.eq("metadata.oid", oid); } else { - // Cleanup the path + // Query using oid and the path fullPrefixPath = fullPrefixPath+folderPath; - + // Remove matching path query = Filters.and( Filters.eq("metadata.oid", oid), @@ -586,6 +606,8 @@ public Set backend_getFileAndFolderPathSet(final String oid, String fold ); } + query = Filters.eq("metadata.oid", oid); + // The return set Set ret = new HashSet<>(); @@ -599,19 +621,30 @@ public Set backend_getFileAndFolderPathSet(final String oid, String fold GridFSFile fileObj = cursor.next(); String fullFilename = fileObj.getFilename(); + // Skip the oid anchor + if( fullFilename.equals(oid) ) { + continue; + } + // Remove the oid prefix - String filepath = fullFilename.substring( fullPrefixPath.length() ); + String filepath = fullFilename.substring( oid.length()+1 ); // Register the validpath ret.add(filepath); + // Prepare a clean path without ending slash + String cleanPath = filepath; + if( cleanPath.endsWith("/") ) { + cleanPath = cleanPath.substring(0, cleanPath.length()-1); + } + // Lets split the filepath - String[] filepathArr = filepath.split("/"); - List filepathList = Arrays.asList(filepathArr); + String[] cleanPathArr = cleanPath.split("/"); + List cleanPathList = Arrays.asList(cleanPathArr); - // Lets handle parent folders - for(int i=1+Math.max(minDepth,0); i<(filepathArr.length-1); ++i) { - ret.add( String.join("/", filepathList.subList(0,i)+"/" ) ); + // Lets handle parent folders, note that i Date: Fri, 5 Aug 2022 09:21:36 +0000 Subject: [PATCH 38/42] Working query for mongodb fileworkspacemap --- src/main/java/picoded/dstack/core/Core_FileWorkspaceMap.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/picoded/dstack/core/Core_FileWorkspaceMap.java b/src/main/java/picoded/dstack/core/Core_FileWorkspaceMap.java index 8c171e3c..f569b414 100755 --- a/src/main/java/picoded/dstack/core/Core_FileWorkspaceMap.java +++ b/src/main/java/picoded/dstack/core/Core_FileWorkspaceMap.java @@ -492,8 +492,8 @@ public long backend_modifiedTimestamp(final String oid, final String filepath) { * - min/max depth * - any / file / folder * - * @param rawSet - * @param folderPath + * @param rawSet (note this expect the full RAW paths, without removing the folderPath prefix) + * @param folderPath the folder path prefix to search and match against, and truncate * @param minDepth (0 = all items, 1 = must be in atleast a folder, 2 = folder, inside a folder) * @param maxDepth * @param pathType (0 = any, 1 = file, 2 = folder) From 0e8368afeda2ec63b541efc35ea92de6e360a336 Mon Sep 17 00:00:00 2001 From: Eugene Cheah Date: Fri, 5 Aug 2022 10:43:59 +0000 Subject: [PATCH 39/42] Working mongodb FileWorkspaceMap prototype (no file cleanup yet) --- .../dstack/core/Core_FileWorkspace.java | 30 ++------ .../dstack/core/Core_FileWorkspaceMap.java | 62 ++++++++++++++-- .../mongodb/MongoDB_FileWorkspaceMap.java | 72 +++++++++++++++---- .../StructSimple_FileWorkspaceMap_test.java | 3 +- 4 files changed, 120 insertions(+), 47 deletions(-) diff --git a/src/main/java/picoded/dstack/core/Core_FileWorkspace.java b/src/main/java/picoded/dstack/core/Core_FileWorkspace.java index 728e3810..b59a757c 100755 --- a/src/main/java/picoded/dstack/core/Core_FileWorkspace.java +++ b/src/main/java/picoded/dstack/core/Core_FileWorkspace.java @@ -122,38 +122,16 @@ protected void setupUninitializedWorkspace() { * @param filePath * @return filePath normalized to remove ending "/" */ - private static String normalizeFilePathString(final String filePath) { - if (filePath == null) { - throw new IllegalArgumentException("Invalid null filePath"); - } - - String res = FileUtil.normalize(filePath, true); - if (res.startsWith("/")) { - res = res.substring(1); - } - if (res.endsWith("/")) { - res = res.substring(0, res.length() - 1); - } - return res; + protected static String normalizeFilePathString(final String filePath) { + return Core_FileWorkspaceMap.normalizeFilePathString(filePath); } /** * @param folderPath * @return folderPath normalized with ending "/" */ - private static String normalizeFolderPathString(final String folderPath) { - if (folderPath == null || folderPath.length() <= 0) { - return "/"; - } - - String res = FileUtil.normalize(folderPath, true); - if (res.startsWith("/")) { - res = res.substring(1); - } - if (!res.endsWith("/")) { - res = res + "/"; - } - return res; + protected static String normalizeFolderPathString(final String folderPath) { + return Core_FileWorkspaceMap.normalizeFolderPathString(folderPath); } // File exists checks diff --git a/src/main/java/picoded/dstack/core/Core_FileWorkspaceMap.java b/src/main/java/picoded/dstack/core/Core_FileWorkspaceMap.java index f569b414..4dc7d96d 100755 --- a/src/main/java/picoded/dstack/core/Core_FileWorkspaceMap.java +++ b/src/main/java/picoded/dstack/core/Core_FileWorkspaceMap.java @@ -1,6 +1,7 @@ package picoded.dstack.core; import picoded.core.conv.ArrayConv; +import picoded.core.file.FileUtil; // Java imports @@ -104,6 +105,48 @@ public void setupWorkspace(String oid) { */ abstract public void backend_setupWorkspace(String oid); + //-------------------------------------------------------------------------- + // File / Folder string normalization + //-------------------------------------------------------------------------- + + /** + * @param filePath + * @return filePath normalized to remove ending "/" + */ + protected static String normalizeFilePathString(final String filePath) { + if (filePath == null) { + throw new IllegalArgumentException("Invalid null filePath"); + } + + String res = FileUtil.normalize(filePath, true); + if (res.startsWith("/")) { + res = res.substring(1); + } + if (res.endsWith("/")) { + res = res.substring(0, res.length() - 1); + } + return res; + } + + /** + * @param folderPath + * @return folderPath normalized with ending "/" + */ + protected static String normalizeFolderPathString(final String folderPath) { + if (folderPath == null || folderPath.length() <= 0) { + return "/"; + } + + String res = FileUtil.normalize(folderPath, true); + if (res.startsWith("/")) { + res = res.substring(1); + } + if (!res.endsWith("/")) { + res = res + "/"; + } + return res; + } + //-------------------------------------------------------------------------- // // Functions, used by FileWorkspace @@ -364,13 +407,13 @@ public void backend_moveFolderPath(final String oid, final String sourceFolder, // Lets sync up all the folders first for(String dir : subPath) { if(dir.endsWith("/")) { - backend_ensureFolderPath(oid, destinationFolder+subPath); + backend_ensureFolderPath(oid, destinationFolder+dir); } } // Lets sync up all the files next for(String file : subPath) { if(!file.endsWith("/")) { - backend_moveFile(oid, sourceFolder+subPath, destinationFolder+subPath); + backend_copyFile(oid, sourceFolder+file, destinationFolder+file); } } // Lets remove the original folders @@ -431,13 +474,13 @@ public void backend_copyFolderPath(final String oid, final String sourceFolder, // Lets sync up all the folders first for(String dir : subPath) { if(dir.endsWith("/")) { - backend_ensureFolderPath(oid, destinationFolder+subPath); + backend_ensureFolderPath(oid, destinationFolder+dir); } } // Lets sync up all the files next for(String file : subPath) { - if(!file.endsWith("/")) { - backend_copyFile(oid, sourceFolder+subPath, destinationFolder+subPath); + if(file.endsWith("/") == false) { + backend_copyFile(oid, sourceFolder+file, destinationFolder+file); } } } @@ -551,12 +594,21 @@ protected Set backend_filterPathSet(final Set rawSet, final Stri } // Alrighto - lets check file / folder type - and add it in + + // Ignore empty, or root path + if(subPath.isEmpty() || subPath.equals("/")) { + continue; + } + + // Expect a folder, reject files if (pathType == 1) { if (subPath.endsWith("/")) { // Not a file - abort! continue; } } + + // Expect files, reject folders if (pathType == 2) { if (!subPath.endsWith("/")) { // Not a folder - abort! diff --git a/src/main/java/picoded/dstack/mongodb/MongoDB_FileWorkspaceMap.java b/src/main/java/picoded/dstack/mongodb/MongoDB_FileWorkspaceMap.java index ff6dc7d8..dc9af2cc 100755 --- a/src/main/java/picoded/dstack/mongodb/MongoDB_FileWorkspaceMap.java +++ b/src/main/java/picoded/dstack/mongodb/MongoDB_FileWorkspaceMap.java @@ -28,6 +28,7 @@ // MongoDB imports import org.bson.Document; import org.bson.types.Binary; +import org.bson.types.ObjectId; import org.bson.conversions.Bson; import com.mongodb.client.*; import com.mongodb.client.gridfs.*; @@ -110,7 +111,7 @@ public void systemSetup() { // We insert a "root" object, to ensure the tables are initialized // --- if(!fullRawPathExist("root")) { - setupAnchorFile("root", "root", "root"); + setupAnchorFile_withFullRawPath("root", "root", "root"); } // Lets setup the index for the metadata fields (which is not enabled by default) @@ -172,7 +173,7 @@ public boolean backend_workspaceExist(String oid) { public void backend_setupWorkspace(String oid) { // We setup a blank file with type root if(!fullRawPathExist(oid)) { - setupAnchorFile(oid, oid, "space"); + setupAnchorFile_withFullRawPath(oid, oid, "space"); } } @@ -219,10 +220,16 @@ protected boolean prefixPathExist(String oid, String path) { // Lets build the query for the "root file" Bson query = null; + // Get the full prefixpath + String fullPrefixPath = oid+"/"+path; + // Remove matching path - query = Filters.and( - Filters.eq("metadata.oid", oid), - Filters.regex("filename", "^"+Pattern.quote(oid+"/"+path)+".*") + query = Filters.or( + Filters.eq("filename", fullPrefixPath), + Filters.and( + Filters.eq("metadata.oid", oid), + Filters.regex("filename", "^"+Pattern.quote(fullPrefixPath)+".*") + ) ); // Lets prepare the search @@ -241,8 +248,9 @@ protected boolean prefixPathExist(String oid, String path) { /** * Setup an empty file, used for various use cases + * The extended funciton name is intentional to avoid confusion of "full path" with "path" */ - public void setupAnchorFile(String oid, String fullPath, String type) { + public void setupAnchorFile_withFullRawPath(String oid, String fullPath, String type) { // In general we will upload a blank file // with the relevent oid, that can be easily lookedup // @@ -255,7 +263,10 @@ public void setupAnchorFile(String oid, String fullPath, String type) { // Prepare the upload options GridFSUploadOptions opt = (new GridFSUploadOptions()).metadata(metadata); - gridFSBucket.uploadFromStream(fullPath, emptyStream, opt); + ObjectId objID = gridFSBucket.uploadFromStream(fullPath, emptyStream, opt); + + // Flush it? + objID.toString(); } catch (IOException e) { throw new RuntimeException(e); } @@ -310,7 +321,6 @@ protected boolean removeFilePath(String oid, String path) { while (cursor.hasNext()) { GridFSFile fileObj = cursor.next(); gridFSBucket.delete(fileObj.getId()); - rmFlag = true; } } @@ -319,8 +329,39 @@ protected boolean removeFilePath(String oid, String path) { return rmFlag; } - protected void performFileCleanup(String oid, String path) { + /** + * Given the current path, enforce the parent pathing dir + * Used mainly to ensure "parent" folder exists on file write/rm + **/ + protected void ensureParentPath(String oid, String path) { + // Does nothing if path is empty + if( path == null || path.equals("/") || path.isEmpty() ) { + return; + } + + // Cleanup ending slash + if( path.endsWith("/") ) { + path = path.substring(0, path.length() - 1); + } + + // Get the parent path + String parPath = normalizeFolderPathString( FileUtil.getParentPath(path) ); + + // Does nothing if folder path is "blank" + if( parPath == null || parPath.equals("/") || parPath.isEmpty() ) { + return; + } + + // Path enforcement + backend_ensureFolderPath(oid, parPath); + } + /** + * Because mongoDB does file versioining on each save, we would need to cleanup + * older file versions where applicable, in a safe way + */ + protected void performVersionedFileCleanup(String oid, String path) { + // @TODO !!! } //-------------------------------------------------------------------------- @@ -383,7 +424,8 @@ public void backend_fileWriteInputStream(final String oid, final String filepath // Prepare the upload options GridFSUploadOptions opt = (new GridFSUploadOptions()).metadata(metadata); - gridFSBucket.uploadFromStream(fullPath, data, opt); + ObjectId objID = gridFSBucket.uploadFromStream(fullPath, data, opt); + objID.toString(); } catch (Exception e) { throw new RuntimeException(e); } finally { @@ -455,6 +497,7 @@ public boolean backend_fileExist(String oid, String filepath) { @Override public void backend_removeFile(String oid, String filepath) { + ensureParentPath(oid, filepath); removeFilePath(oid, filepath); } @@ -473,6 +516,7 @@ public void backend_removeFile(String oid, String filepath) { * @return the stored byte array of the file **/ public void backend_removeFolderPath(final String oid, final String folderPath) { + ensureParentPath(oid, folderPath); removeFilePathRecursively(oid, folderPath); } @@ -504,8 +548,8 @@ public boolean backend_folderPathExist(final String oid, final String folderPath public void backend_ensureFolderPath(final String oid, final String folderPath) { // We setup a blank file with type root, this checks only for the anchor file // if it does not exists, we will make it - if(!fullRawPathExist(oid+"/"+folderPath)) { - setupAnchorFile(oid, folderPath, "dir"); + if(fullRawPathExist(oid+"/"+folderPath) == false) { + setupAnchorFile_withFullRawPath(oid, oid+"/"+folderPath, "dir"); } } @@ -599,15 +643,13 @@ public Set backend_getFileAndFolderPathSet(final String oid, String fold // Query using oid and the path fullPrefixPath = fullPrefixPath+folderPath; - // Remove matching path + // Filter for matching path query = Filters.and( Filters.eq("metadata.oid", oid), Filters.regex("filename", "^"+Pattern.quote(fullPrefixPath)+".*") ); } - query = Filters.eq("metadata.oid", oid); - // The return set Set ret = new HashSet<>(); diff --git a/src/test/java/picoded/dstack/struct/simple/StructSimple_FileWorkspaceMap_test.java b/src/test/java/picoded/dstack/struct/simple/StructSimple_FileWorkspaceMap_test.java index 87db2bc1..9bfc4d31 100755 --- a/src/test/java/picoded/dstack/struct/simple/StructSimple_FileWorkspaceMap_test.java +++ b/src/test/java/picoded/dstack/struct/simple/StructSimple_FileWorkspaceMap_test.java @@ -160,9 +160,10 @@ public void fileWrite_andProperlySetupFolder() { // Remove and assert fileWorkspace.removeFolderPath("test/folder"); + assertFalse(fileWorkspace.fileExist("test/folder/file.txt")); assertFalse(fileWorkspace.folderPathExist("test/folder")); + assertTrue(fileWorkspace.folderPathExist("test")); - assertFalse(fileWorkspace.fileExist("test/folder/file.txt")); } //----------------------------------------------------------------------------------- From b4406c9e481a93a0b19f1f102412c76f86250617 Mon Sep 17 00:00:00 2001 From: Eugene Cheah Date: Mon, 8 Aug 2022 04:53:21 +0000 Subject: [PATCH 40/42] versioned cleanup for mongodb --- .../dstack/core/Core_FileWorkspaceMap.java | 6 + .../mongodb/MongoDB_FileWorkspaceMap.java | 239 +++++++++++++++--- .../dstack/stack/Stack_FileWorkspaceMap.java | 6 +- .../StructSimple_FileWorkspaceMap_test.java | 41 +++ 4 files changed, 262 insertions(+), 30 deletions(-) diff --git a/src/main/java/picoded/dstack/core/Core_FileWorkspaceMap.java b/src/main/java/picoded/dstack/core/Core_FileWorkspaceMap.java index 4dc7d96d..2d28e38a 100755 --- a/src/main/java/picoded/dstack/core/Core_FileWorkspaceMap.java +++ b/src/main/java/picoded/dstack/core/Core_FileWorkspaceMap.java @@ -125,6 +125,12 @@ protected static String normalizeFilePathString(final String filePath) { if (res.endsWith("/")) { res = res.substring(0, res.length() - 1); } + + // Block empty filepath + if( res.isEmpty() ) { + throw new RuntimeException("Empty file path is not allowed"); + } + return res; } diff --git a/src/main/java/picoded/dstack/mongodb/MongoDB_FileWorkspaceMap.java b/src/main/java/picoded/dstack/mongodb/MongoDB_FileWorkspaceMap.java index dc9af2cc..7019f068 100755 --- a/src/main/java/picoded/dstack/mongodb/MongoDB_FileWorkspaceMap.java +++ b/src/main/java/picoded/dstack/mongodb/MongoDB_FileWorkspaceMap.java @@ -8,9 +8,6 @@ import java.util.Map; import java.util.Set; import java.util.regex.Pattern; - -import javax.management.RuntimeErrorException; - import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -359,9 +356,110 @@ protected void ensureParentPath(String oid, String path) { /** * Because mongoDB does file versioining on each save, we would need to cleanup * older file versions where applicable, in a safe way + * + * In general, due to the difficulty of possible race conditions that may occur + * when removing an "old version" immediately, that could be "read" mid-way. + * + * First we scan for the list of all the file versions. + * + * We find the latest that is at-least 10 seconds old (what we consider a safe window) + * and delete all version before it. + * + * If after the above, we found that there are still "10 versions", as there were + * 10 writes in the past 10 seconds. We force a thread.sleep in increments of 1 second, + * and remove any versions that matches the above criteria. Up to a full 10 seconds of delay. + * + * This will forcefully throttle down any write heavy flows, to avoid contentions. + * + * This safety measure is used in addition, to the checks performed on file write */ protected void performVersionedFileCleanup(String oid, String path) { - // @TODO !!! + + // Lets get the list of files and their respective versions + // We query the file table directly, to reduce the required + // back and forth queries + + // Get the full filename + String filename = oid+"/"+path; + + // Get the current timestamp + long now = System.currentTimeMillis(); + long tenSecondsAgo = now - (10 * 1000); + + // Lets fetch the full list in descending date order + FindIterable search = filesCollection.find( Filters.eq("filename", filename) ); + search = search.sort( (new Document()).append("uploadDate", -1) ); + + // Lets remap from cursor to list + List searchList = new ArrayList<>(); + try (MongoCursor cursor = search.iterator()) { + while (cursor.hasNext()) { + searchList.add(cursor.next()); + } + } + + // Safe anchor point, all items after this is "safe to be deleted" + // if this is detected properly (do not delete the safeAnchorPoint file itself) + int safeAnchorPoint = -1; + + // Lets find the document thats atleast 10 seconds old + for( int i=1; i= 1 ) { + // Lets loop through all items after the safeAnchorPoint + while( searchList.size() > (safeAnchorPoint + 1) ) { + // Get and remove the last item + Document doc = searchList.remove( searchList.size() - 1 ); + ObjectId objID = doc.getObjectId("_id"); + + // Lets remove the file (and its chunks) + try { + gridFSBucket.delete(objID); + } catch(Exception e) { + // do nothing, as there could be a race condition delete + // (2 delete by seperate write commands happenign together) + } + } + } + + // If the list is less then 10, lets return + if( searchList.size() <= 10 ) { + return; + } + + // We have more then 10 files, that is less then 10 seconds old + // Lets do a forced 10 seconds halt, so we can forcefully clear the files + try { + Thread.sleep(10 * 1000); + } catch(InterruptedException e) { + throw new RuntimeException(e); + } + + // And clear the various outdated files + // after the latest, and its immediate previous version + while( searchList.size() > 2 ) { + // Get and remove the last item + Document doc = searchList.remove( searchList.size() - 1 ); + ObjectId objID = doc.getObjectId("_id"); + + // Lets remove the file (and its chunks) + try { + gridFSBucket.delete(objID); + } catch(Exception e) { + // do nothing, as there could be a race condition delete + // (2 delete by seperate write commands happenign together) + } + } + } //-------------------------------------------------------------------------- @@ -382,16 +480,111 @@ protected void performVersionedFileCleanup(String oid, String path) { **/ @Override public void backend_fileWrite(String oid, String filepath, byte[] data) { - // Build the input stream - ByteArrayInputStream buffer = null; + + // Build the full path + String fullPath = oid + "/" + filepath; - // Only build if its not null - if (data != null) { - buffer = new ByteArrayInputStream(data); + // + // Due to the rather huge penalty of writing files, without actual content changes, + // and the performance implications of a high number of back to back file changes. + // + // We will employ the following throttling safeguards + // + // 1) Throttling file writes, when the existing file is less then 2 seconds old + // 2) Check against the current values, and skip the write if they match. + // + // This prevents the creation of a "new version" unless its needed. And slow down + // any flooding of back to back file writes. + // + + // 1) Lets check the previous write timing, and throttle it if needed + // --- + + // Lets get the time "NOW" + long now = System.currentTimeMillis(); + + // Lets build the query for the file involved + Bson query = Filters.eq("filename", fullPath); + + // Read timestamp, and objectid + ObjectId readObjId = null; + long readUploadTimestamp = -1; + + // Lets iterate the search result, and return true on an item + try (MongoCursor cursor = gridFSBucket.find(query).limit(1).iterator()) { + if (cursor.hasNext()) { + GridFSFile fileObj = cursor.next(); + readUploadTimestamp = fileObj.getUploadDate().getTime(); + readObjId = fileObj.getObjectId(); + } + } + + // Check if the current file is less then 2 seconds old + // If so, we induce a wait for it to occur (if file exists) + if( readObjId != null && readUploadTimestamp + 2000 >= now ) { + try { + Thread.sleep( Math.min( Math.max( readUploadTimestamp + 2000 - now, 500), 2000 ) ); + } catch(InterruptedException e) { + throw new RuntimeException(e); + } + + // And get the latest objectID again (in case of any changes) + try (MongoCursor cursor = gridFSBucket.find(query).limit(1).iterator()) { + if (cursor.hasNext()) { + GridFSFile fileObj = cursor.next(); + readUploadTimestamp = fileObj.getUploadDate().getTime(); + readObjId = fileObj.getObjectId(); + } + } + } + + // 2) Lets check against current value + // --- + + // Handle null byte[] + if( data == null ) { + data = EmptyArray.BYTE; + } + + // Lets map the current value to an inputstream, in closable blocks + // We intentionally use inputstream, to avoid needing 2 byte[] blocks in memory + // (if file exists) + if( readObjId == null ) { + // does nothing if the object does not exists + } else { + try (ByteArrayInputStream inBuffer = new ByteArrayInputStream(data) ) { + try(InputStream existingValue = gridFSBucket.openDownloadStream(readObjId)) { + if(IOUtils.contentEquals(inBuffer, existingValue)) { + // They are the same, skip the write + return; + } + } + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + // Finally, lets write the update + // --- + + try (ByteArrayInputStream inBuffer = new ByteArrayInputStream(data) ) { + // Setup the metadata for the file + Document metadata = new Document(); + metadata.append("oid", oid); + metadata.append("type", "file"); + + // Prepare the upload options + GridFSUploadOptions opt = (new GridFSUploadOptions()).metadata(metadata); + ObjectId objID = gridFSBucket.uploadFromStream(fullPath, inBuffer, opt); + objID.toString(); + } catch (IOException e) { + throw new RuntimeException(e); } - // Then pump it - backend_fileWriteInputStream(oid, filepath, buffer); + // Perform post file write cleanup (if there was a previous version) + if( readObjId != null ) { + performVersionedFileCleanup(oid, filepath); + } } /** @@ -408,25 +601,11 @@ public void backend_fileWrite(String oid, String filepath, byte[] data) { **/ @Override public void backend_fileWriteInputStream(final String oid, final String filepath, InputStream data) { - // Build the full path - String fullPath = oid + "/" + filepath; - - if (data == null) { - data = new ByteArrayInputStream(EmptyArray.BYTE); - } - - // Write the file + // Converts it to bytearray respectively + byte[] rawBytes = null; try { - // Setup the metadata for the file - Document metadata = new Document(); - metadata.append("oid", oid); - metadata.append("type", "file"); - - // Prepare the upload options - GridFSUploadOptions opt = (new GridFSUploadOptions()).metadata(metadata); - ObjectId objID = gridFSBucket.uploadFromStream(fullPath, data, opt); - objID.toString(); - } catch (Exception e) { + rawBytes = IOUtils.toByteArray(data); + } catch (IOException e) { throw new RuntimeException(e); } finally { try { @@ -435,6 +614,8 @@ public void backend_fileWriteInputStream(final String oid, final String filepath throw new RuntimeException(e); } } + // Does the bytearray writes + backend_fileWrite(oid, filepath, rawBytes); } //-------------------------------------------------------------------------- diff --git a/src/main/java/picoded/dstack/stack/Stack_FileWorkspaceMap.java b/src/main/java/picoded/dstack/stack/Stack_FileWorkspaceMap.java index 4c3437a0..0bce15cf 100755 --- a/src/main/java/picoded/dstack/stack/Stack_FileWorkspaceMap.java +++ b/src/main/java/picoded/dstack/stack/Stack_FileWorkspaceMap.java @@ -223,8 +223,12 @@ public InputStream backend_fileReadInputStream(final String oid, final String fi public void backend_fileWriteInputStream(final String oid, final String filepath, final InputStream data) { + // // Due to the behaviour of how the file data needs to be handled across multiple layers - // we only use an optimized "readStream" call if the filesystem is a single stack layer + // we only use an optimized "writeStream" call ONLY if the filesystem is a single stack layer + // + // Else we will revert to byte[] that can be applied multiple times across the stack + // if (dataLayers.length == 1) { dataLayers[0].backend_fileWriteInputStream(oid, filepath, data); return; diff --git a/src/test/java/picoded/dstack/struct/simple/StructSimple_FileWorkspaceMap_test.java b/src/test/java/picoded/dstack/struct/simple/StructSimple_FileWorkspaceMap_test.java index 9bfc4d31..8d5ad24b 100755 --- a/src/test/java/picoded/dstack/struct/simple/StructSimple_FileWorkspaceMap_test.java +++ b/src/test/java/picoded/dstack/struct/simple/StructSimple_FileWorkspaceMap_test.java @@ -166,6 +166,47 @@ public void fileWrite_andProperlySetupFolder() { assertTrue(fileWorkspace.folderPathExist("test")); } + //----------------------------------------------------------------------------------- + // + // Multiple Writes + // + //----------------------------------------------------------------------------------- + + @Test + public void fileWrite_fiveTimes() { + // Get the file workspace to use + FileWorkspace fileWorkspace = testObj.newEntry(); + assertNotNull(fileWorkspace); + + // Folder does not exist first + assertFalse(fileWorkspace.folderPathExist("test/folder")); + + // Write and read file + for(int i=0; i < 5; ++i) { + fileWorkspace.writeString("test/folder/file.txt", "ver-"+i); + assertEquals("ver-"+i, fileWorkspace.readString("test/folder/file.txt")); + fileWorkspace.writeString("test/folder/file.txt", "ver-"+i); + assertEquals("ver-"+i, fileWorkspace.readString("test/folder/file.txt")); + } + } + @Test + public void fileWrite_twentyTimes() { + // Get the file workspace to use + FileWorkspace fileWorkspace = testObj.newEntry(); + assertNotNull(fileWorkspace); + + // Folder does not exist first + assertFalse(fileWorkspace.folderPathExist("test/folder")); + + // Write and read file + for(int i=0; i < 20; ++i) { + fileWorkspace.writeString("test/folder/file.txt", "ver-"+i); + assertEquals("ver-"+i, fileWorkspace.readString("test/folder/file.txt")); + fileWorkspace.writeString("test/folder/file.txt", "ver-"+i); + assertEquals("ver-"+i, fileWorkspace.readString("test/folder/file.txt")); + } + } + //----------------------------------------------------------------------------------- // // Move test From 8b18bc6d92aabb5cb92230ebb73fb4786fc7f7c4 Mon Sep 17 00:00:00 2001 From: Eugene Cheah Date: Mon, 8 Aug 2022 06:18:04 +0000 Subject: [PATCH 41/42] @TODO optimized getAndAdd operations --- src/main/java/picoded/dstack/KeyLongMap.java | 39 +----- .../dstack/mongodb/MongoDB_KeyLongMap.java | 112 ++++++++++++++++++ 2 files changed, 117 insertions(+), 34 deletions(-) diff --git a/src/main/java/picoded/dstack/KeyLongMap.java b/src/main/java/picoded/dstack/KeyLongMap.java index 7ea870c8..3303f47a 100755 --- a/src/main/java/picoded/dstack/KeyLongMap.java +++ b/src/main/java/picoded/dstack/KeyLongMap.java @@ -180,39 +180,10 @@ default Long removeValue(Object key) { **/ default Long addAndGet(Object key, Object delta) { // - // NOTE : The default implmentation of addAndGet, - // or getAndAdd relies on repetaed tries using - // weakCompareAndSet, while functional. - // Is highly inefficent in most cases + // We simply use get and add, with the delta, + // this reduce the amount of permutation needed to support // - - // Validate and convert the key to String - if (key == null) { - throw new IllegalArgumentException("key cannot be null in addAndGet"); - } - String keyAsString = key.toString(); - - // Attempt to update the key for 5 times before throwing exception - for (int tries = 0; tries < 5; tries++) { - // Retrieve value from key - Long value = getValue(keyAsString); - - // Assume value as 0 if not exist - if (value == null) { - value = new Long(0); - } - - // Calculate the updated value - Long updatedValue = GenericConvert.toLong(delta) + value; - - // Update the value with weakCompareAndSet and return - if (weakCompareAndSet(keyAsString, value, updatedValue)) { - return updatedValue; - } - } - - // Throw exception due to number of retries exceeded the limit - throw new RuntimeException("Number of retries exceeded limit for addAndGet"); + return getAndAdd(key, delta)+GenericConvert.toLong(delta); } /** @@ -233,7 +204,7 @@ default Long getAndAdd(Object key, Object delta) { // Validate and convert the key to String if (key == null) { - throw new IllegalArgumentException("key cannot be null in addAndGet"); + throw new IllegalArgumentException("key cannot be null in"); } String keyAsString = key.toString(); @@ -257,7 +228,7 @@ default Long getAndAdd(Object key, Object delta) { } // Throw exception due to number of retries exceeded the limit - throw new RuntimeException("Number of retries exceeded limit for addAndGet"); + throw new RuntimeException("Number of retries exceeded limit"); } /** diff --git a/src/main/java/picoded/dstack/mongodb/MongoDB_KeyLongMap.java b/src/main/java/picoded/dstack/mongodb/MongoDB_KeyLongMap.java index 4869b423..425e23e4 100644 --- a/src/main/java/picoded/dstack/mongodb/MongoDB_KeyLongMap.java +++ b/src/main/java/picoded/dstack/mongodb/MongoDB_KeyLongMap.java @@ -406,6 +406,118 @@ public boolean weakCompareAndSet(String key, Long expect, Long update) { return false; } + //-------------------------------------------------------------------------- + // + // @TODO : Optimized getAndAdd + // + //-------------------------------------------------------------------------- + + // /** + // * Returns the value, given the key + // * + // * @param key param find the meta key + // * @param delta value to add + // * + // * @return value of the given key after adding + // **/ + // public Long getAndAdd(Object key, Object delta) { + + // // This is the more optimized varient for weakCompareAndSet + // // --- + + // // Validate and convert the key to String + // if (key == null) { + // throw new IllegalArgumentException("key cannot be null in"); + // } + // String keyAsString = key.toString(); + + // // Normalize the delta to a long + // Long deltaLong = GenericConvert.toLong(delta); + + // // Configure this to be an "upsert" query + // FindOneAndUpdateOptions opt = new FindOneAndUpdateOptions(); + // opt.upsert(true); + + // // now timestamp + // Date now = new Date(); + + // // Lets generate the mongodb "update" document rule + // Document updateDoc = new Document(); + + // // Disable expire timestamp when using + // // weakCompareAndSet/getAndAdd/addAndGet + // Document unset_doc = new Document(); + // unset_doc.append("expireAt", ""); + // updateDoc.append("$unset", unset_doc); + + // // Setup the value on update/insert/upsert + // Document inc_doc = new Document(); + // inc_doc.append("val", deltaLong); + // updateDoc.append("$set", set_doc); + + // // + // // In general there are the following compare and set scenerios to handle + // // + // // 1) expecting value is 0 + // // a) existing record is expired + // // b) existing record does not exist + // // c) existing record is NOT expired, and is 0 + // // 2) expecting value is non-zero + // // a) existing record is NOT expired, and is expected value. + // // + + // // Potentially a new upsert, ensure there is something to "update" atleast + // // initializing an empty row if it does not exist + // if (expect == null || expect == 0l) { + // // Expect is now atleast 0 + // expect = 0l; + // } + + // // + // // We update any existing values + // // this handle scenerio 1a, 1c & 2a + // // + // // We can do this safely here, as mongodb handles the expire + // // natively, so we do not need to worry about race conditions. + // // + + // // + // // Upsert the document + // // + // Object ret = collection.findOneAndUpdate(Filters.and(Filters.eq("key", key), // + // Filters.or( // + // // Handles an expired record + // Filters.and(Filters.gt("expireAt", new Date(0l)), Filters.lt("expireAt", now)), + // // Handles an non-expired record + // Filters.and(Filters.eq("val", expect), filterForUnexpired(now)))), updateDoc, opt); + + // // Return true on succesful update + // if (ret != null) { + // return true; + // } + + // // + // // We insert a record if possible, this handle sceneric 1b + // // + // if (expect == 0l) { + // try { + // InsertOneResult res = collection.insertOne( // + // new Document().append("key", key).append("val", update) // + // ); + + // if (res.wasAcknowledged()) { + // return true; + // } + // } catch (Exception e) { + // // This is probably due to a conflicting index + // return false; + // } + // } + + // // All Failed + // return false; + // } + //-------------------------------------------------------------------------- // // Remove call From 2fbda383d135d137e54fe5c5af2354e6980602b8 Mon Sep 17 00:00:00 2001 From: Eugene Cheah Date: Mon, 8 Aug 2022 06:18:50 +0000 Subject: [PATCH 42/42] Added @TODO commentry --- src/main/java/picoded/dstack/mongodb/MongoDB_KeyLongMap.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/main/java/picoded/dstack/mongodb/MongoDB_KeyLongMap.java b/src/main/java/picoded/dstack/mongodb/MongoDB_KeyLongMap.java index 425e23e4..4ae8a57f 100644 --- a/src/main/java/picoded/dstack/mongodb/MongoDB_KeyLongMap.java +++ b/src/main/java/picoded/dstack/mongodb/MongoDB_KeyLongMap.java @@ -438,6 +438,9 @@ public boolean weakCompareAndSet(String key, Long expect, Long update) { // FindOneAndUpdateOptions opt = new FindOneAndUpdateOptions(); // opt.upsert(true); + // // !! THE FOLLOWING BELOW IS INCOMPLETE CODE (AND IS BROKEN) + // // --- + // // now timestamp // Date now = new Date();