Fix issues with object store direct access directory.

plugins/object-store-plugin/src/main/groovy/org/rundeck/plugin/objectstore/ObjectStorePlugin.groovy

@@ -22,6 +22,7 @@ import com.dtolabs.rundeck.plugins.descriptions.PluginDescription
 import com.dtolabs.rundeck.plugins.descriptions.PluginProperty
 import com.dtolabs.rundeck.plugins.storage.StoragePlugin
 import io.minio.MinioClient
+import org.rundeck.plugin.objectstore.directorysource.ObjectStoreDirectAccessDirectorySource
 import org.rundeck.plugin.objectstore.tree.ObjectStoreTree
 import org.rundeck.storage.api.Tree
 import org.rundeck.storage.impl.DelegateTree
@@ -37,20 +38,36 @@ class ObjectStorePlugin extends DelegateTree<ResourceMeta> implements StoragePlu
     String secretKey;
     @PluginProperty(title = 'Access Key', description = 'The access key use by the client to connect to the service')
     String accessKey;
+    @PluginProperty(title = 'Uncached Object Lookup', description = """Use object store directly to list directory resources, check object existence, etc. 
+                                                                    Depending on the directory structure and number of objects, enabling this option could have
+                                                                    performance issues. This option will work better in a cluster because all servers in the cluster will
+                                                                    have coordinated access to the objects managed by the plugins. NOTE: The cached object directory does
+                                                                    not share the cache between servers, so it is not best to use it when operating a Rundeck cluster.""")
+    boolean uncachedObjectLookup = false;
 
     Tree<ResourceMeta> delegateTree
 
     @Override
     Tree<ResourceMeta> getDelegate() {
-        if(!delegateTree) initTree()
+        if (!delegateTree) {
+            initTree()
+        }
         return delegateTree
     }
 
     void initTree() {
-        if(!bucket) throw new IllegalArgumentException("bucket property is required")
-        if(!objectStoreUrl) throw new IllegalArgumentException("objectStoreUrl property is required")
+        if (!bucket) {
+            throw new IllegalArgumentException("bucket property is required")
+        }
+        if (!objectStoreUrl) {
+            throw new IllegalArgumentException("objectStoreUrl property is required")
+        }
 
         MinioClient mClient = new MinioClient(objectStoreUrl, accessKey, secretKey)
-        delegateTree = new ObjectStoreTree(mClient, bucket)
+        if(uncachedObjectLookup) {
+            delegateTree = new ObjectStoreTree(mClient,bucket,new ObjectStoreDirectAccessDirectorySource(mClient,bucket))
+        } else {
+            delegateTree = new ObjectStoreTree(mClient, bucket)
+        }
     }
 }

plugins/object-store-plugin/src/main/groovy/org/rundeck/plugin/objectstore/directorysource/ObjectStoreDirectAccessDirectorySource.groovy

@@ -22,6 +22,7 @@ import io.minio.messages.Item
 import org.rundeck.plugin.objectstore.stream.LazyAccessObjectStoreInputStream
 import org.rundeck.plugin.objectstore.tree.ObjectStoreResource
 import org.rundeck.plugin.objectstore.tree.ObjectStoreTree
+import org.rundeck.plugin.objectstore.tree.ObjectStoreUtils
 import org.rundeck.storage.api.Resource
 
 import java.util.regex.Matcher
@@ -30,6 +31,9 @@ import java.util.regex.Pattern
 /**
  * Uses the object client to directly access the object store to get directory information.
  * For stores with lots of objects this could be a very inefficient directory access mechanism.
+ *
+ * This store works best when the object store is going to be accessed by multiple cluster members
+ * or the object store is regularly updated by third party tools
  */
 class ObjectStoreDirectAccessDirectorySource implements ObjectStoreDirectorySource {
     private static final String DIR_MARKER = "/"
@@ -75,15 +79,16 @@ class ObjectStoreDirectAccessDirectorySource implements ObjectStoreDirectorySour
     Set<Resource<BaseStreamResource>> listSubDirectoriesAt(final String path) {
         def resources = []
         def subdirs = [] as Set
-        Pattern directSubDirMatch = ObjectStoreUtils.createSubdirCheckForPath(path)
+        String lstPath = path == "" ? null : path
+        Pattern directSubDirMatch = ObjectStoreUtils.createSubdirCheckForPath(lstPath)
 
-        mClient.listObjects(bucket, path, true).each { result ->
+        mClient.listObjects(bucket, lstPath, true).each { result ->
             Matcher m = directSubDirMatch.matcher(result.get().objectName())
             if(m.matches()) {
                 subdirs.add(m.group(1))
             }
         }
-        subdirs.sort().each { String dirname -> resources.add(new ObjectStoreResource(dirname, null, true)) }
+        subdirs.sort().each { String dirname -> resources.add(new ObjectStoreResource(path+ "/"+dirname, null, true)) }
         return resources
     }
 
@@ -108,9 +113,12 @@ class ObjectStoreDirectAccessDirectorySource implements ObjectStoreDirectorySour
     @Override
     Set<Resource<BaseStreamResource>> listResourceEntriesAt(final String path) {
         def resources = []
-        mClient.listObjects(bucket, path, true)
+        String lstPath = path == "" ? null : path
+        String rPath = path == "" ?: path+"/"
+
+        mClient.listObjects(bucket, lstPath, true)
                .findAll {
-            !(it.get().objectName().replaceAll(path,"") ==~ ObjectStoreTree.nestedSubDirCheck())
+            !(it.get().objectName().replaceAll(rPath,"").contains("/"))
         }.each { result ->
             resources.add(createResourceListItemWithMetadata(result.get()))
         }

plugins/object-store-plugin/src/main/groovy/org/rundeck/plugin/objectstore/directorysource/ObjectStoreMemoryDirectorySource.groovy

@@ -28,6 +28,9 @@ import org.rundeck.storage.api.Resource
 *
 * If the object store is modified by a third part client this directory store will have to be resynced.
 * Currently the way to resync the directory structure is to restart Rundeck.
+*
+* This source works best when using a single instance of Rundeck, where all updates to the object store
+* will be done through the object tree.
  */
 
 class ObjectStoreMemoryDirectorySource implements ObjectStoreDirectorySource {

plugins/object-store-plugin/src/main/groovy/org/rundeck/plugin/objectstore/tree/ObjectStoreUtils.groovy

@@ -24,6 +24,7 @@ import java.util.regex.Pattern
 
 class ObjectStoreUtils {
     static Pattern createSubdirCheckForPath(String path) {
+        if(!path) return ~/(.*?)\/.*/
         return ~/${path}\/(.*?)\/.*/
     }
 

plugins/object-store-plugin/src/test/groovy/org/rundeck/plugin/objectstore/directorysource/ObjectStoreDirectAccessDirectorySourceTest.groovy

@@ -0,0 +1,124 @@
+/*
+ * Copyright 2018 Rundeck, Inc. (http://rundeck.com)
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.rundeck.plugin.objectstore.directorysource
+
+import io.minio.MinioClient
+import org.rundeck.plugin.objectstore.tree.ObjectStoreTree
+import spock.lang.Shared
+import spock.lang.Specification
+import testhelpers.MinioTestServer
+import testhelpers.MinioTestUtils
+
+class ObjectStoreDirectAccessDirectorySourceTest extends Specification {
+    static MinioTestServer server = new MinioTestServer()
+    static MinioClient mClient
+    @Shared
+    ObjectStoreDirectAccessDirectorySource directory
+
+    def setupSpec() {
+        server.start()
+        String bucket = "direct-access-dir-bucket"
+        mClient = new MinioClient("http://localhost:9000", server.accessKey, server.secretKey)
+        if(!mClient.bucketExists(bucket)) mClient.makeBucket(bucket)
+        directory = new ObjectStoreDirectAccessDirectorySource(mClient, bucket)
+
+        MinioTestUtils.ifNotExistAdd(mClient,bucket,"rootfile.test","data", [:])
+        MinioTestUtils.ifNotExistAdd(mClient,bucket,"etc/framework.properties","data",["description":"rundeck framework property file"])
+        MinioTestUtils.ifNotExistAdd(mClient,bucket,"server/config/rundeck-config.properties","data",["description":"main config properties"])
+        MinioTestUtils.ifNotExistAdd(mClient,bucket,"server/config/realm.properties","data",["description":"security data"])
+        MinioTestUtils.ifNotExistAdd(mClient,bucket,"server/logs/server.log","data",[:])
+        MinioTestUtils.ifNotExistAdd(mClient,bucket,"server/data/grailsdb.mv.db","data",[:])
+        MinioTestUtils.ifNotExistAdd(mClient,bucket,"server/random.file","data",[:])
+        MinioTestUtils.ifNotExistAdd(mClient,bucket,"tobedeleted/delete.me","data",[:])
+    }
+
+    void cleanupSpec() {
+        server.stop()
+    }
+
+    def "CheckPathExists"() {
+        expect:
+        directory.checkPathExists("etc/framework.properties")
+        directory.checkPathExists("server/config")
+        !directory.checkPathExists("this/does/not/exist")
+        !directory.checkPathExists("server/config/nonexistantfile")
+    }
+
+    def "CheckResourceExists"() {
+        expect:
+        directory.checkResourceExists("etc/framework.properties")
+        !directory.checkResourceExists("server/config")
+    }
+
+    def "CheckPathExistsAndIsDirectory"() {
+        expect:
+        directory.checkPathExistsAndIsDirectory("server/config")
+        !directory.checkPathExistsAndIsDirectory("etc/framework.properties")
+    }
+
+    def "GetEntryMetadata"() {
+        expect:
+        directory.getEntryMetadata("etc/framework.properties").description == "rundeck framework property file"
+    }
+
+    def "ListSubDirectoriesAt"() {
+        when:
+        def subdirs = directory.listSubDirectoriesAt("server")
+        then:
+        subdirs.size() == 3
+        subdirs[0].path.name == "config"
+        subdirs[0].path.path == "server/config"
+        subdirs[0].directory
+        subdirs[1].path.name == "data"
+        subdirs[1].path.path == "server/data"
+        subdirs[1].directory
+        subdirs[2].path.name == "logs"
+        subdirs[2].path.path == "server/logs"
+        subdirs[2].directory
+    }
+
+    def "ListSubDirectoriesAtRoot"() {
+        when:
+        def subdirs = directory.listSubDirectoriesAt("")
+        then:
+        subdirs.size() == 3
+        subdirs[0].path.name == "etc"
+        subdirs[0].directory
+        subdirs[1].path.name == "server"
+        subdirs[1].directory
+        subdirs[2].path.name == "tobedeleted"
+        subdirs[2].directory
+    }
+
+    def "ListResourceEntriesAt"() {
+        when:
+        def entries = directory.listResourceEntriesAt("server/config")
+        then:
+        entries.size() == 2
+        entries[0].path.name == "realm.properties"
+        entries[0].contents.meta.description == "security data"
+        entries[1].path.name == "rundeck-config.properties"
+        entries[1].contents.meta.description == "main config properties"
+    }
+
+    def "ListResourceEntries At Root"() {
+        when:
+        def entries = directory.listResourceEntriesAt("")
+        then:
+        entries.size() == 1
+        entries[0].path.name == "rootfile.test"
+    }
+}

plugins/object-store-plugin/src/test/groovy/org/rundeck/plugin/objectstore/directorysource/ObjectStoreMemoryDirectorySourceTest.groovy

@@ -104,10 +104,13 @@ class ObjectStoreMemoryDirectorySourceTest extends Specification {
         then:
         subdirs.size() == 3
         subdirs[0].path.name == "config"
+        subdirs[0].path.path == "server/config"
         subdirs[0].directory
         subdirs[1].path.name == "data"
+        subdirs[1].path.path == "server/data"
         subdirs[1].directory
         subdirs[2].path.name == "logs"
+        subdirs[2].path.path == "server/logs"
         subdirs[2].directory
     }