Object store as a Rundeck plugin

plugins/object-store-plugin/build.gradle

@@ -0,0 +1,99 @@
+plugins {
+    id 'groovy'
+}
+
+ext.pluginClassNames='org.rundeck.plugin.objectstore.ObjectStorePlugin'
+ext.pluginName = 'Object Store Plugin'
+ext.pluginDescription = 'Stores data in an Amazon S3 compliant object store'
+
+configurations{
+    pluginLibs
+
+    //declare compile to extend from pluginLibs so it inherits the dependencies
+    compile {
+        extendsFrom pluginLibs
+    }
+}
+
+dependencies {
+    // Use the latest Groovy version for building this library
+    compile "org.codehaus.groovy:groovy-all:${groovyVersion}"
+    pluginLibs("io.minio:minio:5.0.1")
+
+    // Use the awesome Spock testing and specification framework
+    testCompile 'org.spockframework:spock-core:1.0-groovy-2.4'
+}
+
+repositories {
+    // Use jcenter for resolving your dependencies.
+    // You can declare any Maven/Ivy/file repository here.
+    jcenter()
+    mavenLocal()
+}
+
+task copyToLib(type: Copy) {
+    into "$buildDir/output/lib"
+    from configurations.pluginLibs
+}
+
+jar {
+    from "$buildDir/output"
+    manifest {
+        attributes 'Rundeck-Plugin-Classnames': pluginClassNames
+        attributes 'Rundeck-Plugin-Name': pluginName
+        attributes 'Rundeck-Plugin-Description': pluginDescription
+        def libList = configurations.pluginLibs.collect { 'lib/' + it.name }.join(' ')
+        attributes 'Rundeck-Plugin-Libs': "${libList}"
+    }
+}
+
+//set jar task to depend on copyToLib
+jar.dependsOn(copyToLib)
+
+apply plugin: 'maven'
+task('createPom').doLast {
+    pom {
+        project {
+            artifactId 'rundeck-object-store-plugin'
+            groupId "org.rundeck.plugins"
+            inceptionYear '2018'
+            packaging 'jar'
+            version version
+            name( 'Rundeck '+pluginName)
+            url 'http://rundeck.org'
+            licenses {
+                license {
+                    name 'The Apache Software License, Version 2.0'
+                    url 'http://www.apache.org/licenses/LICENSE-2.0.txt'
+                    distribution 'repo'
+                }
+            }
+            parent{
+                groupId 'com.dtolabs.rundeck'
+                artifactId "rundeck-bundled-plugins"
+                version(version)
+            }
+            build{
+                plugins{
+                    plugin{
+                        groupId 'org.apache.maven.plugins'
+                        artifactId 'maven-jar-plugin'
+                        version '2.3.2'
+                        configuration{
+                            archive{
+                                manifestEntries{
+                                    'Rundeck-Plugin-Classnames'(pluginClassNames)
+                                    'Rundeck-Plugin-File-Version' version
+                                    'Rundeck-Plugin-Version' rundeckPluginVersion
+                                    'Rundeck-Plugin-Archive' 'true'
+                                    'Rundeck-Plugin-Name' pluginName
+                                    'Rundeck-Plugin-Description' pluginDescription
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+        }
+    }.writeTo("pom.xml")
+}

plugins/object-store-plugin/gradle/wrapper/gradle-wrapper.properties

@@ -0,0 +1,5 @@
+distributionBase=GRADLE_USER_HOME
+distributionPath=wrapper/dists
+distributionUrl=https\://services.gradle.org/distributions/gradle-4.7-bin.zip
+zipStoreBase=GRADLE_USER_HOME
+zipStorePath=wrapper/dists

plugins/object-store-plugin/src/main/groovy/org/rundeck/plugin/objectstore/ObjectStorePlugin.groovy

@@ -0,0 +1,73 @@
+/*
+ * Copyright 2018 Rundeck, Inc. (http://rundeck.com)
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.rundeck.plugin.objectstore
+
+import com.dtolabs.rundeck.core.plugins.Plugin
+import com.dtolabs.rundeck.core.storage.ResourceMeta
+import com.dtolabs.rundeck.plugins.ServiceNameConstants
+import com.dtolabs.rundeck.plugins.descriptions.PluginDescription
+import com.dtolabs.rundeck.plugins.descriptions.PluginProperty
+import com.dtolabs.rundeck.plugins.storage.StoragePlugin
+import io.minio.MinioClient
+import org.rundeck.plugin.objectstore.directorysource.ObjectStoreDirectAccessDirectorySource
+import org.rundeck.plugin.objectstore.tree.ObjectStoreTree
+import org.rundeck.storage.api.Tree
+import org.rundeck.storage.impl.DelegateTree
+
+@Plugin(name = 'object', service = ServiceNameConstants.Storage)
+@PluginDescription(title = 'Object Storage', description = 'Use an Amazon S3 compatible object store as storage layer.')
+class ObjectStorePlugin extends DelegateTree<ResourceMeta> implements StoragePlugin {
+    @PluginProperty(title = 'Bucket', description = 'Base bucket into which objects are stored')
+    String bucket;
+    @PluginProperty(title = 'Object Store Url', description = 'The URL endpoint of the s3 compatible service')
+    String objectStoreUrl;
+    @PluginProperty(title = 'Secret Key', description = 'The secret key use by the client to connect to the service')
+    String secretKey;
+    @PluginProperty(title = 'Access Key', description = 'The access key use by the client to connect to the service')
+    String accessKey;
+    @PluginProperty(title = 'Uncached Object Lookup', description = """Use object store directly to list directory resources, check object existence, etc. 
+                                                                    Depending on the directory structure and number of objects, enabling this option could have
+                                                                    performance issues. This option will work better in a cluster because all servers in the cluster will
+                                                                    have coordinated access to the objects managed by the plugins. NOTE: The cached object directory does
+                                                                    not share the cache between servers, so it is not best to use it when operating a Rundeck cluster.""")
+    boolean uncachedObjectLookup = false;
+
+    Tree<ResourceMeta> delegateTree
+
+    @Override
+    Tree<ResourceMeta> getDelegate() {
+        if (!delegateTree) {
+            initTree()
+        }
+        return delegateTree
+    }
+
+    void initTree() {
+        if (!bucket) {
+            throw new IllegalArgumentException("bucket property is required")
+        }
+        if (!objectStoreUrl) {
+            throw new IllegalArgumentException("objectStoreUrl property is required")
+        }
+
+        MinioClient mClient = new MinioClient(objectStoreUrl, accessKey, secretKey)
+        if(uncachedObjectLookup) {
+            delegateTree = new ObjectStoreTree(mClient,bucket,new ObjectStoreDirectAccessDirectorySource(mClient,bucket))
+        } else {
+            delegateTree = new ObjectStoreTree(mClient, bucket)
+        }
+    }
+}

plugins/object-store-plugin/src/main/groovy/org/rundeck/plugin/objectstore/directorysource/ObjectStoreDirectAccessDirectorySource.groovy

@@ -0,0 +1,147 @@
+/*
+ * Copyright 2018 Rundeck, Inc. (http://rundeck.com)
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.rundeck.plugin.objectstore.directorysource
+
+import com.dtolabs.rundeck.core.storage.BaseStreamResource
+import io.minio.MinioClient
+import io.minio.errors.ErrorResponseException
+import io.minio.messages.Item
+import org.rundeck.plugin.objectstore.stream.LazyAccessObjectStoreInputStream
+import org.rundeck.plugin.objectstore.tree.ObjectStoreResource
+import org.rundeck.plugin.objectstore.tree.ObjectStoreTree
+import org.rundeck.plugin.objectstore.tree.ObjectStoreUtils
+import org.rundeck.storage.api.Resource
+
+import java.util.regex.Matcher
+import java.util.regex.Pattern
+
+/**
+ * Uses the object client to directly access the object store to get directory information.
+ * For stores with lots of objects this could be a very inefficient directory access mechanism.
+ *
+ * This store works best when the object store is going to be accessed by multiple cluster members
+ * or the object store is regularly updated by third party tools
+ */
+class ObjectStoreDirectAccessDirectorySource implements ObjectStoreDirectorySource {
+    private static final String DIR_MARKER = "/"
+    private final String bucket
+    private final MinioClient mClient
+
+    ObjectStoreDirectAccessDirectorySource(MinioClient mClient, String bucket) {
+        this.mClient = mClient
+        this.bucket = bucket
+    }
+
+    @Override
+    boolean checkPathExists(final String path) {
+        def items = mClient.listObjects(bucket, path)
+        return items.size() > 0
+    }
+
+    @Override
+    boolean checkResourceExists(final String path) {
+        try {
+            mClient.statObject(bucket, path)
+            return true
+        } catch(ErrorResponseException erex) {
+            if(erex.response.code() == 404) return false
+        }
+
+        return false
+    }
+
+    @Override
+    boolean checkPathExistsAndIsDirectory(final String path) {
+        def items = mClient.listObjects(bucket, path, false)
+        if(items.size() != 1) return false
+        return items[0].get().objectName().endsWith(DIR_MARKER)
+    }
+
+    @Override
+    Map<String, String> getEntryMetadata(final String path) {
+        return ObjectStoreUtils.objectStatToMap(mClient.statObject(bucket, path))
+    }
+
+    @Override
+    Set<Resource<BaseStreamResource>> listSubDirectoriesAt(final String path) {
+        def resources = []
+        def subdirs = [] as Set
+        String lstPath = path == "" ? null : path
+        Pattern directSubDirMatch = ObjectStoreUtils.createSubdirCheckForPath(lstPath)
+
+        mClient.listObjects(bucket, lstPath, true).each { result ->
+            Matcher m = directSubDirMatch.matcher(result.get().objectName())
+            if(m.matches()) {
+                subdirs.add(m.group(1))
+            }
+        }
+        subdirs.sort().each { String dirname -> resources.add(new ObjectStoreResource(path+ "/"+dirname, null, true)) }
+        return resources
+    }
+
+    @Override
+    Set<Resource<BaseStreamResource>> listEntriesAndSubDirectoriesAt(final String path) {
+        def resources = []
+        Pattern directSubDirMatch = ObjectStoreUtils.createSubdirCheckForPath(path)
+        def subdirs = [] as Set
+
+        mClient.listObjects(bucket, path, true).each { result ->
+            Matcher m = directSubDirMatch.matcher(result.get().objectName())
+            if(m.matches()) {
+                subdirs.add(path+DIR_MARKER+m.group(1))
+            } else {
+                resources.add(createResourceListItemWithMetadata(result.get()))
+            }
+        }
+        subdirs.sort().each { String dirname -> resources.add(new ObjectStoreResource(dirname, null, true)) }
+        return resources
+    }
+
+    @Override
+    Set<Resource<BaseStreamResource>> listResourceEntriesAt(final String path) {
+        def resources = []
+        String lstPath = path == "" ? null : path
+        String rPath = path == "" ?: path+"/"
+
+        mClient.listObjects(bucket, lstPath, true)
+               .findAll {
+            !(it.get().objectName().replaceAll(rPath,"").contains("/"))
+        }.each { result ->
+            resources.add(createResourceListItemWithMetadata(result.get()))
+        }
+        return resources
+    }
+
+    private ObjectStoreResource createResourceListItemWithMetadata(final Item item) {
+        BaseStreamResource content = new BaseStreamResource(getEntryMetadata(item.objectName()), new LazyAccessObjectStoreInputStream(mClient, bucket, item.objectName()))
+        return new ObjectStoreResource(item.objectName(), content)
+    }
+
+    @Override
+    void updateEntry(final String fullEntryPath, final Map<String, String> meta) {
+        //no-op no additional action needed
+    }
+
+    @Override
+    void deleteEntry(final String fullEntryPath) {
+        //no-op no additional action needed
+    }
+
+    @Override
+    void resyncDirectory() {
+        //no-op
+    }
+}

plugins/object-store-plugin/src/main/groovy/org/rundeck/plugin/objectstore/directorysource/ObjectStoreDirectorySource.groovy

@@ -0,0 +1,33 @@
+/*
+ * Copyright 2018 Rundeck, Inc. (http://rundeck.com)
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.rundeck.plugin.objectstore.directorysource
+
+import com.dtolabs.rundeck.core.storage.BaseStreamResource
+import org.rundeck.storage.api.Resource
+
+
+interface ObjectStoreDirectorySource {
+    boolean checkPathExists(String path)
+    boolean checkResourceExists(String path)
+    boolean checkPathExistsAndIsDirectory(String path)
+    Map<String,String> getEntryMetadata(String path)
+    Set<Resource<BaseStreamResource>> listSubDirectoriesAt(String path)
+    Set<Resource<BaseStreamResource>> listEntriesAndSubDirectoriesAt(String path)
+    Set<Resource<BaseStreamResource>> listResourceEntriesAt(String path)
+    void updateEntry(String fullEntryPath, Map<String,String> meta)
+    void deleteEntry(String fullEntryPath)
+    void resyncDirectory()
+}