-
Notifications
You must be signed in to change notification settings - Fork 899
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #3908 from rundeck/feature/object-store-rundeck-pl…
…ugin Object store as a Rundeck plugin
- Loading branch information
Showing
25 changed files
with
1,939 additions
and
4 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,99 @@ | ||
plugins { | ||
id 'groovy' | ||
} | ||
|
||
ext.pluginClassNames='org.rundeck.plugin.objectstore.ObjectStorePlugin' | ||
ext.pluginName = 'Object Store Plugin' | ||
ext.pluginDescription = 'Stores data in an Amazon S3 compliant object store' | ||
|
||
configurations{ | ||
pluginLibs | ||
|
||
//declare compile to extend from pluginLibs so it inherits the dependencies | ||
compile { | ||
extendsFrom pluginLibs | ||
} | ||
} | ||
|
||
dependencies { | ||
// Use the latest Groovy version for building this library | ||
compile "org.codehaus.groovy:groovy-all:${groovyVersion}" | ||
pluginLibs("io.minio:minio:5.0.1") | ||
|
||
// Use the awesome Spock testing and specification framework | ||
testCompile 'org.spockframework:spock-core:1.0-groovy-2.4' | ||
} | ||
|
||
repositories { | ||
// Use jcenter for resolving your dependencies. | ||
// You can declare any Maven/Ivy/file repository here. | ||
jcenter() | ||
mavenLocal() | ||
} | ||
|
||
task copyToLib(type: Copy) { | ||
into "$buildDir/output/lib" | ||
from configurations.pluginLibs | ||
} | ||
|
||
jar { | ||
from "$buildDir/output" | ||
manifest { | ||
attributes 'Rundeck-Plugin-Classnames': pluginClassNames | ||
attributes 'Rundeck-Plugin-Name': pluginName | ||
attributes 'Rundeck-Plugin-Description': pluginDescription | ||
def libList = configurations.pluginLibs.collect { 'lib/' + it.name }.join(' ') | ||
attributes 'Rundeck-Plugin-Libs': "${libList}" | ||
} | ||
} | ||
|
||
//set jar task to depend on copyToLib | ||
jar.dependsOn(copyToLib) | ||
|
||
apply plugin: 'maven' | ||
task('createPom').doLast { | ||
pom { | ||
project { | ||
artifactId 'rundeck-object-store-plugin' | ||
groupId "org.rundeck.plugins" | ||
inceptionYear '2018' | ||
packaging 'jar' | ||
version version | ||
name( 'Rundeck '+pluginName) | ||
url 'http://rundeck.org' | ||
licenses { | ||
license { | ||
name 'The Apache Software License, Version 2.0' | ||
url 'http://www.apache.org/licenses/LICENSE-2.0.txt' | ||
distribution 'repo' | ||
} | ||
} | ||
parent{ | ||
groupId 'com.dtolabs.rundeck' | ||
artifactId "rundeck-bundled-plugins" | ||
version(version) | ||
} | ||
build{ | ||
plugins{ | ||
plugin{ | ||
groupId 'org.apache.maven.plugins' | ||
artifactId 'maven-jar-plugin' | ||
version '2.3.2' | ||
configuration{ | ||
archive{ | ||
manifestEntries{ | ||
'Rundeck-Plugin-Classnames'(pluginClassNames) | ||
'Rundeck-Plugin-File-Version' version | ||
'Rundeck-Plugin-Version' rundeckPluginVersion | ||
'Rundeck-Plugin-Archive' 'true' | ||
'Rundeck-Plugin-Name' pluginName | ||
'Rundeck-Plugin-Description' pluginDescription | ||
} | ||
} | ||
} | ||
} | ||
} | ||
} | ||
} | ||
}.writeTo("pom.xml") | ||
} |
Binary file not shown.
5 changes: 5 additions & 0 deletions
5
plugins/object-store-plugin/gradle/wrapper/gradle-wrapper.properties
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,5 @@ | ||
distributionBase=GRADLE_USER_HOME | ||
distributionPath=wrapper/dists | ||
distributionUrl=https\://services.gradle.org/distributions/gradle-4.7-bin.zip | ||
zipStoreBase=GRADLE_USER_HOME | ||
zipStorePath=wrapper/dists |
73 changes: 73 additions & 0 deletions
73
...ject-store-plugin/src/main/groovy/org/rundeck/plugin/objectstore/ObjectStorePlugin.groovy
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,73 @@ | ||
/* | ||
* Copyright 2018 Rundeck, Inc. (http://rundeck.com) | ||
* | ||
* Licensed under the Apache License, Version 2.0 (the "License"); | ||
* you may not use this file except in compliance with the License. | ||
* You may obtain a copy of the License at | ||
* | ||
* http://www.apache.org/licenses/LICENSE-2.0 | ||
* | ||
* Unless required by applicable law or agreed to in writing, software | ||
* distributed under the License is distributed on an "AS IS" BASIS, | ||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
* See the License for the specific language governing permissions and | ||
* limitations under the License. | ||
*/ | ||
package org.rundeck.plugin.objectstore | ||
|
||
import com.dtolabs.rundeck.core.plugins.Plugin | ||
import com.dtolabs.rundeck.core.storage.ResourceMeta | ||
import com.dtolabs.rundeck.plugins.ServiceNameConstants | ||
import com.dtolabs.rundeck.plugins.descriptions.PluginDescription | ||
import com.dtolabs.rundeck.plugins.descriptions.PluginProperty | ||
import com.dtolabs.rundeck.plugins.storage.StoragePlugin | ||
import io.minio.MinioClient | ||
import org.rundeck.plugin.objectstore.directorysource.ObjectStoreDirectAccessDirectorySource | ||
import org.rundeck.plugin.objectstore.tree.ObjectStoreTree | ||
import org.rundeck.storage.api.Tree | ||
import org.rundeck.storage.impl.DelegateTree | ||
|
||
@Plugin(name = 'object', service = ServiceNameConstants.Storage) | ||
@PluginDescription(title = 'Object Storage', description = 'Use an Amazon S3 compatible object store as storage layer.') | ||
class ObjectStorePlugin extends DelegateTree<ResourceMeta> implements StoragePlugin { | ||
@PluginProperty(title = 'Bucket', description = 'Base bucket into which objects are stored') | ||
String bucket; | ||
@PluginProperty(title = 'Object Store Url', description = 'The URL endpoint of the s3 compatible service') | ||
String objectStoreUrl; | ||
@PluginProperty(title = 'Secret Key', description = 'The secret key use by the client to connect to the service') | ||
String secretKey; | ||
@PluginProperty(title = 'Access Key', description = 'The access key use by the client to connect to the service') | ||
String accessKey; | ||
@PluginProperty(title = 'Uncached Object Lookup', description = """Use object store directly to list directory resources, check object existence, etc. | ||
Depending on the directory structure and number of objects, enabling this option could have | ||
performance issues. This option will work better in a cluster because all servers in the cluster will | ||
have coordinated access to the objects managed by the plugins. NOTE: The cached object directory does | ||
not share the cache between servers, so it is not best to use it when operating a Rundeck cluster.""") | ||
boolean uncachedObjectLookup = false; | ||
|
||
Tree<ResourceMeta> delegateTree | ||
|
||
@Override | ||
Tree<ResourceMeta> getDelegate() { | ||
if (!delegateTree) { | ||
initTree() | ||
} | ||
return delegateTree | ||
} | ||
|
||
void initTree() { | ||
if (!bucket) { | ||
throw new IllegalArgumentException("bucket property is required") | ||
} | ||
if (!objectStoreUrl) { | ||
throw new IllegalArgumentException("objectStoreUrl property is required") | ||
} | ||
|
||
MinioClient mClient = new MinioClient(objectStoreUrl, accessKey, secretKey) | ||
if(uncachedObjectLookup) { | ||
delegateTree = new ObjectStoreTree(mClient,bucket,new ObjectStoreDirectAccessDirectorySource(mClient,bucket)) | ||
} else { | ||
delegateTree = new ObjectStoreTree(mClient, bucket) | ||
} | ||
} | ||
} |
147 changes: 147 additions & 0 deletions
147
.../rundeck/plugin/objectstore/directorysource/ObjectStoreDirectAccessDirectorySource.groovy
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,147 @@ | ||
/* | ||
* Copyright 2018 Rundeck, Inc. (http://rundeck.com) | ||
* | ||
* Licensed under the Apache License, Version 2.0 (the "License"); | ||
* you may not use this file except in compliance with the License. | ||
* You may obtain a copy of the License at | ||
* | ||
* http://www.apache.org/licenses/LICENSE-2.0 | ||
* | ||
* Unless required by applicable law or agreed to in writing, software | ||
* distributed under the License is distributed on an "AS IS" BASIS, | ||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
* See the License for the specific language governing permissions and | ||
* limitations under the License. | ||
*/ | ||
package org.rundeck.plugin.objectstore.directorysource | ||
|
||
import com.dtolabs.rundeck.core.storage.BaseStreamResource | ||
import io.minio.MinioClient | ||
import io.minio.errors.ErrorResponseException | ||
import io.minio.messages.Item | ||
import org.rundeck.plugin.objectstore.stream.LazyAccessObjectStoreInputStream | ||
import org.rundeck.plugin.objectstore.tree.ObjectStoreResource | ||
import org.rundeck.plugin.objectstore.tree.ObjectStoreTree | ||
import org.rundeck.plugin.objectstore.tree.ObjectStoreUtils | ||
import org.rundeck.storage.api.Resource | ||
|
||
import java.util.regex.Matcher | ||
import java.util.regex.Pattern | ||
|
||
/** | ||
* Uses the object client to directly access the object store to get directory information. | ||
* For stores with lots of objects this could be a very inefficient directory access mechanism. | ||
* | ||
* This store works best when the object store is going to be accessed by multiple cluster members | ||
* or the object store is regularly updated by third party tools | ||
*/ | ||
class ObjectStoreDirectAccessDirectorySource implements ObjectStoreDirectorySource { | ||
private static final String DIR_MARKER = "/" | ||
private final String bucket | ||
private final MinioClient mClient | ||
|
||
ObjectStoreDirectAccessDirectorySource(MinioClient mClient, String bucket) { | ||
this.mClient = mClient | ||
this.bucket = bucket | ||
} | ||
|
||
@Override | ||
boolean checkPathExists(final String path) { | ||
def items = mClient.listObjects(bucket, path) | ||
return items.size() > 0 | ||
} | ||
|
||
@Override | ||
boolean checkResourceExists(final String path) { | ||
try { | ||
mClient.statObject(bucket, path) | ||
return true | ||
} catch(ErrorResponseException erex) { | ||
if(erex.response.code() == 404) return false | ||
} | ||
|
||
return false | ||
} | ||
|
||
@Override | ||
boolean checkPathExistsAndIsDirectory(final String path) { | ||
def items = mClient.listObjects(bucket, path, false) | ||
if(items.size() != 1) return false | ||
return items[0].get().objectName().endsWith(DIR_MARKER) | ||
} | ||
|
||
@Override | ||
Map<String, String> getEntryMetadata(final String path) { | ||
return ObjectStoreUtils.objectStatToMap(mClient.statObject(bucket, path)) | ||
} | ||
|
||
@Override | ||
Set<Resource<BaseStreamResource>> listSubDirectoriesAt(final String path) { | ||
def resources = [] | ||
def subdirs = [] as Set | ||
String lstPath = path == "" ? null : path | ||
Pattern directSubDirMatch = ObjectStoreUtils.createSubdirCheckForPath(lstPath) | ||
|
||
mClient.listObjects(bucket, lstPath, true).each { result -> | ||
Matcher m = directSubDirMatch.matcher(result.get().objectName()) | ||
if(m.matches()) { | ||
subdirs.add(m.group(1)) | ||
} | ||
} | ||
subdirs.sort().each { String dirname -> resources.add(new ObjectStoreResource(path+ "/"+dirname, null, true)) } | ||
return resources | ||
} | ||
|
||
@Override | ||
Set<Resource<BaseStreamResource>> listEntriesAndSubDirectoriesAt(final String path) { | ||
def resources = [] | ||
Pattern directSubDirMatch = ObjectStoreUtils.createSubdirCheckForPath(path) | ||
def subdirs = [] as Set | ||
|
||
mClient.listObjects(bucket, path, true).each { result -> | ||
Matcher m = directSubDirMatch.matcher(result.get().objectName()) | ||
if(m.matches()) { | ||
subdirs.add(path+DIR_MARKER+m.group(1)) | ||
} else { | ||
resources.add(createResourceListItemWithMetadata(result.get())) | ||
} | ||
} | ||
subdirs.sort().each { String dirname -> resources.add(new ObjectStoreResource(dirname, null, true)) } | ||
return resources | ||
} | ||
|
||
@Override | ||
Set<Resource<BaseStreamResource>> listResourceEntriesAt(final String path) { | ||
def resources = [] | ||
String lstPath = path == "" ? null : path | ||
String rPath = path == "" ?: path+"/" | ||
|
||
mClient.listObjects(bucket, lstPath, true) | ||
.findAll { | ||
!(it.get().objectName().replaceAll(rPath,"").contains("/")) | ||
}.each { result -> | ||
resources.add(createResourceListItemWithMetadata(result.get())) | ||
} | ||
return resources | ||
} | ||
|
||
private ObjectStoreResource createResourceListItemWithMetadata(final Item item) { | ||
BaseStreamResource content = new BaseStreamResource(getEntryMetadata(item.objectName()), new LazyAccessObjectStoreInputStream(mClient, bucket, item.objectName())) | ||
return new ObjectStoreResource(item.objectName(), content) | ||
} | ||
|
||
@Override | ||
void updateEntry(final String fullEntryPath, final Map<String, String> meta) { | ||
//no-op no additional action needed | ||
} | ||
|
||
@Override | ||
void deleteEntry(final String fullEntryPath) { | ||
//no-op no additional action needed | ||
} | ||
|
||
@Override | ||
void resyncDirectory() { | ||
//no-op | ||
} | ||
} |
33 changes: 33 additions & 0 deletions
33
...n/groovy/org/rundeck/plugin/objectstore/directorysource/ObjectStoreDirectorySource.groovy
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,33 @@ | ||
/* | ||
* Copyright 2018 Rundeck, Inc. (http://rundeck.com) | ||
* | ||
* Licensed under the Apache License, Version 2.0 (the "License"); | ||
* you may not use this file except in compliance with the License. | ||
* You may obtain a copy of the License at | ||
* | ||
* http://www.apache.org/licenses/LICENSE-2.0 | ||
* | ||
* Unless required by applicable law or agreed to in writing, software | ||
* distributed under the License is distributed on an "AS IS" BASIS, | ||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
* See the License for the specific language governing permissions and | ||
* limitations under the License. | ||
*/ | ||
package org.rundeck.plugin.objectstore.directorysource | ||
|
||
import com.dtolabs.rundeck.core.storage.BaseStreamResource | ||
import org.rundeck.storage.api.Resource | ||
|
||
|
||
interface ObjectStoreDirectorySource { | ||
boolean checkPathExists(String path) | ||
boolean checkResourceExists(String path) | ||
boolean checkPathExistsAndIsDirectory(String path) | ||
Map<String,String> getEntryMetadata(String path) | ||
Set<Resource<BaseStreamResource>> listSubDirectoriesAt(String path) | ||
Set<Resource<BaseStreamResource>> listEntriesAndSubDirectoriesAt(String path) | ||
Set<Resource<BaseStreamResource>> listResourceEntriesAt(String path) | ||
void updateEntry(String fullEntryPath, Map<String,String> meta) | ||
void deleteEntry(String fullEntryPath) | ||
void resyncDirectory() | ||
} |
Oops, something went wrong.