From 06edb7144118a9c990da149ebd56ced1dee8d02f Mon Sep 17 00:00:00 2001 From: Andrew Gaul Date: Wed, 14 Aug 2013 22:09:07 -0700 Subject: [PATCH] Introduce BlobstoreCli This commit provides an alternative and lighter-weight way to interact with blobstores from the command-line. The really-executable-jar is 6 MB instead of jclouds-cli 30 MB and starts up in 2.5 vs. 3.0 seconds. Further it gives better control over configuration and logging. --- pom.xml | 1 + tools/blobstore-cli/pom.xml | 185 +++++++++ .../main/assembly/jar-with-dependencies.xml | 50 +++ .../jclouds/blobstore/cli/BlobStoreCli.java | 392 ++++++++++++++++++ .../src/main/resources/logback.xml | 36 ++ tools/pom.xml | 35 ++ 6 files changed, 699 insertions(+) create mode 100644 tools/blobstore-cli/pom.xml create mode 100644 tools/blobstore-cli/src/main/assembly/jar-with-dependencies.xml create mode 100644 tools/blobstore-cli/src/main/java/org/jclouds/blobstore/cli/BlobStoreCli.java create mode 100644 tools/blobstore-cli/src/main/resources/logback.xml create mode 100644 tools/pom.xml diff --git a/pom.xml b/pom.xml index 07422d1098d..b66019ec3e3 100644 --- a/pom.xml +++ b/pom.xml @@ -41,6 +41,7 @@ skeletons drivers scriptbuilder + tools allcompute allblobstore allloadbalancer diff --git a/tools/blobstore-cli/pom.xml b/tools/blobstore-cli/pom.xml new file mode 100644 index 00000000000..5fce0141c01 --- /dev/null +++ b/tools/blobstore-cli/pom.xml @@ -0,0 +1,185 @@ + + + 4.0.0 + + org.apache.jclouds + jclouds-project + 2.0.0-SNAPSHOT + ../../project/pom.xml + + org.apache.jclouds.tools + blobstore-cli + jclouds blobstore CLI + command-line access to jclouds blobstore features + jar + + + + The Apache Software License, Version 2.0 + http://www.apache.org/licenses/LICENSE-2.0.txt + repo + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.1 + + 1.6 + 1.6 + + + + + + + org.apache.maven.plugins + maven-jar-plugin + + + + test-jar + + + + + + org.apache.maven.plugins + maven-site-plugin + + + org.apache.maven.plugins + maven-assembly-plugin + 2.5.3 + + + src/main/assembly/jar-with-dependencies.xml + + + + org.jclouds.blobstore.cli.BlobStoreCli + true + + + + + + make-assembly + package + + single + + + + + + org.skife.maven + really-executable-jar-maven-plugin + 1.4.1 + + blobstore + + -XX:+TieredCompilation -XX:TieredStopAtLevel=1 -Xverify:none + + + + package + + really-executable-jar + + + + + + + + + + org.apache.jclouds + jclouds-allblobstore + ${project.version} + + + org.apache.jclouds + jclouds-core + ${project.version} + + + org.apache.jclouds.driver + jclouds-slf4j + ${project.version} + + + + org.apache.jclouds.labs + b2 + ${project.version} + + + + org.apache.jclouds.labs + glacier + ${project.version} + + + + org.apache.jclouds.labs + google-cloud-storage + ${project.version} + + + args4j + args4j + 2.33 + + + ch.qos.logback + logback-classic + 1.1.2 + + + ch.qos.logback + logback-core + 1.1.2 + + + org.clojure + clojure + ${clojure.version} + provided + + + org.clojure + tools.logging + 0.2.3 + provided + + + org.clojure + core.incubator + 0.1.0 + provided + + + + diff --git a/tools/blobstore-cli/src/main/assembly/jar-with-dependencies.xml b/tools/blobstore-cli/src/main/assembly/jar-with-dependencies.xml new file mode 100644 index 00000000000..b3e8f27eaa4 --- /dev/null +++ b/tools/blobstore-cli/src/main/assembly/jar-with-dependencies.xml @@ -0,0 +1,50 @@ + + + jar-with-dependencies + + jar + + false + + + metaInf-services + + + + + / + true + true + runtime + + + + + ${project.basedir}/src/main/config + / + + logback.xml + + true + + + diff --git a/tools/blobstore-cli/src/main/java/org/jclouds/blobstore/cli/BlobStoreCli.java b/tools/blobstore-cli/src/main/java/org/jclouds/blobstore/cli/BlobStoreCli.java new file mode 100644 index 00000000000..7a3b6f4672e --- /dev/null +++ b/tools/blobstore-cli/src/main/java/org/jclouds/blobstore/cli/BlobStoreCli.java @@ -0,0 +1,392 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jclouds.blobstore.cli; + +import java.io.File; +import java.io.BufferedInputStream; +import java.io.FileInputStream; +import java.io.InputStream; +import java.io.IOException; +import java.io.PrintStream; +import java.util.Collections; +import java.util.List; +import java.util.Properties; + +import com.google.common.base.Strings; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Lists; +import com.google.common.hash.Hashing; +import com.google.common.io.ByteSource; +import com.google.common.io.ByteStreams; +import com.google.common.io.Files; +import com.google.inject.Module; + +import org.jclouds.Constants; +import org.jclouds.ContextBuilder; +import org.jclouds.blobstore.BlobStore; +import org.jclouds.blobstore.BlobStoreContext; +import org.jclouds.blobstore.KeyNotFoundException; +import org.jclouds.blobstore.domain.Blob; +import org.jclouds.blobstore.domain.BlobBuilder; +import org.jclouds.blobstore.domain.BlobBuilder.PayloadBlobBuilder; +import org.jclouds.blobstore.domain.PageSet; +import org.jclouds.blobstore.domain.StorageMetadata; +import org.jclouds.blobstore.options.ListContainerOptions; +import org.jclouds.blobstore.options.PutOptions; +import org.jclouds.domain.Location; +import org.jclouds.io.Payload; +import org.jclouds.logging.slf4j.config.SLF4JLoggingModule; +import org.jclouds.util.Closeables2; +import org.kohsuke.args4j.Argument; +import org.kohsuke.args4j.CmdLineException; +import org.kohsuke.args4j.CmdLineParser; +import org.kohsuke.args4j.Option; +import org.kohsuke.args4j.ParserProperties; +import org.kohsuke.args4j.spi.SubCommand; +import org.kohsuke.args4j.spi.SubCommandHandler; +import org.kohsuke.args4j.spi.SubCommands; + +// TODO: map exceptions to errno values like jclouds-cli +// TODO: blobstore-cli uses subcommands, e.g., "container list", instead of +// jclouds-cli tokens, e.g., "container-list". + +public final class BlobStoreCli { + @Option(name = "--properties", usage = "properties") + public File propertiesFile = null; + + @Argument(handler = SubCommandHandler.class, required = true, metaVar = "resource", usage = "resource type") + @SubCommands({ + @SubCommand(name = "blob", impl = BlobCommand.class), + @SubCommand(name = "container", impl = ContainerCommand.class) + }) + BlobStoreCommand value; + + public interface BlobStoreCommand { + void run(BlobStore blobStore) throws IOException; + } + + public static final class BlobCommand implements BlobStoreCommand { + @Argument(handler = SubCommandHandler.class, required = true, metaVar = "action", usage = "blob action") + @SubCommands({ + @SubCommand(name = "get", impl = BlobGetCommand.class), + @SubCommand(name = "list", impl = BlobListCommand.class), + @SubCommand(name = "put", impl = BlobPutCommand.class), + @SubCommand(name = "remove", impl = BlobRemoveCommand.class) + }) + BlobStoreCommand value; + @Override + public void run(BlobStore blobStore) throws IOException { + value.run(blobStore); + } + } + + public static final class ContainerCommand implements BlobStoreCommand { + @Argument(handler = SubCommandHandler.class, required = true, metaVar = "action", usage = "container action") + @SubCommands({ + @SubCommand(name = "clear", impl = ContainerClearCommand.class), + @SubCommand(name = "create", impl = ContainerCreateCommand.class), + @SubCommand(name = "delete", impl = ContainerDeleteCommand.class), + @SubCommand(name = "list", impl = ContainerListCommand.class), + @SubCommand(name = "location-list", impl = ContainerLocationListCommand.class) + }) + BlobStoreCommand value; + @Override + public void run(BlobStore blobStore) throws IOException { + value.run(blobStore); + } + } + + public BlobStoreCli(String[] args) throws CmdLineException { + CmdLineParser parser = new CmdLineParser(this, + ParserProperties.defaults().withUsageWidth(80)); + parser.parseArgument(args); + } + + public static void main(String[] args) throws IOException { + BlobStoreCli cli; + try { + cli = new BlobStoreCli(args); + } catch (CmdLineException cle) { + PrintStream out = System.err; + out.println("Unable to parse command line arguments: " + + cle.getMessage()); + out.println("Valid options include:"); + cle.getParser().printUsage(out); + System.exit(1); + return; + } + + Properties properties = new Properties(System.getProperties()); + if (cli.propertiesFile != null) { + InputStream is = new BufferedInputStream(new FileInputStream(cli.propertiesFile)); + try { + properties.load(is); + } finally { + Closeables2.closeQuietly(is); + } + System.setProperties(properties); + } + + BlobStoreContext blobStoreContext = getBlobStoreContext(properties); + try { + cli.value.run(blobStoreContext.getBlobStore()); + } finally { + blobStoreContext.close(); + } + } + + private static BlobStoreContext getBlobStoreContext(Properties properties) { + String provider = properties.getProperty(Constants.PROPERTY_PROVIDER); + String identity = properties.getProperty(Constants.PROPERTY_IDENTITY); + String credential = properties.getProperty(Constants.PROPERTY_CREDENTIAL); + String endpoint = properties.getProperty(Constants.PROPERTY_ENDPOINT); + if (provider == null || identity == null || credential == null) { + throw new IllegalArgumentException("must provide " + + Constants.PROPERTY_PROVIDER + ", " + + Constants.PROPERTY_IDENTITY + ", and " + + Constants.PROPERTY_CREDENTIAL); + } + + ContextBuilder contextBuilder = ContextBuilder + .newBuilder(provider) + .credentials(identity, credential) + .modules(ImmutableList.of(new SLF4JLoggingModule())) + .overrides(properties); + if (endpoint != null) { + contextBuilder = contextBuilder.endpoint(endpoint); + } + return contextBuilder.build(BlobStoreContext.class); + } + + public static final class BlobGetCommand implements BlobStoreCommand { + @Argument(metaVar = "container-name", usage = "container name", + index = 0, required = true) + public String containerName; + + @Argument(metaVar = "remote-name", usage = "remote blob name", + index = 1, required = true) + public String remoteName; + + @Argument(metaVar = "local-name", usage = "local file name (optional)", + index = 2) + public String localName = null; + + @Override + public void run(BlobStore blobStore) throws IOException { + if (localName == null) { + localName = remoteName; + } + + Blob blob = blobStore.getBlob(containerName, remoteName); + if (blob == null) { + throw new KeyNotFoundException(containerName, remoteName, + "Blob does not exist"); + } + Payload payload = blob.getPayload(); + InputStream is = payload.openStream(); + try { + if (localName.equals("-")) { + ByteStreams.copy(is, System.out); + } else { + Files.asByteSink(new File(localName)).writeFrom(is); + } + } finally { + Closeables2.closeQuietly(is); + Closeables2.closeQuietly(payload); + } + } + } + + public static final class BlobListCommand implements BlobStoreCommand { + @Argument(metaVar = "container-name", usage = "Container name", + index = 0, required = true) + public String containerName; + + @Argument(metaVar = "prefix", usage = "List blobs with only this prefix", + index = 1, required = false) + public String prefix; + + @Option(name = "--detailed", usage = "Display details") + public boolean details = false; + + @Option(name = "--recursive", usage = "List blobs recursively") + public boolean recursive = false; + + @Override + public void run(BlobStore blobStore) { + ListContainerOptions options = new ListContainerOptions() + .delimiter("/") + .prefix(prefix); + if (recursive) { + options.recursive(); + } + + while (true) { + PageSet blobs = blobStore.list( + containerName, options); + for (StorageMetadata sm : blobs) { + if (details) { + // TODO: wonky format + // [type=BLOB, id=null, name=foo, location={scope=PROVIDER, id=s3, description=https://storage.googleapis.com}, uri=https://storage.googleapis.com/gaulbackup2/foo, userMetadata={mode=33204, uid=1000, gid=1000, mtime=1402377504}] + System.out.println(blobStore.blobMetadata( + containerName, sm.getName())); + } else { + System.out.println(sm.getName()); + } + } + + String marker = blobs.getNextMarker(); + if (marker == null) { + break; + } + options.afterMarker(marker); + } + } + } + + public static final class BlobPutCommand implements BlobStoreCommand { + @Argument(metaVar = "container-name", usage = "container name", + index = 0, required = true) + public String containerName; + + @Argument(metaVar = "local-name", usage = "local file name", + index = 1, required = true) + public String localName; + + @Argument(metaVar = "remote-name", + usage = "remote blob name (optional)", index = 2, + required = false) + public String remoteName = null; + + @Option(name = "--multipart-upload", usage = "multipart upload") + public boolean multipartUpload = false; + + @Override + public void run(BlobStore blobStore) throws IOException { + if (remoteName == null) { + remoteName = localName; + } + + BlobBuilder blobBuilder = blobStore.blobBuilder(remoteName); + PayloadBlobBuilder payloadBuilder; + if (localName.equals("-")) { + payloadBuilder = blobBuilder.payload(System.in); + } else { + ByteSource byteSource = Files.asByteSource(new File(localName)); + payloadBuilder = blobBuilder.payload(byteSource) + .contentLength(byteSource.size()); + if (!multipartUpload) { + payloadBuilder = payloadBuilder.contentMD5( + byteSource.hash(Hashing.md5())); + } + } + Blob blob = payloadBuilder.build(); + PutOptions putOptions = new PutOptions(multipartUpload); + blobStore.putBlob(containerName, blob, putOptions); + } + } + + public static final class BlobRemoveCommand implements BlobStoreCommand { + @Argument(metaVar = "container-name", usage = "Container name", + index = 0, required = true) + public String containerName; + + @Argument(metaVar = "remote-name", usage = "remote blob name", + index = 1, required = true) + public String blobName; + + @Override + public void run(BlobStore blobStore) { + blobStore.removeBlob(containerName, blobName); + } + } + + public static final class ContainerClearCommand implements BlobStoreCommand { + @Argument(metaVar = "container-name", usage = "Container name", + required = true) + public String containerName; + + @Override + public void run(BlobStore blobStore) { + blobStore.clearContainer(containerName); + } + } + + public static final class ContainerCreateCommand implements BlobStoreCommand { + @Argument(metaVar = "container-name", usage = "Container name", + required = true) + public String containerName; + + @Option(name = "--location", usage = "Container location") + public String locationString; + + @Override + public void run(BlobStore blobStore) { + Location location = null; + if (!Strings.isNullOrEmpty(locationString)) { + for (Location loc : blobStore.listAssignableLocations()) { + if (loc.getId().equalsIgnoreCase(locationString)) { + location = loc; + break; + } + } + if (location == null) { + throw new IllegalArgumentException("unknown location: " + locationString); + } + } + + boolean created = blobStore.createContainerInLocation(location, containerName); + if (!created) { + System.err.println("container not created; does it already exist?"); + } + } + } + + public static final class ContainerDeleteCommand implements BlobStoreCommand { + @Argument(metaVar = "container-name", usage = "Container name", + required = true) + public String containerName; + + @Override + public void run(BlobStore blobStore) { + blobStore.deleteContainer(containerName); + } + } + + public static final class ContainerListCommand implements BlobStoreCommand { + @Override + public void run(BlobStore blobStore) { + List names = Lists.newArrayList(); + for (StorageMetadata container : blobStore.list()) { + names.add(container.getName()); + } + Collections.sort(names); + for (String name : names) { + System.out.println(name); + } + } + } + + public static final class ContainerLocationListCommand implements BlobStoreCommand { + @Override + public void run(BlobStore blobStore) { + for (Location location : blobStore.listAssignableLocations()) { + System.out.println(location.getId()); + } + } + } +} diff --git a/tools/blobstore-cli/src/main/resources/logback.xml b/tools/blobstore-cli/src/main/resources/logback.xml new file mode 100644 index 00000000000..1ee7ec144b2 --- /dev/null +++ b/tools/blobstore-cli/src/main/resources/logback.xml @@ -0,0 +1,36 @@ + + + + + + - %m%n + + + + + + + + + + + + + diff --git a/tools/pom.xml b/tools/pom.xml new file mode 100644 index 00000000000..f400a07d225 --- /dev/null +++ b/tools/pom.xml @@ -0,0 +1,35 @@ + + + + 4.0.0 + + jclouds-project + org.apache.jclouds + 2.0.0-SNAPSHOT + ../project/pom.xml + + org.apache.jclouds.tools + jclouds-tools + pom + jclouds tools + + blobstore-cli + +