Permalink
Browse files

Implement Librados conncetor for cosbench

  • Loading branch information...
1 parent ab1483d commit a2e79a5cb9a8d68b9afc124fa20fb3cc65bf2e82 Niklas Goerke committed with Niklas974 May 27, 2013
View

Large diffs are not rendered by default.

Oops, something went wrong.
View
@@ -0,0 +1,17 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<projectDescription>
+ <name>cosbench</name>
+ <comment></comment>
+ <projects>
+ </projects>
+ <buildSpec>
+ <buildCommand>
+ <name>org.eclipse.jdt.core.javabuilder</name>
+ <arguments>
+ </arguments>
+ </buildCommand>
+ </buildSpec>
+ <natures>
+ <nature>org.eclipse.jdt.core.javanature</nature>
+ </natures>
+</projectDescription>
@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<classpath>
+ <classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.6"/>
+ <classpathentry kind="con" path="org.eclipse.pde.core.requiredPlugins"/>
+ <classpathentry kind="src" path="src"/>
+ <classpathentry kind="src" path="test"/>
+ <classpathentry kind="lib" path="rados-0.1.1.jar" sourcepath="/rados-java"/>
+ <classpathentry kind="output" path="bin"/>
+</classpath>
@@ -0,0 +1,29 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<projectDescription>
+ <name>cosbench-librados</name>
+ <comment></comment>
+ <projects>
+ <project>cosbench</project>
+ </projects>
+ <buildSpec>
+ <buildCommand>
+ <name>org.eclipse.jdt.core.javabuilder</name>
+ <arguments>
+ </arguments>
+ </buildCommand>
+ <buildCommand>
+ <name>org.eclipse.pde.ManifestBuilder</name>
+ <arguments>
+ </arguments>
+ </buildCommand>
+ <buildCommand>
+ <name>org.eclipse.pde.SchemaBuilder</name>
+ <arguments>
+ </arguments>
+ </buildCommand>
+ </buildSpec>
+ <natures>
+ <nature>org.eclipse.pde.PluginNature</nature>
+ <nature>org.eclipse.jdt.core.javanature</nature>
+ </natures>
+</projectDescription>
@@ -0,0 +1,7 @@
+eclipse.preferences.version=1
+org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled
+org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.6
+org.eclipse.jdt.core.compiler.compliance=1.6
+org.eclipse.jdt.core.compiler.problem.assertIdentifier=error
+org.eclipse.jdt.core.compiler.problem.enumIdentifier=error
+org.eclipse.jdt.core.compiler.source=1.6
@@ -0,0 +1,5 @@
+#Tue May 15 09:34:00 CST 2012
+eclipse.preferences.version=1
+pluginProject.equinox=false
+pluginProject.extensions=false
+resolve.requirebundle=false
@@ -0,0 +1,17 @@
+Manifest-Version: 1.0
+Bundle-ManifestVersion: 2
+Bundle-Name: Librados Storage Client Bundle
+Bundle-SymbolicName: cosbench-librados
+Bundle-Version: 0.3.1.0
+Bundle-Vendor: intel
+Bundle-RequiredExecutionEnvironment: JavaSE-1.6
+Bundle-ClassPath: .,
+ rados-0.1.1.jar,
+ jna-3.5.2.jar
+import-Package: com.intel.cosbench.api.auth,
+ com.intel.cosbench.api.context,
+ com.intel.cosbench.api.storage,
+ com.intel.cosbench.client.http,
+ com.intel.cosbench.config,
+ com.intel.cosbench.config.common,
+ com.intel.cosbench.log
@@ -0,0 +1,17 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<beans xmlns="http://www.springframework.org/schema/beans"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:osgi="http://www.springframework.org/schema/osgi"
+ xsi:schemaLocation="
+ http://www.springframework.org/schema/beans
+ http://www.springframework.org/schema/beans/spring-beans-3.0.xsd
+ http://www.springframework.org/schema/osgi
+ http://www.springframework.org/schema/osgi/spring-osgi.xsd">
+
+ <bean name="storageFactory" class="com.intel.cosbench.api.librados.LibradosStorageFactory" />
+
+ <osgi:service ref="storageFactory" context-class-loader="service-provider"
+ interface="com.intel.cosbench.api.storage.StorageAPIFactory">
+ </osgi:service>
+
+</beans>
@@ -0,0 +1,6 @@
+source.. = src/
+output.. = bin/
+bin.includes = META-INF/,\
+ rados-0.1.1.jar,\
+ jna-3.5.2.jar,\
+ .
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,163 @@
+/**
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+ */
+
+package com.intel.cosbench.api.librados;
+
+import static com.intel.cosbench.client.librados.LibradosConstants.AUTH_PASSWORD_DEFAULT;
+import static com.intel.cosbench.client.librados.LibradosConstants.AUTH_PASSWORD_KEY;
+import static com.intel.cosbench.client.librados.LibradosConstants.AUTH_USERNAME_DEFAULT;
+import static com.intel.cosbench.client.librados.LibradosConstants.AUTH_USERNAME_KEY;
+import static com.intel.cosbench.client.librados.LibradosConstants.ENDPOINT_DEFAULT;
+import static com.intel.cosbench.client.librados.LibradosConstants.ENDPOINT_KEY;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+
+import com.ceph.rados.IoCTX;
+import com.ceph.rados.Rados;
+import com.ceph.rados.RadosException;
+import com.intel.cosbench.api.context.AuthContext;
+import com.intel.cosbench.api.storage.NoneStorage;
+import com.intel.cosbench.api.storage.StorageException;
+import com.intel.cosbench.config.Config;
+import com.intel.cosbench.log.Logger;
+
+/**
+ * LibradosStorage provides methods to access a Storage using librados.
+ * It is based on rados-java
+ * {@link https://github.com/wido/rados-java}
+ *
+ *
+ * @author Niklas Goerke - niklas974@github
+ *
+ */
+public class LibradosStorage extends NoneStorage {
+
+ private String accessKey;
+ private String secretKey;
+ private String endpoint;
+
+ private static Rados client;
+
+ public void init(Config config, Logger logger) {
+ super.init(config, logger);
+
+ this.endpoint = config.get(ENDPOINT_KEY, ENDPOINT_DEFAULT);
+ this.accessKey = config.get(AUTH_USERNAME_KEY, AUTH_USERNAME_DEFAULT);
+ this.secretKey = config.get(AUTH_PASSWORD_KEY, AUTH_PASSWORD_DEFAULT);
+
+ parms.put(ENDPOINT_KEY, endpoint);
+ parms.put(AUTH_USERNAME_KEY, accessKey);
+ parms.put(AUTH_PASSWORD_KEY, secretKey);
+ logger.debug("using storage config: {}", parms);
+
+ if (client == null) {
+ client = new Rados(this.accessKey);
+ try {
+ client.confSet("key", this.secretKey);
+ client.confSet("mon_host", this.endpoint);
+ client.connect();
+ logger.debug("Librados client has been initialized");
+ } catch (RadosException e) {
+ throw new StorageException(e);
+ }
+ }
+ }
+
+ public void setAuthContext(AuthContext info) {
+ super.setAuthContext(info);
+ }
+
+ public void dispose() {
+ super.dispose();
+// client = null;
+ }
+
+ public InputStream getObject(String container, String object, Config config) {
+ super.getObject(container, object, config);
+ InputStream stream;
+
+ logger.info("Retrieving " + container + "\\" + object);
+ IoCTX ioctx;
+ try {
+ ioctx = client.ioCtxCreate(container);
+ long length = ioctx.stat(object).getSize();
+ if (length > (Math.pow(2, 31) - 1)) {
+ throw new StorageException("Object larger than 2GB, handling not implemented");
+ // TODO: implement - read in parts and concatinate
+ }
+
+ byte[] buf = new byte[(int) length];
+ ioctx.read(object, (int) length, 0, buf);
+ stream = new ByteArrayInputStream(buf);
+ } catch (RadosException e) {
+ throw new StorageException(e);
+ }
+ client.ioCtxDestroy(ioctx);
+ return stream;
+ }
+
+ public void createContainer(String container, Config config) {
+ super.createContainer(container, config);
+ try {
+ boolean exists = false;
+ for (String pool : client.poolList()) {
+ if (pool.equals(container)) {
+ exists = true;
+ }
+ }
+ if (!exists) {
+ client.poolCreate(container);
+ }
+ } catch (RadosException e) {
+ throw new StorageException(e);
+ }
+ }
+
+ public void deleteContainer(String container, Config config) {
+ super.deleteContainer(container, config);
+ try {
+ client.poolDelete(container);
+ } catch (RadosException e) {
+ throw new StorageException(e);
+ }
+ }
+
+ public void createObject(String container, String object, InputStream data, long length, Config config) {
+ super.createObject(container, object, data, length, config);
+ byte[] buf = new byte[(int) length];
+ try {
+ data.read(buf, 0, (int) length);
+ IoCTX ioctx = client.ioCtxCreate(container);
+ ioctx.write(object, buf);
+ } catch (RadosException e) {
+ throw new StorageException(e);
+ } catch (IOException e) {
+ throw new StorageException(e);
+ }
+ }
+
+ public void deleteObject(String container, String object, Config config) {
+ super.deleteObject(container, object, config);
+ IoCTX ioctx;
+ try {
+ ioctx = client.ioCtxCreate(container);
+ ioctx.remove(object);
+ } catch (RadosException e) {
+ throw new StorageException(e);
+ }
+ }
+}
@@ -0,0 +1,37 @@
+/**
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+ */
+package com.intel.cosbench.api.librados;
+
+import com.intel.cosbench.api.storage.*;
+
+/**
+ *
+ * @author Niklas Goerke - niklas974@github
+ *
+ */
+
+public class LibradosStorageFactory implements StorageAPIFactory {
+
+ @Override
+ public String getStorageName() {
+ return "librados";
+ }
+
+ @Override
+ public StorageAPI getStorageAPI() {
+ return new LibradosStorage();
+ }
+
+}
@@ -0,0 +1,57 @@
+/**
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+ */
+package com.intel.cosbench.client.librados;
+
+/**
+ *
+ * @author Niklas Goerke - niklas974@github
+ *
+ */
+
+public interface LibradosConstants {
+ // --------------------------------------------------------------------------
+ // CONNECTION
+ // --------------------------------------------------------------------------
+
+ // String CONN_TIMEOUT_KEY = "timeout";
+ // int CONN_TIMEOUT_DEFAULT = 30000;
+ // --------------------------------------------------------------------------
+ // ENDPOINT
+ // --------------------------------------------------------------------------
+ String ENDPOINT_KEY = "endpoint";
+ String ENDPOINT_DEFAULT = "http://s3.amazonaws.com";
+
+ // --------------------------------------------------------------------------
+ // AUTHENTICATION
+ // --------------------------------------------------------------------------
+
+ String AUTH_USERNAME_KEY = "accesskey";
+ String AUTH_USERNAME_DEFAULT = "";
+
+ String AUTH_PASSWORD_KEY = "secretkey";
+ String AUTH_PASSWORD_DEFAULT = "";
+
+ // --------------------------------------------------------------------------
+ // CLIENT CONFIGURATION
+ // --------------------------------------------------------------------------
+
+ // String PROXY_HOST_KEY = "proxyhost";
+ // String PROXY_PORT_KEY = "proxyport";
+
+ // --------------------------------------------------------------------------
+ // CONTEXT NEEDS FROM AUTH MODULE
+ // --------------------------------------------------------------------------
+ // String S3CLIENT_KEY = "s3client";
+}
Oops, something went wrong.

0 comments on commit a2e79a5

Please sign in to comment.