Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

gh-93 New ElasticSearch Database Implementation

  • Loading branch information...
commit 3827d9a303ac58854e3615dcdfad04eedf23746a 1 parent 0caa77f
saden1 authored Michi Mutsuzaki committed
1  .gitignore
View
@@ -0,0 +1 @@
+/elasticsearch/target/
1  CHANGELOG
View
@@ -2,6 +2,7 @@
- gh-76 Implemented OrientDB client (lvca)
- gh-88 YCSB client for Amazon DynamoDB (jananin)
- gh-89 Patch for YCSB Cassandra Client version 1.0.6 (jananin)
+- gh-93 New ElasticSearch Database Implementation (saden1)
0.1.4 - 2/22/12
3  bin/ycsb
View
@@ -29,6 +29,7 @@ DATABASES = {
"cassandra-8" : "com.yahoo.ycsb.db.CassandraClient8",
"cassandra-10" : "com.yahoo.ycsb.db.CassandraClient10",
"dynamodb" : "com.yahoo.ycsb.db.DynamoDBClient",
+ "elasticsearch": "com.yahoo.ycsb.db.ElasticSearchClient",
"gemfire" : "com.yahoo.ycsb.db.GemFireClient",
"hbase" : "com.yahoo.ycsb.db.HBaseClient",
"hypertable" : "com.yahoo.ycsb.db.HypertableClient",
@@ -39,7 +40,7 @@ DATABASES = {
"nosqldb" : "com.yahoo.ycsb.db.NoSqlDbClient",
"orientdb" : "com.yahoo.ycsb.db.OrientDBClient",
"redis" : "com.yahoo.ycsb.db.RedisClient",
- "voldemort" : "com.yahoo.ycsb.db.VoldemortClient",
+ "voldemort" : "com.yahoo.ycsb.db.VoldemortClient",
}
OPTIONS = {
65 elasticsearch/README.md
View
@@ -0,0 +1,65 @@
+## Quick Start
+
+This section describes how to run YCSB on ElasticSearch running locally.
+
+### 1. Set Up YCSB
+
+Clone the YCSB git repository and compile:
+
+ git clone git://github.com/brianfrankcooper/YCSB.git
+ cd YCSB
+ mvn clean package
+
+### 2. Run YCSB
+
+Now you are ready to run! First, load the data:
+
+ ./bin/ycsb load elasticsearch -s -P workloads/workloada
+
+Then, run the workload:
+
+ ./bin/ycsb run elasticsearch -s -P workloads/workloada
+
+For further configuration see below:
+
+### Defaults Configuration
+The default setting for the ElasticSearch node that is created is as follows:
+
+ cluster.name=es.ycsb.cluster
+ node.local=true
+ path.data=$TEMP_DIR/esdata
+ discovery.zen.ping.multicast.enabled=false
+ index.mapping._id.indexed=true
+ index.gateway.type=none
+ gateway.type=none
+ index.number_of_shards=1
+ index.number_of_replicas=0
+ es.index.key=es.ycsb
+
+### Custom Configuration
+If you wish to customize the settings used to create the ElasticSerach node
+you can created a new property file that contains your desired ElasticSearch
+node settings and pass it in via the parameter to 'bin/ycsb' script. Note that
+the default properties will be kept if you don't explicitly overwrite them.
+
+Assuming that we have a properties file named "myproperties.data" that contains
+custom ElasticSearch node configuration you can execute the following to
+pass it into the ElasticSearch client:
+
+
+ ./bin/ycsb run elasticsearch -P workloads/workloada -P myproperties.data -s
+
+
+If you wish to use a in-memory store type rather than the default disk store add
+the following properties to your custom properties file. For a large number of
+insert operations insure that you have sufficient memory on your test system
+otherwise you will run out of memory.
+
+ index.store.type=memory
+ index.store.fs.memory.enabled=true
+ cache.memory.small_buffer_size=4mb
+ cache.memory.large_cache_size=1024mb
+
+If you wish to change the default index name you can set the following property:
+
+ es.index.key=my_index_key
71 elasticsearch/pom.xml
View
@@ -0,0 +1,71 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>com.yahoo.ycsb</groupId>
+ <artifactId>root</artifactId>
+ <version>0.1.4</version>
+ </parent>
+
+ <artifactId>elasticsearch-binding</artifactId>
+ <name>ElasticSearch Binding</name>
+ <packaging>jar</packaging>
+ <properties>
+ <elasticsearch-version>0.19.8</elasticsearch-version>
+ </properties>
+ <repositories>
+ <repository>
+ <id>sonatype-nexus-snapshots</id>
+ <name>Sonatype Nexus Snapshots</name>
+ <url>https://oss.sonatype.org/content/repositories/releases</url>
+ </repository>
+ </repositories>
+ <dependencies>
+ <dependency>
+ <groupId>com.yahoo.ycsb</groupId>
+ <artifactId>core</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.elasticsearch</groupId>
+ <artifactId>elasticsearch</artifactId>
+ <version>${elasticsearch-version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.testng</groupId>
+ <artifactId>testng</artifactId>
+ <version>6.1.1</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-nop</artifactId>
+ <version>1.6.4</version>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <version>${maven.assembly.version}</version>
+ <configuration>
+ <descriptorRefs>
+ <descriptorRef>jar-with-dependencies</descriptorRef>
+ </descriptorRefs>
+ <appendAssemblyId>false</appendAssemblyId>
+ </configuration>
+ <executions>
+ <execution>
+ <phase>package</phase>
+ <goals>
+ <goal>single</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+
+</project>
266 elasticsearch/src/main/java/com/yahoo/ycsb/db/ElasticSearchClient.java
View
@@ -0,0 +1,266 @@
+package com.yahoo.ycsb.db;
+
+import com.yahoo.ycsb.ByteIterator;
+import com.yahoo.ycsb.DB;
+import com.yahoo.ycsb.DBException;
+import com.yahoo.ycsb.StringByteIterator;
+import java.util.HashMap;
+import java.util.Map.Entry;
+import java.util.Properties;
+import java.util.Set;
+import java.util.Vector;
+import org.elasticsearch.action.get.GetResponse;
+import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.client.Requests;
+import static org.elasticsearch.common.settings.ImmutableSettings.*;
+import org.elasticsearch.common.settings.ImmutableSettings.Builder;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import static org.elasticsearch.common.xcontent.XContentFactory.*;
+import static org.elasticsearch.index.query.FilterBuilders.*;
+import static org.elasticsearch.index.query.QueryBuilders.*;
+import org.elasticsearch.index.query.RangeFilterBuilder;
+import org.elasticsearch.node.Node;
+import static org.elasticsearch.node.NodeBuilder.*;
+import org.elasticsearch.search.SearchHit;
+
+/**
+ * ElasticSearch client for YCSB framework.
+ *
+ * <p>Default properties to set:</p> <ul> <li>es.cluster.name = es.ycsb.cluster
+ * <li>es.client = true <li>es.index.key = es.ycsb</ul>
+ *
+ * @author Sharmarke Aden
+ *
+ */
+public class ElasticSearchClient extends DB {
+
+ public static final String DEFAULT_CLUSTER_NAME = "es.ycsb.cluster";
+ public static final String DEFAULT_INDEX_KEY = "es.ycsb";
+ private Node node;
+ private Client client;
+ private String indexKey;
+
+ /**
+ * Initialize any state for this DB. Called once per DB instance; there is
+ * one DB instance per client thread.
+ */
+ @Override
+ public void init() throws DBException {
+ // initialize OrientDB driver
+ Properties props = getProperties();
+ this.indexKey = props.getProperty("es.index.key", DEFAULT_INDEX_KEY);
+ String clusterName = props.getProperty("cluster.name", DEFAULT_CLUSTER_NAME);
+ Boolean newdb = Boolean.parseBoolean(props.getProperty("elasticsearch.newdb", "false"));
+ Builder settings = settingsBuilder()
+ .put("node.local", "true")
+ .put("path.data", System.getProperty("java.io.tmpdir") + "/esdata")
+ .put("discovery.zen.ping.multicast.enabled", "false")
+ .put("index.mapping._id.indexed", "true")
+ .put("index.gateway.type", "none")
+ .put("gateway.type", "none")
+ .put("index.number_of_shards", "1")
+ .put("index.number_of_replicas", "0");
+
+
+ //if properties file contains elasticsearch user defined properties
+ //add it to the settings file (will overwrite the defaults).
+ settings.put(props);
+ System.out.println("ElasticSearch starting node = " + settings.get("cluster.name"));
+ System.out.println("ElasticSearch node data path = " + settings.get("path.data"));
+
+ node = nodeBuilder().clusterName(clusterName).settings(settings).node();
+ node.start();
+ client = node.client();
+
+ if (newdb) {
+ client.admin().indices().prepareDelete(indexKey).execute().actionGet();
+ client.admin().indices().prepareCreate(indexKey).execute().actionGet();
+ } else {
+ boolean exists = client.admin().indices().exists(Requests.indicesExistsRequest(indexKey)).actionGet().isExists();
+ if (!exists) {
+ client.admin().indices().prepareCreate(indexKey).execute().actionGet();
+ }
+ }
+ }
+
+ @Override
+ public void cleanup() throws DBException {
+ if (!node.isClosed()) {
+ client.close();
+ node.stop();
+ node.close();
+ }
+ }
+
+ /**
+ * Insert a record in the database. Any field/value pairs in the specified
+ * values HashMap will be written into the record with the specified record
+ * key.
+ *
+ * @param table The name of the table
+ * @param key The record key of the record to insert.
+ * @param values A HashMap of field/value pairs to insert in the record
+ * @return Zero on success, a non-zero error code on error. See this class's
+ * description for a discussion of error codes.
+ */
+ @Override
+ public int insert(String table, String key, HashMap<String, ByteIterator> values) {
+ try {
+ final XContentBuilder doc = jsonBuilder().startObject();
+
+ for (Entry<String, String> entry : StringByteIterator.getStringMap(values).entrySet()) {
+ doc.field(entry.getKey(), entry.getValue());
+ }
+
+ doc.endObject();
+
+ client.prepareIndex(indexKey, table, key)
+ .setSource(doc)
+ .execute()
+ .actionGet();
+
+ return 0;
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ return 1;
+ }
+
+ /**
+ * Delete a record from the database.
+ *
+ * @param table The name of the table
+ * @param key The record key of the record to delete.
+ * @return Zero on success, a non-zero error code on error. See this class's
+ * description for a discussion of error codes.
+ */
+ @Override
+ public int delete(String table, String key) {
+ try {
+ client.prepareDelete(indexKey, table, key)
+ .execute()
+ .actionGet();
+ return 0;
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ return 1;
+ }
+
+ /**
+ * Read a record from the database. Each field/value pair from the result
+ * will be stored in a HashMap.
+ *
+ * @param table The name of the table
+ * @param key The record key of the record to read.
+ * @param fields The list of fields to read, or null for all of them
+ * @param result A HashMap of field/value pairs for the result
+ * @return Zero on success, a non-zero error code on error or "not found".
+ */
+ @Override
+ public int read(String table, String key, Set<String> fields, HashMap<String, ByteIterator> result) {
+ try {
+ final GetResponse response = client.prepareGet(indexKey, table, key)
+ .execute()
+ .actionGet();
+
+ if (response.isExists()) {
+ if (fields != null) {
+ for (String field : fields) {
+ result.put(field, new StringByteIterator((String) response.getSource().get(field)));
+ }
+ } else {
+ for (String field : response.getSource().keySet()) {
+ result.put(field, new StringByteIterator((String) response.getSource().get(field)));
+ }
+ }
+ return 0;
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ return 1;
+ }
+
+ /**
+ * Update a record in the database. Any field/value pairs in the specified
+ * values HashMap will be written into the record with the specified record
+ * key, overwriting any existing values with the same field name.
+ *
+ * @param table The name of the table
+ * @param key The record key of the record to write.
+ * @param values A HashMap of field/value pairs to update in the record
+ * @return Zero on success, a non-zero error code on error. See this class's
+ * description for a discussion of error codes.
+ */
+ @Override
+ public int update(String table, String key, HashMap<String, ByteIterator> values) {
+ try {
+ final GetResponse response = client.prepareGet(indexKey, table, key)
+ .execute()
+ .actionGet();
+
+ if (response.isExists()) {
+ for (Entry<String, String> entry : StringByteIterator.getStringMap(values).entrySet()) {
+ response.getSource().put(entry.getKey(), entry.getValue());
+ }
+
+ client.prepareIndex(indexKey, table, key)
+ .setSource(response.getSource())
+ .execute()
+ .actionGet();
+
+ return 0;
+ }
+
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ return 1;
+ }
+
+ /**
+ * Perform a range scan for a set of records in the database. Each
+ * field/value pair from the result will be stored in a HashMap.
+ *
+ * @param table The name of the table
+ * @param startkey The record key of the first record to read.
+ * @param recordcount The number of records to read
+ * @param fields The list of fields to read, or null for all of them
+ * @param result A Vector of HashMaps, where each HashMap is a set
+ * field/value pairs for one record
+ * @return Zero on success, a non-zero error code on error. See this class's
+ * description for a discussion of error codes.
+ */
+ @Override
+ public int scan(String table, String startkey, int recordcount, Set<String> fields, Vector<HashMap<String, ByteIterator>> result) {
+ try {
+ final RangeFilterBuilder filter = rangeFilter("_id").gte(startkey);
+ final SearchResponse response = client.prepareSearch(indexKey)
+ .setTypes(table)
+ .setQuery(matchAllQuery())
+ .setFilter(filter)
+ .setSize(recordcount)
+ .execute()
+ .actionGet();
+
+ HashMap<String, ByteIterator> entry;
+
+ for (SearchHit hit : response.getHits()) {
+ entry = new HashMap<String, ByteIterator>(fields.size());
+
+ for (String field : fields) {
+ entry.put(field, new StringByteIterator((String) hit.getSource().get(field)));
+ }
+
+ result.add(entry);
+ }
+
+ return 0;
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ return 1;
+ }
+}
137 elasticsearch/src/test/java/com/yahoo/ycsb/db/ElasticSearchClientTest.java
View
@@ -0,0 +1,137 @@
+/*
+ * To change this template, choose Tools | Templates
+ * and open the template in the editor.
+ */
+package com.yahoo.ycsb.db;
+
+import com.yahoo.ycsb.ByteIterator;
+import com.yahoo.ycsb.DBException;
+import com.yahoo.ycsb.StringByteIterator;
+import java.util.HashMap;
+import java.util.Set;
+import java.util.Vector;
+import static org.testng.AssertJUnit.*;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+/**
+ *
+ * @author saden
+ */
+public class ElasticSearchClientTest {
+
+ protected final static ElasticSearchClient instance = new ElasticSearchClient();
+ protected final static HashMap<String, ByteIterator> MOCK_DATA;
+ protected final static String MOCK_TABLE = "MOCK_TABLE";
+ protected final static String MOCK_KEY0 = "0";
+ protected final static String MOCK_KEY1 = "1";
+ protected final static String MOCK_KEY2 = "2";
+
+ static {
+ MOCK_DATA = new HashMap<String, ByteIterator>(10);
+ for (int i = 1; i <= 10; i++) {
+ MOCK_DATA.put("field" + i, new StringByteIterator("value" + i));
+ }
+ }
+
+ @BeforeClass
+ public static void setUpClass() throws DBException {
+ instance.init();
+ }
+
+ @AfterClass
+ public static void tearDownClass() throws DBException {
+ instance.cleanup();
+ }
+
+ @BeforeMethod
+ public void setUp() {
+ instance.insert(MOCK_TABLE, MOCK_KEY1, MOCK_DATA);
+ instance.insert(MOCK_TABLE, MOCK_KEY2, MOCK_DATA);
+ }
+
+ @AfterMethod
+ public void tearDown() {
+ instance.delete(MOCK_TABLE, MOCK_KEY1);
+ instance.delete(MOCK_TABLE, MOCK_KEY2);
+ }
+
+ /**
+ * Test of insert method, of class ElasticSearchClient.
+ */
+ @Test
+ public void testInsert() {
+ System.out.println("insert");
+ int expResult = 0;
+ int result = instance.insert(MOCK_TABLE, MOCK_KEY0, MOCK_DATA);
+ assertEquals(expResult, result);
+ }
+
+ /**
+ * Test of delete method, of class ElasticSearchClient.
+ */
+ @Test
+ public void testDelete() {
+ System.out.println("delete");
+ int expResult = 0;
+ int result = instance.delete(MOCK_TABLE, MOCK_KEY1);
+ assertEquals(expResult, result);
+ }
+
+ /**
+ * Test of read method, of class ElasticSearchClient.
+ */
+ @Test
+ public void testRead() {
+ System.out.println("read");
+ Set<String> fields = MOCK_DATA.keySet();
+ HashMap<String, ByteIterator> resultParam = new HashMap<String, ByteIterator>(10);
+ int expResult = 0;
+ int result = instance.read(MOCK_TABLE, MOCK_KEY1, fields, resultParam);
+ assertEquals(expResult, result);
+ }
+
+ /**
+ * Test of update method, of class ElasticSearchClient.
+ */
+ @Test
+ public void testUpdate() {
+ System.out.println("update");
+ int i;
+ HashMap<String, ByteIterator> newValues = new HashMap<String, ByteIterator>(10);
+
+ for (i = 1; i <= 10; i++) {
+ newValues.put("field" + i, new StringByteIterator("newvalue" + i));
+ }
+
+ int expResult = 0;
+ int result = instance.update(MOCK_TABLE, MOCK_KEY1, newValues);
+ assertEquals(expResult, result);
+
+ //validate that the values changed
+ HashMap<String, ByteIterator> resultParam = new HashMap<String, ByteIterator>(10);
+ instance.read(MOCK_TABLE, MOCK_KEY1, MOCK_DATA.keySet(), resultParam);
+
+ for (i = 1; i <= 10; i++) {
+ assertEquals("newvalue" + i, resultParam.get("field" + i).toString());
+ }
+
+ }
+
+ /**
+ * Test of scan method, of class ElasticSearchClient.
+ */
+ @Test
+ public void testScan() {
+ System.out.println("scan");
+ int recordcount = 10;
+ Set<String> fields = MOCK_DATA.keySet();
+ Vector<HashMap<String, ByteIterator>> resultParam = new Vector<HashMap<String, ByteIterator>>(10);
+ int expResult = 0;
+ int result = instance.scan(MOCK_TABLE, MOCK_KEY1, recordcount, fields, resultParam);
+ assertEquals(expResult, result);
+ }
+}
3  pom.xml
View
@@ -60,11 +60,12 @@
<modules>
<!--module>build-tools</module-->
+ <module>cassandra</module>
<module>core</module>
<module>hbase</module>
<module>hypertable</module>
- <module>cassandra</module>
<module>dynamodb</module>
+ <module>elasticsearch</module>
<!--<module>gemfire</module>-->
<module>infinispan</module>
<module>jdbc</module>
Please sign in to comment.
Something went wrong with that request. Please try again.