Permalink
Browse files

Cleanup. Add support for removing versioning from voldemort config. M…

…isc. improvements for read-only store. Make external sort multithreaded in the sort phase.
  • Loading branch information...
1 parent 0bacb95 commit 8219a4aed922b0819281cc62e504e4706232c808 @jkreps jkreps committed Jan 19, 2009
Showing with 420 additions and 162 deletions.
  1. +1 −0 .classpath
  2. +6 −3 TODO
  3. +37 −0 bin/build-readonly-store.sh
  4. +2 −2 bin/sort.sh
  5. +1 −0 build.properties
  6. +23 −11 build.xml
  7. +2 −2 build_number.txt
  8. +1 −1 src/java/voldemort/VoldemortException.java
  9. +2 −1 src/java/voldemort/client/AbstractStoreClientFactory.java
  10. +8 −0 src/java/voldemort/client/BootstrapFailureException.java
  11. +11 −1 src/java/voldemort/client/DefaultStoreClient.java
  12. +3 −0 src/java/voldemort/client/HttpStoreClientFactory.java
  13. +2 −1 src/java/voldemort/client/MockStoreClientFactory.java
  14. +6 −0 src/java/voldemort/client/RoutingTier.java
  15. +6 −1 src/java/voldemort/client/SocketStoreClientFactory.java
  16. +10 −7 src/java/voldemort/client/StoreClient.java
  17. +29 −1 src/java/voldemort/client/StoreClientFactory.java
  18. +6 −2 src/java/voldemort/client/UpdateAction.java
  19. +0 −22 src/java/voldemort/client/VoldemortProtocol.java
  20. +2 −2 src/java/voldemort/client/package.html
  21. +30 −12 src/java/voldemort/serialization/SerializerDefinition.java
  22. +6 −0 src/java/voldemort/serialization/json/JsonTypeSerializer.java
  23. +1 −1 src/java/voldemort/server/scheduler/SlopPusherJob.java
  24. +1 −0 src/java/voldemort/store/ErrorCodeMapper.java
  25. +1 −1 src/java/voldemort/store/bdb/BdbStorageEngine.java
  26. +1 −1 src/java/voldemort/store/filesystem/FilesystemStorageEngine.java
  27. +1 −1 src/java/voldemort/store/http/HttpResponseCodeErrorMapper.java
  28. +1 −1 src/java/voldemort/store/memory/InMemoryStorageEngine.java
  29. +1 −1 src/java/voldemort/store/metadata/MetadataStore.java
  30. +1 −1 src/java/voldemort/store/mysql/MysqlStorageEngine.java
  31. +56 −25 src/java/voldemort/store/readonly/ExternalSorter.java
  32. +20 −6 src/java/voldemort/store/readonly/JsonStoreBuilder.java
  33. +5 −3 src/java/voldemort/store/readonly/StringSorter.java
  34. +1 −1 src/java/voldemort/store/routed/RoutedStore.java
  35. +5 −19 src/java/voldemort/utils/ByteUtils.java
  36. +11 −4 src/java/voldemort/versioning/InconsistencyResolver.java
  37. +9 −5 src/java/voldemort/{store → versioning}/ObsoleteVersionException.java
  38. +13 −3 src/java/voldemort/xml/StoreDefinitionsMapper.java
  39. +11 −1 src/java/voldemort/xml/stores.xsd
  40. +4 −0 test/common/voldemort/VoldemortTestConstants.java
  41. +20 −0 test/common/voldemort/config/no-version-store.xml
  42. +1 −1 test/integration/voldemort/MultithreadedStressTest.java
  43. +7 −7 test/integration/voldemort/ReadOnlyStoreBuilder.java
  44. +1 −1 test/integration/voldemort/performance/CacheStorageEnginePerformanceTest.java
  45. +28 −0 test/integration/voldemort/performance/TestSerialization.java
  46. +1 −0 test/unit/voldemort/store/BasicStoreTest.java
  47. +1 −1 test/unit/voldemort/store/memory/CacheStorageEngineTest.java
  48. +15 −8 test/unit/voldemort/store/readonly/ExternalSorterTest.java
  49. +2 −1 test/unit/voldemort/store/readonly/JsonStoreBuilderTest.java
  50. +7 −0 test/unit/voldemort/xml/StoreDefinitionMapperTest.java
View
@@ -28,5 +28,6 @@
<classpathentry kind="lib" path="lib/velocity-1.5.jar"/>
<classpathentry kind="lib" path="lib/xerces.jar"/>
<classpathentry kind="lib" path="lib/colt-1.2.0.jar"/>
+ <classpathentry kind="lib" path="lib/mysql-connector-java-5.1.5-bin.jar"/>
<classpathentry kind="output" path="classes"/>
</classpath>
View
9 TODO
@@ -1,13 +1,14 @@
TODO HIGH
-- push readonly data to voldemort
--- mr task to build stores
--- mechanism to swap table on the fly (HTTP in war, method in vold. server)
+-- m/r task to build stores
+-- JMX mechanism to swap table on the fly (HTTP in war, method in vold. server)
-- initial implementation is in the read-only store using read/write lock
-- rebalancing JMX control
MEDIUM
-- reenable slop
--- Protocolbuffers serialization
+-- pbuffers serialization for network protocol
+-- pbuffers serialization for application level serialization
-- Add setters to StoreClientFactories to allow propery-based configuration
-- Expand socketpool test
-- Slop detecting store test
@@ -19,6 +20,7 @@ MEDIUM
-- RandomAccessFileStore vs. MmapFileStore benchmark
TODO LOW
+-- Add release name to jar file
-- Rewrite socket pool to not use commons-pool because it sucks
-- Support date type formatting
-- Add type enforcement to jsonreader?
@@ -61,6 +63,7 @@ xxx Share threadpool between everyone (socket server, jetty if possible, mina, a
xxx Why is metadata store duplicated in voldemortserver and storageservice
Done
+xxx support bdb write no flush
xxx soft-reference "cache" storage type
xxx Search result tracking test
xxx Change tags to partitions in configuration
@@ -0,0 +1,37 @@
+#!/bin/bash
+
+#
+# Copyright 2008-2009 LinkedIn, Inc
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+if [ $# != 7 ];
+then
+ echo 'USAGE: bin/build-readonly-store.sh cluster.xml store_definitions.xml store_name sort_obj_buffer_size input_data output_dir num_threads'
+ exit 1
+fi
+
+base_dir=$(dirname $0)/..
+
+for file in $base_dir/dist/*.jar;
+do
+ CLASSPATH=$CLASSPATH:$file
+done
+
+for file in $base_dir/lib/*.jar;
+do
+ CLASSPATH=$CLASSPATH:$file
+done
+
+java -Xmx2G -server -cp $CLASSPATH voldemort.store.readonly.JsonStoreBuilder ${1} ${2} ${3} ${4} ${5} ${6} ${7}
View
@@ -18,7 +18,7 @@
if [ $# -gt 2 ];
then
- echo 'USAGE: bin/sort.sh file max_memory_lines'
+ echo 'USAGE: bin/sort.sh file max_memory_lines num_threads'
exit 1
fi
@@ -34,4 +34,4 @@ do
CLASSPATH=$CLASSPATH:$file
done
-java -agentlib:hprof=cpu=samples,depth=10 -Xmx2G -server -cp $CLASSPATH voldemort.store.readonly.StringSorter ${1} ${2}
+java -agentlib:hprof=cpu=samples,depth=10 -Xmx2G -server -cp $CLASSPATH voldemort.store.readonly.StringSorter ${1} ${2} ${3}
View
@@ -16,3 +16,4 @@ tomcat.manager.url=http://localhost:8080/manager
tomcat.manager.username=tomcat
tomcat.manager.password=tomcat
tomcat.context=/voldemort
+curr.release=0.4a
View
@@ -56,6 +56,7 @@
<copy todir="${classes.dir}">
<fileset dir="${java.dir}">
<exclude name="**/*.java"/>
+ <exclude name="**/*.html"/>
</fileset>
</copy>
<javac destdir="${classes.dir}" target="1.5" debug="true" deprecation="true" failonerror="true">
@@ -66,18 +67,20 @@
</target>
<target name="test" depends="build, buildtest" description="Build test jar file">
- <jar destfile="${dist.dir}/${name}-test.jar">
+ <jar destfile="${dist.dir}/${name}-test-${curr.release}.jar">
<fileset dir="${testclasses.dir}"/>
</jar>
</target>
<target name="jar" depends="build" description="Build server jar file">
- <jar destfile="${dist.dir}/${name}.jar">
+ <jar destfile="${dist.dir}/${name}-${curr.release}.jar">
<fileset dir="${classes.dir}">
<include name="**/*.*"/>
</fileset>
- <!-- include src code for ide convenience -->
- <fileset dir="${java.dir}" />
+ <!-- include xsds -->
+ <fileset dir="${java.dir}">
+ <include name="**/*.xsd"/>
+ </fileset>
</jar>
</target>
@@ -136,14 +139,23 @@
</war>
</target>
+ <macrodef name="make-javadocs">
+ <attribute name="dir"/>
+ <attribute name="packagenames"/>
+ <sequential>
+ <replace-dir dir="@{dir}"/>
+ <javadoc sourcepath="${java.dir}" destdir="@{dir}" windowtitle="${display.name}"
+ source="1.5" author="true" version="true" use="true" packagenames="@{packagenames}">
+ <doctitle>${display.name}</doctitle>
+ <bottom>${author}</bottom>
+ <classpath refid="main-classpath"/>
+ </javadoc>
+ </sequential>
+ </macrodef>
+
<target name="docs" description="Create complete Javadoc documentation">
- <replace-dir dir="${javadoc.dir}"/>
- <javadoc sourcepath="${java.dir}" destdir="${javadoc.dir}" windowtitle="${display.name}"
- source="1.5" author="true" version="true" use="true" packagenames="*">
- <doctitle>${display.name}</doctitle>
- <bottom>${author}</bottom>
- <classpath refid="main-classpath"/>
- </javadoc>
+ <make-javadocs dir="${javadoc.dir}/all" packagenames="*"/>
+ <make-javadocs dir="${javadoc.dir}/client" packagenames="voldemort.versioning.*, voldemort.client.*,voldemort.serialization.*"/>
</target>
<target name="redeploy" depends="war, deploy">
View
@@ -1,6 +1,6 @@
#Build Number for ANT. Do not edit!
-#Tue Jan 13 14:19:40 PST 2009
-build.number=143
+#Sun Jan 18 18:24:31 PST 2009
+build.number=167
>>>>>>>=1.8
=\=\=\=\=\=\=
<<<<<<<=build_number.txt
@@ -17,7 +17,7 @@
package voldemort;
/**
- * Base exception
+ * Base exception that all other Voldemort exceptions extend.
*
* @author jay
*
@@ -51,7 +51,8 @@
import com.google.common.collect.Maps;
/**
- * A place to get conveniently pre-assembled StoreClients
+ * A base class for various {@link voldemort.client.StoreClientFactory
+ * StoreClientFactory} implementations
*
* @author jay
*
@@ -18,6 +18,14 @@
import voldemort.VoldemortException;
+/**
+ * The exception thrown when bootstrapping the store from the cluster fails
+ * (e.g. because none of the given nodes could be connected to). This is
+ * generally an unrecoverable failure.
+ *
+ * @author jay
+ *
+ */
public class BootstrapFailureException extends VoldemortException {
private static final long serialVersionUID = 1L;
@@ -21,14 +21,24 @@
import voldemort.cluster.Node;
import voldemort.routing.RoutingStrategy;
import voldemort.serialization.Serializer;
-import voldemort.store.ObsoleteVersionException;
import voldemort.store.Store;
import voldemort.utils.Utils;
import voldemort.versioning.InconsistentDataException;
+import voldemort.versioning.ObsoleteVersionException;
import voldemort.versioning.VectorClock;
import voldemort.versioning.Version;
import voldemort.versioning.Versioned;
+/**
+ * The default {@link voldemort.client.StoreClient StoreClient} implementation
+ * you get back from a {@link voldemort.client.StoreClientFactory
+ * StoreClientFactory}
+ *
+ * @author jay
+ *
+ * @param <K> The key type
+ * @param <V> The value type
+ */
public class DefaultStoreClient<K, V> implements StoreClient<K, V> {
private final Versioned<V> NOT_FOUND = new Versioned<V>(null, null);
@@ -35,6 +35,9 @@
import voldemort.store.http.HttpStore;
/**
+ * A {@link voldemort.client.StoreClientFactory StoreClientFactory} that creates
+ * a remote client that connects and bootstraps itself via HTTP.
+ *
* @author jay
*
*/
@@ -31,7 +31,8 @@
import voldemort.versioning.Versioned;
/**
- * A store client that produces fake, in-memory stores
+ * A store client that produces non-persistent, in-memory stores. This is useful
+ * for unit testing.
*
* @author jay
*
@@ -16,6 +16,12 @@
package voldemort.client;
+/**
+ * Enumerates different routing strategies. Currently just server and client.
+ *
+ * @author jay
+ *
+ */
public enum RoutingTier {
CLIENT("client"),
SERVER("server");
@@ -32,6 +32,11 @@
import voldemort.utils.Utils;
/**
+ * A StoreClientFactory abstracts away the connection pooling, threading, and
+ * bootstrapping mechanism. It can be used to create any number of
+ * {@link voldemort.client.StoreClient StoreClient} instances for different
+ * stores.
+ *
* @author jay
*
*/
@@ -46,7 +51,7 @@
private SocketPool socketPool;
- public SocketStoreClientFactory(String... bootstrapUrls) {
+ public SocketStoreClientFactory(String bootstrapUrls) {
this(DEFAULT_NUM_THREADS,
DEFAULT_NUM_THREADS,
DEFAULT_MAX_QUEUED_REQUESTS,
@@ -16,13 +16,13 @@
package voldemort.client;
-import voldemort.store.ObsoleteVersionException;
+import voldemort.versioning.ObsoleteVersionException;
import voldemort.versioning.Version;
import voldemort.versioning.Versioned;
/**
- * The user-facing interface to a store. Gives basic put/get/delete plus helper
- * functions.
+ * The user-facing interface to a Voldemort store. Gives basic put/get/delete
+ * plus helper functions.
*
* @author jay
*
@@ -105,11 +105,14 @@
/**
* Apply the given action repeatedly until no ObsoleteVersionException is
- * thrown. This is useful for implementing a read-modify-store loop.
+ * thrown. This is useful for implementing a read-modify-store loop that
+ * could be pre-empted by another concurrent update, and should be repeated
+ * until it succeeds.
*
- * @param action The action to apply
- * @return true if the action is successfully applied, false if the default
- * max number of attempts is exceeded
+ * @param action The action to apply. This is meant as a callback for the
+ * user to extend to provide their own logic.
+ * @return true if the action is successfully applied, false if the 3
+ * attempts all result in ObsoleteVersionException
*/
public boolean applyUpdate(UpdateAction<K, V> action);
@@ -21,15 +21,43 @@
/**
* An abstraction the represents a connection to a Voldemort cluster and can be
- * used to create StoreClients to interact with individual stores
+ * used to create {@link voldemort.client.StoreClient StoreClient} instances to
+ * interact with individual stores. The factory abstracts away any connection
+ * pools, thread pools, or other details that will be shared by all the
+ * individual
+ *
+ * {@link voldemort.client.StoreClient StoreClient}
*
* @author jay
*
*/
public interface StoreClientFactory {
+ /**
+ * Get a {@link voldemort.client.StoreClient} for the given store.
+ *
+ * @param <K> The type of the key of the
+ * {@link voldemort.client.StoreClient}
+ * @param <V> The type of the value of the
+ * {@link voldemort.client.StoreClient}
+ * @param storeName The name of the store
+ * @return A fully constructed {@link voldemort.client.StoreClient}
+ */
public <K, V> StoreClient<K, V> getStoreClient(String storeName);
+ /**
+ * Get a {@link voldemort.client.StoreClient} for the given store.
+ *
+ * @param <K> The type of the key of the
+ * {@link voldemort.client.StoreClient}
+ * @param <V> The type of the value of the
+ * {@link voldemort.client.StoreClient}
+ * @param storeName The name of the store
+ * @param inconsistencyResolver The
+ * {@link voldemort.versioning.InconsistencyResolver} that should be
+ * used to resolve inconsistencies.
+ * @return A fully constructed {@link voldemort.client.StoreClient}
+ */
public <K, V> StoreClient<K, V> getStoreClient(String storeName,
InconsistencyResolver<Versioned<V>> inconsistencyResolver);
@@ -17,15 +17,19 @@
package voldemort.client;
/**
- * An update action such as a read-modify-store cycle
+ * An update action such as a read-modify-store cycle. This is meant to be used
+ * as a callback interface to perform a data modification that might involve
+ * obsolete data. The operation will be repeated until it succeeds.
+ *
+ * The update action must be idempotent since it may be called multiple times.
*
* @author jay
*
*/
public abstract class UpdateAction<K, V> {
/**
- * Apply the update sequence
+ * Apply the update operation to the given store client.
*
* @param storeClient The store client to use
*/
Oops, something went wrong.

0 comments on commit 8219a4a

Please sign in to comment.