merge from 0.6

git-svn-id: https://svn.apache.org/repos/asf/cassandra/trunk@939110 13f79535-47bb-0310-9956-ffa450edef68
apache · Apr 28, 2010 · 78cbce1 · 78cbce1
1 parent c03e16d
commit 78cbce1
Show file tree

Hide file tree

Showing 20 changed files with 450 additions and 162 deletions.
diff --git a/CHANGES.txt b/CHANGES.txt
@@ -22,6 +22,12 @@ dev
  * removed multiget thrift method (CASSANDRA-739)
 
 
+0.6.2
+ * fix contrib/word_count build. (CASSANDRA-992)
+ * split CommitLogExecutorService into BatchCommitLogExecutorService and 
+   PeriodicCommitLogExecutorService (CASSANDRA-1014)
+
+
 0.6.1
  * fix NPE in sstable2json when no excluded keys are given (CASSANDRA-934)
  * keep the replica set constant throughout the read repair process

diff --git a/NEWS.txt b/NEWS.txt
@@ -88,6 +88,10 @@ JMX metrics
       a much better estimate of compactions remaining, and the
       progress of the current compaction has been added.
     - commitlog JMX metrics are moved to org.apache.cassandra.db.Commitlog
+    - progress of data streaming during bootstrap, loadbalance, or other
+      data migration, is available under 
+      org.apache.cassandra.streaming.StreamingService.
+      See http://wiki.apache.org/cassandra/Streaming for details.
 
 Installation/Upgrade
 --------------------

diff --git a/build.xml b/build.xml
@@ -44,7 +44,7 @@
     <property name="test.unit.src" value="${test.dir}/unit"/>
     <property name="test.long.src" value="${test.dir}/long"/>
     <property name="dist.dir" value="${build.dir}/dist"/>
-    <property name="version" value="0.6.0-rc1"/>
+    <property name="version" value="0.6.1"/>
     <property name="final.name" value="${ant.project.name}-${version}"/>
     <property name="ivy.version" value="2.1.0" />
     <property name="ivy.url"

diff --git a/contrib/word_count/README.txt b/contrib/word_count/README.txt
@@ -16,3 +16,8 @@ Read the code in src/ for more details.
 
 *If you want to point wordcount at a real cluster, modify the seed
 and listenaddress settings in storage-conf.xml accordingly.
+
+*For Mac users, the storage-conf.xml uses 127.0.0.2 for the 
+word_count_setup. Mac OS X doesn't have that address available.
+To add it, run this before running bin/word_count_setup:
+sudo ifconfig lo0 alias 127.0.0.2 up
diff --git a/contrib/word_count/bin/word_count b/contrib/word_count/bin/word_count
@@ -25,13 +25,16 @@ if [ ! -d $cwd/../../../build/classes ]; then
 fi
 
 # word_count Jar.
-if [ ! -e $cwd/../build/*.jar ]; then
+if [ ! -e $cwd/../build/word_count.jar ]; then
     echo "Unable to locate word_count jar" >&2
     exit 1
 fi
 
-CLASSPATH=$CLASSPATH:`ls -1 $cwd/../build/*.jar`
+CLASSPATH=$CLASSPATH:$cwd/../build/word_count.jar
 CLASSPATH=$CLASSPATH:.:$cwd/../../../build/classes
+for jar in $cwd/../build/lib/jars/*.jar; do
+    CLASSPATH=$CLASSPATH:$jar
+done
 for jar in $cwd/../../../lib/*.jar; do
     CLASSPATH=$CLASSPATH:$jar
 done
@@ -50,4 +53,5 @@ if [ "x$JAVA" = "x" ]; then
     exit 1
 fi
 
+#echo $CLASSPATH
 $JAVA -Xmx1G -ea -cp $CLASSPATH WordCount
diff --git a/contrib/word_count/bin/word_count_setup b/contrib/word_count/bin/word_count_setup
@@ -25,13 +25,16 @@ if [ ! -d $cwd/../../../build/classes ]; then
 fi
 
 # word_count Jar.
-if [ ! -e $cwd/../build/*.jar ]; then
+if [ ! -e $cwd/../build/word_count.jar ]; then
     echo "Unable to locate word_count jar" >&2
     exit 1
 fi
 
-CLASSPATH=$CLASSPATH:`ls -1 $cwd/../build/*.jar`
+CLASSPATH=$CLASSPATH:$cwd/../build/word_count.jar
 CLASSPATH=$CLASSPATH:.:$cwd/../../../build/classes
+for jar in $cwd/../build/lib/jars/*.jar; do
+    CLASSPATH=$CLASSPATH:$jar
+done
 for jar in $cwd/../../../lib/*.jar; do
     CLASSPATH=$CLASSPATH:$jar
 done

diff --git a/contrib/word_count/build.xml b/contrib/word_count/build.xml
@@ -17,39 +17,58 @@
  ~ specific language governing permissions and limitations
  ~ under the License.
  -->
-<project basedir="." default="jar" name="word_count">
+<project default="jar" name="word_count" xmlns:ivy="antlib:org.apache.ivy.ant">
     <property name="cassandra.dir" value="../.." />
-    <property name="cassandra.lib" value="" />
+    <property name="cassandra.dir.lib" value="${cassandra.dir}/lib" />
     <property name="cassandra.classes" value="${cassandra.dir}/build/classes" />
     <property name="build.src" value="${basedir}/src" />
-    <property name="build.out" value="${basedir}/build" />
-    <property name="build.classes" value="${build.out}/classes" />
+    <property name="build.dir" value="${basedir}/build" />
+    <property name="ivy.lib.dir" value="${build.dir}/lib" />
+    <property name="build.classes" value="${build.dir}/classes" />
     <property name="final.name" value="word_count" />
+    <property name="ivy.version" value="2.1.0" />
+    <property name="ivy.url"
+              value="http://repo2.maven.org/maven2/org/apache/ivy/ivy" />
+
+    <condition property="ivy.jar.exists">
+        <available file="${build.dir}/ivy-${ivy.version}.jar" />
+    </condition>
+
+    <path id="autoivy.classpath">
+        <fileset dir="${ivy.lib.dir}">
+            <include name="**/*.jar" />
+        </fileset>
+        <pathelement location="${build.dir}/ivy-${ivy.version}.jar"/>
+    </path>
+
+    <path id="wordcount.build.classpath">
+        <fileset dir="${ivy.lib.dir}">
+            <include name="**/*.jar" />
+        </fileset>
+        <!-- cassandra dependencies -->
+        <fileset dir="${cassandra.dir.lib}">
+            <include name="**/*.jar" />
+        </fileset>
+        <fileset dir="${cassandra.dir}/build/lib/jars">
+            <include name="**/*.jar" />
+        </fileset>
+        <pathelement location="${cassandra.classes}" />
+    </path>
 
     <target name="init">
         <mkdir dir="${build.classes}" />
     </target>
 
-    <target depends="init" name="build">
+    <target depends="init,ivy-retrieve-build" name="build">
         <javac destdir="${build.classes}">
             <src path="${build.src}" />
-            <classpath>
-                <path>
-                    <fileset dir="${cassandra.dir}/lib">
-                        <include name="**/*.jar" />
-                    </fileset>
-                    <fileset dir="${cassandra.dir}/build/lib/jars">
-                        <include name="**/*.jar" />
-                    </fileset>
-                    <pathelement location="${cassandra.classes}" />
-                </path>
-            </classpath>
+            <classpath refid="wordcount.build.classpath" />
         </javac>
     </target>
 
     <target name="jar" depends="build">
         <mkdir dir="${build.classes}/META-INF" />
-        <jar jarfile="${build.out}/${final.name}.jar">
+        <jar jarfile="${build.dir}/${final.name}.jar">
            <fileset dir="${build.classes}" />
            <fileset dir="${cassandra.classes}" />
            <fileset dir="${cassandra.dir}">
@@ -63,6 +82,30 @@
     </target>
 
     <target name="clean">
-        <delete dir="${build.out}" />
+        <delete dir="${build.dir}" />
+    </target>
+
+    <!--
+        Ivy Specific targets
+            to fetch Ivy and this project's dependencies
+    -->
+	<target name="ivy-download" unless="ivy.jar.exists">
+      <echo>Downloading Ivy...</echo>
+      <mkdir dir="${build.dir}" />
+      <get src="${ivy.url}/${ivy.version}/ivy-${ivy.version}.jar"
+           dest="${build.dir}/ivy-${ivy.version}.jar" usetimestamp="true" />
+    </target>
+
+    <target name="ivy-init" depends="ivy-download" unless="ivy.initialized">
+      <mkdir dir="${ivy.lib.dir}"/>
+      <taskdef resource="org/apache/ivy/ant/antlib.xml"
+               uri="antlib:org.apache.ivy.ant"
+               classpathref="autoivy.classpath"/>
+      <property name="ivy.initialized" value="true"/>
+    </target>
+
+    <target name="ivy-retrieve-build" depends="ivy-init">
+      <ivy:retrieve type="jar,source" sync="true"
+             pattern="${ivy.lib.dir}/[type]s/[artifact]-[revision].[ext]" />
     </target>
 </project>
diff --git a/contrib/word_count/ivy.xml b/contrib/word_count/ivy.xml
@@ -0,0 +1,24 @@
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements.  See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership.  The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License.  You may obtain a copy of the License at
+ ~
+ ~    http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing,
+ ~ software distributed under the License is distributed on an
+ ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ ~ KIND, either express or implied.  See the License for the
+ ~ specific language governing permissions and limitations
+ ~ under the License.
+ -->
+<ivy-module version="2.0">
+    <info organisation="apache-cassandra" module="word-count"/>
+    <dependencies>
+        <dependency org="org.apache.hadoop" name="hadoop-core" rev="0.20.2"/>
+    </dependencies>
+</ivy-module>
diff --git a/contrib/word_count/storage-conf.xml b/contrib/word_count/storage-conf.xml
@@ -83,33 +83,38 @@
        ~ An optional `Comment` attribute may be used to attach additional
        ~ human-readable information about the column family to its definition.
        ~ 
-       ~ The optional KeysCachedFraction attribute specifies
-       ~ The fraction of keys per sstable whose locations we keep in
+       ~ The optional KeysCached attribute specifies
+       ~ the number of keys per sstable whose locations we keep in
        ~ memory in "mostly LRU" order.  (JUST the key locations, NOT any
-       ~ column values.) The amount of memory used by the default setting of 
-       ~ 0.01 is comparable to the amount used by the internal per-sstable key
-       ~ index. Consider increasing this if you have fewer, wider rows.
-       ~ Set to 0 to disable entirely.
+       ~ column values.) Specify a fraction (value less than 1), a percentage
+       ~ (ending in a % sign) or an absolute number of keys to cache.
+       ~ KeysCached defaults to 200000 keys.
        ~
        ~ The optional RowsCached attribute specifies the number of rows
-       ~ whose entire contents we cache in memory, either as a fixed number
-       ~ of rows or as a percent of rows in the ColumnFamily.  
-       ~ Do not use this on ColumnFamilies with large rows, or
-       ~ ColumnFamilies with high write:read ratios.  As with key caching,
-       ~ valid values are from 0 to 1.  The default 0 disables it entirely.
+       ~ whose entire contents we cache in memory. Do not use this on
+       ~ ColumnFamilies with large rows, or ColumnFamilies with high write:read
+       ~ ratios. Specify a fraction (value less than 1), a percentage (ending in
+       ~ a % sign) or an absolute number of rows to cache. 
+       ~ RowsCached defaults to 0, i.e., row cache is off by default.
+       ~
+       ~ Remember, when using caches as a percentage, they WILL grow with
+       ~ your data set!
       -->
-      <ColumnFamily CompareWith="BytesType" 
-                    Name="Standard1" 
-                    RowsCached="10%"
-                    KeysCachedFraction="0"/>
-      <ColumnFamily CompareWith="UTF8Type" Name="Standard2"/>
-      <ColumnFamily CompareWith="TimeUUIDType" Name="StandardByUUID1"/>
-      <ColumnFamily ColumnType="Super"
+      <ColumnFamily Name="Standard1" CompareWith="BytesType"/>
+      <ColumnFamily Name="Standard2" 
+                    CompareWith="UTF8Type"
+                    KeysCached="100%"/>
+      <ColumnFamily Name="StandardByUUID1" CompareWith="TimeUUIDType" />
+      <ColumnFamily Name="Super1"
+                    ColumnType="Super"
+                    CompareWith="BytesType"
+                    CompareSubcolumnsWith="BytesType" />
+      <ColumnFamily Name="Super2"
+                    ColumnType="Super"
                     CompareWith="UTF8Type"
                     CompareSubcolumnsWith="UTF8Type"
-                    Name="Super1"
-                    RowsCached="1000"
-                    KeysCachedFraction="0"
+                    RowsCached="10000"
+                    KeysCached="50%"
                     Comment="A column family with supercolumns, whose column and subcolumn names are UTF8 strings"/>
 
       <!--
@@ -134,6 +139,7 @@
        ~ and PropertyFileEndpointSnitch is available in contrib/.
       -->
       <EndpointSnitch>org.apache.cassandra.locator.EndpointSnitch</EndpointSnitch>
+
     </Keyspace>
   </Keyspaces>
 
@@ -188,8 +194,6 @@
   <DataFileDirectories>
       <DataFileDirectory>/var/lib/cassandra/data</DataFileDirectory>
   </DataFileDirectories>
-  <CalloutLocation>/var/lib/cassandra/callouts</CalloutLocation>
-  <StagingFileDirectory>/var/lib/cassandra/staging</StagingFileDirectory>
 
 
   <!--
@@ -205,7 +209,7 @@
   <!-- Miscellaneous -->
 
   <!-- Time to wait for a reply from other nodes before failing the command -->
-  <RpcTimeoutInMillis>5000</RpcTimeoutInMillis>
+  <RpcTimeoutInMillis>10000</RpcTimeoutInMillis>
   <!-- Size to allow commitlog to grow to before creating a new segment -->
   <CommitLogRotationThresholdInMB>128</CommitLogRotationThresholdInMB>
 
@@ -221,6 +225,7 @@
    ~ (hostname, name resolution, etc), and the Right Thing is to use the
    ~ address associated with the hostname (it might not be).
   -->
+  <!--<ListenAddress>localhost</ListenAddress>-->
   <ListenAddress>127.0.0.2</ListenAddress>
   <!-- internal communications port -->
   <StoragePort>7000</StoragePort>
@@ -260,6 +265,14 @@
    -->
   <DiskAccessMode>auto</DiskAccessMode>
 
+  <!--
+   ~ Size of compacted row above which to log a warning.  (If compacted
+   ~ rows do not fit in memory, Cassandra will crash.  This is explained
+   ~ in http://wiki.apache.org/cassandra/CassandraLimitations and is
+   ~ scheduled to be fixed in 0.7.)
+  -->
+  <RowWarningThresholdInMB>512</RowWarningThresholdInMB>
+
   <!--
    ~ Buffer size to use when performing contiguous column slices. Increase
    ~ this to the size of the column slices you typically perform. 
@@ -307,7 +320,7 @@
    ~ ColumnFamily before flushing to disk.  This is also a per-memtable
    ~ setting.  Use with MemtableThroughputInMB to tune memory usage.
   -->
-  <MemtableOperationsInMillions>0.1</MemtableOperationsInMillions>
+  <MemtableOperationsInMillions>0.3</MemtableOperationsInMillions>
   <!--
    ~ The maximum time to leave a dirty memtable unflushed.
    ~ (While any affected columnfamilies have unflushed data from a
@@ -337,7 +350,7 @@
    ~ This is less necessary in Cassandra than in traditional databases
    ~ since replication reduces the odds of losing data from a failure
    ~ after writing the log entry but before it actually reaches the disk.
-   ~ So the other option is "timed," where writes may be acked immediately
+   ~ So the other option is "periodic," where writes may be acked immediately
    ~ and the CommitLog is simply synced every CommitLogSyncPeriodInMS
    ~ milliseconds.
   -->

diff --git a/debian/changelog b/debian/changelog
@@ -1,3 +1,9 @@
+cassandra (0.6.1) unstable; urgency=low
+
+  * New stable point release.
+
+ -- Eric Evans <eevans@apache.org>  Thu, 15 Apr 2010 08:04:20 -0500
+
 cassandra (0.6.0~rc1-1) unstable; urgency=low
 
   * New release candidated.

diff --git a/debian/rules b/debian/rules
@@ -13,7 +13,7 @@ test:
 clean:
 	dh_testdir
 	dh_testroot
-	$(ANT) clean
+	$(ANT) realclean
 	rm -f build-stamp build.properties
 	rm -rf bin/java
 	find -name "*.pyc" -exec rm '{}' ';'

diff --git a/src/java/org/apache/cassandra/db/Table.java b/src/java/org/apache/cassandra/db/Table.java
@@ -340,18 +340,7 @@ public void apply(RowMutation mutation, Object serializedMutation, boolean write
         {
             if (writeCommitLog)
             {
-                Future<CommitLogSegment.CommitLogContext> future = CommitLog.instance().add(mutation, serializedMutation);
-                if (waitForCommitLog)
-                {
-                    try
-                    {
-                        future.get();
-                    }
-                    catch (Exception e)
-                    {
-                        throw new RuntimeException(e);
-                    }
-                }
+                CommitLog.instance().add(mutation, serializedMutation);
             }
 
             DecoratedKey key = StorageService.getPartitioner().decorateKey(mutation.key());