Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

make index_interval configurable per columnfamily

patch by Radim Kolar; reviewed by jbellis for CASSANDRA-3961
  • Loading branch information...
commit 6a5c9ed87995059e8cb824608fd5907432fa7ab4 1 parent 40d2ad1
@jbellis jbellis authored
Showing with 882 additions and 753 deletions.
  1. +1 −0  CHANGES.txt
  2. +0 −12 conf/cassandra.yaml
  3. +0 −5 examples/client_only/conf/cassandra.yaml
  4. +2 −1  interface/cassandra.thrift
  5. +16 −7 interface/thrift/gen-java/org/apache/cassandra/thrift/AuthenticationException.java
  6. +19 −9 interface/thrift/gen-java/org/apache/cassandra/thrift/AuthenticationRequest.java
  7. +643 −645 interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java
  8. +100 −10 interface/thrift/gen-java/org/apache/cassandra/thrift/CfDef.java
  9. +1 −1  interface/thrift/gen-java/org/apache/cassandra/thrift/Constants.java
  10. +1 −1  interface/thrift/gen-java/org/apache/cassandra/thrift/IndexClause.java
  11. +4 −4 interface/thrift/gen-java/org/apache/cassandra/thrift/KsDef.java
  12. +4 −1 pylib/cqlshlib/cql3handling.py
  13. +7 −0 src/java/org/apache/cassandra/cli/CliClient.java
  14. +21 −2 src/java/org/apache/cassandra/config/CFMetaData.java
  15. +0 −2  src/java/org/apache/cassandra/config/Config.java
  16. +0 −5 src/java/org/apache/cassandra/config/DatabaseDescriptor.java
  17. +2 −2 src/java/org/apache/cassandra/db/compaction/CompactionManager.java
  18. +1 −1  src/java/org/apache/cassandra/db/compaction/CompactionTask.java
  19. +1 −1  src/java/org/apache/cassandra/db/compaction/Scrubber.java
  20. +1 −1  src/java/org/apache/cassandra/dht/AbstractByteOrderedPartitioner.java
  21. +1 −1  src/java/org/apache/cassandra/dht/OrderPreservingPartitioner.java
  22. +19 −15 src/java/org/apache/cassandra/io/sstable/IndexSummary.java
  23. +20 −14 src/java/org/apache/cassandra/io/sstable/SSTableReader.java
  24. +1 −1  src/java/org/apache/cassandra/io/sstable/SSTableWriter.java
  25. +5 −5 src/java/org/apache/cassandra/service/StorageService.java
  26. +1 −1  src/java/org/apache/cassandra/thrift/CassandraServer.java
  27. +5 −0 src/resources/org/apache/cassandra/cli/CliHelp.yaml
  28. +2 −2 test/long/org/apache/cassandra/db/compaction/LongCompactionsTest.java
  29. +1 −1  test/unit/org/apache/cassandra/db/KeyCollisionTest.java
  30. +1 −1  test/unit/org/apache/cassandra/db/compaction/CompactionsTest.java
  31. +1 −1  test/unit/org/apache/cassandra/io/LazilyCompactedRowTest.java
  32. +1 −1  test/unit/org/apache/cassandra/io/sstable/SSTableReaderTest.java
View
1  CHANGES.txt
@@ -1,4 +1,5 @@
1.3
+ * make index_interval configurable per columnfamily (CASSANDRA-3961)
* add default_tim_to_live (CASSANDRA-3974)
* add memtable_flush_period_in_ms (CASSANDRA-4237)
View
12 conf/cassandra.yaml
@@ -598,18 +598,6 @@ request_scheduler: org.apache.cassandra.scheduler.NoScheduler
# the request scheduling. Currently the only valid option is keyspace.
# request_scheduler_id: keyspace
-# index_interval controls the sampling of entries from the primrary
-# row index in terms of space versus time. The larger the interval,
-# the smaller and less effective the sampling will be. In technicial
-# terms, the interval coresponds to the number of index entries that
-# are skipped between taking each sample. All the sampled entries
-# must fit in memory. Generally, a value between 128 and 512 here
-# coupled with a large key cache size on CFs results in the best trade
-# offs. This value is not often changed, however if you have many
-# very small rows (many to an OS page), then increasing this will
-# often lower memory usage without a impact on performance.
-index_interval: 128
-
# Enable or disable inter-node encryption
# Default settings are TLS v1, RSA 1024-bit keys (it is imperative that
# users generate their own keys) TLS_RSA_WITH_AES_128_CBC_SHA as the cipher
View
5 examples/client_only/conf/cassandra.yaml
@@ -307,11 +307,6 @@ request_scheduler: org.apache.cassandra.scheduler.NoScheduler
# the request scheduling. Currently the only valid option is keyspace.
# request_scheduler_id: keyspace
-# The Index Interval determines how large the sampling of row keys
-# is for a given SSTable. The larger the sampling, the more effective
-# the index is at the cost of space.
-index_interval: 128
-
# Keyspaces have ColumnFamilies. (Usually 1 KS per application.)
# ColumnFamilies have Rows. (Dozens of CFs per KS.)
# Rows contain Columns. (Many per CF.)
View
3  interface/cassandra.thrift
@@ -55,7 +55,7 @@ namespace rb CassandraThrift
# An effort should be made not to break forward-client-compatibility either
# (e.g. one should avoid removing obsolete fields from the IDL), but no
# guarantees in this respect are made by the Cassandra project.
-const string VERSION = "19.35.0"
+const string VERSION = "19.36.0"
#
@@ -444,6 +444,7 @@ struct CfDef {
37: optional double dclocal_read_repair_chance = 0.0,
38: optional i32 memtable_flush_period_in_ms,
39: optional i32 default_time_to_live,
+ 40: optional i32 index_interval,
/* All of the following are now ignored and unsupplied. */
View
23 interface/thrift/gen-java/org/apache/cassandra/thrift/AuthenticationException.java
@@ -5,7 +5,7 @@
*/
package org.apache.cassandra.thrift;
/*
- *
+ *
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -13,25 +13,34 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- *
+ *
* http://www.apache.org/licenses/LICENSE-2.0
- *
+ *
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
- *
+ *
*/
import org.apache.commons.lang.builder.HashCodeBuilder;
+import java.util.List;
+import java.util.ArrayList;
import java.util.Map;
import java.util.HashMap;
import java.util.EnumMap;
+import java.util.Set;
+import java.util.HashSet;
import java.util.EnumSet;
import java.util.Collections;
+import java.util.BitSet;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* invalid authentication request (invalid keyspace, user does not exist, or credentials invalid)
@@ -106,7 +115,7 @@ public String getFieldName() {
public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
static {
Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.WHY, new org.apache.thrift.meta_data.FieldMetaData("why", org.apache.thrift.TFieldRequirementType.REQUIRED,
+ tmpMap.put(_Fields.WHY, new org.apache.thrift.meta_data.FieldMetaData("why", org.apache.thrift.TFieldRequirementType.REQUIRED,
new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
metaDataMap = Collections.unmodifiableMap(tmpMap);
org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(AuthenticationException.class, metaDataMap);
@@ -267,14 +276,14 @@ public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.t
while (true)
{
field = iprot.readFieldBegin();
- if (field.type == org.apache.thrift.protocol.TType.STOP) {
+ if (field.type == org.apache.thrift.protocol.TType.STOP) {
break;
}
switch (field.id) {
case 1: // WHY
if (field.type == org.apache.thrift.protocol.TType.STRING) {
this.why = iprot.readString();
- } else {
+ } else {
org.apache.thrift.protocol.TProtocolUtil.skip(iprot, field.type);
}
break;
View
28 interface/thrift/gen-java/org/apache/cassandra/thrift/AuthenticationRequest.java
@@ -5,7 +5,7 @@
*/
package org.apache.cassandra.thrift;
/*
- *
+ *
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -13,24 +13,34 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- *
+ *
* http://www.apache.org/licenses/LICENSE-2.0
- *
+ *
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
- *
+ *
*/
+
import org.apache.commons.lang.builder.HashCodeBuilder;
+import java.util.List;
+import java.util.ArrayList;
import java.util.Map;
import java.util.HashMap;
import java.util.EnumMap;
+import java.util.Set;
+import java.util.HashSet;
import java.util.EnumSet;
import java.util.Collections;
+import java.util.BitSet;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* Authentication requests can contain any data, dependent on the IAuthenticator used
@@ -105,9 +115,9 @@ public String getFieldName() {
public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
static {
Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.CREDENTIALS, new org.apache.thrift.meta_data.FieldMetaData("credentials", org.apache.thrift.TFieldRequirementType.REQUIRED,
- new org.apache.thrift.meta_data.MapMetaData(org.apache.thrift.protocol.TType.MAP,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING),
+ tmpMap.put(_Fields.CREDENTIALS, new org.apache.thrift.meta_data.FieldMetaData("credentials", org.apache.thrift.TFieldRequirementType.REQUIRED,
+ new org.apache.thrift.meta_data.MapMetaData(org.apache.thrift.protocol.TType.MAP,
+ new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING),
new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))));
metaDataMap = Collections.unmodifiableMap(tmpMap);
org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(AuthenticationRequest.class, metaDataMap);
@@ -291,7 +301,7 @@ public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.t
while (true)
{
field = iprot.readFieldBegin();
- if (field.type == org.apache.thrift.protocol.TType.STOP) {
+ if (field.type == org.apache.thrift.protocol.TType.STOP) {
break;
}
switch (field.id) {
@@ -310,7 +320,7 @@ public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.t
}
iprot.readMapEnd();
}
- } else {
+ } else {
org.apache.thrift.protocol.TProtocolUtil.skip(iprot, field.type);
}
break;
View
1,288 interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java
643 additions, 645 deletions not shown
View
110 interface/thrift/gen-java/org/apache/cassandra/thrift/CfDef.java
@@ -69,6 +69,7 @@
private static final org.apache.thrift.protocol.TField DCLOCAL_READ_REPAIR_CHANCE_FIELD_DESC = new org.apache.thrift.protocol.TField("dclocal_read_repair_chance", org.apache.thrift.protocol.TType.DOUBLE, (short)37);
private static final org.apache.thrift.protocol.TField MEMTABLE_FLUSH_PERIOD_IN_MS_FIELD_DESC = new org.apache.thrift.protocol.TField("memtable_flush_period_in_ms", org.apache.thrift.protocol.TType.I32, (short)38);
private static final org.apache.thrift.protocol.TField DEFAULT_TIME_TO_LIVE_FIELD_DESC = new org.apache.thrift.protocol.TField("default_time_to_live", org.apache.thrift.protocol.TType.I32, (short)39);
+ private static final org.apache.thrift.protocol.TField INDEX_INTERVAL_FIELD_DESC = new org.apache.thrift.protocol.TField("index_interval", org.apache.thrift.protocol.TType.I32, (short)40);
private static final org.apache.thrift.protocol.TField ROW_CACHE_SIZE_FIELD_DESC = new org.apache.thrift.protocol.TField("row_cache_size", org.apache.thrift.protocol.TType.DOUBLE, (short)9);
private static final org.apache.thrift.protocol.TField KEY_CACHE_SIZE_FIELD_DESC = new org.apache.thrift.protocol.TField("key_cache_size", org.apache.thrift.protocol.TType.DOUBLE, (short)11);
private static final org.apache.thrift.protocol.TField ROW_CACHE_SAVE_PERIOD_IN_SECONDS_FIELD_DESC = new org.apache.thrift.protocol.TField("row_cache_save_period_in_seconds", org.apache.thrift.protocol.TType.I32, (short)19);
@@ -104,6 +105,7 @@
public double dclocal_read_repair_chance; // required
public int memtable_flush_period_in_ms; // required
public int default_time_to_live; // required
+ public int index_interval; // required
/**
* @deprecated
*/
@@ -171,6 +173,7 @@
DCLOCAL_READ_REPAIR_CHANCE((short)37, "dclocal_read_repair_chance"),
MEMTABLE_FLUSH_PERIOD_IN_MS((short)38, "memtable_flush_period_in_ms"),
DEFAULT_TIME_TO_LIVE((short)39, "default_time_to_live"),
+ INDEX_INTERVAL((short)40, "index_interval"),
/**
* @deprecated
*/
@@ -273,6 +276,8 @@ public static _Fields findByThriftId(int fieldId) {
return MEMTABLE_FLUSH_PERIOD_IN_MS;
case 39: // DEFAULT_TIME_TO_LIVE
return DEFAULT_TIME_TO_LIVE;
+ case 40: // INDEX_INTERVAL
+ return INDEX_INTERVAL;
case 9: // ROW_CACHE_SIZE
return ROW_CACHE_SIZE;
case 11: // KEY_CACHE_SIZE
@@ -343,16 +348,17 @@ public String getFieldName() {
private static final int __DCLOCAL_READ_REPAIR_CHANCE_ISSET_ID = 7;
private static final int __MEMTABLE_FLUSH_PERIOD_IN_MS_ISSET_ID = 8;
private static final int __DEFAULT_TIME_TO_LIVE_ISSET_ID = 9;
- private static final int __ROW_CACHE_SIZE_ISSET_ID = 10;
- private static final int __KEY_CACHE_SIZE_ISSET_ID = 11;
- private static final int __ROW_CACHE_SAVE_PERIOD_IN_SECONDS_ISSET_ID = 12;
- private static final int __KEY_CACHE_SAVE_PERIOD_IN_SECONDS_ISSET_ID = 13;
- private static final int __MEMTABLE_FLUSH_AFTER_MINS_ISSET_ID = 14;
- private static final int __MEMTABLE_THROUGHPUT_IN_MB_ISSET_ID = 15;
- private static final int __MEMTABLE_OPERATIONS_IN_MILLIONS_ISSET_ID = 16;
- private static final int __MERGE_SHARDS_CHANCE_ISSET_ID = 17;
- private static final int __ROW_CACHE_KEYS_TO_SAVE_ISSET_ID = 18;
- private BitSet __isset_bit_vector = new BitSet(19);
+ private static final int __INDEX_INTERVAL_ISSET_ID = 10;
+ private static final int __ROW_CACHE_SIZE_ISSET_ID = 11;
+ private static final int __KEY_CACHE_SIZE_ISSET_ID = 12;
+ private static final int __ROW_CACHE_SAVE_PERIOD_IN_SECONDS_ISSET_ID = 13;
+ private static final int __KEY_CACHE_SAVE_PERIOD_IN_SECONDS_ISSET_ID = 14;
+ private static final int __MEMTABLE_FLUSH_AFTER_MINS_ISSET_ID = 15;
+ private static final int __MEMTABLE_THROUGHPUT_IN_MB_ISSET_ID = 16;
+ private static final int __MEMTABLE_OPERATIONS_IN_MILLIONS_ISSET_ID = 17;
+ private static final int __MERGE_SHARDS_CHANCE_ISSET_ID = 18;
+ private static final int __ROW_CACHE_KEYS_TO_SAVE_ISSET_ID = 19;
+ private BitSet __isset_bit_vector = new BitSet(20);
public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
static {
@@ -410,6 +416,8 @@ public String getFieldName() {
new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I32)));
tmpMap.put(_Fields.DEFAULT_TIME_TO_LIVE, new org.apache.thrift.meta_data.FieldMetaData("default_time_to_live", org.apache.thrift.TFieldRequirementType.OPTIONAL,
new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I32)));
+ tmpMap.put(_Fields.INDEX_INTERVAL, new org.apache.thrift.meta_data.FieldMetaData("index_interval", org.apache.thrift.TFieldRequirementType.OPTIONAL,
+ new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I32)));
tmpMap.put(_Fields.ROW_CACHE_SIZE, new org.apache.thrift.meta_data.FieldMetaData("row_cache_size", org.apache.thrift.TFieldRequirementType.OPTIONAL,
new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.DOUBLE)));
tmpMap.put(_Fields.KEY_CACHE_SIZE, new org.apache.thrift.meta_data.FieldMetaData("key_cache_size", org.apache.thrift.TFieldRequirementType.OPTIONAL,
@@ -541,6 +549,7 @@ public CfDef(CfDef other) {
this.dclocal_read_repair_chance = other.dclocal_read_repair_chance;
this.memtable_flush_period_in_ms = other.memtable_flush_period_in_ms;
this.default_time_to_live = other.default_time_to_live;
+ this.index_interval = other.index_interval;
this.row_cache_size = other.row_cache_size;
this.key_cache_size = other.key_cache_size;
this.row_cache_save_period_in_seconds = other.row_cache_save_period_in_seconds;
@@ -598,6 +607,8 @@ public void clear() {
this.memtable_flush_period_in_ms = 0;
setDefault_time_to_liveIsSet(false);
this.default_time_to_live = 0;
+ setIndex_intervalIsSet(false);
+ this.index_interval = 0;
setRow_cache_sizeIsSet(false);
this.row_cache_size = 0.0;
setKey_cache_sizeIsSet(false);
@@ -1232,6 +1243,29 @@ public void setDefault_time_to_liveIsSet(boolean value) {
__isset_bit_vector.set(__DEFAULT_TIME_TO_LIVE_ISSET_ID, value);
}
+ public int getIndex_interval() {
+ return this.index_interval;
+ }
+
+ public CfDef setIndex_interval(int index_interval) {
+ this.index_interval = index_interval;
+ setIndex_intervalIsSet(true);
+ return this;
+ }
+
+ public void unsetIndex_interval() {
+ __isset_bit_vector.clear(__INDEX_INTERVAL_ISSET_ID);
+ }
+
+ /** Returns true if field index_interval is set (has been assigned a value) and false otherwise */
+ public boolean isSetIndex_interval() {
+ return __isset_bit_vector.get(__INDEX_INTERVAL_ISSET_ID);
+ }
+
+ public void setIndex_intervalIsSet(boolean value) {
+ __isset_bit_vector.set(__INDEX_INTERVAL_ISSET_ID, value);
+ }
+
/**
* @deprecated
*/
@@ -1717,6 +1751,14 @@ public void setFieldValue(_Fields field, Object value) {
}
break;
+ case INDEX_INTERVAL:
+ if (value == null) {
+ unsetIndex_interval();
+ } else {
+ setIndex_interval((Integer)value);
+ }
+ break;
+
case ROW_CACHE_SIZE:
if (value == null) {
unsetRow_cache_size();
@@ -1874,6 +1916,9 @@ public Object getFieldValue(_Fields field) {
case DEFAULT_TIME_TO_LIVE:
return Integer.valueOf(getDefault_time_to_live());
+ case INDEX_INTERVAL:
+ return Integer.valueOf(getIndex_interval());
+
case ROW_CACHE_SIZE:
return Double.valueOf(getRow_cache_size());
@@ -1963,6 +2008,8 @@ public boolean isSet(_Fields field) {
return isSetMemtable_flush_period_in_ms();
case DEFAULT_TIME_TO_LIVE:
return isSetDefault_time_to_live();
+ case INDEX_INTERVAL:
+ return isSetIndex_interval();
case ROW_CACHE_SIZE:
return isSetRow_cache_size();
case KEY_CACHE_SIZE:
@@ -2216,6 +2263,15 @@ public boolean equals(CfDef that) {
return false;
}
+ boolean this_present_index_interval = true && this.isSetIndex_interval();
+ boolean that_present_index_interval = true && that.isSetIndex_interval();
+ if (this_present_index_interval || that_present_index_interval) {
+ if (!(this_present_index_interval && that_present_index_interval))
+ return false;
+ if (this.index_interval != that.index_interval)
+ return false;
+ }
+
boolean this_present_row_cache_size = true && this.isSetRow_cache_size();
boolean that_present_row_cache_size = true && that.isSetRow_cache_size();
if (this_present_row_cache_size || that_present_row_cache_size) {
@@ -2433,6 +2489,11 @@ public int hashCode() {
if (present_default_time_to_live)
builder.append(default_time_to_live);
+ boolean present_index_interval = true && (isSetIndex_interval());
+ builder.append(present_index_interval);
+ if (present_index_interval)
+ builder.append(index_interval);
+
boolean present_row_cache_size = true && (isSetRow_cache_size());
builder.append(present_row_cache_size);
if (present_row_cache_size)
@@ -2734,6 +2795,16 @@ public int compareTo(CfDef other) {
return lastComparison;
}
}
+ lastComparison = Boolean.valueOf(isSetIndex_interval()).compareTo(typedOther.isSetIndex_interval());
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ if (isSetIndex_interval()) {
+ lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.index_interval, typedOther.index_interval);
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ }
lastComparison = Boolean.valueOf(isSetRow_cache_size()).compareTo(typedOther.isSetRow_cache_size());
if (lastComparison != 0) {
return lastComparison;
@@ -3064,6 +3135,14 @@ public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.t
org.apache.thrift.protocol.TProtocolUtil.skip(iprot, field.type);
}
break;
+ case 40: // INDEX_INTERVAL
+ if (field.type == org.apache.thrift.protocol.TType.I32) {
+ this.index_interval = iprot.readI32();
+ setIndex_intervalIsSet(true);
+ } else {
+ org.apache.thrift.protocol.TProtocolUtil.skip(iprot, field.type);
+ }
+ break;
case 9: // ROW_CACHE_SIZE
if (field.type == org.apache.thrift.protocol.TType.DOUBLE) {
this.row_cache_size = iprot.readDouble();
@@ -3377,6 +3456,11 @@ public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.
oprot.writeI32(this.default_time_to_live);
oprot.writeFieldEnd();
}
+ if (isSetIndex_interval()) {
+ oprot.writeFieldBegin(INDEX_INTERVAL_FIELD_DESC);
+ oprot.writeI32(this.index_interval);
+ oprot.writeFieldEnd();
+ }
oprot.writeFieldStop();
oprot.writeStructEnd();
}
@@ -3581,6 +3665,12 @@ public String toString() {
sb.append(this.default_time_to_live);
first = false;
}
+ if (isSetIndex_interval()) {
+ if (!first) sb.append(", ");
+ sb.append("index_interval:");
+ sb.append(this.index_interval);
+ first = false;
+ }
if (isSetRow_cache_size()) {
if (!first) sb.append(", ");
sb.append("row_cache_size:");
View
2  interface/thrift/gen-java/org/apache/cassandra/thrift/Constants.java
@@ -44,6 +44,6 @@
public class Constants {
- public static final String VERSION = "19.35.0";
+ public static final String VERSION = "19.36.0";
}
View
2  interface/thrift/gen-java/org/apache/cassandra/thrift/IndexClause.java
@@ -43,7 +43,7 @@
import org.slf4j.LoggerFactory;
/**
- * @Deprecated: use a KeyRange with row_filter in get_range_slices instead
+ * @deprecated use a KeyRange with row_filter in get_range_slices instead
*/
public class IndexClause implements org.apache.thrift.TBase<IndexClause, IndexClause._Fields>, java.io.Serializable, Cloneable {
private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("IndexClause");
View
8 interface/thrift/gen-java/org/apache/cassandra/thrift/KsDef.java
@@ -56,7 +56,7 @@
public String strategy_class; // required
public Map<String,String> strategy_options; // required
/**
- * @deprecated, ignored
+ * @deprecated ignored
*/
public int replication_factor; // required
public List<CfDef> cf_defs; // required
@@ -68,7 +68,7 @@
STRATEGY_CLASS((short)2, "strategy_class"),
STRATEGY_OPTIONS((short)3, "strategy_options"),
/**
- * @deprecated, ignored
+ * @deprecated ignored
*/
REPLICATION_FACTOR((short)4, "replication_factor"),
CF_DEFS((short)5, "cf_defs"),
@@ -319,14 +319,14 @@ public void setStrategy_optionsIsSet(boolean value) {
}
/**
- * @deprecated, ignored
+ * @deprecated ignored
*/
public int getReplication_factor() {
return this.replication_factor;
}
/**
- * @deprecated, ignored
+ * @deprecated ignored
*/
public KsDef setReplication_factor(int replication_factor) {
this.replication_factor = replication_factor;
View
5 pylib/cqlshlib/cql3handling.py
@@ -64,6 +64,7 @@ class Cql3ParsingRuleSet(CqlParsingRuleSet):
('default_validation', 'default_validation_class'),
('replicate_on_write', None),
('compaction_strategy_class', 'compaction_strategy'),
+ ('index_interval', None),
)
old_columnfamily_layout_options = (
@@ -76,6 +77,7 @@ class Cql3ParsingRuleSet(CqlParsingRuleSet):
('gc_grace_seconds', None),
('replicate_on_write', None),
('compaction_strategy_class', None),
+ ('index_interval', None),
)
new_columnfamily_layout_options = (
@@ -88,6 +90,7 @@ class Cql3ParsingRuleSet(CqlParsingRuleSet):
('replicate_on_write', None),
('default_read_consistency', None),
('default_write_consistency', None),
+ ('index_interval', None),
)
old_columnfamily_layout_map_options = (
@@ -490,7 +493,7 @@ def cf_new_prop_val_completer(ctxt, cass):
if this_opt == 'replicate_on_write':
return ["'yes'", "'no'"]
if this_opt in ('min_compaction_threshold', 'max_compaction_threshold',
- 'gc_grace_seconds'):
+ 'gc_grace_seconds', 'index_interval'):
return [Hint('<integer>')]
if this_opt == 'default_read_consistency':
return [cl for cl in CqlRuleSet.consistency_levels if cl != 'ANY']
View
7 src/java/org/apache/cassandra/cli/CliClient.java
@@ -137,6 +137,7 @@ public static String getFunctionNames()
COMPACTION_STRATEGY_OPTIONS,
COMPRESSION_OPTIONS,
BLOOM_FILTER_FP_CHANCE,
+ INDEX_INTERVAL,
MEMTABLE_FLUSH_PERIOD_IN_MS,
CACHING,
DEFAULT_TIME_TO_LIVE
@@ -1334,6 +1335,9 @@ private CfDef updateCfDefAttributes(Tree statement, CfDef cfDefToUpdate)
case DEFAULT_TIME_TO_LIVE:
cfDef.setDefault_time_to_live(Integer.parseInt(mValue));
break;
+ case INDEX_INTERVAL:
+ cfDef.setIndex_interval(Integer.parseInt(mValue));
+ break;
default:
//must match one of the above or we'd throw an exception at the valueOf statement above.
assert(false);
@@ -1861,6 +1865,8 @@ private void showColumnFamily(PrintStream output, CfDef cfDef)
writeAttrRaw(output, false, "compression_options", compOptions.toString());
}
+ if (cfDef.isSetIndex_interval())
+ writeAttr(output, false, "index_interval", cfDef.index_interval);
output.append(";");
output.append(NEWLINE);
@@ -2156,6 +2162,7 @@ private void describeColumnFamily(KsDef ks_def, CfDef cf_def, NodeProbe probe) t
sessionState.out.printf(" Caching: %s%n", cf_def.caching);
sessionState.out.printf(" Default time to live: %s%n", cf_def.default_time_to_live);
sessionState.out.printf(" Bloom Filter FP chance: %s%n", cf_def.isSetBloom_filter_fp_chance() ? cf_def.bloom_filter_fp_chance : "default");
+ sessionState.out.printf(" Index interval: %s%n", cf_def.isSetIndex_interval() ? cf_def.index_interval : "default");
// if we have connection to the cfMBean established
if (cfMBean != null)
View
23 src/java/org/apache/cassandra/config/CFMetaData.java
@@ -76,6 +76,7 @@
public final static ByteBuffer DEFAULT_KEY_NAME = ByteBufferUtil.bytes("KEY");
public final static Caching DEFAULT_CACHING_STRATEGY = Caching.KEYS_ONLY;
public final static int DEFAULT_DEFAULT_TIME_TO_LIVE = 0;
+ public final static int DEFAULT_INDEX_INTERVAL = 128;
// Note that this is the default only for user created tables
public final static String DEFAULT_COMPRESSOR = SnappyCompressor.isAvailable() ? SnappyCompressor.class.getCanonicalName() : null;
@@ -262,6 +263,7 @@ public static Caching fromString(String cache) throws ConfigurationException
private volatile ByteBuffer valueAlias = null;
private volatile Double bloomFilterFpChance = null;
private volatile Caching caching = DEFAULT_CACHING_STRATEGY;
+ private volatile int indexInterval = DEFAULT_INDEX_INTERVAL;
private int memtableFlushPeriod = 0;
private volatile int defaultTimeToLive = DEFAULT_DEFAULT_TIME_TO_LIVE;
@@ -294,6 +296,7 @@ public static Caching fromString(String cache) throws ConfigurationException
public CFMetaData compressionParameters(CompressionParameters prop) {compressionParameters = prop; return this;}
public CFMetaData bloomFilterFpChance(Double prop) {bloomFilterFpChance = prop; return this;}
public CFMetaData caching(Caching prop) {caching = prop; return this;}
+ public CFMetaData indexInterval(int prop) {indexInterval = prop; return this;}
public CFMetaData memtableFlushPeriod(int prop) {memtableFlushPeriod = prop; return this;}
public CFMetaData defaultTimeToLive(int prop) {defaultTimeToLive = prop; return this;}
@@ -435,8 +438,9 @@ static CFMetaData copyOpts(CFMetaData newCFMD, CFMetaData oldCFMD)
.compressionParameters(oldCFMD.compressionParameters)
.bloomFilterFpChance(oldCFMD.bloomFilterFpChance)
.caching(oldCFMD.caching)
- .memtableFlushPeriod(oldCFMD.memtableFlushPeriod)
- .defaultTimeToLive(oldCFMD.defaultTimeToLive);
+ .defaultTimeToLive(oldCFMD.defaultTimeToLive)
+ .indexInterval(oldCFMD.indexInterval)
+ .memtableFlushPeriod(oldCFMD.memtableFlushPeriod);
}
/**
@@ -549,6 +553,11 @@ public Caching getCaching()
return caching;
}
+ public int getIndexInterval()
+ {
+ return indexInterval;
+ }
+
public int getMemtableFlushPeriod()
{
return memtableFlushPeriod;
@@ -598,6 +607,7 @@ else if (obj == null || obj.getClass() != getClass())
.append(memtableFlushPeriod, rhs.memtableFlushPeriod)
.append(caching, rhs.caching)
.append(defaultTimeToLive, rhs.defaultTimeToLive)
+ .append(indexInterval, rhs.indexInterval)
.isEquals();
}
@@ -630,6 +640,7 @@ public int hashCode()
.append(memtableFlushPeriod)
.append(caching)
.append(defaultTimeToLive)
+ .append(indexInterval)
.toHashCode();
}
@@ -713,6 +724,8 @@ public static CFMetaData fromThrift(org.apache.cassandra.thrift.CfDef cf_def) th
newCFMD.defaultTimeToLive(cf_def.default_time_to_live);
if (cf_def.isSetDclocal_read_repair_chance())
newCFMD.dcLocalReadRepairChance(cf_def.dclocal_read_repair_chance);
+ if (cf_def.isSetIndex_interval())
+ newCFMD.indexInterval(cf_def.index_interval);
CompressionParameters cp = CompressionParameters.create(cf_def.compression_options);
@@ -911,6 +924,7 @@ public AbstractCompactionStrategy createCompactionStrategyInstance(ColumnFamilyS
def.setCompression_options(compressionParameters.asThriftOptions());
if (bloomFilterFpChance != null)
def.setBloom_filter_fp_chance(bloomFilterFpChance);
+ def.setIndex_interval(indexInterval);
def.setMemtable_flush_period_in_ms(memtableFlushPeriod);
def.setCaching(caching.toString());
def.setDefault_time_to_live(defaultTimeToLive);
@@ -1260,6 +1274,7 @@ public RowMutation dropFromSchema(long timestamp)
cf.addColumn(DeletedColumn.create(ldt, timestamp, cfName, "value_alias"));
cf.addColumn(DeletedColumn.create(ldt, timestamp, cfName, "column_aliases"));
cf.addColumn(DeletedColumn.create(ldt, timestamp, cfName, "compaction_strategy_options"));
+ cf.addColumn(DeletedColumn.create(ldt, timestamp, cfName, "index_interval"));
for (ColumnDefinition cd : column_metadata.values())
cd.deleteFromSchema(rm, cfName, getColumnDefinitionComparator(cd), timestamp);
@@ -1313,6 +1328,7 @@ private void toSchemaNoColumns(RowMutation rm, long timestamp)
: Column.create(valueAlias, timestamp, cfName, "value_alias"));
cf.addColumn(Column.create(json(aliasesAsStrings(columnAliases)), timestamp, cfName, "column_aliases"));
cf.addColumn(Column.create(json(compactionStrategyOptions), timestamp, cfName, "compaction_strategy_options"));
+ cf.addColumn(Column.create(indexInterval, timestamp, cfName, "index_interval"));
}
// Package protected for use by tests
@@ -1361,6 +1377,8 @@ else if (result.has("key_alias"))
if (result.has("value_alias"))
cfm.valueAlias(result.getBytes("value_alias"));
cfm.compactionStrategyOptions(fromJsonMap(result.getString("compaction_strategy_options")));
+ if (result.has("index_interval"))
+ cfm.indexInterval(result.getInt("index_interval"));
return cfm;
}
@@ -1526,6 +1544,7 @@ public String toString()
.append("memtable_flush_period_in_ms", memtableFlushPeriod)
.append("caching", caching)
.append("defaultTimeToLive", defaultTimeToLive)
+ .append("indexInterval", indexInterval)
.toString();
}
}
View
2  src/java/org/apache/cassandra/config/Config.java
@@ -138,8 +138,6 @@
public InternodeCompression internode_compression = InternodeCompression.none;
- public Integer index_interval = 128;
-
public Double flush_largest_memtables_at = 1.0;
public Double reduce_cache_sizes_at = 1.0;
public double reduce_cache_capacity_to = 0.6;
View
5 src/java/org/apache/cassandra/config/DatabaseDescriptor.java
@@ -1059,11 +1059,6 @@ public static int getMaxHintWindow()
return conf.max_hint_window_in_ms;
}
- public static Integer getIndexInterval()
- {
- return conf.index_interval;
- }
-
public static File getSerializedCachePath(String ksName, String cfName, CacheService.CacheType cacheType, String version)
{
return new File(conf.saved_caches_directory + File.separator + ksName + "-" + cfName + "-" + cacheType + (version == null ? "" : "-" + version + ".db"));
View
4 src/java/org/apache/cassandra/db/compaction/CompactionManager.java
@@ -575,8 +575,8 @@ private void doCleanupCompaction(ColumnFamilyStore cfs, Collection<SSTableReader
long totalkeysWritten = 0;
- int expectedBloomFilterSize = Math.max(DatabaseDescriptor.getIndexInterval(),
- (int)(SSTableReader.getApproximateKeyCount(Arrays.asList(sstable))));
+ int expectedBloomFilterSize = Math.max(cfs.metadata.getIndexInterval(),
+ (int)(SSTableReader.getApproximateKeyCount(Arrays.asList(sstable),cfs.metadata)));
if (logger.isDebugEnabled())
logger.debug("Expected bloom filter size : " + expectedBloomFilterSize);
View
2  src/java/org/apache/cassandra/db/compaction/CompactionTask.java
@@ -119,7 +119,7 @@ protected void runWith(File dataDirectory) throws Exception
long totalkeysWritten = 0;
AbstractCompactionStrategy strategy = cfs.getCompactionStrategy();
- long estimatedTotalKeys = Math.max(DatabaseDescriptor.getIndexInterval(), SSTableReader.getApproximateKeyCount(toCompact));
+ long estimatedTotalKeys = Math.max(cfs.metadata.getIndexInterval(), SSTableReader.getApproximateKeyCount(toCompact, cfs.metadata));
long estimatedSSTables = Math.max(1, SSTable.getTotalBytes(toCompact) / strategy.getMaxSSTableSize());
long keysPerSSTable = (long) Math.ceil((double) estimatedTotalKeys / estimatedSSTables);
if (logger.isDebugEnabled())
View
2  src/java/org/apache/cassandra/db/compaction/Scrubber.java
@@ -88,7 +88,7 @@ public Scrubber(ColumnFamilyStore cfs, SSTableReader sstable, OutputHandler outp
? new ScrubController(cfs)
: new CompactionController(cfs, Collections.singletonList(sstable), CompactionManager.getDefaultGcBefore(cfs));
this.isCommutative = cfs.metadata.getDefaultValidator().isCommutative();
- this.expectedBloomFilterSize = Math.max(DatabaseDescriptor.getIndexInterval(), (int)(SSTableReader.getApproximateKeyCount(toScrub)));
+ this.expectedBloomFilterSize = Math.max(cfs.metadata.getIndexInterval(), (int)(SSTableReader.getApproximateKeyCount(toScrub,cfs.metadata)));
// loop through each row, deserializing to check for damage.
// we'll also loop through the index at the same time, using the position from the index to recover if the
View
2  src/java/org/apache/cassandra/dht/AbstractByteOrderedPartitioner.java
@@ -201,7 +201,7 @@ public boolean preservesOrder()
for (Range<Token> r : sortedRanges)
{
// Looping over every KS:CF:Range, get the splits size and add it to the count
- allTokens.put(r.right, allTokens.get(r.right) + StorageService.instance.getSplits(ks, cfmd.cfName, r, 1).size());
+ allTokens.put(r.right, allTokens.get(r.right) + StorageService.instance.getSplits(ks, cfmd.cfName, r, 1, cfmd).size());
}
}
}
View
2  src/java/org/apache/cassandra/dht/OrderPreservingPartitioner.java
@@ -194,7 +194,7 @@ public StringToken getToken(ByteBuffer key)
for (Range<Token> r : sortedRanges)
{
// Looping over every KS:CF:Range, get the splits size and add it to the count
- allTokens.put(r.right, allTokens.get(r.right) + StorageService.instance.getSplits(ks, cfmd.cfName, r, DatabaseDescriptor.getIndexInterval()).size());
+ allTokens.put(r.right, allTokens.get(r.right) + StorageService.instance.getSplits(ks, cfmd.cfName, r, cfmd.getIndexInterval(), cfmd).size());
}
}
}
View
34 src/java/org/apache/cassandra/io/sstable/IndexSummary.java
@@ -24,7 +24,6 @@
import java.util.ArrayList;
import java.util.List;
-import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.DecoratedKey;
import org.apache.cassandra.dht.IPartitioner;
import org.apache.cassandra.utils.ByteBufferUtil;
@@ -40,21 +39,23 @@
private final ArrayList<Long> positions;
private final ArrayList<DecoratedKey> keys;
private long keysWritten = 0;
+ private int indexInterval;
- public IndexSummary(long expectedKeys)
+ private IndexSummary()
{
- long expectedEntries = expectedKeys / DatabaseDescriptor.getIndexInterval();
+ positions = new ArrayList<Long>();
+ keys = new ArrayList<DecoratedKey>();
+ }
+
+ public IndexSummary(long expectedKeys, int indexInterval)
+ {
+ long expectedEntries = expectedKeys / indexInterval;
if (expectedEntries > Integer.MAX_VALUE)
// TODO: that's a _lot_ of keys, or a very low interval
- throw new RuntimeException("Cannot use index_interval of " + DatabaseDescriptor.getIndexInterval() + " with " + expectedKeys + " (expected) keys.");
+ throw new RuntimeException("Cannot use index_interval of " + indexInterval + " with " + expectedKeys + " (expected) keys.");
positions = new ArrayList<Long>((int)expectedEntries);
keys = new ArrayList<DecoratedKey>((int)expectedEntries);
- }
-
- private IndexSummary()
- {
- positions = new ArrayList<Long>();
- keys = new ArrayList<DecoratedKey>();
+ this.indexInterval = indexInterval;
}
public void incrementRowid()
@@ -64,7 +65,7 @@ public void incrementRowid()
public boolean shouldAddEntry()
{
- return keysWritten % DatabaseDescriptor.getIndexInterval() == 0;
+ return keysWritten % indexInterval == 0;
}
public void addEntry(DecoratedKey key, long indexPosition)
@@ -90,7 +91,11 @@ public long getPosition(int index)
return positions.get(index);
}
- public void complete()
+ public int getIndexInterval() {
+ return indexInterval;
+ }
+
+ public void complete()
{
keys.trimToSize();
positions.trimToSize();
@@ -101,7 +106,7 @@ public void complete()
public void serialize(IndexSummary t, DataOutput dos) throws IOException
{
assert t.keys.size() == t.positions.size() : "keysize and the position sizes are not same.";
- dos.writeInt(DatabaseDescriptor.getIndexInterval());
+ dos.writeInt(t.indexInterval);
dos.writeInt(t.keys.size());
for (int i = 0; i < t.keys.size(); i++)
{
@@ -113,8 +118,7 @@ public void serialize(IndexSummary t, DataOutput dos) throws IOException
public IndexSummary deserialize(DataInput dis, IPartitioner partitioner) throws IOException
{
IndexSummary summary = new IndexSummary();
- if (dis.readInt() != DatabaseDescriptor.getIndexInterval())
- throw new IOException("Cannot read the saved summary because Index Interval changed.");
+ summary.indexInterval = dis.readInt();
int size = dis.readInt();
for (int i = 0; i < size; i++)
View
34 src/java/org/apache/cassandra/io/sstable/SSTableReader.java
@@ -62,7 +62,7 @@
private static final Logger logger = LoggerFactory.getLogger(SSTableReader.class);
// guesstimated size of INDEX_INTERVAL index entries
- private static final int INDEX_FILE_BUFFER_BYTES = 16 * DatabaseDescriptor.getIndexInterval();
+ private static final int INDEX_FILE_BUFFER_BYTES = 16 * CFMetaData.DEFAULT_INDEX_INTERVAL;
/**
* maxDataAge is a timestamp in local server time (e.g. System.currentTimeMilli) which represents an uppper bound
@@ -98,14 +98,14 @@
private final SSTableMetadata sstableMetadata;
- public static long getApproximateKeyCount(Iterable<SSTableReader> sstables)
+ public static long getApproximateKeyCount(Iterable<SSTableReader> sstables, CFMetaData metadata)
{
long count = 0;
for (SSTableReader sstable : sstables)
{
int indexKeyCount = sstable.getKeySamples().size();
- count = count + (indexKeyCount + 1) * DatabaseDescriptor.getIndexInterval();
+ count = count + (indexKeyCount + 1) * metadata.getIndexInterval();
if (logger.isDebugEnabled())
logger.debug("index size for bloom filter calc for file : " + sstable.getFilename() + " : " + count);
}
@@ -354,7 +354,7 @@ private void load(boolean recreatebloom) throws IOException
// try to load summaries from the disk and check if we need
// to read primary index because we should re-create a BloomFilter or pre-load KeyCache
- final boolean summaryLoaded = loadSummary(this, ibuilder, dbuilder);
+ final boolean summaryLoaded = loadSummary(this, ibuilder, dbuilder, metadata);
final boolean readIndex = recreatebloom || !summaryLoaded;
try
{
@@ -367,7 +367,7 @@ private void load(boolean recreatebloom) throws IOException
bf = LegacyBloomFilter.getFilter(estimatedKeys, 15);
if (!summaryLoaded)
- indexSummary = new IndexSummary(estimatedKeys);
+ indexSummary = new IndexSummary(estimatedKeys, metadata.getIndexInterval());
long indexPosition;
while (readIndex && (indexPosition = primaryIndex.getFilePointer()) != indexSize)
@@ -406,7 +406,7 @@ private void load(boolean recreatebloom) throws IOException
saveSummary(this, ibuilder, dbuilder);
}
- public static boolean loadSummary(SSTableReader reader, SegmentedFile.Builder ibuilder, SegmentedFile.Builder dbuilder)
+ public static boolean loadSummary(SSTableReader reader, SegmentedFile.Builder ibuilder, SegmentedFile.Builder dbuilder, CFMetaData metadata)
{
File summariesFile = new File(reader.descriptor.filenameFor(Component.SUMMARY));
if (!summariesFile.exists())
@@ -417,6 +417,14 @@ public static boolean loadSummary(SSTableReader reader, SegmentedFile.Builder ib
{
iStream = new DataInputStream(new FileInputStream(summariesFile));
reader.indexSummary = IndexSummary.serializer.deserialize(iStream, reader.partitioner);
+ if (reader.indexSummary.getIndexInterval() != metadata.getIndexInterval())
+ {
+ iStream.close();
+ logger.debug("Cannot read the saved summary for {} because Index Interval changed from {} to {}.",
+ reader.toString(), reader.indexSummary.getIndexInterval(), metadata.getIndexInterval());
+ FileUtils.deleteWithConfirm(summariesFile);
+ return false;
+ }
reader.first = decodeKey(reader.partitioner, reader.descriptor, ByteBufferUtil.readWithLength(iStream));
reader.last = decodeKey(reader.partitioner, reader.descriptor, ByteBufferUtil.readWithLength(iStream));
ibuilder.deserializeBounds(iStream);
@@ -425,10 +433,8 @@ public static boolean loadSummary(SSTableReader reader, SegmentedFile.Builder ib
catch (IOException e)
{
logger.debug("Cannot deserialize SSTable Summary: ", e);
- // corrupted hence delete it and let it load it now.
- if (summariesFile.exists())
- summariesFile.delete();
-
+ // corrupted; delete it and fall back to creating a new summary
+ FileUtils.deleteWithConfirm(summariesFile);
return false;
}
finally
@@ -530,7 +536,7 @@ public long getBloomFilterSerializedSize()
*/
public long estimatedKeys()
{
- return indexSummary.getKeys().size() * DatabaseDescriptor.getIndexInterval();
+ return indexSummary.getKeys().size() * metadata.getIndexInterval();
}
/**
@@ -543,7 +549,7 @@ public long estimatedKeysForRanges(Collection<Range<Token>> ranges)
List<Pair<Integer, Integer>> sampleIndexes = getSampleIndexesForRanges(indexSummary.getKeys(), ranges);
for (Pair<Integer, Integer> sampleIndexRange : sampleIndexes)
sampleKeyCount += (sampleIndexRange.right - sampleIndexRange.left + 1);
- return Math.max(1, sampleKeyCount * DatabaseDescriptor.getIndexInterval());
+ return Math.max(1, sampleKeyCount * metadata.getIndexInterval());
}
/**
@@ -778,12 +784,12 @@ public RowIndexEntry getPosition(RowPosition key, Operator op, boolean updateCac
// of the next interval).
int i = 0;
Iterator<FileDataInput> segments = ifile.iterator(sampledPosition, INDEX_FILE_BUFFER_BYTES);
- while (segments.hasNext() && i <= DatabaseDescriptor.getIndexInterval())
+ while (segments.hasNext() && i <= metadata.getIndexInterval())
{
FileDataInput in = segments.next();
try
{
- while (!in.isEOF() && i <= DatabaseDescriptor.getIndexInterval())
+ while (!in.isEOF() && i <= metadata.getIndexInterval())
{
i++;
View
2  src/java/org/apache/cassandra/io/sstable/SSTableWriter.java
@@ -438,7 +438,7 @@ public long getOnDiskFilePointer()
indexFile = SequentialWriter.open(new File(descriptor.filenameFor(SSTable.COMPONENT_INDEX)),
!DatabaseDescriptor.populateIOCacheOnFlush());
builder = SegmentedFile.getBuilder(DatabaseDescriptor.getIndexAccessMode());
- summary = new IndexSummary(keyCount);
+ summary = new IndexSummary(keyCount, metadata.getIndexInterval());
double fpChance = metadata.getBloomFilterFpChance();
if (fpChance == 0)
View
10 src/java/org/apache/cassandra/service/StorageService.java
@@ -2475,13 +2475,13 @@ public void setLog4jLevel(String classQualifier, String rawLevel)
* @return list of Token ranges (_not_ keys!) together with estimated key count,
* breaking up the data this node is responsible for into pieces of roughly keysPerSplit
*/
- public List<Pair<Range<Token>, Long>> getSplits(String table, String cfName, Range<Token> range, int keysPerSplit)
+ public List<Pair<Range<Token>, Long>> getSplits(String table, String cfName, Range<Token> range, int keysPerSplit, CFMetaData metadata)
{
Table t = Table.open(table);
ColumnFamilyStore cfs = t.getColumnFamilyStore(cfName);
List<DecoratedKey> keys = keySamples(Collections.singleton(cfs), range);
- final long totalRowCountEstimate = (keys.size() + 1) * DatabaseDescriptor.getIndexInterval();
+ final long totalRowCountEstimate = (keys.size() + 1) * metadata.getIndexInterval();
// splitCount should be much smaller than number of key samples, to avoid huge sampling error
final int minSamplesPerSplit = 4;
@@ -2489,10 +2489,10 @@ public void setLog4jLevel(String classQualifier, String rawLevel)
final int splitCount = Math.max(1, Math.min(maxSplitCount, (int)(totalRowCountEstimate / keysPerSplit)));
List<Token> tokens = keysToTokens(range, keys);
- return getSplits(tokens, splitCount);
+ return getSplits(tokens, splitCount, metadata);
}
- private List<Pair<Range<Token>, Long>> getSplits(List<Token> tokens, int splitCount)
+ private List<Pair<Range<Token>, Long>> getSplits(List<Token> tokens, int splitCount, CFMetaData metadata)
{
final double step = (double) (tokens.size() - 1) / splitCount;
int prevIndex = 0;
@@ -2502,7 +2502,7 @@ public void setLog4jLevel(String classQualifier, String rawLevel)
{
int index = (int) Math.round(i * step);
Token token = tokens.get(index);
- long rowCountEstimate = (index - prevIndex) * DatabaseDescriptor.getIndexInterval();
+ long rowCountEstimate = (index - prevIndex) * metadata.getIndexInterval();
splits.add(Pair.create(new Range<Token>(prevToken, token), rowCountEstimate));
prevIndex = index;
prevToken = token;
View
2  src/java/org/apache/cassandra/thrift/CassandraServer.java
@@ -1224,7 +1224,7 @@ public String describe_snitch() throws TException
Token.TokenFactory tf = StorageService.getPartitioner().getTokenFactory();
Range<Token> tr = new Range<Token>(tf.fromString(start_token), tf.fromString(end_token));
List<Pair<Range<Token>, Long>> splits =
- StorageService.instance.getSplits(state().getKeyspace(), cfName, tr, keys_per_split);
+ StorageService.instance.getSplits(state().getKeyspace(), cfName, tr, keys_per_split, Schema.instance.getCFMetaData(state().getKeyspace(), cfName));
List<CfSplit> result = new ArrayList<CfSplit>(splits.size());
for (Pair<Range<Token>, Long> split : splits)
result.add(new CfSplit(split.left.left.toString(), split.left.right.toString(), split.right));
View
5 src/resources/org/apache/cassandra/cli/CliHelp.yaml
@@ -394,6 +394,11 @@ commands:
- bloom_filter_fp_chance: Desired false positive probability for
sstable row bloom filters. Default is 0.000744.
+ - index_interval: controls the sampling of entries from the primrary
+ row index in terms of space versus time. The larger the interval,
+ the smaller and less effective the sampling will be. All the sampled
+ entries must fit in memory. Default value is 128.
+
- column_type: Type of columns this column family holds, valid values are
Standard and Super. Default is Standard.
View
4 test/long/org/apache/cassandra/db/compaction/LongCompactionsTest.java
@@ -119,7 +119,7 @@ public void testStandardColumnCompactions() throws IOException, ExecutionExcepti
cfs.clearUnsafe();
final int ROWS_PER_SSTABLE = 10;
- final int SSTABLES = DatabaseDescriptor.getIndexInterval() * 3 / ROWS_PER_SSTABLE;
+ final int SSTABLES = cfs.metadata.getIndexInterval() * 3 / ROWS_PER_SSTABLE;
// disable compaction while flushing
cfs.disableAutoCompaction();
@@ -159,7 +159,7 @@ public void testSuperColumnCompactions() throws IOException, ExecutionException,
ColumnFamilyStore cfs = table.getColumnFamilyStore("Super1");
final int ROWS_PER_SSTABLE = 10;
- final int SSTABLES = DatabaseDescriptor.getIndexInterval() * 3 / ROWS_PER_SSTABLE;
+ final int SSTABLES = cfs.metadata.getIndexInterval() * 3 / ROWS_PER_SSTABLE;
//disable compaction while flushing
cfs.disableAutoCompaction();
View
2  test/unit/org/apache/cassandra/db/KeyCollisionTest.java
@@ -192,7 +192,7 @@ public BigIntegerToken getToken(ByteBuffer key)
for (Range<Token> r : sortedRanges)
{
// Looping over every KS:CF:Range, get the splits size and add it to the count
- allTokens.put(r.right, allTokens.get(r.right) + StorageService.instance.getSplits(ks, cfmd.cfName, r, 1).size());
+ allTokens.put(r.right, allTokens.get(r.right) + StorageService.instance.getSplits(ks, cfmd.cfName, r, 1, cfmd).size());
}
}
}
View
2  test/unit/org/apache/cassandra/db/compaction/CompactionsTest.java
@@ -289,7 +289,7 @@ public void testBlacklisting(String compactionStrategy) throws Exception
final ColumnFamilyStore cfs = table.getColumnFamilyStore("Standard1");
final int ROWS_PER_SSTABLE = 10;
- final int SSTABLES = DatabaseDescriptor.getIndexInterval() * 2 / ROWS_PER_SSTABLE;
+ final int SSTABLES = cfs.metadata.getIndexInterval() * 2 / ROWS_PER_SSTABLE;
cfs.setCompactionStrategyClass(compactionStrategy);
View
2  test/unit/org/apache/cassandra/io/LazilyCompactedRowTest.java
@@ -278,7 +278,7 @@ public void testManyRows() throws IOException, ExecutionException, InterruptedEx
ColumnFamilyStore cfs = table.getColumnFamilyStore("Standard1");
final int ROWS_PER_SSTABLE = 10;
- for (int j = 0; j < (DatabaseDescriptor.getIndexInterval() * 3) / ROWS_PER_SSTABLE; j++)
+ for (int j = 0; j < (cfs.metadata.getIndexInterval() * 3) / ROWS_PER_SSTABLE; j++)
{
for (int i = 0; i < ROWS_PER_SSTABLE; i++)
{
View
2  test/unit/org/apache/cassandra/io/sstable/SSTableReaderTest.java
@@ -260,7 +260,7 @@ public void testOpeningSSTable() throws Exception
DecoratedKey firstKey = null, lastKey = null;
long timestamp = System.currentTimeMillis();
- for (int i = 0; i < DatabaseDescriptor.getIndexInterval(); i++)
+ for (int i = 0; i < store.metadata.getIndexInterval(); i++)
{
DecoratedKey key = Util.dk(String.valueOf(i));
if (firstKey == null)
Please sign in to comment.
Something went wrong with that request. Please try again.