Skip to content

Commit

Permalink
merge from 1.0
Browse files Browse the repository at this point in the history
  • Loading branch information
jbellis committed Jul 26, 2012
2 parents 10372c2 + 1ad710b commit 070c1d8
Show file tree
Hide file tree
Showing 8 changed files with 258 additions and 63 deletions.
3 changes: 3 additions & 0 deletions CHANGES.txt
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
* fix nodetool's setcompactionthreshold command (CASSANDRA-4455)
* Ensure compacted files are never used, to avoid counter overcount (CASSANDRA-4436)
Merged from 1.0:
* Push the validation of secondary index values to the SecondaryIndexManager (CASSANDRA-4240)
* allow dropping columns shadowed by not-yet-expired supercolumn or row
tombstones in PrecompactedRow (CASSANDRA-4396)
* fix 1.0.x node join to mixed version cluster, other nodes >= 1.1 (CASSANDRA-4195)
Expand Down Expand Up @@ -156,6 +157,8 @@ Merged from 1.0:
* improve ability of STCS.getBuckets to deal with 100s of 1000s of
sstables, such as when convertinb back from LCS (CASSANDRA-4287)
* Oversize integer in CQL throws NumberFormatException (CASSANDRA-4291)
* fix 1.0.x node join to mixed version cluster, other nodes >= 1.1 (CASSANDRA-4195)
* Fix LCS splitting sstable base on uncompressed size (CASSANDRA-4419)


1.1.0-final
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,8 @@

import org.apache.cassandra.db.DecoratedKey;
import org.apache.cassandra.db.IColumn;
import org.apache.cassandra.thrift.Column;
import org.apache.cassandra.utils.FBUtilities;

/**
* Base class for Secondary indexes that implement a unique index per column
Expand Down Expand Up @@ -60,4 +62,10 @@ public String getNameForSystemTable(ByteBuffer column)
{
return getIndexName();
}

@Override
public boolean validate(Column column)
{
return column.value.remaining() < FBUtilities.MAX_UNSIGNED_SHORT;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
import org.apache.cassandra.db.ColumnFamily;
import org.apache.cassandra.db.DecoratedKey;
import org.apache.cassandra.db.IColumn;
import org.apache.cassandra.thrift.Column;
import org.apache.cassandra.utils.ByteBufferUtil;

/**
Expand Down Expand Up @@ -70,4 +71,10 @@ public String getNameForSystemTable(ByteBuffer columnName)
throw new RuntimeException(e);
}
}

@Override
public boolean validate(Column column)
{
return true;
}
}
3 changes: 3 additions & 0 deletions src/java/org/apache/cassandra/db/index/SecondaryIndex.java
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@
import org.apache.cassandra.db.index.keys.KeysIndex;
import org.apache.cassandra.io.sstable.ReducingKeyIterator;
import org.apache.cassandra.io.sstable.SSTableReader;
import org.apache.cassandra.thrift.Column;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
Expand Down Expand Up @@ -330,4 +331,6 @@ public static SecondaryIndex createInstance(ColumnFamilyStore baseCfs, ColumnDef

return index;
}

public abstract boolean validate(Column column);
}
19 changes: 15 additions & 4 deletions src/java/org/apache/cassandra/db/index/SecondaryIndexManager.java
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,14 @@
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.*;
import java.util.concurrent.*;
import java.util.concurrent.ConcurrentNavigableMap;
import java.util.concurrent.ConcurrentSkipListMap;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;

import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import org.apache.cassandra.config.ColumnDefinition;
import org.apache.cassandra.config.ConfigurationException;
Expand All @@ -31,11 +38,9 @@
import org.apache.cassandra.dht.LocalToken;
import org.apache.cassandra.io.sstable.ReducingKeyIterator;
import org.apache.cassandra.io.sstable.SSTableReader;
import org.apache.cassandra.thrift.Column;
import org.apache.cassandra.thrift.IndexExpression;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
* Manages all the indexes associated with a given CFS
Expand Down Expand Up @@ -606,4 +611,10 @@ public void setIndexRemoved(Collection<ByteBuffer> indexes)
for (ByteBuffer colName : indexes)
indexesByColumn.get(colName).setIndexRemoved(colName);
}

public boolean validate(Column column)
{
SecondaryIndex index = getIndexForColumn(column.name);
return index != null ? index.validate(column) : true;
}
}
14 changes: 7 additions & 7 deletions src/java/org/apache/cassandra/thrift/ThriftValidation.java
Original file line number Diff line number Diff line change
Expand Up @@ -441,13 +441,13 @@ public static void validateColumnData(CFMetaData metadata, Column column, boolea
(isSubColumn ? metadata.subcolumnComparator : metadata.comparator).getString(column.name)));
}

// Indexed column values cannot be larger than 64K. See CASSANDRA-3057 for more details
if (columnDef != null && columnDef.getIndexType() != null && column.value.remaining() > FBUtilities.MAX_UNSIGNED_SHORT)
throw new InvalidRequestException(String.format("Can't index column value of size %d for index %s in CF %s of KS %s",
column.value.remaining(),
columnDef.getIndexName(),
metadata.cfName,
metadata.ksName));
// Indexed column values cannot be larger than 64K. See CASSANDRA-3057/4240 for more details
if (!Table.open(metadata.ksName).getColumnFamilyStore(metadata.cfName).indexManager.validate(column))
throw new InvalidRequestException(String.format("Can't index column value of size %d for index %s in CF %s of KS %s",
column.value.remaining(),
columnDef.getIndexName(),
metadata.cfName,
metadata.ksName));
}

/**
Expand Down
215 changes: 215 additions & 0 deletions test/unit/org/apache/cassandra/db/SecondaryIndexColumnSizeTest.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,215 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.cassandra.db;

import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.List;
import java.util.Set;
import java.util.SortedSet;

import org.junit.Test;

import org.apache.cassandra.config.ConfigurationException;
import org.apache.cassandra.db.index.PerColumnSecondaryIndex;
import org.apache.cassandra.db.index.PerRowSecondaryIndex;
import org.apache.cassandra.db.index.SecondaryIndexSearcher;
import org.apache.cassandra.thrift.Column;
import org.apache.cassandra.utils.ByteBufferUtil;

import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;

public class SecondaryIndexColumnSizeTest
{
@Test
public void test64kColumn()
{
Column column = new Column();
column.name = ByteBufferUtil.bytes("test");

// a byte buffer more than 64k
ByteBuffer buffer = ByteBuffer.allocate(1024 * 65);
buffer.clear();

//read more than 64k
for (int i=0; i<1024*64/4 + 1; i++)
buffer.putInt(0);

// for read
buffer.flip();
column.value = buffer;

MockRowIndex mockRowIndex = new MockRowIndex();
MockColumnIndex mockColumnIndex = new MockColumnIndex();

assertTrue(mockRowIndex.validate(column));
assertFalse(mockColumnIndex.validate(column));

// test less than 64k value
buffer.flip();
buffer.clear();
buffer.putInt(20);
buffer.flip();

assertTrue(mockRowIndex.validate(column));
assertTrue(mockColumnIndex.validate(column));
}

private class MockRowIndex extends PerRowSecondaryIndex
{
@Override
public void applyIndexUpdates(ByteBuffer rowKey, ColumnFamily cf, SortedSet<ByteBuffer> mutatedIndexedColumns, ColumnFamily oldIndexedColumns) throws IOException
{
}

@Override
public void init()
{
}

@Override
public void validateOptions() throws ConfigurationException
{
}

@Override
public String getIndexName()
{
return null;
}

@Override
protected SecondaryIndexSearcher createSecondaryIndexSearcher(Set<ByteBuffer> columns)
{
return null;
}

@Override
public void forceBlockingFlush() throws IOException
{
}

@Override
public long getLiveSize()
{
return 0;
}

@Override
public ColumnFamilyStore getIndexCfs()
{
return null;
}

@Override
public void removeIndex(ByteBuffer columnName) throws IOException
{
}

@Override
public void invalidate()
{
}

@Override
public void truncate(long truncatedAt)
{
}

@Override
public void deleteFromIndex(DecoratedKey<?> key, List<IColumn> indexedColumnsInRow)
{
}

}


private class MockColumnIndex extends PerColumnSecondaryIndex
{
@Override
public void init()
{
}

@Override
public void validateOptions() throws ConfigurationException
{
}

@Override
public String getIndexName()
{
return null;
}

@Override
protected SecondaryIndexSearcher createSecondaryIndexSearcher(Set<ByteBuffer> columns)
{
return null;
}

@Override
public void forceBlockingFlush() throws IOException
{
}

@Override
public long getLiveSize()
{
return 0;
}

@Override
public ColumnFamilyStore getIndexCfs()
{
return null;
}

@Override
public void removeIndex(ByteBuffer columnName) throws IOException
{
}

@Override
public void invalidate()
{
}

@Override
public void truncate(long truncatedAt)
{
}

@Override
public void deleteColumn(DecoratedKey valueKey, ByteBuffer rowKey, IColumn col) throws IOException
{
}

@Override
public void insertColumn(DecoratedKey valueKey, ByteBuffer rowKey, IColumn col) throws IOException
{
}

@Override
public void updateColumn(DecoratedKey valueKey, ByteBuffer rowKey, IColumn col) throws IOException
{
}
}
}
Loading

0 comments on commit 070c1d8

Please sign in to comment.