Skip to content
This repository

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse code

Merge branch 'p/4121/01_support_multiple_tokens_per_host' into refs/t…

…op-bases/p/4122/01_bootstrap_decommission
  • Loading branch information...
commit 3cfc95772056bc3d701401997b9cf8f12da3f7f6 2 parents cd3bfa3 + 198d6d2
Eric Evans eevans authored
3  CHANGES.txt
@@ -50,6 +50,9 @@
50 50 (CASSANDRA-4187)
51 51 * Fix exception during move when localhost is the only source (CASSANDRA-4200)
52 52 * (cql3) Allow paging through non-ordered partitioner results (CASSANDRA-3771)
  53 + * (cql3) Fix drop index (CASSANDRA-4192)
  54 + * (cql3) Don't return range ghosts anymore (CASSANDRA-3982)
  55 + * fix the problem re-creating Keyspace/ColumnFamily (CASSANDRA-4219)
53 56 Merged from 1.0:
54 57 * Fix super columns bug where cache is not updated (CASSANDRA-4190)
55 58 * fix maxTimestamp to include row tombstones (CASSANDRA-4116)
31 src/java/org/apache/cassandra/config/CFMetaData.java
@@ -850,10 +850,18 @@ public ColumnDefinition getColumnDefinitionForIndex(String indexName)
850 850 */
851 851 public void addDefaultIndexNames() throws ConfigurationException
852 852 {
  853 + Set<String> existingNames = existingIndexNames(null);
853 854 for (ColumnDefinition column : column_metadata.values())
854 855 {
855 856 if (column.getIndexType() != null && column.getIndexName() == null)
856   - column.setIndexName(getDefaultIndexName(cfName, comparator, column.name));
  857 + {
  858 + String baseName = getDefaultIndexName(cfName, comparator, column.name);
  859 + String indexName = baseName;
  860 + int i = 0;
  861 + while (existingNames.contains(indexName))
  862 + indexName = baseName + '_' + (++i);
  863 + column.setIndexName(indexName);
  864 + }
857 865 }
858 866 }
859 867
@@ -942,14 +950,7 @@ public CFMetaData validate() throws ConfigurationException
942 950 validateAlias(valueAlias, "Value");
943 951
944 952 // initialize a set of names NOT in the CF under consideration
945   - Set<String> indexNames = new HashSet<String>();
946   - for (ColumnFamilyStore cfs : ColumnFamilyStore.all())
947   - {
948   - if (!cfs.getColumnFamilyName().equals(cfName))
949   - for (ColumnDefinition cd : cfs.metadata.getColumn_metadata().values())
950   - indexNames.add(cd.getIndexName());
951   - }
952   -
  953 + Set<String> indexNames = existingIndexNames(cfName);
953 954 for (ColumnDefinition c : column_metadata.values())
954 955 {
955 956 AbstractType<?> comparator = getColumnDefinitionComparator(c);
@@ -996,6 +997,18 @@ public CFMetaData validate() throws ConfigurationException
996 997 return this;
997 998 }
998 999
  1000 + private static Set<String> existingIndexNames(String cfToExclude)
  1001 + {
  1002 + Set<String> indexNames = new HashSet<String>();
  1003 + for (ColumnFamilyStore cfs : ColumnFamilyStore.all())
  1004 + {
  1005 + if (cfToExclude == null || !cfs.getColumnFamilyName().equals(cfToExclude))
  1006 + for (ColumnDefinition cd : cfs.metadata.getColumn_metadata().values())
  1007 + indexNames.add(cd.getIndexName());
  1008 + }
  1009 + return indexNames;
  1010 + }
  1011 +
999 1012 private static void validateAlias(ByteBuffer alias, String msg) throws ConfigurationException
1000 1013 {
1001 1014 if (alias != null)
2  src/java/org/apache/cassandra/config/Schema.java
@@ -453,7 +453,7 @@ public void updateVersion()
453 453
454 454 for (Row row : SystemTable.serializedSchema())
455 455 {
456   - if (row.cf == null || row.cf.isMarkedForDelete() || row.cf.isEmpty())
  456 + if (row.cf == null || (row.cf.isMarkedForDelete() && row.cf.isEmpty()))
457 457 continue;
458 458
459 459 row.cf.updateDigest(versionDigest);
2  src/java/org/apache/cassandra/cql3/Cql.g
@@ -435,7 +435,7 @@ truncateStatement returns [TruncateStatement stmt]
435 435 cident returns [ColumnIdentifier id]
436 436 : t=IDENT { $id = new ColumnIdentifier($t.text, false); }
437 437 | t=QUOTED_NAME { $id = new ColumnIdentifier($t.text, true); }
438   - | k=unreserved_keyword { $id = new ColumnIdentifier(k, true); }
  438 + | k=unreserved_keyword { $id = new ColumnIdentifier(k, false); }
439 439 ;
440 440
441 441 // Keyspace & Column family names
17 src/java/org/apache/cassandra/cql3/statements/DropIndexStatement.java
@@ -28,12 +28,12 @@
28 28
29 29 public class DropIndexStatement extends SchemaAlteringStatement
30 30 {
31   - public final CharSequence index;
  31 + public final String indexName;
32 32
33 33 public DropIndexStatement(String indexName)
34 34 {
35 35 super(new CFName());
36   - index = indexName;
  36 + this.indexName = indexName;
37 37 }
38 38
39 39 public void announceMigration() throws InvalidRequestException, ConfigurationException
@@ -50,7 +50,7 @@ public void announceMigration() throws InvalidRequestException, ConfigurationExc
50 50 }
51 51
52 52 if (updatedCfm == null)
53   - throw new InvalidRequestException("Index '" + index + "' could not be found in any of the column families of keyspace '" + keyspace() + "'");
  53 + throw new InvalidRequestException("Index '" + indexName + "' could not be found in any of the column families of keyspace '" + keyspace() + "'");
54 54
55 55 MigrationManager.announceColumnFamilyUpdate(updatedCfm);
56 56 }
@@ -59,11 +59,14 @@ private CFMetaData getUpdatedCFMetadata(CFMetaData cfm) throws InvalidRequestExc
59 59 {
60 60 for (ColumnDefinition column : cfm.getColumn_metadata().values())
61 61 {
62   - if (column.getIndexType() != null && column.getIndexName() != null && column.getIndexName().equals(index))
  62 + if (column.getIndexType() != null && column.getIndexName() != null && column.getIndexName().equals(indexName))
63 63 {
64   - column.setIndexName(null);
65   - column.setIndexType(null, null);
66   - return cfm;
  64 + CFMetaData cloned = cfm.clone();
  65 + ColumnDefinition toChange = cloned.getColumn_metadata().get(column.name);
  66 + assert toChange.getIndexName() != null && toChange.getIndexName().equals(indexName);
  67 + toChange.setIndexName(null);
  68 + toChange.setIndexType(null, null);
  69 + return cloned;
67 70 }
68 71 }
69 72
53 src/java/org/apache/cassandra/cql3/statements/SelectStatement.java
@@ -189,7 +189,8 @@ public String columnFamily()
189 189 private List<Row> getSlice(List<ByteBuffer> variables) throws InvalidRequestException, TimedOutException, UnavailableException
190 190 {
191 191 QueryPath queryPath = new QueryPath(columnFamily());
192   - List<ReadCommand> commands = new ArrayList<ReadCommand>();
  192 + Collection<ByteBuffer> keys = getKeys(variables);
  193 + List<ReadCommand> commands = new ArrayList<ReadCommand>(keys.size());
193 194
194 195 // ...a range (slice) of column names
195 196 if (isColumnRange())
@@ -200,7 +201,7 @@ public String columnFamily()
200 201 // Note that we use the total limit for every key. This is
201 202 // potentially inefficient, but then again, IN + LIMIT is not a
202 203 // very sensible choice
203   - for (ByteBuffer key : getKeys(variables))
  204 + for (ByteBuffer key : keys)
204 205 {
205 206 QueryProcessor.validateKey(key);
206 207 QueryProcessor.validateSliceRange(cfDef.cfm, start, finish, isReversed);
@@ -219,7 +220,7 @@ public String columnFamily()
219 220 Collection<ByteBuffer> columnNames = getRequestedColumns(variables);
220 221 QueryProcessor.validateColumnNames(columnNames);
221 222
222   - for (ByteBuffer key: getKeys(variables))
  223 + for (ByteBuffer key: keys)
223 224 {
224 225 QueryProcessor.validateKey(key);
225 226 commands.add(new SliceByNamesReadCommand(keyspace(), key, queryPath, columnNames));
@@ -465,18 +466,18 @@ private boolean isWildcard()
465 466 }
466 467 else
467 468 {
468   - // Adds all (requested) columns
469   - List<Pair<CFDefinition.Name, ColumnIdentifier>> selection = getExpandedSelection();
470   - List<ByteBuffer> columns = new ArrayList<ByteBuffer>(selection.size());
471   - Iterator<Pair<CFDefinition.Name, ColumnIdentifier>> iter = selection.iterator();
  469 + // Adds all columns (even if the user selected a few columns, we
  470 + // need to query all columns to know if the row exists or not).
  471 + // Note that when we allow IS NOT NULL in queries and if all
  472 + // selected name are request 'not null', we will allow to only
  473 + // query those.
  474 + List<ByteBuffer> columns = new ArrayList<ByteBuffer>(cfDef.columns.size());
  475 + Iterator<ColumnIdentifier> iter = cfDef.metadata.keySet().iterator();
472 476 while (iter.hasNext())
473 477 {
474   - CFDefinition.Name name = iter.next().left;
475   - // Skip everything that is not a 'metadata' column
476   - if (name.kind != CFDefinition.Name.Kind.COLUMN_METADATA)
477   - continue;
  478 + ColumnIdentifier name = iter.next();
478 479 ColumnNameBuilder b = iter.hasNext() ? builder.copy() : builder;
479   - ByteBuffer cname = b.add(name.name.key).build();
  480 + ByteBuffer cname = b.add(name.key).build();
480 481 columns.add(cname);
481 482 }
482 483 return columns;
@@ -624,14 +625,19 @@ public IColumn computeNext()
624 625 List<Pair<CFDefinition.Name, ColumnIdentifier>> selection = getExpandedSelection();
625 626 List<Column> thriftColumns = null;
626 627
  628 + // Add schema only once
  629 + for (Pair<CFDefinition.Name, ColumnIdentifier> p : selection)
  630 + addToSchema(schema, p);
  631 +
627 632 for (org.apache.cassandra.db.Row row : rows)
628 633 {
  634 + // Not columns match the query, skip
  635 + if (row.cf == null)
  636 + continue;
  637 +
629 638 if (cfDef.isCompact)
630 639 {
631 640 // One cqlRow per column
632   - if (row.cf == null)
633   - continue;
634   -
635 641 for (IColumn c : columnsInOrder(row.cf, variables))
636 642 {
637 643 if (c.isMarkedForDelete())
@@ -660,7 +666,6 @@ else if (sliceRestriction != null)
660 666 CFDefinition.Name name = p.left;
661 667 ByteBuffer nameAsRequested = p.right.key;
662 668
663   - addToSchema(schema, p);
664 669 Column col = new Column(nameAsRequested);
665 670 switch (name.kind)
666 671 {
@@ -696,9 +701,6 @@ else if (sliceRestriction != null)
696 701 else if (cfDef.isComposite)
697 702 {
698 703 // Sparse case: group column in cqlRow when composite prefix is equal
699   - if (row.cf == null)
700   - continue;
701   -
702 704 CompositeType composite = (CompositeType)cfDef.cfm.comparator;
703 705 int last = composite.types.size() - 1;
704 706
@@ -727,8 +729,12 @@ else if (cfDef.isComposite)
727 729 }
728 730 else
729 731 {
  732 + if (row.cf.getLiveColumnCount() == 0)
  733 + continue;
  734 +
730 735 // Static case: One cqlRow for all columns
731 736 thriftColumns = new ArrayList<Column>(selection.size());
  737 +
732 738 // Respect selection order
733 739 for (Pair<CFDefinition.Name, ColumnIdentifier> p : selection)
734 740 {
@@ -737,15 +743,10 @@ else if (cfDef.isComposite)
737 743
738 744 if (name.kind == CFDefinition.Name.Kind.KEY_ALIAS)
739 745 {
740   - addToSchema(schema, p);
741 746 thriftColumns.add(new Column(nameAsRequested).setValue(row.key.key).setTimestamp(-1L));
742 747 continue;
743 748 }
744 749
745   - if (row.cf == null)
746   - continue;
747   -
748   - addToSchema(schema, p);
749 750 IColumn c = row.cf.getColumn(name.name.key);
750 751 Column col = new Column(name.name.key);
751 752 if (c != null && !c.isMarkedForDelete())
@@ -797,7 +798,6 @@ private CqlRow handleGroup(List<Pair<CFDefinition.Name, ColumnIdentifier>> selec
797 798 CFDefinition.Name name = p.left;
798 799 ByteBuffer nameAsRequested = p.right.key;
799 800
800   - addToSchema(schema, p);
801 801 Column col = new Column(nameAsRequested);
802 802 switch (name.kind)
803 803 {
@@ -813,7 +813,8 @@ private CqlRow handleGroup(List<Pair<CFDefinition.Name, ColumnIdentifier>> selec
813 813 throw new AssertionError();
814 814 case COLUMN_METADATA:
815 815 IColumn c = columns.get(name.name.key);
816   - if (c != null && !c.isMarkedForDelete())
  816 + // We already have excluded deleted columns
  817 + if (c != null)
817 818 col.setValue(value(c)).setTimestamp(c.timestamp());
818 819 break;
819 820 }
22 src/java/org/apache/cassandra/db/Column.java
@@ -124,7 +124,16 @@ public long mostRecentNonGCableChangeAt(int gcbefore)
124 124 return timestamp;
125 125 }
126 126
127   - public int size(TypeSizes typeSizes)
  127 + public int dataSize()
  128 + {
  129 + return name().remaining() + value.remaining() + TypeSizes.NATIVE.sizeof(timestamp);
  130 + }
  131 +
  132 + /*
  133 + * This returns the size of the column when serialized.
  134 + * @see com.facebook.infrastructure.db.IColumn#serializedSize()
  135 + */
  136 + public int serializedSize(TypeSizes sizes)
128 137 {
129 138 /*
130 139 * Size of a column is =
@@ -136,16 +145,7 @@ public int size(TypeSizes typeSizes)
136 145 */
137 146 int nameSize = name.remaining();
138 147 int valueSize = value.remaining();
139   - return typeSizes.sizeof((short) nameSize) + nameSize + 1 + typeSizes.sizeof(timestamp) + typeSizes.sizeof(valueSize) + valueSize;
140   - }
141   -
142   - /*
143   - * This returns the size of the column when serialized.
144   - * @see com.facebook.infrastructure.db.IColumn#serializedSize()
145   - */
146   - public int serializedSize(TypeSizes typeSizes)
147   - {
148   - return size(typeSizes);
  148 + return sizes.sizeof((short) nameSize) + nameSize + 1 + sizes.sizeof(timestamp) + sizes.sizeof(valueSize) + valueSize;
149 149 }
150 150
151 151 public int serializationFlags()
7 src/java/org/apache/cassandra/db/ColumnFamily.java
@@ -249,12 +249,13 @@ public ColumnFamily diff(ColumnFamily cfComposite)
249 249 return null;
250 250 }
251 251
252   - int size(TypeSizes typeSizes)
  252 + /** the size of user-provided data, not including internal overhead */
  253 + int dataSize()
253 254 {
254   - int size = TypeSizes.NATIVE.sizeof(1L) + TypeSizes.NATIVE.sizeof(1); // tombstone tracking
  255 + int size = TypeSizes.NATIVE.sizeof(getMarkedForDeleteAt());
255 256 for (IColumn column : columns)
256 257 {
257   - size += column.size(typeSizes);
  258 + size += column.dataSize();
258 259 }
259 260 return size;
260 261 }
10 src/java/org/apache/cassandra/db/CounterColumn.java
@@ -93,13 +93,19 @@ public long total()
93 93 }
94 94
95 95 @Override
96   - public int size(TypeSizes typeSizes)
  96 + public int dataSize()
97 97 {
98 98 /*
99 99 * A counter column adds to a Column :
100 100 * + 8 bytes for timestampOfLastDelete
101 101 */
102   - return super.size(typeSizes) + typeSizes.sizeof(timestampOfLastDelete);
  102 + return super.dataSize() + TypeSizes.NATIVE.sizeof(timestampOfLastDelete);
  103 + }
  104 +
  105 + @Override
  106 + public int serializedSize(TypeSizes sizes)
  107 + {
  108 + return super.serializedSize(sizes) + sizes.sizeof(timestampOfLastDelete);
103 109 }
104 110
105 111 @Override
2  src/java/org/apache/cassandra/db/DefsTable.java
@@ -152,7 +152,7 @@ public static synchronized void dumpToStorage(Collection<KSMetaData> keyspaces)
152 152
153 153 for (Row row : serializedSchema)
154 154 {
155   - if (row.cf == null || row.cf.isEmpty() || row.cf.isMarkedForDelete())
  155 + if (row.cf == null || (row.cf.isMarkedForDelete() && row.cf.isEmpty()))
156 156 continue;
157 157
158 158 keyspaces.add(KSMetaData.fromSchema(row, serializedColumnFamilies(row.key)));
20 src/java/org/apache/cassandra/db/ExpiringColumn.java
@@ -74,14 +74,18 @@ public int getTimeToLive()
74 74 }
75 75
76 76 @Override
77   - public int size(TypeSizes typeSizes)
78   - {
79   - /*
80   - * An expired column adds to a Column :
81   - * 4 bytes for the localExpirationTime
82   - * + 4 bytes for the timeToLive
83   - */
84   - return super.size(typeSizes) + typeSizes.sizeof(localExpirationTime) + typeSizes.sizeof(timeToLive);
  77 + public int dataSize()
  78 + {
  79 + return super.dataSize() + TypeSizes.NATIVE.sizeof(localExpirationTime) + TypeSizes.NATIVE.sizeof(timeToLive);
  80 + }
  81 +
  82 + @Override
  83 + public int serializedSize(TypeSizes sizes)
  84 + {
  85 + // An expired column adds to a Column :
  86 + // 4 bytes for the localExpirationTime
  87 + // + 4 bytes for the timeToLive
  88 + return super.serializedSize(sizes) + sizes.sizeof(localExpirationTime) + sizes.sizeof(timeToLive);
85 89 }
86 90
87 91 @Override
3  src/java/org/apache/cassandra/db/IColumn.java
@@ -42,7 +42,8 @@
42 42 public long mostRecentLiveChangeAt();
43 43 public long mostRecentNonGCableChangeAt(int gcbefore);
44 44 public ByteBuffer name();
45   - public int size(TypeSizes typeSizes);
  45 + /** the size of user-provided data, not including internal overhead */
  46 + public int dataSize();
46 47 public int serializedSize(TypeSizes typeSizes);
47 48 public int serializationFlags();
48 49 public long timestamp();
6 src/java/org/apache/cassandra/db/Memtable.java
@@ -119,9 +119,7 @@ public Memtable(ColumnFamilyStore cfs)
119 119
120 120 public long getLiveSize()
121 121 {
122   - // 25% fudge factor on the base throughput * liveRatio calculation. (Based on observed
123   - // pre-slabbing behavior -- not sure what accounts for this. May have changed with introduction of slabbing.)
124   - return (long) (currentThroughput.get() * cfs.liveRatio * 1.25);
  122 + return (long) (currentThroughput.get() * cfs.liveRatio);
125 123 }
126 124
127 125 public long getSerializedSize()
@@ -225,7 +223,7 @@ public void run()
225 223
226 224 private void resolve(DecoratedKey key, ColumnFamily cf)
227 225 {
228   - currentThroughput.addAndGet(cf.size(TypeSizes.NATIVE));
  226 + currentThroughput.addAndGet(cf.dataSize());
229 227 currentOperations.addAndGet((cf.getColumnCount() == 0)
230 228 ? cf.isMarkedForDelete() ? 1 : 0
231 229 : cf.getColumnCount());
22 src/java/org/apache/cassandra/db/SuperColumn.java
@@ -102,13 +102,11 @@ public IColumn getSubColumn(ByteBuffer columnName)
102 102 /**
103 103 * This calculates the exact size of the sub columns on the fly
104 104 */
105   - public int size(TypeSizes typeSizes)
  105 + public int dataSize()
106 106 {
107   - int size = 0;
  107 + int size = TypeSizes.NATIVE.sizeof(getMarkedForDeleteAt());
108 108 for (IColumn subColumn : getSubColumns())
109   - {
110   - size += subColumn.serializedSize(typeSizes);
111   - }
  109 + size += subColumn.dataSize();
112 110 return size;
113 111 }
114 112
@@ -116,7 +114,7 @@ public int size(TypeSizes typeSizes)
116 114 * This returns the size of the super-column when serialized.
117 115 * @see org.apache.cassandra.db.IColumn#serializedSize()
118 116 */
119   - public int serializedSize(TypeSizes typeSizes)
  117 + public int serializedSize(TypeSizes sizes)
120 118 {
121 119 /*
122 120 * We need to keep the way we are calculating the column size in sync with the
@@ -130,11 +128,13 @@ public int serializedSize(TypeSizes typeSizes)
130 128 * size(constantSize) of subcolumns.
131 129 */
132 130 int nameSize = name.remaining();
133   - int subColumnsSize = size(typeSizes);
134   - return typeSizes.sizeof((short) nameSize) + nameSize
135   - + typeSizes.sizeof(getLocalDeletionTime())
136   - + typeSizes.sizeof(getMarkedForDeleteAt())
137   - + typeSizes.sizeof(subColumnsSize) + subColumnsSize;
  131 + int subColumnsSize = 0;
  132 + for (IColumn subColumn : getSubColumns())
  133 + subColumnsSize += subColumn.serializedSize(sizes);
  134 + return sizes.sizeof((short) nameSize) + nameSize
  135 + + sizes.sizeof(getLocalDeletionTime())
  136 + + sizes.sizeof(getMarkedForDeleteAt())
  137 + + sizes.sizeof(subColumnsSize) + subColumnsSize;
138 138 }
139 139
140 140 public long timestamp()

0 comments on commit 3cfc957

Please sign in to comment.
Something went wrong with that request. Please try again.