From a6319750e65d0b04aaa211e83ac220875120d9a5 Mon Sep 17 00:00:00 2001 From: Michael Russo Date: Mon, 8 Feb 2016 18:04:36 -0800 Subject: [PATCH 01/30] Initial commit introducing Datastax cassandra driver. --- stack/core/pom.xml | 810 +++++++++--------- .../UniqueValueSerializationStrategyImpl.java | 2 +- stack/corepersistence/common/pom.xml | 40 + .../core/astyanax/CassandraConfig.java | 2 +- .../core/astyanax/CassandraConfigImpl.java | 10 +- .../core/astyanax/CassandraFig.java | 56 +- .../core/datastax/DataStaxCluster.java | 33 + .../datastax/DatastaxSessionProvider.java | 23 + .../datastax/impl/DatastaxClusterImpl.java | 169 ++++ .../persistence/core/guice/CommonModule.java | 4 + .../core/datastax/DatastaxClusterTest.java | 24 + stack/corepersistence/pom.xml | 1 + stack/pom.xml | 6 + stack/test-utils/pom.xml | 4 + 14 files changed, 765 insertions(+), 419 deletions(-) create mode 100644 stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/DataStaxCluster.java create mode 100644 stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/DatastaxSessionProvider.java create mode 100644 stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/impl/DatastaxClusterImpl.java create mode 100644 stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/datastax/DatastaxClusterTest.java diff --git a/stack/core/pom.xml b/stack/core/pom.xml index 76ff69429f..7083687dfb 100644 --- a/stack/core/pom.xml +++ b/stack/core/pom.xml @@ -17,30 +17,30 @@ --> - 4.0.0 - - org.apache.usergrid - usergrid - 2.1.1-SNAPSHOT - ../ - - - usergrid-core - Usergrid Core - Core services for Usergrid system. - jar - - - - - org.apache.maven.plugins - maven-javadoc-plugin - - - - - - + 4.0.0 + + org.apache.usergrid + usergrid + 2.1.1-SNAPSHOT + ../ + + + usergrid-core + Usergrid Core + Core services for Usergrid system. + jar + + + + + org.apache.maven.plugins + maven-javadoc-plugin + + + + + + @@ -55,384 +55,388 @@ - - - src/test/resources - true - - **/*.yaml - **/*.properties - **/*.xml - - - - src/test/resources - true - - largeentity.json - - - - - - - - - - org.apache.maven.plugins - maven-jar-plugin - - - test-jar-execution - package - - test-jar - - - - **/org/apache/usergrid/** - - - - - - - - - - - - - - - com.google.inject.extensions - guice-spring - 4.0-beta5 - - - - - - org.apache.cassandra - cassandra-all - - - - snappy-java - org.xerial.snappy - - - antlr - org.antlr - - - - - - org.apache.cassandra - cassandra-thrift - - - - - snappy-java - org.xerial.snappy - 1.0.5 - - - - org.apache.httpcomponents - httpclient - - - - commons-collections - commons-collections - - - - commons-io - commons-io - - - - commons-codec - commons-codec - - - - commons-lang - commons-lang - - - - commons-cli - commons-cli - - - - commons-beanutils - commons-beanutils - - - - - - com.beust - jcommander - - - - com.sun.mail - javax.mail - - - - javax.activation - activation - - - - javax.persistence - persistence-api - - - - com.fasterxml.uuid - java-uuid-generator - - - - com.hazelcast - hazelcast-all - - - - com.netflix.curator - curator-recipes - - - - com.netflix.astyanax - astyanax-recipes - - - - com.google.guava - guava - - - - - - org.hectorclient - hector-core - - - - org.springframework - spring-core - - - - org.springframework - spring-expression - - - - org.springframework - spring-beans - - - - org.springframework - spring-aop - - - - aopalliance - aopalliance - - - - org.springframework - spring-context - - - - org.springframework - spring-context-support - - - - org.springframework - spring-web - - - - org.yaml - snakeyaml - - - - org.jsoup - jsoup - - - - org.perf4j - perf4j - - - - org.aspectj - aspectjweaver - - - - org.aspectj - aspectjrt - - - - cglib - cglib-nodep - - - - jline - jline - - - - org.jboss.netty - netty - - - - - - org.apache.usergrid - usergrid-config - ${project.version} - test - tests - - - - org.hectorclient - hector-test - test - - - - - org.mockito - mockito-core - ${mockito.version} - test - - - - org.jvnet.mock-javamail - mock-javamail - test - - - - org.apache.usergrid - usergrid-test-utils - ${project.version} - test - - - - org.springframework - spring-test - test - - - - - org.apache.usergrid - common - 2.1.1-SNAPSHOT - test-jar - test - - - - - - org.apache.usergrid - collection - 2.1.1-SNAPSHOT - jar - - - antlr - org.antlr - - - antlr - antlr - - - - - - - org.apache.usergrid - queryindex - 2.1.1-SNAPSHOT - jar - - - - org.apache.usergrid - graph - 2.1.1-SNAPSHOT - jar - - - - org.apache.usergrid - map - 2.1.1-SNAPSHOT - jar - - - - org.apache.usergrid - cache - 2.1.1-SNAPSHOT - - - - org.apache.usergrid - queue - 2.1.1-SNAPSHOT - jar - - - - com.codahale.metrics - metrics-core - ${metrics.version} - - - - com.codahale.metrics - metrics-graphite - ${metrics.version} - - - - - + + + src/test/resources + true + + **/*.yaml + **/*.properties + **/*.xml + + + + src/test/resources + true + + largeentity.json + + + + + + + + + + org.apache.maven.plugins + maven-jar-plugin + + + test-jar-execution + package + + test-jar + + + + **/org/apache/usergrid/** + + + + + + + + + + + + + + + com.google.inject.extensions + guice-spring + 4.0-beta5 + + + + + + org.apache.cassandra + cassandra-all + + + + snappy-java + org.xerial.snappy + + + antlr + org.antlr + + + net.jpountz.lz4 + * + + + + + + org.apache.cassandra + cassandra-thrift + + + + + snappy-java + org.xerial.snappy + 1.0.5 + + + + org.apache.httpcomponents + httpclient + + + + commons-collections + commons-collections + + + + commons-io + commons-io + + + + commons-codec + commons-codec + + + + commons-lang + commons-lang + + + + commons-cli + commons-cli + + + + commons-beanutils + commons-beanutils + + + + + + com.beust + jcommander + + + + com.sun.mail + javax.mail + + + + javax.activation + activation + + + + javax.persistence + persistence-api + + + + com.fasterxml.uuid + java-uuid-generator + + + + com.hazelcast + hazelcast-all + + + + com.netflix.curator + curator-recipes + + + + com.netflix.astyanax + astyanax-recipes + + + + com.google.guava + guava + + + + + + org.hectorclient + hector-core + + + + org.springframework + spring-core + + + + org.springframework + spring-expression + + + + org.springframework + spring-beans + + + + org.springframework + spring-aop + + + + aopalliance + aopalliance + + + + org.springframework + spring-context + + + + org.springframework + spring-context-support + + + + org.springframework + spring-web + + + + org.yaml + snakeyaml + + + + org.jsoup + jsoup + + + + org.perf4j + perf4j + + + + org.aspectj + aspectjweaver + + + + org.aspectj + aspectjrt + + + + cglib + cglib-nodep + + + + jline + jline + + + + org.jboss.netty + netty + + + + + + org.apache.usergrid + usergrid-config + ${project.version} + test + tests + + + + org.hectorclient + hector-test + test + + + + + org.mockito + mockito-core + ${mockito.version} + test + + + + org.jvnet.mock-javamail + mock-javamail + test + + + + org.apache.usergrid + usergrid-test-utils + ${project.version} + test + + + + org.springframework + spring-test + test + + + + + org.apache.usergrid + common + 2.1.1-SNAPSHOT + test-jar + test + + + + + + org.apache.usergrid + collection + 2.1.1-SNAPSHOT + jar + + + antlr + org.antlr + + + antlr + antlr + + + + + + + org.apache.usergrid + queryindex + 2.1.1-SNAPSHOT + jar + + + + org.apache.usergrid + graph + 2.1.1-SNAPSHOT + jar + + + + org.apache.usergrid + map + 2.1.1-SNAPSHOT + jar + + + + org.apache.usergrid + cache + 2.1.1-SNAPSHOT + + + + org.apache.usergrid + queue + 2.1.1-SNAPSHOT + jar + + + + com.codahale.metrics + metrics-core + ${metrics.version} + + + + com.codahale.metrics + metrics-graphite + ${metrics.version} + + + + + diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyImpl.java index 622ebef600..1795511efc 100644 --- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyImpl.java +++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyImpl.java @@ -240,7 +240,7 @@ private MutationBatch doWrite( ApplicationScope applicationScope, UniqueValue un @Override public UniqueValueSet load( final ApplicationScope colScope, final String type, final Collection fields ) throws ConnectionException { - return load( colScope, ConsistencyLevel.valueOf( cassandraFig.getReadCL() ), type, fields ); + return load( colScope, ConsistencyLevel.valueOf( cassandraFig.getAstyanaxReadCL() ), type, fields ); } diff --git a/stack/corepersistence/common/pom.xml b/stack/corepersistence/common/pom.xml index b47706c8d0..6b617cf3f2 100644 --- a/stack/corepersistence/common/pom.xml +++ b/stack/corepersistence/common/pom.xml @@ -55,6 +55,12 @@ org.apache.cassandra cassandra-all ${cassandra.version} + + + net.jpountz.lz4 + * + + @@ -176,6 +182,40 @@ metrics-graphite ${metrics.version} + + + + com.datastax.cassandra + cassandra-driver-core + ${datastax.version} + shaded + + + io.netty + * + + + + + com.datastax.cassandra + cassandra-driver-mapping + ${datastax.version} + + + com.datastax.cassandra + cassandra-driver-core + + + + + + + net.jpountz.lz4 + lz4 + 1.2.0 + + diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/CassandraConfig.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/CassandraConfig.java index dba36461fd..c506d2dd37 100644 --- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/CassandraConfig.java +++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/CassandraConfig.java @@ -37,7 +37,7 @@ public interface CassandraConfig { ConsistencyLevel getReadCL(); /** - * Get the currently configured ReadCL that is more consitent than getReadCL + * Get the currently configured ReadCL that is more consitent than getAstyanaxReadCL * @return */ ConsistencyLevel getConsistentReadCL(); diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/CassandraConfigImpl.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/CassandraConfigImpl.java index 7373322d66..a9b37fd50f 100644 --- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/CassandraConfigImpl.java +++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/CassandraConfigImpl.java @@ -45,13 +45,13 @@ public class CassandraConfigImpl implements CassandraConfig { @Inject public CassandraConfigImpl( final CassandraFig cassandraFig ) { - this.readCl = ConsistencyLevel.valueOf( cassandraFig.getReadCL() ); + this.readCl = ConsistencyLevel.valueOf( cassandraFig.getAstyanaxReadCL() ); - this.writeCl = ConsistencyLevel.valueOf( cassandraFig.getWriteCL() ); + this.writeCl = ConsistencyLevel.valueOf( cassandraFig.getAstyanaxWriteCL() ); this.shardSettings = parseShardSettings( cassandraFig.getShardValues() ); - this.consistentCl = ConsistencyLevel.valueOf(cassandraFig.getConsistentReadCL()); + this.consistentCl = ConsistencyLevel.valueOf(cassandraFig.getAstyanaxConsistentReadCL()); //add the listeners to update the values cassandraFig.addPropertyChangeListener( new PropertyChangeListener() { @@ -59,11 +59,11 @@ public CassandraConfigImpl( final CassandraFig cassandraFig ) { public void propertyChange( final PropertyChangeEvent evt ) { final String propName = evt.getPropertyName(); - if ( CassandraFig.READ_CL.equals( propName ) ) { + if ( CassandraFig.ASTYANAX_READ_CL.equals( propName ) ) { readCl = ConsistencyLevel.valueOf( evt.getNewValue().toString() ); } - else if ( CassandraFig.WRITE_CL.equals( propName ) ) { + else if ( CassandraFig.ASTYANAX_WRITE_CL.equals( propName ) ) { writeCl = ConsistencyLevel.valueOf( evt.getNewValue().toString() ); } else if (CassandraFig.SHARD_VALUES.equals(propName)){ diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/CassandraFig.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/CassandraFig.java index e30ba9ce40..78c4fbc494 100644 --- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/CassandraFig.java +++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/CassandraFig.java @@ -25,17 +25,22 @@ import org.safehaus.guicyfig.Key; - /** * Cassandra configuration interface. */ @FigSingleton public interface CassandraFig extends GuicyFig { + // cassndra properties used by datastax driver + String READ_CL = "cassandra.readcl"; + String WRITE_CL = "cassandra.writecl"; + String STRATEGY = "cassandra.strategy"; + String STRATEGY_OPTIONS = "cassandra.strategy.options"; + // main application cassandra properties - String READ_CONSISTENT_CL = "usergrid.consistent.read.cl"; - String READ_CL = "usergrid.read.cl"; - String WRITE_CL = "usergrid.write.cl"; + String ASTYANAX_READ_CONSISTENT_CL = "usergrid.consistent.read.cl"; + String ASTYANAX_READ_CL = "usergrid.read.cl"; + String ASTYANAX_WRITE_CL = "usergrid.write.cl"; String SHARD_VALUES = "cassandra.shardvalues"; String THRIFT_TRANSPORT_SIZE = "cassandra.thrift.transport.frame"; @@ -54,6 +59,9 @@ public interface CassandraFig extends GuicyFig { // re-usable default values String DEFAULT_CONNECTION_POOLSIZE = "15"; String DEFAULT_LOCKS_EXPIRATION = "3600000"; // 1 hour + String DEFAULT_LOCAL_DC = ""; + String DEFAULT_USERNAME = ""; + String DEFAULT_PASSWORD = ""; @Key( "cassandra.hosts" ) @@ -75,7 +83,16 @@ public interface CassandraFig extends GuicyFig { @Default( "9160" ) int getThriftPort(); + @Key( "cassandra.username" ) + @Default( DEFAULT_USERNAME ) + String getUsername(); + + @Key( "cassandra.password" ) + @Default( DEFAULT_PASSWORD ) + String getPassword(); + @Key( "cassandra.datacenter.local" ) + @Default( DEFAULT_LOCAL_DC ) String getLocalDataCenter(); @Key( "cassandra.connections" ) @@ -86,22 +103,43 @@ public interface CassandraFig extends GuicyFig { @Default( "10000" ) int getTimeout(); + @Key( "cassandra.timeout.pool" ) + @Default( "5000" ) + int getPoolTimeout(); + @Key("cassandra.discovery") @Default( "RING_DESCRIBE" ) String getDiscoveryType(); @Default("CL_LOCAL_QUORUM") - @Key(READ_CL) - String getReadCL(); + @Key(ASTYANAX_READ_CL) + String getAstyanaxReadCL(); @Default("CL_QUORUM") - @Key(READ_CONSISTENT_CL) - String getConsistentReadCL(); + @Key(ASTYANAX_READ_CONSISTENT_CL) + String getAstyanaxConsistentReadCL(); @Default("CL_LOCAL_QUORUM") + @Key(ASTYANAX_WRITE_CL) + String getAstyanaxWriteCL(); + + + @Default("LOCAL_QUORUM") + @Key(READ_CL) + String getReadCl(); + + @Default("LOCAL_QUORUM") @Key(WRITE_CL) - String getWriteCL(); + String getWriteCl(); + + @Default("SimpleStrategy") + @Key( STRATEGY ) + String getStrategy(); + + @Default("replication_factor:1") + @Key( STRATEGY_OPTIONS ) + String getStrategyOptions(); /** * Return the history of all shard values which are immutable. For instance, if shard values diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/DataStaxCluster.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/DataStaxCluster.java new file mode 100644 index 0000000000..7515bf5bb0 --- /dev/null +++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/DataStaxCluster.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.usergrid.persistence.core.datastax; + + +import com.datastax.driver.core.Cluster; +import com.datastax.driver.core.Session; + +public interface DataStaxCluster { + + Cluster getCluster(); + + Session getClusterSession(); + + Session getApplicationSession(); + +} diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/DatastaxSessionProvider.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/DatastaxSessionProvider.java new file mode 100644 index 0000000000..eeca763cbe --- /dev/null +++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/DatastaxSessionProvider.java @@ -0,0 +1,23 @@ +package org.apache.usergrid.persistence.core.datastax; + + +import com.datastax.driver.core.Session; +import com.google.inject.Inject; +import com.google.inject.Provider; + +public class DatastaxSessionProvider implements Provider { + + private final DataStaxCluster dataStaxCluster; + + @Inject + public DatastaxSessionProvider( final DataStaxCluster dataStaxCluster){ + + this.dataStaxCluster = dataStaxCluster; + } + + @Override + public Session get(){ + + return dataStaxCluster.getApplicationSession(); + } +} diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/impl/DatastaxClusterImpl.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/impl/DatastaxClusterImpl.java new file mode 100644 index 0000000000..0582c4e012 --- /dev/null +++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/impl/DatastaxClusterImpl.java @@ -0,0 +1,169 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.usergrid.persistence.core.datastax.impl; + +import com.datastax.driver.core.*; +import com.datastax.driver.core.policies.DCAwareRoundRobinPolicy; +import com.datastax.driver.core.policies.LoadBalancingPolicy; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.inject.Inject; +import com.google.inject.Singleton; +import org.apache.usergrid.persistence.core.astyanax.CassandraFig; +import org.apache.usergrid.persistence.core.datastax.DataStaxCluster; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + + +@Singleton +public class DatastaxClusterImpl implements DataStaxCluster { + + private static final Logger logger = LoggerFactory.getLogger( DatastaxClusterImpl.class ); + + + private final CassandraFig cassandraFig; + private final Cluster cluster; + private Session applicationSession; + private Session clusterSession; + + @Inject + public DatastaxClusterImpl( final CassandraFig cassandraFig ) throws Exception { + this.cassandraFig = cassandraFig; + + ConsistencyLevel defaultConsistencyLevel; + try { + defaultConsistencyLevel = ConsistencyLevel.valueOf(cassandraFig.getReadCl()); + } catch (IllegalArgumentException e){ + + logger.error("Unable to parse provided consistency level in property: {}, defaulting to: {}", + CassandraFig.READ_CL, + ConsistencyLevel.LOCAL_QUORUM); + + defaultConsistencyLevel = ConsistencyLevel.LOCAL_QUORUM; + } + + + LoadBalancingPolicy loadBalancingPolicy; + if( !cassandraFig.getLocalDataCenter().isEmpty() ){ + + loadBalancingPolicy = new DCAwareRoundRobinPolicy.Builder() + .withLocalDc( cassandraFig.getLocalDataCenter() ).build(); + }else{ + loadBalancingPolicy = new DCAwareRoundRobinPolicy.Builder().build(); + } + + final PoolingOptions poolingOptions = new PoolingOptions() + .setCoreConnectionsPerHost(HostDistance.LOCAL, cassandraFig.getConnections() / 2) + .setMaxConnectionsPerHost(HostDistance.LOCAL, cassandraFig.getConnections()) + .setIdleTimeoutSeconds(cassandraFig.getTimeout() / 1000) + .setPoolTimeoutMillis(cassandraFig.getPoolTimeout()); + + final QueryOptions queryOptions = new QueryOptions() + .setConsistencyLevel(defaultConsistencyLevel); + + final Cluster.Builder datastaxCluster = Cluster.builder() + .withClusterName(cassandraFig.getClusterName()) + .addContactPoints(cassandraFig.getHosts().split(",")) + .withCompression(ProtocolOptions.Compression.LZ4) + .withLoadBalancingPolicy(loadBalancingPolicy) + .withPoolingOptions(poolingOptions) + .withQueryOptions(queryOptions) + .withProtocolVersion(ProtocolVersion.NEWEST_SUPPORTED); + + // only add auth credentials if they were provided + if ( !cassandraFig.getUsername().isEmpty() && !cassandraFig.getPassword().isEmpty() ){ + datastaxCluster.withCredentials( + cassandraFig.getUsername(), + cassandraFig.getPassword() + ); + } + + this.cluster = datastaxCluster.build(); + this.clusterSession = cluster.connect(); + logger.info("Initialized datastax client cluster. Hosts={}, Idle Timeout={}s, Request Timeout={}s", + cluster.getMetadata().getAllHosts().toString(), + cluster.getConfiguration().getPoolingOptions().getIdleTimeoutSeconds(), + cluster.getConfiguration().getPoolingOptions().getPoolTimeoutMillis() / 1000); + + logger.info("Creating keyspaces if they do not already exist."); + createKeyspaces(); + this.applicationSession = cluster.connect( "\""+cassandraFig.getApplicationKeyspace()+"\"" ); + + + + + } + + public Cluster getCluster(){ + return cluster; + } + + public Session getClusterSession(){ + return clusterSession; + } + + public Session getApplicationSession(){ + return applicationSession; + } + + private void createKeyspaces() throws Exception{ + + final String createApplicationKeyspace = String.format( + "CREATE KEYSPACE IF NOT EXISTS \"%s\" WITH replication = %s", + cassandraFig.getApplicationKeyspace(), + getFormattedReplication( cassandraFig.getStrategy(), cassandraFig.getStrategyOptions() ) + + ); + + final String createLocksKeyspace = String.format( + "CREATE KEYSPACE IF NOT EXISTS \"%s\" WITH replication = %s", + cassandraFig.getLocksKeyspace(), + getFormattedReplication( + cassandraFig.getLocksKeyspaceStrategy(), + cassandraFig.getLocksKeyspaceReplication() + ) + ); + + logger.info("Creating application keyspace with the following CQL: {}", createApplicationKeyspace); + clusterSession.execute(createApplicationKeyspace); + + logger.info("Creating locks keyspace with the following CQL: {}", createLocksKeyspace); + clusterSession.execute(createLocksKeyspace); + + + } + + private String getFormattedReplication(String strategy, String strategyOptions) throws JsonProcessingException{ + + Map replicationSettings = new HashMap<>(); + replicationSettings.put("class", strategy); + String[] strategyOptionsSplit = strategyOptions.split(","); + for ( String option : strategyOptionsSplit){ + String[] splitOptions = option.split(":"); + replicationSettings.put(splitOptions[0], splitOptions[1]); + } + ObjectMapper mapper = new ObjectMapper(); + return mapper.writeValueAsString(replicationSettings).replace("\"", "'"); + } + +} diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/guice/CommonModule.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/guice/CommonModule.java index f7a9926681..a06a6e7876 100644 --- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/guice/CommonModule.java +++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/guice/CommonModule.java @@ -21,6 +21,7 @@ import com.netflix.astyanax.Keyspace; import org.apache.usergrid.persistence.core.astyanax.*; +import org.apache.usergrid.persistence.core.datastax.impl.DatastaxClusterImpl; import org.safehaus.guicyfig.GuicyFigModule; import org.apache.usergrid.persistence.core.consistency.TimeService; @@ -61,6 +62,9 @@ protected void configure() { // bind our Cassandra cluster to the Astyanax Implementation bind(CassandraCluster.class).to(CassandraClusterImpl.class).asEagerSingleton(); + // bind our Datastax cluster + bind(DatastaxClusterImpl.class).asEagerSingleton(); + // bind our keyspace to the AstyanaxKeyspaceProvider bind(Keyspace.class).toProvider(AstyanaxKeyspaceProvider.class).asEagerSingleton(); diff --git a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/datastax/DatastaxClusterTest.java b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/datastax/DatastaxClusterTest.java new file mode 100644 index 0000000000..477a205a63 --- /dev/null +++ b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/datastax/DatastaxClusterTest.java @@ -0,0 +1,24 @@ +package org.apache.usergrid.persistence.core.datastax; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +public class DatastaxClusterTest { + + //TODO +} diff --git a/stack/corepersistence/pom.xml b/stack/corepersistence/pom.xml index 0d8609b572..869cbd4a50 100644 --- a/stack/corepersistence/pom.xml +++ b/stack/corepersistence/pom.xml @@ -80,6 +80,7 @@ limitations under the License. 3.2.1 2.4 3.1 + 2.1.9 1.4.4 3.1.3 18.0 diff --git a/stack/pom.xml b/stack/pom.xml index f052bf9497..2dd1034f6a 100644 --- a/stack/pom.xml +++ b/stack/pom.xml @@ -467,6 +467,12 @@ snappy-java org.xerial.snappy + + + net.jpountz.lz4 + * + + diff --git a/stack/test-utils/pom.xml b/stack/test-utils/pom.xml index c104f0a0e4..e07abf6739 100644 --- a/stack/test-utils/pom.xml +++ b/stack/test-utils/pom.xml @@ -101,6 +101,10 @@ snappy-java org.xerial.snappy + + net.jpountz.lz4 + * + From 01c4970a80fa080f7deccf3f004112d9112f624d Mon Sep 17 00:00:00 2001 From: Michael Russo Date: Tue, 9 Feb 2016 10:18:27 -0800 Subject: [PATCH 02/30] Integrate datastax cluster into migration manager for creation of keyspaces using database/setup. --- .../usergrid/corepersistence/CpSetup.java | 15 +-- .../persistence/core/datastax/CQLUtils.java | 49 ++++++++ .../datastax/DatastaxSessionProvider.java | 22 +++- .../datastax/impl/DatastaxClusterImpl.java | 68 +++-------- .../persistence/core/guice/CommonModule.java | 10 +- .../schema/MigrationManagerImpl.java | 113 +++++++----------- 6 files changed, 142 insertions(+), 135 deletions(-) create mode 100644 stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/CQLUtils.java diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpSetup.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpSetup.java index e97be3feae..d2c38e426f 100644 --- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpSetup.java +++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpSetup.java @@ -83,6 +83,9 @@ public void initSubsystems() throws Exception { //a no op, creating the injector creates the connections //init our index if required this.emf.initializeManagementIndex(); + + logger.info( "Initialize application keyspace" ); + migrate(); setupStaticKeyspace(); setupSystemKeyspace(); @@ -129,9 +132,7 @@ private void migrate() { @Override public void setupSystemKeyspace() throws Exception { - logger.info( "Initialize system keyspace" ); - - migrate(); + logger.info( "Initialize system tables" ); cass.createColumnFamily( getApplicationKeyspace(), createColumnFamilyDefinition( getApplicationKeyspace(), APPLICATIONS_CF, ComparatorType.BYTESTYPE ) ); @@ -145,7 +146,7 @@ public void setupSystemKeyspace() throws Exception { cass.createColumnFamily( getApplicationKeyspace(), createColumnFamilyDefinition( getApplicationKeyspace(), PRINCIPAL_TOKEN_CF, ComparatorType.UUIDTYPE ) ); - logger.info( "System keyspace initialized" ); + logger.info( "System tables initialized" ); } @@ -167,11 +168,9 @@ public void setupApplicationKeyspace( final UUID applicationId, String applicati @Override public void setupStaticKeyspace() throws Exception { - migrate(); - // Need this legacy stuff for queues - logger.info( "Creating static application keyspace {}", getApplicationKeyspace() ); + logger.info( "Initialize application tables" ); cass.createColumnFamily( getApplicationKeyspace(), createColumnFamilyDefinition( getApplicationKeyspace(), APPLICATIONS_CF, @@ -183,6 +182,8 @@ public void setupStaticKeyspace() throws Exception { cass.createColumnFamilies( getApplicationKeyspace(), getCfDefs( QueuesCF.class, getApplicationKeyspace() ) ); + logger.info( "Application tables initialized" ); + } diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/CQLUtils.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/CQLUtils.java new file mode 100644 index 0000000000..b6639348cf --- /dev/null +++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/CQLUtils.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.usergrid.persistence.core.datastax; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; + +import java.util.HashMap; +import java.util.Map; + +public class CQLUtils { + + public static String getFormattedReplication(String strategy, String strategyOptions) throws JsonProcessingException { + + Map replicationSettings = new HashMap<>(); + replicationSettings.put("class", strategy); + String[] strategyOptionsSplit = strategyOptions.split(","); + for ( String option : strategyOptionsSplit){ + String[] splitOptions = option.split(":"); + replicationSettings.put(splitOptions[0], splitOptions[1]); + } + ObjectMapper mapper = new ObjectMapper(); + return mapper.writeValueAsString(replicationSettings).replace("\"", "'"); + } + + + public static void createColumnFamily(){ + + + } + + +} diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/DatastaxSessionProvider.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/DatastaxSessionProvider.java index eeca763cbe..1b39cb8439 100644 --- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/DatastaxSessionProvider.java +++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/DatastaxSessionProvider.java @@ -1,3 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ package org.apache.usergrid.persistence.core.datastax; @@ -5,12 +23,12 @@ import com.google.inject.Inject; import com.google.inject.Provider; -public class DatastaxSessionProvider implements Provider { +public class DataStaxSessionProvider implements Provider { private final DataStaxCluster dataStaxCluster; @Inject - public DatastaxSessionProvider( final DataStaxCluster dataStaxCluster){ + public DataStaxSessionProvider( final DataStaxCluster dataStaxCluster ){ this.dataStaxCluster = dataStaxCluster; } diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/impl/DatastaxClusterImpl.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/impl/DatastaxClusterImpl.java index 0582c4e012..ffe61e6455 100644 --- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/impl/DatastaxClusterImpl.java +++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/impl/DatastaxClusterImpl.java @@ -21,8 +21,6 @@ import com.datastax.driver.core.*; import com.datastax.driver.core.policies.DCAwareRoundRobinPolicy; import com.datastax.driver.core.policies.LoadBalancingPolicy; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; import com.google.inject.Inject; import com.google.inject.Singleton; import org.apache.usergrid.persistence.core.astyanax.CassandraFig; @@ -30,15 +28,11 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - @Singleton -public class DatastaxClusterImpl implements DataStaxCluster { +public class DataStaxClusterImpl implements DataStaxCluster { - private static final Logger logger = LoggerFactory.getLogger( DatastaxClusterImpl.class ); + private static final Logger logger = LoggerFactory.getLogger( DataStaxClusterImpl.class ); private final CassandraFig cassandraFig; @@ -47,7 +41,7 @@ public class DatastaxClusterImpl implements DataStaxCluster { private Session clusterSession; @Inject - public DatastaxClusterImpl( final CassandraFig cassandraFig ) throws Exception { + public DataStaxClusterImpl(final CassandraFig cassandraFig ) throws Exception { this.cassandraFig = cassandraFig; ConsistencyLevel defaultConsistencyLevel; @@ -99,15 +93,12 @@ public DatastaxClusterImpl( final CassandraFig cassandraFig ) throws Exception { } this.cluster = datastaxCluster.build(); - this.clusterSession = cluster.connect(); - logger.info("Initialized datastax client cluster. Hosts={}, Idle Timeout={}s, Request Timeout={}s", + logger.info("Initialized datastax cluster client. Hosts={}, Idle Timeout={}s, Request Timeout={}s", cluster.getMetadata().getAllHosts().toString(), cluster.getConfiguration().getPoolingOptions().getIdleTimeoutSeconds(), cluster.getConfiguration().getPoolingOptions().getPoolTimeoutMillis() / 1000); - logger.info("Creating keyspaces if they do not already exist."); - createKeyspaces(); - this.applicationSession = cluster.connect( "\""+cassandraFig.getApplicationKeyspace()+"\"" ); + @@ -115,55 +106,26 @@ public DatastaxClusterImpl( final CassandraFig cassandraFig ) throws Exception { } public Cluster getCluster(){ + return cluster; } public Session getClusterSession(){ - return clusterSession; - } - - public Session getApplicationSession(){ - return applicationSession; - } - - private void createKeyspaces() throws Exception{ - - final String createApplicationKeyspace = String.format( - "CREATE KEYSPACE IF NOT EXISTS \"%s\" WITH replication = %s", - cassandraFig.getApplicationKeyspace(), - getFormattedReplication( cassandraFig.getStrategy(), cassandraFig.getStrategyOptions() ) - - ); - - final String createLocksKeyspace = String.format( - "CREATE KEYSPACE IF NOT EXISTS \"%s\" WITH replication = %s", - cassandraFig.getLocksKeyspace(), - getFormattedReplication( - cassandraFig.getLocksKeyspaceStrategy(), - cassandraFig.getLocksKeyspaceReplication() - ) - ); - - logger.info("Creating application keyspace with the following CQL: {}", createApplicationKeyspace); - clusterSession.execute(createApplicationKeyspace); - - logger.info("Creating locks keyspace with the following CQL: {}", createLocksKeyspace); - clusterSession.execute(createLocksKeyspace); + if ( clusterSession == null || clusterSession.isClosed() ){ + clusterSession = cluster.connect(); + } + return clusterSession; } - private String getFormattedReplication(String strategy, String strategyOptions) throws JsonProcessingException{ + public Session getApplicationSession(){ - Map replicationSettings = new HashMap<>(); - replicationSettings.put("class", strategy); - String[] strategyOptionsSplit = strategyOptions.split(","); - for ( String option : strategyOptionsSplit){ - String[] splitOptions = option.split(":"); - replicationSettings.put(splitOptions[0], splitOptions[1]); + if ( applicationSession == null || applicationSession.isClosed() ){ + applicationSession = cluster.connect( "\""+cassandraFig.getApplicationKeyspace()+"\"" ); } - ObjectMapper mapper = new ObjectMapper(); - return mapper.writeValueAsString(replicationSettings).replace("\"", "'"); + return applicationSession; } + } diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/guice/CommonModule.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/guice/CommonModule.java index a06a6e7876..460efa5bec 100644 --- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/guice/CommonModule.java +++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/guice/CommonModule.java @@ -19,9 +19,12 @@ package org.apache.usergrid.persistence.core.guice; +import com.datastax.driver.core.Session; import com.netflix.astyanax.Keyspace; import org.apache.usergrid.persistence.core.astyanax.*; -import org.apache.usergrid.persistence.core.datastax.impl.DatastaxClusterImpl; +import org.apache.usergrid.persistence.core.datastax.DataStaxCluster; +import org.apache.usergrid.persistence.core.datastax.DataStaxSessionProvider; +import org.apache.usergrid.persistence.core.datastax.impl.DataStaxClusterImpl; import org.safehaus.guicyfig.GuicyFigModule; import org.apache.usergrid.persistence.core.consistency.TimeService; @@ -63,7 +66,10 @@ protected void configure() { bind(CassandraCluster.class).to(CassandraClusterImpl.class).asEagerSingleton(); // bind our Datastax cluster - bind(DatastaxClusterImpl.class).asEagerSingleton(); + bind(DataStaxCluster.class).to(DataStaxClusterImpl.class).asEagerSingleton(); + + // bind our Session to the DataStaxSessionProvider + bind(Session.class).toProvider(DataStaxSessionProvider.class).asEagerSingleton(); // bind our keyspace to the AstyanaxKeyspaceProvider bind(Keyspace.class).toProvider(AstyanaxKeyspaceProvider.class).asEagerSingleton(); diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/schema/MigrationManagerImpl.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/schema/MigrationManagerImpl.java index db694fe461..e7e0cb542a 100644 --- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/schema/MigrationManagerImpl.java +++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/schema/MigrationManagerImpl.java @@ -20,22 +20,21 @@ import java.util.Collection; -import java.util.List; -import java.util.Map; import java.util.Set; +import com.datastax.driver.core.Session; +import org.apache.usergrid.persistence.core.astyanax.CassandraFig; +import org.apache.usergrid.persistence.core.datastax.CQLUtils; +import org.apache.usergrid.persistence.core.datastax.DataStaxCluster; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.usergrid.persistence.core.astyanax.MultiTenantColumnFamilyDefinition; -import org.apache.usergrid.persistence.core.migration.util.AstayanxUtils; -import com.google.common.collect.ImmutableMap; import com.google.inject.Inject; import com.google.inject.Singleton; import com.netflix.astyanax.Keyspace; import com.netflix.astyanax.connectionpool.exceptions.ConnectionException; -import com.netflix.astyanax.connectionpool.exceptions.NotFoundException; import com.netflix.astyanax.ddl.ColumnFamilyDefinition; import com.netflix.astyanax.ddl.KeyspaceDefinition; @@ -51,18 +50,20 @@ public class MigrationManagerImpl implements MigrationManager { private static final Logger logger = LoggerFactory.getLogger( MigrationManagerImpl.class ); + private final CassandraFig cassandraFig; private final Set migrations; private final Keyspace keyspace; - - private final MigrationManagerFig fig; + private final DataStaxCluster dataStaxCluster; @Inject - public MigrationManagerImpl( final Keyspace keyspace, final Set migrations, - MigrationManagerFig fig ) { + public MigrationManagerImpl( final CassandraFig cassandraFig, final Keyspace keyspace, + final Set migrations, final DataStaxCluster dataStaxCluster) { + + this.cassandraFig = cassandraFig; this.keyspace = keyspace; this.migrations = migrations; - this.fig = fig; + this.dataStaxCluster = dataStaxCluster; } @@ -72,7 +73,7 @@ public void migrate() throws MigrationException { try { - testAndCreateKeyspace(); + createOrUpdateKeyspace(); for ( Migration migration : migrations ) { @@ -116,86 +117,54 @@ private void testAndCreateColumnFamilyDef( MultiTenantColumnFamilyDefinition col logger.info( "Created column family {}", columnFamily.getColumnFamily().getName() ); - waitForMigration(); + waitForSchemaAgreement(); } /** - * Check if they keyspace exists. If it doesn't create it + * Execute CQL to create the keyspace if it does not already exists. Always update the keyspace with the + * configured strategy options to allow for real time replication updates. + * + * @throws Exception */ - private void testAndCreateKeyspace() throws ConnectionException { - - - KeyspaceDefinition keyspaceDefinition = null; - - try { - keyspaceDefinition = keyspace.describeKeyspace(); - - }catch( NotFoundException nfe){ - //if we execute this immediately after a drop keyspace in 1.2.x, Cassandra is returning the NFE instead of a BadRequestException - //swallow and log, then continue to create the keyspaces. - logger.info( "Received a NotFoundException when attempting to describe keyspace. It does not exist" ); - } - catch(Exception e){ - AstayanxUtils.isKeyspaceMissing("Unable to connect to cassandra", e); - } - - - if ( keyspaceDefinition != null ) { - return; - } - + private void createOrUpdateKeyspace() throws Exception { - ImmutableMap.Builder strategyOptions = getKeySpaceProps(); + Session clusterSession = dataStaxCluster.getClusterSession(); + final String createApplicationKeyspace = String.format( + "CREATE KEYSPACE IF NOT EXISTS \"%s\" WITH replication = %s", + cassandraFig.getApplicationKeyspace(), + CQLUtils.getFormattedReplication( cassandraFig.getStrategy(), cassandraFig.getStrategyOptions() ) - ImmutableMap options = - ImmutableMap.builder().put( "strategy_class", fig.getStrategyClass() ) - .put( "strategy_options", strategyOptions.build() ).build(); + ); + final String updateApplicationKeyspace = String.format( + "ALTER KEYSPACE \"%s\" WITH replication = %s", + cassandraFig.getApplicationKeyspace(), + CQLUtils.getFormattedReplication( cassandraFig.getStrategy(), cassandraFig.getStrategyOptions() ) + ); - keyspace.createKeyspace( options ); + logger.info("Creating application keyspace with the following CQL: {}", createApplicationKeyspace); + clusterSession.execute(createApplicationKeyspace); + logger.info("Updating application keyspace with the following CQL: {}", updateApplicationKeyspace); + clusterSession.execute(updateApplicationKeyspace); - strategyOptions.toString(); + // this session pool is only used when running database setup so close it when finished to clear resources + clusterSession.close(); - logger.info( "Created keyspace {} with options {}", keyspace.getKeyspaceName(), options.toString() ); - - waitForMigration(); + waitForSchemaAgreement(); } /** - * Get keyspace properties + * Wait until all Cassandra nodes agree on the schema. Sleeps 100ms between checks. + * */ - private ImmutableMap.Builder getKeySpaceProps() { - ImmutableMap.Builder keyspaceProps = ImmutableMap.builder(); - - String optionString = fig.getStrategyOptions(); - - if(optionString == null){ - return keyspaceProps; - } - - - - for ( String key : optionString.split( "," ) ) { - - final String[] options = key.split( ":" ); - - keyspaceProps.put( options[0], options[1] ); - } - - return keyspaceProps; - } - - - private void waitForMigration() throws ConnectionException { + private void waitForSchemaAgreement() { while ( true ) { - final Map> versions = keyspace.describeSchemaVersions(); - - if ( versions != null && versions.size() == 1 ) { + if( dataStaxCluster.getCluster().getMetadata().checkSchemaAgreement() ){ return; } @@ -208,4 +177,6 @@ private void waitForMigration() throws ConnectionException { } } } + + } From a62fb0a0c95ece13c84a563e58802230bdcff1cb Mon Sep 17 00:00:00 2001 From: Michael Russo Date: Tue, 9 Feb 2016 17:14:38 -0800 Subject: [PATCH 03/30] Create object for defining our table schema and provide utilities for generating CQL to create or ALTER the table schema. --- stack/corepersistence/common/pom.xml | 11 -- .../persistence/core/datastax/CQLUtils.java | 77 ++++++++++- .../core/datastax/TableDefinition.java | 129 ++++++++++++++++++ .../core/migration/schema/Migration.java | 2 + .../core/datastax/CQLUtilsTest.java | 73 ++++++++++ .../core/datastax/DatastaxClusterTest.java | 4 +- 6 files changed, 281 insertions(+), 15 deletions(-) create mode 100644 stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/TableDefinition.java create mode 100644 stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/datastax/CQLUtilsTest.java diff --git a/stack/corepersistence/common/pom.xml b/stack/corepersistence/common/pom.xml index 6b617cf3f2..983c6652f9 100644 --- a/stack/corepersistence/common/pom.xml +++ b/stack/corepersistence/common/pom.xml @@ -196,17 +196,6 @@ - - com.datastax.cassandra - cassandra-driver-mapping - ${datastax.version} - - - com.datastax.cassandra - cassandra-driver-core - - - diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/CQLUtils.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/CQLUtils.java index b6639348cf..0a7408a0d2 100644 --- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/CQLUtils.java +++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/CQLUtils.java @@ -20,12 +20,35 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.usergrid.persistence.core.util.StringUtils; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.Map; +import java.util.StringJoiner; public class CQLUtils { + + enum ACTION { + CREATE, UPDATE + } + + static String CREATE_TABLE = "CREATE TABLE IF NOT EXISTS"; + static String ALTER_TABLE = "ALTER TABLE"; + static String WITH ="WITH"; + static String AND = "AND"; + static String EQUAL = "="; + static String COMPRESSION = "compression"; + static String COMPACTION = "compaction"; + static String GC_GRACE_SECONDS = "gc_grace_seconds"; + static String PRIMARY_KEY = "PRIMARY KEY"; + static String COMPACT_STORAGE = "COMPACT STORAGE"; + static String CLUSTERING_ORDER_BY = "CLUSTERING ORDER BY"; + + private final static ObjectMapper mapper = new ObjectMapper(); + + public static String getFormattedReplication(String strategy, String strategyOptions) throws JsonProcessingException { Map replicationSettings = new HashMap<>(); @@ -35,13 +58,63 @@ public static String getFormattedReplication(String strategy, String strategyOpt String[] splitOptions = option.split(":"); replicationSettings.put(splitOptions[0], splitOptions[1]); } - ObjectMapper mapper = new ObjectMapper(); return mapper.writeValueAsString(replicationSettings).replace("\"", "'"); } - public static void createColumnFamily(){ + public static String getMapAsCQLString(Map map) throws JsonProcessingException { + + return mapper.writeValueAsString(map).replace("\"", "'"); + } + + + public static String getTableCQL(TableDefinition tableDefinition, ACTION tableAction) throws Exception { + + StringJoiner cql = new StringJoiner(" "); + + if ( tableAction.equals(ACTION.CREATE) ){ + cql.add(CREATE_TABLE); + } else if ( tableAction.equals(ACTION.UPDATE) ){ + cql.add(ALTER_TABLE); + }else{ + throw new Exception("Invalid Action specified. Must of of type CQLUtils.Action"); + } + + cql.add( "\""+tableDefinition.getTableName()+"\"" ); + + + StringJoiner columnsString = new StringJoiner(","); + Map columns = tableDefinition.getColumns(); + columns.forEach( (key, value) -> columnsString.add(key+" "+value)); + columnsString.add(PRIMARY_KEY +" ( "+StringUtils.join(tableDefinition.getPrimaryKeys(), ",") + " )"); + + StringJoiner orderingString = new StringJoiner(" "); + Map ordering = tableDefinition.getClusteringOrder(); + ordering.forEach( (key, value) -> orderingString.add(key+" "+value)); + + if ( tableAction.equals(ACTION.CREATE) ){ + cql.add("(").add(columnsString.toString()).add(")") + .add(WITH) + .add(CLUSTERING_ORDER_BY).add("(").add(orderingString.toString()).add(")") + .add(AND) + .add(COMPACT_STORAGE) + .add(AND); + + } else if ( tableAction.equals(ACTION.UPDATE) ){ + cql.add(WITH); + + } + + + cql.add(COMPACTION).add(EQUAL).add( getMapAsCQLString( tableDefinition.getCompaction() ) ) + .add(AND) + .add(COMPRESSION).add(EQUAL).add( getMapAsCQLString( tableDefinition.getCompression() ) ) + .add(AND) + .add(GC_GRACE_SECONDS).add(EQUAL).add( tableDefinition.getGcGraceSeconds() ); + + + return cql.toString(); } diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/TableDefinition.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/TableDefinition.java new file mode 100644 index 0000000000..58d43a55f7 --- /dev/null +++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/TableDefinition.java @@ -0,0 +1,129 @@ +/* + * + * * Licensed to the Apache Software Foundation (ASF) under one + * * or more contributor license agreements. See the NOTICE file + * * distributed with this work for additional information + * * regarding copyright ownership. The ASF licenses this file + * * to you under the Apache License, Version 2.0 (the + * * "License"); you may not use this file except in compliance + * * with the License. You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, + * * software distributed under the License is distributed on an + * * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * * KIND, either express or implied. See the License for the + * * specific language governing permissions and limitations + * * under the License. + * + */ + +package org.apache.usergrid.persistence.core.datastax; + + +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; + +public class TableDefinition { + + + public enum CacheOption { + + ALL( "ALL" ), + KEYS( "KEYS_ONLY" ), + ROWS( "ROWS_ONLY" ), + NONE( "NONE" ); + + private String value; + + + CacheOption( String value ) { + this.value = value; + } + + + public String getValue() { + return value; + } + } + + + private final String tableName; + private final Collection primaryKeys; + private final Map columns; + private final CacheOption cacheOption; + private final Map compaction; + private final String bloomFilterChance; + private final String readRepairChance; + private final Map compression; + private final String gcGraceSeconds; + private final Map clusteringOrder; + + public TableDefinition( final String tableName, final Collection primaryKeys, + final Map columns, final CacheOption cacheOption, + final Map clusteringOrder){ + + this.tableName = tableName; + this.primaryKeys = primaryKeys; + this.columns = columns; + this.cacheOption = cacheOption; + this.clusteringOrder = clusteringOrder; + + + // this are default settings always used + this.compaction = new HashMap<>(1); + compaction.put( "class", "LeveledCompactionStrategy" ); + this.bloomFilterChance = "0.1d"; + this.readRepairChance = "0.1d"; + this.compression = new HashMap<>(1); + compression.put("sstable_compression", "LZ4Compressor"); + this.gcGraceSeconds = "864000"; + + + + } + + public String getTableName() { + return tableName; + } + + public Collection getPrimaryKeys() { + return primaryKeys; + } + + public Map getColumns() { + return columns; + } + + public CacheOption getCacheOption() { + return cacheOption; + } + + public Map getCompaction() { + return compaction; + } + + public String getBloomFilterChance() { + return bloomFilterChance; + } + + public String getReadRepairChance() { + return readRepairChance; + } + + public Map getCompression() { + return compression; + } + + public String getGcGraceSeconds() { + return gcGraceSeconds; + } + + public Map getClusteringOrder() { + return clusteringOrder; + } + + +} diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/schema/Migration.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/schema/Migration.java index 9938b88c42..3896df249e 100644 --- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/schema/Migration.java +++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/schema/Migration.java @@ -33,4 +33,6 @@ public interface Migration { * Get the column families required for this implementation. If one does not exist it will be created. */ Collection getColumnFamilies(); + + //Collection getTables(); } diff --git a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/datastax/CQLUtilsTest.java b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/datastax/CQLUtilsTest.java new file mode 100644 index 0000000000..8ddfa3f91a --- /dev/null +++ b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/datastax/CQLUtilsTest.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.usergrid.persistence.core.datastax; + + +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.assertTrue; + +public class CQLUtilsTest { + + private static final Logger logger = LoggerFactory.getLogger( CQLUtilsTest.class ); + + @Test + public void testTableCQL() throws Exception { + + + Map columns = new HashMap<>(); + columns.put("key", "blob"); + columns.put("column1", "text"); + columns.put("value", "blob"); + + List primaryKeys = new ArrayList<>(); + primaryKeys.add("key"); + primaryKeys.add("column1"); + + Map clusteringOrder = new HashMap<>(); + clusteringOrder.put("column1", "DESC"); + + + + TableDefinition table1 = new TableDefinition( + "table1", + primaryKeys, + columns, + TableDefinition.CacheOption.KEYS, + clusteringOrder + ); + + String createCQL = CQLUtils.getTableCQL(table1, CQLUtils.ACTION.CREATE); + String updateCQL = CQLUtils.getTableCQL(table1, CQLUtils.ACTION.UPDATE); + + assertTrue( createCQL.contains( CQLUtils.CREATE_TABLE ) && !createCQL.contains( CQLUtils.ALTER_TABLE ) ); + assertTrue( updateCQL.contains( CQLUtils.ALTER_TABLE ) && !updateCQL.contains( CQLUtils.CREATE_TABLE ) ); + //logger.info("CREATE: {}", createCQL); + //logger.info("UPDATE: {}", updateCQL); + + } + +} diff --git a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/datastax/DatastaxClusterTest.java b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/datastax/DatastaxClusterTest.java index 477a205a63..5da23ce92d 100644 --- a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/datastax/DatastaxClusterTest.java +++ b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/datastax/DatastaxClusterTest.java @@ -1,5 +1,3 @@ -package org.apache.usergrid.persistence.core.datastax; - /* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file @@ -18,6 +16,8 @@ * specific language governing permissions and limitations * under the License. */ +package org.apache.usergrid.persistence.core.datastax; + public class DatastaxClusterTest { //TODO From ad4a33713e8280a4d3637bf1395937bd32aa5fca Mon Sep 17 00:00:00 2001 From: Michael Russo Date: Tue, 9 Feb 2016 17:36:49 -0800 Subject: [PATCH 04/30] Add null checks and tests confirming NPEs are thrown. --- .../core/datastax/TableDefinition.java | 8 ++ .../core/datastax/TableDefinitionTest.java | 81 +++++++++++++++++++ 2 files changed, 89 insertions(+) create mode 100644 stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/datastax/TableDefinitionTest.java diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/TableDefinition.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/TableDefinition.java index 58d43a55f7..801eaa710b 100644 --- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/TableDefinition.java +++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/TableDefinition.java @@ -22,6 +22,8 @@ package org.apache.usergrid.persistence.core.datastax; +import com.google.common.base.Preconditions; + import java.util.Collection; import java.util.HashMap; import java.util.Map; @@ -65,6 +67,12 @@ public TableDefinition( final String tableName, final Collection primary final Map columns, final CacheOption cacheOption, final Map clusteringOrder){ + Preconditions.checkNotNull(tableName, "Table name cannot be null"); + Preconditions.checkNotNull(primaryKeys, "Primary Key(s) cannot be null"); + Preconditions.checkNotNull(columns, "Columns cannot be null"); + Preconditions.checkNotNull(cacheOption, "CacheOption cannot be null"); + + this.tableName = tableName; this.primaryKeys = primaryKeys; this.columns = columns; diff --git a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/datastax/TableDefinitionTest.java b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/datastax/TableDefinitionTest.java new file mode 100644 index 0000000000..792864bdd0 --- /dev/null +++ b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/datastax/TableDefinitionTest.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.usergrid.persistence.core.datastax; + + +import org.junit.Test; + +import java.util.ArrayList; +import java.util.HashMap; + +import static org.junit.Assert.assertEquals; + +public class TableDefinitionTest { + + @Test + public void testNullTableName(){ + + try{ + TableDefinition table1 = new TableDefinition(null, null, null, null, null); + } catch (NullPointerException npe){ + assertEquals("Table name cannot be null", npe.getMessage()); + } + + + } + + @Test + public void testNullPrimaryKeys(){ + + try{ + TableDefinition table1 = new TableDefinition("table1", null, null, null, null); + } catch (NullPointerException npe){ + assertEquals("Primary Key(s) cannot be null", npe.getMessage()); + } + + + } + + @Test + public void testNullColumns(){ + + try{ + TableDefinition table1 = new TableDefinition("table1", + new ArrayList<>(), null, null, null); + } catch (NullPointerException npe){ + assertEquals("Columns cannot be null", npe.getMessage()); + } + + + } + + @Test + public void testNullCacheOption(){ + + try{ + TableDefinition table1 = new TableDefinition("table1", + new ArrayList<>(), + new HashMap<>(), null, null); + } catch (NullPointerException npe){ + assertEquals("CacheOption cannot be null", npe.getMessage()); + } + + + } +} From 2203f43336ae86ebcad823fb740207b4eecc36bb Mon Sep 17 00:00:00 2001 From: Michael Russo Date: Thu, 11 Feb 2016 14:49:44 -0800 Subject: [PATCH 05/30] Convert map manager serialization to read and write using CQL using the original Astyanax data modeling in Cassandra (aka no migration required). --- .../core/astyanax/CassandraConfig.java | 19 + .../core/astyanax/CassandraConfigImpl.java | 27 ++ .../persistence/core/datastax/CQLUtils.java | 77 +++- .../core/datastax/TableDefinition.java | 22 +- .../core/astyanax/ColumnNameIteratorTest.java | 16 + .../MultiKeyColumnNameIteratorTest.java | 16 + .../astyanax/MultiRowColumnIteratorTest.java | 16 + .../core/datastax/CQLUtilsTest.java | 15 +- .../core/datastax/TableDefinitionTest.java | 8 +- .../persistence/map/impl/MapManagerImpl.java | 1 - .../map/impl/MapSerializationImpl.java | 391 ++++++++++-------- .../persistence/map/MapManagerTest.java | 16 + 12 files changed, 407 insertions(+), 217 deletions(-) diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/CassandraConfig.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/CassandraConfig.java index c506d2dd37..8cb96ac232 100644 --- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/CassandraConfig.java +++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/CassandraConfig.java @@ -48,6 +48,25 @@ public interface CassandraConfig { */ ConsistencyLevel getWriteCL(); + + /** + * Get the currently configured read CL for DataStax driver + * @return + */ + com.datastax.driver.core.ConsistencyLevel getDataStaxReadCl(); + + /** + * Get the currently configured write CL for DataStax driver + * @return + */ + com.datastax.driver.core.ConsistencyLevel getDataStaxWriteCl(); + + /** + * Get the currently configured consistent read CL for DataStax driver + * @return + */ + com.datastax.driver.core.ConsistencyLevel getDataStaxReadConsistentCl(); + /** * Return the number of shards that has been set in the property file * @return diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/CassandraConfigImpl.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/CassandraConfigImpl.java index a9b37fd50f..15f434cde9 100644 --- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/CassandraConfigImpl.java +++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/CassandraConfigImpl.java @@ -41,6 +41,12 @@ public class CassandraConfigImpl implements CassandraConfig { private int[] shardSettings; private ConsistencyLevel consistentCl; + // DataStax driver's CL + private com.datastax.driver.core.ConsistencyLevel dataStaxReadCl; + private com.datastax.driver.core.ConsistencyLevel dataStaxWriteCl; + private com.datastax.driver.core.ConsistencyLevel dataStaxReadConsistentCl; + + @Inject public CassandraConfigImpl( final CassandraFig cassandraFig ) { @@ -53,6 +59,12 @@ public CassandraConfigImpl( final CassandraFig cassandraFig ) { this.consistentCl = ConsistencyLevel.valueOf(cassandraFig.getAstyanaxConsistentReadCL()); + this.dataStaxReadCl = com.datastax.driver.core.ConsistencyLevel.valueOf( cassandraFig.getReadCl()); + + this.dataStaxReadConsistentCl = com.datastax.driver.core.ConsistencyLevel.valueOf( cassandraFig.getReadCl()); + + this.dataStaxWriteCl = com.datastax.driver.core.ConsistencyLevel.valueOf( cassandraFig.getWriteCl() ); + //add the listeners to update the values cassandraFig.addPropertyChangeListener( new PropertyChangeListener() { @Override @@ -89,6 +101,21 @@ public ConsistencyLevel getWriteCL() { return writeCl; } + @Override + public com.datastax.driver.core.ConsistencyLevel getDataStaxReadCl() { + return dataStaxReadCl; + } + + @Override + public com.datastax.driver.core.ConsistencyLevel getDataStaxWriteCl() { + return dataStaxWriteCl; + } + + @Override + public com.datastax.driver.core.ConsistencyLevel getDataStaxReadConsistentCl() { + return dataStaxReadConsistentCl; + } + @Override public int[] getShardSettings() { diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/CQLUtils.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/CQLUtils.java index 0a7408a0d2..7dee9c8e52 100644 --- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/CQLUtils.java +++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/CQLUtils.java @@ -18,14 +18,14 @@ */ package org.apache.usergrid.persistence.core.datastax; +import com.datastax.driver.core.DataType; +import com.datastax.driver.core.ProtocolVersion; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import org.apache.usergrid.persistence.core.util.StringUtils; -import java.util.HashMap; -import java.util.LinkedHashMap; -import java.util.Map; -import java.util.StringJoiner; +import java.nio.ByteBuffer; +import java.util.*; public class CQLUtils { @@ -45,6 +45,10 @@ enum ACTION { static String PRIMARY_KEY = "PRIMARY KEY"; static String COMPACT_STORAGE = "COMPACT STORAGE"; static String CLUSTERING_ORDER_BY = "CLUSTERING ORDER BY"; + static String COMMA = ","; + static String PAREN_LEFT = "("; + static String PAREN_RIGHT = ")"; + private final static ObjectMapper mapper = new ObjectMapper(); @@ -80,22 +84,28 @@ public static String getTableCQL(TableDefinition tableDefinition, ACTION tableAc throw new Exception("Invalid Action specified. Must of of type CQLUtils.Action"); } - cql.add( "\""+tableDefinition.getTableName()+"\"" ); - + cql.add( quote( tableDefinition.getTableName() ) ); - StringJoiner columnsString = new StringJoiner(","); - Map columns = tableDefinition.getColumns(); - columns.forEach( (key, value) -> columnsString.add(key+" "+value)); - columnsString.add(PRIMARY_KEY +" ( "+StringUtils.join(tableDefinition.getPrimaryKeys(), ",") + " )"); - StringJoiner orderingString = new StringJoiner(" "); - Map ordering = tableDefinition.getClusteringOrder(); - ordering.forEach( (key, value) -> orderingString.add(key+" "+value)); if ( tableAction.equals(ACTION.CREATE) ){ - cql.add("(").add(columnsString.toString()).add(")") + + cql.add(PAREN_LEFT).add( spaceSeparatedKeyValue(tableDefinition.getColumns()) ).add(COMMA) + .add(PRIMARY_KEY) + .add(PAREN_LEFT).add(PAREN_LEFT) + .add( StringUtils.join(tableDefinition.getPartitionKeys(), COMMA) ).add(PAREN_RIGHT); + + if ( tableDefinition.getColumnKeys() != null && !tableDefinition.getColumnKeys().isEmpty() ){ + + cql.add(COMMA).add( StringUtils.join(tableDefinition.getColumnKeys(), COMMA) ); + } + + cql.add(PAREN_RIGHT).add(PAREN_RIGHT) .add(WITH) - .add(CLUSTERING_ORDER_BY).add("(").add(orderingString.toString()).add(")") + .add(CLUSTERING_ORDER_BY) + .add(PAREN_LEFT) + .add( spaceSeparatedKeyValue(tableDefinition.getClusteringOrder()) ) + .add(PAREN_RIGHT) .add(AND) .add(COMPACT_STORAGE) .add(AND); @@ -118,5 +128,42 @@ public static String getTableCQL(TableDefinition tableDefinition, ACTION tableAc } + public static String quote( String value){ + + return "\"" + value + "\""; + + } + + public static String spaceSeparatedKeyValue(Map columns){ + + StringJoiner columnsSchema = new StringJoiner(","); + columns.forEach( (key, value) -> columnsSchema.add(key+" "+value)); + + return columnsSchema.toString(); + + } + + + /** + * Below functions borrowed from Astyanax until the schema is re-written to be more CQL friendly + */ + + public static int getShortLength(ByteBuffer bb) { + int length = (bb.get() & 255) << 8; + return length | bb.get() & 255; + } + + public static ByteBuffer getBytes(ByteBuffer bb, int length) { + ByteBuffer copy = bb.duplicate(); + copy.limit(copy.position() + length); + bb.position(bb.position() + length); + return copy; + } + + public static ByteBuffer getWithShortLength(ByteBuffer bb) { + int length = getShortLength(bb); + return getBytes(bb, length); + } + } diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/TableDefinition.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/TableDefinition.java index 801eaa710b..0635b93dba 100644 --- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/TableDefinition.java +++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/TableDefinition.java @@ -53,7 +53,8 @@ public String getValue() { private final String tableName; - private final Collection primaryKeys; + private final Collection partitionKeys; + private final Collection columnKeys; private final Map columns; private final CacheOption cacheOption; private final Map compaction; @@ -63,18 +64,19 @@ public String getValue() { private final String gcGraceSeconds; private final Map clusteringOrder; - public TableDefinition( final String tableName, final Collection primaryKeys, - final Map columns, final CacheOption cacheOption, - final Map clusteringOrder){ + public TableDefinition( final String tableName, final Collection partitionKeys, + final Collection columnKeys, final Map columns, + final CacheOption cacheOption, final Map clusteringOrder){ Preconditions.checkNotNull(tableName, "Table name cannot be null"); - Preconditions.checkNotNull(primaryKeys, "Primary Key(s) cannot be null"); + Preconditions.checkNotNull(partitionKeys, "Primary Key(s) cannot be null"); Preconditions.checkNotNull(columns, "Columns cannot be null"); Preconditions.checkNotNull(cacheOption, "CacheOption cannot be null"); this.tableName = tableName; - this.primaryKeys = primaryKeys; + this.partitionKeys = partitionKeys; + this.columnKeys = columnKeys; this.columns = columns; this.cacheOption = cacheOption; this.clusteringOrder = clusteringOrder; @@ -97,8 +99,12 @@ public String getTableName() { return tableName; } - public Collection getPrimaryKeys() { - return primaryKeys; + public Collection getPartitionKeys() { + return partitionKeys; + } + + public Collection getColumnKeys() { + return columnKeys; } public Map getColumns() { diff --git a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/astyanax/ColumnNameIteratorTest.java b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/astyanax/ColumnNameIteratorTest.java index 18c93270d8..dccbd4552b 100644 --- a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/astyanax/ColumnNameIteratorTest.java +++ b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/astyanax/ColumnNameIteratorTest.java @@ -93,6 +93,22 @@ public ConsistencyLevel getWriteCL() { return ConsistencyLevel.CL_QUORUM; } + @Override + public com.datastax.driver.core.ConsistencyLevel getDataStaxReadCl() { + return com.datastax.driver.core.ConsistencyLevel.LOCAL_ONE; + } + + @Override + public com.datastax.driver.core.ConsistencyLevel getDataStaxReadConsistentCl() { + return com.datastax.driver.core.ConsistencyLevel.ALL; + } + + + @Override + public com.datastax.driver.core.ConsistencyLevel getDataStaxWriteCl() { + return com.datastax.driver.core.ConsistencyLevel.QUORUM; + } + @Override public int[] getShardSettings() { diff --git a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/astyanax/MultiKeyColumnNameIteratorTest.java b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/astyanax/MultiKeyColumnNameIteratorTest.java index bd1ea557d3..d020949c04 100644 --- a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/astyanax/MultiKeyColumnNameIteratorTest.java +++ b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/astyanax/MultiKeyColumnNameIteratorTest.java @@ -98,6 +98,22 @@ public ConsistencyLevel getWriteCL() { return ConsistencyLevel.CL_QUORUM; } + @Override + public com.datastax.driver.core.ConsistencyLevel getDataStaxReadCl() { + return com.datastax.driver.core.ConsistencyLevel.LOCAL_ONE; + } + + @Override + public com.datastax.driver.core.ConsistencyLevel getDataStaxReadConsistentCl() { + return com.datastax.driver.core.ConsistencyLevel.ALL; + } + + + @Override + public com.datastax.driver.core.ConsistencyLevel getDataStaxWriteCl() { + return com.datastax.driver.core.ConsistencyLevel.QUORUM; + } + @Override public int[] getShardSettings() { diff --git a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/astyanax/MultiRowColumnIteratorTest.java b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/astyanax/MultiRowColumnIteratorTest.java index 9f5741b3d6..8bcdcb207f 100644 --- a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/astyanax/MultiRowColumnIteratorTest.java +++ b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/astyanax/MultiRowColumnIteratorTest.java @@ -101,6 +101,22 @@ public ConsistencyLevel getWriteCL() { return ConsistencyLevel.CL_QUORUM; } + @Override + public com.datastax.driver.core.ConsistencyLevel getDataStaxReadCl() { + return com.datastax.driver.core.ConsistencyLevel.LOCAL_ONE; + } + + @Override + public com.datastax.driver.core.ConsistencyLevel getDataStaxReadConsistentCl() { + return com.datastax.driver.core.ConsistencyLevel.ALL; + } + + + @Override + public com.datastax.driver.core.ConsistencyLevel getDataStaxWriteCl() { + return com.datastax.driver.core.ConsistencyLevel.QUORUM; + } + @Override public int[] getShardSettings() { diff --git a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/datastax/CQLUtilsTest.java b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/datastax/CQLUtilsTest.java index 8ddfa3f91a..76fcefea0d 100644 --- a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/datastax/CQLUtilsTest.java +++ b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/datastax/CQLUtilsTest.java @@ -43,9 +43,11 @@ public void testTableCQL() throws Exception { columns.put("column1", "text"); columns.put("value", "blob"); - List primaryKeys = new ArrayList<>(); - primaryKeys.add("key"); - primaryKeys.add("column1"); + List partitionKeys = new ArrayList<>(); + partitionKeys.add("key"); + + List columnKeys = new ArrayList<>(); + columnKeys.add("column1"); Map clusteringOrder = new HashMap<>(); clusteringOrder.put("column1", "DESC"); @@ -54,7 +56,8 @@ public void testTableCQL() throws Exception { TableDefinition table1 = new TableDefinition( "table1", - primaryKeys, + partitionKeys, + columnKeys, columns, TableDefinition.CacheOption.KEYS, clusteringOrder @@ -65,8 +68,8 @@ public void testTableCQL() throws Exception { assertTrue( createCQL.contains( CQLUtils.CREATE_TABLE ) && !createCQL.contains( CQLUtils.ALTER_TABLE ) ); assertTrue( updateCQL.contains( CQLUtils.ALTER_TABLE ) && !updateCQL.contains( CQLUtils.CREATE_TABLE ) ); - //logger.info("CREATE: {}", createCQL); - //logger.info("UPDATE: {}", updateCQL); + logger.info("CREATE: {}", createCQL); + logger.info("UPDATE: {}", updateCQL); } diff --git a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/datastax/TableDefinitionTest.java b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/datastax/TableDefinitionTest.java index 792864bdd0..3acce696e1 100644 --- a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/datastax/TableDefinitionTest.java +++ b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/datastax/TableDefinitionTest.java @@ -32,7 +32,7 @@ public class TableDefinitionTest { public void testNullTableName(){ try{ - TableDefinition table1 = new TableDefinition(null, null, null, null, null); + TableDefinition table1 = new TableDefinition(null, null, null, null, null, null); } catch (NullPointerException npe){ assertEquals("Table name cannot be null", npe.getMessage()); } @@ -44,7 +44,7 @@ public void testNullTableName(){ public void testNullPrimaryKeys(){ try{ - TableDefinition table1 = new TableDefinition("table1", null, null, null, null); + TableDefinition table1 = new TableDefinition("table1", null, null, null, null, null); } catch (NullPointerException npe){ assertEquals("Primary Key(s) cannot be null", npe.getMessage()); } @@ -57,7 +57,7 @@ public void testNullColumns(){ try{ TableDefinition table1 = new TableDefinition("table1", - new ArrayList<>(), null, null, null); + new ArrayList<>(), null, null, null, null); } catch (NullPointerException npe){ assertEquals("Columns cannot be null", npe.getMessage()); } @@ -71,7 +71,7 @@ public void testNullCacheOption(){ try{ TableDefinition table1 = new TableDefinition("table1", new ArrayList<>(), - new HashMap<>(), null, null); + new ArrayList<>(), new HashMap<>(), null, null); } catch (NullPointerException npe){ assertEquals("CacheOption cannot be null", npe.getMessage()); } diff --git a/stack/corepersistence/map/src/main/java/org/apache/usergrid/persistence/map/impl/MapManagerImpl.java b/stack/corepersistence/map/src/main/java/org/apache/usergrid/persistence/map/impl/MapManagerImpl.java index 501ade7b81..ae050570e2 100644 --- a/stack/corepersistence/map/src/main/java/org/apache/usergrid/persistence/map/impl/MapManagerImpl.java +++ b/stack/corepersistence/map/src/main/java/org/apache/usergrid/persistence/map/impl/MapManagerImpl.java @@ -51,7 +51,6 @@ public String getString( final String key ) { return mapSerialization.getString( scope, key ); } - @Override public String getStringHighConsistency( final String key ) { return mapSerialization.getStringHighConsistency(scope, key); diff --git a/stack/corepersistence/map/src/main/java/org/apache/usergrid/persistence/map/impl/MapSerializationImpl.java b/stack/corepersistence/map/src/main/java/org/apache/usergrid/persistence/map/impl/MapSerializationImpl.java index f90f80cefb..5fc6ee1959 100644 --- a/stack/corepersistence/map/src/main/java/org/apache/usergrid/persistence/map/impl/MapSerializationImpl.java +++ b/stack/corepersistence/map/src/main/java/org/apache/usergrid/persistence/map/impl/MapSerializationImpl.java @@ -20,6 +20,7 @@ package org.apache.usergrid.persistence.map.impl; +import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -28,6 +29,10 @@ import java.util.Map; import java.util.UUID; +import com.datastax.driver.core.*; +import com.datastax.driver.core.querybuilder.Clause; +import com.datastax.driver.core.querybuilder.QueryBuilder; +import com.datastax.driver.core.querybuilder.Using; import org.apache.cassandra.db.marshal.BytesType; import org.apache.cassandra.db.marshal.UTF8Type; @@ -39,6 +44,7 @@ import org.apache.usergrid.persistence.core.astyanax.MultiTenantColumnFamilyDefinition; import org.apache.usergrid.persistence.core.astyanax.ScopedRowKey; import org.apache.usergrid.persistence.core.astyanax.ScopedRowKeySerializer; +import org.apache.usergrid.persistence.core.datastax.CQLUtils; import org.apache.usergrid.persistence.core.shard.ExpandingShardLocator; import org.apache.usergrid.persistence.core.shard.StringHashUtils; import org.apache.usergrid.persistence.map.MapScope; @@ -47,17 +53,9 @@ import com.google.common.hash.Funnel; import com.google.inject.Inject; import com.google.inject.Singleton; -import com.netflix.astyanax.ColumnListMutation; import com.netflix.astyanax.Keyspace; -import com.netflix.astyanax.MutationBatch; -import com.netflix.astyanax.connectionpool.exceptions.ConnectionException; -import com.netflix.astyanax.connectionpool.exceptions.NotFoundException; -import com.netflix.astyanax.model.Column; import com.netflix.astyanax.model.CompositeBuilder; import com.netflix.astyanax.model.CompositeParser; -import com.netflix.astyanax.model.ConsistencyLevel; -import com.netflix.astyanax.model.Row; -import com.netflix.astyanax.model.Rows; import com.netflix.astyanax.serializers.BooleanSerializer; import com.netflix.astyanax.serializers.StringSerializer; @@ -65,6 +63,9 @@ @Singleton public class MapSerializationImpl implements MapSerialization { + private static final String MAP_KEYS_TABLE = CQLUtils.quote("Map_Keys"); + private static final String MAP_ENTRIES_TABLE = CQLUtils.quote("Map_Entries"); + private static final MapKeySerializer KEY_SERIALIZER = new MapKeySerializer(); private static final BucketScopedRowKeySerializer MAP_KEY_SERIALIZER = @@ -81,7 +82,7 @@ public class MapSerializationImpl implements MapSerialization { private static final StringSerializer STRING_SERIALIZER = StringSerializer.get(); - private static final StringResultsBuilder STRING_RESULTS_BUILDER = new StringResultsBuilder(); + private static final StringResultsBuilderCQL STRING_RESULTS_BUILDER_CQL = new StringResultsBuilderCQL(); /** @@ -117,137 +118,111 @@ public class MapSerializationImpl implements MapSerialization { private final Keyspace keyspace; private final CassandraConfig cassandraConfig; + private final Session session; + @Inject - public MapSerializationImpl( final Keyspace keyspace, final CassandraConfig cassandraConfig ) { + public MapSerializationImpl( final Keyspace keyspace, final CassandraConfig cassandraConfig, + final Session session ) { this.keyspace = keyspace; + this.session = session; this.cassandraConfig = cassandraConfig; } @Override public String getString( final MapScope scope, final String key ) { - Column col = getValue( scope, key, cassandraConfig.getReadCL() ); - return ( col != null ) ? col.getStringValue() : null; + + ByteBuffer value = getValueCQL( scope, key, cassandraConfig.getDataStaxReadCl() ) ; + return value != null ? (String)DataType.text().deserialize(value,ProtocolVersion.NEWEST_SUPPORTED ): null; } @Override public String getStringHighConsistency( final MapScope scope, final String key ) { - Column col = getValue( scope, key, cassandraConfig.getConsistentReadCL() ); // TODO: why boolean? - return ( col != null ) ? col.getStringValue() : null; + + ByteBuffer value = getValueCQL( scope, key, cassandraConfig.getDataStaxReadConsistentCl() ) ; + return value != null ? (String)DataType.text().deserialize(value,ProtocolVersion.NEWEST_SUPPORTED ): null; } @Override public Map getStrings( final MapScope scope, final Collection keys ) { - return getValues( scope, keys, STRING_RESULTS_BUILDER ); + return getValuesCQL( scope, keys, STRING_RESULTS_BUILDER_CQL ); } @Override public void putString( final MapScope scope, final String key, final String value ) { - final RowOp op = new RowOp() { - @Override - public void putValue( final ColumnListMutation columnListMutation ) { - columnListMutation.putColumn( true, value ); - } - - @Override - public void putKey( final ColumnListMutation keysMutation ) { - keysMutation.putColumn( key, true ); - } - }; - - - writeString( scope, key, value, op ); + writeStringCQL( scope, key, value, -1 ); } @Override public void putString( final MapScope scope, final String key, final String value, final int ttl ) { - Preconditions.checkArgument( ttl > 0, "ttl must be > than 0" ); - - final RowOp op = new RowOp() { - @Override - public void putValue( final ColumnListMutation columnListMutation ) { - columnListMutation.putColumn( true, value, ttl ); - } - - @Override - public void putKey( final ColumnListMutation keysMutation ) { - keysMutation.putColumn( key, true, ttl ); - } - }; - - - writeString( scope, key, value, op ); + Preconditions.checkArgument( ttl > 0, "ttl must be > than 0" ); + writeStringCQL( scope, key, value, ttl ); } /** * Write our string index with the specified row op */ - private void writeString( final MapScope scope, final String key, final String value, final RowOp rowOp ) { + private void writeStringCQL( final MapScope scope, final String key, final String value, int ttl ) { Preconditions.checkNotNull( scope, "mapscope is required" ); Preconditions.checkNotNull( key, "key is required" ); Preconditions.checkNotNull( value, "value is required" ); - final MutationBatch batch = keyspace.prepareMutationBatch(); - - //add it to the entry - final ScopedRowKey entryRowKey = MapEntryKey.fromKey( scope, key ); + Statement mapEntry; + Statement mapKey; + if (ttl > 0){ + Using timeToLive = QueryBuilder.ttl(ttl); - //serialize to the - // entry + mapEntry = QueryBuilder.insertInto(MAP_ENTRIES_TABLE) + .using(timeToLive) + .value("key", getMapEntryPartitionKey(scope, key)) + .value("column1", DataType.cboolean().serialize(true, ProtocolVersion.NEWEST_SUPPORTED)) + .value("value", DataType.text().serialize(value, ProtocolVersion.NEWEST_SUPPORTED)); - rowOp.putValue( batch.withRow( MAP_ENTRIES, entryRowKey ) ); + final int bucket = BUCKET_LOCATOR.getCurrentBucket( key ); + mapKey = QueryBuilder.insertInto(MAP_KEYS_TABLE) + .using(timeToLive) + .value("key", getMapKeyPartitionKey(scope, key, bucket)) + .value("column1", DataType.cboolean().serialize(true, ProtocolVersion.NEWEST_SUPPORTED)) + .value("value", DataType.text().serialize(value, ProtocolVersion.NEWEST_SUPPORTED)); + }else{ + mapEntry = QueryBuilder.insertInto(MAP_ENTRIES_TABLE) + .value("key", getMapEntryPartitionKey(scope, key)) + .value("column1", DataType.cboolean().serialize(true, ProtocolVersion.NEWEST_SUPPORTED)) + .value("value", DataType.text().serialize(value, ProtocolVersion.NEWEST_SUPPORTED)); - //add it to the keys + // get a bucket number for the map keys table + final int bucket = BUCKET_LOCATOR.getCurrentBucket( key ); - final int bucket = BUCKET_LOCATOR.getCurrentBucket( key ); - - final BucketScopedRowKey keyRowKey = BucketScopedRowKey.fromKey( scope.getApplication(), key, bucket ); - - //serialize to the entry + mapKey = QueryBuilder.insertInto(MAP_KEYS_TABLE) + .value("key", getMapKeyPartitionKey(scope, key, bucket)) + .value("column1", DataType.cboolean().serialize(true, ProtocolVersion.NEWEST_SUPPORTED)) + .value("value", DataType.text().serialize(value, ProtocolVersion.NEWEST_SUPPORTED)); - rowOp.putKey( batch.withRow( MAP_KEYS, keyRowKey ) ); + } + session.execute(mapEntry); + session.execute(mapKey); - executeBatch( batch ); } - /** - * Callbacks for performing row operations - */ - private static interface RowOp { - - /** - * Callback to do the row - * - * @param columnListMutation The column mutation - */ - void putValue( final ColumnListMutation columnListMutation ); - - - /** - * Write the key - */ - void putKey( final ColumnListMutation keysMutation ); - } - @Override public UUID getUuid( final MapScope scope, final String key ) { - Column col = getValue( scope, key, cassandraConfig.getReadCL() ); - return ( col != null ) ? col.getUUIDValue() : null; + ByteBuffer value = getValueCQL( scope, key, cassandraConfig.getDataStaxReadCl() ); + return value != null ? (UUID)DataType.uuid().deserialize(value, ProtocolVersion.NEWEST_SUPPORTED ) : null; } @@ -258,31 +233,34 @@ public void putUuid( final MapScope scope, final String key, final UUID putUuid Preconditions.checkNotNull( key, "key is required" ); Preconditions.checkNotNull( putUuid, "value is required" ); - final MutationBatch batch = keyspace.prepareMutationBatch(); - //add it to the entry - final ScopedRowKey entryRowKey = MapEntryKey.fromKey( scope, key ); + Statement mapEntry = QueryBuilder.insertInto(MAP_ENTRIES_TABLE) + .value("key", getMapEntryPartitionKey(scope, key)) + .value("column1", DataType.cboolean().serialize(true, ProtocolVersion.NEWEST_SUPPORTED)) + .value("value", DataType.uuid().serialize(putUuid, ProtocolVersion.NEWEST_SUPPORTED)); - //serialize to the entry - batch.withRow( MAP_ENTRIES, entryRowKey ).putColumn( true, putUuid ); + session.execute(mapEntry); - //add it to the keys final int bucket = BUCKET_LOCATOR.getCurrentBucket( key ); + Statement mapKey; + mapKey = QueryBuilder.insertInto(MAP_KEYS_TABLE) + .value("key", getMapKeyPartitionKey(scope, key, bucket)) + .value("column1", DataType.text().serialize(key, ProtocolVersion.NEWEST_SUPPORTED)) + .value("value", DataType.serializeValue(null, ProtocolVersion.NEWEST_SUPPORTED)); + + session.execute(mapKey); + } - final BucketScopedRowKey keyRowKey = BucketScopedRowKey.fromKey( scope.getApplication(), key, bucket ); - //serialize to the entry - batch.withRow( MAP_KEYS, keyRowKey ).putColumn( key, true ); - executeBatch( batch ); - } @Override public Long getLong( final MapScope scope, final String key ) { - Column col = getValue( scope, key, cassandraConfig.getReadCL() ); - return ( col != null ) ? col.getLongValue() : null; + + ByteBuffer value = getValueCQL( scope, key, cassandraConfig.getDataStaxReadCl()); + return value != null ? (Long)DataType.bigint().deserialize(value, ProtocolVersion.NEWEST_SUPPORTED ) : null; } @@ -293,46 +271,50 @@ public void putLong( final MapScope scope, final String key, final Long value ) Preconditions.checkNotNull( key, "key is required" ); Preconditions.checkNotNull( value, "value is required" ); - final MutationBatch batch = keyspace.prepareMutationBatch(); + Statement mapEntry = QueryBuilder.insertInto(MAP_ENTRIES_TABLE) + .value("key", getMapEntryPartitionKey(scope, key)) + .value("column1", DataType.cboolean().serialize(true, ProtocolVersion.NEWEST_SUPPORTED)) + .value("value", DataType.bigint().serialize(value, ProtocolVersion.NEWEST_SUPPORTED)); - //add it to the entry - final ScopedRowKey entryRowKey = MapEntryKey.fromKey( scope, key ); + session.execute(mapEntry); - //serialize to the entry - batch.withRow( MAP_ENTRIES, entryRowKey ).putColumn( true, value ); - //add it to the keys final int bucket = BUCKET_LOCATOR.getCurrentBucket( key ); + Statement mapKey; + mapKey = QueryBuilder.insertInto(MAP_KEYS_TABLE) + .value("key", getMapKeyPartitionKey(scope, key, bucket)) + .value("column1", DataType.text().serialize(key, ProtocolVersion.NEWEST_SUPPORTED)) + .value("value", DataType.serializeValue(null, ProtocolVersion.NEWEST_SUPPORTED)); - final BucketScopedRowKey keyRowKey = BucketScopedRowKey.fromKey( scope.getApplication(), key, bucket ); - - //serialize to the entry - batch.withRow( MAP_KEYS, keyRowKey ).putColumn( key, true ); - - executeBatch( batch ); + session.execute(mapKey); } @Override public void delete( final MapScope scope, final String key ) { - final MutationBatch batch = keyspace.prepareMutationBatch(); - final ScopedRowKey entryRowKey = MapEntryKey.fromKey( scope, key ); - //serialize to the entry - batch.withRow( MAP_ENTRIES, entryRowKey ).delete(); - - //add it to the keys, we're not sure which one it may have come from - final int[] buckets = BUCKET_LOCATOR.getAllBuckets( key ); + Statement deleteMapEntry; + Clause equalsEntryKey = QueryBuilder.eq("key", getMapEntryPartitionKey(scope, key)); + deleteMapEntry = QueryBuilder.delete().from(MAP_ENTRIES_TABLE) + .where(equalsEntryKey); + session.execute(deleteMapEntry); - final List> rowKeys = - BucketScopedRowKey.fromRange( scope.getApplication(), key, buckets ); - for ( BucketScopedRowKey rowKey : rowKeys ) { - batch.withRow( MAP_KEYS, rowKey ).deleteColumn( key ); + // not sure which bucket the value is in, execute a delete against them all + final int[] buckets = BUCKET_LOCATOR.getAllBuckets( key ); + List mapKeys = new ArrayList<>(); + for( int bucket : buckets){ + mapKeys.add( getMapKeyPartitionKey(scope, key, bucket)); } - executeBatch( batch ); + Statement deleteMapKey; + Clause inKey = QueryBuilder.in("key", mapKeys); + deleteMapKey = QueryBuilder.delete().from(MAP_KEYS_TABLE) + .where(inKey); + session.execute(deleteMapKey); + + } @@ -353,72 +335,38 @@ public Collection getColumnFamilies() { } - private Column getValue( MapScope scope, String key, final ConsistencyLevel consistencyLevel ) { + private ByteBuffer getValueCQL( MapScope scope, String key, final ConsistencyLevel consistencyLevel ) { - //add it to the entry - final ScopedRowKey entryRowKey = MapEntryKey.fromKey( scope, key ); + Clause in = QueryBuilder.in("key", getMapEntryPartitionKey(scope, key) ); + Statement statement = QueryBuilder.select().all().from(MAP_ENTRIES_TABLE) + .where(in) + .setConsistencyLevel(consistencyLevel); - //now get all columns, including the "old row key value" - try { - final Column result = - keyspace.prepareQuery( MAP_ENTRIES ).setConsistencyLevel( consistencyLevel ).getKey( entryRowKey ).getColumn( true ).execute().getResult(); + ResultSet resultSet = session.execute(statement); + com.datastax.driver.core.Row row = resultSet.one(); - return result; - } - catch ( NotFoundException nfe ) { - //nothing to return - return null; - } - catch ( ConnectionException e ) { - throw new RuntimeException( "Unable to connect to cassandra", e ); - } + return row != null ? row.getBytes("value") : null; } - /** - * Get multiple values, using the string builder - */ - private T getValues( final MapScope scope, final Collection keys, final ResultsBuilder builder ) { - - final List> rowKeys = new ArrayList<>( keys.size() ); + private T getValuesCQL( final MapScope scope, final Collection keys, final ResultsBuilderCQL builder ) { - for ( final String key : keys ) { - //add it to the entry - final ScopedRowKey entryRowKey = MapEntryKey.fromKey( scope, key ); + final List serializedKeys = new ArrayList<>(); - rowKeys.add( entryRowKey ); - } + keys.forEach(key -> serializedKeys.add(getMapEntryPartitionKey(scope,key))); + Clause in = QueryBuilder.in("key", serializedKeys ); + Statement statement = QueryBuilder.select().all().from(MAP_ENTRIES_TABLE) + .where(in); - //now get all columns, including the "old row key value" - try { - final Rows, Boolean> rows = - keyspace.prepareQuery( MAP_ENTRIES ).setConsistencyLevel( cassandraConfig.getReadCL() ).getKeySlice( - rowKeys ).withColumnSlice( true ).execute() - .getResult(); + ResultSet resultSet = session.execute(statement); - return builder.buildResults( rows ); - } - catch ( NotFoundException nfe ) { - //nothing to return - return null; - } - catch ( ConnectionException e ) { - throw new RuntimeException( "Unable to connect to cassandra", e ); - } + return builder.buildResultsCQL( resultSet ); } - private void executeBatch( MutationBatch batch ) { - try { - batch.execute(); - } - catch ( ConnectionException e ) { - throw new RuntimeException( "Unable to connect to cassandra", e ); - } - } /** @@ -491,37 +439,114 @@ public static ScopedRowKey fromKey( final MapScope mapScope, final } + /** * Build the results from the row keys */ - private static interface ResultsBuilder { - public T buildResults( final Rows, Boolean> rows ); + private interface ResultsBuilderCQL { + + T buildResultsCQL( final ResultSet resultSet ); } - public static class StringResultsBuilder implements ResultsBuilder> { + public static class StringResultsBuilderCQL implements ResultsBuilderCQL> { @Override - public Map buildResults( final Rows, Boolean> rows ) { - final int size = rows.size(); + public Map buildResultsCQL( final ResultSet resultSet ) { - final Map results = new HashMap<>( size ); - for ( int i = 0; i < size; i++ ) { + final Map results = new HashMap<>(); - final Row, Boolean> row = rows.getRowByIndex( i ); + resultSet.all().forEach( row -> { - final String value = row.getColumns().getStringValue( true, null ); + @SuppressWarnings("unchecked") + List keys = (List) deserializeMapEntryKey(row.getBytes("key")); + String value = (String)DataType.text().deserialize( row.getBytes("value"), + ProtocolVersion.NEWEST_SUPPORTED ); - if ( value == null ) { - continue; - } + // the actual string key value is the last element + results.put((String)keys.get(keys.size() -1), value); - results.put( row.getKey().getKey().key, value ); - } + }); return results; } } + + private static Object deserializeMapEntryKey(ByteBuffer bb){ + + List stuff = new ArrayList<>(); + while(bb.hasRemaining()){ + ByteBuffer data = CQLUtils.getWithShortLength(bb); + if(stuff.size() == 0){ + stuff.add(DataType.uuid().deserialize(data.slice(), ProtocolVersion.NEWEST_SUPPORTED)); + }else{ + stuff.add(DataType.text().deserialize(data.slice(), ProtocolVersion.NEWEST_SUPPORTED)); + } + byte equality = bb.get(); // we don't use this but take the equality byte off the buffer + + } + + return stuff; + + } + + public static ByteBuffer serializeKeys(UUID ownerUUID, String ownerType, String mapName, String mapKey, + int bucketNumber ){ + + List keys = new ArrayList<>(4); + keys.add(0, ownerUUID); + keys.add(1, ownerType); + keys.add(2, mapName); + keys.add(3, mapKey); + + if( bucketNumber > 0){ + keys.add(4, bucketNumber); + } + + // UUIDs are 16 bytes, allocate the buffer accordingly + int size = 16+ownerType.length()+mapName.length()+mapKey.length(); + if(bucketNumber > 0 ){ + // ints are 4 bytes + size += 4; + } + + // we always need to add length for the 2 byte short and 1 byte equality + size += keys.size()*3; + + ByteBuffer stuff = ByteBuffer.allocate(size); + + for (Object key : keys) { + + ByteBuffer kb = DataType.serializeValue(key, ProtocolVersion.NEWEST_SUPPORTED); + if (kb == null) { + kb = ByteBuffer.allocate(0); + } + + stuff.putShort((short) kb.remaining()); + stuff.put(kb.slice()); + stuff.put((byte) 0); + + + } + stuff.flip(); + return stuff.duplicate(); + + } + + + private ByteBuffer getMapEntryPartitionKey(MapScope scope, String key){ + + return serializeKeys(scope.getApplication().getUuid(), + scope.getApplication().getType(), scope.getName(), key, -1); + + } + + private ByteBuffer getMapKeyPartitionKey(MapScope scope, String key, int bucketNumber){ + + return serializeKeys(scope.getApplication().getUuid(), + scope.getApplication().getType(), scope.getName(), key, bucketNumber); + + } } diff --git a/stack/corepersistence/map/src/test/java/org/apache/usergrid/persistence/map/MapManagerTest.java b/stack/corepersistence/map/src/test/java/org/apache/usergrid/persistence/map/MapManagerTest.java index 41286ab154..2a68247229 100644 --- a/stack/corepersistence/map/src/test/java/org/apache/usergrid/persistence/map/MapManagerTest.java +++ b/stack/corepersistence/map/src/test/java/org/apache/usergrid/persistence/map/MapManagerTest.java @@ -82,6 +82,22 @@ public void writeReadString() { assertEquals( value, returned ); } + @Test + public void writeReadStringWithLongKey() { + MapManager mm = mmf.createMapManager( this.scope ); + + final String key = "key1234567890123456789012345678901234567890123456789012345678901234567890" + + "123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890" + + "123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890"; + final String value = "value"; + + mm.putString( key, value ); + + final String returned = mm.getString( key ); + + assertEquals( value, returned ); + } + @Test public void multiReadNoKey() { From 3f8e7cbaf4a7f8351185e803e7bd43dafc49a9a9 Mon Sep 17 00:00:00 2001 From: Michael Russo Date: Thu, 11 Feb 2016 17:11:59 -0800 Subject: [PATCH 06/30] Reduce number of connections in core tests and ensure the datastax provider is a singleton. --- .../resources/usergrid-custom-test.properties | 2 +- .../datastax/DatastaxSessionProvider.java | 2 + .../datastax/impl/DatastaxClusterImpl.java | 60 +++++++++++++++++-- .../schema/MigrationManagerImpl.java | 2 +- 4 files changed, 59 insertions(+), 7 deletions(-) diff --git a/stack/core/src/test/resources/usergrid-custom-test.properties b/stack/core/src/test/resources/usergrid-custom-test.properties index 0fa95732d0..c8b4d48f89 100644 --- a/stack/core/src/test/resources/usergrid-custom-test.properties +++ b/stack/core/src/test/resources/usergrid-custom-test.properties @@ -15,7 +15,7 @@ # these settings allow tests to run and consistently pass on 16GB MacBook Pro # with ug.heapmax=5000m and ug.heapmin=3000m (set in Maven settings.xml) cassandra.timeout=2000 -cassandra.connections=1000 +cassandra.connections=50 #Not a good number for real systems. Write shards should be 2x cluster size from our tests diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/DatastaxSessionProvider.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/DatastaxSessionProvider.java index 1b39cb8439..5e9a6338b5 100644 --- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/DatastaxSessionProvider.java +++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/DatastaxSessionProvider.java @@ -22,7 +22,9 @@ import com.datastax.driver.core.Session; import com.google.inject.Inject; import com.google.inject.Provider; +import com.google.inject.Singleton; +@Singleton public class DataStaxSessionProvider implements Provider { private final DataStaxCluster dataStaxCluster; diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/impl/DatastaxClusterImpl.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/impl/DatastaxClusterImpl.java index ffe61e6455..a0d65e0d7a 100644 --- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/impl/DatastaxClusterImpl.java +++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/impl/DatastaxClusterImpl.java @@ -24,6 +24,7 @@ import com.google.inject.Inject; import com.google.inject.Singleton; import org.apache.usergrid.persistence.core.astyanax.CassandraFig; +import org.apache.usergrid.persistence.core.datastax.CQLUtils; import org.apache.usergrid.persistence.core.datastax.DataStaxCluster; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -98,18 +99,17 @@ public DataStaxClusterImpl(final CassandraFig cassandraFig ) throws Exception { cluster.getConfiguration().getPoolingOptions().getIdleTimeoutSeconds(), cluster.getConfiguration().getPoolingOptions().getPoolTimeoutMillis() / 1000); - - - - + createOrUpdateKeyspace(); } + @Override public Cluster getCluster(){ return cluster; } + @Override public Session getClusterSession(){ if ( clusterSession == null || clusterSession.isClosed() ){ @@ -119,13 +119,63 @@ public Session getClusterSession(){ return clusterSession; } + @Override public Session getApplicationSession(){ if ( applicationSession == null || applicationSession.isClosed() ){ - applicationSession = cluster.connect( "\""+cassandraFig.getApplicationKeyspace()+"\"" ); + applicationSession = cluster.connect( CQLUtils.quote(cassandraFig.getApplicationKeyspace() ) ); } return applicationSession; } + private void createOrUpdateKeyspace() throws Exception { + + clusterSession = getClusterSession(); + + final String createApplicationKeyspace = String.format( + "CREATE KEYSPACE IF NOT EXISTS \"%s\" WITH replication = %s", + cassandraFig.getApplicationKeyspace(), + CQLUtils.getFormattedReplication( cassandraFig.getStrategy(), cassandraFig.getStrategyOptions() ) + + ); + + final String updateApplicationKeyspace = String.format( + "ALTER KEYSPACE \"%s\" WITH replication = %s", + cassandraFig.getApplicationKeyspace(), + CQLUtils.getFormattedReplication( cassandraFig.getStrategy(), cassandraFig.getStrategyOptions() ) + ); + + logger.info("Creating application keyspace with the following CQL: {}", createApplicationKeyspace); + clusterSession.execute(createApplicationKeyspace); + logger.info("Updating application keyspace with the following CQL: {}", updateApplicationKeyspace); + clusterSession.executeAsync(updateApplicationKeyspace); + + // this session pool is only used when running database setup so close it when finished to clear resources + clusterSession.close(); + + waitForSchemaAgreement(); + } + + /** + * Wait until all Cassandra nodes agree on the schema. Sleeps 100ms between checks. + * + */ + private void waitForSchemaAgreement() { + + while ( true ) { + + if( cluster.getMetadata().checkSchemaAgreement() ){ + return; + } + + //sleep and try it again + try { + Thread.sleep( 100 ); + } + catch ( InterruptedException e ) { + //swallow + } + } + } } diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/schema/MigrationManagerImpl.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/schema/MigrationManagerImpl.java index e7e0cb542a..8eb0576918 100644 --- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/schema/MigrationManagerImpl.java +++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/schema/MigrationManagerImpl.java @@ -147,7 +147,7 @@ private void createOrUpdateKeyspace() throws Exception { logger.info("Creating application keyspace with the following CQL: {}", createApplicationKeyspace); clusterSession.execute(createApplicationKeyspace); logger.info("Updating application keyspace with the following CQL: {}", updateApplicationKeyspace); - clusterSession.execute(updateApplicationKeyspace); + clusterSession.executeAsync(updateApplicationKeyspace); // this session pool is only used when running database setup so close it when finished to clear resources clusterSession.close(); From 3066ab4a59f5fff25f516e6869ea846a48000081 Mon Sep 17 00:00:00 2001 From: Michael Russo Date: Thu, 11 Feb 2016 18:04:35 -0800 Subject: [PATCH 07/30] Standardizing test properties. --- stack/core/src/test/resources/usergrid-custom-test.properties | 1 - .../collection/src/test/resources/dynamic-test.properties | 3 +-- .../collection/src/test/resources/usergrid-CHOP.properties | 3 +-- .../collection/src/test/resources/usergrid-UNIT.properties | 1 - .../persistence/core/datastax/impl/DatastaxClusterImpl.java | 2 +- .../graph/src/test/resources/usergrid-AWS.properties | 3 +-- .../graph/src/test/resources/usergrid-CHOP.properties | 3 +-- .../graph/src/test/resources/usergrid-UNIT.properties | 3 +-- .../queryindex/src/test/resources/dynamic-test.properties | 3 +-- .../queryindex/src/test/resources/usergrid-CHOP.properties | 3 +-- .../queryindex/src/test/resources/usergrid-UNIT.properties | 3 +-- stack/rest/src/test/resources/usergrid-custom-test.properties | 3 +-- .../src/test/resources/usergrid-custom-test.properties | 3 +-- 13 files changed, 11 insertions(+), 23 deletions(-) diff --git a/stack/core/src/test/resources/usergrid-custom-test.properties b/stack/core/src/test/resources/usergrid-custom-test.properties index c8b4d48f89..a9be7ed078 100644 --- a/stack/core/src/test/resources/usergrid-custom-test.properties +++ b/stack/core/src/test/resources/usergrid-custom-test.properties @@ -14,7 +14,6 @@ # these settings allow tests to run and consistently pass on 16GB MacBook Pro # with ug.heapmax=5000m and ug.heapmin=3000m (set in Maven settings.xml) -cassandra.timeout=2000 cassandra.connections=50 diff --git a/stack/corepersistence/collection/src/test/resources/dynamic-test.properties b/stack/corepersistence/collection/src/test/resources/dynamic-test.properties index 6b6f551472..b847f507f7 100644 --- a/stack/corepersistence/collection/src/test/resources/dynamic-test.properties +++ b/stack/corepersistence/collection/src/test/resources/dynamic-test.properties @@ -1,12 +1,11 @@ # The properties are not the actual configuration properties but # safe dynamic property defaults for our testing via IDE or Maven -cassandra.connections=10 +cassandra.connections=50 cassandra.port=9160 cassandra.version=1.2 cassandra.hosts=localhost cassandra.cluster_name=Usergrid collections.keyspace=Usergrid_Collections -cassandra.timeout=5000 collection.stage.transient.timeout=5 # This property is required to be set and cannot be defaulted anywhere diff --git a/stack/corepersistence/collection/src/test/resources/usergrid-CHOP.properties b/stack/corepersistence/collection/src/test/resources/usergrid-CHOP.properties index 78e340035c..d276934363 100644 --- a/stack/corepersistence/collection/src/test/resources/usergrid-CHOP.properties +++ b/stack/corepersistence/collection/src/test/resources/usergrid-CHOP.properties @@ -1,6 +1,6 @@ # These are for CHOP environment settings -cassandra.connections=20 +cassandra.connections=50 cassandra.port=9160 cassandra.version=1.2 @@ -9,7 +9,6 @@ cassandra.version=1.2 cassandra.hosts=${chop.cassandra.hosts} cassandra.cluster_name=Usergrid collections.keyspace=Usergrid_Collections -cassandra.timeout=5000 # This property is required to be set and cannot be defaulted anywhere usergrid.cluster_name=usergrid diff --git a/stack/corepersistence/collection/src/test/resources/usergrid-UNIT.properties b/stack/corepersistence/collection/src/test/resources/usergrid-UNIT.properties index 44dc399527..bec1c88f8c 100644 --- a/stack/corepersistence/collection/src/test/resources/usergrid-UNIT.properties +++ b/stack/corepersistence/collection/src/test/resources/usergrid-UNIT.properties @@ -5,7 +5,6 @@ cassandra.version=1.2 cassandra.hosts=localhost cassandra.cluster_name=Usergrid collections.keyspace=Usergrid_Collections -cassandra.timeout=5000 cassandra.embedded=true diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/impl/DatastaxClusterImpl.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/impl/DatastaxClusterImpl.java index a0d65e0d7a..5fc804b846 100644 --- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/impl/DatastaxClusterImpl.java +++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/impl/DatastaxClusterImpl.java @@ -94,7 +94,7 @@ public DataStaxClusterImpl(final CassandraFig cassandraFig ) throws Exception { } this.cluster = datastaxCluster.build(); - logger.info("Initialized datastax cluster client. Hosts={}, Idle Timeout={}s, Request Timeout={}s", + logger.info("Initialized datastax cluster client. Hosts={}, Idle Timeout={}s, Pool Timeout={}s", cluster.getMetadata().getAllHosts().toString(), cluster.getConfiguration().getPoolingOptions().getIdleTimeoutSeconds(), cluster.getConfiguration().getPoolingOptions().getPoolTimeoutMillis() / 1000); diff --git a/stack/corepersistence/graph/src/test/resources/usergrid-AWS.properties b/stack/corepersistence/graph/src/test/resources/usergrid-AWS.properties index e19f6768bd..e630bb8694 100644 --- a/stack/corepersistence/graph/src/test/resources/usergrid-AWS.properties +++ b/stack/corepersistence/graph/src/test/resources/usergrid-AWS.properties @@ -1,12 +1,11 @@ # Keep nothing but overriding test defaults in here -cassandra.connections=100 +cassandra.connections=50 cassandra.port=9160 cassandra.version=1.2 cassandra.hosts= #cassandra.hosts=localhost cassandra.cluster_name=Usergrid collections.keyspace=Usergrid_Collections -cassandra.timeout=5000 collections.keyspace.strategy.options=us-east:3 diff --git a/stack/corepersistence/graph/src/test/resources/usergrid-CHOP.properties b/stack/corepersistence/graph/src/test/resources/usergrid-CHOP.properties index 78e340035c..d276934363 100644 --- a/stack/corepersistence/graph/src/test/resources/usergrid-CHOP.properties +++ b/stack/corepersistence/graph/src/test/resources/usergrid-CHOP.properties @@ -1,6 +1,6 @@ # These are for CHOP environment settings -cassandra.connections=20 +cassandra.connections=50 cassandra.port=9160 cassandra.version=1.2 @@ -9,7 +9,6 @@ cassandra.version=1.2 cassandra.hosts=${chop.cassandra.hosts} cassandra.cluster_name=Usergrid collections.keyspace=Usergrid_Collections -cassandra.timeout=5000 # This property is required to be set and cannot be defaulted anywhere usergrid.cluster_name=usergrid diff --git a/stack/corepersistence/graph/src/test/resources/usergrid-UNIT.properties b/stack/corepersistence/graph/src/test/resources/usergrid-UNIT.properties index eb060012e4..e1c4c81001 100644 --- a/stack/corepersistence/graph/src/test/resources/usergrid-UNIT.properties +++ b/stack/corepersistence/graph/src/test/resources/usergrid-UNIT.properties @@ -1,11 +1,10 @@ # Keep nothing but overriding test defaults in here -cassandra.connections=40 +cassandra.connections=50 cassandra.port=9160 cassandra.version=1.2 cassandra.hosts=localhost cassandra.cluster_name=Usergrid collections.keyspace=Usergrid_Collections -cassandra.timeout=2000 cassandra.embedded=true diff --git a/stack/corepersistence/queryindex/src/test/resources/dynamic-test.properties b/stack/corepersistence/queryindex/src/test/resources/dynamic-test.properties index ed3209c9b8..4ac97d84cc 100644 --- a/stack/corepersistence/queryindex/src/test/resources/dynamic-test.properties +++ b/stack/corepersistence/queryindex/src/test/resources/dynamic-test.properties @@ -1,12 +1,11 @@ # The properties are not the actual configuration properties but # safe dynamic property defaults for our testing via IDE or Maven -cassandra.connections=10 +cassandra.connections=50 cassandra.port=9160 cassandra.version=1.2 cassandra.hosts=localhost cassandra.cluster_name=Usergrid collections.keyspace=Usergrid_Collections -cassandra.timeout=5000 index.query.limit.default=10 elasticsearch.indexname=QueryIndexTests diff --git a/stack/corepersistence/queryindex/src/test/resources/usergrid-CHOP.properties b/stack/corepersistence/queryindex/src/test/resources/usergrid-CHOP.properties index e0e71c9305..ce40600e8d 100644 --- a/stack/corepersistence/queryindex/src/test/resources/usergrid-CHOP.properties +++ b/stack/corepersistence/queryindex/src/test/resources/usergrid-CHOP.properties @@ -1,5 +1,5 @@ # These are for CHOP environment settings -cassandra.connections=20 +cassandra.connections=50 cassandra.port=9160 cassandra.version=1.2 @@ -8,7 +8,6 @@ cassandra.version=1.2 cassandra.hosts=${chop.cassandra.hosts} cassandra.cluster_name=Usergrid collections.keyspace=Usergrid_Collections -cassandra.timeout=5000 index.query.limit.default=10 diff --git a/stack/corepersistence/queryindex/src/test/resources/usergrid-UNIT.properties b/stack/corepersistence/queryindex/src/test/resources/usergrid-UNIT.properties index ccff66a803..5d80cac49e 100644 --- a/stack/corepersistence/queryindex/src/test/resources/usergrid-UNIT.properties +++ b/stack/corepersistence/queryindex/src/test/resources/usergrid-UNIT.properties @@ -5,8 +5,7 @@ cassandra.hosts=127.0.0.1 cassandra.port=9160 cassandra.version=1.2 cassandra.cluster_name=Usergrid -cassandra.connections=20 -cassandra.timeout=5000 +cassandra.connections=50 collections.keyspace=Usergrid_Collections collections.keyspace.strategy.options=replication_factor:1 diff --git a/stack/rest/src/test/resources/usergrid-custom-test.properties b/stack/rest/src/test/resources/usergrid-custom-test.properties index 2605c28868..13e06a1ec4 100644 --- a/stack/rest/src/test/resources/usergrid-custom-test.properties +++ b/stack/rest/src/test/resources/usergrid-custom-test.properties @@ -13,8 +13,7 @@ # REST module test properties cassandra.startup=external -cassandra.timeout=2000 -cassandra.connections=800 +cassandra.connections=50 elasticsearch.startup=external diff --git a/stack/services/src/test/resources/usergrid-custom-test.properties b/stack/services/src/test/resources/usergrid-custom-test.properties index 1c3d190adf..9ccfc9c1d2 100644 --- a/stack/services/src/test/resources/usergrid-custom-test.properties +++ b/stack/services/src/test/resources/usergrid-custom-test.properties @@ -16,8 +16,7 @@ # with ug.heapmax=5000m and ug.heapmin=3000m (set in Maven settings.xml) cassandra.startup=external elasticsearch.startup=external -cassandra.timeout=2000 -cassandra.connections=1000 +cassandra.connections=50 #Poll interval to check for new jobs in millseconds. 10 milliseconds for testing usergrid.scheduler.job.interval=100 From 25454f950f9b16eb0ede1905dfa6cc64b1530c3a Mon Sep 17 00:00:00 2001 From: Michael Russo Date: Thu, 11 Feb 2016 18:23:55 -0800 Subject: [PATCH 08/30] Fix header in TableDefinition. --- .../core/datastax/TableDefinition.java | 30 +++++++++---------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/TableDefinition.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/TableDefinition.java index 0635b93dba..3dc3145d45 100644 --- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/TableDefinition.java +++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/TableDefinition.java @@ -1,22 +1,20 @@ /* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at * - * * Licensed to the Apache Software Foundation (ASF) under one - * * or more contributor license agreements. See the NOTICE file - * * distributed with this work for additional information - * * regarding copyright ownership. The ASF licenses this file - * * to you under the Apache License, Version 2.0 (the - * * "License"); you may not use this file except in compliance - * * with the License. You may obtain a copy of the License at - * * - * * http://www.apache.org/licenses/LICENSE-2.0 - * * - * * Unless required by applicable law or agreed to in writing, - * * software distributed under the License is distributed on an - * * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * * KIND, either express or implied. See the License for the - * * specific language governing permissions and limitations - * * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package org.apache.usergrid.persistence.core.datastax; From c0f8a4a28576ab58b1eb0b173a73966655a863c1 Mon Sep 17 00:00:00 2001 From: Michael Russo Date: Fri, 12 Feb 2016 16:34:04 -0800 Subject: [PATCH 09/30] MapSerialization schema now created by DataStax driver. Add getTables into Migration interface for creating schema via DataStax driver and CQL. --- .../MvccEntitySerializationStrategyImpl.java | 7 + ...cEntitySerializationStrategyProxyImpl.java | 6 + ...MvccEntitySerializationStrategyV3Impl.java | 7 + .../MvccLogEntrySerializationProxyImpl.java | 6 + ...ccLogEntrySerializationStrategyV1Impl.java | 7 + ...ccLogEntrySerializationStrategyV2Impl.java | 7 + ...ueValueSerializationStrategyProxyImpl.java | 6 + ...niqueValueSerializationStrategyV1Impl.java | 8 + ...niqueValueSerializationStrategyV2Impl.java | 7 + .../persistence/core/datastax/CQLUtils.java | 4 +- .../data/MigrationInfoSerializationImpl.java | 7 + .../core/migration/schema/Migration.java | 3 +- .../schema/MigrationManagerImpl.java | 43 ++++- .../map/impl/MapSerializationImpl.java | 172 +++++------------- .../persistence/map/MapManagerTest.java | 1 + 15 files changed, 153 insertions(+), 138 deletions(-) diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImpl.java index 57199e2b13..3e2d391f84 100644 --- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImpl.java +++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImpl.java @@ -25,6 +25,7 @@ import java.util.List; import java.util.UUID; +import org.apache.usergrid.persistence.core.datastax.TableDefinition; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -342,6 +343,12 @@ public java.util.Collection getColumnFamilies() { return Collections.singleton( cf ); } + @Override + public Collection getTables() { + + return Collections.emptyList(); + } + /** * Do the write on the correct row for the entity id with the operation diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyProxyImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyProxyImpl.java index 1fe342ea5f..19735cc16b 100644 --- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyProxyImpl.java +++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyProxyImpl.java @@ -27,6 +27,7 @@ import org.apache.usergrid.persistence.collection.serialization.MvccEntitySerializationStrategy; import org.apache.usergrid.persistence.collection.serialization.impl.migration.CollectionMigrationPlugin; import org.apache.usergrid.persistence.core.astyanax.MultiTenantColumnFamilyDefinition; +import org.apache.usergrid.persistence.core.datastax.TableDefinition; import org.apache.usergrid.persistence.core.migration.data.MigrationInfoCache; import org.apache.usergrid.persistence.core.migration.data.MigrationRelationship; import org.apache.usergrid.persistence.core.migration.data.VersionedMigrationSet; @@ -186,6 +187,11 @@ public Collection getColumnFamilies() { return Collections.emptyList(); } + @Override + public Collection getTables() { + return Collections.emptyList(); + } + @Override public int getImplementationVersion() { diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV3Impl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV3Impl.java index 57607d0997..2fa5d09b6c 100644 --- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV3Impl.java +++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV3Impl.java @@ -16,6 +16,7 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.annotation.JsonSerialize; import com.netflix.astyanax.serializers.StringSerializer; +import org.apache.usergrid.persistence.core.datastax.TableDefinition; import org.apache.usergrid.persistence.core.metrics.MetricsFactory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -304,6 +305,12 @@ public java.util.Collection getColumnFamilies() { return Collections.singleton( cf ); } + @Override + public Collection getTables() { + + return Collections.emptyList(); + } + /** * Do the write on the correct row for the entity id with the operation diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccLogEntrySerializationProxyImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccLogEntrySerializationProxyImpl.java index 8e34f0164a..b27651d0c8 100644 --- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccLogEntrySerializationProxyImpl.java +++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccLogEntrySerializationProxyImpl.java @@ -30,6 +30,7 @@ import org.apache.usergrid.persistence.collection.serialization.MvccLogEntrySerializationStrategy; import org.apache.usergrid.persistence.collection.serialization.impl.migration.CollectionMigrationPlugin; import org.apache.usergrid.persistence.core.astyanax.MultiTenantColumnFamilyDefinition; +import org.apache.usergrid.persistence.core.datastax.TableDefinition; import org.apache.usergrid.persistence.core.migration.data.MigrationInfoCache; import org.apache.usergrid.persistence.core.migration.data.MigrationRelationship; import org.apache.usergrid.persistence.core.migration.data.VersionedMigrationSet; @@ -146,6 +147,11 @@ public Collection getColumnFamilies() { return Collections.emptyList(); } + @Override + public Collection getTables(){ + return Collections.emptyList(); + } + @Override public int getImplementationVersion() { diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccLogEntrySerializationStrategyV1Impl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccLogEntrySerializationStrategyV1Impl.java index e0c0909c46..62c35bbe4b 100644 --- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccLogEntrySerializationStrategyV1Impl.java +++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccLogEntrySerializationStrategyV1Impl.java @@ -35,6 +35,7 @@ import org.apache.usergrid.persistence.core.astyanax.MultiTenantColumnFamily; import org.apache.usergrid.persistence.core.astyanax.MultiTenantColumnFamilyDefinition; import org.apache.usergrid.persistence.core.astyanax.ScopedRowKey; +import org.apache.usergrid.persistence.core.datastax.TableDefinition; import org.apache.usergrid.persistence.model.entity.Id; import com.google.inject.Inject; @@ -115,4 +116,10 @@ public Collection getColumnFamilies() { return Collections.singleton( cf ); } + + @Override + public Collection getTables() { + + return Collections.emptyList(); + } } diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccLogEntrySerializationStrategyV2Impl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccLogEntrySerializationStrategyV2Impl.java index 1bfc289873..325762db55 100644 --- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccLogEntrySerializationStrategyV2Impl.java +++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccLogEntrySerializationStrategyV2Impl.java @@ -35,6 +35,7 @@ import org.apache.usergrid.persistence.core.astyanax.MultiTenantColumnFamilyDefinition; import org.apache.usergrid.persistence.core.astyanax.ScopedRowKey; import org.apache.usergrid.persistence.core.astyanax.ScopedRowKeySerializer; +import org.apache.usergrid.persistence.core.datastax.TableDefinition; import org.apache.usergrid.persistence.model.entity.Id; import com.google.inject.Inject; @@ -103,4 +104,10 @@ public Collection getColumnFamilies() { return Collections.singleton( cf ); } + + @Override + public Collection getTables() { + + return Collections.emptyList(); + } } diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyProxyImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyProxyImpl.java index 1de4052474..87b164176b 100644 --- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyProxyImpl.java +++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyProxyImpl.java @@ -29,6 +29,7 @@ import org.apache.usergrid.persistence.collection.serialization.UniqueValueSet; import org.apache.usergrid.persistence.collection.serialization.impl.migration.CollectionMigrationPlugin; import org.apache.usergrid.persistence.core.astyanax.MultiTenantColumnFamilyDefinition; +import org.apache.usergrid.persistence.core.datastax.TableDefinition; import org.apache.usergrid.persistence.core.migration.data.MigrationInfoCache; import org.apache.usergrid.persistence.core.migration.data.MigrationRelationship; import org.apache.usergrid.persistence.core.migration.data.VersionedMigrationSet; @@ -171,6 +172,11 @@ public Collection getColumnFamilies() { return Collections.emptyList(); } + @Override + public Collection getTables() { + return Collections.emptyList(); + } + @Override public int getImplementationVersion() { diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyV1Impl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyV1Impl.java index 6a1cb58763..dafbf2b9c0 100644 --- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyV1Impl.java +++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyV1Impl.java @@ -22,6 +22,7 @@ import java.util.Arrays; import java.util.Collection; +import java.util.Collections; import org.apache.cassandra.db.marshal.BytesType; @@ -33,6 +34,7 @@ import org.apache.usergrid.persistence.core.astyanax.MultiTenantColumnFamily; import org.apache.usergrid.persistence.core.astyanax.MultiTenantColumnFamilyDefinition; import org.apache.usergrid.persistence.core.astyanax.ScopedRowKey; +import org.apache.usergrid.persistence.core.datastax.TableDefinition; import org.apache.usergrid.persistence.model.entity.Id; import org.apache.usergrid.persistence.model.field.Field; @@ -99,6 +101,12 @@ public Collection getColumnFamilies() { return Arrays.asList( uniqueLookupCF, uniqueLogCF ); } + @Override + public Collection getTables() { + + return Collections.emptyList(); + } + @Override protected MultiTenantColumnFamily>, EntityVersion> getUniqueValuesCF() { diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyV2Impl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyV2Impl.java index 40622a4a7a..17b8b37f9a 100644 --- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyV2Impl.java +++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyV2Impl.java @@ -22,6 +22,7 @@ import java.util.Arrays; import java.util.Collection; +import java.util.Collections; import org.apache.cassandra.db.marshal.BytesType; @@ -33,6 +34,7 @@ import org.apache.usergrid.persistence.core.astyanax.MultiTenantColumnFamilyDefinition; import org.apache.usergrid.persistence.core.astyanax.ScopedRowKey; import org.apache.usergrid.persistence.core.astyanax.ScopedRowKeySerializer; +import org.apache.usergrid.persistence.core.datastax.TableDefinition; import org.apache.usergrid.persistence.model.entity.Id; import org.apache.usergrid.persistence.model.field.Field; @@ -99,6 +101,11 @@ public Collection getColumnFamilies() { return Arrays.asList( uniqueLookupCF, uniqueLogCF ); } + @Override + public Collection getTables() { + + return Collections.emptyList(); + } @Override protected MultiTenantColumnFamily, EntityVersion> getUniqueValuesCF() { diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/CQLUtils.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/CQLUtils.java index 7dee9c8e52..f2e4f5799b 100644 --- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/CQLUtils.java +++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/CQLUtils.java @@ -30,7 +30,7 @@ public class CQLUtils { - enum ACTION { + public enum ACTION { CREATE, UPDATE } @@ -84,7 +84,7 @@ public static String getTableCQL(TableDefinition tableDefinition, ACTION tableAc throw new Exception("Invalid Action specified. Must of of type CQLUtils.Action"); } - cql.add( quote( tableDefinition.getTableName() ) ); + cql.add( tableDefinition.getTableName() ); diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/MigrationInfoSerializationImpl.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/MigrationInfoSerializationImpl.java index e02cafee2c..18427f71d3 100644 --- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/MigrationInfoSerializationImpl.java +++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/MigrationInfoSerializationImpl.java @@ -27,6 +27,7 @@ import org.apache.cassandra.db.marshal.UTF8Type; import org.apache.usergrid.persistence.core.astyanax.*; +import org.apache.usergrid.persistence.core.datastax.TableDefinition; import org.apache.usergrid.persistence.core.migration.util.AstayanxUtils; import org.apache.usergrid.persistence.model.entity.Id; import org.apache.usergrid.persistence.model.entity.SimpleId; @@ -205,4 +206,10 @@ public Collection getColumnFamilies() { UTF8Type.class.getSimpleName(), BytesType.class.getSimpleName(), MultiTenantColumnFamilyDefinition.CacheOption.KEYS ) ); } + + @Override + public Collection getTables() { + + return Collections.emptyList(); + } } diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/schema/Migration.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/schema/Migration.java index 3896df249e..952f898db0 100644 --- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/schema/Migration.java +++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/schema/Migration.java @@ -22,6 +22,7 @@ import java.util.Collection; import org.apache.usergrid.persistence.core.astyanax.MultiTenantColumnFamilyDefinition; +import org.apache.usergrid.persistence.core.datastax.TableDefinition; /** @@ -34,5 +35,5 @@ public interface Migration { */ Collection getColumnFamilies(); - //Collection getTables(); + Collection getTables(); } diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/schema/MigrationManagerImpl.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/schema/MigrationManagerImpl.java index 8eb0576918..39a78b7488 100644 --- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/schema/MigrationManagerImpl.java +++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/schema/MigrationManagerImpl.java @@ -26,6 +26,7 @@ import org.apache.usergrid.persistence.core.astyanax.CassandraFig; import org.apache.usergrid.persistence.core.datastax.CQLUtils; import org.apache.usergrid.persistence.core.datastax.DataStaxCluster; +import org.apache.usergrid.persistence.core.datastax.TableDefinition; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -79,17 +80,35 @@ public void migrate() throws MigrationException { final Collection columnFamilies = migration.getColumnFamilies(); + final Collection tables = migration.getTables(); - if ( columnFamilies == null || columnFamilies.size() == 0 ) { + + if ((columnFamilies == null || columnFamilies.size() == 0) && + (tables == null || tables.size() == 0)) { logger.warn( - "Class {} implements {} but returns null column families for migration. Either implement this method or remove the interface from the class", - migration.getClass(), Migration.class ); + "Class {} implements {} but returns null for getColumnFamilies and getTables for migration. Either implement this method or remove the interface from the class", + migration.getClass().getSimpleName(), Migration.class.getSimpleName()); continue; } - for ( MultiTenantColumnFamilyDefinition cf : columnFamilies ) { - testAndCreateColumnFamilyDef( cf ); + if (columnFamilies != null && columnFamilies.isEmpty()) { + for (MultiTenantColumnFamilyDefinition cf : columnFamilies) { + testAndCreateColumnFamilyDef(cf); + } + } + + + if ( tables != null && !tables.isEmpty() ) { + for (TableDefinition tableDefinition : tables) { + + createTable(tableDefinition); + + } } + + + + } } catch ( Throwable t ) { @@ -120,6 +139,20 @@ private void testAndCreateColumnFamilyDef( MultiTenantColumnFamilyDefinition col waitForSchemaAgreement(); } + private void createTable(TableDefinition tableDefinition ) throws Exception { + + logger.info("Creating, if not exists, table: {}", tableDefinition.getTableName()); + String CQL = CQLUtils.getTableCQL( tableDefinition, CQLUtils.ACTION.CREATE ); + logger.info( CQL ); + if (logger.isDebugEnabled()){ + logger.debug( CQL ); + } + dataStaxCluster.getApplicationSession() + .execute( CQL ); + + waitForSchemaAgreement(); + } + /** * Execute CQL to create the keyspace if it does not already exists. Always update the keyspace with the diff --git a/stack/corepersistence/map/src/main/java/org/apache/usergrid/persistence/map/impl/MapSerializationImpl.java b/stack/corepersistence/map/src/main/java/org/apache/usergrid/persistence/map/impl/MapSerializationImpl.java index 5fc6ee1959..958b6f226a 100644 --- a/stack/corepersistence/map/src/main/java/org/apache/usergrid/persistence/map/impl/MapSerializationImpl.java +++ b/stack/corepersistence/map/src/main/java/org/apache/usergrid/persistence/map/impl/MapSerializationImpl.java @@ -21,30 +21,17 @@ import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.UUID; +import java.util.*; import com.datastax.driver.core.*; import com.datastax.driver.core.querybuilder.Clause; import com.datastax.driver.core.querybuilder.QueryBuilder; import com.datastax.driver.core.querybuilder.Using; -import org.apache.cassandra.db.marshal.BytesType; -import org.apache.cassandra.db.marshal.UTF8Type; -import org.apache.usergrid.persistence.core.astyanax.BucketScopedRowKey; -import org.apache.usergrid.persistence.core.astyanax.BucketScopedRowKeySerializer; import org.apache.usergrid.persistence.core.astyanax.CassandraConfig; -import org.apache.usergrid.persistence.core.astyanax.CompositeFieldSerializer; -import org.apache.usergrid.persistence.core.astyanax.MultiTenantColumnFamily; import org.apache.usergrid.persistence.core.astyanax.MultiTenantColumnFamilyDefinition; -import org.apache.usergrid.persistence.core.astyanax.ScopedRowKey; -import org.apache.usergrid.persistence.core.astyanax.ScopedRowKeySerializer; import org.apache.usergrid.persistence.core.datastax.CQLUtils; +import org.apache.usergrid.persistence.core.datastax.TableDefinition; import org.apache.usergrid.persistence.core.shard.ExpandingShardLocator; import org.apache.usergrid.persistence.core.shard.StringHashUtils; import org.apache.usergrid.persistence.map.MapScope; @@ -53,51 +40,40 @@ import com.google.common.hash.Funnel; import com.google.inject.Inject; import com.google.inject.Singleton; -import com.netflix.astyanax.Keyspace; -import com.netflix.astyanax.model.CompositeBuilder; -import com.netflix.astyanax.model.CompositeParser; -import com.netflix.astyanax.serializers.BooleanSerializer; -import com.netflix.astyanax.serializers.StringSerializer; @Singleton public class MapSerializationImpl implements MapSerialization { - private static final String MAP_KEYS_TABLE = CQLUtils.quote("Map_Keys"); - private static final String MAP_ENTRIES_TABLE = CQLUtils.quote("Map_Entries"); - - private static final MapKeySerializer KEY_SERIALIZER = new MapKeySerializer(); - - private static final BucketScopedRowKeySerializer MAP_KEY_SERIALIZER = - new BucketScopedRowKeySerializer<>( KEY_SERIALIZER ); - - private static final MapEntrySerializer ENTRY_SERIALIZER = new MapEntrySerializer(); - private static final ScopedRowKeySerializer MAP_ENTRY_SERIALIZER = - new ScopedRowKeySerializer<>( ENTRY_SERIALIZER ); + private static final String MAP_ENTRIES_TABLE = CQLUtils.quote("Map_Entries"); + private static final Collection MAP_ENTRIES_PARTITION_KEYS = Collections.singletonList("key"); + private static final Collection MAP_ENTRIES_COLUMN_KEYS = Collections.singletonList("column1"); + private static final Map MAP_ENTRIES_COLUMNS = + new HashMap() {{ + put( "key", "blob"); + put( "column1", "blob"); + put( "value", "blob"); }}; + private static final Map MAP_ENTRIES_CLUSTERING_ORDER = + new HashMap(){{ put("column1", "ASC"); }}; + private static final String MAP_KEYS_TABLE = CQLUtils.quote("Map_Keys"); + private static final Collection MAP_KEYS_PARTITION_KEYS = Collections.singletonList("key"); + private static final Collection MAP_KEYS_COLUMN_KEYS = Collections.singletonList("column1"); + private static final Map MAP_KEYS_COLUMNS = + new HashMap() {{ + put( "key", "blob"); + put( "column1", "blob"); + put( "value", "blob"); }}; + private static final Map MAP_KEYS_CLUSTERING_ORDER = + new HashMap(){{ put("column1", "ASC"); }}; - private static final BooleanSerializer BOOLEAN_SERIALIZER = BooleanSerializer.get(); - private static final StringSerializer STRING_SERIALIZER = StringSerializer.get(); private static final StringResultsBuilderCQL STRING_RESULTS_BUILDER_CQL = new StringResultsBuilderCQL(); - /** - * CFs where the row key contains the source node id - */ - public static final MultiTenantColumnFamily, Boolean> MAP_ENTRIES = - new MultiTenantColumnFamily<>( "Map_Entries", MAP_ENTRY_SERIALIZER, BOOLEAN_SERIALIZER ); - - - /** - * CFs where the row key contains the source node id - */ - public static final MultiTenantColumnFamily, String> MAP_KEYS = - new MultiTenantColumnFamily<>( "Map_Keys", MAP_KEY_SERIALIZER, STRING_SERIALIZER ); - /** * Number of buckets to hash across. */ @@ -115,16 +91,13 @@ public class MapSerializationImpl implements MapSerialization { private static final ExpandingShardLocator BUCKET_LOCATOR = new ExpandingShardLocator<>( MAP_KEY_FUNNEL, NUM_BUCKETS ); - private final Keyspace keyspace; private final CassandraConfig cassandraConfig; private final Session session; @Inject - public MapSerializationImpl( final Keyspace keyspace, final CassandraConfig cassandraConfig, - final Session session ) { - this.keyspace = keyspace; + public MapSerializationImpl( final CassandraConfig cassandraConfig, final Session session ) { this.session = session; this.cassandraConfig = cassandraConfig; } @@ -321,17 +294,27 @@ public void delete( final MapScope scope, final String key ) { @Override public Collection getColumnFamilies() { - final MultiTenantColumnFamilyDefinition mapEntries = - new MultiTenantColumnFamilyDefinition( MAP_ENTRIES, BytesType.class.getSimpleName(), - BytesType.class.getSimpleName(), BytesType.class.getSimpleName(), - MultiTenantColumnFamilyDefinition.CacheOption.KEYS ); + // This here only until all traces of Astyanax are removed. + return Collections.emptyList(); + + } + + + @Override + public Collection getTables() { + + final TableDefinition mapEntries = + new TableDefinition( MAP_ENTRIES_TABLE, MAP_ENTRIES_PARTITION_KEYS, MAP_ENTRIES_COLUMN_KEYS, + MAP_ENTRIES_COLUMNS, TableDefinition.CacheOption.KEYS, MAP_ENTRIES_CLUSTERING_ORDER); + + final TableDefinition mapKeys = + new TableDefinition( MAP_KEYS_TABLE, MAP_KEYS_PARTITION_KEYS, MAP_KEYS_COLUMN_KEYS, + MAP_KEYS_COLUMNS, TableDefinition.CacheOption.KEYS, MAP_KEYS_CLUSTERING_ORDER); + - final MultiTenantColumnFamilyDefinition mapKeys = - new MultiTenantColumnFamilyDefinition( MAP_KEYS, BytesType.class.getSimpleName(), - UTF8Type.class.getSimpleName(), BytesType.class.getSimpleName(), - MultiTenantColumnFamilyDefinition.CacheOption.KEYS ); return Arrays.asList( mapEntries, mapKeys ); + } @@ -350,6 +333,7 @@ private ByteBuffer getValueCQL( MapScope scope, String key, final ConsistencyLev + private T getValuesCQL( final MapScope scope, final Collection keys, final ResultsBuilderCQL builder ) { final List serializedKeys = new ArrayList<>(); @@ -368,78 +352,6 @@ private T getValuesCQL( final MapScope scope, final Collection keys, - - /** - * Inner class to serialize and edgeIdTypeKey - */ - private static class MapKeySerializer implements CompositeFieldSerializer { - - - @Override - public void toComposite( final CompositeBuilder builder, final String key ) { - builder.addString( key ); - } - - - @Override - public String fromComposite( final CompositeParser composite ) { - final String key = composite.readString(); - - return key; - } - } - - - /** - * Inner class to serialize and edgeIdTypeKey - */ - private static class MapEntrySerializer implements CompositeFieldSerializer { - - @Override - public void toComposite( final CompositeBuilder builder, final MapEntryKey key ) { - - builder.addString( key.mapName ); - builder.addString( key.key ); - } - - - @Override - public MapEntryKey fromComposite( final CompositeParser composite ) { - - final String mapName = composite.readString(); - - final String entryKey = composite.readString(); - - return new MapEntryKey( mapName, entryKey ); - } - } - - - /** - * Entries for serializing map entries and keys to a row - */ - private static class MapEntryKey { - public final String mapName; - public final String key; - - - private MapEntryKey( final String mapName, final String key ) { - this.mapName = mapName; - this.key = key; - } - - - /** - * Create a scoped row key from the key - */ - public static ScopedRowKey fromKey( final MapScope mapScope, final String key ) { - - return ScopedRowKey.fromKey( mapScope.getApplication(), new MapEntryKey( mapScope.getName(), key ) ); - } - } - - - /** * Build the results from the row keys */ diff --git a/stack/corepersistence/map/src/test/java/org/apache/usergrid/persistence/map/MapManagerTest.java b/stack/corepersistence/map/src/test/java/org/apache/usergrid/persistence/map/MapManagerTest.java index 2a68247229..bd109d13c9 100644 --- a/stack/corepersistence/map/src/test/java/org/apache/usergrid/persistence/map/MapManagerTest.java +++ b/stack/corepersistence/map/src/test/java/org/apache/usergrid/persistence/map/MapManagerTest.java @@ -27,6 +27,7 @@ import java.util.concurrent.TimeUnit; import org.junit.Before; +import org.junit.ClassRule; import org.junit.Rule; import org.junit.Test; import org.junit.runner.RunWith; From df7d2ff856d71b1f1b999319c9865ea540d4412a Mon Sep 17 00:00:00 2001 From: Michael Russo Date: Fri, 12 Feb 2016 16:35:36 -0800 Subject: [PATCH 10/30] Remove unused import. --- .../org/apache/usergrid/persistence/map/MapManagerTest.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/stack/corepersistence/map/src/test/java/org/apache/usergrid/persistence/map/MapManagerTest.java b/stack/corepersistence/map/src/test/java/org/apache/usergrid/persistence/map/MapManagerTest.java index bd109d13c9..1ce0a4033c 100644 --- a/stack/corepersistence/map/src/test/java/org/apache/usergrid/persistence/map/MapManagerTest.java +++ b/stack/corepersistence/map/src/test/java/org/apache/usergrid/persistence/map/MapManagerTest.java @@ -27,8 +27,7 @@ import java.util.concurrent.TimeUnit; import org.junit.Before; -import org.junit.ClassRule; -import org.junit.Rule; +\import org.junit.Rule; import org.junit.Test; import org.junit.runner.RunWith; From 29814ef50f47254e45371f355c1c1065ebf1aac4 Mon Sep 17 00:00:00 2001 From: Michael Russo Date: Fri, 12 Feb 2016 19:15:33 -0800 Subject: [PATCH 11/30] Fix issues with adding getTables() to Migration interface. --- .../cache/impl/ScopedCacheSerializationImpl.java | 10 +++++++++- .../impl/UniqueValueSerializationStrategyImpl.java | 14 ++++++++------ .../core/datastax/impl/DatastaxClusterImpl.java | 2 -- .../migration/schema/MigrationManagerImpl.java | 5 +---- .../impl/EdgeMetadataSerializationProxyImpl.java | 6 ++++++ .../impl/EdgeMetadataSerializationV1Impl.java | 8 ++++++++ .../impl/EdgeMetadataSerializationV2Impl.java | 14 ++++++++------ .../serialization/impl/NodeSerializationImpl.java | 7 +++++++ .../shard/impl/EdgeShardSerializationImpl.java | 7 +++++++ .../shard/impl/SizebasedEdgeColumnFamilies.java | 8 ++++++++ .../usergrid/persistence/map/MapManagerTest.java | 2 +- 11 files changed, 63 insertions(+), 20 deletions(-) diff --git a/stack/corepersistence/cache/src/main/java/org/apache/usergrid/persistence/cache/impl/ScopedCacheSerializationImpl.java b/stack/corepersistence/cache/src/main/java/org/apache/usergrid/persistence/cache/impl/ScopedCacheSerializationImpl.java index 1f763b278d..cf835c828e 100644 --- a/stack/corepersistence/cache/src/main/java/org/apache/usergrid/persistence/cache/impl/ScopedCacheSerializationImpl.java +++ b/stack/corepersistence/cache/src/main/java/org/apache/usergrid/persistence/cache/impl/ScopedCacheSerializationImpl.java @@ -39,6 +39,7 @@ import org.apache.cassandra.db.marshal.BytesType; import org.apache.usergrid.persistence.cache.CacheScope; import org.apache.usergrid.persistence.core.astyanax.*; +import org.apache.usergrid.persistence.core.datastax.TableDefinition; import org.apache.usergrid.persistence.core.shard.ExpandingShardLocator; import org.apache.usergrid.persistence.core.shard.StringHashUtils; import org.slf4j.Logger; @@ -47,6 +48,7 @@ import java.io.IOException; import java.util.Arrays; import java.util.Collection; +import java.util.Collections; import java.util.concurrent.Callable; @@ -284,7 +286,13 @@ public Collection getColumnFamilies() { BytesType.class.getSimpleName(), MultiTenantColumnFamilyDefinition.CacheOption.KEYS ); - return Arrays.asList(scopedCache); + return Collections.singletonList(scopedCache); + } + + @Override + public Collection getTables() { + + return Collections.emptyList(); } diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyImpl.java index 1795511efc..197fd4ed9c 100644 --- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyImpl.java +++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyImpl.java @@ -18,13 +18,9 @@ package org.apache.usergrid.persistence.collection.serialization.impl; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Iterator; -import java.util.List; -import java.util.UUID; +import java.util.*; +import org.apache.usergrid.persistence.core.datastax.TableDefinition; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -379,6 +375,12 @@ public Collection getColumnFamilies() { return Arrays.asList( uniqueLookupCF, uniqueLogCF ); } + @Override + public Collection getTables() { + + return Collections.emptyList(); + } + /** * Get the column family for the unique fields diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/impl/DatastaxClusterImpl.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/impl/DatastaxClusterImpl.java index 5fc804b846..43e2eb204d 100644 --- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/impl/DatastaxClusterImpl.java +++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/impl/DatastaxClusterImpl.java @@ -150,8 +150,6 @@ private void createOrUpdateKeyspace() throws Exception { logger.info("Updating application keyspace with the following CQL: {}", updateApplicationKeyspace); clusterSession.executeAsync(updateApplicationKeyspace); - // this session pool is only used when running database setup so close it when finished to clear resources - clusterSession.close(); waitForSchemaAgreement(); } diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/schema/MigrationManagerImpl.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/schema/MigrationManagerImpl.java index 39a78b7488..d746482f2b 100644 --- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/schema/MigrationManagerImpl.java +++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/schema/MigrationManagerImpl.java @@ -91,7 +91,7 @@ public void migrate() throws MigrationException { continue; } - if (columnFamilies != null && columnFamilies.isEmpty()) { + if (columnFamilies != null && !columnFamilies.isEmpty()) { for (MultiTenantColumnFamilyDefinition cf : columnFamilies) { testAndCreateColumnFamilyDef(cf); } @@ -182,9 +182,6 @@ private void createOrUpdateKeyspace() throws Exception { logger.info("Updating application keyspace with the following CQL: {}", updateApplicationKeyspace); clusterSession.executeAsync(updateApplicationKeyspace); - // this session pool is only used when running database setup so close it when finished to clear resources - clusterSession.close(); - waitForSchemaAgreement(); } diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgeMetadataSerializationProxyImpl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgeMetadataSerializationProxyImpl.java index 44d0b73725..46acd1c430 100644 --- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgeMetadataSerializationProxyImpl.java +++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgeMetadataSerializationProxyImpl.java @@ -26,6 +26,7 @@ import java.util.Collections; import java.util.Iterator; +import org.apache.usergrid.persistence.core.datastax.TableDefinition; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -288,6 +289,11 @@ public Collection getColumnFamilies() { return Collections.EMPTY_LIST; } + @Override + public Collection getTables() { + + return Collections.emptyList(); + } diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgeMetadataSerializationV1Impl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgeMetadataSerializationV1Impl.java index da4d044265..870ba164eb 100644 --- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgeMetadataSerializationV1Impl.java +++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgeMetadataSerializationV1Impl.java @@ -22,6 +22,7 @@ import java.util.Arrays; import java.util.Collection; +import java.util.Collections; import java.util.Iterator; import org.apache.cassandra.db.marshal.BytesType; @@ -36,6 +37,7 @@ import org.apache.usergrid.persistence.core.astyanax.ScopedRowKeySerializer; import org.apache.usergrid.persistence.core.astyanax.ScopedRowKey; import org.apache.usergrid.persistence.core.astyanax.StringColumnParser; +import org.apache.usergrid.persistence.core.datastax.TableDefinition; import org.apache.usergrid.persistence.core.migration.schema.Migration; import org.apache.usergrid.persistence.core.scope.ApplicationScope; import org.apache.usergrid.persistence.core.util.ValidationUtils; @@ -376,6 +378,12 @@ public Collection getColumnFamilies() { graphCf( CF_SOURCE_EDGE_ID_TYPES ), graphCf( CF_TARGET_EDGE_ID_TYPES ) ); } + @Override + public Collection getTables() { + + return Collections.emptyList(); + } + /** * Helper to generate an edge definition by the type diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgeMetadataSerializationV2Impl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgeMetadataSerializationV2Impl.java index 2af62a89b6..1aa86cc2d3 100644 --- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgeMetadataSerializationV2Impl.java +++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgeMetadataSerializationV2Impl.java @@ -22,12 +22,7 @@ package org.apache.usergrid.persistence.graph.serialization.impl; -import java.util.Arrays; -import java.util.Collection; -import java.util.Comparator; -import java.util.Iterator; -import java.util.List; -import java.util.UUID; +import java.util.*; import org.apache.cassandra.db.marshal.BytesType; import org.apache.cassandra.db.marshal.UTF8Type; @@ -42,6 +37,7 @@ import org.apache.usergrid.persistence.core.astyanax.MultiTenantColumnFamily; import org.apache.usergrid.persistence.core.astyanax.MultiTenantColumnFamilyDefinition; import org.apache.usergrid.persistence.core.astyanax.StringColumnParser; +import org.apache.usergrid.persistence.core.datastax.TableDefinition; import org.apache.usergrid.persistence.core.migration.schema.Migration; import org.apache.usergrid.persistence.core.scope.ApplicationScope; import org.apache.usergrid.persistence.core.shard.ExpandingShardLocator; @@ -470,6 +466,12 @@ public Collection getColumnFamilies() { graphCf( CF_SOURCE_EDGE_ID_TYPES ), graphCf( CF_TARGET_EDGE_ID_TYPES ) ); } + @Override + public Collection getTables() { + + return Collections.emptyList(); + } + /** * Helper to generate an edge definition by the type diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/NodeSerializationImpl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/NodeSerializationImpl.java index cd803e8e66..8a2da5446f 100644 --- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/NodeSerializationImpl.java +++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/NodeSerializationImpl.java @@ -38,6 +38,7 @@ import org.apache.usergrid.persistence.core.astyanax.MultiTenantColumnFamilyDefinition; import org.apache.usergrid.persistence.core.astyanax.ScopedRowKeySerializer; import org.apache.usergrid.persistence.core.astyanax.ScopedRowKey; +import org.apache.usergrid.persistence.core.datastax.TableDefinition; import org.apache.usergrid.persistence.core.migration.schema.Migration; import org.apache.usergrid.persistence.core.scope.ApplicationScope; import org.apache.usergrid.persistence.core.util.ValidationUtils; @@ -110,6 +111,12 @@ public Collection getColumnFamilies() { MultiTenantColumnFamilyDefinition.CacheOption.ALL ) ); } + @Override + public Collection getTables() { + + return Collections.emptyList(); + } + @Override public MutationBatch mark( final ApplicationScope scope, final Id node, final long timestamp ) { diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/EdgeShardSerializationImpl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/EdgeShardSerializationImpl.java index 120a15ce4b..87309500c7 100644 --- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/EdgeShardSerializationImpl.java +++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/EdgeShardSerializationImpl.java @@ -34,6 +34,7 @@ import org.apache.usergrid.persistence.core.astyanax.MultiTenantColumnFamilyDefinition; import org.apache.usergrid.persistence.core.astyanax.ScopedRowKeySerializer; import org.apache.usergrid.persistence.core.astyanax.ScopedRowKey; +import org.apache.usergrid.persistence.core.datastax.TableDefinition; import org.apache.usergrid.persistence.core.scope.ApplicationScope; import org.apache.usergrid.persistence.core.util.ValidationUtils; import org.apache.usergrid.persistence.graph.GraphFig; @@ -173,6 +174,12 @@ public Collection getColumnFamilies() { MultiTenantColumnFamilyDefinition.CacheOption.KEYS ) ); } + @Override + public Collection getTables() { + + return Collections.emptyList(); + } + diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/SizebasedEdgeColumnFamilies.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/SizebasedEdgeColumnFamilies.java index 9185ac88b1..f4e19d6f18 100644 --- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/SizebasedEdgeColumnFamilies.java +++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/SizebasedEdgeColumnFamilies.java @@ -23,6 +23,7 @@ import java.util.Arrays; import java.util.Collection; +import java.util.Collections; import org.apache.cassandra.db.marshal.BytesType; import org.apache.cassandra.db.marshal.DynamicCompositeType; @@ -32,6 +33,7 @@ import org.apache.usergrid.persistence.core.astyanax.MultiTenantColumnFamilyDefinition; import org.apache.usergrid.persistence.core.astyanax.ScopedRowKeySerializer; import org.apache.usergrid.persistence.core.astyanax.ScopedRowKey; +import org.apache.usergrid.persistence.core.datastax.TableDefinition; import org.apache.usergrid.persistence.graph.serialization.impl.shard.DirectedEdge; import org.apache.usergrid.persistence.graph.serialization.impl.shard.EdgeColumnFamilies; import org.apache.usergrid.persistence.graph.serialization.impl.shard.EdgeRowKey; @@ -143,6 +145,12 @@ public Collection getColumnFamilies() { MultiTenantColumnFamilyDefinition.CacheOption.ALL ) ); } + @Override + public Collection getTables() { + + return Collections.emptyList(); + } + /** * Helper to generate an edge definition by the type diff --git a/stack/corepersistence/map/src/test/java/org/apache/usergrid/persistence/map/MapManagerTest.java b/stack/corepersistence/map/src/test/java/org/apache/usergrid/persistence/map/MapManagerTest.java index 1ce0a4033c..2a68247229 100644 --- a/stack/corepersistence/map/src/test/java/org/apache/usergrid/persistence/map/MapManagerTest.java +++ b/stack/corepersistence/map/src/test/java/org/apache/usergrid/persistence/map/MapManagerTest.java @@ -27,7 +27,7 @@ import java.util.concurrent.TimeUnit; import org.junit.Before; -\import org.junit.Rule; +import org.junit.Rule; import org.junit.Test; import org.junit.runner.RunWith; From 7ff31ebff0cb431c9655764e922d94d7f215a673 Mon Sep 17 00:00:00 2001 From: Michael Russo Date: Fri, 12 Feb 2016 21:45:08 -0800 Subject: [PATCH 12/30] Add tests for DataStaxCluster and improve cluster/connection handling to ensure the cluster is always available through the interface. --- .../persistence/core/datastax/CQLUtils.java | 4 +- .../core/datastax/DataStaxCluster.java | 2 + .../core/datastax/TableDefinition.java | 7 +- .../datastax/impl/DatastaxClusterImpl.java | 138 ++++++++++-------- .../schema/MigrationManagerImpl.java | 66 +-------- .../core/datastax/CQLUtilsTest.java | 30 ++-- .../core/datastax/DatastaxClusterTest.java | 63 +++++++- .../map/impl/MapSerializationImpl.java | 25 ++-- 8 files changed, 186 insertions(+), 149 deletions(-) diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/CQLUtils.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/CQLUtils.java index f2e4f5799b..38b6b7bc0c 100644 --- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/CQLUtils.java +++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/CQLUtils.java @@ -134,10 +134,10 @@ public static String quote( String value){ } - public static String spaceSeparatedKeyValue(Map columns){ + public static String spaceSeparatedKeyValue(Map columns){ StringJoiner columnsSchema = new StringJoiner(","); - columns.forEach( (key, value) -> columnsSchema.add(key+" "+value)); + columns.forEach( (key, value) -> columnsSchema.add(key+" "+String.valueOf(value))); return columnsSchema.toString(); diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/DataStaxCluster.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/DataStaxCluster.java index 7515bf5bb0..206c2a03c0 100644 --- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/DataStaxCluster.java +++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/DataStaxCluster.java @@ -30,4 +30,6 @@ public interface DataStaxCluster { Session getApplicationSession(); + void waitForSchemaAgreement(); + } diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/TableDefinition.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/TableDefinition.java index 3dc3145d45..2f83513329 100644 --- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/TableDefinition.java +++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/TableDefinition.java @@ -20,6 +20,7 @@ package org.apache.usergrid.persistence.core.datastax; +import com.datastax.driver.core.DataType; import com.google.common.base.Preconditions; import java.util.Collection; @@ -53,7 +54,7 @@ public String getValue() { private final String tableName; private final Collection partitionKeys; private final Collection columnKeys; - private final Map columns; + private final Map columns; private final CacheOption cacheOption; private final Map compaction; private final String bloomFilterChance; @@ -63,7 +64,7 @@ public String getValue() { private final Map clusteringOrder; public TableDefinition( final String tableName, final Collection partitionKeys, - final Collection columnKeys, final Map columns, + final Collection columnKeys, final Map columns, final CacheOption cacheOption, final Map clusteringOrder){ Preconditions.checkNotNull(tableName, "Table name cannot be null"); @@ -105,7 +106,7 @@ public Collection getColumnKeys() { return columnKeys; } - public Map getColumns() { + public Map getColumns() { return columns; } diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/impl/DatastaxClusterImpl.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/impl/DatastaxClusterImpl.java index 43e2eb204d..1e9061f5c3 100644 --- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/impl/DatastaxClusterImpl.java +++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/impl/DatastaxClusterImpl.java @@ -37,63 +37,15 @@ public class DataStaxClusterImpl implements DataStaxCluster { private final CassandraFig cassandraFig; - private final Cluster cluster; + private Cluster cluster; private Session applicationSession; private Session clusterSession; @Inject public DataStaxClusterImpl(final CassandraFig cassandraFig ) throws Exception { this.cassandraFig = cassandraFig; + this.cluster = buildCluster(); - ConsistencyLevel defaultConsistencyLevel; - try { - defaultConsistencyLevel = ConsistencyLevel.valueOf(cassandraFig.getReadCl()); - } catch (IllegalArgumentException e){ - - logger.error("Unable to parse provided consistency level in property: {}, defaulting to: {}", - CassandraFig.READ_CL, - ConsistencyLevel.LOCAL_QUORUM); - - defaultConsistencyLevel = ConsistencyLevel.LOCAL_QUORUM; - } - - - LoadBalancingPolicy loadBalancingPolicy; - if( !cassandraFig.getLocalDataCenter().isEmpty() ){ - - loadBalancingPolicy = new DCAwareRoundRobinPolicy.Builder() - .withLocalDc( cassandraFig.getLocalDataCenter() ).build(); - }else{ - loadBalancingPolicy = new DCAwareRoundRobinPolicy.Builder().build(); - } - - final PoolingOptions poolingOptions = new PoolingOptions() - .setCoreConnectionsPerHost(HostDistance.LOCAL, cassandraFig.getConnections() / 2) - .setMaxConnectionsPerHost(HostDistance.LOCAL, cassandraFig.getConnections()) - .setIdleTimeoutSeconds(cassandraFig.getTimeout() / 1000) - .setPoolTimeoutMillis(cassandraFig.getPoolTimeout()); - - final QueryOptions queryOptions = new QueryOptions() - .setConsistencyLevel(defaultConsistencyLevel); - - final Cluster.Builder datastaxCluster = Cluster.builder() - .withClusterName(cassandraFig.getClusterName()) - .addContactPoints(cassandraFig.getHosts().split(",")) - .withCompression(ProtocolOptions.Compression.LZ4) - .withLoadBalancingPolicy(loadBalancingPolicy) - .withPoolingOptions(poolingOptions) - .withQueryOptions(queryOptions) - .withProtocolVersion(ProtocolVersion.NEWEST_SUPPORTED); - - // only add auth credentials if they were provided - if ( !cassandraFig.getUsername().isEmpty() && !cassandraFig.getPassword().isEmpty() ){ - datastaxCluster.withCredentials( - cassandraFig.getUsername(), - cassandraFig.getPassword() - ); - } - - this.cluster = datastaxCluster.build(); logger.info("Initialized datastax cluster client. Hosts={}, Idle Timeout={}s, Pool Timeout={}s", cluster.getMetadata().getAllHosts().toString(), cluster.getConfiguration().getPoolingOptions().getIdleTimeoutSeconds(), @@ -106,14 +58,20 @@ public DataStaxClusterImpl(final CassandraFig cassandraFig ) throws Exception { @Override public Cluster getCluster(){ + // ensure we can build the cluster if it was previously closed + if ( cluster.isClosed() ){ + cluster = buildCluster(); + } + return cluster; } @Override public Session getClusterSession(){ + // always grab cluster from getCluster() in case it was prematurely closed if ( clusterSession == null || clusterSession.isClosed() ){ - clusterSession = cluster.connect(); + clusterSession = getCluster().connect(); } return clusterSession; @@ -122,34 +80,39 @@ public Session getClusterSession(){ @Override public Session getApplicationSession(){ + // always grab cluster from getCluster() in case it was prematurely closed if ( applicationSession == null || applicationSession.isClosed() ){ - applicationSession = cluster.connect( CQLUtils.quote(cassandraFig.getApplicationKeyspace() ) ); + applicationSession = getCluster().connect( CQLUtils.quote(cassandraFig.getApplicationKeyspace() ) ); } return applicationSession; } + + /** + * Execute CQL that will create the keyspace if it doesn't exist and alter it if it does. + * @throws Exception + */ private void createOrUpdateKeyspace() throws Exception { clusterSession = getClusterSession(); final String createApplicationKeyspace = String.format( - "CREATE KEYSPACE IF NOT EXISTS \"%s\" WITH replication = %s", - cassandraFig.getApplicationKeyspace(), + "CREATE KEYSPACE IF NOT EXISTS %s WITH replication = %s", + CQLUtils.quote(cassandraFig.getApplicationKeyspace()), CQLUtils.getFormattedReplication( cassandraFig.getStrategy(), cassandraFig.getStrategyOptions() ) ); final String updateApplicationKeyspace = String.format( - "ALTER KEYSPACE \"%s\" WITH replication = %s", - cassandraFig.getApplicationKeyspace(), + "ALTER KEYSPACE %s WITH replication = %s", + CQLUtils.quote(cassandraFig.getApplicationKeyspace()), CQLUtils.getFormattedReplication( cassandraFig.getStrategy(), cassandraFig.getStrategyOptions() ) ); - logger.info("Creating application keyspace with the following CQL: {}", createApplicationKeyspace); clusterSession.execute(createApplicationKeyspace); - logger.info("Updating application keyspace with the following CQL: {}", updateApplicationKeyspace); clusterSession.executeAsync(updateApplicationKeyspace); + logger.info("Created/Updated keyspace: {}", cassandraFig.getApplicationKeyspace()); waitForSchemaAgreement(); } @@ -158,11 +121,11 @@ private void createOrUpdateKeyspace() throws Exception { * Wait until all Cassandra nodes agree on the schema. Sleeps 100ms between checks. * */ - private void waitForSchemaAgreement() { + public void waitForSchemaAgreement() { while ( true ) { - if( cluster.getMetadata().checkSchemaAgreement() ){ + if( this.cluster.getMetadata().checkSchemaAgreement() ){ return; } @@ -176,4 +139,59 @@ private void waitForSchemaAgreement() { } } + public Cluster buildCluster(){ + + ConsistencyLevel defaultConsistencyLevel; + try { + defaultConsistencyLevel = ConsistencyLevel.valueOf(cassandraFig.getReadCl()); + } catch (IllegalArgumentException e){ + + logger.error("Unable to parse provided consistency level in property: {}, defaulting to: {}", + CassandraFig.READ_CL, + ConsistencyLevel.LOCAL_QUORUM); + + defaultConsistencyLevel = ConsistencyLevel.LOCAL_QUORUM; + } + + + LoadBalancingPolicy loadBalancingPolicy; + if( !cassandraFig.getLocalDataCenter().isEmpty() ){ + + loadBalancingPolicy = new DCAwareRoundRobinPolicy.Builder() + .withLocalDc( cassandraFig.getLocalDataCenter() ).build(); + }else{ + loadBalancingPolicy = new DCAwareRoundRobinPolicy.Builder().build(); + } + + final PoolingOptions poolingOptions = new PoolingOptions() + .setCoreConnectionsPerHost(HostDistance.LOCAL, cassandraFig.getConnections() / 2) + .setMaxConnectionsPerHost(HostDistance.LOCAL, cassandraFig.getConnections()) + .setIdleTimeoutSeconds(cassandraFig.getTimeout() / 1000) + .setPoolTimeoutMillis(cassandraFig.getPoolTimeout()); + + final QueryOptions queryOptions = new QueryOptions() + .setConsistencyLevel(defaultConsistencyLevel); + + Cluster.Builder datastaxCluster = Cluster.builder() + .withClusterName(cassandraFig.getClusterName()) + .addContactPoints(cassandraFig.getHosts().split(",")) + .withCompression(ProtocolOptions.Compression.LZ4) + .withLoadBalancingPolicy(loadBalancingPolicy) + .withPoolingOptions(poolingOptions) + .withQueryOptions(queryOptions) + .withProtocolVersion(ProtocolVersion.NEWEST_SUPPORTED); + + // only add auth credentials if they were provided + if ( !cassandraFig.getUsername().isEmpty() && !cassandraFig.getPassword().isEmpty() ){ + datastaxCluster.withCredentials( + cassandraFig.getUsername(), + cassandraFig.getPassword() + ); + } + + + return datastaxCluster.build(); + + } + } diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/schema/MigrationManagerImpl.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/schema/MigrationManagerImpl.java index d746482f2b..105f93a48d 100644 --- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/schema/MigrationManagerImpl.java +++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/schema/MigrationManagerImpl.java @@ -41,7 +41,7 @@ /** - * Implementation of the migration manager to set up keyspace + * Implementation of the migration manager to set up column families / tables * * @author tnine */ @@ -74,8 +74,6 @@ public void migrate() throws MigrationException { try { - createOrUpdateKeyspace(); - for ( Migration migration : migrations ) { final Collection columnFamilies = migration.getColumnFamilies(); @@ -107,8 +105,6 @@ public void migrate() throws MigrationException { } - - } } catch ( Throwable t ) { @@ -136,77 +132,23 @@ private void testAndCreateColumnFamilyDef( MultiTenantColumnFamilyDefinition col logger.info( "Created column family {}", columnFamily.getColumnFamily().getName() ); - waitForSchemaAgreement(); + dataStaxCluster.waitForSchemaAgreement(); } private void createTable(TableDefinition tableDefinition ) throws Exception { - logger.info("Creating, if not exists, table: {}", tableDefinition.getTableName()); String CQL = CQLUtils.getTableCQL( tableDefinition, CQLUtils.ACTION.CREATE ); - logger.info( CQL ); if (logger.isDebugEnabled()){ logger.debug( CQL ); } dataStaxCluster.getApplicationSession() .execute( CQL ); - waitForSchemaAgreement(); - } - - - /** - * Execute CQL to create the keyspace if it does not already exists. Always update the keyspace with the - * configured strategy options to allow for real time replication updates. - * - * @throws Exception - */ - private void createOrUpdateKeyspace() throws Exception { - - Session clusterSession = dataStaxCluster.getClusterSession(); - - final String createApplicationKeyspace = String.format( - "CREATE KEYSPACE IF NOT EXISTS \"%s\" WITH replication = %s", - cassandraFig.getApplicationKeyspace(), - CQLUtils.getFormattedReplication( cassandraFig.getStrategy(), cassandraFig.getStrategyOptions() ) - - ); - - final String updateApplicationKeyspace = String.format( - "ALTER KEYSPACE \"%s\" WITH replication = %s", - cassandraFig.getApplicationKeyspace(), - CQLUtils.getFormattedReplication( cassandraFig.getStrategy(), cassandraFig.getStrategyOptions() ) - ); + logger.info("Created table: {}", tableDefinition.getTableName()); - logger.info("Creating application keyspace with the following CQL: {}", createApplicationKeyspace); - clusterSession.execute(createApplicationKeyspace); - logger.info("Updating application keyspace with the following CQL: {}", updateApplicationKeyspace); - clusterSession.executeAsync(updateApplicationKeyspace); - - waitForSchemaAgreement(); + dataStaxCluster.waitForSchemaAgreement(); } - /** - * Wait until all Cassandra nodes agree on the schema. Sleeps 100ms between checks. - * - */ - private void waitForSchemaAgreement() { - - while ( true ) { - - if( dataStaxCluster.getCluster().getMetadata().checkSchemaAgreement() ){ - return; - } - - //sleep and try it again - try { - Thread.sleep( 100 ); - } - catch ( InterruptedException e ) { - //swallow - } - } - } - } diff --git a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/datastax/CQLUtilsTest.java b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/datastax/CQLUtilsTest.java index 76fcefea0d..c47312dc19 100644 --- a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/datastax/CQLUtilsTest.java +++ b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/datastax/CQLUtilsTest.java @@ -19,6 +19,7 @@ package org.apache.usergrid.persistence.core.datastax; +import com.datastax.driver.core.DataType; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -38,10 +39,10 @@ public class CQLUtilsTest { public void testTableCQL() throws Exception { - Map columns = new HashMap<>(); - columns.put("key", "blob"); - columns.put("column1", "text"); - columns.put("value", "blob"); + Map columns = new HashMap<>(); + columns.put("key", DataType.Name.BLOB); + columns.put("column1", DataType.Name.TEXT); + columns.put("value", DataType.Name.BLOB); List partitionKeys = new ArrayList<>(); partitionKeys.add("key"); @@ -55,7 +56,7 @@ public void testTableCQL() throws Exception { TableDefinition table1 = new TableDefinition( - "table1", + CQLUtils.quote("table1"), partitionKeys, columnKeys, columns, @@ -66,10 +67,21 @@ public void testTableCQL() throws Exception { String createCQL = CQLUtils.getTableCQL(table1, CQLUtils.ACTION.CREATE); String updateCQL = CQLUtils.getTableCQL(table1, CQLUtils.ACTION.UPDATE); - assertTrue( createCQL.contains( CQLUtils.CREATE_TABLE ) && !createCQL.contains( CQLUtils.ALTER_TABLE ) ); - assertTrue( updateCQL.contains( CQLUtils.ALTER_TABLE ) && !updateCQL.contains( CQLUtils.CREATE_TABLE ) ); - logger.info("CREATE: {}", createCQL); - logger.info("UPDATE: {}", updateCQL); + assertTrue( + createCQL.contains(CQLUtils.CREATE_TABLE ) && + !createCQL.contains( CQLUtils.ALTER_TABLE ) && + createCQL.contains( DataType.Name.BLOB.toString() ) && + createCQL.contains( DataType.Name.TEXT.toString() ) + + ); + assertTrue( + updateCQL.contains( CQLUtils.ALTER_TABLE ) && + !updateCQL.contains( CQLUtils.CREATE_TABLE ) && + !updateCQL.contains( DataType.Name.BLOB.toString() ) && + !updateCQL.contains( DataType.Name.TEXT.toString() ) + ); + logger.info(createCQL); + logger.info(updateCQL); } diff --git a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/datastax/DatastaxClusterTest.java b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/datastax/DatastaxClusterTest.java index 5da23ce92d..81b7d8fe1b 100644 --- a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/datastax/DatastaxClusterTest.java +++ b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/datastax/DatastaxClusterTest.java @@ -18,7 +18,68 @@ */ package org.apache.usergrid.persistence.core.datastax; +import com.datastax.driver.core.Cluster; +import com.datastax.driver.core.Session; +import com.google.inject.Inject; +import org.apache.usergrid.persistence.core.astyanax.CassandraFig; +import org.apache.usergrid.persistence.core.guice.TestCommonModule; +import org.apache.usergrid.persistence.core.test.ITRunner; +import org.apache.usergrid.persistence.core.test.UseModules; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; + +import static org.junit.Assert.*; + +@RunWith( ITRunner.class ) +@UseModules( TestCommonModule.class ) public class DatastaxClusterTest { - //TODO + + @Inject + DataStaxCluster dataStaxCluster; + + @Inject + CassandraFig cassandraFig; + + + @Test + public void testConnectCloseCluster() { + + Cluster cluster = dataStaxCluster.getCluster(); + + assertTrue(!cluster.isClosed()); + + cluster.close(); + assertTrue(cluster.isClosed()); + + // validate getCluster will re-init the cluster + cluster = dataStaxCluster.getCluster(); + assertTrue(!cluster.isClosed()); + + + } + + @Test + public void testGetClusterSession() { + + Session session = dataStaxCluster.getClusterSession(); + String clusterName = session.getCluster().getClusterName(); + String keyspaceName = session.getLoggedKeyspace(); + + // cluster session is not logged to a keyspace + assertNull(keyspaceName); + assertNotNull(clusterName); + } + + @Test + public void testGetApplicationSession() { + + Session session = dataStaxCluster.getApplicationSession(); + String keyspaceName = session.getLoggedKeyspace(); + + + assertEquals(cassandraFig.getApplicationKeyspace(), keyspaceName); + } + } diff --git a/stack/corepersistence/map/src/main/java/org/apache/usergrid/persistence/map/impl/MapSerializationImpl.java b/stack/corepersistence/map/src/main/java/org/apache/usergrid/persistence/map/impl/MapSerializationImpl.java index 958b6f226a..e2ea681e65 100644 --- a/stack/corepersistence/map/src/main/java/org/apache/usergrid/persistence/map/impl/MapSerializationImpl.java +++ b/stack/corepersistence/map/src/main/java/org/apache/usergrid/persistence/map/impl/MapSerializationImpl.java @@ -49,24 +49,25 @@ public class MapSerializationImpl implements MapSerialization { private static final String MAP_ENTRIES_TABLE = CQLUtils.quote("Map_Entries"); private static final Collection MAP_ENTRIES_PARTITION_KEYS = Collections.singletonList("key"); private static final Collection MAP_ENTRIES_COLUMN_KEYS = Collections.singletonList("column1"); - private static final Map MAP_ENTRIES_COLUMNS = - new HashMap() {{ - put( "key", "blob"); - put( "column1", "blob"); - put( "value", "blob"); }}; + private static final Map MAP_ENTRIES_COLUMNS = + new HashMap() {{ + put( "key", DataType.Name.BLOB ); + put( "column1", DataType.Name.BLOB ); + put( "value", DataType.Name.BLOB ); }}; private static final Map MAP_ENTRIES_CLUSTERING_ORDER = - new HashMap(){{ put("column1", "ASC"); }}; + new HashMap(){{ put( "column1", "ASC" ); }}; + private static final String MAP_KEYS_TABLE = CQLUtils.quote("Map_Keys"); private static final Collection MAP_KEYS_PARTITION_KEYS = Collections.singletonList("key"); private static final Collection MAP_KEYS_COLUMN_KEYS = Collections.singletonList("column1"); - private static final Map MAP_KEYS_COLUMNS = - new HashMap() {{ - put( "key", "blob"); - put( "column1", "blob"); - put( "value", "blob"); }}; + private static final Map MAP_KEYS_COLUMNS = + new HashMap() {{ + put( "key", DataType.Name.BLOB ); + put( "column1", DataType.Name.BLOB ); + put( "value", DataType.Name.BLOB ); }}; private static final Map MAP_KEYS_CLUSTERING_ORDER = - new HashMap(){{ put("column1", "ASC"); }}; + new HashMap(){{ put( "column1", "ASC" ); }}; From ecbd495970b73e0c1ed85fa6491c906cac33de6e Mon Sep 17 00:00:00 2001 From: Michael Russo Date: Fri, 12 Feb 2016 22:48:09 -0800 Subject: [PATCH 13/30] Expose creation of keyspace via the interface so other classes can use that method to create the keyspace if for some reason it does not already exist. --- .../usergrid/persistence/core/datastax/DataStaxCluster.java | 2 ++ .../persistence/core/datastax/impl/DatastaxClusterImpl.java | 4 +++- .../core/migration/schema/MigrationManagerImpl.java | 3 ++- .../persistence/core/datastax/DatastaxClusterTest.java | 2 +- 4 files changed, 8 insertions(+), 3 deletions(-) diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/DataStaxCluster.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/DataStaxCluster.java index 206c2a03c0..768a7a27b5 100644 --- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/DataStaxCluster.java +++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/DataStaxCluster.java @@ -30,6 +30,8 @@ public interface DataStaxCluster { Session getApplicationSession(); + void createOrUpdateKeyspace() throws Exception; + void waitForSchemaAgreement(); } diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/impl/DatastaxClusterImpl.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/impl/DatastaxClusterImpl.java index 1e9061f5c3..e15475e207 100644 --- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/impl/DatastaxClusterImpl.java +++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/impl/DatastaxClusterImpl.java @@ -26,6 +26,7 @@ import org.apache.usergrid.persistence.core.astyanax.CassandraFig; import org.apache.usergrid.persistence.core.datastax.CQLUtils; import org.apache.usergrid.persistence.core.datastax.DataStaxCluster; +import org.safehaus.guicyfig.Overrides; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -92,7 +93,8 @@ public Session getApplicationSession(){ * Execute CQL that will create the keyspace if it doesn't exist and alter it if it does. * @throws Exception */ - private void createOrUpdateKeyspace() throws Exception { + @Override + public void createOrUpdateKeyspace() throws Exception { clusterSession = getClusterSession(); diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/schema/MigrationManagerImpl.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/schema/MigrationManagerImpl.java index 105f93a48d..9ab403d3b7 100644 --- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/schema/MigrationManagerImpl.java +++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/schema/MigrationManagerImpl.java @@ -71,9 +71,10 @@ public MigrationManagerImpl( final CassandraFig cassandraFig, final Keyspace key @Override public void migrate() throws MigrationException { - try { + dataStaxCluster.createOrUpdateKeyspace(); + for ( Migration migration : migrations ) { final Collection columnFamilies = migration.getColumnFamilies(); diff --git a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/datastax/DatastaxClusterTest.java b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/datastax/DatastaxClusterTest.java index 81b7d8fe1b..593e71bb84 100644 --- a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/datastax/DatastaxClusterTest.java +++ b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/datastax/DatastaxClusterTest.java @@ -57,7 +57,7 @@ public void testConnectCloseCluster() { cluster = dataStaxCluster.getCluster(); assertTrue(!cluster.isClosed()); - + } @Test From 65e873e0ddae370f07af84824225b1a59f7c1a19 Mon Sep 17 00:00:00 2001 From: Michael Russo Date: Sat, 13 Feb 2016 10:23:33 -0800 Subject: [PATCH 14/30] Let the keyspace be created only if database/setup is called. Maybe later add option for auto-keyspace creation. --- .../persistence/core/datastax/impl/DatastaxClusterImpl.java | 1 - 1 file changed, 1 deletion(-) diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/impl/DatastaxClusterImpl.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/impl/DatastaxClusterImpl.java index e15475e207..702c1e362c 100644 --- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/impl/DatastaxClusterImpl.java +++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/impl/DatastaxClusterImpl.java @@ -52,7 +52,6 @@ public DataStaxClusterImpl(final CassandraFig cassandraFig ) throws Exception { cluster.getConfiguration().getPoolingOptions().getIdleTimeoutSeconds(), cluster.getConfiguration().getPoolingOptions().getPoolTimeoutMillis() / 1000); - createOrUpdateKeyspace(); } From 6ef4c32114044d22d9d74e42033a0cccdd2069ab Mon Sep 17 00:00:00 2001 From: Michael Russo Date: Sat, 13 Feb 2016 21:23:52 -0800 Subject: [PATCH 15/30] Ensure that when specific apps are re-indexed, their index is initialized. --- .../corepersistence/index/ReIndexServiceImpl.java | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java index da2ef1df13..6c6597650c 100644 --- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java +++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java @@ -221,7 +221,13 @@ private Observable getApplications( final Optional } //this is intentional. If else if (appId.isPresent()) { - return Observable.just(appId.get()); + return Observable.just(appId.get()) + .doOnNext(appScope -> { + //make sure index is initialized on rebuild + entityIndexFactory.createEntityIndex( + indexLocationStrategyFactory.getIndexLocationStrategy(appScope) + ).initialize(); + }); } return allApplicationsObservable.getData() From bb225e8a37cfbe1419d98e61247af45a53f39733 Mon Sep 17 00:00:00 2001 From: Michael Russo Date: Sun, 14 Feb 2016 17:24:52 -0800 Subject: [PATCH 16/30] Add caching options to the table creation, supporting pre 2.1 and 2.1 table properties in Cassandra. Don't default the Cassandra version in the tests. --- .../resources/usergrid-default.properties | 2 +- .../test/resources/dynamic-test.properties | 1 - .../test/resources/usergrid-CHOP.properties | 1 - .../test/resources/usergrid-UNIT.properties | 1 - .../persistence/core/datastax/CQLUtils.java | 97 ++++++++++++++-- .../core/datastax/TableDefinition.java | 17 +-- .../datastax/impl/DatastaxClusterImpl.java | 1 - .../schema/MigrationManagerImpl.java | 2 +- .../core/datastax/CQLUtilsTest.java | 105 +++++++++++++++++- .../test/resources/usergrid-UNIT.properties | 1 - .../test/resources/usergrid-AWS.properties | 1 - .../test/resources/usergrid-CHOP.properties | 1 - .../test/resources/usergrid-UNIT.properties | 1 - .../test/resources/dynamic-test.properties | 1 - .../test/resources/usergrid-CHOP.properties | 1 - .../test/resources/usergrid-UNIT.properties | 1 - 16 files changed, 196 insertions(+), 38 deletions(-) diff --git a/stack/config/src/main/resources/usergrid-default.properties b/stack/config/src/main/resources/usergrid-default.properties index 94d02ff3ef..89f5dd9c1a 100644 --- a/stack/config/src/main/resources/usergrid-default.properties +++ b/stack/config/src/main/resources/usergrid-default.properties @@ -59,7 +59,7 @@ usergrid.persistence=CP # Set a property to tell Usergrid which version of cassandra is being used. # -#cassandra.version=1.2 +#cassandra.version=2.1 # Set the Cassandra cluster name that this instance of Usergrid should use. diff --git a/stack/corepersistence/collection/src/test/resources/dynamic-test.properties b/stack/corepersistence/collection/src/test/resources/dynamic-test.properties index b847f507f7..ca203f9829 100644 --- a/stack/corepersistence/collection/src/test/resources/dynamic-test.properties +++ b/stack/corepersistence/collection/src/test/resources/dynamic-test.properties @@ -2,7 +2,6 @@ # safe dynamic property defaults for our testing via IDE or Maven cassandra.connections=50 cassandra.port=9160 -cassandra.version=1.2 cassandra.hosts=localhost cassandra.cluster_name=Usergrid collections.keyspace=Usergrid_Collections diff --git a/stack/corepersistence/collection/src/test/resources/usergrid-CHOP.properties b/stack/corepersistence/collection/src/test/resources/usergrid-CHOP.properties index d276934363..92d0041edd 100644 --- a/stack/corepersistence/collection/src/test/resources/usergrid-CHOP.properties +++ b/stack/corepersistence/collection/src/test/resources/usergrid-CHOP.properties @@ -2,7 +2,6 @@ cassandra.connections=50 cassandra.port=9160 -cassandra.version=1.2 # a comma delimited private IP address list to your chop cassandra cluster # define this in your settings.xml and have it as an always active profile diff --git a/stack/corepersistence/collection/src/test/resources/usergrid-UNIT.properties b/stack/corepersistence/collection/src/test/resources/usergrid-UNIT.properties index bec1c88f8c..4a2d031014 100644 --- a/stack/corepersistence/collection/src/test/resources/usergrid-UNIT.properties +++ b/stack/corepersistence/collection/src/test/resources/usergrid-UNIT.properties @@ -1,7 +1,6 @@ # Keep nothing but overriding test defaults in here cassandra.connections=50 cassandra.port=9160 -cassandra.version=1.2 cassandra.hosts=localhost cassandra.cluster_name=Usergrid collections.keyspace=Usergrid_Collections diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/CQLUtils.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/CQLUtils.java index 38b6b7bc0c..f04e1a9f91 100644 --- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/CQLUtils.java +++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/CQLUtils.java @@ -18,10 +18,10 @@ */ package org.apache.usergrid.persistence.core.datastax; -import com.datastax.driver.core.DataType; -import com.datastax.driver.core.ProtocolVersion; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.inject.Inject; +import org.apache.usergrid.persistence.core.astyanax.CassandraFig; import org.apache.usergrid.persistence.core.util.StringUtils; import java.nio.ByteBuffer; @@ -29,6 +29,8 @@ public class CQLUtils { + private final CassandraFig cassandraFig; + private final static ObjectMapper mapper = new ObjectMapper(); public enum ACTION { CREATE, UPDATE @@ -41,6 +43,7 @@ public enum ACTION { static String EQUAL = "="; static String COMPRESSION = "compression"; static String COMPACTION = "compaction"; + static String CACHING = "caching"; static String GC_GRACE_SECONDS = "gc_grace_seconds"; static String PRIMARY_KEY = "PRIMARY KEY"; static String COMPACT_STORAGE = "COMPACT STORAGE"; @@ -49,8 +52,12 @@ public enum ACTION { static String PAREN_LEFT = "("; static String PAREN_RIGHT = ")"; + @Inject + public CQLUtils ( final CassandraFig cassandraFig ){ - private final static ObjectMapper mapper = new ObjectMapper(); + this.cassandraFig = cassandraFig; + + } public static String getFormattedReplication(String strategy, String strategyOptions) throws JsonProcessingException { @@ -72,7 +79,8 @@ public static String getMapAsCQLString(Map map) throws JsonProce } - public static String getTableCQL(TableDefinition tableDefinition, ACTION tableAction) throws Exception { + public static String getTableCQL( CassandraFig cassandraFig, TableDefinition tableDefinition, + ACTION tableAction) throws Exception { StringJoiner cql = new StringJoiner(" "); @@ -120,9 +128,9 @@ public static String getTableCQL(TableDefinition tableDefinition, ACTION tableAc .add(AND) .add(COMPRESSION).add(EQUAL).add( getMapAsCQLString( tableDefinition.getCompression() ) ) .add(AND) - .add(GC_GRACE_SECONDS).add(EQUAL).add( tableDefinition.getGcGraceSeconds() ); - - + .add(GC_GRACE_SECONDS).add(EQUAL).add( tableDefinition.getGcGraceSeconds() ) + .add(AND) + .add(CACHING).add(EQUAL).add( getCachingOptions( cassandraFig, tableDefinition.getCacheOption() ) ); return cql.toString(); @@ -144,6 +152,81 @@ public static String spaceSeparatedKeyValue(Map columns){ } + public static String getCachingOptions(CassandraFig cassandraFig, TableDefinition.CacheOption cacheOption) throws JsonProcessingException { + + // Cassandra 2.0 and below has a different CQL syntax for caching + if( Double.parseDouble( cassandraFig.getVersion() ) <= 2.0 ){ + + return quote( getLegacyCacheValue( cacheOption ) ); + + } else { + + return getCacheValue( cacheOption ); + } + + } + + + public static String getCacheValue( TableDefinition.CacheOption cacheOption ) throws JsonProcessingException { + + + Map cacheValue = new HashMap<>(2); + switch (cacheOption) { + + case ALL: + cacheValue.put("keys", "ALL"); + cacheValue.put("rows_per_partition", "ALL"); + break; + + case KEYS: + cacheValue.put("keys", "ALL"); + cacheValue.put("rows_per_partition", "NONE"); + break; + + case ROWS: + cacheValue.put("keys", "NONE"); + cacheValue.put("rows_per_partition", "ALL"); + break; + + case NONE: + cacheValue.put("keys", "NONE"); + cacheValue.put("rows_per_partition", "NONE"); + break; + + } + + return getMapAsCQLString( cacheValue ); + + } + + public static String getLegacyCacheValue( TableDefinition.CacheOption cacheOption ){ + + String cacheValue = "none"; // default to no caching + switch (cacheOption) { + + case ALL: + cacheValue = "all"; + break; + + case KEYS: + cacheValue = "keys_only"; + break; + + case ROWS: + cacheValue = "rows_only"; + break; + + case NONE: + cacheValue = "none"; + break; + + } + + return cacheValue; + + } + + /** * Below functions borrowed from Astyanax until the schema is re-written to be more CQL friendly */ diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/TableDefinition.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/TableDefinition.java index 2f83513329..d749d40a08 100644 --- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/TableDefinition.java +++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/TableDefinition.java @@ -32,22 +32,7 @@ public class TableDefinition { public enum CacheOption { - ALL( "ALL" ), - KEYS( "KEYS_ONLY" ), - ROWS( "ROWS_ONLY" ), - NONE( "NONE" ); - - private String value; - - - CacheOption( String value ) { - this.value = value; - } - - - public String getValue() { - return value; - } + ALL, KEYS, ROWS, NONE } diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/impl/DatastaxClusterImpl.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/impl/DatastaxClusterImpl.java index 702c1e362c..875c5c2d66 100644 --- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/impl/DatastaxClusterImpl.java +++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/impl/DatastaxClusterImpl.java @@ -26,7 +26,6 @@ import org.apache.usergrid.persistence.core.astyanax.CassandraFig; import org.apache.usergrid.persistence.core.datastax.CQLUtils; import org.apache.usergrid.persistence.core.datastax.DataStaxCluster; -import org.safehaus.guicyfig.Overrides; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/schema/MigrationManagerImpl.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/schema/MigrationManagerImpl.java index 9ab403d3b7..3630fc8ee6 100644 --- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/schema/MigrationManagerImpl.java +++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/schema/MigrationManagerImpl.java @@ -138,7 +138,7 @@ private void testAndCreateColumnFamilyDef( MultiTenantColumnFamilyDefinition col private void createTable(TableDefinition tableDefinition ) throws Exception { - String CQL = CQLUtils.getTableCQL( tableDefinition, CQLUtils.ACTION.CREATE ); + String CQL = CQLUtils.getTableCQL( cassandraFig, tableDefinition, CQLUtils.ACTION.CREATE ); if (logger.isDebugEnabled()){ logger.debug( CQL ); } diff --git a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/datastax/CQLUtilsTest.java b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/datastax/CQLUtilsTest.java index c47312dc19..5bad54e0c5 100644 --- a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/datastax/CQLUtilsTest.java +++ b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/datastax/CQLUtilsTest.java @@ -20,7 +20,14 @@ import com.datastax.driver.core.DataType; +import com.google.inject.Inject; +import org.apache.usergrid.persistence.core.astyanax.CassandraFig; +import org.apache.usergrid.persistence.core.guice.TestCommonModule; +import org.apache.usergrid.persistence.core.test.ITRunner; +import org.apache.usergrid.persistence.core.test.UseModules; +import org.junit.Before; import org.junit.Test; +import org.junit.runner.RunWith; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -28,13 +35,22 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.concurrent.ExecutionException; import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +@RunWith( ITRunner.class ) +@UseModules( TestCommonModule.class ) public class CQLUtilsTest { private static final Logger logger = LoggerFactory.getLogger( CQLUtilsTest.class ); + @Inject + CassandraFig cassandraFig; + + @Test public void testTableCQL() throws Exception { @@ -64,8 +80,8 @@ public void testTableCQL() throws Exception { clusteringOrder ); - String createCQL = CQLUtils.getTableCQL(table1, CQLUtils.ACTION.CREATE); - String updateCQL = CQLUtils.getTableCQL(table1, CQLUtils.ACTION.UPDATE); + String createCQL = CQLUtils.getTableCQL(cassandraFig, table1, CQLUtils.ACTION.CREATE); + String updateCQL = CQLUtils.getTableCQL(cassandraFig, table1, CQLUtils.ACTION.UPDATE); assertTrue( createCQL.contains(CQLUtils.CREATE_TABLE ) && @@ -85,4 +101,89 @@ public void testTableCQL() throws Exception { } + @Test + public void testLegacyCachingOptions() throws Exception{ + + final CassandraFig cassandraFig = mock(CassandraFig.class); + when(cassandraFig.getVersion()).thenReturn("2.0"); + + Map columns = new HashMap<>(); + columns.put("key", DataType.Name.BLOB); + columns.put("column1", DataType.Name.TEXT); + columns.put("value", DataType.Name.BLOB); + + List partitionKeys = new ArrayList<>(); + partitionKeys.add("key"); + + List columnKeys = new ArrayList<>(); + columnKeys.add("column1"); + + Map clusteringOrder = new HashMap<>(); + clusteringOrder.put("column1", "DESC"); + + + + TableDefinition table1 = new TableDefinition( + CQLUtils.quote("table1"), + partitionKeys, + columnKeys, + columns, + TableDefinition.CacheOption.KEYS, + clusteringOrder + ); + + String createCQL = CQLUtils.getTableCQL(cassandraFig, table1, CQLUtils.ACTION.CREATE); + logger.info(createCQL); + assertTrue( + createCQL.contains( "\"keys_only\"" ) && + !createCQL.contains( "'keys':'ALL'" ) + + ); + + + + } + + @Test + public void testCachingOptions() throws Exception { + + final CassandraFig cassandraFig = mock(CassandraFig.class); + when(cassandraFig.getVersion()).thenReturn("2.1"); + + Map columns = new HashMap<>(); + columns.put("key", DataType.Name.BLOB); + columns.put("column1", DataType.Name.TEXT); + columns.put("value", DataType.Name.BLOB); + + List partitionKeys = new ArrayList<>(); + partitionKeys.add("key"); + + List columnKeys = new ArrayList<>(); + columnKeys.add("column1"); + + Map clusteringOrder = new HashMap<>(); + clusteringOrder.put("column1", "DESC"); + + + + TableDefinition table1 = new TableDefinition( + CQLUtils.quote("table1"), + partitionKeys, + columnKeys, + columns, + TableDefinition.CacheOption.KEYS, + clusteringOrder + ); + + String createCQL = CQLUtils.getTableCQL(cassandraFig, table1, CQLUtils.ACTION.CREATE); + logger.info(createCQL); + assertTrue( + createCQL.contains( "'keys':'ALL'" ) && + !createCQL.contains( "\"keys_only\"" ) + + ); + + + } + } diff --git a/stack/corepersistence/common/src/test/resources/usergrid-UNIT.properties b/stack/corepersistence/common/src/test/resources/usergrid-UNIT.properties index 2004d9e5e7..5f59711ff4 100644 --- a/stack/corepersistence/common/src/test/resources/usergrid-UNIT.properties +++ b/stack/corepersistence/common/src/test/resources/usergrid-UNIT.properties @@ -1,7 +1,6 @@ # Keep nothing but overriding test defaults in here cassandra.connections=50 cassandra.port=9160 -cassandra.version=1.2 cassandra.hosts=localhost cassandra.cluster_name=Usergrid collections.keyspace=Usergrid_Collections diff --git a/stack/corepersistence/graph/src/test/resources/usergrid-AWS.properties b/stack/corepersistence/graph/src/test/resources/usergrid-AWS.properties index e630bb8694..e7378156f8 100644 --- a/stack/corepersistence/graph/src/test/resources/usergrid-AWS.properties +++ b/stack/corepersistence/graph/src/test/resources/usergrid-AWS.properties @@ -1,7 +1,6 @@ # Keep nothing but overriding test defaults in here cassandra.connections=50 cassandra.port=9160 -cassandra.version=1.2 cassandra.hosts= #cassandra.hosts=localhost cassandra.cluster_name=Usergrid diff --git a/stack/corepersistence/graph/src/test/resources/usergrid-CHOP.properties b/stack/corepersistence/graph/src/test/resources/usergrid-CHOP.properties index d276934363..92d0041edd 100644 --- a/stack/corepersistence/graph/src/test/resources/usergrid-CHOP.properties +++ b/stack/corepersistence/graph/src/test/resources/usergrid-CHOP.properties @@ -2,7 +2,6 @@ cassandra.connections=50 cassandra.port=9160 -cassandra.version=1.2 # a comma delimited private IP address list to your chop cassandra cluster # define this in your settings.xml and have it as an always active profile diff --git a/stack/corepersistence/graph/src/test/resources/usergrid-UNIT.properties b/stack/corepersistence/graph/src/test/resources/usergrid-UNIT.properties index e1c4c81001..6f8a7c5568 100644 --- a/stack/corepersistence/graph/src/test/resources/usergrid-UNIT.properties +++ b/stack/corepersistence/graph/src/test/resources/usergrid-UNIT.properties @@ -1,7 +1,6 @@ # Keep nothing but overriding test defaults in here cassandra.connections=50 cassandra.port=9160 -cassandra.version=1.2 cassandra.hosts=localhost cassandra.cluster_name=Usergrid collections.keyspace=Usergrid_Collections diff --git a/stack/corepersistence/queryindex/src/test/resources/dynamic-test.properties b/stack/corepersistence/queryindex/src/test/resources/dynamic-test.properties index 4ac97d84cc..7b869af285 100644 --- a/stack/corepersistence/queryindex/src/test/resources/dynamic-test.properties +++ b/stack/corepersistence/queryindex/src/test/resources/dynamic-test.properties @@ -2,7 +2,6 @@ # safe dynamic property defaults for our testing via IDE or Maven cassandra.connections=50 cassandra.port=9160 -cassandra.version=1.2 cassandra.hosts=localhost cassandra.cluster_name=Usergrid collections.keyspace=Usergrid_Collections diff --git a/stack/corepersistence/queryindex/src/test/resources/usergrid-CHOP.properties b/stack/corepersistence/queryindex/src/test/resources/usergrid-CHOP.properties index ce40600e8d..ff4f3bb264 100644 --- a/stack/corepersistence/queryindex/src/test/resources/usergrid-CHOP.properties +++ b/stack/corepersistence/queryindex/src/test/resources/usergrid-CHOP.properties @@ -1,7 +1,6 @@ # These are for CHOP environment settings cassandra.connections=50 cassandra.port=9160 -cassandra.version=1.2 # a comma delimited private IP address list to your chop cassandra cluster # define this in your settings.xml and have it as an always active profile diff --git a/stack/corepersistence/queryindex/src/test/resources/usergrid-UNIT.properties b/stack/corepersistence/queryindex/src/test/resources/usergrid-UNIT.properties index 5d80cac49e..3c853aceaf 100644 --- a/stack/corepersistence/queryindex/src/test/resources/usergrid-UNIT.properties +++ b/stack/corepersistence/queryindex/src/test/resources/usergrid-UNIT.properties @@ -3,7 +3,6 @@ cassandra.embedded=true cassandra.hosts=127.0.0.1 cassandra.port=9160 -cassandra.version=1.2 cassandra.cluster_name=Usergrid cassandra.connections=50 From 75cc3a43edf3bb6efe32b715823a30872453358b Mon Sep 17 00:00:00 2001 From: Michael Russo Date: Sun, 14 Feb 2016 18:36:03 -0800 Subject: [PATCH 17/30] Move CassandraFig outside of Astyanax package. Update handling of protocol version for datastax cluster. --- .../ApplicationIndexLocationStrategy.java | 2 +- .../IndexLocationStrategyFactoryImpl.java | 2 +- .../ManagementIndexLocationStrategy.java | 2 +- .../cassandra/AstyanaxLockManagerImpl.java | 1 + .../cassandra/CassandraService.java | 3 +- .../index/IndexNamingTest.java | 2 +- .../MvccEntitySerializationStrategyImpl.java | 2 +- ...MvccEntitySerializationStrategyV1Impl.java | 2 +- ...MvccEntitySerializationStrategyV2Impl.java | 2 +- ...MvccEntitySerializationStrategyV3Impl.java | 2 +- .../impl/SettingsValidation.java | 2 +- .../UniqueValueSerializationStrategyImpl.java | 2 +- ...niqueValueSerializationStrategyV1Impl.java | 2 +- ...niqueValueSerializationStrategyV2Impl.java | 2 +- ...ccEntitySerializationStrategyImplTest.java | 2 +- ...MvccEntitySerializationStrategyV2Test.java | 2 +- .../impl/SettingsValidationTest.java | 2 +- .../core/{astyanax => }/CassandraFig.java | 7 ++++- .../core/astyanax/CassandraClusterImpl.java | 1 + .../core/astyanax/CassandraConfigImpl.java | 1 + .../persistence/core/datastax/CQLUtils.java | 11 +++++-- .../datastax/impl/DatastaxClusterImpl.java | 29 +++++++++++++++++-- .../persistence/core/guice/CommonModule.java | 1 + .../schema/MigrationManagerImpl.java | 3 +- .../core/astyanax/ColumnNameIteratorTest.java | 2 +- .../MultiKeyColumnNameIteratorTest.java | 4 +-- .../astyanax/MultiRowColumnIteratorTest.java | 3 +- .../core/datastax/CQLUtilsTest.java | 4 +-- .../core/datastax/DatastaxClusterTest.java | 3 +- .../index/impl/EntityIndexTest.java | 2 +- .../persistence/index/impl/GeoPagingTest.java | 2 +- .../index/impl/IndexLoadTestsIT.java | 2 +- .../index/impl/TestIndexIdentifier.java | 3 +- .../queue/impl/SNSQueueManagerImpl.java | 2 +- 34 files changed, 73 insertions(+), 41 deletions(-) rename stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/{astyanax => }/CassandraFig.java (98%) diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ApplicationIndexLocationStrategy.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ApplicationIndexLocationStrategy.java index e13566d015..53d214438d 100644 --- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ApplicationIndexLocationStrategy.java +++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ApplicationIndexLocationStrategy.java @@ -19,7 +19,7 @@ */ package org.apache.usergrid.corepersistence.index; -import org.apache.usergrid.persistence.core.astyanax.CassandraFig; +import org.apache.usergrid.persistence.core.CassandraFig; import org.apache.usergrid.persistence.core.guicyfig.ClusterFig; import org.apache.usergrid.persistence.core.scope.ApplicationScope; import org.apache.usergrid.persistence.index.IndexAlias; diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexLocationStrategyFactoryImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexLocationStrategyFactoryImpl.java index 6a998909f0..93fb677f9f 100644 --- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexLocationStrategyFactoryImpl.java +++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexLocationStrategyFactoryImpl.java @@ -21,7 +21,7 @@ import com.google.inject.Inject; import org.apache.usergrid.corepersistence.util.CpNamingUtils; -import org.apache.usergrid.persistence.core.astyanax.CassandraFig; +import org.apache.usergrid.persistence.core.CassandraFig; import org.apache.usergrid.persistence.core.guicyfig.ClusterFig; import org.apache.usergrid.persistence.core.scope.ApplicationScope; import org.apache.usergrid.persistence.index.IndexFig; diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ManagementIndexLocationStrategy.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ManagementIndexLocationStrategy.java index 1e13bbf6cb..ffa02cfb57 100644 --- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ManagementIndexLocationStrategy.java +++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ManagementIndexLocationStrategy.java @@ -20,7 +20,7 @@ package org.apache.usergrid.corepersistence.index; import org.apache.usergrid.corepersistence.util.CpNamingUtils; -import org.apache.usergrid.persistence.core.astyanax.CassandraFig; +import org.apache.usergrid.persistence.core.CassandraFig; import org.apache.usergrid.persistence.core.guicyfig.ClusterFig; import org.apache.usergrid.persistence.core.scope.ApplicationScope; import org.apache.usergrid.persistence.index.IndexAlias; diff --git a/stack/core/src/main/java/org/apache/usergrid/locking/cassandra/AstyanaxLockManagerImpl.java b/stack/core/src/main/java/org/apache/usergrid/locking/cassandra/AstyanaxLockManagerImpl.java index a69aee2558..f76704299c 100644 --- a/stack/core/src/main/java/org/apache/usergrid/locking/cassandra/AstyanaxLockManagerImpl.java +++ b/stack/core/src/main/java/org/apache/usergrid/locking/cassandra/AstyanaxLockManagerImpl.java @@ -33,6 +33,7 @@ import org.apache.usergrid.locking.Lock; import org.apache.usergrid.locking.LockManager; import org.apache.usergrid.locking.LockPathBuilder; +import org.apache.usergrid.persistence.core.CassandraFig; import org.apache.usergrid.persistence.core.astyanax.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/CassandraService.java b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/CassandraService.java index 24315332f2..2de41bfaa3 100644 --- a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/CassandraService.java +++ b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/CassandraService.java @@ -17,7 +17,6 @@ package org.apache.usergrid.persistence.cassandra; -import com.google.inject.Inject; import com.google.inject.Injector; import me.prettyprint.cassandra.connection.HConnectionManager; import me.prettyprint.cassandra.model.ConfigurableConsistencyLevel; @@ -36,7 +35,7 @@ import me.prettyprint.hector.api.query.QueryResult; import me.prettyprint.hector.api.query.SliceQuery; import org.apache.usergrid.locking.LockManager; -import org.apache.usergrid.persistence.core.astyanax.CassandraFig; +import org.apache.usergrid.persistence.core.CassandraFig; import org.apache.usergrid.persistence.hector.CountingMutator; import org.apache.usergrid.utils.MapUtils; import org.slf4j.Logger; diff --git a/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/IndexNamingTest.java b/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/IndexNamingTest.java index 4fea8e26f9..1661af431b 100644 --- a/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/IndexNamingTest.java +++ b/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/IndexNamingTest.java @@ -23,7 +23,7 @@ import net.jcip.annotations.NotThreadSafe; import org.apache.usergrid.corepersistence.TestIndexModule; import org.apache.usergrid.corepersistence.util.CpNamingUtils; -import org.apache.usergrid.persistence.core.astyanax.CassandraFig; +import org.apache.usergrid.persistence.core.CassandraFig; import org.apache.usergrid.persistence.core.guicyfig.ClusterFig; import org.apache.usergrid.persistence.core.scope.ApplicationScope; import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl; diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImpl.java index 3e2d391f84..2d7892a58e 100644 --- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImpl.java +++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImpl.java @@ -41,7 +41,7 @@ import org.apache.usergrid.persistence.collection.serialization.MvccEntitySerializationStrategy; import org.apache.usergrid.persistence.collection.serialization.SerializationFig; import org.apache.usergrid.persistence.collection.serialization.impl.util.LegacyScopeUtils; -import org.apache.usergrid.persistence.core.astyanax.CassandraFig; +import org.apache.usergrid.persistence.core.CassandraFig; import org.apache.usergrid.persistence.core.astyanax.ColumnNameIterator; import org.apache.usergrid.persistence.core.astyanax.ColumnParser; import org.apache.usergrid.persistence.core.astyanax.MultiTenantColumnFamily; diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV1Impl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV1Impl.java index 9e29e9c505..ee548e36bd 100644 --- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV1Impl.java +++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV1Impl.java @@ -26,7 +26,7 @@ import org.apache.usergrid.persistence.collection.MvccEntity; import org.apache.usergrid.persistence.collection.exception.DataCorruptionException; import org.apache.usergrid.persistence.collection.serialization.SerializationFig; -import org.apache.usergrid.persistence.core.astyanax.CassandraFig; +import org.apache.usergrid.persistence.core.CassandraFig; import org.apache.usergrid.persistence.core.astyanax.IdRowCompositeSerializer; import org.apache.usergrid.persistence.core.astyanax.MultiTenantColumnFamily; import org.apache.usergrid.persistence.core.astyanax.ScopedRowKey; diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV2Impl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV2Impl.java index 13d9613b4e..e9b078197f 100644 --- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV2Impl.java +++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV2Impl.java @@ -29,7 +29,7 @@ import org.apache.usergrid.persistence.collection.exception.DataCorruptionException; import org.apache.usergrid.persistence.collection.exception.EntityTooLargeException; import org.apache.usergrid.persistence.collection.serialization.SerializationFig; -import org.apache.usergrid.persistence.core.astyanax.CassandraFig; +import org.apache.usergrid.persistence.core.CassandraFig; import org.apache.usergrid.persistence.core.astyanax.FieldBuffer; import org.apache.usergrid.persistence.core.astyanax.FieldBufferBuilder; import org.apache.usergrid.persistence.core.astyanax.FieldBufferParser; diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV3Impl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV3Impl.java index 2fa5d09b6c..ecd0f5c457 100644 --- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV3Impl.java +++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV3Impl.java @@ -32,7 +32,7 @@ import org.apache.usergrid.persistence.collection.mvcc.entity.impl.MvccEntityImpl; import org.apache.usergrid.persistence.collection.serialization.MvccEntitySerializationStrategy; import org.apache.usergrid.persistence.collection.serialization.SerializationFig; -import org.apache.usergrid.persistence.core.astyanax.CassandraFig; +import org.apache.usergrid.persistence.core.CassandraFig; import org.apache.usergrid.persistence.core.astyanax.ColumnParser; import org.apache.usergrid.persistence.core.astyanax.IdRowCompositeSerializer; import org.apache.usergrid.persistence.core.astyanax.MultiTenantColumnFamily; diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/SettingsValidation.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/SettingsValidation.java index 6715cfb712..0f3f6b93e8 100644 --- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/SettingsValidation.java +++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/SettingsValidation.java @@ -21,7 +21,7 @@ import org.apache.usergrid.persistence.collection.serialization.SerializationFig; -import org.apache.usergrid.persistence.core.astyanax.CassandraFig; +import org.apache.usergrid.persistence.core.CassandraFig; import com.google.common.base.Preconditions; import com.google.inject.Inject; diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyImpl.java index 197fd4ed9c..0f27167fe3 100644 --- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyImpl.java +++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyImpl.java @@ -30,7 +30,7 @@ import org.apache.usergrid.persistence.collection.serialization.UniqueValue; import org.apache.usergrid.persistence.collection.serialization.UniqueValueSerializationStrategy; import org.apache.usergrid.persistence.collection.serialization.UniqueValueSet; -import org.apache.usergrid.persistence.core.astyanax.CassandraFig; +import org.apache.usergrid.persistence.core.CassandraFig; import org.apache.usergrid.persistence.core.astyanax.ColumnNameIterator; import org.apache.usergrid.persistence.core.astyanax.ColumnParser; import org.apache.usergrid.persistence.core.astyanax.ColumnTypes; diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyV1Impl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyV1Impl.java index dafbf2b9c0..2235f63aaf 100644 --- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyV1Impl.java +++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyV1Impl.java @@ -28,7 +28,7 @@ import org.apache.usergrid.persistence.collection.serialization.SerializationFig; import org.apache.usergrid.persistence.collection.serialization.impl.util.LegacyScopeUtils; -import org.apache.usergrid.persistence.core.astyanax.CassandraFig; +import org.apache.usergrid.persistence.core.CassandraFig; import org.apache.usergrid.persistence.core.astyanax.ColumnTypes; import org.apache.usergrid.persistence.core.astyanax.IdRowCompositeSerializer; import org.apache.usergrid.persistence.core.astyanax.MultiTenantColumnFamily; diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyV2Impl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyV2Impl.java index 17b8b37f9a..0f233cfd1f 100644 --- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyV2Impl.java +++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyV2Impl.java @@ -27,7 +27,7 @@ import org.apache.cassandra.db.marshal.BytesType; import org.apache.usergrid.persistence.collection.serialization.SerializationFig; -import org.apache.usergrid.persistence.core.astyanax.CassandraFig; +import org.apache.usergrid.persistence.core.CassandraFig; import org.apache.usergrid.persistence.core.astyanax.ColumnTypes; import org.apache.usergrid.persistence.core.astyanax.IdRowCompositeSerializer; import org.apache.usergrid.persistence.core.astyanax.MultiTenantColumnFamily; diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImplTest.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImplTest.java index f6720f970d..ef3aabdca9 100644 --- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImplTest.java +++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImplTest.java @@ -33,7 +33,7 @@ import org.apache.usergrid.persistence.collection.MvccEntity; import org.apache.usergrid.persistence.collection.mvcc.entity.impl.MvccEntityImpl; import org.apache.usergrid.persistence.collection.serialization.MvccEntitySerializationStrategy; -import org.apache.usergrid.persistence.core.astyanax.CassandraFig; +import org.apache.usergrid.persistence.core.CassandraFig; import org.apache.usergrid.persistence.core.guice.MigrationManagerRule; import org.apache.usergrid.persistence.core.scope.ApplicationScope; import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl; diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV2Test.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV2Test.java index f8b65d93d6..2dca27d1a6 100644 --- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV2Test.java +++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV2Test.java @@ -36,7 +36,7 @@ import org.apache.usergrid.persistence.collection.mvcc.entity.impl.MvccEntityImpl; import org.apache.usergrid.persistence.collection.serialization.SerializationFig; import org.apache.usergrid.persistence.collection.util.EntityHelper; -import org.apache.usergrid.persistence.core.astyanax.CassandraFig; +import org.apache.usergrid.persistence.core.CassandraFig; import org.apache.usergrid.persistence.core.guicyfig.SetConfigTestBypass; import org.apache.usergrid.persistence.core.scope.ApplicationScope; import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl; diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/SettingsValidationTest.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/SettingsValidationTest.java index 273bec081c..cf700f393a 100644 --- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/SettingsValidationTest.java +++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/SettingsValidationTest.java @@ -23,7 +23,7 @@ import org.junit.Test; import org.apache.usergrid.persistence.collection.serialization.SerializationFig; -import org.apache.usergrid.persistence.core.astyanax.CassandraFig; +import org.apache.usergrid.persistence.core.CassandraFig; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/CassandraFig.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/CassandraFig.java similarity index 98% rename from stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/CassandraFig.java rename to stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/CassandraFig.java index 78c4fbc494..7a2a9d165b 100644 --- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/CassandraFig.java +++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/CassandraFig.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.usergrid.persistence.core.astyanax; +package org.apache.usergrid.persistence.core; import org.safehaus.guicyfig.Default; @@ -67,6 +67,11 @@ public interface CassandraFig extends GuicyFig { @Key( "cassandra.hosts" ) String getHosts(); + /** + * Valid options are 1.2, 2.0, 2.1 + * + * @return + */ @Key( "cassandra.version" ) @Default( "2.1" ) String getVersion(); diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/CassandraClusterImpl.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/CassandraClusterImpl.java index cc52f1b023..b433ecc728 100644 --- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/CassandraClusterImpl.java +++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/CassandraClusterImpl.java @@ -35,6 +35,7 @@ import com.netflix.astyanax.connectionpool.impl.Slf4jConnectionPoolMonitorImpl; import com.netflix.astyanax.impl.AstyanaxConfigurationImpl; import com.netflix.astyanax.thrift.ThriftFamilyFactory; +import org.apache.usergrid.persistence.core.CassandraFig; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/CassandraConfigImpl.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/CassandraConfigImpl.java index 15f434cde9..0135c95a17 100644 --- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/CassandraConfigImpl.java +++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/CassandraConfigImpl.java @@ -26,6 +26,7 @@ import com.google.inject.Inject; import com.google.inject.Singleton; import com.netflix.astyanax.model.ConsistencyLevel; +import org.apache.usergrid.persistence.core.CassandraFig; /** diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/CQLUtils.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/CQLUtils.java index f04e1a9f91..f65a2605d5 100644 --- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/CQLUtils.java +++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/CQLUtils.java @@ -21,7 +21,7 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.inject.Inject; -import org.apache.usergrid.persistence.core.astyanax.CassandraFig; +import org.apache.usergrid.persistence.core.CassandraFig; import org.apache.usergrid.persistence.core.util.StringUtils; import java.nio.ByteBuffer; @@ -192,6 +192,10 @@ public static String getCacheValue( TableDefinition.CacheOption cacheOption ) th cacheValue.put("keys", "NONE"); cacheValue.put("rows_per_partition", "NONE"); break; + default: + cacheValue.put("keys", "NONE"); + cacheValue.put("rows_per_partition", "NONE"); + break; } @@ -201,7 +205,7 @@ public static String getCacheValue( TableDefinition.CacheOption cacheOption ) th public static String getLegacyCacheValue( TableDefinition.CacheOption cacheOption ){ - String cacheValue = "none"; // default to no caching + String cacheValue; switch (cacheOption) { case ALL: @@ -219,6 +223,9 @@ public static String getLegacyCacheValue( TableDefinition.CacheOption cacheOptio case NONE: cacheValue = "none"; break; + default: + cacheValue = "none"; + break; } diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/impl/DatastaxClusterImpl.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/impl/DatastaxClusterImpl.java index 875c5c2d66..34cfec8e36 100644 --- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/impl/DatastaxClusterImpl.java +++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/impl/DatastaxClusterImpl.java @@ -23,7 +23,7 @@ import com.datastax.driver.core.policies.LoadBalancingPolicy; import com.google.inject.Inject; import com.google.inject.Singleton; -import org.apache.usergrid.persistence.core.astyanax.CassandraFig; +import org.apache.usergrid.persistence.core.CassandraFig; import org.apache.usergrid.persistence.core.datastax.CQLUtils; import org.apache.usergrid.persistence.core.datastax.DataStaxCluster; import org.slf4j.Logger; @@ -179,7 +179,7 @@ public Cluster buildCluster(){ .withLoadBalancingPolicy(loadBalancingPolicy) .withPoolingOptions(poolingOptions) .withQueryOptions(queryOptions) - .withProtocolVersion(ProtocolVersion.NEWEST_SUPPORTED); + .withProtocolVersion(getProtocolVersion(cassandraFig.getVersion())); // only add auth credentials if they were provided if ( !cassandraFig.getUsername().isEmpty() && !cassandraFig.getPassword().isEmpty() ){ @@ -194,4 +194,29 @@ public Cluster buildCluster(){ } + private ProtocolVersion getProtocolVersion(String versionNumber){ + + ProtocolVersion protocolVersion; + switch (versionNumber) { + + case "2.1": + protocolVersion = ProtocolVersion.V3; + break; + case "2.0": + protocolVersion = ProtocolVersion.V2; + break; + case "1.2": + protocolVersion = ProtocolVersion.V1; + break; + default: + protocolVersion = ProtocolVersion.NEWEST_SUPPORTED; + break; + + } + + return protocolVersion; + + + } + } diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/guice/CommonModule.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/guice/CommonModule.java index 460efa5bec..94aac6019d 100644 --- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/guice/CommonModule.java +++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/guice/CommonModule.java @@ -21,6 +21,7 @@ import com.datastax.driver.core.Session; import com.netflix.astyanax.Keyspace; +import org.apache.usergrid.persistence.core.CassandraFig; import org.apache.usergrid.persistence.core.astyanax.*; import org.apache.usergrid.persistence.core.datastax.DataStaxCluster; import org.apache.usergrid.persistence.core.datastax.DataStaxSessionProvider; diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/schema/MigrationManagerImpl.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/schema/MigrationManagerImpl.java index 3630fc8ee6..f5f5d7becf 100644 --- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/schema/MigrationManagerImpl.java +++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/schema/MigrationManagerImpl.java @@ -22,8 +22,7 @@ import java.util.Collection; import java.util.Set; -import com.datastax.driver.core.Session; -import org.apache.usergrid.persistence.core.astyanax.CassandraFig; +import org.apache.usergrid.persistence.core.CassandraFig; import org.apache.usergrid.persistence.core.datastax.CQLUtils; import org.apache.usergrid.persistence.core.datastax.DataStaxCluster; import org.apache.usergrid.persistence.core.datastax.TableDefinition; diff --git a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/astyanax/ColumnNameIteratorTest.java b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/astyanax/ColumnNameIteratorTest.java index dccbd4552b..415eafac6d 100644 --- a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/astyanax/ColumnNameIteratorTest.java +++ b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/astyanax/ColumnNameIteratorTest.java @@ -24,8 +24,8 @@ import java.util.HashMap; +import org.apache.usergrid.persistence.core.CassandraFig; import org.junit.Before; -import org.junit.BeforeClass; import org.junit.Test; import org.junit.runner.RunWith; diff --git a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/astyanax/MultiKeyColumnNameIteratorTest.java b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/astyanax/MultiKeyColumnNameIteratorTest.java index d020949c04..55ed421cab 100644 --- a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/astyanax/MultiKeyColumnNameIteratorTest.java +++ b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/astyanax/MultiKeyColumnNameIteratorTest.java @@ -26,8 +26,8 @@ import java.util.Comparator; import java.util.HashMap; +import org.apache.usergrid.persistence.core.CassandraFig; import org.junit.Before; -import org.junit.BeforeClass; import org.junit.Test; import org.junit.runner.RunWith; @@ -50,8 +50,6 @@ import com.netflix.astyanax.util.RangeBuilder; import rx.Observable; -import rx.functions.Action1; -import rx.functions.Func1; import rx.schedulers.Schedulers; import static org.junit.Assert.assertEquals; diff --git a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/astyanax/MultiRowColumnIteratorTest.java b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/astyanax/MultiRowColumnIteratorTest.java index 8bcdcb207f..d48795ecbc 100644 --- a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/astyanax/MultiRowColumnIteratorTest.java +++ b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/astyanax/MultiRowColumnIteratorTest.java @@ -28,8 +28,8 @@ import java.util.HashMap; import java.util.concurrent.CountDownLatch; +import org.apache.usergrid.persistence.core.CassandraFig; import org.junit.Before; -import org.junit.BeforeClass; import org.junit.Test; import org.junit.runner.RunWith; @@ -53,7 +53,6 @@ import rx.Observable; import rx.Observer; import rx.functions.Action1; -import rx.functions.Func1; import rx.schedulers.Schedulers; import static org.junit.Assert.assertEquals; diff --git a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/datastax/CQLUtilsTest.java b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/datastax/CQLUtilsTest.java index 5bad54e0c5..34dd3703f0 100644 --- a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/datastax/CQLUtilsTest.java +++ b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/datastax/CQLUtilsTest.java @@ -21,11 +21,10 @@ import com.datastax.driver.core.DataType; import com.google.inject.Inject; -import org.apache.usergrid.persistence.core.astyanax.CassandraFig; +import org.apache.usergrid.persistence.core.CassandraFig; import org.apache.usergrid.persistence.core.guice.TestCommonModule; import org.apache.usergrid.persistence.core.test.ITRunner; import org.apache.usergrid.persistence.core.test.UseModules; -import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.slf4j.Logger; @@ -35,7 +34,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.concurrent.ExecutionException; import static org.junit.Assert.assertTrue; import static org.mockito.Mockito.mock; diff --git a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/datastax/DatastaxClusterTest.java b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/datastax/DatastaxClusterTest.java index 593e71bb84..ca7f97a49b 100644 --- a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/datastax/DatastaxClusterTest.java +++ b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/datastax/DatastaxClusterTest.java @@ -21,11 +21,10 @@ import com.datastax.driver.core.Cluster; import com.datastax.driver.core.Session; import com.google.inject.Inject; -import org.apache.usergrid.persistence.core.astyanax.CassandraFig; +import org.apache.usergrid.persistence.core.CassandraFig; import org.apache.usergrid.persistence.core.guice.TestCommonModule; import org.apache.usergrid.persistence.core.test.ITRunner; import org.apache.usergrid.persistence.core.test.UseModules; -import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java index 3978956cd4..2f9b6735c4 100644 --- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java +++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java @@ -29,7 +29,7 @@ import java.util.concurrent.atomic.AtomicLong; import com.google.common.base.Optional; -import org.apache.usergrid.persistence.core.astyanax.CassandraFig; +import org.apache.usergrid.persistence.core.CassandraFig; import org.apache.usergrid.persistence.index.*; import org.apache.usergrid.persistence.model.field.*; import org.junit.Before; diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/GeoPagingTest.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/GeoPagingTest.java index eb6651f255..3d68fe1e31 100644 --- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/GeoPagingTest.java +++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/GeoPagingTest.java @@ -25,7 +25,7 @@ import java.util.*; -import org.apache.usergrid.persistence.core.astyanax.CassandraFig; +import org.apache.usergrid.persistence.core.CassandraFig; import org.apache.usergrid.persistence.index.*; import org.apache.usergrid.persistence.model.entity.SimpleId; import org.junit.Before; diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/IndexLoadTestsIT.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/IndexLoadTestsIT.java index a6cef6034f..afaebb7bb1 100644 --- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/IndexLoadTestsIT.java +++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/IndexLoadTestsIT.java @@ -24,7 +24,7 @@ import java.util.concurrent.atomic.AtomicLong; import org.apache.usergrid.StressTest; -import org.apache.usergrid.persistence.core.astyanax.CassandraFig; +import org.apache.usergrid.persistence.core.CassandraFig; import org.apache.usergrid.persistence.index.*; import org.junit.After; import org.junit.Before; diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/TestIndexIdentifier.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/TestIndexIdentifier.java index 2cc7e62959..ffd25f3d2b 100644 --- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/TestIndexIdentifier.java +++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/TestIndexIdentifier.java @@ -20,9 +20,8 @@ package org.apache.usergrid.persistence.index.impl; -import org.apache.usergrid.persistence.core.astyanax.CassandraFig; +import org.apache.usergrid.persistence.core.CassandraFig; import org.apache.usergrid.persistence.core.scope.ApplicationScope; -import org.apache.usergrid.persistence.core.util.StringUtils; import org.apache.usergrid.persistence.index.IndexAlias; import org.apache.usergrid.persistence.index.IndexFig; import org.apache.usergrid.persistence.index.IndexLocationStrategy; diff --git a/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/SNSQueueManagerImpl.java b/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/SNSQueueManagerImpl.java index 4028d4679d..ed22fea9cf 100644 --- a/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/SNSQueueManagerImpl.java +++ b/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/SNSQueueManagerImpl.java @@ -31,7 +31,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.apache.usergrid.persistence.core.astyanax.CassandraFig; +import org.apache.usergrid.persistence.core.CassandraFig; import org.apache.usergrid.persistence.core.executor.TaskExecutorFactory; import org.apache.usergrid.persistence.core.guicyfig.ClusterFig; import org.apache.usergrid.persistence.queue.Queue; From c3566713b30cc99bb7541f0069bf396008c2fe50 Mon Sep 17 00:00:00 2001 From: Michael Russo Date: Sun, 1 May 2016 14:32:05 +0800 Subject: [PATCH 18/30] Update Usergrid dependencies to 2.2.0-SNAPSHOT. --- stack/core/pom.xml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/stack/core/pom.xml b/stack/core/pom.xml index 75d6007300..4b2252e34a 100644 --- a/stack/core/pom.xml +++ b/stack/core/pom.xml @@ -21,7 +21,7 @@ org.apache.usergrid usergrid - 2.1.1-SNAPSHOT + 2.2.0-SNAPSHOT ../ @@ -362,7 +362,7 @@ org.apache.usergrid common - 2.1.1-SNAPSHOT + 2.2.0-SNAPSHOT test-jar test @@ -372,7 +372,7 @@ org.apache.usergrid collection - 2.1.1-SNAPSHOT + 2.2.0-SNAPSHOT jar @@ -390,34 +390,34 @@ org.apache.usergrid queryindex - 2.1.1-SNAPSHOT + 2.2.0-SNAPSHOT jar org.apache.usergrid graph - 2.1.1-SNAPSHOT + 2.2.0-SNAPSHOT jar org.apache.usergrid map - 2.1.1-SNAPSHOT + 2.2.0-SNAPSHOT jar org.apache.usergrid cache - 2.1.1-SNAPSHOT + 2.2.0-SNAPSHOT org.apache.usergrid queue - 2.1.1-SNAPSHOT + 2.2.0-SNAPSHOT jar From 6a1fd223b3e0f1a677af1d6b0d0c87b25d525da7 Mon Sep 17 00:00:00 2001 From: Michael Russo Date: Sun, 1 May 2016 14:37:08 +0800 Subject: [PATCH 19/30] Fix compile issue post-merge with master to get current. --- .../apache/usergrid/persistence/core/CassandraFig.java | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/CassandraFig.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/CassandraFig.java index 184eeee88b..d8620be2e9 100644 --- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/CassandraFig.java +++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/CassandraFig.java @@ -90,11 +90,11 @@ public interface CassandraFig extends GuicyFig { @Default( "9160" ) int getThriftPort(); - @Key( "cassandra.username" ) + @Key( USERNAME ) @Default( DEFAULT_USERNAME ) String getUsername(); - @Key( "cassandra.password" ) + @Key( PASSWORD ) @Default( DEFAULT_PASSWORD ) String getPassword(); @@ -102,12 +102,6 @@ public interface CassandraFig extends GuicyFig { @Default( DEFAULT_LOCAL_DC ) String getLocalDataCenter(); - @Key( USERNAME ) - String getUsername(); - - @Key( PASSWORD ) - String getPassword(); - @Key( "cassandra.connections" ) @Default( DEFAULT_CONNECTION_POOLSIZE ) int getConnections(); From 4475158f1b48aa485d6af7d90caa91948ac2f46f Mon Sep 17 00:00:00 2001 From: Michael Russo Date: Sun, 1 May 2016 16:56:56 +0800 Subject: [PATCH 20/30] Re-write SCOPED_CACHE serialization to use Datastax driver and CQL. --- .../impl/ScopedCacheSerializationImpl.java | 311 ++++++++++-------- 1 file changed, 170 insertions(+), 141 deletions(-) diff --git a/stack/corepersistence/cache/src/main/java/org/apache/usergrid/persistence/cache/impl/ScopedCacheSerializationImpl.java b/stack/corepersistence/cache/src/main/java/org/apache/usergrid/persistence/cache/impl/ScopedCacheSerializationImpl.java index 1646d369bc..ffa5f1fa4f 100644 --- a/stack/corepersistence/cache/src/main/java/org/apache/usergrid/persistence/cache/impl/ScopedCacheSerializationImpl.java +++ b/stack/corepersistence/cache/src/main/java/org/apache/usergrid/persistence/cache/impl/ScopedCacheSerializationImpl.java @@ -16,10 +16,16 @@ */ package org.apache.usergrid.persistence.cache.impl; +import com.datastax.driver.core.*; +import com.datastax.driver.core.querybuilder.Clause; +import com.datastax.driver.core.querybuilder.QueryBuilder; +import com.datastax.driver.core.querybuilder.Using; import com.fasterxml.jackson.annotation.JsonAutoDetect; import com.fasterxml.jackson.annotation.PropertyAccessor; +import com.fasterxml.jackson.core.JsonParseException; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.JsonMappingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Preconditions; import com.google.common.hash.Funnel; @@ -39,6 +45,7 @@ import org.apache.cassandra.db.marshal.BytesType; import org.apache.usergrid.persistence.cache.CacheScope; import org.apache.usergrid.persistence.core.astyanax.*; +import org.apache.usergrid.persistence.core.datastax.CQLUtils; import org.apache.usergrid.persistence.core.datastax.TableDefinition; import org.apache.usergrid.persistence.core.shard.ExpandingShardLocator; import org.apache.usergrid.persistence.core.shard.StringHashUtils; @@ -46,9 +53,8 @@ import org.slf4j.LoggerFactory; import java.io.IOException; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; +import java.nio.ByteBuffer; +import java.util.*; import java.util.concurrent.Callable; @@ -57,55 +63,50 @@ */ public class ScopedCacheSerializationImpl implements ScopedCacheSerialization { - // row-keys are application ID + consistent hash key - // column names are K key toString() - // column values are serialization of V value - public static final Logger logger = LoggerFactory.getLogger(ScopedCacheSerializationImpl.class); + // row-keys are (app UUID, application type, app UUID as string, consistent hash int as bucket number) + // column names are K key toString() + // column values are serialization of V value - private static final CacheRowKeySerializer ROWKEY_SERIALIZER = new CacheRowKeySerializer(); - - private static final BucketScopedRowKeySerializer BUCKET_ROWKEY_SERIALIZER = - new BucketScopedRowKeySerializer<>( ROWKEY_SERIALIZER ); - - private static final Serializer COLUMN_NAME_SERIALIZER = StringSerializer.get(); + private static final String SCOPED_CACHE_TABLE = CQLUtils.quote("SCOPED_CACHE"); + private static final Collection SCOPED_CACHE_PARTITION_KEYS = Collections.singletonList("key"); + private static final Collection SCOPED_CACHE_COLUMN_KEYS = Collections.singletonList("column1"); + private static final Map SCOPED_CACHE_COLUMNS = + new HashMap() {{ + put( "key", DataType.Name.BLOB ); + put( "column1", DataType.Name.BLOB ); + put( "value", DataType.Name.BLOB ); }}; + private static final Map SCOPED_CACHE_CLUSTERING_ORDER = + new HashMap(){{ put( "column1", "ASC" ); }}; - private static final ObjectSerializer COLUMN_VALUE_SERIALIZER = ObjectSerializer.get(); - public static final MultiTenantColumnFamily, String> SCOPED_CACHE - = new MultiTenantColumnFamily<>( "SCOPED_CACHE", - BUCKET_ROWKEY_SERIALIZER, COLUMN_NAME_SERIALIZER, COLUMN_VALUE_SERIALIZER ); /** Number of buckets to hash across */ private static final int[] NUM_BUCKETS = {20}; /** How to funnel keys for buckets */ - private static final Funnel MAP_KEY_FUNNEL = new Funnel() { - - @Override - public void funnel( final String key, final PrimitiveSink into ) { - into.putString(key, StringHashUtils.UTF8); - } - }; + private static final Funnel MAP_KEY_FUNNEL = + (Funnel) (key, into) -> into.putString(key, StringHashUtils.UTF8); - /** - * Locator to get us all buckets - */ + /** Locator to get us all buckets */ private static final ExpandingShardLocator BUCKET_LOCATOR = new ExpandingShardLocator<>(MAP_KEY_FUNNEL, NUM_BUCKETS); - private final Keyspace keyspace; + private final Session session; + private final CassandraConfig cassandraConfig; private final ObjectMapper MAPPER = new ObjectMapper(); - //------------------------------------------------------------------------------------------ + @Inject - public ScopedCacheSerializationImpl( final Keyspace keyspace ) { - this.keyspace = keyspace; - //MAPPER.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + public ScopedCacheSerializationImpl( final Session session, + final CassandraConfig cassandraConfig ) { + this.session = session; + this.cassandraConfig = cassandraConfig; + MAPPER.enableDefaultTyping(); MAPPER.setVisibility(PropertyAccessor.ALL, JsonAutoDetect.Visibility.NONE); MAPPER.setVisibility(PropertyAccessor.FIELD, JsonAutoDetect.Visibility.ANY); @@ -115,77 +116,76 @@ public ScopedCacheSerializationImpl( final Keyspace keyspace ) { @Override public V readValue(CacheScope scope, K key, TypeReference typeRef ) { + return readValueCQL( scope, key, typeRef); + + } + + + private V readValueCQL(CacheScope scope, K key, TypeReference typeRef){ + Preconditions.checkNotNull(scope, "scope is required"); Preconditions.checkNotNull(key, "key is required"); - // determine bucketed row-key based application UUID - String rowKeyString = scope.getApplication().getUuid().toString(); + final String rowKeyString = scope.getApplication().getUuid().toString(); final int bucket = BUCKET_LOCATOR.getCurrentBucket(rowKeyString); - final BucketScopedRowKey keyRowKey = - BucketScopedRowKey.fromKey(scope.getApplication(), rowKeyString, bucket); // determine column name based on K key to string - String columnName = key.toString(); + final String columnName = key.toString(); - try { - try { - Column result = keyspace.prepareQuery(SCOPED_CACHE) - .getKey(keyRowKey).getColumn( columnName ).execute().getResult(); - - result.getByteBufferValue(); - //V value = MAPPER.readValue(result.getByteArrayValue(), new TypeReference() {}); - V value = MAPPER.readValue(result.getByteArrayValue(), typeRef); - - logger.debug("Read cache item from scope {}\n key/value types {}/{}\n key:value: {}:{}", - scope.getApplication().getUuid(), - key.getClass().getSimpleName(), - value.getClass().getSimpleName(), - key, - value); - - return value; - - } catch (NotFoundException nfe) { - if(logger.isDebugEnabled()) { - logger.debug("Value not found"); - } - - } catch (IOException ioe) { - logger.error("Unable to read cached value", ioe); - throw new RuntimeException("Unable to read cached value", ioe); + final Clause inKey = QueryBuilder.eq("key", getPartitionKey(scope, rowKeyString, bucket) ); + final Clause inColumn = QueryBuilder.eq("column1", DataType.text().serialize(columnName, ProtocolVersion.NEWEST_SUPPORTED) ); + + final Statement statement = QueryBuilder.select().all().from(SCOPED_CACHE_TABLE) + .where(inKey) + .and(inColumn) + .setConsistencyLevel(cassandraConfig.getDataStaxReadCl()); + + final ResultSet resultSet = session.execute(statement); + final com.datastax.driver.core.Row row = resultSet.one(); + + if (row == null){ + + if(logger.isDebugEnabled()){ + logger.debug("Cache value not found for key {}", key ); } - } catch (ConnectionException e) { - throw new RuntimeException("Unable to connect to cassandra", e); + return null; } - if(logger.isDebugEnabled()){ - logger.debug("Cache value not found for key {}", key ); + try { + + return MAPPER.readValue(row.getBytes("value").array(), typeRef); + + } catch (IOException ioe) { + logger.error("Unable to read cached value", ioe); + throw new RuntimeException("Unable to read cached value", ioe); } - return null; + } @Override public V writeValue(CacheScope scope, K key, V value, Integer ttl) { + return writeValueCQL( scope, key, value, ttl); + + } + + private V writeValueCQL(CacheScope scope, K key, V value, Integer ttl) { + Preconditions.checkNotNull( scope, "scope is required"); Preconditions.checkNotNull( key, "key is required" ); Preconditions.checkNotNull( value, "value is required"); Preconditions.checkNotNull( ttl, "ttl is required"); - // determine bucketed row-key based application UUID - String rowKeyString = scope.getApplication().getUuid().toString(); + final String rowKeyString = scope.getApplication().getUuid().toString(); final int bucket = BUCKET_LOCATOR.getCurrentBucket(rowKeyString); - final BucketScopedRowKey keyRowKey = - BucketScopedRowKey.fromKey(scope.getApplication(), rowKeyString, bucket); - // determine column name based on K key to string - String columnName = key.toString(); + final String columnName = key.toString(); // serialize cache item byte[] cacheBytes; @@ -195,127 +195,156 @@ public V writeValue(CacheScope scope, K key, V value, Integer ttl) { throw new RuntimeException("Unable to serialize cache value", jpe); } - // serialize to the entry - final MutationBatch batch = keyspace.prepareMutationBatch(); - batch.withRow(SCOPED_CACHE, keyRowKey).putColumn(columnName, cacheBytes, ttl); + final Using timeToLive = QueryBuilder.ttl(ttl); + + + // convert to ByteBuffer for the blob DataType in Cassandra + final ByteBuffer bb = ByteBuffer.allocate(cacheBytes.length); + bb.put(cacheBytes); + bb.flip(); + + final Statement cacheEntry = QueryBuilder.insertInto(SCOPED_CACHE_TABLE) + .using(timeToLive) + .value("key", getPartitionKey(scope, rowKeyString, bucket)) + .value("column1", DataType.text().serialize(columnName, ProtocolVersion.NEWEST_SUPPORTED)) + .value("value", bb); + - executeBatch(batch); + session.execute(cacheEntry); logger.debug("Wrote cache item to scope {}\n key/value types {}/{}\n key:value: {}:{}", - scope.getApplication().getUuid(), - key.getClass().getSimpleName(), - value.getClass().getSimpleName(), - key, - value); + scope.getApplication().getUuid(), + key.getClass().getSimpleName(), + value.getClass().getSimpleName(), + key, + value); return value; + } + @Override public void removeValue(CacheScope scope, K key) { + removeValueCQL(scope, key); + + } + + + private void removeValueCQL(CacheScope scope, K key) { + Preconditions.checkNotNull( scope, "scope is required"); Preconditions.checkNotNull( key, "key is required" ); // determine bucketed row-key based application UUID - String rowKeyString = scope.getApplication().getUuid().toString(); + final String rowKeyString = scope.getApplication().getUuid().toString(); final int bucket = BUCKET_LOCATOR.getCurrentBucket(rowKeyString); - final BucketScopedRowKey keyRowKey = - BucketScopedRowKey.fromKey(scope.getApplication(), rowKeyString, bucket); - // determine column name based on K key to string - String columnName = key.toString(); + final String columnName = key.toString(); - final MutationBatch batch = keyspace.prepareMutationBatch(); - batch.withRow(SCOPED_CACHE, keyRowKey).deleteColumn(columnName); - executeBatch(batch); - } + final Clause inKey = QueryBuilder.eq("key", getPartitionKey(scope, rowKeyString, bucket) ); + final Clause inColumn = QueryBuilder.eq("column1", DataType.text().serialize(columnName, ProtocolVersion.NEWEST_SUPPORTED) ); + + final Statement statement = QueryBuilder.delete().from(SCOPED_CACHE_TABLE) + .where(inKey) + .and(inColumn); + session.execute(statement); + } + + @Override public void invalidate(CacheScope scope) { + invalidateCQL(scope); + logger.debug("Invalidated scope {}", scope.getApplication().getUuid()); + + } + + private void invalidateCQL(CacheScope scope){ + Preconditions.checkNotNull(scope, "scope is required"); // determine bucketed row-key based application UUID - String rowKeyString = scope.getApplication().getUuid().toString(); + final String rowKeyString = scope.getApplication().getUuid().toString(); final int bucket = BUCKET_LOCATOR.getCurrentBucket(rowKeyString); - final BucketScopedRowKey keyRowKey = - BucketScopedRowKey.fromKey(scope.getApplication(), rowKeyString, bucket); - final MutationBatch batch = keyspace.prepareMutationBatch(); + final Clause inKey = QueryBuilder.eq("key", getPartitionKey(scope, rowKeyString, bucket) ); - batch.withRow(SCOPED_CACHE, keyRowKey).delete(); + final Statement statement = QueryBuilder.delete().from(SCOPED_CACHE_TABLE) + .where(inKey); - final OperationResult result = executeBatch(batch); + session.execute(statement); - logger.debug("Invalidated scope {}", scope.getApplication().getUuid()); } + @Override + public Collection getColumnFamilies() { - private class MutationBatchExec implements Callable { - private final MutationBatch myBatch; - private MutationBatchExec(MutationBatch batch) { - myBatch = batch; - } - @Override - public Void call() throws Exception { - myBatch.execute(); - return null; - } + return Collections.emptyList(); } + @Override + public Collection getTables() { - private OperationResult executeBatch(MutationBatch batch) { - try { - return batch.execute(); + final TableDefinition scopedCache = + new TableDefinition( SCOPED_CACHE_TABLE, SCOPED_CACHE_PARTITION_KEYS, SCOPED_CACHE_COLUMN_KEYS, + SCOPED_CACHE_COLUMNS, TableDefinition.CacheOption.KEYS, SCOPED_CACHE_CLUSTERING_ORDER); - } catch (ConnectionException e) { - throw new RuntimeException("Unable to connect to cassandra", e); - } + return Collections.singletonList(scopedCache); } - //------------------------------------------------------------------------------------------ - @Override - public Collection getColumnFamilies() { - final MultiTenantColumnFamilyDefinition scopedCache = - new MultiTenantColumnFamilyDefinition( SCOPED_CACHE, - BytesType.class.getSimpleName(), - BytesType.class.getSimpleName(), - BytesType.class.getSimpleName(), - MultiTenantColumnFamilyDefinition.CacheOption.KEYS ); + private ByteBuffer getPartitionKey(CacheScope scope, String key, int bucketNumber){ + + return serializeKeys(scope.getApplication().getUuid(), + scope.getApplication().getType(), bucketNumber, key); - return Collections.singletonList(scopedCache); } - @Override - public Collection getTables() { + private static ByteBuffer serializeKeys(UUID ownerUUID, String ownerType, int bucketNumber, String rowKeyString ){ - return Collections.emptyList(); - } + List keys = new ArrayList<>(4); + keys.add(0, ownerUUID); + keys.add(1, ownerType); + keys.add(2, bucketNumber); + keys.add(3, rowKeyString); + // UUIDs are 16 bytes, allocate the buffer accordingly + int size = 16+ownerType.length()+rowKeyString.length(); - /** - * Inner class to serialize cache key - */ - private static class CacheRowKeySerializer implements CompositeFieldSerializer { + // ints are 4 bytes, add for the bucket + size += 4; + + + // we always need to add length for the 2 byte short and 1 byte equality + size += keys.size()*3; + + ByteBuffer stuff = ByteBuffer.allocate(size); + + for (Object key : keys) { + + ByteBuffer kb = DataType.serializeValue(key, ProtocolVersion.NEWEST_SUPPORTED); + if (kb == null) { + kb = ByteBuffer.allocate(0); + } + + stuff.putShort((short) kb.remaining()); + stuff.put(kb.slice()); + stuff.put((byte) 0); - @Override - public void toComposite( final CompositeBuilder builder, final String key ) { - builder.addString(key); - } - @Override - public String fromComposite( final CompositeParser composite ) { - final String key = composite.readString(); - return key; } + stuff.flip(); + return stuff.duplicate(); + } } From f9a29e803cab3940d5e3521b16201cd95067c460 Mon Sep 17 00:00:00 2001 From: Michael Russo Date: Sun, 1 May 2016 16:57:45 +0800 Subject: [PATCH 21/30] Remove Astyanax dependencies for Scoped Cache serialization. --- .../impl/ScopedCacheSerializationImpl.java | 18 +----------------- 1 file changed, 1 insertion(+), 17 deletions(-) diff --git a/stack/corepersistence/cache/src/main/java/org/apache/usergrid/persistence/cache/impl/ScopedCacheSerializationImpl.java b/stack/corepersistence/cache/src/main/java/org/apache/usergrid/persistence/cache/impl/ScopedCacheSerializationImpl.java index ffa5f1fa4f..f8dccccfbf 100644 --- a/stack/corepersistence/cache/src/main/java/org/apache/usergrid/persistence/cache/impl/ScopedCacheSerializationImpl.java +++ b/stack/corepersistence/cache/src/main/java/org/apache/usergrid/persistence/cache/impl/ScopedCacheSerializationImpl.java @@ -22,27 +22,12 @@ import com.datastax.driver.core.querybuilder.Using; import com.fasterxml.jackson.annotation.JsonAutoDetect; import com.fasterxml.jackson.annotation.PropertyAccessor; -import com.fasterxml.jackson.core.JsonParseException; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.JsonMappingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Preconditions; import com.google.common.hash.Funnel; -import com.google.common.hash.PrimitiveSink; import com.google.inject.Inject; -import com.netflix.astyanax.Keyspace; -import com.netflix.astyanax.MutationBatch; -import com.netflix.astyanax.Serializer; -import com.netflix.astyanax.connectionpool.OperationResult; -import com.netflix.astyanax.connectionpool.exceptions.ConnectionException; -import com.netflix.astyanax.connectionpool.exceptions.NotFoundException; -import com.netflix.astyanax.model.Column; -import com.netflix.astyanax.model.CompositeBuilder; -import com.netflix.astyanax.model.CompositeParser; -import com.netflix.astyanax.serializers.ObjectSerializer; -import com.netflix.astyanax.serializers.StringSerializer; -import org.apache.cassandra.db.marshal.BytesType; import org.apache.usergrid.persistence.cache.CacheScope; import org.apache.usergrid.persistence.core.astyanax.*; import org.apache.usergrid.persistence.core.datastax.CQLUtils; @@ -55,7 +40,6 @@ import java.io.IOException; import java.nio.ByteBuffer; import java.util.*; -import java.util.concurrent.Callable; /** @@ -258,7 +242,7 @@ private void removeValueCQL(CacheScope scope, K key) { } - + @Override public void invalidate(CacheScope scope) { From 190e51992c97da7e863d399eafa9cc7455d8121c Mon Sep 17 00:00:00 2001 From: Michael Russo Date: Sun, 1 May 2016 17:37:22 +0800 Subject: [PATCH 22/30] Fix file naming issues ( had to use git mv for git to pickup case changes in filenames). --- ...atastaxSessionProvider.java => DataStaxSessionProvider.java} | 0 .../impl/{DatastaxClusterImpl.java => DataStaxClusterImpl.java} | 0 .../{DatastaxClusterTest.java => DataStaxClusterTest.java} | 2 +- 3 files changed, 1 insertion(+), 1 deletion(-) rename stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/{DatastaxSessionProvider.java => DataStaxSessionProvider.java} (100%) rename stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/impl/{DatastaxClusterImpl.java => DataStaxClusterImpl.java} (100%) rename stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/datastax/{DatastaxClusterTest.java => DataStaxClusterTest.java} (98%) diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/DatastaxSessionProvider.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/DataStaxSessionProvider.java similarity index 100% rename from stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/DatastaxSessionProvider.java rename to stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/DataStaxSessionProvider.java diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/impl/DatastaxClusterImpl.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/impl/DataStaxClusterImpl.java similarity index 100% rename from stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/impl/DatastaxClusterImpl.java rename to stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/impl/DataStaxClusterImpl.java diff --git a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/datastax/DatastaxClusterTest.java b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/datastax/DataStaxClusterTest.java similarity index 98% rename from stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/datastax/DatastaxClusterTest.java rename to stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/datastax/DataStaxClusterTest.java index ca7f97a49b..227f11d9ce 100644 --- a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/datastax/DatastaxClusterTest.java +++ b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/datastax/DataStaxClusterTest.java @@ -32,7 +32,7 @@ @RunWith( ITRunner.class ) @UseModules( TestCommonModule.class ) -public class DatastaxClusterTest { +public class DataStaxClusterTest { @Inject From f28abda041378884999e665e1af073e24557f953 Mon Sep 17 00:00:00 2001 From: Michael Russo Date: Sun, 1 May 2016 17:43:20 +0800 Subject: [PATCH 23/30] Move CassandraConfig out of astyanax package. --- .../persistence/cache/impl/ScopedCacheSerializationImpl.java | 1 + .../collection/mvcc/stage/write/WriteUniqueVerify.java | 2 +- .../collection/mvcc/stage/write/WriteUniqueVerifyTest.java | 2 +- .../persistence/core/{astyanax => }/CassandraConfig.java | 2 +- .../persistence/core/{astyanax => }/CassandraConfigImpl.java | 3 +-- .../persistence/core/astyanax/CassandraClusterImpl.java | 1 + .../apache/usergrid/persistence/core/guice/CommonModule.java | 2 ++ .../persistence/core/astyanax/ColumnNameIteratorTest.java | 1 + .../core/astyanax/MultiKeyColumnNameIteratorTest.java | 1 + .../persistence/core/astyanax/MultiRowColumnIteratorTest.java | 1 + .../serialization/impl/EdgeMetadataSerializationV1Impl.java | 2 +- .../serialization/impl/EdgeMetadataSerializationV2Impl.java | 2 +- .../graph/serialization/impl/EdgeSerializationImpl.java | 2 +- .../graph/serialization/impl/NodeSerializationImpl.java | 2 +- .../impl/shard/impl/EdgeShardSerializationImpl.java | 4 +--- .../impl/shard/impl/ShardedEdgeSerializationImpl.java | 2 +- .../usergrid/persistence/map/impl/MapSerializationImpl.java | 2 +- 17 files changed, 18 insertions(+), 14 deletions(-) rename stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/{astyanax => }/CassandraConfig.java (97%) rename stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/{astyanax => }/CassandraConfigImpl.java (97%) diff --git a/stack/corepersistence/cache/src/main/java/org/apache/usergrid/persistence/cache/impl/ScopedCacheSerializationImpl.java b/stack/corepersistence/cache/src/main/java/org/apache/usergrid/persistence/cache/impl/ScopedCacheSerializationImpl.java index f8dccccfbf..133465045c 100644 --- a/stack/corepersistence/cache/src/main/java/org/apache/usergrid/persistence/cache/impl/ScopedCacheSerializationImpl.java +++ b/stack/corepersistence/cache/src/main/java/org/apache/usergrid/persistence/cache/impl/ScopedCacheSerializationImpl.java @@ -29,6 +29,7 @@ import com.google.common.hash.Funnel; import com.google.inject.Inject; import org.apache.usergrid.persistence.cache.CacheScope; +import org.apache.usergrid.persistence.core.CassandraConfig; import org.apache.usergrid.persistence.core.astyanax.*; import org.apache.usergrid.persistence.core.datastax.CQLUtils; import org.apache.usergrid.persistence.core.datastax.TableDefinition; diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java index d05f8381ea..585c26e27a 100644 --- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java +++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java @@ -35,7 +35,7 @@ import org.apache.usergrid.persistence.collection.serialization.UniqueValueSerializationStrategy; import org.apache.usergrid.persistence.collection.serialization.UniqueValueSet; import org.apache.usergrid.persistence.collection.serialization.impl.UniqueValueImpl; -import org.apache.usergrid.persistence.core.astyanax.CassandraConfig; +import org.apache.usergrid.persistence.core.CassandraConfig; import org.apache.usergrid.persistence.core.scope.ApplicationScope; import org.apache.usergrid.persistence.model.entity.Entity; import org.apache.usergrid.persistence.model.entity.Id; diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerifyTest.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerifyTest.java index b9a1565120..09876fbf34 100644 --- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerifyTest.java +++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerifyTest.java @@ -27,7 +27,7 @@ import org.apache.usergrid.persistence.collection.mvcc.stage.CollectionIoEvent; import org.apache.usergrid.persistence.collection.serialization.SerializationFig; import org.apache.usergrid.persistence.collection.serialization.UniqueValueSerializationStrategy; -import org.apache.usergrid.persistence.core.astyanax.CassandraConfig; +import org.apache.usergrid.persistence.core.CassandraConfig; import org.apache.usergrid.persistence.core.guice.MigrationManagerRule; import org.apache.usergrid.persistence.core.scope.ApplicationScope; import org.apache.usergrid.persistence.core.test.ITRunner; diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/CassandraConfig.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/CassandraConfig.java similarity index 97% rename from stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/CassandraConfig.java rename to stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/CassandraConfig.java index 8cb96ac232..595b65f6f3 100644 --- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/CassandraConfig.java +++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/CassandraConfig.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.usergrid.persistence.core.astyanax; +package org.apache.usergrid.persistence.core; import com.netflix.astyanax.model.ConsistencyLevel; diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/CassandraConfigImpl.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/CassandraConfigImpl.java similarity index 97% rename from stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/CassandraConfigImpl.java rename to stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/CassandraConfigImpl.java index 0135c95a17..5dbf162ccb 100644 --- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/CassandraConfigImpl.java +++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/CassandraConfigImpl.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.usergrid.persistence.core.astyanax; +package org.apache.usergrid.persistence.core; import java.beans.PropertyChangeEvent; @@ -26,7 +26,6 @@ import com.google.inject.Inject; import com.google.inject.Singleton; import com.netflix.astyanax.model.ConsistencyLevel; -import org.apache.usergrid.persistence.core.CassandraFig; /** diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/CassandraClusterImpl.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/CassandraClusterImpl.java index 06fadfc3d8..9f5d4accf9 100644 --- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/CassandraClusterImpl.java +++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/CassandraClusterImpl.java @@ -36,6 +36,7 @@ import com.netflix.astyanax.connectionpool.impl.Slf4jConnectionPoolMonitorImpl; import com.netflix.astyanax.impl.AstyanaxConfigurationImpl; import com.netflix.astyanax.thrift.ThriftFamilyFactory; +import org.apache.usergrid.persistence.core.CassandraConfig; import org.apache.usergrid.persistence.core.CassandraFig; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/guice/CommonModule.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/guice/CommonModule.java index 94aac6019d..46816744f0 100644 --- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/guice/CommonModule.java +++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/guice/CommonModule.java @@ -21,6 +21,8 @@ import com.datastax.driver.core.Session; import com.netflix.astyanax.Keyspace; +import org.apache.usergrid.persistence.core.CassandraConfig; +import org.apache.usergrid.persistence.core.CassandraConfigImpl; import org.apache.usergrid.persistence.core.CassandraFig; import org.apache.usergrid.persistence.core.astyanax.*; import org.apache.usergrid.persistence.core.datastax.DataStaxCluster; diff --git a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/astyanax/ColumnNameIteratorTest.java b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/astyanax/ColumnNameIteratorTest.java index 415eafac6d..caa6294c7c 100644 --- a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/astyanax/ColumnNameIteratorTest.java +++ b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/astyanax/ColumnNameIteratorTest.java @@ -24,6 +24,7 @@ import java.util.HashMap; +import org.apache.usergrid.persistence.core.CassandraConfig; import org.apache.usergrid.persistence.core.CassandraFig; import org.junit.Before; import org.junit.Test; diff --git a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/astyanax/MultiKeyColumnNameIteratorTest.java b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/astyanax/MultiKeyColumnNameIteratorTest.java index 55ed421cab..b31fa2f8b2 100644 --- a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/astyanax/MultiKeyColumnNameIteratorTest.java +++ b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/astyanax/MultiKeyColumnNameIteratorTest.java @@ -26,6 +26,7 @@ import java.util.Comparator; import java.util.HashMap; +import org.apache.usergrid.persistence.core.CassandraConfig; import org.apache.usergrid.persistence.core.CassandraFig; import org.junit.Before; import org.junit.Test; diff --git a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/astyanax/MultiRowColumnIteratorTest.java b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/astyanax/MultiRowColumnIteratorTest.java index 882febda2d..ea5359e6a2 100644 --- a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/astyanax/MultiRowColumnIteratorTest.java +++ b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/astyanax/MultiRowColumnIteratorTest.java @@ -28,6 +28,7 @@ import java.util.HashMap; import java.util.concurrent.CountDownLatch; +import org.apache.usergrid.persistence.core.CassandraConfig; import org.apache.usergrid.persistence.core.CassandraFig; import org.junit.Before; import org.junit.Test; diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgeMetadataSerializationV1Impl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgeMetadataSerializationV1Impl.java index 870ba164eb..d7a5c80a41 100644 --- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgeMetadataSerializationV1Impl.java +++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgeMetadataSerializationV1Impl.java @@ -28,7 +28,7 @@ import org.apache.cassandra.db.marshal.BytesType; import org.apache.cassandra.db.marshal.UTF8Type; -import org.apache.usergrid.persistence.core.astyanax.CassandraConfig; +import org.apache.usergrid.persistence.core.CassandraConfig; import org.apache.usergrid.persistence.core.astyanax.ColumnNameIterator; import org.apache.usergrid.persistence.core.astyanax.CompositeFieldSerializer; import org.apache.usergrid.persistence.core.astyanax.IdRowCompositeSerializer; diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgeMetadataSerializationV2Impl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgeMetadataSerializationV2Impl.java index c9faf5f87e..1f818644c1 100644 --- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgeMetadataSerializationV2Impl.java +++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgeMetadataSerializationV2Impl.java @@ -29,7 +29,7 @@ import org.apache.usergrid.persistence.core.astyanax.BucketScopedRowKey; import org.apache.usergrid.persistence.core.astyanax.BucketScopedRowKeySerializer; -import org.apache.usergrid.persistence.core.astyanax.CassandraConfig; +import org.apache.usergrid.persistence.core.CassandraConfig; import org.apache.usergrid.persistence.core.astyanax.ColumnSearch; import org.apache.usergrid.persistence.core.astyanax.CompositeFieldSerializer; import org.apache.usergrid.persistence.core.astyanax.IdRowCompositeSerializer; diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgeSerializationImpl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgeSerializationImpl.java index 0f4d722413..984365f86a 100644 --- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgeSerializationImpl.java +++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgeSerializationImpl.java @@ -27,7 +27,7 @@ import javax.inject.Inject; import com.google.common.base.Optional; -import org.apache.usergrid.persistence.core.astyanax.CassandraConfig; +import org.apache.usergrid.persistence.core.CassandraConfig; import org.apache.usergrid.persistence.core.consistency.TimeService; import org.apache.usergrid.persistence.core.scope.ApplicationScope; import org.apache.usergrid.persistence.core.util.ValidationUtils; diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/NodeSerializationImpl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/NodeSerializationImpl.java index 8a2da5446f..2429d5aeec 100644 --- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/NodeSerializationImpl.java +++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/NodeSerializationImpl.java @@ -32,7 +32,7 @@ import org.apache.cassandra.db.marshal.BooleanType; import org.apache.cassandra.db.marshal.BytesType; -import org.apache.usergrid.persistence.core.astyanax.CassandraConfig; +import org.apache.usergrid.persistence.core.CassandraConfig; import org.apache.usergrid.persistence.core.astyanax.IdRowCompositeSerializer; import org.apache.usergrid.persistence.core.astyanax.MultiTenantColumnFamily; import org.apache.usergrid.persistence.core.astyanax.MultiTenantColumnFamilyDefinition; diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/EdgeShardSerializationImpl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/EdgeShardSerializationImpl.java index 651bc64c71..098c1526d9 100644 --- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/EdgeShardSerializationImpl.java +++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/EdgeShardSerializationImpl.java @@ -24,10 +24,9 @@ import java.util.Collections; import java.util.Iterator; -import com.google.common.util.concurrent.ExecutionError; import org.apache.cassandra.db.marshal.BytesType; -import org.apache.usergrid.persistence.core.astyanax.CassandraConfig; +import org.apache.usergrid.persistence.core.CassandraConfig; import org.apache.usergrid.persistence.core.astyanax.ColumnNameIterator; import org.apache.usergrid.persistence.core.astyanax.ColumnParser; import org.apache.usergrid.persistence.core.astyanax.ColumnTypes; @@ -41,7 +40,6 @@ import org.apache.usergrid.persistence.graph.GraphFig; import org.apache.usergrid.persistence.graph.serialization.impl.shard.DirectedEdgeMeta; import org.apache.usergrid.persistence.graph.serialization.impl.shard.EdgeShardSerialization; -import org.apache.usergrid.persistence.graph.serialization.impl.shard.NodeShardCache; import org.apache.usergrid.persistence.graph.serialization.impl.shard.Shard; import org.apache.usergrid.persistence.graph.serialization.impl.shard.impl.serialize.EdgeShardRowKeySerializer; import org.apache.usergrid.persistence.graph.serialization.impl.shard.impl.serialize.ShardSerializer; diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardedEdgeSerializationImpl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardedEdgeSerializationImpl.java index 65a6f40238..8259df42d4 100644 --- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardedEdgeSerializationImpl.java +++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardedEdgeSerializationImpl.java @@ -30,7 +30,7 @@ import javax.inject.Inject; import com.google.common.base.Optional; -import org.apache.usergrid.persistence.core.astyanax.CassandraConfig; +import org.apache.usergrid.persistence.core.CassandraConfig; import org.apache.usergrid.persistence.core.astyanax.MultiTenantColumnFamily; import org.apache.usergrid.persistence.core.astyanax.ScopedRowKey; import org.apache.usergrid.persistence.core.consistency.TimeService; diff --git a/stack/corepersistence/map/src/main/java/org/apache/usergrid/persistence/map/impl/MapSerializationImpl.java b/stack/corepersistence/map/src/main/java/org/apache/usergrid/persistence/map/impl/MapSerializationImpl.java index e2ea681e65..735f2b819d 100644 --- a/stack/corepersistence/map/src/main/java/org/apache/usergrid/persistence/map/impl/MapSerializationImpl.java +++ b/stack/corepersistence/map/src/main/java/org/apache/usergrid/persistence/map/impl/MapSerializationImpl.java @@ -28,7 +28,7 @@ import com.datastax.driver.core.querybuilder.QueryBuilder; import com.datastax.driver.core.querybuilder.Using; -import org.apache.usergrid.persistence.core.astyanax.CassandraConfig; +import org.apache.usergrid.persistence.core.CassandraConfig; import org.apache.usergrid.persistence.core.astyanax.MultiTenantColumnFamilyDefinition; import org.apache.usergrid.persistence.core.datastax.CQLUtils; import org.apache.usergrid.persistence.core.datastax.TableDefinition; From ff3f7e899d81d636eddaa203141ee31ef2da097c Mon Sep 17 00:00:00 2001 From: Michael Russo Date: Sun, 1 May 2016 17:48:43 +0800 Subject: [PATCH 24/30] Add consistent read CL for datastax driver implementations. --- .../usergrid/persistence/core/CassandraConfigImpl.java | 2 +- .../org/apache/usergrid/persistence/core/CassandraFig.java | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/CassandraConfigImpl.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/CassandraConfigImpl.java index 5dbf162ccb..e87ebb8587 100644 --- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/CassandraConfigImpl.java +++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/CassandraConfigImpl.java @@ -61,7 +61,7 @@ public CassandraConfigImpl( final CassandraFig cassandraFig ) { this.dataStaxReadCl = com.datastax.driver.core.ConsistencyLevel.valueOf( cassandraFig.getReadCl()); - this.dataStaxReadConsistentCl = com.datastax.driver.core.ConsistencyLevel.valueOf( cassandraFig.getReadCl()); + this.dataStaxReadConsistentCl = com.datastax.driver.core.ConsistencyLevel.valueOf( cassandraFig.getReadClConsistent()); this.dataStaxWriteCl = com.datastax.driver.core.ConsistencyLevel.valueOf( cassandraFig.getWriteCl() ); diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/CassandraFig.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/CassandraFig.java index d8620be2e9..3c58dfba3c 100644 --- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/CassandraFig.java +++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/CassandraFig.java @@ -33,6 +33,7 @@ public interface CassandraFig extends GuicyFig { // cassndra properties used by datastax driver String READ_CL = "cassandra.readcl"; + String READ_CL_CONSISTENT = "cassandra.readcl.consistent"; String WRITE_CL = "cassandra.writecl"; String STRATEGY = "cassandra.strategy"; String STRATEGY_OPTIONS = "cassandra.strategy.options"; @@ -136,6 +137,10 @@ public interface CassandraFig extends GuicyFig { @Key(READ_CL) String getReadCl(); + @Default("QUORUM") + @Key(READ_CL_CONSISTENT) + String getReadClConsistent(); + @Default("LOCAL_QUORUM") @Key(WRITE_CL) String getWriteCl(); From 0c609878e1eccd35f31bc4fdc86bd3fe9da21593 Mon Sep 17 00:00:00 2001 From: Michael Russo Date: Fri, 6 May 2016 22:36:37 +0800 Subject: [PATCH 25/30] Initial UniqueValueSerialization conversion to CQL. --- .../mvcc/stage/write/WriteCommit.java | 18 +- .../mvcc/stage/write/WriteUniqueVerify.java | 23 +- .../UniqueValueSerializationStrategy.java | 6 +- .../UniqueValueSerializationStrategyImpl.java | 287 +++++++++++- ...ueValueSerializationStrategyProxyImpl.java | 31 +- ...niqueValueSerializationStrategyV1Impl.java | 410 +++++++++++++++++- ...niqueValueSerializationStrategyV2Impl.java | 379 +++++++++++++++- .../MvccEntityDataMigrationImpl.java | 26 +- .../mvcc/stage/delete/MarkCommitTest.java | 13 +- .../mvcc/stage/write/WriteCommitTest.java | 15 +- .../stage/write/WriteUniqueVerifyTest.java | 6 +- ...queValueSerializationStrategyImplTest.java | 41 +- ...MvccEntityDataMigrationV1ToV3ImplTest.java | 5 +- .../datastax/impl/DataStaxClusterImpl.java | 3 + 14 files changed, 1169 insertions(+), 94 deletions(-) diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteCommit.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteCommit.java index 7eb96e7ef5..cfac8e484b 100644 --- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteCommit.java +++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteCommit.java @@ -20,6 +20,8 @@ import java.util.UUID; +import com.datastax.driver.core.BatchStatement; +import com.datastax.driver.core.Session; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -67,11 +69,14 @@ public class WriteCommit implements Func1, Collect private final MvccEntitySerializationStrategy entityStrat; + private final Session session; + @Inject public WriteCommit( final MvccLogEntrySerializationStrategy logStrat, final MvccEntitySerializationStrategy entryStrat, - final UniqueValueSerializationStrategy uniqueValueStrat) { + final UniqueValueSerializationStrategy uniqueValueStrat, + final Session session) { Preconditions.checkNotNull( logStrat, "MvccLogEntrySerializationStrategy is required" ); Preconditions.checkNotNull( entryStrat, "MvccEntitySerializationStrategy is required" ); @@ -80,6 +85,7 @@ public WriteCommit( final MvccLogEntrySerializationStrategy logStrat, this.logEntryStrat = logStrat; this.entityStrat = entryStrat; this.uniqueValueStrat = uniqueValueStrat; + this.session = session; } @@ -103,6 +109,8 @@ public CollectionIoEvent call( final CollectionIoEvent i final MvccLogEntry startEntry = new MvccLogEntryImpl( entityId, version, Stage.COMMITTED, MvccLogEntry.State.COMPLETE ); + + MutationBatch logMutation = logEntryStrat.write( applicationScope, startEntry ); // now get our actual insert into the entity data @@ -112,21 +120,23 @@ public CollectionIoEvent call( final CollectionIoEvent i logMutation.mergeShallow( entityMutation ); // re-write the unique values but this time with no TTL + final BatchStatement uniqueBatch = new BatchStatement(); + for ( Field field : EntityUtils.getUniqueFields(mvccEntity.getEntity().get()) ) { UniqueValue written = new UniqueValueImpl( field, entityId,version); - MutationBatch mb = uniqueValueStrat.write(applicationScope, written ); + uniqueBatch.add(uniqueValueStrat.writeCQL(applicationScope, written, -1 )); logger.debug("Finalizing {} unique value {}", field.getName(), field.getValue().toString()); - // merge into our existing mutation batch - logMutation.mergeShallow( mb ); + } try { logMutation.execute(); + session.execute(uniqueBatch); } catch ( ConnectionException e ) { logger.error( "Failed to execute write asynchronously ", e ); diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java index 585c26e27a..8e0b20208f 100644 --- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java +++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java @@ -23,6 +23,8 @@ import java.util.List; import java.util.Map; +import com.datastax.driver.core.BatchStatement; +import com.datastax.driver.core.Session; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -71,14 +73,20 @@ public class WriteUniqueVerify implements Action1> protected final SerializationFig serializationFig; protected final Keyspace keyspace; + + protected final Session session; + private final CassandraConfig cassandraFig; @Inject public WriteUniqueVerify( final UniqueValueSerializationStrategy uniqueValueSerializiationStrategy, - final SerializationFig serializationFig, final Keyspace keyspace, final CassandraConfig cassandraFig ) { + final SerializationFig serializationFig, final Keyspace keyspace, + final CassandraConfig cassandraFig, final Session session ) { + this.keyspace = keyspace; this.cassandraFig = cassandraFig; + this.session = session; Preconditions.checkNotNull( uniqueValueSerializiationStrategy, "uniqueValueSerializationStrategy is required" ); Preconditions.checkNotNull( serializationFig, "serializationFig is required" ); @@ -101,7 +109,7 @@ public void call( final CollectionIoEvent ioevent ) { final ApplicationScope scope = ioevent.getEntityCollection(); - final MutationBatch batch = keyspace.prepareMutationBatch(); + final BatchStatement batch = new BatchStatement(); //allocate our max size, worst case final List uniqueFields = new ArrayList<>( entity.getFields().size() ); @@ -119,9 +127,8 @@ public void call( final CollectionIoEvent ioevent ) { final UniqueValue written = new UniqueValueImpl( field, mvccEntity.getId(), mvccEntity.getVersion() ); // use TTL in case something goes wrong before entity is finally committed - final MutationBatch mb = uniqueValueStrat.write( scope, written, serializationFig.getTimeout() ); + batch.add(uniqueValueStrat.writeCQL( scope, written, serializationFig.getTimeout() )); - batch.mergeShallow( mb ); uniqueFields.add(field); } @@ -131,12 +138,8 @@ public void call( final CollectionIoEvent ioevent ) { } //perform the write - try { - batch.execute(); - } - catch ( ConnectionException ex ) { - throw new RuntimeException( "Unable to write to cassandra", ex ); - } + session.execute(batch); + // use simple thread pool to verify fields in parallel ConsistentReplayCommand cmd = new ConsistentReplayCommand(uniqueValueStrat,cassandraFig,scope, entity.getId().getType(), uniqueFields,entity); diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/UniqueValueSerializationStrategy.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/UniqueValueSerializationStrategy.java index 3645107f2c..56e8b87c73 100644 --- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/UniqueValueSerializationStrategy.java +++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/UniqueValueSerializationStrategy.java @@ -21,6 +21,8 @@ import java.util.Collection; import java.util.Iterator; +import com.datastax.driver.core.BatchStatement; +import org.apache.usergrid.persistence.collection.serialization.impl.UniqueValueSerializationStrategyImpl; import org.apache.usergrid.persistence.core.migration.data.VersionedData; import org.apache.usergrid.persistence.core.migration.schema.Migration; import org.apache.usergrid.persistence.core.scope.ApplicationScope; @@ -46,7 +48,6 @@ public interface UniqueValueSerializationStrategy extends Migration, VersionedDa * * @return MutatationBatch that encapsulates operation, caller may or may not execute. */ - MutationBatch write( ApplicationScope applicationScope, UniqueValue uniqueValue ); /** * Write the specified UniqueValue to Cassandra with optional timeToLive in milliseconds. @@ -56,7 +57,8 @@ public interface UniqueValueSerializationStrategy extends Migration, VersionedDa * @param timeToLive How long object should live in seconds. -1 implies store forever * @return MutatationBatch that encapsulates operation, caller may or may not execute. */ - MutationBatch write( ApplicationScope applicationScope, UniqueValue uniqueValue, int timeToLive ); + + BatchStatement writeCQL(ApplicationScope applicationScope, UniqueValue uniqueValue, int timeToLive ); /** * Load UniqueValue that matches field from collection or null if that value does not exist. diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyImpl.java index 0f27167fe3..27a86095ef 100644 --- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyImpl.java +++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyImpl.java @@ -18,9 +18,19 @@ package org.apache.usergrid.persistence.collection.serialization.impl; +import java.nio.ByteBuffer; import java.util.*; +import com.datastax.driver.core.*; +import com.datastax.driver.core.querybuilder.Clause; +import com.datastax.driver.core.querybuilder.QueryBuilder; +import com.datastax.driver.core.querybuilder.Using; +import com.netflix.astyanax.model.*; +import com.netflix.astyanax.util.RangeBuilder; +import org.apache.usergrid.persistence.core.CassandraConfig; import org.apache.usergrid.persistence.core.datastax.TableDefinition; +import org.apache.usergrid.persistence.model.entity.SimpleId; +import org.apache.usergrid.persistence.model.field.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -40,18 +50,13 @@ import org.apache.usergrid.persistence.core.scope.ApplicationScope; import org.apache.usergrid.persistence.core.util.ValidationUtils; import org.apache.usergrid.persistence.model.entity.Id; -import org.apache.usergrid.persistence.model.field.Field; import com.google.common.base.Preconditions; import com.netflix.astyanax.ColumnListMutation; import com.netflix.astyanax.Keyspace; import com.netflix.astyanax.MutationBatch; import com.netflix.astyanax.connectionpool.exceptions.ConnectionException; -import com.netflix.astyanax.model.Column; -import com.netflix.astyanax.model.ConsistencyLevel; -import com.netflix.astyanax.model.Row; import com.netflix.astyanax.query.RowQuery; -import com.netflix.astyanax.util.RangeBuilder; /** @@ -62,6 +67,9 @@ public abstract class UniqueValueSerializationStrategyImpl private static final Logger log = LoggerFactory.getLogger( UniqueValueSerializationStrategyImpl.class ); + public static final String UUID_TYPE_REVERSED = "UUIDType(reversed=true)"; + + private final MultiTenantColumnFamily, EntityVersion> CF_UNIQUE_VALUES; @@ -70,6 +78,15 @@ public abstract class UniqueValueSerializationStrategyImpl private final MultiTenantColumnFamily, UniqueFieldEntry> CF_ENTITY_UNIQUE_VALUE_LOG ; + private final String TABLE_UNIQUE_VALUES; + private final String TABLE_UNIQUE_VALUES_LOG; + + + private final Map COLUMNS_UNIQUE_VALUES; + private final Map COLUMNS_UNIQUE_VALUES_LOG; + + + public static final int COL_VALUE = 0x0; @@ -77,6 +94,9 @@ public abstract class UniqueValueSerializationStrategyImpl protected final Keyspace keyspace; private final CassandraFig cassandraFig; + private final Session session; + private final CassandraConfig cassandraConfig; + /** * Construct serialization strategy for keyspace. @@ -86,13 +106,24 @@ public abstract class UniqueValueSerializationStrategyImpl * @param serializationFig The serialization configuration */ public UniqueValueSerializationStrategyImpl( final Keyspace keyspace, final CassandraFig cassandraFig, - final SerializationFig serializationFig ) { + final SerializationFig serializationFig, + final Session session, final CassandraConfig cassandraConfig) { this.keyspace = keyspace; this.cassandraFig = cassandraFig; this.serializationFig = serializationFig; + this.session = session; + this.cassandraConfig = cassandraConfig; + CF_UNIQUE_VALUES = getUniqueValuesCF(); CF_ENTITY_UNIQUE_VALUE_LOG = getEntityUniqueLogCF(); + + TABLE_UNIQUE_VALUES = getUniqueValuesTable().getTableName(); + TABLE_UNIQUE_VALUES_LOG = getEntityUniqueLogTable().getTableName(); + + COLUMNS_UNIQUE_VALUES = getUniqueValuesTable().getColumns(); + COLUMNS_UNIQUE_VALUES_LOG = getEntityUniqueLogTable().getColumns(); + } @@ -129,7 +160,6 @@ public void doLog( final ColumnListMutation colMutation ) { } - @Override public MutationBatch write( final ApplicationScope collectionScope, final UniqueValue value, final int timeToLive ) { @@ -163,6 +193,86 @@ public void doLog( final ColumnListMutation colMutation ) { } ); } + @Override + public BatchStatement writeCQL( final ApplicationScope collectionScope, final UniqueValue value, + final int timeToLive ){ + + + Preconditions.checkNotNull( value, "value is required" ); + + BatchStatement batch = new BatchStatement(); + + Using ttl = null; + if(timeToLive > 0){ + + ttl = QueryBuilder.ttl(timeToLive); + + } + + final Id entityId = value.getEntityId(); + final UUID entityVersion = value.getEntityVersion(); + final Field field = value.getField(); + + ValidationUtils.verifyIdentity( entityId ); + ValidationUtils.verifyVersion( entityVersion ); + + final EntityVersion ev = new EntityVersion( entityId, entityVersion ); + final UniqueFieldEntry uniqueFieldEntry = new UniqueFieldEntry( entityVersion, field ); + + ByteBuffer partitionKey = getPartitionKey(collectionScope.getApplication(), value.getEntityId().getType(), + field.getTypeName().toString(), field.getName(), field.getValue()); + + ByteBuffer logPartitionKey = getLogPartitionKey(collectionScope.getApplication(), value.getEntityId()); + + + if(ttl != null) { + + Statement uniqueValueStatement = QueryBuilder.insertInto(TABLE_UNIQUE_VALUES) + .value("key", partitionKey) + .value("column1", serializeUniqueValueColumn(ev)) + .value("value", DataType.serializeValue(COL_VALUE, ProtocolVersion.NEWEST_SUPPORTED)) + .using(ttl); + + batch.add(uniqueValueStatement); + + + }else{ + + Statement uniqueValueStatement = QueryBuilder.insertInto(TABLE_UNIQUE_VALUES) + .value("key", partitionKey) + .value("column1", serializeUniqueValueColumn(ev)) + .value("value", DataType.serializeValue(COL_VALUE, ProtocolVersion.NEWEST_SUPPORTED)); + + batch.add(uniqueValueStatement); + + } + + // we always want to retain the log entry, so never write with the TTL + Statement uniqueValueLogStatement = QueryBuilder.insertInto(TABLE_UNIQUE_VALUES_LOG) + .value("key", logPartitionKey) + .value("column1", serializeUniqueValueLogColumn(uniqueFieldEntry)) + .value("value", DataType.serializeValue(COL_VALUE, ProtocolVersion.NEWEST_SUPPORTED)); + + batch.add(uniqueValueLogStatement); + + + + return batch; + + /** + * @Override + public void doLookup( final ColumnListMutation colMutation ) { + colMutation.putColumn( ev, COL_VALUE ); + } + + + @Override + public void doLog( final ColumnListMutation colMutation ) { + colMutation.putColumn( uniqueFieldEntry, COL_VALUE ); + } + */ + } + @Override public MutationBatch delete( final ApplicationScope scope, UniqueValue value ) { @@ -236,18 +346,26 @@ private MutationBatch doWrite( ApplicationScope applicationScope, UniqueValue un @Override public UniqueValueSet load( final ApplicationScope colScope, final String type, final Collection fields ) throws ConnectionException { - return load( colScope, ConsistencyLevel.valueOf( cassandraFig.getAstyanaxReadCL() ), type, fields ); + return load( colScope, com.netflix.astyanax.model.ConsistencyLevel.valueOf( cassandraFig.getAstyanaxReadCL() ), type, fields ); } @Override - public UniqueValueSet load( final ApplicationScope appScope, final ConsistencyLevel consistencyLevel, + public UniqueValueSet load( final ApplicationScope appScope, final com.netflix.astyanax.model.ConsistencyLevel consistencyLevel, final String type, final Collection fields ) throws ConnectionException { Preconditions.checkNotNull( fields, "fields are required" ); Preconditions.checkArgument( fields.size() > 0, "More than 1 field must be specified" ); + return loadCQL(appScope, com.datastax.driver.core.ConsistencyLevel.LOCAL_QUORUM, type, fields); + //return loadLegacy( appScope, type, fields); + + } + + + private UniqueValueSet loadLegacy(final ApplicationScope appScope, + final String type, final Collection fields) throws ConnectionException { final List> keys = new ArrayList<>( fields.size() ); final Id applicationId = appScope.getApplication(); @@ -265,16 +383,16 @@ public UniqueValueSet load( final ApplicationScope appScope, final ConsistencyLe final UniqueValueSetImpl uniqueValueSet = new UniqueValueSetImpl( fields.size() ); - Iterator, EntityVersion>> results = - keyspace.prepareQuery( CF_UNIQUE_VALUES ).setConsistencyLevel( consistencyLevel ).getKeySlice( keys ) - .withColumnRange( new RangeBuilder().setLimit( 1 ).build() ).execute().getResult().iterator(); + Iterator, EntityVersion>> results = + keyspace.prepareQuery( CF_UNIQUE_VALUES ).setConsistencyLevel(com.netflix.astyanax.model.ConsistencyLevel.CL_LOCAL_QUORUM ).getKeySlice( keys ) + .withColumnRange( new RangeBuilder().setLimit( 1 ).build() ).execute().getResult().iterator(); while ( results.hasNext() ) { - final Row, EntityVersion> unique = results.next(); + final com.netflix.astyanax.model.Row, EntityVersion> unique = results.next(); final Field field = parseRowKey( unique.getKey() ); @@ -296,9 +414,112 @@ public UniqueValueSet load( final ApplicationScope appScope, final ConsistencyLe } return uniqueValueSet; + + } + + private UniqueValueSet loadCQL( final ApplicationScope appScope, final com.datastax.driver.core.ConsistencyLevel consistencyLevel, + final String type, final Collection fields ) throws ConnectionException { + + Preconditions.checkNotNull( fields, "fields are required" ); + Preconditions.checkArgument( fields.size() > 0, "More than 1 field must be specified" ); + + + final Id applicationId = appScope.getApplication(); + + // row key = app UUID + app type + entityType + field type + field name + field value + + List partitionKeys = new ArrayList<>( fields.size() ); + for ( Field field : fields ) { + + //log.info(Bytes.toHexString(getPartitionKey(applicationId, type, field.getTypeName().toString(), field.getName(), field.getValue()))); + + partitionKeys.add(getPartitionKey(applicationId, type, field.getTypeName().toString(), field.getName(), field.getValue())); + + } + + final UniqueValueSetImpl uniqueValueSet = new UniqueValueSetImpl( fields.size() ); + + final Clause inKey = QueryBuilder.in("key", partitionKeys ); + + final Statement statement = QueryBuilder.select().all().from(TABLE_UNIQUE_VALUES) + .where(inKey) + .setConsistencyLevel(com.datastax.driver.core.ConsistencyLevel.LOCAL_QUORUM); + + final ResultSet resultSet = session.execute(statement); + + + Iterator results = resultSet.iterator(); + + + while( results.hasNext() ){ + + final com.datastax.driver.core.Row unique = results.next(); + ByteBuffer partitionKey = unique.getBytes("key"); + ByteBuffer column = unique.getBytesUnsafe("column1"); + + List keyContents = deserializePartitionKey(partitionKey); + List columnContents = deserializeUniqueValueColumn(column); + + Field field = null; + FieldTypeName fieldType; + String name; + String value; + if(this instanceof UniqueValueSerializationStrategyV2Impl) { + + fieldType = FieldTypeName.valueOf((String) keyContents.get(3)); + name = (String) keyContents.get(4); + value = (String) keyContents.get(5); + + }else{ + + fieldType = FieldTypeName.valueOf((String) keyContents.get(5)); + name = (String) keyContents.get(6); + value = (String) keyContents.get(7); + + } + + switch ( fieldType ) { + case BOOLEAN: + field = new BooleanField( name, Boolean.parseBoolean( value ) ); + break; + case DOUBLE: + field = new DoubleField( name, Double.parseDouble( value ) ); + break; + case FLOAT: + field = new FloatField( name, Float.parseFloat( value ) ); + break; + case INTEGER: + field = new IntegerField( name, Integer.parseInt( value ) ); + break; + case LONG: + field = new LongField( name, Long.parseLong( value ) ); + break; + case STRING: + field = new StringField( name, value ); + break; + case UUID: + field = new UUIDField( name, UUID.fromString( value ) ); + break; + } + + final EntityVersion entityVersion = new EntityVersion( + new SimpleId((UUID)columnContents.get(1), (String)columnContents.get(2)), (UUID)columnContents.get(0)); + + + final UniqueValueImpl uniqueValue = + new UniqueValueImpl( field, entityVersion.getEntityId(), entityVersion.getEntityVersion() ); + + uniqueValueSet.addValue(uniqueValue); + + } + + return uniqueValueSet; + } + + @Override public Iterator getAllUniqueFields( final ApplicationScope collectionScope, final Id entityId ) { Preconditions.checkNotNull( collectionScope, "collectionScope is required" ); @@ -378,7 +599,13 @@ public Collection getColumnFamilies() { @Override public Collection getTables() { - return Collections.emptyList(); + final TableDefinition uniqueValues = getUniqueValuesTable(); + + final TableDefinition uniqueValuesLog = getEntityUniqueLogTable(); + + + return Arrays.asList( uniqueValues, uniqueValuesLog ); + } @@ -388,6 +615,12 @@ public Collection getTables() { protected abstract MultiTenantColumnFamily, EntityVersion> getUniqueValuesCF(); + /** + * Get the CQL table definition for the unique values log table + */ + protected abstract TableDefinition getUniqueValuesTable(); + + /** * Generate a key that is compatible with the column family * @@ -405,10 +638,32 @@ public Collection getTables() { protected abstract Field parseRowKey(final ScopedRowKey rowKey); + protected abstract List deserializePartitionKey(ByteBuffer bb); + + + protected abstract Object serializeUniqueValueLogColumn(UniqueFieldEntry fieldEntry); + + protected abstract ByteBuffer getPartitionKey(Id applicationId, String entityType, String fieldType, String fieldName, Object fieldValue ); + + protected abstract ByteBuffer getLogPartitionKey(final Id applicationId, final Id uniqueValueId); + + protected abstract ByteBuffer serializeUniqueValueColumn(EntityVersion entityVersion); + + protected abstract List deserializeUniqueValueColumn(ByteBuffer bb); + + + + + + /** + * Get the column family for the unique field CF + */ + protected abstract MultiTenantColumnFamily, UniqueFieldEntry> getEntityUniqueLogCF(); + /** - * Get the column family for the unique field CF + * Get the CQL table definition for the unique values log table */ - protected abstract MultiTenantColumnFamily, UniqueFieldEntry> getEntityUniqueLogCF(); + protected abstract TableDefinition getEntityUniqueLogTable(); /** * Generate a key that is compatible with the column family diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyProxyImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyProxyImpl.java index 87b164176b..bbfaa2dd37 100644 --- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyProxyImpl.java +++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyProxyImpl.java @@ -24,6 +24,7 @@ import java.util.Collections; import java.util.Iterator; +import com.datastax.driver.core.BatchStatement; import org.apache.usergrid.persistence.collection.serialization.UniqueValue; import org.apache.usergrid.persistence.collection.serialization.UniqueValueSerializationStrategy; import org.apache.usergrid.persistence.collection.serialization.UniqueValueSet; @@ -67,40 +68,22 @@ public UniqueValueSerializationStrategyProxyImpl( final Keyspace keyspace, @Override - public MutationBatch write( final ApplicationScope applicationScope, final UniqueValue uniqueValue ) { - final MigrationRelationship migration = getMigrationRelationShip(); - - if ( migration.needsMigration() ) { - final MutationBatch aggregateBatch = keyspace.prepareMutationBatch(); - - aggregateBatch.mergeShallow( migration.from.write( applicationScope, uniqueValue ) ); - aggregateBatch.mergeShallow( migration.to.write( applicationScope, uniqueValue ) ); - - return aggregateBatch; - } + public BatchStatement writeCQL(final ApplicationScope applicationScope, final UniqueValue uniqueValue, + final int timeToLive ){ - return migration.to.write( applicationScope, uniqueValue ); - } - - - @Override - public MutationBatch write( final ApplicationScope applicationScope, final UniqueValue uniqueValue, - final int timeToLive ) { final MigrationRelationship migration = getMigrationRelationShip(); if ( migration.needsMigration() ) { - final MutationBatch aggregateBatch = keyspace.prepareMutationBatch(); - - aggregateBatch.mergeShallow( migration.from.write( applicationScope, uniqueValue, timeToLive ) ); - aggregateBatch.mergeShallow( migration.to.write( applicationScope, uniqueValue, timeToLive ) ); + migration.from.writeCQL( applicationScope, uniqueValue, timeToLive ); + migration.to.writeCQL( applicationScope, uniqueValue, timeToLive ); - return aggregateBatch; } - return migration.to.write( applicationScope, uniqueValue, timeToLive ); + return migration.to.writeCQL( applicationScope, uniqueValue, timeToLive ); } + @Override public UniqueValueSet load( final ApplicationScope applicationScope, final String type, final Collection fields ) throws ConnectionException { diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyV1Impl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyV1Impl.java index 2235f63aaf..75666faafb 100644 --- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyV1Impl.java +++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyV1Impl.java @@ -20,20 +20,24 @@ package org.apache.usergrid.persistence.collection.serialization.impl; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; +import java.nio.ByteBuffer; +import java.util.*; +import com.datastax.driver.core.DataType; +import com.datastax.driver.core.ProtocolVersion; +import com.datastax.driver.core.Session; import org.apache.cassandra.db.marshal.BytesType; import org.apache.usergrid.persistence.collection.serialization.SerializationFig; import org.apache.usergrid.persistence.collection.serialization.impl.util.LegacyScopeUtils; +import org.apache.usergrid.persistence.core.CassandraConfig; import org.apache.usergrid.persistence.core.CassandraFig; import org.apache.usergrid.persistence.core.astyanax.ColumnTypes; import org.apache.usergrid.persistence.core.astyanax.IdRowCompositeSerializer; import org.apache.usergrid.persistence.core.astyanax.MultiTenantColumnFamily; import org.apache.usergrid.persistence.core.astyanax.MultiTenantColumnFamilyDefinition; import org.apache.usergrid.persistence.core.astyanax.ScopedRowKey; +import org.apache.usergrid.persistence.core.datastax.CQLUtils; import org.apache.usergrid.persistence.core.datastax.TableDefinition; import org.apache.usergrid.persistence.model.entity.Id; import org.apache.usergrid.persistence.model.field.Field; @@ -50,6 +54,40 @@ public class UniqueValueSerializationStrategyV1Impl extends UniqueValueSerializationStrategyImpl, CollectionPrefixedKey> { + + private static final String UNIQUE_VALUES_TABLE = CQLUtils.quote("Unique_Values"); + private static final Collection UNIQUE_VALUES_PARTITION_KEYS = Collections.singletonList("key"); + private static final Collection UNIQUE_VALUES_COLUMN_KEYS = Collections.singletonList("column1"); + private static final Map UNIQUE_VALUES_COLUMNS = + new HashMap() {{ + put( "key", DataType.Name.BLOB ); + put( "column1", DataType.Name.BLOB ); + put( "value", DataType.Name.BLOB ); }}; + private static final Map UNIQUE_VALUES_CLUSTERING_ORDER = + new HashMap(){{ put( "column1", "ASC" ); }}; + + + private static final String UNIQUE_VALUES_LOG_TABLE = CQLUtils.quote("Entity_Unique_Values"); + private static final Collection UNIQUE_VALUES_LOG_PARTITION_KEYS = Collections.singletonList("key"); + private static final Collection UNIQUE_VALUES_LOG_COLUMN_KEYS = Collections.singletonList("column1"); + private static final Map UNIQUE_VALUES_LOG_COLUMNS = + new HashMap() {{ + put( "key", DataType.Name.BLOB ); + put( "column1", DataType.Name.BLOB ); + put( "value", DataType.Name.BLOB ); }}; + private static final Map UNIQUE_VALUES_LOG_CLUSTERING_ORDER = + new HashMap(){{ put( "column1", "ASC" ); }}; + + + private final static TableDefinition uniqueValues = + new TableDefinition( UNIQUE_VALUES_TABLE, UNIQUE_VALUES_PARTITION_KEYS, UNIQUE_VALUES_COLUMN_KEYS, + UNIQUE_VALUES_COLUMNS, TableDefinition.CacheOption.KEYS, UNIQUE_VALUES_CLUSTERING_ORDER); + + private final static TableDefinition uniqueValuesLog = + new TableDefinition( UNIQUE_VALUES_LOG_TABLE, UNIQUE_VALUES_LOG_PARTITION_KEYS, UNIQUE_VALUES_LOG_COLUMN_KEYS, + UNIQUE_VALUES_LOG_COLUMNS, TableDefinition.CacheOption.KEYS, UNIQUE_VALUES_LOG_CLUSTERING_ORDER); + + private static final CollectionScopedRowKeySerializer ROW_KEY_SER = new CollectionScopedRowKeySerializer<>( UniqueFieldRowKeySerializer.get() ); @@ -79,9 +117,11 @@ public class UniqueValueSerializationStrategyV1Impl extends UniqueValueSerializ * @param serializationFig The serialization configuration */ @Inject - public UniqueValueSerializationStrategyV1Impl( final Keyspace keyspace, final CassandraFig cassandraFig, - final SerializationFig serializationFig ) { - super( keyspace, cassandraFig, serializationFig ); + public UniqueValueSerializationStrategyV1Impl(final Keyspace keyspace, final CassandraFig cassandraFig, + final SerializationFig serializationFig, + final Session session, + final CassandraConfig cassandraConfig) { + super( keyspace, cassandraFig, serializationFig, session, cassandraConfig ); } @@ -113,6 +153,12 @@ protected MultiTenantColumnFamily>, En return CF_UNIQUE_VALUES; } + @Override + protected TableDefinition getUniqueValuesTable(){ + + return uniqueValues; + } + @Override protected MultiTenantColumnFamily>, UniqueFieldEntry> @@ -121,6 +167,14 @@ protected MultiTenantColumnFamily>, En } + @Override + protected TableDefinition getEntityUniqueLogTable(){ + + return uniqueValuesLog; + + } + + @Override protected CollectionPrefixedKey createUniqueValueKey( final Id applicationId, final String type, final Field field) { @@ -141,6 +195,242 @@ protected Field parseRowKey( final ScopedRowKey> ro return rowKey.getKey().getSubKey(); } + @Override + protected List deserializePartitionKey(ByteBuffer bb){ + + + /** + * List keys = new ArrayList<>(8); + keys.add(0, appUUID); + keys.add(1, applicationType); + keys.add(2, appUUID); + keys.add(3, applicationType); + keys.add(4, entityType); + keys.add(5, fieldType); + keys.add(6, fieldName); + keys.add(7, fieldValueString); + + */ + + int count = 0; + List stuff = new ArrayList<>(); + while(bb.hasRemaining()){ + ByteBuffer data = CQLUtils.getWithShortLength(bb); + if(count == 0 || count == 2){ + stuff.add(DataType.uuid().deserialize(data.slice(), ProtocolVersion.NEWEST_SUPPORTED)); + }else{ + stuff.add(DataType.text().deserialize(data.slice(), ProtocolVersion.NEWEST_SUPPORTED)); + } + byte equality = bb.get(); // we don't use this but take the equality byte off the buffer + count++; + } + + return stuff; + + } + + @Override + protected Object serializeUniqueValueLogColumn(UniqueFieldEntry fieldEntry){ + + /** + * final UUID version = value.getVersion(); + final Field field = value.getField(); + + final FieldTypeName fieldType = field.getTypeName(); + final String fieldValue = field.getValue().toString().toLowerCase(); + + + DynamicComposite composite = new DynamicComposite( ); + + //we want to sort ascending to descending by version + composite.addComponent( version, UUID_SERIALIZER, ColumnTypes.UUID_TYPE_REVERSED); + composite.addComponent( field.getName(), STRING_SERIALIZER ); + composite.addComponent( fieldValue, STRING_SERIALIZER ); + composite.addComponent( fieldType.name() , STRING_SERIALIZER); + */ + + // values are serialized as strings, not sure why, and always lower cased + String fieldValueString = fieldEntry.getField().getValue().toString().toLowerCase(); + + + List keys = new ArrayList<>(4); + keys.add(fieldEntry.getVersion()); + keys.add(fieldEntry.getField().getName()); + keys.add(fieldValueString); + keys.add(fieldEntry.getField().getTypeName().name()); + + String comparator = UUID_TYPE_REVERSED; + + int size = 16+fieldEntry.getField().getName().length()+fieldEntry.getField().getValue().toString().length()+ + fieldEntry.getField().getTypeName().name().length(); + + // we always need to add length for the 2 byte comparator short, 2 byte length short and 1 byte equality + size += keys.size()*65; + + // uuid type comparator is longest, ensure we allocate buffer using the max size to avoid overflow + size += keys.size()*comparator.length(); + + ByteBuffer stuff = ByteBuffer.allocate(size); + + + for (Object key : keys) { + + if(key.equals(fieldEntry.getVersion())) { + int p = comparator.indexOf("(reversed=true)"); + boolean desc = false; + if (p >= 0) { + comparator = comparator.substring(0, p); + desc = true; + } + + byte a = (byte) 85; // this is the byte value for UUIDType in astyanax used in legacy data + if (desc) { + a = (byte) Character.toUpperCase((char) a); + } + + stuff.putShort((short) ('耀' | a)); + }else{ + comparator = "UTF8Type"; // only strings are being serialized other than UUIDs here + stuff.putShort((short)comparator.length()); + stuff.put(DataType.serializeValue(comparator, ProtocolVersion.NEWEST_SUPPORTED)); + } + + ByteBuffer kb = DataType.serializeValue(key, ProtocolVersion.NEWEST_SUPPORTED); + if (kb == null) { + kb = ByteBuffer.allocate(0); + } + + // put a short that indicates how big the buffer is for this item + stuff.putShort((short) kb.remaining()); + + // put the actual item + stuff.put(kb.slice()); + + // put an equality byte ( again not used by part of legacy thrift Astyanax schema) + stuff.put((byte) 0); + + + } + + stuff.flip(); + return stuff.duplicate(); + + } + + @Override + protected ByteBuffer getPartitionKey(Id applicationId, String entityType, String fieldType, String fieldName, Object fieldValue ){ + + return serializeKey(applicationId.getUuid(), applicationId.getType(), + entityType, fieldType, fieldName, fieldValue); + + } + + @Override + protected ByteBuffer getLogPartitionKey(final Id applicationId, final Id uniqueValueId){ + + return serializeLogKey(applicationId.getUuid(), applicationId.getType(), + uniqueValueId.getUuid(), uniqueValueId.getType()); + + } + + @Override + protected ByteBuffer serializeUniqueValueColumn(EntityVersion entityVersion){ + + /** + * final Id entityId = ev.getEntityId(); + final UUID entityUuid = entityId.getUuid(); + final String entityType = entityId.getType(); + + CompositeBuilder builder = Composites.newDynamicCompositeBuilder(); + + builder.addUUID( entityVersion ); + builder.addUUID( entityUuid ); + builder.addString(entityType ); + */ + + String comparator = "UTF8Type"; + + List keys = new ArrayList<>(3); + keys.add(entityVersion.getEntityVersion()); + keys.add(entityVersion.getEntityId().getUuid()); + keys.add(entityVersion.getEntityId().getType()); + + // UUIDs are 16 bytes + int size = 16+16+entityVersion.getEntityId().getType().length(); + + // we always need to add length for the 2 byte comparator short, 2 byte length short and 1 byte equality + size += keys.size()*5; + + // we always add comparator to the buffer as well + size += keys.size()*comparator.length(); + + ByteBuffer stuff = ByteBuffer.allocate(size); + + for (Object key : keys) { + + if(key instanceof UUID){ + comparator = "UUIDType"; + }else{ + comparator = "UTF8Type"; // if it's not a UUID, the only other thing we're serializing is text + } + + stuff.putShort((short)comparator.length()); + stuff.put(DataType.serializeValue(comparator, ProtocolVersion.NEWEST_SUPPORTED)); + + ByteBuffer kb = DataType.serializeValue(key, ProtocolVersion.NEWEST_SUPPORTED); + if (kb == null) { + kb = ByteBuffer.allocate(0); + } + + // put a short that indicates how big the buffer is for this item + stuff.putShort((short) kb.remaining()); + + // put the actual item + stuff.put(kb.slice()); + + // put an equality byte ( again not used by part of legacy thrift Astyanax schema) + stuff.put((byte) 0); + + + } + + stuff.flip(); + return stuff.duplicate(); + + } + + @Override + protected List deserializeUniqueValueColumn(ByteBuffer bb){ + + List stuff = new ArrayList<>(); + int count = 0; + while(bb.hasRemaining()){ + + // custom columns have a short at beginning for comparator (which we don't use here ) + ByteBuffer comparator = CQLUtils.getWithShortLength(bb); + + ByteBuffer data = CQLUtils.getWithShortLength(bb); + + + // first two composites are UUIDs, rest are strings + if(count == 0) { + stuff.add(new UUID(data.getLong(), data.getLong())); + }else if(count ==1){ + stuff.add(new UUID(data.getLong(), data.getLong())); + }else{ + stuff.add(DataType.text().deserialize(data.duplicate(), ProtocolVersion.NEWEST_SUPPORTED)); + } + + byte equality = bb.get(); // we don't use this but take the equality byte off the buffer + + count++; + } + + return stuff; + + } + + @Override protected CollectionPrefixedKey createEntityUniqueLogKey( final Id applicationId, @@ -163,4 +453,112 @@ protected CollectionPrefixedKey createEntityUniqueLogKey( final Id applicati public int getImplementationVersion() { return CollectionDataVersions.INITIAL.getVersion(); } + + + + private ByteBuffer serializeKey( UUID appUUID, + String applicationType, + String entityType, + String fieldType, + String fieldName, + Object fieldValue ){ + + final String collectionName = LegacyScopeUtils.getCollectionScopeNameFromEntityType( entityType ); + +// final CollectionPrefixedKey uniquePrefixedKey = +// new CollectionPrefixedKey<>( collectionName, applicationId, field ); + +// //read back the id +// final Id orgId = ID_SER.fromComposite( parser ); +// final Id scopeId = ID_SER.fromComposite( parser ); +// final String scopeName = parser.readString(); +// final K value = keySerializer.fromComposite( parser ); + + + // values are serialized as strings, not sure why, and always lower cased + String fieldValueString = fieldValue.toString().toLowerCase(); + + List keys = new ArrayList<>(8); + keys.add(0, appUUID); + keys.add(1, applicationType); + keys.add(2, appUUID); + keys.add(3, applicationType); + keys.add(4, collectionName); + keys.add(5, fieldType); + keys.add(6, fieldName); + keys.add(7, fieldValueString); + + + // UUIDs are 16 bytes, allocate the buffer accordingly + int size = 16 + applicationType.length() + 16 + applicationType.length() + collectionName.length() + + fieldType.length() + fieldName.length()+fieldValueString.length(); + + + // we always need to add length for the 2 byte short and 1 byte equality + size += keys.size()*3; + + ByteBuffer stuff = ByteBuffer.allocate(size); + + for (Object key : keys) { + + ByteBuffer kb = DataType.serializeValue(key, ProtocolVersion.NEWEST_SUPPORTED); + if (kb == null) { + kb = ByteBuffer.allocate(0); + } + + stuff.putShort((short) kb.remaining()); + stuff.put(kb.slice()); + stuff.put((byte) 0); + + + } + stuff.flip(); + return stuff.duplicate(); + + } + + private ByteBuffer serializeLogKey(UUID appUUID, String applicationType, UUID entityId, String entityType){ + + + final String collectionName = LegacyScopeUtils.getCollectionScopeNameFromEntityType( entityType ); +// +// +// final CollectionPrefixedKey collectionPrefixedEntityKey = +// new CollectionPrefixedKey<>( collectionName, applicationId, uniqueValueId ); + + List keys = new ArrayList<>(4); + keys.add(appUUID); + keys.add(applicationType); + keys.add(appUUID); + keys.add(applicationType); + keys.add(collectionName); + keys.add(entityId); + keys.add(entityType); + + int size = 16+applicationType.length()+16+applicationType.length()+collectionName.length()+16+entityType.length(); + + // we always need to add length for the 2 byte short and 1 byte equality + size += keys.size()*3; + + ByteBuffer stuff = ByteBuffer.allocate(size); + + for (Object key : keys) { + + ByteBuffer kb = DataType.serializeValue(key, ProtocolVersion.NEWEST_SUPPORTED); + if (kb == null) { + kb = ByteBuffer.allocate(0); + } + + stuff.putShort((short) kb.remaining()); + stuff.put(kb.slice()); + stuff.put((byte) 0); + + + } + stuff.flip(); + return stuff.duplicate(); + + } + + } diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyV2Impl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyV2Impl.java index 0f233cfd1f..4177c37d90 100644 --- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyV2Impl.java +++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyV2Impl.java @@ -20,13 +20,16 @@ package org.apache.usergrid.persistence.collection.serialization.impl; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; +import java.nio.ByteBuffer; +import java.util.*; +import com.datastax.driver.core.DataType; +import com.datastax.driver.core.ProtocolVersion; +import com.datastax.driver.core.Session; import org.apache.cassandra.db.marshal.BytesType; import org.apache.usergrid.persistence.collection.serialization.SerializationFig; +import org.apache.usergrid.persistence.core.CassandraConfig; import org.apache.usergrid.persistence.core.CassandraFig; import org.apache.usergrid.persistence.core.astyanax.ColumnTypes; import org.apache.usergrid.persistence.core.astyanax.IdRowCompositeSerializer; @@ -34,6 +37,7 @@ import org.apache.usergrid.persistence.core.astyanax.MultiTenantColumnFamilyDefinition; import org.apache.usergrid.persistence.core.astyanax.ScopedRowKey; import org.apache.usergrid.persistence.core.astyanax.ScopedRowKeySerializer; +import org.apache.usergrid.persistence.core.datastax.CQLUtils; import org.apache.usergrid.persistence.core.datastax.TableDefinition; import org.apache.usergrid.persistence.model.entity.Id; import org.apache.usergrid.persistence.model.field.Field; @@ -49,6 +53,37 @@ @Singleton public class UniqueValueSerializationStrategyV2Impl extends UniqueValueSerializationStrategyImpl { + private static final String UNIQUE_VALUES_TABLE = CQLUtils.quote("Unique_Values_V2"); + private static final Collection UNIQUE_VALUES_PARTITION_KEYS = Collections.singletonList("key"); + private static final Collection UNIQUE_VALUES_COLUMN_KEYS = Collections.singletonList("column1"); + private static final Map UNIQUE_VALUES_COLUMNS = + new HashMap() {{ + put( "key", DataType.Name.BLOB ); + put( "column1", DataType.Name.BLOB ); + put( "value", DataType.Name.BLOB ); }}; + private static final Map UNIQUE_VALUES_CLUSTERING_ORDER = + new HashMap(){{ put( "column1", "ASC" ); }}; + + + private static final String UNIQUE_VALUES_LOG_TABLE = CQLUtils.quote("Entity_Unique_Values_V2"); + private static final Collection UNIQUE_VALUES_LOG_PARTITION_KEYS = Collections.singletonList("key"); + private static final Collection UNIQUE_VALUES_LOG_COLUMN_KEYS = Collections.singletonList("column1"); + private static final Map UNIQUE_VALUES_LOG_COLUMNS = + new HashMap() {{ + put( "key", DataType.Name.BLOB ); + put( "column1", DataType.Name.BLOB ); + put( "value", DataType.Name.BLOB ); }}; + private static final Map UNIQUE_VALUES_LOG_CLUSTERING_ORDER = + new HashMap(){{ put( "column1", "ASC" ); }}; + + private final static TableDefinition uniqueValues = + new TableDefinition( UNIQUE_VALUES_TABLE, UNIQUE_VALUES_PARTITION_KEYS, UNIQUE_VALUES_COLUMN_KEYS, + UNIQUE_VALUES_COLUMNS, TableDefinition.CacheOption.KEYS, UNIQUE_VALUES_CLUSTERING_ORDER); + + private final static TableDefinition uniqueValuesLog = + new TableDefinition( UNIQUE_VALUES_LOG_TABLE, UNIQUE_VALUES_LOG_PARTITION_KEYS, UNIQUE_VALUES_LOG_COLUMN_KEYS, + UNIQUE_VALUES_LOG_COLUMNS, TableDefinition.CacheOption.KEYS, UNIQUE_VALUES_LOG_CLUSTERING_ORDER); + private static final ScopedRowKeySerializer ROW_KEY_SER = new ScopedRowKeySerializer<>( UniqueTypeFieldRowKeySerializer.get() ); @@ -80,8 +115,10 @@ public class UniqueValueSerializationStrategyV2Impl extends UniqueValueSerializ */ @Inject public UniqueValueSerializationStrategyV2Impl( final Keyspace keyspace, final CassandraFig cassandraFig, - final SerializationFig serializationFig ) { - super( keyspace, cassandraFig, serializationFig ); + final SerializationFig serializationFig, + final Session session, + final CassandraConfig cassandraConfig) { + super( keyspace, cassandraFig, serializationFig, session, cassandraConfig ); } @@ -104,7 +141,9 @@ public Collection getColumnFamilies() { @Override public Collection getTables() { - return Collections.emptyList(); + + return Arrays.asList( uniqueValues, uniqueValuesLog ); + } @Override @@ -113,6 +152,12 @@ protected MultiTenantColumnFamily, EntityVersion> getUni } + @Override + protected TableDefinition getUniqueValuesTable(){ + return uniqueValues; + } + + @Override protected MultiTenantColumnFamily, UniqueFieldEntry> getEntityUniqueLogCF() { @@ -120,6 +165,13 @@ protected MultiTenantColumnFamily, EntityVersion> getUni } + @Override + protected TableDefinition getEntityUniqueLogTable(){ + return uniqueValuesLog; + } + + + @Override protected TypeField createUniqueValueKey( final Id applicationId, final String type, final Field field) { return new TypeField(type,field); @@ -131,6 +183,238 @@ protected Field parseRowKey( final ScopedRowKey rowKey ) { return rowKey.getKey().getField(); } + @Override + protected List deserializePartitionKey(ByteBuffer bb){ + + + /** + * List keys = new ArrayList<>(6); + keys.add(0, appUUID); // UUID + keys.add(1, applicationType); // String + keys.add(2, entityType); // String + keys.add(3, fieldType); // String + keys.add(4, fieldName); // String + keys.add(5, fieldValueString); // String + + */ + + List stuff = new ArrayList<>(); + while(bb.hasRemaining()){ + ByteBuffer data = CQLUtils.getWithShortLength(bb); + if(stuff.size() == 0){ + stuff.add(DataType.uuid().deserialize(data.slice(), ProtocolVersion.NEWEST_SUPPORTED)); + }else{ + stuff.add(DataType.text().deserialize(data.slice(), ProtocolVersion.NEWEST_SUPPORTED)); + } + byte equality = bb.get(); // we don't use this but take the equality byte off the buffer + + } + + return stuff; + + } + + @Override + protected Object serializeUniqueValueLogColumn(UniqueFieldEntry fieldEntry){ + + /** + * final UUID version = value.getVersion(); + final Field field = value.getField(); + + final FieldTypeName fieldType = field.getTypeName(); + final String fieldValue = field.getValue().toString().toLowerCase(); + + + DynamicComposite composite = new DynamicComposite( ); + + //we want to sort ascending to descending by version + composite.addComponent( version, UUID_SERIALIZER, ColumnTypes.UUID_TYPE_REVERSED); + composite.addComponent( field.getName(), STRING_SERIALIZER ); + composite.addComponent( fieldValue, STRING_SERIALIZER ); + composite.addComponent( fieldType.name() , STRING_SERIALIZER); + */ + + // values are serialized as strings, not sure why, and always lower cased + String fieldValueString = fieldEntry.getField().getValue().toString().toLowerCase(); + + + List keys = new ArrayList<>(4); + keys.add(fieldEntry.getVersion()); + keys.add(fieldEntry.getField().getName()); + keys.add(fieldValueString); + keys.add(fieldEntry.getField().getTypeName().name()); + + String comparator = UUID_TYPE_REVERSED; + + int size = 16+fieldEntry.getField().getName().length()+fieldEntry.getField().getValue().toString().length()+ + fieldEntry.getField().getTypeName().name().length(); + + // we always need to add length for the 2 byte comparator short, 2 byte length short and 1 byte equality + size += keys.size()*65; + + // uuid type comparator is longest, ensure we allocate buffer using the max size to avoid overflow + size += keys.size()*comparator.length(); + + ByteBuffer stuff = ByteBuffer.allocate(size); + + + for (Object key : keys) { + + if(key.equals(fieldEntry.getVersion())) { + int p = comparator.indexOf("(reversed=true)"); + boolean desc = false; + if (p >= 0) { + comparator = comparator.substring(0, p); + desc = true; + } + + byte a = (byte) 85; // this is the byte value for UUIDType in astyanax used in legacy data + if (desc) { + a = (byte) Character.toUpperCase((char) a); + } + + stuff.putShort((short) ('耀' | a)); + }else{ + comparator = "UTF8Type"; // only strings are being serialized other than UUIDs here + stuff.putShort((short)comparator.length()); + stuff.put(DataType.serializeValue(comparator, ProtocolVersion.NEWEST_SUPPORTED)); + } + + ByteBuffer kb = DataType.serializeValue(key, ProtocolVersion.NEWEST_SUPPORTED); + if (kb == null) { + kb = ByteBuffer.allocate(0); + } + + // put a short that indicates how big the buffer is for this item + stuff.putShort((short) kb.remaining()); + + // put the actual item + stuff.put(kb.slice()); + + // put an equality byte ( again not used by part of legacy thrift Astyanax schema) + stuff.put((byte) 0); + + + } + + stuff.flip(); + return stuff.duplicate(); + + } + + @Override + protected ByteBuffer getPartitionKey(Id applicationId, String entityType, String fieldType, String fieldName, Object fieldValue ){ + + return serializeKey(applicationId.getUuid(), applicationId.getType(), + entityType, fieldType, fieldName, fieldValue); + + } + + @Override + protected ByteBuffer getLogPartitionKey(final Id applicationId, final Id uniqueValueId){ + + return serializeLogKey(applicationId.getUuid(), applicationId.getType(), + uniqueValueId.getUuid(), uniqueValueId.getType()); + + } + + @Override + protected ByteBuffer serializeUniqueValueColumn(EntityVersion entityVersion){ + + /** + * final Id entityId = ev.getEntityId(); + final UUID entityUuid = entityId.getUuid(); + final String entityType = entityId.getType(); + + CompositeBuilder builder = Composites.newDynamicCompositeBuilder(); + + builder.addUUID( entityVersion ); + builder.addUUID( entityUuid ); + builder.addString(entityType ); + */ + + String comparator = "UTF8Type"; + + List keys = new ArrayList<>(3); + keys.add(entityVersion.getEntityVersion()); + keys.add(entityVersion.getEntityId().getUuid()); + keys.add(entityVersion.getEntityId().getType()); + + // UUIDs are 16 bytes + int size = 16+16+entityVersion.getEntityId().getType().length(); + + // we always need to add length for the 2 byte comparator short, 2 byte length short and 1 byte equality + size += keys.size()*5; + + // we always add comparator to the buffer as well + size += keys.size()*comparator.length(); + + ByteBuffer stuff = ByteBuffer.allocate(size); + + for (Object key : keys) { + + if(key instanceof UUID){ + comparator = "UUIDType"; + }else{ + comparator = "UTF8Type"; // if it's not a UUID, the only other thing we're serializing is text + } + + stuff.putShort((short)comparator.length()); + stuff.put(DataType.serializeValue(comparator, ProtocolVersion.NEWEST_SUPPORTED)); + + ByteBuffer kb = DataType.serializeValue(key, ProtocolVersion.NEWEST_SUPPORTED); + if (kb == null) { + kb = ByteBuffer.allocate(0); + } + + // put a short that indicates how big the buffer is for this item + stuff.putShort((short) kb.remaining()); + + // put the actual item + stuff.put(kb.slice()); + + // put an equality byte ( again not used by part of legacy thrift Astyanax schema) + stuff.put((byte) 0); + + + } + + stuff.flip(); + return stuff.duplicate(); + + } + + @Override + protected List deserializeUniqueValueColumn(ByteBuffer bb){ + + List stuff = new ArrayList<>(); + int count = 0; + while(bb.hasRemaining()){ + + // custom columns have a short at beginning for comparator (which we don't use here ) + ByteBuffer comparator = CQLUtils.getWithShortLength(bb); + + ByteBuffer data = CQLUtils.getWithShortLength(bb); + + + // first two composites are UUIDs, rest are strings + if(count == 0) { + stuff.add(new UUID(data.getLong(), data.getLong())); + }else if(count ==1){ + stuff.add(new UUID(data.getLong(), data.getLong())); + }else{ + stuff.add(DataType.text().deserialize(data.duplicate(), ProtocolVersion.NEWEST_SUPPORTED)); + } + + byte equality = bb.get(); // we don't use this but take the equality byte off the buffer + + count++; + } + + return stuff; + + } + @Override protected Id createEntityUniqueLogKey( final Id applicationId, final Id uniqueValueId ) { @@ -142,4 +426,87 @@ protected Id createEntityUniqueLogKey( final Id applicationId, final Id uniqueVa public int getImplementationVersion() { return CollectionDataVersions.LOG_REMOVAL.getVersion(); } + + + + // row key = app UUID + app type + app UUID + app type + field type + field name + field value + private ByteBuffer serializeKey(UUID appUUID, + String applicationType, + String entityType, + String fieldType, + String fieldName, + Object fieldValue ){ + + // values are serialized as strings, not sure why, and always lower cased + String fieldValueString = fieldValue.toString().toLowerCase(); + + List keys = new ArrayList<>(6); + keys.add(0, appUUID); + keys.add(1, applicationType); + keys.add(2, entityType); + keys.add(3, fieldType); + keys.add(4, fieldName); + keys.add(5, fieldValueString); + + + // UUIDs are 16 bytes, allocate the buffer accordingly + int size = 16 + applicationType.length() + entityType.length() + fieldType.length() + fieldName.length()+fieldValueString.length(); + + + // we always need to add length for the 2 byte short and 1 byte equality + size += keys.size()*3; + + ByteBuffer stuff = ByteBuffer.allocate(size); + + for (Object key : keys) { + + ByteBuffer kb = DataType.serializeValue(key, ProtocolVersion.NEWEST_SUPPORTED); + if (kb == null) { + kb = ByteBuffer.allocate(0); + } + + stuff.putShort((short) kb.remaining()); + stuff.put(kb.slice()); + stuff.put((byte) 0); + + + } + stuff.flip(); + return stuff.duplicate(); + + } + + private ByteBuffer serializeLogKey(UUID appUUID, String applicationType, UUID entityId, String entityType){ + + List keys = new ArrayList<>(4); + keys.add(appUUID); + keys.add(applicationType); + keys.add(entityId); + keys.add(entityType); + + int size = 16+applicationType.length()+16+entityType.length(); + + // we always need to add length for the 2 byte short and 1 byte equality + size += keys.size()*3; + + ByteBuffer stuff = ByteBuffer.allocate(size); + + for (Object key : keys) { + + ByteBuffer kb = DataType.serializeValue(key, ProtocolVersion.NEWEST_SUPPORTED); + if (kb == null) { + kb = ByteBuffer.allocate(0); + } + + stuff.putShort((short) kb.remaining()); + stuff.put(kb.slice()); + stuff.put((byte) 0); + + + } + stuff.flip(); + return stuff.duplicate(); + + } + } diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/MvccEntityDataMigrationImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/MvccEntityDataMigrationImpl.java index a110ed7215..8d52d8bc86 100644 --- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/MvccEntityDataMigrationImpl.java +++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/MvccEntityDataMigrationImpl.java @@ -26,6 +26,8 @@ import java.util.UUID; import java.util.concurrent.atomic.AtomicLong; +import com.datastax.driver.core.Session; +import org.apache.cassandra.cql.BatchStatement; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -71,6 +73,7 @@ public class MvccEntityDataMigrationImpl implements DataMigration{ private static final Logger logger = LoggerFactory.getLogger( MvccEntityDataMigrationImpl.class ); private final Keyspace keyspace; + private final Session session; private final VersionedMigrationSet allVersions; private final MvccEntitySerializationStrategyV3Impl mvccEntitySerializationStrategyV3; private final UniqueValueSerializationStrategy uniqueValueSerializationStrategy; @@ -80,12 +83,14 @@ public class MvccEntityDataMigrationImpl implements DataMigration{ @Inject public MvccEntityDataMigrationImpl( final Keyspace keyspace, + final Session session, final VersionedMigrationSet allVersions, final MvccEntitySerializationStrategyV3Impl mvccEntitySerializationStrategyV3, final UniqueValueSerializationStrategy uniqueValueSerializationStrategy, final MvccLogEntrySerializationStrategy mvccLogEntrySerializationStrategy, final MigrationDataProvider migrationDataProvider ) { this.keyspace = keyspace; + this.session = session; this.allVersions = allVersions; this.mvccEntitySerializationStrategyV3 = mvccEntitySerializationStrategyV3; this.uniqueValueSerializationStrategy = uniqueValueSerializationStrategy; @@ -163,8 +168,9 @@ public void call( final Subscriber toSaveIds = new ArrayList<>( entities.size() ); + final com.datastax.driver.core.BatchStatement uniqueBatch = new com.datastax.driver.core.BatchStatement(); - for ( EntityToSaveMessage message : entities ) { + for ( EntityToSaveMessage message : entities ) { try { final MutationBatch entityRewrite = migration.to.write(message.scope, message.entity); @@ -197,17 +203,14 @@ public void call( final Subscriber( context, mvccEntityInput ) ).getEvent().getEntity().get(); @@ -116,6 +124,9 @@ protected void validateStage( final CollectionIoEvent event ) { /** * Write up mock mutations so we don't npe on the our operations, but rather on the input */ + + final Session session = mock(Session.class); + final MvccLogEntrySerializationStrategy logStrategy = mock( MvccLogEntrySerializationStrategy.class ); final MutationBatch logMutation = mock( MutationBatch.class ); @@ -131,7 +142,7 @@ protected void validateStage( final CollectionIoEvent event ) { when( mvccEntityStrategy.write( any( ApplicationScope.class ), any( MvccEntity.class ) ) ) .thenReturn( entityMutation ); - new WriteCommit( logStrategy, mvccEntityStrategy, uniqueValueStrategy ).call( event ); + new WriteCommit( logStrategy, mvccEntityStrategy, uniqueValueStrategy, session ).call( event ); } } diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerifyTest.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerifyTest.java index 09876fbf34..3ddc14dfbb 100644 --- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerifyTest.java +++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerifyTest.java @@ -18,6 +18,7 @@ package org.apache.usergrid.persistence.collection.mvcc.stage.write; +import com.datastax.driver.core.Session; import org.junit.Rule; import org.junit.Test; import org.junit.runner.RunWith; @@ -59,7 +60,8 @@ public class WriteUniqueVerifyTest { @Rule public MigrationManagerRule migrationManagerRule; - + @Inject + private Session session; @Inject private SerializationFig fig; @@ -82,7 +84,7 @@ public void testNoFields() throws ConnectionException { final MvccEntity mvccEntity = fromEntity( entity ); // run the stage - WriteUniqueVerify newStage = new WriteUniqueVerify( uvstrat, fig, keyspace,cassandraConfig ); + WriteUniqueVerify newStage = new WriteUniqueVerify( uvstrat, fig, keyspace,cassandraConfig, session ); newStage.call( new CollectionIoEvent<>( collectionScope, mvccEntity ) ) ; diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyImplTest.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyImplTest.java index fcf22cf1fa..3ffdb6581e 100644 --- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyImplTest.java +++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyImplTest.java @@ -23,6 +23,8 @@ import java.util.Iterator; import java.util.UUID; +import com.datastax.driver.core.BatchStatement; +import com.datastax.driver.core.Session; import org.junit.Assert; import org.junit.Before; import org.junit.Rule; @@ -64,6 +66,8 @@ public abstract class UniqueValueSerializationStrategyImplTest { @Rule public MigrationManagerRule migrationManagerRule; + @Inject + private Session session; private UniqueValueSerializationStrategy strategy; @@ -91,7 +95,9 @@ public void testBasicOperation() throws ConnectionException, InterruptedExceptio Id entityId = new SimpleId( UUIDGenerator.newTimeUUID(), "entity" ); UUID version = UUIDGenerator.newTimeUUID(); UniqueValue stored = new UniqueValueImpl( field, entityId, version ); - strategy.write( scope, stored ).execute(); + //strategy.write( scope, stored ).execute(); + BatchStatement batch = strategy.writeCQL(scope, stored, -1); + session.execute(batch); UniqueValueSet fields = strategy.load( scope, entityId.getType(), Collections.singleton( field ) ); @@ -127,7 +133,9 @@ public void testWriteWithTTL() throws InterruptedException, ConnectionException Id entityId = new SimpleId( UUIDGenerator.newTimeUUID(), "entity" ); UUID version = UUIDGenerator.newTimeUUID(); UniqueValue stored = new UniqueValueImpl( field, entityId, version ); - strategy.write( scope, stored, 5 ).execute(); + //strategy.write( scope, stored, 5 ).execute(); + BatchStatement batch = strategy.writeCQL(scope, stored, 5); + session.execute(batch); Thread.sleep( 1000 ); @@ -179,7 +187,10 @@ public void testDelete() throws ConnectionException { Id entityId = new SimpleId( UUIDGenerator.newTimeUUID(), "entity" ); UUID version = UUIDGenerator.newTimeUUID(); UniqueValue stored = new UniqueValueImpl( field, entityId, version ); - strategy.write( scope, stored ).execute(); + + //strategy.write( scope, stored ).execute(); + BatchStatement batch = strategy.writeCQL( scope, stored, -1); + session.execute(batch); strategy.delete( scope, stored ).execute(); @@ -207,8 +218,9 @@ public void testCapitalizationFixes() throws ConnectionException { Id entityId = new SimpleId( UUIDGenerator.newTimeUUID(), "entity" ); UUID version = UUIDGenerator.newTimeUUID(); UniqueValue stored = new UniqueValueImpl( field, entityId, version ); - strategy.write( scope, stored ).execute(); - + //strategy.write( scope, stored ).execute(); + BatchStatement batch = strategy.writeCQL( scope, stored, -1); + session.execute(batch); UniqueValueSet fields = strategy.load( scope, entityId.getType(), Collections.singleton( field ) ); @@ -278,9 +290,13 @@ public void twoFieldsPerVersion() throws ConnectionException, InterruptedExcepti UniqueValue version1Field1Value = new UniqueValueImpl( version1Field1, entityId, version1 ); UniqueValue version1Field2Value = new UniqueValueImpl( version1Field2, entityId, version1 ); - final MutationBatch batch = strategy.write( scope, version1Field1Value ); - batch.mergeShallow( strategy.write( scope, version1Field2Value ) ); + //final MutationBatch batch = strategy.write( scope, version1Field1Value ); + //batch.mergeShallow( strategy.write( scope, version1Field2Value ) ); + final BatchStatement batch = new BatchStatement(); + + batch.add(strategy.writeCQL( scope, version1Field1Value, -1)); + batch.add(strategy.writeCQL( scope, version1Field2Value, -1)); //write V2 of everything final UUID version2 = UUIDGenerator.newTimeUUID(); @@ -292,10 +308,15 @@ public void twoFieldsPerVersion() throws ConnectionException, InterruptedExcepti UniqueValue version2Field1Value = new UniqueValueImpl( version2Field1, entityId, version2 ); UniqueValue version2Field2Value = new UniqueValueImpl( version2Field2, entityId, version2 ); - batch.mergeShallow( strategy.write( scope, version2Field1Value ) ); - batch.mergeShallow( strategy.write( scope, version2Field2Value ) ); + //batch.mergeShallow( strategy.write( scope, version2Field1Value ) ); + //batch.mergeShallow( strategy.write( scope, version2Field2Value ) ); + + batch.add(strategy.writeCQL( scope, version2Field1Value, -1)); + batch.add(strategy.writeCQL( scope, version2Field2Value, -1)); + + session.execute(batch); - batch.execute(); + //batch.execute(); UniqueValueSet fields = strategy.load( scope, entityId.getType(), Arrays.asList( version1Field1, version1Field2 ) ); diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/AbstractMvccEntityDataMigrationV1ToV3ImplTest.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/AbstractMvccEntityDataMigrationV1ToV3ImplTest.java index 9be979b948..046aaf5826 100644 --- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/AbstractMvccEntityDataMigrationV1ToV3ImplTest.java +++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/AbstractMvccEntityDataMigrationV1ToV3ImplTest.java @@ -22,6 +22,7 @@ import java.util.UUID; +import com.datastax.driver.core.Session; import org.junit.Rule; import org.junit.Test; import org.junit.runner.RunWith; @@ -87,6 +88,8 @@ public abstract class AbstractMvccEntityDataMigrationV1ToV3ImplTest @Inject public Keyspace keyspace; @Inject + public Session session; + @Inject public VersionedMigrationSet allVersions; @Inject public MvccEntitySerializationStrategyV3Impl mvccEntitySerializationStrategyV3; @@ -148,7 +151,7 @@ public void testMigration() throws ConnectionException { assertEquals( "Same instance for to", v3Impl.getClass(), tuple.to.getClass() ); - MvccEntityDataMigrationImpl mvccEntityDataMigrationImpl = new MvccEntityDataMigrationImpl(keyspace, allVersions, mvccEntitySerializationStrategyV3, uniqueValueSerializationStrategy, mvccLogEntrySerializationStrategy, migrationProvider); + MvccEntityDataMigrationImpl mvccEntityDataMigrationImpl = new MvccEntityDataMigrationImpl(keyspace, session, allVersions, mvccEntitySerializationStrategyV3, uniqueValueSerializationStrategy, mvccLogEntrySerializationStrategy, migrationProvider); //now migration diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/impl/DataStaxClusterImpl.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/impl/DataStaxClusterImpl.java index 34cfec8e36..c7b736fe7b 100644 --- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/impl/DataStaxClusterImpl.java +++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/impl/DataStaxClusterImpl.java @@ -46,6 +46,9 @@ public DataStaxClusterImpl(final CassandraFig cassandraFig ) throws Exception { this.cassandraFig = cassandraFig; this.cluster = buildCluster(); + // always initialize the keyspaces + this.createOrUpdateKeyspace(); + logger.info("Initialized datastax cluster client. Hosts={}, Idle Timeout={}s, Pool Timeout={}s", cluster.getMetadata().getAllHosts().toString(), cluster.getConfiguration().getPoolingOptions().getIdleTimeoutSeconds(), From 866d11bf3ecf909682f9fbc91195323f26a1e2ad Mon Sep 17 00:00:00 2001 From: Michael Russo Date: Sun, 8 May 2016 15:45:28 +0800 Subject: [PATCH 26/30] Unique value serialization completely converted to use CQL. Complete Astyanax removal still needs to be completed. --- .../EntityCollectionManagerFactoryImpl.java | 9 +- .../impl/EntityCollectionManagerImpl.java | 21 +- .../mvcc/stage/delete/UniqueCleanup.java | 21 +- .../mvcc/stage/write/RollbackAction.java | 44 +- .../mvcc/stage/write/WriteUniqueVerify.java | 15 +- .../UniqueValueSerializationStrategy.java | 19 +- .../impl/AllUniqueFieldsIterator.java | 29 ++ .../serialization/impl/EntityVersion.java | 12 +- .../UniqueValueSerializationStrategyImpl.java | 384 ++++++------------ ...ueValueSerializationStrategyProxyImpl.java | 12 +- ...niqueValueSerializationStrategyV1Impl.java | 125 +++--- ...niqueValueSerializationStrategyV2Impl.java | 109 +++-- .../impl/UniqueValueSetImpl.java | 6 +- .../write/WriteOptimisticVerifyTest.java | 17 +- ...queValueSerializationStrategyImplTest.java | 5 +- 15 files changed, 383 insertions(+), 445 deletions(-) create mode 100644 stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/AllUniqueFieldsIterator.java diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerFactoryImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerFactoryImpl.java index a52ee9c00c..71e56f5eaa 100644 --- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerFactoryImpl.java +++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerFactoryImpl.java @@ -23,6 +23,7 @@ import java.util.concurrent.ExecutionException; +import com.datastax.driver.core.Session; import org.apache.usergrid.persistence.collection.EntityCollectionManager; import org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory; import org.apache.usergrid.persistence.collection.cache.EntityCacheFig; @@ -75,6 +76,7 @@ public class EntityCollectionManagerFactoryImpl implements EntityCollectionManag private final UniqueValueSerializationStrategy uniqueValueSerializationStrategy; private final MvccLogEntrySerializationStrategy mvccLogEntrySerializationStrategy; private final Keyspace keyspace; + private final Session session; private final MetricsFactory metricsFactory; private final RxTaskScheduler rxTaskScheduler; @@ -89,7 +91,7 @@ public EntityCollectionManager load( ApplicationScope scope ) { entitySerializationStrategy, uniqueValueSerializationStrategy, mvccLogEntrySerializationStrategy, keyspace, metricsFactory, serializationFig, - rxTaskScheduler, scope ); + rxTaskScheduler, scope, session ); return target; } @@ -107,7 +109,9 @@ public EntityCollectionManagerFactoryImpl( final WriteStart writeStart, final Wr final UniqueValueSerializationStrategy uniqueValueSerializationStrategy, final MvccLogEntrySerializationStrategy mvccLogEntrySerializationStrategy, final Keyspace keyspace, final EntityCacheFig entityCacheFig, - final MetricsFactory metricsFactory, @CollectionExecutorScheduler final RxTaskScheduler rxTaskScheduler ) { + final MetricsFactory metricsFactory, + @CollectionExecutorScheduler final RxTaskScheduler rxTaskScheduler, + final Session session ) { this.writeStart = writeStart; this.writeVerifyUnique = writeVerifyUnique; @@ -125,6 +129,7 @@ public EntityCollectionManagerFactoryImpl( final WriteStart writeStart, final Wr this.keyspace = keyspace; this.metricsFactory = metricsFactory; this.rxTaskScheduler = rxTaskScheduler; + this.session = session; } @Override public EntityCollectionManager createCollectionManager(ApplicationScope applicationScope) { diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java index e71e6bb730..6d42fa20e3 100644 --- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java +++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java @@ -26,6 +26,8 @@ import java.util.List; import java.util.UUID; +import com.datastax.driver.core.BatchStatement; +import com.datastax.driver.core.Session; import com.netflix.astyanax.model.ConsistencyLevel; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -64,7 +66,6 @@ import org.apache.usergrid.persistence.model.entity.Entity; import org.apache.usergrid.persistence.model.entity.Id; import org.apache.usergrid.persistence.model.field.Field; -import org.apache.usergrid.persistence.model.util.EntityUtils; import org.apache.usergrid.persistence.model.util.UUIDGenerator; import com.codahale.metrics.Timer; @@ -72,7 +73,6 @@ import com.google.inject.Inject; import com.google.inject.assistedinject.Assisted; import com.netflix.astyanax.Keyspace; -import com.netflix.astyanax.MutationBatch; import com.netflix.astyanax.connectionpool.OperationResult; import com.netflix.astyanax.connectionpool.exceptions.ConnectionException; import com.netflix.astyanax.model.ColumnFamily; @@ -81,7 +81,6 @@ import rx.Observable; import rx.Subscriber; -import rx.functions.Action0; /** @@ -114,6 +113,7 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager { private final Keyspace keyspace; + private final Session session; private final Timer writeTimer; private final Timer deleteTimer; private final Timer fieldIdTimer; @@ -136,7 +136,8 @@ public EntityCollectionManagerImpl( final WriteStart writeStart, final WriteUniq final MvccLogEntrySerializationStrategy mvccLogEntrySerializationStrategy, final Keyspace keyspace, final MetricsFactory metricsFactory, final SerializationFig serializationFig, final RxTaskScheduler rxTaskScheduler, - @Assisted final ApplicationScope applicationScope ) { + @Assisted final ApplicationScope applicationScope, + final Session session) { this.uniqueValueSerializationStrategy = uniqueValueSerializationStrategy; this.entitySerializationStrategy = entitySerializationStrategy; this.uniqueCleanup = uniqueCleanup; @@ -157,6 +158,7 @@ public EntityCollectionManagerImpl( final WriteStart writeStart, final WriteUniq this.markCommit = markCommit; this.keyspace = keyspace; + this.session = session; this.applicationScope = applicationScope; @@ -347,8 +349,7 @@ public Observable
getEntitiesFromFields( final String type, final Coll //Load a entity for each entityId we retrieved. final EntitySet entitySet = entitySerializationStrategy.load( applicationScope, entityIds, startTime ); - //now loop through and ensure the entities are there. - final MutationBatch deleteBatch = keyspace.prepareMutationBatch(); + final BatchStatement uniqueDeleteBatch = new BatchStatement(); final MutableFieldSet response = new MutableFieldSet( fields1.size() ); @@ -357,9 +358,8 @@ public Observable
getEntitiesFromFields( final String type, final Coll //bad unique value, delete this, it's inconsistent if ( entity == null || !entity.getEntity().isPresent() ) { - final MutationBatch valueDelete = - uniqueValueSerializationStrategy.delete( applicationScope, expectedUnique ); - deleteBatch.mergeShallow( valueDelete ); + uniqueDeleteBatch.add( + uniqueValueSerializationStrategy.deleteCQL( applicationScope, expectedUnique )); continue; } @@ -371,8 +371,7 @@ public Observable
getEntitiesFromFields( final String type, final Coll } //TODO: explore making this an Async process - //We'll repair it again if we have to - deleteBatch.execute(); + session.execute(uniqueDeleteBatch); return response; } diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/delete/UniqueCleanup.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/delete/UniqueCleanup.java index 8aa5cfc9ca..9f2b9942ea 100644 --- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/delete/UniqueCleanup.java +++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/delete/UniqueCleanup.java @@ -26,6 +26,8 @@ import java.util.List; import java.util.UUID; +import com.datastax.driver.core.BatchStatement; +import com.datastax.driver.core.Session; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -63,6 +65,7 @@ public class UniqueCleanup private final UniqueValueSerializationStrategy uniqueValueSerializationStrategy; private final Keyspace keyspace; + private final Session session; private final SerializationFig serializationFig; @@ -70,12 +73,14 @@ public class UniqueCleanup @Inject public UniqueCleanup( final SerializationFig serializationFig, final UniqueValueSerializationStrategy uniqueValueSerializationStrategy, - final Keyspace keyspace, final MetricsFactory metricsFactory ) { + final Keyspace keyspace, final MetricsFactory metricsFactory, + final Session session ) { this.serializationFig = serializationFig; this.uniqueValueSerializationStrategy = uniqueValueSerializationStrategy; this.keyspace = keyspace; this.uniqueCleanupTimer = metricsFactory.getTimer( UniqueCleanup.class, "uniquecleanup.base" ); + this.session = session; } @@ -127,22 +132,20 @@ protected Iterator getIterator() { //roll them up .doOnNext( uniqueValues -> { - final MutationBatch uniqueCleanupBatch = keyspace.prepareMutationBatch(); + + final BatchStatement uniqueCleanupBatch = new BatchStatement(); for ( UniqueValue value : uniqueValues ) { logger .debug( "Deleting value:{} from application scope: {} ", value, applicationScope ); uniqueCleanupBatch - .mergeShallow( uniqueValueSerializationStrategy.delete( applicationScope, value ) ); + .add( uniqueValueSerializationStrategy.deleteCQL( applicationScope, value ) ); } - try { - uniqueCleanupBatch.execute(); - } - catch ( ConnectionException e ) { - throw new RuntimeException( "Unable to execute batch mutation", e ); - } + + session.execute(uniqueCleanupBatch); + } ).lastOrDefault( Collections.emptyList() ).map( list -> mvccEntityCollectionIoEvent ); return ObservableTimer.time( uniqueValueCleanup, uniqueCleanupTimer ); diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/RollbackAction.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/RollbackAction.java index 23c6dfe961..e5c4c9642b 100644 --- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/RollbackAction.java +++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/RollbackAction.java @@ -18,6 +18,8 @@ package org.apache.usergrid.persistence.collection.mvcc.stage.write; +import com.datastax.driver.core.BatchStatement; +import com.datastax.driver.core.Session; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -51,14 +53,17 @@ public class RollbackAction implements Action1 { private final UniqueValueSerializationStrategy uniqueValueStrat; private final MvccLogEntrySerializationStrategy logEntryStrat; + private final Session session; @Inject - public RollbackAction(MvccLogEntrySerializationStrategy logEntryStrat, - UniqueValueSerializationStrategy uniqueValueStrat ) { + public RollbackAction( final MvccLogEntrySerializationStrategy logEntryStrat, + final UniqueValueSerializationStrategy uniqueValueStrat, + final Session session ) { this.uniqueValueStrat = uniqueValueStrat; this.logEntryStrat = logEntryStrat; + this.session = session; } @@ -72,6 +77,7 @@ public void call( final Throwable t ) { // one batch to handle rollback MutationBatch rollbackMb = null; + final BatchStatement uniqueDeleteBatch = new BatchStatement(); final Optional entity = mvccEntity.getEntity(); if ( entity.isPresent() ) { @@ -83,45 +89,17 @@ public void call( final Throwable t ) { UniqueValue toDelete = new UniqueValueImpl( field, entity.get().getId(), mvccEntity.getVersion() ); - MutationBatch deleteMb = uniqueValueStrat.delete(scope, toDelete ); + uniqueDeleteBatch.add(uniqueValueStrat.deleteCQL(scope, toDelete )); - if ( rollbackMb == null ) { - rollbackMb = deleteMb; - } - else { - rollbackMb.mergeShallow( deleteMb ); - } } } - - if ( rollbackMb != null ) { - try { - rollbackMb.execute(); - } - catch ( ConnectionException ex ) { - throw new RuntimeException( "Error rolling back changes", ex ); - } - } + // execute the batch statements for deleting unique field entries + session.execute(uniqueDeleteBatch); logEntryStrat.delete( scope, entity.get().getId(), mvccEntity.getVersion() ); } } } - - class FieldDeleteResult { - - private final String name; - - - public FieldDeleteResult( String name ) { - this.name = name; - } - - - public String getName() { - return this.name; - } - } } diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java index 8e0b20208f..501950ac93 100644 --- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java +++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java @@ -25,6 +25,7 @@ import com.datastax.driver.core.BatchStatement; import com.datastax.driver.core.Session; +import com.netflix.hystrix.HystrixCommandProperties; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -68,7 +69,9 @@ public class WriteUniqueVerify implements Action1> private final UniqueValueSerializationStrategy uniqueValueStrat; - public static int uniqueVerifyPoolSize = 100; + private static int uniqueVerifyPoolSize = 100; + + private static int uniqueVerifyTimeoutMillis= 5000; protected final SerializationFig serializationFig; @@ -224,8 +227,10 @@ public Map executeStrategy(ConsistencyLevel consistencyLevel){ /** * Command group used for realtime user commands */ - public static final HystrixCommand.Setter - REPLAY_GROUP = HystrixCommand.Setter.withGroupKey( - HystrixCommandGroupKey.Factory.asKey( "uniqueVerify" ) ).andThreadPoolPropertiesDefaults( - HystrixThreadPoolProperties.Setter().withCoreSize( uniqueVerifyPoolSize ) ); + private static final HystrixCommand.Setter + REPLAY_GROUP = HystrixCommand.Setter.withGroupKey( HystrixCommandGroupKey.Factory.asKey( "uniqueVerify" ) ) + .andThreadPoolPropertiesDefaults( + HystrixThreadPoolProperties.Setter().withCoreSize( uniqueVerifyPoolSize ) ) + .andCommandPropertiesDefaults( + HystrixCommandProperties.Setter().withExecutionTimeoutInMilliseconds(uniqueVerifyTimeoutMillis)); } diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/UniqueValueSerializationStrategy.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/UniqueValueSerializationStrategy.java index 56e8b87c73..bb6f5fe37c 100644 --- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/UniqueValueSerializationStrategy.java +++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/UniqueValueSerializationStrategy.java @@ -41,23 +41,14 @@ public interface UniqueValueSerializationStrategy extends Migration, VersionedDa /** - * Write the specified UniqueValue to Cassandra with optional timeToLive in milliseconds. - * - * @param applicationScope scope - * @param uniqueValue Object to be written - * - * @return MutatationBatch that encapsulates operation, caller may or may not execute. - */ - - /** - * Write the specified UniqueValue to Cassandra with optional timeToLive in milliseconds. + * Write the specified UniqueValue to Cassandra with optional timeToLive in milliseconds. -1 is the same as no ttl + * (lives forever) * * @param applicationScope scope * @param uniqueValue Object to be written * @param timeToLive How long object should live in seconds. -1 implies store forever - * @return MutatationBatch that encapsulates operation, caller may or may not execute. + * @return BatchStatement that encapsulates CQL statements, caller may or may not execute. */ - BatchStatement writeCQL(ApplicationScope applicationScope, UniqueValue uniqueValue, int timeToLive ); /** @@ -103,9 +94,9 @@ UniqueValueSet load( ApplicationScope applicationScope, ConsistencyLevel consist * * @param applicationScope The scope of the application * @param uniqueValue Object to be deleted. - * @return MutatationBatch that encapsulates operation, caller may or may not execute. + * @return BatchStatement that encapsulates the CQL statements, caller may or may not execute. */ - MutationBatch delete( ApplicationScope applicationScope, UniqueValue uniqueValue ); + BatchStatement deleteCQL( ApplicationScope applicationScope, UniqueValue uniqueValue); } diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/AllUniqueFieldsIterator.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/AllUniqueFieldsIterator.java new file mode 100644 index 0000000000..ed210e99f3 --- /dev/null +++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/AllUniqueFieldsIterator.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.usergrid.persistence.collection.serialization.impl; + + +import com.datastax.driver.core.Row; +import com.datastax.driver.core.Session; +import com.datastax.driver.core.Statement; +import org.apache.usergrid.persistence.collection.serialization.UniqueValue; + +import java.util.Iterator; + + diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/EntityVersion.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/EntityVersion.java index 274cf5d03d..d451adc26a 100644 --- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/EntityVersion.java +++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/EntityVersion.java @@ -42,7 +42,8 @@ public UUID getEntityVersion() { return entityVersion; } - public boolean equals( Object o ) { + @Override + public boolean equals( final Object o ) { if ( o == null || !(o instanceof EntityVersion) ) { return false; @@ -60,5 +61,12 @@ public boolean equals( Object o ) { return true; } - + + @Override + public int hashCode() { + int result = entityId.hashCode(); + result = 31 * result + entityVersion.hashCode(); + return result; + } + } diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyImpl.java index 27a86095ef..e0a9035a7c 100644 --- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyImpl.java +++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyImpl.java @@ -22,11 +22,10 @@ import java.util.*; import com.datastax.driver.core.*; +import com.datastax.driver.core.Row; import com.datastax.driver.core.querybuilder.Clause; import com.datastax.driver.core.querybuilder.QueryBuilder; import com.datastax.driver.core.querybuilder.Using; -import com.netflix.astyanax.model.*; -import com.netflix.astyanax.util.RangeBuilder; import org.apache.usergrid.persistence.core.CassandraConfig; import org.apache.usergrid.persistence.core.datastax.TableDefinition; import org.apache.usergrid.persistence.model.entity.SimpleId; @@ -41,8 +40,6 @@ import org.apache.usergrid.persistence.collection.serialization.UniqueValueSerializationStrategy; import org.apache.usergrid.persistence.collection.serialization.UniqueValueSet; import org.apache.usergrid.persistence.core.CassandraFig; -import org.apache.usergrid.persistence.core.astyanax.ColumnNameIterator; -import org.apache.usergrid.persistence.core.astyanax.ColumnParser; import org.apache.usergrid.persistence.core.astyanax.ColumnTypes; import org.apache.usergrid.persistence.core.astyanax.MultiTenantColumnFamily; import org.apache.usergrid.persistence.core.astyanax.MultiTenantColumnFamilyDefinition; @@ -56,7 +53,6 @@ import com.netflix.astyanax.Keyspace; import com.netflix.astyanax.MutationBatch; import com.netflix.astyanax.connectionpool.exceptions.ConnectionException; -import com.netflix.astyanax.query.RowQuery; /** @@ -126,73 +122,6 @@ public UniqueValueSerializationStrategyImpl( final Keyspace keyspace, final Cass } - - public MutationBatch write( final ApplicationScope collectionScope, UniqueValue value ) { - - - Preconditions.checkNotNull( value, "value is required" ); - - - final Id entityId = value.getEntityId(); - final UUID entityVersion = value.getEntityVersion(); - final Field field = value.getField(); - - ValidationUtils.verifyIdentity( entityId ); - ValidationUtils.verifyVersion( entityVersion ); - - - final EntityVersion ev = new EntityVersion( entityId, entityVersion ); - final UniqueFieldEntry uniqueFieldEntry = new UniqueFieldEntry( entityVersion, field ); - - return doWrite( collectionScope, value, new RowOp() { - - @Override - public void doLookup( final ColumnListMutation colMutation ) { - colMutation.putColumn( ev, COL_VALUE ); - } - - - @Override - public void doLog( final ColumnListMutation colMutation ) { - colMutation.putColumn( uniqueFieldEntry, COL_VALUE ); - } - } ); - } - - - public MutationBatch write( final ApplicationScope collectionScope, final UniqueValue value, - final int timeToLive ) { - - Preconditions.checkNotNull( value, "value is required" ); - Preconditions.checkArgument( timeToLive > 0, "timeToLive must be greater than 0 is required" ); - - final Id entityId = value.getEntityId(); - final UUID entityVersion = value.getEntityVersion(); - final Field field = value.getField(); - - ValidationUtils.verifyIdentity( entityId ); - ValidationUtils.verifyVersion( entityVersion ); - - final EntityVersion ev = new EntityVersion( entityId, entityVersion ); - final UniqueFieldEntry uniqueFieldEntry = new UniqueFieldEntry( entityVersion, field ); - - return doWrite( collectionScope, value, new RowOp() { - - @Override - public void doLookup( final ColumnListMutation colMutation ) { - colMutation.putColumn( ev, COL_VALUE, timeToLive ); - } - - - //we purposefully leave out TTL. Worst case we issue deletes against tombstoned columns - //best case, we clean up an invalid secondary index entry when the log is used - @Override - public void doLog( final ColumnListMutation colMutation ) { - colMutation.putColumn( uniqueFieldEntry, COL_VALUE ); - } - } ); - } - @Override public BatchStatement writeCQL( final ApplicationScope collectionScope, final UniqueValue value, final int timeToLive ){ @@ -259,26 +188,15 @@ public BatchStatement writeCQL( final ApplicationScope collectionScope, final Un return batch; - /** - * @Override - public void doLookup( final ColumnListMutation colMutation ) { - colMutation.putColumn( ev, COL_VALUE ); - } - - - @Override - public void doLog( final ColumnListMutation colMutation ) { - colMutation.putColumn( uniqueFieldEntry, COL_VALUE ); - } - */ } @Override - public MutationBatch delete( final ApplicationScope scope, UniqueValue value ) { + public BatchStatement deleteCQL( final ApplicationScope scope, UniqueValue value){ Preconditions.checkNotNull( value, "value is required" ); + final BatchStatement batch = new BatchStatement(); final Id entityId = value.getEntityId(); final UUID entityVersion = value.getEntityVersion(); @@ -291,52 +209,31 @@ public MutationBatch delete( final ApplicationScope scope, UniqueValue value ) { final EntityVersion ev = new EntityVersion( entityId, entityVersion ); final UniqueFieldEntry uniqueFieldEntry = new UniqueFieldEntry( entityVersion, field ); - return doWrite( scope, value, new RowOp() { - - @Override - public void doLookup( final ColumnListMutation colMutation ) { - colMutation.deleteColumn( ev ); - } - - - @Override - public void doLog( final ColumnListMutation colMutation ) { - colMutation.deleteColumn( uniqueFieldEntry ); - } - } ); - } + ByteBuffer partitionKey = getPartitionKey( scope.getApplication(), value.getEntityId().getType(), + value.getField().getTypeName().toString(), value.getField().getName(), value.getField().getValue()); - /** - * Do the column update or delete for the given column and row key - * - * @param applicationScope We need to use this when getting the keyspace - * @param uniqueValue The unique value to write - * @param op The operation to write - */ - private MutationBatch doWrite( ApplicationScope applicationScope, UniqueValue uniqueValue, RowOp op ) { - final MutationBatch batch = keyspace.prepareMutationBatch(); + ByteBuffer columnValue = serializeUniqueValueColumn(ev); - final Id applicationId = applicationScope.getApplication(); + final Clause uniqueEqKey = QueryBuilder.eq("key", partitionKey ); + final Clause uniqueEqColumn = QueryBuilder.eq("column1", columnValue ); + Statement uniqueDelete = QueryBuilder.delete().from(TABLE_UNIQUE_VALUES).where(uniqueEqKey).and(uniqueEqColumn); + batch.add(uniqueDelete); - final FieldKey fieldKey = createUniqueValueKey( applicationId, uniqueValue.getEntityId().getType(), uniqueValue.getField() ); - op.doLookup( batch.withRow( CF_UNIQUE_VALUES, ScopedRowKey.fromKey( applicationId, fieldKey ) ) ); + ByteBuffer logPartitionKey = getLogPartitionKey(scope.getApplication(), entityId); + ByteBuffer logColumnValue = serializeUniqueValueLogColumn(uniqueFieldEntry); - final EntityKey entityKey = createEntityUniqueLogKey( applicationId, uniqueValue.getEntityId() ); + final Clause uniqueLogEqKey = QueryBuilder.eq("key", logPartitionKey ); + final Clause uniqueLogEqColumn = QueryBuilder.eq("column1", logColumnValue ); - op.doLog( batch.withRow( CF_ENTITY_UNIQUE_VALUE_LOG, - ScopedRowKey.fromKey( applicationId, entityKey ) ) ); + Statement uniqueLogDelete = QueryBuilder.delete() + .from(TABLE_UNIQUE_VALUES_LOG).where(uniqueLogEqKey).and( uniqueLogEqColumn); + batch.add(uniqueLogDelete); - if ( log.isTraceEnabled() ) { - log.trace( "Writing unique value version={} name={} value={} ", - uniqueValue.getEntityVersion(), uniqueValue.getField().getName(), - uniqueValue.getField().getValue() - ); - } return batch; @@ -364,59 +261,6 @@ public UniqueValueSet load( final ApplicationScope appScope, final com.netflix.a } - private UniqueValueSet loadLegacy(final ApplicationScope appScope, - final String type, final Collection fields) throws ConnectionException { - final List> keys = new ArrayList<>( fields.size() ); - - final Id applicationId = appScope.getApplication(); - - for ( Field field : fields ) { - - final FieldKey key = createUniqueValueKey( applicationId, type, field ); - - - final ScopedRowKey rowKey = - ScopedRowKey.fromKey( applicationId, key ); - - keys.add( rowKey ); - } - - final UniqueValueSetImpl uniqueValueSet = new UniqueValueSetImpl( fields.size() ); - - Iterator, EntityVersion>> results = - keyspace.prepareQuery( CF_UNIQUE_VALUES ).setConsistencyLevel(com.netflix.astyanax.model.ConsistencyLevel.CL_LOCAL_QUORUM ).getKeySlice( keys ) - .withColumnRange( new RangeBuilder().setLimit( 1 ).build() ).execute().getResult().iterator(); - - - while ( results.hasNext() ) - - { - - final com.netflix.astyanax.model.Row, EntityVersion> unique = results.next(); - - - final Field field = parseRowKey( unique.getKey() ); - - final Iterator> columnList = unique.getColumns().iterator(); - - //sanity check, nothing to do, skip it - if ( !columnList.hasNext() ) { - continue; - } - - final EntityVersion entityVersion = columnList.next().getName(); - - - final UniqueValueImpl uniqueValue = - new UniqueValueImpl( field, entityVersion.getEntityId(), entityVersion.getEntityVersion() ); - - uniqueValueSet.addValue( uniqueValue ); - } - - return uniqueValueSet; - - } - private UniqueValueSet loadCQL( final ApplicationScope appScope, final com.datastax.driver.core.ConsistencyLevel consistencyLevel, final String type, final Collection fields ) throws ConnectionException { @@ -460,7 +304,6 @@ private UniqueValueSet loadCQL( final ApplicationScope appScope, final com.datas List keyContents = deserializePartitionKey(partitionKey); List columnContents = deserializeUniqueValueColumn(column); - Field field = null; FieldTypeName fieldType; String name; String value; @@ -478,29 +321,8 @@ private UniqueValueSet loadCQL( final ApplicationScope appScope, final com.datas } - switch ( fieldType ) { - case BOOLEAN: - field = new BooleanField( name, Boolean.parseBoolean( value ) ); - break; - case DOUBLE: - field = new DoubleField( name, Double.parseDouble( value ) ); - break; - case FLOAT: - field = new FloatField( name, Float.parseFloat( value ) ); - break; - case INTEGER: - field = new IntegerField( name, Integer.parseInt( value ) ); - break; - case LONG: - field = new LongField( name, Long.parseLong( value ) ); - break; - case STRING: - field = new StringField( name, value ); - break; - case UUID: - field = new UUIDField( name, UUID.fromString( value ) ); - break; - } + Field field = getField(name, value, fieldType); + final EntityVersion entityVersion = new EntityVersion( new SimpleId((UUID)columnContents.get(1), (String)columnContents.get(2)), (UUID)columnContents.get(0)); @@ -526,59 +348,17 @@ public Iterator getAllUniqueFields( final ApplicationScope collecti Preconditions.checkNotNull( entityId, "entity id is required" ); - final Id applicationId = collectionScope.getApplication(); - - final EntityKey entityKey = createEntityUniqueLogKey( applicationId, entityId ); - - - final ScopedRowKey rowKey = - ScopedRowKey.fromKey( applicationId, entityKey ); - - - RowQuery, UniqueFieldEntry> query = - keyspace.prepareQuery( CF_ENTITY_UNIQUE_VALUE_LOG ).getKey( rowKey ) - .withColumnRange( ( UniqueFieldEntry ) null, null, false, serializationFig.getBufferSize() ); - - return new ColumnNameIterator( query, new UniqueEntryParser( entityId ), false ); - } + Clause inKey = QueryBuilder.in("key", getLogPartitionKey(collectionScope.getApplication(), entityId)); + Statement statement = QueryBuilder.select().all().from(TABLE_UNIQUE_VALUES_LOG) + .where(inKey); - /** - * Simple callback to perform puts and deletes with a common row setup code - */ - private interface RowOp { + return new AllUniqueFieldsIterator(session, statement, entityId); - /** - * Execute the mutation into the lookup CF_UNIQUE_VALUES row - */ - void doLookup( ColumnListMutation colMutation ); - /** - * Execute the mutation into the lCF_ENTITY_UNIQUE_VALUESLUE row - */ - void doLog( ColumnListMutation colMutation ); } - /** - * Converts raw columns to the expected output - */ - private static final class UniqueEntryParser implements ColumnParser { - - private final Id entityId; - - - private UniqueEntryParser( final Id entityId ) {this.entityId = entityId;} - - - @Override - public UniqueValue parseColumn( final Column column ) { - final UniqueFieldEntry entry = column.getName(); - - return new UniqueValueImpl( entry.getField(), entityId, entry.getVersion() ); - } - } - @Override public Collection getColumnFamilies() { @@ -621,27 +401,9 @@ public Collection getTables() { protected abstract TableDefinition getUniqueValuesTable(); - /** - * Generate a key that is compatible with the column family - * - * @param applicationId The applicationId - * @param type The type in the field - * @param field The field we're creating the key for - */ - protected abstract FieldKey createUniqueValueKey(final Id applicationId, final String type, final Field field ); - - /** - * Parse the row key into the field - * @param rowKey - * @return - */ - protected abstract Field parseRowKey(final ScopedRowKey rowKey); - - protected abstract List deserializePartitionKey(ByteBuffer bb); - - protected abstract Object serializeUniqueValueLogColumn(UniqueFieldEntry fieldEntry); + protected abstract ByteBuffer serializeUniqueValueLogColumn(UniqueFieldEntry fieldEntry); protected abstract ByteBuffer getPartitionKey(Id applicationId, String entityType, String fieldType, String fieldName, Object fieldValue ); @@ -651,6 +413,8 @@ public Collection getTables() { protected abstract List deserializeUniqueValueColumn(ByteBuffer bb); + protected abstract List deserializeUniqueValueLogColumn(ByteBuffer bb); + @@ -672,4 +436,100 @@ public Collection getTables() { * @param uniqueValueId The uniqueValue */ protected abstract EntityKey createEntityUniqueLogKey(final Id applicationId, final Id uniqueValueId ); + + + public class AllUniqueFieldsIterator implements Iterable, Iterator { + + private final Session session; + private final Statement query; + private final Id entityId; + + private Iterator sourceIterator; + + + + public AllUniqueFieldsIterator( final Session session, final Statement query, final Id entityId){ + + this.session = session; + this.query = query; + this.entityId = entityId; + + } + + + @Override + public Iterator iterator() { + return this; + } + + @Override + public boolean hasNext() { + + if ( sourceIterator == null ) { + + advanceIterator(); + + return sourceIterator.hasNext(); + } + + return sourceIterator.hasNext(); + } + + @Override + public UniqueValue next() { + + com.datastax.driver.core.Row next = sourceIterator.next(); + + ByteBuffer column = next.getBytesUnsafe("column1"); + + List columnContents = deserializeUniqueValueLogColumn(column); + + UUID version = (UUID) columnContents.get(0); + String name = (String) columnContents.get(1); + String value = (String) columnContents.get(2); + FieldTypeName fieldType = FieldTypeName.valueOf((String) columnContents.get(3)); + + + return new UniqueValueImpl(getField(name, value, fieldType), entityId, version); + + } + + private void advanceIterator() { + + sourceIterator = session.execute(query).iterator(); + } + } + + private Field getField( String name, String value, FieldTypeName fieldType){ + + Field field = null; + + switch ( fieldType ) { + case BOOLEAN: + field = new BooleanField( name, Boolean.parseBoolean( value ) ); + break; + case DOUBLE: + field = new DoubleField( name, Double.parseDouble( value ) ); + break; + case FLOAT: + field = new FloatField( name, Float.parseFloat( value ) ); + break; + case INTEGER: + field = new IntegerField( name, Integer.parseInt( value ) ); + break; + case LONG: + field = new LongField( name, Long.parseLong( value ) ); + break; + case STRING: + field = new StringField( name, value ); + break; + case UUID: + field = new UUIDField( name, UUID.fromString( value ) ); + break; + } + + return field; + + } + } diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyProxyImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyProxyImpl.java index bbfaa2dd37..dc5b48f11d 100644 --- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyProxyImpl.java +++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyProxyImpl.java @@ -125,19 +125,19 @@ public Iterator getAllUniqueFields( final ApplicationScope applicat @Override - public MutationBatch delete( final ApplicationScope applicationScope, final UniqueValue uniqueValue ) { + public BatchStatement deleteCQL( final ApplicationScope applicationScope, final UniqueValue uniqueValue ) { final MigrationRelationship migration = getMigrationRelationShip(); if ( migration.needsMigration() ) { - final MutationBatch aggregateBatch = keyspace.prepareMutationBatch(); + final BatchStatement batch = new BatchStatement(); - aggregateBatch.mergeShallow( migration.from.delete( applicationScope, uniqueValue ) ); - aggregateBatch.mergeShallow( migration.to.delete( applicationScope, uniqueValue ) ); + batch.add(migration.from.deleteCQL( applicationScope, uniqueValue ) ); + batch.add(migration.to.deleteCQL( applicationScope, uniqueValue ) ); - return aggregateBatch; + return batch; } - return migration.to.delete( applicationScope, uniqueValue ); + return migration.to.deleteCQL( applicationScope, uniqueValue ); } diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyV1Impl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyV1Impl.java index 75666faafb..cbd8a3e8b0 100644 --- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyV1Impl.java +++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyV1Impl.java @@ -175,26 +175,6 @@ protected TableDefinition getEntityUniqueLogTable(){ } - @Override - protected CollectionPrefixedKey createUniqueValueKey( final Id applicationId, - final String type, final Field field) { - - - final String collectionName = LegacyScopeUtils.getCollectionScopeNameFromEntityType( type ); - - - final CollectionPrefixedKey uniquePrefixedKey = - new CollectionPrefixedKey<>( collectionName, applicationId, field ); - - return uniquePrefixedKey; - } - - - @Override - protected Field parseRowKey( final ScopedRowKey> rowKey ) { - return rowKey.getKey().getSubKey(); - } - @Override protected List deserializePartitionKey(ByteBuffer bb){ @@ -230,23 +210,23 @@ protected List deserializePartitionKey(ByteBuffer bb){ } @Override - protected Object serializeUniqueValueLogColumn(UniqueFieldEntry fieldEntry){ + protected ByteBuffer serializeUniqueValueLogColumn(UniqueFieldEntry fieldEntry){ /** - * final UUID version = value.getVersion(); - final Field field = value.getField(); + * final UUID version = value.getVersion(); + final Field field = value.getField(); - final FieldTypeName fieldType = field.getTypeName(); - final String fieldValue = field.getValue().toString().toLowerCase(); + final FieldTypeName fieldType = field.getTypeName(); + final String fieldValue = field.getValue().toString().toLowerCase(); - DynamicComposite composite = new DynamicComposite( ); + DynamicComposite composite = new DynamicComposite( ); - //we want to sort ascending to descending by version - composite.addComponent( version, UUID_SERIALIZER, ColumnTypes.UUID_TYPE_REVERSED); - composite.addComponent( field.getName(), STRING_SERIALIZER ); - composite.addComponent( fieldValue, STRING_SERIALIZER ); - composite.addComponent( fieldType.name() , STRING_SERIALIZER); + //we want to sort ascending to descending by version + composite.addComponent( version, UUID_SERIALIZER, ColumnTypes.UUID_TYPE_REVERSED); + composite.addComponent( field.getName(), STRING_SERIALIZER ); + composite.addComponent( fieldValue, STRING_SERIALIZER ); + composite.addComponent( fieldType.name() , STRING_SERIALIZER); */ // values are serialized as strings, not sure why, and always lower cased @@ -337,15 +317,15 @@ protected ByteBuffer getLogPartitionKey(final Id applicationId, final Id uniqueV protected ByteBuffer serializeUniqueValueColumn(EntityVersion entityVersion){ /** - * final Id entityId = ev.getEntityId(); - final UUID entityUuid = entityId.getUuid(); - final String entityType = entityId.getType(); + * final Id entityId = ev.getEntityId(); + final UUID entityUuid = entityId.getUuid(); + final String entityType = entityId.getType(); - CompositeBuilder builder = Composites.newDynamicCompositeBuilder(); + CompositeBuilder builder = Composites.newDynamicCompositeBuilder(); - builder.addUUID( entityVersion ); - builder.addUUID( entityUuid ); - builder.addString(entityType ); + builder.addUUID( entityVersion ); + builder.addUUID( entityUuid ); + builder.addString(entityType ); */ String comparator = "UTF8Type"; @@ -418,7 +398,49 @@ protected List deserializeUniqueValueColumn(ByteBuffer bb){ }else if(count ==1){ stuff.add(new UUID(data.getLong(), data.getLong())); }else{ - stuff.add(DataType.text().deserialize(data.duplicate(), ProtocolVersion.NEWEST_SUPPORTED)); + stuff.add(DataType.text().deserialize(data.slice(), ProtocolVersion.NEWEST_SUPPORTED)); + } + + byte equality = bb.get(); // we don't use this but take the equality byte off the buffer + + count++; + } + + return stuff; + + } + + @Override + protected List deserializeUniqueValueLogColumn(ByteBuffer bb){ + + + /** + * List keys = new ArrayList<>(4); + keys.add(fieldEntry.getVersion()); + keys.add(fieldEntry.getField().getName()); + keys.add(fieldValueString); + keys.add(fieldEntry.getField().getTypeName().name()); + */ + + List stuff = new ArrayList<>(); + int count = 0; + while(bb.hasRemaining()){ + + // the comparator info is different for the UUID reversed type vs. UTF8 type + if(count ==0){ + bb.getShort(); // take the reversed comparator byte off + }else { + ByteBuffer comparator = CQLUtils.getWithShortLength(bb); + } + + ByteBuffer data = CQLUtils.getWithShortLength(bb); + + + // first composite is a UUID, rest are strings + if(count == 0) { + stuff.add(new UUID(data.getLong(), data.getLong())); + }else{ + stuff.add(DataType.text().deserialize(data.slice(), ProtocolVersion.NEWEST_SUPPORTED)); } byte equality = bb.get(); // we don't use this but take the equality byte off the buffer @@ -465,15 +487,15 @@ private ByteBuffer serializeKey( UUID appUUID, final String collectionName = LegacyScopeUtils.getCollectionScopeNameFromEntityType( entityType ); -// final CollectionPrefixedKey uniquePrefixedKey = -// new CollectionPrefixedKey<>( collectionName, applicationId, field ); - -// //read back the id -// final Id orgId = ID_SER.fromComposite( parser ); -// final Id scopeId = ID_SER.fromComposite( parser ); -// final String scopeName = parser.readString(); -// final K value = keySerializer.fromComposite( parser ); + /** + final CollectionPrefixedKey uniquePrefixedKey = + new CollectionPrefixedKey<>( collectionName, applicationId, field ); + final Id orgId = ID_SER.fromComposite( parser ); + final Id scopeId = ID_SER.fromComposite( parser ); + final String scopeName = parser.readString(); + final K value = keySerializer.fromComposite( parser ); + **/ // values are serialized as strings, not sure why, and always lower cased String fieldValueString = fieldValue.toString().toLowerCase(); @@ -521,10 +543,11 @@ private ByteBuffer serializeLogKey(UUID appUUID, String applicationType, UUID en final String collectionName = LegacyScopeUtils.getCollectionScopeNameFromEntityType( entityType ); -// -// -// final CollectionPrefixedKey collectionPrefixedEntityKey = -// new CollectionPrefixedKey<>( collectionName, applicationId, uniqueValueId ); + + /** + final CollectionPrefixedKey collectionPrefixedEntityKey = + new CollectionPrefixedKey<>( collectionName, applicationId, uniqueValueId ); + **/ List keys = new ArrayList<>(4); keys.add(appUUID); diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyV2Impl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyV2Impl.java index 4177c37d90..3e4932aaf5 100644 --- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyV2Impl.java +++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyV2Impl.java @@ -40,7 +40,6 @@ import org.apache.usergrid.persistence.core.datastax.CQLUtils; import org.apache.usergrid.persistence.core.datastax.TableDefinition; import org.apache.usergrid.persistence.model.entity.Id; -import org.apache.usergrid.persistence.model.field.Field; import com.google.inject.Inject; import com.google.inject.Singleton; @@ -171,30 +170,18 @@ protected TableDefinition getEntityUniqueLogTable(){ } - - @Override - protected TypeField createUniqueValueKey( final Id applicationId, final String type, final Field field) { - return new TypeField(type,field); - } - - - @Override - protected Field parseRowKey( final ScopedRowKey rowKey ) { - return rowKey.getKey().getField(); - } - @Override protected List deserializePartitionKey(ByteBuffer bb){ /** - * List keys = new ArrayList<>(6); - keys.add(0, appUUID); // UUID - keys.add(1, applicationType); // String - keys.add(2, entityType); // String - keys.add(3, fieldType); // String - keys.add(4, fieldName); // String - keys.add(5, fieldValueString); // String + * List keys = new ArrayList<>(6); + keys.add(0, appUUID); // UUID + keys.add(1, applicationType); // String + keys.add(2, entityType); // String + keys.add(3, fieldType); // String + keys.add(4, fieldName); // String + keys.add(5, fieldValueString); // String */ @@ -215,23 +202,23 @@ protected List deserializePartitionKey(ByteBuffer bb){ } @Override - protected Object serializeUniqueValueLogColumn(UniqueFieldEntry fieldEntry){ + protected ByteBuffer serializeUniqueValueLogColumn(UniqueFieldEntry fieldEntry){ /** - * final UUID version = value.getVersion(); - final Field field = value.getField(); + * final UUID version = value.getVersion(); + final Field field = value.getField(); - final FieldTypeName fieldType = field.getTypeName(); - final String fieldValue = field.getValue().toString().toLowerCase(); + final FieldTypeName fieldType = field.getTypeName(); + final String fieldValue = field.getValue().toString().toLowerCase(); - DynamicComposite composite = new DynamicComposite( ); + DynamicComposite composite = new DynamicComposite( ); - //we want to sort ascending to descending by version - composite.addComponent( version, UUID_SERIALIZER, ColumnTypes.UUID_TYPE_REVERSED); - composite.addComponent( field.getName(), STRING_SERIALIZER ); - composite.addComponent( fieldValue, STRING_SERIALIZER ); - composite.addComponent( fieldType.name() , STRING_SERIALIZER); + //we want to sort ascending to descending by version + composite.addComponent( version, UUID_SERIALIZER, ColumnTypes.UUID_TYPE_REVERSED); + composite.addComponent( field.getName(), STRING_SERIALIZER ); + composite.addComponent( fieldValue, STRING_SERIALIZER ); + composite.addComponent( fieldType.name() , STRING_SERIALIZER); */ // values are serialized as strings, not sure why, and always lower cased @@ -250,7 +237,7 @@ protected Object serializeUniqueValueLogColumn(UniqueFieldEntry fieldEntry){ fieldEntry.getField().getTypeName().name().length(); // we always need to add length for the 2 byte comparator short, 2 byte length short and 1 byte equality - size += keys.size()*65; + size += keys.size()*5; // uuid type comparator is longest, ensure we allocate buffer using the max size to avoid overflow size += keys.size()*comparator.length(); @@ -322,15 +309,15 @@ protected ByteBuffer getLogPartitionKey(final Id applicationId, final Id uniqueV protected ByteBuffer serializeUniqueValueColumn(EntityVersion entityVersion){ /** - * final Id entityId = ev.getEntityId(); - final UUID entityUuid = entityId.getUuid(); - final String entityType = entityId.getType(); + * final Id entityId = ev.getEntityId(); + final UUID entityUuid = entityId.getUuid(); + final String entityType = entityId.getType(); - CompositeBuilder builder = Composites.newDynamicCompositeBuilder(); + CompositeBuilder builder = Composites.newDynamicCompositeBuilder(); - builder.addUUID( entityVersion ); - builder.addUUID( entityUuid ); - builder.addString(entityType ); + builder.addUUID( entityVersion ); + builder.addUUID( entityUuid ); + builder.addString(entityType ); */ String comparator = "UTF8Type"; @@ -403,7 +390,49 @@ protected List deserializeUniqueValueColumn(ByteBuffer bb){ }else if(count ==1){ stuff.add(new UUID(data.getLong(), data.getLong())); }else{ - stuff.add(DataType.text().deserialize(data.duplicate(), ProtocolVersion.NEWEST_SUPPORTED)); + stuff.add(DataType.text().deserialize(data.slice(), ProtocolVersion.NEWEST_SUPPORTED)); + } + + byte equality = bb.get(); // we don't use this but take the equality byte off the buffer + + count++; + } + + return stuff; + + } + + @Override + protected List deserializeUniqueValueLogColumn(ByteBuffer bb){ + + + /** + * List keys = new ArrayList<>(4); + keys.add(fieldEntry.getVersion()); + keys.add(fieldEntry.getField().getName()); + keys.add(fieldValueString); + keys.add(fieldEntry.getField().getTypeName().name()); + */ + + List stuff = new ArrayList<>(); + int count = 0; + while(bb.hasRemaining()){ + + // the comparator info is different for the UUID reversed type vs. UTF8 type + if(count ==0){ + bb.getShort(); // take the reversed comparator byte off + }else { + ByteBuffer comparator = CQLUtils.getWithShortLength(bb); + } + + ByteBuffer data = CQLUtils.getWithShortLength(bb); + + + // first composite is a UUID, rest are strings + if(count == 0) { + stuff.add(new UUID(data.getLong(), data.getLong())); + }else{ + stuff.add(DataType.text().deserialize(data.slice(), ProtocolVersion.NEWEST_SUPPORTED)); } byte equality = bb.get(); // we don't use this but take the equality byte off the buffer diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSetImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSetImpl.java index 8dd9528bc2..853913b4c6 100644 --- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSetImpl.java +++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSetImpl.java @@ -37,7 +37,11 @@ public UniqueValueSetImpl(final int expectedMaxSize) { public void addValue(UniqueValue value){ - values.put( value.getField().getName(), value ); + values.putIfAbsent( value.getField().getName(), value ); + // ^^ putIfAbsent important here as CQL returns column values differently than Asytanax/thrift due to CQL not + // having a 'column range' for each row slice and all columns are returned. We don't want to overwrite the + // first column values retrieved + } @Override diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteOptimisticVerifyTest.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteOptimisticVerifyTest.java index 6a705e4e1d..148cc0955a 100644 --- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteOptimisticVerifyTest.java +++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteOptimisticVerifyTest.java @@ -21,7 +21,12 @@ import java.util.ArrayList; import java.util.List; +import com.datastax.driver.core.BatchStatement; +import com.datastax.driver.core.Session; +import com.google.inject.Inject; +import org.apache.usergrid.persistence.core.test.ITRunner; import org.junit.Test; +import org.junit.runner.RunWith; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -53,8 +58,6 @@ import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; - - @UseModules( TestCollectionModule.class ) public class WriteOptimisticVerifyTest extends AbstractMvccEntityStageTest { @@ -110,6 +113,7 @@ public void testConflict() throws Exception { when( scope.getApplication() ) .thenReturn( new SimpleId( UUIDGenerator.newTimeUUID(), "organization" ) ); + final Session session = mock(Session.class); // there is an entity final Entity entity = generateEntity(); @@ -135,16 +139,13 @@ public void testConflict() throws Exception { UniqueValueSerializationStrategy uvstrat = mock( UniqueValueSerializationStrategy.class); UniqueValue uv1 = new UniqueValueImpl(entity.getField("name"), entity.getId(), entity.getVersion()); UniqueValue uv2 = new UniqueValueImpl( entity.getField("identifier"), entity.getId(), entity.getVersion()); - MutationBatch mb = mock( MutationBatch.class ); - when( uvstrat.delete(scope, uv1) ).thenReturn(mb); - when( uvstrat.delete(scope, uv2) ).thenReturn(mb); // Run the stage, conflict should be detected final MvccEntity mvccEntity = fromEntity( entity ); boolean conflictDetected = false; WriteOptimisticVerify newStage = new WriteOptimisticVerify( mvccLog ); - RollbackAction rollbackAction = new RollbackAction( mvccLog, uvstrat ); + RollbackAction rollbackAction = new RollbackAction( mvccLog, uvstrat, session ); try { newStage.call( new CollectionIoEvent<>(scope, mvccEntity)); @@ -157,8 +158,8 @@ public void testConflict() throws Exception { assertTrue( conflictDetected ); // check that unique values were deleted - verify( uvstrat, times(1) ).delete(scope, uv1 ); - verify( uvstrat, times(1) ).delete(scope, uv2 ); + verify( uvstrat, times(1) ).deleteCQL(scope, uv1 ); + verify( uvstrat, times(1) ).deleteCQL(scope, uv2 ); } } diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyImplTest.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyImplTest.java index 3ffdb6581e..185cfb74d9 100644 --- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyImplTest.java +++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyImplTest.java @@ -192,7 +192,10 @@ public void testDelete() throws ConnectionException { BatchStatement batch = strategy.writeCQL( scope, stored, -1); session.execute(batch); - strategy.delete( scope, stored ).execute(); + + //strategy.delete( scope, stored ).execute(); + BatchStatement deleteBatch = strategy.deleteCQL(scope, stored); + session.execute(deleteBatch); UniqueValueSet fields = strategy.load( scope, entityId.getType(), Collections.singleton( field ) ); From 24b42856293470f87bba701e01359a2f56a0850a Mon Sep 17 00:00:00 2001 From: Michael Russo Date: Sun, 8 May 2016 22:16:37 +0800 Subject: [PATCH 27/30] Unique Value serialization table creation now via CQL instead of Astyanax/Thrift. --- .../CpEntityManagerFactory.java | 6 ++--- .../UniqueValueSerializationStrategyImpl.java | 26 ++----------------- ...niqueValueSerializationStrategyV1Impl.java | 18 ++++++++++--- ...niqueValueSerializationStrategyV2Impl.java | 18 +++++++++---- .../persistence/core/datastax/CQLUtils.java | 12 ++++++++- 5 files changed, 43 insertions(+), 37 deletions(-) diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java index c3bcaf6358..91a936dae7 100644 --- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java +++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java @@ -530,12 +530,12 @@ public void bootstrap() throws Exception { // Always make sure the database schema is initialized getSetup().initSchema(); - // Make sure the management application is created - initMgmtAppInternal(); - // Roll the new 2.x Migration classes to the latest version supported getSetup().runDataMigration(); + // Make sure the management application is created + initMgmtAppInternal(); + // Ensure management app is initialized getSetup().initMgmtApp(); diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyImpl.java index e0a9035a7c..fe4e06ff48 100644 --- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyImpl.java +++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyImpl.java @@ -361,32 +361,10 @@ public Iterator getAllUniqueFields( final ApplicationScope collecti @Override - public Collection getColumnFamilies() { - - final MultiTenantColumnFamilyDefinition uniqueLookupCF = - new MultiTenantColumnFamilyDefinition( CF_UNIQUE_VALUES, BytesType.class.getSimpleName(), - ColumnTypes.DYNAMIC_COMPOSITE_TYPE, BytesType.class.getSimpleName(), - MultiTenantColumnFamilyDefinition.CacheOption.KEYS ); - - final MultiTenantColumnFamilyDefinition uniqueLogCF = - new MultiTenantColumnFamilyDefinition( CF_ENTITY_UNIQUE_VALUE_LOG, BytesType.class.getSimpleName(), - ColumnTypes.DYNAMIC_COMPOSITE_TYPE, BytesType.class.getSimpleName(), - MultiTenantColumnFamilyDefinition.CacheOption.KEYS ); - - return Arrays.asList( uniqueLookupCF, uniqueLogCF ); - } + public abstract Collection getColumnFamilies(); @Override - public Collection getTables() { - - final TableDefinition uniqueValues = getUniqueValuesTable(); - - final TableDefinition uniqueValuesLog = getEntityUniqueLogTable(); - - - return Arrays.asList( uniqueValues, uniqueValuesLog ); - - } + public abstract Collection getTables(); /** diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyV1Impl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyV1Impl.java index cbd8a3e8b0..73df64a220 100644 --- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyV1Impl.java +++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyV1Impl.java @@ -61,10 +61,10 @@ public class UniqueValueSerializationStrategyV1Impl extends UniqueValueSerializ private static final Map UNIQUE_VALUES_COLUMNS = new HashMap() {{ put( "key", DataType.Name.BLOB ); - put( "column1", DataType.Name.BLOB ); + put( "column1", DataType.Name.CUSTOM ); put( "value", DataType.Name.BLOB ); }}; private static final Map UNIQUE_VALUES_CLUSTERING_ORDER = - new HashMap(){{ put( "column1", "ASC" ); }}; + new HashMap(){{ put( "column1", "ASC" );}}; private static final String UNIQUE_VALUES_LOG_TABLE = CQLUtils.quote("Entity_Unique_Values"); @@ -73,7 +73,7 @@ public class UniqueValueSerializationStrategyV1Impl extends UniqueValueSerializ private static final Map UNIQUE_VALUES_LOG_COLUMNS = new HashMap() {{ put( "key", DataType.Name.BLOB ); - put( "column1", DataType.Name.BLOB ); + put( "column1", DataType.Name.CUSTOM ); put( "value", DataType.Name.BLOB ); }}; private static final Map UNIQUE_VALUES_LOG_CLUSTERING_ORDER = new HashMap(){{ put( "column1", "ASC" ); }}; @@ -138,13 +138,23 @@ public Collection getColumnFamilies() { ColumnTypes.DYNAMIC_COMPOSITE_TYPE, BytesType.class.getSimpleName(), MultiTenantColumnFamilyDefinition.CacheOption.KEYS ); - return Arrays.asList( uniqueLookupCF, uniqueLogCF ); + //return Collections.emptyList(); + + return Arrays.asList(uniqueLookupCF, uniqueLogCF); } @Override public Collection getTables() { + final TableDefinition uniqueValues = getUniqueValuesTable(); + + final TableDefinition uniqueValuesLog = getEntityUniqueLogTable(); + + + //return Arrays.asList( uniqueValues, uniqueValuesLog ); + return Collections.emptyList(); + } diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyV2Impl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyV2Impl.java index 3e4932aaf5..8e13f865c7 100644 --- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyV2Impl.java +++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyV2Impl.java @@ -58,10 +58,10 @@ public class UniqueValueSerializationStrategyV2Impl extends UniqueValueSerializ private static final Map UNIQUE_VALUES_COLUMNS = new HashMap() {{ put( "key", DataType.Name.BLOB ); - put( "column1", DataType.Name.BLOB ); + put( "column1", DataType.Name.CUSTOM ); put( "value", DataType.Name.BLOB ); }}; private static final Map UNIQUE_VALUES_CLUSTERING_ORDER = - new HashMap(){{ put( "column1", "ASC" ); }}; + new HashMap(){{ put( "column1", "ASC" );}}; private static final String UNIQUE_VALUES_LOG_TABLE = CQLUtils.quote("Entity_Unique_Values_V2"); @@ -70,10 +70,10 @@ public class UniqueValueSerializationStrategyV2Impl extends UniqueValueSerializ private static final Map UNIQUE_VALUES_LOG_COLUMNS = new HashMap() {{ put( "key", DataType.Name.BLOB ); - put( "column1", DataType.Name.BLOB ); + put( "column1", DataType.Name.CUSTOM ); put( "value", DataType.Name.BLOB ); }}; private static final Map UNIQUE_VALUES_LOG_CLUSTERING_ORDER = - new HashMap(){{ put( "column1", "ASC" ); }}; + new HashMap(){{ put( "column1", "ASC" );}}; private final static TableDefinition uniqueValues = new TableDefinition( UNIQUE_VALUES_TABLE, UNIQUE_VALUES_PARTITION_KEYS, UNIQUE_VALUES_COLUMN_KEYS, @@ -134,15 +134,23 @@ public Collection getColumnFamilies() { ColumnTypes.DYNAMIC_COMPOSITE_TYPE, BytesType.class.getSimpleName(), MultiTenantColumnFamilyDefinition.CacheOption.KEYS ); - return Arrays.asList( uniqueLookupCF, uniqueLogCF ); + return Collections.emptyList(); + + //return Arrays.asList(uniqueLookupCF, uniqueLogCF); } @Override public Collection getTables() { + final TableDefinition uniqueValues = getUniqueValuesTable(); + + final TableDefinition uniqueValuesLog = getEntityUniqueLogTable(); + return Arrays.asList( uniqueValues, uniqueValuesLog ); + //return Collections.emptyList(); + } @Override diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/CQLUtils.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/CQLUtils.java index f65a2605d5..0d6a312980 100644 --- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/CQLUtils.java +++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/CQLUtils.java @@ -18,6 +18,7 @@ */ package org.apache.usergrid.persistence.core.datastax; +import com.datastax.driver.core.DataType; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.inject.Inject; @@ -52,6 +53,8 @@ public enum ACTION { static String PAREN_LEFT = "("; static String PAREN_RIGHT = ")"; + static String COMPOSITE_TYPE = "'org.apache.cassandra.db.marshal.DynamicCompositeType(a=>org.apache.cassandra.db.marshal.AsciiType,A=>org.apache.cassandra.db.marshal.ReversedType(org.apache.cassandra.db.marshal.AsciiType),b=>org.apache.cassandra.db.marshal.BytesType,B=>org.apache.cassandra.db.marshal.ReversedType(org.apache.cassandra.db.marshal.BytesType),i=>org.apache.cassandra.db.marshal.IntegerType,I=>org.apache.cassandra.db.marshal.ReversedType(org.apache.cassandra.db.marshal.IntegerType),l=>org.apache.cassandra.db.marshal.LongType,L=>org.apache.cassandra.db.marshal.ReversedType(org.apache.cassandra.db.marshal.LongType),s=>org.apache.cassandra.db.marshal.UTF8Type,S=>org.apache.cassandra.db.marshal.ReversedType(org.apache.cassandra.db.marshal.UTF8Type),t=>org.apache.cassandra.db.marshal.TimeUUIDType,T=>org.apache.cassandra.db.marshal.ReversedType(org.apache.cassandra.db.marshal.TimeUUIDType),u=>org.apache.cassandra.db.marshal.UUIDType,U=>org.apache.cassandra.db.marshal.ReversedType(org.apache.cassandra.db.marshal.UUIDType),x=>org.apache.cassandra.db.marshal.LexicalUUIDType,X=>org.apache.cassandra.db.marshal.ReversedType(org.apache.cassandra.db.marshal.LexicalUUIDType))'"; + @Inject public CQLUtils ( final CassandraFig cassandraFig ){ @@ -145,7 +148,14 @@ public static String quote( String value){ public static String spaceSeparatedKeyValue(Map columns){ StringJoiner columnsSchema = new StringJoiner(","); - columns.forEach( (key, value) -> columnsSchema.add(key+" "+String.valueOf(value))); + columns.forEach( (key, value) -> { + + if( value == DataType.Name.CUSTOM ){ + columnsSchema.add(key+" "+COMPOSITE_TYPE); + }else { + columnsSchema.add(key + " " + String.valueOf(value)); + } + }); return columnsSchema.toString(); From 6efb5bad0106cdbe8b0e48843ee68fcfd1aa56d3 Mon Sep 17 00:00:00 2001 From: Michael Russo Date: Sun, 8 May 2016 22:39:54 +0800 Subject: [PATCH 28/30] Remove remaining traces of Astyanax from Unique Value serialization. --- .../impl/EntityCollectionManagerImpl.java | 95 +++++++++---------- .../mvcc/stage/write/WriteUniqueVerify.java | 15 +-- .../UniqueValueSerializationStrategy.java | 16 +--- .../UniqueValueSerializationStrategyImpl.java | 71 +++----------- ...ueValueSerializationStrategyProxyImpl.java | 14 +-- ...niqueValueSerializationStrategyV1Impl.java | 85 ++--------------- ...niqueValueSerializationStrategyV2Impl.java | 67 +------------ 7 files changed, 80 insertions(+), 283 deletions(-) diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java index 6d42fa20e3..291e5dfc9d 100644 --- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java +++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java @@ -296,15 +296,11 @@ public Observable delete( final Collection entries ) public Observable getIdField( final String type, final Field field ) { final List fields = Collections.singletonList( field ); final Observable idObservable = Observable.from( fields ).map( field1 -> { - try { - final UniqueValueSet set = uniqueValueSerializationStrategy.load( applicationScope, type, fields ); - final UniqueValue value = set.getValue( field1.getName() ); - return value == null ? null : value.getEntityId(); - } - catch ( ConnectionException e ) { - logger.error( "Failed to getIdField", e ); - throw new RuntimeException( e ); - } + + final UniqueValueSet set = uniqueValueSerializationStrategy.load( applicationScope, type, fields ); + final UniqueValue value = set.getValue( field1.getName() ); + return value == null ? null : value.getEntityId(); + } ); return ObservableTimer.time( idObservable, fieldIdTimer ); @@ -317,68 +313,63 @@ public Observable getIdField( final String type, final Field field ) { @Override public Observable
getEntitiesFromFields( final String type, final Collection fields ) { final Observable
fieldSetObservable = Observable.just( fields ).map( fields1 -> { - try { - final UUID startTime = UUIDGenerator.newTimeUUID(); + final UUID startTime = UUIDGenerator.newTimeUUID(); - //Get back set of unique values that correspond to collection of fields - UniqueValueSet set = uniqueValueSerializationStrategy.load( applicationScope, type, fields1 ); - - //Short circuit if we don't have any uniqueValues from the given fields. - if ( !set.iterator().hasNext() ) { - return new MutableFieldSet( 0 ); - } + //Get back set of unique values that correspond to collection of fields + UniqueValueSet set = uniqueValueSerializationStrategy.load( applicationScope, type, fields1 ); + //Short circuit if we don't have any uniqueValues from the given fields. + if ( !set.iterator().hasNext() ) { + return new MutableFieldSet( 0 ); + } - //loop through each field, and construct an entity load - List entityIds = new ArrayList<>( fields1.size() ); - List uniqueValues = new ArrayList<>( fields1.size() ); - for ( final Field expectedField : fields1 ) { + //loop through each field, and construct an entity load + List entityIds = new ArrayList<>( fields1.size() ); + List uniqueValues = new ArrayList<>( fields1.size() ); - UniqueValue value = set.getValue( expectedField.getName() ); + for ( final Field expectedField : fields1 ) { - if ( value == null ) { - logger.debug( "Field does not correspond to a unique value" ); - } + UniqueValue value = set.getValue( expectedField.getName() ); - entityIds.add( value.getEntityId() ); - uniqueValues.add( value ); + if ( value == null ) { + logger.debug( "Field does not correspond to a unique value" ); } - //Load a entity for each entityId we retrieved. - final EntitySet entitySet = entitySerializationStrategy.load( applicationScope, entityIds, startTime ); - - final BatchStatement uniqueDeleteBatch = new BatchStatement(); - - final MutableFieldSet response = new MutableFieldSet( fields1.size() ); + entityIds.add( value.getEntityId() ); + uniqueValues.add( value ); + } - for ( final UniqueValue expectedUnique : uniqueValues ) { - final MvccEntity entity = entitySet.getEntity( expectedUnique.getEntityId() ); + //Load a entity for each entityId we retrieved. + final EntitySet entitySet = entitySerializationStrategy.load( applicationScope, entityIds, startTime ); - //bad unique value, delete this, it's inconsistent - if ( entity == null || !entity.getEntity().isPresent() ) { - uniqueDeleteBatch.add( - uniqueValueSerializationStrategy.deleteCQL( applicationScope, expectedUnique )); - continue; - } + final BatchStatement uniqueDeleteBatch = new BatchStatement(); - //TODO, we need to validate the property in the entity matches the property in the unique value + final MutableFieldSet response = new MutableFieldSet( fields1.size() ); + for ( final UniqueValue expectedUnique : uniqueValues ) { + final MvccEntity entity = entitySet.getEntity( expectedUnique.getEntityId() ); - //else add it to our result set - response.addEntity( expectedUnique.getField(), entity ); + //bad unique value, delete this, it's inconsistent + if ( entity == null || !entity.getEntity().isPresent() ) { + uniqueDeleteBatch.add( + uniqueValueSerializationStrategy.deleteCQL( applicationScope, expectedUnique )); + continue; } - //TODO: explore making this an Async process - session.execute(uniqueDeleteBatch); + //TODO, we need to validate the property in the entity matches the property in the unique value - return response; - } - catch ( ConnectionException e ) { - logger.error( "Failed to getIdField", e ); - throw new RuntimeException( e ); + + //else add it to our result set + response.addEntity( expectedUnique.getField(), entity ); } + + //TODO: explore making this an Async process + session.execute(uniqueDeleteBatch); + + return response; + } ); diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java index 501950ac93..5e99a05f17 100644 --- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java +++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java @@ -24,6 +24,7 @@ import java.util.Map; import com.datastax.driver.core.BatchStatement; +import com.datastax.driver.core.ConsistencyLevel; import com.datastax.driver.core.Session; import com.netflix.hystrix.HystrixCommandProperties; import org.slf4j.Logger; @@ -49,9 +50,7 @@ import com.google.inject.Inject; import com.google.inject.Singleton; import com.netflix.astyanax.Keyspace; -import com.netflix.astyanax.MutationBatch; import com.netflix.astyanax.connectionpool.exceptions.ConnectionException; -import com.netflix.astyanax.model.ConsistencyLevel; import com.netflix.hystrix.HystrixCommand; import com.netflix.hystrix.HystrixCommandGroupKey; import com.netflix.hystrix.HystrixThreadPoolProperties; @@ -180,25 +179,21 @@ public ConsistentReplayCommand( UniqueValueSerializationStrategy uniqueValueSeri @Override protected Map run() throws Exception { - return executeStrategy(fig.getReadCL()); + return executeStrategy(fig.getDataStaxReadCl()); } @Override protected Map getFallback() { // fallback with same CL as there are many reasons the 1st execution failed, not just due to consistency problems - return executeStrategy(fig.getReadCL()); + return executeStrategy(fig.getDataStaxReadCl()); } public Map executeStrategy(ConsistencyLevel consistencyLevel){ //allocate our max size, worst case //now get the set of fields back final UniqueValueSet uniqueValues; - try { - uniqueValues = uniqueValueSerializationStrategy.load( scope, consistencyLevel, type, uniqueFields ); - } - catch ( ConnectionException e ) { - throw new RuntimeException( "Unable to read from cassandra", e ); - } + + uniqueValues = uniqueValueSerializationStrategy.load( scope, consistencyLevel, type, uniqueFields ); final Map uniquenessViolations = new HashMap<>( uniqueFields.size() ); diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/UniqueValueSerializationStrategy.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/UniqueValueSerializationStrategy.java index bb6f5fe37c..eb4398595e 100644 --- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/UniqueValueSerializationStrategy.java +++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/UniqueValueSerializationStrategy.java @@ -22,17 +22,13 @@ import java.util.Iterator; import com.datastax.driver.core.BatchStatement; -import org.apache.usergrid.persistence.collection.serialization.impl.UniqueValueSerializationStrategyImpl; +import com.datastax.driver.core.ConsistencyLevel; import org.apache.usergrid.persistence.core.migration.data.VersionedData; import org.apache.usergrid.persistence.core.migration.schema.Migration; import org.apache.usergrid.persistence.core.scope.ApplicationScope; import org.apache.usergrid.persistence.model.entity.Id; import org.apache.usergrid.persistence.model.field.Field; -import com.netflix.astyanax.MutationBatch; -import com.netflix.astyanax.connectionpool.exceptions.ConnectionException; -import com.netflix.astyanax.model.ConsistencyLevel; - /** * Reads and writes to UniqueValues column family. @@ -60,9 +56,8 @@ public interface UniqueValueSerializationStrategy extends Migration, VersionedDa * * @return UniqueValueSet containing fields from the collection that exist in cassandra * - * @throws ConnectionException on error connecting to Cassandra */ - UniqueValueSet load( ApplicationScope applicationScope, String type, Collection fields ) throws ConnectionException; + UniqueValueSet load( ApplicationScope applicationScope, String type, Collection fields ); /** * Load UniqueValue that matches field from collection or null if that value does not exist. @@ -72,10 +67,9 @@ public interface UniqueValueSerializationStrategy extends Migration, VersionedDa * @param type The type the unique value exists within * @param fields Field name/value to search for * @return UniqueValueSet containing fields from the collection that exist in cassandra - * @throws ConnectionException on error connecting to Cassandra */ - UniqueValueSet load( ApplicationScope applicationScope, ConsistencyLevel consistencyLevel, String type, - Collection fields ) throws ConnectionException; + UniqueValueSet load(ApplicationScope applicationScope, ConsistencyLevel consistencyLevel, String type, + Collection fields ); /** @@ -96,7 +90,7 @@ UniqueValueSet load( ApplicationScope applicationScope, ConsistencyLevel consist * @param uniqueValue Object to be deleted. * @return BatchStatement that encapsulates the CQL statements, caller may or may not execute. */ - BatchStatement deleteCQL( ApplicationScope applicationScope, UniqueValue uniqueValue); + BatchStatement deleteCQL( ApplicationScope applicationScope, UniqueValue uniqueValue ); } diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyImpl.java index fe4e06ff48..532015243c 100644 --- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyImpl.java +++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyImpl.java @@ -33,26 +33,18 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.apache.cassandra.db.marshal.BytesType; import org.apache.usergrid.persistence.collection.serialization.SerializationFig; import org.apache.usergrid.persistence.collection.serialization.UniqueValue; import org.apache.usergrid.persistence.collection.serialization.UniqueValueSerializationStrategy; import org.apache.usergrid.persistence.collection.serialization.UniqueValueSet; import org.apache.usergrid.persistence.core.CassandraFig; -import org.apache.usergrid.persistence.core.astyanax.ColumnTypes; -import org.apache.usergrid.persistence.core.astyanax.MultiTenantColumnFamily; import org.apache.usergrid.persistence.core.astyanax.MultiTenantColumnFamilyDefinition; -import org.apache.usergrid.persistence.core.astyanax.ScopedRowKey; import org.apache.usergrid.persistence.core.scope.ApplicationScope; import org.apache.usergrid.persistence.core.util.ValidationUtils; import org.apache.usergrid.persistence.model.entity.Id; import com.google.common.base.Preconditions; -import com.netflix.astyanax.ColumnListMutation; -import com.netflix.astyanax.Keyspace; -import com.netflix.astyanax.MutationBatch; -import com.netflix.astyanax.connectionpool.exceptions.ConnectionException; /** @@ -66,28 +58,17 @@ public abstract class UniqueValueSerializationStrategyImpl public static final String UUID_TYPE_REVERSED = "UUIDType(reversed=true)"; - - private final MultiTenantColumnFamily, EntityVersion> - CF_UNIQUE_VALUES; - - - private final MultiTenantColumnFamily, UniqueFieldEntry> - CF_ENTITY_UNIQUE_VALUE_LOG ; - private final String TABLE_UNIQUE_VALUES; private final String TABLE_UNIQUE_VALUES_LOG; - private final Map COLUMNS_UNIQUE_VALUES; private final Map COLUMNS_UNIQUE_VALUES_LOG; - public static final int COL_VALUE = 0x0; private final SerializationFig serializationFig; - protected final Keyspace keyspace; private final CassandraFig cassandraFig; private final Session session; @@ -97,23 +78,19 @@ public abstract class UniqueValueSerializationStrategyImpl /** * Construct serialization strategy for keyspace. * - * @param keyspace Keyspace in which to store Unique Values. * @param cassandraFig The cassandra configuration * @param serializationFig The serialization configuration */ - public UniqueValueSerializationStrategyImpl( final Keyspace keyspace, final CassandraFig cassandraFig, + public UniqueValueSerializationStrategyImpl( final CassandraFig cassandraFig, final SerializationFig serializationFig, - final Session session, final CassandraConfig cassandraConfig) { - this.keyspace = keyspace; + final Session session, + final CassandraConfig cassandraConfig) { this.cassandraFig = cassandraFig; this.serializationFig = serializationFig; this.session = session; this.cassandraConfig = cassandraConfig; - CF_UNIQUE_VALUES = getUniqueValuesCF(); - CF_ENTITY_UNIQUE_VALUE_LOG = getEntityUniqueLogCF(); - TABLE_UNIQUE_VALUES = getUniqueValuesTable().getTableName(); TABLE_UNIQUE_VALUES_LOG = getEntityUniqueLogTable().getTableName(); @@ -242,27 +219,27 @@ public BatchStatement deleteCQL( final ApplicationScope scope, UniqueValue value @Override public UniqueValueSet load( final ApplicationScope colScope, final String type, final Collection fields ) - throws ConnectionException { - return load( colScope, com.netflix.astyanax.model.ConsistencyLevel.valueOf( cassandraFig.getAstyanaxReadCL() ), type, fields ); + { + return load( colScope, ConsistencyLevel.valueOf( cassandraFig.getReadCl() ), type, fields ); } @Override - public UniqueValueSet load( final ApplicationScope appScope, final com.netflix.astyanax.model.ConsistencyLevel consistencyLevel, - final String type, final Collection fields ) throws ConnectionException { + public UniqueValueSet load( final ApplicationScope appScope, + final ConsistencyLevel consistencyLevel, + final String type, final Collection fields ) { Preconditions.checkNotNull( fields, "fields are required" ); Preconditions.checkArgument( fields.size() > 0, "More than 1 field must be specified" ); - return loadCQL(appScope, com.datastax.driver.core.ConsistencyLevel.LOCAL_QUORUM, type, fields); - - //return loadLegacy( appScope, type, fields); + return loadCQL(appScope, consistencyLevel, type, fields); } - private UniqueValueSet loadCQL( final ApplicationScope appScope, final com.datastax.driver.core.ConsistencyLevel consistencyLevel, - final String type, final Collection fields ) throws ConnectionException { + private UniqueValueSet loadCQL( final ApplicationScope appScope, + final ConsistencyLevel consistencyLevel, + final String type, final Collection fields ) { Preconditions.checkNotNull( fields, "fields are required" ); Preconditions.checkArgument( fields.size() > 0, "More than 1 field must be specified" ); @@ -287,7 +264,7 @@ private UniqueValueSet loadCQL( final ApplicationScope appScope, final com.datas final Statement statement = QueryBuilder.select().all().from(TABLE_UNIQUE_VALUES) .where(inKey) - .setConsistencyLevel(com.datastax.driver.core.ConsistencyLevel.LOCAL_QUORUM); + .setConsistencyLevel(consistencyLevel); final ResultSet resultSet = session.execute(statement); @@ -366,13 +343,6 @@ public Iterator getAllUniqueFields( final ApplicationScope collecti @Override public abstract Collection getTables(); - - /** - * Get the column family for the unique fields - */ - protected abstract MultiTenantColumnFamily, EntityVersion> getUniqueValuesCF(); - - /** * Get the CQL table definition for the unique values log table */ @@ -395,26 +365,11 @@ public Iterator getAllUniqueFields( final ApplicationScope collecti - - - /** - * Get the column family for the unique field CF - */ - protected abstract MultiTenantColumnFamily, UniqueFieldEntry> getEntityUniqueLogCF(); - /** * Get the CQL table definition for the unique values log table */ protected abstract TableDefinition getEntityUniqueLogTable(); - /** - * Generate a key that is compatible with the column family - * - * @param applicationId The applicationId - * @param uniqueValueId The uniqueValue - */ - protected abstract EntityKey createEntityUniqueLogKey(final Id applicationId, final Id uniqueValueId ); - public class AllUniqueFieldsIterator implements Iterable, Iterator { diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyProxyImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyProxyImpl.java index dc5b48f11d..4b5653f593 100644 --- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyProxyImpl.java +++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyProxyImpl.java @@ -25,6 +25,7 @@ import java.util.Iterator; import com.datastax.driver.core.BatchStatement; +import com.datastax.driver.core.ConsistencyLevel; import org.apache.usergrid.persistence.collection.serialization.UniqueValue; import org.apache.usergrid.persistence.collection.serialization.UniqueValueSerializationStrategy; import org.apache.usergrid.persistence.collection.serialization.UniqueValueSet; @@ -40,28 +41,21 @@ import com.google.inject.Inject; import com.google.inject.Singleton; -import com.netflix.astyanax.Keyspace; -import com.netflix.astyanax.MutationBatch; -import com.netflix.astyanax.connectionpool.exceptions.ConnectionException; -import com.netflix.astyanax.model.ConsistencyLevel; @Singleton public class UniqueValueSerializationStrategyProxyImpl implements UniqueValueSerializationStrategy { - protected final Keyspace keyspace; private final VersionedMigrationSet versions; private final MigrationInfoCache migrationInfoCache; @Inject - public UniqueValueSerializationStrategyProxyImpl( final Keyspace keyspace, - final VersionedMigrationSet + public UniqueValueSerializationStrategyProxyImpl( final VersionedMigrationSet allVersions, final MigrationInfoCache migrationInfoCache ) { - this.keyspace = keyspace; this.migrationInfoCache = migrationInfoCache; this.versions = allVersions; } @@ -86,7 +80,7 @@ public BatchStatement writeCQL(final ApplicationScope applicationScope, final Un @Override public UniqueValueSet load( final ApplicationScope applicationScope, final String type, - final Collection fields ) throws ConnectionException { + final Collection fields ) { final MigrationRelationship migration = getMigrationRelationShip(); @@ -100,7 +94,7 @@ public UniqueValueSet load( final ApplicationScope applicationScope, final Strin @Override public UniqueValueSet load( final ApplicationScope applicationScope, final ConsistencyLevel consistencyLevel, - final String type, final Collection fields ) throws ConnectionException { + final String type, final Collection fields ) { final MigrationRelationship migration = getMigrationRelationShip(); diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyV1Impl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyV1Impl.java index 73df64a220..6421869a93 100644 --- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyV1Impl.java +++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyV1Impl.java @@ -26,17 +26,12 @@ import com.datastax.driver.core.DataType; import com.datastax.driver.core.ProtocolVersion; import com.datastax.driver.core.Session; -import org.apache.cassandra.db.marshal.BytesType; import org.apache.usergrid.persistence.collection.serialization.SerializationFig; import org.apache.usergrid.persistence.collection.serialization.impl.util.LegacyScopeUtils; import org.apache.usergrid.persistence.core.CassandraConfig; import org.apache.usergrid.persistence.core.CassandraFig; -import org.apache.usergrid.persistence.core.astyanax.ColumnTypes; -import org.apache.usergrid.persistence.core.astyanax.IdRowCompositeSerializer; -import org.apache.usergrid.persistence.core.astyanax.MultiTenantColumnFamily; import org.apache.usergrid.persistence.core.astyanax.MultiTenantColumnFamilyDefinition; -import org.apache.usergrid.persistence.core.astyanax.ScopedRowKey; import org.apache.usergrid.persistence.core.datastax.CQLUtils; import org.apache.usergrid.persistence.core.datastax.TableDefinition; import org.apache.usergrid.persistence.model.entity.Id; @@ -44,7 +39,6 @@ import com.google.inject.Inject; import com.google.inject.Singleton; -import com.netflix.astyanax.Keyspace; /** @@ -88,80 +82,40 @@ public class UniqueValueSerializationStrategyV1Impl extends UniqueValueSerializ UNIQUE_VALUES_LOG_COLUMNS, TableDefinition.CacheOption.KEYS, UNIQUE_VALUES_LOG_CLUSTERING_ORDER); - private static final CollectionScopedRowKeySerializer ROW_KEY_SER = - new CollectionScopedRowKeySerializer<>( UniqueFieldRowKeySerializer.get() ); - - private static final EntityVersionSerializer ENTITY_VERSION_SER = new EntityVersionSerializer(); - - private static final MultiTenantColumnFamily>, EntityVersion> - CF_UNIQUE_VALUES = new MultiTenantColumnFamily<>( "Unique_Values", ROW_KEY_SER, ENTITY_VERSION_SER ); - - - private static final IdRowCompositeSerializer ID_SER = IdRowCompositeSerializer.get(); - - - private static final CollectionScopedRowKeySerializer ENTITY_ROW_KEY_SER = - new CollectionScopedRowKeySerializer<>( ID_SER ); - - - private static final MultiTenantColumnFamily>, UniqueFieldEntry> - CF_ENTITY_UNIQUE_VALUE_LOG = - new MultiTenantColumnFamily<>( "Entity_Unique_Values", ENTITY_ROW_KEY_SER, UniqueFieldEntrySerializer.get() ); - /** * Construct serialization strategy for keyspace. * - * @param keyspace Keyspace in which to store Unique Values. * @param cassandraFig The cassandra configuration * @param serializationFig The serialization configuration */ @Inject - public UniqueValueSerializationStrategyV1Impl(final Keyspace keyspace, final CassandraFig cassandraFig, - final SerializationFig serializationFig, - final Session session, - final CassandraConfig cassandraConfig) { - super( keyspace, cassandraFig, serializationFig, session, cassandraConfig ); + public UniqueValueSerializationStrategyV1Impl( final CassandraFig cassandraFig, + final SerializationFig serializationFig, + final Session session, + final CassandraConfig cassandraConfig) { + super( cassandraFig, serializationFig, session, cassandraConfig ); } @Override public Collection getColumnFamilies() { - final MultiTenantColumnFamilyDefinition uniqueLookupCF = - new MultiTenantColumnFamilyDefinition( CF_UNIQUE_VALUES, BytesType.class.getSimpleName(), - ColumnTypes.DYNAMIC_COMPOSITE_TYPE, BytesType.class.getSimpleName(), - MultiTenantColumnFamilyDefinition.CacheOption.KEYS ); - - final MultiTenantColumnFamilyDefinition uniqueLogCF = - new MultiTenantColumnFamilyDefinition( CF_ENTITY_UNIQUE_VALUE_LOG, BytesType.class.getSimpleName(), - ColumnTypes.DYNAMIC_COMPOSITE_TYPE, BytesType.class.getSimpleName(), - MultiTenantColumnFamilyDefinition.CacheOption.KEYS ); - - //return Collections.emptyList(); + return Collections.emptyList(); - return Arrays.asList(uniqueLookupCF, uniqueLogCF); } @Override public Collection getTables() { final TableDefinition uniqueValues = getUniqueValuesTable(); - final TableDefinition uniqueValuesLog = getEntityUniqueLogTable(); - - //return Arrays.asList( uniqueValues, uniqueValuesLog ); - - return Collections.emptyList(); + return Arrays.asList( uniqueValues, uniqueValuesLog ); } - @Override - protected MultiTenantColumnFamily>, EntityVersion> getUniqueValuesCF() { - return CF_UNIQUE_VALUES; - } @Override protected TableDefinition getUniqueValuesTable(){ @@ -170,13 +124,6 @@ protected TableDefinition getUniqueValuesTable(){ } - @Override - protected MultiTenantColumnFamily>, UniqueFieldEntry> - getEntityUniqueLogCF() { - return CF_ENTITY_UNIQUE_VALUE_LOG; - } - - @Override protected TableDefinition getEntityUniqueLogTable(){ @@ -463,24 +410,6 @@ protected List deserializeUniqueValueLogColumn(ByteBuffer bb){ } - - @Override - protected CollectionPrefixedKey createEntityUniqueLogKey( final Id applicationId, - final Id uniqueValueId ) { - - - final String collectionName = LegacyScopeUtils.getCollectionScopeNameFromEntityType( uniqueValueId.getType() ); - - - final CollectionPrefixedKey collectionPrefixedEntityKey = - new CollectionPrefixedKey<>( collectionName, applicationId, uniqueValueId ); - - - - return collectionPrefixedEntityKey; - } - - @Override public int getImplementationVersion() { return CollectionDataVersions.INITIAL.getVersion(); diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyV2Impl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyV2Impl.java index 8e13f865c7..6ea5c1e112 100644 --- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyV2Impl.java +++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyV2Impl.java @@ -26,24 +26,17 @@ import com.datastax.driver.core.DataType; import com.datastax.driver.core.ProtocolVersion; import com.datastax.driver.core.Session; -import org.apache.cassandra.db.marshal.BytesType; import org.apache.usergrid.persistence.collection.serialization.SerializationFig; import org.apache.usergrid.persistence.core.CassandraConfig; import org.apache.usergrid.persistence.core.CassandraFig; -import org.apache.usergrid.persistence.core.astyanax.ColumnTypes; -import org.apache.usergrid.persistence.core.astyanax.IdRowCompositeSerializer; -import org.apache.usergrid.persistence.core.astyanax.MultiTenantColumnFamily; import org.apache.usergrid.persistence.core.astyanax.MultiTenantColumnFamilyDefinition; -import org.apache.usergrid.persistence.core.astyanax.ScopedRowKey; -import org.apache.usergrid.persistence.core.astyanax.ScopedRowKeySerializer; import org.apache.usergrid.persistence.core.datastax.CQLUtils; import org.apache.usergrid.persistence.core.datastax.TableDefinition; import org.apache.usergrid.persistence.model.entity.Id; import com.google.inject.Inject; import com.google.inject.Singleton; -import com.netflix.astyanax.Keyspace; /** @@ -84,78 +77,37 @@ public class UniqueValueSerializationStrategyV2Impl extends UniqueValueSerializ UNIQUE_VALUES_LOG_COLUMNS, TableDefinition.CacheOption.KEYS, UNIQUE_VALUES_LOG_CLUSTERING_ORDER); - private static final ScopedRowKeySerializer ROW_KEY_SER = new ScopedRowKeySerializer<>( UniqueTypeFieldRowKeySerializer.get() ); - - - private static final EntityVersionSerializer ENTITY_VERSION_SER = new EntityVersionSerializer(); - - private static final MultiTenantColumnFamily, EntityVersion> - CF_UNIQUE_VALUES = new MultiTenantColumnFamily<>( "Unique_Values_V2", ROW_KEY_SER, ENTITY_VERSION_SER ); - - - private static final IdRowCompositeSerializer ID_SER = IdRowCompositeSerializer.get(); - - - private static final ScopedRowKeySerializer ENTITY_ROW_KEY_SER = - new ScopedRowKeySerializer<>( ID_SER ); - - - private static final MultiTenantColumnFamily, UniqueFieldEntry> - CF_ENTITY_UNIQUE_VALUE_LOG = - new MultiTenantColumnFamily<>( "Entity_Unique_Values_V2", ENTITY_ROW_KEY_SER, UniqueFieldEntrySerializer.get() ); - - /** * Construct serialization strategy for keyspace. * - * @param keyspace Keyspace in which to store Unique Values. * @param cassandraFig The cassandra configuration * @param serializationFig The serialization configuration + * */ @Inject - public UniqueValueSerializationStrategyV2Impl( final Keyspace keyspace, final CassandraFig cassandraFig, + public UniqueValueSerializationStrategyV2Impl( final CassandraFig cassandraFig, final SerializationFig serializationFig, final Session session, final CassandraConfig cassandraConfig) { - super( keyspace, cassandraFig, serializationFig, session, cassandraConfig ); + super( cassandraFig, serializationFig, session, cassandraConfig ); } @Override public Collection getColumnFamilies() { - final MultiTenantColumnFamilyDefinition uniqueLookupCF = - new MultiTenantColumnFamilyDefinition( CF_UNIQUE_VALUES, BytesType.class.getSimpleName(), - ColumnTypes.DYNAMIC_COMPOSITE_TYPE, BytesType.class.getSimpleName(), - MultiTenantColumnFamilyDefinition.CacheOption.KEYS ); - - final MultiTenantColumnFamilyDefinition uniqueLogCF = - new MultiTenantColumnFamilyDefinition( CF_ENTITY_UNIQUE_VALUE_LOG, BytesType.class.getSimpleName(), - ColumnTypes.DYNAMIC_COMPOSITE_TYPE, BytesType.class.getSimpleName(), - MultiTenantColumnFamilyDefinition.CacheOption.KEYS ); - return Collections.emptyList(); - //return Arrays.asList(uniqueLookupCF, uniqueLogCF); } @Override public Collection getTables() { final TableDefinition uniqueValues = getUniqueValuesTable(); - final TableDefinition uniqueValuesLog = getEntityUniqueLogTable(); - return Arrays.asList( uniqueValues, uniqueValuesLog ); - //return Collections.emptyList(); - - } - - @Override - protected MultiTenantColumnFamily, EntityVersion> getUniqueValuesCF() { - return CF_UNIQUE_VALUES; } @@ -165,13 +117,6 @@ protected TableDefinition getUniqueValuesTable(){ } - @Override - protected MultiTenantColumnFamily, UniqueFieldEntry> - getEntityUniqueLogCF() { - return CF_ENTITY_UNIQUE_VALUE_LOG; - } - - @Override protected TableDefinition getEntityUniqueLogTable(){ return uniqueValuesLog; @@ -453,12 +398,6 @@ protected List deserializeUniqueValueLogColumn(ByteBuffer bb){ } - @Override - protected Id createEntityUniqueLogKey( final Id applicationId, final Id uniqueValueId ) { - return uniqueValueId; - } - - @Override public int getImplementationVersion() { return CollectionDataVersions.LOG_REMOVAL.getVersion(); From 324f94d416617492775ce8608ccd7c822d193be7 Mon Sep 17 00:00:00 2001 From: Michael Russo Date: Sun, 8 May 2016 23:49:30 +0800 Subject: [PATCH 29/30] Fix issue with V1 impl of UniqueValue serialization from creating unecessarily large ByteBuffers. --- .../impl/UniqueValueSerializationStrategyV1Impl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyV1Impl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyV1Impl.java index 6421869a93..d305044662 100644 --- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyV1Impl.java +++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyV1Impl.java @@ -202,7 +202,7 @@ protected ByteBuffer serializeUniqueValueLogColumn(UniqueFieldEntry fieldEntry){ fieldEntry.getField().getTypeName().name().length(); // we always need to add length for the 2 byte comparator short, 2 byte length short and 1 byte equality - size += keys.size()*65; + size += keys.size()*5; // uuid type comparator is longest, ensure we allocate buffer using the max size to avoid overflow size += keys.size()*comparator.length(); From 32782a39bf82c1bf606cf587ac4d57ec85b2b15e Mon Sep 17 00:00:00 2001 From: Michael Russo Date: Tue, 26 Jul 2016 11:10:29 -0700 Subject: [PATCH 30/30] Update comment --- .../impl/UniqueValueSerializationStrategyV2Impl.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyV2Impl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyV2Impl.java index 92c0a5bcb9..a5fceeba9a 100644 --- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyV2Impl.java +++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyV2Impl.java @@ -294,7 +294,9 @@ protected ByteBuffer serializeUniqueValueColumn(EntityVersion entityVersion){ for (Object key : keys) { - // custom comparator mappings in CQLUtils.COMPOSITE_TYPE ( more leftover from Asytanax ) + // custom comparator alias to comparator mappings in CQLUtils.COMPOSITE_TYPE ( more leftover from Asytanax ) + // the custom mapping is used for schema creation, but datastax driver does not have the alias concept and + // we must work with the actual types if(key instanceof UUID){ comparator = "UUIDType"; }else{