diff --git a/core/src/integration-test/java/com/scalar/db/storage/cassandra/CassandraEnv.java b/core/src/integration-test/java/com/scalar/db/storage/cassandra/CassandraEnv.java index 2b633986e6..2079381fb0 100644 --- a/core/src/integration-test/java/com/scalar/db/storage/cassandra/CassandraEnv.java +++ b/core/src/integration-test/java/com/scalar/db/storage/cassandra/CassandraEnv.java @@ -24,6 +24,9 @@ public static Properties getProperties(@SuppressWarnings("unused") String testNa props.setProperty(DatabaseConfig.CONTACT_POINTS, contactPoints); props.setProperty(DatabaseConfig.USERNAME, username); props.setProperty(DatabaseConfig.PASSWORD, password); + props.setProperty(DatabaseConfig.CROSS_PARTITION_SCAN, "true"); + props.setProperty(DatabaseConfig.CROSS_PARTITION_SCAN_FILTERING, "false"); + props.setProperty(DatabaseConfig.CROSS_PARTITION_SCAN_ORDERING, "false"); return props; } } diff --git a/core/src/integration-test/java/com/scalar/db/storage/cosmos/CosmosEnv.java b/core/src/integration-test/java/com/scalar/db/storage/cosmos/CosmosEnv.java index c1c0c755b0..ba0c93c653 100644 --- a/core/src/integration-test/java/com/scalar/db/storage/cosmos/CosmosEnv.java +++ b/core/src/integration-test/java/com/scalar/db/storage/cosmos/CosmosEnv.java @@ -28,6 +28,9 @@ public static Properties getProperties(String testName) { props.setProperty(DatabaseConfig.CONTACT_POINTS, contactPoint); props.setProperty(DatabaseConfig.PASSWORD, password); props.setProperty(DatabaseConfig.STORAGE, "cosmos"); + props.setProperty(DatabaseConfig.CROSS_PARTITION_SCAN, "true"); + props.setProperty(DatabaseConfig.CROSS_PARTITION_SCAN_FILTERING, "false"); + props.setProperty(DatabaseConfig.CROSS_PARTITION_SCAN_ORDERING, "false"); if (databasePrefix.isPresent()) { // Add the prefix and testName as a metadata database suffix diff --git a/core/src/integration-test/java/com/scalar/db/storage/dynamo/DynamoEnv.java b/core/src/integration-test/java/com/scalar/db/storage/dynamo/DynamoEnv.java index 25ea50e659..8159cb0ea8 100644 --- a/core/src/integration-test/java/com/scalar/db/storage/dynamo/DynamoEnv.java +++ b/core/src/integration-test/java/com/scalar/db/storage/dynamo/DynamoEnv.java @@ -39,6 +39,9 @@ public static Properties getProperties(String testName) { props.setProperty(DatabaseConfig.USERNAME, accessKeyId); props.setProperty(DatabaseConfig.PASSWORD, secretAccessKey); props.setProperty(DatabaseConfig.STORAGE, "dynamo"); + props.setProperty(DatabaseConfig.CROSS_PARTITION_SCAN, "true"); + props.setProperty(DatabaseConfig.CROSS_PARTITION_SCAN_FILTERING, "false"); + props.setProperty(DatabaseConfig.CROSS_PARTITION_SCAN_ORDERING, "false"); // Add testName as a metadata namespace suffix props.setProperty( diff --git a/core/src/integration-test/java/com/scalar/db/storage/jdbc/JdbcEnv.java b/core/src/integration-test/java/com/scalar/db/storage/jdbc/JdbcEnv.java index c168b31ac9..6a19630dd4 100644 --- a/core/src/integration-test/java/com/scalar/db/storage/jdbc/JdbcEnv.java +++ b/core/src/integration-test/java/com/scalar/db/storage/jdbc/JdbcEnv.java @@ -24,6 +24,9 @@ public static Properties getProperties(String testName) { props.setProperty(DatabaseConfig.USERNAME, username); props.setProperty(DatabaseConfig.PASSWORD, password); props.setProperty(DatabaseConfig.STORAGE, "jdbc"); + props.setProperty(DatabaseConfig.CROSS_PARTITION_SCAN, "true"); + props.setProperty(DatabaseConfig.CROSS_PARTITION_SCAN_FILTERING, "true"); + props.setProperty(DatabaseConfig.CROSS_PARTITION_SCAN_ORDERING, "true"); // Add testName as a metadata schema suffix props.setProperty(JdbcConfig.TABLE_METADATA_SCHEMA, JdbcAdmin.METADATA_SCHEMA + "_" + testName); diff --git a/core/src/integration-test/java/com/scalar/db/storage/multistorage/MultiStorageEnv.java b/core/src/integration-test/java/com/scalar/db/storage/multistorage/MultiStorageEnv.java index 2ec22345b6..a17c71301e 100644 --- a/core/src/integration-test/java/com/scalar/db/storage/multistorage/MultiStorageEnv.java +++ b/core/src/integration-test/java/com/scalar/db/storage/multistorage/MultiStorageEnv.java @@ -41,6 +41,9 @@ public static Properties getPropertiesForCassandra(@SuppressWarnings("unused") S properties.setProperty(DatabaseConfig.USERNAME, username); properties.setProperty(DatabaseConfig.PASSWORD, password); properties.setProperty(DatabaseConfig.STORAGE, "cassandra"); + properties.setProperty(DatabaseConfig.CROSS_PARTITION_SCAN, "true"); + properties.setProperty(DatabaseConfig.CROSS_PARTITION_SCAN_FILTERING, "false"); + properties.setProperty(DatabaseConfig.CROSS_PARTITION_SCAN_ORDERING, "false"); return properties; } @@ -54,6 +57,9 @@ public static Properties getPropertiesForJdbc(String testName) { properties.setProperty(DatabaseConfig.USERNAME, username); properties.setProperty(DatabaseConfig.PASSWORD, password); properties.setProperty(DatabaseConfig.STORAGE, "jdbc"); + properties.setProperty(DatabaseConfig.CROSS_PARTITION_SCAN, "true"); + properties.setProperty(DatabaseConfig.CROSS_PARTITION_SCAN_FILTERING, "true"); + properties.setProperty(DatabaseConfig.CROSS_PARTITION_SCAN_ORDERING, "true"); // Add testName as a metadata schema suffix properties.setProperty( diff --git a/core/src/main/java/com/scalar/db/common/checker/OperationChecker.java b/core/src/main/java/com/scalar/db/common/checker/OperationChecker.java index d24aaeb3fd..b264e5d190 100644 --- a/core/src/main/java/com/scalar/db/common/checker/OperationChecker.java +++ b/core/src/main/java/com/scalar/db/common/checker/OperationChecker.java @@ -13,6 +13,7 @@ import com.scalar.db.api.Selection; import com.scalar.db.api.TableMetadata; import com.scalar.db.common.TableMetadataManager; +import com.scalar.db.config.DatabaseConfig; import com.scalar.db.exception.storage.ExecutionException; import com.scalar.db.io.Column; import com.scalar.db.io.Key; @@ -27,9 +28,11 @@ @ThreadSafe public class OperationChecker { + private final DatabaseConfig config; private final TableMetadataManager tableMetadataManager; - public OperationChecker(TableMetadataManager tableMetadataManager) { + public OperationChecker(DatabaseConfig config, TableMetadataManager tableMetadataManager) { + this.config = config; this.tableMetadataManager = tableMetadataManager; } @@ -122,6 +125,11 @@ public void check(Scan scan) throws ExecutionException { } private void check(ScanAll scanAll) throws ExecutionException { + if (!config.isCrossPartitionScanEnabled()) { + throw new IllegalArgumentException( + "Cross-partition scan is not enabled. Operation: " + scanAll); + } + TableMetadata metadata = getTableMetadata(scanAll); checkProjections(scanAll, metadata); @@ -130,8 +138,16 @@ private void check(ScanAll scanAll) throws ExecutionException { throw new IllegalArgumentException("The limit cannot be negative. Operation: " + scanAll); } + if (!config.isCrossPartitionScanOrderingEnabled() && !scanAll.getOrderings().isEmpty()) { + throw new IllegalArgumentException( + "Cross-partition scan ordering is not enabled. Operation: " + scanAll); + } checkOrderings(scanAll, metadata); + if (!config.isCrossPartitionScanFilteringEnabled() && !scanAll.getConjunctions().isEmpty()) { + throw new IllegalArgumentException( + "Cross-partition scan filtering is not enabled. Operation: " + scanAll); + } checkConjunctions(scanAll, metadata); } diff --git a/core/src/main/java/com/scalar/db/config/DatabaseConfig.java b/core/src/main/java/com/scalar/db/config/DatabaseConfig.java index c5cafa7439..e0bc12c92c 100644 --- a/core/src/main/java/com/scalar/db/config/DatabaseConfig.java +++ b/core/src/main/java/com/scalar/db/config/DatabaseConfig.java @@ -1,6 +1,7 @@ package com.scalar.db.config; import static com.google.common.base.Preconditions.checkArgument; +import static com.scalar.db.config.ConfigUtils.getBoolean; import static com.scalar.db.config.ConfigUtils.getInt; import static com.scalar.db.config.ConfigUtils.getLong; import static com.scalar.db.config.ConfigUtils.getString; @@ -32,6 +33,9 @@ public class DatabaseConfig { private long metadataCacheExpirationTimeSecs; private long activeTransactionManagementExpirationTimeMillis; @Nullable private String defaultNamespaceName; + private boolean crossPartitionScanEnabled; + private boolean crossPartitionScanFilteringEnabled; + private boolean crossPartitionScanOrderingEnabled; public static final String PREFIX = "scalar.db."; public static final String CONTACT_POINTS = PREFIX + "contact_points"; @@ -45,6 +49,10 @@ public class DatabaseConfig { public static final String ACTIVE_TRANSACTION_MANAGEMENT_EXPIRATION_TIME_MILLIS = PREFIX + "active_transaction_management.expiration_time_millis"; public static final String DEFAULT_NAMESPACE_NAME = PREFIX + "default_namespace_name"; + public static final String SCAN_PREFIX = PREFIX + "cross_partition_scan."; + public static final String CROSS_PARTITION_SCAN = SCAN_PREFIX + "enabled"; + public static final String CROSS_PARTITION_SCAN_FILTERING = SCAN_PREFIX + "filtering.enabled"; + public static final String CROSS_PARTITION_SCAN_ORDERING = SCAN_PREFIX + "ordering.enabled"; public DatabaseConfig(File propertiesFile) throws IOException { try (FileInputStream stream = new FileInputStream(propertiesFile)) { @@ -94,6 +102,18 @@ protected void load() { activeTransactionManagementExpirationTimeMillis = getLong(getProperties(), ACTIVE_TRANSACTION_MANAGEMENT_EXPIRATION_TIME_MILLIS, -1); defaultNamespaceName = getString(getProperties(), DEFAULT_NAMESPACE_NAME, null); + crossPartitionScanEnabled = getBoolean(getProperties(), CROSS_PARTITION_SCAN, true); + crossPartitionScanFilteringEnabled = + getBoolean(getProperties(), CROSS_PARTITION_SCAN_FILTERING, false); + crossPartitionScanOrderingEnabled = + getBoolean(getProperties(), CROSS_PARTITION_SCAN_ORDERING, false); + + if (!crossPartitionScanEnabled + && (crossPartitionScanFilteringEnabled || crossPartitionScanOrderingEnabled)) { + throw new IllegalArgumentException( + CROSS_PARTITION_SCAN + + " must be enabled to use cross-partition scan with filtering or ordering"); + } } public List getContactPoints() { @@ -131,4 +151,16 @@ public long getActiveTransactionManagementExpirationTimeMillis() { public Optional getDefaultNamespaceName() { return Optional.ofNullable(defaultNamespaceName); } + + public boolean isCrossPartitionScanEnabled() { + return crossPartitionScanEnabled; + } + + public boolean isCrossPartitionScanFilteringEnabled() { + return crossPartitionScanFilteringEnabled; + } + + public boolean isCrossPartitionScanOrderingEnabled() { + return crossPartitionScanOrderingEnabled; + } } diff --git a/core/src/main/java/com/scalar/db/storage/cassandra/Cassandra.java b/core/src/main/java/com/scalar/db/storage/cassandra/Cassandra.java index 7271ced9f5..62e8d5adfc 100644 --- a/core/src/main/java/com/scalar/db/storage/cassandra/Cassandra.java +++ b/core/src/main/java/com/scalar/db/storage/cassandra/Cassandra.java @@ -20,7 +20,6 @@ import com.scalar.db.common.checker.OperationChecker; import com.scalar.db.config.DatabaseConfig; import com.scalar.db.exception.storage.ExecutionException; -import com.scalar.db.util.ScalarDbUtils; import java.util.List; import java.util.Optional; import javax.annotation.Nonnull; @@ -45,6 +44,13 @@ public class Cassandra extends AbstractDistributedStorage { @Inject public Cassandra(DatabaseConfig config) { super(config); + + if (config.isCrossPartitionScanFilteringEnabled() + || config.isCrossPartitionScanOrderingEnabled()) { + throw new IllegalArgumentException( + "Cross-partition scan with filtering or ordering is not supported in Cassandra"); + } + clusterManager = new ClusterManager(config); Session session = clusterManager.getSession(); @@ -62,7 +68,7 @@ public Cassandra(DatabaseConfig config) { metadataManager = new TableMetadataManager( new CassandraAdmin(clusterManager), config.getMetadataCacheExpirationTimeSecs()); - operationChecker = new OperationChecker(metadataManager); + operationChecker = new OperationChecker(config, metadataManager); } @VisibleForTesting @@ -81,6 +87,10 @@ public Cassandra(DatabaseConfig config) { this.operationChecker = operationChecker; } + // For the SpotBugs warning CT_CONSTRUCTOR_THROW + @Override + protected final void finalize() {} + @Override @Nonnull public Optional get(Get get) throws ExecutionException { @@ -107,11 +117,6 @@ public Scanner scan(Scan scan) throws ExecutionException { scan = copyAndSetTargetToIfNot(scan); operationChecker.check(scan); - if (ScalarDbUtils.isRelational(scan)) { - throw new UnsupportedOperationException( - "Scanning all records with orderings or conditions is not supported in Cassandra"); - } - ResultSet results = handlers.select().handle(scan); return new ScannerImpl( diff --git a/core/src/main/java/com/scalar/db/storage/cassandra/CassandraConfig.java b/core/src/main/java/com/scalar/db/storage/cassandra/CassandraConfig.java new file mode 100644 index 0000000000..f53148b036 --- /dev/null +++ b/core/src/main/java/com/scalar/db/storage/cassandra/CassandraConfig.java @@ -0,0 +1,6 @@ +package com.scalar.db.storage.cassandra; + +public class CassandraConfig { + // just hold the storage name for consistency with other storage + public static final String STORAGE_NAME = "cassandra"; +} diff --git a/core/src/main/java/com/scalar/db/storage/cassandra/CassandraProvider.java b/core/src/main/java/com/scalar/db/storage/cassandra/CassandraProvider.java index 53b951fa3d..313be499c1 100644 --- a/core/src/main/java/com/scalar/db/storage/cassandra/CassandraProvider.java +++ b/core/src/main/java/com/scalar/db/storage/cassandra/CassandraProvider.java @@ -9,7 +9,7 @@ public class CassandraProvider implements DistributedStorageProvider { @Override public String getName() { - return "cassandra"; + return CassandraConfig.STORAGE_NAME; } @Override diff --git a/core/src/main/java/com/scalar/db/storage/cosmos/Cosmos.java b/core/src/main/java/com/scalar/db/storage/cosmos/Cosmos.java index 18f983d60c..bfe1b80f61 100644 --- a/core/src/main/java/com/scalar/db/storage/cosmos/Cosmos.java +++ b/core/src/main/java/com/scalar/db/storage/cosmos/Cosmos.java @@ -20,7 +20,6 @@ import com.scalar.db.common.checker.OperationChecker; import com.scalar.db.config.DatabaseConfig; import com.scalar.db.exception.storage.ExecutionException; -import com.scalar.db.util.ScalarDbUtils; import java.util.List; import java.util.Optional; import javax.annotation.Nonnull; @@ -47,6 +46,13 @@ public class Cosmos extends AbstractDistributedStorage { @Inject public Cosmos(DatabaseConfig databaseConfig) { super(databaseConfig); + + if (databaseConfig.isCrossPartitionScanFilteringEnabled() + || databaseConfig.isCrossPartitionScanOrderingEnabled()) { + throw new IllegalArgumentException( + "Cross-partition scan with filtering or ordering is not supported in Cosmos DB"); + } + CosmosConfig config = new CosmosConfig(databaseConfig); client = @@ -60,7 +66,7 @@ public Cosmos(DatabaseConfig databaseConfig) { TableMetadataManager metadataManager = new TableMetadataManager( new CosmosAdmin(client, config), databaseConfig.getMetadataCacheExpirationTimeSecs()); - operationChecker = new CosmosOperationChecker(metadataManager); + operationChecker = new CosmosOperationChecker(databaseConfig, metadataManager); selectStatementHandler = new SelectStatementHandler(client, metadataManager); putStatementHandler = new PutStatementHandler(client, metadataManager); @@ -88,6 +94,10 @@ public Cosmos(DatabaseConfig databaseConfig) { this.operationChecker = operationChecker; } + // For the SpotBugs warning CT_CONSTRUCTOR_THROW + @Override + protected final void finalize() {} + @Override @Nonnull public Optional get(Get get) throws ExecutionException { @@ -108,11 +118,6 @@ public Scanner scan(Scan scan) throws ExecutionException { scan = copyAndSetTargetToIfNot(scan); operationChecker.check(scan); - if (ScalarDbUtils.isRelational(scan)) { - throw new UnsupportedOperationException( - "Scanning all records with orderings or conditions is not supported in Cosmos DB"); - } - return selectStatementHandler.handle(scan); } diff --git a/core/src/main/java/com/scalar/db/storage/cosmos/CosmosConfig.java b/core/src/main/java/com/scalar/db/storage/cosmos/CosmosConfig.java index 93b4299f92..d0b30df42d 100644 --- a/core/src/main/java/com/scalar/db/storage/cosmos/CosmosConfig.java +++ b/core/src/main/java/com/scalar/db/storage/cosmos/CosmosConfig.java @@ -9,8 +9,8 @@ @Immutable public class CosmosConfig { - - public static final String PREFIX = DatabaseConfig.PREFIX + "cosmos."; + public static final String STORAGE_NAME = "cosmos"; + public static final String PREFIX = DatabaseConfig.PREFIX + STORAGE_NAME + "."; public static final String TABLE_METADATA_DATABASE = PREFIX + "table_metadata.database"; private final String endpoint; @@ -19,8 +19,9 @@ public class CosmosConfig { public CosmosConfig(DatabaseConfig databaseConfig) { String storage = databaseConfig.getStorage(); - if (!"cosmos".equals(storage)) { - throw new IllegalArgumentException(DatabaseConfig.STORAGE + " should be 'cosmos'"); + if (!storage.equals(STORAGE_NAME)) { + throw new IllegalArgumentException( + DatabaseConfig.STORAGE + " should be '" + STORAGE_NAME + "'"); } if (databaseConfig.getContactPoints().isEmpty()) { diff --git a/core/src/main/java/com/scalar/db/storage/cosmos/CosmosOperationChecker.java b/core/src/main/java/com/scalar/db/storage/cosmos/CosmosOperationChecker.java index d6d7bd464a..1b0b32eff5 100644 --- a/core/src/main/java/com/scalar/db/storage/cosmos/CosmosOperationChecker.java +++ b/core/src/main/java/com/scalar/db/storage/cosmos/CosmosOperationChecker.java @@ -7,12 +7,14 @@ import com.scalar.db.api.TableMetadata; import com.scalar.db.common.TableMetadataManager; import com.scalar.db.common.checker.OperationChecker; +import com.scalar.db.config.DatabaseConfig; import com.scalar.db.exception.storage.ExecutionException; import com.scalar.db.io.DataType; public class CosmosOperationChecker extends OperationChecker { - public CosmosOperationChecker(TableMetadataManager metadataManager) { - super(metadataManager); + public CosmosOperationChecker( + DatabaseConfig databaseConfig, TableMetadataManager metadataManager) { + super(databaseConfig, metadataManager); } @Override diff --git a/core/src/main/java/com/scalar/db/storage/cosmos/CosmosProvider.java b/core/src/main/java/com/scalar/db/storage/cosmos/CosmosProvider.java index f01050359a..a926dded40 100644 --- a/core/src/main/java/com/scalar/db/storage/cosmos/CosmosProvider.java +++ b/core/src/main/java/com/scalar/db/storage/cosmos/CosmosProvider.java @@ -9,7 +9,7 @@ public class CosmosProvider implements DistributedStorageProvider { @Override public String getName() { - return "cosmos"; + return CosmosConfig.STORAGE_NAME; } @Override diff --git a/core/src/main/java/com/scalar/db/storage/dynamo/Dynamo.java b/core/src/main/java/com/scalar/db/storage/dynamo/Dynamo.java index 792f406f84..3578bca0c2 100644 --- a/core/src/main/java/com/scalar/db/storage/dynamo/Dynamo.java +++ b/core/src/main/java/com/scalar/db/storage/dynamo/Dynamo.java @@ -17,7 +17,6 @@ import com.scalar.db.common.checker.OperationChecker; import com.scalar.db.config.DatabaseConfig; import com.scalar.db.exception.storage.ExecutionException; -import com.scalar.db.util.ScalarDbUtils; import java.io.IOException; import java.net.URI; import java.util.List; @@ -51,6 +50,13 @@ public class Dynamo extends AbstractDistributedStorage { @Inject public Dynamo(DatabaseConfig databaseConfig) { super(databaseConfig); + + if (databaseConfig.isCrossPartitionScanFilteringEnabled() + || databaseConfig.isCrossPartitionScanOrderingEnabled()) { + throw new IllegalArgumentException( + "Cross-partition scan with filtering or ordering is not supported in DynamoDB"); + } + DynamoConfig config = new DynamoConfig(databaseConfig); DynamoDbClientBuilder builder = DynamoDbClient.builder(); @@ -67,7 +73,7 @@ public Dynamo(DatabaseConfig databaseConfig) { TableMetadataManager metadataManager = new TableMetadataManager( new DynamoAdmin(client, config), databaseConfig.getMetadataCacheExpirationTimeSecs()); - operationChecker = new DynamoOperationChecker(metadataManager); + operationChecker = new DynamoOperationChecker(databaseConfig, metadataManager); selectStatementHandler = new SelectStatementHandler(client, metadataManager, config.getNamespacePrefix()); @@ -98,6 +104,10 @@ public Dynamo(DatabaseConfig databaseConfig) { this.operationChecker = operationChecker; } + // For the SpotBugs warning CT_CONSTRUCTOR_THROW + @Override + protected final void finalize() {} + @Override @Nonnull public Optional get(Get get) throws ExecutionException { @@ -128,11 +138,6 @@ public Scanner scan(Scan scan) throws ExecutionException { scan = copyAndSetTargetToIfNot(scan); operationChecker.check(scan); - if (ScalarDbUtils.isRelational(scan)) { - throw new UnsupportedOperationException( - "Scanning all records with orderings or conditions is not supported in DynamoDB"); - } - return selectStatementHandler.handle(scan); } diff --git a/core/src/main/java/com/scalar/db/storage/dynamo/DynamoConfig.java b/core/src/main/java/com/scalar/db/storage/dynamo/DynamoConfig.java index a0f4c2c627..f8a3af149a 100644 --- a/core/src/main/java/com/scalar/db/storage/dynamo/DynamoConfig.java +++ b/core/src/main/java/com/scalar/db/storage/dynamo/DynamoConfig.java @@ -13,7 +13,8 @@ public class DynamoConfig { private static final Logger logger = LoggerFactory.getLogger(DynamoConfig.class); - public static final String PREFIX = DatabaseConfig.PREFIX + "dynamo."; + public static final String STORAGE_NAME = "dynamo"; + public static final String PREFIX = DatabaseConfig.PREFIX + STORAGE_NAME + "."; public static final String ENDPOINT_OVERRIDE = PREFIX + "endpoint_override"; public static final String TABLE_METADATA_NAMESPACE = PREFIX + "table_metadata.namespace"; public static final String NAMESPACE_PREFIX = PREFIX + "namespace.prefix"; @@ -27,8 +28,9 @@ public class DynamoConfig { public DynamoConfig(DatabaseConfig databaseConfig) { String storage = databaseConfig.getStorage(); - if (!"dynamo".equals(storage)) { - throw new IllegalArgumentException(DatabaseConfig.STORAGE + " should be 'dynamo'"); + if (!storage.equals(STORAGE_NAME)) { + throw new IllegalArgumentException( + DatabaseConfig.STORAGE + " should be '" + STORAGE_NAME + "'"); } if (databaseConfig.getContactPoints().isEmpty()) { diff --git a/core/src/main/java/com/scalar/db/storage/dynamo/DynamoOperationChecker.java b/core/src/main/java/com/scalar/db/storage/dynamo/DynamoOperationChecker.java index 914379c426..a7abd7d046 100644 --- a/core/src/main/java/com/scalar/db/storage/dynamo/DynamoOperationChecker.java +++ b/core/src/main/java/com/scalar/db/storage/dynamo/DynamoOperationChecker.java @@ -8,14 +8,16 @@ import com.scalar.db.common.TableMetadataManager; import com.scalar.db.common.checker.ColumnChecker; import com.scalar.db.common.checker.OperationChecker; +import com.scalar.db.config.DatabaseConfig; import com.scalar.db.exception.storage.ExecutionException; import com.scalar.db.io.DataType; import javax.annotation.concurrent.ThreadSafe; @ThreadSafe public class DynamoOperationChecker extends OperationChecker { - public DynamoOperationChecker(TableMetadataManager metadataManager) { - super(metadataManager); + public DynamoOperationChecker( + DatabaseConfig databaseConfig, TableMetadataManager metadataManager) { + super(databaseConfig, metadataManager); } @Override diff --git a/core/src/main/java/com/scalar/db/storage/dynamo/DynamoProvider.java b/core/src/main/java/com/scalar/db/storage/dynamo/DynamoProvider.java index c3b2249a19..67c263c164 100644 --- a/core/src/main/java/com/scalar/db/storage/dynamo/DynamoProvider.java +++ b/core/src/main/java/com/scalar/db/storage/dynamo/DynamoProvider.java @@ -9,7 +9,7 @@ public class DynamoProvider implements DistributedStorageProvider { @Override public String getName() { - return "dynamo"; + return DynamoConfig.STORAGE_NAME; } @Override diff --git a/core/src/main/java/com/scalar/db/storage/jdbc/JdbcConfig.java b/core/src/main/java/com/scalar/db/storage/jdbc/JdbcConfig.java index 8e53693fd9..a55d56187b 100644 --- a/core/src/main/java/com/scalar/db/storage/jdbc/JdbcConfig.java +++ b/core/src/main/java/com/scalar/db/storage/jdbc/JdbcConfig.java @@ -12,7 +12,8 @@ @Immutable public class JdbcConfig { - public static final String PREFIX = DatabaseConfig.PREFIX + "jdbc."; + public static final String STORAGE_NAME = "jdbc"; + public static final String PREFIX = DatabaseConfig.PREFIX + STORAGE_NAME + "."; public static final String CONNECTION_POOL_MIN_IDLE = PREFIX + "connection_pool.min_idle"; public static final String CONNECTION_POOL_MAX_IDLE = PREFIX + "connection_pool.max_idle"; public static final String CONNECTION_POOL_MAX_TOTAL = PREFIX + "connection_pool.max_total"; @@ -76,12 +77,14 @@ public class JdbcConfig { public JdbcConfig(DatabaseConfig databaseConfig) { String storage = databaseConfig.getStorage(); String transactionManager = databaseConfig.getTransactionManager(); - if (!"jdbc".equals(storage) && !"jdbc".equals(transactionManager)) { + if (!storage.equals(STORAGE_NAME) && !transactionManager.equals(STORAGE_NAME)) { throw new IllegalArgumentException( DatabaseConfig.STORAGE + " or " + DatabaseConfig.TRANSACTION_MANAGER - + " should be 'jdbc'"); + + " should be '" + + STORAGE_NAME + + "'"); } if (databaseConfig.getContactPoints().isEmpty()) { diff --git a/core/src/main/java/com/scalar/db/storage/jdbc/JdbcDatabase.java b/core/src/main/java/com/scalar/db/storage/jdbc/JdbcDatabase.java index 981e572b77..022c9786c9 100644 --- a/core/src/main/java/com/scalar/db/storage/jdbc/JdbcDatabase.java +++ b/core/src/main/java/com/scalar/db/storage/jdbc/JdbcDatabase.java @@ -58,7 +58,7 @@ public JdbcDatabase(DatabaseConfig databaseConfig) { new JdbcAdmin(tableMetadataDataSource, config), databaseConfig.getMetadataCacheExpirationTimeSecs()); - OperationChecker operationChecker = new OperationChecker(tableMetadataManager); + OperationChecker operationChecker = new OperationChecker(databaseConfig, tableMetadataManager); QueryBuilder queryBuilder = new QueryBuilder(rdbEngine); jdbcService = new JdbcService(tableMetadataManager, operationChecker, queryBuilder); } diff --git a/core/src/main/java/com/scalar/db/storage/jdbc/JdbcProvider.java b/core/src/main/java/com/scalar/db/storage/jdbc/JdbcProvider.java index 11166d818d..2bc7869a05 100644 --- a/core/src/main/java/com/scalar/db/storage/jdbc/JdbcProvider.java +++ b/core/src/main/java/com/scalar/db/storage/jdbc/JdbcProvider.java @@ -9,7 +9,7 @@ public class JdbcProvider implements DistributedStorageProvider { @Override public String getName() { - return "jdbc"; + return JdbcConfig.STORAGE_NAME; } @Override diff --git a/core/src/main/java/com/scalar/db/storage/jdbc/JdbcService.java b/core/src/main/java/com/scalar/db/storage/jdbc/JdbcService.java index aacf066b4a..28db40dd8c 100644 --- a/core/src/main/java/com/scalar/db/storage/jdbc/JdbcService.java +++ b/core/src/main/java/com/scalar/db/storage/jdbc/JdbcService.java @@ -18,7 +18,6 @@ import com.scalar.db.storage.jdbc.query.QueryBuilder; import com.scalar.db.storage.jdbc.query.SelectQuery; import com.scalar.db.storage.jdbc.query.UpsertQuery; -import com.scalar.db.util.ScalarDbUtils; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import java.sql.Connection; import java.sql.PreparedStatement; @@ -123,18 +122,9 @@ public List scan(Scan scan, Connection connection) private SelectQuery buildSelectQuery(Scan scan, TableMetadata tableMetadata) { if (scan instanceof ScanAll) { - ScanAll scanAll = (ScanAll) scan; - if (ScalarDbUtils.isRelational(scan)) { - return buildSelectQueryForRelationalScan(scanAll, tableMetadata); - } else { - return buildSelectQueryForScanAll(scanAll, tableMetadata); - } - } else { - return buildSelectQueryForScan(scan, tableMetadata); + return buildSelectQuery((ScanAll) scan, tableMetadata); } - } - private SelectQuery buildSelectQueryForScan(Scan scan, TableMetadata tableMetadata) { return queryBuilder .select(scan.getProjections()) .from(scan.forNamespace().get(), scan.forTable().get(), tableMetadata) @@ -149,15 +139,7 @@ private SelectQuery buildSelectQueryForScan(Scan scan, TableMetadata tableMetada .build(); } - private SelectQuery buildSelectQueryForScanAll(ScanAll scanAll, TableMetadata tableMetadata) { - return queryBuilder - .select(scanAll.getProjections()) - .from(scanAll.forNamespace().get(), scanAll.forTable().get(), tableMetadata) - .limit(scanAll.getLimit()) - .build(); - } - - private SelectQuery buildSelectQueryForRelationalScan(ScanAll scan, TableMetadata tableMetadata) { + private SelectQuery buildSelectQuery(ScanAll scan, TableMetadata tableMetadata) { return queryBuilder .select(scan.getProjections()) .from(scan.forNamespace().get(), scan.forTable().get(), tableMetadata) diff --git a/core/src/main/java/com/scalar/db/storage/multistorage/MultiStorageConfig.java b/core/src/main/java/com/scalar/db/storage/multistorage/MultiStorageConfig.java index c8d9b08655..583f4be89a 100644 --- a/core/src/main/java/com/scalar/db/storage/multistorage/MultiStorageConfig.java +++ b/core/src/main/java/com/scalar/db/storage/multistorage/MultiStorageConfig.java @@ -16,14 +16,13 @@ public class MultiStorageConfig { private static final Logger logger = LoggerFactory.getLogger(MultiStorageConfig.class); + public static final String STORAGE_NAME = "multi-storage"; public static final String PREFIX = DatabaseConfig.PREFIX + "multi_storage."; public static final String STORAGES = PREFIX + "storages"; public static final String TABLE_MAPPING = PREFIX + "table_mapping"; public static final String NAMESPACE_MAPPING = PREFIX + "namespace_mapping"; public static final String DEFAULT_STORAGE = PREFIX + "default_storage"; - private static final String MULTI_STORAGE = "multi-storage"; - private final ImmutableMap databasePropertiesMap; private final ImmutableMap tableStorageMap; private final ImmutableMap namespaceStorageMap; @@ -31,9 +30,9 @@ public class MultiStorageConfig { public MultiStorageConfig(DatabaseConfig databaseConfig) { String storage = databaseConfig.getStorage(); - if (!MULTI_STORAGE.equals(storage)) { + if (!storage.equals(STORAGE_NAME)) { throw new IllegalArgumentException( - DatabaseConfig.STORAGE + " should be '" + MULTI_STORAGE + "'"); + DatabaseConfig.STORAGE + " should be '" + STORAGE_NAME + "'"); } databasePropertiesMap = loadDatabasePropertiesMapping(databaseConfig.getProperties()); @@ -65,9 +64,9 @@ private ImmutableMap loadDatabasePropertiesMapping(Propertie } } - if (dbProps.getProperty(DatabaseConfig.STORAGE).equals(MULTI_STORAGE)) { + if (dbProps.getProperty(DatabaseConfig.STORAGE).equals(STORAGE_NAME)) { throw new IllegalArgumentException( - "Does not support nested " + MULTI_STORAGE + ": " + storage); + "Does not support nested " + STORAGE_NAME + ": " + storage); } builder.put(storage, dbProps); } diff --git a/core/src/main/java/com/scalar/db/storage/multistorage/MultiStorageProvider.java b/core/src/main/java/com/scalar/db/storage/multistorage/MultiStorageProvider.java index d4544561d9..031d951c4d 100644 --- a/core/src/main/java/com/scalar/db/storage/multistorage/MultiStorageProvider.java +++ b/core/src/main/java/com/scalar/db/storage/multistorage/MultiStorageProvider.java @@ -8,7 +8,7 @@ public class MultiStorageProvider implements DistributedStorageProvider { @Override public String getName() { - return "multi-storage"; + return MultiStorageConfig.STORAGE_NAME; } @Override diff --git a/core/src/main/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitConfig.java b/core/src/main/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitConfig.java index f7c66697b2..ca023b689b 100644 --- a/core/src/main/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitConfig.java +++ b/core/src/main/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitConfig.java @@ -5,8 +5,11 @@ import static com.scalar.db.config.ConfigUtils.getString; import com.scalar.db.config.DatabaseConfig; +import com.scalar.db.storage.jdbc.JdbcConfig; +import com.scalar.db.storage.multistorage.MultiStorageConfig; import java.util.Locale; import java.util.Optional; +import java.util.Properties; import javax.annotation.Nullable; import javax.annotation.concurrent.Immutable; import org.slf4j.Logger; @@ -77,6 +80,9 @@ public ConsensusCommitConfig(DatabaseConfig databaseConfig) { "scalar.db.isolation_level", // for backward compatibility Isolation.SNAPSHOT.toString())) .toUpperCase(Locale.ROOT)); + if (isolation.equals(Isolation.SERIALIZABLE)) { + validateCrossPartitionScanConfig(databaseConfig); + } strategy = SerializableStrategy.valueOf( getString( @@ -170,4 +176,30 @@ public boolean isIncludeMetadataEnabled() { public boolean isParallelImplicitPreReadEnabled() { return parallelImplicitPreReadEnabled; } + + private void validateCrossPartitionScanConfig(DatabaseConfig databaseConfig) { + // It might be better to let each storage have metadata (e.g., linearizable cross-partition scan + // is supported or not) and check it rather than checking specific storage types. We will + // revisit here when supporting metadata management in DistributedStorage. + if (databaseConfig.getStorage().equals(MultiStorageConfig.STORAGE_NAME)) { + MultiStorageConfig multiStorageConfig = new MultiStorageConfig(databaseConfig); + for (Properties props : multiStorageConfig.getDatabasePropertiesMap().values()) { + DatabaseConfig c = new DatabaseConfig(props); + if (!c.getStorage().equals(JdbcConfig.STORAGE_NAME) && c.isCrossPartitionScanEnabled()) { + warnCrossPartitionScan(c.getStorage()); + } + } + } else if (!databaseConfig.getStorage().equals(JdbcConfig.STORAGE_NAME) + && databaseConfig.isCrossPartitionScanEnabled()) { + warnCrossPartitionScan(databaseConfig.getStorage()); + } + } + + private void warnCrossPartitionScan(String storageName) { + logger.warn( + "Enabling cross-partition scan for '{}' in production is not recommended " + + "because it makes transaction isolation level lower (i.e., snapshot). " + + "Use it at your own risk only if the consistency does not matter for your transactions", + storageName); + } } diff --git a/core/src/main/java/com/scalar/db/transaction/consensuscommit/Snapshot.java b/core/src/main/java/com/scalar/db/transaction/consensuscommit/Snapshot.java index 9bcd286a93..1e0db045e9 100644 --- a/core/src/main/java/com/scalar/db/transaction/consensuscommit/Snapshot.java +++ b/core/src/main/java/com/scalar/db/transaction/consensuscommit/Snapshot.java @@ -209,9 +209,7 @@ public Optional> get(Scan scan) { } public void verify(Scan scan) { - boolean isRelational = ScalarDbUtils.isRelational(scan); - if ((isRelational && isWriteSetOverlappedWithRelational(scan)) - || (!isRelational && isWriteSetOverlappedWith(scan))) { + if (isWriteSetOverlappedWith(scan)) { throw new IllegalArgumentException("Reading already written data is not allowed"); } } @@ -233,15 +231,13 @@ public void to(MutationComposer composer) } private boolean isWriteSetOverlappedWith(Scan scan) { + if (scan instanceof ScanAll) { + return isWriteSetOverlappedWith((ScanAll) scan); + } + for (Map.Entry entry : writeSet.entrySet()) { Put put = entry.getValue(); - if (scan instanceof ScanAll - && put.forNamespace().equals(scan.forNamespace()) - && put.forTable().equals(scan.forTable())) { - return true; - } - if (!put.forNamespace().equals(scan.forNamespace()) || !put.forTable().equals(scan.forTable()) || !put.getPartitionKey().equals(scan.getPartitionKey())) { @@ -294,7 +290,7 @@ private boolean isWriteSetOverlappedWith(Scan scan) { return false; } - private boolean isWriteSetOverlappedWithRelational(Scan scan) { + private boolean isWriteSetOverlappedWith(ScanAll scan) { for (Map.Entry entry : writeSet.entrySet()) { // We need to consider three cases here to prevent scan-after-write. // 1) A put operation overlaps the scan range regardless of the update (put) results. diff --git a/core/src/main/java/com/scalar/db/transaction/jdbc/JdbcTransactionManager.java b/core/src/main/java/com/scalar/db/transaction/jdbc/JdbcTransactionManager.java index 8c70cc5868..e6d6b2c41e 100644 --- a/core/src/main/java/com/scalar/db/transaction/jdbc/JdbcTransactionManager.java +++ b/core/src/main/java/com/scalar/db/transaction/jdbc/JdbcTransactionManager.java @@ -48,7 +48,7 @@ public JdbcTransactionManager(DatabaseConfig databaseConfig) { new JdbcAdmin(tableMetadataDataSource, config), databaseConfig.getMetadataCacheExpirationTimeSecs()); - OperationChecker operationChecker = new OperationChecker(tableMetadataManager); + OperationChecker operationChecker = new OperationChecker(databaseConfig, tableMetadataManager); QueryBuilder queryBuilder = new QueryBuilder(rdbEngine); jdbcService = new JdbcService(tableMetadataManager, operationChecker, queryBuilder); } diff --git a/core/src/main/java/com/scalar/db/util/ScalarDbUtils.java b/core/src/main/java/com/scalar/db/util/ScalarDbUtils.java index 247c737f94..4db72c32ab 100644 --- a/core/src/main/java/com/scalar/db/util/ScalarDbUtils.java +++ b/core/src/main/java/com/scalar/db/util/ScalarDbUtils.java @@ -244,9 +244,4 @@ public static Column toColumn(Value value) { throw new AssertionError(); } } - - public static boolean isRelational(Scan scan) { - return scan instanceof ScanAll - && (!scan.getOrderings().isEmpty() || !scan.getConjunctions().isEmpty()); - } } diff --git a/core/src/test/java/com/scalar/db/common/checker/OperationCheckerTest.java b/core/src/test/java/com/scalar/db/common/checker/OperationCheckerTest.java index d11dd21ff0..cfa6991380 100644 --- a/core/src/test/java/com/scalar/db/common/checker/OperationCheckerTest.java +++ b/core/src/test/java/com/scalar/db/common/checker/OperationCheckerTest.java @@ -18,9 +18,11 @@ import com.scalar.db.api.PutIfExists; import com.scalar.db.api.PutIfNotExists; import com.scalar.db.api.Scan; +import com.scalar.db.api.Scan.Ordering; import com.scalar.db.api.ScanAll; import com.scalar.db.api.TableMetadata; import com.scalar.db.common.TableMetadataManager; +import com.scalar.db.config.DatabaseConfig; import com.scalar.db.exception.storage.ExecutionException; import com.scalar.db.io.BooleanValue; import com.scalar.db.io.DataType; @@ -50,6 +52,7 @@ public class OperationCheckerTest { private static final String COL2 = "v2"; private static final String COL3 = "v3"; + @Mock private DatabaseConfig databaseConfig; @Mock private TableMetadataManager metadataManager; private OperationChecker operationChecker; @@ -75,7 +78,7 @@ public void setUp() throws Exception { .addSecondaryIndex(COL1) .build()); - operationChecker = new OperationChecker(metadataManager); + operationChecker = new OperationChecker(databaseConfig, metadataManager); } @Test @@ -959,7 +962,7 @@ public void whenCheckingPutOperationWithDeleteIfCondition_shouldThrowIllegalArgu .addClusteringKey(CKEY1) .build()); - operationChecker = new OperationChecker(metadataManager); + operationChecker = new OperationChecker(databaseConfig, metadataManager); Key partitionKey = new Key(PKEY1, (byte[]) null); Key clusteringKey = new Key(CKEY1, new byte[] {1, 1, 1}); @@ -990,7 +993,7 @@ public void whenCheckingPutOperationWithDeleteIfCondition_shouldThrowIllegalArgu .addClusteringKey(CKEY1) .build()); - operationChecker = new OperationChecker(metadataManager); + operationChecker = new OperationChecker(databaseConfig, metadataManager); Key partitionKey = new Key(PKEY1, new byte[0]); Key clusteringKey = new Key(CKEY1, new byte[] {1, 1, 1}); @@ -1021,7 +1024,7 @@ public void whenCheckingPutOperationWithDeleteIfCondition_shouldThrowIllegalArgu .addClusteringKey(CKEY1) .build()); - operationChecker = new OperationChecker(metadataManager); + operationChecker = new OperationChecker(databaseConfig, metadataManager); Key partitionKey = new Key(PKEY1, new byte[] {1, 1, 1}); Key clusteringKey = new Key(CKEY1, (byte[]) null); @@ -1052,7 +1055,7 @@ public void whenCheckingPutOperationWithDeleteIfCondition_shouldThrowIllegalArgu .addClusteringKey(CKEY1) .build()); - operationChecker = new OperationChecker(metadataManager); + operationChecker = new OperationChecker(databaseConfig, metadataManager); Key partitionKey = new Key(PKEY1, new byte[] {1, 1, 1}); Key clusteringKey = new Key(CKEY1, new byte[0]); @@ -1698,6 +1701,8 @@ public void whenCheckingScanAllOperationWithAllValidArguments_shouldNotThrowAnyE .withLimit(limit) .forNamespace(NAMESPACE) .forTable(TABLE_NAME); + when(databaseConfig.isCrossPartitionScanEnabled()).thenReturn(true); + operationChecker = new OperationChecker(databaseConfig, metadataManager); // Act Assert assertThatCode(() -> operationChecker.check(scanAll)).doesNotThrowAnyException(); @@ -1715,6 +1720,8 @@ public void whenCheckingScanAllOperationWithAllValidArguments_shouldNotThrowAnyE .withLimit(limit) .forNamespace(NAMESPACE) .forTable(TABLE_NAME); + when(databaseConfig.isCrossPartitionScanEnabled()).thenReturn(true); + operationChecker = new OperationChecker(databaseConfig, metadataManager); // Act Assert assertThatThrownBy(() -> operationChecker.check(scanAll)) @@ -1733,6 +1740,72 @@ public void whenCheckingScanAllOperationWithAllValidArguments_shouldNotThrowAnyE .withLimit(limit) .forNamespace(NAMESPACE) .forTable(TABLE_NAME); + when(databaseConfig.isCrossPartitionScanEnabled()).thenReturn(true); + operationChecker = new OperationChecker(databaseConfig, metadataManager); + + // Act Assert + assertThatThrownBy(() -> operationChecker.check(scanAll)) + .isInstanceOf(IllegalArgumentException.class); + } + + @Test + public void + whenCheckingScanAllOperationWithCrossPartitionScanDisabled_shouldThrowIllegalArgumentException() { + // Arrange + Scan scanAll = + Scan.newBuilder() + .namespace(NAMESPACE) + .table(TABLE_NAME) + .all() + .projections(Arrays.asList(COL1, COL2, COL3)) + .limit(10) + .build(); + when(databaseConfig.isCrossPartitionScanEnabled()).thenReturn(false); + operationChecker = new OperationChecker(databaseConfig, metadataManager); + + // Act Assert + assertThatThrownBy(() -> operationChecker.check(scanAll)) + .isInstanceOf(IllegalArgumentException.class); + } + + @Test + public void + whenCheckingScanAllOperationWithOrderingsButCrossPartitionScanOrderingDisabled_shouldThrowIllegalArgumentException() { + // Arrange + Scan scanAll = + Scan.newBuilder() + .namespace(NAMESPACE) + .table(TABLE_NAME) + .all() + .projections(Arrays.asList(COL1, COL2, COL3)) + .ordering(Ordering.desc(COL1)) + .limit(10) + .build(); + when(databaseConfig.isCrossPartitionScanEnabled()).thenReturn(true); + when(databaseConfig.isCrossPartitionScanOrderingEnabled()).thenReturn(false); + operationChecker = new OperationChecker(databaseConfig, metadataManager); + + // Act Assert + assertThatThrownBy(() -> operationChecker.check(scanAll)) + .isInstanceOf(IllegalArgumentException.class); + } + + @Test + public void + whenCheckingScanAllOperationWithConjunctionsButCrossPartitionScanFilteringDisabled_shouldThrowIllegalArgumentException() { + // Arrange + Scan scanAll = + Scan.newBuilder() + .namespace(NAMESPACE) + .table(TABLE_NAME) + .all() + .where(ConditionBuilder.column(COL3).isEqualToText("aaa")) + .projections(Arrays.asList(COL1, COL2, COL3)) + .limit(10) + .build(); + when(databaseConfig.isCrossPartitionScanEnabled()).thenReturn(true); + when(databaseConfig.isCrossPartitionScanFilteringEnabled()).thenReturn(false); + operationChecker = new OperationChecker(databaseConfig, metadataManager); // Act Assert assertThatThrownBy(() -> operationChecker.check(scanAll)) diff --git a/core/src/test/java/com/scalar/db/config/DatabaseConfigTest.java b/core/src/test/java/com/scalar/db/config/DatabaseConfigTest.java index 8947673e62..686eb52c08 100644 --- a/core/src/test/java/com/scalar/db/config/DatabaseConfigTest.java +++ b/core/src/test/java/com/scalar/db/config/DatabaseConfigTest.java @@ -12,6 +12,8 @@ public class DatabaseConfigTest { private static final int ANY_PORT = 9999; private static final String ANY_USERNAME = "username"; private static final String ANY_PASSWORD = "password"; + private static final String ANY_TRUE = "true"; + private static final String ANY_FALSE = "false"; @Test public void constructor_PropertiesWithoutPortGiven_ShouldLoadProperly() { @@ -35,6 +37,9 @@ public void constructor_PropertiesWithoutPortGiven_ShouldLoadProperly() { assertThat(config.getTransactionManager()).isEqualTo("consensus-commit"); assertThat(config.getMetadataCacheExpirationTimeSecs()).isEqualTo(-1); assertThat(config.getActiveTransactionManagementExpirationTimeMillis()).isEqualTo(-1); + assertThat(config.isCrossPartitionScanEnabled()).isTrue(); + assertThat(config.isCrossPartitionScanFilteringEnabled()).isFalse(); + assertThat(config.isCrossPartitionScanOrderingEnabled()).isFalse(); } @Test @@ -59,6 +64,9 @@ public void constructor_PropertiesWithoutUsernameGiven_ShouldLoadProperly() { assertThat(config.getMetadataCacheExpirationTimeSecs()).isEqualTo(-1); assertThat(config.getActiveTransactionManagementExpirationTimeMillis()).isEqualTo(-1); assertThat(config.getDefaultNamespaceName()).isEmpty(); + assertThat(config.isCrossPartitionScanEnabled()).isTrue(); + assertThat(config.isCrossPartitionScanFilteringEnabled()).isFalse(); + assertThat(config.isCrossPartitionScanOrderingEnabled()).isFalse(); } @Test @@ -83,6 +91,9 @@ public void constructor_PropertiesWithoutPasswordGiven_ShouldLoadProperly() { assertThat(config.getMetadataCacheExpirationTimeSecs()).isEqualTo(-1); assertThat(config.getActiveTransactionManagementExpirationTimeMillis()).isEqualTo(-1); assertThat(config.getDefaultNamespaceName()).isEmpty(); + assertThat(config.isCrossPartitionScanEnabled()).isTrue(); + assertThat(config.isCrossPartitionScanFilteringEnabled()).isFalse(); + assertThat(config.isCrossPartitionScanOrderingEnabled()).isFalse(); } @Test @@ -109,6 +120,9 @@ public void constructor_PropertiesWithPortGiven_ShouldLoadProperly() { assertThat(config.getMetadataCacheExpirationTimeSecs()).isEqualTo(-1); assertThat(config.getActiveTransactionManagementExpirationTimeMillis()).isEqualTo(-1); assertThat(config.getDefaultNamespaceName()).isEmpty(); + assertThat(config.isCrossPartitionScanEnabled()).isTrue(); + assertThat(config.isCrossPartitionScanFilteringEnabled()).isFalse(); + assertThat(config.isCrossPartitionScanOrderingEnabled()).isFalse(); } @Test @@ -385,4 +399,47 @@ public void constructor_PropertiesWithDefaultNamespaceNameGiven_ShouldLoadProper assertThat(config.getPassword().get()).isEqualTo(ANY_PASSWORD); assertThat(config.getDefaultNamespaceName()).hasValue("ns"); } + + @Test + public void constructor_PropertiesWithCrossPartitionScanSettingsGiven_ShouldLoadProperly() { + // Arrange + Properties props = new Properties(); + props.setProperty(DatabaseConfig.CONTACT_POINTS, ANY_HOST); + props.setProperty(DatabaseConfig.USERNAME, ANY_USERNAME); + props.setProperty(DatabaseConfig.PASSWORD, ANY_PASSWORD); + props.setProperty(DatabaseConfig.CROSS_PARTITION_SCAN, ANY_TRUE); + props.setProperty(DatabaseConfig.CROSS_PARTITION_SCAN_FILTERING, ANY_TRUE); + props.setProperty(DatabaseConfig.CROSS_PARTITION_SCAN_ORDERING, ANY_FALSE); + + // Act + DatabaseConfig config = new DatabaseConfig(props); + + // Assert + assertThat(config.getContactPoints()).isEqualTo(Collections.singletonList(ANY_HOST)); + assertThat(config.getContactPort()).isEqualTo(0); + assertThat(config.getUsername().isPresent()).isTrue(); + assertThat(config.getUsername().get()).isEqualTo(ANY_USERNAME); + assertThat(config.getPassword().isPresent()).isTrue(); + assertThat(config.getPassword().get()).isEqualTo(ANY_PASSWORD); + assertThat(config.isCrossPartitionScanEnabled()).isTrue(); + assertThat(config.isCrossPartitionScanFilteringEnabled()).isTrue(); + assertThat(config.isCrossPartitionScanOrderingEnabled()).isFalse(); + } + + @Test + public void + constructor_PropertiesWithInvalidCrossPartitionScanSettingsGiven_ShouldThrowIllegalArgumentException() { + // Arrange + Properties props = new Properties(); + props.setProperty(DatabaseConfig.CONTACT_POINTS, ANY_HOST); + props.setProperty(DatabaseConfig.USERNAME, ANY_USERNAME); + props.setProperty(DatabaseConfig.PASSWORD, ANY_PASSWORD); + props.setProperty(DatabaseConfig.CROSS_PARTITION_SCAN, ANY_FALSE); + props.setProperty(DatabaseConfig.CROSS_PARTITION_SCAN_FILTERING, ANY_TRUE); + props.setProperty(DatabaseConfig.CROSS_PARTITION_SCAN_ORDERING, ANY_TRUE); + + // Act Assert + assertThatThrownBy(() -> new DatabaseConfig(props)) + .isInstanceOf(IllegalArgumentException.class); + } } diff --git a/core/src/test/java/com/scalar/db/storage/cassandra/CassandraTest.java b/core/src/test/java/com/scalar/db/storage/cassandra/CassandraTest.java deleted file mode 100644 index b0228ea53b..0000000000 --- a/core/src/test/java/com/scalar/db/storage/cassandra/CassandraTest.java +++ /dev/null @@ -1,81 +0,0 @@ -package com.scalar.db.storage.cassandra; - -import static org.assertj.core.api.Assertions.assertThatThrownBy; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -import com.datastax.driver.core.ResultSet; -import com.scalar.db.api.ConditionBuilder; -import com.scalar.db.api.Scan; -import com.scalar.db.api.TableMetadata; -import com.scalar.db.common.TableMetadataManager; -import com.scalar.db.common.checker.OperationChecker; -import com.scalar.db.config.DatabaseConfig; -import com.scalar.db.exception.storage.ExecutionException; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.mockito.Mock; -import org.mockito.MockitoAnnotations; - -public class CassandraTest { - private Cassandra cassandra; - private StatementHandlerManager handlers; - @Mock private DatabaseConfig config; - @Mock private ClusterManager clusterManager; - @Mock private SelectStatementHandler select; - @Mock private InsertStatementHandler insert; - @Mock private UpdateStatementHandler update; - @Mock private DeleteStatementHandler delete; - @Mock private BatchHandler batchHandler; - @Mock private TableMetadataManager metadataManager; - @Mock private OperationChecker operationChecker; - @Mock private ResultSet resultSet; - @Mock private TableMetadata tableMetadata; - - @BeforeEach - public void setUp() throws Exception { - MockitoAnnotations.openMocks(this).close(); - handlers = - StatementHandlerManager.builder() - .select(select) - .insert(insert) - .update(update) - .delete(delete) - .build(); - cassandra = - new Cassandra( - config, clusterManager, handlers, batchHandler, metadataManager, operationChecker); - } - - @Test - public void scan_ScanAllWithoutOrderingAndConditionsGiven_ShouldScanHandled() - throws ExecutionException { - // Arrange - Scan scan = Scan.newBuilder().namespace("namespace").table("table").all().build(); - when(handlers.select().handle(any())).thenReturn(resultSet); - when(metadataManager.getTableMetadata(any())).thenReturn(tableMetadata); - - // Act - cassandra.scan(scan); - - // Assert - verify(handlers.select()).handle(scan); - } - - @Test - public void scan_ScanAllWithConditionsGiven_ShouldThrowUnsupportedOperationException() { - // Arrange - Scan scan = - Scan.newBuilder() - .namespace("namespace") - .table("table") - .all() - .where(ConditionBuilder.column("column").isEqualToInt(1)) - .build(); - - // Act Assert - assertThatThrownBy(() -> cassandra.scan(scan)) - .isInstanceOf(UnsupportedOperationException.class); - } -} diff --git a/core/src/test/java/com/scalar/db/storage/cosmos/CosmosOperationCheckerTest.java b/core/src/test/java/com/scalar/db/storage/cosmos/CosmosOperationCheckerTest.java index 61c3726761..31c67ef572 100644 --- a/core/src/test/java/com/scalar/db/storage/cosmos/CosmosOperationCheckerTest.java +++ b/core/src/test/java/com/scalar/db/storage/cosmos/CosmosOperationCheckerTest.java @@ -17,6 +17,7 @@ import com.scalar.db.api.Put; import com.scalar.db.api.TableMetadata; import com.scalar.db.common.TableMetadataManager; +import com.scalar.db.config.DatabaseConfig; import com.scalar.db.io.DataType; import com.scalar.db.io.Key; import java.nio.charset.StandardCharsets; @@ -33,6 +34,7 @@ public class CosmosOperationCheckerTest { private static final String CKEY1 = "c1"; private static final String COL1 = "v1"; private static final String COL2 = "v2"; + @Mock private DatabaseConfig databaseConfig; @Mock private TableMetadataManager metadataManager; private CosmosOperationChecker operationChecker; @@ -50,7 +52,7 @@ public void setUp() throws Exception { .addSecondaryIndex(COL1) .build(); when(metadataManager.getTableMetadata(any())).thenReturn(tableMetadata); - operationChecker = new CosmosOperationChecker(metadataManager); + operationChecker = new CosmosOperationChecker(databaseConfig, metadataManager); } @Test diff --git a/core/src/test/java/com/scalar/db/storage/cosmos/CosmosTest.java b/core/src/test/java/com/scalar/db/storage/cosmos/CosmosTest.java deleted file mode 100644 index 609292257d..0000000000 --- a/core/src/test/java/com/scalar/db/storage/cosmos/CosmosTest.java +++ /dev/null @@ -1,60 +0,0 @@ -package com.scalar.db.storage.cosmos; - -import static org.assertj.core.api.Assertions.assertThatThrownBy; -import static org.mockito.Mockito.verify; - -import com.azure.cosmos.CosmosClient; -import com.scalar.db.api.ConditionBuilder; -import com.scalar.db.api.Scan; -import com.scalar.db.common.checker.OperationChecker; -import com.scalar.db.config.DatabaseConfig; -import com.scalar.db.exception.storage.ExecutionException; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.mockito.Mock; -import org.mockito.MockitoAnnotations; - -public class CosmosTest { - private Cosmos cosmos; - @Mock private DatabaseConfig config; - @Mock private CosmosClient client; - @Mock private SelectStatementHandler select; - @Mock private PutStatementHandler put; - @Mock private DeleteStatementHandler delete; - @Mock private BatchHandler batch; - @Mock private OperationChecker operationChecker; - - @BeforeEach - public void setUp() throws Exception { - MockitoAnnotations.openMocks(this).close(); - cosmos = new Cosmos(config, client, select, put, delete, batch, operationChecker); - } - - @Test - public void scan_ScanAllWithoutOrderingAndConditionsGiven_ShouldScanHandled() - throws ExecutionException { - // Arrange - Scan scan = Scan.newBuilder().namespace("namespace").table("table").all().build(); - - // Act - cosmos.scan(scan); - - // Assert - verify(select).handle(scan); - } - - @Test - public void scan_ScanAllWithConditionsGiven_ShouldThrowUnsupportedOperationException() { - // Arrange - Scan scan = - Scan.newBuilder() - .namespace("namespace") - .table("table") - .all() - .where(ConditionBuilder.column("column").isEqualToInt(1)) - .build(); - - // Act Assert - assertThatThrownBy(() -> cosmos.scan(scan)).isInstanceOf(UnsupportedOperationException.class); - } -} diff --git a/core/src/test/java/com/scalar/db/storage/dynamo/DynamoOperationCheckerTest.java b/core/src/test/java/com/scalar/db/storage/dynamo/DynamoOperationCheckerTest.java index c2cd1a9958..1ade1bc8d8 100644 --- a/core/src/test/java/com/scalar/db/storage/dynamo/DynamoOperationCheckerTest.java +++ b/core/src/test/java/com/scalar/db/storage/dynamo/DynamoOperationCheckerTest.java @@ -17,6 +17,7 @@ import com.scalar.db.api.Put; import com.scalar.db.api.TableMetadata; import com.scalar.db.common.TableMetadataManager; +import com.scalar.db.config.DatabaseConfig; import com.scalar.db.io.DataType; import com.scalar.db.io.Key; import java.util.Arrays; @@ -34,6 +35,7 @@ public class DynamoOperationCheckerTest { private static final String COL2 = "v2"; private static final String COL3 = "v3"; private static final String COL4 = "v4"; + @Mock private DatabaseConfig databaseConfig; @Mock private TableMetadataManager metadataManager; private DynamoOperationChecker operationChecker; @@ -55,7 +57,7 @@ public void setUp() throws Exception { .addSecondaryIndex(COL4) .build(); when(metadataManager.getTableMetadata(any())).thenReturn(tableMetadata); - operationChecker = new DynamoOperationChecker(metadataManager); + operationChecker = new DynamoOperationChecker(databaseConfig, metadataManager); } @Test diff --git a/core/src/test/java/com/scalar/db/storage/dynamo/DynamoTest.java b/core/src/test/java/com/scalar/db/storage/dynamo/DynamoTest.java deleted file mode 100644 index 385cceda5d..0000000000 --- a/core/src/test/java/com/scalar/db/storage/dynamo/DynamoTest.java +++ /dev/null @@ -1,60 +0,0 @@ -package com.scalar.db.storage.dynamo; - -import static org.assertj.core.api.Assertions.assertThatThrownBy; -import static org.mockito.Mockito.verify; - -import com.scalar.db.api.ConditionBuilder; -import com.scalar.db.api.Scan; -import com.scalar.db.common.checker.OperationChecker; -import com.scalar.db.config.DatabaseConfig; -import com.scalar.db.exception.storage.ExecutionException; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.mockito.Mock; -import org.mockito.MockitoAnnotations; -import software.amazon.awssdk.services.dynamodb.DynamoDbClient; - -public class DynamoTest { - private Dynamo dynamo; - @Mock private DatabaseConfig config; - @Mock private DynamoDbClient client; - @Mock private SelectStatementHandler select; - @Mock private PutStatementHandler put; - @Mock private DeleteStatementHandler delete; - @Mock private BatchHandler batch; - @Mock private OperationChecker operationChecker; - - @BeforeEach - public void setUp() throws Exception { - MockitoAnnotations.openMocks(this).close(); - dynamo = new Dynamo(config, client, select, put, delete, batch, operationChecker); - } - - @Test - public void scan_ScanAllWithoutOrderingAndConditionsGiven_ShouldScanHandled() - throws ExecutionException { - // Arrange - Scan scan = Scan.newBuilder().namespace("namespace").table("table").all().build(); - - // Act - dynamo.scan(scan); - - // Assert - verify(select).handle(scan); - } - - @Test - public void scan_ScanAllWithConditionsGiven_ShouldThrowUnsupportedOperationException() { - // Arrange - Scan scan = - Scan.newBuilder() - .namespace("namespace") - .table("table") - .all() - .where(ConditionBuilder.column("column").isEqualToInt(1)) - .build(); - - // Act Assert - assertThatThrownBy(() -> dynamo.scan(scan)).isInstanceOf(UnsupportedOperationException.class); - } -} diff --git a/core/src/test/java/com/scalar/db/storage/jdbc/JdbcServiceTest.java b/core/src/test/java/com/scalar/db/storage/jdbc/JdbcServiceTest.java index 8538912698..6f58d61f3c 100644 --- a/core/src/test/java/com/scalar/db/storage/jdbc/JdbcServiceTest.java +++ b/core/src/test/java/com/scalar/db/storage/jdbc/JdbcServiceTest.java @@ -141,6 +141,8 @@ public void whenGetScannerExecuted_withScanAll_shouldCallQueryBuilder() throws E when(queryBuilder.select(any())).thenReturn(selectQueryBuilder); when(selectQueryBuilder.from(any(), any(), any())).thenReturn(selectQueryBuilder); + when(selectQueryBuilder.where(any())).thenReturn(selectQueryBuilder); + when(selectQueryBuilder.orderBy(any())).thenReturn(selectQueryBuilder); when(selectQueryBuilder.limit(anyInt())).thenReturn(selectQueryBuilder); when(selectQueryBuilder.build()).thenReturn(selectQuery); @@ -222,6 +224,8 @@ public void whenScanExecuted_withScanAll_shouldCallQueryBuilder() throws Excepti when(queryBuilder.select(any())).thenReturn(selectQueryBuilder); when(selectQueryBuilder.from(any(), any(), any())).thenReturn(selectQueryBuilder); + when(selectQueryBuilder.where(any())).thenReturn(selectQueryBuilder); + when(selectQueryBuilder.orderBy(any())).thenReturn(selectQueryBuilder); when(selectQueryBuilder.limit(anyInt())).thenReturn(selectQueryBuilder); when(selectQueryBuilder.build()).thenReturn(selectQuery); diff --git a/core/src/test/java/com/scalar/db/transaction/consensuscommit/SnapshotTest.java b/core/src/test/java/com/scalar/db/transaction/consensuscommit/SnapshotTest.java index be2eb3764d..ce97f48500 100644 --- a/core/src/test/java/com/scalar/db/transaction/consensuscommit/SnapshotTest.java +++ b/core/src/test/java/com/scalar/db/transaction/consensuscommit/SnapshotTest.java @@ -1322,6 +1322,8 @@ public void verify_ScanAllGivenAndPutInWriteSetInSameTable_ShouldThrowException( .withConsistency(Consistency.LINEARIZABLE) .forNamespace(ANY_NAMESPACE_NAME) .forTable(ANY_TABLE_NAME); + Snapshot.Key key = new Snapshot.Key(scanAll, prepareResult(ANY_ID)); + snapshot.put(scanAll, Collections.singletonList(key)); // Act Assert Throwable thrown = catchThrowable(() -> snapshot.verify(scanAll)); @@ -1344,6 +1346,8 @@ public void verify_ScanAllGivenAndPutInWriteSetInSameTable_ShouldThrowException( .withConsistency(Consistency.LINEARIZABLE) .forNamespace(ANY_NAMESPACE_NAME_2) .forTable(ANY_TABLE_NAME_2); + Snapshot.Key key = new Snapshot.Key(scanAll, prepareResult(ANY_ID)); + snapshot.put(scanAll, Collections.singletonList(key)); // Act Assert Throwable thrown = catchThrowable(() -> snapshot.verify(scanAll)); diff --git a/server/src/integration-test/java/com/scalar/db/server/ServerEnv.java b/server/src/integration-test/java/com/scalar/db/server/ServerEnv.java index 39c2d740bb..920052cf85 100644 --- a/server/src/integration-test/java/com/scalar/db/server/ServerEnv.java +++ b/server/src/integration-test/java/com/scalar/db/server/ServerEnv.java @@ -56,6 +56,9 @@ public static Properties getServer1Properties(String testName) { properties.setProperty(DatabaseConfig.USERNAME, jdbcUsername); properties.setProperty(DatabaseConfig.PASSWORD, jdbcPassword); properties.setProperty(DatabaseConfig.STORAGE, "jdbc"); // use JDBC storage + properties.setProperty(DatabaseConfig.CROSS_PARTITION_SCAN, "true"); + properties.setProperty(DatabaseConfig.CROSS_PARTITION_SCAN_FILTERING, "true"); + properties.setProperty(DatabaseConfig.CROSS_PARTITION_SCAN_ORDERING, "true"); properties.setProperty(ServerConfig.PORT, DEFAULT_GRPC_CONTACT_PORT_SERVER1); properties.setProperty(ServerConfig.PROMETHEUS_EXPORTER_PORT, "-1");