From 0782ebe165948bda1935e9d5c88c8e44a047797c Mon Sep 17 00:00:00 2001 From: Jun Nemoto Date: Fri, 25 Aug 2023 18:50:54 +0900 Subject: [PATCH] Fix to handle version-specific integration tests --- .github/workflows/ci.yaml | 4 +- ...upported_storages_compatibility_check.yaml | 12 +- ...tTableIntegrationTestWithJdbcDatabase.java | 3 +- .../JdbcAdminImportTableIntegrationTest.java | 3 +- .../jdbc/JdbcAdminImportTestUtils.java | 210 +++++++++++++----- .../com/scalar/db/storage/jdbc/JdbcEnv.java | 5 + ...actionAdminImportTableIntegrationTest.java | 3 +- 7 files changed, 173 insertions(+), 67 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 82e292f106..fdf83a2be1 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -240,7 +240,7 @@ jobs: - name: Setup and execute Gradle 'integrationTestJdbc' task uses: gradle/gradle-build-action@v2 with: - arguments: integrationTestJdbc -Dscalardb.jdbc.url=jdbc:postgresql://localhost:5432/ -Dscalardb.jdbc.username=postgres -Dscalardb.jdbc.password=postgres + arguments: integrationTestJdbc -Dscalardb.jdbc.url=jdbc:postgresql://localhost:5432/ -Dscalardb.jdbc.username=postgres -Dscalardb.jdbc.password=postgres -Dscalardb.jdbc.database.version.major=14 - name: Upload Gradle test reports if: always() @@ -284,7 +284,7 @@ jobs: uses: gradle/gradle-build-action@v2 # Addressing the Oracle container with "@localhost" in the JDBC url cause the JDBC connection to fail for an unknown reason. As a workaround, using instead the container ip address is working. with: - arguments: integrationTestJdbc -Dscalardb.jdbc.url=jdbc:oracle:thin:@${{ env.oracle_db_ip }}:1521/XEPDB1 -Dscalardb.jdbc.username=SYSTEM -Dscalardb.jdbc.password=Oracle21 + arguments: integrationTestJdbc -Dscalardb.jdbc.url=jdbc:oracle:thin:@${{ env.oracle_db_ip }}:1521/XEPDB1 -Dscalardb.jdbc.username=SYSTEM -Dscalardb.jdbc.password=Oracle21 -Dscalardb.jdbc.database.version.major=21 - name: Upload Gradle test reports if: always() diff --git a/.github/workflows/supported_storages_compatibility_check.yaml b/.github/workflows/supported_storages_compatibility_check.yaml index d104da5038..0da0f6be57 100644 --- a/.github/workflows/supported_storages_compatibility_check.yaml +++ b/.github/workflows/supported_storages_compatibility_check.yaml @@ -255,7 +255,7 @@ jobs: - name: Setup and execute Gradle 'integrationTestJdbc' task uses: gradle/gradle-build-action@v2 with: - arguments: integrationTestJdbc -Dscalardb.jdbc.url=jdbc:postgresql://localhost:5432/ -Dscalardb.jdbc.username=postgres -Dscalardb.jdbc.password=postgres + arguments: integrationTestJdbc -Dscalardb.jdbc.url=jdbc:postgresql://localhost:5432/ -Dscalardb.jdbc.username=postgres -Dscalardb.jdbc.password=postgres -Dscalardb.jdbc.database.version.major=12 - name: Upload Gradle test reports if: always() @@ -288,7 +288,7 @@ jobs: - name: Setup and execute Gradle 'integrationTestJdbc' task uses: gradle/gradle-build-action@v2 with: - arguments: integrationTestJdbc -Dscalardb.jdbc.url=jdbc:postgresql://localhost:5432/ -Dscalardb.jdbc.username=postgres -Dscalardb.jdbc.password=postgres + arguments: integrationTestJdbc -Dscalardb.jdbc.url=jdbc:postgresql://localhost:5432/ -Dscalardb.jdbc.username=postgres -Dscalardb.jdbc.password=postgres -Dscalardb.jdbc.database.version.major=13 - name: Upload Gradle test reports if: always() @@ -321,7 +321,7 @@ jobs: - name: Setup and execute Gradle 'integrationTestJdbc' task uses: gradle/gradle-build-action@v2 with: - arguments: integrationTestJdbc -Dscalardb.jdbc.url=jdbc:postgresql://localhost:5432/ -Dscalardb.jdbc.username=postgres -Dscalardb.jdbc.password=postgres + arguments: integrationTestJdbc -Dscalardb.jdbc.url=jdbc:postgresql://localhost:5432/ -Dscalardb.jdbc.username=postgres -Dscalardb.jdbc.password=postgres -Dscalardb.jdbc.database.version.major=14 - name: Upload Gradle test reports if: always() @@ -354,7 +354,7 @@ jobs: - name: Setup and execute Gradle 'integrationTestJdbc' task uses: gradle/gradle-build-action@v2 with: - arguments: integrationTestJdbc -Dscalardb.jdbc.url=jdbc:postgresql://localhost:5432/ -Dscalardb.jdbc.username=postgres -Dscalardb.jdbc.password=postgres + arguments: integrationTestJdbc -Dscalardb.jdbc.url=jdbc:postgresql://localhost:5432/ -Dscalardb.jdbc.username=postgres -Dscalardb.jdbc.password=postgres -Dscalardb.jdbc.database.version.major=15 - name: Upload Gradle test reports if: always() @@ -398,7 +398,7 @@ jobs: uses: gradle/gradle-build-action@v2 # Addressing the Oracle container with "@localhost" in the JDBC url cause the JDBC connection to fail for an unknown reason. As a workaround, using instead the container ip address is working. with: - arguments: integrationTestJdbc -Dscalardb.jdbc.url=jdbc:oracle:thin:@${{ env.oracle_db_ip }}:1521/XEPDB1 -Dscalardb.jdbc.username=SYSTEM -Dscalardb.jdbc.password=Oracle + arguments: integrationTestJdbc -Dscalardb.jdbc.url=jdbc:oracle:thin:@${{ env.oracle_db_ip }}:1521/XEPDB1 -Dscalardb.jdbc.username=SYSTEM -Dscalardb.jdbc.password=Oracle -Dscalardb.jdbc.database.version.major=18 - name: Upload Gradle test reports if: always() @@ -440,7 +440,7 @@ jobs: uses: gradle/gradle-build-action@v2 # Addressing the Oracle container with "@localhost" in the JDBC url cause the JDBC connection to fail for an unknown reason. As a workaround, using instead the container ip address is working. with: - arguments: integrationTestJdbc -Dscalardb.jdbc.url=jdbc:oracle:thin:@${{ env.oracle_db_ip }}:1521/FREEPDB1 -Dscalardb.jdbc.username=SYSTEM -Dscalardb.jdbc.password=Oracle + arguments: integrationTestJdbc -Dscalardb.jdbc.url=jdbc:oracle:thin:@${{ env.oracle_db_ip }}:1521/FREEPDB1 -Dscalardb.jdbc.username=SYSTEM -Dscalardb.jdbc.password=Oracle -Dscalardb.jdbc.database.version.major=23 - name: Upload Gradle test reports if: always() diff --git a/core/src/integration-test/java/com/scalar/db/storage/jdbc/ConsensusCommitAdminImportTableIntegrationTestWithJdbcDatabase.java b/core/src/integration-test/java/com/scalar/db/storage/jdbc/ConsensusCommitAdminImportTableIntegrationTestWithJdbcDatabase.java index 2b3611e030..6a4cb5d7f3 100644 --- a/core/src/integration-test/java/com/scalar/db/storage/jdbc/ConsensusCommitAdminImportTableIntegrationTestWithJdbcDatabase.java +++ b/core/src/integration-test/java/com/scalar/db/storage/jdbc/ConsensusCommitAdminImportTableIntegrationTestWithJdbcDatabase.java @@ -25,7 +25,8 @@ protected Properties getProps(String testName) { @Override protected Map createExistingDatabaseWithAllDataTypes() throws SQLException { - return testUtils.createExistingDatabaseWithAllDataTypes(getNamespace()); + return testUtils.createExistingDatabaseWithAllDataTypes( + JdbcEnv.getJdbcDatabaseMajorVersion(), getNamespace()); } @Override diff --git a/core/src/integration-test/java/com/scalar/db/storage/jdbc/JdbcAdminImportTableIntegrationTest.java b/core/src/integration-test/java/com/scalar/db/storage/jdbc/JdbcAdminImportTableIntegrationTest.java index d554b48843..c1684340f5 100644 --- a/core/src/integration-test/java/com/scalar/db/storage/jdbc/JdbcAdminImportTableIntegrationTest.java +++ b/core/src/integration-test/java/com/scalar/db/storage/jdbc/JdbcAdminImportTableIntegrationTest.java @@ -25,7 +25,8 @@ protected Properties getProperties(String testName) { @Override protected Map createExistingDatabaseWithAllDataTypes() throws SQLException { - return testUtils.createExistingDatabaseWithAllDataTypes(getNamespace()); + return testUtils.createExistingDatabaseWithAllDataTypes( + JdbcEnv.getJdbcDatabaseMajorVersion(), getNamespace()); } @Override diff --git a/core/src/integration-test/java/com/scalar/db/storage/jdbc/JdbcAdminImportTestUtils.java b/core/src/integration-test/java/com/scalar/db/storage/jdbc/JdbcAdminImportTestUtils.java index eddcc1050f..e18eaad64e 100644 --- a/core/src/integration-test/java/com/scalar/db/storage/jdbc/JdbcAdminImportTestUtils.java +++ b/core/src/integration-test/java/com/scalar/db/storage/jdbc/JdbcAdminImportTestUtils.java @@ -15,9 +15,11 @@ import java.util.Map; import java.util.Properties; import java.util.stream.Collectors; +import java.util.stream.Stream; import org.apache.commons.dbcp2.BasicDataSource; public class JdbcAdminImportTestUtils { + static final String SUPPORTED_TABLE_NAME = "supported_table"; static final List UNSUPPORTED_DATA_TYPES_MYSQL = Arrays.asList( "BIGINT UNSIGNED", @@ -54,7 +56,6 @@ public class JdbcAdminImportTestUtils { "numeric(8,2)", "path", "pg_lsn", - "pg_snapshot", // after v14 "point", "polygon", "serial", @@ -68,6 +69,8 @@ public class JdbcAdminImportTestUtils { "txid_snapshot", "uuid", "xml"); + static final List UNSUPPORTED_DATA_TYPES_PGSQL_V13_OR_LATER = + Collections.singletonList("pg_snapshot"); static final List UNSUPPORTED_DATA_TYPES_ORACLE = Arrays.asList( "BFILE", @@ -76,13 +79,14 @@ public class JdbcAdminImportTestUtils { "INT", "INTERVAL YEAR(3) TO MONTH", "INTERVAL DAY(2) TO SECOND", - "JSON", "NUMBER(16,0)", "ROWID", "TIMESTAMP", "TIMESTAMP WITH TIME ZONE", "TIMESTAMP WITH LOCAL TIME ZONE", "UROWID"); + static final List UNSUPPORTED_DATA_TYPES_ORACLE_V20_OR_LATER = + Collections.singletonList("JSON"); static final List UNSUPPORTED_DATA_TYPES_MSSQL = Arrays.asList( "date", @@ -109,67 +113,19 @@ public JdbcAdminImportTestUtils(Properties properties) { rdbEngine = RdbEngineFactory.create(config); } - public Map createExistingDatabaseWithAllDataTypes(String namespace) - throws SQLException { - Map results = new HashMap<>(); - List sqls = new ArrayList<>(); - LinkedHashMap goodTableColumns; - TableMetadata goodTableMetadata; - Map badTables; + public Map createExistingDatabaseWithAllDataTypes( + int majorVersion, String namespace) throws SQLException { if (rdbEngine instanceof RdbEngineMysql) { - goodTableColumns = prepareColumnsForMysql(); - goodTableMetadata = prepareTableMetadataForMysql(); - if (JdbcEnv.isMariaDB()) { - badTables = - prepareCreateNonImportableTableSql( - namespace, - UNSUPPORTED_DATA_TYPES_MYSQL.stream() - .filter(type -> !type.equalsIgnoreCase("JSON")) - .collect(Collectors.toList())); - } else { - badTables = prepareCreateNonImportableTableSql(namespace, UNSUPPORTED_DATA_TYPES_MYSQL); - } + return createExistingMysqlDatabaseWithAllDataTypes(namespace); } else if (rdbEngine instanceof RdbEnginePostgresql) { - goodTableColumns = prepareColumnsForPostgresql(); - goodTableMetadata = prepareTableMetadataForPostgresql(); - badTables = prepareCreateNonImportableTableSql(namespace, UNSUPPORTED_DATA_TYPES_PGSQL); + return createExistingPostgresDatabaseWithAllDataTypes(majorVersion, namespace); } else if (rdbEngine instanceof RdbEngineOracle) { - goodTableColumns = prepareColumnsForOracle(); - goodTableMetadata = prepareTableMetadataForOracle(); - badTables = prepareCreateNonImportableTableSql(namespace, UNSUPPORTED_DATA_TYPES_ORACLE); - - // LONG columns must be tested with separated tables since they cannot be coexisted - TableMetadata longRawMetadata = prepareTableMetadataForOracleForLongRaw(); - sqls.add( - prepareCreateTableSql( - namespace, - "good_table_long_raw", - prepareColumnsForOracleLongRaw(), - longRawMetadata.getPartitionKeyNames())); - results.put("good_table_long_raw", longRawMetadata); + return createExistingOracleDatabaseWithAllDataTypes(majorVersion, namespace); } else if (rdbEngine instanceof RdbEngineSqlServer) { - goodTableColumns = prepareColumnsForSqlServer(); - goodTableMetadata = prepareTableMetadataForSqlServer(); - badTables = prepareCreateNonImportableTableSql(namespace, UNSUPPORTED_DATA_TYPES_MSSQL); + return createExistingSqlServerDatabaseWithAllDataTypes(namespace); } else { throw new RuntimeException(); } - - // table with all supported columns - sqls.add( - prepareCreateTableSql( - namespace, "good_table", goodTableColumns, goodTableMetadata.getPartitionKeyNames())); - results.put("good_table", goodTableMetadata); - - // tables with an unsupported column - badTables.forEach( - (table, sql) -> { - sqls.add(sql); - results.put(table, null); - }); - - execute(sqls.toArray(new String[0])); - return results; } public void dropTable(String namespace, String table) throws SQLException { @@ -435,4 +391,146 @@ private String prepareCreateTableSql( + primaryKeys.stream().map(rdbEngine::enclose).collect(Collectors.joining(",")) + "))"; } + + private Map createExistingMysqlDatabaseWithAllDataTypes(String namespace) + throws SQLException { + TableMetadata tableMetadata = prepareTableMetadataForMysql(); + Map supportedTables = + Collections.singletonMap( + SUPPORTED_TABLE_NAME, + prepareCreateTableSql( + namespace, + SUPPORTED_TABLE_NAME, + prepareColumnsForMysql(), + tableMetadata.getPartitionKeyNames())); + Map supportedTableMetadata = + Collections.singletonMap(SUPPORTED_TABLE_NAME, tableMetadata); + + Map unsupportedTables; + if (JdbcEnv.isMariaDB()) { + unsupportedTables = + prepareCreateNonImportableTableSql( + namespace, + UNSUPPORTED_DATA_TYPES_MYSQL.stream() + .filter(type -> !type.equalsIgnoreCase("JSON")) + .collect(Collectors.toList())); + } else { + unsupportedTables = + prepareCreateNonImportableTableSql(namespace, UNSUPPORTED_DATA_TYPES_MYSQL); + } + + return executeCreateTableSql(supportedTables, supportedTableMetadata, unsupportedTables); + } + + private Map createExistingPostgresDatabaseWithAllDataTypes( + int majorVersion, String namespace) throws SQLException { + TableMetadata tableMetadata = prepareTableMetadataForPostgresql(); + Map supportedTables = + Collections.singletonMap( + SUPPORTED_TABLE_NAME, + prepareCreateTableSql( + namespace, + SUPPORTED_TABLE_NAME, + prepareColumnsForPostgresql(), + tableMetadata.getPartitionKeyNames())); + Map supportedTableMetadata = + Collections.singletonMap(SUPPORTED_TABLE_NAME, tableMetadata); + + Map unsupportedTables = + prepareCreateNonImportableTableSql( + namespace, + majorVersion >= 14 + ? Stream.concat( + UNSUPPORTED_DATA_TYPES_PGSQL.stream(), + UNSUPPORTED_DATA_TYPES_PGSQL_V13_OR_LATER.stream()) + .collect(Collectors.toList()) + : UNSUPPORTED_DATA_TYPES_PGSQL); + + return executeCreateTableSql(supportedTables, supportedTableMetadata, unsupportedTables); + } + + private Map createExistingOracleDatabaseWithAllDataTypes( + int majorVersion, String namespace) throws SQLException { + Map supportedTables = new HashMap<>(); + Map supportedTableMetadata = new HashMap<>(); + + TableMetadata tableMetadata = prepareTableMetadataForOracle(); + supportedTables.put( + SUPPORTED_TABLE_NAME, + prepareCreateTableSql( + namespace, + SUPPORTED_TABLE_NAME, + prepareColumnsForOracle(), + tableMetadata.getPartitionKeyNames())); + supportedTableMetadata.put(SUPPORTED_TABLE_NAME, tableMetadata); + + // LONG columns must be tested with separated tables since they cannot be coexisted + TableMetadata longRawTableMetadata = prepareTableMetadataForOracleForLongRaw(); + supportedTables.put( + SUPPORTED_TABLE_NAME + "_long_raw", + prepareCreateTableSql( + namespace, + SUPPORTED_TABLE_NAME + "_long_raw", + prepareColumnsForOracleLongRaw(), + longRawTableMetadata.getPartitionKeyNames())); + supportedTableMetadata.put(SUPPORTED_TABLE_NAME + "_long_raw", longRawTableMetadata); + + Map unsupportedTables = + prepareCreateNonImportableTableSql( + namespace, + majorVersion >= 20 + ? Stream.concat( + UNSUPPORTED_DATA_TYPES_ORACLE.stream(), + UNSUPPORTED_DATA_TYPES_ORACLE_V20_OR_LATER.stream()) + .collect(Collectors.toList()) + : UNSUPPORTED_DATA_TYPES_ORACLE); + + return executeCreateTableSql(supportedTables, supportedTableMetadata, unsupportedTables); + } + + private Map createExistingSqlServerDatabaseWithAllDataTypes( + String namespace) throws SQLException { + TableMetadata tableMetadata = prepareTableMetadataForSqlServer(); + Map supportedTables = + Collections.singletonMap( + SUPPORTED_TABLE_NAME, + prepareCreateTableSql( + namespace, + SUPPORTED_TABLE_NAME, + prepareColumnsForSqlServer(), + tableMetadata.getPartitionKeyNames())); + Map supportedTableMetadata = + Collections.singletonMap(SUPPORTED_TABLE_NAME, tableMetadata); + + Map unsupportedTables = + prepareCreateNonImportableTableSql(namespace, UNSUPPORTED_DATA_TYPES_MSSQL); + + return executeCreateTableSql(supportedTables, supportedTableMetadata, unsupportedTables); + } + + private Map executeCreateTableSql( + Map supportedTables, + Map supportedTableMetadata, + Map unsupportedTables) + throws SQLException { + Map results = new HashMap<>(); + List sqls = new ArrayList<>(); + + // table with all supported columns + supportedTables.forEach( + (table, sql) -> { + sqls.add(sql); + results.put(table, supportedTableMetadata.get(table)); + }); + + // tables with an unsupported column + unsupportedTables.forEach( + (table, sql) -> { + sqls.add(sql); + results.put(table, null); + }); + + execute(sqls.toArray(new String[0])); + return results; + } } diff --git a/core/src/integration-test/java/com/scalar/db/storage/jdbc/JdbcEnv.java b/core/src/integration-test/java/com/scalar/db/storage/jdbc/JdbcEnv.java index 38b2103837..ea627a29c9 100644 --- a/core/src/integration-test/java/com/scalar/db/storage/jdbc/JdbcEnv.java +++ b/core/src/integration-test/java/com/scalar/db/storage/jdbc/JdbcEnv.java @@ -8,6 +8,7 @@ public final class JdbcEnv { private static final String PROP_JDBC_USERNAME = "scalardb.jdbc.username"; private static final String PROP_JDBC_PASSWORD = "scalardb.jdbc.password"; private static final String PROP_JDBC_MARIADB = "scalardb.jdbc.mariadb"; + private static final String PROP_JDBC_DB_MAJOR_VERSION = "scalardb.jdbc.database.version.major"; private static final String DEFAULT_JDBC_URL = "jdbc:mysql://localhost:3306/"; private static final String DEFAULT_JDBC_USERNAME = "root"; @@ -43,4 +44,8 @@ public static boolean isSqlite() { public static boolean isMariaDB() { return Boolean.getBoolean(PROP_JDBC_MARIADB); } + + public static int getJdbcDatabaseMajorVersion() { + return Integer.getInteger(PROP_JDBC_DB_MAJOR_VERSION, Integer.MAX_VALUE); + } } diff --git a/core/src/integration-test/java/com/scalar/db/transaction/jdbc/JdbcTransactionAdminImportTableIntegrationTest.java b/core/src/integration-test/java/com/scalar/db/transaction/jdbc/JdbcTransactionAdminImportTableIntegrationTest.java index b892917f63..fcb1e989d7 100644 --- a/core/src/integration-test/java/com/scalar/db/transaction/jdbc/JdbcTransactionAdminImportTableIntegrationTest.java +++ b/core/src/integration-test/java/com/scalar/db/transaction/jdbc/JdbcTransactionAdminImportTableIntegrationTest.java @@ -30,7 +30,8 @@ protected Properties getProperties(String testName) { @Override protected Map createExistingDatabaseWithAllDataTypes() throws SQLException { - return testUtils.createExistingDatabaseWithAllDataTypes(getNamespace()); + return testUtils.createExistingDatabaseWithAllDataTypes( + JdbcEnv.getJdbcDatabaseMajorVersion(), getNamespace()); } @Override