From 465662e38d197713877fcc2cef295a9f8b3b3827 Mon Sep 17 00:00:00 2001 From: Sebastian Nehls Date: Mon, 28 Jan 2019 15:24:54 +0100 Subject: [PATCH 01/12] convert postgres identifiers to lowercase always --- .../com/exasol/adapter/dialects/impl/PostgreSQLSqlDialect.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jdbc-adapter/virtualschema-jdbc-adapter/src/main/java/com/exasol/adapter/dialects/impl/PostgreSQLSqlDialect.java b/jdbc-adapter/virtualschema-jdbc-adapter/src/main/java/com/exasol/adapter/dialects/impl/PostgreSQLSqlDialect.java index 75adba8d9..5757b6fea 100644 --- a/jdbc-adapter/virtualschema-jdbc-adapter/src/main/java/com/exasol/adapter/dialects/impl/PostgreSQLSqlDialect.java +++ b/jdbc-adapter/virtualschema-jdbc-adapter/src/main/java/com/exasol/adapter/dialects/impl/PostgreSQLSqlDialect.java @@ -345,7 +345,7 @@ public IdentifierCaseHandling getQuotedIdentifierHandling() { @Override public String applyQuote(final String identifier) { - return "\"" + identifier.replace("\"", "\"\"") + "\""; + return "\"" + identifier.toLowerCase().replace("\"", "\"\"") + "\""; } @Override From 677148ca6e4dd9c0173456ce4d20dc2bacc3420a Mon Sep 17 00:00:00 2001 From: Sebastian Nehls Date: Tue, 29 Jan 2019 15:07:34 +0100 Subject: [PATCH 02/12] create virtual schema throws an error message if postgres schema has table containing uppercase characters --- .../dialects/impl/PostgreSQLSqlDialect.java | 14 +++++ .../impl/PostgreSQLSqlDialectTest.java | 53 +++++++++++++++++++ 2 files changed, 67 insertions(+) create mode 100644 jdbc-adapter/virtualschema-jdbc-adapter/src/test/java/com/exasol/adapter/dialects/impl/PostgreSQLSqlDialectTest.java diff --git a/jdbc-adapter/virtualschema-jdbc-adapter/src/main/java/com/exasol/adapter/dialects/impl/PostgreSQLSqlDialect.java b/jdbc-adapter/virtualschema-jdbc-adapter/src/main/java/com/exasol/adapter/dialects/impl/PostgreSQLSqlDialect.java index 5757b6fea..d43111982 100644 --- a/jdbc-adapter/virtualschema-jdbc-adapter/src/main/java/com/exasol/adapter/dialects/impl/PostgreSQLSqlDialect.java +++ b/jdbc-adapter/virtualschema-jdbc-adapter/src/main/java/com/exasol/adapter/dialects/impl/PostgreSQLSqlDialect.java @@ -1,10 +1,12 @@ package com.exasol.adapter.dialects.impl; +import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Types; import java.util.EnumMap; import java.util.Map; +import com.exasol.adapter.AdapterException; import com.exasol.adapter.capabilities.AggregateFunctionCapability; import com.exasol.adapter.capabilities.Capabilities; import com.exasol.adapter.capabilities.LiteralCapability; @@ -298,6 +300,18 @@ public DataType dialectSpecificMapJdbcType(final JdbcTypeDescription jdbcTypeDes return colType; } + @Override + public MappedTable mapTable(final ResultSet tables) throws SQLException { + final String tableName = tables.getString("TABLE_NAME"); + if (!tableName.equals(tableName.toLowerCase())) { + //TODO: think about a good error message + throw new IllegalArgumentException("Table " + tableName + " cannot be used in virtual schema. " + + "Use property POSTGRES_IGNORE_UPPERCASE_TABLES to enforce schema creation."); + } else { + return super.mapTable(tables); + } + } + @Override public Map getScalarFunctionAliases() { diff --git a/jdbc-adapter/virtualschema-jdbc-adapter/src/test/java/com/exasol/adapter/dialects/impl/PostgreSQLSqlDialectTest.java b/jdbc-adapter/virtualschema-jdbc-adapter/src/test/java/com/exasol/adapter/dialects/impl/PostgreSQLSqlDialectTest.java new file mode 100644 index 000000000..3faa2e508 --- /dev/null +++ b/jdbc-adapter/virtualschema-jdbc-adapter/src/test/java/com/exasol/adapter/dialects/impl/PostgreSQLSqlDialectTest.java @@ -0,0 +1,53 @@ +package com.exasol.adapter.dialects.impl; + +import com.exasol.adapter.dialects.SqlDialectContext; +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mock; +import static org.mockito.Mockito.*; + +import java.sql.ResultSet; +import java.sql.SQLException; + +import static org.junit.Assert.*; +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.when; + +public class PostgreSQLSqlDialectTest { + @Mock + SqlDialectContext sqlDialectContext; + + + @Before + public void setUp() throws SQLException { + + } + + @Test + public void applyQuoteOnUpperCase() { + PostgreSQLSqlDialect postgresDialect = new PostgreSQLSqlDialect(sqlDialectContext); + assertEquals("\"abc\"", postgresDialect.applyQuote("ABC")); + } + + @Test + public void applyQuoteOnMixedCase() { + PostgreSQLSqlDialect postgresDialect = new PostgreSQLSqlDialect(sqlDialectContext); + assertEquals("\"abcde\"", postgresDialect.applyQuote("AbCde")); + } + + @Test(expected = IllegalArgumentException.class) + public void mapTableWithUpperCaseCharacters() throws SQLException { + ResultSet resultSet = mock(ResultSet.class); + when(resultSet.getString("TABLE_NAME")).thenReturn("uPPer"); + PostgreSQLSqlDialect postgresDialect = new PostgreSQLSqlDialect(sqlDialectContext); + postgresDialect.mapTable(resultSet); + } + + @Test + public void mapTableWithLowerCaseCharacters() throws SQLException { + ResultSet resultSet = mock(ResultSet.class); + when(resultSet.getString("TABLE_NAME")).thenReturn("lower"); + PostgreSQLSqlDialect postgresDialect = new PostgreSQLSqlDialect(sqlDialectContext); + postgresDialect.mapTable(resultSet); + } +} \ No newline at end of file From a4742605b27071a8d8129bb08222516434a5580d Mon Sep 17 00:00:00 2001 From: Sebastian Nehls Date: Tue, 29 Jan 2019 15:54:58 +0100 Subject: [PATCH 03/12] added ignore_error_list property, implemented it for POSTGRES_IGNORE_UPPERCASE_TABLES --- .../adapter/dialects/AbstractSqlDialect.java | 2 +- .../com/exasol/adapter/dialects/SqlDialect.java | 3 ++- .../adapter/dialects/impl/OracleSqlDialect.java | 4 ++-- .../dialects/impl/PostgreSQLSqlDialect.java | 14 ++++++++++---- .../java/com/exasol/adapter/jdbc/JdbcAdapter.java | 6 ++++-- .../exasol/adapter/jdbc/JdbcAdapterProperties.java | 5 +++++ .../exasol/adapter/jdbc/JdbcMetadataReader.java | 10 +++++----- .../adapter/dialects/impl/ExasolSqlDialectIT.java | 2 +- .../dialects/impl/PostgreSQLSqlDialectTest.java | 4 ++-- 9 files changed, 32 insertions(+), 18 deletions(-) diff --git a/jdbc-adapter/virtualschema-jdbc-adapter/src/main/java/com/exasol/adapter/dialects/AbstractSqlDialect.java b/jdbc-adapter/virtualschema-jdbc-adapter/src/main/java/com/exasol/adapter/dialects/AbstractSqlDialect.java index 8c28c94cf..ab9657601 100644 --- a/jdbc-adapter/virtualschema-jdbc-adapter/src/main/java/com/exasol/adapter/dialects/AbstractSqlDialect.java +++ b/jdbc-adapter/virtualschema-jdbc-adapter/src/main/java/com/exasol/adapter/dialects/AbstractSqlDialect.java @@ -40,7 +40,7 @@ public String getTableCatalogAndSchemaSeparator() { } @Override - public MappedTable mapTable(final ResultSet tables) throws SQLException { + public MappedTable mapTable(final ResultSet tables, String ignoreErrorList) throws SQLException { String commentString = tables.getString("REMARKS"); if (commentString == null) { commentString = ""; diff --git a/jdbc-adapter/virtualschema-jdbc-adapter/src/main/java/com/exasol/adapter/dialects/SqlDialect.java b/jdbc-adapter/virtualschema-jdbc-adapter/src/main/java/com/exasol/adapter/dialects/SqlDialect.java index c6394ae1f..e50678d1d 100644 --- a/jdbc-adapter/virtualschema-jdbc-adapter/src/main/java/com/exasol/adapter/dialects/SqlDialect.java +++ b/jdbc-adapter/virtualschema-jdbc-adapter/src/main/java/com/exasol/adapter/dialects/SqlDialect.java @@ -153,9 +153,10 @@ public String getTableComment() { * @param tables A jdbc Resultset for the * {@link DatabaseMetaData#getTables(String, String, String, String[])} * call, pointing to the current table. + * @param ignoreErrorList * @return An instance of {@link MappedTable} describing the mapped table. */ - public MappedTable mapTable(ResultSet tables) throws SQLException; + public MappedTable mapTable(ResultSet tables, String ignoreErrorList) throws SQLException; /** * @param columns A jdbc Resultset for the diff --git a/jdbc-adapter/virtualschema-jdbc-adapter/src/main/java/com/exasol/adapter/dialects/impl/OracleSqlDialect.java b/jdbc-adapter/virtualschema-jdbc-adapter/src/main/java/com/exasol/adapter/dialects/impl/OracleSqlDialect.java index 8d0ff6ae5..4e99fa1c2 100644 --- a/jdbc-adapter/virtualschema-jdbc-adapter/src/main/java/com/exasol/adapter/dialects/impl/OracleSqlDialect.java +++ b/jdbc-adapter/virtualschema-jdbc-adapter/src/main/java/com/exasol/adapter/dialects/impl/OracleSqlDialect.java @@ -331,7 +331,7 @@ public SchemaOrCatalogSupport supportsJdbcSchemas() { } @Override - public MappedTable mapTable(final ResultSet tables) throws SQLException { + public MappedTable mapTable(final ResultSet tables, String ignoreErrorList) throws SQLException { final String tableName = tables.getString("TABLE_NAME"); if (tableName.startsWith("BIN$")) { // In case of Oracle we may see deleted tables with strange names @@ -341,7 +341,7 @@ public MappedTable mapTable(final ResultSet tables) throws SQLException { System.out.println("Skip table: " + tableName); return MappedTable.createIgnoredTable(); } else { - return super.mapTable(tables); + return super.mapTable(tables, ignoreErrorList); } } diff --git a/jdbc-adapter/virtualschema-jdbc-adapter/src/main/java/com/exasol/adapter/dialects/impl/PostgreSQLSqlDialect.java b/jdbc-adapter/virtualschema-jdbc-adapter/src/main/java/com/exasol/adapter/dialects/impl/PostgreSQLSqlDialect.java index d43111982..de80b32e6 100644 --- a/jdbc-adapter/virtualschema-jdbc-adapter/src/main/java/com/exasol/adapter/dialects/impl/PostgreSQLSqlDialect.java +++ b/jdbc-adapter/virtualschema-jdbc-adapter/src/main/java/com/exasol/adapter/dialects/impl/PostgreSQLSqlDialect.java @@ -6,7 +6,6 @@ import java.util.EnumMap; import java.util.Map; -import com.exasol.adapter.AdapterException; import com.exasol.adapter.capabilities.AggregateFunctionCapability; import com.exasol.adapter.capabilities.Capabilities; import com.exasol.adapter.capabilities.LiteralCapability; @@ -300,15 +299,22 @@ public DataType dialectSpecificMapJdbcType(final JdbcTypeDescription jdbcTypeDes return colType; } + //TODO: write unittest for ignoreerrorlist + //TODO: make return type of ignoreerrorlist an actial List + //TODO: write integrationtest @Override - public MappedTable mapTable(final ResultSet tables) throws SQLException { + public MappedTable mapTable(final ResultSet tables, String ignoreErrorList) throws SQLException { final String tableName = tables.getString("TABLE_NAME"); + //TODO: remove magic value + if (ignoreErrorList.equals("POSTGRES_IGNORE_UPPERCASE_TABLES")) { + return super.mapTable(tables, ignoreErrorList); + } if (!tableName.equals(tableName.toLowerCase())) { //TODO: think about a good error message throw new IllegalArgumentException("Table " + tableName + " cannot be used in virtual schema. " + - "Use property POSTGRES_IGNORE_UPPERCASE_TABLES to enforce schema creation."); + "Set property IGNORE_ERROR_LIST to POSTGRES_IGNORE_UPPERCASE_TABLES to enforce schema creation."); } else { - return super.mapTable(tables); + return super.mapTable(tables, ignoreErrorList); } } diff --git a/jdbc-adapter/virtualschema-jdbc-adapter/src/main/java/com/exasol/adapter/jdbc/JdbcAdapter.java b/jdbc-adapter/virtualschema-jdbc-adapter/src/main/java/com/exasol/adapter/jdbc/JdbcAdapter.java index 42b956ff9..b4e6389f8 100644 --- a/jdbc-adapter/virtualschema-jdbc-adapter/src/main/java/com/exasol/adapter/jdbc/JdbcAdapter.java +++ b/jdbc-adapter/virtualschema-jdbc-adapter/src/main/java/com/exasol/adapter/jdbc/JdbcAdapter.java @@ -135,7 +135,8 @@ private static SchemaMetadata readMetadata(final SchemaMetadataInfo meta, final return JdbcMetadataReader.readRemoteMetadata(connection.getAddress(), connection.getUser(), connection.getPassword(), catalog, schema, tables, JdbcAdapterProperties.getSqlDialectName(meta.getProperties()), - JdbcAdapterProperties.getExceptionHandlingMode(meta.getProperties())); + JdbcAdapterProperties.getExceptionHandlingMode(meta.getProperties()), + JdbcAdapterProperties.getIgnoreErrorList(meta.getProperties())); } private static String handleRefresh(final RefreshRequest request, final ExaMetadata meta) @@ -165,7 +166,8 @@ private static String handleSetProperty(final SetPropertiesRequest request, fina connection.getUser(), connection.getPassword(), JdbcAdapterProperties.getCatalog(newSchemaMeta), JdbcAdapterProperties.getSchema(newSchemaMeta), tableFilter, JdbcAdapterProperties.getSqlDialectName(newSchemaMeta), - JdbcAdapterProperties.getExceptionHandlingMode(newSchemaMeta)); + JdbcAdapterProperties.getExceptionHandlingMode(newSchemaMeta), + JdbcAdapterProperties.getIgnoreErrorList(newSchemaMeta)); return ResponseJsonSerializer.makeSetPropertiesResponse(remoteMeta); } return ResponseJsonSerializer.makeSetPropertiesResponse(null); diff --git a/jdbc-adapter/virtualschema-jdbc-adapter/src/main/java/com/exasol/adapter/jdbc/JdbcAdapterProperties.java b/jdbc-adapter/virtualschema-jdbc-adapter/src/main/java/com/exasol/adapter/jdbc/JdbcAdapterProperties.java index f73cd97c1..bb9c988c1 100644 --- a/jdbc-adapter/virtualschema-jdbc-adapter/src/main/java/com/exasol/adapter/jdbc/JdbcAdapterProperties.java +++ b/jdbc-adapter/virtualschema-jdbc-adapter/src/main/java/com/exasol/adapter/jdbc/JdbcAdapterProperties.java @@ -43,6 +43,7 @@ public final class JdbcAdapterProperties { static final String PROP_EXCLUDED_CAPABILITIES = "EXCLUDED_CAPABILITIES"; static final String PROP_EXCEPTION_HANDLING = "EXCEPTION_HANDLING"; static final String PROP_LOG_LEVEL = "LOG_LEVEL"; + static final String PROP_IGNORE_ERROR_LIST = "IGNORE_ERROR_LIST"; private static final String DEFAULT_LOG_LEVEL = "INFO"; @@ -64,6 +65,10 @@ private static String getProperty(final Map properties, final St } } + public static String getIgnoreErrorList(final Map properties) { + return getProperty(properties, PROP_IGNORE_ERROR_LIST, ""); + } + public static String getCatalog(final Map properties) { return getProperty(properties, PROP_CATALOG_NAME, ""); } diff --git a/jdbc-adapter/virtualschema-jdbc-adapter/src/main/java/com/exasol/adapter/jdbc/JdbcMetadataReader.java b/jdbc-adapter/virtualschema-jdbc-adapter/src/main/java/com/exasol/adapter/jdbc/JdbcMetadataReader.java index a8acf9167..9d57b0dbe 100644 --- a/jdbc-adapter/virtualschema-jdbc-adapter/src/main/java/com/exasol/adapter/jdbc/JdbcMetadataReader.java +++ b/jdbc-adapter/virtualschema-jdbc-adapter/src/main/java/com/exasol/adapter/jdbc/JdbcMetadataReader.java @@ -20,7 +20,7 @@ public class JdbcMetadataReader { public static SchemaMetadata readRemoteMetadata(final String connectionString, final String user, final String password, String catalog, String schema, final List tableFilter, - final String dialectName, final JdbcAdapterProperties.ExceptionHandlingMode exceptionMode) + final String dialectName, final JdbcAdapterProperties.ExceptionHandlingMode exceptionMode, final String ignoreErrorList) throws SQLException, AdapterException { assert (catalog != null); assert (schema != null); @@ -45,7 +45,7 @@ public static SchemaMetadata readRemoteMetadata(final String connectionString, f schema = findSchema(schema, dbMeta, dialect); - final List tables = findTables(catalog, schema, tableFilter, dbMeta, dialect, exceptionMode); + final List tables = findTables(catalog, schema, tableFilter, dbMeta, dialect, exceptionMode, ignoreErrorList); conn.close(); return new SchemaMetadata(SchemaAdapterNotes.serialize(schemaAdapterNotes), tables); @@ -242,8 +242,8 @@ private static String findSchema(final String schema, final DatabaseMetaData dbM } private static List findTables(final String catalog, final String schema, - final List tableFilter, final DatabaseMetaData dbMeta, final SqlDialect dialect, - final JdbcAdapterProperties.ExceptionHandlingMode exceptionMode) throws SQLException { + final List tableFilter, final DatabaseMetaData dbMeta, final SqlDialect dialect, + final JdbcAdapterProperties.ExceptionHandlingMode exceptionMode, String ignoreErrorList) throws SQLException { final List tables = new ArrayList<>(); final String[] supportedTableTypes = { "TABLE", "VIEW", "SYSTEM TABLE" }; @@ -252,7 +252,7 @@ private static List findTables(final String catalog, final String final List tablesMapped = new ArrayList<>(); // List tableComments = new ArrayList<>(); while (resTables.next()) { - final SqlDialect.MappedTable mappedTable = dialect.mapTable(resTables); + final SqlDialect.MappedTable mappedTable = dialect.mapTable(resTables, ignoreErrorList); if (!mappedTable.isIgnored()) { tablesMapped.add(mappedTable); // tableComments.add(mappedTable.getTableComment()); diff --git a/jdbc-adapter/virtualschema-jdbc-adapter/src/test/java/com/exasol/adapter/dialects/impl/ExasolSqlDialectIT.java b/jdbc-adapter/virtualschema-jdbc-adapter/src/test/java/com/exasol/adapter/dialects/impl/ExasolSqlDialectIT.java index fbffbcefd..5d36adf11 100644 --- a/jdbc-adapter/virtualschema-jdbc-adapter/src/test/java/com/exasol/adapter/dialects/impl/ExasolSqlDialectIT.java +++ b/jdbc-adapter/virtualschema-jdbc-adapter/src/test/java/com/exasol/adapter/dialects/impl/ExasolSqlDialectIT.java @@ -308,7 +308,7 @@ public void testDifferentDataTypes() final List tables = new ArrayList<>(Arrays.asList(tableNames)); final SchemaMetadata meta = JdbcMetadataReader.readRemoteMetadata("jdbc:exa:" + getConfig().getExasolAddress(), getConfig().getExasolUser(), getConfig().getExasolPassword(), "EXA_DB", "JDBC_ADAPTER_TEST_SCHEMA", - tables, ExasolSqlDialect.getPublicName(), getConfig().getExceptionHandlingMode()); + tables, ExasolSqlDialect.getPublicName(), getConfig().getExceptionHandlingMode(), ""); if (getConfig().isDebugOn()) { System.out.println("Meta: " + SchemaMetadataSerializer.serialize(meta).build().toString()); } diff --git a/jdbc-adapter/virtualschema-jdbc-adapter/src/test/java/com/exasol/adapter/dialects/impl/PostgreSQLSqlDialectTest.java b/jdbc-adapter/virtualschema-jdbc-adapter/src/test/java/com/exasol/adapter/dialects/impl/PostgreSQLSqlDialectTest.java index 3faa2e508..53c69ee11 100644 --- a/jdbc-adapter/virtualschema-jdbc-adapter/src/test/java/com/exasol/adapter/dialects/impl/PostgreSQLSqlDialectTest.java +++ b/jdbc-adapter/virtualschema-jdbc-adapter/src/test/java/com/exasol/adapter/dialects/impl/PostgreSQLSqlDialectTest.java @@ -40,7 +40,7 @@ public void mapTableWithUpperCaseCharacters() throws SQLException { ResultSet resultSet = mock(ResultSet.class); when(resultSet.getString("TABLE_NAME")).thenReturn("uPPer"); PostgreSQLSqlDialect postgresDialect = new PostgreSQLSqlDialect(sqlDialectContext); - postgresDialect.mapTable(resultSet); + postgresDialect.mapTable(resultSet, ""); } @Test @@ -48,6 +48,6 @@ public void mapTableWithLowerCaseCharacters() throws SQLException { ResultSet resultSet = mock(ResultSet.class); when(resultSet.getString("TABLE_NAME")).thenReturn("lower"); PostgreSQLSqlDialect postgresDialect = new PostgreSQLSqlDialect(sqlDialectContext); - postgresDialect.mapTable(resultSet); + postgresDialect.mapTable(resultSet, ""); } } \ No newline at end of file From 4e752356b200ffb4523493b63c7de579f35b1061 Mon Sep 17 00:00:00 2001 From: Sebastian Nehls Date: Wed, 30 Jan 2019 10:14:58 +0100 Subject: [PATCH 04/12] #74: changed ignore_error_list from String to List --- .../com/exasol/adapter/dialects/AbstractSqlDialect.java | 8 ++------ .../main/java/com/exasol/adapter/dialects/SqlDialect.java | 3 ++- .../exasol/adapter/dialects/impl/OracleSqlDialect.java | 3 ++- .../adapter/dialects/impl/PostgreSQLSqlDialect.java | 3 ++- .../com/exasol/adapter/jdbc/JdbcAdapterProperties.java | 7 +++++-- .../java/com/exasol/adapter/jdbc/JdbcMetadataReader.java | 4 ++-- .../exasol/adapter/dialects/impl/ExasolSqlDialectIT.java | 3 ++- .../adapter/dialects/impl/PostgreSQLSqlDialectTest.java | 5 +++-- 8 files changed, 20 insertions(+), 16 deletions(-) diff --git a/jdbc-adapter/virtualschema-jdbc-adapter/src/main/java/com/exasol/adapter/dialects/AbstractSqlDialect.java b/jdbc-adapter/virtualschema-jdbc-adapter/src/main/java/com/exasol/adapter/dialects/AbstractSqlDialect.java index ab9657601..5c600f9c7 100644 --- a/jdbc-adapter/virtualschema-jdbc-adapter/src/main/java/com/exasol/adapter/dialects/AbstractSqlDialect.java +++ b/jdbc-adapter/virtualschema-jdbc-adapter/src/main/java/com/exasol/adapter/dialects/AbstractSqlDialect.java @@ -3,11 +3,7 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Types; -import java.util.EnumMap; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Map; -import java.util.Set; +import java.util.*; import com.exasol.adapter.jdbc.ColumnAdapterNotes; import com.exasol.adapter.jdbc.JdbcAdapterProperties; @@ -40,7 +36,7 @@ public String getTableCatalogAndSchemaSeparator() { } @Override - public MappedTable mapTable(final ResultSet tables, String ignoreErrorList) throws SQLException { + public MappedTable mapTable(final ResultSet tables, List ignoreErrorList) throws SQLException { String commentString = tables.getString("REMARKS"); if (commentString == null) { commentString = ""; diff --git a/jdbc-adapter/virtualschema-jdbc-adapter/src/main/java/com/exasol/adapter/dialects/SqlDialect.java b/jdbc-adapter/virtualschema-jdbc-adapter/src/main/java/com/exasol/adapter/dialects/SqlDialect.java index e50678d1d..ebcd7a84c 100644 --- a/jdbc-adapter/virtualschema-jdbc-adapter/src/main/java/com/exasol/adapter/dialects/SqlDialect.java +++ b/jdbc-adapter/virtualschema-jdbc-adapter/src/main/java/com/exasol/adapter/dialects/SqlDialect.java @@ -3,6 +3,7 @@ import java.sql.DatabaseMetaData; import java.sql.ResultSet; import java.sql.SQLException; +import java.util.List; import java.util.Map; import com.exasol.adapter.capabilities.Capabilities; @@ -156,7 +157,7 @@ public String getTableComment() { * @param ignoreErrorList * @return An instance of {@link MappedTable} describing the mapped table. */ - public MappedTable mapTable(ResultSet tables, String ignoreErrorList) throws SQLException; + public MappedTable mapTable(ResultSet tables, List ignoreErrorList) throws SQLException; /** * @param columns A jdbc Resultset for the diff --git a/jdbc-adapter/virtualschema-jdbc-adapter/src/main/java/com/exasol/adapter/dialects/impl/OracleSqlDialect.java b/jdbc-adapter/virtualschema-jdbc-adapter/src/main/java/com/exasol/adapter/dialects/impl/OracleSqlDialect.java index 4e99fa1c2..e20e375e7 100644 --- a/jdbc-adapter/virtualschema-jdbc-adapter/src/main/java/com/exasol/adapter/dialects/impl/OracleSqlDialect.java +++ b/jdbc-adapter/virtualschema-jdbc-adapter/src/main/java/com/exasol/adapter/dialects/impl/OracleSqlDialect.java @@ -4,6 +4,7 @@ import java.sql.SQLException; import java.sql.Types; import java.util.EnumMap; +import java.util.List; import java.util.Map; import com.exasol.adapter.capabilities.AggregateFunctionCapability; @@ -331,7 +332,7 @@ public SchemaOrCatalogSupport supportsJdbcSchemas() { } @Override - public MappedTable mapTable(final ResultSet tables, String ignoreErrorList) throws SQLException { + public MappedTable mapTable(final ResultSet tables, List ignoreErrorList) throws SQLException { final String tableName = tables.getString("TABLE_NAME"); if (tableName.startsWith("BIN$")) { // In case of Oracle we may see deleted tables with strange names diff --git a/jdbc-adapter/virtualschema-jdbc-adapter/src/main/java/com/exasol/adapter/dialects/impl/PostgreSQLSqlDialect.java b/jdbc-adapter/virtualschema-jdbc-adapter/src/main/java/com/exasol/adapter/dialects/impl/PostgreSQLSqlDialect.java index de80b32e6..e41f705e7 100644 --- a/jdbc-adapter/virtualschema-jdbc-adapter/src/main/java/com/exasol/adapter/dialects/impl/PostgreSQLSqlDialect.java +++ b/jdbc-adapter/virtualschema-jdbc-adapter/src/main/java/com/exasol/adapter/dialects/impl/PostgreSQLSqlDialect.java @@ -4,6 +4,7 @@ import java.sql.SQLException; import java.sql.Types; import java.util.EnumMap; +import java.util.List; import java.util.Map; import com.exasol.adapter.capabilities.AggregateFunctionCapability; @@ -303,7 +304,7 @@ public DataType dialectSpecificMapJdbcType(final JdbcTypeDescription jdbcTypeDes //TODO: make return type of ignoreerrorlist an actial List //TODO: write integrationtest @Override - public MappedTable mapTable(final ResultSet tables, String ignoreErrorList) throws SQLException { + public MappedTable mapTable(final ResultSet tables, List ignoreErrorList) throws SQLException { final String tableName = tables.getString("TABLE_NAME"); //TODO: remove magic value if (ignoreErrorList.equals("POSTGRES_IGNORE_UPPERCASE_TABLES")) { diff --git a/jdbc-adapter/virtualschema-jdbc-adapter/src/main/java/com/exasol/adapter/jdbc/JdbcAdapterProperties.java b/jdbc-adapter/virtualschema-jdbc-adapter/src/main/java/com/exasol/adapter/jdbc/JdbcAdapterProperties.java index bb9c988c1..4d31e28a9 100644 --- a/jdbc-adapter/virtualschema-jdbc-adapter/src/main/java/com/exasol/adapter/jdbc/JdbcAdapterProperties.java +++ b/jdbc-adapter/virtualschema-jdbc-adapter/src/main/java/com/exasol/adapter/jdbc/JdbcAdapterProperties.java @@ -7,6 +7,7 @@ import java.util.Map; import java.util.logging.Level; import java.util.logging.Logger; +import java.util.stream.Collectors; import com.exasol.ExaConnectionAccessException; import com.exasol.ExaConnectionInformation; @@ -65,8 +66,10 @@ private static String getProperty(final Map properties, final St } } - public static String getIgnoreErrorList(final Map properties) { - return getProperty(properties, PROP_IGNORE_ERROR_LIST, ""); + public static List getIgnoreErrorList(final Map properties) { + String ignoreErrors = getProperty(properties, PROP_IGNORE_ERROR_LIST, ""); + List ignoreErrorsList = Arrays.asList(ignoreErrors.split(",")); + return ignoreErrorsList.stream().map(error -> error.trim()).collect(Collectors.toList()); } public static String getCatalog(final Map properties) { diff --git a/jdbc-adapter/virtualschema-jdbc-adapter/src/main/java/com/exasol/adapter/jdbc/JdbcMetadataReader.java b/jdbc-adapter/virtualschema-jdbc-adapter/src/main/java/com/exasol/adapter/jdbc/JdbcMetadataReader.java index 9d57b0dbe..cdb690de4 100644 --- a/jdbc-adapter/virtualschema-jdbc-adapter/src/main/java/com/exasol/adapter/jdbc/JdbcMetadataReader.java +++ b/jdbc-adapter/virtualschema-jdbc-adapter/src/main/java/com/exasol/adapter/jdbc/JdbcMetadataReader.java @@ -20,7 +20,7 @@ public class JdbcMetadataReader { public static SchemaMetadata readRemoteMetadata(final String connectionString, final String user, final String password, String catalog, String schema, final List tableFilter, - final String dialectName, final JdbcAdapterProperties.ExceptionHandlingMode exceptionMode, final String ignoreErrorList) + final String dialectName, final JdbcAdapterProperties.ExceptionHandlingMode exceptionMode, final List ignoreErrorList) throws SQLException, AdapterException { assert (catalog != null); assert (schema != null); @@ -243,7 +243,7 @@ private static String findSchema(final String schema, final DatabaseMetaData dbM private static List findTables(final String catalog, final String schema, final List tableFilter, final DatabaseMetaData dbMeta, final SqlDialect dialect, - final JdbcAdapterProperties.ExceptionHandlingMode exceptionMode, String ignoreErrorList) throws SQLException { + final JdbcAdapterProperties.ExceptionHandlingMode exceptionMode, List ignoreErrorList) throws SQLException { final List tables = new ArrayList<>(); final String[] supportedTableTypes = { "TABLE", "VIEW", "SYSTEM TABLE" }; diff --git a/jdbc-adapter/virtualschema-jdbc-adapter/src/test/java/com/exasol/adapter/dialects/impl/ExasolSqlDialectIT.java b/jdbc-adapter/virtualschema-jdbc-adapter/src/test/java/com/exasol/adapter/dialects/impl/ExasolSqlDialectIT.java index 5d36adf11..77634da69 100644 --- a/jdbc-adapter/virtualschema-jdbc-adapter/src/test/java/com/exasol/adapter/dialects/impl/ExasolSqlDialectIT.java +++ b/jdbc-adapter/virtualschema-jdbc-adapter/src/test/java/com/exasol/adapter/dialects/impl/ExasolSqlDialectIT.java @@ -9,6 +9,7 @@ import java.sql.Statement; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.List; import org.junit.Assume; @@ -308,7 +309,7 @@ public void testDifferentDataTypes() final List tables = new ArrayList<>(Arrays.asList(tableNames)); final SchemaMetadata meta = JdbcMetadataReader.readRemoteMetadata("jdbc:exa:" + getConfig().getExasolAddress(), getConfig().getExasolUser(), getConfig().getExasolPassword(), "EXA_DB", "JDBC_ADAPTER_TEST_SCHEMA", - tables, ExasolSqlDialect.getPublicName(), getConfig().getExceptionHandlingMode(), ""); + tables, ExasolSqlDialect.getPublicName(), getConfig().getExceptionHandlingMode(), Collections.emptyList()); if (getConfig().isDebugOn()) { System.out.println("Meta: " + SchemaMetadataSerializer.serialize(meta).build().toString()); } diff --git a/jdbc-adapter/virtualschema-jdbc-adapter/src/test/java/com/exasol/adapter/dialects/impl/PostgreSQLSqlDialectTest.java b/jdbc-adapter/virtualschema-jdbc-adapter/src/test/java/com/exasol/adapter/dialects/impl/PostgreSQLSqlDialectTest.java index 53c69ee11..51bd5803d 100644 --- a/jdbc-adapter/virtualschema-jdbc-adapter/src/test/java/com/exasol/adapter/dialects/impl/PostgreSQLSqlDialectTest.java +++ b/jdbc-adapter/virtualschema-jdbc-adapter/src/test/java/com/exasol/adapter/dialects/impl/PostgreSQLSqlDialectTest.java @@ -8,6 +8,7 @@ import java.sql.ResultSet; import java.sql.SQLException; +import java.util.Collections; import static org.junit.Assert.*; import static org.mockito.Matchers.any; @@ -40,7 +41,7 @@ public void mapTableWithUpperCaseCharacters() throws SQLException { ResultSet resultSet = mock(ResultSet.class); when(resultSet.getString("TABLE_NAME")).thenReturn("uPPer"); PostgreSQLSqlDialect postgresDialect = new PostgreSQLSqlDialect(sqlDialectContext); - postgresDialect.mapTable(resultSet, ""); + postgresDialect.mapTable(resultSet, Collections.emptyList()); } @Test @@ -48,6 +49,6 @@ public void mapTableWithLowerCaseCharacters() throws SQLException { ResultSet resultSet = mock(ResultSet.class); when(resultSet.getString("TABLE_NAME")).thenReturn("lower"); PostgreSQLSqlDialect postgresDialect = new PostgreSQLSqlDialect(sqlDialectContext); - postgresDialect.mapTable(resultSet, ""); + postgresDialect.mapTable(resultSet, Collections.emptyList()); } } \ No newline at end of file From 28b8a063ee8900547a360db791578518a93ca80f Mon Sep 17 00:00:00 2001 From: Sebastian Nehls Date: Wed, 30 Jan 2019 10:37:54 +0100 Subject: [PATCH 05/12] #74: added tests, minor refactoring, changed porperty error_list to be case insensitive --- .../dialects/impl/PostgreSQLSqlDialect.java | 16 +++++++++------- .../adapter/jdbc/JdbcAdapterProperties.java | 2 +- .../dialects/impl/PostgreSQLSqlDialectTest.java | 13 +++++++++++++ .../adapter/jdbc/JdbcAdapterPropertiesTest.java | 12 ++++++++++++ 4 files changed, 35 insertions(+), 8 deletions(-) diff --git a/jdbc-adapter/virtualschema-jdbc-adapter/src/main/java/com/exasol/adapter/dialects/impl/PostgreSQLSqlDialect.java b/jdbc-adapter/virtualschema-jdbc-adapter/src/main/java/com/exasol/adapter/dialects/impl/PostgreSQLSqlDialect.java index e41f705e7..ab84aaa16 100644 --- a/jdbc-adapter/virtualschema-jdbc-adapter/src/main/java/com/exasol/adapter/dialects/impl/PostgreSQLSqlDialect.java +++ b/jdbc-adapter/virtualschema-jdbc-adapter/src/main/java/com/exasol/adapter/dialects/impl/PostgreSQLSqlDialect.java @@ -22,6 +22,9 @@ import com.exasol.adapter.sql.ScalarFunction; public class PostgreSQLSqlDialect extends AbstractSqlDialect { + + public static final String POSTGRES_IGNORE_UPPERCASE_TABLES = "POSTGRES_IGNORE_UPPERCASE_TABLES"; + public PostgreSQLSqlDialect(final SqlDialectContext context) { super(context); } @@ -300,18 +303,13 @@ public DataType dialectSpecificMapJdbcType(final JdbcTypeDescription jdbcTypeDes return colType; } - //TODO: write unittest for ignoreerrorlist - //TODO: make return type of ignoreerrorlist an actial List - //TODO: write integrationtest @Override public MappedTable mapTable(final ResultSet tables, List ignoreErrorList) throws SQLException { final String tableName = tables.getString("TABLE_NAME"); - //TODO: remove magic value - if (ignoreErrorList.equals("POSTGRES_IGNORE_UPPERCASE_TABLES")) { + if (ignoreErrorList.contains(POSTGRES_IGNORE_UPPERCASE_TABLES)) { return super.mapTable(tables, ignoreErrorList); } - if (!tableName.equals(tableName.toLowerCase())) { - //TODO: think about a good error message + if (containsUppercaseCharacter(tableName)) { throw new IllegalArgumentException("Table " + tableName + " cannot be used in virtual schema. " + "Set property IGNORE_ERROR_LIST to POSTGRES_IGNORE_UPPERCASE_TABLES to enforce schema creation."); } else { @@ -319,6 +317,10 @@ public MappedTable mapTable(final ResultSet tables, List ignoreErrorList } } + private boolean containsUppercaseCharacter(String tableName) { + return !tableName.equals(tableName.toLowerCase()); + } + @Override public Map getScalarFunctionAliases() { diff --git a/jdbc-adapter/virtualschema-jdbc-adapter/src/main/java/com/exasol/adapter/jdbc/JdbcAdapterProperties.java b/jdbc-adapter/virtualschema-jdbc-adapter/src/main/java/com/exasol/adapter/jdbc/JdbcAdapterProperties.java index 4d31e28a9..52c3048f5 100644 --- a/jdbc-adapter/virtualschema-jdbc-adapter/src/main/java/com/exasol/adapter/jdbc/JdbcAdapterProperties.java +++ b/jdbc-adapter/virtualschema-jdbc-adapter/src/main/java/com/exasol/adapter/jdbc/JdbcAdapterProperties.java @@ -69,7 +69,7 @@ private static String getProperty(final Map properties, final St public static List getIgnoreErrorList(final Map properties) { String ignoreErrors = getProperty(properties, PROP_IGNORE_ERROR_LIST, ""); List ignoreErrorsList = Arrays.asList(ignoreErrors.split(",")); - return ignoreErrorsList.stream().map(error -> error.trim()).collect(Collectors.toList()); + return ignoreErrorsList.stream().map(error -> error.trim().toUpperCase()).collect(Collectors.toList()); } public static String getCatalog(final Map properties) { diff --git a/jdbc-adapter/virtualschema-jdbc-adapter/src/test/java/com/exasol/adapter/dialects/impl/PostgreSQLSqlDialectTest.java b/jdbc-adapter/virtualschema-jdbc-adapter/src/test/java/com/exasol/adapter/dialects/impl/PostgreSQLSqlDialectTest.java index 51bd5803d..57742a0ff 100644 --- a/jdbc-adapter/virtualschema-jdbc-adapter/src/test/java/com/exasol/adapter/dialects/impl/PostgreSQLSqlDialectTest.java +++ b/jdbc-adapter/virtualschema-jdbc-adapter/src/test/java/com/exasol/adapter/dialects/impl/PostgreSQLSqlDialectTest.java @@ -8,7 +8,9 @@ import java.sql.ResultSet; import java.sql.SQLException; +import java.util.ArrayList; import java.util.Collections; +import java.util.List; import static org.junit.Assert.*; import static org.mockito.Matchers.any; @@ -51,4 +53,15 @@ public void mapTableWithLowerCaseCharacters() throws SQLException { PostgreSQLSqlDialect postgresDialect = new PostgreSQLSqlDialect(sqlDialectContext); postgresDialect.mapTable(resultSet, Collections.emptyList()); } + + @Test + public void mapTableWithIgnoreUppercaseCharactersError() throws SQLException { + ResultSet resultSet = mock(ResultSet.class); + when(resultSet.getString("TABLE_NAME")).thenReturn("Upper"); + List ignoreList = new ArrayList<>(); + ignoreList.add("Dummy_Error"); + ignoreList.add("POSTGRES_IGNORE_UPPERCASE_TABLES"); + PostgreSQLSqlDialect postgresDialect = new PostgreSQLSqlDialect(sqlDialectContext); + postgresDialect.mapTable(resultSet, ignoreList); + } } \ No newline at end of file diff --git a/jdbc-adapter/virtualschema-jdbc-adapter/src/test/java/com/exasol/adapter/jdbc/JdbcAdapterPropertiesTest.java b/jdbc-adapter/virtualschema-jdbc-adapter/src/test/java/com/exasol/adapter/jdbc/JdbcAdapterPropertiesTest.java index e4ca68d5f..970df385e 100644 --- a/jdbc-adapter/virtualschema-jdbc-adapter/src/test/java/com/exasol/adapter/jdbc/JdbcAdapterPropertiesTest.java +++ b/jdbc-adapter/virtualschema-jdbc-adapter/src/test/java/com/exasol/adapter/jdbc/JdbcAdapterPropertiesTest.java @@ -2,6 +2,7 @@ import static org.junit.Assert.assertEquals; +import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -246,4 +247,15 @@ public void testNoneAsExceptionValue() throws AdapterException { JdbcAdapterProperties.getExceptionHandlingMode(properties)); JdbcAdapterProperties.checkPropertyConsistency(properties); } + + @Test + public void getIgnoreErrorList() { + Map properties = new HashMap<>(); + properties.put("IGNORE_ERROR_LIST", "ERrror_foo, error_bar , another_error"); + List expectedErrorList = new ArrayList<>(); + expectedErrorList.add("ERRROR_FOO"); + expectedErrorList.add("ERROR_BAR"); + expectedErrorList.add("ANOTHER_ERROR"); + assertEquals(expectedErrorList, JdbcAdapterProperties.getIgnoreErrorList(properties)); + } } From 21e4326ae7d6e9e62bec6d741b801fce7e5c8bc3 Mon Sep 17 00:00:00 2001 From: Sebastian Nehls Date: Mon, 4 Feb 2019 10:40:02 +0100 Subject: [PATCH 06/12] #74: added documentation for postgres identifier handling --- jdbc-adapter/doc/sql_dialects/postgresql.md | 23 ++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/jdbc-adapter/doc/sql_dialects/postgresql.md b/jdbc-adapter/doc/sql_dialects/postgresql.md index d71d7054a..8d05887cb 100644 --- a/jdbc-adapter/doc/sql_dialects/postgresql.md +++ b/jdbc-adapter/doc/sql_dialects/postgresql.md @@ -34,4 +34,25 @@ CREATE VIRTUAL SCHEMA postgres SCHEMA_NAME = 'public' CONNECTION_NAME = 'POSTGRES_DOCKER' ; -``` \ No newline at end of file +``` + +## Postgres identifiers + +In contrast to EXASOL, PostgreSQL does not treat identifiers as specified in the SQL standard. PostgreSQL folds unquoted identifiers to lower case instead of upper case. The adapter can do the identifier conversion as long as there are no identifiers in the PostgreSQL database that contain upper case characters. If that is the case an error will be thrown when creating or refreshing the virtual schema. +In order to create or refresh the virtual schema regrardlessly, you can specifiy that the adapter should ignore this specific error: +```sql +CREATE VIRTUAL SCHEMA postgres + USING adapter.jdbc_adapter + WITH + SQL_DIALECT = 'POSTGRESQL' + CATALOG_NAME = 'postgres' + SCHEMA_NAME = 'public' + CONNECTION_NAME = 'POSTGRES_DOCKER' + IGNORE_ERROR_LIST='POSTGRES_IGNORE_UPPERCASE_TABLES' +; +``` +You can also set this property for an exitsing virtual schema: +```sql +ALTER VIRTUAL SCHEMA postgres SET IGNORE_ERROR_LIST = 'POSTGRES_IGNORE_UPPERCASE_TABLES'; +``` +However you won't be able to query the identifier containing the upper case character. \ No newline at end of file From cacfad9fe90b3208e0de3ce060f8d69b712789e5 Mon Sep 17 00:00:00 2001 From: Sebastian Nehls Date: Tue, 5 Feb 2019 16:04:51 +0100 Subject: [PATCH 07/12] #74: implemented changes from review --- .../adapter/dialects/AbstractSqlDialect.java | 2 +- .../exasol/adapter/dialects/SqlDialect.java | 4 +- .../dialects/impl/OracleSqlDialect.java | 2 +- .../dialects/impl/PostgreSQLSqlDialect.java | 16 +++--- .../adapter/jdbc/JdbcAdapterProperties.java | 49 +++++++++++-------- .../adapter/jdbc/JdbcMetadataReader.java | 33 ++++++------- .../impl/PostgreSQLSqlDialectTest.java | 19 ++++--- .../jdbc/JdbcAdapterPropertiesTest.java | 3 +- 8 files changed, 71 insertions(+), 57 deletions(-) diff --git a/jdbc-adapter/virtualschema-jdbc-adapter/src/main/java/com/exasol/adapter/dialects/AbstractSqlDialect.java b/jdbc-adapter/virtualschema-jdbc-adapter/src/main/java/com/exasol/adapter/dialects/AbstractSqlDialect.java index 5c600f9c7..39c8497a5 100644 --- a/jdbc-adapter/virtualschema-jdbc-adapter/src/main/java/com/exasol/adapter/dialects/AbstractSqlDialect.java +++ b/jdbc-adapter/virtualschema-jdbc-adapter/src/main/java/com/exasol/adapter/dialects/AbstractSqlDialect.java @@ -36,7 +36,7 @@ public String getTableCatalogAndSchemaSeparator() { } @Override - public MappedTable mapTable(final ResultSet tables, List ignoreErrorList) throws SQLException { + public MappedTable mapTable(final ResultSet tables, final List ignoreErrorList) throws SQLException { String commentString = tables.getString("REMARKS"); if (commentString == null) { commentString = ""; diff --git a/jdbc-adapter/virtualschema-jdbc-adapter/src/main/java/com/exasol/adapter/dialects/SqlDialect.java b/jdbc-adapter/virtualschema-jdbc-adapter/src/main/java/com/exasol/adapter/dialects/SqlDialect.java index ebcd7a84c..016ab796e 100644 --- a/jdbc-adapter/virtualschema-jdbc-adapter/src/main/java/com/exasol/adapter/dialects/SqlDialect.java +++ b/jdbc-adapter/virtualschema-jdbc-adapter/src/main/java/com/exasol/adapter/dialects/SqlDialect.java @@ -154,10 +154,10 @@ public String getTableComment() { * @param tables A jdbc Resultset for the * {@link DatabaseMetaData#getTables(String, String, String, String[])} * call, pointing to the current table. - * @param ignoreErrorList + * @param ignoreErrorList The elements of this list suppress certain errors the adapter would throw * @return An instance of {@link MappedTable} describing the mapped table. */ - public MappedTable mapTable(ResultSet tables, List ignoreErrorList) throws SQLException; + public MappedTable mapTable(ResultSet tables, final List ignoreErrorList) throws SQLException; /** * @param columns A jdbc Resultset for the diff --git a/jdbc-adapter/virtualschema-jdbc-adapter/src/main/java/com/exasol/adapter/dialects/impl/OracleSqlDialect.java b/jdbc-adapter/virtualschema-jdbc-adapter/src/main/java/com/exasol/adapter/dialects/impl/OracleSqlDialect.java index e20e375e7..1e0f909f0 100644 --- a/jdbc-adapter/virtualschema-jdbc-adapter/src/main/java/com/exasol/adapter/dialects/impl/OracleSqlDialect.java +++ b/jdbc-adapter/virtualschema-jdbc-adapter/src/main/java/com/exasol/adapter/dialects/impl/OracleSqlDialect.java @@ -332,7 +332,7 @@ public SchemaOrCatalogSupport supportsJdbcSchemas() { } @Override - public MappedTable mapTable(final ResultSet tables, List ignoreErrorList) throws SQLException { + public MappedTable mapTable(final ResultSet tables, final List ignoreErrorList) throws SQLException { final String tableName = tables.getString("TABLE_NAME"); if (tableName.startsWith("BIN$")) { // In case of Oracle we may see deleted tables with strange names diff --git a/jdbc-adapter/virtualschema-jdbc-adapter/src/main/java/com/exasol/adapter/dialects/impl/PostgreSQLSqlDialect.java b/jdbc-adapter/virtualschema-jdbc-adapter/src/main/java/com/exasol/adapter/dialects/impl/PostgreSQLSqlDialect.java index ab84aaa16..56de5937d 100644 --- a/jdbc-adapter/virtualschema-jdbc-adapter/src/main/java/com/exasol/adapter/dialects/impl/PostgreSQLSqlDialect.java +++ b/jdbc-adapter/virtualschema-jdbc-adapter/src/main/java/com/exasol/adapter/dialects/impl/PostgreSQLSqlDialect.java @@ -304,12 +304,9 @@ public DataType dialectSpecificMapJdbcType(final JdbcTypeDescription jdbcTypeDes } @Override - public MappedTable mapTable(final ResultSet tables, List ignoreErrorList) throws SQLException { + public MappedTable mapTable(final ResultSet tables, final List ignoreErrorList) throws SQLException { final String tableName = tables.getString("TABLE_NAME"); - if (ignoreErrorList.contains(POSTGRES_IGNORE_UPPERCASE_TABLES)) { - return super.mapTable(tables, ignoreErrorList); - } - if (containsUppercaseCharacter(tableName)) { + if (!ignoreErrorList.contains(POSTGRES_IGNORE_UPPERCASE_TABLES) && containsUppercaseCharacter(tableName)) { throw new IllegalArgumentException("Table " + tableName + " cannot be used in virtual schema. " + "Set property IGNORE_ERROR_LIST to POSTGRES_IGNORE_UPPERCASE_TABLES to enforce schema creation."); } else { @@ -317,7 +314,7 @@ public MappedTable mapTable(final ResultSet tables, List ignoreErrorList } } - private boolean containsUppercaseCharacter(String tableName) { + private boolean containsUppercaseCharacter(final String tableName) { return !tableName.equals(tableName.toLowerCase()); } @@ -368,7 +365,12 @@ public IdentifierCaseHandling getQuotedIdentifierHandling() { @Override public String applyQuote(final String identifier) { - return "\"" + identifier.toLowerCase().replace("\"", "\"\"") + "\""; + final String lowercaseIdentifier = convertIdentifierToLowerCase(identifier); + return "\"" + lowercaseIdentifier.replace("\"", "\"\"") + "\""; + } + + private String convertIdentifierToLowerCase(final String identifier) { + return identifier.toLowerCase(); } @Override diff --git a/jdbc-adapter/virtualschema-jdbc-adapter/src/main/java/com/exasol/adapter/jdbc/JdbcAdapterProperties.java b/jdbc-adapter/virtualschema-jdbc-adapter/src/main/java/com/exasol/adapter/jdbc/JdbcAdapterProperties.java index 52c3048f5..9934f3fcd 100644 --- a/jdbc-adapter/virtualschema-jdbc-adapter/src/main/java/com/exasol/adapter/jdbc/JdbcAdapterProperties.java +++ b/jdbc-adapter/virtualschema-jdbc-adapter/src/main/java/com/exasol/adapter/jdbc/JdbcAdapterProperties.java @@ -66,27 +66,34 @@ private static String getProperty(final Map properties, final St } } + private static String getProperty(final Map properties, final String name) { + return getProperty(properties, name, ""); + } + public static List getIgnoreErrorList(final Map properties) { - String ignoreErrors = getProperty(properties, PROP_IGNORE_ERROR_LIST, ""); - List ignoreErrorsList = Arrays.asList(ignoreErrors.split(",")); - return ignoreErrorsList.stream().map(error -> error.trim().toUpperCase()).collect(Collectors.toList()); + final String ignoreErrors = getProperty(properties, PROP_IGNORE_ERROR_LIST); + return Arrays. + stream(ignoreErrors.split(",")) + .map(String::trim) + .map(String::toUpperCase) + .collect(Collectors.toList()); } public static String getCatalog(final Map properties) { - return getProperty(properties, PROP_CATALOG_NAME, ""); + return getProperty(properties, PROP_CATALOG_NAME); } public static String getSchema(final Map properties) { - return getProperty(properties, PROP_SCHEMA_NAME, ""); + return getProperty(properties, PROP_SCHEMA_NAME); } public static boolean userSpecifiedConnection(final Map properties) { - final String connName = getProperty(properties, PROP_CONNECTION_NAME, ""); + final String connName = getProperty(properties, PROP_CONNECTION_NAME); return (connName != null && !connName.isEmpty()); } public static String getConnectionName(final Map properties) { - final String connName = getProperty(properties, PROP_CONNECTION_NAME, ""); + final String connName = getProperty(properties, PROP_CONNECTION_NAME); assert (connName != null && !connName.isEmpty()); return connName; } @@ -98,7 +105,7 @@ public static String getConnectionName(final Map properties) { */ public static ExaConnectionInformation getConnectionInformation(final Map properties, final ExaMetadata exaMeta) { - final String connName = getProperty(properties, PROP_CONNECTION_NAME, ""); + final String connName = getProperty(properties, PROP_CONNECTION_NAME); if (connName != null && !connName.isEmpty()) { try { final ExaConnectionInformation connInfo = exaMeta.getConnection(connName); @@ -124,8 +131,8 @@ public static void checkPropertyConsistency(final Map properties private static void checkImportPropertyConsistency(final Map properties, final String propImportFromX, final String propConnection) throws InvalidPropertyException { - final boolean isImport = getProperty(properties, propImportFromX, "").toUpperCase().equals("TRUE"); - final boolean connectionIsEmpty = getProperty(properties, propConnection, "").isEmpty(); + final boolean isImport = getProperty(properties, propImportFromX).toUpperCase().equals("TRUE"); + final boolean connectionIsEmpty = getProperty(properties, propConnection).isEmpty(); if (isImport) { if (connectionIsEmpty) { throw new InvalidPropertyException( @@ -214,23 +221,23 @@ private static void checkMandatoryProperties(final Map propertie } public static boolean isImportFromExa(final Map properties) { - return getProperty(properties, PROP_IMPORT_FROM_EXA, "").toUpperCase().equals("TRUE"); + return getProperty(properties, PROP_IMPORT_FROM_EXA).toUpperCase().equals("TRUE"); } public static boolean isImportFromOra(final Map properties) { - return getProperty(properties, PROP_IMPORT_FROM_ORA, "").toUpperCase().equals("TRUE"); + return getProperty(properties, PROP_IMPORT_FROM_ORA).toUpperCase().equals("TRUE"); } public static String getExaConnectionString(final Map properties) { - return getProperty(properties, PROP_EXA_CONNECTION_STRING, ""); + return getProperty(properties, PROP_EXA_CONNECTION_STRING); } public static String getOraConnectionName(final Map properties) { - return getProperty(properties, PROP_ORA_CONNECTION_NAME, ""); + return getProperty(properties, PROP_ORA_CONNECTION_NAME); } public static List getTableFilter(final Map properties) { - final String tableNames = getProperty(properties, PROP_TABLES, ""); + final String tableNames = getProperty(properties, PROP_TABLES); if (!tableNames.isEmpty()) { final List tables = Arrays.asList(tableNames.split(",")); for (int i = 0; i < tables.size(); ++i) { @@ -243,24 +250,24 @@ public static List getTableFilter(final Map properties) } public static String getExcludedCapabilities(final Map properties) { - return getProperty(properties, PROP_EXCLUDED_CAPABILITIES, ""); + return getProperty(properties, PROP_EXCLUDED_CAPABILITIES); } public static String getDebugAddress(final Map properties) { - return getProperty(properties, PROP_DEBUG_ADDRESS, ""); + return getProperty(properties, PROP_DEBUG_ADDRESS); } public static boolean isLocal(final Map properties) { - return getProperty(properties, PROP_IS_LOCAL, "").toUpperCase().equals("TRUE"); + return getProperty(properties, PROP_IS_LOCAL).toUpperCase().equals("TRUE"); } public static String getSqlDialectName(final Map properties) { - return getProperty(properties, PROP_SQL_DIALECT, ""); + return getProperty(properties, PROP_SQL_DIALECT); } public static SqlDialect getSqlDialect(final Map properties, final SqlDialectContext dialectContext) throws InvalidPropertyException { - final String dialectName = getProperty(properties, PROP_SQL_DIALECT, ""); + final String dialectName = getProperty(properties, PROP_SQL_DIALECT); final SqlDialect dialect = SqlDialects.getInstance().getDialectInstanceForNameWithContext(dialectName, dialectContext); if (dialect == null) { @@ -271,7 +278,7 @@ public static SqlDialect getSqlDialect(final Map properties, fin } public static ExceptionHandlingMode getExceptionHandlingMode(final Map properties) { - final String propertyValue = getProperty(properties, PROP_EXCEPTION_HANDLING, ""); + final String propertyValue = getProperty(properties, PROP_EXCEPTION_HANDLING); if (propertyValue == null || propertyValue.isEmpty()) { return ExceptionHandlingMode.NONE; } diff --git a/jdbc-adapter/virtualschema-jdbc-adapter/src/main/java/com/exasol/adapter/jdbc/JdbcMetadataReader.java b/jdbc-adapter/virtualschema-jdbc-adapter/src/main/java/com/exasol/adapter/jdbc/JdbcMetadataReader.java index cdb690de4..22392cee6 100644 --- a/jdbc-adapter/virtualschema-jdbc-adapter/src/main/java/com/exasol/adapter/jdbc/JdbcMetadataReader.java +++ b/jdbc-adapter/virtualschema-jdbc-adapter/src/main/java/com/exasol/adapter/jdbc/JdbcMetadataReader.java @@ -19,13 +19,12 @@ public class JdbcMetadataReader { private static final Logger LOGGER = Logger.getLogger(JdbcMetadataReader.class.getName()); public static SchemaMetadata readRemoteMetadata(final String connectionString, final String user, - final String password, String catalog, String schema, final List tableFilter, - final String dialectName, final JdbcAdapterProperties.ExceptionHandlingMode exceptionMode, final List ignoreErrorList) + final String password, String catalog, String schema, final List tableFilter, + final String dialectName, final JdbcAdapterProperties.ExceptionHandlingMode exceptionMode, final List ignoreErrorList) throws SQLException, AdapterException { assert (catalog != null); assert (schema != null); - try { - final Connection conn = establishConnection(connectionString, user, password); + try (final Connection conn = establishConnection(connectionString, user, password);) { final DatabaseMetaData dbMeta = conn.getMetaData(); // Retrieve relevant parts of DatabaseMetadata. Will be cached in adapternotes @@ -47,7 +46,6 @@ public static SchemaMetadata readRemoteMetadata(final String connectionString, f final List tables = findTables(catalog, schema, tableFilter, dbMeta, dialect, exceptionMode, ignoreErrorList); - conn.close(); return new SchemaMetadata(SchemaAdapterNotes.serialize(schemaAdapterNotes), tables); } catch (final SQLException e) { e.printStackTrace(); @@ -56,7 +54,7 @@ public static SchemaMetadata readRemoteMetadata(final String connectionString, f } private static Connection establishConnection(final String connectionString, final String user, - final String password) throws SQLException { + final String password) throws SQLException { LOGGER.fine(() -> "Establishing connection with paramters: " + connectionString); final java.util.Properties info = new java.util.Properties(); if (user != null) { @@ -246,21 +244,20 @@ private static List findTables(final String catalog, final String final JdbcAdapterProperties.ExceptionHandlingMode exceptionMode, List ignoreErrorList) throws SQLException { final List tables = new ArrayList<>(); - final String[] supportedTableTypes = { "TABLE", "VIEW", "SYSTEM TABLE" }; + final String[] supportedTableTypes = {"TABLE", "VIEW", "SYSTEM TABLE"}; - final ResultSet resTables = dbMeta.getTables(catalog, schema, null, supportedTableTypes); final List tablesMapped = new ArrayList<>(); - // List tableComments = new ArrayList<>(); - while (resTables.next()) { - final SqlDialect.MappedTable mappedTable = dialect.mapTable(resTables, ignoreErrorList); - if (!mappedTable.isIgnored()) { - tablesMapped.add(mappedTable); - // tableComments.add(mappedTable.getTableComment()); + try (final ResultSet resTables = dbMeta.getTables(catalog, schema, null, supportedTableTypes)) { + // List tableComments = new ArrayList<>(); + while (resTables.next()) { + final SqlDialect.MappedTable mappedTable = dialect.mapTable(resTables, ignoreErrorList); + if (!mappedTable.isIgnored()) { + tablesMapped.add(mappedTable); + // tableComments.add(mappedTable.getTableComment()); + } } } - resTables.close(); - // Columns for (int i = 0; i < tablesMapped.size(); ++i) { final SqlDialect.MappedTable table = tablesMapped.get(i); @@ -300,8 +297,8 @@ private static boolean identifiersAreCaseInsensitive(final SqlDialect dialect) { } private static List readColumns(final DatabaseMetaData dbMeta, final String catalog, - final String schema, final String table, final SqlDialect dialect, - final JdbcAdapterProperties.ExceptionHandlingMode exceptionMode) throws SQLException { + final String schema, final String table, final SqlDialect dialect, + final JdbcAdapterProperties.ExceptionHandlingMode exceptionMode) throws SQLException { final List columns = new ArrayList<>(); try { final ResultSet cols = dbMeta.getColumns(catalog, schema, table, null); diff --git a/jdbc-adapter/virtualschema-jdbc-adapter/src/test/java/com/exasol/adapter/dialects/impl/PostgreSQLSqlDialectTest.java b/jdbc-adapter/virtualschema-jdbc-adapter/src/test/java/com/exasol/adapter/dialects/impl/PostgreSQLSqlDialectTest.java index 57742a0ff..dab8a455b 100644 --- a/jdbc-adapter/virtualschema-jdbc-adapter/src/test/java/com/exasol/adapter/dialects/impl/PostgreSQLSqlDialectTest.java +++ b/jdbc-adapter/virtualschema-jdbc-adapter/src/test/java/com/exasol/adapter/dialects/impl/PostgreSQLSqlDialectTest.java @@ -20,34 +20,41 @@ public class PostgreSQLSqlDialectTest { @Mock SqlDialectContext sqlDialectContext; - @Before public void setUp() throws SQLException { } @Test - public void applyQuoteOnUpperCase() { + public void testApplyQuoteOnUpperCase() { PostgreSQLSqlDialect postgresDialect = new PostgreSQLSqlDialect(sqlDialectContext); assertEquals("\"abc\"", postgresDialect.applyQuote("ABC")); } @Test - public void applyQuoteOnMixedCase() { + public void testApplyQuoteOnMixedCase() { PostgreSQLSqlDialect postgresDialect = new PostgreSQLSqlDialect(sqlDialectContext); assertEquals("\"abcde\"", postgresDialect.applyQuote("AbCde")); } @Test(expected = IllegalArgumentException.class) - public void mapTableWithUpperCaseCharacters() throws SQLException { + public void testMapTableWithUpperCaseCharactersAndNoErrorIgnoredThrowsException() throws SQLException { ResultSet resultSet = mock(ResultSet.class); when(resultSet.getString("TABLE_NAME")).thenReturn("uPPer"); PostgreSQLSqlDialect postgresDialect = new PostgreSQLSqlDialect(sqlDialectContext); postgresDialect.mapTable(resultSet, Collections.emptyList()); } + @Test(expected = IllegalArgumentException.class) + public void testMapTableWithRussianUpperCaseCharactersAndNoErrorIgnoredThrowsException() throws SQLException { + ResultSet resultSet = mock(ResultSet.class); + when(resultSet.getString("TABLE_NAME")).thenReturn("аППер"); + PostgreSQLSqlDialect postgresDialect = new PostgreSQLSqlDialect(sqlDialectContext); + postgresDialect.mapTable(resultSet, Collections.emptyList()); + } + @Test - public void mapTableWithLowerCaseCharacters() throws SQLException { + public void testMapTableWithLowerCaseCharacters() throws SQLException { ResultSet resultSet = mock(ResultSet.class); when(resultSet.getString("TABLE_NAME")).thenReturn("lower"); PostgreSQLSqlDialect postgresDialect = new PostgreSQLSqlDialect(sqlDialectContext); @@ -55,7 +62,7 @@ public void mapTableWithLowerCaseCharacters() throws SQLException { } @Test - public void mapTableWithIgnoreUppercaseCharactersError() throws SQLException { + public void testMapTableWithIgnoreUppercaseCharactersError() throws SQLException { ResultSet resultSet = mock(ResultSet.class); when(resultSet.getString("TABLE_NAME")).thenReturn("Upper"); List ignoreList = new ArrayList<>(); diff --git a/jdbc-adapter/virtualschema-jdbc-adapter/src/test/java/com/exasol/adapter/jdbc/JdbcAdapterPropertiesTest.java b/jdbc-adapter/virtualschema-jdbc-adapter/src/test/java/com/exasol/adapter/jdbc/JdbcAdapterPropertiesTest.java index 970df385e..a6b4e2054 100644 --- a/jdbc-adapter/virtualschema-jdbc-adapter/src/test/java/com/exasol/adapter/jdbc/JdbcAdapterPropertiesTest.java +++ b/jdbc-adapter/virtualschema-jdbc-adapter/src/test/java/com/exasol/adapter/jdbc/JdbcAdapterPropertiesTest.java @@ -251,11 +251,12 @@ public void testNoneAsExceptionValue() throws AdapterException { @Test public void getIgnoreErrorList() { Map properties = new HashMap<>(); - properties.put("IGNORE_ERROR_LIST", "ERrror_foo, error_bar , another_error"); + properties.put("IGNORE_ERROR_LIST", "ERrror_foo, error_bar , another_error, уккщк"); List expectedErrorList = new ArrayList<>(); expectedErrorList.add("ERRROR_FOO"); expectedErrorList.add("ERROR_BAR"); expectedErrorList.add("ANOTHER_ERROR"); + expectedErrorList.add("УККЩК"); assertEquals(expectedErrorList, JdbcAdapterProperties.getIgnoreErrorList(properties)); } } From 8e0985f63194c46b5d279a89bf6999577d048a9e Mon Sep 17 00:00:00 2001 From: Sebastian Nehls Date: Mon, 11 Feb 2019 14:24:48 +0100 Subject: [PATCH 08/12] #74: changed property name to IGNORE_ERRORS and valid value to POSTGRES_UPPERCASE_TABLES --- .../com/exasol/adapter/dialects/impl/PostgreSQLSqlDialect.java | 2 +- .../java/com/exasol/adapter/jdbc/JdbcAdapterProperties.java | 2 +- .../exasol/adapter/dialects/impl/PostgreSQLSqlDialectTest.java | 2 +- .../java/com/exasol/adapter/jdbc/JdbcAdapterPropertiesTest.java | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/jdbc-adapter/virtualschema-jdbc-adapter/src/main/java/com/exasol/adapter/dialects/impl/PostgreSQLSqlDialect.java b/jdbc-adapter/virtualschema-jdbc-adapter/src/main/java/com/exasol/adapter/dialects/impl/PostgreSQLSqlDialect.java index 56de5937d..927841d70 100644 --- a/jdbc-adapter/virtualschema-jdbc-adapter/src/main/java/com/exasol/adapter/dialects/impl/PostgreSQLSqlDialect.java +++ b/jdbc-adapter/virtualschema-jdbc-adapter/src/main/java/com/exasol/adapter/dialects/impl/PostgreSQLSqlDialect.java @@ -23,7 +23,7 @@ public class PostgreSQLSqlDialect extends AbstractSqlDialect { - public static final String POSTGRES_IGNORE_UPPERCASE_TABLES = "POSTGRES_IGNORE_UPPERCASE_TABLES"; + public static final String POSTGRES_IGNORE_UPPERCASE_TABLES = "POSTGRES_UPPERCASE_TABLES"; public PostgreSQLSqlDialect(final SqlDialectContext context) { super(context); diff --git a/jdbc-adapter/virtualschema-jdbc-adapter/src/main/java/com/exasol/adapter/jdbc/JdbcAdapterProperties.java b/jdbc-adapter/virtualschema-jdbc-adapter/src/main/java/com/exasol/adapter/jdbc/JdbcAdapterProperties.java index 9934f3fcd..538d8219c 100644 --- a/jdbc-adapter/virtualschema-jdbc-adapter/src/main/java/com/exasol/adapter/jdbc/JdbcAdapterProperties.java +++ b/jdbc-adapter/virtualschema-jdbc-adapter/src/main/java/com/exasol/adapter/jdbc/JdbcAdapterProperties.java @@ -44,7 +44,7 @@ public final class JdbcAdapterProperties { static final String PROP_EXCLUDED_CAPABILITIES = "EXCLUDED_CAPABILITIES"; static final String PROP_EXCEPTION_HANDLING = "EXCEPTION_HANDLING"; static final String PROP_LOG_LEVEL = "LOG_LEVEL"; - static final String PROP_IGNORE_ERROR_LIST = "IGNORE_ERROR_LIST"; + static final String PROP_IGNORE_ERROR_LIST = "IGNORE_ERRORS"; private static final String DEFAULT_LOG_LEVEL = "INFO"; diff --git a/jdbc-adapter/virtualschema-jdbc-adapter/src/test/java/com/exasol/adapter/dialects/impl/PostgreSQLSqlDialectTest.java b/jdbc-adapter/virtualschema-jdbc-adapter/src/test/java/com/exasol/adapter/dialects/impl/PostgreSQLSqlDialectTest.java index dab8a455b..f6272133a 100644 --- a/jdbc-adapter/virtualschema-jdbc-adapter/src/test/java/com/exasol/adapter/dialects/impl/PostgreSQLSqlDialectTest.java +++ b/jdbc-adapter/virtualschema-jdbc-adapter/src/test/java/com/exasol/adapter/dialects/impl/PostgreSQLSqlDialectTest.java @@ -67,7 +67,7 @@ public void testMapTableWithIgnoreUppercaseCharactersError() throws SQLException when(resultSet.getString("TABLE_NAME")).thenReturn("Upper"); List ignoreList = new ArrayList<>(); ignoreList.add("Dummy_Error"); - ignoreList.add("POSTGRES_IGNORE_UPPERCASE_TABLES"); + ignoreList.add("POSTGRES_UPPERCASE_TABLES"); PostgreSQLSqlDialect postgresDialect = new PostgreSQLSqlDialect(sqlDialectContext); postgresDialect.mapTable(resultSet, ignoreList); } diff --git a/jdbc-adapter/virtualschema-jdbc-adapter/src/test/java/com/exasol/adapter/jdbc/JdbcAdapterPropertiesTest.java b/jdbc-adapter/virtualschema-jdbc-adapter/src/test/java/com/exasol/adapter/jdbc/JdbcAdapterPropertiesTest.java index a6b4e2054..0d949afb6 100644 --- a/jdbc-adapter/virtualschema-jdbc-adapter/src/test/java/com/exasol/adapter/jdbc/JdbcAdapterPropertiesTest.java +++ b/jdbc-adapter/virtualschema-jdbc-adapter/src/test/java/com/exasol/adapter/jdbc/JdbcAdapterPropertiesTest.java @@ -251,7 +251,7 @@ public void testNoneAsExceptionValue() throws AdapterException { @Test public void getIgnoreErrorList() { Map properties = new HashMap<>(); - properties.put("IGNORE_ERROR_LIST", "ERrror_foo, error_bar , another_error, уккщк"); + properties.put("IGNORE_ERRORS", "ERrror_foo, error_bar , another_error, уккщк"); List expectedErrorList = new ArrayList<>(); expectedErrorList.add("ERRROR_FOO"); expectedErrorList.add("ERROR_BAR"); From 4e5917f3af0753cfdb9a0ea8fa5e03de68a4a08b Mon Sep 17 00:00:00 2001 From: Sebastian Nehls Date: Mon, 11 Feb 2019 14:46:08 +0100 Subject: [PATCH 09/12] #74 adapted postgres uppercase tables error message for new property name --- .../com/exasol/adapter/dialects/impl/PostgreSQLSqlDialect.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jdbc-adapter/virtualschema-jdbc-adapter/src/main/java/com/exasol/adapter/dialects/impl/PostgreSQLSqlDialect.java b/jdbc-adapter/virtualschema-jdbc-adapter/src/main/java/com/exasol/adapter/dialects/impl/PostgreSQLSqlDialect.java index 927841d70..7c9bde9c4 100644 --- a/jdbc-adapter/virtualschema-jdbc-adapter/src/main/java/com/exasol/adapter/dialects/impl/PostgreSQLSqlDialect.java +++ b/jdbc-adapter/virtualschema-jdbc-adapter/src/main/java/com/exasol/adapter/dialects/impl/PostgreSQLSqlDialect.java @@ -308,7 +308,7 @@ public MappedTable mapTable(final ResultSet tables, final List ignoreErr final String tableName = tables.getString("TABLE_NAME"); if (!ignoreErrorList.contains(POSTGRES_IGNORE_UPPERCASE_TABLES) && containsUppercaseCharacter(tableName)) { throw new IllegalArgumentException("Table " + tableName + " cannot be used in virtual schema. " + - "Set property IGNORE_ERROR_LIST to POSTGRES_IGNORE_UPPERCASE_TABLES to enforce schema creation."); + "Set property IGNORE_ERRORS to POSTGRES_UPPERCASE_TABLES to enforce schema creation."); } else { return super.mapTable(tables, ignoreErrorList); } From 88977abc2e4d00004202dfadf85f092445cb3089 Mon Sep 17 00:00:00 2001 From: Sebastian Nehls Date: Mon, 11 Feb 2019 14:53:05 +0100 Subject: [PATCH 10/12] 74: changed property prefix from POSTGRES to POSTGRESQL --- .../com/exasol/adapter/dialects/impl/PostgreSQLSqlDialect.java | 2 +- .../exasol/adapter/dialects/impl/PostgreSQLSqlDialectTest.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/jdbc-adapter/virtualschema-jdbc-adapter/src/main/java/com/exasol/adapter/dialects/impl/PostgreSQLSqlDialect.java b/jdbc-adapter/virtualschema-jdbc-adapter/src/main/java/com/exasol/adapter/dialects/impl/PostgreSQLSqlDialect.java index 7c9bde9c4..f8d6a79be 100644 --- a/jdbc-adapter/virtualschema-jdbc-adapter/src/main/java/com/exasol/adapter/dialects/impl/PostgreSQLSqlDialect.java +++ b/jdbc-adapter/virtualschema-jdbc-adapter/src/main/java/com/exasol/adapter/dialects/impl/PostgreSQLSqlDialect.java @@ -23,7 +23,7 @@ public class PostgreSQLSqlDialect extends AbstractSqlDialect { - public static final String POSTGRES_IGNORE_UPPERCASE_TABLES = "POSTGRES_UPPERCASE_TABLES"; + public static final String POSTGRES_IGNORE_UPPERCASE_TABLES = "POSTGRESQL_UPPERCASE_TABLES"; public PostgreSQLSqlDialect(final SqlDialectContext context) { super(context); diff --git a/jdbc-adapter/virtualschema-jdbc-adapter/src/test/java/com/exasol/adapter/dialects/impl/PostgreSQLSqlDialectTest.java b/jdbc-adapter/virtualschema-jdbc-adapter/src/test/java/com/exasol/adapter/dialects/impl/PostgreSQLSqlDialectTest.java index f6272133a..65d07b39b 100644 --- a/jdbc-adapter/virtualschema-jdbc-adapter/src/test/java/com/exasol/adapter/dialects/impl/PostgreSQLSqlDialectTest.java +++ b/jdbc-adapter/virtualschema-jdbc-adapter/src/test/java/com/exasol/adapter/dialects/impl/PostgreSQLSqlDialectTest.java @@ -67,7 +67,7 @@ public void testMapTableWithIgnoreUppercaseCharactersError() throws SQLException when(resultSet.getString("TABLE_NAME")).thenReturn("Upper"); List ignoreList = new ArrayList<>(); ignoreList.add("Dummy_Error"); - ignoreList.add("POSTGRES_UPPERCASE_TABLES"); + ignoreList.add("POSTGRESQL_UPPERCASE_TABLES"); PostgreSQLSqlDialect postgresDialect = new PostgreSQLSqlDialect(sqlDialectContext); postgresDialect.mapTable(resultSet, ignoreList); } From 2cfeb2f7299c961c612b65057b05dcb9598a2427 Mon Sep 17 00:00:00 2001 From: Sebastian Nehls Date: Mon, 11 Feb 2019 15:17:33 +0100 Subject: [PATCH 11/12] 74: added consistency check for IGNORE_ERRORS property --- .../adapter/jdbc/JdbcAdapterProperties.java | 22 ++++++++++++++----- .../jdbc/JdbcAdapterPropertiesTest.java | 10 ++++++++- 2 files changed, 26 insertions(+), 6 deletions(-) diff --git a/jdbc-adapter/virtualschema-jdbc-adapter/src/main/java/com/exasol/adapter/jdbc/JdbcAdapterProperties.java b/jdbc-adapter/virtualschema-jdbc-adapter/src/main/java/com/exasol/adapter/jdbc/JdbcAdapterProperties.java index 538d8219c..e73ecd759 100644 --- a/jdbc-adapter/virtualschema-jdbc-adapter/src/main/java/com/exasol/adapter/jdbc/JdbcAdapterProperties.java +++ b/jdbc-adapter/virtualschema-jdbc-adapter/src/main/java/com/exasol/adapter/jdbc/JdbcAdapterProperties.java @@ -1,10 +1,6 @@ package com.exasol.adapter.jdbc; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; @@ -72,6 +68,9 @@ private static String getProperty(final Map properties, final St public static List getIgnoreErrorList(final Map properties) { final String ignoreErrors = getProperty(properties, PROP_IGNORE_ERROR_LIST); + if (ignoreErrors.trim().isEmpty()) { + return Collections.emptyList(); + } return Arrays. stream(ignoreErrors.split(",")) .map(String::trim) @@ -127,6 +126,19 @@ public static void checkPropertyConsistency(final Map properties checkMandatoryProperties(properties); checkImportPropertyConsistency(properties, PROP_IMPORT_FROM_EXA, PROP_EXA_CONNECTION_STRING); checkImportPropertyConsistency(properties, PROP_IMPORT_FROM_ORA, PROP_ORA_CONNECTION_NAME); + checkIgnoreErrors(properties); + } + + private static void checkIgnoreErrors(final Map properties) throws InvalidPropertyException { + final String dialect = getSqlDialectName(properties); + List errorsToIgnore = getIgnoreErrorList(properties); + for (String errorToIgnore : errorsToIgnore) { + if (!errorToIgnore.startsWith(dialect)) { + throw new InvalidPropertyException( + "Error " + errorToIgnore + " cannot be ignored in " + dialect + " dialect." + ); + } + } } private static void checkImportPropertyConsistency(final Map properties, diff --git a/jdbc-adapter/virtualschema-jdbc-adapter/src/test/java/com/exasol/adapter/jdbc/JdbcAdapterPropertiesTest.java b/jdbc-adapter/virtualschema-jdbc-adapter/src/test/java/com/exasol/adapter/jdbc/JdbcAdapterPropertiesTest.java index 0d949afb6..01ac7ac6c 100644 --- a/jdbc-adapter/virtualschema-jdbc-adapter/src/test/java/com/exasol/adapter/jdbc/JdbcAdapterPropertiesTest.java +++ b/jdbc-adapter/virtualschema-jdbc-adapter/src/test/java/com/exasol/adapter/jdbc/JdbcAdapterPropertiesTest.java @@ -249,7 +249,7 @@ public void testNoneAsExceptionValue() throws AdapterException { } @Test - public void getIgnoreErrorList() { + public void getIgnoreErrors() { Map properties = new HashMap<>(); properties.put("IGNORE_ERRORS", "ERrror_foo, error_bar , another_error, уккщк"); List expectedErrorList = new ArrayList<>(); @@ -259,4 +259,12 @@ public void getIgnoreErrorList() { expectedErrorList.add("УККЩК"); assertEquals(expectedErrorList, JdbcAdapterProperties.getIgnoreErrorList(properties)); } + + @Test(expected = InvalidPropertyException.class) + public void checkIgnoreErrorsConsistency() throws AdapterException { + Map properties = new HashMap<>(); + properties.put("IGNORE_ERRORS", "ORACLE_ERROR"); + properties.put("dialect", "postgresql"); + JdbcAdapterProperties.checkPropertyConsistency(properties); + } } From 18d089414d2cd6723973b1c77a1047d2c91c8ff1 Mon Sep 17 00:00:00 2001 From: Sebastian Nehls Date: Mon, 11 Feb 2019 15:26:25 +0100 Subject: [PATCH 12/12] 74: added documentation for IGNORE_ERRORS property --- jdbc-adapter/README.md | 1 + jdbc-adapter/doc/sql_dialects/postgresql.md | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/jdbc-adapter/README.md b/jdbc-adapter/README.md index 1a8bf80a2..985650330 100644 --- a/jdbc-adapter/README.md +++ b/jdbc-adapter/README.md @@ -136,6 +136,7 @@ Property | Value **ORA_CONNECTION_NAME** | Name of the connection to an Oracle database created with `CREATE CONNECTION`. Used by `IMPORT FROM ORA`. **IS_LOCAL** | Only relevant if your data source is the same Exasol database where you create the virtual schema. Either `TRUE` or `FALSE` (default). If true, you are connecting to the local Exasol database (e.g. for testing purposes). In this case, the adapter can avoid the `IMPORT FROM JDBC` overhead. **EXCEPTION_HANDLING** | Activates or deactivates different exception handling modes. Supported values: `IGNORE_INVALID_VIEWS` and `NONE` (default). Currently this property only affects the Teradata dialect. +**IGNORE_ERRORS** | Is used to ignore errors thrown by the adapter. Supported values: 'POSTGRESQL_UPPERCASE_TABLES' (see PostgreSQL dialect documentation). ## Debugging diff --git a/jdbc-adapter/doc/sql_dialects/postgresql.md b/jdbc-adapter/doc/sql_dialects/postgresql.md index 8d05887cb..fe8e5f4f2 100644 --- a/jdbc-adapter/doc/sql_dialects/postgresql.md +++ b/jdbc-adapter/doc/sql_dialects/postgresql.md @@ -48,11 +48,11 @@ CREATE VIRTUAL SCHEMA postgres CATALOG_NAME = 'postgres' SCHEMA_NAME = 'public' CONNECTION_NAME = 'POSTGRES_DOCKER' - IGNORE_ERROR_LIST='POSTGRES_IGNORE_UPPERCASE_TABLES' + IGNORE_ERRORS = 'POSTGRESQL_UPPERCASE_TABLES' ; ``` You can also set this property for an exitsing virtual schema: ```sql -ALTER VIRTUAL SCHEMA postgres SET IGNORE_ERROR_LIST = 'POSTGRES_IGNORE_UPPERCASE_TABLES'; +ALTER VIRTUAL SCHEMA postgres SET IGNORE_ERRORS = 'POSTGRESQL_UPPERCASE_TABLES'; ``` However you won't be able to query the identifier containing the upper case character. \ No newline at end of file