#369 Type mapping documentation + refactoring (#367)

#369 Type mapping documentation #370 Cleanup of oracle dialect
exasol · Aug 13, 2020 · febc227 · febc227
1 parent 34831c0
commit febc227
Show file tree

Hide file tree

Showing 6 changed files with 64 additions and 89 deletions.
diff --git a/doc/dialects/oracle.md b/doc/dialects/oracle.md
@@ -133,31 +133,46 @@ CREATE VIRTUAL SCHEMA <virtual schema name>
     ORA_CONNECTION_NAME = 'ORA_CONNECTION';
 ```
 
-## Supported capabilities
+## Supported Capabilities
 
 The Oracle dialect does not support all capabilities. A complete list can be found in [OracleSqlDialect.getCapabilities()](../../src/main/java/com/exasol/adapter/dialects/oracle/OracleSqlDialect.java).
 
 ## Type Mappings and Limitations
 
-Oracle data types are mapped to their equivalents in Exasol. The following exceptions apply:
-
-- `NUMBER`, `NUMBER with precision > 36` and `LONG` are casted to `VARCHAR` to prevent a loss of precision. 
-
-    If you want to return a DECIMAL type for these types you can set the property ORACLE_CAST_NUMBER_TO_DECIMAL_WITH_PRECISION_AND_SCALE: 
-
-    `ORACLE_CAST_NUMBER_TO_DECIMAL_WITH_PRECISION_AND_SCALE='36,20'` 
-
-    This will cast NUMBER with precision > 36, NUMBER without precision and LONG to DECIMAL(36,20).
-    Keep in mind that this will yield errors if the data in the Oracle database does not fit into the specified DECIMAL type.
-
-- `DATE` is casted to `TIMESTAMP`. This data type is only supported for positive year values, i.e., years > 0001.
-- `TIMESTAMP WITH [LOCAL] TIME ZONE` is casted to `TIMESTAMP`. 
-- `INTERVAL` is casted to `VARCHAR`.
-- `CLOB`, `BLOB`, and `NCLOB` values are supported up to the maximum size of an Exasol [`VARCHAR`](https://docs.exasol.com/sql_references/data_types/datatypedetails.htm#StringDataType).
-
-    By default an error will be thrown if you try to import a value that is larger. If you want to import from tables which contain bigger values, you need to truncate the values.
-
-- `RAW` and `LONG RAW` are not supported.
+| Orcale Data Type                                                                 | Supported | Converted Exasol Data Type | Comments                                                                                                                                      |
+| -------------------------------------------------------------------------------- | --------- | -------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------- |
+| BINARY\_DOUBLE                                                                   | ✓         | VARCHAR(2000000)           |                                                                                                                                               |
+| BINARY\_FLOAT                                                                    | ✓         | VARCHAR(2000000)           |                                                                                                                                               |
+| BLOB                                                                             | ×         |                            |                                                                                                                                               |
+| BFILE                                                                            | ×         |                            |                                                                                                                                               |
+| CHAR \[(size)\]                                                                  | ✓         | CHAR                       |                                                                                                                                               |
+| CLOB                                                                             | ×         |                            |                                                                                                                                               |
+| DATE                                                                             | ✓         | TIMESTAMP                  | This data type is only supported for positive year values, i.e., years > 0001                                                                 |
+| FLOAT \[(p)\]                                                                    | ✓         | DOUBLE                     |                                                                                                                                               |
+| INTERVAL DAY \[(day\_precision)\] TO SECOND \[(fractional\_seconds\_precision)\] | ✓         | VARCHAR(2000000)           |                                                                                                                                               |
+| INTERVAL YEAR \[(year\_precision)\] TO MONTH                                     | ✓         | VARCHAR(2000000)           |                                                                                                                                               |
+| LONG                                                                             | ✓         | VARCHAR(2000000)           | Casted to VARCHAR to prevent a loss of precision.                                                                                             |
+| LONG RAW                                                                         | ×         |                            |                                                                                                                                               |
+| NCLOB                                                                            | ×         |                            |                                                                                                                                               |
+| NCHAR\[(size)\]                                                                  | ✓         | CHAR                       |                                                                                                                                               |
+| NUMBER \[ (p \[, s\]) \]                                                         | ✓         | NUMBER or VARCHAR(2000000) | NUMBER with precision > 36 are casted to VARCHAR to prevent a loss of precision. [*](#Mapping-of-number-types)                                |
+| NVARCHAR2(size)                                                                  | ✓         | VARCHAR                    |                                                                                                                                               |
+| RAW(size)                                                                        | ×         |                            |                                                                                                                                               |
+| ROWID                                                                            | ×         |                            |                                                                                                                                               |
+| TIMESTAMP \[(fractional\_seconds\_precision)\]                                   | ✓         | TIMESTAMP                  |                                                                                                                                               |
+| TIMESTAMP \[(fractional\_seconds\_precision)\] WITH TIME ZONE                    | ✓         | TIMESTAMP                  |                                                                                                                                               |
+| UROWID \[(size)\]                                                                | ×         |                            |                                                                                                                                               |
+| VARCHAR2(size)                                                                   | ✓         | VARCHAR                    |                                                                                                                                               |
+
+
+
+### Mapping of Number Types:
+
+`NUMBER`, `NUMBER with precision > 36` and `LONG` are casted to `VARCHAR` to prevent a loss of precision. 
+
+If you want to return a DECIMAL type for these types you can set the property ORACLE_CAST_NUMBER_TO_DECIMAL_WITH_PRECISION_AND_SCALE:
+This will cast NUMBER with precision > 36, NUMBER without precision and LONG to DECIMAL(36,20).
+Keep in mind that this will yield errors if the data in the Oracle database does not fit into the specified DECIMAL type. 
 
 ## Testing information
 

diff --git a/src/main/java/com/exasol/adapter/dialects/oracle/OracleColumnMetadataReader.java b/src/main/java/com/exasol/adapter/dialects/oracle/OracleColumnMetadataReader.java
@@ -6,7 +6,6 @@
 import java.sql.Types;
 
 import com.exasol.adapter.AdapterProperties;
-import com.exasol.adapter.BinaryColumnHandling;
 import com.exasol.adapter.dialects.IdentifierConverter;
 import com.exasol.adapter.jdbc.BaseColumnMetadataReader;
 import com.exasol.adapter.jdbc.JdbcTypeDescription;
@@ -20,7 +19,6 @@ public class OracleColumnMetadataReader extends BaseColumnMetadataReader {
     private static final int ORACLE_TIMESTAMP_WITH_TIME_ZONE = -102;
     private static final int ORACLE_BINARY_FLOAT = 100;
     private static final int ORACLE_BINARY_DOUBLE = 101;
-    private static final int ORACLE_CLOB = Types.OTHER;
     private static final int INTERVAL_DAY_TO_SECOND = -104;
     private static final int INTERVAL_YEAR_TO_MONTH = -103;
     static final int ORACLE_MAGIC_NUMBER_SCALE = -127;
@@ -46,16 +44,11 @@ public DataType mapJdbcType(final JdbcTypeDescription jdbcTypeDescription) {
         case ORACLE_TIMESTAMP_WITH_TIME_ZONE:
         case ORACLE_TIMESTAMP_WITH_LOCAL_TIME_ZONE:
             return DataType.createTimestamp(false);
-        case Types.NCLOB:
-        case ORACLE_CLOB:
         case INTERVAL_YEAR_TO_MONTH:
         case INTERVAL_DAY_TO_SECOND:
         case ORACLE_BINARY_FLOAT:
         case ORACLE_BINARY_DOUBLE:
-        case Types.ROWID:
             return DataType.createMaximumSizeVarChar(DataType.ExaCharset.UTF8);
-        case Types.BLOB:
-            return mapBlobType();
         default:
             return super.mapJdbcType(jdbcTypeDescription);
         }
@@ -92,12 +85,4 @@ private DataType getOracleNumberTargetType() {
             return DataType.createMaximumSizeVarChar(DataType.ExaCharset.UTF8);
         }
     }
-
-    private DataType mapBlobType() {
-        if (this.properties.getBinaryColumnHandling() == BinaryColumnHandling.IGNORE) {
-            return DataType.createUnsupported();
-        } else {
-            return DataType.createMaximumSizeVarChar(DataType.ExaCharset.UTF8);
-        }
-    }
 }
diff --git a/src/main/java/com/exasol/adapter/dialects/oracle/OracleSqlGenerationVisitor.java b/src/main/java/com/exasol/adapter/dialects/oracle/OracleSqlGenerationVisitor.java
@@ -4,11 +4,12 @@
 import static com.exasol.adapter.sql.ScalarFunction.*;
 
 import java.util.*;
-import java.util.logging.Logger;
 
 import com.exasol.adapter.AdapterException;
 import com.exasol.adapter.dialects.*;
-import com.exasol.adapter.metadata.*;
+import com.exasol.adapter.metadata.ColumnMetadata;
+import com.exasol.adapter.metadata.DataType;
+import com.exasol.adapter.metadata.TableMetadata;
 import com.exasol.adapter.sql.*;
 
 /**
@@ -18,7 +19,7 @@ public class OracleSqlGenerationVisitor extends SqlGenerationVisitor {
     private boolean requiresSelectListAliasesForLimit = false;
     private static final String TIMESTAMP_FORMAT = "'YYYY-MM-DD HH24:MI:SS.FF3'";
     private static final List<String> TYPE_NAMES_REQUIRING_CAST = List.of("TIMESTAMP", "INTERVAL", "BINARY_FLOAT",
-            "BINARY_DOUBLE", "ROWID", "UROWID", "BLOB");
+            "BINARY_DOUBLE");
     private final Set<AggregateFunction> aggregateFunctionsCast = EnumSet.noneOf(AggregateFunction.class);
     private final Set<ScalarFunction> scalarFunctionsCast = EnumSet.noneOf(ScalarFunction.class);
 
@@ -273,10 +274,6 @@ private String getProjectionString(final SqlColumn column, final String projecti
                     + ")";
         } else if (typeName.equals("NUMBER")) {
             return getNumberProjectionString(column, projectionString, (OracleSqlDialect) dialect);
-        } else if (typeName.equals("ROWID") || typeName.equals("UROWID")) {
-            return "ROWIDTOCHAR(" + projectionString + ")";
-        } else if (typeName.equals("BLOB")) {
-            return "UTL_RAW.CAST_TO_VARCHAR2(UTL_ENCODE.BASE64_DECODE(" + projectionString + "))";
         } else {
             return projectionString;
         }

diff --git a/src/test/java/com/exasol/adapter/dialects/oracle/OracleColumnMetadataReaderTest.java b/src/test/java/com/exasol/adapter/dialects/oracle/OracleColumnMetadataReaderTest.java
@@ -1,11 +1,10 @@
 package com.exasol.adapter.dialects.oracle;
 
 import static com.exasol.adapter.metadata.DataType.createMaximumSizeVarChar;
-import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.equalTo;
 
 import java.sql.Types;
-import java.util.HashMap;
 import java.util.Map;
 
 import org.junit.jupiter.api.BeforeEach;
@@ -15,7 +14,6 @@
 import com.exasol.adapter.dialects.BaseIdentifierConverter;
 import com.exasol.adapter.jdbc.JdbcTypeDescription;
 import com.exasol.adapter.metadata.DataType;
-import com.exasol.adapter.metadata.DataType.ExaCharset;
 
 class OracleColumnMetadataReaderTest {
     private OracleColumnMetadataReader columnMetadataReader;
@@ -44,21 +42,6 @@ void testMapColumnTypeWithMagicScale() {
                 equalTo(createMaximumSizeVarChar(DataType.ExaCharset.UTF8)));
     }
 
-    @Test
-    void testMapBlobMappedToUnsupportedTypeByDefault() {
-        final JdbcTypeDescription typeDescription = new JdbcTypeDescription(Types.BLOB, 0, 0, 0, null);
-        assertThat(this.columnMetadataReader.mapJdbcType(typeDescription), equalTo(DataType.createUnsupported()));
-    }
-
-    @Test
-    void testMapBlobMappedToMaximumSizeVarCharIfBase64EncodingEnabled() {
-        final Map<String, String> rawProperties = new HashMap<>();
-        rawProperties.put("BINARY_COLUMN_HANDLING", "ENCODE_BASE64");
-        final JdbcTypeDescription typeDescription = new JdbcTypeDescription(Types.BLOB, 0, 0, 0, null);
-        assertThat(createParameterizedColumnMetadataReader(rawProperties).mapJdbcType(typeDescription),
-                equalTo(createMaximumSizeVarChar(ExaCharset.UTF8)));
-    }
-
     @Test
     void testMapNumericColumnTypeWithMaximumDecimalPrecision() {
         final int precision = DataType.MAX_EXASOL_DECIMAL_PRECISION;
@@ -77,13 +60,6 @@ void testMapColumnTypeWithZeroPrecision() {
                 equalTo(DataType.createDecimal(DataType.MAX_EXASOL_DECIMAL_PRECISION, scale)));
     }
 
-    @Test
-    void testMapRowId() {
-        final JdbcTypeDescription typeDescription = new JdbcTypeDescription(Types.ROWID, 0, 0, 0, null);
-        assertThat(this.columnMetadataReader.mapJdbcType(typeDescription),
-                equalTo(createMaximumSizeVarChar(DataType.ExaCharset.UTF8)));
-    }
-
     private OracleColumnMetadataReader createParameterizedColumnMetadataReader(
             final Map<String, String> rawProperties) {
         return new OracleColumnMetadataReader(null, new AdapterProperties(rawProperties),

diff --git a/src/test/java/com/exasol/adapter/dialects/oracle/OracleSqlDialectIT.java b/src/test/java/com/exasol/adapter/dialects/oracle/OracleSqlDialectIT.java
@@ -3,8 +3,7 @@
 import static com.exasol.adapter.dialects.IntegrationTestConstants.*;
 import static com.exasol.dbbuilder.dialects.exasol.AdapterScript.Language.JAVA;
 import static com.exasol.matcher.ResultSetMatcher.matchesResultSet;
-import static org.hamcrest.CoreMatchers.containsString;
-import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.CoreMatchers.*;
 import static org.hamcrest.MatcherAssert.assertThat;
 import static org.junit.Assert.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertAll;
@@ -34,7 +33,10 @@
 import com.exasol.bucketfs.BucketAccessException;
 import com.exasol.containers.ExasolContainer;
 import com.exasol.containers.ExasolContainerConstants;
-import com.exasol.dbbuilder.dialects.exasol.*;
+import com.exasol.dbbuilder.dialects.exasol.AdapterScript;
+import com.exasol.dbbuilder.dialects.exasol.ConnectionDefinition;
+import com.exasol.dbbuilder.dialects.exasol.ExasolObjectFactory;
+import com.exasol.dbbuilder.dialects.exasol.ExasolSchema;
 
 /**
  * How to run `OracleSqlDialectIT`: See the documentation <a
@@ -670,6 +672,22 @@ void testCharactersColumns(final String virtualSchemaName, final String columnNa
                             equalTo(expectedColumnType)));
         }
 
+        @ParameterizedTest
+        @CsvSource(value = { //
+                "VIRTUAL_SCHEMA_JDBC, C18", //
+                "VIRTUAL_SCHEMA_JDBC, C19", //
+                "VIRTUAL_SCHEMA_JDBC, C20", //
+                "VIRTUAL_SCHEMA_ORA, C18", //
+                "VIRTUAL_SCHEMA_ORA, C19", //
+                "VIRTUAL_SCHEMA_ORA, C20",//
+        })
+        void testBlobColumns(final String virtualSchemaName, final String columnName) {
+            final String qualifiedTableName = virtualSchemaName + "." + TABLE_ORACLE_ALL_DATA_TYPES;
+            final String query = "SELECT " + columnName + " FROM " + qualifiedTableName;
+            final SQLException exception = assertThrows(SQLException.class, () -> statementExasol.execute(query));
+            assertThat(exception.getMessage(), startsWith("object " + columnName + " not found"));
+        }
+
         @ParameterizedTest
         @CsvSource(value = { //
                 "VIRTUAL_SCHEMA_JDBC | C5 | VARCHAR(2000000) UTF8 | 123456789012345678901234567890123456", //

diff --git a/src/test/java/com/exasol/adapter/dialects/oracle/OracleSqlGenerationVisitorTest.java b/src/test/java/com/exasol/adapter/dialects/oracle/OracleSqlGenerationVisitorTest.java
@@ -12,7 +12,9 @@
 import java.math.BigDecimal;
 import java.sql.Connection;
 import java.sql.SQLException;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
 
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
@@ -147,24 +149,6 @@ void testVisitSqlSelectListSelectStarWithTimestamp() throws AdapterException {
                 "TO_TIMESTAMP(TO_CHAR(\"test_column\", 'YYYY-MM-DD HH24:MI:SS.FF3'), 'YYYY-MM-DD HH24:MI:SS.FF3')"));
     }
 
-    @CsvSource({ "ROWID", "UROWID" })
-    @ParameterizedTest
-    void testVisitSqlSelectListSelectStarNumberCastRowidToChar(final String dataType) throws AdapterException {
-        final SqlSelectList selectList = createSqlSelectStarListWithOneColumn(
-                "{\"jdbcDataType\":2, \"typeName\":\"" + dataType + "\"}",
-                DataType.createVarChar(50, DataType.ExaCharset.UTF8), "test_column");
-        assertThat(this.visitor.visit(selectList), equalTo("ROWIDTOCHAR(\"test_column\")"));
-    }
-
-    @Test
-    void testVisitSqlSelectListSelectStarNumberCastBlobToChar() throws AdapterException {
-        final SqlSelectList selectList = createSqlSelectStarListWithOneColumn(
-                "{\"jdbcDataType\":2, \"typeName\":\"BLOB\"}", DataType.createVarChar(50, DataType.ExaCharset.UTF8),
-                "test_column");
-        assertThat(this.visitor.visit(selectList),
-                equalTo("UTL_RAW.CAST_TO_VARCHAR2(UTL_ENCODE.BASE64_DECODE(\"test_column\"))"));
-    }
-
     @Test
     void testVisitSqlSelectListSelectStarNumberCastToDecimal() throws AdapterException {
         final SqlSelectList selectList = SqlSelectList.createSelectStarSelectList();