Merge remote-tracking branch 'origin/master' into

enhancement/121_Request_dispatching_and_remote_logging

Conflicts:
	jdbc-adapter/pom.xml
	jdbc-adapter/virtualschema-jdbc-adapter/src/main/java/com/exasol/adapter/jdbc/JdbcAdapter.java
	jdbc-adapter/virtualschema-jdbc-adapter/src/test/java/com/exasol/adapter/jdbc/JdbcAdapterTest.java

jdbc-adapter/doc/deploying_the_virtual_schema_adapter.md

@@ -23,7 +23,7 @@ cd virtual-schemas/jdbc-adapter/
 mvn clean -DskipTests package
 ```
 
-The resulting fat JAR is stored in `virtualschema-jdbc-adapter-dist/target/virtualschema-jdbc-adapter-dist-1.7.1.jar`.
+The resulting fat JAR is stored in `virtualschema-jdbc-adapter-dist/target/virtualschema-jdbc-adapter-dist-1.8.2.jar`.
 
 ## Uploading the Adapter JAR Archive
 
@@ -42,8 +42,8 @@ Following steps are required to upload a file to a bucket:
 1. Now upload the file into this bucket, e.g. using curl (adapt the hostname, BucketFS port, bucket name and bucket write password).
 
 ```bash
-curl -X PUT -T virtualschema-jdbc-adapter-dist/target/virtualschema-jdbc-adapter-dist-1.7.1.jar \
- http://w:write-password@your.exasol.host.com:2580/bucket1/virtualschema-jdbc-adapter-dist-1.7.1.jar
+curl -X PUT -T virtualschema-jdbc-adapter-dist/target/virtualschema-jdbc-adapter-dist-1.8.2.jar \
+ http://w:write-password@your.exasol.host.com:2580/bucket1/virtualschema-jdbc-adapter-dist-1.8.2.jar
 ```
 
 See chapter 3.6.4. "The synchronous cluster file system BucketFS" in the EXASolution User Manual for more details about BucketFS.
@@ -75,7 +75,7 @@ CREATE JAVA ADAPTER SCRIPT adapter.jdbc_adapter AS
 
   // This will add the adapter jar to the classpath so that it can be used inside the adapter script
   // Replace the names of the bucketfs and the bucket with the ones you used.
-  %jar /buckets/your-bucket-fs/your-bucket/virtualschema-jdbc-adapter-dist-1.7.1.jar;
+  %jar /buckets/your-bucket-fs/your-bucket/virtualschema-jdbc-adapter-dist-1.8.2.jar;
 
   // You have to add all files of the data source jdbc driver here (e.g. Hive JDBC driver files)
   %jar /buckets/your-bucket-fs/your-bucket/name-of-data-source-jdbc-driver.jar;

jdbc-adapter/doc/developing_an_sql_dialect.md

@@ -292,7 +292,7 @@ CREATE OR REPLACE JAVA ADAPTER SCRIPT adapter.jdbc_adapter
 
   // This will add the adapter jar to the classpath so that it can be used inside the adapter script
   // Replace the names of the bucketfs and the bucket with the ones you used.
-  %jar /buckets/bucketfs1/bucket1/virtualschema-jdbc-adapter-dist-1.7.1.jar;
+  %jar /buckets/bucketfs1/bucket1/virtualschema-jdbc-adapter-dist-1.8.2.jar;
 
   // You have to add all files of the data source jdbc driver here (e.g. MySQL or Hive)
 

jdbc-adapter/doc/sql_dialects/db2.md

@@ -46,7 +46,7 @@ CREATE or replace JAVA ADAPTER SCRIPT adapter.jdbc_adapter AS
 
   // This will add the adapter jar to the classpath so that it can be used inside the adapter script
   // Replace the names of the bucketfs and the bucket with the ones you used.
-  %jar /buckets/bucketfs1/bucket1/virtualschema-jdbc-adapter-dist-1.7.1.jar;
+  %jar /buckets/bucketfs1/bucket1/virtualschema-jdbc-adapter-dist-1.8.2.jar;
 
   // DB2 Driver files
   %jar /buckets/bucketfs1/bucket1/db2jcc4.jar;

jdbc-adapter/doc/sql_dialects/exasol.md

@@ -17,7 +17,7 @@ After uploading the adapter jar, the adapter script can be created as follows:
 CREATE SCHEMA adapter;
 CREATE JAVA ADAPTER SCRIPT adapter.jdbc_adapter AS
   %scriptclass com.exasol.adapter.jdbc.JdbcAdapter;
-  %jar /buckets/your-bucket-fs/your-bucket/virtualschema-jdbc-adapter-dist-1.7.1.jar;
+  %jar /buckets/your-bucket-fs/your-bucket/virtualschema-jdbc-adapter-dist-1.8.2.jar;
 /
 ```
 

jdbc-adapter/doc/sql_dialects/hive.md

@@ -23,7 +23,7 @@ CREATE SCHEMA adapter;
 CREATE  JAVA  ADAPTER SCRIPT jdbc_adapter AS
   %scriptclass com.exasol.adapter.jdbc.JdbcAdapter;
 
-  %jar /buckets/bucketfs1/bucket1/virtualschema-jdbc-adapter-dist-1.7.1.jar;
+  %jar /buckets/bucketfs1/bucket1/virtualschema-jdbc-adapter-dist-1.8.2.jar;
 
   %jar /buckets/bucketfs1/bucket1/hive_metastore.jar;
   %jar /buckets/bucketfs1/bucket1/hive_service.jar;

jdbc-adapter/doc/sql_dialects/impala.md

@@ -22,7 +22,7 @@ CREATE SCHEMA adapter;
 CREATE  JAVA  ADAPTER SCRIPT jdbc_adapter AS
   %scriptclass com.exasol.adapter.jdbc.JdbcAdapter;
 
-  %jar /buckets/bucketfs1/bucket1/virtualschema-jdbc-adapter-dist-1.7.1.jar;
+  %jar /buckets/bucketfs1/bucket1/virtualschema-jdbc-adapter-dist-1.8.2.jar;
 
   %jar /buckets/bucketfs1/bucket1/hive_metastore.jar;
   %jar /buckets/bucketfs1/bucket1/hive_service.jar;

jdbc-adapter/doc/sql_dialects/oracle.md

@@ -28,7 +28,7 @@ CREATE JAVA ADAPTER SCRIPT adapter.jdbc_oracle AS
 
   // You need to replace `your-bucket-fs` and `your-bucket` to match the actual location
   // of the adapter jar.
-  %jar /buckets/your-bucket-fs/your-bucket/virtualschema-jdbc-adapter-dist-1.7.1.jar;
+  %jar /buckets/your-bucket-fs/your-bucket/virtualschema-jdbc-adapter-dist-1.8.2.jar;
 
   // Add the oracle jdbc driver to the classpath
   %jar /buckets/bucketfs1/bucket1/ojdbc7-12.1.0.2.jar

jdbc-adapter/doc/sql_dialects/postgresql.md

@@ -15,7 +15,7 @@ CREATE OR REPLACE JAVA ADAPTER SCRIPT adapter.jdbc_adapter
 
   // This will add the adapter jar to the classpath so that it can be used inside the adapter script
   // Replace the names of the bucketfs and the bucket with the ones you used.
-  %jar /buckets/bucketfs1/bucket1/virtualschema-jdbc-adapter-dist-1.7.1.jar;
+  %jar /buckets/bucketfs1/bucket1/virtualschema-jdbc-adapter-dist-1.8.2.jar;
 
   // You have to add all files of the data source jdbc driver here (e.g. MySQL or Hive)
   %jar /buckets/bucketfs1/bucket1/postgresql-42.0.0.jar;

jdbc-adapter/doc/sql_dialects/redshift.md

@@ -21,7 +21,7 @@ CREATE OR REPLACE JAVA ADAPTER SCRIPT adapter.jdbc_adapter
 
   // This will add the adapter jar to the classpath so that it can be used inside the adapter script
   // Replace the names of the bucketfs and the bucket with the ones you used.
-  %jar /buckets/bucketfs1/bucket1/virtualschema-jdbc-adapter-dist-1.7.1.jar;
+  %jar /buckets/bucketfs1/bucket1/virtualschema-jdbc-adapter-dist-1.8.2.jar;
 
   // You have to add all files of the data source jdbc driver here (e.g. MySQL or Hive)
 

jdbc-adapter/doc/sql_dialects/sql_server.md

@@ -17,7 +17,7 @@ CREATE OR REPLACE JAVA ADAPTER SCRIPT adapter.sql_server_jdbc_adapter
 
   // This will add the adapter jar to the classpath so that it can be used inside the adapter script
   // Replace the names of the bucketfs and the bucket with the ones you used.
-  %jar /buckets/bucketfs1/bucket1/virtualschema-jdbc-adapter-dist-1.7.1.jar;
+  %jar /buckets/bucketfs1/bucket1/virtualschema-jdbc-adapter-dist-1.8.2.jar;
 
   // You have to add all files of the data source jdbc driver here 
   %jar /buckets/bucketfs1/bucket1/jtds.jar;

jdbc-adapter/doc/sql_dialects/sybase.md

@@ -18,7 +18,7 @@ CREATE OR REPLACE JAVA ADAPTER SCRIPT adapter.jdbc_adapter
   AS
 
   %scriptclass com.exasol.adapter.jdbc.JdbcAdapter;
-  %jar /buckets/bucketfs1/virtualschema/virtualschema-jdbc-adapter-dist-1.7.1.jar;
+  %jar /buckets/bucketfs1/virtualschema/virtualschema-jdbc-adapter-dist-1.8.2.jar;
   %jar /buckets/bucketfs1/virtualschema/jtds-1.3.1.jar;
 /
 ```

jdbc-adapter/doc/sql_dialects/teradata.md

@@ -22,7 +22,7 @@ CREATE OR REPLACE JAVA ADAPTER SCRIPT adapter.jdbc_adapter
 
   // This will add the adapter jar to the classpath so that it can be used inside the adapter script
   // Replace the names of the bucketfs and the bucket with the ones you used.
-  %jar /buckets/bucketfs1/bucket1/virtualschema-jdbc-adapter-dist-1.7.1.jar;
+  %jar /buckets/bucketfs1/bucket1/virtualschema-jdbc-adapter-dist-1.8.2.jar;
 
   // You have to add all files of the data source jdbc driver here (e.g. MySQL or Hive)
   %jar /buckets/bucketfs1/bucket1/terajdbc4.jar;

jdbc-adapter/integration-test-data/integration-test-db2.yaml

@@ -5,7 +5,7 @@ general:
   debugAddress:    '192.168.0.12:3000'  # Address which will be defined as DEBUG_ADDRESS in the virtual schemas
   bucketFsUrl:  http://exasol-host:2580/bucket1
   bucketFsPassword: bucket1
-  jdbcAdapterPath:  /buckets/bucketfs1/bucket1/virtualschema-jdbc-adapter-dist-1.7.1.jar
+  jdbcAdapterPath:  /buckets/bucketfs1/bucket1/virtualschema-jdbc-adapter-dist-1.8.2.jar
 
 exasol:
   runIntegrationTests: true

jdbc-adapter/integration-test-data/integration-test-sample.yaml

@@ -5,7 +5,7 @@ general:
   debugAddress:    '192.168.0.12:3000'  # Address which will be defined as DEBUG_ADDRESS in the virtual schemas
   bucketFsUrl:  http://exasol-host:2580/bucket1
   bucketFsPassword: bucket1
-  jdbcAdapterPath:  /buckets/bucketfs1/bucket1/virtualschema-jdbc-adapter-dist-1.7.1.jar
+  jdbcAdapterPath:  /buckets/bucketfs1/bucket1/virtualschema-jdbc-adapter-dist-1.8.2.jar
 
 exasol:
   runIntegrationTests: true

jdbc-adapter/integration-test-data/integration-test-travis.yaml

@@ -1,11 +1,12 @@
 # Configuration file for integration tests run by `run_integration_tests.sh`
 
 general:
-  debug:            false
-  debugAddress:     ''
-  bucketFsUrl:      http://127.0.0.1:6594/default
-  bucketFsPassword: write
-  jdbcAdapterPath:  /buckets/bfsdefault/default/virtualschema-jdbc-adapter-dist-1.7.1.jar
+  debug:                   false
+  debugAddress:            ''
+  bucketFsUrl:             http://127.0.0.1:6594/default
+  bucketFsPassword:        write
+  jdbcAdapterPath:         /buckets/bfsdefault/default/virtualschema-jdbc-adapter-dist-1.8.2.jar
+  additionalJDBCDriverDir: /var/tmp/vstest/drivers/
 
 exasol:
   runIntegrationTests: true
@@ -24,3 +25,13 @@ postgresql:
   dockerPortMapping:      45432:5432
   dockerName:             testpg
   dockerConnectionString: jdbc:postgresql://DBHOST:5432/postgres
+
+oracle:
+  runIntegrationTests:    false
+  jdbcDriverPath:         /buckets/bfsdefault/default/drivers/jdbc/ORACLE/ojdbc7.jar;
+  connectionString:       jdbc:oracle:thin:@localhost:1521/XE
+  user:                   system
+  password:               myorapwd
+  dockerName:             myora
+  dockerConnectionString: jdbc:oracle:thin:@DBHOST:1521/XE
+  instantclientDir:       /var/tmp/vstest/instantclient/

jdbc-adapter/integration-test-data/oracle-testdata.sql

@@ -91,3 +91,13 @@ INSERT INTO LOADER.TYPE_TEST (c3, c5, c7, c_binfloat, c17) VALUES (
   -- c_float126
   -- c_long
 );
+
+create table ts_t(a timestamp, b timestamp with local time zone, c timestamp with time zone);
+insert into ts_t values (timestamp '2018-01-01 11:00:00', timestamp '2018-01-01 11:00:00 +01:00', timestamp '2018-01-01 11:00:00 +01:00');
+
+create table t1(x int, y varchar(100));
+insert into t1 values (1,'aaa');
+insert into t1 values (2,'bbb');
+create table t2(x int, y varchar(100));
+insert into t2 values (2,'bbb');
+insert into t2 values (3,'ccc');

jdbc-adapter/integration-test-data/run_integration_tests.sh

@@ -41,12 +41,31 @@ deploy_jdbc_drivers() {
 	bucket_fs_url=$(awk '/bucketFsUrl/{print $NF}' $config)
 	bfs_url_no_http=$(echo $bucket_fs_url | awk -F/ '{for(i=3;i<=NF;++i)printf "%s/",$i}')
 	bucket_fs_pwd=$(awk '/bucketFsPassword/{print $NF}' $config)
-	bucket_fs_upload_url=http://w:$bucket_fs_pwd@$bfs_url_no_http/drivers/jdbc/
+	bucket_fs_upload_url=http://w:$bucket_fs_pwd@$bfs_url_no_http/drivers/
+	#upload drivers that are part of the repository
 	for d in $jdbc_driver_dir/*
 	do
 	    db_driver=$(basename $d)
-	    find $jdbc_driver_dir/$db_driver -type f -exec curl -X PUT -T {} $bucket_fs_upload_url/$db_driver/ \;
+	    find $jdbc_driver_dir/$db_driver -type f -exec curl -X PUT -T {} $bucket_fs_upload_url/jdbc/$db_driver/ \;
 	done
+	#upload additional (local) drivers
+	additional_jdbc_driver_dir=$(awk '/additionalJDBCDriverDir/{print $NF}' $config)
+	if [ -d "$additional_jdbc_driver_dir" ]; then
+	    for d in $additional_jdbc_driver_dir/*
+	    do
+	        db_driver=$(basename $d)
+	        find $additional_jdbc_driver_dir/$db_driver -type f -exec curl -X PUT -T {} $bucket_fs_upload_url/jdbc/$db_driver/ \;
+	    done
+	fi
+	#deploy oracle instantclient
+	instantclient_dir=$(awk '/instantclientDir/{print $NF}' $config)
+	instantclient_path=$instantclient_dir/instantclient-basic-linux.x64-12.1.0.2.0.zip
+	if [ -f $instantclient_path ]; then
+	    curl -X PUT -T $instantclient_path $bucket_fs_upload_url/oracle/
+	fi
+	#workaround for https://github.com/exasol/docker-db/issues/26
+	docker exec -d exasoldb mkdir -p /exa/data/bucketfs/default/drivers
+	docker exec -d exasoldb ln -s /exa/data/bucketfs/bfsdefault/.dest/default/drivers/jdbc /exa/data/bucketfs/default/drivers/jdbc
 }
 
 replace_hosts_with_ips_in_config() {
@@ -55,6 +74,7 @@ replace_hosts_with_ips_in_config() {
 
 start_remote_dbs() {
 	$docker_helper --run $config
+	sleep 10
 }
 
 cleanup_remote_dbs() {

jdbc-adapter/integration-test-data/socker.py

@@ -7,21 +7,31 @@
 
 def docker_run(config):
     for db, properties in config.items():
-        if 'dockerImage' in properties:
-            cmd = "docker run -d -p {port_map} --name {name} {image}:{version}".format(
-                port_map = properties['dockerPortMapping'],
-                name = properties['dockerName'],
-                image = properties['dockerImage'],
-                version = properties['dockerImageVersion'])
-            print(cmd)
-            run(cmd)
+        if properties.get('runIntegrationTests', False):
+            if 'dockerImage' in properties:
+                cmd = "docker run -d -p {port_map} --name {name} {image}:{version}".format(
+                    port_map = properties['dockerPortMapping'],
+                    name = properties['dockerName'],
+                    image = properties['dockerImage'],
+                    version = properties['dockerImageVersion'])
+                print(cmd)
+                run(cmd)
+            elif 'dockerName' in properties:
+                cmd = "docker start {name}".format(name = properties['dockerName'])
+                print(cmd)
+                run(cmd)
 
 def docker_rm(config):
     for db, properties in config.items():
-        if 'dockerImage' in properties:
-            cmd = "docker rm -f {name}".format(name = properties['dockerName'])
-            print(cmd)
-            run(cmd)
+        if properties.get('runIntegrationTests', False):
+            if 'dockerImage' in properties:
+                cmd = "docker rm -f {name}".format(name = properties['dockerName'])
+                print(cmd)
+                run(cmd)
+            elif 'dockerName' in properties:
+                cmd = "docker stop {name}".format(name = properties['dockerName'])
+                print(cmd)
+                run(cmd)
 
 def run(cmd):
     try: 
@@ -45,11 +55,12 @@ def run(cmd):
 
 def replace_hosts_in(config):
     for db, properties in config.items():
-        if 'dockerImage' in properties:
-            container_ip = get_ip_for(properties['dockerName'])
-            conn_string_with_ip = properties['dockerConnectionString'].replace(
-                'DBHOST',container_ip)
-            properties['dockerConnectionString'] = conn_string_with_ip
+        if properties.get('runIntegrationTests', False):
+            if 'dockerName' in properties:
+                container_ip = get_ip_for(properties['dockerName'])
+                conn_string_with_ip = properties['dockerConnectionString'].replace(
+                    'DBHOST',container_ip)
+                properties['dockerConnectionString'] = conn_string_with_ip
     return yaml.dump(config, default_flow_style=False)
 
 def get_ip_for(docker_name):

jdbc-adapter/local/integration-test-config.yaml

@@ -5,7 +5,7 @@ general:
   debugAddress:    '10.44.1.228:3000'  # Address which will be defined as DEBUG_ADDRESS in the virtual schemas
   bucketFsUrl:      http://localhost:2580/jars
   bucketFsPassword: public
-  jdbcAdapterPath:  /buckets/bfsdefault/jars/virtualschema-jdbc-adapter-dist-1.7.1.jar
+  jdbcAdapterPath:  /buckets/bfsdefault/jars/virtualschema-jdbc-adapter-dist-1.8.2.jar
 
 exasol:
   runIntegrationTests: true

jdbc-adapter/pom.xml

@@ -10,7 +10,7 @@
         <module>virtualschema-jdbc-adapter-dist</module>
     </modules>
     <properties>
-        <product.version>1.8.0</product.version>
+        <product.version>1.9.0</product.version>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
         <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
         <java.version>1.8</java.version>
@@ -117,4 +117,4 @@
             </plugin>
         </plugins>
     </build>
-</project>
+</project>

jdbc-adapter/virtualschema-jdbc-adapter/src/main/java/com/exasol/adapter/dialects/SqlGenerationVisitor.java

@@ -82,6 +82,10 @@ public SqlGenerationVisitor(final SqlDialect dialect, final SqlGenerationContext
         checkDialectAliases();
     }
 
+    protected SqlDialect getDialect() {
+        return dialect;
+    }
+
     protected void checkDialectAliases() {
         // Check if dialect provided invalid aliases, which would never be applied.
         for (final ScalarFunction function : this.dialect.getScalarFunctionAliases().keySet()) {

jdbc-adapter/virtualschema-jdbc-adapter/src/main/java/com/exasol/adapter/dialects/impl/OracleSqlDialect.java

@@ -2,6 +2,7 @@
 
 import com.exasol.adapter.capabilities.Capabilities;
 import com.exasol.adapter.dialects.*;
+import com.exasol.adapter.jdbc.ConnectionInformation;
 import com.exasol.adapter.metadata.DataType;
 import com.exasol.adapter.sql.AggregateFunction;
 import com.exasol.adapter.sql.ScalarFunction;
@@ -203,4 +204,21 @@ public String getStringLiteral(final String value) {
         return "'" + value.replace("'", "''") + "'";
     }
 
+    @Override
+    public String generatePushdownSql(final ConnectionInformation connectionInformation, final String columnDescription, final String pushdownSql) {
+        final ImportType importType = getContext().getImportType();
+        if (importType == ImportType.JDBC) {
+            return super.generatePushdownSql(connectionInformation, columnDescription, pushdownSql);
+        } else {
+            if ((importType != ImportType.ORA)) {
+                throw new AssertionError("OracleSqlDialect has wrong ImportType");
+            }
+            final StringBuilder oracleImportQuery = new StringBuilder();
+            oracleImportQuery.append("IMPORT FROM ORA AT ").append(connectionInformation.getOraConnectionName()).append(" ");
+            oracleImportQuery.append(connectionInformation.getCredentials());
+            oracleImportQuery.append(" STATEMENT '").append(pushdownSql.replace("'", "''")).append("'");
+            return oracleImportQuery.toString();
+        }
+    }
+
 }

jdbc-adapter/virtualschema-jdbc-adapter/src/main/java/com/exasol/adapter/dialects/impl/OracleSqlGenerationVisitor.java

@@ -1,10 +1,7 @@
 package com.exasol.adapter.dialects.impl;
 
 import com.exasol.adapter.AdapterException;
-import com.exasol.adapter.dialects.SqlDialect;
-import com.exasol.adapter.dialects.SqlGenerationContext;
-import com.exasol.adapter.dialects.SqlGenerationHelper;
-import com.exasol.adapter.dialects.SqlGenerationVisitor;
+import com.exasol.adapter.dialects.*;
 import com.exasol.adapter.jdbc.ColumnAdapterNotes;
 import com.exasol.adapter.metadata.ColumnMetadata;
 import com.exasol.adapter.metadata.DataType;
@@ -511,8 +508,9 @@ private String getColumnProjectionString(SqlColumn column, String projString) th
         if (!isDirectlyInSelectList) {
             return projString;
         }
-        String typeName = ColumnAdapterNotes.deserialize(column.getMetadata().getAdapterNotes(), column.getMetadata().getName()).getTypeName();
-        if (typeName.startsWith("TIMESTAMP") ||
+        final AbstractSqlDialect dialect = (AbstractSqlDialect) getDialect();
+        final String typeName = ColumnAdapterNotes.deserialize(column.getMetadata().getAdapterNotes(), column.getMetadata().getName()).getTypeName();
+        if ((typeName.startsWith("TIMESTAMP") && dialect.getContext().getImportType() == ImportType.JDBC) ||
             typeName.startsWith("INTERVAL") ||
             typeName.equals("BINARY_FLOAT") ||
             typeName.equals("BINARY_DOUBLE") ||
@@ -540,7 +538,11 @@ private boolean nodeRequiresCast(SqlNode node) throws AdapterException {
             if (typeName.equals("NUMBER") && column.getMetadata().getType().getExaDataType() == DataType.ExaDataType.VARCHAR) {
                 return true;
             } else {
-                return TYPE_NAMES_REQUIRING_CAST.contains(typeName);
+                for (final String typeRequiringCast : TYPE_NAMES_REQUIRING_CAST) {
+                    if (typeName.startsWith(typeRequiringCast)) {
+                        return true;
+                    }
+                }
             }
         }
         return false;