Enhancement/131 refactor metadata reading (#153)

* #45: Rewrote parts of the JDBC adapter to use the new commons code. * #121: Refactored request dispatching * #124: Read Redshift metadata * #131: Refactored general metadata reading * #155: Moved validation logic to dialects * #163: Fixed Hive integration tests
exasol · May 2, 2019 · adbc826 · adbc826
1 parent 598b153
commit adbc826
Show file tree

Hide file tree

Showing 212 changed files with 10,612 additions and 6,630 deletions.
diff --git a/.gitignore b/.gitignore
@@ -3,7 +3,7 @@
 # Eclipse and Maven
 .classpath
 .project
-.settings
+# .settings : we need Eclipse settings for code formatter and clean-up rules
 target
 .cache
 dependency-reduced-pom.xml
@@ -18,4 +18,4 @@ dependency-reduced-pom.xml
 *.swp
 **/local
 Scripts
-.dbeaver*
+.dbeaver*
diff --git a/jdbc-adapter/.settings/org.eclipse.core.resources.prefs b/jdbc-adapter/.settings/org.eclipse.core.resources.prefs
@@ -0,0 +1,2 @@
+eclipse.preferences.version=1
+encoding/<project>=UTF-8
diff --git a/jdbc-adapter/doc/deploying_the_virtual_schema_adapter.md b/jdbc-adapter/doc/deploying_the_virtual_schema_adapter.md
@@ -23,7 +23,7 @@ cd virtual-schemas/jdbc-adapter/
 mvn clean -DskipTests package
 ```
 
-The resulting fat JAR is stored in `virtualschema-jdbc-adapter-dist/target/virtualschema-jdbc-adapter-dist-1.9.1.jar`.
+The resulting fat JAR is stored in `virtualschema-jdbc-adapter-dist/target/virtualschema-jdbc-adapter-dist-1.10.0.jar`.
 
 ## Uploading the Adapter JAR Archive
 
@@ -42,8 +42,8 @@ Following steps are required to upload a file to a bucket:
 1. Now upload the file into this bucket, e.g. using curl (adapt the hostname, BucketFS port, bucket name and bucket write password).
 
 ```bash
-curl -X PUT -T virtualschema-jdbc-adapter-dist/target/virtualschema-jdbc-adapter-dist-1.9.1.jar \
- http://w:write-password@your.exasol.host.com:2580/bucket1/virtualschema-jdbc-adapter-dist-1.9.1.jar
+curl -X PUT -T virtualschema-jdbc-adapter-dist/target/virtualschema-jdbc-adapter-dist-1.10.0.jar \
+ http://w:write-password@your.exasol.host.com:2580/bucket1/virtualschema-jdbc-adapter-dist-1.10.0.jar
 ```
 
 See chapter 3.6.4. "The synchronous cluster file system BucketFS" in the EXASolution User Manual for more details about BucketFS.
@@ -75,7 +75,7 @@ CREATE JAVA ADAPTER SCRIPT adapter.jdbc_adapter AS
 
   // This will add the adapter jar to the classpath so that it can be used inside the adapter script
   // Replace the names of the bucketfs and the bucket with the ones you used.
-  %jar /buckets/your-bucket-fs/your-bucket/virtualschema-jdbc-adapter-dist-1.9.1.jar;
+  %jar /buckets/your-bucket-fs/your-bucket/virtualschema-jdbc-adapter-dist-1.10.0.jar;
 
   // You have to add all files of the data source jdbc driver here (e.g. Hive JDBC driver files)
   %jar /buckets/your-bucket-fs/your-bucket/name-of-data-source-jdbc-driver.jar;

diff --git a/jdbc-adapter/doc/developing_an_sql_dialect.md b/jdbc-adapter/doc/developing_an_sql_dialect.md
@@ -25,7 +25,7 @@ To register your new dialect add it to the list in [sql_dialects.properties](../
 ```properties
 com.exasol.adapter.dialects.supported=\
 ...
-com.exasol.adapter.dialects.impl.MyAweSomeSqlDialect
+com.exasol.adapter.dialects.myawesomedialect.MyAweSomeSqlDialect
 ```
 
 For tests or in case you want to exclude existing dialects in certain scenarios you can override the contents of this file by setting the system property `com.exasol.adapter.dialects.supported`.
@@ -292,7 +292,7 @@ CREATE OR REPLACE JAVA ADAPTER SCRIPT adapter.jdbc_adapter
 
   // This will add the adapter jar to the classpath so that it can be used inside the adapter script
   // Replace the names of the bucketfs and the bucket with the ones you used.
-  %jar /buckets/bucketfs1/bucket1/virtualschema-jdbc-adapter-dist-1.9.1.jar;
+  %jar /buckets/bucketfs1/bucket1/virtualschema-jdbc-adapter-dist-1.10.0.jar;
 
   // You have to add all files of the data source jdbc driver here (e.g. MySQL or Hive)
 

diff --git a/jdbc-adapter/doc/sql_dialects/db2.md b/jdbc-adapter/doc/sql_dialects/db2.md
@@ -46,7 +46,7 @@ CREATE or replace JAVA ADAPTER SCRIPT adapter.jdbc_adapter AS
 
   // This will add the adapter jar to the classpath so that it can be used inside the adapter script
   // Replace the names of the bucketfs and the bucket with the ones you used.
-  %jar /buckets/bucketfs1/bucket1/virtualschema-jdbc-adapter-dist-1.9.1.jar;
+  %jar /buckets/bucketfs1/bucket1/virtualschema-jdbc-adapter-dist-1.10.0.jar;
 
   // DB2 Driver files
   %jar /buckets/bucketfs1/bucket1/db2jcc4.jar;

diff --git a/jdbc-adapter/doc/sql_dialects/exasol.md b/jdbc-adapter/doc/sql_dialects/exasol.md
@@ -17,7 +17,7 @@ After uploading the adapter jar, the adapter script can be created as follows:
 CREATE SCHEMA adapter;
 CREATE JAVA ADAPTER SCRIPT adapter.jdbc_adapter AS
   %scriptclass com.exasol.adapter.jdbc.JdbcAdapter;
-  %jar /buckets/your-bucket-fs/your-bucket/virtualschema-jdbc-adapter-dist-1.9.1.jar;
+  %jar /buckets/your-bucket-fs/your-bucket/virtualschema-jdbc-adapter-dist-1.10.0.jar;
 /
 ```
 

diff --git a/jdbc-adapter/doc/sql_dialects/hive.md b/jdbc-adapter/doc/sql_dialects/hive.md
@@ -23,7 +23,7 @@ CREATE SCHEMA adapter;
 CREATE  JAVA  ADAPTER SCRIPT jdbc_adapter AS
   %scriptclass com.exasol.adapter.jdbc.JdbcAdapter;
 
-  %jar /buckets/bucketfs1/bucket1/virtualschema-jdbc-adapter-dist-1.9.1.jar;
+  %jar /buckets/bucketfs1/bucket1/virtualschema-jdbc-adapter-dist-1.10.0.jar;
 
   %jar /buckets/bucketfs1/bucket1/HiveJDBC41.jar;
 /

diff --git a/jdbc-adapter/doc/sql_dialects/impala.md b/jdbc-adapter/doc/sql_dialects/impala.md
@@ -22,7 +22,7 @@ CREATE SCHEMA adapter;
 CREATE  JAVA  ADAPTER SCRIPT jdbc_adapter AS
   %scriptclass com.exasol.adapter.jdbc.JdbcAdapter;
 
-  %jar /buckets/bucketfs1/bucket1/virtualschema-jdbc-adapter-dist-1.9.1.jar;
+  %jar /buckets/bucketfs1/bucket1/virtualschema-jdbc-adapter-dist-1.10.0.jar;
 
   %jar /buckets/bucketfs1/bucket1/hive_metastore.jar;
   %jar /buckets/bucketfs1/bucket1/hive_service.jar;

diff --git a/jdbc-adapter/doc/sql_dialects/oracle.md b/jdbc-adapter/doc/sql_dialects/oracle.md
@@ -32,7 +32,7 @@ CREATE JAVA ADAPTER SCRIPT adapter.jdbc_oracle AS
 
   // You need to replace `your-bucket-fs` and `your-bucket` to match the actual location
   // of the adapter jar.
-  %jar /buckets/your-bucket-fs/your-bucket/virtualschema-jdbc-adapter-dist-1.9.1.jar;
+  %jar /buckets/your-bucket-fs/your-bucket/virtualschema-jdbc-adapter-dist-1.10.0.jar;
 
   // Add the oracle jdbc driver to the classpath
   %jar /buckets/bucketfs1/bucket1/ojdbc7-12.1.0.2.jar

diff --git a/jdbc-adapter/doc/sql_dialects/postgresql.md b/jdbc-adapter/doc/sql_dialects/postgresql.md
@@ -15,7 +15,7 @@ CREATE OR REPLACE JAVA ADAPTER SCRIPT adapter.jdbc_adapter
 
   // This will add the adapter jar to the classpath so that it can be used inside the adapter script
   // Replace the names of the bucketfs and the bucket with the ones you used.
-  %jar /buckets/bucketfs1/bucket1/virtualschema-jdbc-adapter-dist-1.9.1.jar;
+  %jar /buckets/bucketfs1/bucket1/virtualschema-jdbc-adapter-dist-1.10.0.jar;
 
   // You have to add all files of the data source jdbc driver here (e.g. MySQL or Hive)
   %jar /buckets/bucketfs1/bucket1/postgresql-42.0.0.jar;

diff --git a/jdbc-adapter/doc/sql_dialects/redshift.md b/jdbc-adapter/doc/sql_dialects/redshift.md
@@ -21,7 +21,7 @@ CREATE OR REPLACE JAVA ADAPTER SCRIPT adapter.jdbc_adapter
 
   // This will add the adapter jar to the classpath so that it can be used inside the adapter script
   // Replace the names of the bucketfs and the bucket with the ones you used.
-  %jar /buckets/bucketfs1/bucket1/virtualschema-jdbc-adapter-dist-1.9.1.jar;
+  %jar /buckets/bucketfs1/bucket1/virtualschema-jdbc-adapter-dist-1.10.0.jar;
 
   // You have to add all files of the data source jdbc driver here (e.g. MySQL or Hive)
 

diff --git a/jdbc-adapter/doc/sql_dialects/sql_server.md b/jdbc-adapter/doc/sql_dialects/sql_server.md
@@ -17,7 +17,7 @@ CREATE OR REPLACE JAVA ADAPTER SCRIPT adapter.sql_server_jdbc_adapter
 
   // This will add the adapter jar to the classpath so that it can be used inside the adapter script
   // Replace the names of the bucketfs and the bucket with the ones you used.
-  %jar /buckets/bucketfs1/bucket1/virtualschema-jdbc-adapter-dist-1.9.1.jar;
+  %jar /buckets/bucketfs1/bucket1/virtualschema-jdbc-adapter-dist-1.10.0.jar;
 
   // You have to add all files of the data source jdbc driver here 
   %jar /buckets/bucketfs1/bucket1/jtds.jar;

diff --git a/jdbc-adapter/doc/sql_dialects/sybase.md b/jdbc-adapter/doc/sql_dialects/sybase.md
@@ -18,7 +18,7 @@ CREATE OR REPLACE JAVA ADAPTER SCRIPT adapter.jdbc_adapter
   AS
 
   %scriptclass com.exasol.adapter.jdbc.JdbcAdapter;
-  %jar /buckets/bucketfs1/virtualschema/virtualschema-jdbc-adapter-dist-1.9.1.jar;
+  %jar /buckets/bucketfs1/virtualschema/virtualschema-jdbc-adapter-dist-1.10.0.jar;
   %jar /buckets/bucketfs1/virtualschema/jtds-1.3.1.jar;
 /
 ```

diff --git a/jdbc-adapter/doc/sql_dialects/teradata.md b/jdbc-adapter/doc/sql_dialects/teradata.md
@@ -22,7 +22,7 @@ CREATE OR REPLACE JAVA ADAPTER SCRIPT adapter.jdbc_adapter
 
   // This will add the adapter jar to the classpath so that it can be used inside the adapter script
   // Replace the names of the bucketfs and the bucket with the ones you used.
-  %jar /buckets/bucketfs1/bucket1/virtualschema-jdbc-adapter-dist-1.9.1.jar;
+  %jar /buckets/bucketfs1/bucket1/virtualschema-jdbc-adapter-dist-1.10.0.jar;
 
   // You have to add all files of the data source jdbc driver here (e.g. MySQL or Hive)
   %jar /buckets/bucketfs1/bucket1/terajdbc4.jar;

diff --git a/jdbc-adapter/integration-test-data/integration-test-db2.yaml b/jdbc-adapter/integration-test-data/integration-test-db2.yaml
@@ -5,7 +5,7 @@ general:
   debugAddress:    '192.168.0.12:3000'  # Address which will be defined as DEBUG_ADDRESS in the virtual schemas
   bucketFsUrl:  http://exasol-host:2580/bucket1
   bucketFsPassword: bucket1
-  jdbcAdapterPath:  /buckets/bucketfs1/bucket1/virtualschema-jdbc-adapter-dist-1.9.1.jar
+  jdbcAdapterPath:  /buckets/bucketfs1/bucket1/virtualschema-jdbc-adapter-dist-1.10.0.jar
 
 exasol:
   runIntegrationTests: true

diff --git a/jdbc-adapter/integration-test-data/integration-test-sample.yaml b/jdbc-adapter/integration-test-data/integration-test-sample.yaml
@@ -5,7 +5,7 @@ general:
   debugAddress:    '192.168.0.12:3000'  # Address which will be defined as DEBUG_ADDRESS in the virtual schemas
   bucketFsUrl:  http://exasol-host:2580/bucket1
   bucketFsPassword: bucket1
-  jdbcAdapterPath:  /buckets/bucketfs1/bucket1/virtualschema-jdbc-adapter-dist-1.9.1.jar
+  jdbcAdapterPath:  /buckets/bucketfs1/bucket1/virtualschema-jdbc-adapter-dist-1.10.0.jar
 
 exasol:
   runIntegrationTests: true

diff --git a/jdbc-adapter/integration-test-data/integration-test-travis.yaml b/jdbc-adapter/integration-test-data/integration-test-travis.yaml
@@ -4,7 +4,7 @@ general:
   debug:                   false
   debugAddress:            ''
   bucketFsUrl:             http://127.0.0.1:6594/default
-  jdbcAdapterPath:         /buckets/bfsdefault/default/virtualschema-jdbc-adapter-dist-1.9.1.jar
+  jdbcAdapterPath:         /buckets/bfsdefault/default/virtualschema-jdbc-adapter-dist-1.10.0.jar
   additionalJDBCDriverDir: /vagrant/drivers/
 
 exasol:

diff --git a/jdbc-adapter/integration-test-data/socker.py b/jdbc-adapter/integration-test-data/socker.py
@@ -72,6 +72,7 @@ def docker_rm(config):
                 run(cmd)
 
 def run(cmd, working_dir=None):
+    p = None
     try: 
         p = subprocess.Popen(
             cmd,
@@ -94,12 +95,11 @@ def run(cmd, working_dir=None):
 
 def replace_hosts_in(config):
     for db, properties in config.items():
-        if properties.get('runIntegrationTests', False):
-            if 'dockerName' in properties:
-                container_ip = get_ip_for(properties['dockerName'])
-                conn_string_with_ip = properties['dockerConnectionString'].replace(
-                    'DBHOST',container_ip)
-                properties['dockerConnectionString'] = conn_string_with_ip
+        if properties.get('runIntegrationTests', False) and 'dockerName' in properties:
+            container_ip = get_ip_for(properties['dockerName'])
+            conn_string_with_ip = properties['dockerConnectionString'].replace(
+                'DBHOST',container_ip)
+            properties['dockerConnectionString'] = conn_string_with_ip
     return yaml.dump(config, default_flow_style=False)
 
 def get_ip_for(docker_name):

diff --git a/jdbc-adapter/launch/virtual-schema all tests.launch b/jdbc-adapter/launch/virtual-schema all tests.launch
@@ -9,7 +9,14 @@
 <listAttribute key="org.eclipse.debug.ui.favoriteGroups">
 <listEntry value="org.eclipse.eclemma.ui.launchGroup.coverage"/>
 </listAttribute>
+<listAttribute key="org.eclipse.eclemma.core.SCOPE_IDS">
+<listEntry value="=virtualschema-jdbc-adapter/src\/main\/java"/>
+</listAttribute>
 <stringAttribute key="org.eclipse.jdt.junit.CONTAINER" value="=virtualschema-jdbc-adapter"/>
+<stringAttribute key="org.eclipse.jdt.junit.EXCLUDE_TAGS" value="integration"/>
+<booleanAttribute key="org.eclipse.jdt.junit.HAS_EXCLUDE_TAGS" value="true"/>
+<booleanAttribute key="org.eclipse.jdt.junit.HAS_INCLUDE_TAGS" value="false"/>
+<stringAttribute key="org.eclipse.jdt.junit.INCLUDE_TAGS" value=""/>
 <booleanAttribute key="org.eclipse.jdt.junit.KEEPRUNNING_ATTR" value="false"/>
 <stringAttribute key="org.eclipse.jdt.junit.TESTNAME" value=""/>
 <stringAttribute key="org.eclipse.jdt.junit.TEST_KIND" value="org.eclipse.jdt.junit.loader.junit5"/>

diff --git a/jdbc-adapter/launch/virtual-schema clean package.launch b/jdbc-adapter/launch/virtual-schema clean package.launch
@@ -0,0 +1,20 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<launchConfiguration type="org.eclipse.m2e.Maven2LaunchConfigurationType">
+<booleanAttribute key="M2_DEBUG_OUTPUT" value="false"/>
+<stringAttribute key="M2_GOALS" value="clean package"/>
+<booleanAttribute key="M2_NON_RECURSIVE" value="false"/>
+<booleanAttribute key="M2_OFFLINE" value="false"/>
+<stringAttribute key="M2_PROFILES" value=""/>
+<listAttribute key="M2_PROPERTIES"/>
+<stringAttribute key="M2_RUNTIME" value="EMBEDDED"/>
+<booleanAttribute key="M2_SKIP_TESTS" value="false"/>
+<intAttribute key="M2_THREADS" value="1"/>
+<booleanAttribute key="M2_UPDATE_SNAPSHOTS" value="false"/>
+<stringAttribute key="M2_USER_SETTINGS" value=""/>
+<booleanAttribute key="M2_WORKSPACE_RESOLUTION" value="false"/>
+<listAttribute key="org.eclipse.debug.ui.favoriteGroups">
+<listEntry value="org.eclipse.debug.ui.launchGroup.run"/>
+</listAttribute>
+<stringAttribute key="org.eclipse.jdt.launching.JRE_CONTAINER" value="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/java-1.8.0-openjdk-1.8.0.201.b09-2.fc29.x86_64"/>
+<stringAttribute key="org.eclipse.jdt.launching.WORKING_DIRECTORY" value="${workspace_loc:/jdbc-adapter}"/>
+</launchConfiguration>
diff --git a/jdbc-adapter/launch/virtual-schema clean.launch b/jdbc-adapter/launch/virtual-schema clean.launch
@@ -0,0 +1,20 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<launchConfiguration type="org.eclipse.m2e.Maven2LaunchConfigurationType">
+<booleanAttribute key="M2_DEBUG_OUTPUT" value="false"/>
+<stringAttribute key="M2_GOALS" value="clean"/>
+<booleanAttribute key="M2_NON_RECURSIVE" value="false"/>
+<booleanAttribute key="M2_OFFLINE" value="false"/>
+<stringAttribute key="M2_PROFILES" value=""/>
+<listAttribute key="M2_PROPERTIES"/>
+<stringAttribute key="M2_RUNTIME" value="EMBEDDED"/>
+<booleanAttribute key="M2_SKIP_TESTS" value="false"/>
+<intAttribute key="M2_THREADS" value="1"/>
+<booleanAttribute key="M2_UPDATE_SNAPSHOTS" value="false"/>
+<stringAttribute key="M2_USER_SETTINGS" value=""/>
+<booleanAttribute key="M2_WORKSPACE_RESOLUTION" value="false"/>
+<listAttribute key="org.eclipse.debug.ui.favoriteGroups">
+<listEntry value="org.eclipse.debug.ui.launchGroup.run"/>
+</listAttribute>
+<stringAttribute key="org.eclipse.jdt.launching.JRE_CONTAINER" value="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/java-1.8.0-openjdk-1.8.0.201.b09-2.fc29.x86_64"/>
+<stringAttribute key="org.eclipse.jdt.launching.WORKING_DIRECTORY" value="${workspace_loc:/jdbc-adapter}"/>
+</launchConfiguration>
diff --git a/jdbc-adapter/local/integration-test-config.yaml b/jdbc-adapter/local/integration-test-config.yaml
@@ -5,7 +5,7 @@ general:
   debugAddress:    '10.44.1.228:3000'  # Address which will be defined as DEBUG_ADDRESS in the virtual schemas
   bucketFsUrl:      http://localhost:2580/jars
   bucketFsPassword: public
-  jdbcAdapterPath:  /buckets/bfsdefault/jars/virtualschema-jdbc-adapter-dist-1.9.1.jar
+  jdbcAdapterPath:  /buckets/bfsdefault/jars/virtualschema-jdbc-adapter-dist-1.10.0.jar
 
 exasol:
   runIntegrationTests: true

diff --git a/jdbc-adapter/pom.xml b/jdbc-adapter/pom.xml
@@ -10,12 +10,14 @@
         <module>virtualschema-jdbc-adapter-dist</module>
     </modules>
     <properties>
-        <product.version>1.9.1</product.version>
+        <product.version>1.10.0</product.version>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
         <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
         <java.version>1.8</java.version>
         <junit.version>5.4.0</junit.version>
         <junit.platform.version>1.3.2</junit.platform.version>
+        <maven.surefire.version>2.22.1</maven.surefire.version>
+        <vscommon.version>4.0.0</vscommon.version>
     </properties>
     <distributionManagement>
         <repository>
@@ -47,12 +49,7 @@
         <dependency>
             <groupId>com.exasol</groupId>
             <artifactId>virtual-schema-common-java</artifactId>
-            <version>2.1.0</version>
-        </dependency>
-        <dependency>
-            <groupId>com.google.guava</groupId>
-            <artifactId>guava</artifactId>
-            <version>18.0</version>
+            <version>${vscommon.version}</version>
         </dependency>
         <dependency>
             <groupId>org.hamcrest</groupId>

diff --git a/jdbc-adapter/tools/udf_debug.py b/jdbc-adapter/tools/udf_debug.py
diff --git a/jdbc-adapter/virtualschema-jdbc-adapter-dist/.settings/org.eclipse.core.resources.prefs b/jdbc-adapter/virtualschema-jdbc-adapter-dist/.settings/org.eclipse.core.resources.prefs
@@ -0,0 +1,2 @@
+eclipse.preferences.version=1
+encoding/<project>=UTF-8