FEATURE: TMA-1630 Support Snowflake, BigQuery as input source

Dockerfile

@@ -43,16 +43,24 @@ RUN groupadd -g 48 apache \
 USER apache
 
 ADD ./bin ./bin
+ADD --chown=apache:apache ./ci ./ci
 ADD --chown=apache:apache ./lib ./lib
 ADD ./SDK_VERSION .
 ADD ./VERSION .
 ADD ./Gemfile .
 ADD ./gooddata.gemspec .
 
-RUN mkdir -p tmp
-COPY spec/lcm/redshift_driver_pom.xml tmp/pom.xml
-RUN mvn -f tmp/pom.xml clean install -P binary-packaging
-RUN cp -rf tmp/target/*.jar ./lib/gooddata/cloud_resources/redshift/drivers/
+#build redshift dependencies
+RUN mvn -f ci/redshift/pom.xml clean install -P binary-packaging
+RUN cp -rf ci/redshift/target/*.jar ./lib/gooddata/cloud_resources/redshift/drivers/
+
+#build snowflake dependencies
+RUN mvn -f ci/snowflake/pom.xml clean install -P binary-packaging
+RUN cp -rf ci/snowflake/target/*.jar ./lib/gooddata/cloud_resources/snowflake/drivers/
+
+#build bigquery dependencies
+RUN mvn -f ci/bigquery/pom.xml clean install -P binary-packaging
+RUN cp -rf ci/bigquery/target/*.jar ./lib/gooddata/cloud_resources/bigquery/drivers/
 
 RUN bundle install
 

ci/bigquery/pom.xml

@@ -0,0 +1,54 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <groupId>com.gooddata.lcm</groupId>
+    <artifactId>lcm-bigquery-driver</artifactId>
+    <version>1.0-SNAPSHOT</version>
+
+    <dependencies>
+        <dependency>
+            <groupId>com.google.auth</groupId>
+            <artifactId>google-auth-library-oauth2-http</artifactId>
+            <version>0.16.2</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.commons</groupId>
+            <artifactId>commons-text</artifactId>
+            <version>1.7</version>
+        </dependency>
+        <dependency>
+            <groupId>com.google.cloud</groupId>
+            <artifactId>google-cloud-bigquery</artifactId>
+            <version>1.102.0</version>
+        </dependency>
+    </dependencies>
+
+    <profiles>
+        <profile>
+            <id>binary-packaging</id>
+            <build>
+                <plugins>
+                    <plugin>
+                        <artifactId>maven-dependency-plugin</artifactId>
+                        <executions>
+                            <execution>
+                                <phase>package</phase>
+                                <goals>
+                                    <goal>copy-dependencies</goal>
+                                </goals>
+                                <configuration>
+                                    <outputDirectory>${project.build.directory}</outputDirectory>
+                                    <!-- compile scope gives runtime and compile dependencies (skips test deps) -->
+                                    <includeScope>runtime</includeScope>
+                                </configuration>
+                            </execution>
+                        </executions>
+                    </plugin>
+                </plugins>
+            </build>
+        </profile>
+    </profiles>
+</project>

ci/snowflake/pom.xml

@@ -0,0 +1,57 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <groupId>com.gooddata.lcm</groupId>
+    <artifactId>lcm-snowflake-driver</artifactId>
+    <version>1.0-SNAPSHOT</version>
+
+    <dependencies>
+        <dependency>
+            <groupId>net.snowflake</groupId>
+            <artifactId>snowflake-jdbc</artifactId>
+            <version>3.6.22</version>
+        </dependency>
+        <dependency>
+            <groupId>org.slf4j</groupId>
+            <artifactId>slf4j-api</artifactId>
+            <version>1.7.2</version>
+        </dependency>
+    </dependencies>
+
+    <profiles>
+        <profile>
+            <id>binary-packaging</id>
+            <build>
+                <plugins>
+                    <plugin>
+                        <artifactId>maven-dependency-plugin</artifactId>
+                        <executions>
+                            <execution>
+                                <phase>package</phase>
+                                <goals>
+                                    <goal>copy-dependencies</goal>
+                                </goals>
+                                <configuration>
+                                    <outputDirectory>${project.build.directory}</outputDirectory>
+                                    <!-- compile scope gives runtime and compile dependencies (skips test deps) -->
+                                    <includeScope>runtime</includeScope>
+                                </configuration>
+                            </execution>
+                        </executions>
+                    </plugin>
+                </plugins>
+            </build>
+        </profile>
+    </profiles>
+
+    <repositories>
+        <repository>
+            <id>my-repo1</id>
+            <name>my custom repo</name>
+            <url>https://repository.mulesoft.org/nexus/content/repositories/public/</url>
+        </repository>
+    </repositories>
+</project>

lcm.rake

@@ -118,7 +118,7 @@ end
 namespace :docker do
   desc 'Build Docker image'
   task :build do
-    Rake::Task["maven:build_redshift"].invoke
+    Rake::Task["maven:build_dependencies"].invoke
     system('docker build -f Dockerfile.jruby -t gooddata/appstore .')
   end
 
@@ -129,11 +129,15 @@ namespace :docker do
 end
 
 namespace :maven do
-  task :build_redshift do
-    system("cp -rf spec/lcm/redshift_driver_pom.xml tmp/pom.xml")
-    system('mvn -f tmp/pom.xml clean install -P binary-packaging')
-    system('cp -rf tmp/target/*.jar lib/gooddata/cloud_resources/redshift/drivers/')
-    system('rm -rf lib/gooddata/cloud_resources/redshift/drivers/lcm-redshift-driver*.jar')
+  task :build_dependencies do
+    system('mvn -f ci/snowflake/pom.xml clean install -P binary-packaging')
+    system('cp -rf ci/snowflake/target/*.jar lib/gooddata/cloud_resources/snowflake/drivers/')
+
+    system('mvn -f ci/bigquery/pom.xml clean install -P binary-packaging')
+    system('cp -rf ci/bigquery/target/*.jar lib/gooddata/cloud_resources/bigquery/drivers/')
+
+    system('mvn -f ci/redshift/pom.xml clean install -P binary-packaging')
+    system('cp -rf ci/redshift/target/*.jar lib/gooddata/cloud_resources/redshift/drivers/')
   end
 end
 

lib/gooddata/cloud_resources/bigquery/bigquery_client.rb

@@ -0,0 +1,85 @@
+# encoding: UTF-8
+#
+# Copyright (c) 2010-2019 GoodData Corporation. All rights reserved.
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+require 'securerandom'
+require 'java'
+require 'pathname'
+require_relative '../cloud_resource_client'
+
+base = Pathname(__FILE__).dirname.expand_path
+Dir.glob(base + 'drivers/*.jar').each do |file|
+  require file unless file.start_with?('lcm-bigquery-driver')
+end
+
+java_import 'com.google.auth.oauth2.ServiceAccountCredentials'
+java_import 'com.google.cloud.bigquery.BigQuery'
+java_import 'com.google.cloud.bigquery.BigQueryOptions'
+java_import 'com.google.cloud.bigquery.FieldList'
+java_import 'com.google.cloud.bigquery.FieldValue'
+java_import 'com.google.cloud.bigquery.FieldValueList'
+java_import 'com.google.cloud.bigquery.QueryJobConfiguration'
+java_import 'com.google.cloud.bigquery.TableResult'
+java_import 'org.apache.commons.text.StringEscapeUtils'
+
+module GoodData
+  module CloudResources
+    class BigQueryClient < CloudResourceClient
+      class << self
+        def accept?(type)
+          type == 'bigquery'
+        end
+      end
+
+      def initialize(options = {})
+        raise("Data Source needs a client to BigQuery to be able to query the storage but 'bigquery_client' is empty.") unless options['bigquery_client']
+
+        if options['bigquery_client']['connection'].is_a?(Hash)
+          @project = options['bigquery_client']['connection']['project']
+          @schema = options['bigquery_client']['connection']['schema'] || 'public'
+          @authentication = options['bigquery_client']['connection']['authentication']
+        else
+          raise('Missing connection info for BigQuery client')
+
+        end
+      end
+
+      def realize_query(query, _params)
+        GoodData.gd_logger.info("Realize SQL query: type=bigquery status=started")
+
+        client = create_client
+        filename = "#{SecureRandom.urlsafe_base64(6)}_#{Time.now.to_i}.csv"
+        measure = Benchmark.measure do
+          query_config = QueryJobConfiguration.newBuilder(query).setDefaultDataset(@schema).build
+          table_result = client.query(query_config)
+
+          if table_result.getTotalRows > 0
+            result = table_result.iterateAll
+            field_list = table_result.getSchema.getFields
+            col_count = field_list.size
+            CSV.open(filename, 'wb', :force_quotes => true) do |csv|
+              csv << Array(1..col_count).map { |i| field_list.get(i - 1).getName } # build the header
+              result.each do |row|
+                csv << Array(1..col_count).map { |i| row.get(i - 1).getStringValue }
+              end
+            end
+          end
+        end
+        GoodData.gd_logger.info("Realize SQL query: type=redshift status=finished duration=#{measure.real}")
+        filename
+      end
+
+      private
+
+      def create_client
+        GoodData.logger.info "Setting up connection to BigQuery"
+        client_email = @authentication['serviceAccount']['clientEmail']
+        private_key = @authentication['serviceAccount']['privateKey']
+        credentials = ServiceAccountCredentials.fromPkcs8(nil, client_email, StringEscapeUtils.unescapeJson(private_key), nil, nil)
+        BigQueryOptions.newBuilder.setProjectId(@project).setCredentials(credentials).build.getService
+      end
+    end
+  end
+end

lib/gooddata/cloud_resources/snowflake/snowflake_client.rb

@@ -0,0 +1,83 @@
+# encoding: UTF-8
+#
+# Copyright (c) 2010-2019 GoodData Corporation. All rights reserved.
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+require 'securerandom'
+require 'java'
+require 'pathname'
+require_relative '../cloud_resource_client'
+
+base = Pathname(__FILE__).dirname.expand_path
+Dir.glob(base + 'drivers/*.jar').each do |file|
+  require file unless file.start_with?('lcm-snowflake-driver')
+end
+
+module GoodData
+  module CloudResources
+    class SnowflakeClient < CloudResourceClient
+      class << self
+        def accept?(type)
+          type == 'snowflake'
+        end
+      end
+
+      def initialize(options = {})
+        raise("Data Source needs a client to Snowflake to be able to query the storage but 'snowflake_client' is empty.") unless options['snowflake_client']
+
+        if options['snowflake_client']['connection'].is_a?(Hash)
+          @database = options['snowflake_client']['connection']['database']
+          @schema = options['snowflake_client']['connection']['schema'] || 'public'
+          @warehouse = options['snowflake_client']['connection']['warehouse']
+          @url = options['snowflake_client']['connection']['url']
+          @authentication = options['snowflake_client']['connection']['authentication']
+        else
+          raise('Missing connection info for Snowflake client')
+
+        end
+
+        Java.net.snowflake.client.jdbc.SnowflakeDriver
+      end
+
+      def realize_query(query, _params)
+        GoodData.gd_logger.info("Realize SQL query: type=snowflake status=started")
+
+        connect
+        filename = "#{SecureRandom.urlsafe_base64(6)}_#{Time.now.to_i}.csv"
+        measure = Benchmark.measure do
+          statement = @connection.create_statement
+
+          has_result = statement.execute(query)
+          if has_result
+            result = statement.get_result_set
+            metadata = result.get_meta_data
+            col_count = metadata.column_count
+            CSV.open(filename, 'wb', :force_quotes => true) do |csv|
+              csv << Array(1..col_count).map { |i| metadata.get_column_name(i) } # build the header
+              csv << Array(1..col_count).map { |i| result.get_string(i) } while result.next
+            end
+          end
+        end
+        GoodData.gd_logger.info("Realize SQL query: type=snowflake status=finished duration=#{measure.real}")
+        filename
+      ensure
+        @connection.close unless @connection.nil?
+        @connection = nil
+      end
+
+      def connect
+        GoodData.logger.info "Setting up connection to Snowflake #{@url}"
+
+        prop = java.util.Properties.new
+        prop.setProperty('user', @authentication['basic']['userName'])
+        prop.setProperty('password', @authentication['basic']['password'])
+        prop.setProperty('schema', @schema)
+        prop.setProperty('warehouse', @warehouse)
+        prop.setProperty('db', @database)
+
+        @connection = java.sql.DriverManager.getConnection(@url, prop)
+      end
+    end
+  end
+end

lib/gooddata/helpers/data_helper.rb

@@ -44,7 +44,7 @@ def realize(params = {})
           realize_link
         when 's3'
           realize_s3(params)
-        when 'redshift'
+        when 'redshift', 'snowflake', 'bigquery'
           raise GoodData::InvalidEnvError, "DataSource does not support type \"#{source}\" on the platform #{RUBY_PLATFORM}" unless RUBY_PLATFORM =~ /java/
 
           require_relative '../cloud_resources/cloud_resources'

spec/data/bigquery_data.csv

@@ -0,0 +1,5 @@
+"EmployeeID","Fullname","Salary","DOB","Departments","x__client_id","x__timestamp","x__deleted"
+"E101","Name 01","4000","1970-01-20","101","mtt_testing_01","1557396000.001","false"
+"E102","Name 02","3000","1918-05-26","102","mtt_testing_01","1557396000.002","false"
+"E103","Name 03","5000","1960-02-03","103","mtt_testing_01","1557396000.003","false"
+"E104","Name 04","5000","1960-02-03","104","MTT-Client02","1557396000.004","false"

spec/data/snowflake_data.csv

@@ -0,0 +1,2 @@
+"CP__CUSTKEY","A__NAME","D__DATEADDED","X__TIMESTAMP"
+"cus3","phong","2019-03-05",""