FEATURE: MSF-22129 add support mysql input source

Dockerfile

@@ -70,6 +70,10 @@ RUN cp -rf ci/postgresql/target/*.jar ./lib/gooddata/cloud_resources/postgresql/
 RUN mvn -f ci/mssql/pom.xml clean install -P binary-packaging
 RUN cp -rf ci/mssql/target/*.jar ./lib/gooddata/cloud_resources/mssql/drivers/
 
+#build mysql dependencies
+RUN mvn -f ci/mysql/pom.xml clean install -P binary-packaging
+RUN cp -rf ci/mysql/target/*.jar ./lib/gooddata/cloud_resources/mysql/drivers/
+
 RUN bundle install
 
 ARG GIT_COMMIT=unspecified

ci/mysql/pom.xml

@@ -0,0 +1,57 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <groupId>com.gooddata.lcm</groupId>
+    <artifactId>lcm-mysql-driver</artifactId>
+    <version>1.0-SNAPSHOT</version>
+
+    <dependencies>
+        <dependency>
+            <groupId>mysql</groupId>
+            <artifactId>mysql-connector-java</artifactId>
+            <version>8.0.25</version>
+        </dependency>
+        <dependency>
+            <groupId>org.slf4j</groupId>
+            <artifactId>slf4j-api</artifactId>
+            <version>1.7.2</version>
+        </dependency>
+    </dependencies>
+
+    <profiles>
+        <profile>
+            <id>binary-packaging</id>
+            <build>
+                <plugins>
+                    <plugin>
+                        <artifactId>maven-dependency-plugin</artifactId>
+                        <executions>
+                            <execution>
+                                <phase>package</phase>
+                                <goals>
+                                    <goal>copy-dependencies</goal>
+                                </goals>
+                                <configuration>
+                                    <outputDirectory>${project.build.directory}</outputDirectory>
+                                    <!-- compile scope gives runtime and compile dependencies (skips test deps) -->
+                                    <includeScope>runtime</includeScope>
+                                </configuration>
+                            </execution>
+                        </executions>
+                    </plugin>
+                </plugins>
+            </build>
+        </profile>
+    </profiles>
+
+    <repositories>
+        <repository>
+            <id>my-repo1</id>
+            <name>my custom repo</name>
+            <url>https://repository.mulesoft.org/nexus/content/repositories/public/</url>
+        </repository>
+    </repositories>
+</project>

lib/gooddata/cloud_resources/mysql/mysql_client.rb

@@ -0,0 +1,111 @@
+# encoding: UTF-8
+# frozen_string_literal: true
+#
+# Copyright (c) 2021 GoodData Corporation. All rights reserved.
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+require 'securerandom'
+require 'java'
+require 'pathname'
+require_relative '../cloud_resource_client'
+
+base = Pathname(__FILE__).dirname.expand_path
+Dir.glob(base + 'drivers/*.jar').each do |file|
+  require file unless file.start_with?('lcm-mysql-driver')
+end
+
+module GoodData
+  module CloudResources
+    class MysqlClient < CloudResourceClient
+      JDBC_MYSQL_PATTERN = %r{jdbc:mysql:\/\/([^:^\/]+)(:([0-9]+))?(\/)?}
+      MYSQL_DEFAULT_PORT = 3306
+      JDBC_MYSQL_PROTOCOL = 'jdbc:mysql://'
+      VERIFY_FULL = 'VERIFY_IDENTITY'
+      PREFER = 'PREFERRED'
+      REQUIRE = 'REQUIRED'
+      MYSQL_FETCH_SIZE = 1000
+
+      class << self
+        def accept?(type)
+          type == 'mysql'
+        end
+      end
+
+      def initialize(options = {})
+        raise("Data Source needs a client to Mysql to be able to query the storage but 'mysql_client' is empty.") unless options['mysql_client']
+
+        if options['mysql_client']['connection'].is_a?(Hash)
+          @database = options['mysql_client']['connection']['database']
+          @authentication = options['mysql_client']['connection']['authentication']
+          @ssl_mode = options['mysql_client']['connection']['sslMode']
+          raise "SSL Mode should be prefer, require and verify-full" unless @ssl_mode == 'prefer' || @ssl_mode == 'require' || @ssl_mode == 'verify-full'
+
+          @url = build_url(options['mysql_client']['connection']['url'])
+        else
+          raise('Missing connection info for Mysql client')
+        end
+
+        Java.com.mysql.cj.jdbc.Driver
+      end
+
+      def realize_query(query, _params)
+        GoodData.gd_logger.info("Realize SQL query: type=mysql status=started")
+
+        connect
+        filename = "#{SecureRandom.urlsafe_base64(6)}_#{Time.now.to_i}.csv"
+        measure = Benchmark.measure do
+          statement = @connection.create_statement
+          statement.set_fetch_size(MYSQL_FETCH_SIZE)
+          has_result = statement.execute(query)
+          if has_result
+            result = statement.get_result_set
+            metadata = result.get_meta_data
+            col_count = metadata.column_count
+            CSV.open(filename, 'wb') do |csv|
+              csv << Array(1..col_count).map { |i| metadata.get_column_name(i) } # build the header
+              csv << Array(1..col_count).map { |i| result.get_string(i)&.to_s } while result.next
+            end
+          end
+        end
+        GoodData.gd_logger.info("Realize SQL query: type=mysql status=finished duration=#{measure.real}")
+        filename
+      ensure
+        @connection&.close
+        @connection = nil
+      end
+
+      def connect
+        GoodData.logger.info "Setting up connection to Mysql #{@url}"
+
+        prop = java.util.Properties.new
+        prop.setProperty('user', @authentication['basic']['userName'])
+        prop.setProperty('password', @authentication['basic']['password'])
+
+        @connection = java.sql.DriverManager.getConnection(@url, prop)
+        @connection.set_auto_commit(false)
+      end
+
+      def build_url(url)
+        matches = url.scan(JDBC_MYSQL_PATTERN)
+        raise 'Cannot reach the url' unless matches
+
+        host = matches[0][0]
+        port = matches[0][2]&.to_i || MYSQL_DEFAULT_PORT
+
+        "#{JDBC_MYSQL_PROTOCOL}#{host}:#{port}/#{@database}?sslmode=#{get_ssl_mode(@ssl_mode)}&useCursorFetch=true"
+      end
+
+      def get_ssl_mode(ssl_mode)
+        mode = PREFER
+        if ssl_mode == 'verify-full'
+          mode =  VERIFY_FULL
+        elsif ssl_mode == 'require'
+          mode =  REQUIRE
+        end
+
+        mode
+      end
+    end
+  end
+end

lib/gooddata/cloud_resources/postgresql/postgresql_client.rb

@@ -98,7 +98,6 @@ def build_url(url)
 
         host = matches[0][0]
         port = matches[0][2]&.to_i || POSTGRES_DEFAULT_PORT
-        raise "Custom port #{port} is not supported. Remove it or use the default port '5432'" if POSTGRES_DEFAULT_PORT != port
 
         "#{JDBC_POSTGRES_PROTOCOL}#{host}:#{port}/#{@database}?sslmode=#{@ssl_mode}#{VERIFY_FULL == @ssl_mode ? SSL_JAVA_FACTORY : ''}"
       end

lib/gooddata/helpers/data_helper.rb

@@ -44,7 +44,7 @@ def realize(params = {})
           realize_link
         when 's3'
           realize_s3(params)
-        when 'redshift', 'snowflake', 'bigquery', 'postgresql', 'mssql'
+        when 'redshift', 'snowflake', 'bigquery', 'postgresql', 'mssql', 'mysql'
           raise GoodData::InvalidEnvError, "DataSource does not support type \"#{source}\" on the platform #{RUBY_PLATFORM}" unless RUBY_PLATFORM =~ /java/
           require_relative '../cloud_resources/cloud_resources'
           realize_cloud_resource(source, params)

lib/gooddata/lcm/actions/update_metric_formats.rb

@@ -75,7 +75,7 @@ def validate_input_source(input_source)
 
           modified_input_source = input_source
           case type
-          when 'ads', 'redshift', 'snowflake', 'bigquery', 'postgresql', 'mssql'
+          when 'ads', 'redshift', 'snowflake', 'bigquery', 'postgresql', 'mssql', 'mysql'
             if metric_format[:query].blank?
               GoodData.logger.warn("The metric input_source '#{type}' is missing property 'query'")
               return nil

spec/data/mysql_data.csv

@@ -0,0 +1,3 @@
+segment_id,client_id,project_title,project_id,project_token
+Segment,Client1,Client-1,,token
+Segment,Client2,Client-2,,token

spec/environment/secrets.yaml

@@ -8,6 +8,7 @@ global:
   redshift_secret_key: 1JHYwvoIQinjgdKJXQL6RNGFhx4o4M9DiQf9q4jV+dq6xCPLDBCP/8tXBd0H9y8xdOOI78mY/aOQWjiPzgizLA==
   snowflake_password: 1zg1PDRMQq2DhBG3SwQOA8/POUkeek3gurrmV4MT2Go=
   blob_storage_connection: Md/faNEbH3YOsmVCDaUJEH4/eHkABgp2X1V6BIZyMbuMxlAdlCFxY8gLqM1sJUEt2txBp7I6PmDdnG34+wV1nawRO3U9WAwr8wTPI57pkcNj0fpFN9KLycNA8ms6cVklxFlgO1WmCOqBL+wBnIbqRZ8sl9wx2BTFebt8QQSLucGMZtY0oDjy/YeG6SqH+HCzEW70ipU3whVXWJkZStIK8cHy9uxJZF88uqpphJFTQAFMwgCQQ9+vEF+mpt4xaWtF2KnRkif2a2OuYSsEStuA/A==
+  mysql_connection: PsmFcHvUtff5A2OGYWaI1+KKkQsDrThUf46DAR/m6v69yPBLI65jCZO25XV6R4xU
 development:
   dev_token: 8qWaLsyWwAUJ7MJJTBdriUvtaWKNidnzmfxVThCrL0c=
   prod_token: RitXvhFjpJ8KEpqUqZm57iV3bwVU1zBGDrXNklvwkaE=

spec/lcm/integration/spec/others/data_helper_spec.rb

@@ -179,6 +179,26 @@
   },
 }
 
+mysql_basic_params = {
+    "input_source"=> {
+        "type"=> "mysql",
+        "query"=> "SELECT DISTINCT * FROM clients",
+    },
+    "mysql_client" => {
+        "connection" => {
+            "url" => "jdbc:mysql://msf-test-database.na.intgdc.com:1435",
+            "database" => "integration_test",
+            "authentication" => {
+                "basic" => {
+                    "userName" => "mysql_integration_test",
+                    "password" => ConnectionHelper::SECRETS[:mysql_connection],
+                }
+            },
+            "sslMode" => "prefer"
+        },
+    },
+}
+
 describe 'data helper', :vcr do
 
   it 'connect to redshift with IAM authentication' do
@@ -256,4 +276,11 @@
     data = File.open('spec/data/mssql_data.csv').read
     expect(data).to eq File.open(file_path).read
   end
+
+  it 'connect to mysql with BASIC authentication' do
+    data_helper = GoodData::Helpers::DataSource.new(mysql_basic_params['input_source'])
+    file_path = data_helper.realize(mysql_basic_params)
+    data = File.open('spec/data/mysql_data.csv').read
+    expect(data).to eq File.open(file_path).read
+  end
 end