From 536ec7cf07bece0ed93cf2a87aa3dc0b7aaf5a38 Mon Sep 17 00:00:00 2001 From: Tu Quoc Tran Date: Thu, 25 Nov 2021 19:17:28 +0700 Subject: [PATCH] FEATURE: MSF-22129 add support mysql input source --- Dockerfile | 4 + ci/mysql/pom.xml | 57 +++++++++ .../cloud_resources/mysql/drivers/.gitkeepme | 0 .../cloud_resources/mysql/mysql_client.rb | 111 ++++++++++++++++++ .../postgresql/postgresql_client.rb | 1 - lib/gooddata/helpers/data_helper.rb | 2 +- .../lcm/actions/update_metric_formats.rb | 2 +- spec/data/mysql_data.csv | 3 + spec/environment/secrets.yaml | 1 + .../spec/others/data_helper_spec.rb | 27 +++++ 10 files changed, 205 insertions(+), 3 deletions(-) create mode 100644 ci/mysql/pom.xml create mode 100644 lib/gooddata/cloud_resources/mysql/drivers/.gitkeepme create mode 100644 lib/gooddata/cloud_resources/mysql/mysql_client.rb create mode 100644 spec/data/mysql_data.csv diff --git a/Dockerfile b/Dockerfile index 005e8361b..e9323cc49 100644 --- a/Dockerfile +++ b/Dockerfile @@ -70,6 +70,10 @@ RUN cp -rf ci/postgresql/target/*.jar ./lib/gooddata/cloud_resources/postgresql/ RUN mvn -f ci/mssql/pom.xml clean install -P binary-packaging RUN cp -rf ci/mssql/target/*.jar ./lib/gooddata/cloud_resources/mssql/drivers/ +#build mysql dependencies +RUN mvn -f ci/mysql/pom.xml clean install -P binary-packaging +RUN cp -rf ci/mysql/target/*.jar ./lib/gooddata/cloud_resources/mysql/drivers/ + RUN bundle install ARG GIT_COMMIT=unspecified diff --git a/ci/mysql/pom.xml b/ci/mysql/pom.xml new file mode 100644 index 000000000..ea8a7a9ea --- /dev/null +++ b/ci/mysql/pom.xml @@ -0,0 +1,57 @@ + + + 4.0.0 + + com.gooddata.lcm + lcm-mysql-driver + 1.0-SNAPSHOT + + + + mysql + mysql-connector-java + 8.0.25 + + + org.slf4j + slf4j-api + 1.7.2 + + + + + + binary-packaging + + + + maven-dependency-plugin + + + package + + copy-dependencies + + + ${project.build.directory} + + runtime + + + + + + + + + + + + my-repo1 + my custom repo + https://repository.mulesoft.org/nexus/content/repositories/public/ + + + diff --git a/lib/gooddata/cloud_resources/mysql/drivers/.gitkeepme b/lib/gooddata/cloud_resources/mysql/drivers/.gitkeepme new file mode 100644 index 000000000..e69de29bb diff --git a/lib/gooddata/cloud_resources/mysql/mysql_client.rb b/lib/gooddata/cloud_resources/mysql/mysql_client.rb new file mode 100644 index 000000000..b717d7b01 --- /dev/null +++ b/lib/gooddata/cloud_resources/mysql/mysql_client.rb @@ -0,0 +1,111 @@ +# encoding: UTF-8 +# frozen_string_literal: true +# +# Copyright (c) 2021 GoodData Corporation. All rights reserved. +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +require 'securerandom' +require 'java' +require 'pathname' +require_relative '../cloud_resource_client' + +base = Pathname(__FILE__).dirname.expand_path +Dir.glob(base + 'drivers/*.jar').each do |file| + require file unless file.start_with?('lcm-mysql-driver') +end + +module GoodData + module CloudResources + class MysqlClient < CloudResourceClient + JDBC_MYSQL_PATTERN = %r{jdbc:mysql:\/\/([^:^\/]+)(:([0-9]+))?(\/)?} + MYSQL_DEFAULT_PORT = 3306 + JDBC_MYSQL_PROTOCOL = 'jdbc:mysql://' + VERIFY_FULL = 'VERIFY_IDENTITY' + PREFER = 'PREFERRED' + REQUIRE = 'REQUIRED' + MYSQL_FETCH_SIZE = 1000 + + class << self + def accept?(type) + type == 'mysql' + end + end + + def initialize(options = {}) + raise("Data Source needs a client to Mysql to be able to query the storage but 'mysql_client' is empty.") unless options['mysql_client'] + + if options['mysql_client']['connection'].is_a?(Hash) + @database = options['mysql_client']['connection']['database'] + @authentication = options['mysql_client']['connection']['authentication'] + @ssl_mode = options['mysql_client']['connection']['sslMode'] + raise "SSL Mode should be prefer, require and verify-full" unless @ssl_mode == 'prefer' || @ssl_mode == 'require' || @ssl_mode == 'verify-full' + + @url = build_url(options['mysql_client']['connection']['url']) + else + raise('Missing connection info for Mysql client') + end + + Java.com.mysql.cj.jdbc.Driver + end + + def realize_query(query, _params) + GoodData.gd_logger.info("Realize SQL query: type=mysql status=started") + + connect + filename = "#{SecureRandom.urlsafe_base64(6)}_#{Time.now.to_i}.csv" + measure = Benchmark.measure do + statement = @connection.create_statement + statement.set_fetch_size(MYSQL_FETCH_SIZE) + has_result = statement.execute(query) + if has_result + result = statement.get_result_set + metadata = result.get_meta_data + col_count = metadata.column_count + CSV.open(filename, 'wb') do |csv| + csv << Array(1..col_count).map { |i| metadata.get_column_name(i) } # build the header + csv << Array(1..col_count).map { |i| result.get_string(i)&.to_s } while result.next + end + end + end + GoodData.gd_logger.info("Realize SQL query: type=mysql status=finished duration=#{measure.real}") + filename + ensure + @connection&.close + @connection = nil + end + + def connect + GoodData.logger.info "Setting up connection to Mysql #{@url}" + + prop = java.util.Properties.new + prop.setProperty('user', @authentication['basic']['userName']) + prop.setProperty('password', @authentication['basic']['password']) + + @connection = java.sql.DriverManager.getConnection(@url, prop) + @connection.set_auto_commit(false) + end + + def build_url(url) + matches = url.scan(JDBC_MYSQL_PATTERN) + raise 'Cannot reach the url' unless matches + + host = matches[0][0] + port = matches[0][2]&.to_i || MYSQL_DEFAULT_PORT + + "#{JDBC_MYSQL_PROTOCOL}#{host}:#{port}/#{@database}?sslmode=#{get_ssl_mode(@ssl_mode)}&useCursorFetch=true" + end + + def get_ssl_mode(ssl_mode) + mode = PREFER + if ssl_mode == 'verify-full' + mode = VERIFY_FULL + elsif ssl_mode == 'require' + mode = REQUIRE + end + + mode + end + end + end +end diff --git a/lib/gooddata/cloud_resources/postgresql/postgresql_client.rb b/lib/gooddata/cloud_resources/postgresql/postgresql_client.rb index 4aa7ea07f..be20e0582 100644 --- a/lib/gooddata/cloud_resources/postgresql/postgresql_client.rb +++ b/lib/gooddata/cloud_resources/postgresql/postgresql_client.rb @@ -98,7 +98,6 @@ def build_url(url) host = matches[0][0] port = matches[0][2]&.to_i || POSTGRES_DEFAULT_PORT - raise "Custom port #{port} is not supported. Remove it or use the default port '5432'" if POSTGRES_DEFAULT_PORT != port "#{JDBC_POSTGRES_PROTOCOL}#{host}:#{port}/#{@database}?sslmode=#{@ssl_mode}#{VERIFY_FULL == @ssl_mode ? SSL_JAVA_FACTORY : ''}" end diff --git a/lib/gooddata/helpers/data_helper.rb b/lib/gooddata/helpers/data_helper.rb index ee6b9293d..76eb5f310 100644 --- a/lib/gooddata/helpers/data_helper.rb +++ b/lib/gooddata/helpers/data_helper.rb @@ -44,7 +44,7 @@ def realize(params = {}) realize_link when 's3' realize_s3(params) - when 'redshift', 'snowflake', 'bigquery', 'postgresql', 'mssql' + when 'redshift', 'snowflake', 'bigquery', 'postgresql', 'mssql', 'mysql' raise GoodData::InvalidEnvError, "DataSource does not support type \"#{source}\" on the platform #{RUBY_PLATFORM}" unless RUBY_PLATFORM =~ /java/ require_relative '../cloud_resources/cloud_resources' realize_cloud_resource(source, params) diff --git a/lib/gooddata/lcm/actions/update_metric_formats.rb b/lib/gooddata/lcm/actions/update_metric_formats.rb index e75a6f60b..b237869e4 100644 --- a/lib/gooddata/lcm/actions/update_metric_formats.rb +++ b/lib/gooddata/lcm/actions/update_metric_formats.rb @@ -75,7 +75,7 @@ def validate_input_source(input_source) modified_input_source = input_source case type - when 'ads', 'redshift', 'snowflake', 'bigquery', 'postgresql', 'mssql' + when 'ads', 'redshift', 'snowflake', 'bigquery', 'postgresql', 'mssql', 'mysql' if metric_format[:query].blank? GoodData.logger.warn("The metric input_source '#{type}' is missing property 'query'") return nil diff --git a/spec/data/mysql_data.csv b/spec/data/mysql_data.csv new file mode 100644 index 000000000..bc7ebef4a --- /dev/null +++ b/spec/data/mysql_data.csv @@ -0,0 +1,3 @@ +segment_id,client_id,project_title,project_id,project_token +Segment,Client1,Client-1,,token +Segment,Client2,Client-2,,token diff --git a/spec/environment/secrets.yaml b/spec/environment/secrets.yaml index 30efd62d6..54af7f4af 100644 --- a/spec/environment/secrets.yaml +++ b/spec/environment/secrets.yaml @@ -8,6 +8,7 @@ global: redshift_secret_key: 1JHYwvoIQinjgdKJXQL6RNGFhx4o4M9DiQf9q4jV+dq6xCPLDBCP/8tXBd0H9y8xdOOI78mY/aOQWjiPzgizLA== snowflake_password: 1zg1PDRMQq2DhBG3SwQOA8/POUkeek3gurrmV4MT2Go= blob_storage_connection: Md/faNEbH3YOsmVCDaUJEH4/eHkABgp2X1V6BIZyMbuMxlAdlCFxY8gLqM1sJUEt2txBp7I6PmDdnG34+wV1nawRO3U9WAwr8wTPI57pkcNj0fpFN9KLycNA8ms6cVklxFlgO1WmCOqBL+wBnIbqRZ8sl9wx2BTFebt8QQSLucGMZtY0oDjy/YeG6SqH+HCzEW70ipU3whVXWJkZStIK8cHy9uxJZF88uqpphJFTQAFMwgCQQ9+vEF+mpt4xaWtF2KnRkif2a2OuYSsEStuA/A== + mysql_connection: PsmFcHvUtff5A2OGYWaI1+KKkQsDrThUf46DAR/m6v69yPBLI65jCZO25XV6R4xU development: dev_token: 8qWaLsyWwAUJ7MJJTBdriUvtaWKNidnzmfxVThCrL0c= prod_token: RitXvhFjpJ8KEpqUqZm57iV3bwVU1zBGDrXNklvwkaE= diff --git a/spec/lcm/integration/spec/others/data_helper_spec.rb b/spec/lcm/integration/spec/others/data_helper_spec.rb index dc1ff563c..dd27bf7af 100644 --- a/spec/lcm/integration/spec/others/data_helper_spec.rb +++ b/spec/lcm/integration/spec/others/data_helper_spec.rb @@ -179,6 +179,26 @@ }, } +mysql_basic_params = { + "input_source"=> { + "type"=> "mysql", + "query"=> "SELECT DISTINCT * FROM clients", + }, + "mysql_client" => { + "connection" => { + "url" => "jdbc:mysql://msf-test-database.na.intgdc.com:1435", + "database" => "integration_test", + "authentication" => { + "basic" => { + "userName" => "mysql_integration_test", + "password" => ConnectionHelper::SECRETS[:mysql_connection], + } + }, + "sslMode" => "prefer" + }, + }, +} + describe 'data helper', :vcr do it 'connect to redshift with IAM authentication' do @@ -256,4 +276,11 @@ data = File.open('spec/data/mssql_data.csv').read expect(data).to eq File.open(file_path).read end + + it 'connect to mysql with BASIC authentication' do + data_helper = GoodData::Helpers::DataSource.new(mysql_basic_params['input_source']) + file_path = data_helper.realize(mysql_basic_params) + data = File.open('spec/data/mysql_data.csv').read + expect(data).to eq File.open(file_path).read + end end