FEATURE: TMA-1630 Bump sdk version to 3.7.15

.rubocop.yml

@@ -2,6 +2,7 @@ AllCops:
   DisplayCopNames: true
   Exclude:
       - 'spec/helpers/appstore_project_helper.rb'
+  TargetRubyVersion: 2.3
 
 AbcSize:
   Enabled: false

CHANGELOG.md

@@ -1,4 +1,20 @@
 # GoodData Ruby SDK Changelog
+## 2.1.9
+ - FEATURE: TMA-1076 support new version api 2
+ - BUGFIX: TMA-1637 handle input_source of dynamic params
+ - BUGFIX: TMA-1636 Build csv file with force_quotes
+ - FEATURE: TMA-1614 Support redshift input source
+ - FEATURE: TMA-1259 Start using dataproduct in NFS release table
+ - FEATURE: MSF-16455 support yellow WARNING status into RubySDK
+ - CONFIG: TMA-1625 update version lock for test docker images
+ - BUGFIX: TMA-1602 User filter brick failed - K8s bricks don't show error properly
+ - BUGFIX: TMA-1593 Increase java heap space during execute bricks
+ - BUGFIX: TMA-1558 K8s bricks don't show error when config invalid
+ - BUGFIX: TMA-1596 The error propagated from a LCM brick is shown multiple times in the log
+ - BUGFIX: TMA-1582 show synchronize ldm mode is running
+ - FEATURE: TMA-1588 support schedule param include deprecated
+ - FEATURE: TMA-1597 Logging lcm execution result
+
 ## 2.1.8
  - FEATURE: TMA-1604 Upgrade Restforce version to 3.x
 

VERSION

@@ -1 +1 @@
-3.7.14
+3.7.15

lib/gooddata/cloud_resources/bigquery/bigquery_client.rb

@@ -1,4 +1,5 @@
 # encoding: UTF-8
+# frozen_string_literal: true
 #
 # Copyright (c) 2010-2019 GoodData Corporation. All rights reserved.
 # This source code is licensed under the BSD-style license found in the
@@ -55,19 +56,19 @@ def realize_query(query, _params)
           query_config = QueryJobConfiguration.newBuilder(query).setDefaultDataset(@schema).build
           table_result = client.query(query_config)
 
-          if table_result.getTotalRows > 0
+          if table_result.getTotalRows.positive?
             result = table_result.iterateAll
             field_list = table_result.getSchema.getFields
             col_count = field_list.size
-            CSV.open(filename, 'wb', :force_quotes => true) do |csv|
+            CSV.open(filename, 'wb') do |csv|
               csv << Array(1..col_count).map { |i| field_list.get(i - 1).getName } # build the header
               result.each do |row|
-                csv << Array(1..col_count).map { |i| row.get(i - 1).getStringValue }
+                csv << Array(1..col_count).map { |i| row.get(i - 1).getValue&.to_s }
               end
             end
           end
         end
-        GoodData.gd_logger.info("Realize SQL query: type=redshift status=finished duration=#{measure.real}")
+        GoodData.gd_logger.info("Realize SQL query: type=bigquery status=finished duration=#{measure.real}")
         filename
       end
 

lib/gooddata/cloud_resources/redshift/redshift_client.rb

@@ -1,4 +1,5 @@
 # encoding: UTF-8
+# frozen_string_literal: true
 #
 # Copyright (c) 2010-2019 GoodData Corporation. All rights reserved.
 # This source code is licensed under the BSD-style license found in the
@@ -57,9 +58,9 @@ def realize_query(query, _params)
             result = statement.get_result_set
             metadata = result.get_meta_data
             col_count = metadata.column_count
-            CSV.open(filename, 'wb', :force_quotes => true) do |csv|
+            CSV.open(filename, 'wb') do |csv|
               csv << Array(1..col_count).map { |i| metadata.get_column_name(i) } # build the header
-              csv << Array(1..col_count).map { |i| result.get_string(i) } while result.next
+              csv << Array(1..col_count).map { |i| result.get_string(i)&.to_s } while result.next
             end
           end
         end

lib/gooddata/cloud_resources/snowflake/snowflake_client.rb

@@ -1,4 +1,5 @@
 # encoding: UTF-8
+# frozen_string_literal: true
 #
 # Copyright (c) 2010-2019 GoodData Corporation. All rights reserved.
 # This source code is licensed under the BSD-style license found in the
@@ -53,16 +54,16 @@ def realize_query(query, _params)
             result = statement.get_result_set
             metadata = result.get_meta_data
             col_count = metadata.column_count
-            CSV.open(filename, 'wb', :force_quotes => true) do |csv|
+            CSV.open(filename, 'wb') do |csv|
               csv << Array(1..col_count).map { |i| metadata.get_column_name(i) } # build the header
-              csv << Array(1..col_count).map { |i| result.get_string(i) } while result.next
+              csv << Array(1..col_count).map { |i| result.get_string(i)&.to_s } while result.next
             end
           end
         end
         GoodData.gd_logger.info("Realize SQL query: type=snowflake status=finished duration=#{measure.real}")
         filename
       ensure
-        @connection.close unless @connection.nil?
+        @connection&.close
         @connection = nil
       end
 

lib/gooddata/lcm/actions/collect_clients.rb

@@ -67,11 +67,11 @@ def call(params)
         end
 
         def collect_clients(params, segment_names = nil)
-          client_id_column = params.client_id_column || 'client_id'
-          segment_id_column = params.segment_id_column || 'segment_id'
-          project_id_column = params.project_id_column || 'project_id'
-          project_title_column = params.project_title_column || 'project_title'
-          project_token_column = params.project_token_column || 'project_token'
+          client_id_column = params.client_id_column&.downcase || 'client_id'
+          segment_id_column = params.segment_id_column&.downcase || 'segment_id'
+          project_id_column = params.project_id_column&.downcase || 'project_id'
+          project_title_column = params.project_title_column&.downcase || 'project_title'
+          project_token_column = params.project_token_column&.downcase || 'project_token'
           client = params.gdc_gd_client
 
           clients = []
@@ -82,7 +82,7 @@ def collect_clients(params, segment_names = nil)
           end
           GoodData.logger.debug("Input data: #{input_data.read}")
           GoodData.logger.debug("Segment names: #{segment_names}")
-          CSV.foreach(input_data, :headers => true, :return_headers => false, encoding: 'utf-8') do |row|
+          CSV.foreach(input_data, :headers => true, :return_headers => false, :header_converters => :downcase, :encoding => 'utf-8') do |row|
             GoodData.logger.debug("Processing row: #{row}")
             segment_name = row[segment_id_column]
             GoodData.logger.debug("Segment name: #{segment_name}")

lib/gooddata/lcm/actions/collect_dynamic_schedule_params.rb

@@ -38,11 +38,11 @@ class << self
         def call(params)
           return [] unless params.dynamic_params
 
-          schedule_title_column = params.schedule_title_column || 'schedule_title'
-          client_id_column = params.client_id_column || 'client_id'
-          param_name_column = params.param_name_column || 'param_name'
-          param_value_column = params.param_value_column || 'param_value'
-          param_secure_column = params.param_secure_column || 'param_secure'
+          schedule_title_column = params.schedule_title_column&.downcase || 'schedule_title'
+          client_id_column = params.client_id_column&.downcase || 'client_id'
+          param_name_column = params.param_name_column&.downcase || 'param_name'
+          param_value_column = params.param_value_column&.downcase || 'param_value'
+          param_secure_column = params.param_secure_column&.downcase || 'param_secure'
 
           encryption_key = params.dynamic_params_encryption_key || ''
           exist_encryption_key = encryption_key.blank? ? false : true
@@ -59,7 +59,7 @@ def call(params)
           schedule_hidden_params = {}
           exist_param_secure = false
 
-          CSV.foreach(input_data, :headers => true, :return_headers => false, encoding: 'utf-8') do |row|
+          CSV.foreach(input_data, :headers => true, :return_headers => false, :header_converters => :downcase, :encoding => 'utf-8') do |row|
             is_param_secure = row[param_secure_column] == 'true'
             is_decrypt_secure_value = is_param_secure && exist_encryption_key ? true : false
             exist_param_secure = true if is_param_secure

lib/gooddata/lcm/actions/collect_users_brick_users.rb

@@ -35,7 +35,7 @@ class CollectUsersBrickUsers < BaseAction
       class << self
         def call(params)
           users_brick_users = []
-          login_column = params.users_brick_config.login_column || 'login'
+          login_column = params.users_brick_config.login_column&.downcase || 'login'
           users_brick_data_source = GoodData::Helpers::DataSource.new(params.users_brick_config.input_source)
 
           users_brick_data_source_file = without_check(PARAMS, params) do
@@ -45,14 +45,15 @@ def call(params)
             )
           end
           CSV.foreach(users_brick_data_source_file,
-                      headers: true,
-                      return_headers: false,
-                      encoding: 'utf-8') do |row|
-            pid = row[params.multiple_projects_column]
+                      :headers => true,
+                      :return_headers => false,
+                      :header_converters => :downcase,
+                      :encoding => 'utf-8') do |row|
+            pid = row[params.multiple_projects_column&.downcase]
             fail "The set multiple_projects_column '#{params.multiple_projects_column}' of the users input is empty" if !pid && MULTIPLE_COLUMN_MODES.include?(params.sync_mode)
 
             users_brick_users << {
-              login: row[login_column].downcase,
+              login: row[login_column].nil? ? nil : row[login_column].strip.downcase,
               pid: pid
             }
           end

lib/gooddata/lcm/actions/set_master_project.rb

@@ -0,0 +1,76 @@
+# encoding: UTF-8
+# frozen_string_literal: true
+#
+# Copyright (c) 2010-2017 GoodData Corporation. All rights reserved.
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+require_relative 'base_action'
+
+module GoodData
+  module LCM2
+    class SetMasterProject < BaseAction
+      DESCRIPTION = 'Set master project'
+
+      PARAMS = define_params(self) do
+        description 'Organization Name'
+        param :organization, instance_of(Type::StringType), required: false
+
+        description 'Domain'
+        param :domain, instance_of(Type::StringType), required: false
+
+        description 'ADS Client'
+        param :ads_client, instance_of(Type::AdsClientType), required: false
+
+        description 'Table Name'
+        param :release_table_name, instance_of(Type::StringType), required: false
+
+        description 'Segments to manage'
+        param :segments, array_of(instance_of(Type::SegmentType)), required: true
+
+        description 'DataProduct to manage'
+        param :data_product, instance_of(Type::GDDataProductType), required: false
+
+        description 'Released master project should be used in next rollout'
+        param :set_master_project, instance_of(Type::StringType), required: false
+      end
+
+      class << self
+        def call(params)
+          results = []
+          domain_name = params.organization || params.domain
+          data_product = params.data_product
+          params.segments.each do |segment_in|
+            version = get_latest_version(params, domain_name, data_product.data_product_id, segment_in.segment_id) + 1
+            segment_in[:data_product_id] = data_product.data_product_id
+            segment_in[:master_pid] = params.set_master_project
+            segment_in[:version] = version
+            segment_in[:timestamp] = Time.now.utc.iso8601
+
+            results << {
+              data_product_id: data_product.data_product_id,
+              segment_id: segment_in.segment_id,
+              version: version
+            }
+          end
+          results
+        end
+
+        def get_latest_version(params, domain_name, data_product_id, segment_id)
+          if params.ads_client
+            current_master = GoodData::LCM2::Helpers.latest_master_project_from_ads(
+              params.release_table_name,
+              params.ads_client,
+              segment_id
+            )
+          else
+            current_master = GoodData::LCM2::Helpers.latest_master_project_from_nfs(domain_name, data_product_id, segment_id)
+          end
+          return 0 unless current_master
+
+          current_master[:version].to_i
+        end
+      end
+    end
+  end
+end

lib/gooddata/lcm/actions/synchronize_user_filters.rb

@@ -230,8 +230,8 @@ def load_data(params, symbolized_config)
           begin
             GoodData.logger.info('Start reading data')
             row_count = 0
-            CSV.foreach(tmp, headers: csv_with_headers, return_headers: false, encoding: 'utf-8') do |row|
-              filters << row.to_hash.merge(pid: row[multiple_projects_column])
+            CSV.foreach(tmp, :headers => csv_with_headers, :return_headers => false, :header_converters => :downcase, :encoding => 'utf-8') do |row|
+              filters << row.to_hash.merge(pid: row[multiple_projects_column.downcase])
               row_count += 1
               GoodData.logger.info("Read #{row_count} rows") if (row_count % 50_000).zero?
             end

lib/gooddata/lcm/actions/synchronize_users.rb

@@ -347,38 +347,33 @@ def call(params)
         end
 
         def load_data(params, data_source)
-          first_name_column           = params.first_name_column || 'first_name'
-          last_name_column            = params.last_name_column || 'last_name'
-          login_column                = params.login_column || 'login'
-          password_column             = params.password_column || 'password'
-          email_column                = params.email_column || 'email'
-          role_column                 = params.role_column || 'role'
-          sso_provider_column         = params.sso_provider_column || 'sso_provider'
-          authentication_modes_column = params.authentication_modes_column || 'authentication_modes'
-          user_groups_column          = params.user_groups_column || 'user_groups'
-          language_column             = params.language_column || 'language'
-          company_column              = params.company_column || 'company'
-          position_column             = params.position_column || 'position'
-          country_column              = params.country_column || 'country'
-          phone_column                = params.phone_column || 'phone'
-          ip_whitelist_column         = params.ip_whitelist_column || 'ip_whitelist'
+          first_name_column           = params.first_name_column&.downcase || 'first_name'
+          last_name_column            = params.last_name_column&.downcase || 'last_name'
+          login_column                = params.login_column&.downcase || 'login'
+          password_column             = params.password_column&.downcase || 'password'
+          email_column                = params.email_column&.downcase || 'email'
+          role_column                 = params.role_column&.downcase || 'role'
+          sso_provider_column         = params.sso_provider_column&.downcase || 'sso_provider'
+          authentication_modes_column = params.authentication_modes_column&.downcase || 'authentication_modes'
+          user_groups_column          = params.user_groups_column&.downcase || 'user_groups'
+          language_column             = params.language_column&.downcase || 'language'
+          company_column              = params.company_column&.downcase || 'company'
+          position_column             = params.position_column&.downcase || 'position'
+          country_column              = params.country_column&.downcase || 'country'
+          phone_column                = params.phone_column&.downcase || 'phone'
+          ip_whitelist_column         = params.ip_whitelist_column&.downcase || 'ip_whitelist'
 
           sso_provider = params.sso_provider
           authentication_modes = params.authentication_modes || []
 
-          dwh = params.ads_client
-          if dwh
-            data = dwh.execute_select(params.input_source.query)
-          else
-            tmp = without_check(PARAMS, params) do
-              File.open(data_source.realize(params), 'r:UTF-8')
-            end
+          tmp = without_check(PARAMS, params) do
+            File.open(data_source.realize(params), 'r:UTF-8')
+          end
 
-            begin
-              data = read_csv_file(tmp)
-            rescue Exception => e # rubocop:disable RescueException
-              fail "There was an error during loading users from csv file. Message: #{e.message}. Error: #{e}"
-            end
+          begin
+            data = read_csv_file(tmp)
+          rescue Exception => e # rubocop:disable RescueException
+            fail "There was an error during loading users from csv file. Message: #{e.message}. Error: #{e}"
           end
 
           data.map do |row|
@@ -398,12 +393,18 @@ def load_data(params, data_source)
             ip_whitelist = row[ip_whitelist_column] || row[ip_whitelist_column.to_sym]
             ip_whitelist = ip_whitelist.split(',').map(&:strip) if ip_whitelist
 
+            user_login = row[login_column] || row[login_column.to_sym]
+            user_login = user_login.strip unless user_login.nil?
+
+            user_email = row[email_column] || row[login_column] || row[email_column.to_sym] || row[login_column.to_sym]
+            user_email = user_email.strip unless user_email.nil?
+
             {
               :first_name => row[first_name_column] || row[first_name_column.to_sym],
               :last_name => row[last_name_column] || row[last_name_column.to_sym],
-              :login => row[login_column] || row[login_column.to_sym],
+              :login => user_login,
               :password => row[password_column] || row[password_column.to_sym],
-              :email => row[email_column] || row[login_column] || row[email_column.to_sym] || row[login_column.to_sym],
+              :email => user_email,
               :role => row[role_column] || row[role_column.to_sym],
               :sso_provider => sso_provider || row[sso_provider_column] || row[sso_provider_column.to_sym],
               :authentication_modes => modes,
@@ -424,7 +425,7 @@ def read_csv_file(path)
           res = []
           row_count = 0
 
-          CSV.foreach(path, :headers => true) do |row|
+          CSV.foreach(path, :headers => true, :header_converters => :downcase, :encoding => 'utf-8') do |row|
             if block_given?
               data = yield row
             else