diff --git a/.rubocop.yml b/.rubocop.yml index 22a89362a..04f5bf9d4 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -2,6 +2,7 @@ AllCops: DisplayCopNames: true Exclude: - 'spec/helpers/appstore_project_helper.rb' + TargetRubyVersion: 2.3 AbcSize: Enabled: false diff --git a/CHANGELOG.md b/CHANGELOG.md index 0a18c25ba..825d7e0e6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,20 @@ # GoodData Ruby SDK Changelog +## 2.1.9 + - FEATURE: TMA-1076 support new version api 2 + - BUGFIX: TMA-1637 handle input_source of dynamic params + - BUGFIX: TMA-1636 Build csv file with force_quotes + - FEATURE: TMA-1614 Support redshift input source + - FEATURE: TMA-1259 Start using dataproduct in NFS release table + - FEATURE: MSF-16455 support yellow WARNING status into RubySDK + - CONFIG: TMA-1625 update version lock for test docker images + - BUGFIX: TMA-1602 User filter brick failed - K8s bricks don't show error properly + - BUGFIX: TMA-1593 Increase java heap space during execute bricks + - BUGFIX: TMA-1558 K8s bricks don't show error when config invalid + - BUGFIX: TMA-1596 The error propagated from a LCM brick is shown multiple times in the log + - BUGFIX: TMA-1582 show synchronize ldm mode is running + - FEATURE: TMA-1588 support schedule param include deprecated + - FEATURE: TMA-1597 Logging lcm execution result + ## 2.1.8 - FEATURE: TMA-1604 Upgrade Restforce version to 3.x diff --git a/VERSION b/VERSION index 35c6ac517..03d7b8fb9 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.7.14 +3.7.15 diff --git a/lib/gooddata/cloud_resources/bigquery/bigquery_client.rb b/lib/gooddata/cloud_resources/bigquery/bigquery_client.rb index 620a19704..f9dffe8a5 100644 --- a/lib/gooddata/cloud_resources/bigquery/bigquery_client.rb +++ b/lib/gooddata/cloud_resources/bigquery/bigquery_client.rb @@ -1,4 +1,5 @@ # encoding: UTF-8 +# frozen_string_literal: true # # Copyright (c) 2010-2019 GoodData Corporation. All rights reserved. # This source code is licensed under the BSD-style license found in the @@ -55,19 +56,19 @@ def realize_query(query, _params) query_config = QueryJobConfiguration.newBuilder(query).setDefaultDataset(@schema).build table_result = client.query(query_config) - if table_result.getTotalRows > 0 + if table_result.getTotalRows.positive? result = table_result.iterateAll field_list = table_result.getSchema.getFields col_count = field_list.size - CSV.open(filename, 'wb', :force_quotes => true) do |csv| + CSV.open(filename, 'wb') do |csv| csv << Array(1..col_count).map { |i| field_list.get(i - 1).getName } # build the header result.each do |row| - csv << Array(1..col_count).map { |i| row.get(i - 1).getStringValue } + csv << Array(1..col_count).map { |i| row.get(i - 1).getValue&.to_s } end end end end - GoodData.gd_logger.info("Realize SQL query: type=redshift status=finished duration=#{measure.real}") + GoodData.gd_logger.info("Realize SQL query: type=bigquery status=finished duration=#{measure.real}") filename end diff --git a/lib/gooddata/cloud_resources/redshift/redshift_client.rb b/lib/gooddata/cloud_resources/redshift/redshift_client.rb index d245e0643..41c58d341 100644 --- a/lib/gooddata/cloud_resources/redshift/redshift_client.rb +++ b/lib/gooddata/cloud_resources/redshift/redshift_client.rb @@ -1,4 +1,5 @@ # encoding: UTF-8 +# frozen_string_literal: true # # Copyright (c) 2010-2019 GoodData Corporation. All rights reserved. # This source code is licensed under the BSD-style license found in the @@ -57,9 +58,9 @@ def realize_query(query, _params) result = statement.get_result_set metadata = result.get_meta_data col_count = metadata.column_count - CSV.open(filename, 'wb', :force_quotes => true) do |csv| + CSV.open(filename, 'wb') do |csv| csv << Array(1..col_count).map { |i| metadata.get_column_name(i) } # build the header - csv << Array(1..col_count).map { |i| result.get_string(i) } while result.next + csv << Array(1..col_count).map { |i| result.get_string(i)&.to_s } while result.next end end end diff --git a/lib/gooddata/cloud_resources/snowflake/snowflake_client.rb b/lib/gooddata/cloud_resources/snowflake/snowflake_client.rb index 12b2c0f20..14d86aa56 100644 --- a/lib/gooddata/cloud_resources/snowflake/snowflake_client.rb +++ b/lib/gooddata/cloud_resources/snowflake/snowflake_client.rb @@ -1,4 +1,5 @@ # encoding: UTF-8 +# frozen_string_literal: true # # Copyright (c) 2010-2019 GoodData Corporation. All rights reserved. # This source code is licensed under the BSD-style license found in the @@ -53,16 +54,16 @@ def realize_query(query, _params) result = statement.get_result_set metadata = result.get_meta_data col_count = metadata.column_count - CSV.open(filename, 'wb', :force_quotes => true) do |csv| + CSV.open(filename, 'wb') do |csv| csv << Array(1..col_count).map { |i| metadata.get_column_name(i) } # build the header - csv << Array(1..col_count).map { |i| result.get_string(i) } while result.next + csv << Array(1..col_count).map { |i| result.get_string(i)&.to_s } while result.next end end end GoodData.gd_logger.info("Realize SQL query: type=snowflake status=finished duration=#{measure.real}") filename ensure - @connection.close unless @connection.nil? + @connection&.close @connection = nil end diff --git a/lib/gooddata/lcm/actions/collect_clients.rb b/lib/gooddata/lcm/actions/collect_clients.rb index 5f05b80f9..a4f24ac8c 100644 --- a/lib/gooddata/lcm/actions/collect_clients.rb +++ b/lib/gooddata/lcm/actions/collect_clients.rb @@ -67,11 +67,11 @@ def call(params) end def collect_clients(params, segment_names = nil) - client_id_column = params.client_id_column || 'client_id' - segment_id_column = params.segment_id_column || 'segment_id' - project_id_column = params.project_id_column || 'project_id' - project_title_column = params.project_title_column || 'project_title' - project_token_column = params.project_token_column || 'project_token' + client_id_column = params.client_id_column&.downcase || 'client_id' + segment_id_column = params.segment_id_column&.downcase || 'segment_id' + project_id_column = params.project_id_column&.downcase || 'project_id' + project_title_column = params.project_title_column&.downcase || 'project_title' + project_token_column = params.project_token_column&.downcase || 'project_token' client = params.gdc_gd_client clients = [] @@ -82,7 +82,7 @@ def collect_clients(params, segment_names = nil) end GoodData.logger.debug("Input data: #{input_data.read}") GoodData.logger.debug("Segment names: #{segment_names}") - CSV.foreach(input_data, :headers => true, :return_headers => false, encoding: 'utf-8') do |row| + CSV.foreach(input_data, :headers => true, :return_headers => false, :header_converters => :downcase, :encoding => 'utf-8') do |row| GoodData.logger.debug("Processing row: #{row}") segment_name = row[segment_id_column] GoodData.logger.debug("Segment name: #{segment_name}") diff --git a/lib/gooddata/lcm/actions/collect_dynamic_schedule_params.rb b/lib/gooddata/lcm/actions/collect_dynamic_schedule_params.rb index ad2c0d50b..abb3e2575 100644 --- a/lib/gooddata/lcm/actions/collect_dynamic_schedule_params.rb +++ b/lib/gooddata/lcm/actions/collect_dynamic_schedule_params.rb @@ -38,11 +38,11 @@ class << self def call(params) return [] unless params.dynamic_params - schedule_title_column = params.schedule_title_column || 'schedule_title' - client_id_column = params.client_id_column || 'client_id' - param_name_column = params.param_name_column || 'param_name' - param_value_column = params.param_value_column || 'param_value' - param_secure_column = params.param_secure_column || 'param_secure' + schedule_title_column = params.schedule_title_column&.downcase || 'schedule_title' + client_id_column = params.client_id_column&.downcase || 'client_id' + param_name_column = params.param_name_column&.downcase || 'param_name' + param_value_column = params.param_value_column&.downcase || 'param_value' + param_secure_column = params.param_secure_column&.downcase || 'param_secure' encryption_key = params.dynamic_params_encryption_key || '' exist_encryption_key = encryption_key.blank? ? false : true @@ -59,7 +59,7 @@ def call(params) schedule_hidden_params = {} exist_param_secure = false - CSV.foreach(input_data, :headers => true, :return_headers => false, encoding: 'utf-8') do |row| + CSV.foreach(input_data, :headers => true, :return_headers => false, :header_converters => :downcase, :encoding => 'utf-8') do |row| is_param_secure = row[param_secure_column] == 'true' is_decrypt_secure_value = is_param_secure && exist_encryption_key ? true : false exist_param_secure = true if is_param_secure diff --git a/lib/gooddata/lcm/actions/collect_users_brick_users.rb b/lib/gooddata/lcm/actions/collect_users_brick_users.rb index 93ff1bb0f..21938746c 100644 --- a/lib/gooddata/lcm/actions/collect_users_brick_users.rb +++ b/lib/gooddata/lcm/actions/collect_users_brick_users.rb @@ -35,7 +35,7 @@ class CollectUsersBrickUsers < BaseAction class << self def call(params) users_brick_users = [] - login_column = params.users_brick_config.login_column || 'login' + login_column = params.users_brick_config.login_column&.downcase || 'login' users_brick_data_source = GoodData::Helpers::DataSource.new(params.users_brick_config.input_source) users_brick_data_source_file = without_check(PARAMS, params) do @@ -45,14 +45,15 @@ def call(params) ) end CSV.foreach(users_brick_data_source_file, - headers: true, - return_headers: false, - encoding: 'utf-8') do |row| - pid = row[params.multiple_projects_column] + :headers => true, + :return_headers => false, + :header_converters => :downcase, + :encoding => 'utf-8') do |row| + pid = row[params.multiple_projects_column&.downcase] fail "The set multiple_projects_column '#{params.multiple_projects_column}' of the users input is empty" if !pid && MULTIPLE_COLUMN_MODES.include?(params.sync_mode) users_brick_users << { - login: row[login_column].downcase, + login: row[login_column].nil? ? nil : row[login_column].strip.downcase, pid: pid } end diff --git a/lib/gooddata/lcm/actions/set_master_project.rb b/lib/gooddata/lcm/actions/set_master_project.rb new file mode 100644 index 000000000..58a5b9a2a --- /dev/null +++ b/lib/gooddata/lcm/actions/set_master_project.rb @@ -0,0 +1,76 @@ +# encoding: UTF-8 +# frozen_string_literal: true +# +# Copyright (c) 2010-2017 GoodData Corporation. All rights reserved. +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +require_relative 'base_action' + +module GoodData + module LCM2 + class SetMasterProject < BaseAction + DESCRIPTION = 'Set master project' + + PARAMS = define_params(self) do + description 'Organization Name' + param :organization, instance_of(Type::StringType), required: false + + description 'Domain' + param :domain, instance_of(Type::StringType), required: false + + description 'ADS Client' + param :ads_client, instance_of(Type::AdsClientType), required: false + + description 'Table Name' + param :release_table_name, instance_of(Type::StringType), required: false + + description 'Segments to manage' + param :segments, array_of(instance_of(Type::SegmentType)), required: true + + description 'DataProduct to manage' + param :data_product, instance_of(Type::GDDataProductType), required: false + + description 'Released master project should be used in next rollout' + param :set_master_project, instance_of(Type::StringType), required: false + end + + class << self + def call(params) + results = [] + domain_name = params.organization || params.domain + data_product = params.data_product + params.segments.each do |segment_in| + version = get_latest_version(params, domain_name, data_product.data_product_id, segment_in.segment_id) + 1 + segment_in[:data_product_id] = data_product.data_product_id + segment_in[:master_pid] = params.set_master_project + segment_in[:version] = version + segment_in[:timestamp] = Time.now.utc.iso8601 + + results << { + data_product_id: data_product.data_product_id, + segment_id: segment_in.segment_id, + version: version + } + end + results + end + + def get_latest_version(params, domain_name, data_product_id, segment_id) + if params.ads_client + current_master = GoodData::LCM2::Helpers.latest_master_project_from_ads( + params.release_table_name, + params.ads_client, + segment_id + ) + else + current_master = GoodData::LCM2::Helpers.latest_master_project_from_nfs(domain_name, data_product_id, segment_id) + end + return 0 unless current_master + + current_master[:version].to_i + end + end + end + end +end diff --git a/lib/gooddata/lcm/actions/synchronize_user_filters.rb b/lib/gooddata/lcm/actions/synchronize_user_filters.rb index 13bfd87d1..0bb73f8af 100644 --- a/lib/gooddata/lcm/actions/synchronize_user_filters.rb +++ b/lib/gooddata/lcm/actions/synchronize_user_filters.rb @@ -230,8 +230,8 @@ def load_data(params, symbolized_config) begin GoodData.logger.info('Start reading data') row_count = 0 - CSV.foreach(tmp, headers: csv_with_headers, return_headers: false, encoding: 'utf-8') do |row| - filters << row.to_hash.merge(pid: row[multiple_projects_column]) + CSV.foreach(tmp, :headers => csv_with_headers, :return_headers => false, :header_converters => :downcase, :encoding => 'utf-8') do |row| + filters << row.to_hash.merge(pid: row[multiple_projects_column.downcase]) row_count += 1 GoodData.logger.info("Read #{row_count} rows") if (row_count % 50_000).zero? end diff --git a/lib/gooddata/lcm/actions/synchronize_users.rb b/lib/gooddata/lcm/actions/synchronize_users.rb index ca9cefcc1..377741388 100644 --- a/lib/gooddata/lcm/actions/synchronize_users.rb +++ b/lib/gooddata/lcm/actions/synchronize_users.rb @@ -347,38 +347,33 @@ def call(params) end def load_data(params, data_source) - first_name_column = params.first_name_column || 'first_name' - last_name_column = params.last_name_column || 'last_name' - login_column = params.login_column || 'login' - password_column = params.password_column || 'password' - email_column = params.email_column || 'email' - role_column = params.role_column || 'role' - sso_provider_column = params.sso_provider_column || 'sso_provider' - authentication_modes_column = params.authentication_modes_column || 'authentication_modes' - user_groups_column = params.user_groups_column || 'user_groups' - language_column = params.language_column || 'language' - company_column = params.company_column || 'company' - position_column = params.position_column || 'position' - country_column = params.country_column || 'country' - phone_column = params.phone_column || 'phone' - ip_whitelist_column = params.ip_whitelist_column || 'ip_whitelist' + first_name_column = params.first_name_column&.downcase || 'first_name' + last_name_column = params.last_name_column&.downcase || 'last_name' + login_column = params.login_column&.downcase || 'login' + password_column = params.password_column&.downcase || 'password' + email_column = params.email_column&.downcase || 'email' + role_column = params.role_column&.downcase || 'role' + sso_provider_column = params.sso_provider_column&.downcase || 'sso_provider' + authentication_modes_column = params.authentication_modes_column&.downcase || 'authentication_modes' + user_groups_column = params.user_groups_column&.downcase || 'user_groups' + language_column = params.language_column&.downcase || 'language' + company_column = params.company_column&.downcase || 'company' + position_column = params.position_column&.downcase || 'position' + country_column = params.country_column&.downcase || 'country' + phone_column = params.phone_column&.downcase || 'phone' + ip_whitelist_column = params.ip_whitelist_column&.downcase || 'ip_whitelist' sso_provider = params.sso_provider authentication_modes = params.authentication_modes || [] - dwh = params.ads_client - if dwh - data = dwh.execute_select(params.input_source.query) - else - tmp = without_check(PARAMS, params) do - File.open(data_source.realize(params), 'r:UTF-8') - end + tmp = without_check(PARAMS, params) do + File.open(data_source.realize(params), 'r:UTF-8') + end - begin - data = read_csv_file(tmp) - rescue Exception => e # rubocop:disable RescueException - fail "There was an error during loading users from csv file. Message: #{e.message}. Error: #{e}" - end + begin + data = read_csv_file(tmp) + rescue Exception => e # rubocop:disable RescueException + fail "There was an error during loading users from csv file. Message: #{e.message}. Error: #{e}" end data.map do |row| @@ -398,12 +393,18 @@ def load_data(params, data_source) ip_whitelist = row[ip_whitelist_column] || row[ip_whitelist_column.to_sym] ip_whitelist = ip_whitelist.split(',').map(&:strip) if ip_whitelist + user_login = row[login_column] || row[login_column.to_sym] + user_login = user_login.strip unless user_login.nil? + + user_email = row[email_column] || row[login_column] || row[email_column.to_sym] || row[login_column.to_sym] + user_email = user_email.strip unless user_email.nil? + { :first_name => row[first_name_column] || row[first_name_column.to_sym], :last_name => row[last_name_column] || row[last_name_column.to_sym], - :login => row[login_column] || row[login_column.to_sym], + :login => user_login, :password => row[password_column] || row[password_column.to_sym], - :email => row[email_column] || row[login_column] || row[email_column.to_sym] || row[login_column.to_sym], + :email => user_email, :role => row[role_column] || row[role_column.to_sym], :sso_provider => sso_provider || row[sso_provider_column] || row[sso_provider_column.to_sym], :authentication_modes => modes, @@ -424,7 +425,7 @@ def read_csv_file(path) res = [] row_count = 0 - CSV.foreach(path, :headers => true) do |row| + CSV.foreach(path, :headers => true, :header_converters => :downcase, :encoding => 'utf-8') do |row| if block_given? data = yield row else diff --git a/lib/gooddata/lcm/lcm2.rb b/lib/gooddata/lcm/lcm2.rb index 2937b197e..577ef238a 100644 --- a/lib/gooddata/lcm/lcm2.rb +++ b/lib/gooddata/lcm/lcm2.rb @@ -106,6 +106,14 @@ def respond_to_missing?(name, *_args) UpdateReleaseTable ], + release_set_master_project: [ + EnsureReleaseTable, + CollectDataProduct, + SegmentsFilter, + SetMasterProject, + UpdateReleaseTable + ], + provision: [ EnsureReleaseTable, CollectDataProduct, @@ -271,8 +279,14 @@ def perform(mode, params = {}) GoodData.gd_logger.brick = mode + final_mode = if params.set_master_project && mode == 'release' + 'release_set_master_project' + else + mode + end + # Get actions for mode specified - actions = get_mode_actions(mode) + actions = get_mode_actions(final_mode) if params.actions actions = params.actions.map do |action| @@ -305,6 +319,12 @@ def perform(mode, params = {}) skip_actions.include?(action.name.split('::').last) end + sync_mode = params.fetch(:sync_mode, nil) + if mode == 'users' && %w[add_to_organization remove_from_organization].include?(sync_mode) + actions = actions.reject do |action| + %w[CollectDataProduct CollectSegments].include?(action.name.split('::').last) + end + end check_unused_params(actions, params) print_action_names(mode, actions) diff --git a/spec/data/bigquery_data.csv b/spec/data/bigquery_data.csv index 1feee6615..62009d2f2 100644 --- a/spec/data/bigquery_data.csv +++ b/spec/data/bigquery_data.csv @@ -1,5 +1,5 @@ -"EmployeeID","Fullname","Salary","DOB","Departments","x__client_id","x__timestamp","x__deleted" -"E101","Name 01","4000","1970-01-20","101","mtt_testing_01","1557396000.001","false" -"E102","Name 02","3000","1918-05-26","102","mtt_testing_01","1557396000.002","false" -"E103","Name 03","5000","1960-02-03","103","mtt_testing_01","1557396000.003","false" -"E104","Name 04","5000","1960-02-03","104","MTT-Client02","1557396000.004","false" +EmployeeID,Fullname,Salary,DOB,Departments,x__client_id,x__timestamp,x__deleted +E101,Name 01,4000,1970-01-20,101,mtt_testing_01,1557396000.0,false +E102,Name 02,3000,1918-05-26,102,mtt_testing_01,1557396000.0,false +E103,Name 03,5000,1960-02-03,103,mtt_testing_01,1557396000.0,false +E104,"Name, 04",5000,,104,MTT-Client02,1557396000.0,false diff --git a/spec/data/redshift_data.csv b/spec/data/redshift_data.csv index d77d89b55..9d79accef 100644 --- a/spec/data/redshift_data.csv +++ b/spec/data/redshift_data.csv @@ -1,3 +1,3 @@ -"login","client_id","role","user_groups" -"nahunta@flsb11.com","client01","admin","GROUP_01" -"rhabille@workwolf.site","client02","admin","GROUP_02" +login,client_id,role,user_groups +nahunta@flsb11.com,client01,admin,GROUP_01 +rhabille@workwolf.site,client02,admin,GROUP_02 diff --git a/spec/data/redshift_data2.csv b/spec/data/redshift_data2.csv index 84b903174..a336170f7 100644 --- a/spec/data/redshift_data2.csv +++ b/spec/data/redshift_data2.csv @@ -1,3 +1,3 @@ -"login","client_id","role","user_groups" -"nahunta@flsb11.com","client01","admin","GROUP_01, GROUP_02" -"rhabille@workwolf.site","client02","admin","GROUP_02" +login,client_id,role,user_groups +nahunta@flsb11.com,client01,admin,"GROUP_01, GROUP_02" +rhabille@workwolf.site,client02,admin,GROUP_02 diff --git a/spec/data/snowflake_data.csv b/spec/data/snowflake_data.csv index 66e158c3e..f5c8b33a2 100644 --- a/spec/data/snowflake_data.csv +++ b/spec/data/snowflake_data.csv @@ -1,2 +1,2 @@ -"CP__CUSTKEY","A__NAME","D__DATEADDED","X__TIMESTAMP" -"cus3","phong","2019-03-05","" +CP__CUSTKEY,A__NAME,D__DATEADDED,X__TIMESTAMP +cus3,phong,, diff --git a/spec/lcm/integration/brick_runner.rb b/spec/lcm/integration/brick_runner.rb index e196f16b2..ec593c436 100644 --- a/spec/lcm/integration/brick_runner.rb +++ b/spec/lcm/integration/brick_runner.rb @@ -10,8 +10,10 @@ def users_brick(opts) def release_brick(opts) result = GoodData::Bricks::Pipeline.release_brick_pipeline.call script_params opts - master_ids = result[:results]['CreateSegmentMasters'].map { |r| r[:master_pid] } - fetch_projects(master_ids, opts) if opts[:client] + unless opts[:set_master_project] + master_ids = result[:results]['CreateSegmentMasters'].map { |r| r[:master_pid] } + fetch_projects(master_ids, opts) if opts[:client] + end end def provisioning_brick(opts) diff --git a/spec/lcm/integration/data_helper_spec.rb b/spec/lcm/integration/data_helper_spec.rb index 6892b1edb..d332d5989 100644 --- a/spec/lcm/integration/data_helper_spec.rb +++ b/spec/lcm/integration/data_helper_spec.rb @@ -138,19 +138,21 @@ } }, "project"=> "gdc-us-dev", - "schema"=> "mtt_team_checklist" + "schema"=> "lcm_test" } }, "input_source"=> { "type"=> "bigquery", - "query"=> "select * from employees;" + "query"=> "select * from employees order by EmployeeID" } } + describe 'data helper', :vcr do it 'connect to redshift with IAM authentication' do data_helper = GoodData::Helpers::DataSource.new(iam_params['input_source']) file_path = data_helper.realize(iam_params) + puts "redshift iam: #{file_path}" data = File.open('spec/data/redshift_data2.csv').read expect(data).to eq File.open(file_path).read end @@ -158,6 +160,7 @@ it 'connect to redshift with BASIC authentication' do data_helper = GoodData::Helpers::DataSource.new(basic_params['input_source']) file_path = data_helper.realize(basic_params) + puts "redshift basic 1: #{file_path}" data = File.open('spec/data/redshift_data2.csv').read expect(data).to eq File.open(file_path).read end @@ -165,6 +168,7 @@ it 'connect to redshift with BASIC authentication without schema' do data_helper = GoodData::Helpers::DataSource.new(basic_params_without_schema['input_source']) file_path = data_helper.realize(basic_params_without_schema) + puts "redshift basic 2: #{file_path}" data = File.open('spec/data/redshift_data.csv').read expect(data).to eq File.open(file_path).read end @@ -172,6 +176,7 @@ it 'connect to redshift with BASIC authentication and dynamic source' do data_helper = GoodData::Helpers::DataSource.new(basic_params_dynamic_source['dynamic_params']['input_source']) file_path = data_helper.realize(basic_params_dynamic_source) + puts "redshift basic 3: #{file_path}" data = File.open('spec/data/redshift_data2.csv').read expect(data).to eq File.open(file_path).read end @@ -179,6 +184,7 @@ it 'connect to redshift with BASIC authentication and url has parameter' do data_helper = GoodData::Helpers::DataSource.new(basic_params_url_parameters['input_source']) file_path = data_helper.realize(basic_params_url_parameters) + puts "redshift basic 4: #{file_path}" data = File.open('spec/data/redshift_data2.csv').read expect(data).to eq File.open(file_path).read end @@ -186,6 +192,7 @@ it 'connect to snowflake with BASIC authentication' do data_helper = GoodData::Helpers::DataSource.new(snowflake_basic_params['input_source']) file_path = data_helper.realize(snowflake_basic_params) + puts "snowflake: #{file_path}" data = File.open('spec/data/snowflake_data.csv').read expect(data).to eq File.open(file_path).read end @@ -198,6 +205,7 @@ data_helper = GoodData::Helpers::DataSource.new(bigquery_basic_params['input_source']) file_path = data_helper.realize(bigquery_basic_params) + puts "bigquery: #{file_path}" data = File.open('spec/data/bigquery_data.csv').read expect(data).to eq File.open(file_path).read end diff --git a/spec/lcm/integration/lcm_release_set_master_project_for_ads_spec.rb b/spec/lcm/integration/lcm_release_set_master_project_for_ads_spec.rb new file mode 100644 index 000000000..0d1d3132a --- /dev/null +++ b/spec/lcm/integration/lcm_release_set_master_project_for_ads_spec.rb @@ -0,0 +1,55 @@ +# encoding: UTF-8 +# +# Copyright (c) 2010-2017 GoodData Corporation. All rights reserved. +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +require_relative 'support/constants' +require_relative 'support/configuration_helper' +require_relative 'support/lcm_helper' +require_relative 'brick_runner' +require_relative 'shared_contexts_for_lcm' + +$master_projects = [] +$client_projects = [] + +describe 'Release brick with ADS', :vcr do + include_context 'lcm bricks', + ads: true + + it 'set master project - first run' do + @test_context[:master_project_id] = 'fake_project_id' + @test_context[:data_product] = 'LCM_DATA_PRODUCT_' + SecureRandom.urlsafe_base64(5).gsub('-', '_') + @test_context[:release_table_name] = "#{@test_context[:data_product]}_LCM_RELEASE" + BrickRunner.release_brick context: @test_context, template_path: '../params/release_brick_set_master_project.json.erb', client: @prod_rest_client, set_master_project: true + segments = JSON.parse(@test_context[:segments]) + segments.each do |s| + latest_master = GoodData::LCM2::Helpers.latest_master_project_from_ads(@test_context[:release_table_name], @ads_client, s['segment_id']) + expect(latest_master[:version].to_i).to eq 1 + expect(latest_master[:master_project_id]).to eq 'fake_project_id' + end + + end + + it 'set master project - second run' do + @test_context[:master_project_id] = 'project_id_1' + @test_context[:data_product] = 'LCM_DATA_PRODUCT_' + SecureRandom.urlsafe_base64(5).gsub('-', '_') + @test_context[:release_table_name] = "#{@test_context[:data_product]}_LCM_RELEASE" + BrickRunner.release_brick context: @test_context, template_path: '../params/release_brick_set_master_project.json.erb', client: @prod_rest_client, set_master_project: true + segments = JSON.parse(@test_context[:segments]) + segments.each do |s| + latest_master = GoodData::LCM2::Helpers.latest_master_project_from_ads(@test_context[:release_table_name], @ads_client, s['segment_id']) + expect(latest_master[:version].to_i).to eq 1 + expect(latest_master[:master_project_id]).to eq 'project_id_1' + end + + @test_context[:master_project_id] = 'project_id_2' + BrickRunner.release_brick context: @test_context, template_path: '../params/release_brick_set_master_project.json.erb', client: @prod_rest_client, set_master_project: true + segments.each do |s| + latest_master = GoodData::LCM2::Helpers.latest_master_project_from_ads(@test_context[:release_table_name], @ads_client, s['segment_id']) + expect(latest_master[:version].to_i).to eq 2 + expect(latest_master[:master_project_id]).to eq 'project_id_2' + end + end + +end diff --git a/spec/lcm/integration/lcm_release_set_master_project_for_nfs_spec.rb b/spec/lcm/integration/lcm_release_set_master_project_for_nfs_spec.rb new file mode 100644 index 000000000..49f7fae01 --- /dev/null +++ b/spec/lcm/integration/lcm_release_set_master_project_for_nfs_spec.rb @@ -0,0 +1,55 @@ +# encoding: UTF-8 +# +# Copyright (c) 2010-2017 GoodData Corporation. All rights reserved. +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +require_relative 'support/constants' +require_relative 'support/configuration_helper' +require_relative 'support/lcm_helper' +require_relative 'brick_runner' +require_relative 'shared_contexts_for_lcm' + +$master_projects = [] +$client_projects = [] + +describe 'Release brick with NFS', :vcr do + include_context 'lcm bricks', + ads: false + + it 'set master project - first run' do + @test_context[:master_project_id] = 'fake_project_id' + @test_context[:data_product] = 'LCM_DATA_PRODUCT_' + SecureRandom.urlsafe_base64(5).gsub('-', '_') + puts "data_product 1: #{@test_context[:data_product]}" + BrickRunner.release_brick context: @test_context, template_path: '../params/release_brick_set_master_project.json.erb', client: @prod_rest_client, set_master_project: true + segments = JSON.parse(@test_context[:segments]) + segments.each do |s| + latest_master = GoodData::LCM2::Helpers.latest_master_project_from_nfs(@config[:prod_organization], @test_context[:data_product], s['segment_id']) + expect(latest_master[:version].to_i).to eq 1 + expect(latest_master[:master_project_id]).to eq 'fake_project_id' + end + + end + + it 'set master project - second run' do + @test_context[:master_project_id] = 'project_id_1' + @test_context[:data_product] = 'LCM_DATA_PRODUCT_' + SecureRandom.urlsafe_base64(5).gsub('-', '_') + puts "data_product 2: #{@test_context[:data_product]}" + BrickRunner.release_brick context: @test_context, template_path: '../params/release_brick_set_master_project.json.erb', client: @prod_rest_client, set_master_project: true + segments = JSON.parse(@test_context[:segments]) + segments.each do |s| + latest_master = GoodData::LCM2::Helpers.latest_master_project_from_nfs(@config[:prod_organization], @test_context[:data_product], s['segment_id']) + expect(latest_master[:version].to_i).to eq 1 + expect(latest_master[:master_project_id]).to eq 'project_id_1' + end + + @test_context[:master_project_id] = 'project_id_2' + BrickRunner.release_brick context: @test_context, template_path: '../params/release_brick_set_master_project.json.erb', client: @prod_rest_client, set_master_project: true + segments.each do |s| + latest_master = GoodData::LCM2::Helpers.latest_master_project_from_nfs(@config[:prod_organization], @test_context[:data_product], s['segment_id']) + expect(latest_master[:version].to_i).to eq 2 + expect(latest_master[:master_project_id]).to eq 'project_id_2' + end + end + +end diff --git a/spec/lcm/integration/params/release_brick_set_master_project.json.erb b/spec/lcm/integration/params/release_brick_set_master_project.json.erb new file mode 100644 index 000000000..746a52802 --- /dev/null +++ b/spec/lcm/integration/params/release_brick_set_master_project.json.erb @@ -0,0 +1,22 @@ +{ + "organization": "<%= config[:prod_organization] %>", + "CLIENT_GDC_PROTOCOL": "https", + "CLIENT_GDC_HOSTNAME": "<%= config[:prod_server] %>", + "gd_encoded_params": { + <% if defined? jdbc_url %> + "ads_client": { + "username": "${GDC_USERNAME}", + "password": "${GDC_PASSWORD}", + "jdbc_url": "<%= jdbc_url %>" + }, + <% end %> + "SEGMENTS": <%= segments %>, + "DATA_PRODUCT": "<%= data_product %>" + }, + "release_table_name": "<%= release_table_name %>", + "fail_early": "true", + "strict": "true", + "GDC_USERNAME": "<%= config[:username] %>", + "GDC_PASSWORD": "<%= config[:password] %>", + "set_master_project": "<%= master_project_id %>" +} diff --git a/spec/lcm/integration/shared_contexts_for_lcm.rb b/spec/lcm/integration/shared_contexts_for_lcm.rb index 102080259..e1b843735 100644 --- a/spec/lcm/integration/shared_contexts_for_lcm.rb +++ b/spec/lcm/integration/shared_contexts_for_lcm.rb @@ -274,7 +274,7 @@ def create_workspace_csv(workspaces, client_id_column) begin GoodData.logger.info("Deleting segments") domain = @rest_client.domain(@config[:prod_organization]) - data_product = domain.data_products(@data_product_id) + data_product = domain.data_products(@test_context[:data_product]) data_product.delete(force: true) rescue StandardError => e GoodData.logger.warn("Failed to delete segments. #{e}")