Skip to content

Commit

Permalink
Merge pull request #594 from Crown-Commercial-Service/add-prod-live-f…
Browse files Browse the repository at this point in the history
…eatures-back-in

Add-prod-live-features-back-in
  • Loading branch information
tberey committed Jul 3, 2023
2 parents 1de53f7 + f0a0ef9 commit f484712
Show file tree
Hide file tree
Showing 13 changed files with 126 additions and 182 deletions.
4 changes: 4 additions & 0 deletions Gemfile
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,10 @@ gem 'rollbar'
gem 'webmock', '>= 3.12.1'
# aws ssm
gem 'aws-sdk-ssm'
# azure
gem 'azure-storage-blob'
# cron job scheduling
gem 'rufus-scheduler'

group :development, :test do
gem 'listen', '~> 3.2'
Expand Down
21 changes: 21 additions & 0 deletions Gemfile.lock
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,14 @@ GEM
aws-sigv4 (~> 1.1)
aws-sigv4 (1.2.3)
aws-eventstream (~> 1, >= 1.0.2)
azure-storage-blob (2.0.3)
azure-storage-common (~> 2.0)
nokogiri (~> 1, >= 1.10.8)
azure-storage-common (2.0.4)
faraday (~> 1.0)
faraday_middleware (~> 1.0, >= 1.0.0.rc1)
net-http-persistent (~> 4.0)
nokogiri (~> 1, >= 1.10.8)
better_html (1.0.16)
actionview (>= 4.0)
activesupport (>= 4.0)
Expand All @@ -89,6 +97,7 @@ GEM
byebug (11.1.3)
coderay (1.1.3)
concurrent-ruby (1.1.10)
connection_pool (2.4.1)
crack (0.4.5)
rexml
crass (1.0.6)
Expand All @@ -100,6 +109,8 @@ GEM
dotenv (= 2.7.6)
railties (>= 3.2)
erubi (1.11.0)
et-orbi (1.2.7)
tzinfo
factory_bot (6.2.1)
activesupport (>= 5.0.0)
factory_bot_rails (6.2.0)
Expand All @@ -119,6 +130,9 @@ GEM
path_expander (~> 1.0)
ruby_parser (~> 3.1, > 3.1.0)
sexp_processor (~> 4.8)
fugit (1.8.1)
et-orbi (~> 1, >= 1.2.7)
raabro (~> 1.4)
globalid (1.0.1)
activesupport (>= 5.0)
graphlient (0.5.0)
Expand Down Expand Up @@ -166,6 +180,8 @@ GEM
msgpack (1.4.2)
multi_json (1.15.0)
multipart-post (2.1.1)
net-http-persistent (4.0.2)
connection_pool (~> 2.2)
net-imap (0.3.2)
date
net-protocol
Expand Down Expand Up @@ -195,6 +211,7 @@ GEM
public_suffix (4.0.6)
puma (4.3.12)
nio4r (~> 2.0)
raabro (1.4.0)
racc (1.6.2)
rack (2.2.6.4)
rack-cors (1.1.1)
Expand Down Expand Up @@ -303,6 +320,8 @@ GEM
ruby2_keywords (0.0.4)
ruby_parser (3.15.1)
sexp_processor (~> 4.9)
rufus-scheduler (3.9.1)
fugit (~> 1.1, >= 1.1.6)
sexp_processor (4.15.2)
simplecov (0.21.2)
docile (~> 1.1)
Expand Down Expand Up @@ -342,6 +361,7 @@ PLATFORMS
DEPENDENCIES
activerecord-postgis-adapter (>= 6.0.1)
aws-sdk-ssm
azure-storage-blob
bootsnap (>= 1.4.2)
brakeman
byebug
Expand Down Expand Up @@ -370,6 +390,7 @@ DEPENDENCIES
rubocop-performance (>= 1.10.2)
rubocop-rails (>= 2.14.2)
rubocop-rspec (>= 2.2.0)
rufus-scheduler
simplecov
sqlite3 (~> 1.4)
webmock (>= 3.12.1)
Expand Down
16 changes: 2 additions & 14 deletions app/controllers/api/v1/create_organisations_controller.rb
Original file line number Diff line number Diff line change
Expand Up @@ -4,17 +4,13 @@ class CreateOrganisationsController < ActionController::API
include Authorize::Token
rescue_from ApiValidations::ApiError, with: :return_error_code
before_action :validate_api_key
# This is checking for the dummy org (id: 111111111) in params. Must be done first, to prevent external api call.
before_action :mock_id_check
before_action :validate_params

attr_accessor :ccs_org_id, :api_result

def index
result = search_scheme_api unless @is_mock_id
result = search_scheme_api
generate_record(result)
# If the dummy org (id: 111111111) has been found, this will add it to db, and return the ccs_org_id to be rendered.
result = Common::ApiHelper.add_dummy_org(api_key_to_string, params[:identifier][:scheme], true) if @is_mock_id
render_results(result)
end

Expand All @@ -33,8 +29,7 @@ def render_results(result)
if result.blank? || params[:identifier][:scheme].upcase == Common::AdditionalIdentifier::SCHEME_PPON
render json: '', status: :not_found
else
# @ccs_org_id is empty if dummy org (id: 111111111) was found. In this case the 'result' variable holds the ccs_org_id instead.
render json: { organisationId: @ccs_org_id || result }, status: :created
render json: { organisationId: @ccs_org_id }, status: :created
end
end

Expand Down Expand Up @@ -78,17 +73,10 @@ def additional_identifiers
end

def validate_params
return if @is_mock_id

validate = ApiValidations::CreateOrganisation.new(params)
render json: validate.errors, status: :bad_request unless validate.valid?
end

# This is checking for the dummy org (id: 111111111) in params. Sets global variable to true or false, for the rest of controller behavior.
def mock_id_check
@is_mock_id = Common::ApiHelper.find_mock_organisation(params[:identifier][:scheme], params[:identifier][:id]) if params.present?
end

def return_error_code(code)
if code.to_s.length > 3
render json: { organisationId: code }, status: '409'.freeze
Expand Down
19 changes: 1 addition & 18 deletions app/controllers/api/v1/data_migration_controller.rb
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,6 @@ class DataMigrationController < ActionController::API
include Authorize::AuthorizationMethods
rescue_from ApiValidations::ApiError, with: :return_error_code
before_action :validate_integrating_service_user
# This is checking for the dummy org (id: 111111111) in params. must be done first, to stop external api calls.
before_action :mock_id_check
before_action :create_ccs_org_id

attr_accessor :ccs_org_id, :salesforce_result, :api_result, :sales_force_organisation_created
Expand All @@ -20,12 +18,10 @@ def create_ccs_org_id
@duns_scheme = Common::AdditionalIdentifier::SCHEME_DANDB
@coh_scheme = Common::AdditionalIdentifier::SCHEME_COMPANIES_HOUSE
@sf_scheme = Common::SalesforceSearchIds::SFID
params[:account_id_type] = params[:account_id_type].downcase.delete('-') unless @is_mock_id
params[:account_id_type] = params[:account_id_type].downcase.delete('-')
end

def create_org_profile
return mock_id_dm_helper if @is_mock_id

schemes_list = Common::AdditionalIdentifier.new
create_from_schemes if schemes_list.schemes.include? params[:account_id_type].to_s
create_from_salesforce if Common::SalesforceSearchIds.account_id_types_salesforce.include? params[:account_id_type].to_s
Expand All @@ -45,14 +41,6 @@ def return_success
render json: '', status: :not_found if @api_result.blank?
end

def mock_id_dm_helper
# If the dummy org (id: 111111111) has been found, this will add it to db, and return the ccs_org_id to be rendered.
ccs_org_id = Common::ApiHelper.add_dummy_org(nil, params[:account_id_type], true)
response_body = Common::ApiHelper.return_mock_organisation(params[:scheme])
response_body['organisationID'] = ccs_org_id.to_s
render json: response_body, status: :created
end

def create_from_salesforce
salesforce_api_search
@sales_force_organisation_created = create_organisation if create_org_check
Expand Down Expand Up @@ -250,11 +238,6 @@ def validate_salesforce
validate_additional_schemes(user_params) if user_params[:scheme] == Common::AdditionalIdentifier::SCHEME_CCS
end
end

# This is checking for the dummy org (id: 111111111) in params. Sets global variable to true or false, for the rest of controller flow.
def mock_id_check
@is_mock_id = Common::ApiHelper.find_mock_organisation(params[:account_id_type], params[:account_id]) if params.present?
end
end
end
end
2 changes: 0 additions & 2 deletions app/controllers/api/v1/manage_organisations_controller.rb
Original file line number Diff line number Diff line change
Expand Up @@ -26,8 +26,6 @@ def validate_params
private

def api_result
return Common::ApiHelper.return_mock_organisation(params[:scheme]) if Common::ApiHelper.find_mock_organisation(params[:scheme], params[:id])

search_api_with_params = SearchApi.new(params[:id], params[:scheme], params[:ccs_org_id])
search_api_with_params.call
end
Expand Down
5 changes: 1 addition & 4 deletions app/controllers/api/v1/organisations_controller.rb
Original file line number Diff line number Diff line change
Expand Up @@ -19,15 +19,12 @@ def search_organisation
def validate_params
params[:id] = params[:id].upcase if params[:scheme].upcase == Common::AdditionalIdentifier::SCHEME_NHS
validate = ApiValidations::Scheme.new(params, return_organisation_id: true)
render json: validate.errors, status: :bad_request unless params[:id] == Common::AdditionalIdentifier::MOCK_ID || validate.valid?
render json: validate.errors, status: :bad_request unless validate.valid?
end

private

def api_result
# Check for dummy org (id: 111111111), to handle that seperately to ordinary processing.
return Common::ApiHelper.return_mock_organisation(params[:scheme]) if Common::ApiHelper.find_mock_organisation(params[:scheme], params[:id])

search_api_with_params = SearchApi.new(params[:id], params[:scheme], return_organisation_id: true)
search_api_with_params.call
end
Expand Down
30 changes: 7 additions & 23 deletions app/controllers/api/v1/update_organisations_controller.rb
Original file line number Diff line number Diff line change
Expand Up @@ -4,33 +4,22 @@ class UpdateOrganisationsController < ActionController::API
include Authorize::AuthorizationMethods
rescue_from ApiValidations::ApiError, with: :return_error_code
before_action :validate_ccs_org_user_or_api_key
# This is checking for the dummy org (id: 111111111) in params. must be done first, to stop external api call.
before_action :mock_id_check
before_action :validate_params

attr_accessor :ccs_org_id, :api_result

def index
result = set_results
result = search_scheme_api

Common::SalesforceHelper.new(result, params[:ccs_org_id]).insert_salesforce_record if result.present?

save_or_update_organisation_scheme if result.present?

if result.blank?
render json: '', status: :not_found
else
# If mock id is used, then @ccs_org_id will be empty, and id will contained in 'result' variable instead.
render json: { organisationId: @ccs_org_id || result }, status: :ok
end
end

def set_results
if @is_mock_id
result = Common::ApiHelper.update_dummy_org(params[:ccs_org_id], params[:scheme])
else
result = search_scheme_api
return if result.blank?

Common::SalesforceHelper.new(result, params[:ccs_org_id]).insert_salesforce_record
save_or_update_organisation_scheme
render json: { organisationId: @ccs_org_id }, status: :ok
end
result
end

def validate_params
Expand All @@ -51,11 +40,6 @@ def check_ppon_identifier(ccs_org_id)
end
end

# This is checking for the dummy org (id: 111111111) in params. Sets global variable to true or false, for the rest of controller behavoir.
def mock_id_check
@is_mock_id = Common::ApiHelper.find_mock_organisation(params[:scheme], params[:id]) if params.present?
end

private

def save_or_update_organisation_scheme
Expand Down
81 changes: 81 additions & 0 deletions app/models/export.rb
Original file line number Diff line number Diff line change
@@ -0,0 +1,81 @@
class Export
include ActionController::MimeResponds
include Authorize::Token
require 'csv'
require 'date'
require 'fileutils'
require 'azure/storage/blob'

def self.begin
Rails.logger.info 'BEGINNING DATA EXPORT JOB...'

directory = Rails.root.join('public/export').to_s
file_path = Rails.root.join("public/export/#{Date.yesterday}_Organisations.csv").to_s

FileUtils.mkdir_p directory unless File.directory?(directory)

csv_generated = generate_csv_export(file_path)
return upload_to_azure(file_path) if csv_generated

Rails.logger.info 'NO DATA TO EXPORT - JOB COMPLETE'
nil
end

def self.generate_csv_export(file_path)
Rails.logger.info "GENERATING #{file_path}"

organisations = find_organisations
return failed unless organisations.any?

CSV.open(file_path, 'w') do |writer|
writer << organisations.first.attributes.map { |a, _v| a }
organisations.each do |s|
writer << s.attributes.map { |_a, v| v }
end
end

success
end

def self.upload_to_azure(file_path)
Rails.logger.info "UPLOADING #{file_path} > #{azure_container_name}"

file_name = "#{Date.yesterday}_Organisations.csv"
file_content = File.binread(file_path)
azure_client.create_block_blob(azure_container_name, file_name, file_content)

delete_file(file_path)
Rails.logger.info 'DATA EXPORTED - JOB COMPLETE'
end

def self.azure_client
@azure_client ||= Azure::Storage::Blob::BlobService.create(azure_client_config)
end

def self.azure_client_config
{
storage_account_name: ENV.fetch('AZURE_ACCOUNT_NAME', nil),
storage_access_key: ENV.fetch('AZURE_ACCOUNT_KEY', nil)
}
end

def self.azure_container_name
ENV.fetch('AZURE_CONTAINER_NAME', 'ContainerName')
end

def self.delete_file(file_path)
File.delete(file_path) if File.exist?(file_path)
end

def self.find_organisations
OrganisationSchemeIdentifier.where(updated_at: Date.yesterday.all_day)
end

def self.success
true
end

def self.failed
false
end
end
1 change: 0 additions & 1 deletion app/services/common/additional_identifier.rb
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@ class AdditionalIdentifier
DANDB_ENG_WALES_CHARITY_NUMBER_CODE = 33463
DANDB_SCOTTISH_CHARITY_NUMBER_CODE = 33461
DANDB_NORTHERN_IRELAND_CHARITY_NUMBER_CODE = 33462
MOCK_ID = '111111111'.freeze
MOCK_SF_ID = 'NSO7IUSHF98HFP9WEH7NSF~56734565478'.freeze
SCHEME_ENG_WALES_CHARITY = 'GB-CHC'.freeze
SCHEME_NORTHEN_IRELAND_CHARITY = 'GB-NIC'.freeze
Expand Down
Loading

0 comments on commit f484712

Please sign in to comment.