Merge pull request #594 from Crown-Commercial-Service/add-prod-live-f…

…eatures-back-in

Add-prod-live-features-back-in

Gemfile

@@ -43,6 +43,10 @@ gem 'rollbar'
 gem 'webmock', '>= 3.12.1'
 # aws ssm
 gem 'aws-sdk-ssm'
+# azure
+gem 'azure-storage-blob'
+# cron job scheduling
+gem 'rufus-scheduler'
 
 group :development, :test do
   gem 'listen', '~> 3.2'

Gemfile.lock

@@ -74,6 +74,14 @@ GEM
       aws-sigv4 (~> 1.1)
     aws-sigv4 (1.2.3)
       aws-eventstream (~> 1, >= 1.0.2)
+    azure-storage-blob (2.0.3)
+      azure-storage-common (~> 2.0)
+      nokogiri (~> 1, >= 1.10.8)
+    azure-storage-common (2.0.4)
+      faraday (~> 1.0)
+      faraday_middleware (~> 1.0, >= 1.0.0.rc1)
+      net-http-persistent (~> 4.0)
+      nokogiri (~> 1, >= 1.10.8)
     better_html (1.0.16)
       actionview (>= 4.0)
       activesupport (>= 4.0)
@@ -89,6 +97,7 @@ GEM
     byebug (11.1.3)
     coderay (1.1.3)
     concurrent-ruby (1.1.10)
+    connection_pool (2.4.1)
     crack (0.4.5)
       rexml
     crass (1.0.6)
@@ -100,6 +109,8 @@ GEM
       dotenv (= 2.7.6)
       railties (>= 3.2)
     erubi (1.11.0)
+    et-orbi (1.2.7)
+      tzinfo
     factory_bot (6.2.1)
       activesupport (>= 5.0.0)
     factory_bot_rails (6.2.0)
@@ -119,6 +130,9 @@ GEM
       path_expander (~> 1.0)
       ruby_parser (~> 3.1, > 3.1.0)
       sexp_processor (~> 4.8)
+    fugit (1.8.1)
+      et-orbi (~> 1, >= 1.2.7)
+      raabro (~> 1.4)
     globalid (1.0.1)
       activesupport (>= 5.0)
     graphlient (0.5.0)
@@ -166,6 +180,8 @@ GEM
     msgpack (1.4.2)
     multi_json (1.15.0)
     multipart-post (2.1.1)
+    net-http-persistent (4.0.2)
+      connection_pool (~> 2.2)
     net-imap (0.3.2)
       date
       net-protocol
@@ -195,6 +211,7 @@ GEM
     public_suffix (4.0.6)
     puma (4.3.12)
       nio4r (~> 2.0)
+    raabro (1.4.0)
     racc (1.6.2)
     rack (2.2.6.4)
     rack-cors (1.1.1)
@@ -303,6 +320,8 @@ GEM
     ruby2_keywords (0.0.4)
     ruby_parser (3.15.1)
       sexp_processor (~> 4.9)
+    rufus-scheduler (3.9.1)
+      fugit (~> 1.1, >= 1.1.6)
     sexp_processor (4.15.2)
     simplecov (0.21.2)
       docile (~> 1.1)
@@ -342,6 +361,7 @@ PLATFORMS
 DEPENDENCIES
   activerecord-postgis-adapter (>= 6.0.1)
   aws-sdk-ssm
+  azure-storage-blob
   bootsnap (>= 1.4.2)
   brakeman
   byebug
@@ -370,6 +390,7 @@ DEPENDENCIES
   rubocop-performance (>= 1.10.2)
   rubocop-rails (>= 2.14.2)
   rubocop-rspec (>= 2.2.0)
+  rufus-scheduler
   simplecov
   sqlite3 (~> 1.4)
   webmock (>= 3.12.1)

app/controllers/api/v1/create_organisations_controller.rb

@@ -4,17 +4,13 @@ class CreateOrganisationsController < ActionController::API
       include Authorize::Token
       rescue_from ApiValidations::ApiError, with: :return_error_code
       before_action :validate_api_key
-      # This is checking for the dummy org (id: 111111111) in params. Must be done first, to prevent external api call.
-      before_action :mock_id_check
       before_action :validate_params
 
       attr_accessor :ccs_org_id, :api_result
 
       def index
-        result = search_scheme_api unless @is_mock_id
+        result = search_scheme_api
         generate_record(result)
-        # If the dummy org (id: 111111111) has been found, this will add it to db, and return the ccs_org_id to be rendered.
-        result = Common::ApiHelper.add_dummy_org(api_key_to_string, params[:identifier][:scheme], true) if @is_mock_id
         render_results(result)
       end
 
@@ -33,8 +29,7 @@ def render_results(result)
         if result.blank? || params[:identifier][:scheme].upcase == Common::AdditionalIdentifier::SCHEME_PPON
           render json: '', status: :not_found
         else
-          # @ccs_org_id is empty if dummy org (id: 111111111) was found. In this case the 'result' variable holds the ccs_org_id instead.
-          render json: { organisationId: @ccs_org_id || result }, status: :created
+          render json: { organisationId: @ccs_org_id }, status: :created
         end
       end
 
@@ -78,17 +73,10 @@ def additional_identifiers
       end
 
       def validate_params
-        return if @is_mock_id
-
         validate = ApiValidations::CreateOrganisation.new(params)
         render json: validate.errors, status: :bad_request unless validate.valid?
       end
 
-      # This is checking for the dummy org (id: 111111111) in params. Sets global variable to true or false, for the rest of controller behavior.
-      def mock_id_check
-        @is_mock_id = Common::ApiHelper.find_mock_organisation(params[:identifier][:scheme], params[:identifier][:id]) if params.present?
-      end
-
       def return_error_code(code)
         if code.to_s.length > 3
           render json: { organisationId: code }, status: '409'.freeze

app/controllers/api/v1/data_migration_controller.rb

@@ -4,8 +4,6 @@ class DataMigrationController < ActionController::API
       include Authorize::AuthorizationMethods
       rescue_from ApiValidations::ApiError, with: :return_error_code
       before_action :validate_integrating_service_user
-      # This is checking for the dummy org (id: 111111111) in params. must be done first, to stop external api calls.
-      before_action :mock_id_check
       before_action :create_ccs_org_id
 
       attr_accessor :ccs_org_id, :salesforce_result, :api_result, :sales_force_organisation_created
@@ -20,12 +18,10 @@ def create_ccs_org_id
         @duns_scheme = Common::AdditionalIdentifier::SCHEME_DANDB
         @coh_scheme = Common::AdditionalIdentifier::SCHEME_COMPANIES_HOUSE
         @sf_scheme = Common::SalesforceSearchIds::SFID
-        params[:account_id_type] = params[:account_id_type].downcase.delete('-') unless @is_mock_id
+        params[:account_id_type] = params[:account_id_type].downcase.delete('-')
       end
 
       def create_org_profile
-        return mock_id_dm_helper if @is_mock_id
-
         schemes_list = Common::AdditionalIdentifier.new
         create_from_schemes if schemes_list.schemes.include? params[:account_id_type].to_s
         create_from_salesforce if Common::SalesforceSearchIds.account_id_types_salesforce.include? params[:account_id_type].to_s
@@ -45,14 +41,6 @@ def return_success
         render json: '', status: :not_found if @api_result.blank?
       end
 
-      def mock_id_dm_helper
-        # If the dummy org (id: 111111111) has been found, this will add it to db, and return the ccs_org_id to be rendered.
-        ccs_org_id = Common::ApiHelper.add_dummy_org(nil, params[:account_id_type], true)
-        response_body = Common::ApiHelper.return_mock_organisation(params[:scheme])
-        response_body['organisationID'] = ccs_org_id.to_s
-        render json: response_body, status: :created
-      end
-
       def create_from_salesforce
         salesforce_api_search
         @sales_force_organisation_created = create_organisation if create_org_check
@@ -250,11 +238,6 @@ def validate_salesforce
           validate_additional_schemes(user_params) if user_params[:scheme] == Common::AdditionalIdentifier::SCHEME_CCS
         end
       end
-
-      # This is checking for the dummy org (id: 111111111) in params. Sets global variable to true or false, for the rest of controller flow.
-      def mock_id_check
-        @is_mock_id = Common::ApiHelper.find_mock_organisation(params[:account_id_type], params[:account_id]) if params.present?
-      end
     end
   end
 end

app/controllers/api/v1/manage_organisations_controller.rb

@@ -26,8 +26,6 @@ def validate_params
       private
 
       def api_result
-        return Common::ApiHelper.return_mock_organisation(params[:scheme]) if Common::ApiHelper.find_mock_organisation(params[:scheme], params[:id])
-
         search_api_with_params = SearchApi.new(params[:id], params[:scheme], params[:ccs_org_id])
         search_api_with_params.call
       end

app/controllers/api/v1/organisations_controller.rb

@@ -19,15 +19,12 @@ def search_organisation
       def validate_params
         params[:id] = params[:id].upcase if params[:scheme].upcase == Common::AdditionalIdentifier::SCHEME_NHS
         validate = ApiValidations::Scheme.new(params, return_organisation_id: true)
-        render json: validate.errors, status: :bad_request unless params[:id] == Common::AdditionalIdentifier::MOCK_ID || validate.valid?
+        render json: validate.errors, status: :bad_request unless validate.valid?
       end
 
       private
 
       def api_result
-        # Check for dummy org (id: 111111111), to handle that seperately to ordinary processing.
-        return Common::ApiHelper.return_mock_organisation(params[:scheme]) if Common::ApiHelper.find_mock_organisation(params[:scheme], params[:id])
-
         search_api_with_params = SearchApi.new(params[:id], params[:scheme], return_organisation_id: true)
         search_api_with_params.call
       end

app/controllers/api/v1/update_organisations_controller.rb

@@ -4,33 +4,22 @@ class UpdateOrganisationsController < ActionController::API
       include Authorize::AuthorizationMethods
       rescue_from ApiValidations::ApiError, with: :return_error_code
       before_action :validate_ccs_org_user_or_api_key
-      # This is checking for the dummy org (id: 111111111) in params. must be done first, to stop external api call.
-      before_action :mock_id_check
       before_action :validate_params
 
       attr_accessor :ccs_org_id, :api_result
 
       def index
-        result = set_results
+        result = search_scheme_api
+
+        Common::SalesforceHelper.new(result, params[:ccs_org_id]).insert_salesforce_record if result.present?
+
+        save_or_update_organisation_scheme if result.present?
+
         if result.blank?
           render json: '', status: :not_found
         else
-          # If mock id is used, then @ccs_org_id will be empty, and id will contained in 'result' variable instead.
-          render json: { organisationId: @ccs_org_id || result }, status: :ok
-        end
-      end
-
-      def set_results
-        if @is_mock_id
-          result = Common::ApiHelper.update_dummy_org(params[:ccs_org_id], params[:scheme])
-        else
-          result = search_scheme_api
-          return if result.blank?
-
-          Common::SalesforceHelper.new(result, params[:ccs_org_id]).insert_salesforce_record
-          save_or_update_organisation_scheme
+          render json: { organisationId: @ccs_org_id }, status: :ok
         end
-        result
       end
 
       def validate_params
@@ -51,11 +40,6 @@ def check_ppon_identifier(ccs_org_id)
         end
       end
 
-      # This is checking for the dummy org (id: 111111111) in params. Sets global variable to true or false, for the rest of controller behavoir.
-      def mock_id_check
-        @is_mock_id = Common::ApiHelper.find_mock_organisation(params[:scheme], params[:id]) if params.present?
-      end
-
       private
 
       def save_or_update_organisation_scheme

app/models/export.rb

@@ -0,0 +1,81 @@
+class Export
+  include ActionController::MimeResponds
+  include Authorize::Token
+  require 'csv'
+  require 'date'
+  require 'fileutils'
+  require 'azure/storage/blob'
+
+  def self.begin
+    Rails.logger.info 'BEGINNING DATA EXPORT JOB...'
+
+    directory = Rails.root.join('public/export').to_s
+    file_path = Rails.root.join("public/export/#{Date.yesterday}_Organisations.csv").to_s
+
+    FileUtils.mkdir_p directory unless File.directory?(directory)
+
+    csv_generated = generate_csv_export(file_path)
+    return upload_to_azure(file_path) if csv_generated
+
+    Rails.logger.info 'NO DATA TO EXPORT - JOB COMPLETE'
+    nil
+  end
+
+  def self.generate_csv_export(file_path)
+    Rails.logger.info "GENERATING #{file_path}"
+
+    organisations = find_organisations
+    return failed unless organisations.any?
+
+    CSV.open(file_path, 'w') do |writer|
+      writer << organisations.first.attributes.map { |a, _v| a }
+      organisations.each do |s|
+        writer << s.attributes.map { |_a, v| v }
+      end
+    end
+
+    success
+  end
+
+  def self.upload_to_azure(file_path)
+    Rails.logger.info "UPLOADING #{file_path} > #{azure_container_name}"
+
+    file_name = "#{Date.yesterday}_Organisations.csv"
+    file_content = File.binread(file_path)
+    azure_client.create_block_blob(azure_container_name, file_name, file_content)
+
+    delete_file(file_path)
+    Rails.logger.info 'DATA EXPORTED - JOB COMPLETE'
+  end
+
+  def self.azure_client
+    @azure_client ||= Azure::Storage::Blob::BlobService.create(azure_client_config)
+  end
+
+  def self.azure_client_config
+    {
+      storage_account_name: ENV.fetch('AZURE_ACCOUNT_NAME', nil),
+      storage_access_key: ENV.fetch('AZURE_ACCOUNT_KEY', nil)
+    }
+  end
+
+  def self.azure_container_name
+    ENV.fetch('AZURE_CONTAINER_NAME', 'ContainerName')
+  end
+
+  def self.delete_file(file_path)
+    File.delete(file_path) if File.exist?(file_path)
+  end
+
+  def self.find_organisations
+    OrganisationSchemeIdentifier.where(updated_at: Date.yesterday.all_day)
+  end
+
+  def self.success
+    true
+  end
+
+  def self.failed
+    false
+  end
+end

app/services/common/additional_identifier.rb

@@ -4,7 +4,6 @@ class AdditionalIdentifier
     DANDB_ENG_WALES_CHARITY_NUMBER_CODE = 33463
     DANDB_SCOTTISH_CHARITY_NUMBER_CODE = 33461
     DANDB_NORTHERN_IRELAND_CHARITY_NUMBER_CODE = 33462
-    MOCK_ID = '111111111'.freeze
     MOCK_SF_ID = 'NSO7IUSHF98HFP9WEH7NSF~56734565478'.freeze
     SCHEME_ENG_WALES_CHARITY = 'GB-CHC'.freeze
     SCHEME_NORTHEN_IRELAND_CHARITY = 'GB-NIC'.freeze