15/05/24 update existing company jobs

.gitignore

@@ -47,3 +47,6 @@
 .idea/modules.xml
 .idea/vcs.xml
 .idea/.gitignore
+
+# ignore .jar files
+*.jar

app/controllers/job_applications_controller.rb

@@ -35,6 +35,7 @@ def new
             core_field: details["core_field"]
           )
         end
+        p job_application.application_responses
         job_application
       end
     else

app/jobs/apply_job.rb

@@ -48,8 +48,6 @@ def assign_values_to_form(application, user)
         application_criteria[key]['value'] = custom_fields[key]
       end
     end
-    puts "Here are the application criteria:"
-    p application_criteria
     return application_criteria
   end
 end

app/jobs/scrape_true_up_job.rb

@@ -129,6 +129,6 @@ def alt_identifier
   def store_to_database
     puts "\nFound #{@companies_added} new company ats identifiers."
     puts "\nStoring the information in CSV format..."
-    save_ats_list
+    save_ats_list(@companies)
   end
 end

app/jobs/update_existing_company_jobs.rb

@@ -1,6 +1,5 @@
 class UpdateExistingCompanyJobs < ApplicationJob
   include CompanyCsv
-  include Relevant
 
   # Why do we get all the job urls?
   # Wouldn't the best approach here be to create a csv with all the company data from the respective APIs
@@ -15,64 +14,86 @@ class UpdateExistingCompanyJobs < ApplicationJob
   def perform
     puts "Beginning jobs updater for companies already seeded to the DB..."
 
-    @job_urls = Job.all.to_set(&:posting_url)
+    # Compile a list of urls (unique ids) of jobs from last update that may no longer be live
+    @job_urls_from_last_update = Job.all.to_set(&:posting_url)
+    @company_ids = ats_list
+    @invalid_company_ids = ats_hash
 
     fetch_jobs_and_companies
-    destroy_defunct_jobs
-    # update_ats_identifiers_list
+    mark_defunct_jobs
+    update_lists
+
+    puts "Completed jobs updater."
   end
 
   private
 
   def fetch_jobs_and_companies
-    ats_list.each do |ats_name, ats_identifiers|
-      # only prepared to handle some ATS systems at the moment
-      next unless ['Greenhouse', 'Lever'].include?(ats_name)
-
+    @company_ids.each do |ats_name, ats_identifiers|
       puts "Scanning #{ats_name} jobs:"
       @ats = ApplicantTrackingSystem.find_by(name: ats_name)
 
       ats_identifiers.each do |ats_identifier|
         puts "Looking at jobs with #{ats_identifier}..."
 
         # Find or create the company
-        next puts "Problem with #{ats_identifier}" unless (company = @ats.find_or_create_company(ats_identifier))
+        unless (company = @ats.find_or_create_company(ats_identifier))
+          puts "Problem with #{ats_identifier}"
+          add_to_invalid_ids(ats_name, ats_identifier)
+          next
+        end
+
+        begin
+          company_jobs = company.create_all_relevant_jobs
+        rescue NoDataReturnedError => e
+          puts e.message
+          add_to_invalid_ids(ats_name, ats_identifier)
+          next
+        end
+
+        # Cross each live job off the list
+        company_jobs.each { |job| @job_urls_from_last_update.delete(job.posting_url) }
+      end
+    end
+  end
 
-        company_jobs = @ats.fetch_company_jobs(ats_identifier)
+  def mark_defunct_jobs
+    puts "Marking jobs that are no longer live:"
+    @job_urls_from_last_update.each do |posting_url|
+      job = Job.find_by(posting_url:)
+      job.live = false
+      puts "No longer live: #{job.title}"
+    end
+  end
 
-        next unless company_jobs
+  def update_lists
+    remove_invalid_ids_from_company_ids
+    update_company_ids
+    update_invalid_ids
+  end
 
-        # Create new jobs using AtsSystem method
-        company_jobs.each do |job_data|
-          @ats.find_or_create_job_by_data(company, job_data) if relevant?(job_data)
-          @job_urls.delete(@ats.fetch_url(job_data))
-        rescue StandardError => e
-          puts "Error: #{e}"
-        end
+  def remove_invalid_ids_from_company_ids
+    @invalid_company_ids.each do |ats_name, list|
+      list.each do |company_id|
+        @company_ids[ats_name].delete(company_id)
       end
     end
   end
 
-  def relevant?(job_data)
-    # TODO: call the Relevant module method instead
-    title, job_location, remote = @ats.fetch_title_and_location(job_data)
-
-    (title &&
-      remote &&
-      JOB_TITLE_KEYWORDS.any? { |keyword| title.downcase.match?(keyword) }) ||
-      (title &&
-      job_location &&
-      JOB_LOCATION_KEYWORDS.any? { |keyword| job_location.downcase.match?(keyword) } &&
-      JOB_TITLE_KEYWORDS.any? { |keyword| title.downcase.match?(keyword) })
+  def update_company_ids
+    save_ats_list(@company_ids)
   end
 
-  # TODO: Update this so that the jobs are kept but are no longer live on the site
+  def add_to_invalid_ids(ats_name, ats_identifier)
+    puts "#{ats_identifier} is an invalid id for #{ats_name}"
+    @invalid_company_ids[ats_name] << ats_identifier
+  end
 
-  def destroy_defunct_jobs
-    puts "Deleting jobs that are no longer live:"
-    @job_urls.each do |posting_url|
-      Job.find_by(posting_url:).destroy
-      puts "Destroyed #{posting_url}"
+  def update_invalid_ids
+    invalid_ids_from_last_update = load_from_csv('invalid_ids')
+    @invalid_company_ids.merge!(invalid_ids_from_last_update) do |_k, new_set, old_set|
+      new_set.merge(old_set)
     end
+    save_ats_list(@invalid_company_ids, 'invalid_ids')
   end
 end

app/models/applicant_tracking_system.rb

@@ -190,6 +190,12 @@ def fetch_title_and_location(data)
     refer_to_module(defined?(super) ? super : nil, __method__)
   end
 
+  def mark_job_expired(job)
+    p "Job with ID #{job.ats_job_id} is expired."
+    job.live = false
+    return nil
+  end
+
   private
 
   def job_details(new_job, data)
@@ -205,6 +211,7 @@ def fetch_job_data(job)
   end
 
   def fetch_additional_fields(job, data)
+    p job
     get_application_criteria(job, data)
     p "Getting form fields for #{job.title}..."
     job.save! # must save before passing to Sidekiq job

app/models/company.rb

@@ -14,9 +14,11 @@ class Company < ApplicationRecord
   # multisearchable against: [:name]
 
   def create_all_relevant_jobs
-    jobs_created = 0
+    jobs_found_or_created = []
     ats = applicant_tracking_system
     all_jobs = ats.fetch_company_jobs(ats_identifier)
+    raise NoDataReturnedError, "The API returned no jobs data for #{name}" unless all_jobs
+
     all_jobs.each do |job_data|
       details = ats.fetch_title_and_location(job_data)
       next unless relevant?(*details)
@@ -28,9 +30,10 @@ def create_all_relevant_jobs
       else
         job = ats.find_or_create_job_by_data(self, job_data)
       end
-      jobs_created += 1 if job&.persisted?
+      jobs_found_or_created << job if job&.persisted?
     end
-    puts "Created #{jobs_created} new jobs with #{name}."
+    puts "Found or created #{jobs_found_or_created.size} new jobs with #{name}."
+    jobs_found_or_created
   end
 
   private

app/models/concerns/ats/bamboohr/application_fields.rb

@@ -37,6 +37,7 @@ def build_additional_field(data)
           question_set.merge!(
             name => {
               interaction:,
+              label: field['question'],
               locators: field['id'],
               required: field['isRequired'],
               options:
@@ -46,7 +47,7 @@ def build_additional_field(data)
       end
 
       def fetch_field_attributes(field)
-        name = field['question']
+        name = field['question'].downcase.gsub(' ', '_').gsub(/[^a-z0-9_-]/, '')
         options = field['options']&.map { |option| option['text'] } unless field['options'].blank?
         interaction = FIELD_TYPES[field['type']]
         if interaction == :boolean
@@ -64,66 +65,133 @@ def fetch_field_attributes(field)
       }
 
       CORE_FIELDS = {
-        'First Name' => {
+        first_name: {
           interaction: :input,
-          locators: 'firstName'
+          locators: 'firstName',
+          label: 'First Name',
+          core_field: true
         },
-        'Last Name' => {
+        last_name: {
           interaction: :input,
-          locators: 'lastName'
+          locators: 'lastName',
+          label: 'Last Name',
+          core_field: true
         },
-        'Email' => {
+        email: {
           interaction: :input,
-          locators: 'email'
+          locators: 'email',
+          label: 'Email',
+          core_field: true
         },
-        'Phone' => {
+        phone_number: {
           interaction: :input,
-          locators: 'phone'
+          locators: 'phone',
+          label: 'Phone',
+          core_field: true
         },
-        'Address' => {
+        address: {
           interaction: :input,
-          locators: 'streetAddress'
+          locators: 'streetAddress',
+          label: 'Address',
+          core_field: true
         },
-        'City' => {
+        city: {
           interaction: :input,
-          locators: 'city'
+          locators: 'city',
+          label: 'City',
+          core_field: true
         },
-        'State' => {
+        state: {
           interaction: :select,
-          locators: 'state'
-          # locators: 'fab-SelectToggle__placeholder'
+          locators: 'state',
+          label: 'State',
+          core_field: true
         },
-        'ZIP' => {
+        post_code: {
           interaction: :input,
-          locators: 'zip'
+          locators: 'zip',
+          label: 'ZIP',
+          core_field: true
         },
-        'Country' => {
+        country: {
           interaction: :input,
-          locators: 'countryId'
+          locators: 'countryId',
+          label: 'Country',
+          core_field: true
         },
-        'Website, Blog, or Portfolio' => {
+        website_url: {
           interaction: :input,
-          locators: 'websiteUrl'
+          locators: 'websiteUrl',
+          label: 'Website, Blog, or Portfolio',
+          core_field: true
         },
-        'LinkedIn Profile URL' => {
+        linkedin_url: {
           interaction: :input,
-          locators: 'linkedinUrl'
+          locators: 'linkedinUrl',
+          label: 'LinkedIn Profile URL',
+          core_field: true
         },
-        'Cover Letter' => {
+        cover_letter: {
           interaction: :upload,
-          locators: 'coverLetterFileId'
+          locators: 'coverLetterFileId',
+          label: 'Cover Letter',
+          core_field: true
         },
-        'Resume' => {
+        resume: {
           interaction: :upload,
-          locators: 'resumeFileId'
+          locators: 'resumeFileId',
+          label: 'Resume',
+          core_field: true
         },
-        'Date Available' => {
+        education_level: {
+          interaction: :select,
+          locators: 'educationLevelId',
+          label: 'Highest Education Obtained',
+          core_field: true
+        },
+        education_school: {
+          interaction: :input,
+          locators: 'educationInstitutionName',
+          label: 'College/University',
+          core_field: true
+        },
+        date_available: {
           interaction: :date,
-          locators: 'dateAvailable'
+          locators: 'dateAvailable',
+          label: 'Date Available',
+          core_field: true
         },
-        'Desired Pay' => {
+        desired_pay: {
           interaction: :input,
-          locators: 'desiredPay'
+          locators: 'desiredPay',
+          label: 'Desired Pay',
+          core_field: true
+        },
+        referred_by: {
+          interaction: :input,
+          locators: 'referredBy',
+          label: 'Who referred you for this position?',
+          core_field: true
+        },
+        gender: {
+          interaction: :select,
+          locators: 'genderId',
+          label: 'Gender'
+        },
+        ethnicity: {
+          interaction: :select,
+          locators: 'ethnicityId',
+          label: 'Ethnicity'
+        },
+        disability: {
+          interaction: :select,
+          locators: 'disabilityId',
+          label: 'Disability'
+        },
+        veteran_status: {
+          interaction: :select,
+          locators: 'veteranStatusId',
+          label: 'Veteran Status'
         }
       }
     end