Wrote ats methods for TrueUp

app/models/concerns/ats/trueup/application_fields.rb

@@ -0,0 +1,11 @@
+module Ats
+  module Trueup
+    module ApplicationFields
+      def get_application_criteria(job, _data)
+        p "Getting TrueUp application criteria"
+        job.application_criteria = {} # TODO: fix this
+        job.save
+      end
+    end
+  end
+end

app/models/concerns/ats/trueup/company_details.rb

@@ -1,14 +1,25 @@
 module Ats
   module Trueup
     module CompanyDetails
-      def company_details_from_data(data)
-        p "Fetching company details from data"
+      def fetch_company_id(data)
+        p "Fetching company ID"
+        data['company_id']
+      end
+
+      def company_details(ats_identifier)
+        data = Importer::Scraper::TrueUpCompanyDetails.call(ats_identifier)
         {
           name: data['company'],
-          url_website: data['company_url_clean'],
-          description: Flipper.enabled?(:company_description) ? data['business_description_short'] : 'Not added yet'
+          url_website: data['company_url_clean'], # TODO: make this the long form
+          description: Flipper.enabled?(:company_description) ? data['business_description_long'] : 'Not added yet',
+          industry: data['company_classification'],
+          location: data['hq_location']
         }
       end
+
+      def company_details_from_data(_data)
+        {}
+      end
     end
   end
 end

app/models/concerns/ats/trueup/job_details.rb

@@ -0,0 +1,83 @@
+module Ats
+  module Trueup
+    module JobDetails
+      include ActionView::Helpers::NumberHelper
+
+      def fetch_id(job_data)
+        p "Fetching ID"
+        job_data['unique_id']
+      end
+
+      def job_url_api(base_url, _company_id, _job_id)
+        p "Fetching job URL"
+        return base_url
+      end
+
+      def job_details(_job, data)
+        p "Fetching job details"
+        {
+          title: data['title'],
+          description: build_description(data),
+          posting_url: data['url'],
+          date_posted: (Date.parse(data['updated_at']) if data['updated_at']),
+          salary: build_salary(data),
+          non_geocoded_location_string: data['location']
+        }
+        # associate_technologies(job, data)
+      end
+
+      private
+
+      def build_description(data)
+        title = data['title']
+        company_name = data['company_short']
+        business_description = data['business_description_short']
+        location = data['location']
+        apply_url = data['url']
+        "<p>#{company_name}, a #{business_description&.downcase} company, is hiring a #{title} in #{location}.</p><p>To apply, visit <a href=\"#{apply_url}\">#{apply_url}</a>.</p>"
+      end
+
+      def build_salary(data)
+        # assumes that the salary is listed in GBP
+
+        min = data['salary_range_min']
+        max = data['salary_range_max']
+        return unless min || max
+
+        salary_low = number_with_delimiter(min)
+        salary_high = number_with_delimiter(max)
+        salary_low == salary_high ? "£#{salary_low} GBP" : "£#{salary_low} - £#{salary_high} GBP"
+      end
+    end
+  end
+end
+
+{
+  "job_id"=>"1818f540-831b-4487-8a55-c87fe36080cb",
+  "company_id"=>"lemfi",
+  "title"=>"Senior Software Engineer- Golang",
+  "url"=>"https://lemonade-technology-inc.breezy.hr/p/524f20b229a3-senior-software-engineer-golang",
+  "updated_at"=>"2024-07-03T23:36:07.000Z",
+  "updated_at_timestamp"=>1720049767,
+  "location"=>"United Kingdom (remote)",
+  "unique_id"=>"br-lemfi-524f20b229a3",
+  "salary_range_min"=>nil,
+  "salary_range_max"=>nil,
+  "company_url_clean"=>"lemfi.com",
+  "company_short"=>"LemFi",
+  "company"=>"LemFi",
+  "business_description_short"=>"Remittance payments",
+  "valuation"=>nil,
+  "job_openings_total"=>24,
+  "date_founded"=>"2020-01-01T00:00:00.000Z",
+  "trajectory_score"=>nil,
+  "ai_ready"=>1,
+  "sponsored"=>0,
+  "promoted"=>0,
+  "description_tags"=>["GoLang", "AWS", "PostgreSQL", "Docker", "GitHub"],
+  "company_investor_details"=>[{"name"=>"Y Combinator", "url"=>"ycombinator.com", "tag_id"=>"yc"}],
+  "company_description_plus"=>["🌱 Early-stage startup", "Total funding $34M", "Last raised 10mo ago ($33M Series A)", "5 yrs old", "110 employees"],
+  "ats_ux_warning"=>"",
+  "objectID"=>"12479004002",
+  "_highlightResult"=>{"title"=>{"value"=>"Senior Software Engineer- Golang", "matchLevel"=>"none", "matchedWords"=>[]}, "company"=>{"value"=>"LemFi", "matchLevel"=>"none", "matchedWords"=>[]}}
+}

app/services/faraday_helpers.rb

@@ -1,4 +1,11 @@
 module FaradayHelpers
+  def faraday_request(details)
+    endpoint = details[:endpoint]
+    verb = details[:verb]
+    options = details[:options]
+    fetch_json(endpoint, verb, options)
+  end
+
   def fetch_json(url, verb = :get, options = {})
     fetch_data(url, :parse_json, verb, options)
   end

app/services/importer/api/job_postings_api.rb

@@ -42,6 +42,17 @@ def process
         log_final_counts
       end
 
+      def count_redirects
+        @redirect_jobs.count
+      end
+
+      def create_company_and_job(job_data)
+        company = CompanyCreator.call(ats: @ats, data: job_data)
+        p "Company: #{company&.name}"
+        job = JobCreator.call(ats: @ats, company:, data: job_data)
+        p "Job: #{job&.title}"
+      end
+
       def extract_jobs_from_data(data)
         data
       end
@@ -52,40 +63,32 @@ def fetch_jobs_data
       end
 
       def fetch_and_save_remote_data
-        jobs_data = new_faraday_request
+        jobs_data = faraday_request(@api_details)
         return unless jobs_data
 
         save_jobs_data(jobs_data)
 
         jobs_data
       end
 
-      def new_faraday_request
-        endpoint = @api_details[:endpoint]
-        verb = @api_details[:verb]
-        options = @api_details[:options]
-        fetch_json(endpoint, verb, options)
-      end
-
-      def save_jobs_data(jobs_data)
-        @local_storage.save_jobs_data(jobs_data)
+      def log_initial_counts
+        p "Initial companies: #{@initial_companies}"
+        p "Initial jobs: #{@initial_jobs}"
       end
 
-      def set_initial_counts
-        @initial_companies = Company.count
-        @initial_jobs = Job.count
+      def log_fetched_jobs_data(jobs_data)
+        p "Fetched #{jobs_data.count} jobs from #{@ats.name}"
+        log_redirects
       end
 
-      def sort_by_hosted(jobs_data)
-        @redirect_jobs, @hosted_jobs = jobs_data.partition { |job_data| redirect?(job_data) }
+      def log_redirects
+        no_of_redirect_jobs = count_redirects
+        p "Redirected links: #{no_of_redirect_jobs}"
       end
 
-      # def redirect?(_job_data)
-      #   false
-      # end
-
-      def count_redirects
-        @redirect_jobs.count
+      def log_final_counts
+        p "Imported #{Company.count - @initial_companies} companies from #{@ats.name}"
+        p "Imported #{Job.count - @initial_jobs} jobs from #{@ats.name}"
       end
 
       def process_jobs
@@ -105,32 +108,21 @@ def process_redirect_jobs
         p "Redirected jobs processed."
       end
 
-      def create_company_and_job(job_data)
-        # TODO: Refactor to call CompanyCreator and JobCreator as service classes
-        company = CompanyCreator.call(ats: @ats, data: job_data)
-        p "Company: #{company&.name}"
-        job = JobCreator.call(ats: @ats, company:, data: job_data)
-        p "Job: #{job&.title}"
-      end
-
-      def log_initial_counts
-        p "Initial companies: #{@initial_companies}"
-        p "Initial jobs: #{@initial_jobs}"
-      end
+      # def redirect?(_job_data)
+      #   false
+      # end
 
-      def log_fetched_jobs_data(jobs_data)
-        p "Fetched #{jobs_data.count} jobs from #{@ats.name}"
-        log_redirects
+      def save_jobs_data(jobs_data)
+        @local_storage.save_jobs_data(jobs_data)
       end
 
-      def log_redirects
-        no_of_redirect_jobs = count_redirects
-        p "Redirected links: #{no_of_redirect_jobs}"
+      def set_initial_counts
+        @initial_companies = Company.count
+        @initial_jobs = Job.count
       end
 
-      def log_final_counts
-        p "Imported #{Company.count - @initial_companies} companies from #{@ats.name}"
-        p "Imported #{Job.count - @initial_jobs} jobs from #{@ats.name}"
+      def sort_by_hosted(jobs_data)
+        @redirect_jobs, @hosted_jobs = jobs_data.partition { |job_data| redirect?(job_data) }
       end
     end
   end

app/services/importer/api/true_up.rb

@@ -10,8 +10,6 @@ def initialize
         super(@ats, api_details, Url::CreateTrueupJobFromUrlJob)
       end
 
-      private
-
       def api_details
         {
           endpoint:,
@@ -24,14 +22,38 @@ def api_details
         }
       end
 
+      private
+
       def body
-        hits_per_page = 2000
+        # TODO: figure out how to most efficiently formulate this request to get all jobs, despite the hits_per_page limit
+
+        hits_per_page = 300 # This is limited by the API
         max_values_per_facet = 2000
         {
           requests: [
             {
               indexName: 'job',
               params: "facetFilters=[[\"job_locations_combined:🇬🇧 London, UK\",\"job_locations_combined:🇬🇧 United Kingdom (remote)\"],[\"job_categories_lvl0:Engineering (Software)\"]]&facets=[\"job_categories_lvl0\",\"job_locations_combined\"]&hitsPerPage=#{hits_per_page}&maxValuesPerFacet=#{max_values_per_facet}&query=&tagFilters="
+            },
+            {
+              indexName: 'job',
+              params: "facetFilters=[[\"job_locations_combined:🇬🇧 London, UK\",\"job_locations_combined:🇬🇧 United Kingdom (remote)\"]]&facets=[\"job_categories_lvl0\"]&hitsPerPage=#{hits_per_page}&maxValuesPerFacet=#{max_values_per_facet}&page=0&query="
+            },
+            {
+              indexName: 'job',
+              params: "facetFilters=[[\"job_categories_lvl0:Engineering (Software)\"]]&facets=job_locations_combined&hitsPerPage=#{hits_per_page}&maxValuesPerFacet=#{max_values_per_facet}&page=0&query="
+            },
+            {
+              indexName: 'job',
+              params: "facetFilters=[[\"job_locations_combined:🇬🇧 London, UK\",\"job_locations_combined:🇬🇧 United Kingdom (remote)\"],[\"job_categories_lvl0:Engineering (Software)\"]]&facets=[\"company_stage\",\"company_valuation\",\"curated_company_lists\",\"customer_type\",\"description_tags\",\"job_categories_lvl0\",\"job_categories_lvl1\",\"job_locations_combined\",\"level\",\"public_private\",\"themes\",\"top_investors\"]&highlightPostTag=__/ais-highlight__&highlightPreTag=__ais-highlight__&hitsPerPage=#{hits_per_page}&maxValuesPerFacet=#{max_values_per_facet}&page=0&query=&tagFilters="
+            },
+            {
+              indexName: 'job',
+              params: "facetFilters=[[\"job_locations_combined:🇬🇧 London, UK\",\"job_locations_combined:🇬🇧 United Kingdom (remote)\"]]&facets=[\"job_categories_lvl0\"]&highlightPostTag=__/ais-highlight__&highlightPreTag=__ais-highlight__&hitsPerPage=#{hits_per_page}&maxValuesPerFacet=#{max_values_per_facet}&page=0&query="
+            },
+            {
+              indexName: 'job',
+              params: "facetFilters=[[\"job_categories_lvl0:Engineering (Software)\"]]&facets=job_locations_combined&highlightPostTag=__/ais-highlight__&highlightPreTag=__ais-highlight__&hitsPerPage=#{hits_per_page}&maxValuesPerFacet=#{max_values_per_facet}&page=0&query="
             }
           ]
         }.to_json
@@ -42,7 +64,13 @@ def endpoint
       end
 
       def extract_jobs_from_data(data)
-        data.dig('results', 0, 'hits')
+        results = data&.dig('results')
+
+        jobs = results&.inject([]) do |total, job_group|
+          total + (job_group['hits'] || [])
+        end
+
+        jobs.uniq
       end
 
       def fetch_ats

app/services/importer/scraper/true_up_company_details.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module Importer
+  module Scraper
+    class TrueUpCompanyDetails < ApplicationTask
+      def initialize(ats_identifier)
+        @url = "https://www.trueup.io/co/#{ats_identifier}"
+      end
+
+      def call
+        return false unless processable
+
+        process
+      end
+
+      private
+
+      def processable
+        true
+      end
+
+      def process
+        return {} unless page_script
+
+        parse_company_data_from_script
+      end
+
+      def page_script
+        html = URI.parse(@url).open
+        doc = Nokogiri::HTML(html)
+        @page_script = doc.at_css('script#__NEXT_DATA__')
+      end
+
+      def parse_company_data_from_script
+        result = @page_script.content.strip
+        data = JSON.parse(result)
+        data&.dig('props', 'pageProps', 'companyInfo')
+      end
+    end
+  end
+end

app/tasks/devit_job_fetcher.rb

@@ -27,7 +27,5 @@ def process_as_devit_job
     p "Company: #{company&.name}"
     job = JobCreator.call(ats:, company:, data: @job_data)
     p "Job: #{job&.title}"
-  rescue StandardError => e
-    Rails.logger.error "Error creating company and job: #{e.message}"
   end
 end

app/tasks/trueup_job_fetcher.rb

@@ -20,6 +20,18 @@ def process
     # TODO: Create company separately, taking advantage of TrueUp company API?
 
     url = @job_data['url']
-    Url::CreateJobFromUrl.new(url).create_company_then_job
+    _ats, _company, job = Url::CreateJobFromUrl.new(url).create_company_then_job
+    process_as_true_up_job unless job&.persisted?
+  end
+
+  private
+
+  def process_as_true_up_job
+    ats = ApplicantTrackingSystem.find_by(name: 'TrueUp')
+    company = CompanyCreator.call(ats:, data: @job_data, apply_with_cheddar: false)
+    p "Company: #{company&.name}"
+
+    job = JobCreator.call(ats:, company:, data: @job_data)
+    p "Job: #{job&.title}"
   end
 end