Mix.install([
{:spider_man, "~> 0.3"},
{:floki, "~> 0.31"},
{:nimble_csv, "~> 1.1"},
{:timex, "~> 3.0"},
{:kino, "~> 0.8.0"}
])Build settings for spider
base_url = "https://elixirjobs.net/"
requester_options = [
base_url: base_url,
middlewares: [
{SpiderMan.Middleware.UserAgent,
[
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.93 Safari/537.36",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.114 Safari/537.36",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.82 Safari/537.36",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4389.82 Safari/537.36",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4389.82 Safari/537.36",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.94 Safari/537.36"
]},
{Tesla.Middleware.Headers,
[
{"referer", base_url},
{"accept",
"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9"},
{"accept-encoding", "gzip, deflate"},
{"accept-language", "zh-CN,zh;q=0.9,zh-TW;q=0.8,en;q=0.7"}
]},
Tesla.Middleware.DecompressResponse
]
]
settings = [
log2file: false,
downloader_options: [requester: {SpiderMan.Requester.Finch, requester_options}],
spider_options: [pipelines: []],
item_processor_options: [
storage: [
{SpiderMan.Storage.ETS, "./data/jobs.ets"},
{SpiderMan.Storage.CSV,
file: "./data/jobs.csv",
headers: [
:posted_at,
:title,
:company,
:location,
:workplace,
:type,
:source_url,
:source_description,
:page_number
]}
]
]
]Prepare callbacks for spider
import SpiderMan
import SpiderMan.Utils
require Logger
spider = SpiderList.ElixirJobs
init = fn state ->
build_request(base_url)
|> set_flag(:first_page)
|> then(&SpiderMan.insert_request(spider, &1))
state
end
handle_list_page = fn body, n ->
Logger.info("processing page #{n}")
{:ok, document} = Floki.parse_document(body)
jobs =
Floki.find(document, ".offers-index")
|> hd()
|> Floki.children(include_text: false)
|> Enum.filter(&match?({"a", _, _}, &1))
items =
Enum.map(jobs, fn job ->
title = Floki.find(job, ".title strong") |> Floki.text() |> String.trim()
sub_title = Floki.find(job, ".title small") |> Floki.text() |> String.trim()
source_url = Floki.attribute(job, "a", "href") |> hd()
[company, location | _] = sub_title |> String.split(" - ")
[_, date, _, workplace, _, type] =
Floki.find(job, ".control .tag")
|> Enum.map(&(&1 |> Floki.text() |> String.trim()))
full_date =
case Timex.parse(date, "{D} {Mshort}") do
{:ok, date} ->
year = Timex.today().year
{:ok, full_date} = Date.new(year, date.month, date.day)
full_date |> Date.to_string()
{:error, "Expected end of input at line 1, column 6"} ->
{:ok, date} = Timex.parse(date, "{D} {Mshort} {YYYY}")
date |> Date.to_string()
{:error, "Expected end of input at line 1, column 5"} ->
{:ok, date} = Timex.parse(date, "{D} {Mshort} {YYYY}")
date |> Date.to_string()
{:error, "Expected `day of month` at line 1, column 1."} ->
Logger.info("date: #{date}")
Date.utc_today() |> Date.to_string()
end
source_description = ""
build_item(
source_url,
%{
posted_at: full_date,
title: title,
company: company,
location: location,
workplace: workplace,
type: type,
source_url: base_url <> String.slice(source_url, 1..-1),
source_description: source_description,
page_number: n
}
)
end)
%{items: items}
end
handle_response = fn
%{env: env, flag: :first_page}, _context ->
# total_page =
# Regex.run(~r/Showing page 1 of (\d+)/, env.body, capture: :all_but_first)
# |> hd()
# |> String.to_integer()
total_page = 25
Logger.info("total: #{total_page}")
requests =
Enum.map(2..total_page, fn n ->
build_request("/?page=#{n}")
|> set_flag({:list_page, n})
end)
handle_list_page.(env.body, 1)
|> Map.put(:requests, requests)
%{env: env, flag: {:list_page, n}}, _context ->
handle_list_page.(env.body, n)
end
callbacks = [init: init, handle_response: handle_response]
{:ok, settings} = SpiderMan.CommonSpider.check_callbacks_and_merge_settings(callbacks, settings)Run the spider
# Delete previous dumps
File.rm_rf("./data/jobs.csv")
File.rm_rf("./data/jobs.ets")
SpiderMan.run_until_zero(spider, settings, 5_000)Sort the csv by date descending
alias NimbleCSV.RFC4180, as: CSV
headers = [
:posted_at,
:title,
:company,
:location,
:workplace,
:type,
:source_url,
:source_description,
:page_number
]
header_row = CSV.dump_to_iodata([headers])
sorted_path = "./data/jobs-sorted.csv"
File.rm_rf(sorted_path)
io_device = File.open!(sorted_path, [:write, :append, :binary, :utf8])
csv =
"./data/jobs.csv"
|> File.read!()
|> CSV.parse_string()
|> Enum.sort_by(& &1, :desc)
|> CSV.dump_to_iodata()
:ok = IO.write(io_device, header_row)
:ok = IO.write(io_device, csv)
:ok = File.close(io_device)data =
"./data/jobs-sorted.csv"
|> File.read!()
|> CSV.parse_string()
data
|> Enum.with_index()
|> Enum.map(fn {row, index} ->
%{
id: index,
posted_at: Enum.at(row, 0),
title: Enum.at(row, 1),
company: Enum.at(row, 2),
location: Enum.at(row, 3),
workplace: Enum.at(row, 4),
type: Enum.at(row, 5),
source_url: Enum.at(row, 6),
source_description: Enum.at(row, 7),
page_number: Enum.at(row, 8)
}
end)
|> Kino.DataTable.new()