-
Notifications
You must be signed in to change notification settings - Fork 0
/
pages.rb
54 lines (47 loc) · 1.27 KB
/
pages.rb
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
class Pages
require 'rest-client'
require 'nokogiri'
require 'open-uri'
require 'json'
require 'fileutils'
require 'net/http'
require_relative 'logging'
include Logging
def initialize(args)
@title = args[:title]
@ids_file = args[:ids_file]
@sleep = 3
end
def run
destination = "pages/#{@title}"
FileUtils.mkdir_p(destination)
save_all destination, @ids_file
end
private
def save_to(file_path, id)
logger.info "going to create #{file_path}".green
return false if File.file?(file_path)
FileUtils.touch(file_path)
store = File.open(file_path, 'w')
url = 'https://www.glassdoor.de/job-listing/software-engineer-codeship-frontend-cloudbees-JV_IC2622109_KO2.htm?jl=' + id
logger.info "trying #{url}"
store.puts Net::HTTP.get(URI.parse(url))
store.close
logger.info "#{url} done!".green
end
def save_all(destination, ids_file)
list = open(ids_file).gets.split(',')
listing_ids = []
counter = 0
begin
list.each do |id|
logger.info "#{counter} of #{list.size}"
file_path = "./#{destination}/#{id}"
sleep @sleep if save_to file_path.strip, id.strip
counter += 1
end
rescue StandardError => e
logger.warn "can not save: #{e}"
end
end
end