/
http_client.rb
85 lines (66 loc) · 1.83 KB
/
http_client.rb
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
# frozen_string_literal: true
require "down/http"
require "http"
require "uri"
module Miteru
class HTTPClient
DEFAULT_UA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36"
URLSCAN_UA = "miteru/#{Miteru::VERSION}"
attr_reader :ssl_context
def initialize
ctx = OpenSSL::SSL::SSLContext.new
ctx.verify_mode = OpenSSL::SSL::VERIFY_NONE
@ssl_context = ctx
end
def download(url, destination)
down = Down::Http.new(**default_options) { |client| client.headers(**default_headers) }
down.download(url, destination: destination)
destination
end
def head(url, options = {})
options = options.merge default_options
HTTP.follow
.timeout(3)
.headers(urlscan_url?(url) ? urlscan_headers : default_headers)
.head(url, options)
end
def get(url, options = {})
options = options.merge default_options
HTTP.follow
.timeout(write: 2, connect: 5, read: 10)
.headers(urlscan_url?(url) ? urlscan_headers : default_headers)
.get(url, options)
end
def post(url, options = {})
HTTP.post url, options
end
class << self
def download(url, base_dir = "/tmp")
new.download(url, base_dir)
end
def get(url, options = {})
new.get url, options
end
def post(url, options = {})
new.post url, options
end
def head(url, options = {})
new.head url, options
end
end
private
def default_headers
{ user_agent: DEFAULT_UA }
end
def default_options
{ ssl_context: ssl_context }
end
def urlscan_headers
{ user_agent: URLSCAN_UA }
end
def urlscan_url?(url)
uri = URI(url)
uri.hostname == "urlscan.io"
end
end
end