-
-
Notifications
You must be signed in to change notification settings - Fork 9.3k
/
curl.rb
192 lines (152 loc) 路 5.93 KB
/
curl.rb
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
# frozen_string_literal: true
require "open3"
def curl_executable
@curl ||= [
ENV["HOMEBREW_CURL"],
which("curl"),
"/usr/bin/curl",
].compact.map { |c| Pathname(c) }.find(&:executable?)
raise "no executable curl was found" unless @curl
@curl
end
def curl_args(*extra_args, show_output: false, user_agent: :default)
args = []
# do not load .curlrc unless requested (must be the first argument)
args << "-q" unless ENV["HOMEBREW_CURLRC"]
args << "--show-error"
args << "--user-agent" << case user_agent
when :browser, :fake
HOMEBREW_USER_AGENT_FAKE_SAFARI
when :default
HOMEBREW_USER_AGENT_CURL
else
user_agent
end
unless show_output
args << "--fail"
args << "--progress-bar" unless ARGV.verbose?
args << "--verbose" if ENV["HOMEBREW_CURL_VERBOSE"]
args << "--silent" unless $stdout.tty?
end
args << "--retry" << ENV["HOMEBREW_CURL_RETRIES"] if ENV["HOMEBREW_CURL_RETRIES"]
args + extra_args
end
def curl(*args)
# SSL_CERT_FILE can be incorrectly set by users or portable-ruby and screw
# with SSL downloads so unset it here.
system_command! curl_executable,
args: curl_args(*args),
print_stdout: true,
env: { "SSL_CERT_FILE" => nil }
end
def curl_download(*args, to: nil, **options)
destination = Pathname(to)
destination.dirname.mkpath
range_stdout = curl_output("--location", "--range", "0-1",
"--dump-header", "-",
"--write-out", "%{http_code}",
"--output", "/dev/null", *args, **options).stdout
headers, _, http_status = range_stdout.partition("\r\n\r\n")
supports_partial_download = http_status.to_i == 206 # Partial Content
if supports_partial_download &&
destination.exist? &&
destination.size == %r{^.*Content-Range: bytes \d+-\d+/(\d+)\r\n.*$}m.match(headers)&.[](1)&.to_i
return # We've already downloaded all the bytes
end
continue_at = if destination.exist? && supports_partial_download
"-"
else
0
end
curl("--location", "--remote-time", "--continue-at", continue_at.to_s, "--output", destination, *args, **options)
end
def curl_output(*args, **options)
system_command(curl_executable,
args: curl_args(*args, show_output: true, **options),
print_stderr: false)
end
def curl_check_http_content(url, user_agents: [:default], check_content: false, strict: false)
return unless url.start_with? "http"
details = nil
user_agent = nil
hash_needed = url.start_with?("http:")
user_agents.each do |ua|
details = curl_http_content_headers_and_checksum(url, hash_needed: hash_needed, user_agent: ua)
user_agent = ua
break if details[:status].to_s.start_with?("2")
end
unless details[:status]
# Hack around https://github.com/Homebrew/brew/issues/3199
return if MacOS.version == :el_capitan
return "The URL #{url} is not reachable"
end
unless details[:status].start_with? "2"
return "The URL #{url} is not reachable (HTTP status code #{details[:status]})"
end
if url.start_with?("https://") && ENV["HOMEBREW_NO_INSECURE_REDIRECT"] &&
!details[:final_url].start_with?("https://")
return "The URL #{url} redirects back to HTTP"
end
return unless hash_needed
secure_url = url.sub "http", "https"
secure_details =
curl_http_content_headers_and_checksum(secure_url, hash_needed: true, user_agent: user_agent)
if !details[:status].to_s.start_with?("2") ||
!secure_details[:status].to_s.start_with?("2")
return
end
etag_match = details[:etag] &&
details[:etag] == secure_details[:etag]
content_length_match =
details[:content_length] &&
details[:content_length] == secure_details[:content_length]
file_match = details[:file_hash] == secure_details[:file_hash]
if (etag_match || content_length_match || file_match) &&
secure_details[:final_url].start_with?("https://") &&
url.start_with?("http://")
return "The URL #{url} should use HTTPS rather than HTTP"
end
return unless check_content
no_protocol_file_contents = %r{https?:\\?/\\?/}
details[:file] = details[:file].gsub(no_protocol_file_contents, "/")
secure_details[:file] = secure_details[:file].gsub(no_protocol_file_contents, "/")
# Check for the same content after removing all protocols
if (details[:file] == secure_details[:file]) &&
secure_details[:final_url].start_with?("https://") &&
url.start_with?("http://")
return "The URL #{url} should use HTTPS rather than HTTP"
end
return unless strict
# Same size, different content after normalization
# (typical causes: Generated ID, Timestamp, Unix time)
if details[:file].length == secure_details[:file].length
return "The URL #{url} may be able to use HTTPS rather than HTTP. Please verify it in a browser."
end
lenratio = (100 * secure_details[:file].length / details[:file].length).to_i
return unless (90..110).cover?(lenratio)
"The URL #{url} may be able to use HTTPS rather than HTTP. Please verify it in a browser."
end
def curl_http_content_headers_and_checksum(url, hash_needed: false, user_agent: :default)
max_time = hash_needed ? "600" : "25"
output, = curl_output(
"--connect-timeout", "15", "--include", "--max-time", max_time, "--location", url,
user_agent: user_agent
)
status_code = :unknown
while status_code == :unknown || status_code.to_s.start_with?("3")
headers, _, output = output.partition("\r\n\r\n")
status_code = headers[%r{HTTP\/.* (\d+)}, 1]
final_url = headers[/^Location:\s*(.*)$/i, 1]&.chomp
end
output_hash = Digest::SHA256.digest(output) if hash_needed
final_url ||= url
{
url: url,
final_url: final_url,
status: status_code,
etag: headers[%r{ETag: ([wW]\/)?"(([^"]|\\")*)"}, 2],
content_length: headers[/Content-Length: (\d+)/, 1],
file_hash: output_hash,
file: output,
}
end