-
-
Notifications
You must be signed in to change notification settings - Fork 9.4k
/
curl.rb
227 lines (179 loc) 路 6.9 KB
/
curl.rb
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
# frozen_string_literal: true
require "open3"
def curl_executable
@curl ||= [
ENV["HOMEBREW_CURL"],
which("curl"),
"/usr/bin/curl",
].compact.map { |c| Pathname(c) }.find(&:executable?)
raise "no executable curl was found" unless @curl
@curl
end
def curl_args(*extra_args, show_output: false, user_agent: :default)
args = []
# do not load .curlrc unless requested (must be the first argument)
args << "--disable" unless Homebrew::EnvConfig.curlrc?
args << "--globoff"
args << "--show-error"
args << "--user-agent" << case user_agent
when :browser, :fake
HOMEBREW_USER_AGENT_FAKE_SAFARI
when :default
HOMEBREW_USER_AGENT_CURL
else
user_agent
end
unless show_output
args << "--fail"
args << "--progress-bar" unless Homebrew.args.verbose?
args << "--verbose" if Homebrew::EnvConfig.curl_verbose?
args << "--silent" unless $stdout.tty?
end
args << "--retry" << Homebrew::EnvConfig.curl_retries
args + extra_args
end
def curl(*args, secrets: [], **options)
# SSL_CERT_FILE can be incorrectly set by users or portable-ruby and screw
# with SSL downloads so unset it here.
system_command! curl_executable,
args: curl_args(*args, **options),
print_stdout: true,
env: { "SSL_CERT_FILE" => nil },
secrets: secrets
end
def curl_download(*args, to: nil, partial: true, **options)
destination = Pathname(to)
destination.dirname.mkpath
if partial
range_stdout = curl_output("--location", "--range", "0-1",
"--dump-header", "-",
"--write-out", "%\{http_code}",
"--output", "/dev/null", *args, **options).stdout
headers, _, http_status = range_stdout.partition("\r\n\r\n")
supports_partial_download = http_status.to_i == 206 # Partial Content
if supports_partial_download &&
destination.exist? &&
destination.size == %r{^.*Content-Range: bytes \d+-\d+/(\d+)\r\n.*$}m.match(headers)&.[](1)&.to_i
return # We've already downloaded all the bytes
end
else
supports_partial_download = false
end
continue_at = if destination.exist? && supports_partial_download
"-"
else
0
end
curl("--location", "--remote-time", "--continue-at", continue_at.to_s, "--output", destination, *args, **options)
rescue ErrorDuringExecution => e
# This is a workaround for https://github.com/curl/curl/issues/1618.
raise unless e.status.exitstatus == 56 # Unexpected EOF
raise if args.include?("--http1.1")
out = curl_output("-V").stdout
# If `curl` doesn't support HTTP2, the exception is unrelated to this bug.
raise unless out.include?("HTTP2")
# The bug is fixed in `curl` >= 7.60.0.
curl_version = out[/curl (\d+(\.\d+)+)/, 1]
raise if Gem::Version.new(curl_version) >= Gem::Version.new("7.60.0")
args << "--http1.1"
retry
end
def curl_output(*args, secrets: [], **options)
system_command curl_executable,
args: curl_args(*args, show_output: true, **options),
print_stderr: false,
secrets: secrets
end
def curl_check_http_content(url, user_agents: [:default], check_content: false, strict: false)
return unless url.start_with? "http"
details = nil
user_agent = nil
hash_needed = url.start_with?("http:")
user_agents.each do |ua|
details = curl_http_content_headers_and_checksum(url, hash_needed: hash_needed, user_agent: ua)
user_agent = ua
break if http_status_ok?(details[:status])
end
unless details[:status]
# Hack around https://github.com/Homebrew/brew/issues/3199
return if MacOS.version == :el_capitan
return "The URL #{url} is not reachable"
end
unless http_status_ok?(details[:status])
return "The URL #{url} is not reachable (HTTP status code #{details[:status]})"
end
if url.start_with?("https://") && Homebrew::EnvConfig.no_insecure_redirect? &&
!details[:final_url].start_with?("https://")
return "The URL #{url} redirects back to HTTP"
end
return unless hash_needed
secure_url = url.sub "http", "https"
secure_details =
curl_http_content_headers_and_checksum(secure_url, hash_needed: true, user_agent: user_agent)
if !http_status_ok?(details[:status]) ||
!http_status_ok?(secure_details[:status])
return
end
etag_match = details[:etag] &&
details[:etag] == secure_details[:etag]
content_length_match =
details[:content_length] &&
details[:content_length] == secure_details[:content_length]
file_match = details[:file_hash] == secure_details[:file_hash]
if (etag_match || content_length_match || file_match) &&
secure_details[:final_url].start_with?("https://") &&
url.start_with?("http://")
return "The URL #{url} should use HTTPS rather than HTTP"
end
return unless check_content
no_protocol_file_contents = %r{https?:\\?/\\?/}
details[:file] = details[:file].gsub(no_protocol_file_contents, "/")
secure_details[:file] = secure_details[:file].gsub(no_protocol_file_contents, "/")
# Check for the same content after removing all protocols
if (details[:file] == secure_details[:file]) &&
secure_details[:final_url].start_with?("https://") &&
url.start_with?("http://")
return "The URL #{url} should use HTTPS rather than HTTP"
end
return unless strict
# Same size, different content after normalization
# (typical causes: Generated ID, Timestamp, Unix time)
if details[:file].length == secure_details[:file].length
return "The URL #{url} may be able to use HTTPS rather than HTTP. Please verify it in a browser."
end
lenratio = (100 * secure_details[:file].length / details[:file].length).to_i
return unless (90..110).cover?(lenratio)
"The URL #{url} may be able to use HTTPS rather than HTTP. Please verify it in a browser."
end
def curl_http_content_headers_and_checksum(url, hash_needed: false, user_agent: :default)
file = Tempfile.new.tap(&:close)
max_time = hash_needed ? "600" : "25"
output, = curl_output(
"--dump-header", "-", "--output", file.path, "--include", "--location",
"--connect-timeout", "15", "--max-time", max_time, url,
user_agent: user_agent
)
status_code = :unknown
while status_code == :unknown || status_code.to_s.start_with?("3")
headers, _, output = output.partition("\r\n\r\n")
status_code = headers[%r{HTTP/.* (\d+)}, 1]
location = headers[/^Location:\s*(.*)$/i, 1]
final_url = location.chomp if location
end
output_hash = Digest::SHA256.file(file.path) if hash_needed
final_url ||= url
{
url: url,
final_url: final_url,
status: status_code,
etag: headers[%r{ETag: ([wW]/)?"(([^"]|\\")*)"}, 2],
content_length: headers[/Content-Length: (\d+)/, 1],
file_hash: output_hash,
file: output,
}
ensure
file.unlink
end
def http_status_ok?(status)
(100..299).cover?(status.to_i)
end