Merge pull request #13236 from samford/parse_curl_response-handle-dup…

…licate-headers

`parse_curl_response`: Handle duplicate headers

Library/Homebrew/test/utils/curl_spec.rb

@@ -4,6 +4,115 @@
 require "utils/curl"
 
 describe "Utils::Curl" do
+  let(:details) {
+    details = {
+      normal:     {},
+      cloudflare: {},
+      incapsula:  {},
+    }
+
+    details[:normal][:no_cookie] = {
+      url:            "https://www.example.com/",
+      final_url:      nil,
+      status:         "403",
+      headers:        {
+        "age"            => "123456",
+        "cache-control"  => "max-age=604800",
+        "content-type"   => "text/html; charset=UTF-8",
+        "date"           => "Wed, 1 Jan 2020 01:23:45 GMT",
+        "etag"           => "\"3147526947+ident\"",
+        "expires"        => "Wed, 31 Jan 2020 01:23:45 GMT",
+        "last-modified"  => "Wed, 1 Jan 2020 00:00:00 GMT",
+        "server"         => "ECS (dcb/7EA2)",
+        "vary"           => "Accept-Encoding",
+        "x-cache"        => "HIT",
+        "content-length" => "3",
+      },
+      etag:           "3147526947+ident",
+      content_length: "3",
+      file:           "...",
+      file_hash:      nil,
+    }
+
+    details[:normal][:ok] = Marshal.load(Marshal.dump(details[:normal][:no_cookie]))
+    details[:normal][:ok][:status] = "200"
+
+    details[:normal][:single_cookie] = Marshal.load(Marshal.dump(details[:normal][:no_cookie]))
+    details[:normal][:single_cookie][:headers]["set-cookie"] = "a_cookie=for_testing"
+
+    details[:normal][:multiple_cookies] = Marshal.load(Marshal.dump(details[:normal][:no_cookie]))
+    details[:normal][:multiple_cookies][:headers]["set-cookie"] = [
+      "first_cookie=for_testing",
+      "last_cookie=also_for_testing",
+    ]
+
+    details[:normal][:blank_headers] = Marshal.load(Marshal.dump(details[:normal][:no_cookie]))
+    details[:normal][:blank_headers][:headers] = {}
+
+    details[:cloudflare][:single_cookie] = {
+      url:            "https://www.example.com/",
+      final_url:      nil,
+      status:         "403",
+      headers:        {
+        "date"            => "Wed, 1 Jan 2020 01:23:45 GMT",
+        "content-type"    => "text/plain; charset=UTF-8",
+        "content-length"  => "16",
+        "x-frame-options" => "SAMEORIGIN",
+        "referrer-policy" => "same-origin",
+        "cache-control"   => "private, max-age=0, no-store, no-cache, must-revalidate, post-check=0, pre-check=0",
+        "expires"         => "Thu, 01 Jan 1970 00:00:01 GMT",
+        "expect-ct"       => "max-age=604800, report-uri=\"https://report-uri.cloudflare.com/cdn-cgi/beacon/expect-ct\"",
+        "set-cookie"      => "__cf_bm=0123456789abcdef; path=/; expires=Wed, 31-Jan-20 01:23:45 GMT;" \
+                             " domain=www.example.com; HttpOnly; Secure; SameSite=None",
+        "server"          => "cloudflare",
+        "cf-ray"          => "0123456789abcdef-IAD",
+        "alt-svc"         => "h3=\":443\"; ma=86400, h3-29=\":443\"; ma=86400",
+      },
+      etag:           nil,
+      content_length: "16",
+      file:           "error code: 1020",
+      file_hash:      nil,
+    }
+
+    details[:cloudflare][:multiple_cookies] = Marshal.load(Marshal.dump(details[:cloudflare][:single_cookie]))
+    details[:cloudflare][:multiple_cookies][:headers]["set-cookie"] = [
+      "first_cookie=for_testing",
+      "__cf_bm=abcdef0123456789; path=/; expires=Thu, 28-Apr-22 18:38:40 GMT; domain=www.example.com; HttpOnly;" \
+      " Secure; SameSite=None",
+      "last_cookie=also_for_testing",
+    ]
+
+    details[:cloudflare][:no_server] = Marshal.load(Marshal.dump(details[:cloudflare][:single_cookie]))
+    details[:cloudflare][:no_server][:headers].delete("server")
+
+    details[:cloudflare][:wrong_server] = Marshal.load(Marshal.dump(details[:cloudflare][:single_cookie]))
+    details[:cloudflare][:wrong_server][:headers]["server"] = "nginx 1.2.3"
+
+    # TODO: Make the Incapsula test data more realistic once we can find an
+    # example website to reference.
+    details[:incapsula][:single_cookie_visid_incap] = Marshal.load(Marshal.dump(details[:normal][:no_cookie]))
+    details[:incapsula][:single_cookie_visid_incap][:headers]["set-cookie"] = "visid_incap_something=something"
+
+    details[:incapsula][:single_cookie_incap_ses] = Marshal.load(Marshal.dump(details[:normal][:no_cookie]))
+    details[:incapsula][:single_cookie_incap_ses][:headers]["set-cookie"] = "incap_ses_something=something"
+
+    details[:incapsula][:multiple_cookies_visid_incap] = Marshal.load(Marshal.dump(details[:normal][:no_cookie]))
+    details[:incapsula][:multiple_cookies_visid_incap][:headers]["set-cookie"] = [
+      "first_cookie=for_testing",
+      "visid_incap_something=something",
+      "last_cookie=also_for_testing",
+    ]
+
+    details[:incapsula][:multiple_cookies_incap_ses] = Marshal.load(Marshal.dump(details[:normal][:no_cookie]))
+    details[:incapsula][:multiple_cookies_incap_ses][:headers]["set-cookie"] = [
+      "first_cookie=for_testing",
+      "incap_ses_something=something",
+      "last_cookie=also_for_testing",
+    ]
+
+    details
+  }
+
   let(:location_urls) {
     %w[
       https://example.com/example/
@@ -112,6 +221,24 @@
       },
     }
 
+    response_hash[:duplicate_header] = {
+      status_code: "200",
+      status_text: "OK",
+      headers:     {
+        "cache-control"  => "max-age=604800",
+        "content-type"   => "text/html; charset=UTF-8",
+        "date"           => "Wed, 1 Jan 2020 01:23:45 GMT",
+        "expires"        => "Wed, 31 Jan 2020 01:23:45 GMT",
+        "last-modified"  => "Thu, 1 Jan 2019 01:23:45 GMT",
+        "content-length" => "123",
+        "set-cookie"     => [
+          "example1=first",
+          "example2=second; Expires Wed, 31 Jan 2020 01:23:45 GMT",
+          "example3=third",
+        ],
+      },
+    }
+
     response_hash
   }
 
@@ -144,6 +271,13 @@
       #{response_text[:ok]}
     EOS
 
+    response_text[:duplicate_header] = response_text[:ok].sub(
+      /\r\n\Z/,
+      "Set-Cookie: #{response_hash[:duplicate_header][:headers]["set-cookie"][0]}\r\n" \
+      "Set-Cookie: #{response_hash[:duplicate_header][:headers]["set-cookie"][1]}\r\n" \
+      "Set-Cookie: #{response_hash[:duplicate_header][:headers]["set-cookie"][2]}\r\n\r\n",
+    )
+
     response_text
   }
 
@@ -269,6 +403,46 @@
     end
   end
 
+  describe "url_protected_by_cloudflare?" do
+    it "returns `true` when a URL is protected by Cloudflare" do
+      expect(url_protected_by_cloudflare?(details[:cloudflare][:single_cookie])).to be(true)
+      expect(url_protected_by_cloudflare?(details[:cloudflare][:multiple_cookies])).to be(true)
+    end
+
+    it "returns `false` when a URL is not protected by Cloudflare" do
+      expect(url_protected_by_cloudflare?(details[:cloudflare][:no_server])).to be(false)
+      expect(url_protected_by_cloudflare?(details[:cloudflare][:wrong_server])).to be(false)
+      expect(url_protected_by_cloudflare?(details[:normal][:no_cookie])).to be(false)
+      expect(url_protected_by_cloudflare?(details[:normal][:ok])).to be(false)
+      expect(url_protected_by_cloudflare?(details[:normal][:single_cookie])).to be(false)
+      expect(url_protected_by_cloudflare?(details[:normal][:multiple_cookies])).to be(false)
+    end
+
+    it "returns `false` when response headers are blank" do
+      expect(url_protected_by_cloudflare?(details[:normal][:blank_headers])).to be(false)
+    end
+  end
+
+  describe "url_protected_by_incapsula?" do
+    it "returns `true` when a URL is protected by Cloudflare" do
+      expect(url_protected_by_incapsula?(details[:incapsula][:single_cookie_visid_incap])).to be(true)
+      expect(url_protected_by_incapsula?(details[:incapsula][:single_cookie_incap_ses])).to be(true)
+      expect(url_protected_by_incapsula?(details[:incapsula][:multiple_cookies_visid_incap])).to be(true)
+      expect(url_protected_by_incapsula?(details[:incapsula][:multiple_cookies_incap_ses])).to be(true)
+    end
+
+    it "returns `false` when a URL is not protected by Incapsula" do
+      expect(url_protected_by_incapsula?(details[:normal][:no_cookie])).to be(false)
+      expect(url_protected_by_incapsula?(details[:normal][:ok])).to be(false)
+      expect(url_protected_by_incapsula?(details[:normal][:single_cookie])).to be(false)
+      expect(url_protected_by_incapsula?(details[:normal][:multiple_cookies])).to be(false)
+    end
+
+    it "returns `false` when response headers are blank" do
+      expect(url_protected_by_incapsula?(details[:normal][:blank_headers])).to be(false)
+    end
+  end
+
   describe "#parse_curl_output" do
     it "returns a correct hash when curl output contains response(s) and body" do
       expect(parse_curl_output("#{response_text[:ok]}#{body[:default]}"))
@@ -312,6 +486,7 @@
     it "returns a correct hash when given HTTP response text" do
       expect(parse_curl_response(response_text[:ok])).to eq(response_hash[:ok])
       expect(parse_curl_response(response_text[:redirection])).to eq(response_hash[:redirection])
+      expect(parse_curl_response(response_text[:duplicate_header])).to eq(response_hash[:duplicate_header])
     end
 
     it "returns an empty hash when given an empty string" do

Library/Homebrew/utils/curl.rb

@@ -198,20 +198,40 @@ def curl_output(*args, **options)
     end
 
     # Check if a URL is protected by CloudFlare (e.g. badlion.net and jaxx.io).
+    # @param details [Hash] Response information from
+    #  `#curl_http_content_headers_and_checksum`.
+    # @return [true, false] Whether a response contains headers indicating that
+    #   the URL is protected by Cloudflare.
+    sig { params(details: T::Hash[Symbol, T.untyped]).returns(T::Boolean) }
     def url_protected_by_cloudflare?(details)
       return false if details[:headers].blank?
-      return unless [403, 503].include?(details[:status].to_i)
+      return false unless [403, 503].include?(details[:status].to_i)
 
-      details[:headers].fetch("set-cookie", nil)&.match?(/^(__cfduid|__cf_bm)=/i) &&
-        details[:headers].fetch("server", nil)&.match?(/^cloudflare/i)
+      set_cookie_header = Array(details[:headers]["set-cookie"])
+      has_cloudflare_cookie_header = set_cookie_header.compact.any? do |cookie|
+        cookie.match?(/^(__cfduid|__cf_bm)=/i)
+      end
+
+      server_header = Array(details[:headers]["server"])
+      has_cloudflare_server = server_header.compact.any? do |server|
+        server.match?(/^cloudflare/i)
+      end
+
+      has_cloudflare_cookie_header && has_cloudflare_server
     end
 
     # Check if a URL is protected by Incapsula (e.g. corsair.com).
+    # @param details [Hash] Response information from
+    #  `#curl_http_content_headers_and_checksum`.
+    # @return [true, false] Whether a response contains headers indicating that
+    #   the URL is protected by Incapsula.
+    sig { params(details: T::Hash[Symbol, T.untyped]).returns(T::Boolean) }
     def url_protected_by_incapsula?(details)
       return false if details[:headers].blank?
       return false if details[:status].to_i != 403
 
-      details[:headers].fetch("set-cookie", nil)&.match?(/^(visid_incap|incap_ses)_/i)
+      set_cookie_header = Array(details[:headers]["set-cookie"])
+      set_cookie_header.compact.any? { |cookie| cookie.match?(/^(visid_incap|incap_ses)_/i) }
     end
 
     def curl_check_http_content(url, url_type, specs: {}, user_agents: [:default],
@@ -484,10 +504,25 @@ def parse_curl_response(response_text)
       response_text = response_text.sub(%r{^HTTP/.* (\d+).*$\s*}, "")
 
       # Create a hash from the header lines
-      response[:headers] =
-        response_text.split("\r\n")
-                     .to_h { |header| header.split(/:\s*/, 2) }
-                     .transform_keys(&:downcase)
+      response[:headers] = {}
+      response_text.split("\r\n").each do |line|
+        header_name, header_value = line.split(/:\s*/, 2)
+        next if header_name.blank?
+
+        header_name = header_name.strip.downcase
+        header_value&.strip!
+
+        case response[:headers][header_name]
+        when nil
+          response[:headers][header_name] = header_value
+        when String
+          response[:headers][header_name] = [response[:headers][header_name], header_value]
+        when Array
+          response[:headers][header_name].push(header_value)
+        end
+
+        response[:headers][header_name]
+      end
 
       response
     end