add allowAnyHttpsCertificate flag on Request

.github/workflows/build.yml

@@ -17,6 +17,7 @@ jobs:
 
     steps:
     - uses: actions/checkout@v2
+    - uses: actions/setup-python@v4
     - uses: jiro4989/setup-nim-action@v1
       with:
         nim-version: ${{ matrix.nim-version }}

src/puppy/common.nim

@@ -15,6 +15,8 @@ type
     timeout*: float32
     verb*: string
     body*: string
+    when defined(puppyLibcurl) or (defined(windows) or not defined(macosx)):
+      allowAnyHttpsCertificate*: bool
 
   Response* = ref object
     headers*: seq[Header]

src/puppy/platforms/linux/platform.nim

@@ -16,8 +16,6 @@ proc fetch*(req: Request): Response {.raises: [PuppyError].} =
     count: int,
     outstream: pointer
   ): int {.cdecl.} =
-    if size != 1:
-      raise newException(PuppyError, "Unexpected curl write callback size")
     let
       outbuf = cast[ptr StringWrap](outstream)
       i = outbuf.str.len
@@ -74,6 +72,10 @@ proc fetch*(req: Request): Response {.raises: [PuppyError].} =
   # Follow redirects by default.
   discard curl.easy_setopt(OPT_FOLLOWLOCATION, 1)
 
+  if req.allowAnyHttpsCertificate:
+    discard curl.easy_setopt(OPT_SSL_VERIFYPEER, 0)
+    discard curl.easy_setopt(OPT_SSL_VERIFYHOST, 0)
+
   let
     ret = curl.easy_perform()
     headerData = headerWrap.str

src/puppy/platforms/win32/platform.nim

@@ -114,9 +114,43 @@ proc fetch*(req: Request): Response {.raises: [PuppyError].} =
       req.body.len.DWORD,
       0
     ) == 0:
-      raise newException(
-        PuppyError, "WinHttpSendRequest error: " & $GetLastError()
-      )
+      let error = GetLastError()
+      if error in {ERROR_WINHTTP_SECURE_FAILURE, ERROR_INTERNET_INVALID_CA} and
+        req.allowAnyHttpsCertificate:
+        # If this is a certificate error but we should allow any HTTPS cert,
+        # we need to set some options and retry sending the request.
+        # https://stackoverflow.com/questions/19338395/how-do-you-use-winhttp-to-do-ssl-with-a-self-signed-cert
+        var flags: DWORD =
+          SECURITY_FLAG_IGNORE_UNKNOWN_CA or
+          SECURITY_FLAG_IGNORE_CERT_WRONG_USAGE or
+          SECURITY_FLAG_IGNORE_CERT_CN_INVALID or
+          SECURITY_FLAG_IGNORE_CERT_DATE_INVALID
+        if WinHttpSetOption(
+          hRequest,
+          WINHTTP_OPTION_SECURITY_FLAGS,
+          flags.addr,
+          sizeof(flags).DWORD
+        ) == 0:
+          raise newException(
+            PuppyError, "WinHttpSetOption error: " & $GetLastError()
+          )
+
+        if WinHttpSendRequest(
+          hRequest,
+          nil,
+          0,
+          req.body.cstring,
+          req.body.len.DWORD,
+          req.body.len.DWORD,
+          0
+        ) == 0:
+          raise newException(
+            PuppyError, "WinHttpSendRequest error: " & $GetLastError()
+          )
+      else:
+        raise newException(
+          PuppyError, "WinHttpSendRequest error: " & $GetLastError()
+        )
 
     if WinHttpReceiveResponse(hRequest, nil) == 0:
       raise newException(
@@ -209,7 +243,7 @@ proc fetch*(req: Request): Response {.raises: [PuppyError].} =
         break
 
       if i == result.body.len:
-        result.body.setLen(min(i * 2, i + 100 * 1024 * 1024))
+        result.body.setLen(i * 2)
 
     result.body.setLen(i)
 

src/puppy/platforms/win32/windefs.nim

@@ -33,6 +33,14 @@ const
   WINHTTP_QUERY_FLAG_NUMBER* = 0x20000000
   WINHTTP_QUERY_RAW_HEADERS_CRLF* = 22
   ERROR_INSUFFICIENT_BUFFER* = 122
+  ERROR_WINHTTP_SECURE_FAILURE* = 12175
+  ERROR_INTERNET_INVALID_CA* = 12045
+  WINHTTP_OPTION_SECURITY_FLAGS* = 31
+  SECURITY_FLAG_IGNORE_UNKNOWN_CA* = 0x00000100
+  # SECURITY_FLAG_IGNORE_WRONG_USAGE* = 0x00000200
+  SECURITY_FLAG_IGNORE_CERT_WRONG_USAGE* = 0x00000200
+  SECURITY_FLAG_IGNORE_CERT_CN_INVALID* = 0x00001000
+  SECURITY_FLAG_IGNORE_CERT_DATE_INVALID* = 0x00002000
 
 {.push importc, stdcall.}
 
@@ -88,6 +96,13 @@ proc WinHttpOpenRequest*(
   dwFlags: DWORD
 ): HINTERNET {.dynlib: "winhttp".}
 
+proc WinHttpSetOption* (
+  hInternet: HINTERNET,
+  dwOption: DWORD,
+  lpBuffer: LPVOID,
+  dwBufferLength: DWORD
+): BOOL {.dynlib: "winhttp".}
+
 proc WinHttpAddRequestHeaders*(
   hRequest: HINTERNET,
   lpszHeaders: LPCWSTR,

tests/data/ssl.crt

@@ -0,0 +1,21 @@
+-----BEGIN CERTIFICATE-----
+MIIDazCCAlOgAwIBAgIUL6WPqwJb9MBbrqL3v1gIX42hySMwDQYJKoZIhvcNAQEL
+BQAwRTELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM
+GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDAeFw0yMjAyMjMwNzI1MjFaFw0yMzAy
+MjMwNzI1MjFaMEUxCzAJBgNVBAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEw
+HwYDVQQKDBhJbnRlcm5ldCBXaWRnaXRzIFB0eSBMdGQwggEiMA0GCSqGSIb3DQEB
+AQUAA4IBDwAwggEKAoIBAQCqA9pvOjC8Jg7wiX3aiXedg7LKS/V3coINi/Xzh7zY
+1YERUyda/nRpFjmONtJkRipxA59ZZfNfMLoOtl5BhcE4LlLC81m48KQt7/eR3SMx
+tkqFByOA5WltulhXHKghyJhyjy0UDIrBHy5Ic7X8dkR1n9spqpfWyM6VMyDOX9Fl
+8N5H03ioicvjZ4n6Mrc2sCMl39G5GFyUgBX6UAAsPg4xrfPL5Yz+TGSeCV7UtFDc
+nfpR4l5WQERqU4IJCNQj5MxweNZNa00sTOP5s7zuzFiGVQqkCz7FWzpZVUbMggBo
+0UfhCNlnFGPMzddG/IsN0ojCRFzBufFa3jC5AzuhwZk/AgMBAAGjUzBRMB0GA1Ud
+DgQWBBSodOQnZd0tc3IMtnEDIoPzw8tMWzAfBgNVHSMEGDAWgBSodOQnZd0tc3IM
+tnEDIoPzw8tMWzAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQAm
+9IJBZ0xiQUSsUyBe0bZ+1iBkzX8ci1df1rqGooVIIkF3L5HYq8MME0oi5YdQrpqJ
+I4OArRBwz06SUqMP16KTibOgnc8EYM0ZdhqkdK5dZQnyCBpExtlz73U0ELnMvdGE
+a+1ZsnPKXgYXaEfbWWs5EN6avbAkURuzzt4eg7t4XCxm8T/MJqL7D3xoT7ion1v0
+iXsdbBVoC5A0nvPWdraOvB7PI4Z951bywuj/7VRIPlMYUFmrYbOGo5oJudZHwrl8
+N3vDHkgr5GFeB5cyqlPjUNrfjxTQP6Mim3AtaHC30NhhABmxQVjj53Hn+oDuNeF6
+GSj2EtzNm76AU377GrBi
+-----END CERTIFICATE-----

tests/data/ssl.key

@@ -0,0 +1,28 @@
+-----BEGIN PRIVATE KEY-----
+MIIEwAIBADANBgkqhkiG9w0BAQEFAASCBKowggSmAgEAAoIBAQCqA9pvOjC8Jg7w
+iX3aiXedg7LKS/V3coINi/Xzh7zY1YERUyda/nRpFjmONtJkRipxA59ZZfNfMLoO
+tl5BhcE4LlLC81m48KQt7/eR3SMxtkqFByOA5WltulhXHKghyJhyjy0UDIrBHy5I
+c7X8dkR1n9spqpfWyM6VMyDOX9Fl8N5H03ioicvjZ4n6Mrc2sCMl39G5GFyUgBX6
+UAAsPg4xrfPL5Yz+TGSeCV7UtFDcnfpR4l5WQERqU4IJCNQj5MxweNZNa00sTOP5
+s7zuzFiGVQqkCz7FWzpZVUbMggBo0UfhCNlnFGPMzddG/IsN0ojCRFzBufFa3jC5
+AzuhwZk/AgMBAAECggEBAKZMLpUtbg4bi/FsC1Z/sCi6cV++/NNhhiSKCoGy+918
+uUqg85Xl3ygLPTEGHrVGjK0OxgdD2dH6b4OEjp24n068wOc/8Tsc5vqoBpj+nTY7
+AJkuamPiAkX6R/6tYSfqdnNX6Nf7jJ1qSnND+3Z+mGgVfOI8o1jMAoWeBTDYOJJk
+eFc0e2O+vBburjj//daVizq47tEIoVRRd+nmsnaMXFuvZh283W6ds1D63HLLZatS
+Jwbycxj0PHQOSEUmeRbmcO6XJtfPuD6mXumtMUgb0OMQtnIj3tY3sWDEAM4cWrP2
+9HZKNFDkoD+Xb7HXcFY9MfvaMzdB/MP+LKo947fiGAECgYEA1Dy+S0bPSrA9X2hh
+vbXBSZvqQsx2xSti9Ern9J/c9ExDe5jsRLTwONBayQ4ctIeuEKBKtr6dY96cKLEY
+QuB40Qzej3ht0nGhx8S57U6rLugIbwVVrvRfPZ7PDWDTGvK4a3s3hZgvxmzTrYIA
+6ZzsfH01pWSgFJ9/66wci56K8b8CgYEAzRJY/iEvVujwMR7HRF0bDwTYzIMsPJgp
+grKz31fcgUesVttUCMKG0Ge+L515xPFjPG9TgFCFwDaQSomwGm1OwAqYTmA5w+WU
+iFncBgs7KhHx5rfwX9JtiEaouNI3R4po00WGA+zz0UYRjVCJHrf+MyV537VSI4FG
+XrP6uiYkOIECgYEAs31/ref/rXmpHcQITUmmYttCXiXPGGbd9B5ZVu/QDKdmtuOY
+hW7Ebjf/X2PY8PCCTDtTlINWVjzQsjU7gGuYoauRmaJOtpg1Kt58I27RpQTFBSds
+1F6FIXbqQrUtM/Ar+XImfYw8c0JcLrPwk6GL+qhlsy+LloVhyO0w4v89IL8CgYEA
+gkx7IRWix50AKKW+xRBHhhZ1ThS2gdXI4lN7eJiR8c7BkPqQ/XPkRvzz2bs8SMd7
+X0X5D1maclP5AHNV4qS7WcghmAMKEQ+Jfc1iwLBYKlX2lrsezzOcBu+merCPETS/
+gCX3jfz7umfD9T9LsKoFqSfRtTO3efnE5Z2D3M0pTIECgYEAwxe+eDuqKVNEorJn
+/3mN5SzC0kXMOiT3UEgMcPAJaizzNjD7d0oIz7afzgJJggGsnerFRFe3Bmw0GJJ5
+lVyanZW3kumPw1B86kakkiaeEAy3uAD6QPcU6vlSlwREkxAujr/K2sCaNtpYwWly
+SCT+0J/TkI28lbnzl1eSG2aQUbA=
+-----END PRIVATE KEY-----

tests/https_server.py

@@ -0,0 +1,46 @@
+from http.server import HTTPServer, BaseHTTPRequestHandler
+import ssl
+
+host = ("0.0.0.0", 443)
+
+class ExecuteServer(BaseHTTPRequestHandler):
+    def do_GET(self):
+        if self.path == "/connect":
+            try:
+                self.send_response(200)
+                self.send_header("Content-type", "text/html")
+                self.end_headers()
+                self.wfile.write(bytes("test", "utf8"))
+            except:
+                pass
+
+    def do_POST(self):
+        if self.path == "/plain":
+            content_len = int(self.headers.get("Content-Length"))
+            post_body = self.rfile.read(content_len)
+            self.send_response(200)
+            self.send_header("Content-type", "text/html")
+            self.end_headers()
+
+            post_body = str(post_body.decode())
+            print(post_body)
+
+    def log_message(self, format, *args):
+        pass
+
+def start_server():
+    server = HTTPServer(host, ExecuteServer)
+    server.socket = ssl.wrap_socket(server.socket,
+                                     server_side=True,
+                                     certfile="tests/data/ssl.crt",
+                                     keyfile="tests/data/ssl.key",
+                                     ssl_version=ssl.PROTOCOL_TLS)
+
+    print(f"Starting server, listening on port {host[1]}")
+    server.serve_forever()
+
+def main():
+    start_server()
+
+if __name__ == "__main__":
+    main()

tests/test.nim

@@ -49,6 +49,34 @@ block:
   doAssert res.headers.len > 0
   doAssert res.body != ""
 
+when defined(windows):
+  block:
+    let httpsServer = startProcess(
+      "python tests/https_server.py",
+      options = {poEvalCommand, poParentStreams}
+    )
+
+    # Wait for server to start, Python is mega slow
+    sleep(4000)
+
+    try:
+      echo "# allowAnyHttpsCertificate"
+      let res = fetch(
+        Request(
+          url: parseUrl("https://127.0.0.1/connect"),
+          verb: "get",
+          allowAnyHttpsCertificate: true,
+        )
+      )
+      echo "res.code: ", res.code
+      echo "res.headers: ", res.headers
+      echo "res.body.len: ", res.body.len
+      doAssert res.code == 200
+      doAssert res.headers.len > 0
+      doAssert res.body != ""
+    finally:
+      httpsServer.terminate()
+
 # test headers
 
 block: