python: Support http/1.1 (#24)

* python: Support http/1.1

* http.py: Handle unsupported case

Author: corona10

.github/workflows/python.yml

@@ -18,7 +18,11 @@ jobs:
         uses: actions/setup-python@v2
         with:
           python-version: 3.9
+      - name: Install requirements
+        run: pip install -r requirements.txt
       - name: Install Black
-        run: pip install black
+        run: pip install black pytest
       - name: Run black --check .
         run: black --check .
+      - name: Test with pytest
+        run: pytest -v

python/http.py

@@ -1,6 +1,7 @@
 import gzip
 import socket
 import ssl
+import sys
 import zlib
 
 try:
@@ -40,11 +41,17 @@ def request(url):
 
 def _get_headers_and_body(sock, host, port, path):
     sock.connect((host, port))
+    accept_encoding = ["gzip", "deflate"]
+    if brotli:
+        accept_encoding.append("br")
+    accept_encoding = ",".join(accept_encoding)
 
     # 5. Send request
-    sock.send(f"GET {path} HTTP/1.0\r\n".encode())
+    sock.send(f"GET {path} HTTP/1.1\r\n".encode())
     sock.send(f"Host: {host}\r\n".encode())
-    sock.send("Accept-Encoding: br,gzip,deflate\r\n".encode())
+    sock.send(f"Connection: close\r\n".encode())
+    sock.send(f"User-Agent: Mozilla/5.0 ({sys.platform})\r\n".encode())
+    sock.send(f"Accept-Encoding: {accept_encoding}\r\n".encode())
     sock.send("\r\n".encode())
 
     # 6. Receive response
@@ -63,11 +70,18 @@ def _get_headers_and_body(sock, host, port, path):
             line = response.readline().decode()
             if line == "\r\n":
                 break
-
             header, value = line.split(":", 1)
             headers[header.lower()] = value.strip()
 
-        body = response.read()
+        if "transfer-encoding" in headers:
+            encoding = headers["transfer-encoding"]
+            if encoding == "chunked":
+                body = unchunked(response)
+            else:
+                raise RuntimeError(f"Unsupported transfer-encoding: {encoding}")
+        else:
+            body = response.read()
+
         if "content-encoding" in headers:
             encoding = headers["content-encoding"]
             body = decompress(body, encoding)
@@ -77,6 +91,23 @@ def _get_headers_and_body(sock, host, port, path):
         return headers, body
 
 
+def unchunked(response):
+    ret = b""
+
+    def get_chunk_size():
+        chunk_size = response.readline().rstrip()
+        return int(chunk_size, 16)
+
+    while True:
+        chunk_size = get_chunk_size()
+        if chunk_size == 0:
+            break
+        else:
+            ret += response.read(chunk_size)
+            response.read(2)
+    return ret
+
+
 def decompress(data, encoding):
     if encoding == "gzip":
         return gzip.decompress(data)

python/tests/__init__.py

@@ -0,0 +1,3 @@
+import sys
+
+sys.path.append("../")

python/tests/test_http.py

@@ -0,0 +1,32 @@
+import unittest
+
+from http import request
+
+
+class RequestTest(unittest.TestCase):
+    def test_http_request(self):
+        http_sites = ["http://www.google.com/", "http://example.com/"]
+        for site in http_sites:
+            headers, body = request(site)
+            self.assertGreater(len(body), 0)
+            self.assertIn("content-type", headers)
+
+    def test_https_request(self):
+        https_sites = [
+            "https://www.google.com/",
+            "https://www.facebook.com/",
+            "https://example.com/",
+        ]
+        for site in https_sites:
+            headers, body = request(site)
+            self.assertGreater(len(body), 0)
+            self.assertIn("content-type", headers)
+
+    def test_data_request(self):
+        headers, body = request("data:text/html,Hello world")
+        self.assertEqual(body, "Hello world")
+        self.assertEqual(headers["content-type"], "text/html")
+
+
+if __name__ == "__main__":
+    unittest.main()