Skip to content

Commit 033e99f

Browse files
ScutoidzzScutoidzz
committed
first commit
0 parents  commit 033e99f

13 files changed

Lines changed: 58 additions & 0 deletions

‎processor/__init__.py‎

Whitespace-only changes.

‎processor/__pycache__/__init__.cpython-313.pyc‎

136 Bytes
Binary file not shown.

‎processor/__pycache__/pull.cpython-313.pyc‎

1.68 KB
Binary file not shown.

‎processor/grab/__init__.py‎

Whitespace-only changes.

‎processor/grab/__pycache__/__init__.cpython-313.pyc‎

141 Bytes
Binary file not shown.

‎processor/grab/__pycache__/pull.cpython-313.pyc‎

2.6 KB
Binary file not shown.

‎processor/grab/errors.json‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
{
2+
"Authentication required": "401 Unauthorized",
3+
"Name or server not known": "404 Not Found",
4+
"Connection refused": "503 Service Unavailable"
5+
}

‎processor/grab/info.json‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
{
2+
"version": "0.0.1",
3+
"ua": "Mip/0.0.1"
4+
}

‎processor/grab/pull.py‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
import ssl
2+
import socket
3+
import logging
4+
import json
5+
import os
6+
from urllib.parse import urlparse
7+
8+
def pull_raw(url):
9+
parsed = urlparse(url)
10+
https_status = parsed.scheme == "https"
11+
host = parsed.netloc
12+
13+
if https_status:
14+
with open(os.path.join(os.path.dirname(__file__), "info.json"), "r") as f:
15+
info = json.load(f)
16+
context = ssl.create_default_context()
17+
sock = socket.create_connection((host, 443))
18+
19+
sock = context.wrap_socket(sock, server_hostname=host)
20+
21+
sock.sendall(b"GET / HTTP/1.1\r\nHost: " + host.encode() + b"\r\nUser-Agent: " + info["ua"].encode() + b"\r\n\r\n")
22+
23+
response = sock.recv(4096)
24+
25+
if "Name or service not known" in response.decode():
26+
logging.error("Name or service not known")
27+
response = b"<!DOCTYPE html><html><head><title>404</title></head><body><h1>404 Not Found</h1><p>Couldn't find the site you were looking for.</p></body></html>"
28+
return response, https_status
29+
30+
else:
31+
try:
32+
with open(os.path.join(os.path.dirname(__file__), "info.json"), "r") as f:
33+
info = json.load(f)
34+
sock = socket.create_connection((host, 80))
35+
sock.sendall(b"GET / HTTP/1.1\r\nHost: " + host.encode() + b"\r\nUser-Agent: " + info["ua"].encode() + b"\r\n\r\n")
36+
response = sock.recv(4096)
37+
except Exception as e:
38+
logging.error(e)
39+
response = b"<!DOCTYPE html><html><head><title>500</title></head><body><h1>500 Internal Server Error</h1><p>Couldn't find the site you were looking for.</p></body></html>"
40+
return response, False
41+
if "Name or service not known" in response.decode():
42+
logging.error("Name or service not known")
43+
response = b"<!DOCTYPE html><html><head><title>404</title></head><body><h1>404 Not Found</h1><p>Couldn't find the site you were looking for.</p></body></html>"
44+
# For now until I implement actual rendering
45+
sock.close()
46+
return response, False

‎processor/tests/__init__.py‎

Whitespace-only changes.

0 commit comments

Comments
 (0)