From 73a7764698f6a7694c46cba1c062d07ac3f2d61b Mon Sep 17 00:00:00 2001 From: masterluo Date: Wed, 26 Apr 2023 19:12:27 +0800 Subject: [PATCH 1/9] to: update Makefile - add clean - add upload --- Makefile | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/Makefile b/Makefile index 8013354..83bc5c5 100644 --- a/Makefile +++ b/Makefile @@ -1,15 +1,26 @@ PYTHON := $(shell command -v python3) +CLEAN_PATHS := $(PWD)/build $(PWD)/dist $(PWD)/*.egg-info +WHL_PATH := $(wildcard dist/*.whl) + +.PHONY: clean +clean: + @-rm -rf $(CLEAN_PATHS) fmt: - command -v black || $(PYTHON) -m pip install -r requirements.txt - black ja3requests + @command -v black || $(PYTHON) -m pip install -r requirements.txt + $(shell black ja3requests) lint: - command -v pylint || $(PYTHON) -m pip install -r requirements.txt - pylint ja3requests + @command -v pylint || $(PYTHON) -m pip install -r requirements.txt + $(shell pylint ja3requests) + +.PHONY: dist +dist: + @if [ -f 'setup.py' ]; then $(PYTHON) setup.py sdist;fi -package: - if [ -f 'setup.py' ]; then $(PYTHON) setup.py sdist;fi +.PHONY: build +build: dist + @if [ -f 'setup.py' ]; then $(PYTHON) setup.py bdist_wheel;fi -package_whl: - if [ -f 'setup.py' ]; then $(PYTHON) setup.py bdist_wheel;fi +upload: + @if [ -f '$(WHL_PATH)' ];then twine upload $(wildcard dist/*); else echo "File not existed.";fi From baf5ef59998548846304d3443c4337999954bc54 Mon Sep 17 00:00:00 2001 From: masterluo Date: Fri, 28 Apr 2023 18:57:31 +0800 Subject: [PATCH 2/9] to: Request class - add basic request class - add request class --- ja3requests/base/__init__.py | 1 + ja3requests/base/_request.py | 93 ++++++++++++++++++++++++++++++++++++ ja3requests/exceptions.py | 6 +++ ja3requests/request.py | 50 +++++++++++++++++++ ja3requests/sessions.py | 20 +++++++- 5 files changed, 168 insertions(+), 2 deletions(-) create mode 100644 ja3requests/base/_request.py create mode 100644 ja3requests/exceptions.py create mode 100644 ja3requests/request.py diff --git a/ja3requests/base/__init__.py b/ja3requests/base/__init__.py index e9de5d3..8fc2183 100644 --- a/ja3requests/base/__init__.py +++ b/ja3requests/base/__init__.py @@ -6,3 +6,4 @@ """ from ._sessions import BaseSession +from ._request import BaseRequest diff --git a/ja3requests/base/_request.py b/ja3requests/base/_request.py new file mode 100644 index 0000000..426303f --- /dev/null +++ b/ja3requests/base/_request.py @@ -0,0 +1,93 @@ +""" +ja3Requests.base._request +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Basic Request +""" + + +class BaseRequest: + + def __init__(self): + + self._method = None + self._url = None + self._headers = None + self._params = None + self._data = None + self._cookies = None + self._files = None + self._auth = None + self._json = None + + @property + def method(self): + return self._method + + @method.setter + def method(self, attr): + self._method = attr + + @property + def url(self): + return self._url + + @url.setter + def url(self, attr): + self._url = attr + + @property + def headers(self): + return self._headers + + @headers.setter + def headers(self, attr): + self._headers = attr + + @property + def params(self): + return self._params + + @params.setter + def params(self, attr): + self._params = attr + + @property + def data(self): + return self._data + + @data.setter + def data(self, attr): + self._data = attr + + @property + def cookies(self): + return self._cookies + + @cookies.setter + def cookies(self, attr): + self._cookies = attr + + @property + def files(self): + return self._files + + @files.setter + def files(self, attr): + self._files = attr + + @property + def auth(self): + return self._auth + + @auth.setter + def auth(self, attr): + self._auth = attr + + @property + def json(self): + return self._json + + @json.setter + def json(self, attr): + self._json = attr diff --git a/ja3requests/exceptions.py b/ja3requests/exceptions.py new file mode 100644 index 0000000..60b5a3d --- /dev/null +++ b/ja3requests/exceptions.py @@ -0,0 +1,6 @@ + + +class NotAllowRequestMethod(ValueError): + """ + If the request method not allow raise it. + """ diff --git a/ja3requests/request.py b/ja3requests/request.py new file mode 100644 index 0000000..6cbe9be --- /dev/null +++ b/ja3requests/request.py @@ -0,0 +1,50 @@ +""" +ja3requests.request +~~~~~~~~~~~~~~~~~~~ + +This module create a request struct and ready request object. +""" +from .base import BaseRequest +from .exceptions import NotAllowRequestMethod +from http.cookiejar import CookieJar +from typing import Any, AnyStr, Dict, List, Union, ByteString, Tuple + + +class Request(BaseRequest): + + def __init__( + self, + method: AnyStr, + url: AnyStr, + params: Union[Dict[AnyStr, Any], ByteString] = None, + data: Union[Dict[AnyStr, Any], List, Tuple, ByteString] = None, + headers: Dict[AnyStr, AnyStr] = None, + cookies: Union[Dict[AnyStr, AnyStr], CookieJar] = None, + auth: Tuple = None, + json: Dict[AnyStr, AnyStr] = None, + ): + super().__init__() + self.method = method + self.url = url + self.params = params + self.data = data + self.headers = headers + self.cookies = cookies + self.auth = auth + self.json = json + + def __repr__(self): + return f"" + + def check_method(self): + + if self.method == "" or self.method not in [ + "GET", + "OPTIONS", + "HEAD", + "POST", + "PUT", + "PATCH", + "DELETE", + ]: + raise NotAllowRequestMethod(self.method) diff --git a/ja3requests/sessions.py b/ja3requests/sessions.py index 2db6bb1..1273fa6 100644 --- a/ja3requests/sessions.py +++ b/ja3requests/sessions.py @@ -52,10 +52,10 @@ def request( timeout: float = None, allow_redirects: bool = True, proxies: Dict[AnyStr, AnyStr] = None, - # json: = None, + json: Dict[AnyStr, AnyStr] = None, ): """ - Request + Instantiating a request class and ready request to send. :param method: :param url: :param params: @@ -66,6 +66,7 @@ def request( :param timeout: :param allow_redirects: :param proxies: + :param json: :return: """ @@ -77,6 +78,8 @@ def get(self, url, **kwargs): :return: """ + return self.request("GET", url, **kwargs) + def options(self, url, **kwargs): """ Send a OPTIONS request. @@ -85,6 +88,8 @@ def options(self, url, **kwargs): :return: """ + return self.request("OPTIONS", url, **kwargs) + def head(self, url, **kwargs): """ Send a HEAD request. @@ -93,6 +98,9 @@ def head(self, url, **kwargs): :return: """ + kwargs.setdefault("allow_redirects", False) + return self.request("HEAD", url, **kwargs) + def post(self, url, data=None, json=None, **kwargs): """ Send a POST request. @@ -103,6 +111,8 @@ def post(self, url, data=None, json=None, **kwargs): :return: """ + return self.request("POST", url, data=data, json=json, **kwargs) + def put(self, url, data=None, **kwargs): """ Send a PUT request. @@ -112,6 +122,8 @@ def put(self, url, data=None, **kwargs): :return: """ + return self.request("PUT", url, data=data, **kwargs) + def patch(self, url, data=None, **kwargs): """ Send a PATCH request. @@ -121,6 +133,8 @@ def patch(self, url, data=None, **kwargs): :return: """ + return self.request("PATCH", url, data=data, **kwargs) + def delete(self, url, data=None, **kwargs): """ Send a DELETE request. @@ -130,6 +144,8 @@ def delete(self, url, data=None, **kwargs): :return: """ + return self.request("DELETE", url, **kwargs) + def send(self): """ Send a ready request. From 5509b9651978818bb9b0047e24e2e44e5eb92cb6 Mon Sep 17 00:00:00 2001 From: masterluo Date: Thu, 4 May 2023 19:00:04 +0800 Subject: [PATCH 3/9] to: ReadyRequest - ReadyRequest - add RequestException - add MissingSchema --- ja3requests/exceptions.py | 31 ++++++++++++++++++++++++++- ja3requests/request.py | 44 +++++++++++++++++++++++++++++++++++---- 2 files changed, 70 insertions(+), 5 deletions(-) diff --git a/ja3requests/exceptions.py b/ja3requests/exceptions.py index 60b5a3d..8fa4418 100644 --- a/ja3requests/exceptions.py +++ b/ja3requests/exceptions.py @@ -1,6 +1,35 @@ +""" +ja3requests.exceptions +~~~~~~~~~~~~~~~~~~~~~~ +This module contains the set of Requests' exceptions. +""" -class NotAllowRequestMethod(ValueError): + +class RequestException(IOError): + """ + There was an ambiguous exception that occurred while handling your request. + """ + + def __init__(self, *args, **kwargs): + """ + Initialize RequestException with `request` and `response` objects. + """ + response = kwargs.pop("response", None) + self.response = response + self.request = kwargs.pop("request", None) + if response is not None and not self.request and hasattr(response, "request"): + self.request = self.response.request + super().__init__(*args, **kwargs) + + +class NotAllowRequestMethod(RequestException, ValueError): """ If the request method not allow raise it. """ + + +class MissingSchema(RequestException, ValueError): + """ + The URL scheme (e.g. http or https) is missing. + """ diff --git a/ja3requests/request.py b/ja3requests/request.py index 6cbe9be..5e4820f 100644 --- a/ja3requests/request.py +++ b/ja3requests/request.py @@ -5,12 +5,13 @@ This module create a request struct and ready request object. """ from .base import BaseRequest -from .exceptions import NotAllowRequestMethod +from .exceptions import NotAllowRequestMethod, MissingSchema +from urllib.parse import urlparse from http.cookiejar import CookieJar from typing import Any, AnyStr, Dict, List, Union, ByteString, Tuple -class Request(BaseRequest): +class ReadyRequest(BaseRequest): def __init__( self, @@ -33,10 +34,16 @@ def __init__( self.auth = auth self.json = json + self.ready_method() + def __repr__(self): - return f"" + return f"" - def check_method(self): + def ready_method(self): + """ + Ready request method and check request method whether allow used. + :return: + """ if self.method == "" or self.method not in [ "GET", @@ -48,3 +55,32 @@ def check_method(self): "DELETE", ]: raise NotAllowRequestMethod(self.method) + + self.method = self.method.upper() + + def ready_url(self): + """ + Ready http url and check url whether valid. + :return: + """ + + if self.url == "": + raise ValueError("The request url is required.") + + # Remove whitespaces for url + self.url.strip() + + parse = urlparse(self.url) + + # Check HTTP schemes, just allow http or https + if parse.scheme == "": + raise MissingSchema(f"Invalid URL '{self.url}': No scheme supplied. Perhaps you meant http://{parse.netloc} or https://{parse.netloc}") + + + + def ready(self): + """ + Make a ready request to send. + :return: + """ + From 623e6deb11dcf41178daeea77916629257ce499b Mon Sep 17 00:00:00 2001 From: masterluo Date: Fri, 5 May 2023 19:08:56 +0800 Subject: [PATCH 4/9] to: ready url - request ready url --- ja3requests/base/_request.py | 18 ++++++++++++++++++ ja3requests/exceptions.py | 14 ++++++++++---- ja3requests/request.py | 37 +++++++++++++++++++++++++++++------- 3 files changed, 58 insertions(+), 11 deletions(-) diff --git a/ja3requests/base/_request.py b/ja3requests/base/_request.py index 426303f..d0a7649 100644 --- a/ja3requests/base/_request.py +++ b/ja3requests/base/_request.py @@ -12,6 +12,8 @@ def __init__(self): self._method = None self._url = None + self._scheme = None + self._port = None self._headers = None self._params = None self._data = None @@ -36,6 +38,22 @@ def url(self): def url(self, attr): self._url = attr + @property + def scheme(self): + return self._scheme + + @scheme.setter + def scheme(self, attr): + self._scheme = attr + + @property + def port(self): + return self._port + + @port.setter + def port(self, attr): + self._port = attr + @property def headers(self): return self._headers diff --git a/ja3requests/exceptions.py b/ja3requests/exceptions.py index 8fa4418..72492b3 100644 --- a/ja3requests/exceptions.py +++ b/ja3requests/exceptions.py @@ -23,13 +23,19 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) -class NotAllowRequestMethod(RequestException, ValueError): +class NotAllowedRequestMethod(RequestException, ValueError): """ - If the request method not allow raise it. + If the request method not allowed and raise it. """ -class MissingSchema(RequestException, ValueError): +class MissingScheme(RequestException, ValueError): """ - The URL scheme (e.g. http or https) is missing. + The URL scheme (e.g. http or https) is missing and raise it. """ + + +class NotAllowedScheme(RequestException, ValueError): + """ + If the scheme not allowed and raise it. + """ \ No newline at end of file diff --git a/ja3requests/request.py b/ja3requests/request.py index 5e4820f..426291c 100644 --- a/ja3requests/request.py +++ b/ja3requests/request.py @@ -5,7 +5,7 @@ This module create a request struct and ready request object. """ from .base import BaseRequest -from .exceptions import NotAllowRequestMethod, MissingSchema +from .exceptions import NotAllowedRequestMethod, MissingScheme, NotAllowedScheme from urllib.parse import urlparse from http.cookiejar import CookieJar from typing import Any, AnyStr, Dict, List, Union, ByteString, Tuple @@ -17,7 +17,7 @@ def __init__( self, method: AnyStr, url: AnyStr, - params: Union[Dict[AnyStr, Any], ByteString] = None, + params: Union[Dict[AnyStr, Any], List[Tuple[AnyStr, Any]], ByteString] = None, data: Union[Dict[AnyStr, Any], List, Tuple, ByteString] = None, headers: Dict[AnyStr, AnyStr] = None, cookies: Union[Dict[AnyStr, AnyStr], CookieJar] = None, @@ -54,7 +54,7 @@ def ready_method(self): "PATCH", "DELETE", ]: - raise NotAllowRequestMethod(self.method) + raise NotAllowedRequestMethod(self.method) self.method = self.method.upper() @@ -72,11 +72,34 @@ def ready_url(self): parse = urlparse(self.url) - # Check HTTP schemes, just allow http or https + # Check HTTP scheme if parse.scheme == "": - raise MissingSchema(f"Invalid URL '{self.url}': No scheme supplied. Perhaps you meant http://{parse.netloc} or https://{parse.netloc}") - - + raise MissingScheme( + f"Invalid URL {self.url!r}: No scheme supplied. " + f"Perhaps you meant http://{self.url} or https://{self.url}" + ) + + # Just allow http or https + if parse.scheme.lower() not in ["http", "https"]: + raise NotAllowedScheme( + f"Schema: {parse.scheme} not allowed." + ) + + self.scheme = parse.scheme + if self.scheme == "https": + self.port = 443 + + if parse.netloc != "" and ":" in parse.netloc: + port = parse.netloc.split(":")[-1] + self.port = int(port) + else: + self.port = 80 + + def ready_params(self): + """ + Ready params. + :return: + """ def ready(self): """ From 7331f7515ac99c599f790ecd6a6c9f904b5e57f4 Mon Sep 17 00:00:00 2001 From: masterluo Date: Tue, 9 May 2023 17:47:28 +0800 Subject: [PATCH 5/9] to: ReadyRequest&Request - add ready request - add simple request --- ja3requests/exceptions.py | 6 +++ ja3requests/request.py | 89 ++++++++++++++++++++++++++++++++++++--- ja3requests/sessions.py | 51 ++++++++++++++++++++-- 3 files changed, 136 insertions(+), 10 deletions(-) diff --git a/ja3requests/exceptions.py b/ja3requests/exceptions.py index 72492b3..f15c5cf 100644 --- a/ja3requests/exceptions.py +++ b/ja3requests/exceptions.py @@ -38,4 +38,10 @@ class MissingScheme(RequestException, ValueError): class NotAllowedScheme(RequestException, ValueError): """ If the scheme not allowed and raise it. + """ + + +class InvalidParams(RequestException, ValueError): + """ + If request params invalid and raise it. """ \ No newline at end of file diff --git a/ja3requests/request.py b/ja3requests/request.py index 426291c..b7e3f7a 100644 --- a/ja3requests/request.py +++ b/ja3requests/request.py @@ -5,9 +5,9 @@ This module create a request struct and ready request object. """ from .base import BaseRequest -from .exceptions import NotAllowedRequestMethod, MissingScheme, NotAllowedScheme -from urllib.parse import urlparse +from .exceptions import NotAllowedRequestMethod, MissingScheme, NotAllowedScheme, InvalidParams from http.cookiejar import CookieJar +from urllib.parse import urlparse, urlencode from typing import Any, AnyStr, Dict, List, Union, ByteString, Tuple @@ -17,7 +17,7 @@ def __init__( self, method: AnyStr, url: AnyStr, - params: Union[Dict[AnyStr, Any], List[Tuple[AnyStr, Any]], ByteString] = None, + params: Union[Dict[Any, Any], List[Tuple[Any, Any]], Tuple[Tuple[Any, Any]], ByteString, AnyStr] = None, data: Union[Dict[AnyStr, Any], List, Tuple, ByteString] = None, headers: Dict[AnyStr, AnyStr] = None, cookies: Union[Dict[AnyStr, AnyStr], CookieJar] = None, @@ -34,8 +34,6 @@ def __init__( self.auth = auth self.json = json - self.ready_method() - def __repr__(self): return f"" @@ -80,7 +78,7 @@ def ready_url(self): ) # Just allow http or https - if parse.scheme.lower() not in ["http", "https"]: + if parse.scheme not in ["http", "https"]: raise NotAllowedScheme( f"Schema: {parse.scheme} not allowed." ) @@ -100,10 +98,89 @@ def ready_params(self): Ready params. :return: """ + if self.params: + parse = urlparse(self.url) + + if isinstance(self.params, str): + params = self.params + elif isinstance(self.params, bytes): + params = self.params.decode() + elif isinstance(self.params, (dict, list, tuple)): + params = urlencode(self.params) + else: + raise InvalidParams(f"Invalid params: {self.params!r}") + + if params.startswith("?"): + params = params.replace("?", "") + + if parse.query != "": + self.url = "&" + params + else: + self.url = "?" + params + + def ready_data(self): + """ + Todo: Ready form data. + :return: + """ + + def ready_headers(self): + """ + Todo: Ready http headers. + :return: + """ + + def ready_cookies(self): + """ + Todo: Ready http cookies. + :return: + """ + + def ready_auth(self): + """ + Todo: Ready http authenticator + :return: + """ + + def ready_json(self): + """ + Todo: Ready post json. + :return: + """ def ready(self): """ Make a ready request to send. :return: """ + self.ready_method() + self.ready_url() + self.ready_params() + self.ready_data() + self.ready_headers() + self.ready_cookies() + self.ready_auth() + self.ready_json() + + def request(self): + + req = Request() + req.clone(self) + + return req + + +class Request(BaseRequest): + + def __repr__(self): + return f"" + + def clone(self, ready_request: ReadyRequest): + + for k, v in ready_request.__dict__.items(): + setattr(self, k, v) + + def send(self): + + pass diff --git a/ja3requests/sessions.py b/ja3requests/sessions.py index 1273fa6..5e9ae2f 100644 --- a/ja3requests/sessions.py +++ b/ja3requests/sessions.py @@ -12,6 +12,7 @@ from .base import BaseSession from .utils import default_headers from .const import DEFAULT_REDIRECT_LIMIT +from .request import ReadyRequest, Request # Preferred clock, based on which one is more accurate on a given system. if sys.platform == "win32": @@ -32,13 +33,36 @@ def __init__(self): self.headers = default_headers() self.max_redirects = DEFAULT_REDIRECT_LIMIT - def ready_request(self, request): + def ready( + self, + method, + url, + params, + data, + headers, + cookies, + auth, + json + ): """ Ready to send request. - :param request: :return: """ + req = ReadyRequest( + method=method, + url=url, + params=params, + data=data, + headers=headers, + cookies=cookies, + auth=auth, + json=json, + ) + req.ready() + + return req + def request( self, method: AnyStr, @@ -69,6 +93,21 @@ def request( :param json: :return: """ + ready_request = self.ready( + method=method, + url=url, + params=params, + data=data, + headers=headers, + cookies=cookies, + auth=auth, + json=json + ) + + req = ready_request.request() + response = self.send(req) + + return response def get(self, url, **kwargs): """ @@ -146,8 +185,12 @@ def delete(self, url, data=None, **kwargs): return self.request("DELETE", url, **kwargs) - def send(self): + def send(self, request: Request): """ - Send a ready request. + Send request. :return: """ + + response = request.send() + + return response From fbc13c4f270323a2c05a70508aa6d030641385f4 Mon Sep 17 00:00:00 2001 From: masterluo Date: Wed, 10 May 2023 19:09:10 +0800 Subject: [PATCH 6/9] to:socket - add HTTPConnection - create socket - socket exceptions --- ja3requests/base/__init__.py | 1 + ja3requests/base/_connection.py | 85 +++++++++++++++++++++++++++ ja3requests/connections.py | 42 ++++++++++++++ ja3requests/protocol/__init__.py | 0 ja3requests/protocol/exceptions.py | 34 +++++++++++ ja3requests/protocol/sockets.py | 93 ++++++++++++++++++++++++++++++ 6 files changed, 255 insertions(+) create mode 100644 ja3requests/base/_connection.py create mode 100644 ja3requests/connections.py create mode 100644 ja3requests/protocol/__init__.py create mode 100644 ja3requests/protocol/exceptions.py create mode 100644 ja3requests/protocol/sockets.py diff --git a/ja3requests/base/__init__.py b/ja3requests/base/__init__.py index 8fc2183..6f03805 100644 --- a/ja3requests/base/__init__.py +++ b/ja3requests/base/__init__.py @@ -7,3 +7,4 @@ from ._sessions import BaseSession from ._request import BaseRequest +from ._connection import BaseHttpConnection diff --git a/ja3requests/base/_connection.py b/ja3requests/base/_connection.py new file mode 100644 index 0000000..3918bf4 --- /dev/null +++ b/ja3requests/base/_connection.py @@ -0,0 +1,85 @@ +""" +ja3Requests.base._connection +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Basic HTTP Connection +""" + + +class BaseHttpConnection: + + def __init__(self): + + self._scheme = "http" + self._http_version = "HTTP/1.1" + self._port = 80 + self._source_address = None + self._destination_address = None + self._timeout = None + self._proxy = None + self._proxy_username = None + self._proxy_password = None + + @property + def scheme(self): + return self._scheme + + @scheme.setter + def scheme(self, attr): + self._scheme = attr + + @property + def http_version(self): + return self._http_version + + @property + def port(self): + return self._port + + @port.setter + def port(self, attr): + self._port = attr + + @property + def source_address(self): + return self._source_address + + @property + def destination_address(self): + return self._destination_address + + @destination_address.setter + def destination_address(self, attr): + self._destination_address = attr + + @property + def timeout(self): + return self._timeout + + @timeout.setter + def timeout(self, attr): + self._timeout = attr + + @property + def proxy(self): + return self._proxy + + @proxy.setter + def proxy(self, attr): + self._proxy = attr + + @property + def proxy_username(self): + return self._proxy_username + + @proxy_username.setter + def proxy_username(self, attr): + self._proxy_username = attr + + @property + def proxy_password(self): + return self._proxy_password + + @proxy_password.setter + def proxy_password(self, attr): + self._proxy_password = attr diff --git a/ja3requests/connections.py b/ja3requests/connections.py new file mode 100644 index 0000000..8ce30e5 --- /dev/null +++ b/ja3requests/connections.py @@ -0,0 +1,42 @@ +""" +ja3requests.connections +~~~~~~~~~~~~~~~~~~~~~~~ + +This module contains HTTP connection and HTTPS connection. +""" + +from .base import BaseHttpConnection +from .protocol.sockets import create_connection +from .protocol.exceptions import SocketTimeout, ConnectTimeoutError + + +class HTTPConnection(BaseHttpConnection): + + def __init__(self): + + super().__init__() + + def _new_conn(self): + """ + Establish a socket connection + :return: socket connection + """ + + try: + conn = create_connection( + (self.destination_address, self.port), + self.timeout, + self.source_address, + ) + except SocketTimeout: + raise ConnectTimeoutError( + f"Connection to {self.destination_address} timeout out. timeout={self.timeout}" + ) + + return conn + + def connect(self): + + conn = self._new_conn() + + diff --git a/ja3requests/protocol/__init__.py b/ja3requests/protocol/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/ja3requests/protocol/exceptions.py b/ja3requests/protocol/exceptions.py new file mode 100644 index 0000000..224d461 --- /dev/null +++ b/ja3requests/protocol/exceptions.py @@ -0,0 +1,34 @@ +""" +ja3requests.protocol.exceptions +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +This module contains socket exceptions. +""" + + +class SocketException(Exception): + """ + Base exception used by this module. + """ + + +class SocketTimeout(OSError): + """ Timeout expired. """ + + +class LocationParseError(SocketException, ValueError): + """ + Socket host encode error. + """ + + +class SocketTimeoutError(SocketException): + """ + Raised when a socket timeout error occurs. + """ + + +class ConnectTimeoutError(SocketTimeoutError): + """ + Raised when a socket timeout occurs while connecting to a server + """ \ No newline at end of file diff --git a/ja3requests/protocol/sockets.py b/ja3requests/protocol/sockets.py new file mode 100644 index 0000000..aa9bd45 --- /dev/null +++ b/ja3requests/protocol/sockets.py @@ -0,0 +1,93 @@ +""" +ja3requests.protocol.sockets +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +This module contains socket dependencies. +""" + +import socket +from .exceptions import LocationParseError + + +def create_connection( + address, + timeout=socket._GLOBAL_DEFAULT_TIMEOUT, + source_address=None, + socket_options=None, +): + if socket_options is None: + socket_options = [(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)] + + err = None + host, port = address + + family = allowed_gai_family() + + try: + host.encode("idna") + except UnicodeError: + raise LocationParseError(f"{host!r}, label empty or too long") + + for res in socket.getaddrinfo(host, port, family, socket.SOCK_STREAM): + _family, _type, _proto, _canonname, _addr = res + sock = None + try: + sock = socket.socket(_family, _type, _proto) + + # If provided, set socket level options before connecting. + _set_socket_options(sock, socket_options) + + if timeout is not socket._GLOBAL_DEFAULT_TIMEOUT: + sock.settimeout(timeout) + if source_address: + sock.bind(source_address) + sock.connect(_addr) + return sock + + except socket.error as e: + err = e + if sock is not None: + sock.close() + + if err is not None: + raise err + + raise socket.error("getaddrinfo returns an empty list") + + +def _set_socket_options(sock, options): + + if options is None: + return + + for opt in options: + sock.setsockopt(*opt) + + +def allowed_gai_family(): + + family = socket.AF_INET + if HAS_IPV6: + family = socket.AF_UNSPEC + return family + + +def _has_ipv6(host): + """Returns True if the system can bind an IPv6 address.""" + sock = None + has_ipv6 = False + + if socket.has_ipv6: + try: + sock = socket.socket(socket.AF_INET6) + sock.bind((host, 0)) + has_ipv6 = True + except Exception: + pass + + if sock: + sock.close() + return has_ipv6 + + +HAS_IPV6 = _has_ipv6("::1") From af381fa8e530845b54827bcdc1e55b53d0b83e34 Mon Sep 17 00:00:00 2001 From: lxjmaster <3795001669@qq.com> Date: Thu, 11 May 2023 01:31:26 +0800 Subject: [PATCH 7/9] to: add connection - add connection - add connections.connect - add request.create_connect --- ja3requests/base/_connection.py | 9 +++++++++ ja3requests/base/_request.py | 6 ++++++ ja3requests/connections.py | 28 ++++++++++++++++++++++++++-- ja3requests/request.py | 21 ++++++++++++++++++++- 4 files changed, 61 insertions(+), 3 deletions(-) diff --git a/ja3requests/base/_connection.py b/ja3requests/base/_connection.py index 3918bf4..0a25026 100644 --- a/ja3requests/base/_connection.py +++ b/ja3requests/base/_connection.py @@ -19,6 +19,7 @@ def __init__(self): self._proxy = None self._proxy_username = None self._proxy_password = None + self._connection = None @property def scheme(self): @@ -83,3 +84,11 @@ def proxy_password(self): @proxy_password.setter def proxy_password(self, attr): self._proxy_password = attr + + @property + def connection(self): + return self._connection + + @connection.setter + def connection(self, attr): + self._connection = attr diff --git a/ja3requests/base/_request.py b/ja3requests/base/_request.py index d0a7649..1f694a5 100644 --- a/ja3requests/base/_request.py +++ b/ja3requests/base/_request.py @@ -109,3 +109,9 @@ def json(self): @json.setter def json(self, attr): self._json = attr + + def is_http(self): + return self._scheme == "http" + + def is_https(self): + return self._scheme == "https" diff --git a/ja3requests/connections.py b/ja3requests/connections.py index 8ce30e5..0802058 100644 --- a/ja3requests/connections.py +++ b/ja3requests/connections.py @@ -35,8 +35,32 @@ def _new_conn(self): return conn - def connect(self): + def _ready_connect(self, **kwargs): - conn = self._new_conn() + pass + + def connect( + self, + scheme=None, + port=None, + source_address=None, + destination_address=None, + timeout=None, + proxy=None, + proxy_username=None, + proxy_password=None, + ): + self._ready_connect( + scheme=scheme, + port=port, + source_address=source_address, + destination_address=destination_address, + timeout=timeout, + proxy=proxy, + proxy_username=proxy_username, + proxy_password=proxy_password, + ) + conn = self._new_conn() + self.connection = conn diff --git a/ja3requests/request.py b/ja3requests/request.py index b7e3f7a..593ab8e 100644 --- a/ja3requests/request.py +++ b/ja3requests/request.py @@ -5,6 +5,7 @@ This module create a request struct and ready request object. """ from .base import BaseRequest +from .connections import HTTPConnection from .exceptions import NotAllowedRequestMethod, MissingScheme, NotAllowedScheme, InvalidParams from http.cookiejar import CookieJar from urllib.parse import urlparse, urlencode @@ -182,5 +183,23 @@ def clone(self, ready_request: ReadyRequest): def send(self): - pass + conn = self.create_connect() + conn.connect( + self.scheme, + self.port, + ) + # conn.send() + + def create_connect(self): + + if self.is_http(): + conn = HTTPConnection() + elif self.is_https(): + # TODO: HTTPS + # conn = HTTPSConnection() + raise NotImplementedError("HTTPSConnection not implemented yet.") + else: + raise MissingScheme(f"Scheme: {self.scheme}, parse scheme failed, can't create connection.") + + return conn From fc69d5c233803145b559122f0d54f0f329d93ddd Mon Sep 17 00:00:00 2001 From: masterluo Date: Thu, 11 May 2023 17:12:19 +0800 Subject: [PATCH 8/9] to: perfect the connection - conenction add source_address. - request add source. - connections perfect send function. - connections add close function for connection close. - connections add _ready_connect for ready connect arguments - request prefect send function --- ja3requests/base/_connection.py | 4 +++ ja3requests/base/_request.py | 27 +++++++++++++++ ja3requests/connections.py | 60 ++++++++++++++++++++++++++++++++- ja3requests/request.py | 28 +++++++++++++-- test/test_ready_request.py | 23 +++++++++++++ test/test_session.py | 15 +++++++++ 6 files changed, 154 insertions(+), 3 deletions(-) create mode 100644 test/test_ready_request.py create mode 100644 test/test_session.py diff --git a/ja3requests/base/_connection.py b/ja3requests/base/_connection.py index 0a25026..d222631 100644 --- a/ja3requests/base/_connection.py +++ b/ja3requests/base/_connection.py @@ -45,6 +45,10 @@ def port(self, attr): def source_address(self): return self._source_address + @source_address.setter + def source_address(self, attr): + self._source_address = attr + @property def destination_address(self): return self._destination_address diff --git a/ja3requests/base/_request.py b/ja3requests/base/_request.py index 1f694a5..ff5815b 100644 --- a/ja3requests/base/_request.py +++ b/ja3requests/base/_request.py @@ -11,6 +11,7 @@ class BaseRequest: def __init__(self): self._method = None + self._source = None self._url = None self._scheme = None self._port = None @@ -21,6 +22,8 @@ def __init__(self): self._files = None self._auth = None self._json = None + self._timeout = None + self._proxies = None @property def method(self): @@ -30,6 +33,14 @@ def method(self): def method(self, attr): self._method = attr + @property + def source(self): + return self._source + + @source.setter + def source(self, attr): + self._source = attr + @property def url(self): return self._url @@ -110,6 +121,22 @@ def json(self): def json(self, attr): self._json = attr + @property + def timeout(self): + return self._timeout + + @timeout.setter + def timeout(self, attr): + self._timeout = attr + + @property + def proxies(self): + return self._proxies + + @proxies.setter + def proxies(self, attr): + self._proxies = attr + def is_http(self): return self._scheme == "http" diff --git a/ja3requests/connections.py b/ja3requests/connections.py index 0802058..c7718c7 100644 --- a/ja3requests/connections.py +++ b/ja3requests/connections.py @@ -16,6 +16,9 @@ def __init__(self): super().__init__() + def __del__(self): + self.close() + def _new_conn(self): """ Establish a socket connection @@ -36,8 +39,42 @@ def _new_conn(self): return conn def _ready_connect(self, **kwargs): + """ + Ready http connection. + :param kwargs: + :return: + """ + if kwargs.get("scheme", None): + self.scheme = kwargs["scheme"] + + if kwargs.get("port", None): + self.port = kwargs["port"] + + if kwargs.get("source_address", None): + self.source_address = kwargs["source_address"] + + if kwargs.get("destination_address", None): + self.destination_address = kwargs["destination_address"] + + # Remove scheme + if self.destination_address.startswith("http://"): + self.destination_address = self.destination_address.replace("http://", "") + + # Remove port + if ":" in self.destination_address: + self.destination_address = self.destination_address.split(":")[0] + + if kwargs.get("timeout", None): + self.timeout = kwargs["timeout"] - pass + if kwargs.get("proxy", None): + self.proxy = kwargs["proxy"] + + if kwargs.get("proxy_username", None): + self.proxy_username = kwargs["proxy_username"] + + if kwargs.get("proxy_password", None): + self.proxy_password = kwargs["proxy_password"] def connect( self, @@ -64,3 +101,24 @@ def connect( conn = self._new_conn() self.connection = conn + def send(self): + """ + Send socket. + :return: + """ + self.connection.sendall( + "GET / HTTP/1.1\r\n\r\n".encode() + ) + response_data = b"" + while True: + data = self.connection.recv(1024) + if not data: + break + response_data += data + + return response_data + + def close(self): + + if self.connection: + self.connection.close() diff --git a/ja3requests/request.py b/ja3requests/request.py index 593ab8e..a70180b 100644 --- a/ja3requests/request.py +++ b/ja3requests/request.py @@ -182,16 +182,32 @@ def clone(self, ready_request: ReadyRequest): setattr(self, k, v) def send(self): + """ + Connection sending. + :return: + """ conn = self.create_connect() + proxy, proxy_username, proxy_password = self.parse_proxies() conn.connect( self.scheme, self.port, - + self.source, + self.url, + self.timeout, + proxy, + proxy_username, + proxy_password ) - # conn.send() + response = conn.send() + + return response def create_connect(self): + """ + Create http connection or https connection by request scheme. + :return: + """ if self.is_http(): conn = HTTPConnection() @@ -203,3 +219,11 @@ def create_connect(self): raise MissingScheme(f"Scheme: {self.scheme}, parse scheme failed, can't create connection.") return conn + + def parse_proxies(self): + """ + TODO + Parse proxy, proxy's username and password. if proxies is set. + :return: + """ + return None, None, None diff --git a/test/test_ready_request.py b/test/test_ready_request.py new file mode 100644 index 0000000..e2088d5 --- /dev/null +++ b/test/test_ready_request.py @@ -0,0 +1,23 @@ +import unittest +from ja3requests.request import ReadyRequest + + +class TestReadyRequest(unittest.TestCase): + + request = ReadyRequest( + "GET", + "http://www.baidu.com" + ) + + def test_ready(self): + + self.request.ready() + print(self.request.scheme, self.request.url) + + def test_request(self): + req = self.request.request() + req.send() + + +if __name__ == '__main__': + unittest.main() diff --git a/test/test_session.py b/test/test_session.py new file mode 100644 index 0000000..4eaaa0e --- /dev/null +++ b/test/test_session.py @@ -0,0 +1,15 @@ +import unittest +from ja3requests.sessions import Session + + +class TestSession(unittest.TestCase): + + session = Session() + + def test_get(self): + + self.session.get("http://localhost:8080/") + + +if __name__ == '__main__': + unittest.main() From 9f0a292e0e4a2d90f56680cf8834d7c081fec41e Mon Sep 17 00:00:00 2001 From: masterluo Date: Thu, 11 May 2023 17:17:28 +0800 Subject: [PATCH 9/9] release: change version --- ja3requests/__version__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ja3requests/__version__.py b/ja3requests/__version__.py index 826e93f..3f67b0a 100644 --- a/ja3requests/__version__.py +++ b/ja3requests/__version__.py @@ -8,7 +8,7 @@ __title__ = "ja3requests" __description__ = "An http request library that can customize ja3 fingerprints." __url__ = "https://github.com/lxjmaster/ja3requests" -__version__ = "0.0.1" +__version__ = "0.0.2" __author__ = "Mast Luo" __author_email__ = "379501669@qq.com" __license__ = "Apache-2.0 license"