diff --git a/README-zh.md b/README-zh.md index 4541f4c..8a1427d 100644 --- a/README-zh.md +++ b/README-zh.md @@ -1,20 +1,18 @@ - - # Ja3Requests **Ja3Requests**是一个可以自定义ja3指纹（tls指纹）和HTTP2指纹的请求库 [English Document](README.md) -```python +```pycon >>> import ja3requests >>> session = ja3requests.Session() ->>> response = session.get("http://www.baidu.com") +>>> response = session.get("http://www.baidu.com/") >>> response >>> response.status_code 200 >>> response.headers -[{'Bdqid': '0xdc8736c700095118'}, {'Connection': 'keep-alive'},...] +{'Content-Length': '405968', 'Content-Type': 'text/html; charset=utf-8', 'Server': 'BWS/1.1', 'Vary': 'Accept-Encoding', 'X-Ua-Compatible': 'IE=Edge,chrome=1', ...} >>> response.text ' + +# Or set cookies in headers = {"Cookies": "sessionId=xxxx; userId=xxxx;...."} + +response = session.get("http://example.com/", cookies=cookies) +print(response) +``` + + +### 允许重定向 + +```python +import ja3requests + +session = ja3requests.session() + +# Default allow_redirects=True +response = session.get("http://example.com/", allow_redirects=False) +print(response) +``` + + ## 参考 - [HTTP](https://developer.mozilla.org/en-US/docs/Web/HTTP) - [HTTP-RFC](https://www.rfc-editor.org/rfc/rfc2068.html) \ No newline at end of file diff --git a/README.md b/README.md index e05233b..03f5bd9 100644 --- a/README.md +++ b/README.md @@ -1,20 +1,20 @@ # Ja3Requests -**Ja3Requests** is an http request library that can customize ja3 or h2 fingerprints. +**Ja3Requests** is a http request library that can customize ja3 or h2 fingerprints. [中文文档](README-zh.md) -```python +```pycon >>> import ja3requests >>> session = ja3requests.Session() ->>> response = session.get("http://www.baidu.com") +>>> response = session.get("http://www.baidu.com/") >>> response >>> response.status_code 200 >>> response.headers -[{'Bdqid': '0xdc8736c700095118'}, {'Connection': 'keep-alive'},...] +{'Content-Length': '405968', 'Content-Type': 'text/html; charset=utf-8', 'Server': 'BWS/1.1', 'Vary': 'Accept-Encoding', 'X-Ua-Compatible': 'IE=Edge,chrome=1', ...} >>> response.text ' + +# Or set cookies in headers = {"Cookies": "sessionId=xxxx; userId=xxxx;...."} + +response = session.get("http://example.com/", cookies=cookies) +print(response) +``` + + +### Allow Redirects + +```python +import ja3requests + +session = ja3requests.session() + +# Default allow_redirects=True +response = session.get("http://example.com/", allow_redirects=False) +print(response) +``` + + ## Reference - [HTTP](https://developer.mozilla.org/en-US/docs/Web/HTTP) - [HTTP-RFC](https://www.rfc-editor.org/rfc/rfc2068.html) \ No newline at end of file diff --git a/ja3requests/__init__.py b/ja3requests/__init__.py index aa2ff15..c9386de 100644 --- a/ja3requests/__init__.py +++ b/ja3requests/__init__.py @@ -1,3 +1,11 @@ +""" +Ja3Requests.__init__ +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Ja3Request +""" + + from .sessions import Session diff --git a/ja3requests/__version__.py b/ja3requests/__version__.py index 937d695..80f9b64 100644 --- a/ja3requests/__version__.py +++ b/ja3requests/__version__.py @@ -1,14 +1,14 @@ """ -ja3requests.__version__ +Ja3Requests.__version__ ~~~~~~~~~~~~~~~~~~~~~~~ Version information. """ __title__ = "ja3requests" -__description__ = "An http request library that can customize ja3 or h2 fingerprints." +__description__ = "A http request library that can customize JA3 or H2 fingerprints." __url__ = "https://github.com/lxjmaster/ja3requests" -__version__ = "1.0.2" +__version__ = "1.1.0" __author__ = "Mast Luo" __author_email__ = "379501669@qq.com" __license__ = "Apache-2.0 license" diff --git a/ja3requests/base/__contexts.py b/ja3requests/base/__contexts.py new file mode 100644 index 0000000..c9dcf9f --- /dev/null +++ b/ja3requests/base/__contexts.py @@ -0,0 +1,480 @@ +"""" +Ja3Requests.base.__contexts +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Basic of Context. +""" +from urllib.parse import urlparse, urlencode, parse_qsl +from abc import ABC, abstractmethod +from typing import AnyStr, Dict +from json import dumps +import mimetypes + + +class BaseContext(ABC): + """ + Basic connection context. + """ + + def __init__(self): + self._protocol = None + self._version = None + self._method = None + self._destination_address = None + self._path = None + self._port = None + self._headers = None + self._data = None + self._json = None + self._files = None + self._body = None + self._start_line = None + self._message = None + self._source_address = None + self._timeout = None + self._proxy = None + self._cookies = None + + @property + def protocol(self): + """ + Protocol + :return: + """ + return self._protocol + + @protocol.setter + def protocol(self, attr): + """ + Set protocol + :param attr: + :return: + """ + self._protocol = attr + + @property + def version(self): + """ + Version + :return: + """ + return self._version + + @version.setter + def version(self, attr): + """ + Set version + :param attr: + :return: + """ + self._version = attr + + @property + def method(self) -> AnyStr: + """ + Method + :return: + """ + return self._method + + @method.setter + def method(self, attr: AnyStr): + """ + Set method + :param attr: + :return: + """ + self._method = attr + + @property + def destination_address(self) -> AnyStr: + """ + Context property destination_address + :return: + """ + return self._destination_address + + @destination_address.setter + def destination_address(self, attr: AnyStr): + """ + Conntext property destination_address set + :param attr: + :return: + """ + self._destination_address = attr + + @property + def path(self) -> AnyStr: + """ + Context property path + :return: + """ + return self._path + + @path.setter + def path(self, attr: AnyStr): + """ + Context property path set + :param attr: + :return: + """ + self._path = attr + + @property + def port(self) -> int: + """ + Context property port + :return: + """ + return self._port + + @port.setter + def port(self, attr: int): + """ + Context property port set + :param attr: + :return: + """ + self._port = attr + + @property + def start_line(self) -> AnyStr: + """ + Start line + :return: + """ + return ( + self._start_line + if self._start_line + else " ".join([self.method, self.path, self.version]) + ) + + @start_line.setter + def start_line(self, attr: AnyStr): + """ + Set start line + :param attr: + :return: + """ + if attr: + parse = urlparse(attr) + self.destination_address = parse.hostname + self.path = parse.path + if self.path == "": + self.path = "/" + + if parse.query != "": + self.path += "?" + parse.query + + self._start_line = " ".join([self.method, self.path, self.version]) + + @property + def headers(self) -> Dict: + """ + Headers + :return: + """ + return self._headers + + @headers.setter + def headers(self, attr: Dict): + """ + Set headers + :param attr: + :return: + """ + headers = attr + if headers: + if not headers.get("Host", None): + if self.destination_address: + headers.update({"Host": self.destination_address}) + + if self.method in ["POST", "PUT"]: + if not headers.get("Content-Type", None): + if self.data: + headers.update( + {"Content-Type": "application/x-www-form-urlencoded"} + ) + + if self.json: + headers.update({"Content-Type": "application/json"}) + + if self.files: + headers.update( + {"Content-Type": 'multipart/form-data;boundary="boundary"'} + ) + else: + content_type = headers["Content-Type"] + if "multipart/form-data" in content_type.lower(): + headers.update( + {"Content-Type": 'multipart/form-data;boundary="boundary"'} + ) + + if not headers.get("Content-Type", None): + headers.update( + {"Content-Type": "application/x-www-form-urlencoded"} + ) + + self._headers = headers + + @property + def data(self) -> AnyStr: + """ + Context property data + :return: + """ + return self._data + + @data.setter + def data(self, attr): + """ + Context property data set + :param attr: + :return: + """ + data = attr + if isinstance(data, (dict, list, tuple)): + data = urlencode(data) + elif isinstance(data, bytes): + data = data.decode() + + self._data = data + + @property + def json(self) -> AnyStr: + """ + Context property json + :return: + """ + return self._json + + @json.setter + def json(self, attr): + """ + Context property json set + :param attr: + :return: + """ + json = attr + if isinstance(json, dict): + json = dumps(json) + elif isinstance(json, bytes): + json = json.decode() + + self._json = json + + @property + def files(self): + """ + Context property files + :return: + """ + return self._files + + @files.setter + def files(self, attr): + """ + Context property files set + :param attr: + :return: + """ + self._files = attr + + @property + def body(self) -> AnyStr: + """ + Body + :return: + """ + + return self._body + + @body.setter + def body(self, attr): + """ + Set body + :param attr: + :return: + """ + body = attr + if ( + self.headers.get("Content-Type", "") + == 'multipart/form-data;boundary="boundary"' + ): + body_list = parse_qsl(body) + form_data = "--boundary" + for name, value in body_list: + content = f'\r\nContent-Disposition: form-data; name="{name}"\r\n\r\n{value}\r\n--boundary' + form_data += content + + if self.files: + for name, file in self.files.items(): + for f in file: + mime_type, _ = mimetypes.guess_type(f["file_name"]) + file_name = f["file_name"] + content = f'\r\nContent-Disposition: form-data; name="{name}"; filename="{file_name}"'.encode() + if mime_type: + content += f"\r\nContent-Type: {mime_type}".encode() + + content += b"\r\n\r\n" + content += f["content"] + content += b"\r\n--boundary" + form_data = ( + form_data.encode() + if isinstance(form_data, str) + else form_data + ) + form_data += content + + form_data += b"--" + body = form_data + + self.headers.update({"Content-Length": len(body)}) + + self._body = body + + @property + def message(self) -> AnyStr: + """ + Message + :return: + """ + if self.data: + self.body = self.data + if self.json: + self.headers.update({"Content-Type": "application/json"}) + self.body = self.json.encode() + + message = b"" + if self._message: + message = self._message + else: + if self.start_line: + message += self.start_line.encode() + if self.headers: + message += b"\r\n" + message += "\r\n".join( + [f"{k}: {v}" for k, v in self.headers.items()] + ).encode() + + message += b"\r\n\r\n" + + if self.body: + message += self.body + + self._message = message + + return self._message + + @message.setter + def message(self, attr: AnyStr): + """ + Set message + :param attr: + :return: + """ + self._message = attr + + @property + def source_address(self): + """ + Context property source_address + :return: + """ + return self._source_address + + @source_address.setter + def source_address(self, attr): + """ + Context property source_address setter + :param attr: + :return: + """ + self._source_address = attr + + @property + def timeout(self): + """ + Context property timeout + :return: + """ + return self._timeout + + @timeout.setter + def timeout(self, attr): + """ + Context property timeout set + :param attr: + :return: + """ + self._timeout = attr + + @property + def proxy(self): + """ + Context property proxy + :return: + """ + proxy = None + if self._proxy: + if "@" in self._proxy: + proxy = self._proxy.split("@")[-1] + else: + proxy = self._proxy + + return proxy + + @proxy.setter + def proxy(self, attr): + """ + Context property proxy set + :param attr: + :return: + """ + self._proxy = attr + + @property + def proxy_auth(self): + """ + Context property proxy auth + :return: + """ + proxy_auth = None + if self._proxy: + if "@" in self._proxy: + proxy_auth = self._proxy.split("@")[0] + + return proxy_auth + + @property + def cookies(self): + """ + Context property cookies + :return: + """ + return self._cookies + + @cookies.setter + def cookies(self, attr: Dict): + """ + Context property cookies set + :param attr: + :return: + """ + cookies = attr + if isinstance(cookies, dict): + cookies_list = [f"{k}={v};" for k, v in cookies.items()] + self._cookies = " ".join(cookies_list) + else: + self._cookies = None + + if self._cookies: + self.headers.setdefault("Cookie", self._cookies) + + @abstractmethod + def set_payload(self, *args, **kwargs): + """ + Set context payload + :return: + """ + raise NotImplementedError("set_payload method must be implemented by subclass.") diff --git a/ja3requests/base/__init__.py b/ja3requests/base/__init__.py index fbda07e..92059b0 100644 --- a/ja3requests/base/__init__.py +++ b/ja3requests/base/__init__.py @@ -1,12 +1,13 @@ """ -ja3requests.base +Ja3Requests.base ~~~~~~~~~~~~~~~~ Basic module. """ -from ._context import BaseContext -from ._request import BaseRequest -from ._sessions import BaseSession -from ._response import BaseResponse -from ._connection import BaseHttpConnection + +from .__contexts import BaseContext +from .__requests import BaseRequest +from .__sockets import BaseSocket +from .__sessions import BaseSession +from .__response import BaseResponse diff --git a/ja3requests/base/__requests.py b/ja3requests/base/__requests.py new file mode 100644 index 0000000..1062dca --- /dev/null +++ b/ja3requests/base/__requests.py @@ -0,0 +1,414 @@ +""" +Ja3Requests.base.__requests +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Basic of Request. +""" + + +import os +from io import IOBase +from abc import ABC, abstractmethod +from http.cookiejar import CookieJar +from urllib.parse import urlparse, urlencode +from typing import Any, AnyStr, List, Dict, Tuple, Union +from ja3requests.const import DEFAULT_HTTP_SCHEME, DEFAULT_HTTP_PORT +from ja3requests.exceptions import InvalidParams, InvalidData +from ja3requests.utils import ( + default_headers, + dict_from_cookie_string, + dict_from_cookiejar, +) + + +class BaseRequest(ABC): + """ + Basic of Request + """ + + def __init__(self): + self._scheme = None + self._schema = None + self._port = None + self._method = None + self._url = None + self._params = None + self._data = None + self._files = None + self._headers = None + self._cookies = None + self._auth = None + self._json = None + self._proxy = None + self._timeout = None + + @property + def schema(self) -> AnyStr: + """ + Request property schema + :return: + """ + return self._schema + + @schema.setter + def schema(self, attr: AnyStr): + """ + Request property schema set + :param attr: + :return: + """ + self._schema = attr if attr else DEFAULT_HTTP_SCHEME + + @property + def port(self) -> int: + """ + Request property port + :return: + """ + return self._port + + @port.setter + def port(self, attr: int): + """ + Request property port set + :param attr: + :return: + """ + self._port = attr if attr else DEFAULT_HTTP_PORT + + @property + def method(self) -> AnyStr: + """ + Request property method + :return: + """ + return self._method + + @method.setter + def method(self, attr: AnyStr): + """ + Request property method set + :param attr: + :return: + """ + self._method = attr.upper() + + @property + def url(self) -> AnyStr: + """ + Request property url + :return: + """ + return self._url + + @url.setter + def url(self, attr: AnyStr): + """ " + Request property url set + """ + self._url = attr + if self._url: + parse = urlparse(self._url) + self.schema = parse.scheme + if self.schema == "https": + self.port = 443 + + if parse.netloc != "" and ":" in parse.netloc: + port = parse.netloc.split(":")[-1] + self.port = int(port) + else: + self.port = 80 + + @property + def params(self): + """ + Request property params + :return: + """ + return self._params + + @params.setter + def params( + self, + attr: Union[ + Dict[AnyStr, Any], List[Tuple[Any, Any]], Tuple[Tuple[Any, Any]], AnyStr + ], + ): + """ + Request property params set + :param attr: + :return: + """ + self._params = attr + if self._params: + if isinstance(self._params, str): + self._params = self._params + elif isinstance(self._params, bytes): + self._params = self._params.decode() + else: + try: + self._params = urlencode(self._params) + except TypeError as err: + raise InvalidParams(f"Invalid params: {self._params!r}") from err + + if self._params.startswith("?"): + self._params = self._params.replace("?", "") + + parse = urlparse(self.url) + + if parse.query != "": + self.url += "&" + self._params + else: + self.url += "?" + self._params + + @property + def data(self): + """ + Request property data + :return: + """ + return self._data + + @data.setter + def data( + self, + attr: Union[ + Dict[AnyStr, Any], + List[Tuple[AnyStr, Any]], + Tuple[Tuple[AnyStr, Any]], + AnyStr, + ], + ): + """ + Request property data set + :param attr: + :return: + """ + self._data = attr + if self._data: + if isinstance(self._data, str): + self._data = self._data + elif isinstance(self._data, bytes): + self._data = self._data.decode() + else: + try: + self._data = urlencode(self._data) + except TypeError as err: + raise InvalidData(f"Invalid data: {self._data!r}") from err + + if not self.headers: + self.headers = default_headers() + + content_type = self.headers.get("Content-Type", "") + if content_type == "": + self.headers["Content-Type"] = "application/x-www-form-urlencoded" + + self.headers["Content-Length"] = len(self._data) + + @property + def files(self): + """ + Request property files + :return: + """ + return self._files + + @files.setter + def files(self, attr): + """ + Request property files set + :param attr: + :return: + """ + new_files = None + files = attr + if files: + new_files = {} + for name, file in files.items(): + if not isinstance(file, list): + file = [file] + + for f in file: + if isinstance(f, (str, bytes)): + with open(f, "rb+") as f_obj: + item = { + "file_name": os.path.basename(f_obj.name), + "content": f_obj.read(), + } + elif isinstance(f, IOBase): + item = { + "file_name": os.path.basename( + f.name if hasattr(f, "name") else "" + ), + "content": f.read(), + } + else: + continue + + if not new_files.get(name, None): + new_files.update({name: [item]}) + else: + new_files[name].append(item) + + self._files = new_files + + @property + def headers(self): + """ + Request property headers + :return: + """ + return self._headers if self._headers else default_headers() + + @headers.setter + def headers(self, attr: Dict[AnyStr, AnyStr]): + """ + Request property headers set + :param attr: + :return: + """ + self._headers = attr + if not self._headers: + self._headers = default_headers() + + headers = {} + for header, value in self._headers.items(): + header = header.title() + headers.update({header: value}) + + self._headers = headers + + @property + def cookies(self) -> Dict | None: + """ + Request property cookies + :return: + """ + return self._cookies + + @cookies.setter + def cookies(self, attr: Union[Dict[AnyStr, AnyStr], CookieJar, AnyStr]): + """ + Request property cookies set + :param attr: + :return: + """ + cookies = attr + if cookies: + if isinstance(cookies, (bytes, str)): + cookies = dict_from_cookie_string(cookies) + elif isinstance(cookies, CookieJar): + cookies = dict_from_cookiejar(cookies) + + self._cookies = cookies + + @property + def auth(self): + """ + Request property auth + :return: + """ + return self._auth + + @auth.setter + def auth(self, attr: Tuple): + """ + Request property auth set + :param attr: + :return: + """ + self._auth = attr + + @property + def json(self): + """ + Request property json + :return: + """ + return self._json + + @json.setter + def json(self, attr: Dict[AnyStr, AnyStr]): + """ + Request property json set + :param attr: + :return: + """ + self._json = attr + + @property + def proxy(self): + """ + Request property proxy + :return: + """ + return self._proxy + + @proxy.setter + def proxy(self, attr): + """ + Request property proxy set + :param attr: + :return: + """ + self._proxy = attr + if self._proxy: + proxy = self._proxy.get(self.schema, None) + else: + proxy = None + + self._proxy = proxy + + @property + def timeout(self): + """ + Request property timeout + :return: + """ + return self._timeout + + @timeout.setter + def timeout(self, attr): + """ + Request property timeout set + :param attr: + :return: + """ + self._timeout = attr + + def set_payload(self, **kwargs): + """ + Set request payload + :param kwargs: + :return: + """ + for k, v in kwargs.items(): + setattr(self, k, v) + + def is_http(self): + """ + Is http + :return: + """ + self.schema = ( + self.schema.decode() if isinstance(self.schema, bytes) else self.schema + ) + return self.schema.lower() == "http" + + def is_https(self): + """ + Is https + :return: + """ + self.schema = ( + self.schema.decode() if isinstance(self.schema, bytes) else self.schema + ) + return self.schema.lower() == "https" + + @abstractmethod + def send(self): + """ + Request send + :return: + """ + raise NotImplementedError("send method must be implemented by subclass.") diff --git a/ja3requests/base/_response.py b/ja3requests/base/__response.py similarity index 98% rename from ja3requests/base/_response.py rename to ja3requests/base/__response.py index 9b7917e..fc67d77 100644 --- a/ja3requests/base/_response.py +++ b/ja3requests/base/__response.py @@ -1,8 +1,8 @@ """ -ja3Requests.base._response +Ja3Requests.base.__response ~~~~~~~~~~~~~~~~~~~~~~~~~~ -Basic Response +Basic of Response. """ diff --git a/ja3requests/base/_sessions.py b/ja3requests/base/__sessions.py similarity index 68% rename from ja3requests/base/_sessions.py rename to ja3requests/base/__sessions.py index ef09db9..acdb8b8 100644 --- a/ja3requests/base/_sessions.py +++ b/ja3requests/base/__sessions.py @@ -1,17 +1,24 @@ """ -ja3Requests.base._sessions +Ja3Requests.base.__sessions ~~~~~~~~~~~~~~~~~~~~~~~~~~ -Basic Session +Basic of Session. """ +from ja3requests.const import DEFAULT_REDIRECT_LIMIT +from ja3requests.cookies import Ja3RequestsCookieJar, CookieJar +from ja3requests.utils import dict_from_cookie_string, add_dict_to_cookiejar + + class BaseSession: """ The basic request session. """ def __init__(self): + self._request = None + self._response = None self._headers = None self._cookies = None self._auth = None @@ -24,6 +31,30 @@ def __init__(self): self._h2_window_update = None self._h2_headers = None + @property + def Request(self): + """ + Session property Request + :return: + """ + return self._request + + @Request.setter + def Request(self, attr): + self._request = attr + + @property + def response(self): + """ + Session property response + :return: + """ + return self._response + + @response.setter + def response(self, attr): + self._response = attr + @property def headers(self): """Headers @@ -31,6 +62,10 @@ def headers(self): >>> {'Accept': '*/*', 'Accept-Encoding': 'gzip,deflate'} :return: """ + if not self._headers: + if self.Request: + self._headers = self.Request.headers + return self._headers @headers.setter @@ -42,14 +77,42 @@ def headers(self, attr): """ self._headers = attr + @staticmethod + def resolve_cookies(cj: Ja3RequestsCookieJar, cookie): + """ + Collection session cookies + :param cj: + :param cookie: + :return: + """ + if isinstance(cookie, (bytes, str)): + cookies_dict = dict_from_cookie_string(cookie) + cj = add_dict_to_cookiejar(cj, cookies_dict) + elif isinstance(cookie, dict): + cj = add_dict_to_cookiejar(cj, cookie) + elif isinstance(cookie, CookieJar): + cj.update(cookie) + + return cj + @property def cookies(self): """Cookies Http cookies. - >>> CookieJar({}) + >>> :return: """ - return self._cookies + cookies = Ja3RequestsCookieJar() + if self._cookies: + cookies = self.resolve_cookies(cookies, self._cookies) + + if self.Request.cookies: + cookies = self.resolve_cookies(cookies, self.Request.cookies) + + if self.response.cookies: + cookies = self.resolve_cookies(cookies, self.response.cookies) + + return cookies @cookies.setter def cookies(self, attr): @@ -58,7 +121,7 @@ def cookies(self, attr): :param attr: :return: """ - self.cookies = attr + self._cookies = attr @property def auth(self): @@ -66,6 +129,10 @@ def auth(self): >>> {'user': 'xxx', 'password': 'xxx'} :return: """ + if not self._auth: + if self.Request: + self._auth = self.Request.auth + return self._auth @auth.setter @@ -84,7 +151,11 @@ def proxies(self): >>> {'http': 'user:password@host:port', 'https': 'user:password@host:port'} :return: """ - return self._cookies + if not self._proxies: + if self.Request: + self._proxies = self.Request.proxies + + return self._proxies @proxies.setter def proxies(self, attr): @@ -102,6 +173,10 @@ def params(self): >>> {'page': 1, 'per_page': 10} :return: """ + if not self._params: + if self.Request: + self._params = self.Request.params + return self._params @params.setter @@ -120,6 +195,13 @@ def max_redirects(self): >>> 5 :return: """ + if not self._max_redirects: + if self.Request: + self._max_redirects = self.Request.max_redirects + + if not self._max_redirects: + self._max_redirects = DEFAULT_REDIRECT_LIMIT + return self._max_redirects @max_redirects.setter @@ -138,6 +220,13 @@ def allow_redirect(self): >>> True or False. :return: """ + if not self._allow_redirect: + if self.Request: + self._allow_redirect = self.Request.allow_redirect + + if not self._allow_redirect: + self._allow_redirect = True + return self._allow_redirect @allow_redirect.setter diff --git a/ja3requests/base/__sockets.py b/ja3requests/base/__sockets.py new file mode 100644 index 0000000..ebea015 --- /dev/null +++ b/ja3requests/base/__sockets.py @@ -0,0 +1,68 @@ +""" +Ja3Requests.base.__sockets +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Basic of Socket. +""" + +import socket +from abc import ABC, abstractmethod +from ja3requests.base.__contexts import BaseContext +from ja3requests.protocol.sockets import create_connection +from ja3requests.protocol.exceptions import ( + SocketTimeout, + ConnectTimeoutError, +) +from ja3requests.utils import Retry + + +class BaseSocket(ABC): + """ + Basic socket class + """ + + def __init__(self, context: BaseContext): + self.context = context + self._conn = None + + @property + def conn(self): + """ + Socket property conn + :return: + """ + return self._conn + + @conn.setter + def conn(self, attr): + """ + Socket property conn set + :param attr: + :return: + """ + self._conn = attr + + @abstractmethod + def new_conn(self): + """ + New socket connection + :return: + """ + raise NotImplementedError("new_conn method must be implemented by subclass.") + + def _new_conn(self, dest_address, port): + try: + retry = Retry() + conn = retry.do( + create_connection, + socket.error, + (dest_address, port), + self.context.timeout, + self.context.source_address, + ) + except SocketTimeout as err: + raise ConnectTimeoutError( + f"Connection to {dest_address}:{port} timeout out." + ) from err + + return conn diff --git a/ja3requests/base/_connection.py b/ja3requests/base/_connection.py deleted file mode 100644 index 16f5238..0000000 --- a/ja3requests/base/_connection.py +++ /dev/null @@ -1,230 +0,0 @@ -""" -ja3Requests.base._connection -~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Basic HTTP Connection -""" - - -class BaseHttpConnection: - """ - Basic HTTP Connection - """ - - def __init__(self): - self._scheme = None - self._host = None - self._port = None - self._source_address = None - self._destination_address = None - self._path = None - self._timeout = None - self._proxy = None - self._proxy_username = None - self._proxy_password = None - self._connection = None - self._is_close = None - - @property - def scheme(self): - """ - Scheme - :return: - """ - return self._scheme - - @scheme.setter - def scheme(self, attr): - """ - Set Scheme - :param attr: - :return: - """ - self._scheme = attr - - @property - def host(self): - """ - Host - :return: - """ - return self._host - - @host.setter - def host(self, attr): - """ - Set Host - :param attr: - :return: - """ - self._host = attr - - @property - def port(self): - """ - Port - :return: - """ - return self._port - - @port.setter - def port(self, attr): - """ - Set Port - :param attr: - :return: - """ - self._port = attr - - @property - def source_address(self): - """ - Source Address - :return: - """ - return self._source_address - - @source_address.setter - def source_address(self, attr): - """ - Set Source Address - :param attr: - :return: - """ - self._source_address = attr - - @property - def destination_address(self): - """ - Destination Address - :return: - """ - return self._destination_address - - @destination_address.setter - def destination_address(self, attr): - """ - Set Destination Address - :param attr: - :return: - """ - self._destination_address = attr - - @property - def path(self): - """ - Path - :return: - """ - return self._path - - @path.setter - def path(self, attr): - """ - Set Path - :param attr: - :return: - """ - self._path = attr - - @property - def timeout(self): - """ - Timeout - :return: - """ - return self._timeout - - @timeout.setter - def timeout(self, attr): - """ - Set Timeout - :param attr: - :return: - """ - self._timeout = attr - - @property - def proxy(self): - """ - Proxy - :return: - """ - return self._proxy - - @proxy.setter - def proxy(self, attr): - """ - Set Proxy - :param attr: - :return: - """ - self._proxy = attr - - @property - def proxy_username(self): - """ - Proxy username - :return: - """ - return self._proxy_username - - @proxy_username.setter - def proxy_username(self, attr): - """ - Set Proxy Username - :param attr: - :return: - """ - self._proxy_username = attr - - @property - def proxy_password(self): - """ - Proxy Password - :return: - """ - return self._proxy_password - - @proxy_password.setter - def proxy_password(self, attr): - """ - Set Proxy Password - :param attr: - :return: - """ - self._proxy_password = attr - - @property - def connection(self): - """ - Connection - :return: - """ - return self._connection - - @connection.setter - def connection(self, attr): - """ - Set Connection - :param attr: - :return: - """ - self._connection = attr - - @property - def is_close(self): - """ - Connection is closed - :return: - """ - return self._is_close - - @is_close.setter - def is_close(self, attr): - """ - Set connection close - :param attr: - :return: - """ - self._is_close = attr diff --git a/ja3requests/base/_context.py b/ja3requests/base/_context.py deleted file mode 100644 index 6660d05..0000000 --- a/ja3requests/base/_context.py +++ /dev/null @@ -1,140 +0,0 @@ -"""" -ja3Requests.base._context -~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Basic Context -""" - - -class BaseContext: - """ - Basic connection context. - """ - - def __init__(self): - self._protocol = None - self._version = None - self._start_line = None - self._method = None - self._headers = None - self._body = None - self._message = None - - @property - def protocol(self): - """ - Protocol - :return: - """ - return self._protocol - - @protocol.setter - def protocol(self, attr): - """ - Set protocol - :param attr: - :return: - """ - self._protocol = attr - - @property - def version(self): - """ - Version - :return: - """ - return self._version - - @version.setter - def version(self, attr): - """ - Set version - :param attr: - :return: - """ - self._version = attr - - @property - def start_line(self): - """ - Start line - :return: - """ - return self._start_line - - @start_line.setter - def start_line(self, attr): - """ - Set start line - :param attr: - :return: - """ - self._start_line = attr - - @property - def method(self): - """ - Method - :return: - """ - return self._method - - @method.setter - def method(self, attr): - """ - Set method - :param attr: - :return: - """ - self._method = attr - - @property - def headers(self): - """ - Headers - :return: - """ - return self._headers - - @headers.setter - def headers(self, attr): - """ - Set headers - :param attr: - :return: - """ - self._headers = attr - - @property - def body(self): - """ - Body - :return: - """ - return self._body - - @body.setter - def body(self, attr): - """ - Set body - :param attr: - :return: - """ - self._body = attr - - @property - def message(self): - """ - Message - :return: - """ - return self._message - - @message.setter - def message(self, attr): - """ - Set message - :param attr: - :return: - """ - self._message = attr diff --git a/ja3requests/base/_request.py b/ja3requests/base/_request.py deleted file mode 100644 index 4000390..0000000 --- a/ja3requests/base/_request.py +++ /dev/null @@ -1,287 +0,0 @@ -""" -ja3Requests.base._request -~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Basic Request -""" - - -class BaseRequest: - """ - The basic request. - """ - - def __init__(self): - self._method = None - self._source = None - self._url = None - self._scheme = None - self._port = None - self._headers = None - self._params = None - self._data = None - self._cookies = None - self._files = None - self._auth = None - self._json = None - self._timeout = None - self._proxies = None - - @property - def method(self): - """ - Request method - >>> "GET" - :return: - """ - return self._method - - @method.setter - def method(self, attr): - """ - Set request method. - :param attr: - :return: - """ - self._method = attr - - @property - def source(self): - """ - Source Address. - :return: - """ - return self._source - - @source.setter - def source(self, attr): - """ - Set source address. - :param attr: - :return: - """ - self._source = attr - - @property - def url(self): - """ - Request url. - :return: - """ - return self._url - - @url.setter - def url(self, attr): - """ - Set request url. - :param attr: - :return: - """ - self._url = attr - - @property - def scheme(self): - """ - Request Scheme. eg. HTTP, HTTPS - :return: - """ - return self._scheme - - @scheme.setter - def scheme(self, attr): - """ - Set scheme. - :param attr: - :return: - """ - self._scheme = attr - - @property - def port(self): - """ - Remote address port. - :return: - """ - return self._port - - @port.setter - def port(self, attr): - """ - Set port. - :param attr: - :return: - """ - self._port = attr - - @property - def headers(self): - """Headers - Request headers. - >>> {"Host": "www.example.com", "Accept": "*/*"} - :return: - """ - return self._headers - - @headers.setter - def headers(self, attr): - """ - Set request headers. - :param attr: - :return: - """ - self._headers = attr - - @property - def params(self): - """ - Request params. eg. ?page=1&page_size=10&desc=1 - >>> [("page", 1), ("page_size", 10),] - :return: - """ - return self._params - - @params.setter - def params(self, attr): - """ - Set params. - :param attr: - :return: - """ - self._params = attr - - @property - def data(self): - """ - Post request data. - >>> {"username": "admin", "password": "admin"} - :return: - """ - return self._data - - @data.setter - def data(self, attr): - """ - Set post request data. - :param attr: - :return: - """ - self._data = attr - - @property - def cookies(self): - """ - Request cookies. - >>> {"UUID": "xxxxxxx"} - :return: - """ - return self._cookies - - @cookies.setter - def cookies(self, attr): - """ - Set request cookies. - :param attr: - :return: - """ - self._cookies = attr - - @property - def files(self): - """ - Request files. - :return: - """ - return self._files - - @files.setter - def files(self, attr): - """ - Set files. - :param attr: - :return: - """ - self._files = attr - - @property - def auth(self): - """ - Request Authorization. - >>> {"username": "admin", "password": "admin"} - :return: - """ - return self._auth - - @auth.setter - def auth(self, attr): - """ - Set authorization. - :param attr: - :return: - """ - self._auth = attr - - @property - def json(self): - """ - Post json. - :return: - """ - return self._json - - @json.setter - def json(self, attr): - """ - Set json for post request. - :param attr: - :return: - """ - self._json = attr - - @property - def timeout(self): - """ - Request timeout. - :return: - """ - return self._timeout - - @timeout.setter - def timeout(self, attr): - """ - Set request timeout. - :param attr: - :return: - """ - self._timeout = attr - - @property - def proxies(self): - """ - Request proxies. - >>> {"http": "username:password@host:port", "https": "username:password@host:port"} - :return: - """ - return self._proxies - - @proxies.setter - def proxies(self, attr): - """ - Set proxies. - :param attr: - :return: - """ - self._proxies = attr - - def is_http(self): - """ - Is http request. - :return: - """ - return self._scheme == "http" - - def is_https(self): - """ - Is https request. - :return: - """ - return self._scheme == "https" diff --git a/ja3requests/connections.py b/ja3requests/connections.py deleted file mode 100644 index 740443d..0000000 --- a/ja3requests/connections.py +++ /dev/null @@ -1,149 +0,0 @@ -""" -ja3requests.connections -~~~~~~~~~~~~~~~~~~~~~~~ - -This module contains HTTP connection and HTTPS connection. -""" - - -from .response import HTTPResponse -from .exceptions import InvalidHost -from .base import BaseHttpConnection -from .protocol.sockets import create_connection -from .const import DEFAULT_HTTP_SCHEME -from .const import DEFAULT_HTTP_PORT -from .protocol.exceptions import SocketTimeout, ConnectTimeoutError - - -class HTTPConnection(BaseHttpConnection): - """ - HTTP connection. - """ - - def __init__(self): - super().__init__() - self.scheme = DEFAULT_HTTP_SCHEME - self.port = DEFAULT_HTTP_PORT - self.is_close = False - - def __del__(self): - self.close() - - def _new_conn(self): - """ - Establish a socket connection - :return: socket connection - """ - - try: - conn = create_connection( - (self.destination_address, self.port), - self.timeout, - self.source_address, - ) - except SocketTimeout as err: - raise ConnectTimeoutError( - f"Connection to {self.destination_address} timeout out. timeout={self.timeout}" - ) from err - - return conn - - def _ready_connect(self, **kwargs): - """ - Ready http connection. - :param kwargs: - :return: - """ - self.scheme = kwargs["scheme"] if kwargs.get("scheme", None) else self.scheme - self.port = kwargs["port"] if kwargs.get("port", None) else self.port - self.source_address = ( - kwargs["source_address"] - if kwargs.get("source_address", None) - else self.source_address - ) - self.timeout = ( - kwargs["timeout"] if kwargs.get("timeout", None) else self.timeout - ) - self.proxy = kwargs["proxy"] if kwargs.get("proxy", None) else self.proxy - self.proxy_username = ( - kwargs["proxy_username"] - if kwargs.get("proxy_username", None) - else self.proxy_username - ) - self.proxy_password = ( - kwargs["proxy_password"] - if kwargs.get("proxy_password", None) - else self.proxy_password - ) - - if kwargs.get("host", None): - host = kwargs["host"].replace("http://", "").split("/") - if len(host) > 0: - self.host = host[0] - self.path = "/" + "/".join(host[1:]) - if ":" in self.host: - self.destination_address = self.host.split(":")[0] - if self.port is None: - self.port = self.host.split(":")[1] - else: - self.destination_address = self.host - else: - raise InvalidHost( - f"Invalid Host: {kwargs['host']!r}, can not parse destination address or path." - ) - - def connect( - self, - scheme=None, - port=None, - source_address=None, - host=None, - timeout=None, - proxy=None, - proxy_username=None, - proxy_password=None, - ): - """ - Create an http connection. - :param scheme: - :param port: - :param source_address: - :param host: - :param timeout: - :param proxy: - :param proxy_username: - :param proxy_password: - :return: - """ - self._ready_connect( - scheme=scheme, - port=port, - source_address=source_address, - host=host, - timeout=timeout, - proxy=proxy, - proxy_username=proxy_username, - proxy_password=proxy_password, - ) - conn = self._new_conn() - self.connection = conn - - def send(self, context): - """ - Send socket. - :return: - """ - self.connection.sendall(context.message) - - response = HTTPResponse(sock=self.connection, method=context.method) - response.begin() - - return response - - def close(self): - """ - Close connection. - :return: - """ - if self.connection: - self.connection.close() diff --git a/ja3requests/const.py b/ja3requests/const.py index 1fef383..a6c3833 100644 --- a/ja3requests/const.py +++ b/ja3requests/const.py @@ -1,43 +1,17 @@ """ -ja3requests.const +Ja3Requests.const ~~~~~~~~~~~~~~~~~ A constant module. """ -import sys - -class _Const: - class ConstError(TypeError): - """ - Const Error - """ - - class ConstCaseError(ConstError): - """ - Const Case Error - """ - - def __setattr__(self, key, value): - if self.__dict__.get(key) is not None: - raise self.ConstError(f"The constant {key} already exists") - - if not key.isupper(): - raise self.ConstCaseError(f"{key}-constants need to be capitalized.") - - self.__dict__[key] = value - - -const = _Const() - -const.MAX_LINE = 65536 -const.MAX_HEADERS = 100 -const.DEFAULT_CHUNKED_SIZE = 2048 -const.DEFAULT_HTTP_SCHEME = "http" -const.DEFAULT_HTTPS_SCHEME = "https" -const.DEFAULT_HTTP_PORT = 80 -const.DEFAULT_HTTPS_PORT = 443 -const.DEFAULT_REDIRECT_LIMIT = 8 # max redirect - -sys.modules[__name__] = const +MAX_LINE = 65536 +MAX_HEADERS = 100 +DEFAULT_CHUNKED_SIZE = 2048 +DEFAULT_HTTP_SCHEME = "http" +DEFAULT_HTTPS_SCHEME = "https" +DEFAULT_HTTP_PORT = 80 +DEFAULT_HTTPS_PORT = 443 +DEFAULT_REDIRECT_LIMIT = 8 # max redirect +DEFAULT_MAX_RETRY_LIMIT = 3 diff --git a/ja3requests/context.py b/ja3requests/context.py deleted file mode 100644 index 4e5444a..0000000 --- a/ja3requests/context.py +++ /dev/null @@ -1,66 +0,0 @@ -""" -ja3requests.context -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -HTTP Context and HTTPS Context -""" - - -from .base import BaseContext - - -DEFAULT_HTTP_CONTEXT_PROTOCOL = 11 -DEFAULT_HTTP_VERSION = "HTTP/1.1" - - -class HTTPContext(BaseContext): - """ - HTTPContext - """ - - def __init__(self, connection): - super().__init__() - self.protocol = DEFAULT_HTTP_CONTEXT_PROTOCOL - self.version = DEFAULT_HTTP_VERSION - self.connection = connection - - @property - def message(self): - """ - HTTP Context message to send - :return: - """ - self.start_line = " ".join([self.method, self.connection.path, self.version]) - self._message = "\r\n".join([self.start_line, self.put_headers()]) - self._message += "\r\n\r\n" - - if self.body: - self._message += self.body - - print(self._message) - - return self._message.encode() - - def set_payload(self, **kwargs): - """ - Set context payload - :param kwargs: - :return: - """ - for k, v in kwargs.items(): - if hasattr(self, k): - setattr(self, k, v) - - def put_headers(self): - """ - Set context headers - :return: - """ - headers = "" - if self.headers is not None: - if not self.headers.get("host", None): - self.headers["host"] = self.connection.host - - headers = "\r\n".join([f"{k}: {v}" for k, v in self.headers.items()]) - - return headers diff --git a/ja3requests/contexts/__init__.py b/ja3requests/contexts/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/ja3requests/contexts/context.py b/ja3requests/contexts/context.py new file mode 100644 index 0000000..a99131b --- /dev/null +++ b/ja3requests/contexts/context.py @@ -0,0 +1,41 @@ +""" +Ja3Requests.contexts.context +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +HTTP Context and HTTPS Context +""" + + +from ja3requests.base import BaseContext + + +DEFAULT_HTTP_CONTEXT_PROTOCOL = 11 +DEFAULT_HTTP_VERSION = "HTTP/1.1" + + +class HTTPContext(BaseContext): + """ + HTTPContext + """ + + def __init__(self): + super().__init__() + self.protocol = DEFAULT_HTTP_CONTEXT_PROTOCOL + self.version = DEFAULT_HTTP_VERSION + + def set_payload(self, **kwargs): + """ + Set context payload + :return: + """ + for k, v in kwargs.items(): + setattr(self, k, v) + + +class HTTPSContext(BaseContext): + """ + HTTPS Context + """ + + def set_payload(self, request): + pass diff --git a/ja3requests/cookies.py b/ja3requests/cookies.py new file mode 100644 index 0000000..664a150 --- /dev/null +++ b/ja3requests/cookies.py @@ -0,0 +1,643 @@ +""" +Ja3Requests.cookies +~~~~~~~~~~~~~~~~~~~ + +This module contains Request or Response Cookies. +""" + + +from http.cookiejar import CookieJar, Cookie +from http import cookies +from typing import MutableMapping +from urllib.parse import urlparse, urlunparse +import threading +import calendar +import copy +import time + + +def to_native_string(string, encoding="ascii"): + """Given a string object, regardless of type, returns a representation of + that string in the native string type, encoding and decoding where + necessary. This assumes ASCII unless told otherwise. + """ + if isinstance(string, str): + out = string + else: + out = string.decode(encoding) + + return out + + +class MockRequest: + """Wraps a `Ja3Requests.Request` to mimic a `urllib2.Request`. + + The code in `CookieJar` expects this interface in order to correctly + manage cookie policies, i.e., determine whether a cookie can be set, given the + domains of the request and the cookie. + + The original request object is read-only. The client is responsible for collecting + the new headers via `get_new_headers()` and interpreting them appropriately. You + probably want `get_cookie_header`, defined below. + """ + + def __init__(self, request): + self._r = request + self._new_headers = {} + self.type = urlparse(self._r.url).scheme + + def get_type(self): + """ + Get schema type + :return: + """ + return self.type + + def get_host(self): + """ + Get host + :return: + """ + return urlparse(self._r.url).netloc + + def get_origin_req_host(self): + """ + Get host + :return: + """ + return self.get_host() + + def get_full_url(self): + """Only return the response's URL if the user hadn't set the Host""" + + # header + if not self._r.headers.get("Host"): + return self._r.url + # If they did set it, retrieve it and reconstruct the expected domain + host = to_native_string(self._r.headers["Host"], encoding="utf-8") + parsed = urlparse(self._r.url) + # Reconstruct the URL as we expect it + return urlunparse( + [ + parsed.scheme, + host, + parsed.path, + parsed.params, + parsed.query, + parsed.fragment, + ] + ) + + @staticmethod + def is_unverifiable(): + """ + Check unverifiable + :return: + """ + return True + + def has_header(self, name): + """ + Check header + :param name: + :return: + """ + return name in self._r.headers or name in self._new_headers + + def get_header(self, name, default=None): + """ + Get header + :param name: + :param default: + :return: + """ + return self._r.headers.get(name, self._new_headers.get(name, default)) + + def add_header(self, key, val): + """cookie has no legitimate use for this method; add it back if you find one.""" + raise NotImplementedError( + "Cookie headers should be added with add_unredirected_header()" + ) + + def add_unredirected_header(self, name, value): + """ + Add unredirected header + :param name: + :param value: + :return: + """ + self._new_headers[name] = value + + def get_new_headers(self): + """ + Get new headers + :return: + """ + return self._new_headers + + @property + def unverifiable(self): + """ + Get unverifiable + :return: + """ + return self.is_unverifiable() + + @property + def origin_req_host(self): + """ + Get host + :return: + """ + return self.get_origin_req_host() + + @property + def host(self): + """ + Host + :return: + """ + return self.get_host() + + +class MockResponse: + """ + Basically, expose the parsed HTTP headers from the server response + the way `cookie` expects to see them. + """ + + def __init__(self, response): + """Make a MockResponse for `cookie` to read. + + :param response: Response + """ + self.response = response + + def info(self): + """ + Info method of Response + :return: + """ + return self.response.headers + + def getheaders(self, name): + """ + Get header + :param name: + :return: + """ + self.response.headers.get(name) + + +def extract_cookies_to_jar(jar, request, response): + """Extract the cookies from the response into a CookieJar. + + :param jar: CookieJar (not necessarily a RequestsCookieJar) + :param request: our own Ja3Requests.Request object + :param response: our own Ja3Requests.Response object + """ + + req = MockRequest(request) + res = MockResponse(response) + + jar.extract_cookies(res, req) + + +def get_cookie_header(jar, request): + """ + Produce an appropriate Cookie header string to be sent with `request`, or None. + + :rtype: str + """ + r = MockRequest(request) + jar.add_cookie_header(r) + return r.get_new_headers().get("Cookie") + + +def remove_cookie_by_name(cookiejar, name, domain=None, path=None): + """Unsets a cookie by name, by default over all domains and paths. + + Wraps CookieJar.clear(), is O(n). + """ + clear_ables = [] + for cookie in cookiejar: + if cookie.name != name: + continue + if domain is not None and domain != cookie.domain: + continue + if path is not None and path != cookie.path: + continue + clear_ables.append((cookie.domain, cookie.path, cookie.name)) + + for _domain, _path, _name in clear_ables: + cookiejar.clear(_domain, _path, _name) + + +class CookieConflictError(RuntimeError): + """There are two cookies that meet the criteria specified in the cookie jar. + Use .get and .set and include domain and path args in order to be more specific. + """ + + +class Ja3RequestsCookieJar(CookieJar, MutableMapping): + """Compatibility class; is a CookieJar, but exposes a dict + interface. + + This is the CookieJar we create by default for requests and sessions that + don't specify one, since some clients may expect `response.cookies` and + `session.cookies` to support dict operations. + + Requests does not use the dict interface internally; it's just for + compatibility with external client code. All requests code should work + out of the box with externally provided instances of ``CookieJar``, e.g. + ``LWPCookieJar`` and ``FileCookieJar``. + + Unlike a regular CookieJar, this class is pickleable. + """ + + def get(self, key, default=None, domain=None, path=None): + """Dict-like get() that also supports optional domain and path args in + order to resolve naming collisions from using one cookie jar over + multiple domains. + + .. warning:: operation is O(n), not O(1). + """ + try: + return self._find_no_duplicates(key, domain, path) + except KeyError: + return default + + def set(self, name, value, **kwargs): + """Dict-like set() that also supports optional domain and path args in + order to resolve naming collisions from using one cookie jar over + multiple domains. + """ + # support client code that unsets cookies by assignment of a None value: + if value is None: + remove_cookie_by_name( + self, name, domain=kwargs.get("domain"), path=kwargs.get("path") + ) + return None + + if isinstance(value, cookies.Morsel): + c = morsel_to_cookie(value) + else: + c = create_cookie(name, value, **kwargs) + + self.set_cookie(c) + + return c + + def iterkeys(self): + """Dict-like iterkeys() that returns an iterator of names of cookies + from the jar. + + .. see also:: itervalues() and iteritems(). + """ + for cookie in iter(self): + yield cookie.name + + def keys(self): + """Dict-like keys() that returns a list of names of cookies from the + jar. + + .. see also:: values() and items(). + """ + return list(self.iterkeys()) + + def itervalues(self): + """Dict-like itervalues() that returns an iterator of values of cookies + from the jar. + + .. see also:: iterkeys() and iteritems(). + """ + for cookie in iter(self): + yield cookie.value + + def values(self): + """Dict-like values() that returns a list of values of cookies from the + jar. + + .. see also:: keys() and items(). + """ + return list(self.itervalues()) + + def iteritems(self): + """Dict-like iteritems() that returns an iterator of name-value tuples + from the jar. + + .. see also:: iterkeys() and itervalues(). + """ + for cookie in iter(self): + yield cookie.name, cookie.value + + def items(self): + """Dict-like items() that returns a list of name-value tuples from the + jar. Allows client-code to call ``dict(Ja3RequestsCookieJar)`` and get a + vanilla python dict of key value pairs. + + .. see also:: keys() and values(). + """ + return list(self.iteritems()) + + def list_domains(self): + """Utility method to list all the domains in the jar.""" + domains = [] + for cookie in iter(self): + if cookie.domain not in domains: + domains.append(cookie.domain) + return domains + + def list_paths(self): + """Utility method to list all the paths in the jar.""" + paths = [] + for cookie in iter(self): + if cookie.path not in paths: + paths.append(cookie.path) + return paths + + def multiple_domains(self): + """Returns True if there are multiple domains in the jar. + Returns False otherwise. + + :rtype: bool + """ + domains = [] + for cookie in iter(self): + if cookie.domain is not None and cookie.domain in domains: + return True + domains.append(cookie.domain) + return False # there is only one domain in jar + + def get_dict(self, domain=None, path=None): + """Takes as an argument an optional domain and path and returns a plain + old Python dict of name-value pairs of cookies that meet the + requirements. + + :rtype: dict + """ + dictionary = {} + for cookie in iter(self): + if (domain is None or cookie.domain == domain) and ( + path is None or cookie.path == path + ): + dictionary[cookie.name] = cookie.value + return dictionary + + def __contains__(self, name): + """ + Class builtin method + :param name: + :return: + """ + try: + return super().__contains__(name) + except CookieConflictError: + return True + + def __getitem__(self, name): + """Dict-like __getitem__() for compatibility with client code. Throws + exception if there are more than one cookie with name. In that case, + use the more explicit get() method instead. + + .. warning:: operation is O(n), not O(1). + """ + return self._find_no_duplicates(name) + + def __setitem__(self, name, value): + """Dict-like __setitem__ for compatibility with client code. Throws + exception if there is already a cookie of that name in the jar. In that + case, use the more explicit set() method instead. + """ + self.set(name, value) + + def __delitem__(self, name): + """Deletes a cookie given a name. Wraps ``CookieJar``'s + ``remove_cookie_by_name()``. + """ + remove_cookie_by_name(self, name) + + def set_cookie(self, cookie): + """ + Set cookie + :param cookie: + :return: + """ + if ( + hasattr(cookie.value, "startswith") + and cookie.value.startswith('"') + and cookie.value.endswith('"') + ): + cookie.value = cookie.value.replace('\\"', "") + return super().set_cookie(cookie) + + def update(self, other): + """Updates this jar with cookies from another CookieJar or dict-like""" + if isinstance(other, CookieJar): + for cookie in other: + self.set_cookie(copy.copy(cookie)) + else: + super().update(other) + + def _find(self, name, domain=None, path=None): + """Requests uses this method internally to get cookie values. + + If there are conflicting cookies, _find arbitrarily chooses one. + See _find_no_duplicates if you want an exception thrown if there are + conflicting cookies. + + :param name: a string containing name of cookie + :param domain: (optional) string containing domain of cookie + :param path: (optional) string containing path of cookie + :return: .Value + """ + for cookie in iter(self): + if cookie.name == name: + if domain is None or cookie.domain == domain: + if path is None or cookie.path == path: + return cookie.value + + raise KeyError(f"name={name!r}, domain={domain!r}, path={path!r}") + + def _find_no_duplicates(self, name, domain=None, path=None): + """Both ``__get_item__`` and ``get`` call this function: it's never + used elsewhere in Requests. + + :param name: a string containing name of cookie + :param domain: (optional) string containing domain of cookie + :param path: (optional) string containing path of cookie + :raises KeyError: if cookie is not found + :raises CookieConflictError: if there are multiple cookies + that match name and optionally domain and path + :return: cookie.Value + """ + to_return = None + for cookie in iter(self): + if cookie.name == name: + if domain is None or cookie.domain == domain: + if path is None or cookie.path == path: + if to_return is not None: + # if there are multiple cookies that meet passed in criteria + raise CookieConflictError( + f"There are multiple cookies with name, {name!r}" + ) + # we will eventually return this as long as no cookie conflict + to_return = cookie.value + + if to_return: + return to_return + raise KeyError(f"name={name!r}, domain={domain!r}, path={path!r}") + + def __getstate__(self): + """Unlike a normal CookieJar, this class is pickleable.""" + state = self.__dict__.copy() + # remove the unpickable RLock object + state.pop("_cookies_lock") + return state + + def __setstate__(self, state): + """Unlike a normal CookieJar, this class is pickleable.""" + self.__dict__.update(state) + if "_cookies_lock" not in self.__dict__: + self._cookies_lock = threading.RLock() + + def copy(self): + """Return a copy of this RequestsCookieJar.""" + new_cj = Ja3RequestsCookieJar() + new_cj.set_policy(self.get_policy()) + new_cj.update(self) + return new_cj + + def get_policy(self): + """Return the CookiePolicy instance used.""" + return self._policy + + +def _copy_cookie_jar(jar): + """ + Copy CookieJar + :param jar: + :return: + """ + if jar is None: + return None + + if hasattr(jar, "copy"): + # We're dealing with an instance of RequestsCookieJar + return jar.copy() + # We're dealing with a generic CookieJar instance + new_jar = copy.copy(jar) + new_jar.clear() + for cookie in jar: + new_jar.set_cookie(copy.copy(cookie)) + return new_jar + + +def create_cookie(name, value, **kwargs): + """Make a cookie from underspecified parameters. + + By default, the pair of `name` and `value` will be set for the domain '' + and sent on every request (this is sometimes called a "super-cookie"). + """ + result = { + "version": 0, + "name": name, + "value": value, + "port": None, + "domain": "", + "path": "/", + "secure": False, + "expires": None, + "discard": True, + "comment": None, + "comment_url": None, + "rest": {"HttpOnly": None}, + "rfc2109": False, + } + + bad_args = set(kwargs) - set(result) + if bad_args: + raise TypeError( + f"create_cookie() got unexpected keyword arguments: {list(bad_args)}" + ) + + result.update(kwargs) + result["port_specified"] = bool(result["port"]) + result["domain_specified"] = bool(result["domain"]) + result["domain_initial_dot"] = result["domain"].startswith(".") + result["path_specified"] = bool(result["path"]) + + return Cookie(**result) + + +def morsel_to_cookie(morsel): + """Convert a Morsel object into a Cookie containing the one k/v pair.""" + + expires = None + if morsel["max-age"]: + try: + expires = int(time.time() + int(morsel["max-age"])) + except ValueError as err: + raise TypeError(f"max-age: {morsel['max-age']} must be integer") from err + elif morsel["expires"]: + time_template = "%a, %d-%b-%Y %H:%M:%S GMT" + expires = calendar.timegm(time.strptime(morsel["expires"], time_template)) + + return create_cookie( + comment=morsel["comment"], + comment_url=bool(morsel["comment"]), + discard=False, + domain=morsel["domain"], + expires=expires, + name=morsel.key, + path=morsel["path"], + port=None, + rest={"HttpOnly": morsel["httponly"]}, + rfc2109=False, + secure=bool(morsel["secure"]), + value=morsel.value, + version=morsel["version"] or 0, + ) + + +def cookiejar_from_dict(cookie_dict, cookiejar=None, overwrite=True): + """Returns a CookieJar from a key/value dictionary. + + :param cookie_dict: Dict of key/values to insert into CookieJar. + :param cookiejar: (optional) A cookiejar to add the cookies to. + :param overwrite: (optional) If False, will not replace cookies + already in the jar with new ones. + :rtype: CookieJar + """ + if cookiejar is None: + cookiejar = Ja3RequestsCookieJar() + + if cookie_dict is not None: + names_from_jar = [cookie.name for cookie in cookiejar] + for name in cookie_dict: + if overwrite or (name not in names_from_jar): + cookiejar.set_cookie(create_cookie(name, cookie_dict[name])) + + return cookiejar + + +def merge_cookies(cookiejar, _cookies): + """Add cookies to cookiejar and returns a merged CookieJar. + + :param cookiejar: CookieJar object to add the cookies to. + :param _cookies: Dictionary or CookieJar object to be added. + :rtype: CookieJar + """ + if not isinstance(cookiejar, CookieJar): + raise ValueError("You can only merge into CookieJar") + + if isinstance(_cookies, dict): + cookiejar = cookiejar_from_dict(_cookies, cookiejar=cookiejar, overwrite=False) + elif isinstance(_cookies, CookieJar): + try: + cookiejar.update(_cookies) + except AttributeError: + for cookie_in_jar in _cookies: + cookiejar.set_cookie(cookie_in_jar) + + return cookiejar diff --git a/ja3requests/exceptions.py b/ja3requests/exceptions.py index 573c7aa..b65e1ed 100644 --- a/ja3requests/exceptions.py +++ b/ja3requests/exceptions.py @@ -1,5 +1,5 @@ """ -ja3requests.exceptions +Ja3Requests.exceptions ~~~~~~~~~~~~~~~~~~~~~~ This module contains the set of Requests' exceptions. @@ -47,6 +47,12 @@ class InvalidParams(RequestException, ValueError): """ +class InvalidData(RequestException, ValueError): + """ + If request data invalid and raise it. + """ + + class InvalidHost(RequestException, ValueError): """ Raised it while host can not parse. @@ -65,6 +71,12 @@ class InvalidResponseHeaders(RequestException, ValueError): """ +class MaxRetriedException(RuntimeError): + """ + Raised it when retried + """ + + class IssueError(ValueError): """ This situation may not be considered yet, please issue it diff --git a/ja3requests/protocol/exceptions.py b/ja3requests/protocol/exceptions.py index 5fce210..b8ac0e3 100644 --- a/ja3requests/protocol/exceptions.py +++ b/ja3requests/protocol/exceptions.py @@ -1,10 +1,18 @@ """ -ja3requests.protocol.exceptions -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Ja3Requests.protocol.exceptions +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ This module contains socket exceptions. """ +import socket + + +class SocketError(socket.error): + """ + Socket error + """ + class SocketException(Exception): """ @@ -34,6 +42,18 @@ class ConnectTimeoutError(SocketTimeoutError): """ +class ProxyError(socket.error): + """ + Raised when a proxy error + """ + + +class ProxyTimeoutError(ProxyError): + """ + Raised when a proxy socket timeout occurs while connecting to a server + """ + + class ReadTimeout(SocketTimeoutError): """ Raised when socket receive timeout. diff --git a/ja3requests/protocol/sockets.py b/ja3requests/protocol/sockets.py index 77f3122..df16337 100644 --- a/ja3requests/protocol/sockets.py +++ b/ja3requests/protocol/sockets.py @@ -1,6 +1,6 @@ # pylint: skip-file """ -ja3requests.protocol.sockets +Ja3Requests.protocol.sockets ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ This module contains socket dependencies. @@ -98,3 +98,19 @@ def _has_ipv6(host): HAS_IPV6 = _has_ipv6("::1") + + +if __name__ == '__main__': + sock = create_connection(("127.0.0.1", 7890)) + sock.sendall("CONNECT ifconfig.me:80 HTTP/1.1\r\n\r\n".encode()) + response = sock.recv(4096).decode("utf8") + if "200" in response: + sock.sendall( + # "GET / HTTP/1.1\r\nAccept: */*\r\nConnection: keep-alive\r\nUser-Agent: Python/3.11.6 (Darwin; macOS-13.3.1-x86_64-i386-64bit) Ja3Requests/1.0.2\r\nHost: ifconfig.me\r\n\r\n".encode() + "GET / HTTP/1.1\r\nAccept: */*\r\nConnection: keep-alive\r\nUser-Agent: Python/3.11.6 (Darwin; macOS-13.3.1-x86_64-i386-64bit) Ja3Requests/1.0.2\r\nHost: ifconfig.me\r\n\r\n".encode() + ) + a = 0 + while a < 50: + response = sock.recv(4096).decode("utf8") + a += 1 + print(response) diff --git a/ja3requests/request.py b/ja3requests/request.py deleted file mode 100644 index 4a7dc39..0000000 --- a/ja3requests/request.py +++ /dev/null @@ -1,300 +0,0 @@ -""" -ja3requests.request -~~~~~~~~~~~~~~~~~~~ - -This module create a request struct and ready request object. -""" - - -import warnings -from http.cookiejar import CookieJar -from urllib.parse import urlparse, urlencode -from typing import Any, AnyStr, Dict, List, Union, ByteString, Tuple -from .base import BaseRequest -from .utils import default_headers -from .context import HTTPContext -from .connections import HTTPConnection -from .exceptions import ( - NotAllowedRequestMethod, - MissingScheme, - NotAllowedScheme, - InvalidParams, -) - - -class ReadyRequest(BaseRequest): - """ - Ready a request, e.g.(check url, check params) - """ - - def __init__( - self, - method: AnyStr, - url: AnyStr, - params: Union[ - Dict[Any, Any], - List[Tuple[Any, Any]], - Tuple[Tuple[Any, Any]], - ByteString, - AnyStr, - ] = None, - data: Union[Dict[AnyStr, Any], List, Tuple, ByteString] = None, - headers: Dict[AnyStr, AnyStr] = None, - cookies: Union[Dict[AnyStr, AnyStr], CookieJar] = None, - auth: Tuple = None, - json: Dict[AnyStr, AnyStr] = None, - ): - super().__init__() - self.method = method - self.url = url - self.params = params - self.data = data - self.headers = headers - self.cookies = cookies - self.auth = auth - self.json = json - - def __repr__(self): - return f"" - - def ready_method(self): - """ - Ready request method and check request method whether allow used. - :return: - """ - - if self.method == "" or self.method not in [ - "GET", - "OPTIONS", - "HEAD", - "POST", - "PUT", - "PATCH", - "DELETE", - ]: - raise NotAllowedRequestMethod(self.method) - - self.method = self.method.upper() - - def ready_url(self): - """ - Ready http url and check url whether valid. - :return: - """ - - if self.url == "": - raise ValueError("The request url is required.") - - # Remove whitespaces for url - self.url.strip() - - parse = urlparse(self.url) - - # Check HTTP scheme - if parse.scheme == "": - raise MissingScheme( - f"Invalid URL {self.url!r}: No scheme supplied. " - f"Perhaps you meant http://{self.url} or https://{self.url}" - ) - - # Just allow http or https - if parse.scheme not in ["http", "https"]: - raise NotAllowedScheme(f"Schema: {parse.scheme} not allowed.") - - self.scheme = parse.scheme - if self.scheme == "https": - self.port = 443 - - if parse.netloc != "" and ":" in parse.netloc: - port = parse.netloc.split(":")[-1] - self.port = int(port) - else: - self.port = 80 - - def ready_params(self): - """ - Ready params. - :return: - """ - if self.params: - parse = urlparse(self.url) - - if isinstance(self.params, str): - params = self.params - elif isinstance(self.params, bytes): - params = self.params.decode() - elif isinstance(self.params, (dict, list, tuple)): - params = urlencode(self.params) - else: - raise InvalidParams(f"Invalid params: {self.params!r}") - - if params.startswith("?"): - params = params.replace("?", "") - - if parse.query != "": - self.url = "&" + params - else: - self.url = "?" + params - - def ready_data(self): - """ - Ready form data. - :return: - """ - if self.data: - if self.headers is not None: - content_type = self.headers.get("Content-Type", "") - if content_type == "": - self.headers["Content-Type"] = content_type = "application/x-www-form-urlencoded" - else: - self.headers = default_headers() - self.headers["Content-Type"] = content_type = "application/x-www-form-urlencoded" - - if content_type == "application/x-www-form-urlencoded": - self.data = urlencode(self.data) - self.headers["Content-Length"] = len(self.data) - - print(self.data) - - def ready_headers(self): - """ - Ready http headers. - :return: - """ - - # Default headers - if self.headers is None: - self.headers = default_headers() - - # Check duplicate default item - new_headers = {} - header_list = [] - for k, v in self.headers.items(): - header = k.title() - if header in header_list: - warnings.warn( - f"Duplicate header: {k}, you should check the request headers.", - RuntimeWarning, - ) - - header_list.append(header) - new_headers[header] = v - - self.headers = new_headers - del new_headers - del header_list - - def ready_cookies(self): - """ - Todo: Ready http cookies. - :return: - """ - - def ready_auth(self): - """ - Todo: Ready http authenticator - :return: - """ - - def ready_json(self): - """ - Todo: Ready post json. - :return: - """ - - def ready(self): - """ - Make a ready request to send. - :return: - """ - self.ready_method() - self.ready_url() - self.ready_params() - self.ready_headers() - self.ready_data() - self.ready_cookies() - self.ready_auth() - self.ready_json() - - def request(self): - """ - Create a Request object. - :return: - """ - req = Request() - req.clone(self) - - return req - - -class Request(BaseRequest): - """ - Request object to send. - """ - - def __repr__(self): - return f"" - - def clone(self, ready_request: ReadyRequest): - """ - Clone arguments from ReadyRequest - :param ready_request: - :return: - """ - for k, v in ready_request.__dict__.items(): - setattr(self, k, v) - - def send(self): - """ - Connection sending. - :return: - """ - - conn = self.create_connect() - proxy, proxy_username, proxy_password = self.parse_proxies() - conn.connect( - self.scheme, - self.port, - self.source, - self.url, - self.timeout, - proxy, - proxy_username, - proxy_password, - ) - context = HTTPContext(conn) - context.set_payload( - method=self.method, - headers=self.headers, - body=self.data, - ) - response = conn.send(context) - - return response - - def create_connect(self): - """ - Create http connection or https connection by request scheme. - :return: - """ - - if self.is_http(): - conn = HTTPConnection() - elif self.is_https(): - # TODO: HTTPS - # conn = HTTPSConnection() - raise NotImplementedError("HTTPS not implemented yet.") - else: - raise MissingScheme( - f"Scheme: {self.scheme}, parse scheme failed, can't create connection." - ) - - return conn - - def parse_proxies(self): - """ - TODO - Parse proxy, proxy's username and password. if proxies is set. - :return: - """ - return None, None, None diff --git a/ja3requests/requests/__init__.py b/ja3requests/requests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/ja3requests/requests/http.py b/ja3requests/requests/http.py new file mode 100644 index 0000000..636e99e --- /dev/null +++ b/ja3requests/requests/http.py @@ -0,0 +1,60 @@ +""" +Ja3Requests.requests.http +~~~~~~~~~~~~~~~~~~~~~~~~~ + +This module of HTTP Request. +""" + + +from ja3requests.base import BaseRequest +from ja3requests.contexts.context import HTTPContext +from ja3requests.sockets.http import HttpSocket +from ja3requests.sockets.proxy import ProxySocket +from ja3requests.const import DEFAULT_HTTP_SCHEME, DEFAULT_HTTP_PORT +from ja3requests.response import HTTPResponse + + +class HttpRequest(BaseRequest): + """ + HTTP Request + """ + + def __init__(self): + super().__init__() + self.scheme = DEFAULT_HTTP_SCHEME + self.port = DEFAULT_HTTP_PORT + + @staticmethod + def create_connection(context: HTTPContext): + """ + create a new connection by context + :param context: + :return: + """ + if context.proxy: + sock = ProxySocket(context) + else: + sock = HttpSocket(context) + + return sock.new_conn() + + def send(self): + context = HTTPContext() + context.set_payload( + method=self.method, + start_line=self.url, + port=self.port, + data=self.data, + files=self.files, + headers=self.headers, + timeout=self.timeout, + json=self.json, + proxy=self.proxy, + cookies=self.cookies, + ) + sock = self.create_connection(context) + sock.send() + response = HTTPResponse(sock.conn) + response.begin() + + return response diff --git a/ja3requests/requests/https.py b/ja3requests/requests/https.py new file mode 100644 index 0000000..b96e21e --- /dev/null +++ b/ja3requests/requests/https.py @@ -0,0 +1,24 @@ +""" +Ja3Requests.requests.https +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +This module of HTTPS Request. +""" + + +from ja3requests.const import DEFAULT_HTTPS_SCHEME, DEFAULT_HTTPS_PORT +from ja3requests.base import BaseRequest + + +class HttpsRequest(BaseRequest): + """ + HTTPS Request + """ + + def __init__(self): + super().__init__() + self.scheme = DEFAULT_HTTPS_SCHEME + self.port = DEFAULT_HTTPS_PORT + + def send(self): + pass diff --git a/ja3requests/requests/request.py b/ja3requests/requests/request.py new file mode 100644 index 0000000..e1da0ca --- /dev/null +++ b/ja3requests/requests/request.py @@ -0,0 +1,374 @@ +""" +Ja3Requests.requests.request +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +This module of Request. +""" + + +import os +import warnings +from io import IOBase +from http.cookiejar import CookieJar +from urllib.parse import urlparse, parse_qs +from typing import Any, AnyStr, List, Dict, Tuple, Union +from ja3requests.requests.https import HttpsRequest +from ja3requests.requests.http import HttpRequest +from ja3requests.exceptions import ( + NotAllowedRequestMethod, + MissingScheme, + NotAllowedScheme, + InvalidParams, + InvalidData, +) + + +class Request: + """ + Request + """ + + def __init__( + self, + method: AnyStr, + url: AnyStr, + params: Union[ + Dict[AnyStr, Any], + List[Tuple[Any, Any]], + Tuple[Tuple[Any, Any]], + AnyStr, + ] = None, + data: Union[Dict[AnyStr, Any], List, Tuple, AnyStr] = None, + headers: Dict[AnyStr, AnyStr] = None, + cookies: Union[Dict[AnyStr, AnyStr], CookieJar, AnyStr] = None, + files: Dict[AnyStr, Union[List[Union[AnyStr, IOBase]], IOBase, AnyStr]] = None, + auth: Tuple = None, + json: Dict[AnyStr, AnyStr] = None, + proxies: Dict[AnyStr, AnyStr] = None, + timeout: float = None, + ): + self.method = method + self.url = url + self.params = params + self.data = data + self.headers = headers + self.cookies = cookies + self.files = files + self.auth = auth + self.json = json + self.proxies = proxies + self.timeout = timeout + + def __repr__(self): + return f"" + + def request(self): + """ + Make a ready request to send. + :return: + """ + method = self.__ready_method() + schema, url = self.__ready_url() + params = self.__ready_params() + data = self.__ready_data() + _json = self.__ready_json() + files = self.__ready_files() + headers = self.__ready_headers() + cookies = self.__ready_cookies() + auth = self.__ready_auth() + proxies = self.__read_proxies() + + if schema == "http": + req = HttpRequest() + req.set_payload( + method=method, + url=url, + params=params, + data=data, + files=files, + headers=headers, + cookies=cookies, + auth=auth, + json=_json, + proxy=proxies, + timeout=self.timeout, + ) + return req + + if schema == "https": + req = HttpsRequest() + req.set_payload( + method=method, + url=url, + params=params, + data=data, + files=files, + headers=headers, + cookies=cookies, + auth=auth, + json=_json, + proxies=proxies, + timeout=self.timeout, + ) + return req + + raise NotAllowedScheme(f"Schema: {schema} not allowed.") + + def __ready_method(self): + """ + Ready request method and check request method whether allow used. + :return: + """ + + method = self.method.upper() + if method == "" or method not in [ + "GET", + "OPTIONS", + "HEAD", + "POST", + "PUT", + "PATCH", + "DELETE", + ]: + raise NotAllowedRequestMethod(method) + + return method + + def __ready_url(self): + """ + Ready http url and check url whether valid. + :return: + """ + url = self.url + + if not url or url == "": + raise ValueError("The request url is require.") + + # Remove whitespaces for url + url = url.strip() + + parse = urlparse(url) + + # Check HTTP scheme + if parse.scheme == "": + raise MissingScheme( + f"Invalid URL {self.url!r}: No scheme supplied. " + f"Perhaps you meant http://{self.url} or https://{self.url}" + ) + + # Just allow http or https + if parse.scheme not in ["http", "https"]: + raise NotAllowedScheme(f"Schema: {parse.scheme} not allowed.") + + return parse.scheme, url + + def __ready_params(self): + """ + Ready params. + :return: + """ + params = self.params + if not params: + return params + + # parse = urlparse(self.url) + if not isinstance(params, (str, bytes, dict, list, tuple)): + raise InvalidParams(f"Invalid params: {self.params!r}") + + return params + + def __ready_headers(self): + """ + Ready http headers. + :return: + """ + + headers = self.headers + if not headers: + return headers + + # Check duplicate default item + header_list = [] + for k, _ in headers.items(): + if k.lower() in header_list: + warnings.warn( + f"Duplicate header: {k}, you should check the request headers.", + RuntimeWarning, + ) + header_list.append(k.lower()) + + return headers + + def __ready_data(self): + """ + Ready form data. + :return: + """ + data = self.data + if not data: + return data + + if self.json: + raise InvalidData( + "Only one of the data and json parameters can be used at the same time" + ) + + if self.method.upper() not in ["POST", "PUT"]: + warnings.warn( + f"The {self.method.upper()} method does not process data." + f"Maybe you request the POST/PUT method?", + RuntimeWarning, + ) + + if not isinstance(data, (dict, list, tuple, bytes, str)): + raise InvalidData(f"Invalid data: {data!r}") + + if isinstance(data, (list, tuple)): + if len(data) < 1: + raise InvalidData( + f"Invalid data: {data!r}. The data parameter of iterable type is empty" + ) + + if not all(list(map(lambda x: isinstance(x, tuple), data))): + raise InvalidData( + f"Invalid data: {data!r}. The data parameter item of iterable type must be a tuple" + ) + + if isinstance(data, (bytes, str)): + try: + parse_qs(data) + except AttributeError as err: + raise InvalidData(f"Invalid data: {data!r}") from err + + return data + + def __ready_cookies(self): + """ + :return: + """ + + cookies = self.cookies + if not cookies: + return cookies + + if not isinstance(cookies, (dict, CookieJar, bytes, str)): + raise AttributeError( + f"Invalid cookies: {cookies!r}." + "Cookies type only support dict, CookieJar, bytes, str" + ) + + if isinstance(cookies, (dict, bytes, str)): + if len(cookies) < 1: + raise AttributeError("Invalid cookies, it's empty.") + + return cookies + + def __ready_auth(self): + """ + Todo: Ready http authenticator + :return: + """ + + auth = self.auth + + return auth + + def __ready_json(self): + """ + Ready post json. + :return: + """ + + _json = self.json + if not _json: + return _json + + if self.data or self.files: + raise ValueError( + "Only one of the data/files and json parameters can be used at the same time" + ) + + if self.method.upper() not in ["POST", "PUT"]: + warnings.warn( + f"The {self.method.upper()} method does not process data." + f"Maybe you request the POST/PUT method?", + RuntimeWarning, + ) + + if not isinstance(self.json, (dict, str, bytes)): + raise ValueError(f"Invalid json: {self.json!r}") + + if self.headers: + for name, value in self.headers.items(): + if name.title() == "Content-Type" and value == "multipart/form-data": + warnings.warn( + "When sending a json data, the Content-Type header should be set to application/json", + RuntimeWarning, + ) + break + + return _json + + def __ready_files(self): + """ + Ready post file + :return: + """ + files = self.files + if not files: + return files + + if not isinstance(files, dict): + raise AttributeError( + "The files parameter is invalid, reference structure: {'file': FileObject}" + ) + + for _, file in files.items(): + if isinstance(file, list): + for f in file: + if isinstance(f, (str, bytes)) and not os.path.isfile(f): + raise AttributeError(f"{f} is not a file") + if isinstance(f, IOBase) and not f.readable(): + raise AttributeError("IO object is not readable") + + if isinstance(file, (str, bytes)) and not os.path.isfile(file): + raise AttributeError(f"{file} is not a file") + + if isinstance(file, IOBase) and not file.readable(): + raise AttributeError("IO object is not readable") + + if self.headers: + for name, value in self.headers.items(): + if name.title() == "Content-Type" and value != "multipart/form-data": + warnings.warn( + "When sending a files data, the Content-Type header should be set to multipart/form-data", + RuntimeWarning, + ) + break + + return files + + def __read_proxies(self): + """ + Read proxies + :return: + """ + proxies = self.proxies + if not proxies: + return proxies + + if not isinstance(proxies, dict): + raise AttributeError( + f"Invalid proxies attribute: {proxies!r}." + "The property structure should look like " + "{'http': 'username:password@host:port', 'https': 'username:password@host:port'}" + ) + + for schema in proxies: + if schema not in ("http", "https"): + raise AttributeError( + f"Invalid proxy schema: {schema!r}.", + "The schema is only support http or https.", + ) + + return proxies diff --git a/ja3requests/response.py b/ja3requests/response.py index 043cc88..b6b2926 100644 --- a/ja3requests/response.py +++ b/ja3requests/response.py @@ -1,6 +1,6 @@ """ -ja3requests.response -~~~~~~~~~~~~~~~~~~~~~~~ +Ja3Requests.response +~~~~~~~~~~~~~~~~~~~~ This module contains response. """ @@ -10,9 +10,11 @@ import gzip import zlib import brotli -from .base import BaseResponse -from .const import MAX_LINE, MAX_HEADERS -from .exceptions import InvalidStatusLine, InvalidResponseHeaders, IssueError +from ja3requests.base import BaseResponse +from ja3requests.cookies import Ja3RequestsCookieJar +from ja3requests.utils import add_dict_to_cookiejar +from ja3requests.const import MAX_LINE, MAX_HEADERS +from ja3requests.exceptions import InvalidStatusLine, InvalidResponseHeaders, IssueError class HTTPResponse(BaseResponse): @@ -169,50 +171,89 @@ def begin(self): self._content_length = int(headers.get(b"content-length", 0)) + @property + def raw_headers(self): + """ + Raw response headers + :return: + """ + headers = [] + if self.headers: + headers_raw = self.headers.decode() + header_list = headers_raw.split("\r\n") + for header_item in header_list: + if header_item == "": + continue + name, value = header_item.split(": ", 1) + headers.append({name.strip(): value.strip()}) + + return headers + class Response(BaseResponse): """Response """ - def __init__(self, response=None): + def __init__(self, request=None, response=None): super().__init__() + self.request = request self.response = response + self.body = self.response.read_body() if self.response else b"" def __repr__(self): + """ + Response repr + :return: + """ return f"" @property - def headers(self): + def cookies(self): """ - Response Headers. + Response cookie property :return: """ - headers = [] - if self.response is None: - return headers - headers_raw = self.response.headers.decode() - header_list = headers_raw.split("\r\n") - for header_item in header_list: - if header_item == "": - continue - name, value = header_item.split(": ", 1) - headers.append({name.strip(): value.strip()}) + cookies = Ja3RequestsCookieJar() + if self.response.raw_headers: + for header in self.response.raw_headers: + set_cookie = header.get("Set-Cookie", None) + if set_cookie is None: + set_cookie = header.get("set-cookie", None) - return headers + if set_cookie: + cookie_item = set_cookie.split(";") + if len(cookie_item) > 0: + cookie = cookie_item[0].split("=") + if len(cookie) == 2: + cookies = add_dict_to_cookiejar( + cookies, {cookie[0].strip(): cookie[1].strip()} + ) + + return cookies @property - def body(self): + def headers(self): """ - Response Body. + Response Headers. :return: """ - body = b"" - if self.response is None: - return body + headers = {} + if not self.response.raw_headers: + return headers + + for header in self.response.raw_headers: + set_cookie = header.get("Set-Cookie", None) + if set_cookie is None: + set_cookie = header.get("set-cookie", None) - return self.response.read_body() + if set_cookie: + continue + + headers.update(header) + + return headers @property def status_code(self): @@ -248,3 +289,24 @@ def json(self): :return: """ return json.loads(self.body) + + @property + def is_redirected(self): + """ + Response property of has redirected + :return: + """ + + return 300 <= self.status_code < 400 + + @property + def location(self): + """ + Response redirected location + :return: + """ + location = self.headers.get("Location", None) + if not location: + location = self.headers.get("location", None) + + return location diff --git a/ja3requests/sessions.py b/ja3requests/sessions.py index 72b0871..f62e368 100644 --- a/ja3requests/sessions.py +++ b/ja3requests/sessions.py @@ -1,5 +1,5 @@ """ -ja3Requests.sessions +Ja3Requests.sessions ~~~~~~~~~~~~~~~~~~~~ This module provides a Session object to manage and persist settings across @@ -7,13 +7,15 @@ """ import sys import time +from io import IOBase from http.cookiejar import CookieJar from typing import AnyStr, Any, Dict, ByteString, Union, List, Tuple -from .base import BaseSession -from .response import Response -from .utils import default_headers -from .const import DEFAULT_REDIRECT_LIMIT -from .request import ReadyRequest, Request +from ja3requests.base import BaseSession +from ja3requests.response import Response +from ja3requests.const import DEFAULT_REDIRECT_LIMIT +from ja3requests.base import BaseRequest +from ja3requests.requests.request import Request +from ja3requests.exceptions import MaxRetriedException # Preferred clock, based on which one is more accurate on a given system. if sys.platform == "win32": @@ -28,46 +30,21 @@ class Session(BaseSession): Provides cookie persistence, connection-pooling, and configuration. """ - def __init__(self): - super().__init__() - - self.headers = default_headers() - self.max_redirects = DEFAULT_REDIRECT_LIMIT - - def ready(self, method, url, params, data, headers, cookies, auth, json): - """ - Ready to send request. - :return: - """ - - req = ReadyRequest( - method=method, - url=url, - params=params, - data=data, - headers=headers, - cookies=cookies, - auth=auth, - json=json, - ) - req.ready() - - return req - def request( self, method: AnyStr, url: AnyStr, params: Union[Dict[AnyStr, Any], ByteString] = None, - data: Union[Dict[AnyStr, Any], List, Tuple, ByteString] = None, + data: Union[ + Dict[Any, Any], List[Tuple[Any, Any]], Tuple[Tuple[Any, Any]], AnyStr + ] = None, headers: Dict[AnyStr, AnyStr] = None, - cookies: Union[Dict[AnyStr, AnyStr], CookieJar] = None, - # files = None, + cookies: Union[Dict[AnyStr, AnyStr], CookieJar, AnyStr] = None, + files: Dict[AnyStr, Union[List[Union[AnyStr, IOBase]], IOBase, AnyStr]] = None, auth: Tuple = None, - timeout: float = None, - allow_redirects: bool = True, proxies: Dict[AnyStr, AnyStr] = None, - json: Dict[AnyStr, AnyStr] = None, + json: Union[Dict[AnyStr, AnyStr], AnyStr] = None, + **kwargs ): """ Instantiating a request class and ready request to send. @@ -77,38 +54,45 @@ def request( :param data: :param headers: :param cookies: + :param files: :param auth: - :param timeout: - :param allow_redirects: :param proxies: :param json: :return: """ - ready_request = self.ready( + + self.Request = Request( method=method, url=url, params=params, data=data, headers=headers, cookies=cookies, + files=files, auth=auth, json=json, + proxies=proxies, ) - req = ready_request.request() - response = self.send(req) + kwargs.setdefault("timeout", None) + kwargs.setdefault("allow_redirects", True) + + req = self.Request.request() + response = self.send(req, **kwargs) return response - def get(self, url, **kwargs): + def get(self, url, params=None, headers=None, **kwargs): """ Send a GET request. :param url: + :param params: + :param headers: :param kwargs: :return: """ - return self.request("GET", url, **kwargs) + return self.request("GET", url, params=params, headers=headers, **kwargs) def options(self, url, **kwargs): """ @@ -131,58 +115,93 @@ def head(self, url, **kwargs): kwargs.setdefault("allow_redirects", False) return self.request("HEAD", url, **kwargs) - def post(self, url, data=None, json=None, **kwargs): + def post(self, url, data=None, json=None, files=None, headers=None, **kwargs): """ Send a POST request. :param url: :param data: :param json: + :param files: + :param headers: :param kwargs: :return: """ - return self.request("POST", url, data=data, json=json, **kwargs) + return self.request( + "POST", url, data=data, json=json, files=files, headers=headers, **kwargs + ) - def put(self, url, data=None, **kwargs): + def put(self, url, **kwargs): """ Send a PUT request. :param url: - :param data: :param kwargs: :return: """ - return self.request("PUT", url, data=data, **kwargs) + return self.request("PUT", url, **kwargs) - def patch(self, url, data=None, **kwargs): + def patch(self, url, **kwargs): """ Send a PATCH request. :param url: - :param data: :param kwargs: :return: """ - return self.request("PATCH", url, data=data, **kwargs) + return self.request("PATCH", url, **kwargs) - def delete(self, url, data=None, **kwargs): + def delete(self, url, **kwargs): """ Send a DELETE request. :param url: - :param data: :param kwargs: :return: """ return self.request("DELETE", url, **kwargs) - def send(self, request: Request): + def send(self, request: BaseRequest, **kwargs): """ Send request. :return: """ + if not isinstance(request, BaseRequest): + raise ValueError("You can only send HttpRequest/HttpsRequest.") + rep = request.send() - response = Response(rep) + response = Response(request, rep) + allow_redirects = kwargs.get("allow_redirects", True) + if allow_redirects and response.is_redirected: + response = self.resolve_redirects(response.location, **kwargs) + + self.response = response + + return response + + def resolve_redirects(self, url, **kwargs): + """ + Handle response redirects + :param url: + :param kwargs: + :return: + """ + send_kwargs = kwargs + + for _ in range(DEFAULT_REDIRECT_LIMIT): + req = Request( + method="GET", + url=url, + headers=self.Request.headers, + cookies=self.Request.cookies, + proxies=self.Request.proxies, + ).request() + + response = self.send(req, **send_kwargs) + if 400 <= response.status_code or response.status_code < 300: + break + else: + raise MaxRetriedException("Too many redirects") return response diff --git a/ja3requests/sockets/__init__.py b/ja3requests/sockets/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/ja3requests/sockets/http.py b/ja3requests/sockets/http.py new file mode 100644 index 0000000..d354f29 --- /dev/null +++ b/ja3requests/sockets/http.py @@ -0,0 +1,28 @@ +""" +Ja3Requests.sockets.http +~~~~~~~~~~~~~~~~~~~~~~~~ + +This module of HTTP Socket. +""" + + +from ja3requests.base import BaseSocket + + +class HttpSocket(BaseSocket): + """ + HTTP Socket + """ + + def new_conn(self): + self.conn = self._new_conn(self.context.destination_address, self.context.port) + return self + + def send(self): + """ + Connection send message + :return: + """ + self.conn.sendall(self.context.message) + + return self.conn diff --git a/ja3requests/sockets/https.py b/ja3requests/sockets/https.py new file mode 100644 index 0000000..7c3aaca --- /dev/null +++ b/ja3requests/sockets/https.py @@ -0,0 +1,18 @@ +""" +Ja3Requests.sockets.https +~~~~~~~~~~~~~~~~~~~~~~~~~ + +This module of HTTPS Socket. +""" + + +from ja3requests.base import BaseSocket + + +class HttpsSocket(BaseSocket): + """ + HTTPS Socket + """ + + def new_conn(self): + pass diff --git a/ja3requests/sockets/proxy.py b/ja3requests/sockets/proxy.py new file mode 100644 index 0000000..7c9a2d9 --- /dev/null +++ b/ja3requests/sockets/proxy.py @@ -0,0 +1,102 @@ +""" +Ja3Requests.sockets.proxy +~~~~~~~~~~~~~~~~~~~~~~~~~ + +This module of Proxy Socket. +""" + + +from base64 import b64encode +from ja3requests.base import BaseSocket +from ja3requests.protocol.exceptions import ( + SocketException, + ProxyError, + ProxyTimeoutError, +) + + +class ProxySocket(BaseSocket): + """ + Proxy Socket + """ + + def __init__(self, context): + super().__init__(context) + if self.context.proxy: + self.proxy_host, self.proxy_port = self.context.proxy.split(":") + else: + self.proxy_host, self.proxy_port = None, None + + if self.context.proxy_auth: + if ":" in self.context.proxy_auth: + ( + self.proxy_username, + self.proxy_password, + ) = self.context.proxy_auth.split(":") + else: + self.proxy_username, self.proxy_password = self.context.proxy_auth, None + else: + self.proxy_username, self.proxy_password = None, None + + def new_conn(self): + if not self.proxy_host and not self.proxy_port: + raise SocketException("The proxy socket must require host and port.") + + self.conn = self._new_conn(self.proxy_host, self.proxy_port) + + message = [ + f"CONNECT {self.context.destination_address}:{self.context.port} HTTP/1.1", + f"Host: {self.context.destination_address}", + ] + if auth := self.context.headers.get("Proxy-Authorization", None): + message.append(f"Proxy-Authorization: Basic {auth}") + else: + auth = "" + if self.proxy_username: + auth += self.proxy_username + if self.proxy_password: + auth += f":{self.proxy_password}" + + if len(auth) > 0: + message.append( + f"Proxy-Authorization: Basic {b64encode(auth.encode()).decode()}" + ) + + message = "\r\n".join(message) + message += "\r\n\r\n" + + try: + self.conn.send(message.encode()) + status_line = self.conn.recv(4096).decode() + proto, status_code, _ = status_line.split(" ", 2) + except (TimeoutError, ConnectionRefusedError, UnicodeError) as err: + raise ProxyTimeoutError("Proxy server connection time out") from err + + if not proto.startswith("HTTP/"): + raise ProxyError("Proxy server does not appear to be an HTTP proxy") + + status_code = int(status_code) + if status_code != 200: + error = "" + # Tunnel connection failed: 502 Proxy Bad Server + if status_code in (400, 403, 405): + error = "The HTTP proxy server may not be supported" + + elif status_code in (407,): + error = f"Tunnel connection failed: status_code = {status_code}, Unauthorized" + + else: + error = f"Tunnel connection failed: status_code = {status_code}" + + raise ProxyError(error) + + return self + + def send(self): + """ + Connection send message + :return: + """ + self.conn.sendall(self.context.message) + + return self.conn diff --git a/ja3requests/utils.py b/ja3requests/utils.py index 2689d55..877ef4d 100644 --- a/ja3requests/utils.py +++ b/ja3requests/utils.py @@ -1,5 +1,5 @@ """ -ja3requests.utils +Ja3Requests.utils ~~~~~~~~~~~~~~~~~ This module provides utility functions. @@ -8,6 +8,9 @@ import platform from base64 import b64encode from typing import Union, AnyStr, List +from .const import DEFAULT_MAX_RETRY_LIMIT +from .exceptions import MaxRetriedException +from .cookies import cookiejar_from_dict from .__version__ import __version__ @@ -87,3 +90,116 @@ def default_headers(): """ return make_headers(keep_alive=True) + + +class SingletonMeta(type): + """ + SingletonMeta Class + """ + + _instance = {} + + def __call__(cls, *args, **kwargs): + if cls not in cls._instance: + cls._instance[cls] = super(SingletonMeta, cls).__call__(*args, **kwargs) + + return cls._instance[cls] + + +class Retry(metaclass=SingletonMeta): + """ + Retry Class + """ + + _tasks = {} + + def do(self, obj, exception, *args, **kwargs): + """ + Method of run retry task + :param obj: + :param exception: + :param args: + :param kwargs: + :return: + """ + if obj not in self._tasks: + self._tasks[obj] = Task( + obj, DEFAULT_MAX_RETRY_LIMIT, exception, *args, **kwargs + ) + + while self._tasks[obj].times > 0: + result = self._tasks[obj].retry() + if result != 0: + return result + + raise MaxRetriedException(f"Max retries exceeded with {obj!r}") + + +class Task: + """ + Retry Task + """ + + def __init__(self, task, times, exception, *args, **kwargs): + self.task = task + self.times = times + self.exception = exception + self.args = args + self.kwargs = kwargs + + def retry(self): + """ + retry method + :return: + """ + self.times -= 1 + try: + return self.task(*self.args, **self.kwargs) + except self.exception: + return 0 + + +def dict_from_cookie_string(cookie_string: AnyStr): + """Returns a key/value dictionary from a cookie string like name1=value1;name2=value2;... + + :param cookie_string: + :return: dict + """ + + cookie_dict = {} + if isinstance(cookie_string, bytes): + cookie_string = cookie_string.decode() + + cookie_list = cookie_string.split(";") + for cookie in cookie_list: + cookie = cookie.strip() + name, value = cookie.split("=") + cookie_dict.setdefault(name, value) + + return cookie_dict + + +def dict_from_cookiejar(cj): + """Returns a key/value dictionary from a CookieJar. + + :param cj: CookieJar object to extract cookies from. + :rtype: dict + """ + + cookie_dict = {} + + for cookie in cj: + cookie_dict[cookie.name] = cookie.value + + return cookie_dict + + +def add_dict_to_cookiejar(cj, cookie_dict): + """Returns a CookieJar from a key/value dictionary. + + :param cj: CookieJar to insert cookies into. + :param cookie_dict: Dict of key/values to insert into CookieJar. + :rtype: CookieJar + """ + + return cookiejar_from_dict(cookie_dict, cj) diff --git a/main.py b/main.py index f135ac2..1cf4a1f 100644 --- a/main.py +++ b/main.py @@ -5,10 +5,20 @@ "connection": "keep-alive", "Accept-Encoding": "deflate, br, gzip" } + +data = { + "username": "admin", + "password": "admin", + "args": ["1", "2", "3"] + } +headers = { + "content-type": "multipart/form-data" +} with Session() as session: # response = session.get("http://127.0.0.1:8080", headers=headers) # response = session.get("http://www.baidu.com", headers=headers) - response = session.get("http://www.aliyun.com") + # response = session.get("http://www.aliyun.com") + response = session.post("http://127.0.0.1:8080/login", json=data, headers=headers) print(response.headers) print(response) # print(response.status_code) diff --git a/setup.py b/setup.py index 5890949..31073a6 100644 --- a/setup.py +++ b/setup.py @@ -55,7 +55,14 @@ def run_tests(self): author=about["__author__"], author_email=about["__author_email__"], url=about["__url__"], - packages=["ja3requests", "ja3requests/base", "ja3requests/protocol"], + packages=[ + "ja3requests", + "ja3requests/base", + "ja3requests/contexts", + "ja3requests/protocol", + "ja3requests/requests", + "ja3requests/sockets" + ], package_dir={"ja3requests": "ja3requests"}, zip_safe=False, include_package_data=True, diff --git a/test/test_session.py b/test/test_session.py index 4c58859..1f5de6d 100644 --- a/test/test_session.py +++ b/test/test_session.py @@ -1,29 +1,128 @@ import unittest from ja3requests.sessions import Session +import requests +from io import BufferedRandom, TextIOWrapper, BytesIO, IOBase +import mimetypes class TestSession(unittest.TestCase): session = Session() + headers = { + "connection": "close" + } def test_get(self): - headers = { - "connection": "close" - } - self.session.get("http://www.baidu.com") + # response = self.session.get("http://www.baidu.com", headers=headers) + response = requests.get("http://www.baidu.com") + print(response) + print(response.status_code) + print(response.headers) + print(response.text) + + def test_request_index(self): + + response = self.session.get("http://127.0.0.1:8080/", headers=self.headers) + # response = requests.get("http://127.0.0.1:8000/") + print(response) + print(response.status_code) + print(response.headers) + print(response.text) + + def test_request1(self): + response = self.session.get("http://127.0.0.1:8080/test1?page=1&limit=100", headers=self.headers) + # response = requests.get("http://127.0.0.1:8000/") + print(response) + print(response.status_code) + print(response.headers) + print(response.text) + + def test_request2(self): + response = self.session.get("http://127.0.0.1:8080/test2/hello/9", headers=self.headers) + # response = requests.get("http://127.0.0.1:8000/") + print(response) + print(response.status_code) + print(response.headers) + print(response.text) def test_post_data(self): data = { "username": "admin", - "password": "admin" + "password": "admin", + } + headers = { + "content-type": "multipart/form-data" + } + f = open("test.txt", "rb+") + + response = self.session.post("http://127.0.0.1:8080/login", json=data, files={"file": f}, headers=headers) + # response = requests.post("http://127.0.0.1:8080/login", data=data, files={"file": f}) + + print(response) + print(response.status_code) + print(response.headers) + print(response.json()) + f.close() + + def test_post_multi_files(self): + + data = { + "name": "test", + "project_type": 1 } - response = self.session.post("http://127.0.0.1:8080/login", data=data) + files = { + "documents": ["/Users/pledgebox/Projects/ja3requests/test/test.txt", "/Users/pledgebox/Projects/ja3requests/test/1.csv"] + } + + response = self.session.post("http://127.0.0.1:8080/api/v1/project/create", data=data, files=files) print(response) print(response.status_code) print(response.headers) + print(response.json()) + + def test_r(self): + response = requests.post("https://baidu.com", data={"a": 1}) + print(response.text) + + def test_proxy(self): + + proxies = { + "http": "127.0.0.1:7890", + "https": "127.0.0.1:7890" + } + # proxies = { + # "http": "9jjmn:uweo3gw@169.197.83.75:6887", + # "https": "9jjmn:uweo3gw@169.197.83.75:6887" + # } + response = self.session.get("http://ifconfig.me", proxies=proxies) + print(response) + print(response.status_code) print(response.content) + print(response.headers) + print(response.text) + # print(response.json()) + + def test_redirect(self): + + response = self.session.get("http://127.0.0.1:8080/redirect") + print(response) + print(response.status_code) + print(response.content) + print(response.headers) + print(response.text) + + def test_cookies(self): + + response = self.session.get("http://127.0.0.1:8080/cookies", cookies="visitor_id=46f759b7a4163bb6cdb75496d0f20d9d4c923c99e66e70b51a42a2565e51bbb2; x-spec-id=b37a2959d518daa20e1a925235efda7b; _ga=GA1.1.1180078114.1700553181; _gcl_au=1.1.939381393.1700794101; _fbp=fb.1.1700794100765.899406327; _tt_enable_cookie=1; _ttp=RFpK3PVl4QTlExYptLDW7bC8EtS; permutive-id=f06468b0-cfd1-44b8-ab89-31dae4a29dfd; __stripe_mid=2c79449d-c8c7-47b6-96b1-ad9b7e2cd2852694b5; __ssid=1bfc66fd636400f35038cffd272dc33; cto_bundle=ZSrZj19vNENzVndIRW54VklPeVpHRU01QWw4eVFBQ2lFOHF6TnV3JTJCYmRZSmxCd3N4eExhTEFKV3Z4Z3JqOHMwMDclMkZyYkJGRnc4TERJNDRoSjl4UWlEZ2Q0ODd4Mzg0UUxBM1NhUWJsMHhwc2h3bmlXOXVQa1l4VSUyRlE0czYlMkIyVE1ENkIybHBrak9YS2QlMkZmRXRoQ3UwRUxZMVoweGlZejV2Tm1kN3dod2lMbmJmSlI4JTNE; _ga_QETRR7E37F=GS1.1.1700794116.1.1.1700794291.28.0.0; localCurrencyIsoCode=USD; romref_referer_host=www.indiegogo.com; optimizelyEndUserId=oeu1704278791748r0.3067598211937421; optimizelySegments=%7B%222354810435%22%3A%22true%22%7D; optimizelyBuckets=%7B%7D; _ga_39QX3WF5EB=GS1.1.1704279034.1.1.1704279038.0.0.0; _session_id=cc851bc32228e4c6d9f596336d3f0b5e; analytics_session_id=e05d0a8d53bdbca46ecae3e8f106986809b4e77dd8264b3f4aefd05182a3c7bf; romref=shr-hmco; cohort=www.indiegogo.com%7Cdir-XXXX%7Cshr-hmco%7Cref-XXXX%7Cshr-hmhd%7Cshr-pies%7Cshr-hmco; recent_project_ids=2870703%262557678%262501412%262262076%262865436%262871402%262889001%26332405%26245753%262850915%262863909%262854586%262830934%262869077; __stripe_sid=490fec6f-b50b-4fb6-a4e0-7a3958ab62e015210d; _ga_DTZH7F2EYR=GS1.1.1704941476.8.1.1704941526.10.0.0; _ga_CLN2NQBG5Y=GS1.1.1704941496.1.1.1704941711.0.0.0") + print(self.session.cookies) + # print(response) + # print(response.status_code) + # print(response.content) + # print(response.headers) + # print(response.cookies) + # print(response.text) if __name__ == '__main__':