diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..8013354 --- /dev/null +++ b/Makefile @@ -0,0 +1,15 @@ +PYTHON := $(shell command -v python3) + +fmt: + command -v black || $(PYTHON) -m pip install -r requirements.txt + black ja3requests + +lint: + command -v pylint || $(PYTHON) -m pip install -r requirements.txt + pylint ja3requests + +package: + if [ -f 'setup.py' ]; then $(PYTHON) setup.py sdist;fi + +package_whl: + if [ -f 'setup.py' ]; then $(PYTHON) setup.py bdist_wheel;fi diff --git a/ja3requests/__init__.py b/ja3requests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/ja3requests/__version__.py b/ja3requests/__version__.py new file mode 100644 index 0000000..826e93f --- /dev/null +++ b/ja3requests/__version__.py @@ -0,0 +1,15 @@ +""" +ja3requests.__version__ +~~~~~~~~~~~~~~~~~~~~~~~ + +Version information. +""" + +__title__ = "ja3requests" +__description__ = "An http request library that can customize ja3 fingerprints." +__url__ = "https://github.com/lxjmaster/ja3requests" +__version__ = "0.0.1" +__author__ = "Mast Luo" +__author_email__ = "379501669@qq.com" +__license__ = "Apache-2.0 license" +__copyright__ = "Copyright Mast Luo" diff --git a/ja3requests/base/__init__.py b/ja3requests/base/__init__.py new file mode 100644 index 0000000..e9de5d3 --- /dev/null +++ b/ja3requests/base/__init__.py @@ -0,0 +1,8 @@ +""" +ja3requests.base +~~~~~~~~~~~~~~~~ + +Basic module. +""" + +from ._sessions import BaseSession diff --git a/ja3requests/base/_sessions.py b/ja3requests/base/_sessions.py new file mode 100644 index 0000000..bf318ce --- /dev/null +++ b/ja3requests/base/_sessions.py @@ -0,0 +1,294 @@ +""" +ja3Requests.base._sessions +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Basic Session +""" + + +class BaseSession: + """ + The base request session. + """ + + def __init__(self): + self._headers = None + self._cookies = None + self._auth = None + self._proxies = None + self._params = None + self._max_redirects = None + self._allow_redirect = None + self._ja3_text = None + self._h2_settings = None + self._h2_window_update = None + self._h2_headers = None + + @property + def headers(self): + """Headers + Http headers. + >>> {'Accept': '*/*', 'Accept-Encoding': 'gzip,deflate'} + :return: + """ + return self._headers + + @headers.setter + def headers(self, attr): + """ + Set Headers + :param attr: + :return: + """ + self._headers = attr + + @property + def cookies(self): + """Cookies + Http cookies. + >>> CookieJar({}) + :return: + """ + return self._cookies + + @cookies.setter + def cookies(self, attr): + """ + Set Cookies + :param attr: + :return: + """ + self.cookies = attr + + @property + def auth(self): + """Auth + >>> {'user': 'xxx', 'password': 'xxx'} + :return: + """ + return self._auth + + @auth.setter + def auth(self, attr): + """ + Set Auth + :param attr: + :return: + """ + self._auth = attr + + @property + def proxies(self): + """Proxies + Http proxy server. + >>> {'http': 'user:password@host:port', 'https': 'user:password@host:port'} + :return: + """ + return self._cookies + + @proxies.setter + def proxies(self, attr): + """ + Set Proxies + :param attr: + :return: + """ + self._proxies = attr + + @property + def params(self): + """Params. + Request Params. ?page=1&per_page=10 + >>> {'page': 1, 'per_page': 10} + :return: + """ + return self._params + + @params.setter + def params(self, attr): + """ + Set Params + :param attr: + :return: + """ + self._params = attr + + @property + def max_redirects(self): + """Max Redirects. + The max for redirect times. + >>> 5 + :return: + """ + return self._max_redirects + + @max_redirects.setter + def max_redirects(self, attr): + """ + Set Max Redirects + :param attr: + :return: + """ + self._max_redirects = attr + + @property + def allow_redirect(self): + """Allow Redirect. + Whether allow redirect. + >>> True or False. + :return: + """ + return self._allow_redirect + + @allow_redirect.setter + def allow_redirect(self, attr): + """ + Set Allow Redirect + :param attr: + :return: + """ + self._allow_redirect = attr + + @property + def ja3_text(self): + """Ja3 Text. + The TLS fingerprint ja3 text. + >>> "771,4865-4866-4867-49195-49199-49196-49200-52393-52392-49171-49172-156-157-47-53,17513-27-0-13-35-43-65281-23-51-5-45-11-16-10-18-21,29-23-24,0" + :return: + """ + return self._ja3_text + + @ja3_text.setter + def ja3_text(self, attr): + """ + Set Ja3 Text + :param attr: + :return: + """ + self._ja3_text = attr + + @property + def h2_settings(self): + """H2 Settings. + The htp2 fingerprint SETTINGS. + >>> {"1": "65535", "2": "0", "3": "1000", "4": "6291456", "6": "262144"} + :return: + """ + return self._h2_settings + + @h2_settings.setter + def h2_settings(self, attr): + """ + Set H2 Settings + :param attr: + :return: + """ + self._h2_settings = attr + + @property + def h2_window_update(self): + """H2 Window Update. + The http2 fingerprint WINDOW_UPDATE. + >>> "15663105" + :return: + """ + return self._h2_window_update + + @h2_window_update.setter + def h2_window_update(self, attr): + """ + Set Window Update + :param attr: + :return: + """ + self._h2_window_update = attr + + @property + def h2_headers(self): + """H2 Headers. + The http2 fingerprint HEADERS. + :method + :authority + :scheme + :path + >>> "m,a,s,p" + :return: + """ + return self._h2_headers + + @h2_headers.setter + def h2_headers(self, attr): + """ + Set H2 Headers + :param attr: + :return: + """ + self._h2_headers = attr + + def __enter__(self): + return self + + def __exit__(self, *args, **kwargs): + self.close(*args, **kwargs) + + def close(self, *args, **kwargs): + """ + Close session. + :param args: + :param kwargs: + :return: + """ + + def request(self, *args, **kwargs): + """ + Request + :return: + """ + + def get(self, *args, **kwargs): + """ + GET Method. + :return: + """ + + def options(self, *args, **kwargs): + """ + OPTIONS Method. + :return: + """ + + def head(self, *args, **kwargs): + """ + HEAD Method. + :return: + """ + + def post(self, *args, **kwargs): + """ + POST Method. + :return: + """ + + def put(self, *args, **kwargs): + """ + PUT Method. + :return: + """ + + def patch(self, *args, **kwargs): + """ + PATCH Method. + :return: + """ + + def delete(self, *args, **kwargs): + """ + DELETE Method. + :return: + """ + + def send(self, *args, **kwargs): + """ + Send + :return: + """ diff --git a/ja3requests/const.py b/ja3requests/const.py new file mode 100644 index 0000000..6f27e96 --- /dev/null +++ b/ja3requests/const.py @@ -0,0 +1,36 @@ +""" +ja3requests.const +~~~~~~~~~~~~~~~~~ + +A constant module. +""" + +import sys + + +class _Const: + class ConstError(TypeError): + """ + Const Error + """ + + class ConstCaseError(ConstError): + """ + Const Case Error + """ + + def __setattr__(self, key, value): + if self.__dict__.get(key) is not None: + raise self.ConstError(f"The constant {key} already exists") + + if not key.isupper(): + raise self.ConstCaseError(f"{key}-constants need to be capitalized.") + + self.__dict__[key] = value + + +const = _Const() + +const.DEFAULT_REDIRECT_LIMIT = 8 # max redirect + +sys.modules[__name__] = const diff --git a/ja3requests/sessions.py b/ja3requests/sessions.py new file mode 100644 index 0000000..2db6bb1 --- /dev/null +++ b/ja3requests/sessions.py @@ -0,0 +1,137 @@ +""" +ja3Requests.sessions +~~~~~~~~~~~~~~~~~~~~ + +This module provides a Session object to manage and persist settings across +ja3Requests. +""" +import sys +import time +from http.cookiejar import CookieJar +from typing import AnyStr, Any, Dict, ByteString, Union, List, Tuple +from .base import BaseSession +from .utils import default_headers +from .const import DEFAULT_REDIRECT_LIMIT + +# Preferred clock, based on which one is more accurate on a given system. +if sys.platform == "win32": + preferred_clock = time.perf_counter +else: + preferred_clock = time.time + + +class Session(BaseSession): + """A Ja3Request session. + + Provides cookie persistence, connection-pooling, and configuration. + """ + + def __init__(self): + super().__init__() + + self.headers = default_headers() + self.max_redirects = DEFAULT_REDIRECT_LIMIT + + def ready_request(self, request): + """ + Ready to send request. + :param request: + :return: + """ + + def request( + self, + method: AnyStr, + url: AnyStr, + params: Union[Dict[AnyStr, Any], ByteString] = None, + data: Union[Dict[AnyStr, Any], List, Tuple, ByteString] = None, + headers: Dict[AnyStr, AnyStr] = None, + cookies: Union[Dict[AnyStr, AnyStr], CookieJar] = None, + # files = None, + auth: Tuple = None, + timeout: float = None, + allow_redirects: bool = True, + proxies: Dict[AnyStr, AnyStr] = None, + # json: = None, + ): + """ + Request + :param method: + :param url: + :param params: + :param data: + :param headers: + :param cookies: + :param auth: + :param timeout: + :param allow_redirects: + :param proxies: + :return: + """ + + def get(self, url, **kwargs): + """ + Send a GET request. + :param url: + :param kwargs: + :return: + """ + + def options(self, url, **kwargs): + """ + Send a OPTIONS request. + :param url: + :param kwargs: + :return: + """ + + def head(self, url, **kwargs): + """ + Send a HEAD request. + :param url: + :param kwargs: + :return: + """ + + def post(self, url, data=None, json=None, **kwargs): + """ + Send a POST request. + :param url: + :param data: + :param json: + :param kwargs: + :return: + """ + + def put(self, url, data=None, **kwargs): + """ + Send a PUT request. + :param url: + :param data: + :param kwargs: + :return: + """ + + def patch(self, url, data=None, **kwargs): + """ + Send a PATCH request. + :param url: + :param data: + :param kwargs: + :return: + """ + + def delete(self, url, data=None, **kwargs): + """ + Send a DELETE request. + :param url: + :param data: + :param kwargs: + :return: + """ + + def send(self): + """ + Send a ready request. + :return: + """ diff --git a/ja3requests/utils.py b/ja3requests/utils.py new file mode 100644 index 0000000..a5086a1 --- /dev/null +++ b/ja3requests/utils.py @@ -0,0 +1,89 @@ +""" +ja3requests.utils +~~~~~~~~~~~~~~~~~ + +This module provides utility functions. +""" + +import platform +from base64 import b64encode +from typing import Union, AnyStr, List +from .__version__ import __version__ + + +ACCEPT_ENCODING = "gzip,deflate" + + +def b(s: AnyStr): # pylint: disable=C + """ + String encode latin1 + :param s: + :return: + """ + return s.encode("latin1") + + +def default_user_agent(agent: AnyStr = "Ja3Requests"): + """ + Return a string representing the default user agent. + :param agent: + :return: str + """ + + return f"Python/{platform.python_version()} ({platform.system()}; {platform.platform()}) {agent}/{__version__}" + + +def make_headers( + keep_alive: bool = None, + accept_encoding: Union[AnyStr, List[AnyStr]] = None, + user_agent: AnyStr = None, + basic_auth: AnyStr = None, + proxy_basic_auth: AnyStr = None, + disable_cache: bool = None, +): + """ + Shortcuts for generating request headers. + :param keep_alive: + :param accept_encoding: + :param user_agent: + :param basic_auth: username:password + :param proxy_basic_auth: username:password + :param disable_cache: + :return: dict + """ + headers = {"Accept": "*/*"} + if accept_encoding: + if isinstance(accept_encoding, str): + pass + elif isinstance(accept_encoding, list): + accept_encoding = ",".join(accept_encoding) + else: + accept_encoding = ACCEPT_ENCODING + headers["Accept-Encoding"] = accept_encoding + + headers["User-Agent"] = user_agent if user_agent else default_user_agent() + + if keep_alive: + headers["Connection"] = "keep-alive" + + if basic_auth: + headers["Authorization"] = "Basic " + b64encode(b(basic_auth)).decode("utf-8") + + if proxy_basic_auth: + headers["Proxy-Authorization"] = "Basic " + b64encode( + b(proxy_basic_auth) + ).decode("utf-8") + + if disable_cache: + headers["Cache-Control"] = "no-cache" + + return headers + + +def default_headers(): + """ + Return default headers. + :return: + """ + + return make_headers(keep_alive=True) diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..5ec8f56 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,25 @@ +[tool.black] +skip-string-normalization = true + +[tool.pylint] +disable = [ + "invalid-name", + "arguments-differ", + "no-name-in-module", + "too-many-arguments", + "too-few-public-methods", + "too-many-public-methods", + "too-many-instance-attributes", + "attribute-defined-outside-init", +# "too-many-return-statements", +# "broad-exception-caught", +# "no-member", +# "wrong-import-order", +# "consider-using-f-string", +] +#argument-rgx = "[a-z0-9]+(_[a-z]+)*" +#attr-rgx = "_?_?[a-z0-9]+(_[a-z]+)*_?_?" +#variable-rgx = "_?_?[a-z0-9]+(_[a-z]+)*_?_?" +max-line-length = 240 +#ignore = "test" +#min-similarity-lines = 20 diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000..4ce232a --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,4 @@ +black +pylint +pytest +twine diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..e69de29 diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..6e1df57 --- /dev/null +++ b/setup.py @@ -0,0 +1,66 @@ +import os +import sys +from setuptools import setup +from setuptools.command.test import test as TestCommand + + +class PyTest(TestCommand): + user_options = [("pytest-args=", "a", "Arguments to pass into py.test")] + + def initialize_options(self): + TestCommand.initialize_options(self) + try: + from multiprocessing import cpu_count + + self.pytest_args = ["-n", str(cpu_count()), "--boxed"] + except (ImportError, NotImplementedError): + self.pytest_args = ["-n", "1", "--boxed"] + + def finalize_options(self): + TestCommand.finalize_options(self) + self.test_args = [] + self.test_suite = True + + def run_tests(self): + import pytest + + errno = pytest.main(self.pytest_args) + sys.exit(errno) + + +with open("requirements.txt", "r", encoding="utf8") as f: + requires = f.readlines() + +with open("requirements-dev.txt", "r", encoding="utf8") as f: + test_requirements = f.readlines() + +about = {} +here = os.path.abspath(os.path.dirname(__file__)) +version_file = os.path.join(here, "ja3requests", "__version__.py") +with open(version_file, "r", encoding="utf8") as f: + exec(f.read(), about) + +with open("README.md", "r", encoding="utf8") as f: + readme = f.read() + + +setup( + name=about["__title__"], + version=about["__version__"], + description=about["__description__"], + long_description=readme, + long_description_content_type="text/markdown", + keywords=["pip", "ja3requests", "ja3", "requests"], + license=about["__license__"], + author=about["__author__"], + author_email=about["__author_email__"], + url=about["__url__"], + packages=["ja3requests"], + package_dir={"ja3requests": "ja3requests"}, + zip_safe=False, + include_package_data=True, + platforms="any", + install_requires=requires, + tests_require=test_requirements, + cmdclass={"test": PyTest}, +) diff --git a/test/test_utils.py b/test/test_utils.py new file mode 100644 index 0000000..2ca8105 --- /dev/null +++ b/test/test_utils.py @@ -0,0 +1,14 @@ +import unittest +from ja3requests.utils import default_headers + + +class TestUtils(unittest.TestCase): + + def test_default_headers(self): + + result = default_headers() + print(result) + + +if __name__ == '__main__': + unittest.main()