From 9e3e4340b012168fc7507c0d6c59f96c9ed5814a Mon Sep 17 00:00:00 2001 From: kaliiiiiiiiii Date: Tue, 27 Jun 2023 09:55:30 +0200 Subject: [PATCH] dynamic proxies now officially supported:) --- README.md | 29 +++++++++--- src/selenium_profiles/scripts/profiles.py | 44 +---------------- .../scripts/{proxy_extension.py => proxy.py} | 47 ++----------------- src/selenium_profiles/webdriver.py | 22 +++++++-- 4 files changed, 48 insertions(+), 94 deletions(-) rename src/selenium_profiles/scripts/{proxy_extension.py => proxy.py} (71%) diff --git a/README.md b/README.md index e2756f9..aa168bd 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ * **Undetected** by Google, Cloudflare, creep-js .. * [Modifying headers](#Modify-headers) supported using [Selenium-Interceptor](https://github.com/kaliiiiiiiiii/Selenium-Interceptor) or seleniumwire * [Touch Actions](#Touch_actions) -* [proxies with authentication](https://github.com/kaliiiiiiiiii/Selenium-Profiles/discussions/6#discussioncomment-4704385) +* dynamic proxies with authentication * making single [POST](https://github.com/kaliiiiiiiiii/Selenium-Profiles/discussions/11#discussioncomment-4797109), GET or other requests using `driver.profiles.fetch(url)` ([syntax](https://developer.mozilla.org/en-US/docs/Web/API/fetch#syntax)) * headless unofficially supported * apply profile on already running driver with `driver.profiles.apply(profiles.Android())` @@ -76,11 +76,6 @@ profile = \ "gpu": False, "proxy": "http://example-proxy.com:9000", # note: auth not supported, "extension_paths": ["path/to/extension_1", ...], # directory, .crx or .zip - "auth_proxy": { - "host":"host", "port":9000, - "username":"user", "password":"password", - "temp_dir": "C:/Downloads/proxy_extension" - }, "args": ["--my-arg1", ...], "capabilities": {"cap_1":"val_1", "cap_2":"val_2"}, "experimental_options":{"option1":"value1", "option2":"value2"}, @@ -116,6 +111,10 @@ profile = \ "bitness": "", "wow64": False} } + }, +"proxy":{ + "proxy":"socks5://user1:pass@example_jost.com:5001", + "bypass_list":["localhost"] } } ``` @@ -206,6 +205,24 @@ input("Press ENTER to exit") driver.quit() # Execute on the End! ``` +### Set proxies dynamically or with options +```python +from selenium_profiles.webdriver import Chrome +from selenium_profiles.profiles import profiles + +profile = profiles.Windows() # or .Android() +profile["proxy"] = { + "proxy":"http://user1:pass1@example_host.com:41149" + } + +driver = Chrome(profile=profile, injector_options=True) + +driver.profiles.proxy.set_single("http://user2:pass2@example_host.com:41149") +print(driver.profiles.proxy.proxy) + +driver.quit() # Execute on the End! +``` + ### To export a profile: go to [https://js.do/kaliiiiiiiiiii/get_profile](https://js.do/kaliiiiiiiiiii/get_profile) in your browser and copy the text. diff --git a/src/selenium_profiles/scripts/profiles.py b/src/selenium_profiles/scripts/profiles.py index b9af898..6babcd3 100644 --- a/src/selenium_profiles/scripts/profiles.py +++ b/src/selenium_profiles/scripts/profiles.py @@ -231,7 +231,7 @@ def __init__(self, options, options_profile: dict or None = None, duplicate_poli self._profile_keys = ["sandbox", "window_size", "headless", "load_images", "incognito", "touch", "app", "gpu", "proxy", "args", "capabilities", "experimental_options", "adb", "adb_package", "use_running_app", - "extension_paths", "auth_proxy" + "extension_paths", ] self._supported_duplicate_policies = ["raise", "replace", "warn-replace", "skip", "warn-skip", "add", "warn-add"] @@ -257,7 +257,6 @@ def apply(self, options_profile): self.update_experimental_options(profile["experimental_options"]) self.adb_remote(profile["adb"], package=profile["adb_package"], use_running_app=profile["use_running_app"]) self.add_extensions(profile["extension_paths"], adb=profile["adb"]) - self.auth_proxy(profile["auth_proxy"]) return self.Options # noinspection PyIncorrectDocstring @@ -543,44 +542,3 @@ def add_extensions(self, extension_paths: None or list = None, adb: bool or None self.add_argument('--load-extension=' + extension_path) else: raise LookupError("Extension-path doesn't exist") - - def auth_proxy(self, config: dict = None): - """ - :param config: dict => - { - host: str - port: int - username: str | optional - password: str | optional - scheme: str | optional - temp_dir: str | optional - } - """ - from selenium_profiles.scripts.proxy_extension import make_extension - - if config: - auth_proxy = defaultdict(lambda: None) - auth_proxy.update(config) - - valid_key(auth_proxy.keys(), ["host", "port", "username", "password", "scheme", "temp_dir"], - 'profile_options auth_proxy => profile["options"]["auth_proxy"]') - - host = auth_proxy["host"] - port = auth_proxy["port"] - username = auth_proxy["username"] - password = auth_proxy["password"] - scheme = auth_proxy["scheme"] - temp_dir = auth_proxy["temp_dir"] - - if not host: - raise ValueError("value 'host' is required") - if not port: - raise ValueError("value 'port' is required") - - if not scheme: - scheme = "http" - - # noinspection PyTypeChecker - path = make_extension(host=host, port=port, username=str(username), password=str(password), - scheme=scheme, temp_dir=temp_dir) - self.add_extensions(extension_paths=[path]) diff --git a/src/selenium_profiles/scripts/proxy_extension.py b/src/selenium_profiles/scripts/proxy.py similarity index 71% rename from src/selenium_profiles/scripts/proxy_extension.py rename to src/selenium_profiles/scripts/proxy.py index a08fcb3..541ce42 100644 --- a/src/selenium_profiles/scripts/proxy_extension.py +++ b/src/selenium_profiles/scripts/proxy.py @@ -1,46 +1,5 @@ -from selenium_profiles.utils.utils import read, write, sel_profiles_path from collections import defaultdict -from selenium_profiles.webdriver import profiles - - -def make_extension(host:str, port:int, username:str or None = None, password:str or None=None, scheme:str="http", temp_dir:str=None): - # noinspection GrazieInspection - """ - :param host: ip or url | str - :param port: int - :param username: str, optional - :param password: optional - :param scheme: "http" | "https" | "quic" | "socks4" | "socks5" - :param temp_dir: str, optional - - usage: - - host = "3.144.32.194" # IP - port = 80 - user = "user1" - password = "my_pass" - scheme="http" - - auth_proxy = {"host":host,"port":port,"username":user, "password":password, "scheme":scheme, "temp_dir": "C:/Downloads/proxy_extension"} - - profile["options"]["extensions"] = {"auth_proxy":auth_proxy} - """ - schemes = ["http" , "https" , "quic" , "socks4" , "socks5"] - if temp_dir is None: - temp_dir = sel_profiles_path()+"files/tmp/proxy_extension/" - else: - if temp_dir[-1] != "/" or temp_dir[-1] != "\\": - temp_dir += "/" - if scheme in schemes: - background_js = read("files/proxy_extension/background.js", sel_root=True) % (scheme, host, str(port), str(username), str(password)) - manifest_json = read("files/proxy_extension/manifest.json", sel_root=True) - write(temp_dir+"background.js",content=background_js, sel_root=False) - write(temp_dir+"manifest.json", content=manifest_json, sel_root=False) - return temp_dir[:-1] # remove "/" in the end - else: - raise ValueError("scheme needs to be: "+str(schemes)+",but got \""+scheme+'"') - class DynamicProxy: def __init__(self, driver, injector=None): self._injector = injector @@ -57,6 +16,7 @@ def __init__(self, driver, injector=None): raise ModuleNotFoundError("either seleniumwire or selenium-injecter is needed for dynamic proxies") def str2val(self, url): + from urllib.parse import unquote creds = "" try: scheme, url = url.split("://") @@ -69,6 +29,7 @@ def str2val(self, url): values = {"host": host, "port": port, "scheme": scheme} if creds: username, password = creds.split(":") + username, password = unquote(username), unquote(password) if not (username and password): raise ValueError creds = {"password": password, "username": username} @@ -91,7 +52,9 @@ def val2str(self, proxy_dict: dict or None, creds: dict = None): return parsed # noinspection PyDefaultArgument - def set_single(self, proxy, bypass_list:list = ["localhost", "127.0.0.1"]): + def set_single(self, proxy, bypass_list:list or None = ["localhost", "127.0.0.1"]): + if not bypass_list: + bypass_list = ["localhost", "127.0.0.1"] if self._seleniumwire: self._driver.proxy = {"http":proxy, "https":proxy, "no_proxy":",".join(bypass_list)} elif self._injector: diff --git a/src/selenium_profiles/webdriver.py b/src/selenium_profiles/webdriver.py index c2b679e..175d808 100644 --- a/src/selenium_profiles/webdriver.py +++ b/src/selenium_profiles/webdriver.py @@ -1,6 +1,5 @@ import warnings from collections import defaultdict -from selenium_profiles.utils.utils import valid_key import typing from selenium.webdriver.chrome.service import Service as ChromeService @@ -22,6 +21,8 @@ def __init__(self, profile: dict = None, chrome_binary: str = None, executable_p import undetected_chromedriver as uc_webdriver from seleniumwire import webdriver as wire_webdriver + from selenium_profiles.utils.utils import valid_key + if not base_drivers: base_drivers = tuple() @@ -58,7 +59,7 @@ def __init__(self, profile: dict = None, chrome_binary: str = None, executable_p if not profile: profile = {} - valid_key(profile.keys(), ["cdp", "options"], "profile (selenium-profiles)") + valid_key(profile.keys(), ["cdp", "options", "proxy"], "profile (selenium-profiles)") if type(seleniumwire_options) is dict: kwargs.update({"seleniumwire_options": seleniumwire_options}) @@ -68,6 +69,14 @@ def __init__(self, profile: dict = None, chrome_binary: str = None, executable_p defdict = defaultdict(lambda: None) defdict.update(profile) profile = defdict + proxy = defaultdict(lambda :None) + # noinspection PyTypeChecker + if profile["proxy"]: + # noinspection PyTypeChecker + proxy.update(profile["proxy"]) + + if proxy["proxy"] and (not seleniumwire_options): + injector_options = True # sandbox handling for google-colab if is_colab(): @@ -165,6 +174,13 @@ def __init__(self, profile: dict = None, chrome_binary: str = None, executable_p self.execute_cdp_cmd("Page.addScriptToEvaluateOnNewDocument", {"source": "(function(){%s})()" % (utils_js + self.profiles.injector.connection_js + config)}) + if proxy["proxy"]: + from selenium_profiles.utils.utils import valid_key + # noinspection PyUnresolvedReferences + valid_key(proxy.keys(), ["proxy", "bypass_list"], '"profiles["proxy"]"') + # noinspection PyUnresolvedReferences + self.profiles.proxy.set_single(proxy["proxy"], bypass_list=proxy["bypass_list"]) + def get_cookies(self, urls:typing.List[str] = None) -> typing.List[dict]: arg = {} if urls: @@ -200,7 +216,7 @@ def __init__(self, driver, profile, cdp_handler=None, selenium_injector=None): from selenium_interceptor.interceptor import cdp_listener from selenium_profiles.scripts.driver_utils import requests, actions - from selenium_profiles.scripts.proxy_extension import DynamicProxy + from selenium_profiles.scripts.proxy import DynamicProxy self._driver = driver self._profile = profile