diff --git a/HISTORY.txt b/HISTORY.txt index f2ac01f..e54fb52 100644 --- a/HISTORY.txt +++ b/HISTORY.txt @@ -19,6 +19,10 @@ TBD use Px. Disallow any external clients unless --gateway specified along with a restrictive --allow definition - PR20 - Changed --socktimeout from int to float +- Added support to discover proxy info from Internet Options - Automatic Config + URL with PAC files, WPAD, or static proxy definition +- Added --proxyreload flag to configure interval between rediscovery of proxy + info v0.3.0 - 2018-02-19 - Fixed issue 9 - Added support for winkerberos to workaround pywin32 bug diff --git a/README.txt b/README.txt index 11e3daa..b8b7e16 100644 --- a/README.txt +++ b/README.txt @@ -1,24 +1,26 @@ -Px is a HTTP(s) proxy server that allows applications to authenticate through an NTLM proxy -server, typically used in corporate deployments, without having to deal with the actual NTLM -handshake. It is primarily designed to run on Windows systems and authenticates on behalf -of the application using the currently logged in Windows user account. +Px is a HTTP(s) proxy server that allows applications to authenticate through +an NTLM proxy server, typically used in corporate deployments, without having +to deal with the actual NTLM handshake. It is primarily designed to run on +Windows systems and authenticates on behalf of the application using the +currently logged in Windows user account. Px is very similar to "NTLM Authorization Proxy Server" (http://ntlmaps.sourceforge.net/) -and Cntlm (http://cntlm.sourceforge.net/) in that it sits between the corporate proxy and -applications and offloads the NTLM authentication. The primary difference in Px is to use -the currently logged in user's credentials to log in automatically rather than requiring the -user to provide the username, password (hash) and domain information. This is -accomplished by using Microsoft SSPI to generate the tokens and signatures required to -authenticate with the NTLM proxy. - -NTLMAps and Cntlm were designed for non-Windows users stuck behind a corporate proxy. -As a result, they require the user to provide the correct credentials to authenticate. On -Windows, the user has already logged in with his credentials so Px is designed for Windows -users who would like to use tools that aren't designed to deal with NTLM authentication, -without having to supply and maintain the credentials within Px. - -The following link from Microsoft provides a good starting point to understand how NTLM -authentication works: +and Cntlm (http://cntlm.sourceforge.net/) in that it sits between the corporate +proxy and applications and offloads the NTLM authentication. The primary +difference in Px is to use the currently logged in user's credentials to log in +automatically rather than requiring the user to provide the username, password +(hash) and domain information. This is accomplished by using Microsoft SSPI to +generate the tokens and signatures required to authenticate with the NTLM proxy. + +NTLMAps and Cntlm were designed for non-Windows users stuck behind a corporate +proxy. As a result, they require the user to provide the correct credentials +to authenticate. On Windows, the user has already logged in with his credentials +so Px is designed for Windows users who would like to use tools that aren't +designed to deal with NTLM authentication, without having to supply and maintain +the credentials within Px. + +The following link from Microsoft provides a good starting point to understand +how NTLM authentication works: https://msdn.microsoft.com/en-us/library/dd925287.aspx @@ -39,30 +41,32 @@ Px can be obtained in multiple ways:- Running the source directly requires Python and all dependencies installed. -Once downloaded, extract to a folder of choice and use the --save and --install commands -as documented below. +Once downloaded, extract to a folder of choice and use the --save and --install +commands as documented below. Configuration -Px requires only one piece of information in order to function - the server name and port of -the NTLM proxy server. This needs to be configured in px.ini. Without this, Px will not work -and exit immediately. +Px requires only one piece of information in order to function - the server +name and port of the NTLM proxy server. This needs to be configured in px.ini. +If not specified, Px will check Internet Options for any proxy definitions and +use them. Without this, Px will not work and exit immediately. -The noproxy capability allows Px to connect to hosts in the configured subnets directly, -bypassing the NTLM proxy altogether. This allows clients to connect to hosts within the -intranet without requiring additional configuration for each client or at the NTLM proxy. -If noproxy is defined, the NTLM proxy is optional - this allows Px to run as a regular -proxy full time if required. +The noproxy capability allows Px to connect to hosts in the configured subnets +directly, bypassing the NTLM proxy altogether. This allows clients to connect +to hosts within the intranet without requiring additional configuration for +each client or at the NTLM proxy. If noproxy is defined, the NTLM proxy is +optional - this allows Px to run as a regular proxy full time if required. -There are a few other settings to tweak in the INI file but most are self-explanatory. All -settings can be specified on the command line for convenience. The INI file can also be -created or updated from the command line using --save. +There are a few other settings to tweak in the INI file but most are obvious. +All settings can be specified on the command line for convenience. The INI file +can also be created or updated from the command line using --save. -The binary distribution of Px runs in the background once started and can be quit by -running "px --quit". When run directly using Python, use CTRL-C to quit. +The binary distribution of Px runs in the background once started and can be +quit by running "px --quit". When run directly using Python, use CTRL-C to quit. -Px can also be setup to automatically run on startup with the --install flag. This is done -by adding an entry into the Window registry which can be removed with --uninstall. +Px can also be setup to automatically run on startup with the --install flag. +This is done by adding an entry into the Window registry which can be removed +with --uninstall. Usage @@ -145,6 +149,11 @@ Configuration: --socktimeout= settings:socktimeout= Timeout in seconds for connections before giving up. Valid float, default: 5 + --proxyreload= settings:proxyreload= + Time interval in seconds before reloading proxy info. Valid int, default: 60 + Proxy info is reloaded from a PAC file found via WPAD or AutoConfig URL, or + manual proxy info defined in Internet Options + --foreground settings:foreground= Run in foreground when frozen or with pythonw.exe. 0 or 1, default: 0 Px will attach to the console and write to it even though the prompt is @@ -176,7 +185,7 @@ Examples px --proxy=proxyserver.com:80 --gateway NOTE: - In Docker for Windows you need to set your proxy to http://:3128 (or actual port + In Docker for Windows you need to set your proxy to http://:3128 (oractual port Px is listening to) and be aware of https://github.com/docker/for-win/issues/1380. Workaround: docker build --build-arg http_proxy=http://:3128 --build-arg @@ -184,24 +193,25 @@ NOTE: Dependencies -Px doesn't have any GUI and runs completely in the background. It is distributed using -Python 3.x and PyInstaller to have a self-contained executable but can also be run using a -Python distribution with the following additional packages. +Px doesn't have any GUI and runs completely in the background. It is distributed +using Python 3.x and PyInstaller to have a self-contained executable but can +also be run using a Python distribution with the following additional packages. - netaddr, psutil, winkerberos + netaddr, psutil, pypac, winkerberos futures on Python 2.x -In order to make Px a capable proxy server, it is designed to run in multiple processes. The -number of parallel workers or processes is configurable. However, this only works on Python -3.3+ since that's when support was added to share sockets across processes in Windows. On -older versions of Python, Px will run multi-threaded but in a single process. The number of -threads per process is also configurable. +In order to make Px a capable proxy server, it is designed to run in multiple +processes. The number of parallel workers or processes is configurable. However, +this only works on Python 3.3+ since that's when support was added to share +sockets across processes in Windows. On older versions of Python, Px will run +multi-threaded but in a single process. The number of threads per process is +also configurable. Feedback -Px is definitely a work in progress and any feedback or suggestions are welcome. It is hosted -on GitHub (https://github.com/genotrance/px) with an MIT license so issues, forks and PRs are -most appreciated. +Px is definitely a work in progress and any feedback or suggestions are welcome. +It is hosted on GitHub (https://github.com/genotrance/px) with an MIT license +so issues, forks and PRs are most appreciated. Credits diff --git a/build.bat b/build.bat index 7cbd67c..6646a29 100644 --- a/build.bat +++ b/build.bat @@ -4,11 +4,9 @@ rmdir /s /q build rmdir /s /q __pycache__ rmdir /s /q dist -pyinstaller --clean --noupx -w -F -i px.ico px.py +pyinstaller --clean --noupx -w -F -i px.ico --add-data "c:\Miniconda\Lib\site-packages\tld\res\effective_tld_names.dat.txt;tld\res" px.py copy px.ini dist\. copy *.txt dist\. -rem upx --best dist\px.exe -o dist\px.tmp -rem move /Y dist\px.tmp dist\px.exe del /q px.spec rmdir /s /q build diff --git a/px.ini b/px.ini index 5f4f0aa..6ccbc6d 100644 --- a/px.ini +++ b/px.ini @@ -57,6 +57,11 @@ idle = 30 ; Timeout in seconds for connections before giving up socktimeout = 5.0 +; Time interval in seconds before refreshing proxy info. Valid int, default: 60 +; Proxy info reloaded from a PAC file found via WPAD or AutoConfig URL, or +; manual proxy info defined in Internet Options +proxyreload = 60 + ; Run in foreground when frozen or with pythonw.exe. 0 or 1, default: 0 ; Px will attach to the console and write to it even though the prompt is ; available for further commands. CTRL-C in the console will exit Px diff --git a/px.py b/px.py index 1a0b8b8..94566b9 100644 --- a/px.py +++ b/px.py @@ -147,9 +147,14 @@ --idle= settings:idle= Idle timeout in seconds for HTTP connect sessions. Valid integer, default: 30 - --socktimeout= settings:socktimeout= + --socktimeout= settings:socktimeout= Timeout in seconds for connections before giving up. Valid float, default: 5 + --proxyreload= settings:proxyreload= + Time interval in seconds before refreshing proxy info. Valid int, default: 60 + Proxy info reloaded from a PAC file found via WPAD or AutoConfig URL, or + manual proxy info defined in Internet Options + --foreground settings:foreground= Run in foreground when frozen or with pythonw.exe. 0 or 1, default: 0 Px will attach to the console and write to it even though the prompt is @@ -160,6 +165,12 @@ Logs are written to working directory and over-written on startup A log is automatically created if Px crashes for some reason""" % __version__ +# Proxy modes - source of proxy info +MODE_NONE = 0 +MODE_CONFIG = 1 +MODE_PAC = 2 +MODE_MANUAL = 3 + class State(object): allow = netaddr.IPGlob("*.*.*.*") config = None @@ -168,6 +179,8 @@ class State(object): logger = None noproxy = netaddr.IPSet([]) pac = None + proxy_mode = MODE_NONE + proxy_refresh = None proxy_server = [] stdout = None useragent = "" @@ -192,16 +205,19 @@ def write(self, data): self.file.write(data) except: pass - self.file.flush() if self.stdout is not None: self.stdout.write(data) + self.flush() def flush(self): self.file.flush() + if self.stdout is not None: + self.stdout.flush() def dprint(*objs): if State.logger != None: print(multiprocessing.current_process().name + ": " + threading.current_thread().name + ": " + str(int(time.time())) + ": " + sys._getframe(1).f_code.co_name + ": ", end="") print(*objs) + sys.stdout.flush() def dfile(): return "debug-%s.log" % multiprocessing.current_process().name @@ -610,29 +626,45 @@ def fwd_resp(self, resp, headers, body): dprint("Done") def get_destination(self): + netloc = self.path + path = "/" + if self.command != "CONNECT": + parse = urlparse.urlparse(self.path, allow_fragments=False) + if parse.netloc: + netloc = parse.netloc + if ":" not in netloc: + port = parse.port + if not port: + if parse.scheme == "http": + port = 80 + elif parse.scheme == "https": + port = 443 + netloc = netloc + ":" + str(port) + + path = parse.path or "/" + if parse.params: + path = path + ";" + parse.params + if parse.query: + path = path + "?" + parse.query + dprint(netloc) + + if State.proxy_mode != MODE_CONFIG: + load_proxy() + + direct = -1 + if State.proxy_mode == MODE_PAC: + pac_proxy, direct = find_proxy_for_url(netloc) + if direct == 0: + ipport = netloc.split(":") + ipport[1] = int(ipport[1]) + dprint("Direct connection from PAC") + return tuple(ipport) + + if pac_proxy: + dprint("Proxy from PAC = " + str(pac_proxy)) + State.proxy_server = pac_proxy + if State.noproxy.size: - netloc = self.path - path = "/" - if self.command != "CONNECT": - parse = urlparse.urlparse(self.path, allow_fragments=False) - if parse.netloc: - netloc = parse.netloc - if ":" not in netloc: - port = parse.port - if not port: - if parse.scheme == "http": - port = 80 - elif parse.scheme == "https": - port = 443 - netloc = netloc + ":" + str(port) - - path = parse.path or "/" - if parse.params: - path = path + ";" + parse.params - if parse.query: - path = path + "?" + parse.query - - dprint(netloc) addr = [] spl = netloc.split(":", 1) try: @@ -739,18 +771,62 @@ def run_pool(): ### # Parse settings and command line -def find_proxy(): +def load_proxy(): # Return if proxies specified in Px config - if State.config.get("proxy", "server") != "": + # Check if need to refresh + if (State.proxy_mode == MODE_CONFIG or + (State.proxy_refresh is not None and + time.time() - State.proxy_refresh < State.config.getint("settings", "proxyreload"))): + dprint("Skip proxy refresh") return + # Reset proxy server list + State.proxy_mode = MODE_NONE + State.proxy_server = [] + # First use pypac for WPAD or PAC config if "pypac" in sys.modules: State.pac = pypac.get_pac() + # Fallback for file:/// URLs + if State.pac is None: + acu = pypac.windows.autoconfig_url_from_registry() + if acu: + acu = acu.replace("file:///", "") + if ":" not in acu: + acu = "c:\\" + acu + if os.path.isfile(acu): + dprint("Loading " + acu) + with open(acu, "r") as f: + State.pac = pypac.parser.PACFile(f.read()) # Fall back to any manual proxies defined in Internet Options - parse_proxy(",".join([urlparse.urlparse(i).netloc - for i in set(urlrequest.getproxies().values())])) + if State.pac is None: + parse_proxy(",".join([urlparse.urlparse(i).netloc + for i in set(urlrequest.getproxies().values())])) + + if State.proxy_server: + State.proxy_mode = MODE_MANUAL + else: + State.proxy_mode = MODE_PAC + + dprint("Proxy mode = " + str(State.proxy_mode)) + State.proxy_refresh = time.time() + +def find_proxy_for_url(netloc): + pac_proxy = [] + direct = -1 + count = 0 + if State.pac is not None: + for i in pypac.parser.parse_pac_value( + State.pac.find_proxy_for_url(netloc, netloc)): + if i != "DIRECT": + pac_proxy.append(tuple(urlparse.urlparse(i).netloc.split(":"))) + else: + direct = count + count += 1 + dprint("pypac = " + str(pac_proxy), str(direct)) + + return pac_proxy, direct def parse_proxy(proxystrs): if not proxystrs: @@ -887,19 +963,12 @@ def parse_config(): State.config.add_section("proxy") cfg_str_init("proxy", "server", "", parse_proxy) - cfg_int_init("proxy", "port", "3128") - cfg_str_init("proxy", "listen", "127.0.0.1") - cfg_str_init("proxy", "allow", "*.*.*.*", parse_allow) - cfg_int_init("proxy", "gateway", "0") - cfg_int_init("proxy", "hostonly", "0") - cfg_str_init("proxy", "noproxy", "", parse_noproxy) - cfg_str_init("proxy", "useragent", "", set_useragent) # [settings] section @@ -910,6 +979,7 @@ def parse_config(): cfg_int_init("settings", "threads", "5") cfg_int_init("settings", "idle", "30") cfg_float_init("settings", "socktimeout", "5.0") + cfg_int_init("settings", "proxyreload", "60") cfg_int_init("settings", "foreground", "0") cfg_int_init("settings", "log", "0" if State.logger is None else "1") @@ -933,7 +1003,7 @@ def parse_config(): elif "--useragent=" in sys.argv[i]: cfg_str_init("proxy", "useragent", val, set_useragent, True) else: - for j in ["workers", "threads", "idle"]: + for j in ["workers", "threads", "idle", "proxyreload"]: if "--" + j + "=" in sys.argv[i]: cfg_int_init("settings", j, val, True) @@ -982,8 +1052,12 @@ def parse_config(): elif "--save" in sys.argv: save() - find_proxy() - if not State.pac and not State.proxy_server and not State.config.get("proxy", "noproxy"): + if State.proxy_server: + State.proxy_mode = MODE_CONFIG + else: + load_proxy() + + if State.proxy_mode == MODE_NONE and not State.config.get("proxy", "noproxy"): print("No proxy server or noproxy list defined") sys.exit()