From 4b66260cb7c88a41a971d2458d75efd556f2e00e Mon Sep 17 00:00:00 2001 From: Sylvain MARIE Date: Fri, 27 Nov 2020 11:51:16 +0100 Subject: [PATCH 1/4] New helper method `set_http_proxy` in `Session` to set proxies in a more user friendly way --- requests/sessions.py | 85 ++++++++++++++++++++++++++++++++++++++++++ tests/test_requests.py | 35 +++++++++++++++++ 2 files changed, 120 insertions(+) diff --git a/requests/sessions.py b/requests/sessions.py index fdf7e9fe35..bb93e9cf2e 100644 --- a/requests/sessions.py +++ b/requests/sessions.py @@ -698,6 +698,91 @@ def send(self, request, **kwargs): return r + def set_http_proxy(self, + http_scheme='http', # type: str + http_host=None, # type: str + http_port=80, # type: int + http_url=None, # type: str + use_http_proxy_for_https_requests=False, # type: bool + https_scheme='https', # type: str + https_host=None, # type: str + https_port=443, # type: int + https_url=None, # type: str + replace=False # type: bool + ): + """Update or replace self.proxies with the provided proxy information. + + This method updates or replaces (depending on the value of `replace`) the dictionary in `self.proxies` with the + provided information. For each kind of connection (http and https), there are two ways to pass the information: + either as an url string (`http_url`, `https_url`), or split in schema/host/port, with sensible defaults. + In addition if the exact same proxy information is to be used for http and https, you can pass only the http + one and set `use_http_proxy_for_https_requests` to True. + + See :ref:`Proxies documentation <_proxies>` for details. + + :param http_host: (optional) a string indicating the http proxy host, for example '10.10.1.10' or 'acme.com'. + :param http_port: (optional) an int indicating the http proxy port, for example `3128`. + :param http_scheme: (optional) a string indicating the scheme to use for http proxy. By default this is 'http' + but you can consider using 'socks5', 'socks5h'. See documentation for details. + :param http_url: (optional) a string indicating the full http proxy url. For example 'http://10.10.1.10:3128' + or 'http://user:pass@10.10.1.10:3128/' or 'socks5://user:pass@host:port'. + Only one of {http_scheme + http_host + http_port} or {http_url} should be provided. + :param use_http_proxy_for_https_requests: (optional) a boolean indicating whether the information provided for + the http proxy should be copied for the https proxy. Note that the full url will be copied including the + scheme (so by default 'http'). + :param https_host: (optional) a string indicating the https proxy host, for example '10.10.1.10' or 'acme.com'. + :param https_port: (optional) an int indicating the https proxy port, for example `3128`. + :param https_scheme: (optional) a string indicating the scheme to use for https proxy. By default this is + 'https' but you can consider using 'socks5', 'socks5h'. See documentation for details. + :param https_url: (optional) a string indicating the full https proxy url. For example 'https://10.10.1.10:3128' + or 'http://user:pass@10.10.1.10:3128/' or 'socks5://user:pass@host:port'. + Only one of {https_scheme + https_host + https_port} or {https_url} should be provided. + :param replace: (optional) a boolean indicating if the provided information should replace the existing one + (True) or just update it (False, default). + :return: + """ + proxies = dict() + + # HTTPS + if http_host is not None: + # (a) scheme + host + port + if http_url is not None: + raise ValueError("Only one of `http_host` and `http_url` should be provided") + proxies['http'] = "%s://%s:%s" % (http_scheme, http_host, http_port) + elif http_url is not None: + # (b) full url + proxies['http'] = http_url + elif http_port != 80 or http_scheme != 'http': + raise ValueError("An `http_host` should be provided if you wish to change `http_port` or `http_scheme`") + + # HTTPS + if use_http_proxy_for_https_requests: + # (a) copy the information from http + if https_host is not None or https_url is not None or https_port != 443 or https_scheme != "https": + raise ValueError("`use_http_proxy_for_https_requests` was set to `True` but custom information for " + "https was provided.") + try: + proxies['https'] = proxies['http'] + except KeyError: + raise ValueError("`use_http_proxy_for_https_requests` was set to `True` but no information was " + "provided for the http proxy") + elif https_host is not None: + # (b) scheme + host + port + if https_url is not None: + raise ValueError("Only one of `https_host` and `https_url` should be provided") + proxies['https'] = '%s://%s:%s' % (https_scheme, https_host, https_port) + elif https_url is not None: + # (c) full url + proxies['https'] = https_url + elif https_port != 443 or https_scheme != 'https': + raise ValueError("An `https_host` should be provided if you wish to change `https_port` or `https_scheme`") + + # Replace or update (default) the configuration + if replace: + self.proxies = proxies + else: + self.proxies.update(proxies) + def merge_environment_settings(self, url, proxies, stream, verify, cert): """ Check the environment and merge it with some settings. diff --git a/tests/test_requests.py b/tests/test_requests.py index 5b6a7f5847..b0abe1be1d 100644 --- a/tests/test_requests.py +++ b/tests/test_requests.py @@ -151,6 +151,41 @@ def test_whitespaces_are_removed_from_url(self): request = requests.Request('GET', ' http://example.com').prepare() assert request.url == 'http://example.com/' + def test_proxies_shortcut(self): + """Test that the set_http_proxy helper function works correctly""" + s = requests.Session() + + # nominal + s.set_http_proxy(http_scheme='socks5', http_host='acme.com', http_port=999) + assert s.proxies == {'http': 'socks5://acme.com:999'} + s.set_http_proxy(https_scheme='socks5h', https_host='acme.org', https_port=80) + assert s.proxies == {'http': 'socks5://acme.com:999', 'https': 'socks5h://acme.org:80'} + + # one can not specify a scheme or a port without passing a host + with pytest.raises(ValueError): + s.set_http_proxy(http_scheme='socks5') + with pytest.raises(ValueError): + s.set_http_proxy(https_scheme='socks5') + with pytest.raises(ValueError): + s.set_http_proxy(http_port=999) + with pytest.raises(ValueError): + s.set_http_proxy(https_port=999) + + # use_http_proxy_for_https_requests requires http related information to be present + with pytest.raises(ValueError): + s.set_http_proxy(use_http_proxy_for_https_requests=True) + + # reuse http info for https + s.set_http_proxy(http_url='http://10.10.10.10:80', use_http_proxy_for_https_requests=True) + assert s.proxies == {'http': 'http://10.10.10.10:80', 'https': 'http://10.10.10.10:80'} + + # replace instead of update + s.set_http_proxy(https_url='http://10.10.10.20:80', replace=True) + assert s.proxies == {'https': 'http://10.10.10.20:80'} + + # https url + s.set_http_proxy(https_url='http://10.10.10.20:80', replace=True) + @pytest.mark.parametrize('scheme', ('http://', 'HTTP://', 'hTTp://', 'HttP://')) def test_mixed_case_scheme_acceptable(self, httpbin, scheme): s = requests.Session() From 2fddbe36063812788a175617aceafe2965d40646 Mon Sep 17 00:00:00 2001 From: Sylvain MARIE Date: Fri, 27 Nov 2020 11:51:47 +0100 Subject: [PATCH 2/4] Updated proxies documentation --- AUTHORS.rst | 1 + docs/user/advanced.rst | 35 ++++++++++++++++++++++++++++++++++- 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/AUTHORS.rst b/AUTHORS.rst index 1c03a68319..f434910468 100644 --- a/AUTHORS.rst +++ b/AUTHORS.rst @@ -190,3 +190,4 @@ Patches and Suggestions - Antti Kaihola (`@akaihola `_) - "Dull Bananas" (`@dullbananas `_) - Alessio Izzo (`@aless10 `_) +- Sylvain Marié (`@smarie `_) diff --git a/docs/user/advanced.rst b/docs/user/advanced.rst index 7cb872e48a..54cebaaa9b 100644 --- a/docs/user/advanced.rst +++ b/docs/user/advanced.rst @@ -589,8 +589,31 @@ If you need to use a proxy, you can configure individual requests with the requests.get('http://example.org', proxies=proxies) +Alternatively you can configure it once for an entire +:class:`Session `, either manually:: + + import requests + + proxies = { + 'http': 'http://10.10.1.10:3128', + 'https': 'http://10.10.1.10:1080', + } + session = request.Session() + session.proxies.update(proxies) + + session.get('http://example.org') + +or with the helper method :func:`set_http_proxy `:: + + import requests + + session = request.Session() + session.set_http_proxy(http_url='http://10.10.1.10:3128', https_url='http://10.10.1.10:1080') + + session.get('http://example.org') + You can also configure proxies by setting the environment variables -``HTTP_PROXY`` and ``HTTPS_PROXY``. +``HTTP_PROXY``, ``HTTPS_PROXY``, ``NO_PROXY`` and ``CURL_CA_BUNDLE``. :: @@ -615,6 +638,16 @@ any request to the given scheme and exact hostname. Note that proxy URLs must include the scheme. +Finally, note that using a proxy for https connections typically requires your local machine to trust the +proxy's root certificate. By default the list of certificates trusted by Requests can be found with:: + + from requests.utils import DEFAULT_CA_BUNDLE_PATH + print(DEFAULT_CA_BUNDLE_PATH) + +You override this default certificate bundle by setting the standard ``CURL_CA_BUNDLE`` environment variable +to another file path. + + SOCKS ^^^^^ From d1ca591c0fc0cfac634c74ed8b283c47ecffa9f8 Mon Sep 17 00:00:00 2001 From: Sylvain MARIE Date: Fri, 27 Nov 2020 17:05:17 +0100 Subject: [PATCH 3/4] Revert "New helper method `set_http_proxy` in `Session` to set proxies in a more user friendly way" This reverts commit 4b66260c --- requests/sessions.py | 85 ------------------------------------------ tests/test_requests.py | 35 ----------------- 2 files changed, 120 deletions(-) diff --git a/requests/sessions.py b/requests/sessions.py index bb93e9cf2e..fdf7e9fe35 100644 --- a/requests/sessions.py +++ b/requests/sessions.py @@ -698,91 +698,6 @@ def send(self, request, **kwargs): return r - def set_http_proxy(self, - http_scheme='http', # type: str - http_host=None, # type: str - http_port=80, # type: int - http_url=None, # type: str - use_http_proxy_for_https_requests=False, # type: bool - https_scheme='https', # type: str - https_host=None, # type: str - https_port=443, # type: int - https_url=None, # type: str - replace=False # type: bool - ): - """Update or replace self.proxies with the provided proxy information. - - This method updates or replaces (depending on the value of `replace`) the dictionary in `self.proxies` with the - provided information. For each kind of connection (http and https), there are two ways to pass the information: - either as an url string (`http_url`, `https_url`), or split in schema/host/port, with sensible defaults. - In addition if the exact same proxy information is to be used for http and https, you can pass only the http - one and set `use_http_proxy_for_https_requests` to True. - - See :ref:`Proxies documentation <_proxies>` for details. - - :param http_host: (optional) a string indicating the http proxy host, for example '10.10.1.10' or 'acme.com'. - :param http_port: (optional) an int indicating the http proxy port, for example `3128`. - :param http_scheme: (optional) a string indicating the scheme to use for http proxy. By default this is 'http' - but you can consider using 'socks5', 'socks5h'. See documentation for details. - :param http_url: (optional) a string indicating the full http proxy url. For example 'http://10.10.1.10:3128' - or 'http://user:pass@10.10.1.10:3128/' or 'socks5://user:pass@host:port'. - Only one of {http_scheme + http_host + http_port} or {http_url} should be provided. - :param use_http_proxy_for_https_requests: (optional) a boolean indicating whether the information provided for - the http proxy should be copied for the https proxy. Note that the full url will be copied including the - scheme (so by default 'http'). - :param https_host: (optional) a string indicating the https proxy host, for example '10.10.1.10' or 'acme.com'. - :param https_port: (optional) an int indicating the https proxy port, for example `3128`. - :param https_scheme: (optional) a string indicating the scheme to use for https proxy. By default this is - 'https' but you can consider using 'socks5', 'socks5h'. See documentation for details. - :param https_url: (optional) a string indicating the full https proxy url. For example 'https://10.10.1.10:3128' - or 'http://user:pass@10.10.1.10:3128/' or 'socks5://user:pass@host:port'. - Only one of {https_scheme + https_host + https_port} or {https_url} should be provided. - :param replace: (optional) a boolean indicating if the provided information should replace the existing one - (True) or just update it (False, default). - :return: - """ - proxies = dict() - - # HTTPS - if http_host is not None: - # (a) scheme + host + port - if http_url is not None: - raise ValueError("Only one of `http_host` and `http_url` should be provided") - proxies['http'] = "%s://%s:%s" % (http_scheme, http_host, http_port) - elif http_url is not None: - # (b) full url - proxies['http'] = http_url - elif http_port != 80 or http_scheme != 'http': - raise ValueError("An `http_host` should be provided if you wish to change `http_port` or `http_scheme`") - - # HTTPS - if use_http_proxy_for_https_requests: - # (a) copy the information from http - if https_host is not None or https_url is not None or https_port != 443 or https_scheme != "https": - raise ValueError("`use_http_proxy_for_https_requests` was set to `True` but custom information for " - "https was provided.") - try: - proxies['https'] = proxies['http'] - except KeyError: - raise ValueError("`use_http_proxy_for_https_requests` was set to `True` but no information was " - "provided for the http proxy") - elif https_host is not None: - # (b) scheme + host + port - if https_url is not None: - raise ValueError("Only one of `https_host` and `https_url` should be provided") - proxies['https'] = '%s://%s:%s' % (https_scheme, https_host, https_port) - elif https_url is not None: - # (c) full url - proxies['https'] = https_url - elif https_port != 443 or https_scheme != 'https': - raise ValueError("An `https_host` should be provided if you wish to change `https_port` or `https_scheme`") - - # Replace or update (default) the configuration - if replace: - self.proxies = proxies - else: - self.proxies.update(proxies) - def merge_environment_settings(self, url, proxies, stream, verify, cert): """ Check the environment and merge it with some settings. diff --git a/tests/test_requests.py b/tests/test_requests.py index b0abe1be1d..5b6a7f5847 100644 --- a/tests/test_requests.py +++ b/tests/test_requests.py @@ -151,41 +151,6 @@ def test_whitespaces_are_removed_from_url(self): request = requests.Request('GET', ' http://example.com').prepare() assert request.url == 'http://example.com/' - def test_proxies_shortcut(self): - """Test that the set_http_proxy helper function works correctly""" - s = requests.Session() - - # nominal - s.set_http_proxy(http_scheme='socks5', http_host='acme.com', http_port=999) - assert s.proxies == {'http': 'socks5://acme.com:999'} - s.set_http_proxy(https_scheme='socks5h', https_host='acme.org', https_port=80) - assert s.proxies == {'http': 'socks5://acme.com:999', 'https': 'socks5h://acme.org:80'} - - # one can not specify a scheme or a port without passing a host - with pytest.raises(ValueError): - s.set_http_proxy(http_scheme='socks5') - with pytest.raises(ValueError): - s.set_http_proxy(https_scheme='socks5') - with pytest.raises(ValueError): - s.set_http_proxy(http_port=999) - with pytest.raises(ValueError): - s.set_http_proxy(https_port=999) - - # use_http_proxy_for_https_requests requires http related information to be present - with pytest.raises(ValueError): - s.set_http_proxy(use_http_proxy_for_https_requests=True) - - # reuse http info for https - s.set_http_proxy(http_url='http://10.10.10.10:80', use_http_proxy_for_https_requests=True) - assert s.proxies == {'http': 'http://10.10.10.10:80', 'https': 'http://10.10.10.10:80'} - - # replace instead of update - s.set_http_proxy(https_url='http://10.10.10.20:80', replace=True) - assert s.proxies == {'https': 'http://10.10.10.20:80'} - - # https url - s.set_http_proxy(https_url='http://10.10.10.20:80', replace=True) - @pytest.mark.parametrize('scheme', ('http://', 'HTTP://', 'hTTp://', 'HttP://')) def test_mixed_case_scheme_acceptable(self, httpbin, scheme): s = requests.Session() From f02a80cbe85c16c99cd0c4df63aeb56c0044865a Mon Sep 17 00:00:00 2001 From: Sylvain MARIE Date: Fri, 27 Nov 2020 17:44:13 +0100 Subject: [PATCH 4/4] Updated proxies documentation --- docs/user/advanced.rst | 48 ++++++++++++++++++++++++------------------ 1 file changed, 28 insertions(+), 20 deletions(-) diff --git a/docs/user/advanced.rst b/docs/user/advanced.rst index 54cebaaa9b..d930a68d25 100644 --- a/docs/user/advanced.rst +++ b/docs/user/advanced.rst @@ -590,7 +590,7 @@ If you need to use a proxy, you can configure individual requests with the requests.get('http://example.org', proxies=proxies) Alternatively you can configure it once for an entire -:class:`Session `, either manually:: +:class:`Session `:: import requests @@ -603,19 +603,12 @@ Alternatively you can configure it once for an entire session.get('http://example.org') -or with the helper method :func:`set_http_proxy `:: - - import requests - - session = request.Session() - session.set_http_proxy(http_url='http://10.10.1.10:3128', https_url='http://10.10.1.10:1080') - - session.get('http://example.org') - -You can also configure proxies by setting the environment variables -``HTTP_PROXY``, ``HTTPS_PROXY``, ``NO_PROXY`` and ``CURL_CA_BUNDLE``. - -:: +When the proxies configuration is not overridden in python as shown above, +by default Requests relies on the proxy configuration defined by standard +environment variables ``http_proxy``, ``https_proxy``, ``no_proxy`` and +``curl_ca_bundle``. Uppercase variants of these variables are also supported. +You can therefore set them to configure Requests (only set the ones relevant +to your needs):: $ export HTTP_PROXY="http://10.10.1.10:3128" $ export HTTPS_PROXY="http://10.10.1.10:1080" @@ -624,9 +617,17 @@ You can also configure proxies by setting the environment variables >>> import requests >>> requests.get('http://example.org') -To use HTTP Basic Auth with your proxy, use the `http://user:password@host/` syntax:: +To use HTTP Basic Auth with your proxy, use the `http://user:password@host/` +syntax in any of the above configuration entries:: + + $ export HTTPS_PROXY="http://user:pass@10.10.1.10:1080" + + $ python + >>> proxies = {'http': 'http://user:pass@10.10.1.10:3128/'} - proxies = {'http': 'http://user:pass@10.10.1.10:3128/'} +.. warning:: Storing sensitive username and password information in an + environment variable or a version-controled file is a security risk and is + highly discouraged. To give a proxy for a specific scheme and host, use the `scheme://hostname` form for the key. This will match for @@ -638,15 +639,22 @@ any request to the given scheme and exact hostname. Note that proxy URLs must include the scheme. -Finally, note that using a proxy for https connections typically requires your local machine to trust the -proxy's root certificate. By default the list of certificates trusted by Requests can be found with:: +Finally, note that using a proxy for https connections typically requires your +local machine to trust the proxy's root certificate. By default the list of +certificates trusted by Requests can be found with:: from requests.utils import DEFAULT_CA_BUNDLE_PATH print(DEFAULT_CA_BUNDLE_PATH) -You override this default certificate bundle by setting the standard ``CURL_CA_BUNDLE`` environment variable -to another file path. +You override this default certificate bundle by setting the standard +``curl_ca_bundle`` environment variable to another file path:: + $ export curl_ca_bundle="/usr/local/myproxy_info/cacert.pem" + $ export https_proxy="http://10.10.1.10:1080" + + $ python + >>> import requests + >>> requests.get('https://example.org') SOCKS ^^^^^