-
-
Notifications
You must be signed in to change notification settings - Fork 206
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
warcserver: SOCKS proxy: - add support for running warcserver through a socks proxy specified via SOCKS_HOST and SOCKS_PORT - move socks patch setup, http max_header adjustment to http module - logging: print stack trace only if debugging - add pysocks to extra_requirements, enable in ci - add simple test (not actual proxy) to check that connection through proxy is attempted - docs: add SOCKS proxy section to docs
- Loading branch information
Showing
6 changed files
with
131 additions
and
8 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -4,3 +4,4 @@ boto3 | |
uwsgi | ||
git+https://github.com/t0m/pyamf.git@python3 | ||
git+https://github.com/esnme/ultrajson.git | ||
pysocks |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,9 +1,78 @@ | ||
from requests.adapters import HTTPAdapter | ||
import requests | ||
import os | ||
|
||
import six.moves.http_client | ||
six.moves.http_client._MAXHEADERS = 10000 | ||
|
||
SOCKS_PROXIES = None | ||
orig_getaddrinfo = None | ||
|
||
|
||
#============================================================================= | ||
class DefaultAdapters(object): | ||
live_adapter = HTTPAdapter(max_retries=3) | ||
remote_adapter = HTTPAdapter(max_retries=3) | ||
|
||
requests.packages.urllib3.disable_warnings() | ||
|
||
|
||
#============================================================================= | ||
def patch_socks(): | ||
try: | ||
import socks | ||
except ImportError: #pragma: no cover | ||
print('Ignoring SOCKS_HOST: PySocks must be installed to use SOCKS proxy') | ||
return | ||
|
||
import socket | ||
|
||
socks_host = os.environ.get('SOCKS_HOST') | ||
socks_port = os.environ.get('SOCKS_PORT', 9050) | ||
|
||
# Set socks proxy and wrap the urllib module | ||
socks.set_default_proxy(socks.PROXY_TYPE_SOCKS5, socks_host, socks_port, True) | ||
#socket.socket = socks.socksocket # sets default socket to be the sockipy socket | ||
|
||
# store original getaddrinfo | ||
global orig_getaddrinfo | ||
orig_getaddrinfo = socks.socket.getaddrinfo | ||
|
||
# Perform DNS resolution through socket | ||
def getaddrinfo(*args): | ||
if args[0] in ('127.0.0.1', 'localhost'): | ||
res = orig_getaddrinfo(*args) | ||
|
||
else: | ||
res = [(socket.AF_INET, socket.SOCK_STREAM, 6, '', (args[0], args[1]))] | ||
|
||
return res | ||
|
||
socks.socket.getaddrinfo = getaddrinfo | ||
|
||
socks_url = 'socks5h://{0}:{1}'.format(socks_host, socks_port) | ||
|
||
global SOCKS_PROXIES | ||
SOCKS_PROXIES = {'http': socks_url, | ||
'https': socks_url} | ||
|
||
# ============================================================================= | ||
def unpatch_socks(): | ||
global orig_getaddrinfo | ||
if not orig_getaddrinfo: | ||
return | ||
|
||
import socks | ||
socks.socket.getaddrinfo = orig_getaddrinfo | ||
orig_getaddrinfo = None | ||
|
||
global SOCKS_PROXIES | ||
SOCKS_PROXIES = None | ||
|
||
|
||
# ============================================================================= | ||
if os.environ.get('SOCKS_HOST'): | ||
patch_socks() | ||
|
||
|
||
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,38 @@ | ||
from .base_config_test import BaseConfigTest, fmod_sl | ||
|
||
import pywb.warcserver.http as pywb_http | ||
import os | ||
import socket | ||
import gevent | ||
import pytest | ||
|
||
|
||
# ============================================================================ | ||
class TestSOCKSProxy(BaseConfigTest): | ||
@classmethod | ||
def setup_class(cls): | ||
os.environ['SOCKS_HOST'] = 'localhost' | ||
os.environ['SOCKS_PORT'] = '8080' | ||
|
||
pywb_http.patch_socks() | ||
import pywb.warcserver.resource.responseloader | ||
pywb.warcserver.resource.responseloader.SOCKS_PROXIES = pywb_http.SOCKS_PROXIES | ||
super(TestSOCKSProxy, cls).setup_class('config_test.yaml') | ||
|
||
@classmethod | ||
def teardown_class(cls): | ||
pywb_http.unpatch_socks() | ||
super(TestSOCKSProxy, cls).teardown_class() | ||
|
||
def test_socks_proxy_set(self): | ||
assert pywb_http.SOCKS_PROXIES == {'http': 'socks5h://localhost:8080', | ||
'https': 'socks5h://localhost:8080' | ||
} | ||
|
||
def test_socks_attempt_connect(self, fmod_sl): | ||
pytest.importorskip('socks') | ||
# no proxy is set, expect to fail if socks is being used | ||
resp = self.get('/live/{0}http://httpbin.org/get', fmod_sl, status=400) | ||
assert resp.status_int == 400 | ||
|
||
|