Skip to content
Browse files

Allow headers sent out by proxy to be modified from the config file

  • Loading branch information...
1 parent d70f74b commit 581f0a4825488a05fe4d17f37c8484967424913d @nibrahim nibrahim committed May 9, 2012
Showing with 15 additions and 6 deletions.
  1. +6 −0 config.yml
  2. +6 −2 liveweb/config.py
  3. +3 −4 liveweb/proxy.py
View
6 config.yml
@@ -5,6 +5,10 @@ storage:
max_file_size: 100000000 # 100MB approx.
pattern: "liveweb-%(seq)s-%(timestamp)s.arc.gz"
+extra_headers:
+ Connection: "keep-alive"
+
+
user_agent: "ia_archiver(OS-Wayback)"
# Max size of ARC record that can be stored in cache
@@ -22,3 +26,5 @@ timeout: 1
#cache:
# type: redis
+
+# Extra headers which will be sent by the proxy to the remote site.
View
8 liveweb/config.py
@@ -7,14 +7,15 @@
import logging
user_agent = "ia_archiver(OS-Wayback)"
+extra_headers = {}
M = 1024 * 1024
# Max size of ARC record that can be stored in cache
max_cacheable_size = 10 * M
# timeout in seconds
-timeout = 60
+timeout = 600
dns_timeout = None
connect_timeout = None
@@ -68,9 +69,12 @@ def load(filename):
return
d = yaml.safe_load(open(filename))
-
+
+ print 'Here', d
if "max_cacheable_size" in d:
d['max_cacheable_size'] = _parse_size(d['max_cacheable_size'])
+
+
# update config
globals().update(d)
View
7 liveweb/proxy.py
@@ -132,10 +132,9 @@ def _urlopen(url):
Called by urlopen and test cases.
"""
- headers = {
- "User-Agent": config.user_agent,
- "Connection": "close"
- }
+ headers = config.get("extra_headers",{})
+ headers['User-Agent'] = config.user_agent
+
type, host, selector = split_type_host(url)
conn = ProxyHTTPConnection(host, url=url)

0 comments on commit 581f0a4

Please sign in to comment.
Something went wrong with that request. Please try again.