Allow headers sent out by proxy to be modified from the config file

config.yml

@@ -5,6 +5,10 @@ storage:
     max_file_size: 100000000 # 100MB approx.
     pattern: "liveweb-%(seq)s-%(timestamp)s.arc.gz"
 
+extra_headers:
+  Connection: "keep-alive"
+
+
 user_agent: "ia_archiver(OS-Wayback)"
 
 # Max size of ARC record that can be stored in cache
@@ -22,3 +26,5 @@ timeout: 1
 
 #cache:
 #    type: redis
+
+# Extra headers which will be sent by the proxy to the remote site.

liveweb/config.py

@@ -7,14 +7,15 @@
 import logging
 
 user_agent = "ia_archiver(OS-Wayback)"
+extra_headers = {}
 
 M = 1024 * 1024
 
 # Max size of ARC record that can be stored in cache
 max_cacheable_size = 10 * M
 
 # timeout in seconds
-timeout = 60
+timeout = 600
 
 dns_timeout = None
 connect_timeout = None
@@ -68,9 +69,12 @@ def load(filename):
         return
 
     d = yaml.safe_load(open(filename))
-
+
+    print 'Here', d
     if "max_cacheable_size" in d:
         d['max_cacheable_size'] = _parse_size(d['max_cacheable_size'])
+
+
 
     # update config
     globals().update(d)

liveweb/proxy.py

@@ -132,10 +132,9 @@ def _urlopen(url):
     
     Called by urlopen and test cases.
     """
-    headers = {
-        "User-Agent": config.user_agent,
-        "Connection": "close"
-    }
+    headers = config.get("extra_headers",{})
+    headers['User-Agent'] = config.user_agent
+
     type, host, selector = split_type_host(url)
 
     conn = ProxyHTTPConnection(host, url=url)