Skip to content
This repository

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse code

news: patch for scrapelib, requests upgrade

  • Loading branch information...
commit a3c345be6ee7e3648c230a95a2321e5f60dbea19 1 parent d4a3560
Thom Neale authored March 27, 2013

Showing 1 changed file with 6 additions and 10 deletions. Show diff stats Hide diff stats

  1. 16  scripts/newsblogs/fetch.py
16  scripts/newsblogs/fetch.py
@@ -27,18 +27,9 @@
27 27
 logger.addHandler(ch)
28 28
 
29 29
 
30  
-request_defaults = {
31  
-    #'proxies': {"http": "localhost:8001"},
32  
-    'timeout': 15.0,
33  
-    'user_agent': ('Mozilla/5.0 (X11; Ubuntu; Linux i686; rv:10.0.2) '
34  
-                   'Gecko/20100101 Firefox/10.0.2'),
35  
-    'follow_robots': False,
36  
-    }
37  
-
38  
-
39 30
 if __name__ == '__main__':
40 31
 
41  
-    session = scrapelib.Scraper(**request_defaults)
  32
+    session = scrapelib.Scraper()
42 33
     session.headers = {
43 34
         'Accept': ('text/html,application/xhtml+xml,application/'
44 35
                    'xml;q=0.9,*/*;q=0.8'),
@@ -46,6 +37,11 @@
46 37
         'Accept-Language': 'en-us,en;q=0.5',
47 38
         'Connection': 'keep-alive',
48 39
         }
  40
+    session.user_agent = (
  41
+        'Mozilla/5.0 (X11; Ubuntu; Linux i686; rv:10.0.2) '
  42
+        'Gecko/20100101 Firefox/10.0.2')
  43
+    session.timeout = 15.0
  44
+    session.follow_robots = False
49 45
 
50 46
     def fetch(url):
51 47
         logger.info('trying %r' % url)

0 notes on commit a3c345b

Please sign in to comment.
Something went wrong with that request. Please try again.