Skip to content
This repository

Full Proxy support via environment variables #152

Closed
wants to merge 6 commits into from

3 participants

shogo82148 Joshua Roesslein Alexandru Stanciu
shogo82148

Hello
I wrote proxy support and fixed authentication error for streaming.

  • improved error handling for APIMethods
  • proxy support for streaming
  • fixed authentication error for streaming
    • I tryed to connect user stream using Python 2.6.6, but Twitter always returns 401 error.

This is including pull request #80.

Joshua Roesslein joshthecoder closed this December 25, 2012
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
This page is out of date. Refresh to see the latest.
23  tweepy/api.py
@@ -42,7 +42,7 @@ def __init__(self, auth_handler=None,
42 42
     home_timeline = bind_api(
43 43
         path = '/statuses/home_timeline.json',
44 44
         payload_type = 'status', payload_list = True,
45  
-        allowed_param = ['since_id', 'max_id', 'count', 'page'],
  45
+        allowed_param = ['since_id', 'max_id', 'count', 'page', 'trim_user', 'include_entities'],
46 46
         require_auth = True
47 47
     )
48 48
 
@@ -50,7 +50,7 @@ def __init__(self, auth_handler=None,
50 50
     friends_timeline = bind_api(
51 51
         path = '/statuses/friends_timeline.json',
52 52
         payload_type = 'status', payload_list = True,
53  
-        allowed_param = ['since_id', 'max_id', 'count', 'page'],
  53
+        allowed_param = ['since_id', 'max_id', 'count', 'page', 'trim_user', 'include_entities'],
54 54
         require_auth = True
55 55
     )
56 56
 
@@ -58,15 +58,14 @@ def __init__(self, auth_handler=None,
58 58
     user_timeline = bind_api(
59 59
         path = '/statuses/user_timeline.json',
60 60
         payload_type = 'status', payload_list = True,
61  
-        allowed_param = ['id', 'user_id', 'screen_name', 'since_id',
62  
-                          'max_id', 'count', 'page', 'include_rts']
  61
+        allowed_param = ['id', 'user_id', 'screen_name', 'since_id', 'max_id', 'count', 'page', 'include_rts', 'trim_user', 'include_entities']
63 62
     )
64 63
 
65 64
     """ statuses/mentions """
66 65
     mentions = bind_api(
67 66
         path = '/statuses/mentions.json',
68 67
         payload_type = 'status', payload_list = True,
69  
-        allowed_param = ['since_id', 'max_id', 'count', 'page'],
  68
+        allowed_param = ['since_id', 'max_id', 'count', 'page', 'trim_user', 'include_entities'],
70 69
         require_auth = True
71 70
     )
72 71
 
@@ -74,7 +73,7 @@ def __init__(self, auth_handler=None,
74 73
     retweeted_by = bind_api(
75 74
         path = '/statuses/{id}/retweeted_by.json',
76 75
         payload_type = 'status', payload_list = True,
77  
-        allowed_param = ['id', 'count', 'page'],
  76
+        allowed_param = ['id', 'count', 'page', 'include_entities'],
78 77
         require_auth = True
79 78
     )
80 79
 
@@ -98,7 +97,7 @@ def __init__(self, auth_handler=None,
98 97
     retweeted_by_me = bind_api(
99 98
         path = '/statuses/retweeted_by_me.json',
100 99
         payload_type = 'status', payload_list = True,
101  
-        allowed_param = ['since_id', 'max_id', 'count', 'page'],
  100
+        allowed_param = ['since_id', 'max_id', 'count', 'page', 'trim_user', 'include_entities'],
102 101
         require_auth = True
103 102
     )
104 103
 
@@ -106,7 +105,7 @@ def __init__(self, auth_handler=None,
106 105
     retweeted_to_me = bind_api(
107 106
         path = '/statuses/retweeted_to_me.json',
108 107
         payload_type = 'status', payload_list = True,
109  
-        allowed_param = ['since_id', 'max_id', 'count', 'page'],
  108
+        allowed_param = ['since_id', 'max_id', 'count', 'page', 'trim_user', 'include_entities'],
110 109
         require_auth = True
111 110
     )
112 111
 
@@ -114,7 +113,7 @@ def __init__(self, auth_handler=None,
114 113
     retweets_of_me = bind_api(
115 114
         path = '/statuses/retweets_of_me.json',
116 115
         payload_type = 'status', payload_list = True,
117  
-        allowed_param = ['since_id', 'max_id', 'count', 'page'],
  116
+        allowed_param = ['since_id', 'max_id', 'count', 'page', 'trim_user', 'include_entities'],
118 117
         require_auth = True
119 118
     )
120 119
 
@@ -156,7 +155,7 @@ def __init__(self, auth_handler=None,
156 155
     retweets = bind_api(
157 156
         path = '/statuses/retweets/{id}.json',
158 157
         payload_type = 'status', payload_list = True,
159  
-        allowed_param = ['id', 'count'],
  158
+        allowed_param = ['id', 'count', 'trim_user', 'include_entities'],
160 159
         require_auth = True
161 160
     )
162 161
 
@@ -381,7 +380,7 @@ def update_profile_background_image(self, filename, *args, **kargs):
381 380
     favorites = bind_api(
382 381
         path = '/favorites.json',
383 382
         payload_type = 'status', payload_list = True,
384  
-        allowed_param = ['id', 'page']
  383
+        allowed_param = ['id', 'page', 'include_entities']
385 384
     )
386 385
 
387 386
     """ favorites/create """
@@ -567,7 +566,7 @@ def update_list(self, slug, *args, **kargs):
567 566
     list_timeline = bind_api(
568 567
         path = '/{owner}/lists/{slug}/statuses.json',
569 568
         payload_type = 'status', payload_list = True,
570  
-        allowed_param = ['owner', 'slug', 'since_id', 'max_id', 'per_page', 'page']
  569
+        allowed_param = ['owner', 'slug', 'since_id', 'max_id', 'per_page', 'page', 'include_entities']
571 570
     )
572 571
 
573 572
     get_list = bind_api(
59  tweepy/binder.py
@@ -7,6 +7,8 @@
7 7
 import time
8 8
 import re
9 9
 
  10
+from urllib2 import Request, HTTPError, urlopen
  11
+
10 12
 from tweepy.error import TweepError
11 13
 from tweepy.utils import convert_to_utf8_str
12 14
 from tweepy.models import Model
@@ -126,14 +128,7 @@ def execute(self):
126 128
             # Continue attempting request until successful
127 129
             # or maximum number of retries is reached.
128 130
             retries_performed = 0
129  
-            while retries_performed < self.retry_count + 1:
130  
-                # Open connection
131  
-                # FIXME: add timeout
132  
-                if self.api.secure:
133  
-                    conn = httplib.HTTPSConnection(self.host)
134  
-                else:
135  
-                    conn = httplib.HTTPConnection(self.host)
136  
-
  131
+            while True:
137 132
                 # Apply authentication
138 133
                 if self.api.auth:
139 134
                     self.api.auth.apply_auth(
@@ -142,36 +137,38 @@ def execute(self):
142 137
                     )
143 138
 
144 139
                 # Execute request
  140
+                # FIXME: add timeout
145 141
                 try:
146  
-                    conn.request(self.method, url, headers=self.headers, body=self.post_data)
147  
-                    resp = conn.getresponse()
  142
+                    req = Request(url=self.scheme + self.host + url, headers=self.headers, data=self.post_data)
  143
+                    req.get_method = lambda: self.method
  144
+                    resp = urlopen(req)
  145
+                    break
  146
+
  147
+                except HTTPError, e:
  148
+                    # continue request loop if retry error code
  149
+                    retry = retries_performed < self.retry_count
  150
+                    if self.retry_errors:
  151
+                        if e.code not in self.retry_errors: retry = False
  152
+                    if retry:
  153
+                        retries_performed += 1
  154
+                        time.sleep(self.retry_delay)
  155
+                        continue
  156
+
  157
+                    # Retry failed. throw an exception
  158
+                    try:
  159
+                        error_msg = self.api.parser.parse_error(e.read())
  160
+                    except Exception:
  161
+                        error_msg = "Twitter error response: status code = %s" % e.code
  162
+                    raise TweepError(error_msg)
  163
+
148 164
                 except Exception, e:
  165
+                    # any other exception is fatal
149 166
                     raise TweepError('Failed to send request: %s' % e)
150 167
 
151  
-                # Exit request loop if non-retry error code
152  
-                if self.retry_errors:
153  
-                    if resp.status not in self.retry_errors: break
154  
-                else:
155  
-                    if resp.status == 200: break
156  
-
157  
-                # Sleep before retrying request again
158  
-                time.sleep(self.retry_delay)
159  
-                retries_performed += 1
160  
-
161  
-            # If an error was returned, throw an exception
162  
-            self.api.last_response = resp
163  
-            if resp.status != 200:
164  
-                try:
165  
-                    error_msg = self.api.parser.parse_error(resp.read())
166  
-                except Exception:
167  
-                    error_msg = "Twitter error response: status code = %s" % resp.status
168  
-                raise TweepError(error_msg, resp)
169  
-
170 168
             # Parse the response payload
  169
+            self.api.last_response = resp
171 170
             result = self.api.parser.parse(self, resp.read())
172 171
 
173  
-            conn.close()
174  
-
175 172
             # Store result into cache if one is available.
176 173
             if self.use_cache and self.api.cache and self.method == 'GET' and result:
177 174
                 self.api.cache.store(url, result)
97  tweepy/streaming.py
@@ -2,11 +2,13 @@
2 2
 # Copyright 2009-2010 Joshua Roesslein
3 3
 # See LICENSE for details.
4 4
 
  5
+import sys
5 6
 import httplib
6 7
 from socket import timeout
7 8
 from threading import Thread
8 9
 from time import sleep
9 10
 import urllib
  11
+import urllib2
10 12
 
11 13
 from tweepy.models import Status
12 14
 from tweepy.api import API
@@ -94,71 +96,69 @@ def _run(self):
94 96
         error_counter = 0
95 97
         conn = None
96 98
         exception = None
  99
+        resp = None
97 100
         while self.running:
98 101
             if self.retry_count is not None and error_counter > self.retry_count:
99 102
                 # quit if error count greater than retry count
100 103
                 break
101 104
             try:
102  
-                if self.scheme == "http":
103  
-                    conn = httplib.HTTPConnection(self.host)
  105
+                if self.body:
  106
+                    method = 'POST'
104 107
                 else:
105  
-                    conn = httplib.HTTPSConnection(self.host)
106  
-                self.auth.apply_auth(url, 'POST', self.headers, self.parameters)
107  
-                conn.connect()
108  
-                conn.sock.settimeout(self.timeout)
109  
-                conn.request('POST', self.url, self.body, headers=self.headers)
110  
-                resp = conn.getresponse()
111  
-                if resp.status != 200:
112  
-                    if self.listener.on_error(resp.status) is False:
113  
-                        break
114  
-                    error_counter += 1
115  
-                    sleep(self.retry_time)
  108
+                    method = 'GET'
  109
+                self.auth.apply_auth(url, method, self.headers, self.parameters)
  110
+                req = urllib2.Request(url, data = self.body, headers = self.headers)
  111
+                
  112
+                if sys.version_info >= (2, 6, 0):
  113
+                    resp = urllib2.urlopen(req, timeout = self.timeout)
116 114
                 else:
117  
-                    error_counter = 0
118  
-                    self._read_loop(resp)
119  
-            except timeout:
120  
-                if self.listener.on_timeout() == False:
  115
+                    resp = urllib2.urlopen(req)
  116
+                error_counter = 0
  117
+                self._read_loop(resp)
  118
+                
  119
+            except urllib2.HTTPError, e:
  120
+                if self.listener.on_error(e.code) is False:
121 121
                     break
122  
-                if self.running is False:
  122
+                error_counter += 1
  123
+                if error_counter > self.retry_count:
  124
+                    self.running = False
  125
+                    raise e
  126
+                sleep(self.retry_time)
  127
+            except urllib2.URLError, e:
  128
+                if isinstance(e.reason, timeout):
  129
+                    if self.listener.on_timeout() is False:
  130
+                        break
  131
+                    sleep(self.snooze_time)
  132
+                else:
123 133
                     break
124  
-                conn.close()
125  
-                sleep(self.snooze_time)
126 134
             except Exception, exception:
127 135
                 # any other exception is fatal, so kill loop
128 136
                 break
  137
+            finally:
  138
+                if resp:
  139
+                    resp.close()
  140
+                resp = None
129 141
 
130 142
         # cleanup
131 143
         self.running = False
132  
-        if conn:
133  
-            conn.close()
134 144
 
135 145
         if exception:
136  
-            raise
137  
-
138  
-    def _data(self, data):
139  
-        for d in [dt for dt in data.split('\n') if dt]:
140  
-            if self.listener.on_data(d) is False:
141  
-                self.running = False
  146
+            raise exception
142 147
 
143 148
     def _read_loop(self, resp):
144  
-        buf = ''
145  
-        while self.running and not resp.isclosed():
146  
-            c = resp.read(self.buffer_size)
147  
-            idx = c.rfind('\n')
148  
-            if idx > -1:
149  
-                # There is an index. Store the tail part for later,
150  
-                # and process the head part as messages. We use idx + 1
151  
-                # as we dont' actually want to store the newline.
152  
-                data = buf + c[:idx]
153  
-                buf = c[idx + 1:]
154  
-                self._data(data)
155  
-            else:
156  
-                # No newline found, so we add this to our accumulated
157  
-                # buffer
158  
-                buf += c
159  
-
160  
-        if resp.isclosed():
161  
-            self.on_closed(resp)
  149
+        while self.running:
  150
+            # read length
  151
+            data = ''
  152
+            while True:
  153
+                c = resp.read(1)
  154
+                if c == '\n':
  155
+                    break
  156
+                data += c
  157
+            data = data.strip()
  158
+
  159
+            # read data and pass into listener
  160
+            if self.listener.on_data(data) is False:
  161
+                self.running = False
162 162
 
163 163
     def _start(self, async):
164 164
         self.running = True
@@ -167,10 +167,6 @@ def _start(self, async):
167 167
         else:
168 168
             self._run()
169 169
 
170  
-    def on_closed(self, resp):
171  
-        """ Called when the response has been closed by Twitter """
172  
-        pass
173  
-
174 170
     def userstream(self, count=None, async=False, secure=True):
175 171
         if self.running:
176 172
             raise TweepError('Stream object already connected!')
@@ -228,4 +224,3 @@ def disconnect(self):
228 224
         if self.running is False:
229 225
             return
230 226
         self.running = False
231  
-
Commit_comment_tip

Tip: You can add notes to lines in a file. Hover to the left of a line to make a note

Something went wrong with that request. Please try again.