Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP

Loading…

Full Proxy support via environment variables #152

Closed
wants to merge 6 commits into from

3 participants

shogo82148 Joshua Roesslein Alexandru Stanciu
shogo82148

Hello
I wrote proxy support and fixed authentication error for streaming.

  • improved error handling for APIMethods
  • proxy support for streaming
  • fixed authentication error for streaming
    • I tryed to connect user stream using Python 2.6.6, but Twitter always returns 401 error.

This is including pull request #80.

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
Commits on Mar 8, 2012
  1. Alexandru Stanciu shogo82148

    Using urllib2 for executing APIMethods through proxies

    ducu authored shogo82148 committed
    Signed-off-by: ducu <alexandru.stanciu@gmail.com>
  2. Alexandru Stanciu shogo82148

    Adjusted for py versions older then 2.6.6

    ducu authored shogo82148 committed
  3. Alexandru Stanciu shogo82148

    Adding params such as 'include_entities'

    ducu authored shogo82148 committed
  4. Alexandru Stanciu shogo82148

    Adding 'trim_user' and 'include_entities' args.

    ducu authored shogo82148 committed
  5. shogo82148
Commits on Mar 9, 2012
  1. shogo82148
This page is out of date. Refresh to see the latest.
Showing with 85 additions and 94 deletions.
  1. +11 −12 tweepy/api.py
  2. +28 −31 tweepy/binder.py
  3. +46 −51 tweepy/streaming.py
23 tweepy/api.py
View
@@ -42,7 +42,7 @@ def __init__(self, auth_handler=None,
home_timeline = bind_api(
path = '/statuses/home_timeline.json',
payload_type = 'status', payload_list = True,
- allowed_param = ['since_id', 'max_id', 'count', 'page'],
+ allowed_param = ['since_id', 'max_id', 'count', 'page', 'trim_user', 'include_entities'],
require_auth = True
)
@@ -50,7 +50,7 @@ def __init__(self, auth_handler=None,
friends_timeline = bind_api(
path = '/statuses/friends_timeline.json',
payload_type = 'status', payload_list = True,
- allowed_param = ['since_id', 'max_id', 'count', 'page'],
+ allowed_param = ['since_id', 'max_id', 'count', 'page', 'trim_user', 'include_entities'],
require_auth = True
)
@@ -58,15 +58,14 @@ def __init__(self, auth_handler=None,
user_timeline = bind_api(
path = '/statuses/user_timeline.json',
payload_type = 'status', payload_list = True,
- allowed_param = ['id', 'user_id', 'screen_name', 'since_id',
- 'max_id', 'count', 'page', 'include_rts']
+ allowed_param = ['id', 'user_id', 'screen_name', 'since_id', 'max_id', 'count', 'page', 'include_rts', 'trim_user', 'include_entities']
)
""" statuses/mentions """
mentions = bind_api(
path = '/statuses/mentions.json',
payload_type = 'status', payload_list = True,
- allowed_param = ['since_id', 'max_id', 'count', 'page'],
+ allowed_param = ['since_id', 'max_id', 'count', 'page', 'trim_user', 'include_entities'],
require_auth = True
)
@@ -74,7 +73,7 @@ def __init__(self, auth_handler=None,
retweeted_by = bind_api(
path = '/statuses/{id}/retweeted_by.json',
payload_type = 'status', payload_list = True,
- allowed_param = ['id', 'count', 'page'],
+ allowed_param = ['id', 'count', 'page', 'include_entities'],
require_auth = True
)
@@ -98,7 +97,7 @@ def __init__(self, auth_handler=None,
retweeted_by_me = bind_api(
path = '/statuses/retweeted_by_me.json',
payload_type = 'status', payload_list = True,
- allowed_param = ['since_id', 'max_id', 'count', 'page'],
+ allowed_param = ['since_id', 'max_id', 'count', 'page', 'trim_user', 'include_entities'],
require_auth = True
)
@@ -106,7 +105,7 @@ def __init__(self, auth_handler=None,
retweeted_to_me = bind_api(
path = '/statuses/retweeted_to_me.json',
payload_type = 'status', payload_list = True,
- allowed_param = ['since_id', 'max_id', 'count', 'page'],
+ allowed_param = ['since_id', 'max_id', 'count', 'page', 'trim_user', 'include_entities'],
require_auth = True
)
@@ -114,7 +113,7 @@ def __init__(self, auth_handler=None,
retweets_of_me = bind_api(
path = '/statuses/retweets_of_me.json',
payload_type = 'status', payload_list = True,
- allowed_param = ['since_id', 'max_id', 'count', 'page'],
+ allowed_param = ['since_id', 'max_id', 'count', 'page', 'trim_user', 'include_entities'],
require_auth = True
)
@@ -156,7 +155,7 @@ def __init__(self, auth_handler=None,
retweets = bind_api(
path = '/statuses/retweets/{id}.json',
payload_type = 'status', payload_list = True,
- allowed_param = ['id', 'count'],
+ allowed_param = ['id', 'count', 'trim_user', 'include_entities'],
require_auth = True
)
@@ -381,7 +380,7 @@ def update_profile_background_image(self, filename, *args, **kargs):
favorites = bind_api(
path = '/favorites.json',
payload_type = 'status', payload_list = True,
- allowed_param = ['id', 'page']
+ allowed_param = ['id', 'page', 'include_entities']
)
""" favorites/create """
@@ -567,7 +566,7 @@ def update_list(self, slug, *args, **kargs):
list_timeline = bind_api(
path = '/{owner}/lists/{slug}/statuses.json',
payload_type = 'status', payload_list = True,
- allowed_param = ['owner', 'slug', 'since_id', 'max_id', 'per_page', 'page']
+ allowed_param = ['owner', 'slug', 'since_id', 'max_id', 'per_page', 'page', 'include_entities']
)
get_list = bind_api(
59 tweepy/binder.py
View
@@ -7,6 +7,8 @@
import time
import re
+from urllib2 import Request, HTTPError, urlopen
+
from tweepy.error import TweepError
from tweepy.utils import convert_to_utf8_str
from tweepy.models import Model
@@ -126,14 +128,7 @@ def execute(self):
# Continue attempting request until successful
# or maximum number of retries is reached.
retries_performed = 0
- while retries_performed < self.retry_count + 1:
- # Open connection
- # FIXME: add timeout
- if self.api.secure:
- conn = httplib.HTTPSConnection(self.host)
- else:
- conn = httplib.HTTPConnection(self.host)
-
+ while True:
# Apply authentication
if self.api.auth:
self.api.auth.apply_auth(
@@ -142,36 +137,38 @@ def execute(self):
)
# Execute request
+ # FIXME: add timeout
try:
- conn.request(self.method, url, headers=self.headers, body=self.post_data)
- resp = conn.getresponse()
+ req = Request(url=self.scheme + self.host + url, headers=self.headers, data=self.post_data)
+ req.get_method = lambda: self.method
+ resp = urlopen(req)
+ break
+
+ except HTTPError, e:
+ # continue request loop if retry error code
+ retry = retries_performed < self.retry_count
+ if self.retry_errors:
+ if e.code not in self.retry_errors: retry = False
+ if retry:
+ retries_performed += 1
+ time.sleep(self.retry_delay)
+ continue
+
+ # Retry failed. throw an exception
+ try:
+ error_msg = self.api.parser.parse_error(e.read())
+ except Exception:
+ error_msg = "Twitter error response: status code = %s" % e.code
+ raise TweepError(error_msg)
+
except Exception, e:
+ # any other exception is fatal
raise TweepError('Failed to send request: %s' % e)
- # Exit request loop if non-retry error code
- if self.retry_errors:
- if resp.status not in self.retry_errors: break
- else:
- if resp.status == 200: break
-
- # Sleep before retrying request again
- time.sleep(self.retry_delay)
- retries_performed += 1
-
- # If an error was returned, throw an exception
- self.api.last_response = resp
- if resp.status != 200:
- try:
- error_msg = self.api.parser.parse_error(resp.read())
- except Exception:
- error_msg = "Twitter error response: status code = %s" % resp.status
- raise TweepError(error_msg, resp)
-
# Parse the response payload
+ self.api.last_response = resp
result = self.api.parser.parse(self, resp.read())
- conn.close()
-
# Store result into cache if one is available.
if self.use_cache and self.api.cache and self.method == 'GET' and result:
self.api.cache.store(url, result)
97 tweepy/streaming.py
View
@@ -2,11 +2,13 @@
# Copyright 2009-2010 Joshua Roesslein
# See LICENSE for details.
+import sys
import httplib
from socket import timeout
from threading import Thread
from time import sleep
import urllib
+import urllib2
from tweepy.models import Status
from tweepy.api import API
@@ -94,71 +96,69 @@ def _run(self):
error_counter = 0
conn = None
exception = None
+ resp = None
while self.running:
if self.retry_count is not None and error_counter > self.retry_count:
# quit if error count greater than retry count
break
try:
- if self.scheme == "http":
- conn = httplib.HTTPConnection(self.host)
+ if self.body:
+ method = 'POST'
else:
- conn = httplib.HTTPSConnection(self.host)
- self.auth.apply_auth(url, 'POST', self.headers, self.parameters)
- conn.connect()
- conn.sock.settimeout(self.timeout)
- conn.request('POST', self.url, self.body, headers=self.headers)
- resp = conn.getresponse()
- if resp.status != 200:
- if self.listener.on_error(resp.status) is False:
- break
- error_counter += 1
- sleep(self.retry_time)
+ method = 'GET'
+ self.auth.apply_auth(url, method, self.headers, self.parameters)
+ req = urllib2.Request(url, data = self.body, headers = self.headers)
+
+ if sys.version_info >= (2, 6, 0):
+ resp = urllib2.urlopen(req, timeout = self.timeout)
else:
- error_counter = 0
- self._read_loop(resp)
- except timeout:
- if self.listener.on_timeout() == False:
+ resp = urllib2.urlopen(req)
+ error_counter = 0
+ self._read_loop(resp)
+
+ except urllib2.HTTPError, e:
+ if self.listener.on_error(e.code) is False:
break
- if self.running is False:
+ error_counter += 1
+ if error_counter > self.retry_count:
+ self.running = False
+ raise e
+ sleep(self.retry_time)
+ except urllib2.URLError, e:
+ if isinstance(e.reason, timeout):
+ if self.listener.on_timeout() is False:
+ break
+ sleep(self.snooze_time)
+ else:
break
- conn.close()
- sleep(self.snooze_time)
except Exception, exception:
# any other exception is fatal, so kill loop
break
+ finally:
+ if resp:
+ resp.close()
+ resp = None
# cleanup
self.running = False
- if conn:
- conn.close()
if exception:
- raise
-
- def _data(self, data):
- for d in [dt for dt in data.split('\n') if dt]:
- if self.listener.on_data(d) is False:
- self.running = False
+ raise exception
def _read_loop(self, resp):
- buf = ''
- while self.running and not resp.isclosed():
- c = resp.read(self.buffer_size)
- idx = c.rfind('\n')
- if idx > -1:
- # There is an index. Store the tail part for later,
- # and process the head part as messages. We use idx + 1
- # as we dont' actually want to store the newline.
- data = buf + c[:idx]
- buf = c[idx + 1:]
- self._data(data)
- else:
- # No newline found, so we add this to our accumulated
- # buffer
- buf += c
-
- if resp.isclosed():
- self.on_closed(resp)
+ while self.running:
+ # read length
+ data = ''
+ while True:
+ c = resp.read(1)
+ if c == '\n':
+ break
+ data += c
+ data = data.strip()
+
+ # read data and pass into listener
+ if self.listener.on_data(data) is False:
+ self.running = False
def _start(self, async):
self.running = True
@@ -167,10 +167,6 @@ def _start(self, async):
else:
self._run()
- def on_closed(self, resp):
- """ Called when the response has been closed by Twitter """
- pass
-
def userstream(self, count=None, async=False, secure=True):
if self.running:
raise TweepError('Stream object already connected!')
@@ -228,4 +224,3 @@ def disconnect(self):
if self.running is False:
return
self.running = False
-
Something went wrong with that request. Please try again.