Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP

Loading…

Full Proxy support via environment variables #152

Closed
wants to merge 6 commits into from

3 participants

@shogo82148

Hello
I wrote proxy support and fixed authentication error for streaming.

  • improved error handling for APIMethods
  • proxy support for streaming
  • fixed authentication error for streaming
    • I tryed to connect user stream using Python 2.6.6, but Twitter always returns 401 error.

This is including pull request #80.

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
Commits on Mar 8, 2012
  1. @ducu @shogo82148

    Using urllib2 for executing APIMethods through proxies

    ducu authored shogo82148 committed
    Signed-off-by: ducu <alexandru.stanciu@gmail.com>
  2. @ducu @shogo82148

    Adjusted for py versions older then 2.6.6

    ducu authored shogo82148 committed
  3. @ducu @shogo82148

    Adding params such as 'include_entities'

    ducu authored shogo82148 committed
  4. @ducu @shogo82148

    Adding 'trim_user' and 'include_entities' args.

    ducu authored shogo82148 committed
  5. @shogo82148
Commits on Mar 9, 2012
  1. @shogo82148
This page is out of date. Refresh to see the latest.
Showing with 85 additions and 94 deletions.
  1. +11 −12 tweepy/api.py
  2. +28 −31 tweepy/binder.py
  3. +46 −51 tweepy/streaming.py
View
23 tweepy/api.py
@@ -42,7 +42,7 @@ def __init__(self, auth_handler=None,
home_timeline = bind_api(
path = '/statuses/home_timeline.json',
payload_type = 'status', payload_list = True,
- allowed_param = ['since_id', 'max_id', 'count', 'page'],
+ allowed_param = ['since_id', 'max_id', 'count', 'page', 'trim_user', 'include_entities'],
require_auth = True
)
@@ -50,7 +50,7 @@ def __init__(self, auth_handler=None,
friends_timeline = bind_api(
path = '/statuses/friends_timeline.json',
payload_type = 'status', payload_list = True,
- allowed_param = ['since_id', 'max_id', 'count', 'page'],
+ allowed_param = ['since_id', 'max_id', 'count', 'page', 'trim_user', 'include_entities'],
require_auth = True
)
@@ -58,15 +58,14 @@ def __init__(self, auth_handler=None,
user_timeline = bind_api(
path = '/statuses/user_timeline.json',
payload_type = 'status', payload_list = True,
- allowed_param = ['id', 'user_id', 'screen_name', 'since_id',
- 'max_id', 'count', 'page', 'include_rts']
+ allowed_param = ['id', 'user_id', 'screen_name', 'since_id', 'max_id', 'count', 'page', 'include_rts', 'trim_user', 'include_entities']
)
""" statuses/mentions """
mentions = bind_api(
path = '/statuses/mentions.json',
payload_type = 'status', payload_list = True,
- allowed_param = ['since_id', 'max_id', 'count', 'page'],
+ allowed_param = ['since_id', 'max_id', 'count', 'page', 'trim_user', 'include_entities'],
require_auth = True
)
@@ -74,7 +73,7 @@ def __init__(self, auth_handler=None,
retweeted_by = bind_api(
path = '/statuses/{id}/retweeted_by.json',
payload_type = 'status', payload_list = True,
- allowed_param = ['id', 'count', 'page'],
+ allowed_param = ['id', 'count', 'page', 'include_entities'],
require_auth = True
)
@@ -98,7 +97,7 @@ def __init__(self, auth_handler=None,
retweeted_by_me = bind_api(
path = '/statuses/retweeted_by_me.json',
payload_type = 'status', payload_list = True,
- allowed_param = ['since_id', 'max_id', 'count', 'page'],
+ allowed_param = ['since_id', 'max_id', 'count', 'page', 'trim_user', 'include_entities'],
require_auth = True
)
@@ -106,7 +105,7 @@ def __init__(self, auth_handler=None,
retweeted_to_me = bind_api(
path = '/statuses/retweeted_to_me.json',
payload_type = 'status', payload_list = True,
- allowed_param = ['since_id', 'max_id', 'count', 'page'],
+ allowed_param = ['since_id', 'max_id', 'count', 'page', 'trim_user', 'include_entities'],
require_auth = True
)
@@ -114,7 +113,7 @@ def __init__(self, auth_handler=None,
retweets_of_me = bind_api(
path = '/statuses/retweets_of_me.json',
payload_type = 'status', payload_list = True,
- allowed_param = ['since_id', 'max_id', 'count', 'page'],
+ allowed_param = ['since_id', 'max_id', 'count', 'page', 'trim_user', 'include_entities'],
require_auth = True
)
@@ -156,7 +155,7 @@ def __init__(self, auth_handler=None,
retweets = bind_api(
path = '/statuses/retweets/{id}.json',
payload_type = 'status', payload_list = True,
- allowed_param = ['id', 'count'],
+ allowed_param = ['id', 'count', 'trim_user', 'include_entities'],
require_auth = True
)
@@ -381,7 +380,7 @@ def update_profile_background_image(self, filename, *args, **kargs):
favorites = bind_api(
path = '/favorites.json',
payload_type = 'status', payload_list = True,
- allowed_param = ['id', 'page']
+ allowed_param = ['id', 'page', 'include_entities']
)
""" favorites/create """
@@ -567,7 +566,7 @@ def update_list(self, slug, *args, **kargs):
list_timeline = bind_api(
path = '/{owner}/lists/{slug}/statuses.json',
payload_type = 'status', payload_list = True,
- allowed_param = ['owner', 'slug', 'since_id', 'max_id', 'per_page', 'page']
+ allowed_param = ['owner', 'slug', 'since_id', 'max_id', 'per_page', 'page', 'include_entities']
)
get_list = bind_api(
View
59 tweepy/binder.py
@@ -7,6 +7,8 @@
import time
import re
+from urllib2 import Request, HTTPError, urlopen
+
from tweepy.error import TweepError
from tweepy.utils import convert_to_utf8_str
from tweepy.models import Model
@@ -126,14 +128,7 @@ def execute(self):
# Continue attempting request until successful
# or maximum number of retries is reached.
retries_performed = 0
- while retries_performed < self.retry_count + 1:
- # Open connection
- # FIXME: add timeout
- if self.api.secure:
- conn = httplib.HTTPSConnection(self.host)
- else:
- conn = httplib.HTTPConnection(self.host)
-
+ while True:
# Apply authentication
if self.api.auth:
self.api.auth.apply_auth(
@@ -142,36 +137,38 @@ def execute(self):
)
# Execute request
+ # FIXME: add timeout
try:
- conn.request(self.method, url, headers=self.headers, body=self.post_data)
- resp = conn.getresponse()
+ req = Request(url=self.scheme + self.host + url, headers=self.headers, data=self.post_data)
+ req.get_method = lambda: self.method
+ resp = urlopen(req)
+ break
+
+ except HTTPError, e:
+ # continue request loop if retry error code
+ retry = retries_performed < self.retry_count
+ if self.retry_errors:
+ if e.code not in self.retry_errors: retry = False
+ if retry:
+ retries_performed += 1
+ time.sleep(self.retry_delay)
+ continue
+
+ # Retry failed. throw an exception
+ try:
+ error_msg = self.api.parser.parse_error(e.read())
+ except Exception:
+ error_msg = "Twitter error response: status code = %s" % e.code
+ raise TweepError(error_msg)
+
except Exception, e:
+ # any other exception is fatal
raise TweepError('Failed to send request: %s' % e)
- # Exit request loop if non-retry error code
- if self.retry_errors:
- if resp.status not in self.retry_errors: break
- else:
- if resp.status == 200: break
-
- # Sleep before retrying request again
- time.sleep(self.retry_delay)
- retries_performed += 1
-
- # If an error was returned, throw an exception
- self.api.last_response = resp
- if resp.status != 200:
- try:
- error_msg = self.api.parser.parse_error(resp.read())
- except Exception:
- error_msg = "Twitter error response: status code = %s" % resp.status
- raise TweepError(error_msg, resp)
-
# Parse the response payload
+ self.api.last_response = resp
result = self.api.parser.parse(self, resp.read())
- conn.close()
-
# Store result into cache if one is available.
if self.use_cache and self.api.cache and self.method == 'GET' and result:
self.api.cache.store(url, result)
View
97 tweepy/streaming.py
@@ -2,11 +2,13 @@
# Copyright 2009-2010 Joshua Roesslein
# See LICENSE for details.
+import sys
import httplib
from socket import timeout
from threading import Thread
from time import sleep
import urllib
+import urllib2
from tweepy.models import Status
from tweepy.api import API
@@ -94,71 +96,69 @@ def _run(self):
error_counter = 0
conn = None
exception = None
+ resp = None
while self.running:
if self.retry_count is not None and error_counter > self.retry_count:
# quit if error count greater than retry count
break
try:
- if self.scheme == "http":
- conn = httplib.HTTPConnection(self.host)
+ if self.body:
+ method = 'POST'
else:
- conn = httplib.HTTPSConnection(self.host)
- self.auth.apply_auth(url, 'POST', self.headers, self.parameters)
- conn.connect()
- conn.sock.settimeout(self.timeout)
- conn.request('POST', self.url, self.body, headers=self.headers)
- resp = conn.getresponse()
- if resp.status != 200:
- if self.listener.on_error(resp.status) is False:
- break
- error_counter += 1
- sleep(self.retry_time)
+ method = 'GET'
+ self.auth.apply_auth(url, method, self.headers, self.parameters)
+ req = urllib2.Request(url, data = self.body, headers = self.headers)
+
+ if sys.version_info >= (2, 6, 0):
+ resp = urllib2.urlopen(req, timeout = self.timeout)
else:
- error_counter = 0
- self._read_loop(resp)
- except timeout:
- if self.listener.on_timeout() == False:
+ resp = urllib2.urlopen(req)
+ error_counter = 0
+ self._read_loop(resp)
+
+ except urllib2.HTTPError, e:
+ if self.listener.on_error(e.code) is False:
break
- if self.running is False:
+ error_counter += 1
+ if error_counter > self.retry_count:
+ self.running = False
+ raise e
+ sleep(self.retry_time)
+ except urllib2.URLError, e:
+ if isinstance(e.reason, timeout):
+ if self.listener.on_timeout() is False:
+ break
+ sleep(self.snooze_time)
+ else:
break
- conn.close()
- sleep(self.snooze_time)
except Exception, exception:
# any other exception is fatal, so kill loop
break
+ finally:
+ if resp:
+ resp.close()
+ resp = None
# cleanup
self.running = False
- if conn:
- conn.close()
if exception:
- raise
-
- def _data(self, data):
- for d in [dt for dt in data.split('\n') if dt]:
- if self.listener.on_data(d) is False:
- self.running = False
+ raise exception
def _read_loop(self, resp):
- buf = ''
- while self.running and not resp.isclosed():
- c = resp.read(self.buffer_size)
- idx = c.rfind('\n')
- if idx > -1:
- # There is an index. Store the tail part for later,
- # and process the head part as messages. We use idx + 1
- # as we dont' actually want to store the newline.
- data = buf + c[:idx]
- buf = c[idx + 1:]
- self._data(data)
- else:
- # No newline found, so we add this to our accumulated
- # buffer
- buf += c
-
- if resp.isclosed():
- self.on_closed(resp)
+ while self.running:
+ # read length
+ data = ''
+ while True:
+ c = resp.read(1)
+ if c == '\n':
+ break
+ data += c
+ data = data.strip()
+
+ # read data and pass into listener
+ if self.listener.on_data(data) is False:
+ self.running = False
def _start(self, async):
self.running = True
@@ -167,10 +167,6 @@ def _start(self, async):
else:
self._run()
- def on_closed(self, resp):
- """ Called when the response has been closed by Twitter """
- pass
-
def userstream(self, count=None, async=False, secure=True):
if self.running:
raise TweepError('Stream object already connected!')
@@ -228,4 +224,3 @@ def disconnect(self):
if self.running is False:
return
self.running = False
-
Something went wrong with that request. Please try again.