Skip to content

Commit

Permalink
[fix] - imdb_list - Enabled imdb_list to cache credentials per login …
Browse files Browse the repository at this point in the history 
…name (#1209). Fixes #1109

* Enabled imdb_list to cache credentials per login name

* Forgot to use class value for cookies

* Refactored plugin to use table and not persistence

* Fixed list_id caching

* Correctly accessed session

* Enabled imdb_list to cache credentials per login name

* Forgot to use class value for cookies

* Refactored plugin to use table and not persistence

* Fixed list_id caching

* Correctly accessed session

* Added cached credentials test

* Correctly pass cookies to all methods

* - Changed user_id column type to string
- Redid cassettes

* Redid cassettes, again

* Reredid cassettes
  • Loading branch information
@liiight
liiight committed May 30, 2016
1 parent 2c32324 commit ef91017
Show file tree
Hide file tree
Showing 3 changed files with 951 additions and 813 deletions.
176 changes: 124 additions & 52 deletions flexget/plugins/list/imdb_list.py
View file
Expand Up @@ -10,16 +10,53 @@

from requests.exceptions import RequestException

from flexget import plugin
from sqlalchemy import Column, Unicode, String
from sqlalchemy.orm import relation
from sqlalchemy.schema import ForeignKey

from flexget import plugin, db_schema
from flexget.entry import Entry
from flexget.event import event
from flexget.plugin import PluginError
from flexget.utils.requests import Session, TimedLimiter
from flexget.manager import Session
from flexget.utils.database import json_synonym
from flexget.utils.requests import Session as RequestSession, TimedLimiter
from flexget.utils.soup import get_soup

log = logging.getLogger('imdb_list')
IMMUTABLE_LISTS = ['ratings', 'checkins']

Base = db_schema.versioned_base('imdb_list', 0)


class IMDBListUser(Base):
__tablename__ = "imdb_list_user"

user_id = Column(String, primary_key=True)
user_name = Column(Unicode)
_cookies = Column('cookies', Unicode)
cookies = json_synonym('_cookies')

lists = relation('IMDBListList', backref='imdb_user', cascade='all, delete, delete-orphan')

def __init__(self, user_name, user_id, cookies):
self.user_name = user_name
self.user_id = user_id
self.cookies = cookies


class IMDBListList(Base):
__tablename__ = "imdb_list_lists"

list_id = Column(Unicode, primary_key=True)
list_name = Column(Unicode)
user_id = Column(String, ForeignKey('imdb_list_user.user_id'))

def __init__(self, list_id, list_name, user_id):
self.list_id = list_id
self.list_name = list_name
self.user_id = user_id


if PY3:
csv_reader = csv.reader
Expand Down Expand Up @@ -49,11 +86,12 @@ class ImdbEntrySet(MutableSet):

def __init__(self, config):
self.config = config
self._session = Session()
self._session = RequestSession()
self._session.add_domain_limiter(TimedLimiter('imdb.com', '5 seconds'))
self._session.headers = {'Accept-Language': config.get('force_language', 'en-us')}
self.user_id = None
self.list_id = None
self.cookies = None
self._items = None
self._authenticated = False

Expand All @@ -64,48 +102,75 @@ def session(self):
return self._session

def authenticate(self):
"""Authenticates a session with imdb, and grabs any IDs needed for getting/modifying list."""
try:
r = self._session.get(
'https://www.imdb.com/ap/signin?openid.return_to=https%3A%2F%2Fwww.imdb.com%2Fap-signin-'
'handler&openid.identity=http%3A%2F%2Fspecs.openid.net%2Fauth%2F2.0%2Fidentifier_select&'
'openid.assoc_handle=imdb_mobile_us&openid.mode=checkid_setup&openid.claimed_id=http%3A%'
'2F%2Fspecs.openid.net%2Fauth%2F2.0%2Fidentifier_select&openid.ns=http%3A%2F%2Fspecs.ope'
'nid.net%2Fauth%2F2.0')
except RequestException as e:
raise PluginError(e.args[0])
soup = get_soup(r.content)
inputs = soup.select('form#ap_signin_form input')
data = dict((i['name'], i.get('value')) for i in inputs if i.get('name'))
data['email'] = self.config['login']
data['password'] = self.config['password']
log.debug('email=%s, password=%s', data['email'], data['password'])
d = self._session.post('https://www.imdb.com/ap/signin', data=data)
# Get user id by extracting from redirect url
r = self._session.head('http://www.imdb.com/profile', allow_redirects=False)
if not r.headers.get('location') or 'login' in r.headers['location']:
raise plugin.PluginError('Login to imdb failed. Check your credentials.')
self.user_id = re.search('ur\d+(?!\d)', r.headers['location']).group()
# Get list ID
if self.config['list'] == 'watchlist':
data = {'consts[]': 'tt0133093', 'tracking_tag': 'watchlistRibbon'}
wl_data = self._session.post('http://www.imdb.com/list/_ajax/watchlist_has', data=data).json()
try:
self.list_id = wl_data['list_id']
except KeyError:
raise PluginError('No list ID could be received. Please initialize list by '
'manually adding an item to it and try again')
elif self.config['list'] in IMMUTABLE_LISTS or self.config['list'].startswith('ls'):
self.list_id = self.config['list']
else:
data = {'tconst': 'tt0133093'}
list_data = self._session.post('http://www.imdb.com/list/_ajax/wlb_dropdown', data=data).json()
for li in list_data['items']:
if li['wlb_text'] == self.config['list']:
self.list_id = li['data_list_id']
break
else:
raise plugin.PluginError('Could not find list %s' % self.config['list'])
"""Authenticates a session with IMDB, and grabs any IDs needed for getting/modifying list."""
cached_credentials = False
with Session() as session:
user = session.query(IMDBListUser).filter(IMDBListUser.user_name == self.config.get('login')).one_or_none()
if user and user.cookies and user.user_id:
log.debug('login credentials found in cache, testing')
self.cookies = user.cookies
self.user_id = user.user_id
r = self._session.head('http://www.imdb.com/profile', allow_redirects=False, cookies=self.cookies)
if not r.headers.get('location') or 'login' in r.headers['location']:
log.debug('cache credentials expired')
else:
cached_credentials = True
if not cached_credentials:
log.debug('user credentials not found in cache or outdated, fetching from IMDB')
try:
r = self._session.get(
'https://www.imdb.com/ap/signin?openid.return_to=https%3A%2F%2Fwww.imdb.com%2Fap-signin-'
'handler&openid.identity=http%3A%2F%2Fspecs.openid.net%2Fauth%2F2.0%2Fidentifier_select&'
'openid.assoc_handle=imdb_mobile_us&openid.mode=checkid_setup&openid.claimed_id=http%3A%'
'2F%2Fspecs.openid.net%2Fauth%2F2.0%2Fidentifier_select&openid.ns=http%3A%2F%2Fspecs.ope'
'nid.net%2Fauth%2F2.0')
except RequestException as e:
raise PluginError(e.args[0])
soup = get_soup(r.content)
inputs = soup.select('form#ap_signin_form input')
data = dict((i['name'], i.get('value')) for i in inputs if i.get('name'))
data['email'] = self.config['login']
data['password'] = self.config['password']
log.debug('email=%s, password=%s', data['email'], data['password'])
d = self._session.post('https://www.imdb.com/ap/signin', data=data)
# Get user id by extracting from redirect url
r = self._session.head('http://www.imdb.com/profile', allow_redirects=False)
if not r.headers.get('location') or 'login' in r.headers['location']:
raise plugin.PluginError('Login to IMDB failed. Check your credentials.')
self.user_id = re.search('ur\d+(?!\d)', r.headers['location']).group()
self.cookies = dict(d.cookies)
# Get list ID
if user:
for list in user.lists:
if self.config['list'] == list.list_name:
log.debug('found list ID %s matching list name %s in cache', list.list_id, list.list_name)
self.list_id = list.list_id
if not self.list_id:
log.debug('could not find list ID in cache, fetching from IMDB')
if self.config['list'] == 'watchlist':
data = {'consts[]': 'tt0133093', 'tracking_tag': 'watchlistRibbon'}
wl_data = self._session.post('http://www.imdb.com/list/_ajax/watchlist_has', data=data).json()
try:
self.list_id = wl_data['list_id']
except KeyError:
raise PluginError('No list ID could be received. Please initialize list by '
'manually adding an item to it and try again')
elif self.config['list'] in IMMUTABLE_LISTS or self.config['list'].startswith('ls'):
self.list_id = self.config['list']
else:
data = {'tconst': 'tt0133093'}
list_data = self._session.post('http://www.imdb.com/list/_ajax/wlb_dropdown', data=data).json()
for li in list_data['items']:
if li['wlb_text'] == self.config['list']:
self.list_id = li['data_list_id']
break
else:
raise plugin.PluginError('Could not find list %s' % self.config['list'])

user = IMDBListUser(self.config['login'], self.user_id, self.cookies)
list = IMDBListList(self.list_id, self.config['list'], self.user_id)
user.lists.append(list)
session.merge(user)

self._authenticated = True

Expand All @@ -117,7 +182,7 @@ def items(self):
if self._items is None:
try:
r = self.session.get('http://www.imdb.com/list/export?list_id=%s&author_id=%s' %
(self.list_id, self.user_id))
(self.list_id, self.user_id), cookies=self.cookies)
except RequestException as e:
raise PluginError(e.args[0])
lines = r.iter_lines(decode_unicode=True)
Expand Down Expand Up @@ -183,11 +248,13 @@ def discard(self, entry):
item_ids = None
if self.config['list'] == 'watchlist':
data = {'consts[]': entry['imdb_id'], 'tracking_tag': 'watchlistRibbon'}
status = self.session.post('http://www.imdb.com/list/_ajax/watchlist_has', data=data).json()
status = self.session.post('http://www.imdb.com/list/_ajax/watchlist_has', data=data,
cookies=self.cookies).json()
item_ids = status.get('has', {}).get(entry['imdb_id'])
else:
data = {'tconst': entry['imdb_id']}
status = self.session.post('http://www.imdb.com/list/_ajax/wlb_dropdown', data=data).json()
status = self.session.post('http://www.imdb.com/list/_ajax/wlb_dropdown', data=data,
cookies=self.cookies).json()
for a_list in status['items']:
if a_list['data_list_id'] == self.list_id:
item_ids = a_list['data_list_item_ids']
Expand All @@ -201,22 +268,27 @@ def discard(self, entry):
'ref_tag': 'title'
}
for item_id in item_ids:
self.session.post('http://www.imdb.com/list/_ajax/edit', data=dict(data, list_item_id=item_id))
# We don't need to invalidate our cache if we remove the item
self._items = [i for i in self._items if i['imdb_id'] != entry['imdb_id']] if self._items else None
log.debug('found movie %s with ID %s in list %s, removing', entry['title'], entry['imdb_id'], self.list_id)
self.session.post('http://www.imdb.com/list/_ajax/edit', data=dict(data, list_item_id=item_id),
cookies=self.cookies)
# We don't need to invalidate our cache if we remove the item
self._items = [i for i in self._items if i['imdb_id'] != entry['imdb_id']] if self._items else None

def add(self, entry):
if self.config['list'] in IMMUTABLE_LISTS:
raise plugin.PluginError('%s lists are not modifiable' % ' and '.join(IMMUTABLE_LISTS))
if 'imdb_id' not in entry:
log.warning('Cannot add %s to imdb_list because it does not have an imdb_id', entry['title'])
return
# Manually calling authenticate to fetch list_id and cookies
self.authenticate()
data = {
'const': entry['imdb_id'],
'list_id': self.list_id,
'ref_tag': 'title'
}
self.session.post('http://www.imdb.com/list/_ajax/edit', data=data)
log.debug('adding title %s with ID %s to imdb %s', entry['title'], entry['imdb_id'], self.list_id)
self.session.post('http://www.imdb.com/list/_ajax/edit', data=data, cookies=self.cookies)
# Invalidate cache so that new movie info will be grabbed
self.invalidate_cache()

Expand Down

0 comments on commit ef91017

Please sign in to comment.