diff --git a/COPYING b/COPYING index 4ec8c3f7..dd1a5364 100644 --- a/COPYING +++ b/COPYING @@ -615,5 +615,3 @@ reviewing courts shall apply local law that most closely approximates an absolute waiver of all civil liability in connection with the Program, unless a warranty or assumption of liability accompanies a copy of the Program in return for a fee. - - END OF TERMS AND CONDITIONS diff --git a/bookie/__init__.py b/bookie/__init__.py index 249088cd..85dbc41b 100644 --- a/bookie/__init__.py +++ b/bookie/__init__.py @@ -1,10 +1,24 @@ +from pyramid.authentication import AuthTktAuthenticationPolicy +from pyramid.authorization import ACLAuthorizationPolicy from pyramid.config import Configurator -from pyramid.session import UnencryptedCookieSessionFactoryConfig from sqlalchemy import engine_from_config +from bookie.lib.access import RequestWithUserAttribute from bookie.models import initialize_sql +from bookie.models.auth import UserMgr from bookie.routes import build_routes +from pyramid.security import Allow +from pyramid.security import Everyone +from pyramid.security import ALL_PERMISSIONS + + +class RootFactory(object): + __acl__ = [ (Allow, Everyone, ALL_PERMISSIONS)] + + def __init__(self, request): + self.__dict__.update(request.matchdict) + def main(global_config, **settings): """ This function returns a Pyramid WSGI application. @@ -12,9 +26,16 @@ def main(global_config, **settings): engine = engine_from_config(settings, 'sqlalchemy.') initialize_sql(engine) - unencrypt = UnencryptedCookieSessionFactoryConfig('itsaseekreet') + authn_policy = AuthTktAuthenticationPolicy(settings.get('auth.secret'), + callback=UserMgr.auth_groupfinder) + authz_policy = ACLAuthorizationPolicy() + + config = Configurator(settings=settings, + root_factory='bookie.RootFactory', + authentication_policy=authn_policy, + authorization_policy=authz_policy) + config.set_request_factory(RequestWithUserAttribute) - config = Configurator(settings=settings, session_factory=unencrypt) config = build_routes(config) config.add_static_view('static', 'bookie:static') config.scan('bookie.views') diff --git a/bookie/lib/access.py b/bookie/lib/access.py index b50387b9..215b1985 100644 --- a/bookie/lib/access.py +++ b/bookie/lib/access.py @@ -1,17 +1,22 @@ """Handle auth and authz activities in bookie""" import logging +from pyramid.decorator import reify from pyramid.httpexceptions import HTTPForbidden -from pyramid.settings import asbool +from pyramid.request import Request +from pyramid.security import unauthenticated_userid + +from bookie.models.auth import UserMgr + LOG = logging.getLogger(__name__) -class Authorize(object): +class ApiAuthorize(object): """Context manager to check if the user is authorized use: - with Authorize(some_key): + with ApiAuthorize(some_key): # do work Will return NotAuthorized if it fails @@ -34,17 +39,56 @@ def __exit__(self, exc_type, exc_value, traceback): """No cleanup work to do after usage""" pass -def edit_enabled(settings): - """Is the config .ini setting for allowing edit enabled? +class ReqAuthorize(object): + """Context manager to check if the user is logged in - If the .ini setting for ui edits is not true, then no authed + use: + with ReqAuthorize(request): + # do work + + Will return NotAuthorized if it fails """ - allow_edit = asbool(settings.get('allow_edit', False)) - if allow_edit: - return True - else: - return False + def __init__(self, request, username=None): + """Create the context manager""" + LOG.debug('USER') + LOG.debug(request.user) + + self.request = request + self.username = username + + def __enter__(self): + """Verify api key set in constructor""" + if self.request.user is None: + LOG.error('Invalid Request: Not Logged in!') + raise HTTPForbidden('Invalid Authorization') + + # if we have a username we're told to check against, make sure the + # username matches + if self.username is not None and self.username != self.request.user.username: + LOG.error('Invalid Request: Wrong Username!') + raise HTTPForbidden('Invalid Authorization') + + def __exit__(self, exc_type, exc_value, traceback): + """No cleanup work to do after usage""" + pass +class RequestWithUserAttribute(Request): + @reify + def user(self): + # + # dbconn = self.registry.settings['dbconn'] + LOG.debug('in Request with Attribute') + user_id = unauthenticated_userid(self) + LOG.debug(user_id) + if user_id is not None: + LOG.debug('user_id is not none') + + # this should return None if the user doesn't exist + # in the database + user = UserMgr.get(user_id=user_id) + LOG.debug(user) + return user diff --git a/bookie/lib/importer.py b/bookie/lib/importer.py index 4316e287..cff54612 100644 --- a/bookie/lib/importer.py +++ b/bookie/lib/importer.py @@ -8,9 +8,10 @@ class Importer(object): """The actual factory object we use for handling imports""" - def __init__(self, import_io): + def __init__(self, import_io, username=None): """work on getting an importer instance""" self.file_handle = import_io + self.username = username def __new__(cls, *args, **kwargs): """Overriding new we return a subclass based on the file content""" @@ -42,7 +43,7 @@ def save_bookmark(self, url, desc, ext, tags, dt=None, fulltext=None): :param fulltext: Fulltext handler instance used to store that info """ - BmarkMgr.store(url, desc, ext, tags, dt=dt, fulltext=fulltext) + BmarkMgr.store(url, self.username, desc, ext, tags, dt=dt, fulltext=fulltext) class DelImporter(Importer): diff --git a/bookie/models/__init__.py b/bookie/models/__init__.py index 35e8caa7..99b549f0 100644 --- a/bookie/models/__init__.py +++ b/bookie/models/__init__.py @@ -151,7 +151,7 @@ def from_string(tag_str): return tag_objects @staticmethod - def find(order_by=None, tags=None): + def find(order_by=None, tags=None, username=None): """Find all of the tags in the system""" qry = Tag.query @@ -159,6 +159,12 @@ def find(order_by=None, tags=None): # limit to only the tag names in this list qry = qry.filter(Tag.name.in_(tags)) + if username: + # then we'll need to bind to bmarks to be able to limit on the + # username field + bmark = aliased(Bmark) + qry = qry.join((bmark, Tag.bmark)).filter(bmark.username==username) + if order_by is not None: qry = qry.order_by(order_by) else: @@ -388,28 +394,43 @@ class BmarkMgr(object): """Class to handle non-instance Bmark functions""" @staticmethod - def get_by_url(url): + def get_by_url(url, username=None): """Get a bmark from the system via the url""" # normalize the url clean_url = BmarkTools.normalize_url(url) - return Bmark.query.join(Bmark.hashed).\ + + qry = Bmark.query.join(Bmark.hashed).\ options(contains_eager(Bmark.hashed)).\ - filter(Hashed.url == clean_url).one() + filter(Hashed.url == clean_url) + + if username: + qry = qry.filter(Bmark.username==username) + + return qry.one() @staticmethod - def get_by_hash(hash_id): + def get_by_hash(hash_id, username=None): """Get a bmark from the system via the hash_id""" # normalize the url - return Bmark.query.join(Bmark.hashed).\ + qry = Bmark.query.join(Bmark.hashed).\ options(contains_eager(Bmark.hashed)).\ - filter(Hashed.hash_id == hash_id).first() + filter(Hashed.hash_id == hash_id) + + if username: + qry = qry.filter(Bmark.username == username) + + return qry.first() @staticmethod - def find(limit=50, order_by=None, page=0, tags=None, with_tags=True): + def find(limit=50, order_by=None, page=0, tags=None, with_tags=True, + username=None): """Search for specific sets of bookmarks""" qry = Bmark.query offset = limit * page + if username: + qry = qry.filter(Bmark.username == username) + if order_by is None: order_by = Bmark.stored.desc() @@ -495,7 +516,7 @@ def popular(limit=50, page=0, with_tags=False): return res @staticmethod - def store(url, desc, ext, tags, dt=None, fulltext=None): + def store(url, username, desc, ext, tags, dt=None, fulltext=None): """Store a bookmark :param url: bookmarked url @@ -506,6 +527,7 @@ def store(url, desc, ext, tags, dt=None, fulltext=None): """ mark = Bmark(url, + username, desc=desc, ext=ext, tags=tags, @@ -524,9 +546,14 @@ def store(url, desc, ext, tags, dt=None, fulltext=None): return mark @staticmethod - def hash_list(): + def hash_list(username=None): """Get a list of the hash_ids we have stored""" - return DBSession.query(Bmark.hash_id).all() + qry = DBSession.query(Bmark.hash_id) + + if username: + qry = qry.filter(Bmark.username==username) + + return qry.all() class BmarkTools(object): @@ -560,6 +587,8 @@ class Bmark(Base): stored = Column(DateTime, default=datetime.now) updated = Column(DateTime, onupdate=datetime.now) clicks = Column(Integer, default=0) + username = Column(Unicode(255), ForeignKey('users.username'), + nullable=False,) # DON"T USE tag_str = Column(UnicodeText()) @@ -579,7 +608,10 @@ class Bmark(Base): single_parent=True, ) - def __init__(self, url, desc=None, ext=None, tags=None): + user = relation("User", + backref="bmark") + + def __init__(self, url, username, desc=None, ext=None, tags=None): """Create a new bmark instance :param url: string of the url to be added as a bookmark @@ -597,6 +629,7 @@ def __init__(self, url, desc=None, ext=None, tags=None): else: self.hashed = existing + self.username = username self.description = desc self.extended = ext diff --git a/bookie/models/auth.py b/bookie/models/auth.py new file mode 100644 index 00000000..0e962688 --- /dev/null +++ b/bookie/models/auth.py @@ -0,0 +1,157 @@ +""" +Sample SQLAlchemy-powered model definition for the repoze.what SQL plugin. + +This model definition has been taken from a quickstarted TurboGears 2 project, +but it's absolutely independent of TurboGears. + +""" + +import bcrypt +import hashlib +import logging +import random + +from sqlalchemy import Column +from sqlalchemy import DateTime +from sqlalchemy import Integer +from sqlalchemy import Unicode +from sqlalchemy import Boolean +from sqlalchemy.orm import synonym + +from bookie.models import Base + +LOG = logging.getLogger(__name__) +GROUPS = ['admin', 'user'] + + +def get_random_word(wordLen): + word = '' + for i in xrange(wordLen): + word += random.choice('ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789/&=') + return word + + +class UserMgr(object): + """ Wrapper for static/combined operations of User object""" + + @staticmethod + def get_list(ignore_activated=False): + """Get a list of all of the user accounts""" + user_query = User.query.order_by(User.username) + + if not ignore_activated: + user_query = user_query.filter(User.activated == True) + + return user_query.all() + + @staticmethod + def get(user_id=None, username=None): + """Get the user instance for this information + + :param user_id: integer id of the user in db + :param username: string user's name + :param inactive: default to only get activated true + + """ + user_query = User.query + + if username is not None: + return user_query.filter(User.username == username).first() + + if user_id is not None: + return user_query.filter(User.id == user_id).first() + + return None + + @staticmethod + def auth_groupfinder(userid, request): + """Pyramid wants to know what groups a user is in + + We need to pull this from the User object that we've stashed in the request + object + + """ + LOG.debug('GROUP FINDER') + LOG.debug(userid) + LOG.debug(request) + user = request.user + LOG.debug(user) + if user is not None: + if user.is_admin: + return 'admin' + else: + return 'user' + return None + + +class User(Base): + """Basic User def""" + __tablename__ = 'users' + + id = Column(Integer, autoincrement=True, primary_key=True) + username = Column(Unicode(255), unique=True) + _password = Column('password', Unicode(60)) + email = Column(Unicode(255), unique=True) + activated = Column(Boolean, default=False) + is_admin = Column(Boolean, default=False) + last_login = Column(DateTime) + api_key = Column(Unicode(12)) + + def _set_password(self, password): + """Hash password on the fly.""" + hashed_password = password + + if isinstance(password, unicode): + password_8bit = password.encode('UTF-8') + else: + password_8bit = password + + # Hash a password for the first time, with a randomly-generated salt + salt = bcrypt.gensalt(10) + hashed_password = bcrypt.hashpw(password_8bit, salt) + + # Make sure the hased password is an UTF-8 object at the end of the + # process because SQLAlchemy _wants_ a unicode object for Unicode + # fields + if not isinstance(hashed_password, unicode): + hashed_password = hashed_password.decode('UTF-8') + + self._password = hashed_password + + def _get_password(self): + """Return the password hashed""" + return self._password + + password = synonym('_password', descriptor=property(_get_password, + _set_password)) + + def validate_password(self, password): + """ + Check the password against existing credentials. + + :param password: the password that was provided by the user to + try and authenticate. This is the clear text version that we will + need to match against the hashed one in the database. + :type password: unicode object. + :return: Whether the password is valid. + + """ + # the password might be null as in the case of morpace employees logging + # in via ldap. We check for that here and return them as an incorrect + # login + if self.password: + salt = self.password[:29] + return self.password == bcrypt.hashpw(password, salt) + else: + return False + + def deactivate(self): + """In case we need to disable the login""" + self.activated = False + + @staticmethod + def gen_api_key(): + """Generate a 12 char api key for the user to use""" + m = hashlib.sha256() + m.update(get_random_word(12)) + return unicode(m.hexdigest()[:12]) diff --git a/bookie/models/fulltext.py b/bookie/models/fulltext.py index 1098eb79..77e5937f 100644 --- a/bookie/models/fulltext.py +++ b/bookie/models/fulltext.py @@ -43,7 +43,7 @@ class SqliteFulltext(object): Storing is done automatically via the before_insert mapper hook on Bmark obj """ - def search(self, phrase, content=False): + def search(self, phrase, content=False, username=None): """Perform the search on the index""" #we need to adjust the phrase to be a set of OR per word phrase = " OR ".join(phrase.split()) @@ -59,9 +59,14 @@ def search(self, phrase, content=False): ext = SqliteBmarkFT.query.\ filter(SqliteBmarkFT.extended.match(phrase)) - res = desc.union(tag_str, ext).join(SqliteBmarkFT.bmark).\ - options(contains_eager(SqliteBmarkFT.bmark)).\ - order_by('bmarks.stored').all() + bmark = aliased(Bmark) + qry = desc.union(tag_str, ext).join((bmark, SqliteBmarkFT.bmark)).\ + options(contains_eager(SqliteBmarkFT.bmark, alias=bmark)) + + if username: + qry = qry.filter(bmark.username==username) + + res = qry.order_by(bmark.stored).all() # everyone else sends a list of bmarks, so need to get our bmarks # out of the result set @@ -84,6 +89,9 @@ def search(self, phrase, content=False): hashed.bmark, alias=bmarks)) + if username: + qry = qry.filter(bmark.username==username) + res = qry.order_by(bmarks.stored).all() for read in res: readable_res.append(read.hashed.bmark[0]) @@ -98,7 +106,7 @@ class MySqlFulltext(object): Columns: bid, description, extended, tags """ - def search(self, phrase, content=False): + def search(self, phrase, content=False, username=None): """Perform the search on the index""" #we need to adjust the phrase to be a set of OR per word phrase = " OR ".join(phrase.split()) @@ -114,7 +122,14 @@ def search(self, phrase, content=False): ext = Bmark.query.\ filter(Bmark.extended.match(phrase)) - results.update(set([bmark for bmark in desc.union(tag_str, ext).order_by(Bmark.stored).all()])) + qry = desc.union(tag_str, ext) + + if username: + qry = qry.filter(Bmark.username==username) + + res = qry.order_by(Bmark.stored).all() + + results.update(set([bmark for bmark in qry.all()])) readable_res = [] if content: @@ -130,6 +145,8 @@ def search(self, phrase, content=False): options(contains_eager(Readable.hashed, hashed.bmark, alias=bmarks)) + if username: + qry = qry.filter(bmarks.username==username) res = qry.order_by(bmarks.stored).all() for read in res: @@ -147,7 +164,7 @@ class PgSqlFulltext(object): """ - def search(self, phrase, content=False): + def search(self, phrase, content=False, username=None): """Need to perform searches against the three columns""" phrase = " | ".join(phrase.split()) @@ -165,9 +182,14 @@ def search(self, phrase, content=False): ids = set([r.bid for r in res]) - results.update(set([bmark for bmark in Bmark.query.join(Bmark.tags).\ - options(contains_eager(Bmark.tags)).\ - filter(Bmark.bid.in_(ids)).all()])) + qry = Bmark.query.join(Bmark.tags).\ + options(contains_eager(Bmark.tags)).\ + filter(Bmark.bid.in_(ids)) + + if username: + qry = qry.filter(Bmark.username==username) + + results.update(set([bmark for bmark in qry.all()])) readable_res = [] if content: @@ -182,9 +204,14 @@ def search(self, phrase, content=False): ids = set([r.hash_id for r in res]) - readable_res = [bmark for bmark in Bmark.query.join(Bmark.tags).\ - options(contains_eager(Bmark.tags)).\ - filter(Bmark.hash_id.in_(ids)).all()] + qry = Bmark.query.join(Bmark.tags).\ + options(contains_eager(Bmark.tags)).\ + filter(Bmark.hash_id.in_(ids)) + + if username: + qry = qry.filter(Bmark.username==username) + + readable_res = [bmark for bmark in qry.all()] results.update(set(readable_res)) return sorted(list(results), key=lambda res: res.stored, reverse=True) diff --git a/bookie/routes.py b/bookie/routes.py index 75fcda36..a29bcb94 100644 --- a/bookie/routes.py +++ b/bookie/routes.py @@ -73,11 +73,15 @@ def build_routes(config): config.add_route("home", "/") + # auth routes + config.add_route("login", "/login") + config.add_route("logout", "/logout") + # DELAPI Routes - config.add_route("del_post_add", "/delapi/posts/add") - config.add_route("del_post_delete", "/delapi/posts/delete") - config.add_route("del_post_get", "/delapi/posts/get") - config.add_route("del_tag_complete", "/delapi/tags/complete") + # config.add_route("del_post_add", "/{username}/delapi/posts/add") + # config.add_route("del_post_delete", "/{username}/delapi/posts/delete") + # config.add_route("del_post_get", "/{username}/delapi/posts/get") + # config.add_route("del_tag_complete", "/{username}/delapi/tags/complete") # bmark routes config.add_route("bmark_recent", "/recent") @@ -85,45 +89,76 @@ def build_routes(config): config.add_route("bmark_popular", "/popular") config.add_route("bmark_popular_tags", "/popular/*tags") - - config.add_route("bmark_delete", "/bmark/delete") - config.add_route("bmark_confirm_delete", "/bmark/confirm/delete/{bid}") config.add_route("bmark_readable", "/bmark/readable/{hash_id}") + # user based bmark routes + config.add_route("user_bmark_recent", "/{username}/recent") + config.add_route("user_bmark_recent_tags", "/{username}/recent/*tags") + + config.add_route("user_bmark_popular", "/{username}/popular") + config.add_route("user_bmark_popular_tags", "/{username}/popular/*tags") + + # config.add_route("bmark_delete", "/bmark/delete") + # config.add_route("bmark_confirm_delete", "/bmark/confirm/delete/{bid}") # tag related routes - # config.add_route("tag_list", "/tags") - # config.add_route("tag_bmarks_ajax", "/tags/*tags", xhr=True) + config.add_route("tag_list", "/tags") config.add_route("tag_bmarks", "/tags/*tags") - config.add_route("import", "/import") + # user tag related + config.add_route("user_tag_list", "/{username}/tags") + config.add_route("user_tag_bmarks", "/{username}/tags/*tags") + + config.add_route("user_import", "/{username}/import") config.add_route("search", "/search") + config.add_route("user_search", "/{username}/search") + config.add_route("search_results", "/results") + config.add_route("user_search_results", "/{username}/results") # matches based on the header # HTTP_X_REQUESTED_WITH + # ajax versions are used in the mobile search interface config.add_route("search_results_ajax", "/results*terms", xhr=True) config.add_route("search_results_rest", "/results*terms") + config.add_route("user_search_results_ajax", "/{username}/results*terms", xhr=True) + config.add_route("user_search_results_rest", "/{username}/results*terms") + + # removed the overall export. We're not going to have a link for exporting + # all in one swoop. It'll kill things + config.add_route("user_export", "/{username}/export") - config.add_route("export", "/export") config.add_route("redirect", "/redirect/{hash_id}") + config.add_route("user_redirect", "/{username}/redirect/{hash_id}") # MOBILE routes config.add_route("mobile", "/m") + config.add_route("user_mobile", "/{username}/m") # API config.add_route('api_bmark_recent', '/api/v1/bmarks/recent') config.add_route('api_bmark_popular', '/api/v1/bmarks/popular') config.add_route("api_bmark_search", "/api/v1/bmarks/search/*terms") - config.add_route("api_bmark_sync", "/api/v1/bmarks/sync") - config.add_route("api_bmark_add", "/api/v1/bmarks/add") - config.add_route("api_bmark_remove", "/api/v1/bmarks/remove") - - # this route must be last, none of the above will look like hashes (22char) - # so it's safe to have as a kind of default route at the end + config.add_route('api_bmark_get_readable', '/api/v1/bmarks/get_readable') + config.add_route('api_bmark_readable', '/api/v1/bmarks/readable') + + config.add_route("user_api_bmark_recent", "/{username}/api/v1/bmarks/recent") + config.add_route("user_api_bmark_popular", "/{username}/api/v1/bmarks/popular") + config.add_route("user_api_bmark_search", "/{username}/api/v1/bmarks/search/*terms") + config.add_route("user_api_bmark_sync", "/{username}/api/v1/bmarks/sync") + config.add_route("user_api_bmark_add", "/{username}/api/v1/bmarks/add") + config.add_route("user_api_bmark_remove", "/{username}/api/v1/bmarks/remove") + + # # this route must be last, none of the above will look like hashes (22char) + # # so it's safe to have as a kind of default route at the end config.add_route("api_bmark_hash", "/api/v1/bmarks/{hash_id}") + config.add_route("user_api_bmark_hash", "/{username}/api/v1/bmarks/{hash_id}") - # api calls for tag relation information + # # api calls for tag relation information config.add_route("api_tag_complete", "/api/v1/tags/complete") + config.add_route("user_api_tag_complete", "/{username}/api/v1/tags/complete") + + # these are single word matching, they must be after /recent /popular etc + config.add_route("user_home", "/{username}") return config diff --git a/bookie/static/css/bookie.css b/bookie/static/css/bookie.css index a8b745ef..e3218f61 100644 --- a/bookie/static/css/bookie.css +++ b/bookie/static/css/bookie.css @@ -1,4 +1,5 @@ -/* BASIC PROPERTIES */ + + BODY { } diff --git a/bookie/static/js/bookie.js b/bookie/static/js/bookie.js index d0e3af72..ddb2951c 100644 --- a/bookie/static/js/bookie.js +++ b/bookie/static/js/bookie.js @@ -21,7 +21,6 @@ var bookie = (function ($b, $) { $b.events = { 'LOAD': 'load', 'TAG_FILTER': 'tag_filter', - 'SEARCH': 'search' }; @@ -110,7 +109,6 @@ var bookie = (function ($b, $) { // bind some other events we might want read to go out of the gates $($b.EVENTID).bind($b.events.TAG_FILTER, $b.ui.init_tag_filter); - $($b.EVENTID).bind($b.events.SEARCH, $b.call.search); // now trigger the load since we're ready to go from here $($b.EVENTID).trigger($b.events.LOAD); diff --git a/bookie/static/js/mobile.js b/bookie/static/js/mobile.js index 4cbebf86..ecde258e 100644 --- a/bookie/static/js/mobile.js +++ b/bookie/static/js/mobile.js @@ -233,7 +233,11 @@ var bookie = (function ($b, $) { } }, 'complete': function () { - $.mobile.changePage('#results', 'slide', back=false, changeHash=false); + console.log('fired complete'); + $.mobile.changePage('#results', + 'slide', + back=false, + changeHash=false); $.mobile.pageLoading(true); } } @@ -257,11 +261,13 @@ var bookie = (function ($b, $) { $($b.ui.results.title_id).html(title); // this isn't always init'd so need to init it first - $(data_home).listview('refresh'); + console.log(data_home); + + $(data_home).listview('destroy').listview(); // now bind the swipe event to allow following of the links $('.bookmark_link').bind('click', function (ev) { - ev.preventdefault(); + ev.preventDefault(); // the url we need to call is /redirect/hash_id var hash_id = $(this).attr('data-hash'), url = APP_URL + "/redirect/" + hash_id, @@ -290,6 +296,7 @@ var bookie = (function ($b, $) { 'go_search': $b.events.SEARCH, }, + nav_buttons = function (id, event_id, data_home, callback) { if (callback !== undefined) { $($b.EVENTID).bind(event_id, callback); @@ -297,7 +304,6 @@ var bookie = (function ($b, $) { $('.' + id).bind('click', function (ev) { $b.mobilestate.clear(); - console.log(callback); if (callback !== undefined) { ev.preventDefault(); diff --git a/bookie/templates/auth/login.mako b/bookie/templates/auth/login.mako new file mode 100644 index 00000000..28337650 --- /dev/null +++ b/bookie/templates/auth/login.mako @@ -0,0 +1,17 @@ +<%inherit file="/main_wrap.mako" /> +<%def name="title()">Login + +

Log In

+ +

${message}

+ +
+ +
diff --git a/bookie/templates/bmark/func.mako b/bookie/templates/bmark/func.mako index 33930f31..840791af 100644 --- a/bookie/templates/bmark/func.mako +++ b/bookie/templates/bmark/func.mako @@ -13,8 +13,6 @@ % endfor - - <%def name="display_bmark_list(bmark_list)"> @@ -38,11 +36,10 @@ <% is_new = (last_date != bmark.stored.strftime("%m/%d")) %> -
@@ -56,29 +53,48 @@
- R - % if allow_edit: - E - X + % if user: + E + % endif
+ <% + if request.user: + route = 'user_tag_bmarks' + username = request.user.username + else: + route = 'tag_bmarks' + username = None + %> + % for tag in bmark.tags: ${tag} + href="${request.route_url(route, + tags=[tag], + page=prev, + username=username)}">${tag} % endfor
@@ -100,7 +116,7 @@
-<%def name="bmarknextprev(page, max_count, count, next_url, url_params=None, tags=None)"> +<%def name="bmarknextprev(page, max_count, count, next_url, url_params=None, tags=None, username=None)"> <% if max_count == count: show_next = True @@ -116,21 +132,21 @@ %> % if page != 0: - Prev % endif % if show_next: - Next % endif -<%def name="tag_filter(url, tags=None)"> +<%def name="tag_filter(url, tags=None, username=None)">
+ action="${request.route_url(url, tags=tags, username=username)}" method="GET"> Tags  <%def name="title()">Displaying: ${bmark.url} -

Displaying: ${bmark.bmark[0].description}

+<% + username = None + if request.user: + username = request.user.username +%> + +

Displaying: ${bmark.bmark[0].description}

+ % else: + href="${request.route_url('redirect', hash_id=bmark.hash_id)}"> + % endif + ${bmark.bmark[0].description}
% if bmark.readable: ${bmark.readable.content} diff --git a/bookie/templates/bmark/recent.mako b/bookie/templates/bmark/recent.mako index 5491ae53..b50248e9 100644 --- a/bookie/templates/bmark/recent.mako +++ b/bookie/templates/bmark/recent.mako @@ -11,16 +11,21 @@ else: url = 'bmark_recent' + username = None + if request.user: + url = 'user_' + url + username = request.user.username + %>
- ${tag_filter(url, tags=tags)} + ${tag_filter(url, tags=tags, username=username)}
Showing ${max_count} bookmarks
 
- ${bmarknextprev(page, max_count, count, url, tags=tags)} + ${bmarknextprev(page, max_count, count, url, tags=tags, username=username)}
@@ -30,6 +35,6 @@
 
- ${bmarknextprev(page, max_count, count, url, tags=tags)} + ${bmarknextprev(page, max_count, count, url, tags=tags, username=username)}
diff --git a/bookie/templates/main_wrap.mako b/bookie/templates/main_wrap.mako index 73b1d1c4..7bb34368 100644 --- a/bookie/templates/main_wrap.mako +++ b/bookie/templates/main_wrap.mako @@ -18,10 +18,11 @@ @@ -29,7 +30,7 @@ diff --git a/bookie/templates/mobile/index.mako b/bookie/templates/mobile/index.mako index d765df7a..dc8324b7 100644 --- a/bookie/templates/mobile/index.mako +++ b/bookie/templates/mobile/index.mako @@ -187,7 +187,9 @@
-
    +
      +
    • No results
diff --git a/bookie/templates/mobile_wrap.mako b/bookie/templates/mobile_wrap.mako index 1aae7120..5dad52b7 100644 --- a/bookie/templates/mobile_wrap.mako +++ b/bookie/templates/mobile_wrap.mako @@ -22,12 +22,17 @@ 'console_log': logger } + % if hasattr(self, 'header'): ${self.header()} % endif + @@ -45,6 +50,7 @@ // do not do form submissions via ajax by default. We catch and // override them to handle things manually $.mobile.ajaxFormsEnabled = false; + $('#results_list').listview(); bookie.init(); }); @@ -53,7 +59,7 @@ }); - - + + diff --git a/bookie/templates/tag/list.mako b/bookie/templates/tag/list.mako index 617df4e3..3bf880e6 100644 --- a/bookie/templates/tag/list.mako +++ b/bookie/templates/tag/list.mako @@ -2,11 +2,20 @@ <%def name="title()">Your Tags

${tag_count} Tags

- +<% + username = None + if request.user: + username = request.user.username +%>
% for tag in tag_list:
- ${tag.name} + % if username: + ${tag.name} + % else: + ${tag.name} + % endif
% endfor
diff --git a/bookie/templates/utils/import.mako b/bookie/templates/utils/import.mako index f9f33016..875a002c 100644 --- a/bookie/templates/utils/import.mako +++ b/bookie/templates/utils/import.mako @@ -18,18 +18,15 @@
- +
  • -
  • - - -
  • -
  • @@ -52,9 +49,6 @@

    To get an html file from Google Bookmarks, click on Export Bookmarks under the Tools heading on the sidebar.

    -

    API Key

    -

    "API Key" is the secret key specific to this Bookie installation. This setting is in the .ini file associated with this installation (usually found in the Bookie/ directory). -

    Note that the import process might take a bit of time on large sets of bookmarks.

diff --git a/bookie/templates/utils/search.mako b/bookie/templates/utils/search.mako index 3a1d8c25..84b32652 100644 --- a/bookie/templates/utils/search.mako +++ b/bookie/templates/utils/search.mako @@ -7,7 +7,14 @@

Search

- +
diff --git a/bookie/tests/__init__.py b/bookie/tests/__init__.py index b293ba3d..9a795bca 100644 --- a/bookie/tests/__init__.py +++ b/bookie/tests/__init__.py @@ -12,6 +12,7 @@ from bookie.models import Hashed from bookie.models import Tag, bmarks_tags from bookie.models import SqliteBmarkFT +from bookie.models.auth import User global_config = {} diff --git a/bookie/tests/test_api/__init__.py b/bookie/tests/test_api/__init__.py index aaa5e3d2..34bc90f9 100644 --- a/bookie/tests/test_api/__init__.py +++ b/bookie/tests/test_api/__init__.py @@ -15,6 +15,9 @@ BMARKUS_HASH = 'c5c21717c99797' LOG = logging.getLogger(__name__) +API_KEY = None + + class BookieAPITest(unittest.TestCase): """Test the Bookie API""" @@ -25,6 +28,10 @@ def setUp(self): self.testapp = TestApp(app) testing.setUp() + global API_KEY + res = DBSession.execute("SELECT api_key FROM users WHERE username = 'admin'").fetchone() + API_KEY = res['api_key'] + def tearDown(self): """We need to empty the bmarks table on each run""" testing.tearDown() @@ -34,29 +41,13 @@ def _get_good_request(self, content=False, second_bmark=False): """Return the basics for a good add bookmark request""" session = DBSession() - if second_bmark: - prms = { - 'url': u'http://bmark.us', - 'description': u'Bookie', - 'extended': u'Extended notes', - 'tags': u'bookmarks', - 'api_key': u'testapi', - } - - # if we want to test the readable fulltext side we want to make sure we - # pass content into the new bookmark - prms['content'] = "

Second bookmark man

" - - req_params = urllib.urlencode(prms) - res = self.testapp.get('/delapi/posts/add?' + req_params) - # the main bookmark, added second to prove popular will sort correctly prms = { 'url': u'http://google.com', 'description': u'This is my google desc', 'extended': u'And some extended notes about it in full form', 'tags': u'python search', - 'api_key': u'testapi', + 'api_key': API_KEY, } # if we want to test the readable fulltext side we want to make sure we @@ -65,16 +56,58 @@ def _get_good_request(self, content=False, second_bmark=False): prms['content'] = "

There's some content in here dude

" req_params = urllib.urlencode(prms) - res = self.testapp.get('/delapi/posts/add?' + req_params) + res = self.testapp.post('/admin/api/v1/bmarks/add?', + params=req_params) + + if second_bmark: + prms = { + 'url': u'http://bmark.us', + 'description': u'Bookie', + 'extended': u'Extended notes', + 'tags': u'bookmarks', + 'api_key': API_KEY, + } + + # if we want to test the readable fulltext side we want to make sure we + # pass content into the new bookmark + prms['content'] = "

Second bookmark man

" + + req_params = urllib.urlencode(prms) + res = self.testapp.post('/admin/api/v1/bmarks/add?', + params=req_params) session.flush() transaction.commit() return res + def test_add_bookmark(self): + """We should be able to add a new bookmark to the system""" + # we need to know what the current admin's api key is so we can try to + # add + res = DBSession.execute("SELECT api_key FROM users WHERE username = 'admin'").fetchone() + key = res['api_key'] + + test_bmark = { + 'url': u'http://bmark.us', + 'description': u'Bookie', + 'extended': u'Extended notes', + 'tags': u'bookmarks', + 'api_key': key, + } + + res = self.testapp.post('/admin/api/v1/bmarks/add', + params=test_bmark, + status=200) + + ok_('"success": true' in res.body, + "Should have a success of true: " + res.body) + ok_('message": "done"' in res.body, + "Should have a done message: " + res.body) + def test_bookmark_fetch(self): """Test that we can get a bookmark and it's details""" self._get_good_request(content=True) - res = self.testapp.get('/api/v1/bmarks/' + GOOGLE_HASH) + res = self.testapp.get('/admin/api/v1/bmarks/' + GOOGLE_HASH) eq_(res.status, "200 OK", msg='Get status is 200, ' + res.status) @@ -101,7 +134,7 @@ def test_bookmark_fetch_fail(self): self._get_good_request() # test that we only get one resultback - res = self.testapp.get('/api/v1/bmarks/' + BMARKUS_HASH, status=200) + res = self.testapp.get('/admin/api/v1/bmarks/' + BMARKUS_HASH, status=200) ok_('"success": false' in res.body, "Should have a false success" + res.body) @@ -109,7 +142,7 @@ def test_bookmark_fetch_fail(self): def test_bookmark_recent(self): """Test that we can get list of bookmarks with details""" self._get_good_request(content=True) - res = self.testapp.get('/api/v1/bmarks/recent') + res = self.testapp.get('/admin/api/v1/bmarks/recent') eq_(res.status, "200 OK", msg='Get status is 200, ' + res.status) @@ -130,10 +163,10 @@ def test_bookmark_popular(self): self._get_good_request(content=True, second_bmark=True) # we want to make sure the click count of 0 is greater than 1 - res = self.testapp.get('/redirect/' + GOOGLE_HASH) - res = self.testapp.get('/redirect/' + GOOGLE_HASH) + res = self.testapp.get('/admin/redirect/' + GOOGLE_HASH) + res = self.testapp.get('/admin/redirect/' + GOOGLE_HASH) - res = self.testapp.get('/api/v1/bmarks/popular') + res = self.testapp.get('/admin/api/v1/bmarks/popular') eq_(res.status, "200 OK", msg='Get status is 200, ' + res.status) @@ -148,7 +181,9 @@ def test_bookmark_popular(self): bmark2 = bmark_list[1] eq_(GOOGLE_HASH, bmark1[u'hash_id'], - "The hash_id should match: " + str(bmark1[u'hash_id'])) + "The hash_id {0} should match: {1} ".format( + str(GOOGLE_HASH), + str(bmark1[u'hash_id']))) ok_('clicks' in bmark1, "The clicks field should be in there") @@ -162,7 +197,7 @@ def test_paging_results(self): self._get_good_request(content=True, second_bmark=True) # test that we only get one resultback - res = self.testapp.get('/api/v1/bmarks/recent?page=0&count=1') + res = self.testapp.get('/admin/api/v1/bmarks/recent?page=0&count=1') eq_(res.status, "200 OK", msg='Get status is 200, ' + res.status) @@ -172,7 +207,7 @@ def test_paging_results(self): eq_(len(bmarks), 1, "We should only have one result in this page") - res = self.testapp.get('/api/v1/bmarks/recent?page=1&count=1') + res = self.testapp.get('/admin/api/v1/bmarks/recent?page=1&count=1') eq_(res.status, "200 OK", msg='Get status is 200, ' + res.status) @@ -183,7 +218,7 @@ def test_paging_results(self): eq_(len(bmarks), 1, "We should only have one result in the second page") - res = self.testapp.get('/api/v1/bmarks/recent?page=2&count=1') + res = self.testapp.get('/admin/api/v1/bmarks/recent?page=2&count=1') eq_(res.status, "200 OK", msg='Get status is 200, ' + res.status) @@ -199,7 +234,9 @@ def test_bookmark_sync(self): self._get_good_request(content=True, second_bmark=True) # test that we only get one resultback - res = self.testapp.get('/api/v1/bmarks/sync') + res = self.testapp.get('/admin/api/v1/bmarks/sync', + params={'api_key': API_KEY}, + status=200) eq_(res.status, "200 OK", msg='Get status is 200, ' + res.status) @@ -216,10 +253,10 @@ def test_bookmark_add(self): 'description': u'Bookie', 'extended': u'Extended notes', 'tags': u'bookmarks', - 'api_key': u'testapi', + 'api_key': API_KEY, } - res = self.testapp.post('/api/v1/bmarks/add', params=test_bmark, + res = self.testapp.post('/admin/api/v1/bmarks/add', params=test_bmark, status=200) ok_('"success": true' in res.body, @@ -237,7 +274,7 @@ def test_bookmark_add_bad_key(self): 'api_key': u'badkey', } - self.testapp.post('/api/v1/bmarks/add', params=test_bmark, + self.testapp.post('/admin/api/v1/bmarks/add', params=test_bmark, status=403) def test_bookmark_toread(self): @@ -248,10 +285,10 @@ def test_bookmark_toread(self): 'description': u'Bookie', 'extended': u'Extended notes', 'tags': u'bookmarks !toread', - 'api_key': u'testapi', + 'api_key': API_KEY, } - res = self.testapp.post('/api/v1/bmarks/add', params=test_bmark, + res = self.testapp.post('/admin/api/v1/bmarks/add', params=test_bmark, status=200) ok_('"success": true' in res.body, @@ -270,15 +307,15 @@ def test_bookmark_update_toread(self): 'description': u'Bookie', 'extended': u'Extended notes', 'tags': u'bookmarks', - 'api_key': u'testapi', + 'api_key': API_KEY, } - res = self.testapp.post('/api/v1/bmarks/add', params=test_bmark, + res = self.testapp.post('/admin/api/v1/bmarks/add', params=test_bmark, status=200) test_bmark['tags'] = u'!toread' - res = self.testapp.post('/api/v1/bmarks/add', params=test_bmark, + res = self.testapp.post('/admin/api/v1/bmarks/add', params=test_bmark, status=200) ok_('toread' in res.body, @@ -291,9 +328,9 @@ def test_bookmark_remove(self): self._get_good_request(content=True, second_bmark=True) # now let's delete the google bookmark - res = self.testapp.post('/api/v1/bmarks/remove', params = { + res = self.testapp.post('/admin/api/v1/bmarks/remove', params = { 'url': u'http://google.com', - 'api_key': 'testapi', + 'api_key': API_KEY }, status=200) ok_('success": true' in res.body, @@ -302,7 +339,9 @@ def test_bookmark_remove(self): # we're going to cheat like mad, use the sync call to get the hash_ids # of bookmarks in the system and verify that only the bmark.us hash_id # is in the response body - res = self.testapp.get('/api/v1/bmarks/sync', status=200) + res = self.testapp.get('/admin/api/v1/bmarks/sync', + params={'api_key': API_KEY}, + status=200) ok_(GOOGLE_HASH not in res.body, "Should not have the google hash: " + res.body) @@ -317,7 +356,7 @@ def test_bookmark_tag_complete(self): """ self._get_good_request(second_bmark=True) - res = self.testapp.get('/api/v1/tags/complete', + res = self.testapp.get('/admin/api/v1/tags/complete', params={'tag': 'py'}, status=200) ok_('python' in res.body, @@ -325,7 +364,7 @@ def test_bookmark_tag_complete(self): # we shouldn't get python as an option if we supply bookmarks as the # current tag. No bookmarks have both bookmarks & python as tags - res = self.testapp.get('/api/v1/tags/complete', + res = self.testapp.get('/admin/api/v1/tags/complete', params={'tag': 'py', 'current': 'bookmarks'}, status=200) diff --git a/bookie/tests/test_auth/__init__.py b/bookie/tests/test_auth/__init__.py new file mode 100644 index 00000000..c17eca74 --- /dev/null +++ b/bookie/tests/test_auth/__init__.py @@ -0,0 +1,85 @@ +"""Test the auth related web calls""" +import logging +import transaction + +from nose.tools import ok_, eq_ +from pyramid import testing +from unittest import TestCase + +# from bookie.lib import access +# from bookie.models import DBSession + +LOG = logging.getLogger(__name__) + + +class TestAuthWeb(TestCase): + """Testing web calls""" + + def setUp(self): + from pyramid.paster import get_app + from bookie.tests import BOOKIE_TEST_INI + app = get_app(BOOKIE_TEST_INI, 'main') + from webtest import TestApp + self.testapp = TestApp(app) + testing.setUp() + + def tearDown(self): + """We need to empty the bmarks table on each run""" + testing.tearDown() + # session = DBSession() + # Bmark.query.delete() + # Tag.query.delete() + # Hashed.query.delete() + # session.execute(bmarks_tags.delete()) + # session.flush() + # transaction.commit() + + def test_login_url(self): + """Verify we get the login form""" + res = self.testapp.get('/login', status=200) + + body_str = "Log In" + form_str = 'name="login"' + + ok_(body_str in res.body, + msg="Request should contain Log In: " + res.body) + + # there should be a login form on there + ok_(form_str in res.body, + msg="The login input should be visible in the body:" + res.body) + + def test_login_success(self): + """Verify a good login""" + + # the migrations add a default admin account + user_data = {'login': 'admin', + 'password': 'admin', + 'form.submitted': 'true'} + + res = self.testapp.post('/login', + params=user_data) + eq_(res.status, "302 Found", + msg='status is 302 Found, ' + res.status) + + # should end up back at the recent page + res = res.follow() + ok_('recent' in str(res), + "Should have 'recent' in the resp: " + str(res)) + + def test_login_failure(self): + """Verify a bad login""" + + # the migrations add a default admin account + user_data = {'login': 'admin', + 'password': 'wrongpass', + 'form.submitted': 'true'} + + res = self.testapp.post('/login', + params=user_data) + + eq_(res.status, "200 OK", + msg='status is 200 OK, ' + res.status) + + # should end up back at login with an error message + ok_('Failed login' in str(res), + "Should have 'Failed login' in the resp: " + str(res)) diff --git a/bookie/tests/test_auth/test_model.py b/bookie/tests/test_auth/test_model.py new file mode 100644 index 00000000..ee20b71c --- /dev/null +++ b/bookie/tests/test_auth/test_model.py @@ -0,0 +1,69 @@ +"""Test the Auth model setup""" +import transaction +from unittest import TestCase +from nose.tools import ok_, eq_ + +from bookie.models import DBSession +from bookie.models.auth import User +from bookie.models.auth import UserMgr + +from bookie.tests import empty_db + +class TestPassword(TestCase): + """Test password checks""" + pass + + +class TestAuthUser(TestCase): + """Test User Model""" + test_hash = '$2a$10$FMFKEYqC7kifFTm05iag7etE17Q0AyKvtX88XUdUcM7rvpz48He92' + test_password = 'testing' + + def test_password_set(self): + """Make sure we get the proper hashed password""" + tst = User() + tst.password = self.test_password + + eq_(len(tst.password), 60, + "Hashed should be 60 char long: " + tst.password) + eq_('$2a$', tst.password[:4], + "Hash should start with the right complexity: " + tst.password[:4]) + + def test_password_match(self): + """Try to match a given hash""" + + tst = User() + tst._password = self.test_hash + + ok_(tst._password == self.test_hash, "Setting should have hash") + ok_(tst.password == self.test_hash, "Getting should have hash") + ok_(tst.validate_password(self.test_password), + "The password should check out against the given hash: " + tst.password) + + +class TestAuthMgr(TestCase): + """Test User Manager""" + + def test_get_id(self): + """Fetching user by the id""" + # the migration adds an initial admin user to the system + user = UserMgr.get(user_id=1) + eq_(user.id, 1, + "Should have a user id of 1: " + str(user.id)) + eq_(user.username, 'admin', + "Should have a username of admin: " + user.username) + + def test_get_username(self): + """Fetching the user by the username""" + user = UserMgr.get(username='admin') + eq_(user.id, 1, + "Should have a user id of 1: " + str(user.id)) + eq_(user.username, 'admin', + "Should have a username of admin: " + user.username) + + def test_get_bad_user(self): + """We shouldn't get a hit if the user is inactive""" + user = UserMgr.get(username='noexist') + + eq_(user, None, + "Should not find a non-existant user: " + str(user)) diff --git a/bookie/tests/test_delicious/__init__.py b/bookie/tests/test_delicious/__init__.py index 52ec0ebc..31a48690 100644 --- a/bookie/tests/test_delicious/__init__.py +++ b/bookie/tests/test_delicious/__init__.py @@ -1,394 +1,394 @@ -"""Test that we're meeting delicious API specifications""" -from datetime import datetime, timedelta -import transaction -import unittest -import urllib -from nose.tools import ok_, eq_ -from pyramid import testing - -from bookie.models import DBSession -from bookie.models import Bmark, NoResultFound -from bookie.models import Hashed -from bookie.models import Tag, bmarks_tags -from bookie.models import SqliteBmarkFT - -from bookie.tests import BOOKIE_TEST_INI - -GOOGLE_HASH = 'aa2239c17609b2' - - -class DelPostTest(unittest.TestCase): - """Test post related calls""" - - def setUp(self): - from pyramid.paster import get_app - app = get_app(BOOKIE_TEST_INI, 'main') - from webtest import TestApp - self.testapp = TestApp(app) - testing.setUp() - - def tearDown(self): - """We need to empty the bmarks table on each run""" - testing.tearDown() - - if BOOKIE_TEST_INI == 'test.ini': - SqliteBmarkFT.query.delete() - Bmark.query.delete() - Tag.query.delete() - Hashed.query.delete() - - DBSession.execute(bmarks_tags.delete()) - DBSession.flush() - transaction.commit() - - def _get_good_request(self, content=False): - """Return the basics for a good add bookmark request""" - session = DBSession() - prms = { - 'url': u'http://google.com', - 'description': u'This is my google desc', - 'extended': u'And some extended notes about it in full form', - 'tags': u'python search', - 'api_key': u'testapi', - } - - # if we want to test the readable fulltext side we want to make sure we - # pass content into the new bookmark - if content: - prms['content'] = "

There's some content in here dude

" - - req_params = urllib.urlencode(prms) - res = self.testapp.get('/delapi/posts/add?' + req_params) - session.flush() - transaction.commit() - return res - - def test_post_add_fail(self): - """Basic add of a new post failing - - Failed response: - - - Not supporting optional params right now - replace, shared - - """ - failed = '' - prms = { - 'url': '', - 'description': '', - 'extended': '', - 'tags': '', - 'api_key': u'testapi', - } - - req_params = urllib.urlencode(prms) - - res = self.testapp.get('/delapi/posts/add?' + req_params) - eq_(res.status, "200 OK", msg='Post Add status is 200, ' + res.status) - eq_(res.body, failed, msg="Request should return failed msg: " + res.body) - - def test_post_add_success(self): - """Basic add of a new post working - - Success response: - - - Not supporting optional params right now - replace, shared - - """ - success = '' - res = self._get_good_request() - - eq_(res.status, "200 OK", msg='Post Add status is 200, ' + res.status) - eq_(res.body, success, msg="Request should return done msg") - - def test_new_bmark(self): - # go save the thing - self._get_good_request() - - try: - res = Bmark.query.filter(Bmark.hash_id == GOOGLE_HASH).one() - ok_(res, 'We found a result in the db for this bookmark') - ok_('extended' in res.extended, - 'Extended value was set to bookmark') - - # make sure our hash was stored correctly - ok_(res.hashed.url == 'http://google.com', - "The hashed object got the url") - - ok_(res.hashed.clicks == 0, "No clicks on the url yet") - - if res: - return True - else: - assert False, "Could not find our bookmark we saved" - except NoResultFound: - assert False, "No result found for the bookmark" - - def test_post_add_with_dt(self): - """Make sure if we provide a date it works - - Success response: - - - Not supporting optional params right now - replace, shared - - """ - - success = '' - session = DBSession() - - # pick a date that is tomorrow - today = datetime.now() - yesterday = today - timedelta(days=1) - dt = yesterday.strftime("%Y-%m-%dT%H:%M:%SZ") - prms = { - 'url': u'http://google.com', - 'description': u'This is my google desc', - 'extended': u'And some extended notes about it in full form', - 'tags': u'python search', - 'dt': dt, - 'api_key': u'testapi', - } - - req_params = urllib.urlencode(prms) - res = self.testapp.get('/delapi/posts/add?' + req_params) - session.flush() - - eq_(res.status, "200 OK", msg='Post Add status is 200, ' + res.status) - eq_(res.body, success, msg="Request should return done msg") - - # now pull up the bmark and check the date is yesterday - res = Bmark.query.filter(Bmark.hash_id == GOOGLE_HASH).one() - eq_(res.stored.strftime('%Y-%m-%d'), yesterday.strftime('%Y-%m-%d'), - "The stored date {0} is the same as the requested {1}".format( - res.stored, - yesterday)) - - def test_new_bmark_tags(self): - """Manually check db for new bmark tags set""" - self._get_good_request() - - res = Bmark.query.filter(Bmark.hash_id == GOOGLE_HASH).one() - - ok_('python' in res.tags, 'Found the python tag in the bmark') - ok_('search' in res.tags, 'Found the search tag in the bmark') - - def test_skip_dupe_tags(self): - """Make sure we don't end up with duplicate tags in the system""" - self._get_good_request() - self._get_good_request() - - all_tags = Tag.query.all() - - ok_(len(all_tags) == 2, 'We only have two tags in the system') - - def test_datestimes_set(self): - """Test that we get the new datetime fields as we work""" - - # we've got some issue with mysql truncating the timestamp to not - # include seconds, so we allow for a one minute leeway in the - # timestamp. Enough to know it's set and close enough for government - # use - now = datetime.now() - timedelta(minutes=1) - self._get_good_request() - res = Bmark.query.filter(Bmark.hash_id == GOOGLE_HASH).one() - - ok_(res.stored >= now, - "Stored time is now or close to now {0}--{1}".format(res.stored, now)) - - res.hash_id = u"Somethingnew.com" - DBSession.flush() - - # now hopefully have an updated value - ok_(res.updated >= now, - "Stored time, after update, is now or close to now {0}--{1}".format(res.updated, now)) - - def test_remove_bmark(self): - """Remove a bmark from the system - - We want to make sure we store content in here to make sure all the - delete cascades are operating properly - - """ - res1 = self._get_good_request(content=True) - ok_('done' in res1.body, res1.body) - - # now send in the delete squad - prms = { - 'url': u'http://google.com', - 'api_key': u'testapi', - } - - req_params = urllib.urlencode(prms) - - res = self.testapp.get('/delapi/posts/delete?' + req_params) - eq_(res.status, "200 OK", 'Post Delete status is 200, ' + res.status) - ok_('done' in res.body, "Request should return done msg: " + res.body) - - # now make sure our hashed object is gone as well. - res = Hashed.query.get(GOOGLE_HASH) - ok_(not res, "We didn't get our hash object") - - def test_get_post_byurl(self): - """Verify we can fetch a post back via a url - - While this is delicious api compat, we're going to default to json - response I think - - We'll add xml support to the output later on - - """ - self._get_good_request() - prms = { - 'url': u'http://google.com', - } - - req_params = urllib.urlencode(prms) - - res = self.testapp.get('/delapi/posts/get?' + req_params) - - ok_('href' in res.body, "we have an href link in there") - ok_('python' in res.body, "we have the python tag") - ok_('search' in res.body, "we have the search tag") - - def test_update_post(self): - """Updates allowed over the last one - - If you /post/add to an existing bookmark, the new data overrides the - old data - - """ - self._get_good_request() - - # now build a new version of the request we can check - session = DBSession() - prms = { - 'url': u'http://google.com', - 'description': u'This is my updated google desc', - 'extended': 'updated extended notes about it in full form', - 'tags': u'python search updated', - 'api_key': u'testapi', - } - - req_params = urllib.urlencode(prms) - self.testapp.get('/delapi/posts/add?' + req_params) - session.flush() - - res = Bmark.query.filter(Bmark.hash_id == GOOGLE_HASH).one() - - ok_('updated' in res.description, - 'Updated description took: ' + res.description) - ok_('updated' in res.extended, - 'Updated extended took: ' + res.extended) - ok_('python' in res.tags, 'Found the python tag in the bmark') - ok_('search' in res.tags, 'Found the search tag in the bmark') - ok_('updated' in res.tags, 'Found the updated tag in the bmark') - - - def test_tag_with_space(self): - """Test that we strip out spaces from tags and don't get empty tags - - """ - self._get_good_request() - - # now build a new version of the request we can check - session = DBSession() - prms = { - 'url': u'http://google.com', - 'description': u'This is my updated google desc', - 'extended': 'updated extended notes about it in full form', - 'tags': u'python search updated ', - 'api_key': u'testapi', - } - - req_params = urllib.urlencode(prms) - self.testapp.get('/delapi/posts/add?' + req_params) - session.flush() - - res = Bmark.query.filter(Bmark.hash_id == GOOGLE_HASH).one() - - ok_(len(res.tags) == 3, - 'Should only have 3 tags: ' + str([str(t) for t in res.tags])) - - for tag in res.tags: - ok_(tag[0] != " ", "Tag should not start with a space") - ok_(tag[-1] != " ", "Tag should not end with a space") - - - def test_tag_completion(self): - """Make sure we can get good completion suggestions""" - # add the default bookmark which tags tags of python and search - self._get_good_request() - - # now try to get completion suggestions - resp = self.testapp.get('/delapi/tags/complete?tag=py') - - eq_(resp.status, "200 OK", "Status of a completion request should be 200") - ok_('python' in resp.body, - "The tag python should be in the response body: " + resp.body) - - # now try to get completion suggestions - resp = self.testapp.get('/delapi/tags/complete?tag=test') - - eq_(resp.status, "200 OK", "Status of a completion request should be 200") - ok_('python' not in resp.body, - "The tag python should not be in the response body: " + resp.body) - -class DelImportTest(unittest.TestCase): - """Test that we can successfully import data from delicious""" - - def setUp(self): - from pyramid.paster import get_app - from bookie.tests import BOOKIE_TEST_INI - app = get_app(BOOKIE_TEST_INI, 'main') - from webtest import TestApp - self.testapp = TestApp(app) - testing.setUp() - - def tearDown(self): - """We need to empty the bmarks table on each run""" - testing.tearDown() - - session = DBSession() - Bmark.query.delete() - Tag.query.delete() - session.execute(bmarks_tags.delete()) - session.flush() - transaction.commit() - - def test_import(self): - """Grab our test data file, import it, and check it out""" - # need to start work on adding this, but passing for build now - assert True - - -class GBookmarkImportTest(unittest.TestCase): - """Test that we can successfully import data from delicious""" - - def setUp(self): - from pyramid.paster import get_app - from bookie.tests import BOOKIE_TEST_INI - app = get_app(BOOKIE_TEST_INI, 'main') - from webtest import TestApp - self.testapp = TestApp(app) - testing.setUp() - - def tearDown(self): - """We need to empty the bmarks table on each run""" - testing.tearDown() - - session = DBSession() - Bmark.query.delete() - Tag.query.delete() - session.execute(bmarks_tags.delete()) - session.flush() - transaction.commit() - - def test_import(self): - """Grab our test data file, import it, and check it out""" - # need to start work on adding this, but passing for build now - assert True +# """Test that we're meeting delicious API specifications""" +# from datetime import datetime, timedelta +# import transaction +# import unittest +# import urllib +# from nose.tools import ok_, eq_ +# from pyramid import testing +# +# from bookie.models import DBSession +# from bookie.models import Bmark, NoResultFound +# from bookie.models import Hashed +# from bookie.models import Tag, bmarks_tags +# from bookie.models import SqliteBmarkFT +# +# from bookie.tests import BOOKIE_TEST_INI +# +# GOOGLE_HASH = 'aa2239c17609b2' +# +# +# class DelPostTest(unittest.TestCase): +# """Test post related calls""" +# +# def setUp(self): +# from pyramid.paster import get_app +# app = get_app(BOOKIE_TEST_INI, 'main') +# from webtest import TestApp +# self.testapp = TestApp(app) +# testing.setUp() +# +# def tearDown(self): +# """We need to empty the bmarks table on each run""" +# testing.tearDown() +# +# if BOOKIE_TEST_INI == 'test.ini': +# SqliteBmarkFT.query.delete() +# Bmark.query.delete() +# Tag.query.delete() +# Hashed.query.delete() +# +# DBSession.execute(bmarks_tags.delete()) +# DBSession.flush() +# transaction.commit() +# +# def _get_good_request(self, content=False): +# """Return the basics for a good add bookmark request""" +# session = DBSession() +# prms = { +# 'url': u'http://google.com', +# 'description': u'This is my google desc', +# 'extended': u'And some extended notes about it in full form', +# 'tags': u'python search', +# 'api_key': u'testapi', +# } +# +# # if we want to test the readable fulltext side we want to make sure we +# # pass content into the new bookmark +# if content: +# prms['content'] = "

There's some content in here dude

" +# +# req_params = urllib.urlencode(prms) +# res = self.testapp.get('/delapi/posts/add?' + req_params) +# session.flush() +# transaction.commit() +# return res +# +# def test_post_add_fail(self): +# """Basic add of a new post failing +# +# Failed response: +# +# +# Not supporting optional params right now +# replace, shared +# +# """ +# failed = '' +# prms = { +# 'url': '', +# 'description': '', +# 'extended': '', +# 'tags': '', +# 'api_key': u'testapi', +# } +# +# req_params = urllib.urlencode(prms) +# +# res = self.testapp.get('/delapi/posts/add?' + req_params) +# eq_(res.status, "200 OK", msg='Post Add status is 200, ' + res.status) +# eq_(res.body, failed, msg="Request should return failed msg: " + res.body) +# +# def test_post_add_success(self): +# """Basic add of a new post working +# +# Success response: +# +# +# Not supporting optional params right now +# replace, shared +# +# """ +# success = '' +# res = self._get_good_request() +# +# eq_(res.status, "200 OK", msg='Post Add status is 200, ' + res.status) +# eq_(res.body, success, msg="Request should return done msg") +# +# def test_new_bmark(self): +# # go save the thing +# self._get_good_request() +# +# try: +# res = Bmark.query.filter(Bmark.hash_id == GOOGLE_HASH).one() +# ok_(res, 'We found a result in the db for this bookmark') +# ok_('extended' in res.extended, +# 'Extended value was set to bookmark') +# +# # make sure our hash was stored correctly +# ok_(res.hashed.url == 'http://google.com', +# "The hashed object got the url") +# +# ok_(res.hashed.clicks == 0, "No clicks on the url yet") +# +# if res: +# return True +# else: +# assert False, "Could not find our bookmark we saved" +# except NoResultFound: +# assert False, "No result found for the bookmark" +# +# def test_post_add_with_dt(self): +# """Make sure if we provide a date it works +# +# Success response: +# +# +# Not supporting optional params right now +# replace, shared +# +# """ +# +# success = '' +# session = DBSession() +# +# # pick a date that is tomorrow +# today = datetime.now() +# yesterday = today - timedelta(days=1) +# dt = yesterday.strftime("%Y-%m-%dT%H:%M:%SZ") +# prms = { +# 'url': u'http://google.com', +# 'description': u'This is my google desc', +# 'extended': u'And some extended notes about it in full form', +# 'tags': u'python search', +# 'dt': dt, +# 'api_key': u'testapi', +# } +# +# req_params = urllib.urlencode(prms) +# res = self.testapp.get('/delapi/posts/add?' + req_params) +# session.flush() +# +# eq_(res.status, "200 OK", msg='Post Add status is 200, ' + res.status) +# eq_(res.body, success, msg="Request should return done msg") +# +# # now pull up the bmark and check the date is yesterday +# res = Bmark.query.filter(Bmark.hash_id == GOOGLE_HASH).one() +# eq_(res.stored.strftime('%Y-%m-%d'), yesterday.strftime('%Y-%m-%d'), +# "The stored date {0} is the same as the requested {1}".format( +# res.stored, +# yesterday)) +# +# def test_new_bmark_tags(self): +# """Manually check db for new bmark tags set""" +# self._get_good_request() +# +# res = Bmark.query.filter(Bmark.hash_id == GOOGLE_HASH).one() +# +# ok_('python' in res.tags, 'Found the python tag in the bmark') +# ok_('search' in res.tags, 'Found the search tag in the bmark') +# +# def test_skip_dupe_tags(self): +# """Make sure we don't end up with duplicate tags in the system""" +# self._get_good_request() +# self._get_good_request() +# +# all_tags = Tag.query.all() +# +# ok_(len(all_tags) == 2, 'We only have two tags in the system') +# +# def test_datestimes_set(self): +# """Test that we get the new datetime fields as we work""" +# +# # we've got some issue with mysql truncating the timestamp to not +# # include seconds, so we allow for a one minute leeway in the +# # timestamp. Enough to know it's set and close enough for government +# # use +# now = datetime.now() - timedelta(minutes=1) +# self._get_good_request() +# res = Bmark.query.filter(Bmark.hash_id == GOOGLE_HASH).one() +# +# ok_(res.stored >= now, +# "Stored time is now or close to now {0}--{1}".format(res.stored, now)) +# +# res.hash_id = u"Somethingnew.com" +# DBSession.flush() +# +# # now hopefully have an updated value +# ok_(res.updated >= now, +# "Stored time, after update, is now or close to now {0}--{1}".format(res.updated, now)) +# +# def test_remove_bmark(self): +# """Remove a bmark from the system +# +# We want to make sure we store content in here to make sure all the +# delete cascades are operating properly +# +# """ +# res1 = self._get_good_request(content=True) +# ok_('done' in res1.body, res1.body) +# +# # now send in the delete squad +# prms = { +# 'url': u'http://google.com', +# 'api_key': u'testapi', +# } +# +# req_params = urllib.urlencode(prms) +# +# res = self.testapp.get('/delapi/posts/delete?' + req_params) +# eq_(res.status, "200 OK", 'Post Delete status is 200, ' + res.status) +# ok_('done' in res.body, "Request should return done msg: " + res.body) +# +# # now make sure our hashed object is gone as well. +# res = Hashed.query.get(GOOGLE_HASH) +# ok_(not res, "We didn't get our hash object") +# +# def test_get_post_byurl(self): +# """Verify we can fetch a post back via a url +# +# While this is delicious api compat, we're going to default to json +# response I think +# +# We'll add xml support to the output later on +# +# """ +# self._get_good_request() +# prms = { +# 'url': u'http://google.com', +# } +# +# req_params = urllib.urlencode(prms) +# +# res = self.testapp.get('/delapi/posts/get?' + req_params) +# +# ok_('href' in res.body, "we have an href link in there") +# ok_('python' in res.body, "we have the python tag") +# ok_('search' in res.body, "we have the search tag") +# +# def test_update_post(self): +# """Updates allowed over the last one +# +# If you /post/add to an existing bookmark, the new data overrides the +# old data +# +# """ +# self._get_good_request() +# +# # now build a new version of the request we can check +# session = DBSession() +# prms = { +# 'url': u'http://google.com', +# 'description': u'This is my updated google desc', +# 'extended': 'updated extended notes about it in full form', +# 'tags': u'python search updated', +# 'api_key': u'testapi', +# } +# +# req_params = urllib.urlencode(prms) +# self.testapp.get('/delapi/posts/add?' + req_params) +# session.flush() +# +# res = Bmark.query.filter(Bmark.hash_id == GOOGLE_HASH).one() +# +# ok_('updated' in res.description, +# 'Updated description took: ' + res.description) +# ok_('updated' in res.extended, +# 'Updated extended took: ' + res.extended) +# ok_('python' in res.tags, 'Found the python tag in the bmark') +# ok_('search' in res.tags, 'Found the search tag in the bmark') +# ok_('updated' in res.tags, 'Found the updated tag in the bmark') +# +# +# def test_tag_with_space(self): +# """Test that we strip out spaces from tags and don't get empty tags +# +# """ +# self._get_good_request() +# +# # now build a new version of the request we can check +# session = DBSession() +# prms = { +# 'url': u'http://google.com', +# 'description': u'This is my updated google desc', +# 'extended': 'updated extended notes about it in full form', +# 'tags': u'python search updated ', +# 'api_key': u'testapi', +# } +# +# req_params = urllib.urlencode(prms) +# self.testapp.get('/delapi/posts/add?' + req_params) +# session.flush() +# +# res = Bmark.query.filter(Bmark.hash_id == GOOGLE_HASH).one() +# +# ok_(len(res.tags) == 3, +# 'Should only have 3 tags: ' + str([str(t) for t in res.tags])) +# +# for tag in res.tags: +# ok_(tag[0] != " ", "Tag should not start with a space") +# ok_(tag[-1] != " ", "Tag should not end with a space") +# +# +# def test_tag_completion(self): +# """Make sure we can get good completion suggestions""" +# # add the default bookmark which tags tags of python and search +# self._get_good_request() +# +# # now try to get completion suggestions +# resp = self.testapp.get('/delapi/tags/complete?tag=py') +# +# eq_(resp.status, "200 OK", "Status of a completion request should be 200") +# ok_('python' in resp.body, +# "The tag python should be in the response body: " + resp.body) +# +# # now try to get completion suggestions +# resp = self.testapp.get('/delapi/tags/complete?tag=test') +# +# eq_(resp.status, "200 OK", "Status of a completion request should be 200") +# ok_('python' not in resp.body, +# "The tag python should not be in the response body: " + resp.body) +# +# class DelImportTest(unittest.TestCase): +# """Test that we can successfully import data from delicious""" +# +# def setUp(self): +# from pyramid.paster import get_app +# from bookie.tests import BOOKIE_TEST_INI +# app = get_app(BOOKIE_TEST_INI, 'main') +# from webtest import TestApp +# self.testapp = TestApp(app) +# testing.setUp() +# +# def tearDown(self): +# """We need to empty the bmarks table on each run""" +# testing.tearDown() +# +# session = DBSession() +# Bmark.query.delete() +# Tag.query.delete() +# session.execute(bmarks_tags.delete()) +# session.flush() +# transaction.commit() +# +# def test_import(self): +# """Grab our test data file, import it, and check it out""" +# # need to start work on adding this, but passing for build now +# assert True +# +# +# class GBookmarkImportTest(unittest.TestCase): +# """Test that we can successfully import data from delicious""" +# +# def setUp(self): +# from pyramid.paster import get_app +# from bookie.tests import BOOKIE_TEST_INI +# app = get_app(BOOKIE_TEST_INI, 'main') +# from webtest import TestApp +# self.testapp = TestApp(app) +# testing.setUp() +# +# def tearDown(self): +# """We need to empty the bmarks table on each run""" +# testing.tearDown() +# +# session = DBSession() +# Bmark.query.delete() +# Tag.query.delete() +# session.execute(bmarks_tags.delete()) +# session.flush() +# transaction.commit() +# +# def test_import(self): +# """Grab our test data file, import it, and check it out""" +# # need to start work on adding this, but passing for build now +# assert True diff --git a/bookie/tests/test_utils/test_export.py b/bookie/tests/test_utils/test_export.py index e978c634..51b60715 100644 --- a/bookie/tests/test_utils/test_export.py +++ b/bookie/tests/test_utils/test_export.py @@ -14,7 +14,7 @@ LOG = logging.getLogger(__name__) - +API_KEY = None class TestExport(unittest.TestCase): """Test the web export""" @@ -27,11 +27,12 @@ def _get_good_request(self): 'description': u'This is my google desc', 'extended': u'And some extended notes about it in full form', 'tags': u'python search', - 'api_key': u'testapi', + 'api_key': API_KEY, } req_params = urllib.urlencode(prms) - res = self.testapp.get('/delapi/posts/add?' + req_params) + res = self.testapp.get('/admin/api/v1/bmarks/add?', + params=req_params,) session.flush() transaction.commit() return res @@ -43,6 +44,9 @@ def setUp(self): from webtest import TestApp self.testapp = TestApp(app) testing.setUp() + global API_KEY + res = DBSession.execute("SELECT api_key FROM users WHERE username = 'admin'").fetchone() + API_KEY = res['api_key'] def tearDown(self): """We need to empty the bmarks table on each run""" @@ -58,8 +62,8 @@ def test_export(self): """Test that we can upload/import our test file""" self._get_good_request() - res = self.testapp.post('/export', - status=200) + res = self.testapp.post('/admin/export', + status=200) ok_("google.com" in res.body, msg='Google is in the exported body: ' + res.body) diff --git a/bookie/tests/test_utils/test_fulltext.py b/bookie/tests/test_utils/test_fulltext.py index 304ac1be..4c5d2c66 100644 --- a/bookie/tests/test_utils/test_fulltext.py +++ b/bookie/tests/test_utils/test_fulltext.py @@ -14,6 +14,9 @@ from bookie.models.fulltext import get_fulltext_handler from bookie.models.fulltext import SqliteFulltext +API_KEY = None + + class TestFulltext(TestCase): """Test that our fulltext classes function""" @@ -25,6 +28,10 @@ def setUp(self): from webtest import TestApp self.testapp = TestApp(app) testing.setUp() + global API_KEY + if API_KEY is None: + res = DBSession.execute("SELECT api_key FROM users WHERE username = 'admin'").fetchone() + API_KEY = res['api_key'] def tearDown(self): """Tear down each test""" @@ -45,14 +52,15 @@ def _get_good_request(self, new_tags=None): 'description': u'This is my google desc', 'extended': u'And some extended notes about it in full form', 'tags': u'python search', - 'api_key': u'testapi', + 'api_key': API_KEY, } if new_tags: prms['tags'] = new_tags req_params = urllib.urlencode(prms) - res = self.testapp.get('/delapi/posts/add?' + req_params) + res = self.testapp.get('/admin/api/v1/bmarks/add', + params=req_params) session.flush() transaction.commit() return res @@ -71,7 +79,7 @@ def test_sqlite_save(self): # first let's add a bookmark we can search on res = self._get_good_request() - search_res = self.testapp.get('/results?search=google') + search_res = self.testapp.get('/admin/results?search=google') ok_(search_res.status == '200 OK', "Status is 200: " + search_res.status) @@ -79,7 +87,7 @@ def test_sqlite_save(self): ok_('my google desc' in search_res.body, "We should find our description on the page: " + search_res.body) - search_res = self.testapp.get('/results?search=python') + search_res = self.testapp.get('/admin/results?search=python') ok_(search_res.status == '200 OK', "Status is 200: " + search_res.status) @@ -87,7 +95,7 @@ def test_sqlite_save(self): ok_('my google desc' in search_res.body, "Tag search should find our description on the page: " + search_res.body) - search_res = self.testapp.get('/results?search=extended%20notes') + search_res = self.testapp.get('/admin/results?search=extended%20notes') ok_(search_res.status == '200 OK', "Status is 200: " + search_res.status) @@ -107,7 +115,7 @@ def test_sqlite_update(self): # now we need to do another request with updated tag string self._get_good_request(new_tags="google books icons") - search_res = self.testapp.get('/results?search=icon') + search_res = self.testapp.get('/admin/results?search=icon') ok_(search_res.status == '200 OK', "Status is 200: " + search_res.status) @@ -119,7 +127,7 @@ def test_restlike_search(self): # first let's add a bookmark we can search on self._get_good_request() - search_res = self.testapp.get('/results/google') + search_res = self.testapp.get('/admin/results/google') ok_(search_res.status == '200 OK', "Status is 200: " + search_res.status) @@ -127,7 +135,7 @@ def test_restlike_search(self): ok_('my google desc' in search_res.body, "We should find our description on the page: " + search_res.body) - search_res = self.testapp.get('/results/python') + search_res = self.testapp.get('/admin/results/python') ok_(search_res.status == '200 OK', "Status is 200: " + search_res.status) @@ -135,7 +143,7 @@ def test_restlike_search(self): ok_('my google desc' in search_res.body, "Tag search should find our description on the page: " + search_res.body) - search_res = self.testapp.get('/results/extended/notes') + search_res = self.testapp.get('/admin/results/extended/notes') ok_(search_res.status == '200 OK', "Status is 200: " + search_res.status) @@ -149,7 +157,7 @@ def test_ajax_search(self): self._get_good_request() search_res = self.testapp.get( - '/results/google', + '/admin/results/google', headers = { 'X-Requested-With': 'XMLHttpRequest', 'Accept': 'application/json' @@ -171,4 +179,3 @@ def test_ajax_search(self): ok_('message' in search_res.body, "We should see a message bit in the json: " + search_res.body) - diff --git a/bookie/tests/test_utils/test_imports.py b/bookie/tests/test_utils/test_imports.py index e59f3825..4a03d52f 100644 --- a/bookie/tests/test_utils/test_imports.py +++ b/bookie/tests/test_utils/test_imports.py @@ -21,6 +21,8 @@ LOG = logging.getLogger(__name__) +API_KEY = None + def _delicious_data_test(): """Test that we find the correct set of declicious data after import""" @@ -93,7 +95,7 @@ def test_factory_gives_delicious(self): del_file = os.path.join(loc, 'delicious.html') with open(del_file) as del_io: - imp = Importer(del_io) + imp = Importer(del_io, username="admin") ok_(isinstance(imp, DelImporter), "Instance should be a delimporter instance") @@ -104,7 +106,7 @@ def test_factory_gives_google(self): google_file = os.path.join(loc, 'googlebookmarks.html') with open(google_file) as google_io: - imp = Importer(google_io) + imp = Importer(google_io, username="admin") ok_(isinstance(imp, GBookmarkImporter), "Instance should be a GBookmarkImporter instance") @@ -156,7 +158,7 @@ def test_is_not_delicious_file(self): def test_import_process(self): """Verify importer inserts the correct records""" good_file = self._get_del_file() - imp = Importer(good_file) + imp = Importer(good_file, username="admin") imp.process() # now let's do some db sanity checks @@ -199,7 +201,7 @@ def test_is_not_google_file(self): def test_import_process(self): """Verify importer inserts the correct google bookmarks""" good_file = self._get_google_file() - imp = Importer(good_file) + imp = Importer(good_file, username="admin") imp.process() # now let's do some db sanity checks @@ -229,15 +231,26 @@ def tearDown(self): def test_delicious_import(self): """Test that we can upload/import our test file""" + # first let's login to the site so we can get in + self.testapp.post('/login', + params={ + "login": "admin", + "password": "admin", + "form.submitted": "Log In", + }, + status=302) + session = DBSession() loc = os.path.dirname(__file__) del_file = open(os.path.join(loc, 'delicious.html')) + res = DBSession.execute("SELECT api_key FROM users WHERE username = 'admin'").fetchone() + API_KEY = res['api_key'] post = { - 'api_key': 'testapi', + 'api_key': API_KEY, } - res = self.testapp.post('/import', + res = self.testapp.post('/admin/import', params=post, upload_files=[('import_file', 'delicious.html', @@ -254,15 +267,26 @@ def test_delicious_import(self): def test_google_import(self): """Test that we can upload our google file""" + self.testapp.post('/login', + params={ + "login": "admin", + "password": "admin", + "form.submitted": "Log In", + }, + status=302) + session = DBSession() loc = os.path.dirname(__file__) del_file = open(os.path.join(loc, 'googlebookmarks.html')) + res = DBSession.execute("SELECT api_key FROM users WHERE username = 'admin'").fetchone() + API_KEY = res['api_key'] post = { - 'api_key': 'testapi', + 'api_key': API_KEY, } - res = self.testapp.post('/import', + + res = self.testapp.post('/admin/import', params=post, upload_files=[('import_file', 'googlebookmarks.html', diff --git a/bookie/tests/test_utils/test_readable.py b/bookie/tests/test_utils/test_readable.py index f423dc9a..e43c6424 100644 --- a/bookie/tests/test_utils/test_readable.py +++ b/bookie/tests/test_utils/test_readable.py @@ -21,7 +21,7 @@ LOG = logging.getLogger(__file__) - +API_KEY = None class TestReadable(TestCase): """Test that our fulltext classes function""" @@ -104,6 +104,10 @@ def setUp(self): from webtest import TestApp self.testapp = TestApp(app) testing.setUp() + global API_KEY + if API_KEY is None: + res = DBSession.execute("SELECT api_key FROM users WHERE username = 'admin'").fetchone() + API_KEY = res['api_key'] def tearDown(self): """Tear down each test""" @@ -124,12 +128,13 @@ def _get_good_request(self): 'description': u'This is my google desc', 'extended': u'And some extended notes about it in full form', 'tags': u'python search', - 'api_key': u'testapi', + 'api_key': API_KEY, 'content': 'bmark content is the best kind of content man', } req_params = urllib.urlencode(prms) - res = self.testapp.get('/delapi/posts/add?' + req_params) + res = self.testapp.get('/admin/api/v1/bmarks/add', + params=req_params) session.flush() transaction.commit() return res @@ -148,7 +153,7 @@ def test_sqlite_save(self): # first let's add a bookmark we can search on self._get_good_request() - search_res = self.testapp.get('/results?search=bmark&content=1') + search_res = self.testapp.get('/admin/results?search=bmark&content=1') ok_(search_res.status == '200 OK', "Status is 200: " + search_res.status) @@ -161,7 +166,7 @@ def test_restlike_search(self): # first let's add a bookmark we can search on self._get_good_request() - search_res = self.testapp.get('/results/bmark?content=1') + search_res = self.testapp.get('/admin/results/bmark?content=1') ok_(search_res.status == '200 OK', "Status is 200: " + search_res.status) diff --git a/bookie/tests/test_webviews/__init__.py b/bookie/tests/test_webviews/__init__.py index a67b80b5..243ff017 100644 --- a/bookie/tests/test_webviews/__init__.py +++ b/bookie/tests/test_webviews/__init__.py @@ -25,6 +25,7 @@ def _add_bmark(self): log = logging.getLogger(__name__) log.error('called to add bmark') bmark_us = Bmark('http://bmark.us', + username="admin", desc="Bookie Website", ext= "Bookie Documentation Home", tags = "bookmarks") @@ -56,11 +57,8 @@ def tearDown(self): def test_bookmark_recent(self): """Verify we can call the /recent url """ self._add_bmark() - body_str = "Recent Bookmarks" - delete_str = "/bmark/confirm/delete" - access.edit_enabled = Mock(return_value=True) res = self.testapp.get('/recent') eq_(res.status, "200 OK", @@ -68,11 +66,6 @@ def test_bookmark_recent(self): ok_(body_str in res.body, msg="Request should contain body_str: " + res.body) - # there should be a delete link for the default bookie bookmark in the - # body as well - ok_(delete_str in res.body, - msg="The delete link should be visible in the body:" + res.body) - def test_recent_page(self): """We should be able to page through the list""" body_str = "Prev" @@ -83,26 +76,13 @@ def test_recent_page(self): ok_(body_str in res.body, msg="Page 1 should contain body_str: " + res.body) - def test_allow_edit_requests(self): - """Verify that if allow_edit is false we don't get edit/delete links""" - self._add_bmark() - delete_str = "/bmark/confirm/delete" - - access.edit_enabled = Mock(return_value=False) - - res = self.testapp.get('/recent') - - # the delete link should not render if allow_edits is false - ok_(delete_str not in res.body, - msg="The delete link should NOT be visible:" + res.body) - - def test_delete_auth_failed(self): + def test_import_auth_failed(self): """Veryify that without the right API key we get forbidden""" post = { 'api_key': 'wrong_key' } - res = self.testapp.post('/import', params=post, status=403) + res = self.testapp.post('/admin/import', params=post, status=403) eq_(res.status, "403 Forbidden", msg='Import status is 403, ' + res.status) @@ -112,9 +92,6 @@ def test_bookmark_tag(self): self._add_bmark() body_str = "Bookmarks: bookmarks" - delete_str = "/bmark/confirm/delete" - - access.edit_enabled = Mock(return_value=True) res = self.testapp.get('/tags/bookmarks') eq_(res.status, "200 OK", @@ -122,20 +99,12 @@ def test_bookmark_tag(self): ok_(body_str in res.body, msg="Request should contain body_str: " + res.body) - # there should be a delete link for the default bookie bookmark in the - # body as well - ok_(delete_str in res.body, - msg="Tag view delete link should be visible in the body:" + res.body) - def test_bookmark_tag_no_edits(self): """Verify the tags view""" self._add_bmark() - access.edit_enabled = Mock(return_value=False) - delete_str = "/bmark/confirm/delete" res = self.testapp.get('/tags/bookmarks') - # The delete link should not render if allow_edits is false ok_(delete_str not in res.body, msg="Tag view delete link should NOT be visible:" + res.body) diff --git a/bookie/views/__init__.py b/bookie/views/__init__.py index 1ecfb18c..02f1d134 100644 --- a/bookie/views/__init__.py +++ b/bookie/views/__init__.py @@ -1,9 +1,25 @@ """Basic views with no home""" from pyramid.httpexceptions import HTTPFound +from pyramid.httpexceptions import HTTPNotFound from pyramid.view import view_config +from bookie.models.auth import UserMgr @view_config(route_name="home") +@view_config(route_name="user_home") def home(request): """Inital / view for now until we find a better one""" - return HTTPFound(location=request.route_url("bmark_recent")) + rdict = request.matchdict + username = rdict.get('username', None) + + if not username: + return HTTPFound(location=request.route_url("bmark_recent")) + else: + # we need to see if we have a user by this name + user = UserMgr.get(username=username) + + if not user: + return HTTPNotFound() + else: + return HTTPFound(location=request.route_url("user_bmark_recent", + username=username)) diff --git a/bookie/views/api.py b/bookie/views/api.py index cf98ac72..08cb2620 100644 --- a/bookie/views/api.py +++ b/bookie/views/api.py @@ -2,27 +2,32 @@ import logging from datetime import datetime +from pyramid.settings import asbool from pyramid.view import view_config from StringIO import StringIO -from bookie.lib.access import Authorize +from bookie.lib.access import ApiAuthorize from bookie.lib.readable import ReadContent from bookie.lib.tagcommands import Commander from bookie.models import Bmark from bookie.models import BmarkMgr from bookie.models import DBSession +from bookie.models import Hashed from bookie.models import NoResultFound from bookie.models import Readable from bookie.models import TagMgr +from bookie.models.auth import UserMgr from bookie.models.fulltext import get_fulltext_handler LOG = logging.getLogger(__name__) RESULTS_MAX = 10 +HARD_MAX = 100 @view_config(route_name="api_bmark_recent", renderer="morjson") +@view_config(route_name="user_api_bmark_recent", renderer="morjson") def bmark_recent(request): """Get a list of the bmarks for the api call""" rdict = request.matchdict @@ -31,6 +36,12 @@ def bmark_recent(request): # check if we have a page count submitted page = int(params.get('page', '0')) count = int(params.get('count', RESULTS_MAX)) + username = rdict.get('username', None) + + # thou shalt not have more then the HARD MAX + # @todo move this to the .ini as a setting + if count > HARD_MAX: + count = HARD_MAX # do we have any tags to filter upon tags = rdict.get('tags', None) @@ -47,7 +58,8 @@ def bmark_recent(request): order_by=Bmark.stored.desc(), tags=tags, page=page, - with_tags=True) + with_tags=True, + username=username) result_set = [] @@ -73,6 +85,7 @@ def bmark_recent(request): @view_config(route_name="api_bmark_popular", renderer="morjson") +@view_config(route_name="user_api_bmark_popular", renderer="morjson") def bmark_popular(request): """Get a list of the bmarks for the api call""" rdict = request.matchdict @@ -81,6 +94,12 @@ def bmark_popular(request): # check if we have a page count submitted page = int(params.get('page', '0')) count = int(params.get('count', RESULTS_MAX)) + username = rdict.get('username', None) + + # thou shalt not have more then the HARD MAX + # @todo move this to the .ini as a setting + if count > HARD_MAX: + count = HARD_MAX # do we have any tags to filter upon tags = rdict.get('tags', None) @@ -96,7 +115,8 @@ def bmark_popular(request): popular_list = BmarkMgr.find(limit=count, order_by=Bmark.clicks.desc(), tags=tags, - page=page) + page=page, + username=username) result_set = [] for res in popular_list: @@ -120,28 +140,37 @@ def bmark_popular(request): return ret -@view_config(route_name="api_bmark_sync", renderer="morjson") +@view_config(route_name="user_api_bmark_sync", renderer="morjson") def bmark_sync(request): """Return a list of the bookmarks we know of in the system For right now, send down a list of hash_ids """ + rdict = request.matchdict + params = request.params - hash_list = BmarkMgr.hash_list() + username = rdict.get('username', None) + user = UserMgr.get(username=username) - ret = { - 'success': True, - 'message': "", - 'payload': { - 'hash_list': [hash[0] for hash in hash_list] + with ApiAuthorize(user.api_key, + params.get('api_key', None)): + + hash_list = BmarkMgr.hash_list(username=username) + + ret = { + 'success': True, + 'message': "", + 'payload': { + 'hash_list': [hash[0] for hash in hash_list] + } } - } - return ret + return ret @view_config(route_name="api_bmark_hash", renderer="morjson") +@view_config(route_name="user_api_bmark_hash", renderer="morjson") def bmark_get(request): """Return a bookmark requested via hash_id @@ -152,6 +181,7 @@ def bmark_get(request): rdict = request.matchdict hash_id = rdict.get('hash_id', None) + username = rdict.get('username', None) if not hash_id: return { @@ -160,7 +190,9 @@ def bmark_get(request): 'payload': {} } - bookmark = BmarkMgr.get_by_hash(hash_id) + bookmark = BmarkMgr.get_by_hash(hash_id, + username=username) + if not bookmark: # then not found ret = { @@ -187,18 +219,23 @@ def bmark_get(request): return ret -@view_config(route_name="api_bmark_add", renderer="morjson") +@view_config(route_name="user_api_bmark_add", renderer="morjson") def bmark_add(request): """Add a new bookmark to the system""" params = request.params + rdict = request.matchdict + + username = rdict.get("username", None) + user = UserMgr.get(username=username) - with Authorize(request.registry.settings.get('api_key', ''), - params.get('api_key', None)): + with ApiAuthorize(user.api_key, + params.get('api_key', None)): if 'url' in params and params['url']: # check if we already have this try: - mark = BmarkMgr.get_by_url(params['url']) + mark = BmarkMgr.get_by_url(params['url'], + username=username) mark.description = params.get('description', mark.description) mark.extended = params.get('extended', mark.extended) @@ -243,7 +280,10 @@ def bmark_add(request): False) fulltext = get_fulltext_handler(conn_str) + LOG.debug('Username') + LOG.debug(username) mark = BmarkMgr.store(params['url'], + username, params.get('description', ''), params.get('extended', ''), params.get('tags', ''), @@ -255,8 +295,6 @@ def bmark_add(request): commander = Commander(mark) mark = commander.process() - - # if we have content, stick it on the object here if 'content' in request.params: content = StringIO(request.params['content']) @@ -289,16 +327,22 @@ def bmark_add(request): } -@view_config(route_name="api_bmark_remove", renderer="morjson") +@view_config(route_name="user_api_bmark_remove", renderer="morjson") def bmark_remove(request): """Remove this bookmark from the system""" params = request.params + rdict = request.matchdict + + username = rdict.get("username", None) + user = UserMgr.get(username=username) + + with ApiAuthorize(user.api_key, + params.get('api_key', None)): - with Authorize(request.registry.settings.get('api_key', ''), - params.get('api_key', None)): if 'url' in params and params['url']: try: - bmark = BmarkMgr.get_by_url(params['url']) + bmark = BmarkMgr.get_by_url(params['url'], + username=username) session = DBSession() session.delete(bmark) @@ -320,6 +364,7 @@ def bmark_remove(request): @view_config(route_name="api_tag_complete", renderer="morjson") +@view_config(route_name="user_api_tag_complete", renderer="morjson") def tag_complete(request): """Complete a tag based on the given text @@ -350,3 +395,94 @@ def tag_complete(request): } return ret + + +@view_config(route_name="api_bmark_get_readable", renderer="morjson") +def to_readable(request): + """Get a list of urls, hash_ids we need to readable parse""" + url_list = Hashed.query.outerjoin(Readable).\ + filter(Readable.imported == None).all() + + ret = { + 'success': True, + 'message': "", + 'payload': { + 'urls': [dict(h) for h in url_list] + } + } + + return ret + +@view_config(route_name="api_bmark_readable", renderer="morjson") +def readable(request): + """Take the html given and parse the content in there for readable + + :@param hash_id: POST the hash_id of the bookmark we're readable'ing + :@param content: POST the html of the page in question + + """ + params = request.POST + success = params.get('success', None) + + if success is None: + ret = { + 'success': False, + 'message': "Please submit success data", + 'payload': {} + } + + hashed = Hashed.query.get(params.get('hash_id', None)) + + if hashed: + success = asbool(success) + LOG.debug(success) + if success: + # if we have content, stick it on the object here + if 'content' in params: + content = StringIO(params['content']) + content.seek(0) + parsed = ReadContent.parse(content, content_type="text/html") + + hashed.readable = Readable() + hashed.readable.content = parsed.content + hashed.readable.content_type = parsed.content_type + hashed.readable.status_code = 200 + hashed.readable.status_message = "API Parsed" + + ret = { + 'success': True, + 'message': "Parsed url: " + hashed.url, + 'payload': {} + } + else: + ret = { + 'success': False, + 'message': "Missing content for hash id", + 'payload': { + 'hash_id': params.get('hash_id') + } + } + + else: + # success was false for some reason + # could be an image, 404, error, bad domain... + # need info for content_type, status_code, status_message + hashed.readable = Readable() + hashed.readable.content_type = params.get('content_type', "Unknown") + hashed.readable.status_code = params.get('status_code', 999) + hashed.readable.status_message = params.get('status_message', "Missing message") + + ret = { + 'success': True, + 'message': "Stored unsuccessful content fetching result", + 'payload': dict(params) + } + + else: + ret = { + 'success': False, + 'message': "Missing hash_id to parse", + 'payload': {} + } + + return ret diff --git a/bookie/views/auth.py b/bookie/views/auth.py new file mode 100644 index 00000000..69606b34 --- /dev/null +++ b/bookie/views/auth.py @@ -0,0 +1,88 @@ +import logging + +from pyramid.httpexceptions import HTTPFound +from pyramid.renderers import render_to_response +from pyramid.security import remember +from pyramid.security import forget +from pyramid.url import route_url +from pyramid.view import view_config + +from bookie.models.auth import UserMgr + + +LOG = logging.getLogger(__name__) + + +@view_config(route_name="login", renderer="/auth/login.mako") +def login(request): + """Login the user to the system + + If not POSTed then show the form + If error, display the form with the error message + If successful, forward the user to their /recent + + Note: the came_from stuff we're not using atm. We'll clean out if we keep + things this way + + """ + login_url = route_url('login', request) + referrer = request.url + if referrer == login_url: + referrer = '/' # never use the login form itself as came_from + + came_from = request.params.get('came_from', referrer) + + message = '' + login = '' + password = '' + + if 'form.submitted' in request.params: + login = request.params['login'] + password = request.params['password'] + + LOG.debug(login) + auth = UserMgr.get(username=login) + LOG.debug(auth) + LOG.debug(UserMgr.get_list()) + + if auth and auth.validate_password(password): + # We use the Primary Key as our identifier once someone has authenticated rather than the + # username. You can change what is returned as the userid by altering what is passed to + # remember. + headers = remember(request, auth.id, max_age='86400') + + # we're always going to return a user to their own /recent after a + # login + return HTTPFound(location=request.route_url('user_bmark_recent', + username=auth.username), + headers=headers) + message = 'Failed login' + + return { + 'message': message, + 'came_from': came_from, + 'login': login, + 'password': password, + } + + +@view_config(route_name="logout", renderer="/auth/login.mako") +def logout(request): + headers = forget(request) + return HTTPFound(location = route_url('home', request), + headers = headers) + + +def forbidden_view(request): + login_url = route_url('login', request) + referrer = request.url + if referrer == login_url: + referrer = '/' # never use the login form itself as came_from + came_from = request.params.get('came_from', referrer) + return render_to_response('/auth/login.mako', dict( + message = '', + url = request.application_url + '/login', + came_from = came_from, + login = '', + password = '', + ), request=request) diff --git a/bookie/views/bmarks.py b/bookie/views/bmarks.py index 81c8040f..76f8526f 100644 --- a/bookie/views/bmarks.py +++ b/bookie/views/bmarks.py @@ -1,6 +1,7 @@ """Controllers related to viewing lists of bookmarks""" import logging +from pyramid import security from pyramid.httpexceptions import HTTPForbidden from pyramid.httpexceptions import HTTPFound from pyramid.httpexceptions import HTTPNotFound @@ -18,11 +19,14 @@ @view_config(route_name="bmark_recent", renderer="/bmark/recent.mako") @view_config(route_name="bmark_recent_tags", renderer="/bmark/recent.mako") +@view_config(route_name="user_bmark_recent", renderer="/bmark/recent.mako") +@view_config(route_name="user_bmark_recent_tags", renderer="/bmark/recent.mako") def recent(request): """Most recent list of bookmarks capped at MAX""" rdict = request.matchdict params = request.params + LOG.debug('in recent!') # check if we have a page count submitted page = int(params.get('page', '0')) @@ -32,6 +36,13 @@ def recent(request): if isinstance(tags, str): tags = [tags] + # check for auth related stuff + # are we looking for a specific user + if 'username' in rdict: + username = rdict.get('username') + else: + username = None + # if we don't have tags, we might have them sent by a non-js browser as a # string in a query string if not tags and 'tag_filter' in params: @@ -40,9 +51,8 @@ def recent(request): recent_list = BmarkMgr.find(limit=RESULTS_MAX, order_by=Bmark.stored.desc(), tags=tags, - page=page) - - + page=page, + username=username) ret = { 'bmarks': recent_list, @@ -50,14 +60,15 @@ def recent(request): 'count': len(recent_list), 'page': page, 'tags': tags, - 'allow_edit': access.edit_enabled(request.registry.settings), } return ret @view_config(route_name="bmark_popular", renderer="/bmark/popular.mako") +@view_config(route_name="user_bmark_popular", renderer="/bmark/popular.mako") @view_config(route_name="bmark_popular_tags", renderer="/bmark/popular.mako") +@view_config(route_name="user_bmark_popular_tags", renderer="/bmark/popular.mako") def popular(request): """Most popular list of bookmarks capped at MAX""" rdict = request.matchdict @@ -70,6 +81,13 @@ def popular(request): if isinstance(tags, str): tags = [tags] + # check for auth related stuff + # are we looking for a specific user + if 'username' in rdict: + username = rdict.get('username') + else: + username = None + # if we don't have tags, we might have them sent by a non-js browser as a # string in a query string if not tags and 'tag_filter' in params: @@ -78,7 +96,8 @@ def popular(request): recent_list = BmarkMgr.find(limit=RESULTS_MAX, order_by=Bmark.clicks.desc(), tags=tags, - page=page) + page=page, + username=username, ) return { 'bmarks': recent_list, @@ -86,7 +105,7 @@ def popular(request): 'count': len(recent_list), 'page': page, 'tags': tags, - 'allow_edit': access.edit_enabled(request.registry.settings), + 'user': request.user, } @@ -95,9 +114,6 @@ def delete(request): """Remove the bookmark in question""" rdict = request.POST - if not access.edit_enabled(request.registry.settings): - raise HTTPForbidden("Auth to edit is not enabled") - # make sure we have an id value bid = int(rdict.get('bid', 0)) diff --git a/bookie/views/delapi.py b/bookie/views/delapi.py index 8eb50d51..a5616d15 100644 --- a/bookie/views/delapi.py +++ b/bookie/views/delapi.py @@ -8,7 +8,7 @@ from pyramid.view import view_config -from bookie.lib.access import Authorize +from bookie.lib.access import ApiAuthorize from bookie.lib.readable import ReadContent from bookie.models import DBSession, NoResultFound from bookie.models import BmarkMgr @@ -31,7 +31,7 @@ def posts_add(request): """ params = request.params - with Authorize(request.registry.settings.get('api_key', ''), + with ApiAuthorize(request.registry.settings.get('api_key', ''), params.get('api_key', None)): request.response_content_type = 'text/xml' @@ -96,7 +96,7 @@ def posts_delete(request): params = request.params request.response_content_type = 'text/xml' - with Authorize(request.registry.settings.get('api_key', ''), + with ApiAuthorize(request.registry.settings.get('api_key', ''), params.get('api_key', None)): if 'url' in params and params['url']: try: diff --git a/bookie/views/mobile.py b/bookie/views/mobile.py index a15bd7f4..39f3ee3d 100644 --- a/bookie/views/mobile.py +++ b/bookie/views/mobile.py @@ -12,12 +12,11 @@ @view_config(route_name="mobile", renderer="/mobile/index.mako") +@view_config(route_name="user_mobile", renderer="/mobile/index.mako") def mobile_index(request): - """Mobile index page""" - # tags_found = TagMgr.find() + """Mobile index page + + The content is loaded via ajax calls so we just return the base html/js - return { - 'test': 'Testing' - # 'tag_list': tags_found, - # 'tag_count': len(tags_found), - } + """ + return {} diff --git a/bookie/views/tags.py b/bookie/views/tags.py index 840e1d7e..baebdbaa 100644 --- a/bookie/views/tags.py +++ b/bookie/views/tags.py @@ -13,9 +13,13 @@ @view_config(route_name="tag_list", renderer="/tag/list.mako") +@view_config(route_name="user_tag_list", renderer="/tag/list.mako") def tag_list(request): """Display a list of your tags""" - tags_found = TagMgr.find() + rdict = request.matchdict + username = rdict.get("username", None) + + tags_found = TagMgr.find(username=username) return { 'tag_list': tags_found, @@ -23,16 +27,17 @@ def tag_list(request): } -@view_config(route_name="tag_bmarks_ajax", renderer="morjson") @view_config(route_name="tag_bmarks", renderer="/tag/bmarks_wrap.mako") +@view_config(route_name="user_tag_bmarks", renderer="/tag/bmarks_wrap.mako") def bmark_list(request): """Display the list of bookmarks for this tag""" - route_name = request.matched_route.name rdict = request.matchdict params = request.params # check if we have a page count submitted tags = rdict.get('tags') + username = rdict.get("username", None) + page = int(params.get('page', 0)) # verify the tag exists before we go on @@ -44,32 +49,13 @@ def bmark_list(request): bmarks = BmarkMgr.find(tags=tags, limit=RESULTS_MAX, - page=page,) + page=page, + username=username) - if 'ajax' in route_name: - html = render('bookie:templates/tag/bmarks.mako', - { - 'tags': tags, - 'bmark_list': bmarks, - 'max_count': RESULTS_MAX, - 'count': len(bmarks), - 'page': page, - 'allow_edit': access.edit_enabled(request.registry.settings), - }, - request=request) - return { - 'success': True, - 'message': "", - 'payload': { - 'html': html, - } - } - - else: - return {'tags': tags, - 'bmark_list': bmarks, - 'max_count': RESULTS_MAX, - 'count': len(bmarks), - 'page': page, - 'allow_edit': access.edit_enabled(request.registry.settings), - } + return { + 'tags': tags, + 'bmark_list': bmarks, + 'max_count': RESULTS_MAX, + 'count': len(bmarks), + 'page': page, + } diff --git a/bookie/views/utils.py b/bookie/views/utils.py index 71c56f1b..4b7d8baa 100644 --- a/bookie/views/utils.py +++ b/bookie/views/utils.py @@ -3,12 +3,11 @@ from pyramid.httpexceptions import HTTPFound from pyramid.httpexceptions import HTTPNotFound -from pyramid.renderers import render from pyramid.settings import asbool from pyramid.view import view_config from bookie.lib.importer import Importer -from bookie.lib.access import Authorize +from bookie.lib.access import ReqAuthorize from bookie.models import Bmark from bookie.models import Hashed from bookie.models.fulltext import get_fulltext_handler @@ -16,25 +15,24 @@ LOG = logging.getLogger(__name__) -@view_config(route_name="import", renderer="/utils/import.mako") +@view_config(route_name="user_import", renderer="/utils/import.mako") def import_bmarks(request): """Allow users to upload a delicious bookmark export""" - data = {} - post = request.POST - LOG.error(request.registry.settings.get('api_key', '')) - LOG.error(post.get('api_key')) - if post: - # we have some posted values - with Authorize(request.registry.settings.get('api_key', ''), - post.get('api_key', None)): - - # if auth fails, it'll raise an HTTPForbidden exception + rdict = request.matchdict + username = rdict.get('username') + + # if auth fails, it'll raise an HTTPForbidden exception + with ReqAuthorize(request): + data = {} + post = request.POST + if post: + # we have some posted values files = post.get('import_file', None) if files is not None: # upload is there for use # process the file using the import script - importer = Importer(files.file) + importer = Importer(files.file, username=username) # we want to store fulltext info so send that along to the # import processor @@ -57,23 +55,26 @@ def import_bmarks(request): data['error'] = None return data - else: - # just display the form - return {} + else: + # just display the form + return {} @view_config(route_name="search", renderer="/utils/search.mako") +@view_config(route_name="user_search", renderer="/utils/search.mako") def search(request): """Display the search form to the user""" - return { - - } + return {} @view_config(route_name="search_results", renderer="/utils/results_wrap.mako") -@view_config(route_name="search_results_ajax", renderer="morjson") +@view_config(route_name="user_search_results", renderer="/utils/results_wrap.mako") @view_config(route_name="api_bmark_search", renderer="morjson") +@view_config(route_name="user_api_bmark_search", renderer="morjson") +@view_config(route_name="search_results_ajax", renderer="morjson") +@view_config(route_name="user_search_results_ajax", renderer="morjson") @view_config(route_name="search_results_rest", renderer="/utils/results_wrap.mako") +@view_config(route_name="user_search_results_rest", renderer="/utils/results_wrap.mako") def search_results(request): """Search for the query terms in the matchdict/GET params @@ -99,6 +100,8 @@ def search_results(request): else: phrase = rdict.get('search', '') + username = rdict.get('username', None) + # with content is always in the get string with_content = asbool(rdict.get('content', False)) LOG.debug('with_content') @@ -112,7 +115,7 @@ def search_results(request): page = params.get('page', None) count = params.get('count', None) - res_list = searcher.search(phrase, content=with_content) + res_list = searcher.search(phrase, content=with_content, username=username) # we're going to fake this since we dont' have a good way to do this query # side @@ -150,11 +153,15 @@ def search_results(request): } -@view_config(route_name="export", renderer="/utils/export.mako") +@view_config(route_name="user_export", renderer="/utils/export.mako") def export(request): """Handle exporting a user's bookmarks to file""" - bmark_list = Bmark.query.join(Bmark.tags).all() + rdict = request.matchdict + username = rdict.get('username') + + bmark_list = Bmark.query.join(Bmark.tags).filter(Bmark.username==username).all() request.response_content_type = 'text/html' + headers = [('Content-Disposition', 'attachment; filename="bookie_export.html"')] setattr(request, 'response_headerlist', headers) @@ -164,6 +171,7 @@ def export(request): @view_config(route_name="redirect", renderer="/utils/redirect.mako") +@view_config(route_name="user_redirect", renderer="/utils/redirect.mako") def redirect(request): """Handle redirecting to the selected url @@ -172,6 +180,7 @@ def redirect(request): """ rdict = request.matchdict hash_id = rdict.get('hash_id', None) + username = rdict.get('username', None) hashed = Hashed.query.get(hash_id) @@ -181,7 +190,10 @@ def redirect(request): hashed.clicks = hashed.clicks + 1 - bookmark = Bmark.query.filter(Bmark.hash_id==hash_id).one() - bookmark.clicks = bookmark.clicks + 1 + if username is not None: + bookmark = Bmark.query.\ + filter(Bmark.hash_id==hash_id).\ + filter(Bmark.username==username).one() + bookmark.clicks = bookmark.clicks + 1 return HTTPFound(location=hashed.url) diff --git a/docs/install.rst b/docs/install.rst index 9c2faab5..ba572219 100644 --- a/docs/install.rst +++ b/docs/install.rst @@ -6,13 +6,13 @@ If you don't use the `bootstrap.py` to install Bookie, you can perform its steps manually. The idea is to setup a virtualenv_ to install Bookie into. The list of app packages required for development work are in the `requirements.txt` file with hard locked versions that help make sure things -work. +work. :: # install the required packages to build bookie # (just needs to be run once) - $ sudo apt-get install build-essential libxslt1-dev libxml2-dev python-dev + $ sudo apt-get install build-essential libxslt1-dev libxml2-dev python-dev git # Mysql & Postgresql users $ sudo apt-get install libmysqlclient-dev @@ -53,4 +53,4 @@ http://127.0.0.1:6543 -.. _virtualenv: +.. _virtualenv: diff --git a/docs/started.rst b/docs/started.rst index fa394ce1..a7e9de7a 100644 --- a/docs/started.rst +++ b/docs/started.rst @@ -16,6 +16,7 @@ There are some required packages that need to be installed so you can build book - python-dev - libxslt1-dev - libxml2-dev +- git Note: right we we support three databases - mysql, postgres, and sqlite - and the database bindings need to be built into the virtualenv. We're hoping to `clean this up some`_ some going forward. @@ -36,7 +37,7 @@ If you're running Ubuntu 10.10 (Maverick), here's some actual commands to get yo # install the required packages to build bookie # (just needs to be run once) - $ sudo apt-get install build-essential libxslt1-dev libxml2-dev python-dev + $ sudo apt-get install build-essential libxslt1-dev libxml2-dev python-dev git # Mysql & Postgresql users $ sudo apt-get install libmysqlclient-dev diff --git a/extensions/chrome_ext/lib/bookie.api.js b/extensions/chrome_ext/lib/bookie.api.js index d22e2aad..ae909030 100644 --- a/extensions/chrome_ext/lib/bookie.api.js +++ b/extensions/chrome_ext/lib/bookie.api.js @@ -237,9 +237,12 @@ var bookie = (function (opts) { * @param callbacks is an object of success, complete, error callbacks * */ - $b.api.sync = function (callbacks) { + $b.api.sync = function (api_key, callbacks) { opts = { url: "/api/v1/bmarks/sync", + data: { + 'api_key': api_key + }, success: callbacks.success }; diff --git a/extensions/chrome_ext/manifest.json b/extensions/chrome_ext/manifest.json index f8f4b206..e2aeb9fd 100644 --- a/extensions/chrome_ext/manifest.json +++ b/extensions/chrome_ext/manifest.json @@ -1,6 +1,6 @@ { "name": "Bookie", - "version": "0.2.16", + "version": "0.3.0", "description": "Bookie Bookmarks", "permissions": [ "tabs", diff --git a/extensions/chrome_ext/options.html b/extensions/chrome_ext/options.html index 0a7aa5bf..2551170b 100644 --- a/extensions/chrome_ext/options.html +++ b/extensions/chrome_ext/options.html @@ -171,7 +171,7 @@ $("#syncBtn").click(function() { $('#circle').show(); - bookie.api.sync({ + bookie.api.sync(localStorage.getItem('api_key'), { success: function (data) { var bkg, hash_id; bkg = chrome.extension.getBackgroundPage(); @@ -196,7 +196,7 @@

Bookie Settings

Connection - + @@ -218,12 +218,12 @@

Server Sync

Note: your Bookie url and api key settings must - be set and saved before using Sync

+ be set and saved before using Sync

This will download information about your bookmarks from the - bookie server instance into your browser. This helps us notify you - when you're on a page you've already bookmarked. In the future, - this might be able to sync Chrome bookmarks with your Bookie - bookmarks.

+ bookie server instance into your browser. This helps us notify you + when you're on a page you've already bookmarked. In the future, + this might be able to sync Chrome bookmarks with your Bookie + bookmarks.

diff --git a/fabfile/__init__.py b/fabfile/__init__.py index 95a612b2..bdf4393b 100644 --- a/fabfile/__init__.py +++ b/fabfile/__init__.py @@ -17,6 +17,7 @@ "setup.py", ] # IMPORT the rest of the commands we have available to us +from admin import * from docs import * from database import * from development import * diff --git a/fabfile/admin.py b/fabfile/admin.py new file mode 100644 index 00000000..8e74c317 --- /dev/null +++ b/fabfile/admin.py @@ -0,0 +1,45 @@ +"""Fabric commands to help add new users to the system""" +from database import sample +from fabric.api import env +from fabric.api import require +from utils import parse_ini + + +def new_user(username, email): + """Add new user function, pass username, email + + :param username: string of new user + :param email: string of new email + + """ + require('hosts', provided_by=[sample]) + require('ini', provided_by=[sample]) + + parse_ini(env["ini_file"]) + + import transaction + from bookie.models import initialize_sql + from sqlalchemy import create_engine + + engine = create_engine(env.ini.get('app:bookie', 'sqlalchemy.url')) + initialize_sql(engine) + + from bookie.models import DBSession + from bookie.models.auth import get_random_word, User + sess = DBSession() + + u = User() + u.username = unicode(username) + passwd = get_random_word(8) + u.password = passwd + u.email = unicode(email) + u.activated = True + u.is_admin = False + u.api_key = User.gen_api_key() + + print dict(u) + print passwd + + sess.add(u) + sess.flush() + transaction.commit() diff --git a/fabfile/database.py b/fabfile/database.py index dbdb0f94..aa9cfc32 100644 --- a/fabfile/database.py +++ b/fabfile/database.py @@ -183,6 +183,7 @@ def db_init_bookmark(): from bookie.models import Bmark bmark_us = Bmark(u'http://bmark.us', + u'admin', desc=u"Bookie Website", ext= u"Bookie Documentation Home", tags = u"bookmarks") diff --git a/migrations/versions/009_Add_user_db_table.py b/migrations/versions/009_Add_user_db_table.py new file mode 100644 index 00000000..b0a00e5a --- /dev/null +++ b/migrations/versions/009_Add_user_db_table.py @@ -0,0 +1,30 @@ +from sqlalchemy import * +from migrate import * + +def upgrade(migrate_engine): + """Add the users table we'll use """ + meta = MetaData(migrate_engine) + user = Table('users', meta, + Column('id', Integer, autoincrement=True, primary_key=True), + Column('username', Unicode(255), unique=True), + Column('password', Unicode(60)), + Column('email', Unicode(255), unique=True), + Column('activated', Boolean, server_default="0"), + Column('is_admin', Boolean, server_default="0"), + Column('last_login', DateTime), + ) + + user.create() + + # adding an initial user account with user/pass combo of admin:admin + migrate_engine.execute(user.insert().values(username=u'admin', + password=u'$2a$10$LoSEVbN6833RtwbGQlMhJOROgkjHNH4gjmzkLrIxOX1xLXNvaKFyW', + email=u'testing@dummy.com', + activated=True, + is_admin=True)) + +def downgrade(migrate_engine): + """And the big drop""" + meta = MetaData(migrate_engine) + user = Table('users', meta) + user.drop() diff --git a/migrations/versions/010_Add_user_to_the_bmark_object.py b/migrations/versions/010_Add_user_to_the_bmark_object.py new file mode 100644 index 00000000..f41f1671 --- /dev/null +++ b/migrations/versions/010_Add_user_to_the_bmark_object.py @@ -0,0 +1,25 @@ +from sqlalchemy import * +from migrate import * + +def upgrade(migrate_engine): + """Add the username field to the bmarks table""" + meta = MetaData(migrate_engine) + bmarks = Table('bmarks', meta, autoload=True) + + # we can't have nullable at first, we need to set the values + username = Column('username', Unicode(255)) + create_column(username, bmarks) + + # all current bookmarks need to be owned by 'admin' our default user + migrate_engine.execute(bmarks.update().values(username=u'admin')) + + # now we add on the nullable=False + alter_column('username', nullable=False, table=bmarks) + +def downgrade(migrate_engine): + meta = MetaData(migrate_engine) + bmarks = Table('bmarks', meta, autoload=True) + username = Column('username', Unicode(255), nullable=False) + + drop_column(username, bmarks) + diff --git a/migrations/versions/011_api_key_is_user_specific.py b/migrations/versions/011_api_key_is_user_specific.py new file mode 100644 index 00000000..b823a465 --- /dev/null +++ b/migrations/versions/011_api_key_is_user_specific.py @@ -0,0 +1,26 @@ +from sqlalchemy import * +from migrate import * +from bookie.models.auth import User + +def upgrade(migrate_engine): + """Need to add a col to the users for their api key""" + meta = MetaData(migrate_engine) + + users = Table('users', meta, autoload=True) + api_key = Column('api_key', Unicode(12)) + create_column(api_key, users) + + # now add an api key for our admin user + migrate_engine.execute(users.update().\ + where(users.c.username==u'admin').\ + values(api_key=User.gen_api_key())) + + +def downgrade(migrate_engine): + """And bye bye api key column""" + meta = MetaData(migrate_engine) + + users = Table('users', meta, autoload=True) + api_key = Column('api_key', Unicode(12)) + + drop_column(api_key, users) diff --git a/requirements.txt b/requirements.txt index f0517363..7d0fdf12 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,7 +5,7 @@ colorama -e git+https://github.com/dcramer/decruft.git#egg=decruft Fabric==0.9.2 Jinja2==2.5.5 -Mako==0.3.6 +Mako==0.4.1 MarkupSafe==0.12 Paste==1.7.5.1 PasteDeploy==1.3.4 @@ -27,6 +27,7 @@ mock==0.7.0 nose==1.0.0 paramiko==1.7.6 pep8==0.6.1 +py-bcrypt==0.2.0 pycrypto==2.0.1 pylint==0.23.0 pyramid==1.0 diff --git a/scripts/readability/README.rst b/scripts/readability/README.rst new file mode 100644 index 00000000..e954107e --- /dev/null +++ b/scripts/readability/README.rst @@ -0,0 +1,44 @@ +Readable Parsing +================= + +The system will handle fetching the html content of pages and running that +through a readable filter. We're using the library Decruft. Once parsed, we +store that content so you can search and pull that up later. + +There are currently three ways to load that content into the system. + +1. Google Chrome Extension +--------------------------- +The chrome extension supports a checkbox in the options that sends the current +page's html along for the ride when you add or edit a bookmark. In this way the +content is ready for your use right away. + +2. existing.py +--------------- +`existing.py` is a sample script writting in python that fetches a list of +unparsed urls from your install and starts fetching/parsing them. It was the +first script made to do it and is synchronous. On large bookmark lists it +might take a while for this to run. It was averaging some 1 bookmark/s on my +test system. + +3. Let's complicate it, node.js, beanstalkd, and the api +--------------------------------------------------------- +In order to have a method that was more performant, there's a system you can +use to really crank through these. There's much more setup involved. + +The system is built around a new pair of API calls that will return a list of +unparsed urls and that you can feed information about an attempt to load html +content. The `readable_producer.js` is a node.js script that will run through +the list of bookmarks to parse and async fetch their content. If the content is +there and ok, it'll place that into a beanstalkd queue. If not, it'll create a +list of what went wrong and stick that in the queue. + +The `readable_consumer.py` is meant to be run several times to read items off +the queue and to make API calls to the bookie installation. It will send the +content to bookie to run through the parser and store in the database. Since +this is sync code, we want to run multiple versions of this. In testing, I was +able to run 4 against a sqlite database, and 8 against postgresql backed bookie +install. + +This method of running could be scaled well over 5 urls parsed and put into the +bookie database per second. diff --git a/scripts/readability/nlogger.json b/scripts/readability/nlogger.json new file mode 100644 index 00000000..e57dafb2 --- /dev/null +++ b/scripts/readability/nlogger.json @@ -0,0 +1,6 @@ +{ + "color": "auto", + "level": { + "*": "warn" + } +} diff --git a/scripts/readability/readable_consumer.py b/scripts/readability/readable_consumer.py new file mode 100644 index 00000000..c1c63700 --- /dev/null +++ b/scripts/readability/readable_consumer.py @@ -0,0 +1,43 @@ +import beanstalkc +import json +import urllib +import urllib2 + +SERVER = '127.0.0.5' +PORT = 11300 + +# setup connection +bean = beanstalkc.Connection(host="localhost", + port=11300, + parse_yaml=lambda x: x.split("\n")) + +def post_readable(data): + """Send off the parsing request to the web server""" + url = 'http://127.0.0.1:6543/api/v1/bmarks/readable' + + if 'content' in data: + data['content'] = data['content'].encode('utf-8') + + http_data = urllib.urlencode(data) + + try: + req = urllib2.Request(url, http_data) + response = urllib2.urlopen(req) + res = response.read() + assert "true" in str(res) + except Exception, exc: + print "FAILED: " + data['hash_id'] + print str(exc) + +bean.watch('default') + +while True: + job = bean.reserve() + j = json.loads(urllib.unquote(job.body)) + if 'hash_id' in j: + print j['hash_id'] + post_readable(j) + else: + print "ERROR: missing fields -- " + str(j['hash_id']) + job.delete() + diff --git a/scripts/readability/readable_producer.js b/scripts/readability/readable_producer.js new file mode 100644 index 00000000..faf47aef --- /dev/null +++ b/scripts/readability/readable_producer.js @@ -0,0 +1,299 @@ +/** + * Readable Producer: + * Check with the bookie api for bookmarks that we still need to parse + * Process those, fetch their content, and pass to the consumer via the + * beanstalkd message queue + * + * Requirements: + * npm install nodestalker request nlogger + * + * + */ +var bs = require('nodestalker'), + http = require('http'), + log = require('nlogger').logger(module), + qs = require('querystring'), + req = require('get'), + url = require('url'), + util = require('util'); + +var bean_client = bs.Client(), + inspect = util.inspect, + API = "http://127.0.0.1:6543/api/v1/bmarks", + TIMEOUT = 25; + +/** + * Bind this to the global connection instance of the queue + * + */ +bean_client.addListener('error', function(data) { + log.error('QUEUE ERROR'); + log.error(data); +}); + +bean_client.use('default').onSuccess(function (data) { + log.info('connected to default tube for queue'); +}); + + +/** + * Used to hack the .extend method onto objects for options and such + * + */ +Object.defineProperty(Object.prototype, "extend", { + enumerable: false, + value: function(from) { + var props = Object.getOwnPropertyNames(from); + var dest = this; + props.forEach(function(name) { + if (name in dest) { + var destination = Object.getOwnPropertyDescriptor(from, name); + Object.defineProperty(dest, name, destination); + } + }); + return this; + } +}); + + +var ResponseData = function (hashed) { + var that = {}; + that.hash_id = hashed.hash_id; + that.url = hashed.url; + that.content = undefined; + that.success = true; + that.status_code = undefined; + that.status_message = undefined; + return that; +}; + +/** + * Handle the fetching and processing of the content of a bookmark + * + */ +var BookieContent = function (opts) { + var defaults = { + bookieurl: 'http://127,.0.0.1:6543/', + queue_tube: 'default', + queue_conn: undefined, + }; + + if (typeof opts === 'object') { + opts = defaults.extend(opts); + } else { + opts = defaults; + } + + var that = {}; + that.opts = opts; + + that.queue_content = function (hash_id, content) { + var escaped, + post_data = JSON.stringify({'success': true, 'hash_id': hash_id, 'content': content}), + qclient = that.opts.queue_conn; + + try { + escaped = qs.escape(post_data); + + qclient.put(escaped). + onSuccess(function(data) { + log.info("Added to queue: " + hash_id); + }); + } catch (err) { + log.error('escaping url content'); + log.error(err); + log.error(post_data.substr(0,50)); + } + }; + + /** + * If there's a problem parsing, fetching, or otherwise with this url + * Queue up that we'd like to store a failed fetch so that we don't + * reprocess for now + * + */ + that.queue_error = function (resp) { + var escaped, + data = { + 'success': false, + 'hash_id': resp.hash_id, + 'content_type': resp.content_type, + 'status_code': resp.status_code, + 'status_message': resp.status_message + }, + post_data = JSON.stringify(data), + qclient = that.opts.queue_conn; + + try { + escaped = qs.escape(post_data); + + qclient.put(escaped). + onSuccess(function(data) { + log.info("Added to error queue: " + data.hash_id); + }); + } catch (err) { + log.error('escaping error url content'); + log.error(err); + log.error(post_data.substr(0,100)); + } + }; + + /** + * We can only fetch html and parse content for html pages + * + * Exclude images, pdfs + * + */ + that.ext_is_valid = function (ext, resp_data) { + switch(ext) { + case '.png': + case '.jpg': + case '.gif': + resp_data.success = false; + resp_data.content_type = 'image/' + ext.substr(1); + resp_data.status_code = 1; + resp_data.status_message = 'url skipped'; + break; + case '.mp3': + resp_data.success = false; + resp_data.content_type = 'audio/' + ext.substr(1); + resp_data.status_code = 1; + resp_data.status_message = 'url skipped'; + break; + case '.mp4': + case '.mpg': + case '.mov': + case '.wmv': + resp_data.success = false; + resp_data.content_type = 'video/' + ext.substr(1); + resp_data.status_code = 1; + resp_data.status_message = 'url skipped'; + break; + case '.pdf': + resp_data.success = false; + resp_data.content_type = 'application/' + ext.substr(1); + resp_data.status_code = 1; + resp_data.status_message = 'url skipped'; + break; + default: + // all is well + break; + } + + return resp_data; + }; + + that.fetch_url = function (hashed) { + var hash_id = hashed.hash_id, + someUri = hashed.url, + ext = someUri.substr(-4), + resp = that.ext_is_valid(ext, hashed); + + if (resp.success) { + log.info("Fetching content for url: " + someUri); + var req_data = {'uri': someUri, + "max_redirs": 10 + }, + req_callback = function (error, body) { + if (error) { + log.info('FOUND ERROR'); + resp.success = false; + resp.status_code = error.statusCode; + resp.status_message = error.message; + + // if we got here assume it was html + // content type + resp.content_type = 'text/html'; + that.queue_error(resp); + + } else { + log.info("Fetched " + someUri + " OK!"); + resp.sucess = true; + that.queue_content(hash_id, body); + } + }, + dl = new req(req_data); + + dl.asString(req_callback); + } else { + // then we skipped this because it was an image/binary send to + // queue to store result + log.info('Skipping non html file: ' + someUri); + that.queue_error(resp); + } + }; + + return that; +}; + + +/** + * Handle API calls to the bookie instance + * + */ +var BookieAPI = function (opts) { + var defaults = { + bookieurl: 'http://127,.0.0.1:6543/', + }; + + if (typeof opts === 'object') { + opts = defaults.extend(opts); + } else { + opts = defaults; + } + + log.info('Using API url: ' + opts.bookieurl); + + var that = {}; + that.opts = opts; + + that.processArray = function(items, process) { + var todo = items.concat(); + log.info("Processing"); + + setTimeout(function() { + process(ResponseData(todo.shift())); + if(todo.length > 0) { + setTimeout(arguments.callee, TIMEOUT); + } + }, 25); + }; + + that.getReadableList = function() { + var req_data = {uri: that.opts.bookieurl + '/get_readable', + method: "GET" + }, + req_callback = function (error, body) { + if (error) { + log.error('fetching readable list'); + log.error(error); + } + + var res = JSON.parse(body); + that.processArray(res.payload.urls, function (hashed) { + log.info(inspect(hashed)); + content = BookieContent({ + bookieurl: API, + queue_conn: bean_client + }); + + content.fetch_url(hashed); + }); + }; + + log.info("Requesting list of bookmarks to readable"); + var dl = new req(req_data); + dl.asString(req_callback); + + }; + + return that; + +}; + + +// let's get this party started +log.info('Starting up'); + +var bookie_api = BookieAPI({'bookieurl': API}); +bookie_api.getReadableList(); diff --git a/test.ini b/test.ini index 3c308cea..f89145d7 100644 --- a/test.ini +++ b/test.ini @@ -10,10 +10,7 @@ mako.directories = bookie:templates sqlalchemy.url = sqlite:///test_bookie.db # sqlalchemy.echo = true -# custom setting for allowing the edit links/etc you should not set this to -# true and use a tool to make secure api calls bookie is hosted on a public -# site this should be safe to use if running locally -allow_edit = 1 +auth.secret=supersecretthing_changeme # api_key needs to be submitted with all API requests as a form of weak # security to start out. It should be pretty long and not shared. We encourage diff --git a/test_mysql.ini b/test_mysql.ini index 5879a9d2..e4476e36 100644 --- a/test_mysql.ini +++ b/test_mysql.ini @@ -9,10 +9,7 @@ default_locale_name = en mako.directories = bookie:templates sqlalchemy.url = mysql://jenkins_bookie:bookie@127.0.0.1:3306/jenkins_bookie -# custom setting for allowing the edit links/etc you should not set this to -# true and use a tool to make secure api calls bookie is hosted on a public -# site this should be safe to use if running locally -allow_edit = 1 +auth.secret=supersecretthing_changeme # api_key needs to be submitted with all API requests as a form of weak # security to start out. It should be pretty long and not shared. We encourage diff --git a/test_psql.ini b/test_psql.ini index 72da66fb..082fcab2 100644 --- a/test_psql.ini +++ b/test_psql.ini @@ -9,10 +9,7 @@ default_locale_name = en mako.directories = bookie:templates sqlalchemy.url = postgresql://jenkins_bookie:bookie@127.0.0.1:5432/jenkins_bookie -# custom setting for allowing the edit links/etc you should not set this to -# true and use a tool to make secure api calls bookie is hosted on a public -# site this should be safe to use if running locally -allow_edit = 1 +auth.secret=supersecretthing_changeme # api_key needs to be submitted with all API requests as a form of weak # security to start out. It should be pretty long and not shared. We encourage