Permalink
Browse files

initial release. working qr/url shortening creation. asynchronous data

saving on couchdb via PUSH/PULL model. Thos model need to be reworked to
a XREQ system.
  • Loading branch information...
0 parents commit 2c9e18d5f3ef7d361241b7f1306b953e80f23ef0 @benoitc committed Jan 24, 2011
@@ -0,0 +1,15 @@
+*.gem
+*.swp
+*.pyc
+*.pyo
+*#*
+build
+dist
+setuptools-*
+.svn/*
+.DS_Store
+*.so
+qrurl.egg-info
+nohup.out
+.coverage
+
22 LICENSE
@@ -0,0 +1,22 @@
+20011 (c) Benoît Chesneau <benoitc@e-engura.org>
+
+Permission is hereby granted, free of charge, to any person
+obtaining a copy of this software and associated documentation
+files (the "Software"), to deal in the Software without
+restriction, including without limitation the rights to use,
+copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the
+Software is furnished to do so, subject to the following
+conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+OTHER DEALINGS IN THE SOFTWARE.
@@ -0,0 +1,11 @@
+include NOTICE
+include LICENSE
+include README.rst
+include TODO.txt
+include THANKS
+include bootstrap.py
+include buildout.cfg
+recursive-include debian *
+recursive-include examples *
+recursive-include tests *
+recursive-include doc *
7 NOTICE
@@ -0,0 +1,7 @@
+qrurl
+-----
+
+2011 (c) Benoît Chesneau <benoitc@e-engura.org>
+
+qrurl is released under the MIT license. See the LICENSE file for the
+complete license.
No changes.
@@ -0,0 +1,7 @@
+#!/usr/bin/env sh
+# This file is part of qrurl released under the MIT license.
+# See the NOTICE for more information.
+
+PYTHON_BIN=`which python`
+
+$PYTHON_BIN -OO -c "import qrurl.consumer" $@
@@ -0,0 +1,8 @@
+# -*- coding: utf-8 -
+#
+# This file is part of qrurl released under the MIT license.
+# See the NOTICE for more information.
+
+version_info = (0, 1, 0)
+__version__ = ".".join(map(str, version_info))
+
@@ -0,0 +1,66 @@
+# -*- coding: utf-8 -
+#
+# This file is part of qrurl released under the MIT license.
+# See the NOTICE for more information.
+
+import hashlib
+import re
+import time
+
+from qrurl.base62 import b62encode
+
+URL_PATTERN = ur'^[a-z]+://([^/:]+\.[a-z]{2,10}|([0-9]{1,3}\.){3}[0-9]{1,3})(:[0-9]+)?(\/.*)?$'
+RE_URL = re.compile(URL_PATTERN, re.IGNORECASE)
+
+def key(*args):
+ return ':'.join(["URL"] + [arg.upper() for arg in args])
+
+def valid_url(url):
+ if not RE_URL.match(url):
+ raise ValueError("invalid url")
+ return url
+
+def create_url(conn, sock, url):
+ url = valid_url(url)
+
+ uri_key = key(hashlib.sha1(url).hexdigest())
+
+ uri_id = str(b62encode(int(conn.incr(key("id")))))
+ conn.set(uri_key, uri_id)
+ conn.set(key("hash", uri_id), uri_key)
+ t = int(time.time())
+ data_key = key("uid", "data", uri_id)
+ conn.hset(data_key, 'date_creation', t)
+ conn.hset(data_key, 'url', url)
+
+ # send url creation to consumers
+ sock.send("URL:%s" % uri_id)
+ return uri_id
+
+def get_url(conn, uid):
+ u = conn.hget(key("uid", "data", uid), 'url')
+ if u:
+ return u
+
+def get_url_info(conn, uid):
+ u = conn.hgetall(key("uid", "data", uid))
+ if u:
+ return u
+
+def set_click(conn, uid, info):
+ conn.hmset(key("uid", "click"), info)
+
+def get_click(conn, uid):
+ click = conn.hgetall(key("uid", "click", uid))
+ if click:
+ return click
+
+def add_consumer(conn, consumer_uri):
+ return conn.sadd("consumers", consumer_uri)
+
+def get_consumers(conn):
+ return conn.smembers("consumers")
+
+def remove_consumer(conn, consumer_uri):
+ return conn.srem("consumers", consumer_uri)
+
@@ -0,0 +1,46 @@
+# -*- coding: utf-8 -
+#
+# This file is part of friendpaste released under Apache License, Version 2.0.
+# See the NOTICE for more information.
+
+import string
+
+
+BASE62_CHARS = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
+BASE62_VALUES = ''.join([chr(i) for i in range(62)])
+
+E_BASE62_PRIMITIVES = dict([(BASE62_CHARS[i], i) for i in range(62)])
+D_BASE62_PRIMITIVES = string.maketrans(BASE62_VALUES, BASE62_CHARS)
+
+def b62encode(value):
+ """ encode int/long to base62 stsring.
+
+ >>> b62encode(1123458)
+ '4iGI'
+ """
+ number = value
+ result = ''
+ while number != 0:
+ result = D_BASE62_PRIMITIVES[number % 62] + result
+ number /= 62
+ return result
+
+def b62decode(value):
+ """ decode string to int/long
+
+ >>> b62decode('4iGI')
+ 1123458
+ """
+ value = str(value)
+ i = 0
+ i_out = 0
+ for c in value[::-1]:
+ place = 62 ** i
+ i_out += int(E_BASE62_PRIMITIVES[c]) * place
+ i += 1
+ return i_out
+
+
+if __name__ == '__main__':
+ import doctest
+ doctest.testmod()
@@ -0,0 +1,158 @@
+# -*- coding: utf-8 -
+#
+# This file is part of qrurl released under the MIT license.
+# See the NOTICE for more information.
+
+import json
+from optparse import OptionParser, make_option
+import sys
+import time
+
+import couchdbkit
+import gevent
+from gevent import monkey; monkey.patch_socket()
+from gevent_zeromq import zmq
+import redis
+
+from qrurl import api
+
+class Consumer(object):
+ """ simple consumer class saving infos from redis on couchdb to
+ process them later and allows their replication to final users """
+
+
+ def __init__(self, couch_uri="http://127.0.0.1:5984/lqr",
+ redis_host="127.0.0.1", redis_port=6379, redis_db=0,
+ zmq_uri="tcp://127.0.0.1:5000"):
+
+ self.conf = dict(
+ couch_uri = couch_uri,
+ redis_host = redis_host,
+ redis_port = redis_port,
+ redis_db = redis_db,
+ zmq_uri = zmq_uri
+ )
+
+ # set redis connections
+ self.conn_redis = redis.Redis(host=redis_host, port=redis_port,
+ db=redis_db)
+
+ # set couch database
+ self.couch_db = couchdbkit.Database(couch_uri, create=True)
+ self.docs = []
+ self.last_updated = 0
+
+ # create zmq context
+ self.zmq_ctx = zmq.Context()
+ self.zmq_sock = self.zmq_ctx.socket(zmq.PULL)
+ self.zmq_sock.bind(zmq_uri)
+
+ # we are alive
+ self.alive = True
+
+ # register consumer so it can receive messages
+ self.register()
+
+ # start doc loop
+ self.start_batch()
+
+
+ def start_batch(self):
+ """Start the batch Greenlet used to send docs each mn"""
+
+ def batch():
+ while self.alive:
+ gevent.sleep(60.0)
+ if len(self.docs) > 0:
+ self.couch_db.save_docs(self.docs)
+ self.docs = []
+
+ return gevent.spawn(batch)
+
+ def stop(self):
+ self.alive = False
+
+
+ def register(self):
+ api.add_consumer(self.conn_redis, self.conf['zmq_uri'])
+
+ def unregister(self):
+ api.remove_consumer(self.conn_redis, self.conf['zmq_uri'])
+
+ try:
+ self.zmq_sock.close()
+ self.zmq_ctx.term()
+ except:
+ pass
+
+ def handle_message(self, msg):
+ kind, uri = msg.split(":", 1)
+
+ print uri
+ if kind == "URL":
+ doc = api.get_url_info(self.conn_redis, uri)
+ print "got doc"
+ if kind == "CLICK":
+ doc = api.get_click(self.conn_redis, uri)
+
+ diff = time.time() - self.last_updated
+ if len(self.docs) == 100 or (len(self.docs) > 0 and diff >= 60):
+ # we update each 100 docs or each minutes
+ self.couch_db.save_docs(self.docs)
+ self.docs = []
+
+ if not doc:
+ return
+ self.docs.append(doc)
+ self.last_updated = time.time()
+
+ def listen(self):
+ while True:
+ msg = self.zmq_sock.recv()
+ if not msg:
+ break
+ print "got msg %s" % msg
+ gevent.spawn(self.handle_message, msg)
+
+ self.alive = False
+
+
+def run():
+ usage = "usage: %prog [options]"
+ options = [
+ make_option("--with-couch", action="store",
+ type="string", dest="couch_uri", help="couchdb uri"),
+ make_option("--with-redis-host", action="store",
+ type="string", dest="redis_host", help="redis host"),
+ make_option("--with-redis-port", action="store",
+ type="string", dest="redis_port", help="redis port"),
+ make_option("--with-redis-db", action="store", type="int",
+ dest="redis_db", help="redis db"),
+ make_option("--with-zmq", action="store", type="string",
+ dest="zmq_uri", help="zmq uri")
+
+ ]
+ parser = OptionParser(usage=usage, option_list=options)
+ opts, args = parser.parse_args()
+
+ # settings default here is not needed but while we are here...
+ couchdb_uri = opts.couch_uri or "http://127.0.0.1:5984/lqr"
+ redis_host = opts.redis_host or "127.0.0.1"
+ redis_port = opts.redis_port or 6379
+ redis_db = opts.redis_db or 0
+ zmq_uri = opts.zmq_uri or "tcp://127.0.0.1:5000"
+
+ consumer = Consumer(couch_uri=couchdb_uri, redis_host=redis_host,
+ redis_port=redis_port, redis_db=redis_db, zmq_uri=zmq_uri)
+
+ try:
+ consumer.listen()
+ except (KeyboardInterrupt, SystemExit):
+ consumer.stop()
+ finally:
+ consumer.unregister()
+
+ sys.exit(0)
+
+if __name__ in ('__main__', 'qrurl.consumer'):
+ run()
Oops, something went wrong.

0 comments on commit 2c9e18d

Please sign in to comment.