From 23f4032e145d870530b3eba986554d899a1fbf17 Mon Sep 17 00:00:00 2001 From: Amit Tripathi Date: Sun, 26 Nov 2017 00:22:10 +0530 Subject: [PATCH] diff short code for http and non http fix, clickmeta test case, fixes --- requirements.txt | 5 --- src/pygmy/app/link.py | 10 ++++-- src/pygmy/config/pygmy.cfg | 3 +- src/pygmy/exception/auth.py | 3 +- src/pygmy/model/link.py | 16 +++++++-- src/pygmy/tests/pygmy_test.cfg | 3 +- src/pygmy/tests/test_click_stats.py | 36 +++++++++++++++++++ src/pygmy/tests/test_url_shorten_unshorten.py | 22 +++++++++--- src/pygmy/utilities/urls.py | 5 ++- src/pygmy/validator/link.py | 2 +- src/pyui/restclient/pygmy.py | 2 +- 11 files changed, 85 insertions(+), 22 deletions(-) create mode 100644 src/pygmy/tests/test_click_stats.py diff --git a/requirements.txt b/requirements.txt index a5c0955..a800604 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,3 @@ -appdirs==1.4.3 appnope==0.1.0 bcrypt==3.1.4 certifi==2017.11.5 @@ -7,12 +6,10 @@ chardet==3.0.4 click==6.7 decorator==4.1.2 Django==1.11.7 -ecdsa==0.13 Flask==0.12.2 Flask-Cors==3.0.3 Flask-JWT-Extended==3.3.4 Flask-Script==2.0.6 -future==0.16.0 geoip2==2.6.0 idna==2.6 ipython==6.2.1 @@ -30,10 +27,8 @@ pickleshare==0.7.4 prompt-toolkit==1.0.15 ptyprocess==0.5.2 pycparser==2.18 -pycrypto==2.6.1 Pygments==2.2.0 PyJWT==1.5.3 -pyparsing==2.2.0 pytest==3.2.5 pytz==2017.3 requests==2.18.4 diff --git a/src/pygmy/app/link.py b/src/pygmy/app/link.py index 27a5457..221a12f 100644 --- a/src/pygmy/app/link.py +++ b/src/pygmy/app/link.py @@ -30,7 +30,7 @@ def shorten(long_url, short_code=None, expire_after=None, description=None, query_dict.update(dict(short_code=short_code, is_custom=True)) insert_dict = query_dict if secret_key: - query_dict.update(secret_key=secret_key, is_protected=True) + query_dict.update(secret_key=str(secret_key), is_protected=True) insert_dict = query_dict if expire_after: insert_dict['expire_after'] = expire_after @@ -71,7 +71,7 @@ def unshorten(short_url, secret_key=None, if url_manager.has_expired(): raise LinkExpired if link.is_protected: - if not secret_key or link.secret_key != secret_key: + if not secret_key or str(link.secret_key) != str(secret_key): raise URLAuthFailed(short_url) if request: save_clickmeta(request, link) @@ -93,7 +93,7 @@ def resolve_short(short_code, request=None, secret_key=None): if manager.has_expired() is True: raise LinkExpired('Link has expired') if link.is_protected: - if not secret_key or link.secret_key != secret_key: + if not secret_key or str(link.secret_key) != str(secret_key): raise URLAuthFailed(short_code) # put stats if request: @@ -147,3 +147,7 @@ def save_clickmeta(request, link): else: data = parse_request(request) return ClickMetaManager().add(link_id=link.id, **data) + + +def click(short_code): + """Add method to mimic click url""" diff --git a/src/pygmy/config/pygmy.cfg b/src/pygmy/config/pygmy.cfg index fceeb28..2cef342 100644 --- a/src/pygmy/config/pygmy.cfg +++ b/src/pygmy/config/pygmy.cfg @@ -4,7 +4,8 @@ host = 0.0.0.0 port = 9119 debug = False flask_secret = CvJHGFVBj*&^TRGBHDdBV836bdy73JJDHGV -url = 127.0.0.1 +short_url = 127.0.0.1 +short_url_schema = http:// [database] file_name: pygmy.db diff --git a/src/pygmy/exception/auth.py b/src/pygmy/exception/auth.py index 513dbb2..c85f42c 100644 --- a/src/pygmy/exception/auth.py +++ b/src/pygmy/exception/auth.py @@ -8,5 +8,4 @@ def __init__(self, url): self.url = url def __str__(self): - return ("Invalid/missing secret key for" - " protected url {0}.").format(self.url) \ No newline at end of file + return "Invalid/missing secret key for protected url" diff --git a/src/pygmy/model/link.py b/src/pygmy/model/link.py index 4e94c60..44119d0 100644 --- a/src/pygmy/model/link.py +++ b/src/pygmy/model/link.py @@ -7,6 +7,7 @@ from sqlalchemy.orm import relationship from sqlalchemy import (Column, String, Integer, Boolean, BigInteger, Unicode, DateTime) +from urllib.parse import urlparse from pygmy.database.base import Model from pygmy.database.dbutil import dbconnection, utcnow @@ -142,6 +143,8 @@ def has_expired(self): @staticmethod def crc32(long_url): + if not urlparse(long_url).scheme: + long_url = 'http://{}'.format(long_url) return binascii.crc32(str.encode(long_url)) @property @@ -155,6 +158,8 @@ def add(self, db, long_url, **kwargs): if db.bind.name == 'mysql': kwargs['created_at'] = datetime.datetime.utcnow() kwargs['updated_at'] = datetime.datetime.utcnow() + if not urlparse(long_url).scheme: + long_url = 'http://{}'.format(long_url) self.link = Link(long_url=long_url, long_url_hash=self.crc32(long_url), **kwargs) db.add(self.link) @@ -180,6 +185,8 @@ def update(self, db, **kwargs): @dbconnection def get(self, db, long_url, is_default=True): + if not urlparse(long_url).scheme: + long_url = 'http://{}'.format(long_url) query_dict = dict(long_url_hash=self.crc32(long_url), long_url=long_url, is_default=is_default) @@ -246,9 +253,12 @@ def find(self, db, **kwargs): long_url query for performance optimization. """ query_dict = dict() - if kwargs.get('long_url'): - query_dict['long_url_hash'] = self.crc32(kwargs.get('long_url')) - query_dict['long_url'] = kwargs.get('long_url') + long_url = kwargs.get('long_url') + if long_url: + if not urlparse(long_url).scheme: + long_url = 'http://{}'.format(long_url) + query_dict['long_url_hash'] = self.crc32(long_url) + query_dict['long_url'] = long_url query_dict.update(self.build_query_dict(**kwargs)) # build sqlalchmey and query query = [getattr(Link, k) == v for k, v in query_dict.items()] diff --git a/src/pygmy/tests/pygmy_test.cfg b/src/pygmy/tests/pygmy_test.cfg index 91d14b8..0d599aa 100644 --- a/src/pygmy/tests/pygmy_test.cfg +++ b/src/pygmy/tests/pygmy_test.cfg @@ -4,7 +4,8 @@ host = 0.0.0.0 port = 9118 debug = False flask_secret = CvJHGFVBj*&^TRGBHDdBV836bdy73JJDHGV -url = 127.0.0.1 +short_url = 127.0.0.1 +short_url_schema = http:// [database] file_name: pygmy_test.db diff --git a/src/pygmy/tests/test_click_stats.py b/src/pygmy/tests/test_click_stats.py new file mode 100644 index 0000000..977c90a --- /dev/null +++ b/src/pygmy/tests/test_click_stats.py @@ -0,0 +1,36 @@ +import os +import tempfile +from unittest import TestCase + +from pygmy.app.link import shorten, unshorten, link_stats +from pygmy.core.initialize import initialize_test +from pygmy.config import config + + +class URLClickStatsTest(TestCase): + """Test for clickmeta i.e. click stats""" + + DBPath = None + + def setUp(self): + self.long_url = 'https://example.com' + + @classmethod + def setUpClass(cls): + currdir = os.path.dirname(os.path.abspath(__file__)) + config_path = currdir + '/pygmy_test.cfg' + db_path = tempfile.NamedTemporaryFile(suffix='.db').name + cls.DBPath = "sqlite:///{}".format(db_path) + initialize_test(config_path, db_url=cls.DBPath) + + def test_config(self): + assert config is not None + assert config.db is not None + assert self.DBPath is not None + + def test_clickmeta(self): + data = shorten(self.long_url) + assert isinstance(data, dict) is True + assert link_stats(data['short_code'] + 'abc+') is None + stats = link_stats(data['short_code'] + '+') + assert stats is not None diff --git a/src/pygmy/tests/test_url_shorten_unshorten.py b/src/pygmy/tests/test_url_shorten_unshorten.py index d48a2bb..e5a35e8 100644 --- a/src/pygmy/tests/test_url_shorten_unshorten.py +++ b/src/pygmy/tests/test_url_shorten_unshorten.py @@ -5,6 +5,7 @@ from pygmy.core.initialize import initialize_test from pygmy.app.link import shorten, unshorten from pygmy.config import config +from pygmy.exception.auth import URLAuthFailed class URLShortenUnshortenTestCases(TestCase): @@ -13,7 +14,7 @@ class URLShortenUnshortenTestCases(TestCase): DBPath = None def setUp(self): - self.long_url = 'https://github.com' + self.long_url = 'https://example.com' @classmethod def setUpClass(cls): @@ -28,7 +29,7 @@ def test_config(self): assert config.db is not None assert self.DBPath is not None - def test_long_url_shorten(self): + def test_aaa_long_url_shorten(self): data = shorten(self.long_url) assert isinstance(data, dict) is True assert data['short_code'] == 'b' @@ -45,13 +46,26 @@ def test_custom_short_url(self): assert udata['long_url'] == self.long_url def test_secret_short_url(self): - secret_key = 'safe' + secret_key = 123 data = shorten(self.long_url, secret_key=secret_key) assert data['is_protected'] is True - assert data['secret_key'] == secret_key + assert data['secret_key'] == str(secret_key) + with self.assertRaises(URLAuthFailed): + unshorten(data['short_code']) + unshorten(data['short_code'], secret_key='safe') + udata = unshorten(data['short_code'], secret_key='123') + assert udata['long_url'] == data['long_url'] def test_short_url_unshorten(self): data = shorten(self.long_url) udata = unshorten(data['short_code']) assert isinstance(udata, dict) assert udata['long_url'] == self.long_url + + def test_http_non_http_url(self): + urls = [ + 'http://example.com', + 'example.com' + ] + response = set(shorten(u)['short_code'] for u in urls) + assert len(response) == 1 diff --git a/src/pygmy/utilities/urls.py b/src/pygmy/utilities/urls.py index 8b0c727..18fd769 100644 --- a/src/pygmy/utilities/urls.py +++ b/src/pygmy/utilities/urls.py @@ -26,7 +26,10 @@ def validate_url(url): def make_short_url(short_path): - short_url = urljoin(config.pygmy['url'], short_path) + short_url = urljoin( + config.pygmy['short_url_schema'], + config.pygmy['short_url'], + short_path) return short_url diff --git a/src/pygmy/validator/link.py b/src/pygmy/validator/link.py index 71d782c..2811f83 100644 --- a/src/pygmy/validator/link.py +++ b/src/pygmy/validator/link.py @@ -36,7 +36,7 @@ class LinkSchema(Schema): short_code = fields.Str(required=False, allow_none=True, validate=is_valid_custom_code_or_secret) - short_url = fields.Method('short_url_path', load_only=True) + short_url = fields.Method('short_url_path', dump_only=True) description = fields.Str(required=False, allow_none=True) secret_key = fields.Str(required=False, allow_none=True, diff --git a/src/pyui/restclient/pygmy.py b/src/pyui/restclient/pygmy.py index 32618a2..9c48a7f 100644 --- a/src/pyui/restclient/pygmy.py +++ b/src/pyui/restclient/pygmy.py @@ -222,7 +222,7 @@ def list_links(self, access_token): raise ObjectNotFound(resp_obj) links = resp_obj for link in links: - if link.get('short_url'): + if link.get('short_code'): link['short_url'] = self.HOSTNAME + '/' + link['short_code'] return links