Merge pull request #23 from mkouhei/devel

release 0.3.0

.gitignore

@@ -6,3 +6,5 @@ shiori/static/twbs
 shiori/static/underscore
 .tox
 .coverage
+dummy_*.json
+celerybeat-schedule

.travis.yml

@@ -1,8 +1,10 @@
 language: python
 python: 2.7
-before_script:
-  - pip install python-coveralls
+env:
+  - TOX_ENV=py27
+install:
+  - pip install coveralls tox
 script:
-  - python setup.py test
+  - tox -e $TOX_ENV
 after_success:
-  - coveralls
+  - coveralls --verbose

docs/HISTORY.rst

@@ -1,6 +1,21 @@
 History
 -------
 
+0.3.0 (2014-04-03)
+^^^^^^^^^^^^^^^^^^
+
+* Change API path /v1 to /api
+* Change behavior of Category API
+* Change behavior of Tag API
+* Added FeedSubscription and CralingHistory models
+* Added API of FeedSubscription
+* Added view of subscribed feeds list and manage view
+* Added register bookmarks from subscribed feeds
+* Added feed parser and register of bookmark from subscribed feed by celery
+* Added list display at Django admin
+* Using fake name generator for test
+* Change using fixtures for User, Category, Tag
+
 0.2.7 (2014-03-15)
 ^^^^^^^^^^^^^^^^^^
 

docs/README.rst

@@ -28,9 +28,18 @@ Requirements
 * Python 2.7
 * Django (>= 1.6)
 * Django REST framework (>= 2.3.12)
+* shortuuid (== 0.3.2)
+  (this is work-around for the garbage (v0.4 not released) is left in the PyPI.)
 * django-shortuuidfield (>= 0.1.2)
+* django-jsonfield
 * python-openid (>= 2.2.5)
 * django_openid_auth (>= 0.5)
+* south
+* netaddr
+* lxml
+* defusedxml
+* requests
+* celery
 * jQuery (>= 1.7.2)
 * underscore.js (>= 1.5.2)
 * backbone.js (>= 1.1.0)
@@ -52,9 +61,11 @@ Install Debian packages that Shiori depends on
 Shiori depends on as following.::
 
   $ sudo apt-get install python-django python-djangorestframework \
-  python-django-shortuuidfield python-django-auth-openid \
-  python-django-notification \
-  libjs-jquery libjs-underscore libjs-json libjs-twitter-bootstrap
+  python-django-shortuuidfield python-django-auth-openid python-celery \
+  python-lxml python-defusedxml python-netaddr python-django-south \
+  libjs-jquery libjs-underscore libjs-json libjs-twitter-bootstrap \
+  libxml2-dev libxslt1-dev python-requests python-dev libpython2.7-dev \
+  python-django-jsonfield
 
 
 Update and rebuild libjs-backbone
@@ -141,6 +152,11 @@ Run server.::
   $ python /path/to/shiori/manage.py runserver
 
 
+Start Celery.::
+
+  $ celery -A shiori.core worker --beat -l info
+
+
 Development
 -----------
 
@@ -151,7 +167,7 @@ You copy pre-commit hook scripts after git clone.::
 Next install python 2.7 later and setuptools, pytest, pep8.
 Below way is for Debian GNU/Linux Sid system.::
 
-  $ sudo apt-get install python python-setuptools python-pytest pep8
+  $ sudo apt-get install python python-setuptools python-pytest pep8 libxml2-dev libxml2 python-dev
 
 Then checkout 'devel' branch for development, commit your changes.
 Before pull request, execute git rebase.

docs/TODO.rst

@@ -3,7 +3,6 @@ TODO
 
 * Add appending signature to bookmark and validation signature with GnuPG.
 * Add linking social web services with OAuth.
-* Add Feed parser and adding new item to bookmark automatically.
 * Add job scedular for feed parser and push message to social web services.
 * Add applying reST style description.
 * Add unit test code.

requirements.txt

@@ -2,6 +2,14 @@ pep8>=1.3
 pytest
 django >= 1.6
 djangorestframework
+shortuuid == 0.3.2
 django-shortuuidfield
+django-jsonfield
 python-openid
+south
+celery
+lxml
+defusedxml
+requests
+netaddr
 -e bzr+http://bazaar.launchpad.net/~andrewsomething/django-openid-auth/1252445#egg=django-openid-auth

setup.py

@@ -42,7 +42,6 @@ def run_tests(self):
     "Programming Language :: Python",
     "Programming Language :: Python :: 2.7",
     "Programming Language :: Python :: Implementation :: CPython",
-    "Programming Language :: Python :: Implementation :: PyPy",
     "Topic :: Internet",
     "Topic :: Internet :: WWW/HTTP",
     "Topic :: Internet :: WWW/HTTP :: WSGI :: Application",
@@ -57,12 +56,18 @@ def run_tests(self):
 requires = ['setuptools',
             'django',
             'djangorestframework',
+            'shortuuid == 0.3.2',
             'django-shortuuidfield',
+            'django-jsonfield',
             'python-openid',
-            'django-openid-auth']
+            'django-openid-auth',
+            'lxml',
+            'defusedxml',
+            'requests',
+            'netaddr']
 
 setup(name='shiori',
-      version='0.2.7',
+      version='0.3.0',
       description='bookmarking tool based on Web UI and JSON REST API',
       long_description=long_description,
       author='Kouhei Maeda',

shiori/api/permissions.py

@@ -12,3 +12,15 @@ def has_object_permission(self, request, view, obj):
             return True
 
         return obj.owner == request.user
+
+
+class IsAuthenticatedAndCreateReadOnly(permissions.BasePermission):
+    def has_permission(self, request, view):
+        if request.method in permissions.SAFE_METHODS:
+            return True
+        elif request.method == 'POST':
+            if request.user.is_authenticated():
+                return True
+        else:
+            if request.user.is_superuser:
+                return True

shiori/api/serializers.py

@@ -1,6 +1,10 @@
 # -*- coding: utf-8 -*-
 from rest_framework import serializers
-from shiori.bookmark.models import Category, Tag, Bookmark, BookmarkTag
+from shiori.bookmark.models import (Category,
+                                    Tag,
+                                    Bookmark,
+                                    BookmarkTag,
+                                    FeedSubscription)
 
 
 class CategorySerializer(serializers.ModelSerializer):
@@ -40,3 +44,14 @@ class BookmarkTagSerializer(serializers.ModelSerializer):
     class Meta:
         model = BookmarkTag
         fields = ('id', 'bookmark', 'tag')
+
+
+class FeedSubscriptionSerializer(serializers.ModelSerializer):
+    owner = serializers.Field(source='owner.username')
+    default_category = serializers.SlugRelatedField(many=False,
+                                                    slug_field='category')
+    category_id = serializers.Field(source='category.id')
+
+    class Meta:
+        model = FeedSubscription
+        fields = ('id', 'url', 'name', 'owner', 'default_category')

shiori/api/urls.py

@@ -8,6 +8,7 @@
 router.register(r'tags', views.TagViewSet)
 router.register(r'bookmarks', views.BookmarkViewSet)
 router.register(r'bookmark_tags', views.BookmarkTagViewSet)
+router.register(r'feed_subscription', views.FeedSubscriptionViewSet)
 
 urlpatterns = patterns('shiori.api.views',
                        url(r'^', include(router.urls)),

shiori/api/views.py

@@ -5,20 +5,27 @@
 from rest_framework import viewsets
 from rest_framework.response import Response
 from rest_framework.permissions import (IsAuthenticated,
-                                        IsAuthenticatedOrReadOnly)
+                                        IsAuthenticatedOrReadOnly,
+                                        IsAdminUser)
 from rest_framework.exceptions import NotAuthenticated
-from shiori.bookmark.models import Category, Tag, Bookmark, BookmarkTag
-from shiori.api.permissions import IsOwnerOrReadOnly
+from shiori.bookmark.models import (Category,
+                                    Tag,
+                                    Bookmark,
+                                    BookmarkTag,
+                                    FeedSubscription)
+from shiori.api.permissions import (IsOwnerOrReadOnly,
+                                    IsAuthenticatedAndCreateReadOnly)
 from shiori.api.serializers import (CategorySerializer,
                                     TagSerializer,
                                     BookmarkSerializer,
-                                    BookmarkTagSerializer)
+                                    BookmarkTagSerializer,
+                                    FeedSubscriptionSerializer)
 
 
 class CategoryViewSet(viewsets.ModelViewSet):
     queryset = Category.objects.all()
     serializer_class = CategorySerializer
-    permission_classes = (IsAuthenticatedOrReadOnly,)
+    permission_classes = (IsAuthenticatedAndCreateReadOnly,)
 
     def pre_save(self, obj):
         obj.category = escape(self.request.DATA.get('category'))
@@ -27,7 +34,7 @@ def pre_save(self, obj):
 class TagViewSet(viewsets.ModelViewSet):
     queryset = Tag.objects.all()
     serializer_class = TagSerializer
-    permission_classes = (IsAuthenticatedOrReadOnly,)
+    permission_classes = (IsAuthenticatedAndCreateReadOnly,)
     paginate_by = 50
 
     def pre_save(self, obj):
@@ -63,3 +70,18 @@ class BookmarkTagViewSet(viewsets.ModelViewSet):
     queryset = BookmarkTag.objects.all()
     serializer_class = BookmarkTagSerializer
     permission_classes = (IsAuthenticatedOrReadOnly,)
+
+
+class FeedSubscriptionViewSet(viewsets.ModelViewSet):
+    queryset = FeedSubscription.objects.all()
+    serializer_class = FeedSubscriptionSerializer
+    permission_classes = (IsAuthenticated,)
+
+    def pre_save(self, obj):
+        if isinstance(self.request.user, AnonymousUser):
+            raise NotAuthenticated
+        else:
+            obj.owner = self.request.user
+
+    def get_queryset(self):
+        return FeedSubscription.objects.filter(owner=self.request.user)

shiori/bookmark/admin.py

@@ -1,9 +1,36 @@
 # -*- coding: utf-8 -*-
 from django.contrib import admin
-from shiori.bookmark.models import Bookmark, Category, Tag, BookmarkTag
+from shiori.bookmark.models import (Bookmark,
+                                    Category,
+                                    Tag,
+                                    BookmarkTag,
+                                    FeedSubscription,
+                                    CrawlingHistory)
 
 
-admin.site.register(Bookmark)
+class BookmarkAdmin(admin.ModelAdmin):
+    list_display = ('title', 'category', 'registered_datetime',
+                    'owner', 'is_hide')
+
+
+class BookmarkTagAdmin(admin.ModelAdmin):
+    list_display = ('bookmark', 'tag')
+
+
+class FeedSubscriptionAdmin(admin.ModelAdmin):
+    list_display = ('name', 'owner', 'default_category')
+
+
+class CrawlingHistoryAdmin(admin.ModelAdmin):
+    list_display = ('get_name', 'update_datetime', 'result')
+
+    def get_name(self, obj):
+        return obj.feed.name
+
+
+admin.site.register(Bookmark, BookmarkAdmin)
 admin.site.register(Category)
 admin.site.register(Tag)
-admin.site.register(BookmarkTag)
+admin.site.register(BookmarkTag, BookmarkTagAdmin)
+admin.site.register(FeedSubscription, FeedSubscriptionAdmin)
+admin.site.register(CrawlingHistory, CrawlingHistoryAdmin)

shiori/bookmark/agents/feed.py

@@ -0,0 +1,55 @@
+# -*- coding: utf-8 -*-
+from django.contrib.auth.models import User
+from django.db import IntegrityError
+import json
+from shiori.bookmark.models import (Bookmark,
+                                    Category,
+                                    FeedSubscription,
+                                    CrawlingHistory)
+from shiori.bookmark.agents.feed_parser import FeedParser
+
+
+def register_bookmarks():
+
+    for feed in FeedSubscription.objects.all():
+        result = fetch_feeds(url=feed.url,
+                             category=feed.default_category,
+                             owner=feed.owner)
+        CrawlingHistory(feed=feed, result=json.dumps(result)).save()
+
+
+def fetch_feeds(**kwargs):
+    result = []
+    for entry in FeedParser(kwargs.get('url')).retrieve_items():
+        rc, msg = add_item(url=entry.get('link'),
+                           title=entry.get('title'),
+                           category=kwargs.get('category'),
+                           owner=kwargs.get('owner'))
+        if rc is False and "already registered:" in msg:
+            break
+        result.append(dict(link=entry.get('link'),
+                           rc=rc,
+                           msg=msg))
+    return result
+
+
+def add_categories(url):
+    [add_category(entry.get('category'))
+     for entry
+     in FeedParser(url).retrieve_items()
+     if entry.get('category')]
+
+
+def add_item(**kwargs):
+    category = Category.objects.get(category=kwargs.get('category'))
+    owner = User.objects.get(username=kwargs.get('owner'))
+
+    bookmark = Bookmark(url=kwargs.get('url'),
+                        title=kwargs.get('title'),
+                        category=category,
+                        owner=kwargs.get('owner'))
+    try:
+        bookmark.save()
+        return True, ''
+    except IntegrityError as error:
+        return False, "already registered: %s" % error