Data Digest

django_th/management/commands/send_digest.py

@@ -0,0 +1,44 @@
+#!/usr/bin/env python
+# coding: utf-8
+from __future__ import unicode_literals
+import arrow
+
+from django.conf import settings
+from django.core.management.base import BaseCommand
+from django.core.mail import send_mail
+from django.template.loader import render_to_string
+
+from django_th.models import Digest
+
+
+class Command(BaseCommand):
+
+    help = 'Trigger all data from cache in version 2'
+
+    def handle(self, *args, **options):
+        """
+            get all the digest data to send to each user
+        """
+        now = arrow.utcnow().to(settings.TIME_ZONE)
+        now = now.date()
+
+        digest = Digest.objects.filter(date_end=str(now)).order_by('user',
+                                                                   'date_end')
+        users = digest.distinct('user')
+
+        subject = 'Your digester'
+
+        msg_plain = render_to_string('digest/email.txt',
+                                     {'digest': digest,
+                                      'subject': subject})
+        msg_html = render_to_string('digest/email.html',
+                                    {'digest': digest,
+                                     'subject': subject})
+        message = msg_plain
+        from_email = settings.ADMINS
+        recipient_list = ()
+        for user in users:
+            recipient_list += (user.user.email,)
+
+        send_mail(subject, message, from_email, recipient_list,
+                  html_message=msg_html)

django_th/migrations/0011_digest.py

@@ -0,0 +1,37 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.5 on 2017-09-16 08:35
+from __future__ import unicode_literals
+
+from django.conf import settings
+from django.db import migrations, models
+import django.db.models.deletion
+import uuid
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+        ('django_th', '0010_userservice_token'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='userservice',
+            name='duration',
+            field=models.CharField(choices=[('d', 'Day'), ('w', 'Week'), ('m', 'Month'), ('n', 'None')], default='n',
+                                   max_length=1),
+        ),
+        migrations.CreateModel(
+            name='Digest',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('title', models.CharField(max_length=600)),
+                ('link', models.URLField()),
+                ('duration', models.CharField(max_length=1)),
+                ('date_end', models.DateField()),
+                ('provider', models.CharField(max_length=40)),
+                ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
+            ],
+        ),
+    ]

django_th/models/__init__.py

@@ -1,13 +1,18 @@
 # coding: utf-8
+import arrow
+
 from django.conf import settings
 from django.contrib.auth.models import User
+from django.core.exceptions import ObjectDoesNotExist
 from django.db import models
 from django.db.models.signals import post_save
-from django.core.exceptions import ObjectDoesNotExist
+from django.dispatch import receiver
 from django.utils.timezone import now
 from django.utils.translation import ugettext_lazy as _
+
 from logging import getLogger
 
+from django_th.signals import digest_event
 from django_th.tools import warn_user_and_admin
 
 logger = getLogger('django_th.trigger_happy')
@@ -46,6 +51,17 @@ class UserService(models.Model):
     """
         UserService a model to link service and user
     """
+    DAY = 'd'
+    WEEK = 'w'
+    MONTH = 'm'
+    NONE = 'n'
+    DURATION = (
+        (DAY, _('Day')),
+        (WEEK, _('Week')),
+        (MONTH, _('Month')),
+        (NONE, _('None'))
+    )
+
     user = models.ForeignKey(User)
     token = models.CharField(max_length=255, blank=True)
     name = models.ForeignKey(
@@ -59,6 +75,7 @@ class UserService(models.Model):
         _('client id'), max_length=255, default='', blank=True)
     client_secret = models.CharField(
         _('client secret'), max_length=255, default='', blank=True)
+    duration = models.CharField(max_length=1, choices=DURATION, default=NONE)
 
     def show(self):
         """
@@ -105,6 +122,32 @@ def __str__(self):
                                       self.description)
 
 
+class Digest(models.Model):
+    """
+    Digest service to store the data from other service
+    """
+    user = models.ForeignKey(User)
+    title = models.CharField(max_length=600)
+    link = models.URLField()
+    duration = models.CharField(max_length=1)
+    date_end = models.DateField()
+    provider = models.CharField(max_length=40)
+
+    def show(self):
+        """
+
+        :return: string representing object
+        """
+        return "Digest %s - %s - %s - %s - %s - %s" % (
+            self.user, self.provider, self.title, self.link, self.duration,
+            self.date_end)
+
+    def __str__(self):
+        return "%s - %s - %s - %s - %s - %s" % (
+            self.user, self.provider, self.title, self.link, self.duration,
+            self.date_end)
+
+
 def update_result(trigger_id, msg, status):
     """
     :param trigger_id: trigger id
@@ -153,3 +196,40 @@ def th_create_user_profile(sender, instance, created, **kwargs):
 
 post_save.connect(th_create_user_profile, sender=User,
                   dispatch_uid="create_user_profile")
+
+
+@receiver(digest_event)
+def digest_save(sender, **kwargs):
+    """
+
+    :param sender:
+    :param kwargs:
+    :return:
+    """
+    # set the deadline of the publication of the digest data
+    duration = kwargs.get('duration')
+    if duration not in ('d', 'w', 'm'):
+        return
+    # get the current date
+    now = arrow.utcnow().to(settings.TIME_ZONE)
+
+    # set the deadline
+    if duration == 'd':
+        # set tomorrow
+        tomorrow = now.shift(days=+1)
+        date_end = tomorrow.date()  # noqa extrat the date part
+    elif duration == 'w':
+        # set next week
+        next_week = now.shift(weeks=+1)
+        date_end = next_week.date()
+    else:
+        # set next month
+        next_month = now.shift(months=+1)
+        date_end = next_month.date()
+
+    Digest.objects.create(user=kwargs.get('user'),
+                          title=kwargs.get('title'),
+                          link=kwargs.get('link'),
+                          duration=duration,
+                          date_end=str(date_end),
+                          provider=sender)

django_th/services/services.py

@@ -13,6 +13,7 @@
     from django.db.models.loading import get_model
 
 # django_th stuff
+from django_th import signals
 from django_th.models import UserService, ServicesActivated, TriggerService
 from django_th.publishing_limit import PublishingLimit
 from django_th.html_entities import HtmlEntities
@@ -308,3 +309,14 @@ def reset_failed(self, pk):
             consumer_failed=0, provider_failed=0)
         TriggerService.objects.filter(provider__name__id=pk).update(
             consumer_failed=0, provider_failed=0)
+
+    def send_signal(self, trigger_id, title, link=''):
+
+        t = TriggerService.objects.get(id=trigger_id)
+
+        if t.provider.duration != 'n':
+
+            kwargs = {'user': t.user, 'title': title,
+                      'link': link, 'duration': t.provider.duration}
+
+            signals.digest_event.send(sender=t.provider.name, **kwargs)

django_th/signals.py

@@ -0,0 +1,3 @@
+from django.dispatch import Signal
+
+digest_event = Signal(providing_args=["user", "title", "link", "duration"])

django_th/templates/digest/email.html

@@ -0,0 +1,6 @@
+<h1>Digest of your service </h1>
+<ul>
+{% for data in digest %}
+    <li><a href="{{ data.link }}">{{ data.title }}</a></li>
+{% endfor %}
+</ul>

django_th/templates/digest/email.txt

@@ -0,0 +1,5 @@
+Digest of your service
+
+{% for data in digest %}
+{{ data.title }} - {{ data.link }}
+{% endfor %}

django_th/templates/services/add_service.html

@@ -17,11 +17,20 @@ <h3>{% trans 'Creation of your service' %}</h3>
         <legend>{% trans 'Select the service you want to add' %}</legend>
         {{ form.non_field_errors }}
             <div class="form-group">
-                <label class="col-sm-2 control-label" for="id_service">{% trans 'Service' %}</label>
+                <div class="col-sm-12"><h4>Service</h4></div>
+                <label class="col-sm-2 control-label" for="id_service">{% trans 'Name' %}</label>
                 <div class="col-sm-6">{{ form.name }}</div>
                 <div class="col-sm-offset-2 col-sm-8">{{ form.name.errors }}</div>
             </div>
             <div class="form-group">
+                <div class="col-sm-12"><h4>Digest retention</h4></div>
+                <label class="col-sm-2 control-label" for="id_duration">{% trans 'Duration' %}</label>
+                <div class="col-sm-6">{{ form.duration }}</div>
+                <div class="col-sm-offset-2 col-sm-8">{{ form.duration.errors }}</div>
+                <div class="col-sm-12">{% trans 'Choose the duration of the retention for the digester of that service. After that duration, you will receive the complet bucket of the grabbed data to your email account' %}</div>
+            </div>
+            <div class="form-group">
+                <div class="col-sm-12"><h4>Authentication</h4></div>
                 <label class="col-sm-2 control-label" for="id_token">{% trans 'User Token' %}</label>
                 <div class="col-sm-6">{{ form.token }}</div>
                 <div class="col-sm-offset-2 col-sm-8">Actually, this field is only mandatory for the Instapush service, otherwise the other services will go to the website of each service to get a new token it self</div>

django_th/templates/services/edit_service.html

@@ -10,7 +10,17 @@ <h3>{% trans 'Modification of your service' %} {{ object.name|cut:'Service' }}<
         <fieldset>
         <legend>{% trans 'Set the parameters of your service' %}</legend>
         {{ form.non_field_errors }}
-            {%  if object.name = 'ServiceInstapush' %}
+            <div class="form-group">
+                <div class="col-sm-12"><h4>Digest retention</h4></div>
+                <label class="col-sm-2 control-label" for="id_duration">{% trans 'Duration' %}</label>
+                <div class="col-sm-6">{{ form.duration }}</div>
+                <div class="col-sm-offset-2 col-sm-8">{{ form.duration.errors }}</div>
+                <div class="col-sm-12">{% trans 'Choose the duration of the retention for the digester of that service. After that duration, you will receive the complet bucket of the grabbed data to your email account' %}</div>
+            </div>
+            <div class="form-group">
+                <div class="col-sm-12"><h4>Authentication</h4></div>
+            </div>
+            {%  if object.name == 'ServiceInstapush' %}
             <div class="form-group">
                 <label class="col-sm-2 control-label" for="id_token">{% trans 'User Token' %}</label>
                 <div class="col-sm-6">{{ form.token }}</div>

django_th/tests/test_models_and_services.py

@@ -38,7 +38,8 @@ def test_valid_form(self):
                     'username': 'johndoe',
                     'password': 'password',
                     'client_id': 'the_id',
-                    'client_secret': 'the_secret'
+                    'client_secret': 'the_secret',
+                    'duration': 'd',
                     }
             data2 = {'user': u.user, 'name': u.name, 'token': u.token,
                      'host': 'http://localhost/',

django_th/views_userservices.py

@@ -158,7 +158,7 @@ class UserServiceUpdateView(UserServiceMixin, UpdateView):
     """
     fields = ['username', 'password',
               'client_secret', 'client_id', 'host',
-              'token']
+              'token', 'duration']
     template_name = "services/edit_service.html"
 
     def get_success_url(self):

th_rss/my_rss.py

@@ -1,11 +1,11 @@
 # coding: utf-8
+import arrow
 import datetime
 import time
-import arrow
+from logging import getLogger
 
 # django classes
 from django.conf import settings
-from logging import getLogger
 from django.core.cache import caches
 
 # django_th classes
@@ -87,6 +87,9 @@ def read_data(self, **kwargs):
                now >= published >= date_triggered:
                 my_feeds.append(entry)
 
+                # digester
+                self.send_signal(trigger_id, entry.title, entry.link)
+
         cache.set('th_rss_' + str(trigger_id), my_feeds)
         cache.set('th_rss_uuid_{}'.format(rss.uuid), my_feeds)
         # return the data

th_todoist/my_todoist.py

@@ -76,9 +76,12 @@ def read_data(self, **kwargs):
                     for project in items.get('projects'):
                         if item.get('project_id') == project.get('id'):
                             project_name = project.get('name')
-                    data.append({'title': "From TodoIst Project {0}"
-                                          ":".format(project_name),
+                    title = 'From TodoIst Project {0}:'.format(project_name)
+                    data.append({'title': title,
                                  'content': item.get('content')})
+                    # digester
+                    self.send_signal(trigger_id, title, link='')
+
             cache.set('th_todoist_' + str(trigger_id), data)
         except AttributeError:
             logger.error(items)

th_twitter/my_twitter.py

@@ -177,6 +177,8 @@ def _get_tweets(twitter_obj, search):
                                               'content': s['text'],
                                               'link': url,
                                               'my_date': my_date})
+                            # digester
+                            self.send_signal(trigger_id, title, url)
                     cache.set('th_twitter_' + str(trigger_id), my_tweets)
                     Twitter.objects.filter(trigger_id=trigger_id).update(
                         since_id=since_id,

th_wallabag/my_wallabag.py

@@ -95,6 +95,8 @@ def read_data(self, **kwargs):
                 if created_at > date_triggered:
                     data.append({'title': d.get('title'),
                                  'content': d.get('content')})
+                    # digester
+                    self.send_signal(self.trigger_id, d.get('title'), link='')
             if len(data) > 0:
                 cache.set('th_wallabag_' + str(self.trigger_id), data)
         except Exception as e: