Merge pull request #2 from Dainius-P/develop

Develop

README.md

@@ -8,7 +8,8 @@ A dashboard specifically used for scrapyd servers to manipulate scrapy spiders (
 - [x] Tasks (add, remove, list)
 - [x] Authentification (Using the Django authentication system)
 - [ ] Scheduled tasks (add, remove, list)
-- [ ] Multi user support
+- [ ] View logs
+- [ ] Multiple independent user support
 - [ ] Task performance details
 
 ### Demo
@@ -42,8 +43,9 @@ urlpatterns = [
 
 ### Built With
 
-* [Django](https://www.djangoproject.com/) - The web framework used
-* [ScrapyD](https://github.com/scrapy/scrapyd) - A service daemon to run Scrapy spiders
+* [Django](https://www.djangoproject.com/) - The web framework used.
+* [ScrapyD](https://github.com/scrapy/scrapyd) - A service daemon to run Scrapy spiders.
+* [LogParser](https://github.com/my8100/logparser) - A tool for parsing Scrapy log files.
 
 ### Authors
 

scrapyd_dash/migrations/0001_initial.py

@@ -1,4 +1,4 @@
-# Generated by Django 2.2.2 on 2019-07-24 11:25
+# Generated by Django 2.2.2 on 2019-08-21 07:41
 
 import django.core.validators
 from django.db import migrations, models
@@ -13,29 +13,6 @@ class Migration(migrations.Migration):
     ]
 
     operations = [
-        migrations.CreateModel(
-            name='ScheduledTasks',
-            fields=[
-                ('id', models.PositiveIntegerField(primary_key=True, serialize=False)),
-                ('name', models.CharField(max_length=256, null=True)),
-                ('create_datetime', models.DateTimeField(auto_now_add=True)),
-                ('update_datetime', models.DateTimeField(auto_now=True)),
-                ('project', models.CharField(max_length=256)),
-                ('spider', models.CharField(max_length=256)),
-                ('year', models.PositiveIntegerField()),
-                ('month', models.PositiveIntegerField(validators=[django.core.validators.MaxValueValidator(12)])),
-                ('day', models.PositiveIntegerField(validators=[django.core.validators.MaxValueValidator(32)])),
-                ('week', models.PositiveIntegerField()),
-                ('day_of_week', models.PositiveIntegerField(validators=[django.core.validators.MaxValueValidator(7)])),
-                ('hour', models.PositiveIntegerField(validators=[django.core.validators.MaxValueValidator(24)])),
-                ('minute', models.PositiveIntegerField(validators=[django.core.validators.MaxValueValidator(60)])),
-                ('second', models.PositiveIntegerField(validators=[django.core.validators.MaxValueValidator(60)])),
-                ('deleted', models.BooleanField(default=False)),
-            ],
-            options={
-                'db_table': 'scrapyd_dash_scheduled_tasks',
-            },
-        ),
         migrations.CreateModel(
             name='ScrapydProject',
             fields=[
@@ -83,6 +60,7 @@ class Migration(migrations.Migration):
                 ('create_datetime', models.DateTimeField(auto_now_add=True)),
                 ('update_datetime', models.DateTimeField(auto_now=True)),
                 ('deleted', models.BooleanField(default=False)),
+                ('stopping', models.BooleanField(default=False)),
                 ('project', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='scrapyd_dash.ScrapydProject')),
                 ('server', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='scrapyd_dash.ScrapydServer')),
             ],
@@ -103,6 +81,31 @@ class Migration(migrations.Migration):
             name='server',
             field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='scrapyd_dash.ScrapydServer'),
         ),
+        migrations.CreateModel(
+            name='ScheduledTask',
+            fields=[
+                ('name', models.CharField(max_length=256, primary_key=True, serialize=False)),
+                ('create_datetime', models.DateTimeField(auto_now_add=True)),
+                ('update_datetime', models.DateTimeField(auto_now=True)),
+                ('spider', models.CharField(max_length=256)),
+                ('year', models.PositiveIntegerField(blank=True, null=True)),
+                ('month', models.PositiveIntegerField(blank=True, null=True, validators=[django.core.validators.MaxValueValidator(12)])),
+                ('day', models.PositiveIntegerField(blank=True, null=True, validators=[django.core.validators.MaxValueValidator(32)])),
+                ('week', models.PositiveIntegerField(blank=True, null=True)),
+                ('day_of_week', models.PositiveIntegerField(blank=True, null=True, validators=[django.core.validators.MaxValueValidator(7)])),
+                ('hour', models.PositiveIntegerField(blank=True, null=True, validators=[django.core.validators.MaxValueValidator(24)])),
+                ('minute', models.PositiveIntegerField(blank=True, null=True, validators=[django.core.validators.MaxValueValidator(60)])),
+                ('last_run', models.DateTimeField(blank=True, null=True)),
+                ('next_run', models.DateTimeField(blank=True, null=True)),
+                ('project', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='scrapyd_dash.ScrapydProject')),
+                ('server', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='scrapyd_dash.ScrapydServer')),
+                ('tasks', models.ManyToManyField(blank=True, to='scrapyd_dash.Task')),
+            ],
+            options={
+                'db_table': 'scrapyd_dash_scheduled_tasks',
+                'ordering': ['-create_datetime'],
+            },
+        ),
         migrations.AlterUniqueTogether(
             name='scrapydproject',
             unique_together={('server', 'name')},

scrapyd_dash/models.py

@@ -92,31 +92,47 @@ class Meta:
         ordering = ['-create_datetime']
         db_table = 'scrapyd_dash_tasks'
 
-class ScheduledTasks(models.Model):
-    id = models.PositiveIntegerField(primary_key=True)
-    name = models.CharField(max_length=256, null=True)
+class ScheduledTask(models.Model):
+    name = models.CharField(max_length=256, primary_key=True)
+    tasks = models.ManyToManyField(Task,
+                              blank=True)
+
     create_datetime = models.DateTimeField(auto_now_add=True)
     update_datetime = models.DateTimeField(auto_now=True)
 
-    project = models.CharField(max_length=256, null=False)
-    spider = models.CharField(max_length=256, null=False)
+    project = models.ForeignKey(ScrapydProject,
+                                on_delete=models.CASCADE,
+                                null=False,
+                                blank=False)
+    spider = models.CharField(max_length=256, null=False, blank=False)
+
+    server = models.ForeignKey(ScrapydServer,
+                               on_delete=models.CASCADE,
+                               null=False,
+                               blank=False)
 
-    year = models.PositiveIntegerField(null=False)
-    month = models.PositiveIntegerField(null=False,
+    year = models.PositiveIntegerField(null=True, blank=True)
+    month = models.PositiveIntegerField(null=True, blank=True,
                                  validators=[MaxValueValidator(12)])
-    day = models.PositiveIntegerField(null=False,
+    day = models.PositiveIntegerField(null=True, blank=True,
                                validators=[MaxValueValidator(32)])
-    week = models.PositiveIntegerField(null=False)
-    day_of_week = models.PositiveIntegerField(null=False,
+    week = models.PositiveIntegerField(null=True, blank=True)
+    day_of_week = models.PositiveIntegerField(null=True, blank=True,
                                        validators=[MaxValueValidator(7)])
-    hour = models.PositiveIntegerField(null=False,
+    hour = models.PositiveIntegerField(null=True, blank=True,
                                 validators=[MaxValueValidator(24)])
-    minute = models.PositiveIntegerField(null=False,
-                                  validators=[MaxValueValidator(60)])
-    second = models.PositiveIntegerField(null=False,
+    minute = models.PositiveIntegerField(null=True, blank=True,
                                   validators=[MaxValueValidator(60)])
 
-    deleted = models.BooleanField(null=False, default=False)
+    last_run = models.DateTimeField(null=True, blank=True)
+    next_run = models.DateTimeField(null=True, blank=True)
+
+    def print_last_run(self):
+        return self.last_run.strftime('%Y-%m-%d %H:%M')
+
+    def print_next_run(self):
+        return self.next_run.strftime('%Y-%m-%d %H:%M')
 
     class Meta:
+        ordering = ['-create_datetime']
         db_table = 'scrapyd_dash_scheduled_tasks'

scrapyd_dash/operations/scheduled_tasks.py

@@ -0,0 +1,70 @@
+from ..models import ScheduledTask, Task
+from .tasks_add import add_task
+from datetime import timedelta, datetime
+
+def generate_first_run(s_task, now):
+    if s_task.hour and s_task.hour != now.hour:
+        return 0
+    elif s_task.minute and s_task.minute != now.minute:
+        return 0
+    elif s_task.day_of_week and s_task.day_of_week != int(now.strftime('%w')):
+        return 0
+    elif s_task.week and s_task.week != int(now.strftime('%U')):
+        return 0
+    elif s_task.day and s_task.day != now.day:
+        return 0
+    elif s_task.month and s_task.month != now.month:
+        return 0
+    elif s_task.year and s_task.year != now.year:
+        return 0
+
+"""
+Check scheduled tasks
+"""
+
+def check_scheduled():
+    s_tasks = ScheduledTask.objects.all()
+    now = datetime.now()
+    print(now)
+
+    for s_task in s_tasks:
+        if not s_task.next_run and not generate_first_run:
+            continue
+
+        if s_task.next_run and now.replace(microsecond=0, second=0) != s_task.next_run:
+            continue
+
+        task_resp = add_task(s_task.project,
+                             s_task.spider,
+                             s_task.server)
+
+        s_task.last_run = now
+
+        """
+        if day of week is picked
+        """
+        if s_task.day_of_week:
+            day = now.day + 7
+        else:
+            day = now.day
+
+        s_task.next_run = datetime(now.year + 1 if s_task.month else now.year,
+                                   now.month + 1 if s_task.day else now.month,
+                                   now.day + 1 if s_task.hour else day,
+                                   now.hour + 1 if s_task.minute else now.hour,
+                                   now.minute,
+                                   0)
+
+
+        s_task.save()
+
+        task = Task.objects.create(
+            id=task_resp.get("jobid"),
+            name="scheduled_%s" % (s_task.name),
+            project=s_task.project,
+            spider=s_task.spider,
+            server=s_task.server
+        )
+
+        s_task.tasks.add(task)
+        s_task.save()

scrapyd_dash/operations/tasks_list.py

@@ -66,7 +66,6 @@ def save_tasks(tasks):
             Task.objects.update_or_create(
                 id=j.get("id"),
                 defaults={
-                    "name": j.get("name", "default"),
                     "status": task.get("status"),
                     "server": task.get("server"),
                     "project": task.get("project"),

scrapyd_dash/templates/base.html

@@ -17,16 +17,11 @@
     <script src="https://ajax.googleapis.com/ajax/libs/jquery/3.4.1/jquery.min.js"></script>
     <script type="text/javascript" src="{% static 'js/bootstrap-notify.min.js' %}"></script>
     <script type="text/javascript" src="{% static 'js/script.js' %}"></script>
-    <link rel="stylesheet" href="https://unpkg.com/bootstrap-table@1.15.3/dist/bootstrap-table.min.css">
 
     <link rel="stylesheet" href="{% static 'vendors/bootstrap/dist/css/bootstrap.min.css' %}">
     <link rel="stylesheet" href="{% static 'vendors/font-awesome/css/font-awesome.min.css' %}">
-
-
     <link rel="stylesheet" href="{% static 'css/style.css' %}">
-
     <link href='https://fonts.googleapis.com/css?family=Open+Sans:400,600,700,800' rel='stylesheet' type='text/css'>
-
 </head>
 
 <body>
@@ -59,6 +54,9 @@
                     <li>
                          <a href="{% url 'tasks' %}" > <i class="menu-icon fa fa-tasks"></i>Tasks</a>
                     </li>
+                    <li>
+                         <a href="{% url 'scheduled_tasks' %}" > <i class="menu-icon fa fa-calendar"></i>Scheduled Tasks</a>
+                    </li>
                     <li>
                         <h3 class="menu-title">Extras</h3>
                     </li>