Manual verification #5, Closes #121 Redundancy limits

README.md

@@ -34,6 +34,33 @@ Setting `MOONSHEEP['DEV_ROTATE_TASKS'] = True` has the following effects:
    If you have subtasks then most likely saving will fail because of some missing models.
    It is recommended to write project tests covering saving data and creating dependant tasks.
 
+## Settings
+
+Can be overriden in your project's settings as follows:
+```python
+from moonsheep.settings import *  # NOQA
+
+MOONSHEEP.update({
+    'DEV_ROTATE_TASKS': False,
+    'MIN_ENTRIES_TO_CROSSCHECK': 1,
+})
+```
+
+Apart from `MOONSHEEP`-specific settings, the import will also bring in defaults for `REST_FRAMEWORK`.
+
+Settings:
+- [`DEV_ROTATE_TASKS`](#DEV_ROTATE_TASKS) - support for developing tasks, check details above
+- `MIN_ENTRIES_TO_CROSSCHECK` - number of entries for a task needed to run cross-checking (defaults to 3)
+- `MIN_ENTRIES_TO_MARK_DIRTY` - number of entries for a task at the point where when if crosschecking fails 
+   then the task will be marked as `dirty`. It won't be server anymore to users and will be brought
+   for a moderator attention. (defaults to 4)
+- `USER_AUTHENTICATION` - methods to handle user logins, see [below](#Users_&_authentication) for details
+  - `nickname` - generate pseudonymous nicknames, so you can show statistics to users, 
+  but don't have to keep their peronal data
+
+    you can set `'FAKER_LOCALE': 'it_IT'` to change the language of generated names
+  - `anonymous` - users don't need to login; cookies are sent anyhow to trace entries by each user
+
 ## Importing documents
 
 Configuring backend:

moonsheep/json_field.py

@@ -0,0 +1,85 @@
+import json
+from json import JSONDecodeError
+
+from django.core.serializers.json import DjangoJSONEncoder
+from django.db import models
+from django.contrib.postgres.fields import JSONField as PostgresJSONField
+
+
+class JSONField(PostgresJSONField):
+    def __init__(self, encoder=DjangoJSONEncoder, **options):
+        super().__init__(encoder=encoder, **options)
+
+    def from_db_value(self, value, expression, connection):
+        if isinstance(value, str):
+            # TODO test datetime saving `object_hook`
+            # TODO most likely we would have to add some guessing for the types encoded by DjangoJSONEncoder
+            try:
+                return json.loads(value)
+            except JSONDecodeError as e:
+                # TODO this should be handled by validation
+                e.msg += ': ' + value[max(0, e.pos - 10):e.pos] + '>here>' + value[e.pos:min(e.pos + 10, len(value))]
+
+        return value
+
+
+# TODO delete when squashing migrations
+class JSONTextField(models.TextField):
+    """
+    JSONField is a generic textfield that neatly serializes/unserializes
+    JSON objects seamlessly.
+    Django snippet #1478, Credit: https://stackoverflow.com/a/41839021/803174
+
+    example:
+        class Page(models.Model):
+            data = JSONField(blank=True, null=True)
+
+
+        page = Page.objects.get(pk=5)
+        page.data = {'title': 'test', 'type': 3}
+        page.save()
+    """
+
+    # TODO in Django Admin the value is shown as serialized dict with single quotes, it should have been json
+    def to_python(self, value):
+        if value == "":
+            return None
+
+        if isinstance(value, str):
+            # TODO test datetime saving `object_hook`
+            # TODO most likely we would have to add some guessing for the types encoded by DjangoJSONEncoder
+            try:
+                return json.loads(value)
+            except JSONDecodeError as e:
+                # TODO this should be handled by validation
+                e.msg += ': ' + value[max(0, e.pos - 10):e.pos] + '>here>' + value[e.pos:min(e.pos + 10, len(value))]
+
+        return value
+
+    def to_json(self, value):
+        if value == "":
+            return None
+        if isinstance(value, dict):
+            value = json.dumps(value, cls=DjangoJSONEncoder)
+        return value
+
+    def from_db_value(self, value, *args):
+        return self.to_python(value)
+
+    def value_to_string(self, obj):
+        # Used by seralization framework: manage.py dumpdata
+        value = self.value_from_object(obj)
+
+        return str(self.to_json(value))
+
+    def get_prep_value(self, value):
+        # Use by lookups (equals to, etc.)
+        value = super().get_prep_value(value)
+
+        if isinstance(value, dict):
+            return self.to_json(value)
+
+        return self.to_python(value)
+
+    # def get_db_prep_save(self, value, *args, **kwargs):
+    #     return self.to_json(value)

moonsheep/migrations/0005_auto_20191114_1640.py

@@ -0,0 +1,37 @@
+# Generated by Django 2.2.5 on 2019-11-14 16:40
+
+import django.core.serializers.json
+from django.db import migrations
+import moonsheep.json_field
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('moonsheep', '0004_auto_20191106_1758'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='entry',
+            name='data2',
+            field=moonsheep.json_field.JSONField(default={}, encoder=django.core.serializers.json.DjangoJSONEncoder),
+            preserve_default=False,
+        ),
+        migrations.AddField(
+            model_name='task',
+            name='params2',
+            field=moonsheep.json_field.JSONField(default={}, encoder=django.core.serializers.json.DjangoJSONEncoder),
+            preserve_default=False,
+        ),
+        migrations.AlterField(
+            model_name='entry',
+            name='data',
+            field=moonsheep.json_field.JSONTextField(),
+        ),
+        migrations.AlterField(
+            model_name='task',
+            name='params',
+            field=moonsheep.json_field.JSONTextField(blank=True),
+        ),
+    ]

moonsheep/migrations/0006_auto_20191114_1642.py

@@ -0,0 +1,26 @@
+# Generated by Django 2.2.5 on 2019-11-14 16:42
+
+from django.db import migrations
+
+def copy_json_fields(apps, schema_editor):
+    # We can't import the Person model directly as it may be a newer
+    # version than this migration expects. We use the historical version.
+    Task = apps.get_model('moonsheep', 'Task')
+    for t in Task.objects.all():
+        t.params2 = t.params
+        t.save()
+
+    Entry = apps.get_model('moonsheep', 'Entry')
+    for e in Entry.objects.all():
+        e.data2 = e.data
+        e.save()
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('moonsheep', '0005_auto_20191114_1640'),
+    ]
+
+    operations = [
+        migrations.RunPython(copy_json_fields),
+    ]

moonsheep/migrations/0007_auto_20191114_1648.py

@@ -0,0 +1,31 @@
+# Generated by Django 2.2.5 on 2019-11-14 16:48
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('moonsheep', '0006_auto_20191114_1642'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='entry',
+            name='data',
+        ),
+        migrations.RemoveField(
+            model_name='task',
+            name='params',
+        ),
+        migrations.RenameField(
+            model_name='entry',
+            old_name='data2',
+            new_name='data'
+        ),
+        migrations.RenameField(
+            model_name='task',
+            old_name='params2',
+            new_name='params',
+        ),
+    ]

moonsheep/migrations/0008_auto_20191114_1653.py

@@ -0,0 +1,19 @@
+# Generated by Django 2.2.5 on 2019-11-14 16:53
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('moonsheep', '0007_auto_20191114_1648'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='task',
+            name='parent',
+            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='children', to='moonsheep.Task'),
+        ),
+    ]

moonsheep/models.py

@@ -1,14 +1,14 @@
-import json
 import random
 
 from django.contrib.auth.base_user import BaseUserManager
 from django.contrib.auth.models import AbstractUser
 from django.core import validators
-from django.core.serializers.json import DjangoJSONEncoder
 from django.db import models
 from django.utils.text import slugify
 from django.utils.translation import ugettext_lazy as _
 
+from moonsheep.json_field import JSONField
+
 
 def generate_password(bits=160):
     return "%x" % random.getrandbits(bits)
@@ -85,46 +85,6 @@ class User(AbstractUser):
     PSEUDONYMOUS_DOMAIN = "@pseudonymous.moonsheep.org"
 
 
-class JSONField(models.TextField):
-    """
-    JSONField is a generic textfield that neatly serializes/unserializes
-    JSON objects seamlessly.
-    Django snippet #1478, Credit: https://stackoverflow.com/a/41839021/803174
-
-    example:
-        class Page(models.Model):
-            data = JSONField(blank=True, null=True)
-
-
-        page = Page.objects.get(pk=5)
-        page.data = {'title': 'test', 'type': 3}
-        page.save()
-    """
-
-    def to_python(self, value):
-        if value == "":
-            return None
-
-        try:
-            if isinstance(value, str):
-                # TODO test datetime saving `object_hook`
-                # TODO most likely we would have to add some guessing for the types encoded by DjangoJSONEncoder
-                return json.loads(value)
-        except ValueError:
-            pass
-        return value
-
-    def from_db_value(self, value, *args):
-        return self.to_python(value)
-
-    def get_db_prep_save(self, value, *args, **kwargs):
-        if value == "":
-            return None
-        if isinstance(value, dict):
-            value = json.dumps(value, cls=DjangoJSONEncoder)
-        return value
-
-
 class Task(models.Model):
     """
     A specific Task that users will work on.
@@ -136,12 +96,13 @@ class Task(models.Model):
     type = models.CharField(verbose_name=_("Type"), max_length=255)  # , choices=[(t, t) for t in TASK_TYPES])
     """Full reference (with module) to task class name"""
 
-    params = JSONField(blank=True)
+    params = JSONField()
     """Params specifying the task, that will be passed to user"""
 
-    parent = models.ForeignKey('Task', models.CASCADE, null=True, related_name="children")
+    parent = models.ForeignKey('Task', models.CASCADE, null=True, blank=True, related_name="children")
     """Set if this task is a child of another"""
 
+    # TODO can we make it "dynamic"?
     doc_id = models.IntegerField()
     """Pointing to document_id being processed by this task"""
 

moonsheep/settings.py

@@ -9,6 +9,7 @@
 MOONSHEEP = {
     'DEV_ROTATE_TASKS': False,
     'MIN_ENTRIES_TO_CROSSCHECK': 3,
+    'MIN_ENTRIES_TO_MARK_DIRTY': 4,
     'FAKER_LOCALE': 'it_IT',  # See supported locales at https://github.com/joke2k/faker#localization
     'USER_AUTHENTICATION': 'nickname'  # available settings: 'nickname', 'anonymous', TODO email #60
 }

moonsheep/static/js/manual-verification.js

@@ -0,0 +1,18 @@
+function support_manual_verification(data) {
+    for (let [fld, values] of Object.entries(data)) {
+        let input = jQuery(`input[name="${fld}"]`)
+        if (values.length == 1) {
+            // set chosen value
+            input.val(values[0])
+            // color it
+            input.css('outline', 'lime solid 3px') // TODO set class
+
+        } else if (values.length > 0) {
+            // color it
+            input.css('outline', 'orangered solid 3px') // TODO set class
+            // set hover / popup
+            // TODO https://www.w3schools.com/css/css_tooltip.asp
+        }
+    }
+}
+

moonsheep/tasks.py

@@ -6,6 +6,7 @@
 from django.utils.decorators import classproperty
 
 from moonsheep import statistics
+from moonsheep.json_field import JSONField
 from moonsheep.models import Task, Entry
 from moonsheep.settings import MOONSHEEP
 from .mapper import klass_from_name
@@ -17,7 +18,8 @@
 
 # TODO rename to TaskType? add ABC class?
 class AbstractTask(object):
-    N_ANSWERS = 1
+    params: JSONField
+    instance: Task
 
     def __init__(self, instance: Task):
         self.instance = instance
@@ -95,6 +97,9 @@ def verify_and_save(self, task_id: int) -> bool:
 
                 statistics.update_total_progress(self.instance)
 
+            elif entries_count >= MOONSHEEP['MIN_ENTRIES_TO_MARK_DIRTY']:
+                self.instance.state = Task.DIRTY
+
         # Entry was added so we should update progress if it's not at 100 already
         if self.instance.own_progress != 100:
             self.instance.own_progress = 95 * (