Merge pull request #37 from l0rb/master

corpus upload
interrogator · Dec 11, 2019 · 4255499 · 4255499
2 parents fb300a9 + 648ce53
commit 4255499
Show file tree

Hide file tree

Showing 11 changed files with 213 additions and 11 deletions.
diff --git a/buzzword/templates/base.html b/buzzword/templates/base.html
@@ -13,7 +13,7 @@
   <body>
   {% block body %}
   {% include "nav.html" %}
-  <div class="container">
+  <div class="container" style="margin-top:56px"><!-- 56px is the height of the nav -->
     {% block content %}
     {% endblock %}
     </div>

diff --git a/buzzword/templates/explore/upload_corpus.html b/buzzword/templates/explore/upload_corpus.html
@@ -0,0 +1,20 @@
+{% extends "base.html" %}
+
+{% block title %}buzzword: upload corpus{% endblock %}
+
+
+{% block content %}
+
+<table class="table">
+<form method="POST" enctype="multipart/form-data">
+    {% csrf_token %}
+    {{ form.as_table }}
+    <tr>
+        <td>
+            <button type="submit" class="btn btn-primary">upload corpus</button>
+        </td>
+    </tr>
+</form>
+</table>
+
+{% endblock %}
diff --git a/buzzword/templates/nav.html b/buzzword/templates/nav.html
@@ -14,6 +14,9 @@
             <li class="nav-item active">
                 <a class="nav-link" href="/">Home <span class="sr-only">(current)</span></a>
             </li>
+            <li class="nav-item">
+                <a class="nav-link" href="{% url "explore:upload_corpus" %}">Upload</a>
+            </li>
             <li class="nav-item">
                 <a class="nav-link" href="xxx">Members View</a>
             </li>

diff --git a/explore/migrations/0005_auto_20191210_1410.py b/explore/migrations/0005_auto_20191210_1410.py
@@ -0,0 +1,30 @@
+# Generated by Django 3.0 on 2019-12-10 14:10
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('explore', '0004_auto_20191205_0900'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Language',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('name', models.CharField(max_length=255)),
+            ],
+        ),
+        migrations.AlterModelOptions(
+            name='corpus',
+            options={'verbose_name_plural': 'Corpora'},
+        ),
+        migrations.AddField(
+            model_name='corpus',
+            name='language_link',
+            field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to='explore.Language'),
+        ),
+    ]
diff --git a/explore/migrations/0006_transfer_language.py b/explore/migrations/0006_transfer_language.py
@@ -0,0 +1,31 @@
+# Generated by Django 3.0 on 2019-12-10 14:10
+
+from django.db import migrations
+
+def transfer_lang(apps, schema_editor):
+    Corpus = apps.get_model("explore", "Corpus")
+    Language = apps.get_model("explore", "Language")
+    for corpus in Corpus.objects.all():
+        language, _ = Language.objects.get_or_create(name=corpus.language)
+        corpus.language_link = language
+        corpus.save()
+        language.save()
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("explore", "0005_auto_20191210_1410"),
+    ]
+
+    operations = [
+        migrations.RunPython(transfer_lang),
+        migrations.RemoveField(
+            model_name="corpus",
+            name="language"
+        ),
+        migrations.RenameField(
+            model_name="corpus",
+            old_name="language_link",
+            new_name="language",
+        ),
+    ]
diff --git a/explore/migrations/0007_auto_20191211_1501.py b/explore/migrations/0007_auto_20191211_1501.py
@@ -0,0 +1,38 @@
+# Generated by Django 3.0 on 2019-12-11 15:01
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('explore', '0006_transfer_language'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='corpus',
+            name='date',
+            field=models.DateField(blank=True, null=True),
+        ),
+        migrations.AlterField(
+            model_name='corpus',
+            name='desc',
+            field=models.TextField(blank=True, default=''),
+        ),
+        migrations.AlterField(
+            model_name='corpus',
+            name='initial_query',
+            field=models.TextField(blank=True, null=True),
+        ),
+        migrations.AlterField(
+            model_name='corpus',
+            name='initial_table',
+            field=models.TextField(blank=True, null=True),
+        ),
+        migrations.AlterField(
+            model_name='corpus',
+            name='url',
+            field=models.URLField(blank=True, max_length=255, null=True),
+        ),
+    ]
diff --git a/explore/migrations/0008_corpus_parsed.py b/explore/migrations/0008_corpus_parsed.py
@@ -0,0 +1,18 @@
+# Generated by Django 3.0 on 2019-12-11 15:15
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('explore', '0007_auto_20191211_1501'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='corpus',
+            name='parsed',
+            field=models.BooleanField(default=False),
+        ),
+    ]
diff --git a/explore/models.py b/explore/models.py
@@ -3,12 +3,21 @@
 from django.db import models
 
 from explorer.parts.strings import _slug_from_name
+
+# difference between blank and null:    
+# https://docs.djangoproject.com/en/3.0/ref/models/fields/#blank
 
 def _string_or_none(jsonfield):
     if not jsonfield:
         return
     return json.dumps(jsonfield)
 
+class Language(models.Model):
+    name = models.CharField(max_length=255)
+
+    def __str__(self):
+        return self.name
+
 class Corpus(models.Model):
     class Meta:
         verbose_name_plural = "Corpora" # can't tolerate "Corpuss"
@@ -17,18 +26,19 @@ class Meta:
         max_length=255, unique=True
     )  # this can't be null because a name needs to exist
     name = models.CharField(max_length=255)
-    language = models.CharField(max_length=255)  # probably turn this into a model later
+    language = models.ForeignKey(Language, on_delete=models.SET_NULL, null=True)
     path = models.TextField()
-    desc = models.TextField(default="")
+    desc = models.TextField(default="", blank=True)
     length = models.BigIntegerField(null=True)
     add_governor = models.BooleanField(null=True)
     # drop_columns = array -> needs to be relation
     disabled = models.BooleanField(default=False)
-    date = models.DateField(null=True)
+    date = models.DateField(null=True, blank=True)
     load = models.BooleanField(default=True)
-    url = models.URLField(max_length=255, null=True)
-    initial_query = models.TextField(null=True)
-    initial_table = models.TextField(null=True)
+    url = models.URLField(max_length=255, null=True, blank=True)
+    initial_query = models.TextField(null=True, blank=True)
+    initial_table = models.TextField(null=True, blank=True)
+    parsed = models.BooleanField(default=False)
 
     @classmethod
     def from_json(cls, jsondata, corpus_name):
@@ -74,6 +84,7 @@ def from_json(cls, jsondata, corpus_name):
             url=url,
             initial_query=initial_query,
             initial_table=initial_table,
+            parsed=True, # assuming all files that are provided this way are already parsed
         )
         corp.save()
 

diff --git a/explore/urls.py b/explore/urls.py
@@ -2,7 +2,8 @@
 
 from . import views
 
-app_name = 'explore'
+app_name = "explore"
 urlpatterns = [
-    path('<str:slug>/', views.explore, name="explore"),
+    path("<str:slug>/", views.explore, name="explore"),
+    path("upload_corpus", views.upload_corpus, name="upload_corpus"),
 ]
diff --git a/explore/views.py b/explore/views.py
@@ -1,7 +1,57 @@
-from django.shortcuts import render
+from django.shortcuts import render, redirect
 from explorer.parts.main import app, load_layout
+from django import forms
+from explore.models import Corpus
+from explorer.parts.strings import _slug_from_name
 
+def _make_path(slug):
+    return f"storage/{slug}"
+
+def _store_corpus_file(corpus_file, slug):
+    """
+        store the corpus file sent by the user
+        in the local storage. fails if a corpus with
+        the same slug already exists.
+    """
+    path = _make_path(slug)
+    with open(_make_path(slug), "xb") as storage:
+        for chunk in corpus_file.chunks():
+            storage.write(chunk)
+    return path
+
+def _start_parse_corpus_job(corpus):
+    # todo: implement this function
+    pass
 
 def explore(request, slug):
     app = load_layout(slug)
-    return render(request, 'explore/explore.html')
+    return render(request, "explore/explore.html")
+
+def upload_corpus(request):
+    class UploadCorpusForm(forms.ModelForm):
+        class Meta:
+            model = Corpus
+            fields = ["name", "desc", "language", "date", "url"]
+        corpus_file = forms.FileField()
+
+    if request.method == "POST":
+        form = UploadCorpusForm(request.POST, request.FILES)
+        if form.is_valid():
+            slug = _slug_from_name(form.cleaned_data["name"])
+            try:
+                path = _store_corpus_file(request.FILES["corpus_file"], slug)
+            except:
+                # TODO: was not able to store corpus file. possibly duplicate slug. handle gracefully
+                raise
+            corpus = form.save(commit=False)
+            corpus.slug = slug
+            corpus.path = path
+            corpus.save()
+            _start_parse_corpus_job(corpus)
+            return redirect("/")
+    else:
+        form = UploadCorpusForm()
+
+    return render(request, "explore/upload_corpus.html", {
+        "form": form,
+    })
diff --git a/storage/.gitkeep b/storage/.gitkeep