Browse files

Merge pull request #11 from nyaruka/csv-import

Looks great! Reviewed by @nicpottier
  • Loading branch information...
2 parents 92b536c + 61b0619 commit 14839b37a5a5e9c3599afc32a301c7e7e7391aba @ericnewcomer ericnewcomer committed Apr 26, 2012
View
2 pip-requires.txt
@@ -1,4 +1,6 @@
django>=1.4
django-guardian>=1.0.2
django_compressor
+django-celery
pytz
+redis==2.4.9
View
11 smartmin/__init__.py
@@ -1 +1,12 @@
__version__ = '1.4.0'
+
+def class_from_string(class_name):
+ """
+ Used to load a class object dynamically by name
+ """
+ parts = class_name.split('.')
+ module = ".".join(parts[:-1])
+ m = __import__(module)
+ for comp in parts[1:]:
+ m = getattr(m, comp)
+ return m
View
0 smartmin/csv_imports/__init__.py
No changes.
View
39 smartmin/csv_imports/models.py
@@ -0,0 +1,39 @@
+import datetime
+from django.db import models, transaction
+from smartmin import class_from_string
+
+from smartmin.models import SmartModel
+from .tasks import csv_import
+
+class ImportTask(SmartModel):
+ csv_file = models.FileField(upload_to="csv_imports", verbose_name="Import file", help_text="A comma delimited file of records to import")
+ model_class = models.CharField(max_length=255, help_text="The model we are importing for")
+ import_log = models.TextField()
+ task_id = models.CharField(null=True, max_length=64)
+
+ def start(self):
+ self.log("Queued import at %s" % datetime.datetime.now())
+ result = csv_import.delay(self)
+ self.task_id = result.task_id
+ self.save()
+
+ def done(self):
+ if self.task_id:
+ result = csv_import.AsyncResult(self.task_id)
+ return result.ready()
+
+ def status(self):
+ status = "PENDING"
+ print self.task_id
+ if self.task_id:
+ result = csv_import.AsyncResult(self.task_id)
+ status = result.state
+ return status
+
+ def log(self, message):
+ self.import_log += "%s\n" % message
+ self.modified_on = datetime.datetime.now()
+ self.save()
+
+ def __unicode__(self):
+ return "%s Import" % class_from_string(self.model_class)._meta.verbose_name.title()
View
45 smartmin/csv_imports/tasks.py
@@ -0,0 +1,45 @@
+import StringIO
+from celery.task import task
+from datetime import datetime
+from smartmin import class_from_string
+
+@task(track_started=True)
+def csv_import(task): #pragma: no cover
+ from django.db import transaction
+
+ transaction.enter_transaction_management()
+ transaction.managed()
+
+ log = StringIO.StringIO()
+
+ try:
+ task.task_id = csv_import.request.id
+ task.log("Started import at %s" % datetime.now())
+ task.log("--------------------------------")
+ task.save()
+
+ transaction.commit()
+
+ model = class_from_string(task.model_class)
+ records = model.import_csv(task.csv_file.file, task.created_by, log)
+
+ task.log(log.getvalue())
+ task.log("Import finished at %s" % datetime.now())
+ task.log("%d record(s) added." % len(records))
+
+ transaction.commit()
+
+ except Exception as e:
+ import traceback
+ traceback.print_exc(e)
+
+ task.log("\nError: %s\n" % e)
+ task.log(log.getvalue())
+ transaction.commit()
+
+ raise e
+
+ finally:
+ transaction.leave_transaction_management()
+
+ return task
View
13 smartmin/csv_imports/tests.py
@@ -0,0 +1,13 @@
+"""
+This file demonstrates writing tests using the unittest module. These will pass
+when you run "manage.py test".
+
+Replace this with more appropriate tests for your application.
+"""
+
+from django.test import TestCase
+
+class ImportTest(TestCase):
+ def test_csv_import(self):
+ pass
+
View
3 smartmin/csv_imports/urls.py
@@ -0,0 +1,3 @@
+from smartmin.csv_imports.views import ImportTaskCRUDL
+
+urlpatterns = ImportTaskCRUDL().as_urlpatterns()
View
22 smartmin/csv_imports/views.py
@@ -0,0 +1,22 @@
+# Create your views here.
+from smartmin import class_from_string
+from smartmin.csv_imports.models import ImportTask
+from smartmin.views import SmartCRUDL, SmartListView, SmartReadView
+
+class ImportTaskCRUDL(SmartCRUDL):
+ model = ImportTask
+ actions = ('read', 'list')
+
+ class Read(SmartReadView):
+ def derive_refresh(self):
+ if self.object.status() in ["PENDING", "RUNNING", "STARTED"]:
+ return 2000
+ else:
+ return 0
+
+ class List(SmartListView):
+ fields = ('status', 'type', 'csv_file', 'created_on', 'created_by')
+ link_fields = ('csv_file',)
+
+ def get_type(self, obj):
+ return class_from_string(obj.model_class)._meta.verbose_name_plural.title()
View
53 smartmin/models.py
@@ -1,3 +1,5 @@
+import csv
+import traceback
from django.db import models
from django.contrib.auth.models import User
@@ -23,6 +25,57 @@ class SmartModel(models.Model):
class Meta:
abstract = True
+ @classmethod
+ def prepare_fields(cls, field_dict):
+ return field_dict
+
+ @classmethod
+ def create_instance(cls, field_dict):
+ return cls.objects.create(**field_dict)
+
+ @classmethod
+ def import_csv(cls, file, user, log=None):
+
+ reader = open(file.name, "rU")
+ dialect = csv.Sniffer().sniff(reader.read(1024))
+ reader.seek(0)
+ reader = csv.reader(reader, dialect)
+
+ # read in our header
+ line_number = 0
+
+ header = reader.next()
+ line_number += 1
+ while header is not None and len(header[0]) > 1 and header[0][0] == "#":
+ header = reader.next()
+ line_number += 1
+
+ # do some sanity checking to make sure they uploaded the right kind of file
+ if len(header) < 1:
+ raise Exception("Invalid header for import file")
+
+ records = []
+ for row in reader:
+ # make sure there are same number of fields
+ if len(row) != len(header):
+ raise Exception("Line %d: The number of fields for this row is incorrect. Expected %d but found %d." % (line_number, len(header), len(row)))
+
+ field_values = dict(zip(header, row))
+ field_values['created_by'] = user
+ field_values['modified_by'] = user
+ try:
+ field_values = cls.prepare_fields(field_values)
+ records.append(cls.create_instance(field_values))
+ except Exception as e:
+ if log:
+ traceback.print_exc(log)
+ raise Exception("Line %d: %s" % (line_number, str(e)))
+
+ line_number += 1
+
+ return records
+
+
class ActiveManager(models.Manager):
"""
A manager that only selects items which are still active.
View
BIN smartmin/static/img/smartmin/loading.gif
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
View
47 smartmin/templates/csv_imports/importtask_read.html
@@ -0,0 +1,47 @@
+{% extends "smartmin/read.html" %}
+
+{% block content %}
+{% block pjax %}
+<div id="pjax">
+<div class="row">
+ <div class="span10">
+ <table class="table table-striped">
+ <tbody>
+ <tr>
+ <td class="bold">Status</td>
+ <td>{{ object.status }}
+ {% if object.status == 'PENDING' or object.status == 'RUNNING' or object.status == 'STARTED' %}
+ <img class="pull-right" src="{{ STATIC_URL }}img/smartmin/loading.gif">
+ {% endif %}
+ </td>
+ </tr>
+ <tr>
+ <td class="bold">File</td>
+ <td>{{ object.csv_file }}</td>
+ </tr>
+ </tbody>
+ </table>
+ <pre>{{ object.import_log }}</pre>
+ </div>
+</div>
+
+</div>
+{% endblock %}
+{% endblock %}
+
+{% block extra-style %}
+<style>
+ td.bold {
+ font-weight: bold;
+ text-align: right;
+ }
+
+ div.buttons {
+ padding-bottom: 5px;
+ }
+
+ div.loading {
+ padding-top: 10px;
+ }
+</style>
+{% endblock %}
View
52 smartmin/views.py
@@ -22,6 +22,8 @@
from django.contrib.auth.models import User
import string
+from smartmin.csv_imports.models import ImportTask
+from smartmin.csv_imports.tasks import csv_import
import widgets
def smart_url(url, id=None):
@@ -54,13 +56,16 @@ class SmartView(object):
# set by our CRUDL
url_name = None
+ # if we are part of a CRUDL, we keep a reference to it here, set by CRUDL
+ crudl = None
+
def __init__(self, *args):
"""
There are a few variables we want to mantain in the instance, not the
class.
"""
self.extra_context = {}
- return super(SmartView, self).__init__(*args)
+ super(SmartView, self).__init__()
def derive_title(self):
"""
@@ -1151,6 +1156,27 @@ def derive_title(self):
else:
return self.title
+class SmartCSVImportView(SmartCreateView):
+ success_url = 'id@csv_imports.importtask_read'
+
+ fields = ('csv_file',)
+
+ def derive_title(self):
+ return "Import %s" % self.crudl.model._meta.verbose_name_plural.title()
+
+ def pre_save(self, obj):
+ obj = super(SmartCSVImportView, self).pre_save(obj)
+ obj.model_class = "%s.%s" % (self.crudl.model.__module__, self.crudl.model.__name__)
+ return obj
+
+ def post_save(self, task):
+ task = super(SmartCSVImportView, self).post_save(task)
+
+ # kick off our CSV import
+ task.start()
+
+ return task
+
class SmartCRUDL(object):
actions = ('create', 'read', 'update', 'delete', 'list')
model_name = None
@@ -1256,32 +1282,32 @@ def view_for_action(self, action):
# set permissions if appropriate
if self.permissions:
options['permission'] = self.permission_for_action(action)
-
+
if action == 'create':
view = type("%sCreateView" % self.model_name, (SmartCreateView,),
- options)
+ options)
elif action == 'read':
if 'update' in self.actions:
options['edit_button'] = True
view = type("%sReadView" % self.model_name, (SmartReadView,),
- options)
+ options)
elif action == 'update':
if 'delete' in self.actions:
options['delete_url'] = "id@%s.%s_delete" % (self.module_name, self.model_name.lower())
-
+
view = type("%sUpdateView" % self.model_name, (SmartUpdateView,),
- options)
+ options)
elif action == 'delete':
if 'list' in self.actions:
options['cancel_url'] = "@%s.%s_list" % (self.module_name, self.model_name.lower())
options['redirect_url'] = "@%s.%s_list" % (self.module_name, self.model_name.lower())
-
+
view = type("%sDeleteView" % self.model_name, (SmartDeleteView,),
- options)
+ options)
elif action == 'list':
if 'read' in self.actions:
@@ -1293,9 +1319,13 @@ def view_for_action(self, action):
if 'create' in self.actions:
options['add_button'] = True
-
+
view = type("%sListView" % self.model_name, (SmartListView,),
- options)
+ options)
+
+ elif action == 'csv_import':
+ options['model'] = ImportTask
+ view = type("%sCSVImportView" % self.model_name, (SmartCSVImportView,), options)
if not view:
# couldn't find a view? blow up
@@ -1308,6 +1338,8 @@ def view_for_action(self, action):
if not getattr(view, 'template_name', None):
view.template_name = self.template_for_action(action)
+ view.crudl = self
+
return view
def pattern_for_view(self, view, action):
View
7 test-runner/blog/models.py
@@ -12,7 +12,12 @@ class Post(SmartModel):
objects = models.Manager()
active = ActiveManager()
-
+
+ @classmethod
+ def pre_create_instance(cls, field_dict):
+ field_dict['body'] = "Body: %s" % field_dict['body']
+ return field_dict
+
def __unicode__(self):
return self.title
View
5 test-runner/blog/test_files/posts.csv
@@ -0,0 +1,5 @@
+title,body,order,tags
+"My first post","The body of my first post",0,"tag1 tag2"
+"My 2nd post","The body of my post",0,"tag1 tag2"
+"My 3rd post","The body of my post",0,"tag1 tag2"
+"My 4th post","The body of my post",0,"tag1 tag2"
View
2 test-runner/blog/views.py
@@ -35,7 +35,7 @@ class Create(SmartCreateView):
class PostCRUDL(SmartCRUDL):
model = Post
actions = ('create', 'read', 'update', 'delete', 'list', 'author',
- 'exclude', 'exclude2', 'readonly', 'readonly2', 'messages')
+ 'exclude', 'exclude2', 'readonly', 'readonly2', 'messages', 'csv_import')
class List(SmartListView):
fields = ('title', 'tags', 'created_on', 'created_by')
View
17 test-runner/settings.py
@@ -138,6 +138,10 @@
'django.contrib.admin',
# Uncomment the next line to enable admin documentation:
# 'django.contrib.admindocs',
+
+ 'smartmin.csv_imports',
+
+ 'djcelery',
)
# A sample logging configuration. The only tangible logging
@@ -203,3 +207,16 @@
TEST_RUNNER = 'django_nose.NoseTestSuiteRunner'
+
+#-----------------------------------------------------------------------------------
+# Async tasks with django-celery
+#-----------------------------------------------------------------------------------
+import djcelery
+djcelery.setup_loader()
+
+CELERY_RESULT_BACKEND = 'database'
+
+BROKER_BACKEND = 'redis'
+BROKER_HOST = 'localhost'
+BROKER_PORT = 6379
+BROKER_VHOST = '4'
View
3 test-runner/urls.py
@@ -8,8 +8,9 @@
# Examples:
# url(r'^$', 'proj.views.home', name='home'),
- url(r'^users/', include('smartmin.users.urls')),
+ url(r'^users/', include('smartmin.users.urls')),
url(r'^blog/', include('blog.urls')),
+ url(r'^csv_imports/', include('smartmin.csv_imports.urls')),
# Uncomment the admin/doc line below to enable admin documentation:
# url(r'^admin/doc/', include('django.contrib.admindocs.urls')),

0 comments on commit 14839b3

Please sign in to comment.