output as zip file and get error count

openspending/admin/helpers.py

@@ -1,9 +1,15 @@
+import logging
 import tempfile
 import zipfile
 import io
+import os
 import requests
+import csv
+
+from flask import current_app, url_for
+
+log = logging.getLogger(__name__)
 
-from flask import current_app
 
 OUTPUT_TEXT = """
     %(label)s
@@ -13,6 +19,8 @@
 """
 
 
+#from openspending.admin.helpers import LoadReport;from openspending.model import Dataset;a=Dataset.all()[6];lr =LoadReport(a);lr.get_output()
+
 class LoadReport(object):
     def __init__(self, dataset=None):
         if not dataset:
@@ -25,22 +33,28 @@ def __init__(self, dataset=None):
 
         self._buildzf()
         self._write_basefile()
-        self._write_preloaddata()
+        #self._write_preloaddata()
+        self._write_logs()
+        self._write_loaded_data()
 
     def _buildzf(self):
         self.namedfile = tempfile.NamedTemporaryFile(mode='wb', delete=False)
-        self.zf = zipfile.ZipFile(f, "w")
+        self.zf = zipfile.ZipFile(self.namedfile, "w")
 
     def _write_basefile(self):
-        zf.writestr("datainfo.csv", OUTPUT_TEXT%dict(label=self.dataset.label))
+        self.zf.writestr("metadata.csv", OUTPUT_TEXT%dict(label=self.dataset.label))
 
     def _write_logs(self):
 
+        if not self.dataset.source.runs.first():
+            self.zf.writestr("errorlog.csv", "This dataset has not been loaded yet.")
+            return 
+
         datalogs = self.dataset.source.runs.first().records_as_json()
 
 
         if not len(datalogs):
-            zf.writestr("loadinglog.csv", "All is well")
+            self.zf.writestr("errorlog.csv", "No Errors to report")
             return
 
 
@@ -52,16 +66,17 @@ def _write_logs(self):
 
         for row in datalogs:
             dw.writerow(row)
-        self.zf.writestr("loadinglog.csv", outputfile.getvalue())
+        self.zf.writestr("errorlog.csv", outputfile.getvalue())
 
     def _write_preloaddata(self):
         preloadvalue = self.dataset.source.getORFile().getvalue()
         self.zf.writestr("preloadvalue.csv", preloadvalue)
 
 
     def _write_loaded_data(self):
-        current_app.test_client()
+        client = current_app.test_client()
 
+        url = "/api/slicer/cube/geometry/cubes_aggregate?cubes=" + self.dataset.name + "&drilldown=geometry__time|geometry__country_level0@name&format=csv"
 
         # Fill in your details here to be posted to the login form.
         LOCKDOWN_FORCE = current_app.config.get("LOCKDOWNUSER", False)
@@ -73,59 +88,35 @@ def _write_loaded_data(self):
                 'password': LOCKDOWNPASSWORD
             }
 
-            # Use 'with' to ensure the session context is closed after use.
-            with requests.Session() as s:
-                try:
-                    p = s.post('http://localhost:5000/lockdown', data=payload)
+            client.post(url_for('home.lockdown'), data=payload)
 
-                    # An authorised request.
-                    postloadvalue = s.get(url).content
-                except Exception, e:
-                    log.warn("could not get authorized postload value " + str(e))
-        else:
-
-            try:
-                postloadvalue = requests.get(url).content
-            except Exception, e:
-                log.warn("Could Not find post load content for " + dataset.name)
+        print client
+        try:
+            postloadvalue = client.get(url).data
+        except Exception, e:
+            log.warn("Could Not find post load content for " + dataset.name)
 
         try:
-            zf.writestr(dataset.name + "/postloadvalue.csv", postloadvalue)
+            self.zf.writestr("postloadvalue.csv", postloadvalue)
         except Exception, e:
+            print e
             log.warn("could not write postload value")
 
-
-
-
-
-
-
-
-def output_logs(dataset=None):
-    """
-        zipfile
-         |
-          - dataset info
-          - log records 
-          - output
-    """
-
-    if not dataset:
-        return "No dataset"
-
-    f = tempfile.NamedTemporaryFile(mode='wb', delete=False)
-
-
-
-    zf = zipfile.ZipFile(f, "w")
-
-
-
-
-
-    #write openrefine output
-
-
-    url = "http://localhost:5000/api/slicer/cube/geometry/cubes_aggregate?cubes=" + dataset.name + "&drilldown=geometry__time|geometry__country_level0@name&format=csv"
+    def get_output(self):
+        #return the zip value as a string
+        self.zf.close()
+        self.namedfile.close()
+        returnstring = ""
+        with open(self.namedfile.name, 'rb') as f:
+            returnstring = f.read()
+        self.cleanup()
+        return returnstring
+
+    def cleanup(self):
+        try:
+            os.remove(self.namedfile.name)
+        except:
+            log.info("cannot remove temporary file")
+        #shutils.rm(self.namedfile.path
 
 

openspending/admin/routes.py

@@ -9,7 +9,7 @@
 from wtforms.validators import DataRequired, ValidationError
 from flask import flash, current_app
 import flask_admin as admin
-from flask_admin import form, AdminIndexView,expose
+from flask_admin import form, AdminIndexView,expose, BaseView
 
 from flask_admin.model.form import InlineFormAdmin
 from flask_admin.contrib.sqla.form import InlineModelConverter
@@ -22,6 +22,7 @@
 
 from openspending.model import Account
 from openspending.auth import require
+from openspending.admin.helpers import LoadReport
 from werkzeug.security import generate_password_hash
 
 from jinja2 import Markup
@@ -362,12 +363,21 @@ class QAListView(sqla.ModelView):
     # columns list Data source link to admin page, has data, source_url, run log with cleaned and source, date injested
     def is_accessible(self):
         return require.account.is_admin()
+    can_delete = False
+    can_create = False
+    can_edit= False
+
     column_formatters = dict(name=macro('render_qalist'),
-                            source_url=macro('render_sourceurl'))
+                            source_url=macro('render_sourceurl'),
+                            report_url = macro('render_report'),
+                            number_errors=macro('num_log_records'))
     #column_formatters = dict(dataset_admin_url=macro('render_price'))
-    column_list= ('name', 'source_url', )
+    column_list= ('name', 'source_url', 'report_url', 'number_errors',)
     list_template = 'adminsection/qalist.html'
 
+
+
+
 class IndexView(AdminIndexView):
 
     def is_accessible(self):
@@ -379,7 +389,10 @@ def index(self):
 
 def register_admin(app, db):
 
-    from openspending.model import Source, Dataset, DataOrg, MetadataOrg, Account, Run, SourceFile, LogRecord, Dataview, Feedback, Tags
+    from openspending.model import Source, Dataset, \
+                                    DataOrg, MetadataOrg, Account, \
+                                    Run, SourceFile, LogRecord, Dataview, \
+                                    Feedback, Tags
 
     # flaskadmin = admin.Admin(app,
     #                     name="FIND Admin", 

openspending/templates/adminsection/qalist.html

@@ -15,6 +15,21 @@
 
 {% endmacro %}
 
+{% macro render_report(model, column) %}
+    <a target='_blank' href='{{ url_for("findadmin.report", id=model.dataset.id) }}'>QA Report</a>
+
+{% endmacro %}
+
+{% macro num_log_records(model, column) %}
+    {% if model.dataset.source.runs.first() %}
+        {{ model.dataset.source.runs.first().records.count() }}
+    {% else %}
+        Error
+    {% endif %}
+
+{% endmacro %}
+
+
 {% macro render_sourceurl(model, column) %}
     {% if model.rawfile %}
         <a target="_blank" href="{{ model.rawfile.get_url() }}">Uploaded File</a>

openspending/views/admin.py

@@ -1,5 +1,5 @@
 import colander
-from flask import Blueprint, render_template, request, redirect
+from flask import Blueprint, render_template, request, redirect, Response
 from flask.ext.login import current_user
 from flask import current_app
 
@@ -8,6 +8,9 @@
 
 from wtforms import Form, TextField, PasswordField, validators
 
+from openspending.model import Dataset
+from openspending.admin.helpers import LoadReport
+
 
 
 blueprint = Blueprint('findadmin', __name__)
@@ -21,3 +24,19 @@ def dataloader():
     	return redirect("/login", code=302)
     return render_template('findadmin/index.html')
 
+
+
+
+
+@blueprint.route('/findadmin/report')
+def report():
+    dataset_id = request.args.get("id", None)
+    if not dataset_id:
+        raise
+    dataset = Dataset.by_id(dataset_id)
+    if not dataset:
+        raise
+    lr = LoadReport(dataset)
+    return Response(lr.get_output(),
+                mimetype='application/zip',
+                headers={'Content-Disposition':'attachment;filename=%s.zip'%dataset.name})