Allow users to import CSV as datasource

docs/faq.rst

@@ -45,6 +45,15 @@ visualizations.
 https://github.com/airbnb/superset/issues?q=label%3Aexample+is%3Aclosed
 
 
+Can I upload and visualize csv data?
+-------------------------------------
+
+Yes, using the ``Add CSV Table to Database`` button on the ``Sources -> Database``
+page. This brings up a wizard allowing you to configure the csv to dataframe
+conversion, and to select which database to add the csv table to. The table can
+then be loaded like any other on the ``Sources -> Tables`` page.
+
+
 Why are my queries timing out?
 ------------------------------
 

superset/config.py

@@ -152,6 +152,10 @@
 ENABLE_CORS = False
 CORS_OPTIONS = {}
 
+# Allowed format types for upload on Database view
+# TODO: Add processing of other spreadsheet formats (xls, xlsx etc)
+ALLOWED_EXTENSIONS = set(['csv'])
+
 
 # ---------------------------------------------------
 # List of viz_types not allowed in your environment

superset/forms.py

@@ -10,13 +10,21 @@
 import math
 
 from flask_babel import lazy_gettext as _
+
+from flask_appbuilder.fieldwidgets import BS3TextFieldWidget
+from flask_appbuilder.forms import DynamicForm
+
+from flask_wtf.file import FileField, FileAllowed, FileRequired
+
 from wtforms import (
     Form, SelectMultipleField, SelectField, TextField, TextAreaField,
-    BooleanField, IntegerField, HiddenField, DecimalField)
+    BooleanField, IntegerField, HiddenField, DecimalField, StringField)
 from wtforms import validators, widgets
+from wtforms.validators import DataRequired, Optional, NumberRange
 
 from superset import app
 
+
 config = app.config
 
 TIMESTAMP_CHOICES = [
@@ -1150,3 +1158,141 @@ def add_to_form(attrs):
                 'description': _("Time related form attributes"),
             },) + tuple(QueryForm.fieldsets)
         return QueryForm
+
+
+class CsvToDatabaseForm(DynamicForm):
+    # These are the fields exposed by Pandas read_csv()
+    csv_file = FileField(_('CSV File'),
+        description=_('Select a CSV file to be uploaded to a database.'),
+        validators=[FileRequired(), FileAllowed(['csv'], _('CSV Files Only!'))])
+    sep = StringField(_('Delimiter'),
+        description=_('Delimiter used by CSV file (for whitespace use \s+).'),
+        validators=[DataRequired()],
+        widget=BS3TextFieldWidget())
+    header = IntegerField(_('Header Row'),
+        description=_('Row containing the headers to use as column names (0 is '
+                    'first line of data). Leave empty if there is no header row.'),
+        validators=[Optional(), NumberRange(0, 1E+20)],
+        widget=BS3TextFieldWidget(),
+        filters=[lambda x: x or None])
+    names = StringField(_('Column Names'),
+        description=_('List of comma-separated column names to use if header row '
+                        'not specified above. Leave empty if header field populated.'),
+        validators=[Optional()],
+        widget=BS3TextFieldWidget(),
+        filters=[lambda x: x or None])
+    index_col = IntegerField(_('Index Column'),
+        description=_('Column to use as the row labels of the dataframe. ' 
+                        'Leave empty if no index column.'),
+        validators=[Optional(), NumberRange(0, 1E+20)],
+        widget=BS3TextFieldWidget(),
+        filters=[lambda x: x or None])
+    squeeze = BetterBooleanField(_('Squeeze'),
+        description=_('Parse the data as a series (specify this option if the '
+                        'data contains only one column.)'))
+    prefix = StringField(_('Prefix'),
+        description=_('Prefix to add to column numbers when no header '
+                        '(e.g. "X" for "X0, X1").'),
+        validators=[Optional()],
+        widget=BS3TextFieldWidget(),
+        filters=[lambda x: x or None])
+    mangle_dupe_cols = BetterBooleanField(_('Mangle Duplicate Columns'),
+        description=_('Specify duplicate columns as '
+                       '"X.0, X.1".'))
+    skipinitialspace = BetterBooleanField(_('Skip Initial Space'),
+        description=_('Skip spaces after delimiter.'))
+    skiprows = IntegerField(_('Skip Rows'),
+        description=_('Number of rows to skip at start of file.'),
+        validators=[Optional(), NumberRange(0, 1E+20)],
+        widget=BS3TextFieldWidget(),
+        filters=[lambda x: x or None])
+    nrows = IntegerField(_('Rows to Read'),
+        description=_('Number of rows of file to read.'),
+        validators=[Optional(), NumberRange(0, 1E+20)],
+        widget=BS3TextFieldWidget(),
+        filters=[lambda x: x or None])
+    skip_blank_lines = BetterBooleanField(_('Skip Blank Lines'),
+        description=_('Skip blank lines rather than '
+                       'interpreting them as NaN values.'))
+    parse_dates = BetterBooleanField(_('Parse Dates'),
+        description=_('Parse date values.'))
+    infer_datetime_format = BetterBooleanField(_('Infer Datetime Format'),
+        description=_('Use Pandas to interpret the '
+                         'datetime format '
+                         'automatically.'))
+    dayfirst = BetterBooleanField(_('Day First'),
+        description=_('Use DD/MM (European/International) date format.'))
+    thousands = StringField(_('Thousands Separator'),
+        description=_('Separator for values in thousands.'),
+        validators=[Optional()],
+        widget=BS3TextFieldWidget(),
+        filters=[lambda x: x or None])
+    decimal = StringField(_('Decimal Character'),
+        description=_('Character to interpret as decimal point.'),
+        validators=[Optional()],
+        widget=BS3TextFieldWidget(),
+        filters=[lambda x: x or '.'])
+    quotechar = StringField(_('Quote Character'),
+        description=_('Character used to denote the start and end of a '
+                        'quoted item.'),
+        validators=[Optional()],
+        widget=BS3TextFieldWidget(),
+        filters=[lambda x: x or "'"])
+    escapechar = StringField(_('Escape Character'),
+        description=_('Character used to escape a quoted item.'),
+        validators=[Optional()],
+        widget=BS3TextFieldWidget(),
+        filters=[lambda x: x or None])
+    comment = StringField(_('Comment Character'),
+        description=_('Character used to denote the start of a comment.'),
+        validators=[Optional()],
+        widget=BS3TextFieldWidget(),
+        filters=[lambda x: x or None])
+    encoding = StringField(_('Encoding'),
+        description=_('Encoding to use for UTF when reading/writing (e.g. "utf-8").'),
+        validators=[Optional()],
+        widget=BS3TextFieldWidget(),
+        filters=[lambda x: x or None])
+    error_bad_lines = BetterBooleanField(_('Error On Bad Lines'),
+        description=_('Error on bad lines (e.g. a line with '
+                        'too many commas). If false these bad '
+                        'lines will instead be dropped from '
+                        'the resulting dataframe.'))
+
+    # These are the fields exposed by Pandas .to_sql()
+    name = StringField(_('Table Name'),
+        description=_('Name of table to be created from csv data.'),
+        validators=[DataRequired()],
+        widget=BS3TextFieldWidget())
+    con = StringField(_('Database URI'),
+        description=_('URI of database in which to add above table.'),
+        validators=[DataRequired()],
+        widget=BS3TextFieldWidget())
+    schema = StringField(_('Schema'),
+        description=_('Specify a schema (if database flavour supports this).'),
+        validators=[Optional()],
+        widget=BS3TextFieldWidget(),
+        filters=[lambda x: x or None])
+    if_exists = SelectField(_('Table Exists'),
+        description=_('If table exists do one of the following: Fail (do '
+                     'nothing), Replace (drop and recreate table) or Append '
+                     '(insert data).'),
+        choices=[('fail', _('Fail')),
+                ('replace', _('Replace')),
+                ('append', _('Append'))],
+        validators=[DataRequired()])
+    index = BetterBooleanField(_('Dataframe Index'),
+        description=_('Write dataframe index as a column.'))
+    index_label = StringField(_('Column Label(s)'),
+        description=_('Column label for index column(s). If None is given '
+                      'and Dataframe Index is True, Index Names are used.'),
+        validators=[Optional()],
+        widget=BS3TextFieldWidget(),
+        filters=[lambda x: x or None])
+    chunksize = IntegerField(_('Chunksize'),
+        description=_('If empty, all rows will be written at once. Otherwise, '
+                       'rows will be written in batches of this '
+                       'many rows at a time.'),
+        validators=[Optional()],
+        widget=BS3TextFieldWidget(),
+        filters=[lambda x: x or None])

superset/templates/superset/widgets/list.html

@@ -0,0 +1,10 @@
+{% import "appbuilder/general/lib.html" as lib %}
+{% extends "appbuilder/general/widgets/list.html" %}
+
+    {% block list_header %}
+        {{ super() }}
+        <a href="/csvtodatabaseview/form" class="btn btn-sm btn-primary" id="uploadbutton"
+           title="Add a new database table based on a local CSV file" rel="tooltip" data-toggle="tooltip">
+           <i class="fa fa-table"> Add CSV Table to Database </i>
+        </a>
+    {% endblock %}