geonode/contrib/datatables/admin.py

@@ -0,0 +1,32 @@
+from django.contrib import admin
+from geonode.layers.models import Attribute
+from .models import DataTable, TableJoin, JoinTarget, JoinTargetFormatType, GeocodeType
+
+class DataTableAdmin(admin.ModelAdmin):
+    model = DataTable
+    list_display = (
+        'id',
+        'title',
+        'table_name',
+        'tablespace')
+    list_display_links = ('title',)
+
+
+class TableJoinAdmin(admin.ModelAdmin):
+    model = TableJoin 
+
+class JoinTargetAdmin(admin.ModelAdmin):
+    model = JoinTarget
+
+    def formfield_for_foreignkey(self, db_field, request, **kwargs):
+        if db_field.name == 'attribute':
+            kwargs["queryset"] = Attribute.objects.filter(
+                resource=request.GET.get('layer'))
+        return super(JoinTargetAdmin, self).formfield_for_foreignkey(
+            db_field, request, **kwargs)
+
+admin.site.register(DataTable, DataTableAdmin)
+admin.site.register(TableJoin, TableJoinAdmin)
+admin.site.register(JoinTarget, JoinTargetAdmin)
+admin.site.register(JoinTargetFormatType)
+admin.site.register(GeocodeType)

geonode/contrib/datatables/forms.py

@@ -0,0 +1,38 @@
+from django import forms
+from django.utils.translation import ugettext, ugettext_lazy as _
+
+"""
+Specify Data Table Delimiter Choices
+"""
+#   Delimiter choices
+#   For each delimiter, specify a "varname", "value", and "friendly_name"
+#
+DELIMITER_TYPE_INFO = ( dict(varname='COMMA', value=',', friendly_name='Comma Separated (.csv)'),
+                        dict(varname='TAB', value='\t', friendly_name='Tab Separated (.tab)'),
+                    )
+assert len(DELIMITER_TYPE_INFO) > 1, "DELIMITER_TYPE_INFO must have at least one value"
+
+DELIMITER_TYPE_CHOICES = [ (dt['varname'], dt['friendly_name']) for dt in DELIMITER_TYPE_INFO] # choices for Form
+DELIMITER_VALUE_LOOKUP = dict( (dt['varname'], dt['value']) for dt in DELIMITER_TYPE_INFO)  # value look up for form "clean"
+DEFAULT_DELIMITER = DELIMITER_TYPE_INFO[0]['varname']   # form default value
+
+class UploadDataTableForm(forms.Form):
+    title = forms.CharField(max_length=255)
+    uploaded_file = forms.FileField()
+    delimiter_type = forms.ChoiceField(choices=DELIMITER_TYPE_CHOICES, initial=DEFAULT_DELIMITER, required=False)
+    no_header_row = forms.BooleanField(initial=False, required=False) 
+
+    def clean_delimiter_type(self):
+        """
+        Return actual delimiter value.  e.g. form may have "COMMA" but return ","
+        """
+        delim = self.cleaned_data.get('delimiter_type', None)
+        if delim is None or len(delim) ==0:
+            # If no delim is specified, default to a comma
+            delim_value = DELIMITER_VALUE_LOOKUP.get(DEFAULT_DELIMITER, None) 
+        else:
+            delim_value = DELIMITER_VALUE_LOOKUP.get(delim, None)
+        if delim_value is None:
+            raise forms.ValidationError(_('Invalid value'))
+
+        return delim_value

geonode/contrib/datatables/models.py

@@ -0,0 +1,140 @@
+import psycopg2
+from django.db import models
+from django.db.models import signals
+from geonode.base.models import ResourceBase
+from geonode.layers.models import Attribute, AttributeManager
+from geonode.layers.models import Layer
+from django.utils.text import slugify
+
+TRANSFORMATION_FUNCTIONS = []
+
+class DataTable(ResourceBase):
+
+    """
+    DataTable (inherits ResourceBase fields)
+    """
+
+    # internal fields
+    table_name = models.CharField(max_length=255)
+    tablespace = models.CharField(max_length=255)
+    uploaded_file = models.FileField(upload_to="datatables")
+    create_table_sql = models.TextField(null=True, blank=True)
+
+    @property
+    def attributes(self):
+        return self.attribute_set.exclude(attribute='the_geom')
+
+    objects = AttributeManager()
+
+    def __unicode__(self):
+        return self.table_name
+
+    def remove_table(self):
+        conn = psycopg2.connect("dbname=geonode user=geonode")
+        cur = conn.cursor()
+        cur.execute('drop table if exists %s;' % self.table_name)
+        conn.commit()
+        cur.close()
+        conn.close() 
+
+class GeocodeType(models.Model):
+    name = models.CharField(max_length=255, unique=True, help_text='Examples: US Census Block, US County FIPS code, US Zip code, etc')
+    description = models.CharField(max_length=255, blank=True, help_text='Short description for end user')
+    sort_order = models.IntegerField(default=10)
+
+    def __unicode__(self):
+        return self.name
+
+    class Meta:
+        ordering = ('sort_order', 'name')
+
+class JoinTargetFormatType(models.Model):
+    name = models.CharField(max_length=255, help_text='Census Tract (6 digits, no decimal)') 
+    description_shorthand = models.CharField(max_length=255, help_text='dddddd') 
+    clean_steps = models.TextField(help_text='verbal description. e.g. Remove non integers. Check for empty string. Pad with zeros until 6 digits.')
+    regex_replacement_string = models.CharField(help_text='"[^0-9]"; Usage: re.sub("[^0-9]", "", "1234.99"'\
+                                , max_length=255)
+    python_code_snippet = models.TextField(blank=True)
+    tranformation_function_name = models.CharField(max_length=255, blank=True, choices=TRANSFORMATION_FUNCTIONS)
+
+    def __unicode__(self):
+        return self.name
+
+class JoinTarget(models.Model):
+    """
+    JoinTarget
+    """
+
+    layer = models.ForeignKey(Layer)
+    attribute = models.ForeignKey(Attribute)
+    geocode_type = models.ForeignKey(GeocodeType, on_delete=models.PROTECT)
+    type = models.ForeignKey(JoinTargetFormatType, null=True, blank=True)
+    year = models.IntegerField(null=True, blank=True)
+
+    def __unicode__(self):
+        return self.layer.title
+
+    def as_json(self):
+        if self.type:
+            type = {'name':self.type.name, 'description':self.type.description_shorthand, 'clean_steps':self.type.clean_steps}
+        else:
+            type = None
+        return dict(
+            id=self.id, layer=self.layer.typename,
+            attribute={'attribute':self.attribute.attribute, 'type':self.attribute.attribute_type},
+            type=type,
+            geocode_type=self.geocode_type.name)
+
+class TableJoin(models.Model):
+    """
+    TableJoin 
+    """
+
+    datatable = models.ForeignKey(DataTable)
+    source_layer = models.ForeignKey(Layer, related_name="source_layer")
+    table_attribute = models.ForeignKey(Attribute, related_name="table_attribute")
+    layer_attribute = models.ForeignKey(Attribute, related_name="layer_attribute")
+    view_name = models.CharField(max_length=255, null=True, blank=True)
+    view_sql = models.TextField(null=True, blank=True)
+    join_layer = models.ForeignKey(Layer, related_name="join_layer", null=True, blank=True)
+    matched_records_count = models.IntegerField(null=True, blank=True)
+    unmatched_records_count = models.IntegerField(null=True, blank=True)
+    unmatched_records_list = models.TextField(null=True, blank=True)
+
+    def __unicode__(self):
+        return self.view_name
+
+    def remove_joins(self):
+        conn = psycopg2.connect("dbname=geonode user=geonode")
+        cur = conn.cursor()
+        cur.execute('drop view if exists %s;' % self.view_name)
+        cur.execute('drop materialized view if exists %s;' % self.view_name.replace('view_', ''))
+        conn.commit()
+        cur.close()
+        conn.close() 
+
+    def as_json(self):
+        return dict(
+            id=self.id, datable=self.datatable.table_name, source_layer=self.source_layer.typename, join_layer=self.join_layer.typename,
+            table_attribute={'attribute':self.table_attribute.attribute, 'type':self.table_attribute.attribute_type},
+            layer_attribute={'attribute':self.layer_attribute.attribute, 'type':self.layer_attribute.attribute_type},
+            view_name=self.view_name, 
+            matched_records_count=self.matched_records_count,
+            unmatched_records_count=self.unmatched_records_count,
+            unmatched_records_list=self.unmatched_records_list)
+
+def pre_delete_datatable(instance, sender, **kwargs):
+    """
+    Remove the table from the Database
+    """
+    instance.remove_table()    
+
+
+def pre_delete_tablejoin(instance, sender, **kwargs):
+    """
+    Remove the existing join in the database
+    """
+    instance.remove_joins()
+
+signals.pre_delete.connect(pre_delete_tablejoin, sender=TableJoin)
+signals.pre_delete.connect(pre_delete_datatable, sender=DataTable)

geonode/contrib/datatables/tests/check_overlap.py

@@ -0,0 +1,20 @@
+layer_tractce = """724100|759100|091300|110501|030500|010500|650102|010404|321200|812201|710200|801902|070900|050500|760100|101101|510503|812404|180500|120600|383700|091700|070200|030100|383800|510900|250400|140201|082100|530600|507104|350900|170300|342201|385100|510200|322100|339400|510501|507103|510100|354800|510502|645101|650101|650201|650202|731102|510600|510700|510800|511000|511100|801604|501202|545200|506102|504101|732902|508200|525101|524102|652000|523202|732500|522101|544200|542300|510400|524101|545400|505200|506101|501201|542200|530400|731900|012601|709701|732201|732202|732203|710100|732400|732600|732700|732901|350500|733000|733101|710400|710500|722100|060400|755100|729200|730401|730100|730402|351404|730200|730300|710600|733102|731400|711000|710800|715100|716100|716300|730700|350600|730802|730901|731600|730902|731101|652100|718100|950307|731700|418004|418102|417702|417701|731800|732001|709202|020301|726200|727100|020302|728200|732002|729100|120201|010206|206100|012001|000504|020303|060501|050101|170502|000703|010802|709600|110607|120104|980300|739300|010204|739400|000803|000704|744300|744400|040100|746100|748100|010405|081001|020101|160101|750100|752100|754200|419600|420100|353102|731203|368901|317301|754300|339802|341101|754400|353101|350103|320103|757200|757300|342202|633200|352102|370202|384001|342101|731204|757400|388300|732302|356701|655400|160502|600100|561200|705100|985600|761100|610204|320102|221700|221800|250200|728400|653203|611201|751102|203302|742402|251100|731002|251300|015002|015300|010208|010304|160300|732802|738201|203301|613901|250600|250700|250800|250900|251000|736200|737200|736300|736400|251400|251500|050901|981502|051000|180301|170601|160601|170501|180101|630102|630101|640901|611101|641101|251600|737300|738100|061101|251800|252101|180400|060301|252202|252400|080100|100700|120103|130406|160501|130404|160602|170702|010408|010701|110403|010203|040801|071201|120301|252501|721102|252601|254100|120105|140105|721101|254402|709502|732801|260200|354000|741102|260600|747102|758101|761300|731001|260100|260401|260700|010300|747101|260500|352600|260800|260900|261000|352700|261101|326102|326101|341102|351203|352800|366202|388200|353000|981300|000802|981100|981800|981700|981501|990101|981201|040800|041200|041300|368902|935200|353200|921400|352400|352500|369000|331101|331102|331200|353300|353500|353600|353700|353900|354100|369100|354600|331300|332100|332300|333200|354200|354300|354400|333300|333400|354700|333502|333600|354500|354900|370101|370300|334100|335200|334200|356100|355000|334400|335100|370400|373100|373300|373400|311900|312000|312100|335400|336100|336300|336401|356300|356400|356500|356601|356602|312300|373500|312200|312400|312501|312502|337101|313102|314101|351000|337102|337201|336402|357100|337202|337300|338100|338200|357200|357300|357400|357500|338400|338500|338300|357600|351100|357700|358300|357800|358100|339100|339200|339300|358400|351300|358500|314200|314102|339700|315403|339500|339900|340000|340100|813802|813702|813701|358600|316400|315500|317203|316500|341300|317101|341400|341500|341600|350700|341700|341800|361200|358700|359100|359300|360100|318400|321300|342300|342400|362100|363102|363201|363202|364101|364102|350200|321400|321500|350800|321600|322200|365202|367100|368101|368300|322300|652200|323100|351500|368400|652300|368600|368800|652500|373600|373700|373800|373900|811500|374200|374400|374600|382100|801102|382200|382300|382400|383300|801200|382500|383200|206800|383501|383502|810901|810412|810902|206900|810602|810800|651700|811000|813207|813301|650900|651001|651002|651100|651200|419800|651800|442102|456402|651900|266200|266400|264100|651300|651500|651600|400100|400300|400400|400500|400600|400700|400800|652600|652700|652800|653102|655100|900200|933300|934200|934300|900300|933400|922300|646101|900100|812901|801300|802500|810700|810601|812500|811800|812600|812902|813601|812101|810414|801601|810100|812702|812401|811101|092101|813206|811400|401200|801605|811102|810200|813000|811200|800201|812800|813403|931100|931300|813602|800500|935300|800600|800700|900500|800800|920101|900400|900900|920102|922200|913100|922100|800900|914100|081200|931400|900800|932200|900700|801101|935100|911100|912100|374500|811301|811302|811600|811700|811900|655300|333501|652400|442202|361100|221902|923100|220102|215102|211402|731500|211401|710700|221901|709702|040702|041100|040600|040200|040400|040100|215101|120500|160200|070600|921300|655200|632100|388100|645102|921500|110301|041000|800202|040300|040900|010600|541100|315100|040701|650800|040501|041502|041400|335302|950400|342102|221600|070402|140107|950500|650400|650500|650600|732301|341901|738202|351403|742401|758102|351204|741101|761200|383102|383101|713100|081700|726100|712101|081800|335301|324102|318100|981202|708100|700100|317302|363103|363104|324101|709201|350104|341902|320104|385202|356702|707200|540102|707100|707300|707500|411301|416101|100603|010103|352101|382601|382602|051101|981600|090901|110103|384002|981000|170701|385201|980101|091001|251700|339801|456302|366201|370201|980000|040401|265101|220101|261102|202102|202101|441202|410300|442201|418003|990003|709501|520201|751101|013700|012900|013800|011100|521102|013900|725100|703100|707400|014600|091500|509101|011400|511200|010600|521201|011500|010400|014100|511500|014500|010500|160400|010100|013400|011300|012102|511301|010306|511302|011002|011700|012200|511400|204500|012800|091600|013500|543100|509102|511701|014300|014700|014402|511702|091800|013002|521202|012502|014002|011801|011802|091900|012101|092000|015001|013200|011200|010800|012700|011600|092200|015200|204600|013100|013300|092300|010900|010700|015100|014900|801901|013600|265102|520100|268200|268400|210400|250500|731300|750300|080601|740200|520202|521101|030200|523201|530300|525300|030300|526100|506204|530100|530200|530500|644200|544100|545100|511600|503101|506202|530700|140202|501102|204102|503102|504102|506203|510300|523100|030400|522102|508101|081900|545300|201100|561100|644102|501101|508102|060600|505102|060700|060800|507101|560100|250300|260402|251200|061000|082000|253201|254301|070500|266100|270100|990100|070700|203100|642400|644101|739500|070800|739100|180200|060101|735100|728100|740102|000503|742300|749100|723100|702200|701100|753200|704200|703300|140300|204200|730801|204300|710300|206200|736100|703200|731202|206000|744101|711100|080300|140400|717100|203200|204101|090100|745100|757500|750200|092400|170100|170200|080401|170400|204400|080500|204701|204702|205100|140106|110201|080801|080900|980700|170800|205200|205300|205400|205500|020200|719100|756102|709400|740101|743100|730600|206600|730500|040300|050200|728300|050300|712102|050400|050600|756101|757100|754100|744102|050700|010104|736500|753100|010702|010801|010403|749200|040200|822603|709100|744200|051200|060200|737100|735200|755200|061200|070101|070300|081100|442101|456101|456102|456200|420201|206700|420202|420302|421100|206300|206400|206500|301101|800400|208101|207000|100100|208400|207100|207200|100200|081300|081400|081500|211300|440100|100300|456401|457100|421200|422100|422200|422301|422302|090600|209100|210100|210200|822700|090200|090300|090400|090700|091100|091200|091400|100400|210300|210700|210500|210600|211100|211200|253100|422501|422700|100500|100601|268100|210800|332200|422800|100800|202200|417801|417802|417901|417902|100900|101001|990000|210900|101002|101102|110401|110502|110601|418002|120400|217600|212100|269100|263100|250100|419100|419200|120700|130100|130200|223300|254200|456301|254302|267102|253203|253202|205900|208200|266300|801402|130300|130402|140102|268300|222100|223200|213100|253204|253205|214100|252603|419300|208102|252201|254403|223100|800300|209200|419400|252300|254401|802000|267101|252102|419500|260302|614102|208300|218100|221100|252602|071101|260301|000201|217100|205600|252502|000202|613100|216100|820102|000401|820500|217201|821602|822300|821300|820101|820202|820204|820203|822500|820300|822200|821400|217202|820400|820600|000402|820801|820700|822401|820802|217300|217400|217500|221500|221300|221400|612100|812403|000100|820900|000301|000302|821000|000602|000701|821100|821200|821500|641000|000502|000601|640300|650700|821901|822000|802200|821700|822402|205700|205800|642200|640100|642300|613800|631800|640400|641700|640500|630400|650300|617101|653101|610202|640600|613300|640700|653301|631200|612200|641400|616100|641500|617102|632200|641600|630300|641800|613400|651400|645103|641900|654200|641200|642500|985500|654100|802300|600202|610100|613700|613600|418101|614000|631500|614101|631600|615100|642000|631300|631400|631700|642100|631100|411302|441204|413402|416102|441203|413401|630200|640200|640800|633100|641300|457200|801401|801501|801502|801503|322400|812300|821904|812701|812903|822606|813204|813205|821601|040502|822601|800102|368200|813303|800101|813801|801700|812103|812001|801800|801602|801603|822605|041501|802400|014800|417200|012002|417400|012602|812202|813209|417501|813208|990000|720100|802100|802601|802602|417601|415102|417602|415200|810404|416300|401000|739200|402102|416400|417100|810300|810403|990000|402200|950100|419700|406102|706100|950200|402400|040600|403100|404202|410400|404301|404302|413100|415101|403400|408102|406101|422502|402300|200100|408101|310300|423100|418200|300100|411100|402500|310500|422400|339600|443102|500101|443101|407100|416200|414100|417502|409102|403500|404400|410100|411200|262100|530802|530902|500104|417300|415300|500103|525204|442103|525203|404100|405100|402101|525104|542101|530901|540103|530801|540101|542102|403300|422600|414200|420301|409101|400200|401100|502200|412300|404201|200400|412100|412200|413200|413300|812104|813304|821903|200200|400900|200300|414300|413500|716200|360200|368500|352200|328100|316102|342500|352900|900600|310602|901100|310700|311100|316202|318200|318300|317202|311200|374000|311300|361300|341200|317201|383400|327103|365201|314301|366100|353800|316201|350300|387100|383902|311400|383600|311500|330100|333100|374300|353400|316300|373200|332400|368700|314302|352300|342600|367200|330200|317102|350400|365100|370102|924100|925100|387202|327102|321100|386100|317103|374800|313101|334300|315200|383901|315402|374100|813401|926100|933200|932300|310100|813404|813500|301102|310200|761400|325100|310400|310601|336200|368102|327101|315401|387201|374700|316101|812002|813102|813101|311600|311700|311800|502101|502102|505101|646104|646103|653304|600203|600204|610203|611202|611102|653204|613902|990000""".split('|')
+
+print len(layer_tractce)
+layer_tractce = set(layer_tractce)
+print len(layer_tractce)
+
+csv_tract = """100|201|302|501|602|702|10101|10202|10401|10500|10800|20200|30200|30500|40100|40200|40300|40400|50100|50200|50400|50500|51100|60200|60300|60700|61000|70100|70400|70600|70800|71200|80400|80800|202|301|401|402|502|601|701|801|802|10102|10201|10300|10402|10600|10700|20100|20300|30100|50600|50700|30300|30400|40600|40800|50300|50900|51000|51200|60100|60400|60500|70500|60600|60800|61100|80300|61200|70200|70300|70700|70900|71100|80100|80500|80600|80900|81000|81100|81200|82100|81300|81400|81500|81700|81800|81900|82000|90100|90200|90300|90400|90600|90700|91100|91300|91600|91800|92200|92400|100300|100601|100900|101101|110200|110401|110502|120102|120500|120600|130200|91500|140102|140201|90900|91000|91200|91400|91700|91900|92000|92100|92300|100100|100200|100400|100500|100602|100700|100800|101001|101002|101102|110100|110300|110402|110501|110601|110602|120101|120200|120300|120400|120700|130100|130300|130401|130402|140103|140104|140202|140300|140400""".split('|')
+csv_tract = set(csv_tract)
+
+# Matches if csv file strips leading zeros
+matches = [cval for cval in csv_tract if cval in layer_tractce]
+print 'worst case match: ', len(matches)
+
+
+# Matches if csv file includes leading zeros
+csv_tract_zfill = [ val.zfill(6) for val in csv_tract]
+print 'csv tracts: %s' % len(csv_tract_zfill)
+matches2 = [cval for cval in csv_tract_zfill if cval in layer_tractce]
+print 'best case match: ', len(matches2)
+