Permalink
Browse files

Merge branch 'master' of https://github.com/ireapps/census

  • Loading branch information...
Curt Merrill
Curt Merrill committed Jun 20, 2011
2 parents 1df3fc0 + b2cdeaf commit 368e769736b510c640c3d0f6b5e922a6519b4ced
Showing with 133 additions and 2 deletions.
  1. +3 −0 censusweb/api/urls.py
  2. +106 −1 censusweb/api/utils.py
  3. +24 −1 censusweb/api/views.py
View
@@ -18,6 +18,9 @@
url(r'^data/(?P<geoids>[,\d]+)\.csv$', views.data_as_csv, name="data_as_csv"),
url(r'^data/(?P<geoids>[,\d]+)\.(?P<format>kml|kmz)$', views.data_as_kml, name="data_as_kml"),
url(r'^data/(?P<geoids>[,\d]+)\.html$', views.data, name="data"),
+ url(r'^util/create_table/(?P<aggregate>(all_files|all_tables))\.sql$', views.generate_sql, name="generate_sql"), # order matters. keep this first to catch only numbers before tables
+ url(r'^util/create_table/(?P<file_ids>[,\d{1,2}]+)\.sql$', views.generate_sql, name="generate_sql"), # order matters. keep this first to catch only numbers before tables
+ url(r'^util/create_table/(?P<table_ids>[,\w]+)\.sql$', views.generate_sql, name="generate_sql"),
# Generate CSV/JSON for all elements in a given region (used from within Query Builder)
#/internal/download_data_for_region/10.csv (or .json)
View
@@ -2,6 +2,12 @@
import simplejson
+from sqlalchemy import Column, MetaData, Table
+from sqlalchemy import Float, Integer, String
+from sqlalchemy.schema import CreateTable
+
+from api import mongoutils
+
from django.conf import settings
import requests
@@ -50,4 +56,103 @@ def fetch_geographies(geoids):
def fetch_labels(dataset):
url = '%s/%s_labels.jsonp' % (settings.API_URL, dataset)
return _api_fetch(url)
-
+
+def generate_create_sql_by_file(file_numbers=None):
+ if file_numbers is None:
+ file_numbers = range(1,48)
+
+ statements = []
+ for file_number in file_numbers:
+ sql_table = _create_base_table('file%s' % file_number)
+ for table in SF1_FILE_SEGMENTS[file_number]:
+ _add_sql_columns_for_table(sql_table,table)
+ statements.append(unicode(CreateTable(sql_table).compile(dialect=None)).strip() + ';')
+
+ return "\n\n".join(statements)
+
+def generate_sql_by_table(table_codes=None):
+ statements = []
+ if table_codes is None:
+ table_codes = []
+ for f in SF1_FILE_SEGMENTS[1:]:
+ table_codes.extend(f)
+
+ statements = []
+ for table_code in table_codes:
+ sql_table = _create_base_table(table_code)
+ _add_sql_columns_for_table(sql_table,table_code)
+ statements.append(unicode(CreateTable(sql_table).compile(dialect=None)).strip() + ';')
+
+ return "\n\n".join(statements)
+
+def _create_base_table(name):
+ metadata = MetaData()
+ sql_table = Table(name, metadata)
+ sql_table.append_column(Column('FILEID', String(length=6), nullable=False))
+ sql_table.append_column(Column('STUSAB', String(length=2), nullable=False))
+ sql_table.append_column(Column('CHARITER', String(length=3), nullable=False))
+ sql_table.append_column(Column('CIFSN', String(length=3), nullable=False))
+ sql_table.append_column(Column('LOGRECNO', String(length=7), nullable=False))
+
+ return sql_table
+
+def _add_sql_columns_for_table(sql_table,code):
+ labels = mongoutils.get_labelset()
+ table_labels = labels['tables'][code]
+ if table_labels['name'].find('AVERAGE') != -1 or table_labels['name'].find('MEDIAN') != -1:
+ col_type = Float
+ else:
+ col_type = Integer
+ for label in sorted(table_labels['labels']):
+ sql_table.append_column(Column(label, col_type(), nullable=False))
+
+SF1_FILE_SEGMENTS = [
+ [ 'no file zero, this is a place_holder'],
+ ['P1'], # file 1
+ ['P2'], # file 2
+ ['P3', 'P4', 'P5', 'P6', 'P7', 'P8', 'P9'], # file 3
+ ['P10', 'P11', 'P12', 'P13', 'P14'], # file 4
+ ['P15', 'P16', 'P17', 'P18', 'P19', 'P20', 'P21', 'P22', 'P23', 'P24', 'P25', 'P26', 'P27', 'P28', 'P29', 'P30'], # file 5
+ ['P31', 'P32', 'P33', 'P34', 'P35', 'P36', 'P37', 'P38', 'P39', 'P40', 'P41', 'P42', 'P43', 'P44', 'P45', 'P46', 'P47', 'P48', 'P49'], # file 6
+ ['P50', 'P51', 'P12A', 'P12B', 'P12C', 'P12D', 'P12E'], # file 7
+ ['P12F', 'P12G', 'P12H', 'P12I', 'P13A', 'P13B', 'P13C', 'P13D', 'P13E', 'P13F', 'P13G', 'P13H', 'P13I', 'P16A', 'P16B', 'P16C', 'P16D', 'P16E', 'P16F', 'P16G', 'P16H', 'P16I', 'P17A'], # file 8
+ ['P17B', 'P17C', 'P17D', 'P17E', 'P17F', 'P17G', 'P17H', 'P17I', 'P18A', 'P18B', 'P18C', 'P18D', 'P18E', 'P18F', 'P18G', 'P18H', 'P18I', 'P28A', 'P28B', 'P28C', 'P28D', 'P28E', 'P28F', 'P28G', 'P28H', 'P28I'], # file 9
+ ['P29A', 'P29B', 'P29C', 'P29D', 'P29E', 'P29F', 'P29G', 'P29H', 'P29I'], # file 10
+ ['P31A', 'P31B', 'P31C', 'P31D', 'P31E', 'P31F', 'P31G', 'P31H', 'P31I', 'P34A', 'P34B', 'P34C', 'P34D', 'P34E'], # file 11
+ ['P34F', 'P34G', 'P34H', 'P34I', 'P35A', 'P35B', 'P35C', 'P35D', 'P35E', 'P35F', 'P35G', 'P35H', 'P35I', 'P36A', 'P36B', 'P36C', 'P36D', 'P36E', 'P36F', 'P36G', 'P36H', 'P36I', 'P37A', 'P37B', 'P37C', 'P37D', 'P37E', 'P37F', 'P37G', 'P37H', 'P37I', 'P38A', 'P38B', 'P38C', 'P38D', 'P38E'], # file 12
+ ['P38F', 'P38G', 'P38H', 'P38I', 'P39A', 'P39B', 'P39C', 'P39D', 'P39E', 'P39F', 'P39G', 'P39H'], # file 13
+ ['P39I'], # file 14
+ ['PCT1', 'PCT2', 'PCT3', 'PCT4', 'PCT5', 'PCT6', 'PCT7', 'PCT8'], # file 15
+ ['PCT9', 'PCT10', 'PCT11'], # file 16
+ ['PCT12'], # file 17
+ ['PCT13', 'PCT14', 'PCT15', 'PCT16', 'PCT17', 'PCT18', 'PCT19', 'PCT20'], # file 18
+ ['PCT21','PCT22'], # file 19
+ ['PCT12A'], # file 20
+ ['PCT12B'], # file 21
+ ['PCT12C'], # file 22
+ ['PCT12D'], # file 23
+ ['PCT12E'], # file 24
+ ['PCT12F'], # file 25
+ ['PCT12G'], # file 26
+ ['PCT12H'], # file 27
+ ['PCT12I'], # file 28
+ ['PCT12J'], # file 29
+ ['PCT12K'], # file 30
+ ['PCT12L'], # file 31
+ ['PCT12M'], # file 32
+ ['PCT12N'], # file 33
+ ['PCT12O'], # file 34
+ ['PCT13A','PCT13B','PCT13C','PCT13D','PCT13E'], # file 35
+ ['PCT13F','PCT13G','PCT13H','PCT13I','PCT14A','PCT14B','PCT14C','PCT14D','PCT14E','PCT14F','PCT14G','PCT14H','PCT14I','PCT19A','PCT19B'], # file 36
+ ['PCT19C','PCT19D','PCT19E','PCT19F','PCT19G','PCT19H','PCT19I','PCT20A','PCT20B','PCT20C','PCT20D','PCT20E'], # file 37
+ ['PCT20F','PCT20G','PCT20H','PCT20I','PCT22A','PCT22B','PCT22C','PCT22D','PCT22E','PCT22F'], # file 38
+ ['PCT22G','PCT22H','PCT22I'], # file 39
+ ['PCO1','PCO2','PCO3','PCO4','PCO5','PCO6'], # file 40
+ ['PCO7','PCO8','PCO9','PCO10'], # file 41
+ ['H1'], # file 42
+ ['H2'], # file 43
+ ['H3','H4','H5','H6','H7','H8','H9','H10','H11','H12','H13','H14','H15','H16','H17','H18','H19','H20','H21','H22','H11A','H11B', 'H11C','H11D', 'H11E','H11F'], # file 44
+ ['H11G','H11H','H11I','H12A','H12B','H12C','H12D','H12E','H12F','H12G','H12H','H12I','H16A','H16B','H16C','H16D','H16E','H16F','H16G','H16H','H16I','H17A','H17B','H17C'], # file 45
+ ['H17D','H17E','H17F','H17G','H17H','H17I'], # file 46
+ ['HCT1','HCT2','HCT3','HCT4'], # file 47
+]
View
@@ -1,7 +1,8 @@
import simplejson
from django.conf import settings
-from django.http import HttpResponse
+from django.http import HttpResponse, HttpResponseNotFound
+
from django.shortcuts import render_to_response
from django.contrib.gis.shortcuts import render_to_kml, render_to_kmz
from django.template import RequestContext, Template, Context
@@ -224,3 +225,25 @@ def _build_kml_context_for_template(b, j, tables, labelset):
return kml_context
+def generate_sql(request, file_ids=None, table_ids=None, aggregate=None):
+ if aggregate == 'all_files':
+ sql = utils.generate_create_sql_by_file()
+ return HttpResponse(sql,mimetype='text/plain')
+ elif aggregate == 'all_tables':
+ sql = utils.generate_sql_by_table()
+ return HttpResponse(sql,mimetype='text/plain')
+ elif aggregate is not None:
+ return HttpResponseNotFound()
+
+ if file_ids:
+ ids = map(int,file_ids.split(','))
+ sql = utils.generate_create_sql_by_file(file_numbers=ids)
+ return HttpResponse(sql,mimetype='text/plain')
+
+ if table_ids:
+ table_ids = table_ids.split(',')
+ sql = utils.generate_sql_by_table(table_ids)
+ return HttpResponse(sql,mimetype='text/plain')
+
+ return HttpResponseNotFound()
+

0 comments on commit 368e769

Please sign in to comment.