Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Newer
Older
100644 1033 lines (874 sloc) 42.607 kb
99a5c0c @slinkp Adding copyrights and GPL v3 everywhere
slinkp authored
1 # Copyright 2007,2008,2009,2011 Everyblock LLC, OpenPlans, and contributors
2 #
3 # This file is part of ebpub
4 #
5 # ebpub is free software: you can redistribute it and/or modify
6 # it under the terms of the GNU General Public License as published by
7 # the Free Software Foundation, either version 3 of the License, or
8 # (at your option) any later version.
9 #
10 # ebpub is distributed in the hope that it will be useful,
11 # but WITHOUT ANY WARRANTY; without even the implied warranty of
12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 # GNU General Public License for more details.
14 #
15 # You should have received a copy of the GNU General Public License
16 # along with ebpub. If not, see <http://www.gnu.org/licenses/>.
17 #
18
5c9826f initial import
Don Kukral authored
19 from django.contrib.gis.db import models
20 from django.contrib.gis.db.models import Count
ff0a9b6 @slinkp Change newsitem_detail URL: remove useless date arguments.
slinkp authored
21 from django.core import urlresolvers
a9a4d9f @slinkp ValueError -> ValidatonError to make admin form nicer
slinkp authored
22 from django.core.exceptions import ValidationError
5c9826f initial import
Don Kukral authored
23 from django.db import connection, transaction
220ca2c @ltucker Add some extra info to improve location and place tagging, split Misspel...
ltucker authored
24 from ebpub.geocoder.parser.parsing import normalize
5c9826f initial import
Don Kukral authored
25 from ebpub.streets.models import Block
5309287 @slinkp Overhaul of olwidget integration.
slinkp authored
26 from ebpub.utils.geodjango import flatten_geomcollection
27 from ebpub.utils.geodjango import ensure_valid
5c9826f initial import
Don Kukral authored
28 from ebpub.utils.text import slugify
de7c1fe @slinkp Yet more de-hardcoded URLs.
slinkp authored
29
5c9826f initial import
Don Kukral authored
30 import datetime
5309287 @slinkp Overhaul of olwidget integration.
slinkp authored
31 import logging
f65f953 @slinkp Fix two errors triggered when NewsItems are saved for a schema with no s...
slinkp authored
32 import re
5c9826f initial import
Don Kukral authored
33
5309287 @slinkp Overhaul of olwidget integration.
slinkp authored
34 logger = logging.getLogger('ebpub.db.models')
5c9826f initial import
Don Kukral authored
35
0eb2538 @slinkp Monkeypatches that need settings can't be imported before settings are c...
slinkp authored
36 # Need these monkeypatches for "natural key" support during fixture load/dump.
37 import ebpub.monkeypatches
38 ebpub.monkeypatches.patch_once()
3b0dc05 @slinkp Update stuff using Query.extra_where to use QuerySet.extra(where...). A...
slinkp authored
39
9e0becb @slinkp Deleted SchemaInfo, moved all that metadata into Schema. Observed perfor...
slinkp authored
40 FREQUENCY_CHOICES = ('Hourly', 'Throughout the day', 'Daily', 'Twice a week', 'Weekly', 'Twice a month', 'Monthly', 'Quarterly', 'Sporadically', 'No longer updated')
41 FREQUENCY_CHOICES = [(a, a) for a in FREQUENCY_CHOICES]
42
f65f953 @slinkp Fix two errors triggered when NewsItems are saved for a schema with no s...
slinkp authored
43 logger = logging.getLogger('ebpub.db.models')
0eb2538 @slinkp Monkeypatches that need settings can't be imported before settings are c...
slinkp authored
44
1fa4fec @slinkp SchemaField admin: limit choices of real_name to valid values
slinkp authored
45 def get_valid_real_names():
46 """
47 Field names of ``Attribute``, suitable for use as
48 ``SchemaField.real_name``.
49 """
50 for name in sorted(Attribute._meta.get_all_field_names()):
51 if re.search(r'\d\d$', name):
52 yield name
0eb2538 @slinkp Monkeypatches that need settings can't be imported before settings are c...
slinkp authored
53
54
5c9826f initial import
Don Kukral authored
55 def field_mapping(schema_id_list):
56 """
57 Given a list of schema IDs, returns a dictionary of dictionaries, mapping
58 schema_ids to dictionaries mapping the fields' name->real_name.
59 Example return value:
60 {1: {u'crime_type': 'varchar01', u'crime_date', 'date01'},
61 2: {u'permit_number': 'varchar01', 'to_date': 'date01'},
62 }
63 """
64 result = {}
65 for sf in SchemaField.objects.filter(schema__id__in=(schema_id_list)).values('schema', 'name', 'real_name'):
66 result.setdefault(sf['schema'], {})[sf['name']] = sf['real_name']
67 return result
68
baec0a7 @slinkp Use natural keys for serialization wherever possible; it makes fixtures ...
slinkp authored
69
5c9826f initial import
Don Kukral authored
70 class SchemaManager(models.Manager):
baec0a7 @slinkp Use natural keys for serialization wherever possible; it makes fixtures ...
slinkp authored
71
72 def get_by_natural_key(self, slug):
73 return self.get(slug=slug)
74
9e0becb @slinkp Deleted SchemaInfo, moved all that metadata into Schema. Observed perfor...
slinkp authored
75 def get_query_set(self):
fdaebb9 @slinkp comment re #82
slinkp authored
76 """Warning: This breaks manage.py dumpdata.
77 See bug #82.
78
79 """
9e0becb @slinkp Deleted SchemaInfo, moved all that metadata into Schema. Observed perfor...
slinkp authored
80 return super(SchemaManager, self).get_query_set().defer(
81 'short_description',
82 'summary',
83 'source',
84 'grab_bag_headline',
85 'grab_bag',
86 'short_source',
87 'update_frequency',
88 'intro',
89 )
90
91
baec0a7 @slinkp Use natural keys for serialization wherever possible; it makes fixtures ...
slinkp authored
92 class SchemaPublicManager(SchemaManager):
93
5c9826f initial import
Don Kukral authored
94 def get_query_set(self):
95 return super(SchemaManager, self).get_query_set().filter(is_public=True)
96
97 class Schema(models.Model):
188c1d2 @slinkp Much commenting on how NewsItems work
slinkp authored
98 """
99 Describes a type of NewsItem. A NewsItem has exactly one Schema,
100 which describes its attributes, via associated SchemaFields.
57a93d2 @slinkp another comment
slinkp authored
101
102 nb. to get all NewsItem instances for a Schema, you can do the usual as per
103 http://docs.djangoproject.com/en/dev/topics/db/queries/#backwards-related-objects:
104 schema.newsitem_set.all()
188c1d2 @slinkp Much commenting on how NewsItems work
slinkp authored
105 """
5c9826f initial import
Don Kukral authored
106 name = models.CharField(max_length=32)
107 plural_name = models.CharField(max_length=32)
108 indefinite_article = models.CharField(max_length=2) # 'a' or 'an'
e3056db @slinkp Use SlugField for slugs.
slinkp authored
109 slug = models.SlugField(max_length=32, unique=True)
5c9826f initial import
Don Kukral authored
110 min_date = models.DateField() # the earliest available NewsItem.pub_date for this Schema
111 last_updated = models.DateField()
dcf27b3 make schema creation slightly less detailed by adding some defaults.
Luke Tucker authored
112 date_name = models.CharField(max_length=32, default='Date') # human-readable name for the NewsItem.item_date field
113 date_name_plural = models.CharField(max_length=32, default='Dates')
114 importance = models.SmallIntegerField(default=0) # bigger number is more important
115 is_public = models.BooleanField(db_index=True, default=False)
116 is_special_report = models.BooleanField(default=False)
5c9826f initial import
Don Kukral authored
117
118 # whether RSS feed should collapse many of these into one
dcf27b3 make schema creation slightly less detailed by adding some defaults.
Luke Tucker authored
119 can_collapse = models.BooleanField(default=False)
5c9826f initial import
Don Kukral authored
120
121 # whether a newsitem_detail page exists for NewsItems of this Schema
dcf27b3 make schema creation slightly less detailed by adding some defaults.
Luke Tucker authored
122 has_newsitem_detail = models.BooleanField(default=False)
5c9826f initial import
Don Kukral authored
123
124 # whether aggregate charts are allowed for this Schema
dcf27b3 make schema creation slightly less detailed by adding some defaults.
Luke Tucker authored
125 allow_charting = models.BooleanField(default=False)
5c9826f initial import
Don Kukral authored
126
c0c0edb @slinkp formatting
slinkp authored
127 # whether attributes should be preloaded for NewsItems of this
128 # Schema, in the list view
dcf27b3 make schema creation slightly less detailed by adding some defaults.
Luke Tucker authored
129 uses_attributes_in_list = models.BooleanField(default=False)
5c9826f initial import
Don Kukral authored
130
131 # number of records to show on place_overview
dcf27b3 make schema creation slightly less detailed by adding some defaults.
Luke Tucker authored
132 number_in_overview = models.SmallIntegerField(default=5)
8a05a15 @ltucker Refactoring maps to prepare for additional map functionality
ltucker authored
133
134 map_icon_url = models.TextField(blank=True, null=True)
135 map_color = models.CharField(max_length=255, blank=True, null=True, help_text="CSS Color used on maps to display this type of news. eg #FF0000")
136
5c9826f initial import
Don Kukral authored
137
baec0a7 @slinkp Use natural keys for serialization wherever possible; it makes fixtures ...
slinkp authored
138 objects = SchemaManager()
139 public_objects = SchemaPublicManager()
5c9826f initial import
Don Kukral authored
140
141 def __unicode__(self):
142 return self.name
143
baec0a7 @slinkp Use natural keys for serialization wherever possible; it makes fixtures ...
slinkp authored
144 def natural_key(self):
145 return (self.slug,)
146
5c9826f initial import
Don Kukral authored
147 def url(self):
2d83842 @slinkp De-hardcode more model URLs. Refs #69
slinkp authored
148 return urlresolvers.reverse('ebpub-schema-detail', args=(self.slug,))
5c9826f initial import
Don Kukral authored
149
150 def icon_slug(self):
151 if self.is_special_report:
152 return 'special-report'
153 return self.slug
154
baec0a7 @slinkp Use natural keys for serialization wherever possible; it makes fixtures ...
slinkp authored
155
9e0becb @slinkp Deleted SchemaInfo, moved all that metadata into Schema. Observed perfor...
slinkp authored
156 # Metadata fields moved from SchemaInfo
a03aaad More defaults
Luke Tucker authored
157 short_description = models.TextField(blank=True, default='')
158 summary = models.TextField(blank=True, default='')
159 source = models.TextField(blank=True, default='')
160 grab_bag_headline = models.CharField(max_length=128, blank=True, default='')
161 grab_bag = models.TextField(blank=True, default='') # TODO: what does this field mean?
162 short_source = models.CharField(max_length=128, blank=True, default='')
9e0becb @slinkp Deleted SchemaInfo, moved all that metadata into Schema. Observed perfor...
slinkp authored
163 update_frequency = models.CharField(max_length=64, blank=True, default='',
164 choices=FREQUENCY_CHOICES)
a03aaad More defaults
Luke Tucker authored
165 intro = models.TextField(blank=True, default='')
5c9826f initial import
Don Kukral authored
166
a11a8b0 @slinkp Lots more columns, filters, and usable default ordering in admin UI; clo...
slinkp authored
167 class Meta:
168 ordering = ('name',)
baec0a7 @slinkp Use natural keys for serialization wherever possible; it makes fixtures ...
slinkp authored
169
170 class SchemaFieldManager(models.Manager):
171
172 def get_by_natural_key(self, schema_slug, real_name):
173 return self.get(schema__slug=schema_slug, real_name=real_name)
174
5c9826f initial import
Don Kukral authored
175 class SchemaField(models.Model):
188c1d2 @slinkp Much commenting on how NewsItems work
slinkp authored
176 """
177 Describes the meaning of one Attribute field for one Schema type.
178 """
baec0a7 @slinkp Use natural keys for serialization wherever possible; it makes fixtures ...
slinkp authored
179 objects = SchemaFieldManager()
180
5c9826f initial import
Don Kukral authored
181 schema = models.ForeignKey(Schema)
c0a594d @slinkp Replace SchemaField.name with SchemaField.slug, since it was already a p...
slinkp authored
182
183 pretty_name = models.CharField(
184 max_length=32,
185 help_text="Human-readable name of this field, for display."
186 )
187 pretty_name_plural = models.CharField(
188 max_length=32,
189 help_text="Plural human-readable name"
190 )
2d4bb55 @slinkp Partial revert of c0a594d69e89ba31b7a57 - removing "name" was a bad idea...
slinkp authored
191
192 name = models.SlugField(max_length=32)
c0a594d @slinkp Replace SchemaField.name with SchemaField.slug, since it was already a p...
slinkp authored
193
194 real_name = models.CharField(
195 max_length=10,
196 help_text="Column name in the Attribute model. 'varchar01', 'varchar02', etc.",
1fa4fec @slinkp SchemaField admin: limit choices of real_name to valid values
slinkp authored
197 choices=((name, name) for name in get_valid_real_names()),
c0a594d @slinkp Replace SchemaField.name with SchemaField.slug, since it was already a p...
slinkp authored
198 )
199 display = models.BooleanField(
200 default=True,
201 help_text='Whether to display value on the public site.'
202 )
203 is_lookup = models.BooleanField(
204 default=False,
205 help_text='Whether the value is a foreign key to Lookup.'
206 )
207 is_filter = models.BooleanField(
208 default=False,
209 help_text='Whether to link to list of items with the same value in this field. Assumes is_lookup=True.'
210 )
211 is_charted = models.BooleanField(
212 default=False,
213 help_text='Whether the schema detail view displays a chart for this field; also see "trends" tabs on place overview page.'
214 )
a03aaad More defaults
Luke Tucker authored
215 display_order = models.SmallIntegerField(default=10)
c0a594d @slinkp Replace SchemaField.name with SchemaField.slug, since it was already a p...
slinkp authored
216 is_searchable = models.BooleanField(
217 default=False,
218 help_text="Whether the value is searchable by content. Doesn't make sense if is_lookup=True."
219 )
5c9826f initial import
Don Kukral authored
220
baec0a7 @slinkp Use natural keys for serialization wherever possible; it makes fixtures ...
slinkp authored
221 def natural_key(self):
222 return (self.schema.slug, self.real_name)
223
90b5568 @slinkp Enforce documented uniqueness for SchemaField schema + real_name. And s...
slinkp authored
224 class Meta(object):
225 unique_together = (('schema', 'real_name'),)
a11a8b0 @slinkp Lots more columns, filters, and usable default ordering in admin UI; clo...
slinkp authored
226 ordering = ('pretty_name',)
90b5568 @slinkp Enforce documented uniqueness for SchemaField schema + real_name. And s...
slinkp authored
227
5c9826f initial import
Don Kukral authored
228 def __unicode__(self):
229 return u'%s - %s' % (self.schema, self.name)
230
764e1b9 @slinkp Use modern @property syntax; simplify SchemaField.is_type()
slinkp authored
231 @property
232 def datatype(self):
5c9826f initial import
Don Kukral authored
233 return self.real_name[:-2]
234
235 def is_type(self, *data_types):
236 """
237 Returns True if this SchemaField is of *any* of the given data types.
238
239 Allowed values are 'varchar', 'date', 'time', 'datetime', 'bool', 'int'.
240 """
764e1b9 @slinkp Use modern @property syntax; simplify SchemaField.is_type()
slinkp authored
241 return self.datatype in data_types
5c9826f initial import
Don Kukral authored
242
243 def is_many_to_many_lookup(self):
244 """
245 Returns True if this SchemaField is a many-to-many lookup.
246 """
247 return self.is_lookup and not self.is_type('int')
0f82c51 @slinkp Default News and GeoReport schemas are now set up by ebpub migrations;
slinkp authored
248 is_many_to_many_lookup.boolean = True
5c9826f initial import
Don Kukral authored
249
250 def all_lookups(self):
251 if not self.is_lookup:
252 raise ValueError('SchemaField.all_lookups() can only be called if is_lookup is True')
253 return Lookup.objects.filter(schema_field__id=self.id).order_by('name')
254
255 def browse_by_title(self):
256 "Returns FOO in 'Browse by FOO', for this SchemaField."
257 if self.is_type('bool'):
258 return u'whether they %s' % self.pretty_name_plural
259 return self.pretty_name
260
261 def smart_pretty_name(self):
262 """
263 Returns the pretty name for this SchemaField, taking into account
264 many-to-many fields.
265 """
266 if self.is_many_to_many_lookup():
267 return self.pretty_name_plural
268 return self.pretty_name
269
baec0a7 @slinkp Use natural keys for serialization wherever possible; it makes fixtures ...
slinkp authored
270
271 class LocationTypeManager(models.Manager):
272 def get_by_natural_key(self, slug):
273 return self.get(slug=slug)
274
5c9826f initial import
Don Kukral authored
275 class LocationType(models.Model):
276 name = models.CharField(max_length=255) # e.g., "Ward" or "Congressional District"
277 plural_name = models.CharField(max_length=64) # e.g., "Wards"
f5a8696 @slinkp Docs: A LOT more about setting up geographic information.
slinkp authored
278 scope = models.CharField(max_length=64,
279 help_text='e.g., "Chicago" or "U.S.A."')
e3056db @slinkp Use SlugField for slugs.
slinkp authored
280 slug = models.SlugField(max_length=32, unique=True)
c450ce2 @slinkp Minor LocationType admin UI tweaks
slinkp authored
281 is_browsable = models.BooleanField(
282 default=True, help_text="Whether this is displayed on location_type_list.") # XXX unused??
283 is_significant = models.BooleanField(
284 default=True,
285 help_text="Whether this is used to display aggregates, shows up in 'nearby locations', etc."
286 )
5c9826f initial import
Don Kukral authored
287
288 def __unicode__(self):
289 return u'%s, %s' % (self.name, self.scope)
290
291 def url(self):
2d83842 @slinkp De-hardcode more model URLs. Refs #69
slinkp authored
292 return urlresolvers.reverse('ebpub-loc-type-detail', args=(self.slug,))
5c9826f initial import
Don Kukral authored
293
baec0a7 @slinkp Use natural keys for serialization wherever possible; it makes fixtures ...
slinkp authored
294 def natural_key(self):
295 return (self.slug,)
296
a11a8b0 @slinkp Lots more columns, filters, and usable default ordering in admin UI; clo...
slinkp authored
297 class Meta:
298 ordering = ('name',)
299
baec0a7 @slinkp Use natural keys for serialization wherever possible; it makes fixtures ...
slinkp authored
300 objects = LocationTypeManager()
301
302
303 class LocationManager(models.GeoManager):
304 def get_by_natural_key(self, slug, location_type_slug):
1b4c87c @ltucker fix Location get_by_natural_key, typo in lookup
ltucker authored
305 return self.get(slug=slug, location_type__slug=location_type_slug)
baec0a7 @slinkp Use natural keys for serialization wherever possible; it makes fixtures ...
slinkp authored
306
5c9826f initial import
Don Kukral authored
307 class Location(models.Model):
308 name = models.CharField(max_length=255) # e.g., "35th Ward"
309 normalized_name = models.CharField(max_length=255, db_index=True)
e3056db @slinkp Use SlugField for slugs.
slinkp authored
310 slug = models.SlugField(max_length=32, db_index=True)
5c9826f initial import
Don Kukral authored
311 location_type = models.ForeignKey(LocationType)
312 location = models.GeometryField(null=True)
313 display_order = models.SmallIntegerField()
314 city = models.CharField(max_length=255)
315 source = models.CharField(max_length=64)
09d1204 @slinkp update comment, trigger sql moved
slinkp authored
316 # In square meters. This is populated by a trigger in ebpub/db/migrations/0004_st_intersects_patch.py
317 area = models.FloatField(blank=True, null=True)
5c9826f initial import
Don Kukral authored
318 population = models.IntegerField(blank=True, null=True) # from the 2000 Census
319 user_id = models.IntegerField(blank=True, null=True)
320 is_public = models.BooleanField()
321 description = models.TextField(blank=True)
322 creation_date = models.DateTimeField(blank=True, null=True)
323 last_mod_date = models.DateTimeField(blank=True, null=True)
baec0a7 @slinkp Use natural keys for serialization wherever possible; it makes fixtures ...
slinkp authored
324 objects = LocationManager()
5c9826f initial import
Don Kukral authored
325
1334338 @slinkp Remove Location.centroid, we can just use Location.location.centroid ins...
slinkp authored
326 @property
327 def centroid(self):
328 # For backward compatibility.
329 import warnings
330 warnings.warn(
331 "Location.centroid is deprecated. Use Location.location.centroid instead.",
332 DeprecationWarning)
333 return self.location.centroid
334
e3056db @slinkp Use SlugField for slugs.
slinkp authored
335 def clean(self):
a9a4d9f @slinkp ValueError -> ValidatonError to make admin form nicer
slinkp authored
336 try:
337 self.location = ensure_valid(flatten_geomcollection(self.location))
338 except ValueError, e:
339 raise ValidationError(str(e))
340
5c9826f initial import
Don Kukral authored
341 class Meta:
342 unique_together = (('slug', 'location_type'),)
a11a8b0 @slinkp Lots more columns, filters, and usable default ordering in admin UI; clo...
slinkp authored
343 ordering = ('slug',)
5c9826f initial import
Don Kukral authored
344
baec0a7 @slinkp Use natural keys for serialization wherever possible; it makes fixtures ...
slinkp authored
345 def natural_key(self):
346 return (self.slug, self.location_type.slug)
347
5c9826f initial import
Don Kukral authored
348 def __unicode__(self):
349 return self.name
350
351 def url(self):
9c02185 @slinkp Change many generated URLs to use urlresolvers.reverse(). Closes #69
slinkp authored
352 return urlresolvers.reverse('ebpub-location-timeline',
2d83842 @slinkp De-hardcode more model URLs. Refs #69
slinkp authored
353 args=(self.location_type.slug, self.slug))
5c9826f initial import
Don Kukral authored
354
355 def rss_url(self):
2d83842 @slinkp De-hardcode more model URLs. Refs #69
slinkp authored
356 return urlresolvers.reverse('ebpub-location-rss',
357 args=(self.location_type.slug, self.slug))
5c9826f initial import
Don Kukral authored
358
359
2d83842 @slinkp De-hardcode more model URLs. Refs #69
slinkp authored
360 def alert_url(self):
361 return urlresolvers.reverse('ebpub-location-alerts',
362 args=(self.location_type.slug, self.slug))
5c9826f initial import
Don Kukral authored
363
364 # Give Location objects a "pretty_name" attribute for interoperability with
365 # Block objects. (Parts of our app accept either a Block or Location.)
764e1b9 @slinkp Use modern @property syntax; simplify SchemaField.is_type()
slinkp authored
366 @property
367 def pretty_name(self):
5c9826f initial import
Don Kukral authored
368 return self.name
369
764e1b9 @slinkp Use modern @property syntax; simplify SchemaField.is_type()
slinkp authored
370 @property
371 def is_custom(self):
5c9826f initial import
Don Kukral authored
372 return self.location_type.slug == 'custom'
764e1b9 @slinkp Use modern @property syntax; simplify SchemaField.is_type()
slinkp authored
373
5c9826f initial import
Don Kukral authored
374
220ca2c @ltucker Add some extra info to improve location and place tagging, split Misspel...
ltucker authored
375 class LocationSynonymManager(models.Manager):
376 def get_canonical(self, name):
377 """
378 Returns the canonical normalized spelling of the given location name.
379 If the given location name is already correctly spelled, then it's returned as-is.
380 """
381 try:
382 normalized_name = normalize(name)
383 return self.get(normalized_name=normalized_name).location.normalized_name
384 except self.model.DoesNotExist:
385 return normalized_name
386
387 class LocationSynonym(models.Model):
388 """
389 represents a synonym for a Location
390 """
391 pretty_name = models.CharField(max_length=255)
392 normalized_name = models.CharField(max_length=255, db_index=True)
393 location = models.ForeignKey(Location)
394 objects = LocationSynonymManager()
395
396 def save(self):
9238a42 @slinkp Also normalize LocationSynonyms, and document all this stuff. Closes #80
slinkp authored
397 # Not doing this in clean() because we really don't want there to be
398 # any way to get this wrong.
399 if self.normalized_name:
400 self.normalized_name = normalize(self.normalized_name)
401 else:
220ca2c @ltucker Add some extra info to improve location and place tagging, split Misspel...
ltucker authored
402 self.normalized_name = normalize(self.pretty_name)
403 super(LocationSynonym, self).save()
404
405 def __unicode__(self):
406 return self.pretty_name
407
5c9826f initial import
Don Kukral authored
408 class AttributesDescriptor(object):
409 """
410 This class provides the functionality that makes the attributes available
411 as `attributes` on a model instance.
412 """
413 def __get__(self, instance, instance_type=None):
414 if instance is None:
415 raise AttributeError("%s must be accessed via instance" % self.__class__.__name__)
416 if not hasattr(instance, '_attributes_cache'):
a726543 @ltucker be nicer when there are no schemafields
ltucker authored
417 select_dict = field_mapping([instance.schema_id]).get(instance.schema_id, {})
5c9826f initial import
Don Kukral authored
418 instance._attributes_cache = AttributeDict(instance.id, instance.schema_id, select_dict)
419 return instance._attributes_cache
420
421 def __set__(self, instance, value):
422 if instance is None:
423 raise AttributeError("%s must be accessed via instance" % self.__class__.__name__)
424 if not isinstance(value, dict):
425 raise ValueError('Only a dictionary is allowed')
78fac6b @slinkp Rename openblockapi.authorization to openblockapi.auth; also fix a KeyEr...
slinkp authored
426 mapping = field_mapping([instance.schema_id]).get(instance.schema_id, {}).items()
427 if not mapping:
428 if value:
429 logger.warn("Can't save non-empty attributes dict with an empty schema")
430 return
5c9826f initial import
Don Kukral authored
431 values = [value.get(k, None) for k, v in mapping]
432 cursor = connection.cursor()
433 cursor.execute("""
434 UPDATE %s
435 SET %s
436 WHERE news_item_id = %%s
437 """ % (Attribute._meta.db_table, ','.join(['%s=%%s' % v for k, v in mapping])),
438 values + [instance.id])
439 # If no records were updated, that means the DB doesn't yet have a
440 # row in the attributes table for this news item. Do an INSERT.
441 if cursor.rowcount < 1:
442 cursor.execute("""
443 INSERT INTO %s (news_item_id, schema_id, %s)
444 VALUES (%%s, %%s, %s)""" % (Attribute._meta.db_table, ','.join([v for k, v in mapping]), ','.join(['%s' for k in mapping])),
445 [instance.id, instance.schema_id] + values)
446 transaction.commit_unless_managed()
447
448 class AttributeDict(dict):
449 """
450 A dictionary-like object that serves as a wrapper around attributes for a
451 given NewsItem.
452 """
453 def __init__(self, news_item_id, schema_id, mapping):
454 dict.__init__(self)
455 self.news_item_id = news_item_id
456 self.schema_id = schema_id
457 self.mapping = mapping # name -> real_name dictionary
458 self.cached = False
459
460 def __do_query(self):
461 if not self.cached:
fb48fba @slinkp seeclickfix scraper now uses a new schema.
slinkp authored
462 attr_values = Attribute.objects.filter(news_item__id=self.news_item_id).extra(select=self.mapping).values(*self.mapping.keys())
463 # Rarely, we might have added the first SchemaField for this
464 # Schema *after* the NewsItem was scraped. In that case
465 # attr_values will be empty list.
466 if attr_values:
467 self.update(attr_values[0])
5c9826f initial import
Don Kukral authored
468 self.cached = True
469
ded7330 @slinkp Checking size of NewsItem.attributes is now reliable.
slinkp authored
470 def __len__(self):
471 # So len(self) and bool(self) work.
472 self.__do_query()
473 return dict.__len__(self)
474
338de4b @ltucker output extension schema fields in items_json
ltucker authored
475 def keys(self, *args, **kwargs):
476 self.__do_query()
477 return dict.keys(self, *args, **kwargs)
478
479 def items(self, *args, **kwargs):
480 self.__do_query()
481 return dict.items(self, *args, **kwargs)
482
5c9826f initial import
Don Kukral authored
483 def get(self, *args, **kwargs):
484 self.__do_query()
485 return dict.get(self, *args, **kwargs)
486
487 def __getitem__(self, name):
488 self.__do_query()
489 return dict.__getitem__(self, name)
490
491 def __setitem__(self, name, value):
188c1d2 @slinkp Much commenting on how NewsItems work
slinkp authored
492 # TODO: refactor, code overlaps largely with AttributeDescriptor.__set__
5c9826f initial import
Don Kukral authored
493 cursor = connection.cursor()
494 real_name = self.mapping[name]
495 cursor.execute("""
496 UPDATE %s
497 SET %s = %%s
498 WHERE news_item_id = %%s
499 """ % (Attribute._meta.db_table, real_name), [value, self.news_item_id])
500 # If no records were updated, that means the DB doesn't yet have a
501 # row in the attributes table for this news item. Do an INSERT.
502 if cursor.rowcount < 1:
503 cursor.execute("""
504 INSERT INTO %s (news_item_id, schema_id, %s)
505 VALUES (%%s, %%s, %%s)""" % (Attribute._meta.db_table, real_name),
506 [self.news_item_id, self.schema_id, value])
507 transaction.commit_unless_managed()
508 dict.__setitem__(self, name, value)
509
510 class NewsItemQuerySet(models.query.GeoQuerySet):
b3fcffa @slinkp Fix #146: breakage was introduced in changeset a11a8b00 when we added a ...
slinkp authored
511
5c9826f initial import
Don Kukral authored
512 def prepare_attribute_qs(self):
513 clone = self._clone()
514 if 'db_attribute' not in clone.query.extra_tables:
3b0dc05 @slinkp Update stuff using Query.extra_where to use QuerySet.extra(where...). A...
slinkp authored
515 clone = clone.extra(tables=('db_attribute',))
516 # extra_where went away in Django 1.1.
517 # This seems to be the correct replacement as per
518 # http://docs.djangoproject.com/en/dev/ref/models/querysets/
519 clone = clone.extra(where=('db_newsitem.id = db_attribute.news_item_id',))
5c9826f initial import
Don Kukral authored
520 return clone
521
522 def by_attribute(self, schema_field, att_value, is_lookup=False):
523 """
524 Returns a QuerySet of NewsItems whose attribute value for the given
525 SchemaField is att_value. If att_value is a list, this will do the
526 equivalent of an "OR" search, returning all NewsItems that have an
527 attribute value in the att_value list.
528
529 This handles many-to-many lookups correctly behind the scenes.
530
531 If is_lookup is True, then att_value is treated as the 'code' of a
532 Lookup object, and the Lookup's ID will be retrieved for use in the
533 query.
534 """
a8bc11d @slinkp More hacking on import scripts: factoring into separate scripts, adding ...
slinkp authored
535
5c9826f initial import
Don Kukral authored
536 clone = self.prepare_attribute_qs()
537 real_name = str(schema_field.real_name)
538 if not isinstance(att_value, (list, tuple)):
539 att_value = [att_value]
540 if is_lookup:
c0a594d @slinkp Replace SchemaField.name with SchemaField.slug, since it was already a p...
slinkp authored
541 if not isinstance(att_value[0], Lookup):
542 # Assume all are Lookup.code values.
543 att_value = Lookup.objects.filter(schema_field__id=schema_field.id, code__in=att_value)
5c9826f initial import
Don Kukral authored
544 if not att_value:
545 # If the lookup values don't exist, then there aren't any
546 # NewsItems with this attribute value. Note that we aren't
547 # using QuerySet.none() here, because we want the result to
548 # be a NewsItemQuerySet, and none() returns a normal QuerySet.
3b0dc05 @slinkp Update stuff using Query.extra_where to use QuerySet.extra(where...). A...
slinkp authored
549 clone = clone.extra(where=('1=0',))
5c9826f initial import
Don Kukral authored
550 return clone
551 att_value = [val.id for val in att_value]
552 if schema_field.is_many_to_many_lookup():
553 # We have to use a regular expression search to look for all rows
554 # with the given att_value *somewhere* in the column. The [[:<:]]
555 # thing is a word boundary.
556 for value in att_value:
557 if not str(value).isdigit():
558 raise ValueError('Only integer strings allowed for att_value in many-to-many SchemaFields')
3b0dc05 @slinkp Update stuff using Query.extra_where to use QuerySet.extra(where...). A...
slinkp authored
559 clone = clone.extra(where=("db_attribute.%s ~ '[[:<:]]%s[[:>:]]'" % (real_name, '|'.join([str(val) for val in att_value])),))
5c9826f initial import
Don Kukral authored
560 elif None in att_value:
561 if att_value != [None]:
562 raise ValueError('by_attribute() att_value list cannot have more than one element if it includes None')
3b0dc05 @slinkp Update stuff using Query.extra_where to use QuerySet.extra(where...). A...
slinkp authored
563 clone = clone.extra(where=("db_attribute.%s IS NULL" % real_name,))
5c9826f initial import
Don Kukral authored
564 else:
3b0dc05 @slinkp Update stuff using Query.extra_where to use QuerySet.extra(where...). A...
slinkp authored
565 clone = clone.extra(where=("db_attribute.%s IN (%s)" % (real_name, ','.join(['%s' for val in att_value])),),
566 params=tuple(att_value))
5c9826f initial import
Don Kukral authored
567 return clone
568
569 def date_counts(self):
570 """
571 Returns a dictionary mapping {item_date: count}.
572 """
573 # TODO: values + annotate doesn't seem to play nice with GeoQuerySet
574 # at the moment. This is the changeset where it broke:
575 # http://code.djangoproject.com/changeset/10326
576 from django.db.models.query import QuerySet
577 qs = QuerySet.values(self, 'item_date').annotate(count=models.Count('id'))
578 return dict([(v['item_date'], v['count']) for v in qs])
579
580 def top_lookups(self, schema_field, count):
581 """
582 Returns a list of {lookup, count} dictionaries representing the top
583 Lookups for this QuerySet.
584 """
585 real_name = "db_attribute." + str(schema_field.real_name)
586 if schema_field.is_many_to_many_lookup():
b3fcffa @slinkp Fix #146: breakage was introduced in changeset a11a8b00 when we added a ...
slinkp authored
587 # First prepare a subquery to get a *single* count of
588 # attribute rows that match each relevant m2m lookup
589 # value. It's very important to get a single row here or
590 # else we get a DataBaseError with "more than one row
591 # returned by a subquery used as an expression". (Bug #146)
592 clone = self.prepare_attribute_qs()
593 clone = clone.filter(schema__id=schema_field.schema_id)
594 # This is a regex search for the lookup id.
5c9826f initial import
Don Kukral authored
595 clone = clone.extra(where=[real_name + " ~ ('[[:<:]]' || db_lookup.id || '[[:>:]]')"])
596 # We want to count the current queryset and get a single
597 # row for injecting into the subsequent Lookup query, but
598 # we don't want Django's aggregation support to
599 # automatically group by fields that aren't relevant and
600 # would cause multiple rows as a result. So we call
601 # `values()' on a field that we're already filtering by,
602 # in this case, schema, as essentially a harmless identify
603 # function.
b3fcffa @slinkp Fix #146: breakage was introduced in changeset a11a8b00 when we added a ...
slinkp authored
604 # See http://docs.djangoproject.com/en/dev/topics/db/aggregation/#values
605 clone = clone.values('schema')
606
607 # Fix #146: Having any `ORDER BY foo` in this subquery causes
608 # Django to also add a `GROUP BY foo`, which potentially
609 # returns multiple rows. So, remove the ordering.
610 clone = clone.order_by()
611 clone = clone.annotate(count=Count('schema'))
612 # Unusual: We don't run the clone query, we just stuff its SQL
613 # into our Lookup qs.
5c9826f initial import
Don Kukral authored
614 qs = Lookup.objects.filter(schema_field__id=schema_field.id)
615 qs = qs.extra(select={'lookup_id': 'id', 'item_count': clone.values('count').query})
616 else:
b3fcffa @slinkp Fix #146: breakage was introduced in changeset a11a8b00 when we added a ...
slinkp authored
617 # Counts of attribute rows matching each relevant Lookup.
618 # Much easier when is_many_to_many_lookup == False :-)
5c9826f initial import
Don Kukral authored
619 qs = self.prepare_attribute_qs().extra(select={'lookup_id': real_name})
620 qs.query.group_by = [real_name]
621 qs = qs.values('lookup_id').annotate(item_count=Count('id'))
b3fcffa @slinkp Fix #146: breakage was introduced in changeset a11a8b00 when we added a ...
slinkp authored
622
623 qs = qs.values('lookup_id', 'item_count').order_by('-item_count')
624 ids_and_counts = [(v['lookup_id'], v['item_count']) for v in qs
625 if v['item_count']]
626 ids_and_counts = ids_and_counts[:count]
5c9826f initial import
Don Kukral authored
627 lookup_objs = Lookup.objects.in_bulk([i[0] for i in ids_and_counts])
b3fcffa @slinkp Fix #146: breakage was introduced in changeset a11a8b00 when we added a ...
slinkp authored
628 return [{'lookup': lookup_objs[i[0]], 'count': i[1]} for i in ids_and_counts
629 if not None in i]
5c9826f initial import
Don Kukral authored
630
631 def text_search(self, schema_field, query):
632 """
633 Returns a QuerySet of NewsItems whose attribute for
634 a given schema field matches a text search query.
635 """
636 clone = self.prepare_attribute_qs()
637 query = query.lower()
3b0dc05 @slinkp Update stuff using Query.extra_where to use QuerySet.extra(where...). A...
slinkp authored
638
639 clone = clone.extra(where=("db_attribute." + str(schema_field.real_name) + " ILIKE %s",),
640 params=("%%%s%%" % query,))
5c9826f initial import
Don Kukral authored
641 return clone
642
643 class NewsItemManager(models.GeoManager):
644 def get_query_set(self):
645 return NewsItemQuerySet(self.model)
646
647 def by_attribute(self, *args, **kwargs):
648 return self.get_query_set().by_attribute(*args, **kwargs)
649
650 def text_search(self, *args, **kwargs):
651 return self.get_query_set().text_search(*args, **kwargs)
652
653 def date_counts(self, *args, **kwargs):
654 return self.get_query_set().date_counts(*args, **kwargs)
655
656 def top_lookups(self, *args, **kwargs):
657 return self.get_query_set().top_lookups(*args, **kwargs)
658
659 class NewsItem(models.Model):
188c1d2 @slinkp Much commenting on how NewsItems work
slinkp authored
660 """
661 Lowest common denominator metadata for News-like things.
662
663 self.schema and self.attributes are used for extended metadata;
664 If all you want is to examine the attributes, self.attributes
665 can be treated like a dict.
666 (Internally it's a bit complicated. See the Schema, SchemaField, and
667 Attribute models, plus AttributeDescriptor, for how it all works.)
668
669 NewsItems have several distinct notions of location:
670
671 * The NewsItemLocation model is for fast lookups of NewsItems to
672 all Locations where the .location fields overlap. This is set
673 by a sql trigger whenever self.location changes; not set by any
674 python code. Used in various views for filtering.
675
676 * self.location is typically a point, and is used in views for
677 filtering newsitems. Theoretically (untested!!) could also be a
678 GeometryCollection, for news items that mention multiple
679 places. This is typically set during scraping, by geocoding if
680 not provided in the source data.
681
682 * self.location_object is a Location and a) is usually Null in
683 practice, and b) is only needed by self.location_url(), so we
684 can link back to a location view from a newsitem view. It would
685 be set during scraping. (Example use case: NYC crime
686 aggregates, where there's no location or address data for the
687 "news item" other than which precinct it occurs in.
688 eg. http://nyc.everyblock.com/crime/by-date/2010/8/23/3364632/ )
689
690 * self.block is optionally one Block. Also set during
691 scraping/geocoding. So far can't find anything that actually
692 uses these.
693
4e12c6e @slinkp Add more help text to NewsItem fields... spread the knowledge
slinkp authored
694 * self.location_name is a human-readable version of the location;
695 it can be anything, but typically it describes an address,
696 block, geographic area, or landmark.
697
188c1d2 @slinkp Much commenting on how NewsItems work
slinkp authored
698 """
699
baec0a7 @slinkp Use natural keys for serialization wherever possible; it makes fixtures ...
slinkp authored
700 # We don't have a natural_key() method because we don't know for
701 # sure that anything other than ID will be unique.
702
5c9826f initial import
Don Kukral authored
703 schema = models.ForeignKey(Schema)
704 title = models.CharField(max_length=255)
705 description = models.TextField()
4e12c6e @slinkp Add more help text to NewsItem fields... spread the knowledge
slinkp authored
706 url = models.TextField(
707 blank=True,
708 help_text="link to original source for this news")
c59165d @slinkp Add a bit of help text to NewsItem models.
slinkp authored
709 pub_date = models.DateTimeField(
710 db_index=True,
711 help_text='Date/time this Item was added to the OpenBlock site.') # TODO: default to now()
712 item_date = models.DateField(
713 db_index=True,
714 help_text='Date (no time) this Item occurred, or was published on the original source site.') # TODO: default to now()
4e12c6e @slinkp Add more help text to NewsItem fields... spread the knowledge
slinkp authored
715 location = models.GeometryField(blank=True, null=True, spatial_index=True,
716 help_text="Coordinates where this news occurred.")
717 location_name = models.CharField(max_length=255,
718 help_text="Human-readable address or name of place where this news item occurred.")
719 location_object = models.ForeignKey(Location, blank=True, null=True,
720 help_text="Optional reference to a Location where this item occurred")
721 block = models.ForeignKey(Block, blank=True, null=True,
722 help_text="Optional reference to a Block. Not really used")
723
5c9826f initial import
Don Kukral authored
724 objects = NewsItemManager()
188c1d2 @slinkp Much commenting on how NewsItems work
slinkp authored
725 attributes = AttributesDescriptor() # Treat it like a dict.
5c9826f initial import
Don Kukral authored
726
5309287 @slinkp Overhaul of olwidget integration.
slinkp authored
727
728 def clean(self):
729 self.location = ensure_valid(flatten_geomcollection(self.location))
730
a11a8b0 @slinkp Lots more columns, filters, and usable default ordering in admin UI; clo...
slinkp authored
731 class Meta:
732 ordering = ('title',)
733
5c9826f initial import
Don Kukral authored
734 def __unicode__(self):
92b5f2b @ltucker display "Untitled News Item" if item has no title, fixes non-linked item...
ltucker authored
735 return self.title or 'Untitled News Item'
5c9826f initial import
Don Kukral authored
736
737 def item_url(self):
2d83842 @slinkp De-hardcode more model URLs. Refs #69
slinkp authored
738 return urlresolvers.reverse('ebpub-newsitem-detail',
739 args=[self.schema.slug, self.id], kwargs={})
5c9826f initial import
Don Kukral authored
740
741 def item_url_with_domain(self):
742 from django.conf import settings
2aa82d7 @slinkp on second thought, EB_DOMAIN and EB_FULL_DOMAIN have no reason to be dis...
slinkp authored
743 return 'http://%s%s' % (settings.EB_DOMAIN, self.item_url())
5c9826f initial import
Don Kukral authored
744
745 def item_date_url(self):
5d323be @slinkp Remove one more ad-hoc URL generator. Refs #69
slinkp authored
746 from ebpub.db.schemafilters import FilterChain
747 chain = FilterChain(schema=self.schema)
748 chain.add('date', self.item_date)
749 return chain.make_url()
5c9826f initial import
Don Kukral authored
750
751 def location_url(self):
752 if self.location_object_id is not None:
753 return self.location_object.url()
754 return None
755
756 def attributes_for_template(self):
757 """
758 Return a list of AttributeForTemplate objects for this NewsItem. The
759 objects are ordered by SchemaField.display_order.
760 """
761 fields = SchemaField.objects.filter(schema__id=self.schema_id).select_related().order_by('display_order')
3403be1 added check to validate fields exists
Everyblock User authored
762 if not fields:
763 return []
f65f953 @slinkp Fix two errors triggered when NewsItems are saved for a schema with no s...
slinkp authored
764 if not self.attributes:
765 logger.warn("%s has fields in its schema, but no attributes!" % self)
5c9826f initial import
Don Kukral authored
766 return []
a7b3132 @slinkp Add tests for populate_attributes_if_needed. And fix #72. Mostly saner...
slinkp authored
767 return [AttributeForTemplate(f, self.attributes) for f in fields]
5c9826f initial import
Don Kukral authored
768
de7c1fe @slinkp Yet more de-hardcoded URLs.
slinkp authored
769
5c9826f initial import
Don Kukral authored
770 class AttributeForTemplate(object):
684c02e @slinkp Remove unneeded SchemaFieldInfo model. Refs #50. help_text wasn't even u...
slinkp authored
771 def __init__(self, schema_field, attribute_row):
5c9826f initial import
Don Kukral authored
772 self.sf = schema_field
2d4bb55 @slinkp Partial revert of c0a594d69e89ba31b7a57 - removing "name" was a bad idea...
slinkp authored
773 self.raw_value = attribute_row[schema_field.name]
5c9826f initial import
Don Kukral authored
774 self.schema_slug = schema_field.schema.slug
775 self.is_lookup = schema_field.is_lookup
776 self.is_filter = schema_field.is_filter
777 if self.is_lookup:
a7b3132 @slinkp Add tests for populate_attributes_if_needed. And fix #72. Mostly saner...
slinkp authored
778 # Earlier queries may have already looked up Lookup instances.
779 # Don't do unnecessary work.
780 if isinstance(self.raw_value, Lookup):
781 self.values = [self.raw_value]
782 elif (isinstance(self.raw_value, list) and self.raw_value
783 and isinstance(self.raw_value[0], Lookup)):
784 self.values = self.raw_values
785 elif self.raw_value == '':
5c9826f initial import
Don Kukral authored
786 self.values = []
787 elif self.sf.is_many_to_many_lookup():
788 try:
789 id_values = map(int, self.raw_value.split(','))
790 except ValueError:
791 self.values = []
792 else:
793 lookups = Lookup.objects.in_bulk(id_values)
794 self.values = [lookups[i] for i in id_values]
795 else:
796 self.values = [Lookup.objects.get(id=self.raw_value)]
797 else:
798 self.values = [self.raw_value]
799
800 def value_list(self):
801 """
9b9be15 @slinkp more comments on non-obvious code
slinkp authored
802 Returns a list of {value, url, description} dictionaries
803 representing each value for this attribute.
5c9826f initial import
Don Kukral authored
804 """
805 from django.utils.dateformat import format, time_format
9b9be15 @slinkp more comments on non-obvious code
slinkp authored
806 # Setting these to [None] ensures that zip() returns a list
807 # of at least length one.
5c9826f initial import
Don Kukral authored
808 urls = [None]
809 descriptions = [None]
810 if self.is_filter:
6a39ff1 @slinkp * Rename SchemaFilterChain to FilterChain - schema is optional now.
slinkp authored
811 from ebpub.db.schemafilters import FilterChain
812 chain = FilterChain(schema=self.sf.schema)
de7c1fe @slinkp Yet more de-hardcoded URLs.
slinkp authored
813 chain.base_url = self.sf.schema.url()
5c9826f initial import
Don Kukral authored
814 if self.is_lookup:
de7c1fe @slinkp Yet more de-hardcoded URLs.
slinkp authored
815 urls = [chain.replace(self.sf, look).make_url() if look else None
816 for look in self.values]
817 else:
818 urls = [chain.replace(self.sf, self.raw_value).make_url()]
5c9826f initial import
Don Kukral authored
819 if self.is_lookup:
820 values = [val and val.name or 'None' for val in self.values]
821 descriptions = [val and val.description or None for val in self.values]
822 elif isinstance(self.raw_value, datetime.datetime):
823 values = [format(self.raw_value, 'F j, Y, P')]
824 elif isinstance(self.raw_value, datetime.date):
825 values = [format(self.raw_value, 'F j, Y')]
826 elif isinstance(self.raw_value, datetime.time):
827 values = [time_format(self.raw_value, 'P')]
828 elif self.raw_value is True:
829 values = ['Yes']
830 elif self.raw_value is False:
831 values = ['No']
832 elif self.raw_value is None:
833 values = ['N/A']
834 else:
835 values = [self.raw_value]
836 return [{'value': value, 'url': url, 'description': description} for value, url, description in zip(values, urls, descriptions)]
837
838 class Attribute(models.Model):
188c1d2 @slinkp Much commenting on how NewsItems work
slinkp authored
839 """
840 Extended metadata for NewsItems.
841
842 Each row contains all the extra metadata for one NewsItem
843 instance. The field names are generic, so in order to know what
844 they mean, you must look at the SchemaFields for the Schema for
845 that NewsItem. eg. newsitem.
846
847 """
476e10c @slinkp Changing Attributes.newsitem from ForeignKey to OneToOneField. Not clear...
slinkp authored
848 news_item = models.OneToOneField(NewsItem, primary_key=True, unique=True)
5c9826f initial import
Don Kukral authored
849 schema = models.ForeignKey(Schema)
850 # All data-type field names must end in two digits, because the code assumes this.
e229d7a @ltucker add a bit more wiggle room in the attributes for larger fields since it ...
ltucker authored
851 varchar01 = models.CharField(max_length=4096, blank=True, null=True)
852 varchar02 = models.CharField(max_length=4096, blank=True, null=True)
853 varchar03 = models.CharField(max_length=4096, blank=True, null=True)
854 varchar04 = models.CharField(max_length=4096, blank=True, null=True)
855 varchar05 = models.CharField(max_length=4096, blank=True, null=True)
5c9826f initial import
Don Kukral authored
856 date01 = models.DateField(blank=True, null=True)
857 date02 = models.DateField(blank=True, null=True)
858 date03 = models.DateField(blank=True, null=True)
859 date04 = models.DateField(blank=True, null=True)
860 date05 = models.DateField(blank=True, null=True)
861 time01 = models.TimeField(blank=True, null=True)
862 time02 = models.TimeField(blank=True, null=True)
863 datetime01 = models.DateTimeField(blank=True, null=True)
864 datetime02 = models.DateTimeField(blank=True, null=True)
865 datetime03 = models.DateTimeField(blank=True, null=True)
866 datetime04 = models.DateTimeField(blank=True, null=True)
867 bool01 = models.NullBooleanField(blank=True)
868 bool02 = models.NullBooleanField(blank=True)
869 bool03 = models.NullBooleanField(blank=True)
870 bool04 = models.NullBooleanField(blank=True)
871 bool05 = models.NullBooleanField(blank=True)
872 int01 = models.IntegerField(blank=True, null=True)
873 int02 = models.IntegerField(blank=True, null=True)
874 int03 = models.IntegerField(blank=True, null=True)
875 int04 = models.IntegerField(blank=True, null=True)
876 int05 = models.IntegerField(blank=True, null=True)
877 int06 = models.IntegerField(blank=True, null=True)
878 int07 = models.IntegerField(blank=True, null=True)
879 text01 = models.TextField(blank=True, null=True)
e229d7a @ltucker add a bit more wiggle room in the attributes for larger fields since it ...
ltucker authored
880 text02 = models.TextField(blank=True, null=True)
5c9826f initial import
Don Kukral authored
881
882 def __unicode__(self):
883 return u'Attributes for news item %s' % self.news_item_id
884
885 class LookupManager(models.Manager):
baec0a7 @slinkp Use natural keys for serialization wherever possible; it makes fixtures ...
slinkp authored
886
887 def get_by_natural_key(self, slug, schema_field__slug,
888 schema_field__real_name):
889 return self.get(slug=slug, schema_field__slug=schema_field__slug,
890 schema_field__real_name=schema_field__real_name)
891
5c9826f initial import
Don Kukral authored
892 def get_or_create_lookup(self, schema_field, name, code=None, description='', make_text_slug=True, logger=None):
893 """
894 Returns the Lookup instance matching the given SchemaField, name and
895 Lookup.code, creating it (with the given name/code/description) if it
896 doesn't already exist.
897
898 If make_text_slug is True, then a slug will be created from the given
899 name. If it's False, then the slug will be the Lookup's ID.
900 """
901 def log_info(message):
902 if logger is None:
903 return
904 logger.info(message)
905 def log_warn(message):
906 if logger is None:
907 return
908 logger.warn(message)
909 code = code or name # code defaults to name if it wasn't provided
910 try:
911 obj = Lookup.objects.get(schema_field__id=schema_field.id, code=code)
912 except Lookup.DoesNotExist:
913 if make_text_slug:
914 slug = slugify(name)
915 if len(slug) > 32:
916 # Only bother to warn if we're actually going to use the slug.
917 if make_text_slug:
918 log_warn("Trimming slug %r to %r in order to fit 32-char limit." % (slug, slug[:32]))
919 slug = slug[:32]
920 else:
921 # To avoid integrity errors in the slug when creating the Lookup,
922 # use a temporary dummy slug that's guaranteed not to be in use.
923 # We'll change it back immediately afterward.
924 slug = '__3029j3f029jf029jf029__'
925 if len(name) > 255:
926 old_name = name
927 name = name[:250] + '...'
928 # Save the full name in the description.
929 if not description:
930 description = old_name
931 log_warn("Trimming name %r to %r in order to fit 255-char limit." % (old_name, name))
932 obj = Lookup(schema_field_id=schema_field.id, name=name, code=code, slug=slug, description=description)
933 obj.save()
934 if not make_text_slug:
935 # Set the slug to the ID.
936 obj.slug = obj.id
937 obj.save()
938 log_info('Created %s %r' % (schema_field.name, name))
939 return obj
940
941 class Lookup(models.Model):
942 schema_field = models.ForeignKey(SchemaField)
943 name = models.CharField(max_length=255)
944 # `code` is the optional internal code to use during retrieval.
945 # For example, in scraping Chicago crimes, we use the crime type code
946 # to find the appropriate crime type in this table. We can't use `name`
947 # in that case, because we've massaged `name` to use a "prettier"
948 # formatting than exists in the data source.
949 code = models.CharField(max_length=255, blank=True)
e3056db @slinkp Use SlugField for slugs.
slinkp authored
950 slug = models.SlugField(max_length=32, db_index=True)
5c9826f initial import
Don Kukral authored
951 description = models.TextField(blank=True)
952
953 objects = LookupManager()
954
955 class Meta:
956 unique_together = (('slug', 'schema_field'),)
a11a8b0 @slinkp Lots more columns, filters, and usable default ordering in admin UI; clo...
slinkp authored
957 ordering = ('slug',)
5c9826f initial import
Don Kukral authored
958
baec0a7 @slinkp Use natural keys for serialization wherever possible; it makes fixtures ...
slinkp authored
959 def natural_key(self):
960 return (self.slug, self.schema_field.schema.slug,
961 self.schema_field.real_name)
962
5c9826f initial import
Don Kukral authored
963 def __unicode__(self):
964 return u'%s - %s' % (self.schema_field, self.name)
965
966 class NewsItemLocation(models.Model):
967 news_item = models.ForeignKey(NewsItem)
968 location = models.ForeignKey(Location)
969
970 class Meta:
971 unique_together = (('news_item', 'location'),)
972
973 def __unicode__(self):
974 return u'%s - %s' % (self.news_item, self.location)
975
976 class AggregateBaseClass(models.Model):
977 schema = models.ForeignKey(Schema)
978 total = models.IntegerField()
979
980 class Meta:
981 abstract = True
982
983 class AggregateAll(AggregateBaseClass):
984 # Total items in the schema.
985 pass
986
987 class AggregateDay(AggregateBaseClass):
988 # Total items in the schema with item_date on the given day
989 date_part = models.DateField(db_index=True)
990
991 class AggregateLocation(AggregateBaseClass):
992 # Total items in the schema in location, summed over that last 30 days
993 location_type = models.ForeignKey(LocationType)
994 location = models.ForeignKey(Location)
995
996 class AggregateLocationDay(AggregateBaseClass):
997 # Total items in the schema in location with item_date on the given day
998 location_type = models.ForeignKey(LocationType)
999 location = models.ForeignKey(Location)
1000 date_part = models.DateField(db_index=True)
1001
1002 class AggregateFieldLookup(AggregateBaseClass):
1003 # Total items in the schema with schema_field's value = lookup
1004 schema_field = models.ForeignKey(SchemaField)
1005 lookup = models.ForeignKey(Lookup)
1006
1007 class SearchSpecialCase(models.Model):
1008 query = models.CharField(max_length=64, unique=True)
1009 redirect_to = models.CharField(max_length=255, blank=True)
1010 title = models.CharField(max_length=128, blank=True)
1011 body = models.TextField(blank=True)
1012
1013 def __unicode__(self):
1014 return self.query
1015
1016 class DataUpdate(models.Model):
1017 # Keeps track of each time we update our data.
1018 schema = models.ForeignKey(Schema)
1019 update_start = models.DateTimeField() # When the scraper/importer started running.
1020 update_finish = models.DateTimeField() # When the scraper/importer finished.
1021 num_added = models.IntegerField()
1022 num_changed = models.IntegerField()
1023 num_deleted = models.IntegerField()
1024 num_skipped = models.IntegerField()
1025 got_error = models.BooleanField()
1026
1027 def __unicode__(self):
1028 return u'%s started on %s' % (self.schema.name, self.update_start)
1029
1030 def total_time(self):
1031 return self.update_finish - self.update_start
5309287 @slinkp Overhaul of olwidget integration.
slinkp authored
1032
Something went wrong with that request. Please try again.