Merge 58595b0 into 0a92a1a

pupa/importers/base.py

@@ -10,7 +10,7 @@
 from pupa.exceptions import DuplicateItemError
 from pupa.utils import get_pseudo_id, utcnow
 from pupa.exceptions import UnresolvedIdError, DataImportError
-
+from pupa.models import Identifier
 
 def omnihash(obj):
     """ recursively hash unhashable objects """
@@ -249,6 +249,9 @@ def import_item(self, data):
         except self.model_class.DoesNotExist:
             obj = None
 
+        # remove pupa_id which does not belong in the OCD data models
+        pupa_id = data.pop('pupa_id', None)
+
         # pull related fields off
         related = {}
         for field in self.related_models:
@@ -281,6 +284,14 @@ def import_item(self, data):
                                                                            self.model_class))
             self._create_related(obj, related, self.related_models)
 
+        if pupa_id:
+            try:
+                p_id = Identifier.objects.get(identifier = pupa_id)
+            except Identifier.DoesNotExist:
+                p_id = Identifier(identifier = pupa_id,
+                                  content_object = obj)
+                p_id.save()
+
         return obj.id, what
 
     def _update_related(self, obj, related, subfield_dict):

pupa/importers/events.py

@@ -1,6 +1,7 @@
 from .base import BaseImporter
 from pupa.utils import fix_bill_id, get_pseudo_id, _make_pseudo_id
 from pupa.utils.event import read_event_iso_8601
+from pupa.models import Identifier
 from opencivicdata.models import (Event, EventLocation, EventSource, EventDocument,
                                   EventDocumentLink, EventLink, EventParticipant, EventMedia,
                                   EventMediaLink, EventAgendaItem, EventRelatedEntity,
@@ -38,14 +39,21 @@ def __init__(self, jurisdiction_id, org_importer, person_importer, bill_importer
         self.vote_event_importer = vote_event_importer
 
     def get_object(self, event):
-        spec = {
-            'name': event['name'],
-            'description': event['description'],
-            'start_time': event['start_time'],
-            'end_time': event['end_time'],
-            'timezone': event['timezone'],
-            'jurisdiction_id': self.jurisdiction_id
-        }
+        if event.get('pupa_id'):
+            try:
+                e = Identifier.objects.get(identifier=event['pupa_id'])
+                spec = {'id': e.object_id}
+            except Identifier.DoesNotExist:
+                return None
+        else:
+            spec = {
+                'name': event['name'],
+                'description': event['description'],
+                'start_time': event['start_time'],
+                'end_time': event['end_time'],
+                'timezone': event['timezone'],
+                'jurisdiction_id': self.jurisdiction_id
+            }
         return self.model_class.objects.get(**spec)
 
     def get_location(self, location_data):

pupa/importers/vote_events.py

@@ -1,5 +1,6 @@
 from opencivicdata.models import VoteEvent, VoteCount, PersonVote, VoteSource, BillAction
 from pupa.utils import fix_bill_id, get_pseudo_id, _make_pseudo_id
+from ..models import Identifier
 
 from .base import BaseImporter
 from ..exceptions import InvalidVoteEventError
@@ -40,7 +41,13 @@ def get_object(self, vote_event):
                 )
             spec['bill_id'] = vote_event['bill_id']
 
-        if vote_event['identifier']:
+        if vote_event.get('pupa_id'):
+            try:
+                ve = Identifier.objects.get(identifier=vote_event['pupa_id'])
+                spec = {'id': ve.object_id}
+            except Identifier.DoesNotExist:
+                return None
+        elif vote_event['identifier']:
             # if there's an identifier, just use it and the bill_id and the session
             spec['identifier'] = vote_event['identifier']
         else:

pupa/migrations/0004_identifier.py

@@ -0,0 +1,26 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.10.5 on 2017-05-22 15:51
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('contenttypes', '0002_remove_content_type_name'),
+        ('pupa', '0003_auto_20151118_0408'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Identifier',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('identifier', models.CharField(max_length=500)),
+                ('object_id', models.PositiveIntegerField()),
+                ('content_type', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='contenttypes.ContentType')),
+            ],
+        ),
+    ]

pupa/migrations/0005_auto_20170522_1935.py

@@ -0,0 +1,25 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.10.5 on 2017-05-22 19:35
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('pupa', '0004_identifier'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='identifier',
+            name='identifier',
+            field=models.CharField(max_length=300),
+        ),
+        migrations.AlterField(
+            model_name='identifier',
+            name='object_id',
+            field=models.CharField(max_length=300),
+        ),
+    ]

pupa/models.py

@@ -1,4 +1,6 @@
 from django.db import models
+from django.contrib.contenttypes.fields import GenericForeignKey
+from django.contrib.contenttypes.models import ContentType    
 from opencivicdata.models import Jurisdiction
 
 
@@ -45,3 +47,12 @@ class ImportObjects(models.Model):
     noop_count = models.PositiveIntegerField()
     start_time = models.DateTimeField()
     end_time = models.DateTimeField()
+
+class Identifier(models.Model):
+    identifier = models.CharField(max_length=300)
+    content_type = models.ForeignKey(ContentType)
+    object_id = models.CharField(max_length=300)
+    content_object = GenericForeignKey('content_type', 'object_id')
+
+    def __str__(self):              # __unicode__ on Python 2
+        return self.identifier

pupa/scrape/schemas/event.py

@@ -179,6 +179,7 @@
         },
         "sources": sources,
         "extras": extras,
+        'pupa_id': {"type": ["string", "null"]},
     },
     "type": "object"
 }

pupa/scrape/schemas/vote_event.py

@@ -39,5 +39,6 @@
 
         'sources': sources,
         'extras': extras,
+        'pupa_id': {"type": ["string", "null"]},
     }
 }

pupa/settings.py

@@ -6,7 +6,9 @@
 
 DATABASE_URL = os.environ.get('DATABASE_URL', 'postgis://pupa:pupa@localhost/opencivicdata')
 SECRET_KEY = 'non-secret'
-INSTALLED_APPS = ('opencivicdata.apps.BaseConfig', 'pupa',)
+INSTALLED_APPS = ('django.contrib.contenttypes',
+                  'opencivicdata.apps.BaseConfig',
+                  'pupa')
 
 # scrape settings
 

pupa/tests/django_settings.py

@@ -1,6 +1,8 @@
 # django settings for tests
 SECRET_KEY = 'test'
-INSTALLED_APPS = ('opencivicdata.apps.BaseConfig',)
+INSTALLED_APPS = ('django.contrib.contenttypes',
+                  'opencivicdata.apps.BaseConfig',
+                  'pupa')
 DATABASES = {
     'default': {
         'ENGINE': 'django.contrib.gis.db.backends.postgis',

pupa/tests/importers/test_event_importer.py

@@ -195,6 +195,35 @@ def test_full_event():
     assert result['event']['update'] == 1
 
 
+@pytest.mark.django_db
+def test_pupa_identifier_event():
+    create_jurisdiction()
+    george = Person.objects.create(id='gw', name='George Washington')
+    o = Organization.objects.create(name='Merica', jurisdiction_id='jid')
+    Membership.objects.create(person=george, organization=o)
+
+    event = ge()
+    event.pupa_id = 'foo'
+
+    result = EventImporter('jid', oi, pi, bi, vei).import_data([event.as_dict()])
+    assert result['event']['insert'] == 1
+
+    result = EventImporter('jid', oi, pi, bi, vei).import_data([event.as_dict()])
+    assert result['event']['noop'] == 1
+
+    event.name="America's Anniversary",
+    event.location['name'] = "United States of America"
+    result = EventImporter('jid', oi, pi, bi, vei).import_data([event.as_dict()])
+    assert result['event']['update'] == 1
+
+    event.pupa_id = 'bar'
+    result = EventImporter('jid', oi, pi, bi, vei).import_data([event.as_dict()])
+    assert result['event']['insert'] == 1
+
+
+
+
+
 @pytest.mark.django_db
 def test_bad_event_time():
     create_jurisdiction()

pupa/tests/importers/test_vote_event_importer.py

@@ -106,6 +106,53 @@ def test_vote_event_identifier_dedupe():
     assert what == 'insert'
     assert VoteEvent.objects.count() == 2
 
+@pytest.mark.django_db
+def test_vote_event_pupa_identifier_dedupe():
+    j = create_jurisdiction()
+    j.legislative_sessions.create(name='1900', identifier='1900')
+    org = Organization.objects.create(id='org-id', name='Legislature',
+                                      classification='legislature',
+                                      jurisdiction=j)
+
+    vote_event = ScrapeVoteEvent(legislative_session='1900', start_date='2013',
+                                 classification='anything', result='passed',
+                                 motion_text='a vote on something',
+                                 identifier='Roll Call No. 1')
+    vote_event.pupa_id = 'foo'
+
+    dmi = DumbMockImporter()
+    oi = OrganizationImporter('jid')
+    bi = BillImporter('jid', dmi, oi)
+
+    _, what = VoteEventImporter('jid', dmi, oi, bi).import_item(vote_event.as_dict())
+    assert what == 'insert'
+    assert VoteEvent.objects.count() == 1
+
+    # same exact vote event, no changes
+    _, what = VoteEventImporter('jid', dmi, oi, bi).import_item(vote_event.as_dict())
+    assert what == 'noop'
+    assert VoteEvent.objects.count() == 1
+
+    # new info, update
+    vote_event.result = 'failed'
+    _, what = VoteEventImporter('jid', dmi, oi, bi).import_item(vote_event.as_dict())
+    assert what == 'update'
+    assert VoteEvent.objects.count() == 1
+
+    # new bill identifier, update
+    vote_event.identifier = 'First Roll Call'
+    _, what = VoteEventImporter('jid', dmi, oi, bi).import_item(vote_event.as_dict())
+    assert what == 'update'
+    assert VoteEvent.objects.count() == 1
+
+    # new pupa identifier, insert
+    vote_event.pupa_id = 'bar'
+    _, what = VoteEventImporter('jid', dmi, oi, bi).import_item(vote_event.as_dict())
+    assert what == 'insert'
+    assert VoteEvent.objects.count() == 2
+
+
+
 
 @pytest.mark.django_db
 def test_vote_event_bill_id_dedupe():