Updated a lot of things

Signed-off-by: Matheus Fernandes <matheus.souza.fernandes@gmail.com>

src/pygov_br/django_apps/base.py

@@ -1,4 +1,5 @@
 # -*- coding: utf-8 -*-
+from django.db import transaction
 import abc
 import click
 
@@ -43,6 +44,7 @@ def _get_object(self, data):
         self.after_save_object(obj)
         return obj
 
+    @transaction.atomic
     def _fill_model(self, model_class, data):
         obj_dict = {}
 
@@ -58,5 +60,7 @@ def _fill_model(self, model_class, data):
                 obj_dict[field] = cleaned_field
             except KeyError:
                 continue
+            except AttributeError:
+                continue
         obj, created = model_class.objects.get_or_create(**obj_dict)
         return obj

src/pygov_br/django_apps/camara_deputados/data_importer/deputy.py

@@ -15,6 +15,12 @@ class PartyImporter(BaseDataImporter):
     def get_model(self):
         return models.Party
 
+    def clean_initials(self, data):
+        queryset = models.Party.objects.filter(initials=data)
+        if queryset.exists():
+            return data + '*'
+        return data
+
     def get_data(self):
         return cd.deputies.parties()
 

src/pygov_br/django_apps/camara_deputados/data_importer/proposal.py

@@ -0,0 +1,68 @@
+from django.db import transaction
+from pygov_br.django_apps.base import BaseDataImporter
+from pygov_br.django_apps.camara_deputados import models
+from pygov_br.camara_deputados import cd
+
+
+class ProposalTypeImporter(BaseDataImporter):
+
+    field_relation = {
+        "id": "id",
+        "is_active": "ativa",
+        "description": "descricao",
+        "gender": "genero",
+        "initials": "tipoSigla",
+    }
+
+    def get_model(self):
+        return models.ProposalType
+
+    def get_data(self):
+        return cd.proposals.types()
+
+
+class ProposalImporter(BaseDataImporter):
+
+    field_relation = {
+        "id": "id",
+        "year": "ano",
+        "number": "numero",
+        "submission_date": "datApresentacao",
+        "name": "nome",
+        "summary": "txtEmenta",
+        "summary_explanation": "txtExplicacaoEmenta",
+        "proposal_type": "tipoProposicao",
+    }
+
+    def get_model(self):
+        return models.Proposal
+
+    def get_data(self):
+        return cd.proposals.filter(proposal_type='PL', year=2016)
+
+    def clean_author(self, data):
+        if data['idecadastro']:
+            return models.Deputy.objects.get(enrollment_id=data['idecadastro'])
+        else:
+            return None
+
+    def clean_proposal_type(self, data):
+        return models.ProposalType.objects.get(initials=data['sigla'])
+
+    @transaction.atomic
+    def after_save_object(self, obj):
+        extra_info = cd.proposals.get(proposal_type=obj.proposal_type.initials,
+                                      proposal_number=obj.number,
+                                      year=obj.year)
+        obj.indexes = extra_info['Indexacao']
+        obj.link = extra_info['LinkInteiroTeor']
+        if extra_info['ideCadastro']:
+            try:
+                author = models.Deputy.objects.get(
+                    id=extra_info['ideCadastro']
+                )
+                obj.author = author
+            except models.Deputy.DoesNotExist:
+                print('nao existe', extra_info['ideCadastro'])
+
+        obj.save()

src/pygov_br/django_apps/camara_deputados/data_importer/session.py

@@ -13,7 +13,7 @@ class SessionImporter(BaseDataImporter):
     }
 
     def get_data(self):
-        return cd.sessions.speeches('01/01/2015', '01/04/2015')
+        return cd.sessions.speeches('15/01/2016', '15/12/2016')
 
     def get_model(self):
         return models.Session
@@ -38,7 +38,7 @@ class SpeechImporter(BaseDataImporter):
     }
 
     def get_data(self):
-        sessions = cd.sessions.speeches('01/01/2015', '01/04/2015')
+        sessions = cd.sessions.speeches('15/01/2016', '15/12/2016')
         speeches_list = []
         for session in sessions:
             session_code = session['codigo']
@@ -54,13 +54,22 @@ def get_data(self):
                         }
             else:
                 speeches = phases['discursos']['discurso']
-                for speech in speeches:
+                if isinstance(speeches, list):
+                    for speech in speeches:
+                        speech['codigoSessao'] = session_code
+                        speech['faseSessao'] = {
+                            'codigo': phases['codigo'],
+                            'descricao': phases['descricao']
+                        }
+                    speeches_list += speeches
+                else:
+                    speech = speeches
                     speech['codigoSessao'] = session_code
                     speech['faseSessao'] = {
                         'codigo': phases['codigo'],
                         'descricao': phases['descricao']
                     }
-            speeches_list += speeches
+                    speeches_list.append(speech)
         return speeches_list
 
     def get_model(self):
@@ -85,11 +94,11 @@ def clean_session_phase(self, data):
     def clean_order(self, data):
         return data['numero']
 
-    def after_save_object(self, obj):
-        speech = cd.sessions.full_speech(
-            obj.session.code, obj.order,
-            obj.quarter_number, obj.insertion_number
-        )
-        if speech['discurso'] is not None:
-            obj.full_text = speech['discurso']
-            obj.save()
+    # def after_save_object(self, obj):
+    #     speech = cd.sessions.full_speech(
+    #         obj.session.code, obj.order,
+    #         obj.quarter_number, obj.insertion_number
+    #     )
+    #     if speech['discurso'] is not None:
+    #         obj.full_text = speech['discurso']
+    #         obj.save()

src/pygov_br/django_apps/camara_deputados/migrations/0002_auto_20170719_2250.py

@@ -0,0 +1,52 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11 on 2017-07-19 22:50
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('camara_deputados', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Proposal',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('year', models.IntegerField()),
+                ('number', models.IntegerField()),
+                ('submission_date', models.DateField()),
+                ('name', models.CharField(max_length=500)),
+                ('summary', models.TextField()),
+                ('summary_explanation', models.TextField()),
+                ('author', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='camara_deputados.Deputy')),
+            ],
+            options={
+                'verbose_name': 'Proposal',
+                'verbose_name_plural': 'Proposals',
+            },
+        ),
+        migrations.CreateModel(
+            name='ProposalType',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('is_active', models.BooleanField()),
+                ('description', models.CharField(max_length=500)),
+                ('gender', models.CharField(max_length=2)),
+                ('initials', models.CharField(max_length=50)),
+            ],
+            options={
+                'verbose_name': 'Proposal Type',
+                'verbose_name_plural': 'Proposal Types',
+            },
+        ),
+        migrations.AddField(
+            model_name='proposal',
+            name='proposal_type',
+            field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='camara_deputados.ProposalType'),
+        ),
+    ]

src/pygov_br/django_apps/camara_deputados/migrations/0003_auto_20170719_2307.py

@@ -0,0 +1,25 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11 on 2017-07-19 23:07
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('camara_deputados', '0002_auto_20170719_2250'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='proposal',
+            name='summary',
+            field=models.TextField(blank=True, null=True),
+        ),
+        migrations.AlterField(
+            model_name='proposal',
+            name='summary_explanation',
+            field=models.TextField(blank=True, null=True),
+        ),
+    ]

src/pygov_br/django_apps/camara_deputados/migrations/0004_auto_20170719_2314.py

@@ -0,0 +1,21 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11 on 2017-07-19 23:14
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('camara_deputados', '0003_auto_20170719_2307'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='proposal',
+            name='author',
+            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='camara_deputados.Deputy'),
+        ),
+    ]

src/pygov_br/django_apps/camara_deputados/migrations/0005_auto_20170719_2330.py

@@ -0,0 +1,25 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11 on 2017-07-19 23:30
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('camara_deputados', '0004_auto_20170719_2314'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='proposal',
+            name='indexes',
+            field=models.CharField(blank=True, max_length=500, null=True),
+        ),
+        migrations.AddField(
+            model_name='proposal',
+            name='link',
+            field=models.CharField(blank=True, max_length=500, null=True),
+        ),
+    ]

src/pygov_br/django_apps/camara_deputados/migrations/0006_auto_20170720_2347.py

@@ -0,0 +1,21 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11 on 2017-07-20 23:47
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('camara_deputados', '0005_auto_20170719_2330'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='proposal',
+            name='author',
+            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='proposals', to='camara_deputados.Deputy'),
+        ),
+    ]

src/pygov_br/django_apps/camara_deputados/models.py

@@ -201,6 +201,54 @@ def __str__(self):
     session = models.ForeignKey('Session', related_name='speeches')
     session_phase = models.ForeignKey('SessionPhase', related_name='speeches')
 
+    @property
+    def text_url(self):
+        url = 'http://www.camara.leg.br/internet/sitaqweb/TextoHTML.asp?' \
+              'etapa={}&nuSessao={}&nuQuarto={}&nuOrador={}&nuInsercao={}&' \
+              'dtHorarioQuarto={}'
+        return url.format(
+            self.session.number,
+            self.session.code,
+            self.quarter_number,
+            self.order,
+            self.insertion_number,
+            self.initial_time.strftime('%H:%M')
+        )
+
     def save(self, *args, **kwargs):
         if self.author is not None:
             super(Speech, self).save(*args, **kwargs)
+
+
+class ProposalType(models.Model):
+
+    class Meta:
+        verbose_name = "Proposal Type"
+        verbose_name_plural = "Proposal Types"
+
+    is_active = models.BooleanField()
+    description = models.CharField(max_length=500)
+    gender = models.CharField(max_length=2)
+    initials = models.CharField(max_length=50)
+
+    def __str__(self):
+        return self.description
+
+
+class Proposal(models.Model):
+
+    class Meta:
+        verbose_name = "Proposal"
+        verbose_name_plural = "Proposals"
+
+    year = models.IntegerField()
+    number = models.IntegerField()
+    author = models.ForeignKey(Deputy, blank=True, null=True,
+                               related_name='proposals')
+    submission_date = models.DateField()
+    name = models.CharField(max_length=500)
+    summary = models.TextField(blank=True, null=True)
+    summary_explanation = models.TextField(blank=True, null=True)
+    proposal_type = models.ForeignKey(ProposalType)
+    indexes = models.CharField(max_length=500, blank=True, null=True)
+    link = models.CharField(max_length=500, blank=True, null=True)