From acefa543922f19ab00e5b76bb2a53b23a8933328 Mon Sep 17 00:00:00 2001 From: aniversarioperu Date: Sun, 24 Jan 2016 21:10:31 -0500 Subject: [PATCH] use haystack for cazador --- manolo/apps/cazador/forms.py | 19 +++++ manolo/apps/cazador/management/__init__.py | 0 .../cazador/management/commands/__init__.py | 0 .../management/commands/import_data.py | 73 +++++++++++++++++++ .../apps/cazador/migrations/0001_initial.py | 21 ++++++ manolo/apps/cazador/models.py | 6 +- manolo/apps/cazador/search_indexes.py | 9 +++ .../cazador/templates/cazador/results.html | 4 +- manolo/apps/cazador/views.py | 20 +++-- manolo/apps/visitors/models.py | 1 - .../search/indexes/cazador/cazador_text.txt | 1 + requirements/base.txt | 1 + 12 files changed, 144 insertions(+), 11 deletions(-) create mode 100644 manolo/apps/cazador/forms.py create mode 100644 manolo/apps/cazador/management/__init__.py create mode 100644 manolo/apps/cazador/management/commands/__init__.py create mode 100644 manolo/apps/cazador/management/commands/import_data.py create mode 100644 manolo/apps/cazador/migrations/0001_initial.py create mode 100644 manolo/apps/cazador/search_indexes.py create mode 100644 manolo/templates/search/indexes/cazador/cazador_text.txt diff --git a/manolo/apps/cazador/forms.py b/manolo/apps/cazador/forms.py new file mode 100644 index 0000000..4fb6a2f --- /dev/null +++ b/manolo/apps/cazador/forms.py @@ -0,0 +1,19 @@ +from haystack.forms import HighlightedSearchForm + + +class CazadorForm(HighlightedSearchForm): + def search(self): + sqs = super(CazadorForm, self).search() + + if not self.is_valid(): + return self.no_query_found() + + if not self.cleaned_data.get('q'): + return self.no_query_found() + + sqs = self.searchqueryset.auto_query(self.cleaned_data['q']) + + if self.load_all: + sqs = sqs.load_all() + + return sqs diff --git a/manolo/apps/cazador/management/__init__.py b/manolo/apps/cazador/management/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/manolo/apps/cazador/management/commands/__init__.py b/manolo/apps/cazador/management/commands/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/manolo/apps/cazador/management/commands/import_data.py b/manolo/apps/cazador/management/commands/import_data.py new file mode 100644 index 0000000..7857918 --- /dev/null +++ b/manolo/apps/cazador/management/commands/import_data.py @@ -0,0 +1,73 @@ +"""Add deudores al Estado to index""" +from tqdm import tqdm + +from django.core.management.base import BaseCommand, CommandError + +from cazador.models import Cazador + + +DBS = ['deudores', 'candidato_2014', 'narcoindultos', 'redam'] + + +class Command(BaseCommand): + def add_arguments(self, parser): + parser.add_argument('-i', + action='store', + dest='input_file', + help='Enter filename with data to import.', + ) + parser.add_argument('-db', + action='store', + dest='database', + help='Enter database, options: {}'.format( + ', '.join(DBS))) + + def handle(self, *args, **options): + if not options['input_file']: + raise CommandError("Input Filename does not exist.") + if options['database'] not in DBS: + raise CommandError("Name of database is incorrect. Use one of the " + "following options: {}".format( + ', '.join(DBS))) + + input_file = options['input_file'] + database = options['database'] + self.process_file(input_file, database) + + def process_file(self, input_file, database): + print(input_file, database) + with open(input_file, "r") as handle: + data = handle.readlines() + + entries = [] + for raw_line in tqdm(data): + line = raw_line.strip() + fields = line.split("\t") + if database == 'candidato_2014': + c = Cazador( + raw_data=" ".join([ + fields[2], + fields[3], + fields[4], + fields[1], + fields[7], + ]), + source=database, + ) + elif database == 'redam': + c = Cazador( + raw_data=", ".join(fields), + source=database, + ) + elif database == 'narcoindultos': + c = Cazador( + raw_data=" ".join(fields), + source=database, + ) + else: + c = Cazador( + raw_data=" ".join([fields[2], fields[10], fields[1]]), + source=database, + ) + entries.append(c) + Cazador.objects.bulk_create(entries) diff --git a/manolo/apps/cazador/migrations/0001_initial.py b/manolo/apps/cazador/migrations/0001_initial.py new file mode 100644 index 0000000..68eeac5 --- /dev/null +++ b/manolo/apps/cazador/migrations/0001_initial.py @@ -0,0 +1,21 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import models, migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ] + + operations = [ + migrations.CreateModel( + name='Cazador', + fields=[ + ('id', models.AutoField(serialize=False, primary_key=True)), + ('raw_data', models.TextField()), + ('source', models.TextField()), + ], + ), + ] diff --git a/manolo/apps/cazador/models.py b/manolo/apps/cazador/models.py index bd4b2ab..8b9e350 100644 --- a/manolo/apps/cazador/models.py +++ b/manolo/apps/cazador/models.py @@ -2,4 +2,8 @@ from django.db import models -# Create your models here. + +class Cazador(models.Model): + id = models.AutoField(primary_key=True) + raw_data = models.TextField() + source = models.TextField() diff --git a/manolo/apps/cazador/search_indexes.py b/manolo/apps/cazador/search_indexes.py new file mode 100644 index 0000000..b40d5d2 --- /dev/null +++ b/manolo/apps/cazador/search_indexes.py @@ -0,0 +1,9 @@ +from haystack import indexes +from .models import Cazador + + +class CazadorIndex(indexes.SearchIndex, indexes.Indexable): + text = indexes.CharField(document=True, use_template=True) + + def get_model(self): + return Cazador diff --git a/manolo/apps/cazador/templates/cazador/results.html b/manolo/apps/cazador/templates/cazador/results.html index 4a0e8d1..c9cd7c3 100644 --- a/manolo/apps/cazador/templates/cazador/results.html +++ b/manolo/apps/cazador/templates/cazador/results.html @@ -24,8 +24,8 @@ {% for result in results %} - {{ result.1 }} - {{ result.3 }} + {{ result.source }} + {{ result.raw_data }} {% endfor %} diff --git a/manolo/apps/cazador/views.py b/manolo/apps/cazador/views.py index b1acd9f..ba2766a 100644 --- a/manolo/apps/cazador/views.py +++ b/manolo/apps/cazador/views.py @@ -1,13 +1,19 @@ -from django.shortcuts import render_to_response, redirect +from django.shortcuts import render_to_response -from .utils import search +from cazador.forms import CazadorForm +from django.views.decorators.csrf import csrf_exempt +@csrf_exempt def index(request): - if 'q' in request.GET: - name = request.GET['q'].strip() - results = search(name) + form = CazadorForm(request.GET) + query = request.GET['q'] - return render_to_response('cazador/results.html', {'results': results}) + all_items = form.search() + all_items = [i.object for i in all_items] - return render_to_response('cazador/index.html') + return render_to_response("cazador/results.html", + { + "results": all_items, + } + ) diff --git a/manolo/apps/visitors/models.py b/manolo/apps/visitors/models.py index 05b8f5c..48264b3 100644 --- a/manolo/apps/visitors/models.py +++ b/manolo/apps/visitors/models.py @@ -1,6 +1,5 @@ # -*- coding: utf-8 -*- from django.db import models -import hashlib class Visitor(models.Model): diff --git a/manolo/templates/search/indexes/cazador/cazador_text.txt b/manolo/templates/search/indexes/cazador/cazador_text.txt new file mode 100644 index 0000000..18e84e4 --- /dev/null +++ b/manolo/templates/search/indexes/cazador/cazador_text.txt @@ -0,0 +1 @@ +{{ object.raw_data }} diff --git a/requirements/base.txt b/requirements/base.txt index 631864b..585e9b4 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -25,3 +25,4 @@ python-slugify==0.1.0 six==1.9 elasticsearch==1.4.0 wheel==0.24.0 +tqdm