diff --git a/CHANGELOG.md b/CHANGELOG.md index 1dc965935..ed0e5c9fe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,11 +17,13 @@ - Add endpoints to show the watchers list for issues, tasks and user stories. - Add headers to allow threading for notification emails about changes to issues, tasks, user stories, and wiki pages. (thanks to [@brett](https://github.com/brettp)). - Add externall apps: now Taiga can integrate with hundreds of applications and service. +- Improving searching system, now full text searchs are supported - i18n. - Add polish (pl) translation. - Add portuguese (Brazil) (pt_BR) translation. - Add russian (ru) translation. + ### Misc - API: Mixin fields 'users', 'members' and 'memberships' in ProjectDetailSerializer. - API: Add stats/system resource with global server stats (total project, total users....) diff --git a/taiga/base/filters.py b/taiga/base/filters.py index e0de384cb..d662b47d2 100644 --- a/taiga/base/filters.py +++ b/taiga/base/filters.py @@ -24,7 +24,7 @@ from taiga.base import exceptions as exc from taiga.base.api.utils import get_object_or_404 - +from taiga.base.utils.db import to_tsquery logger = logging.getLogger(__name__) @@ -487,11 +487,11 @@ class QFilter(FilterBackend): def filter_queryset(self, request, queryset, view): q = request.QUERY_PARAMS.get('q', None) if q: - if q.isdigit(): - qs_args = [Q(ref=q)] - else: - qs_args = [Q(subject__icontains=x) for x in q.split()] + table = queryset.model._meta.db_table + where_clause = ("to_tsvector('english_nostop', coalesce({table}.subject, '') || ' ' || " + "coalesce({table}.ref) || ' ' || " + "coalesce({table}.description, '')) @@ to_tsquery('english_nostop', %s)".format(table=table)) - queryset = queryset.filter(reduce(operator.and_, qs_args)) + queryset = queryset.extra(where=[where_clause], params=[to_tsquery(q)]) return queryset diff --git a/taiga/base/utils/db.py b/taiga/base/utils/db.py index 40afea817..2762833e5 100644 --- a/taiga/base/utils/db.py +++ b/taiga/base/utils/db.py @@ -125,3 +125,9 @@ def update_in_bulk_with_ids(ids, list_of_new_values, model): """ for id, new_values in zip(ids, list_of_new_values): model.objects.filter(id=id).update(**new_values) + + +def to_tsquery(text): + # We want to transform a query like "exam proj" (should find "project example") to something like proj:* & exam:* + search_elems = ["{}:*".format(search_elem) for search_elem in text.split(" ")] + return " & ".join(search_elems) diff --git a/taiga/projects/migrations/0026_auto_20150911_1237.py b/taiga/projects/migrations/0026_auto_20150911_1237.py new file mode 100644 index 000000000..073c2349b --- /dev/null +++ b/taiga/projects/migrations/0026_auto_20150911_1237.py @@ -0,0 +1,29 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import connection +from django.db import migrations + + +def create_postgres_search_dictionary(apps, schema_editor): + sql=""" +CREATE TEXT SEARCH DICTIONARY english_stem_nostop ( + Template = snowball, + Language = english +); +CREATE TEXT SEARCH CONFIGURATION public.english_nostop ( COPY = pg_catalog.english ); +ALTER TEXT SEARCH CONFIGURATION public.english_nostop +ALTER MAPPING FOR asciiword, asciihword, hword_asciipart, hword, hword_part, word WITH english_stem_nostop; +""" + cursor = connection.cursor() + cursor.execute(sql) + +class Migration(migrations.Migration): + + dependencies = [ + ('projects', '0025_auto_20150901_1600'), + ] + + operations = [ + migrations.RunPython(create_postgres_search_dictionary), + ] diff --git a/taiga/searches/services.py b/taiga/searches/services.py index 495e298d0..dcac7f33e 100644 --- a/taiga/searches/services.py +++ b/taiga/searches/services.py @@ -16,20 +16,20 @@ from django.apps import apps from django.conf import settings - +from taiga.base.utils.db import to_tsquery MAX_RESULTS = getattr(settings, "SEARCHES_MAX_RESULTS", 150) def search_user_stories(project, text): model_cls = apps.get_model("userstories", "UserStory") - where_clause = ("to_tsvector(coalesce(userstories_userstory.subject) || ' ' || " + where_clause = ("to_tsvector('english_nostop', coalesce(userstories_userstory.subject) || ' ' || " "coalesce(userstories_userstory.ref) || ' ' || " "coalesce(userstories_userstory.description, '')) " - "@@ plainto_tsquery(%s)") + "@@ to_tsquery('english_nostop', %s)") if text: - return (model_cls.objects.extra(where=[where_clause], params=[text]) + return (model_cls.objects.extra(where=[where_clause], params=[to_tsquery(text)]) .filter(project_id=project.pk)[:MAX_RESULTS]) return model_cls.objects.filter(project_id=project.pk)[:MAX_RESULTS] @@ -37,12 +37,12 @@ def search_user_stories(project, text): def search_tasks(project, text): model_cls = apps.get_model("tasks", "Task") - where_clause = ("to_tsvector(coalesce(tasks_task.subject, '') || ' ' || " + where_clause = ("to_tsvector('english_nostop', coalesce(tasks_task.subject, '') || ' ' || " "coalesce(tasks_task.ref) || ' ' || " - "coalesce(tasks_task.description, '')) @@ plainto_tsquery(%s)") + "coalesce(tasks_task.description, '')) @@ to_tsquery('english_nostop', %s)") if text: - return (model_cls.objects.extra(where=[where_clause], params=[text]) + return (model_cls.objects.extra(where=[where_clause], params=[to_tsquery(text)]) .filter(project_id=project.pk)[:MAX_RESULTS]) return model_cls.objects.filter(project_id=project.pk)[:MAX_RESULTS] @@ -50,12 +50,12 @@ def search_tasks(project, text): def search_issues(project, text): model_cls = apps.get_model("issues", "Issue") - where_clause = ("to_tsvector(coalesce(issues_issue.subject) || ' ' || " + where_clause = ("to_tsvector('english_nostop', coalesce(issues_issue.subject) || ' ' || " "coalesce(issues_issue.ref) || ' ' || " - "coalesce(issues_issue.description, '')) @@ plainto_tsquery(%s)") + "coalesce(issues_issue.description, '')) @@ to_tsquery('english_nostop', %s)") if text: - return (model_cls.objects.extra(where=[where_clause], params=[text]) + return (model_cls.objects.extra(where=[where_clause], params=[to_tsquery(text)]) .filter(project_id=project.pk)[:MAX_RESULTS]) return model_cls.objects.filter(project_id=project.pk)[:MAX_RESULTS] @@ -63,12 +63,12 @@ def search_issues(project, text): def search_wiki_pages(project, text): model_cls = apps.get_model("wiki", "WikiPage") - where_clause = ("to_tsvector(coalesce(wiki_wikipage.slug) || ' ' || " + where_clause = ("to_tsvector('english_nostop', coalesce(wiki_wikipage.slug) || ' ' || " "coalesce(wiki_wikipage.content, '')) " - "@@ plainto_tsquery(%s)") + "@@ to_tsquery('english_nostop', %s)") if text: - return (model_cls.objects.extra(where=[where_clause], params=[text]) + return (model_cls.objects.extra(where=[where_clause], params=[to_tsquery(text)]) .filter(project_id=project.pk)[:MAX_RESULTS]) return model_cls.objects.filter(project_id=project.pk)[:MAX_RESULTS] diff --git a/tests/integration/test_searches.py b/tests/integration/test_searches.py index ec6743ad3..db026a2a2 100644 --- a/tests/integration/test_searches.py +++ b/tests/integration/test_searches.py @@ -124,10 +124,11 @@ def test_search_text_query_in_my_project(client, searches_initial_data): response = client.get(reverse("search-list"), {"project": data.project1.id, "text": "back"}) assert response.status_code == 200 - assert response.data["count"] == 2 + assert response.data["count"] == 3 assert len(response.data["userstories"]) == 1 assert len(response.data["tasks"]) == 1 - assert len(response.data["issues"]) == 0 + # Back is a backend substring + assert len(response.data["issues"]) == 1 assert len(response.data["wikipages"]) == 0