Refactoring search system

CHANGELOG.md

@@ -17,11 +17,13 @@
 - Add endpoints to show the watchers list for issues, tasks and user stories.
 - Add headers to allow threading for notification emails about changes to issues, tasks, user stories, and wiki pages. (thanks to [@brett](https://github.com/brettp)).
 - Add externall apps: now Taiga can integrate with hundreds of applications and service.
+- Improving searching system, now full text searchs are supported
 - i18n.
   - Add polish (pl) translation.
   - Add portuguese (Brazil) (pt_BR) translation.
   - Add russian (ru) translation.
 
+
 ### Misc
 - API: Mixin fields 'users', 'members' and 'memberships' in ProjectDetailSerializer.
 - API: Add stats/system resource with global server stats (total project, total users....)

taiga/base/filters.py

@@ -24,7 +24,7 @@ from django.utils.translation import ugettext as _
 
 from taiga.base import exceptions as exc
 from taiga.base.api.utils import get_object_or_404
-
+from taiga.base.utils.db import to_tsquery
 
 logger = logging.getLogger(__name__)
 
@@ -487,11 +487,11 @@ class QFilter(FilterBackend):
     def filter_queryset(self, request, queryset, view):
         q = request.QUERY_PARAMS.get('q', None)
         if q:
-            if q.isdigit():
+            table = queryset.model._meta.db_table
-                qs_args = [Q(ref=q)]
+            where_clause = ("to_tsvector('english_nostop', coalesce({table}.subject, '') || ' ' || "
-            else:
+                            "coalesce({table}.ref) || ' ' || "
-                qs_args = [Q(subject__icontains=x) for x in q.split()]
+                            "coalesce({table}.description, '')) @@ to_tsquery('english_nostop', %s)".format(table=table))
 
-            queryset = queryset.filter(reduce(operator.and_, qs_args))
+            queryset = queryset.extra(where=[where_clause], params=[to_tsquery(q)])
 
         return queryset

taiga/base/utils/db.py

@@ -125,3 +125,9 @@ def update_in_bulk_with_ids(ids, list_of_new_values, model):
     """
     for id, new_values in zip(ids, list_of_new_values):
         model.objects.filter(id=id).update(**new_values)
+
+
+def to_tsquery(text):
+    # We want to transform a query like "exam proj" (should find "project example") to something like proj:* & exam:*
+    search_elems = ["{}:*".format(search_elem) for search_elem in text.split(" ")]
+    return " & ".join(search_elems)

taiga/projects/migrations/0026_auto_20150911_1237.py

@@ -0,0 +1,29 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import connection
+from django.db import migrations
+
+
+def create_postgres_search_dictionary(apps, schema_editor):
+    sql="""
+CREATE TEXT SEARCH DICTIONARY english_stem_nostop (
+    Template = snowball,
+    Language = english
+);
+CREATE TEXT SEARCH CONFIGURATION public.english_nostop ( COPY = pg_catalog.english );
+ALTER TEXT SEARCH CONFIGURATION public.english_nostop
+ALTER MAPPING FOR asciiword, asciihword, hword_asciipart, hword, hword_part, word WITH english_stem_nostop;
+"""
+    cursor = connection.cursor()
+    cursor.execute(sql)
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('projects', '0025_auto_20150901_1600'),
+    ]
+
+    operations = [
+        migrations.RunPython(create_postgres_search_dictionary),
+    ]

taiga/searches/services.py

@@ -16,20 +16,20 @@
 
 from django.apps import apps
 from django.conf import settings
-
+from taiga.base.utils.db import to_tsquery
 
 MAX_RESULTS = getattr(settings, "SEARCHES_MAX_RESULTS", 150)
 
 
 def search_user_stories(project, text):
     model_cls = apps.get_model("userstories", "UserStory")
-    where_clause = ("to_tsvector(coalesce(userstories_userstory.subject) || ' ' || "
+    where_clause = ("to_tsvector('english_nostop', coalesce(userstories_userstory.subject) || ' ' || "
                                 "coalesce(userstories_userstory.ref) || ' ' || "
                                 "coalesce(userstories_userstory.description, '')) "
-                    "@@ plainto_tsquery(%s)")
+                    "@@ to_tsquery('english_nostop', %s)")
 
     if text:
-        return (model_cls.objects.extra(where=[where_clause], params=[text])
+        return (model_cls.objects.extra(where=[where_clause], params=[to_tsquery(text)])
                                  .filter(project_id=project.pk)[:MAX_RESULTS])
 
     return model_cls.objects.filter(project_id=project.pk)[:MAX_RESULTS]
@@ -37,12 +37,12 @@ def search_user_stories(project, text):
 
 def search_tasks(project, text):
     model_cls = apps.get_model("tasks", "Task")
-    where_clause = ("to_tsvector(coalesce(tasks_task.subject, '') || ' ' || "
+    where_clause = ("to_tsvector('english_nostop', coalesce(tasks_task.subject, '') || ' ' || "
                     "coalesce(tasks_task.ref) || ' ' || "
-                    "coalesce(tasks_task.description, '')) @@ plainto_tsquery(%s)")
+                    "coalesce(tasks_task.description, '')) @@ to_tsquery('english_nostop', %s)")
 
     if text:
-        return (model_cls.objects.extra(where=[where_clause], params=[text])
+        return (model_cls.objects.extra(where=[where_clause], params=[to_tsquery(text)])
                                  .filter(project_id=project.pk)[:MAX_RESULTS])
 
     return model_cls.objects.filter(project_id=project.pk)[:MAX_RESULTS]
@@ -50,12 +50,12 @@ def search_tasks(project, text):
 
 def search_issues(project, text):
     model_cls = apps.get_model("issues", "Issue")
-    where_clause = ("to_tsvector(coalesce(issues_issue.subject) || ' ' || "
+    where_clause = ("to_tsvector('english_nostop', coalesce(issues_issue.subject) || ' ' || "
                     "coalesce(issues_issue.ref) || ' ' || "
-                    "coalesce(issues_issue.description, '')) @@ plainto_tsquery(%s)")
+                    "coalesce(issues_issue.description, '')) @@ to_tsquery('english_nostop', %s)")
 
     if text:
-        return (model_cls.objects.extra(where=[where_clause], params=[text])
+        return (model_cls.objects.extra(where=[where_clause], params=[to_tsquery(text)])
                                  .filter(project_id=project.pk)[:MAX_RESULTS])
 
     return model_cls.objects.filter(project_id=project.pk)[:MAX_RESULTS]
@@ -63,12 +63,12 @@ def search_issues(project, text):
 
 def search_wiki_pages(project, text):
     model_cls = apps.get_model("wiki", "WikiPage")
-    where_clause = ("to_tsvector(coalesce(wiki_wikipage.slug) || ' ' || "
+    where_clause = ("to_tsvector('english_nostop', coalesce(wiki_wikipage.slug) || ' ' || "
                                 "coalesce(wiki_wikipage.content, '')) "
-                    "@@ plainto_tsquery(%s)")
+                    "@@ to_tsquery('english_nostop', %s)")
 
     if text:
-        return (model_cls.objects.extra(where=[where_clause], params=[text])
+        return (model_cls.objects.extra(where=[where_clause], params=[to_tsquery(text)])
                                  .filter(project_id=project.pk)[:MAX_RESULTS])
 
     return model_cls.objects.filter(project_id=project.pk)[:MAX_RESULTS]

tests/integration/test_searches.py

@@ -124,10 +124,11 @@ def test_search_text_query_in_my_project(client, searches_initial_data):
 
     response = client.get(reverse("search-list"), {"project": data.project1.id, "text": "back"})
     assert response.status_code == 200
-    assert response.data["count"] == 2
+    assert response.data["count"] == 3
     assert len(response.data["userstories"]) == 1
     assert len(response.data["tasks"]) == 1
-    assert len(response.data["issues"]) == 0
+    # Back is a backend substring
+    assert len(response.data["issues"]) == 1
     assert len(response.data["wikipages"]) == 0