Refactoring search system
Alejandro Alonso
David Barragán Merino
parent
faf7621a0b
commit
275b2950ef
|
|
@ -17,11 +17,13 @@
|
|||
- Add endpoints to show the watchers list for issues, tasks and user stories.
|
||||
- Add headers to allow threading for notification emails about changes to issues, tasks, user stories, and wiki pages. (thanks to [@brett](https://github.com/brettp)).
|
||||
- Add externall apps: now Taiga can integrate with hundreds of applications and service.
|
||||
- Improving searching system, now full text searchs are supported
|
||||
- i18n.
|
||||
- Add polish (pl) translation.
|
||||
- Add portuguese (Brazil) (pt_BR) translation.
|
||||
- Add russian (ru) translation.
|
||||
|
||||
|
||||
### Misc
|
||||
- API: Mixin fields 'users', 'members' and 'memberships' in ProjectDetailSerializer.
|
||||
- API: Add stats/system resource with global server stats (total project, total users....)
|
||||
|
|
|
|||
|
|
@ -24,7 +24,7 @@ from django.utils.translation import ugettext as _
|
|||
|
||||
from taiga.base import exceptions as exc
|
||||
from taiga.base.api.utils import get_object_or_404
|
||||
|
||||
from taiga.base.utils.db import to_tsquery
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
|
@ -487,11 +487,11 @@ class QFilter(FilterBackend):
|
|||
def filter_queryset(self, request, queryset, view):
|
||||
q = request.QUERY_PARAMS.get('q', None)
|
||||
if q:
|
||||
if q.isdigit():
|
||||
qs_args = [Q(ref=q)]
|
||||
else:
|
||||
qs_args = [Q(subject__icontains=x) for x in q.split()]
|
||||
table = queryset.model._meta.db_table
|
||||
where_clause = ("to_tsvector('english_nostop', coalesce({table}.subject, '') || ' ' || "
|
||||
"coalesce({table}.ref) || ' ' || "
|
||||
"coalesce({table}.description, '')) @@ to_tsquery('english_nostop', %s)".format(table=table))
|
||||
|
||||
queryset = queryset.filter(reduce(operator.and_, qs_args))
|
||||
queryset = queryset.extra(where=[where_clause], params=[to_tsquery(q)])
|
||||
|
||||
return queryset
|
||||
|
|
|
|||
|
|
@ -125,3 +125,9 @@ def update_in_bulk_with_ids(ids, list_of_new_values, model):
|
|||
"""
|
||||
for id, new_values in zip(ids, list_of_new_values):
|
||||
model.objects.filter(id=id).update(**new_values)
|
||||
|
||||
|
||||
def to_tsquery(text):
|
||||
# We want to transform a query like "exam proj" (should find "project example") to something like proj:* & exam:*
|
||||
search_elems = ["{}:*".format(search_elem) for search_elem in text.split(" ")]
|
||||
return " & ".join(search_elems)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,29 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
from __future__ import unicode_literals
|
||||
|
||||
from django.db import connection
|
||||
from django.db import migrations
|
||||
|
||||
|
||||
def create_postgres_search_dictionary(apps, schema_editor):
|
||||
sql="""
|
||||
CREATE TEXT SEARCH DICTIONARY english_stem_nostop (
|
||||
Template = snowball,
|
||||
Language = english
|
||||
);
|
||||
CREATE TEXT SEARCH CONFIGURATION public.english_nostop ( COPY = pg_catalog.english );
|
||||
ALTER TEXT SEARCH CONFIGURATION public.english_nostop
|
||||
ALTER MAPPING FOR asciiword, asciihword, hword_asciipart, hword, hword_part, word WITH english_stem_nostop;
|
||||
"""
|
||||
cursor = connection.cursor()
|
||||
cursor.execute(sql)
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('projects', '0025_auto_20150901_1600'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.RunPython(create_postgres_search_dictionary),
|
||||
]
|
||||
|
|
@ -16,20 +16,20 @@
|
|||
|
||||
from django.apps import apps
|
||||
from django.conf import settings
|
||||
|
||||
from taiga.base.utils.db import to_tsquery
|
||||
|
||||
MAX_RESULTS = getattr(settings, "SEARCHES_MAX_RESULTS", 150)
|
||||
|
||||
|
||||
def search_user_stories(project, text):
|
||||
model_cls = apps.get_model("userstories", "UserStory")
|
||||
where_clause = ("to_tsvector(coalesce(userstories_userstory.subject) || ' ' || "
|
||||
where_clause = ("to_tsvector('english_nostop', coalesce(userstories_userstory.subject) || ' ' || "
|
||||
"coalesce(userstories_userstory.ref) || ' ' || "
|
||||
"coalesce(userstories_userstory.description, '')) "
|
||||
"@@ plainto_tsquery(%s)")
|
||||
"@@ to_tsquery('english_nostop', %s)")
|
||||
|
||||
if text:
|
||||
return (model_cls.objects.extra(where=[where_clause], params=[text])
|
||||
return (model_cls.objects.extra(where=[where_clause], params=[to_tsquery(text)])
|
||||
.filter(project_id=project.pk)[:MAX_RESULTS])
|
||||
|
||||
return model_cls.objects.filter(project_id=project.pk)[:MAX_RESULTS]
|
||||
|
|
@ -37,12 +37,12 @@ def search_user_stories(project, text):
|
|||
|
||||
def search_tasks(project, text):
|
||||
model_cls = apps.get_model("tasks", "Task")
|
||||
where_clause = ("to_tsvector(coalesce(tasks_task.subject, '') || ' ' || "
|
||||
where_clause = ("to_tsvector('english_nostop', coalesce(tasks_task.subject, '') || ' ' || "
|
||||
"coalesce(tasks_task.ref) || ' ' || "
|
||||
"coalesce(tasks_task.description, '')) @@ plainto_tsquery(%s)")
|
||||
"coalesce(tasks_task.description, '')) @@ to_tsquery('english_nostop', %s)")
|
||||
|
||||
if text:
|
||||
return (model_cls.objects.extra(where=[where_clause], params=[text])
|
||||
return (model_cls.objects.extra(where=[where_clause], params=[to_tsquery(text)])
|
||||
.filter(project_id=project.pk)[:MAX_RESULTS])
|
||||
|
||||
return model_cls.objects.filter(project_id=project.pk)[:MAX_RESULTS]
|
||||
|
|
@ -50,12 +50,12 @@ def search_tasks(project, text):
|
|||
|
||||
def search_issues(project, text):
|
||||
model_cls = apps.get_model("issues", "Issue")
|
||||
where_clause = ("to_tsvector(coalesce(issues_issue.subject) || ' ' || "
|
||||
where_clause = ("to_tsvector('english_nostop', coalesce(issues_issue.subject) || ' ' || "
|
||||
"coalesce(issues_issue.ref) || ' ' || "
|
||||
"coalesce(issues_issue.description, '')) @@ plainto_tsquery(%s)")
|
||||
"coalesce(issues_issue.description, '')) @@ to_tsquery('english_nostop', %s)")
|
||||
|
||||
if text:
|
||||
return (model_cls.objects.extra(where=[where_clause], params=[text])
|
||||
return (model_cls.objects.extra(where=[where_clause], params=[to_tsquery(text)])
|
||||
.filter(project_id=project.pk)[:MAX_RESULTS])
|
||||
|
||||
return model_cls.objects.filter(project_id=project.pk)[:MAX_RESULTS]
|
||||
|
|
@ -63,12 +63,12 @@ def search_issues(project, text):
|
|||
|
||||
def search_wiki_pages(project, text):
|
||||
model_cls = apps.get_model("wiki", "WikiPage")
|
||||
where_clause = ("to_tsvector(coalesce(wiki_wikipage.slug) || ' ' || "
|
||||
where_clause = ("to_tsvector('english_nostop', coalesce(wiki_wikipage.slug) || ' ' || "
|
||||
"coalesce(wiki_wikipage.content, '')) "
|
||||
"@@ plainto_tsquery(%s)")
|
||||
"@@ to_tsquery('english_nostop', %s)")
|
||||
|
||||
if text:
|
||||
return (model_cls.objects.extra(where=[where_clause], params=[text])
|
||||
return (model_cls.objects.extra(where=[where_clause], params=[to_tsquery(text)])
|
||||
.filter(project_id=project.pk)[:MAX_RESULTS])
|
||||
|
||||
return model_cls.objects.filter(project_id=project.pk)[:MAX_RESULTS]
|
||||
|
|
|
|||
|
|
@ -124,10 +124,11 @@ def test_search_text_query_in_my_project(client, searches_initial_data):
|
|||
|
||||
response = client.get(reverse("search-list"), {"project": data.project1.id, "text": "back"})
|
||||
assert response.status_code == 200
|
||||
assert response.data["count"] == 2
|
||||
assert response.data["count"] == 3
|
||||
assert len(response.data["userstories"]) == 1
|
||||
assert len(response.data["tasks"]) == 1
|
||||
assert len(response.data["issues"]) == 0
|
||||
# Back is a backend substring
|
||||
assert len(response.data["issues"]) == 1
|
||||
assert len(response.data["wikipages"]) == 0
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue