Task #3526: Filter projects by text query (order by ranking name > tags > description

taiga/base/filters.py

@@ -33,7 +33,6 @@ logger = logging.getLogger(__name__)
 # Base and Mixins
 #####################################################################
 
-
 class BaseFilterBackend(object):
     """
     A base class from which all filter backend classes should inherit.
@@ -182,6 +181,10 @@ class CanViewMilestonesFilterBackend(PermissionBasedFilterBackend):
     permission = "view_milestones"
 
 
+#####################################################################
+# Attachments filters
+#####################################################################
+
 class PermissionBasedAttachmentFilterBackend(PermissionBasedFilterBackend):
     permission = None
 
@@ -208,52 +211,9 @@ class CanViewWikiAttachmentFilterBackend(PermissionBasedAttachmentFilterBackend)
     permission = "view_wiki_pages"
 
 
-class CanViewProjectObjFilterBackend(FilterBackend):
+#####################################################################
-    def filter_queryset(self, request, queryset, view):
+# User filters
-        project_id = None
+#####################################################################
-        if (hasattr(view, "filter_fields") and "project" in view.filter_fields and
-                "project" in request.QUERY_PARAMS):
-            try:
-                project_id = int(request.QUERY_PARAMS["project"])
-            except:
-                logger.error("Filtering project diferent value than an integer: {}".format(
-                    request.QUERY_PARAMS["project"]
-                ))
-                raise exc.BadRequest(_("'project' must be an integer value."))
-
-        qs = queryset
-
-        if request.user.is_authenticated() and request.user.is_superuser:
-            qs = qs
-        elif request.user.is_authenticated():
-            membership_model = apps.get_model("projects", "Membership")
-            memberships_qs = membership_model.objects.filter(user=request.user)
-            if project_id:
-                memberships_qs = memberships_qs.filter(project_id=project_id)
-            memberships_qs = memberships_qs.filter(Q(role__permissions__contains=['view_project']) |
-                                                   Q(is_owner=True))
-
-            projects_list = [membership.project_id for membership in memberships_qs]
-
-            qs = qs.filter((Q(id__in=projects_list) |
-                            Q(public_permissions__contains=["view_project"])))
-        else:
-            qs = qs.filter(anon_permissions__contains=["view_project"])
-
-        return super().filter_queryset(request, qs.distinct(), view)
-
-
-class IsProjectMemberFilterBackend(FilterBackend):
-    def filter_queryset(self, request, queryset, view):
-        if request.user.is_authenticated() and request.user.is_superuser:
-            queryset = queryset
-        elif request.user.is_authenticated():
-            queryset = queryset.filter(project__members=request.user)
-        else:
-            queryset = queryset.none()
-
-        return super().filter_queryset(request, queryset.distinct(), view)
-
 
 class MembersFilterBackend(PermissionBasedFilterBackend):
     permission = "view_project"
@@ -309,6 +269,10 @@ class MembersFilterBackend(PermissionBasedFilterBackend):
         return qs.distinct()
 
 
+#####################################################################
+# Webhooks  filters
+#####################################################################
+
 class BaseIsProjectAdminFilterBackend(object):
     def get_project_ids(self, request, view):
         project_id = None
@@ -358,7 +322,7 @@ class IsProjectAdminFromWebhookLogFilterBackend(FilterBackend, BaseIsProjectAdmi
 
 
 #####################################################################
-# Generic Attributes filters
+# Generic Attributes filters (for User Stories, Tasks and Issues)
 #####################################################################
 
 class BaseRelatedFieldsFilter(FilterBackend):
@@ -451,7 +415,6 @@ class TagsFilter(FilterBackend):
         return super().filter_queryset(request, queryset, view)
 
 
-
 class WatchersFilter(FilterBackend):
     filter_name = 'watchers'
 
@@ -487,9 +450,12 @@ class QFilter(FilterBackend):
         q = request.QUERY_PARAMS.get('q', None)
         if q:
             table = queryset.model._meta.db_table
-            where_clause = ("to_tsvector('english_nostop', coalesce({table}.subject, '') || ' ' || "
+            where_clause = ("""
-                            "coalesce({table}.ref) || ' ' || "
+                to_tsvector('english_nostop',
-                            "coalesce({table}.description, '')) @@ to_tsquery('english_nostop', %s)".format(table=table))
+                            coalesce({table}.subject, '') || ' ' ||
+                            coalesce({table}.ref) || ' ' ||
+                            coalesce({table}.description, '')) @@ to_tsquery('english_nostop', %s)
+            """.format(table=table))
 
             queryset = queryset.extra(where=[where_clause], params=[to_tsquery(q)])
 

taiga/projects/api.py

@@ -16,8 +16,8 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import uuid
-
 from easy_thumbnails.source_generators import pil_image
+from dateutil.relativedelta import relativedelta
 
 from django.apps import apps
 from django.db.models import signals, Prefetch
@@ -51,12 +51,12 @@ from taiga.projects.issues.models import Issue
 from taiga.projects.likes.mixins.viewsets import LikedResourceMixin, FansViewSetMixin
 from taiga.permissions import service as permissions_service
 
-from . import serializers
+from . import filters as project_filters
 from . import models
 from . import permissions
+from . import serializers
 from . import services
 
-from dateutil.relativedelta import relativedelta
 
 ######################################################
 ## Project
@@ -67,7 +67,8 @@ class ProjectViewSet(LikedResourceMixin, HistoryResourceMixin, ModelCrudViewSet)
     admin_serializer_class = serializers.ProjectDetailAdminSerializer
     list_serializer_class = serializers.ProjectSerializer
     permission_classes = (permissions.ProjectPermission, )
-    filter_backends = (filters.CanViewProjectObjFilterBackend,)
+    filter_backends = (project_filters.QFilter,
+                       project_filters.CanViewProjectObjFilterBackend)
 
     filter_fields = (("member", "members"),
                      "is_looking_for_people",
@@ -87,7 +88,7 @@ class ProjectViewSet(LikedResourceMixin, HistoryResourceMixin, ModelCrudViewSet)
                        "total_activity_last_year")
 
     def _get_order_by_field_name(self):
-        order_by_query_param = filters.CanViewProjectObjFilterBackend.order_by_query_param
+        order_by_query_param = project_filters.CanViewProjectObjFilterBackend.order_by_query_param
         order_by = self.request.QUERY_PARAMS.get(order_by_query_param, None)
         if order_by is not None and order_by.startswith("-"):
             return order_by[1:]

taiga/projects/filters.py

@@ -0,0 +1,101 @@
+# Copyright (C) 2014-2015 Andrey Antukh <niwi@niwi.be>
+# Copyright (C) 2014-2015 Jesús Espino <jespinog@gmail.com>
+# Copyright (C) 2014-2015 David Barragán <bameda@dbarragan.com>
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+import logging
+
+from django.apps import apps
+from django.db.models import Q
+from django.utils.translation import ugettext as _
+
+from taiga.base import exceptions as exc
+from taiga.base.filters import FilterBackend
+from taiga.base.utils.db import to_tsquery
+
+logger = logging.getLogger(__name__)
+
+
+class CanViewProjectObjFilterBackend(FilterBackend):
+    def filter_queryset(self, request, queryset, view):
+        project_id = None
+
+        # Filter by filter_fields
+        if (hasattr(view, "filter_fields") and "project" in view.filter_fields and
+                "project" in request.QUERY_PARAMS):
+            try:
+                project_id = int(request.QUERY_PARAMS["project"])
+            except:
+                logger.error("Filtering project diferent value than an integer: {}".format(
+                    request.QUERY_PARAMS["project"]
+                ))
+                raise exc.BadRequest(_("'project' must be an integer value."))
+
+        qs = queryset
+
+        # Filter by user permissions
+        if request.user.is_authenticated() and request.user.is_superuser:
+            # superuser
+            qs = qs
+        elif request.user.is_authenticated():
+            # projet members
+            membership_model = apps.get_model("projects", "Membership")
+            memberships_qs = membership_model.objects.filter(user=request.user)
+            if project_id:
+                memberships_qs = memberships_qs.filter(project_id=project_id)
+            memberships_qs = memberships_qs.filter(Q(role__permissions__contains=['view_project']) |
+                                                   Q(is_owner=True))
+
+            projects_list = [membership.project_id for membership in memberships_qs]
+
+            qs = qs.filter((Q(id__in=projects_list) |
+                            Q(public_permissions__contains=["view_project"])))
+        else:
+            # external users / anonymous
+            qs = qs.filter(anon_permissions__contains=["view_project"])
+
+        return super().filter_queryset(request, qs.distinct(), view)
+
+
+class QFilter(FilterBackend):
+    def filter_queryset(self, request, queryset, view):
+        # NOTE: See migtration 0033_text_search_indexes
+        q = request.QUERY_PARAMS.get('q', None)
+        if q:
+            tsquery = "to_tsquery('english_nostop', %s)"
+            tsquery_params = [to_tsquery(q)]
+            tsvector = """
+                 setweight(to_tsvector('english_nostop',
+                                       coalesce(projects_project.name, '')), 'A') ||
+                 setweight(to_tsvector('english_nostop',
+                                       coalesce(inmutable_array_to_string(projects_project.tags), '')), 'B') ||
+                 setweight(to_tsvector('english_nostop',
+                                       coalesce(projects_project.description, '')), 'C')
+            """
+
+            select = {
+                "rank": "ts_rank({tsvector},{tsquery})".format(tsquery=tsquery,
+                                                               tsvector=tsvector),
+            }
+            select_params = tsquery_params
+            where = ["{tsvector} @@ {tsquery}".format(tsquery=tsquery,
+                                                      tsvector=tsvector),]
+            params = tsquery_params
+            order_by = ["-rank", ]
+
+            queryset = queryset.extra(select=select,
+                                      select_params=select_params,
+                                      where=where,
+                                      params=params,
+                                      order_by=order_by)
+        return queryset

taiga/projects/migrations/0033_text_search_indexes.py

@@ -0,0 +1,50 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations
+
+
+DROP_INMUTABLE_ARRAY_TO_STRING_FUNCTION = """
+    DROP FUNCTION IF EXISTS inmutable_array_to_string(text[]) CASCADE
+"""
+
+
+# NOTE: This function is needed by taiga.projects.filters.QFilter
+CREATE_INMUTABLE_ARRAY_TO_STRING_FUNCTION = """
+    CREATE OR REPLACE FUNCTION inmutable_array_to_string(text[])
+                       RETURNS text
+                      LANGUAGE sql
+                     IMMUTABLE AS $$SELECT array_to_string($1, ' ', '')$$
+"""
+
+
+DROP_INDEX = """
+    DROP INDEX IF EXISTS projects_project_textquery_idx;
+"""
+
+
+# NOTE: This index is needed by taiga.projects.filters.QFilter
+CREATE_INDEX = """
+    CREATE INDEX projects_project_textquery_idx
+              ON projects_project
+           USING gin((setweight(to_tsvector('english_nostop',
+                                coalesce(projects_project.name, '')), 'A') ||
+                      setweight(to_tsvector('english_nostop',
+                                 coalesce(inmutable_array_to_string(projects_project.tags), '')), 'B') ||
+                      setweight(to_tsvector('english_nostop',
+                                coalesce(projects_project.description, '')), 'C')));
+"""
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('projects', '0032_auto_20151202_1151'),
+    ]
+
+    operations = [
+        migrations.RunSQL([DROP_INMUTABLE_ARRAY_TO_STRING_FUNCTION, CREATE_INMUTABLE_ARRAY_TO_STRING_FUNCTION],
+                          [DROP_INMUTABLE_ARRAY_TO_STRING_FUNCTION]),
+        migrations.RunSQL([DROP_INDEX, CREATE_INDEX],
+                          [DROP_INDEX]),
+    ]

tests/integration/test_projects.py

@@ -509,3 +509,23 @@ def test_project_list_without_search_query_order_by_name(client):
     assert response.data[0]["id"] == project1.id
     assert response.data[1]["id"] == project2.id
     assert response.data[2]["id"] == project3.id
+
+
+def test_project_list_with_search_query_order_by_ranking(client):
+    user = f.UserFactory.create(is_superuser=True)
+    project3 = f.create_project(name="test 3 - word", description="description 3", tags=["tag3"])
+    project1 = f.create_project(name="test 1", description="description 1 - word", tags=["tag1"])
+    project2 = f.create_project(name="test 2", description="description 2", tags=["word", "tag2"])
+    project4 = f.create_project(name="test 4", description="description 4", tags=["tag4"])
+    project5 = f.create_project(name="test 5", description="description 5", tags=["tag5"])
+
+    url = reverse("projects-list")
+
+    client.login(user)
+    response = client.json.get(url, {"q": "word"})
+
+    assert response.status_code == 200
+    assert len(response.data) == 3
+    assert response.data[0]["id"] == project3.id
+    assert response.data[1]["id"] == project2.id
+    assert response.data[2]["id"] == project1.id