From 50e00b6d454d72bf8d221160f998d85cfe941a43 Mon Sep 17 00:00:00 2001 From: Alejandro Alonso Date: Tue, 1 Dec 2015 11:00:51 +0100 Subject: [PATCH] Task #3517 #3516: Order by fans and activity (last week/moth/year/all time) --- requirements.txt | 1 + taiga/base/api/mixins.py | 10 +- taiga/export_import/dump_service.py | 1 + taiga/projects/api.py | 83 ++++++--- .../migrations/0002_auto_20151130_2230.py | 25 +++ taiga/projects/likes/mixins/serializers.py | 10 +- taiga/projects/likes/mixins/viewsets.py | 14 -- taiga/projects/likes/models.py | 21 --- taiga/projects/likes/services.py | 31 +--- taiga/projects/likes/utils.py | 77 -------- .../migrations/0030_auto_20151128_0757.py | 164 ++++++++++++++++++ taiga/projects/milestones/api.py | 2 +- taiga/projects/models.py | 79 +++++++++ taiga/projects/notifications/mixins.py | 15 +- taiga/projects/notifications/utils.py | 75 +------- taiga/projects/serializers.py | 21 ++- taiga/timeline/signals.py | 2 + taiga/users/services.py | 8 +- tests/factories.py | 9 - .../test_projects_resource.py | 4 - tests/integration/test_fan_projects.py | 16 -- tests/integration/test_notifications.py | 18 -- tests/integration/test_totals_projects.py | 154 ++++++++++++++++ tests/integration/test_users.py | 6 +- 24 files changed, 538 insertions(+), 308 deletions(-) create mode 100644 taiga/projects/likes/migrations/0002_auto_20151130_2230.py delete mode 100644 taiga/projects/likes/utils.py create mode 100644 taiga/projects/migrations/0030_auto_20151128_0757.py create mode 100644 tests/integration/test_totals_projects.py diff --git a/requirements.txt b/requirements.txt index 8b54d05a..8501843c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -33,3 +33,4 @@ django-transactional-cleanup==0.1.15 lxml==3.5.0 git+https://github.com/Xof/django-pglocks.git@dbb8d7375066859f897604132bd437832d2014ea pyjwkest==1.0.9 +python-dateutil==2.4.2 diff --git a/taiga/base/api/mixins.py b/taiga/base/api/mixins.py index 80e14220..27c05675 100644 --- a/taiga/base/api/mixins.py +++ b/taiga/base/api/mixins.py @@ -81,7 +81,7 @@ def _get_validation_exclusions(obj, pk=None, slug_field=None, lookup_field=None) return [field.name for field in obj._meta.fields if field.name not in include] -class CreateModelMixin(object): +class CreateModelMixin: """ Create a model instance. """ @@ -107,7 +107,7 @@ class CreateModelMixin(object): return {} -class ListModelMixin(object): +class ListModelMixin: """ List a queryset. """ @@ -137,7 +137,7 @@ class ListModelMixin(object): return response.Ok(serializer.data) -class RetrieveModelMixin(object): +class RetrieveModelMixin: """ Retrieve a model instance. """ @@ -153,7 +153,7 @@ class RetrieveModelMixin(object): return response.Ok(serializer.data) -class UpdateModelMixin(object): +class UpdateModelMixin: """ Update a model instance. """ @@ -220,7 +220,7 @@ class UpdateModelMixin(object): obj.full_clean(exclude) -class DestroyModelMixin(object): +class DestroyModelMixin: """ Destroy a model instance. """ diff --git a/taiga/export_import/dump_service.py b/taiga/export_import/dump_service.py index e6980cf7..8029fa0f 100644 --- a/taiga/export_import/dump_service.py +++ b/taiga/export_import/dump_service.py @@ -183,4 +183,5 @@ def dict_to_project(data, owner=None): if service.get_errors(clear=False): raise TaigaImportError(_("error importing timelines")) + proj.refresh_totals() return proj diff --git a/taiga/projects/api.py b/taiga/projects/api.py index fedaf793..d80a5509 100644 --- a/taiga/projects/api.py +++ b/taiga/projects/api.py @@ -17,9 +17,13 @@ import uuid -from django.db.models import signals +from django.apps import apps +from django.db.models import signals, Prefetch +from django.db.models import Value as V +from django.db.models.functions import Coalesce from django.core.exceptions import ValidationError from django.utils.translation import ugettext as _ +from django.utils import timezone from taiga.base import filters from taiga.base import response @@ -32,12 +36,9 @@ from taiga.base.api.utils import get_object_or_404 from taiga.base.utils.slug import slugify_uniquely from taiga.projects.history.mixins import HistoryResourceMixin +from taiga.projects.notifications.models import NotifyPolicy from taiga.projects.notifications.mixins import WatchedResourceMixin, WatchersViewSetMixin from taiga.projects.notifications.choices import NotifyLevel -from taiga.projects.notifications.utils import ( - attach_project_total_watchers_attrs_to_queryset, - attach_project_is_watcher_to_queryset, - attach_notify_level_to_project_queryset) from taiga.projects.mixins.ordering import BulkUpdateOrderMixin from taiga.projects.mixins.on_destroy import MoveOnDestroyMixin @@ -53,6 +54,7 @@ from . import models from . import permissions from . import services +from dateutil.relativedelta import relativedelta ###################################################### ## Project @@ -64,6 +66,7 @@ class ProjectViewSet(LikedResourceMixin, HistoryResourceMixin, ModelCrudViewSet) list_serializer_class = serializers.ProjectSerializer permission_classes = (permissions.ProjectPermission, ) filter_backends = (filters.CanViewProjectObjFilterBackend,) + filter_fields = (('member', 'members'), 'is_looking_for_people', 'is_featured', @@ -71,34 +74,68 @@ class ProjectViewSet(LikedResourceMixin, HistoryResourceMixin, ModelCrudViewSet) 'is_kanban_activated') order_by_fields = ("memberships__user_order", - "total_fans") + "total_fans", + "total_fans_last_week", + "total_fans_last_month", + "total_fans_last_year", + "total_activity", + "total_activity_last_week", + "total_activity_last_month", + "total_activity_last_year") + + def _get_order_by_field_name(self): + order_by_query_param = filters.CanViewProjectObjFilterBackend.order_by_query_param + order_by = self.request.QUERY_PARAMS.get(order_by_query_param, None) + if order_by is not None and order_by.startswith("-"): + return order_by[1:] def get_queryset(self): qs = super().get_queryset() - qs = self.attach_likes_attrs_to_queryset(qs) - qs = attach_project_total_watchers_attrs_to_queryset(qs) - if self.request.user.is_authenticated(): - qs = attach_project_is_watcher_to_queryset(qs, self.request.user) - qs = attach_notify_level_to_project_queryset(qs, self.request.user) + + # Prefetch doesn't work correctly if then if the field is filtered later (it generates more queries) + # so we add some custom prefetching + qs = qs.prefetch_related("members") + qs = qs.prefetch_related(Prefetch("notify_policies", + NotifyPolicy.objects.exclude(notify_level=NotifyLevel.none), to_attr="valid_notify_policies")) + + Milestone = apps.get_model("milestones", "Milestone") + qs = qs.prefetch_related(Prefetch("milestones", + Milestone.objects.filter(closed=True), to_attr="closed_milestones")) + + # If filtering an activity period we must exclude the activities not updated recently enough + now = timezone.now() + order_by_field_name = self._get_order_by_field_name() + if order_by_field_name == "total_fans_last_week": + qs = qs.filter(totals_updated_datetime__gte=now-relativedelta(weeks=1)) + elif order_by_field_name == "total_fans_last_month": + qs = qs.filter(totals_updated_datetime__gte=now-relativedelta(months=1)) + elif order_by_field_name == "total_fans_last_year": + qs = qs.filter(totals_updated_datetime__gte=now-relativedelta(years=1)) + elif order_by_field_name == "total_activity_last_week": + qs = qs.filter(totals_updated_datetime__gte=now-relativedelta(weeks=1)) + elif order_by_field_name == "total_activity_last_month": + qs = qs.filter(totals_updated_datetime__gte=now-relativedelta(months=1)) + elif order_by_field_name == "total_activity_last_year": + qs = qs.filter(totals_updated_datetime__gte=now-relativedelta(years=1)) return qs def get_serializer_class(self): + serializer_class = self.serializer_class + if self.action == "list": - return self.list_serializer_class - elif self.action == "create": - return self.serializer_class + serializer_class = self.list_serializer_class + elif self.action != "create": + if self.action == "by_slug": + slug = self.request.QUERY_PARAMS.get("slug", None) + project = get_object_or_404(models.Project, slug=slug) + else: + project = self.get_object() - if self.action == "by_slug": - slug = self.request.QUERY_PARAMS.get("slug", None) - project = get_object_or_404(models.Project, slug=slug) - else: - project = self.get_object() + if permissions_service.is_project_owner(self.request.user, project): + serializer_class = self.admin_serializer_class - if permissions_service.is_project_owner(self.request.user, project): - return self.admin_serializer_class - - return self.serializer_class + return serializer_class @detail_route(methods=["POST"]) def watch(self, request, pk=None): diff --git a/taiga/projects/likes/migrations/0002_auto_20151130_2230.py b/taiga/projects/likes/migrations/0002_auto_20151130_2230.py new file mode 100644 index 00000000..fba65742 --- /dev/null +++ b/taiga/projects/likes/migrations/0002_auto_20151130_2230.py @@ -0,0 +1,25 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('likes', '0001_initial'), + ] + + operations = [ + migrations.AlterUniqueTogether( + name='likes', + unique_together=set([]), + ), + migrations.RemoveField( + model_name='likes', + name='content_type', + ), + migrations.DeleteModel( + name='Likes', + ), + ] diff --git a/taiga/projects/likes/mixins/serializers.py b/taiga/projects/likes/mixins/serializers.py index 1897181e..537cef13 100644 --- a/taiga/projects/likes/mixins/serializers.py +++ b/taiga/projects/likes/mixins/serializers.py @@ -20,12 +20,10 @@ from taiga.base.api import serializers class FanResourceSerializerMixin(serializers.ModelSerializer): is_fan = serializers.SerializerMethodField("get_is_fan") - total_fans = serializers.SerializerMethodField("get_total_fans") def get_is_fan(self, obj): - # The "is_fan" attribute is attached in the get_queryset of the viewset. - return getattr(obj, "is_fan", False) or False + if "request" in self.context: + user = self.context["request"].user + return user.is_authenticated() and user.is_fan(obj) - def get_total_fans(self, obj): - # The "total_fans" attribute is attached in the get_queryset of the viewset. - return getattr(obj, "total_fans", 0) or 0 + return False diff --git a/taiga/projects/likes/mixins/viewsets.py b/taiga/projects/likes/mixins/viewsets.py index 3261deb0..0b1b1831 100644 --- a/taiga/projects/likes/mixins/viewsets.py +++ b/taiga/projects/likes/mixins/viewsets.py @@ -24,23 +24,9 @@ from taiga.base.decorators import detail_route from taiga.projects.likes import serializers from taiga.projects.likes import services -from taiga.projects.likes.utils import attach_total_fans_to_queryset, attach_is_fan_to_queryset class LikedResourceMixin: - # Note: Update get_queryset method: - # def get_queryset(self): - # qs = super().get_queryset() - # return self.attach_likes_attrs_to_queryset(qs) - - def attach_likes_attrs_to_queryset(self, queryset): - qs = attach_total_fans_to_queryset(queryset) - - if self.request.user.is_authenticated(): - qs = attach_is_fan_to_queryset(self.request.user, qs) - - return qs - @detail_route(methods=["POST"]) def like(self, request, pk=None): obj = self.get_object() diff --git a/taiga/projects/likes/models.py b/taiga/projects/likes/models.py index 7e9d94f9..d5c4119f 100644 --- a/taiga/projects/likes/models.py +++ b/taiga/projects/likes/models.py @@ -22,27 +22,6 @@ from django.db import models from django.utils.translation import ugettext_lazy as _ -class Likes(models.Model): - content_type = models.ForeignKey("contenttypes.ContentType") - object_id = models.PositiveIntegerField() - content_object = generic.GenericForeignKey("content_type", "object_id") - count = models.PositiveIntegerField(null=False, blank=False, default=0, verbose_name=_("count")) - - class Meta: - verbose_name = _("Likes") - verbose_name_plural = _("Likes") - unique_together = ("content_type", "object_id") - - @property - def project(self): - if hasattr(self.content_object, 'project'): - return self.content_object.project - return None - - def __str__(self): - return self.count - - class Like(models.Model): content_type = models.ForeignKey("contenttypes.ContentType") object_id = models.PositiveIntegerField() diff --git a/taiga/projects/likes/services.py b/taiga/projects/likes/services.py index f136bad5..cf35746d 100644 --- a/taiga/projects/likes/services.py +++ b/taiga/projects/likes/services.py @@ -21,7 +21,7 @@ from django.db.transaction import atomic from django.apps import apps from django.contrib.auth import get_user_model -from .models import Likes, Like +from .models import Like def add_like(obj, user): @@ -36,12 +36,9 @@ def add_like(obj, user): obj_type = apps.get_model("contenttypes", "ContentType").objects.get_for_model(obj) with atomic(): like, created = Like.objects.get_or_create(content_type=obj_type, object_id=obj.id, user=user) - if not created: - return + if like.project is not None: + like.project.refresh_totals() - likes, _ = Likes.objects.get_or_create(content_type=obj_type, object_id=obj.id) - likes.count = F('count') + 1 - likes.save() return like @@ -60,11 +57,12 @@ def remove_like(obj, user): if not qs.exists(): return + like = qs.first() + project = like.project qs.delete() - likes, _ = Likes.objects.get_or_create(content_type=obj_type, object_id=obj.id) - likes.count = F('count') - 1 - likes.save() + if project is not None: + project.refresh_totals() def get_fans(obj): @@ -78,21 +76,6 @@ def get_fans(obj): return get_user_model().objects.filter(likes__content_type=obj_type, likes__object_id=obj.id) -def get_likes(obj): - """Get the number of likes an object has. - - :param obj: Any Django model instance. - - :return: Number of likes or `0` if the object has no likes at all. - """ - obj_type = apps.get_model("contenttypes", "ContentType").objects.get_for_model(obj) - - try: - return Likes.objects.get(content_type=obj_type, object_id=obj.id).count - except Likes.DoesNotExist: - return 0 - - def get_liked(user_or_id, model): """Get the objects liked by an user. diff --git a/taiga/projects/likes/utils.py b/taiga/projects/likes/utils.py deleted file mode 100644 index c9706d76..00000000 --- a/taiga/projects/likes/utils.py +++ /dev/null @@ -1,77 +0,0 @@ -# Copyright (C) 2014-2016 Andrey Antukh -# Copyright (C) 2014-2016 Jesús Espino -# Copyright (C) 2014-2016 David Barragán -# Copyright (C) 2014-2016 Alejandro Alonso -# Copyright (C) 2014-2016 Anler Hernández -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . - -from django.apps import apps - - -def attach_total_fans_to_queryset(queryset, as_field="total_fans"): - """Attach likes count to each object of the queryset. - - Because of laziness of like objects creation, this makes much simpler and more efficient to - access to liked-object number of likes. - - (The other way was to do it in the serializer with some try/except blocks and additional - queries) - - :param queryset: A Django queryset object. - :param as_field: Attach the likes-count as an attribute with this name. - - :return: Queryset object with the additional `as_field` field. - """ - model = queryset.model - type = apps.get_model("contenttypes", "ContentType").objects.get_for_model(model) - sql = """SELECT coalesce(SUM(total_fans), 0) FROM ( - SELECT coalesce(likes_likes.count, 0) total_fans - FROM likes_likes - WHERE likes_likes.content_type_id = {type_id} - AND likes_likes.object_id = {tbl}.id - ) as e""" - - sql = sql.format(type_id=type.id, tbl=model._meta.db_table) - qs = queryset.extra(select={as_field: sql}) - return qs - - -def attach_is_fan_to_queryset(user, queryset, as_field="is_fan"): - """Attach is_like boolean to each object of the queryset. - - Because of laziness of like objects creation, this makes much simpler and more efficient to - access to likes-object and check if the curren user like it. - - (The other way was to do it in the serializer with some try/except blocks and additional - queries) - - :param user: A users.User object model - :param queryset: A Django queryset object. - :param as_field: Attach the boolean as an attribute with this name. - - :return: Queryset object with the additional `as_field` field. - """ - model = queryset.model - type = apps.get_model("contenttypes", "ContentType").objects.get_for_model(model) - sql = ("""SELECT CASE WHEN (SELECT count(*) - FROM likes_like - WHERE likes_like.content_type_id = {type_id} - AND likes_like.object_id = {tbl}.id - AND likes_like.user_id = {user_id}) > 0 - THEN TRUE - ELSE FALSE - END""") - sql = sql.format(type_id=type.id, tbl=model._meta.db_table, user_id=user.id) - qs = queryset.extra(select={as_field: sql}) - return qs diff --git a/taiga/projects/migrations/0030_auto_20151128_0757.py b/taiga/projects/migrations/0030_auto_20151128_0757.py new file mode 100644 index 00000000..5f515029 --- /dev/null +++ b/taiga/projects/migrations/0030_auto_20151128_0757.py @@ -0,0 +1,164 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import connection, migrations, models +from django.utils.timezone import utc +import datetime + + +def update_totals(apps, schema_editor): + model = apps.get_model("projects", "Project") + type = apps.get_model("contenttypes", "ContentType").objects.get_for_model(model) + sql=""" + UPDATE projects_project + SET + totals_updated_datetime = totals.totals_updated_datetime, + total_fans = totals.total_fans, + total_fans_last_week = totals.total_fans_last_week, + total_fans_last_month = totals.total_fans_last_month, + total_fans_last_year = totals.total_fans_last_year, + total_activity = totals.total_activity, + total_activity_last_week = totals.total_activity_last_week, + total_activity_last_month = totals.total_activity_last_month, + total_activity_last_year = totals.total_activity_last_year + FROM ( + WITH + totals_activity AS (SELECT + split_part(timeline_timeline.namespace, ':', 2)::integer as project_id, + count(timeline_timeline.namespace) total_activity, + MAX (created) updated_datetime + FROM timeline_timeline + WHERE namespace LIKE 'project:%' + GROUP BY namespace), + totals_activity_week AS (SELECT + split_part(timeline_timeline.namespace, ':', 2)::integer as project_id, + count(timeline_timeline.namespace) total_activity_last_week + FROM timeline_timeline + WHERE namespace LIKE 'project:%' + AND timeline_timeline.created > current_date - interval '7' day + GROUP BY namespace), + totals_activity_month AS (SELECT + split_part(timeline_timeline.namespace, ':', 2)::integer as project_id, + count(timeline_timeline.namespace) total_activity_last_month + FROM timeline_timeline + WHERE namespace LIKE 'project:%' + AND timeline_timeline.created > current_date - interval '30' day + GROUP BY namespace), + totals_activity_year AS (SELECT + split_part(timeline_timeline.namespace, ':', 2)::integer as project_id, + count(timeline_timeline.namespace) total_activity_last_year + FROM timeline_timeline + WHERE namespace LIKE 'project:%' + AND timeline_timeline.created > current_date - interval '365' day + GROUP BY namespace), + totals_fans AS (SELECT + object_id as project_id, + COUNT(likes_like.object_id) total_fans, + MAX (created_date) updated_datetime + FROM likes_like + WHERE content_type_id = {type_id} + GROUP BY object_id), + totals_fans_week AS (SELECT + object_id as project_id, + COUNT(likes_like.object_id) total_fans_last_week + FROM likes_like + WHERE content_type_id = {type_id} + AND likes_like.created_date > current_date - interval '7' day + GROUP BY object_id), + totals_fans_month AS (SELECT + object_id as project_id, + COUNT(likes_like.object_id) total_fans_last_month + FROM likes_like + WHERE content_type_id = {type_id} + AND likes_like.created_date > current_date - interval '30' day + GROUP BY object_id), + totals_fans_year AS (SELECT + object_id as project_id, + COUNT(likes_like.object_id) total_fans_last_year + FROM likes_like + WHERE content_type_id = {type_id} + AND likes_like.created_date > current_date - interval '365' day + GROUP BY object_id) + SELECT + totals_activity.project_id, + COALESCE(total_activity, 0) total_activity, + COALESCE(total_activity_last_week, 0) total_activity_last_week, + COALESCE(total_activity_last_month, 0) total_activity_last_month, + COALESCE(total_activity_last_year, 0) total_activity_last_year, + COALESCE(total_fans, 0) total_fans, + COALESCE(total_fans_last_week, 0) total_fans_last_week, + COALESCE(total_fans_last_month, 0) total_fans_last_month, + COALESCE(total_fans_last_year, 0) total_fans_last_year, + totals_activity.updated_datetime totals_updated_datetime + FROM totals_activity + LEFT JOIN totals_fans ON totals_activity.project_id = totals_fans.project_id + LEFT JOIN totals_fans_week ON totals_activity.project_id = totals_fans_week.project_id + LEFT JOIN totals_fans_month ON totals_activity.project_id = totals_fans_month.project_id + LEFT JOIN totals_fans_year ON totals_activity.project_id = totals_fans_year.project_id + LEFT JOIN totals_activity_week ON totals_activity.project_id = totals_activity_week.project_id + LEFT JOIN totals_activity_month ON totals_activity.project_id = totals_activity_month.project_id + LEFT JOIN totals_activity_year ON totals_activity.project_id = totals_activity_year.project_id + ) totals + WHERE projects_project.id = totals.project_id + """.format(type_id=type.id) + + cursor = connection.cursor() + cursor.execute(sql) + + +class Migration(migrations.Migration): + + dependencies = [ + ('projects', '0029_project_is_looking_for_people'), + ('timeline', '0004_auto_20150603_1312'), + ] + + operations = [ + migrations.AddField( + model_name='project', + name='total_activity', + field=models.PositiveIntegerField(default=0, verbose_name='count', db_index=True), + ), + migrations.AddField( + model_name='project', + name='total_activity_last_month', + field=models.PositiveIntegerField(default=0, verbose_name='activity last month', db_index=True), + ), + migrations.AddField( + model_name='project', + name='total_activity_last_week', + field=models.PositiveIntegerField(default=0, verbose_name='activity last week', db_index=True), + ), + migrations.AddField( + model_name='project', + name='total_activity_last_year', + field=models.PositiveIntegerField(default=0, verbose_name='activity last year', db_index=True), + ), + migrations.AddField( + model_name='project', + name='total_fans', + field=models.PositiveIntegerField(default=0, verbose_name='count', db_index=True), + ), + migrations.AddField( + model_name='project', + name='total_fans_last_month', + field=models.PositiveIntegerField(default=0, verbose_name='fans last month', db_index=True), + ), + migrations.AddField( + model_name='project', + name='total_fans_last_week', + field=models.PositiveIntegerField(default=0, verbose_name='fans last week', db_index=True), + ), + migrations.AddField( + model_name='project', + name='total_fans_last_year', + field=models.PositiveIntegerField(default=0, verbose_name='fans last year', db_index=True), + ), + migrations.AddField( + model_name='project', + name='totals_updated_datetime', + field=models.DateTimeField(default=datetime.datetime(2015, 11, 28, 7, 57, 11, 743976, tzinfo=utc), auto_now_add=True, verbose_name='updated date time', db_index=True), + preserve_default=False, + ), + migrations.RunPython(update_totals), + ] diff --git a/taiga/projects/milestones/api.py b/taiga/projects/milestones/api.py index 8f5a1bbd..2823f762 100644 --- a/taiga/projects/milestones/api.py +++ b/taiga/projects/milestones/api.py @@ -84,7 +84,7 @@ class MilestoneViewSet(HistoryResourceMixin, WatchedResourceMixin, ModelCrudView if self.request.user.is_authenticated(): us_qs = attach_is_voter_to_queryset(self.request.user, us_qs) - us_qs = attach_is_watcher_to_queryset(self.request.user, us_qs) + us_qs = attach_is_watcher_to_queryset(us_qs, self.request.user) qs = qs.prefetch_related(Prefetch("user_stories", queryset=us_qs)) diff --git a/taiga/projects/models.py b/taiga/projects/models.py index 6579a1ed..0f9a2fa3 100644 --- a/taiga/projects/models.py +++ b/taiga/projects/models.py @@ -46,8 +46,12 @@ from taiga.projects.notifications.services import ( set_notify_policy_level_to_ignore, create_notify_policy_if_not_exists) +from taiga.timeline.service import build_project_namespace + from . import choices +from dateutil.relativedelta import relativedelta + class Membership(models.Model): # This model stores all project memberships. Also @@ -198,6 +202,36 @@ class Project(ProjectDefaults, TaggedMixin, models.Model): tags_colors = TextArrayField(dimension=2, default=[], null=False, blank=True, verbose_name=_("tags colors")) + + #Totals: + totals_updated_datetime = models.DateTimeField(null=False, blank=False, auto_now_add=True, + verbose_name=_("updated date time"), db_index=True) + + total_fans = models.PositiveIntegerField(null=False, blank=False, default=0, + verbose_name=_("count"), db_index=True) + + total_fans_last_week = models.PositiveIntegerField(null=False, blank=False, default=0, + verbose_name=_("fans last week"), db_index=True) + + total_fans_last_month = models.PositiveIntegerField(null=False, blank=False, default=0, + verbose_name=_("fans last month"), db_index=True) + + total_fans_last_year = models.PositiveIntegerField(null=False, blank=False, default=0, + verbose_name=_("fans last year"), db_index=True) + + total_activity = models.PositiveIntegerField(null=False, blank=False, default=0, + verbose_name=_("count"), db_index=True) + + total_activity_last_week = models.PositiveIntegerField(null=False, blank=False, default=0, + verbose_name=_("activity last week"), db_index=True) + + total_activity_last_month = models.PositiveIntegerField(null=False, blank=False, default=0, + verbose_name=_("activity last month"), db_index=True) + + total_activity_last_year = models.PositiveIntegerField(null=False, blank=False, default=0, + verbose_name=_("activity last year"), db_index=True) + + _cached_user_stories = None _importing = None class Meta: @@ -233,6 +267,51 @@ class Project(ProjectDefaults, TaggedMixin, models.Model): super().save(*args, **kwargs) + def refresh_totals(self, save=True): + now = timezone.now() + self.totals_updated_datetime = now + + Like = apps.get_model("likes", "Like") + content_type = apps.get_model("contenttypes", "ContentType").objects.get_for_model(Project) + qs = Like.objects.filter(content_type=content_type, object_id=self.id) + + self.total_fans = qs.count() + + qs_week = qs.filter(created_date__gte=now-relativedelta(weeks=1)) + self.total_fans_last_week = qs_week.count() + + qs_month = qs.filter(created_date__gte=now-relativedelta(months=1)) + self.total_fans_last_month = qs_month.count() + + qs_year = qs.filter(created_date__gte=now-relativedelta(years=1)) + self.total_fans_last_year = qs_year.count() + + tl_model = apps.get_model("timeline", "Timeline") + namespace = build_project_namespace(self) + + qs = tl_model.objects.filter(namespace=namespace) + self.total_activity = qs.count() + + qs_week = qs.filter(created__gte=now-relativedelta(weeks=1)) + self.total_activity_last_week = qs_week.count() + + qs_month = qs.filter(created__gte=now-relativedelta(months=1)) + self.total_activity_last_month = qs_month.count() + + qs_year = qs.filter(created__gte=now-relativedelta(years=1)) + self.total_activity_last_year = qs_year.count() + + if save: + self.save() + + @property + def cached_user_stories(self): + print(1111111, self._cached_user_stories) + if self._cached_user_stories is None: + self._cached_user_stories = list(self.user_stories.all()) + + return self._cached_user_stories + def get_roles(self): return self.roles.all() diff --git a/taiga/projects/notifications/mixins.py b/taiga/projects/notifications/mixins.py index ba316eac..726e639b 100644 --- a/taiga/projects/notifications/mixins.py +++ b/taiga/projects/notifications/mixins.py @@ -53,12 +53,12 @@ class WatchedResourceMixin: _not_notify = False def attach_watchers_attrs_to_queryset(self, queryset): - qs = attach_watchers_to_queryset(queryset) - qs = attach_total_watchers_to_queryset(qs) + queryset = attach_watchers_to_queryset(queryset) + queryset = attach_total_watchers_to_queryset(queryset) if self.request.user.is_authenticated(): - qs = attach_is_watcher_to_queryset(self.request.user, qs) + queryset = attach_is_watcher_to_queryset(queryset, self.request.user) - return qs + return queryset @detail_route(methods=["POST"]) def watch(self, request, pk=None): @@ -187,8 +187,11 @@ class WatchedResourceModelSerializer(serializers.ModelSerializer): total_watchers = serializers.SerializerMethodField("get_total_watchers") def get_is_watcher(self, obj): - # The "is_watcher" attribute is attached in the get_queryset of the viewset. - return getattr(obj, "is_watcher", False) or False + if "request" in self.context: + user = self.context["request"].user + return user.is_authenticated() and user.is_watcher(obj) + + return False def get_total_watchers(self, obj): # The "total_watchers" attribute is attached in the get_queryset of the viewset. diff --git a/taiga/projects/notifications/utils.py b/taiga/projects/notifications/utils.py index b985aa0d..991228b0 100644 --- a/taiga/projects/notifications/utils.py +++ b/taiga/projects/notifications/utils.py @@ -41,11 +41,11 @@ def attach_watchers_to_queryset(queryset, as_field="watchers"): return qs -def attach_is_watcher_to_queryset(user, queryset, as_field="is_watcher"): +def attach_is_watcher_to_queryset(queryset, user, as_field="is_watcher"): """Attach is_watcher boolean to each object of the queryset. - :param user: A users.User object model :param queryset: A Django queryset object. + :param user: A users.User object model :param as_field: Attach the boolean as an attribute with this name. :return: Queryset object with the additional `as_field` field. @@ -83,74 +83,3 @@ def attach_total_watchers_to_queryset(queryset, as_field="total_watchers"): sql = sql.format(type_id=type.id, tbl=model._meta.db_table) qs = queryset.extra(select={as_field: sql}) return qs - - -def attach_project_is_watcher_to_queryset(queryset, user, as_field="is_watcher"): - """Attach is_watcher boolean to each object of the projects queryset. - - :param user: A users.User object model - :param queryset: A Django projects queryset object. - :param as_field: Attach the boolean as an attribute with this name. - - :return: Queryset object with the additional `as_field` field. - """ - model = queryset.model - type = apps.get_model("contenttypes", "ContentType").objects.get_for_model(model) - sql = ("""SELECT CASE WHEN (SELECT count(*) - FROM notifications_notifypolicy - WHERE notifications_notifypolicy.project_id = {tbl}.id - AND notifications_notifypolicy.user_id = {user_id} - AND notifications_notifypolicy.notify_level != {ignore_notify_level}) > 0 - - THEN TRUE - ELSE FALSE - END""") - sql = sql.format(tbl=model._meta.db_table, user_id=user.id, ignore_notify_level=NotifyLevel.none) - qs = queryset.extra(select={as_field: sql}) - return qs - - -def attach_project_total_watchers_attrs_to_queryset(queryset, as_field="total_watchers"): - """Attach watching user ids to each project of the queryset. - - :param queryset: A Django projects queryset object. - :param as_field: Attach the watchers as an attribute with this name. - - :return: Queryset object with the additional `as_field` field. - """ - model = queryset.model - type = apps.get_model("contenttypes", "ContentType").objects.get_for_model(model) - - sql = ("""SELECT count(user_id) - FROM notifications_notifypolicy - WHERE notifications_notifypolicy.project_id = {tbl}.id - AND notifications_notifypolicy.notify_level != {ignore_notify_level}""") - sql = sql.format(tbl=model._meta.db_table, ignore_notify_level=NotifyLevel.none) - qs = queryset.extra(select={as_field: sql}) - - return qs - - -def attach_notify_level_to_project_queryset(queryset, user): - """ - Function that attach "notify_level" attribute on each queryset - result for query notification level of current user for each - project in the most efficient way. - - :param queryset: A Django queryset object. - :param user: A User model object. - - :return: Queryset object with the additional `as_field` field. - """ - user_id = getattr(user, "id", None) or "NULL" - default_level = NotifyLevel.involved - - sql = strip_lines(""" - COALESCE((SELECT notifications_notifypolicy.notify_level - FROM notifications_notifypolicy - WHERE notifications_notifypolicy.project_id = projects_project.id - AND notifications_notifypolicy.user_id = {user_id}), - {default_level}) - """) - sql = sql.format(user_id=user_id, default_level=default_level) - return queryset.extra(select={"notify_level": sql}) diff --git a/taiga/projects/serializers.py b/taiga/projects/serializers.py index 5d9bb5cc..0259a6b8 100644 --- a/taiga/projects/serializers.py +++ b/taiga/projects/serializers.py @@ -26,6 +26,7 @@ from taiga.base.fields import PgArrayField from taiga.base.fields import TagsField from taiga.base.fields import TagsColorsField +from taiga.projects.notifications.choices import NotifyLevel from taiga.users.services import get_photo_or_gravatar_url from taiga.users.serializers import UserSerializer from taiga.users.serializers import UserBasicInfoSerializer @@ -318,6 +319,7 @@ class ProjectSerializer(FanResourceSerializerMixin, WatchedResourceModelSerializ tags_colors = TagsColorsField(required=False) total_closed_milestones = serializers.SerializerMethodField("get_total_closed_milestones") notify_level = serializers.SerializerMethodField("get_notify_level") + total_watchers = serializers.SerializerMethodField("get_total_watchers") class Meta: model = models.Project @@ -336,10 +338,27 @@ class ProjectSerializer(FanResourceSerializerMixin, WatchedResourceModelSerializ return False def get_total_closed_milestones(self, obj): + # The "closed_milestone" attribute can be attached in the get_queryset method of the viewset. + qs_closed_milestones = getattr(obj, "closed_milestones", None) + if qs_closed_milestones is not None: + return qs_closed_milestones + return obj.milestones.filter(closed=True).count() def get_notify_level(self, obj): - return getattr(obj, "notify_level", None) + if "request" in self.context: + user = self.context["request"].user + return user.is_authenticated() and user.get_notify_level(obj) + + return None + + def get_total_watchers(self, obj): + # The "valid_notify_policies" attribute can be attached in the get_queryset method of the viewset. + qs_valid_notify_policies = getattr(obj, "valid_notify_policies", None) + if qs_valid_notify_policies is not None: + return len(qs_valid_notify_policies) + + return obj.notify_policies.exclude(notify_level=NotifyLevel.none).count() class ProjectDetailSerializer(ProjectSerializer): diff --git a/taiga/timeline/signals.py b/taiga/timeline/signals.py index 77ed6695..5edc3850 100644 --- a/taiga/timeline/signals.py +++ b/taiga/timeline/signals.py @@ -46,6 +46,8 @@ def _push_to_timelines(project, user, obj, event_type, created_datetime, extra_d namespace=build_project_namespace(project), extra_data=extra_data) + project.refresh_totals() + if hasattr(obj, "get_related_people"): related_people = obj.get_related_people() diff --git a/taiga/users/services.py b/taiga/users/services.py index b40798a0..a592698c 100644 --- a/taiga/users/services.py +++ b/taiga/users/services.py @@ -224,7 +224,7 @@ def _build_watched_sql_for_projects(for_user): tags, notifications_notifypolicy.project_id AS object_id, projects_project.id AS project, slug, projects_project.name, null::text AS subject, notifications_notifypolicy.created_at as created_date, - coalesce(watchers, 0) AS total_watchers, coalesce(likes_likes.count, 0) AS total_fans, null::integer AS total_voters, + coalesce(watchers, 0) AS total_watchers, projects_project.total_fans AS total_fans, null::integer AS total_voters, null::integer AS assigned_to, null::text as status, null::text as status_color FROM notifications_notifypolicy INNER JOIN projects_project @@ -235,8 +235,6 @@ def _build_watched_sql_for_projects(for_user): GROUP BY project_id ) type_watchers ON projects_project.id = type_watchers.project_id - LEFT JOIN likes_likes - ON (projects_project.id = likes_likes.object_id AND {project_content_type_id} = likes_likes.content_type_id) WHERE notifications_notifypolicy.user_id = {for_user_id} AND notifications_notifypolicy.notify_level != {none_notify_level} @@ -254,7 +252,7 @@ def _build_liked_sql_for_projects(for_user): tags, likes_like.object_id AS object_id, projects_project.id AS project, slug, projects_project.name, null::text AS subject, likes_like.created_date, - coalesce(watchers, 0) AS total_watchers, coalesce(likes_likes.count, 0) AS total_fans, + coalesce(watchers, 0) AS total_watchers, projects_project.total_fans AS total_fans, null::integer AS assigned_to, null::text as status, null::text as status_color FROM likes_like INNER JOIN projects_project @@ -265,8 +263,6 @@ def _build_liked_sql_for_projects(for_user): GROUP BY project_id ) type_watchers ON projects_project.id = type_watchers.project_id - LEFT JOIN likes_likes - ON (projects_project.id = likes_likes.object_id AND {project_content_type_id} = likes_likes.content_type_id) WHERE likes_like.user_id = {for_user_id} AND {project_content_type_id} = likes_like.content_type_id """ sql = sql.format( diff --git a/tests/factories.py b/tests/factories.py index ee2f4c5c..41f8c3a2 100644 --- a/tests/factories.py +++ b/tests/factories.py @@ -423,15 +423,6 @@ class LikeFactory(Factory): user = factory.SubFactory("tests.factories.UserFactory") -class LikesFactory(Factory): - class Meta: - model = "likes.Likes" - strategy = factory.CREATE_STRATEGY - - content_type = factory.SubFactory("tests.factories.ContentTypeFactory") - object_id = factory.Sequence(lambda n: n) - - class VoteFactory(Factory): class Meta: model = "votes.Vote" diff --git a/tests/integration/resources_permissions/test_projects_resource.py b/tests/integration/resources_permissions/test_projects_resource.py index e604ac7f..023fee94 100644 --- a/tests/integration/resources_permissions/test_projects_resource.py +++ b/tests/integration/resources_permissions/test_projects_resource.py @@ -77,10 +77,6 @@ def data(): f.LikeFactory(content_type=project_ct, object_id=m.private_project2.pk, user=m.project_member_with_perms) f.LikeFactory(content_type=project_ct, object_id=m.private_project2.pk, user=m.project_owner) - f.LikesFactory(content_type=project_ct, object_id=m.public_project.pk, count=2) - f.LikesFactory(content_type=project_ct, object_id=m.private_project1.pk, count=2) - f.LikesFactory(content_type=project_ct, object_id=m.private_project2.pk, count=2) - return m diff --git a/tests/integration/test_fan_projects.py b/tests/integration/test_fan_projects.py index d5377e75..dc9b2fc5 100644 --- a/tests/integration/test_fan_projects.py +++ b/tests/integration/test_fan_projects.py @@ -76,26 +76,10 @@ def test_get_project_fan(client): assert response.data['id'] == like.user.id -def test_get_project_total_fans(client): - user = f.UserFactory.create() - project = f.create_project(owner=user) - f.MembershipFactory.create(project=project, user=user, is_owner=True) - url = reverse("projects-detail", args=(project.id,)) - - f.LikesFactory.create(content_object=project, count=5) - - client.login(user) - response = client.get(url) - - assert response.status_code == 200 - assert response.data['total_fans'] == 5 - - def test_get_project_is_fan(client): user = f.UserFactory.create() project = f.create_project(owner=user) f.MembershipFactory.create(project=project, user=user, is_owner=True) - f.LikesFactory.create(content_object=project) url_detail = reverse("projects-detail", args=(project.id,)) url_like = reverse("projects-like", args=(project.id,)) url_unlike = reverse("projects-unlike", args=(project.id,)) diff --git a/tests/integration/test_notifications.py b/tests/integration/test_notifications.py index f4006836..42058f07 100644 --- a/tests/integration/test_notifications.py +++ b/tests/integration/test_notifications.py @@ -53,24 +53,6 @@ def mail(): return mail -def test_attach_notify_level_to_project_queryset(): - project1 = f.ProjectFactory.create() - f.ProjectFactory.create() - - qs = project1.__class__.objects.order_by("id") - qs = utils.attach_notify_level_to_project_queryset(qs, project1.owner) - - assert len(qs) == 2 - assert qs[0].notify_level == NotifyLevel.involved - assert qs[1].notify_level == NotifyLevel.involved - - services.create_notify_policy(project1, project1.owner, NotifyLevel.all) - qs = project1.__class__.objects.order_by("id") - qs = utils.attach_notify_level_to_project_queryset(qs, project1.owner) - assert qs[0].notify_level == NotifyLevel.all - assert qs[1].notify_level == NotifyLevel.involved - - def test_create_retrieve_notify_policy(): project = f.ProjectFactory.create() diff --git a/tests/integration/test_totals_projects.py b/tests/integration/test_totals_projects.py new file mode 100644 index 00000000..8947d6ec --- /dev/null +++ b/tests/integration/test_totals_projects.py @@ -0,0 +1,154 @@ +# Copyright (C) 2014-2015 Andrey Antukh +# Copyright (C) 2014-2015 Jesús Espino +# Copyright (C) 2014-2015 David Barragán +# Copyright (C) 2014-2015 Anler Hernández +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +import pytest + +import datetime + +from .. import factories as f + +from taiga.projects.history.choices import HistoryType +from taiga.projects.models import Project + +from django.core.urlresolvers import reverse + +pytestmark = pytest.mark.django_db + +def test_project_totals_updated_on_activity(client): + project = f.create_project() + totals_updated_datetime = project.totals_updated_datetime + now = datetime.datetime.now() + assert project.total_activity == 0 + + totals_updated_datetime = project.totals_updated_datetime + us = f.UserStoryFactory.create(project=project, owner=project.owner) + f.HistoryEntryFactory.create( + user={"pk": project.owner.id}, + comment="", + type=HistoryType.change, + key="userstories.userstory:{}".format(us.id), + is_hidden=False, + diff=[], + created_at=now - datetime.timedelta(days=3) + ) + + project = Project.objects.get(id=project.id) + assert project.total_activity == 1 + assert project.total_activity_last_week == 1 + assert project.total_activity_last_month == 1 + assert project.total_activity_last_year == 1 + assert project.totals_updated_datetime > totals_updated_datetime + + totals_updated_datetime = project.totals_updated_datetime + f.HistoryEntryFactory.create( + user={"pk": project.owner.id}, + comment="", + type=HistoryType.change, + key="userstories.userstory:{}".format(us.id), + is_hidden=False, + diff=[], + created_at=now - datetime.timedelta(days=13) + ) + + project = Project.objects.get(id=project.id) + assert project.total_activity == 2 + assert project.total_activity_last_week == 1 + assert project.total_activity_last_month == 2 + assert project.total_activity_last_year == 2 + assert project.totals_updated_datetime > totals_updated_datetime + + totals_updated_datetime = project.totals_updated_datetime + f.HistoryEntryFactory.create( + user={"pk": project.owner.id}, + comment="", + type=HistoryType.change, + key="userstories.userstory:{}".format(us.id), + is_hidden=False, + diff=[], + created_at=now - datetime.timedelta(days=33) + ) + + project = Project.objects.get(id=project.id) + assert project.total_activity == 3 + assert project.total_activity_last_week == 1 + assert project.total_activity_last_month == 2 + assert project.total_activity_last_year == 3 + assert project.totals_updated_datetime > totals_updated_datetime + + totals_updated_datetime = project.totals_updated_datetime + f.HistoryEntryFactory.create( + user={"pk": project.owner.id}, + comment="", + type=HistoryType.change, + key="userstories.userstory:{}".format(us.id), + is_hidden=False, + diff=[], + created_at=now - datetime.timedelta(days=380) + ) + + project = Project.objects.get(id=project.id) + assert project.total_activity == 4 + assert project.total_activity_last_week == 1 + assert project.total_activity_last_month == 2 + assert project.total_activity_last_year == 3 + assert project.totals_updated_datetime > totals_updated_datetime + + + +def test_project_totals_updated_on_like(client): + project = f.create_project() + f.MembershipFactory.create(project=project, user=project.owner, is_owner=True) + + totals_updated_datetime = project.totals_updated_datetime + now = datetime.datetime.now() + assert project.total_activity == 0 + + now = datetime.datetime.now() + totals_updated_datetime = project.totals_updated_datetime + us = f.UserStoryFactory.create(project=project, owner=project.owner) + + l = f.LikeFactory.create(content_object=project) + l.created_date=now-datetime.timedelta(days=13) + l.save() + + l = f.LikeFactory.create(content_object=project) + l.created_date=now-datetime.timedelta(days=33) + l.save() + + l = f.LikeFactory.create(content_object=project) + l.created_date=now-datetime.timedelta(days=633) + l.save() + + project.refresh_totals() + project = Project.objects.get(id=project.id) + + assert project.total_fans == 3 + assert project.total_fans_last_week == 0 + assert project.total_fans_last_month == 1 + assert project.total_fans_last_year == 2 + assert project.totals_updated_datetime > totals_updated_datetime + + client.login(project.owner) + url_like = reverse("projects-like", args=(project.id,)) + response = client.post(url_like) + print(response.data) + + project = Project.objects.get(id=project.id) + assert project.total_fans == 4 + assert project.total_fans_last_week == 1 + assert project.total_fans_last_month == 2 + assert project.total_fans_last_year == 3 + assert project.totals_updated_datetime > totals_updated_datetime diff --git a/tests/integration/test_users.py b/tests/integration/test_users.py index 0410b579..e1e91d92 100644 --- a/tests/integration/test_users.py +++ b/tests/integration/test_users.py @@ -388,7 +388,6 @@ def test_get_liked_list(): membership = f.MembershipFactory(project=project, role=role, user=fan_user) content_type = ContentType.objects.get_for_model(project) f.LikeFactory(content_type=content_type, object_id=project.id, user=fan_user) - f.LikesFactory(content_type=content_type, object_id=project.id, count=1) assert len(get_liked_list(fan_user, viewer_user)) == 1 assert len(get_liked_list(fan_user, viewer_user, type="project")) == 1 @@ -495,8 +494,8 @@ def test_get_liked_list_valid_info(): project = f.ProjectFactory(is_private=False, name="Testing project", tags=['test', 'tag']) content_type = ContentType.objects.get_for_model(project) like = f.LikeFactory(content_type=content_type, object_id=project.id, user=fan_user) - f.LikesFactory(content_type=content_type, object_id=project.id, count=1) - + project.refresh_totals() + raw_project_like_info = get_liked_list(fan_user, viewer_user)[0] project_like_info = LikedObjectSerializer(raw_project_like_info).data @@ -762,7 +761,6 @@ def test_get_liked_list_permissions(): membership = f.MembershipFactory(project=project, role=role, user=viewer_priviliged_user) content_type = ContentType.objects.get_for_model(project) f.LikeFactory(content_type=content_type, object_id=project.id, user=fan_user) - f.LikesFactory(content_type=content_type, object_id=project.id, count=1) #If the project is private a viewer user without any permission shouldn' see # any vote