Improving the stats API performance

taiga/projects/models.py

@@ -271,31 +271,6 @@ class Project(ProjectDefaults, TaggedMixin, models.Model):
         rp_query = rp_query.exclude(role__id__in=roles.values_list("id", flat=True))
         rp_query.delete()
 
-    def _get_user_stories_points(self, user_stories):
-        role_points = [us.role_points.all() for us in user_stories]
-        flat_role_points = itertools.chain(*role_points)
-        flat_role_dicts = map(lambda x: {x.role_id: x.points.value if x.points.value else 0},
-                              flat_role_points)
-        return dict_sum(*flat_role_dicts)
-
-    def _get_points_increment(self, client_requirement, team_requirement):
-        last_milestones = self.milestones.order_by('-estimated_finish')
-        last_milestone = last_milestones[0] if last_milestones else None
-        if last_milestone:
-            user_stories = self.user_stories.filter(
-                created_date__gte=last_milestone.estimated_finish,
-                client_requirement=client_requirement,
-                team_requirement=team_requirement
-            )
-        else:
-            user_stories = self.user_stories.filter(
-                client_requirement=client_requirement,
-                team_requirement=team_requirement
-            )
-        user_stories = user_stories.prefetch_related('role_points', 'role_points__points')
-        return self._get_user_stories_points(user_stories)
-
-
     @property
     def project(self):
         return self
@@ -304,45 +279,6 @@ class Project(ProjectDefaults, TaggedMixin, models.Model):
     def project(self):
         return self
 
-    @property
-    def future_team_increment(self):
-        team_increment = self._get_points_increment(False, True)
-        shared_increment = {key: value / 2 for key, value in self.future_shared_increment.items()}
-        return dict_sum(team_increment, shared_increment)
-
-    @property
-    def future_client_increment(self):
-        client_increment = self._get_points_increment(True, False)
-        shared_increment = {key: value / 2 for key, value in self.future_shared_increment.items()}
-        return dict_sum(client_increment, shared_increment)
-
-    @property
-    def future_shared_increment(self):
-        return self._get_points_increment(True, True)
-
-    @property
-    def closed_points(self):
-        return self.calculated_points["closed"]
-
-    @property
-    def defined_points(self):
-        return self.calculated_points["defined"]
-
-    @property
-    def assigned_points(self):
-        return self.calculated_points["assigned"]
-
-    @property
-    def calculated_points(self):
-        user_stories = self.user_stories.all().prefetch_related('role_points', 'role_points__points')
-        closed_user_stories = user_stories.filter(is_closed=True)
-        assigned_user_stories = user_stories.filter(milestone__isnull=False)
-        return {
-            "defined": self._get_user_stories_points(user_stories),
-            "closed": self._get_user_stories_points(closed_user_stories),
-            "assigned": self._get_user_stories_points(assigned_user_stories),
-        }
-
     def _get_q_watchers(self):
         return Q(notify_policies__project_id=self.id) & ~Q(notify_policies__notify_level=NotifyLevel.none)
 

taiga/projects/services/stats.py

@@ -19,94 +19,11 @@ from django.db.models import Q, Count
 from django.apps import apps
 import datetime
 import copy
+import collections
 
 from taiga.projects.history.models import HistoryEntry
 from taiga.projects.userstories.models import RolePoints
 
-
-def _get_total_story_points(project):
-    return (project.total_story_points if project.total_story_points not in [None, 0] else
-                    sum(project.calculated_points["defined"].values()))
-
-def _get_total_milestones(project):
-    return (project.total_milestones if project.total_milestones not in [None, 0] else
-                    project.milestones.count())
-
-def _get_milestones_stats_for_backlog(project):
-    """
-    Get collection of stats for each millestone of project.
-    Data returned by this function are used on backlog.
-    """
-    current_evolution = 0
-    current_team_increment = 0
-    current_client_increment = 0
-
-    optimal_points_per_sprint = 0
-
-    total_story_points = _get_total_story_points(project)
-    total_milestones = _get_total_milestones(project)
-
-    if total_story_points and total_milestones:
-        optimal_points_per_sprint = total_story_points / total_milestones
-
-    future_team_increment = sum(project.future_team_increment.values())
-    future_client_increment = sum(project.future_client_increment.values())
-
-    milestones = project.milestones.order_by('estimated_start').\
-            prefetch_related("user_stories",
-                             "user_stories__role_points",
-                             "user_stories__role_points__points")
-
-    milestones = list(milestones)
-    milestones_count = len(milestones)
-    optimal_points = 0
-    team_increment = 0
-    client_increment = 0
-
-    for current_milestone in range(0, max(milestones_count, total_milestones)):
-        optimal_points = (total_story_points -
-                            (optimal_points_per_sprint * current_milestone))
-
-        evolution = (total_story_points - current_evolution
-                        if current_evolution is not None else None)
-
-        if current_milestone < milestones_count:
-            ml = milestones[current_milestone]
-
-            milestone_name = ml.name
-            team_increment = current_team_increment
-            client_increment = current_client_increment
-
-            current_evolution += sum(ml.closed_points.values())
-            current_team_increment += sum(ml.team_increment_points.values())
-            current_client_increment += sum(ml.client_increment_points.values())
-
-        else:
-            milestone_name = _("Future sprint")
-            team_increment = current_team_increment + future_team_increment,
-            client_increment = current_client_increment + future_client_increment,
-            current_evolution = None
-
-        yield {
-            'name': milestone_name,
-            'optimal': optimal_points,
-            'evolution': evolution,
-            'team-increment': team_increment,
-            'client-increment': client_increment,
-        }
-
-    optimal_points -= optimal_points_per_sprint
-    evolution = (total_story_points - current_evolution
-                    if current_evolution is not None and total_story_points else None)
-    yield {
-        'name': _('Project End'),
-        'optimal': optimal_points,
-        'evolution': evolution,
-        'team-increment': team_increment,
-        'client-increment': client_increment,
-    }
-
-
 def _count_status_object(status_obj, counting_storage):
     if status_obj.id in counting_storage:
         counting_storage[status_obj.id]['count'] += 1
@@ -218,35 +135,217 @@ def get_stats_for_project_issues(project):
     return project_issues_stats
 
 
+def _get_milestones_stats_for_backlog(project, milestones):
+    """
+    Calculates the stats associated to the milestones parameter.
+
+    - project is a Project model instance
+        We assume this object have also the following numeric attributes:
+        - _defined_points
+        - _future_team_increment
+        - _future_client_increment
+
+    - milestones is a sorted dict of Milestone model instances sorted by estimated_start.
+        We assume this objects have also the following numeric attributes:
+        - _closed_points
+        - _team_increment_points
+        - _client_increment_points
+
+    The returned result is a list of dicts where each entry contains the following keys:
+        - name
+        - optimal
+        - evolution
+        - team
+        - client
+    """
+    current_evolution = 0
+    current_team_increment = 0
+    current_client_increment = 0
+    optimal_points_per_sprint = 0
+    optimal_points = 0
+    team_increment = 0
+    client_increment = 0
+
+    total_story_points = project.total_story_points\
+        if project.total_story_points not in [None, 0] else project._defined_points
+
+    total_milestones = project.total_milestones\
+        if project.total_milestones not in [None, 0] else len(milestones)
+
+    if total_story_points and total_milestones:
+        optimal_points_per_sprint = total_story_points / total_milestones
+
+    milestones_count = len(milestones)
+    milestones_stats = []
+    for current_milestone_pos in range(0, max(milestones_count, total_milestones)):
+        optimal_points = (total_story_points -
+                            (optimal_points_per_sprint * current_milestone_pos))
+
+        evolution = (total_story_points - current_evolution
+                        if current_evolution is not None else None)
+
+        if current_milestone_pos < milestones_count:
+            current_milestone = list(milestones.values())[current_milestone_pos]
+            milestone_name = current_milestone.name
+            team_increment = current_team_increment
+            client_increment = current_client_increment
+            current_evolution += current_milestone._closed_points
+            current_team_increment += current_milestone._team_increment_points
+            current_client_increment += current_milestone._client_increment_points
+
+        else:
+            milestone_name = _("Future sprint")
+            team_increment = current_team_increment + project._future_team_increment,
+            client_increment = current_client_increment + project._future_client_increment,
+            current_evolution = None
+
+        milestones_stats.append({
+            'name': milestone_name,
+            'optimal': optimal_points,
+            'evolution': evolution,
+            'team-increment': team_increment,
+            'client-increment': client_increment,
+        })
+
+    optimal_points -= optimal_points_per_sprint
+    evolution = (total_story_points - current_evolution
+                    if current_evolution is not None and total_story_points else None)
+
+    milestones_stats.append({
+        'name': _('Project End'),
+        'optimal': optimal_points,
+        'evolution': evolution,
+        'team-increment': team_increment,
+        'client-increment': client_increment,
+    })
+
+    return milestones_stats
+
+
 def get_stats_for_project(project):
-    project = apps.get_model("projects", "Project").objects.\
+    # Let's fetch all the estimations related to a project with all the necesary
-        prefetch_related("milestones",
+    # related data
-                                     "user_stories").\
+    role_points = RolePoints.objects.filter(
-        get(id=project.id)
+        user_story__project = project,
+    ).prefetch_related(
+        "user_story",
+        "user_story__assigned_to",
+        "user_story__milestone",
+        "user_story__status",
+        "role",
+        "points")
 
-    points = project.calculated_points
+    # Data inicialization
-    closed_points = sum(points["closed"].values())
+    project._closed_points = 0
-    closed_points_from_closed_milestones = sum(RolePoints.objects.filter(
+    project._closed_points_per_role = {}
-        Q(user_story__project=project) & Q(user_story__milestone__closed=True)
+    project._closed_points_from_closed_milestones = 0
-    ).exclude(points__value__isnull=True).values_list("points__value", flat=True))
+    project._defined_points = 0
+    project._defined_points_per_role = {}
+    project._assigned_points = 0
+    project._assigned_points_per_role = {}
+    project._future_team_increment = 0
+    project._future_client_increment = 0
 
-    closed_milestones = project.milestones.filter(closed=True).count()
+    # The key will be the milestone id and it will be ordered by estimated_start
+    milestones = collections.OrderedDict()
+    for milestone in project.milestones.order_by("estimated_start"):
+        milestone._closed_points = 0
+        milestone._team_increment_points = 0
+        milestone._client_increment_points = 0
+        milestones[milestone.id] = milestone
+
+    def _find_milestone_for_userstory(user_story):
+        for m in milestones.values():
+            if m.estimated_finish > user_story.created_date.date() and\
+               m.estimated_start <= user_story.created_date.date():
+
+              return m
+
+        return None
+
+    def _update_team_increment(milestone, value):
+        if milestone:
+            milestones[milestone.id]._team_increment_points += value
+        else:
+            project._future_team_increment += value
+
+    def _update_client_increment(milestone, value):
+        if milestone:
+            milestones[milestone.id]._client_increment_points += value
+        else:
+            project._future_client_increment += value
+
+    # Iterate over all the project estimations and update our stats
+    for role_point in role_points:
+        role_id = role_point.role.id
+        points_value = role_point.points.value
+        milestone = role_point.user_story.milestone
+        is_team_requirement = role_point.user_story.team_requirement
+        is_client_requirement = role_point.user_story.client_requirement
+        us_milestone = _find_milestone_for_userstory(role_point.user_story)
+
+        # None estimations doesn't affect to project stats
+        if points_value is None:
+            continue
+
+        # Total defined points
+        project._defined_points += points_value
+
+        # Defined points per role
+        project._defined_points_for_role = project._defined_points_per_role.get(role_id, 0)
+        project._defined_points_for_role += points_value
+        project._defined_points_per_role[role_id] = project._defined_points_for_role
+
+        # Closed points
+        if role_point.user_story.is_closed:
+            project._closed_points += points_value
+            closed_points_for_role = project._closed_points_per_role.get(role_id, 0)
+            closed_points_for_role += points_value
+            project._closed_points_per_role[role_id] = closed_points_for_role
+
+            if milestone is not None:
+                milestones[milestone.id]._closed_points += points_value
+
+        if milestone is not None and milestone.closed:
+            project._closed_points_from_closed_milestones += points_value
+
+        # Assigned to milestone points
+        if role_point.user_story.milestone is not None:
+            project._assigned_points += points_value
+            assigned_points_for_role = project._assigned_points_per_role.get(role_id, 0)
+            assigned_points_for_role += points_value
+            project._assigned_points_per_role[role_id] = assigned_points_for_role
+
+        # Extra requirements
+        if is_team_requirement and is_client_requirement:
+            _update_team_increment(us_milestone, points_value/2)
+            _update_client_increment(us_milestone, points_value/2)
+
+        if is_team_requirement and not is_client_requirement:
+            _update_team_increment(us_milestone, points_value)
+
+        if not is_team_requirement and is_client_requirement:
+            _update_client_increment(us_milestone, points_value)
+
+    # Speed calculations
     speed = 0
-
+    closed_milestones = len([m for m in milestones.values() if m.closed])
     if closed_milestones != 0:
-        speed = closed_points_from_closed_milestones / closed_milestones
+        speed = project._closed_points_from_closed_milestones / closed_milestones
+
+    milestones_stats = _get_milestones_stats_for_backlog(project, milestones)
 
     project_stats = {
         'name': project.name,
         'total_milestones': project.total_milestones,
         'total_points': project.total_story_points,
-        'closed_points': closed_points,
+        'closed_points': project._closed_points,
-        'closed_points_per_role': points["closed"],
+        'closed_points_per_role': project._closed_points_per_role,
-        'defined_points': sum(points["defined"].values()),
+        'defined_points': project._defined_points,
-        'defined_points_per_role': points["defined"],
+        'defined_points_per_role': project._defined_points_per_role,
-        'assigned_points': sum(points["assigned"].values()),
+        'assigned_points': project._assigned_points,
-        'assigned_points_per_role': points["assigned"],
+        'assigned_points_per_role': project._assigned_points_per_role,
-        'milestones': _get_milestones_stats_for_backlog(project),
+        'milestones': milestones_stats,
         'speed': speed,
     }
     return project_stats

tests/integration/test_stats.py

@@ -86,36 +86,46 @@ def data():
 
 
 def test_project_defined_points(client, data):
-    assert data.project.defined_points == {data.role1.pk: 63}
+    project_stats = get_stats_for_project(data.project)
+    assert project_stats["defined_points_per_role"] == {data.role1.pk: 63}
     data.role_points1.role = data.role2
     data.role_points1.save()
-    assert data.project.defined_points == {data.role1.pk: 62, data.role2.pk: 1}
+    project_stats = get_stats_for_project(data.project)
+    assert project_stats["defined_points_per_role"] == {data.role1.pk: 62, data.role2.pk: 1}
 
 
 def test_project_closed_points(client, data):
-    assert data.project.closed_points == {}
+    project_stats = get_stats_for_project(data.project)
+    assert project_stats["closed_points_per_role"] == {}
     data.role_points1.role = data.role2
     data.role_points1.save()
-    assert data.project.closed_points == {}
+    project_stats = get_stats_for_project(data.project)
+    assert project_stats["closed_points_per_role"] == {}
     data.user_story1.is_closed = True
     data.user_story1.save()
-    assert data.project.closed_points == {data.role2.pk: 1}
+    project_stats = get_stats_for_project(data.project)
+    assert project_stats["closed_points_per_role"] == {data.role2.pk: 1}
     data.user_story2.is_closed = True
     data.user_story2.save()
-    assert data.project.closed_points == {data.role1.pk: 2, data.role2.pk: 1}
+    project_stats = get_stats_for_project(data.project)
+    assert project_stats["closed_points_per_role"] == {data.role1.pk: 2, data.role2.pk: 1}
     data.user_story3.is_closed = True
     data.user_story3.save()
-    assert data.project.closed_points == {data.role1.pk: 6, data.role2.pk: 1}
+    project_stats = get_stats_for_project(data.project)
+    assert project_stats["closed_points_per_role"] == {data.role1.pk: 6, data.role2.pk: 1}
     data.user_story4.is_closed = True
     data.user_story4.save()
-    assert data.project.closed_points == {data.role1.pk: 14, data.role2.pk: 1}
+    project_stats = get_stats_for_project(data.project)
+    assert project_stats["closed_points_per_role"] == {data.role1.pk: 14, data.role2.pk: 1}
 
     data.user_story5.is_closed = True
     data.user_story5.save()
-    assert data.project.closed_points == {data.role1.pk: 30, data.role2.pk: 1}
+    project_stats = get_stats_for_project(data.project)
+    assert project_stats["closed_points_per_role"] == {data.role1.pk: 30, data.role2.pk: 1}
     data.user_story6.is_closed = True
     data.user_story6.save()
-    assert data.project.closed_points == {data.role1.pk: 62, data.role2.pk: 1}
+    project_stats = get_stats_for_project(data.project)
+    assert project_stats["closed_points_per_role"] == {data.role1.pk: 62, data.role2.pk: 1}
 
     project_stats = get_stats_for_project(data.project)
     assert project_stats["closed_points"] == 63
@@ -123,19 +133,25 @@ def test_project_closed_points(client, data):
 
 
 def test_project_assigned_points(client, data):
-    assert data.project.assigned_points == {data.role1.pk: 48}
+    project_stats = get_stats_for_project(data.project)
+    assert project_stats["assigned_points_per_role"] == {data.role1.pk: 48}
     data.role_points1.role = data.role2
     data.role_points1.save()
-    assert data.project.assigned_points == {data.role1.pk: 48}
+    project_stats = get_stats_for_project(data.project)
+    assert project_stats["assigned_points_per_role"] == {data.role1.pk: 48}
     data.user_story1.milestone = data.milestone
     data.user_story1.save()
-    assert data.project.assigned_points == {data.role1.pk: 48, data.role2.pk: 1}
+    project_stats = get_stats_for_project(data.project)
+    assert project_stats["assigned_points_per_role"] == {data.role1.pk: 48, data.role2.pk: 1}
     data.user_story2.milestone = data.milestone
     data.user_story2.save()
-    assert data.project.assigned_points == {data.role1.pk: 50, data.role2.pk: 1}
+    project_stats = get_stats_for_project(data.project)
+    assert project_stats["assigned_points_per_role"] == {data.role1.pk: 50, data.role2.pk: 1}
     data.user_story3.milestone = data.milestone
     data.user_story3.save()
-    assert data.project.assigned_points == {data.role1.pk: 54, data.role2.pk: 1}
+    project_stats = get_stats_for_project(data.project)
+    assert project_stats["assigned_points_per_role"] == {data.role1.pk: 54, data.role2.pk: 1}
     data.user_story4.milestone = data.milestone
     data.user_story4.save()
-    assert data.project.assigned_points == {data.role1.pk: 62, data.role2.pk: 1}
+    project_stats = get_stats_for_project(data.project)
+    assert project_stats["assigned_points_per_role"] == {data.role1.pk: 62, data.role2.pk: 1}