From 172ce73007d4b7fee01f49e0f7e7d52dac204b89 Mon Sep 17 00:00:00 2001 From: Alejandro Alonso Date: Fri, 5 Feb 2016 08:11:22 +0100 Subject: [PATCH] Improving CSV reports performance --- taiga/projects/issues/services.py | 24 +++++++++++--- taiga/projects/tasks/services.py | 24 +++++++++++--- taiga/projects/userstories/services.py | 44 +++++++++++++++++++------- 3 files changed, 70 insertions(+), 22 deletions(-) diff --git a/taiga/projects/issues/services.py b/taiga/projects/issues/services.py index 29452c8b..6ae35b2a 100644 --- a/taiga/projects/issues/services.py +++ b/taiga/projects/issues/services.py @@ -28,7 +28,9 @@ from taiga.base.utils import db, text from taiga.projects.issues.apps import ( connect_issues_signals, disconnect_issues_signals) -from taiga.projects.votes import services as votes_services +from taiga.projects.votes.utils import attach_total_voters_to_queryset +from taiga.projects.notifications.utils import attach_watchers_to_queryset + from . import models @@ -88,9 +90,21 @@ def issues_to_csv(project, queryset): "attachments", "external_reference", "tags", "watchers", "voters", "created_date", "modified_date", "finished_date"] - for custom_attr in project.issuecustomattributes.all(): + + custom_attrs = project.issuecustomattributes.all() + for custom_attr in custom_attrs: fieldnames.append(custom_attr.name) + queryset = queryset.prefetch_related("attachments", + "generated_user_stories", + "custom_attributes_values") + queryset = queryset.select_related("owner", + "assigned_to", + "status", + "project") + queryset = attach_total_voters_to_queryset(queryset) + queryset = attach_watchers_to_queryset(queryset) + writer = csv.DictWriter(csv_data, fieldnames=fieldnames) writer.writeheader() for issue in queryset: @@ -111,14 +125,14 @@ def issues_to_csv(project, queryset): "attachments": issue.attachments.count(), "external_reference": issue.external_reference, "tags": ",".join(issue.tags or []), - "watchers": [u.id for u in issue.get_watchers()], - "voters": votes_services.get_voters(issue).count(), + "watchers": issue.watchers, + "voters": issue.total_voters, "created_date": issue.created_date, "modified_date": issue.modified_date, "finished_date": issue.finished_date, } - for custom_attr in project.issuecustomattributes.all(): + for custom_attr in custom_attrs: value = issue.custom_attributes_values.attributes_values.get(str(custom_attr.id), None) issue_data[custom_attr.name] = value diff --git a/taiga/projects/tasks/services.py b/taiga/projects/tasks/services.py index d40394a2..7fad684d 100644 --- a/taiga/projects/tasks/services.py +++ b/taiga/projects/tasks/services.py @@ -24,7 +24,8 @@ from taiga.projects.tasks.apps import ( connect_tasks_signals, disconnect_tasks_signals) from taiga.events import events -from taiga.projects.votes import services as votes_services +from taiga.projects.votes.utils import attach_total_voters_to_queryset +from taiga.projects.notifications.utils import attach_watchers_to_queryset from . import models @@ -99,9 +100,22 @@ def tasks_to_csv(project, queryset): "status", "is_iocaine", "is_closed", "us_order", "taskboard_order", "attachments", "external_reference", "tags", "watchers", "voters"] - for custom_attr in project.taskcustomattributes.all(): + + custom_attrs = project.taskcustomattributes.all() + for custom_attr in custom_attrs: fieldnames.append(custom_attr.name) + queryset = queryset.prefetch_related("attachments", + "custom_attributes_values") + queryset = queryset.select_related("milestone", + "owner", + "assigned_to", + "status", + "project") + + queryset = attach_total_voters_to_queryset(queryset) + queryset = attach_watchers_to_queryset(queryset) + writer = csv.DictWriter(csv_data, fieldnames=fieldnames) writer.writeheader() for task in queryset: @@ -123,10 +137,10 @@ def tasks_to_csv(project, queryset): "attachments": task.attachments.count(), "external_reference": task.external_reference, "tags": ",".join(task.tags or []), - "watchers": [u.id for u in task.get_watchers()], - "voters": votes_services.get_voters(task).count(), + "watchers": task.watchers, + "voters": task.total_voters, } - for custom_attr in project.taskcustomattributes.all(): + for custom_attr in custom_attrs: value = task.custom_attributes_values.attributes_values.get(str(custom_attr.id), None) task_data[custom_attr.name] = value diff --git a/taiga/projects/userstories/services.py b/taiga/projects/userstories/services.py index edf98b64..f7685481 100644 --- a/taiga/projects/userstories/services.py +++ b/taiga/projects/userstories/services.py @@ -32,7 +32,8 @@ from taiga.projects.userstories.apps import ( disconnect_userstories_signals) from taiga.events import events -from taiga.projects.votes import services as votes_services +from taiga.projects.votes.utils import attach_total_voters_to_queryset +from taiga.projects.notifications.utils import attach_watchers_to_queryset from . import models @@ -132,8 +133,11 @@ def userstories_to_csv(project,queryset): fieldnames = ["ref", "subject", "description", "milestone", "owner", "owner_full_name", "assigned_to", "assigned_to_full_name", "status", "is_closed"] - for role in project.roles.filter(computable=True).order_by('name'): + + roles = project.roles.filter(computable=True).order_by('slug') + for role in roles: fieldnames.append("{}-points".format(role.slug)) + fieldnames.append("total-points") fieldnames += ["backlog_order", "sprint_order", "kanban_order", @@ -143,9 +147,26 @@ def userstories_to_csv(project,queryset): "tags", "watchers", "voters"] - for custom_attr in project.userstorycustomattributes.all(): + custom_attrs = project.userstorycustomattributes.all() + for custom_attr in custom_attrs: fieldnames.append(custom_attr.name) + queryset = queryset.prefetch_related("role_points", + "role_points__points", + "role_points__role", + "tasks", + "attachments", + "custom_attributes_values") + queryset = queryset.select_related("milestone", + "project", + "status", + "owner", + "assigned_to", + "generated_from_issue") + + queryset = attach_total_voters_to_queryset(queryset) + queryset = attach_watchers_to_queryset(queryset) + writer = csv.DictWriter(csv_data, fieldnames=fieldnames) writer.writeheader() for us in queryset: @@ -173,18 +194,17 @@ def userstories_to_csv(project,queryset): "external_reference": us.external_reference, "tasks": ",".join([str(task.ref) for task in us.tasks.all()]), "tags": ",".join(us.tags or []), - "watchers": [u.id for u in us.get_watchers()], - "voters": votes_services.get_voters(us).count(), + "watchers": us.watchers, + "voters": us.total_voters, } - for role in us.project.roles.filter(computable=True).order_by('name'): - if us.role_points.filter(role_id=role.id).count() == 1: - row["{}-points".format(role.slug)] = us.role_points.get(role_id=role.id).points.value - else: - row["{}-points".format(role.slug)] = 0 + us_role_points_by_role_id = {us_rp.role.id: us_rp.points.value for us_rp in us.role_points.all()} + for role in roles: + row["{}-points".format(role.slug)] = us_role_points_by_role_id.get(role.id, 0) + row['total-points'] = us.get_total_points() - for custom_attr in project.userstorycustomattributes.all(): + for custom_attr in custom_attrs: value = us.custom_attributes_values.attributes_values.get(str(custom_attr.id), None) row[custom_attr.name] = value @@ -274,7 +294,7 @@ def _get_userstories_assigned_to(project, queryset): "full_name": full_name or "", "count": count, }) - + if id is None: none_valued_added = True