From 4d3e65c60328b3b239b4d121e33a1c0917461ae1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jes=C3=BAs=20Espino?= Date: Thu, 24 Nov 2016 11:35:28 +0100 Subject: [PATCH] Move the timeline rebuild to the timeline library --- .../commands/change_project_slug.py | 4 +- .../_rebuild_timeline_for_user_creation.py | 97 ------------- .../management/commands/rebuild_timeline.py | 112 +------------- taiga/timeline/rebuilder.py | 137 ++++++++++++++++++ taiga/timeline/service.py | 5 +- taiga/timeline/signals.py | 12 +- 6 files changed, 152 insertions(+), 215 deletions(-) delete mode 100644 taiga/timeline/management/commands/_rebuild_timeline_for_user_creation.py create mode 100644 taiga/timeline/rebuilder.py diff --git a/taiga/projects/management/commands/change_project_slug.py b/taiga/projects/management/commands/change_project_slug.py index 8e8d012e..e27dc77f 100644 --- a/taiga/projects/management/commands/change_project_slug.py +++ b/taiga/projects/management/commands/change_project_slug.py @@ -23,7 +23,7 @@ from django.test.utils import override_settings from taiga.base.utils.slug import slugify_uniquely from taiga.projects.models import Project from taiga.projects.history.models import HistoryEntry -from taiga.timeline.management.commands.rebuild_timeline import generate_timeline +from taiga.timeline.rebuilder import rebuild_timeline class Command(BaseCommand): @@ -58,4 +58,4 @@ class Command(BaseCommand): # Regenerate timeline self.stdout.write(self.style.SUCCESS("-> Regenerate timeline entries.")) - generate_timeline(None, None, project.id) + rebuild_timeline(None, None, project.id) diff --git a/taiga/timeline/management/commands/_rebuild_timeline_for_user_creation.py b/taiga/timeline/management/commands/_rebuild_timeline_for_user_creation.py deleted file mode 100644 index f3a5fa57..00000000 --- a/taiga/timeline/management/commands/_rebuild_timeline_for_user_creation.py +++ /dev/null @@ -1,97 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright (C) 2014-2016 Andrey Antukh -# Copyright (C) 2014-2016 Jesús Espino -# Copyright (C) 2014-2016 David Barragán -# Copyright (C) 2014-2016 Alejandro Alonso -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . - -# Examples: -# python manage.py rebuild_timeline_for_user_creation --settings=settings.local_timeline - -from django.contrib.auth import get_user_model -from django.contrib.contenttypes.models import ContentType -from django.core.management.base import BaseCommand -from django.db.models import Model -from django.test.utils import override_settings - -from taiga.timeline.service import _get_impl_key_from_model, -from taiga.timeline.service import _timeline_impl_map, -from taiga.timeline.service import extract_user_info) -from taiga.timeline.models import Timeline -from taiga.timeline.signals import _push_to_timelines - -from unittest.mock import patch - -import gc - - -class BulkCreator(object): - def __init__(self): - self.timeline_objects = [] - self.created = None - - def create_element(self, element): - self.timeline_objects.append(element) - if len(self.timeline_objects) > 1000: - self.flush() - - def flush(self): - Timeline.objects.bulk_create(self.timeline_objects, batch_size=1000) - del self.timeline_objects - self.timeline_objects = [] - gc.collect() - -bulk_creator = BulkCreator() - - -def custom_add_to_object_timeline(obj:object, instance:object, event_type:str, created_datetime:object, - namespace:str="default", extra_data:dict={}): - assert isinstance(obj, Model), "obj must be a instance of Model" - assert isinstance(instance, Model), "instance must be a instance of Model" - event_type_key = _get_impl_key_from_model(instance.__class__, event_type) - impl = _timeline_impl_map.get(event_type_key, None) - - bulk_creator.create_element(Timeline( - content_object=obj, - namespace=namespace, - event_type=event_type_key, - project=None, - data=impl(instance, extra_data=extra_data), - data_content_type = ContentType.objects.get_for_model(instance.__class__), - created=created_datetime, - )) - - -def generate_timeline(): - with patch('taiga.timeline.service._add_to_object_timeline', new=custom_add_to_object_timeline): - # Users api wasn't a HistoryResourceMixin so we can't interate on the HistoryEntries in this case - users = get_user_model().objects.order_by("date_joined") - for user in users.iterator(): - print("User:", user.date_joined) - extra_data = { - "values_diff": {}, - "user": extract_user_info(user), - } - _push_to_timelines(None, user, user, "create", user.date_joined, extra_data=extra_data) - del extra_data - - bulk_creator.flush() - - -class Command(BaseCommand): - help = 'Regenerate project timeline' - - @override_settings(DEBUG=False) - def handle(self, *args, **options): - generate_timeline() diff --git a/taiga/timeline/management/commands/rebuild_timeline.py b/taiga/timeline/management/commands/rebuild_timeline.py index cd14795a..fdd2dbd8 100644 --- a/taiga/timeline/management/commands/rebuild_timeline.py +++ b/taiga/timeline/management/commands/rebuild_timeline.py @@ -21,122 +21,14 @@ # python manage.py rebuild_timeline --settings=settings.local_timeline --purge # python manage.py rebuild_timeline --settings=settings.local_timeline --initial_date 2014-10-02 -from django.contrib.contenttypes.models import ContentType -from django.core.exceptions import ObjectDoesNotExist from django.core.management.base import BaseCommand -from django.db.models import Model from django.test.utils import override_settings -from taiga.projects.models import Project -from taiga.projects.history.models import HistoryEntry from taiga.timeline.models import Timeline -from taiga.timeline.service import _get_impl_key_from_model,_timeline_impl_map, extract_user_info -from taiga.timeline.signals import on_new_history_entry, _push_to_timelines +from taiga.timeline.rebuilder import rebuild_timeline -from unittest.mock import patch from optparse import make_option -import gc - - -class BulkCreator(object): - def __init__(self): - self.timeline_objects = [] - self.created = None - - def create_element(self, element): - self.timeline_objects.append(element) - if len(self.timeline_objects) > 999: - self.flush() - - def flush(self): - Timeline.objects.bulk_create(self.timeline_objects, batch_size=1000) - del self.timeline_objects - self.timeline_objects = [] - gc.collect() - -bulk_creator = BulkCreator() - - -def custom_add_to_object_timeline(obj:object, instance:object, event_type:str, created_datetime:object, - namespace:str="default", extra_data:dict={}): - assert isinstance(obj, Model), "obj must be a instance of Model" - assert isinstance(instance, Model), "instance must be a instance of Model" - event_type_key = _get_impl_key_from_model(instance.__class__, event_type) - impl = _timeline_impl_map.get(event_type_key, None) - - bulk_creator.create_element(Timeline( - content_object=obj, - namespace=namespace, - event_type=event_type_key, - project=instance.project, - data=impl(instance, extra_data=extra_data), - data_content_type=ContentType.objects.get_for_model(instance.__class__), - created=created_datetime, - )) - - -def generate_timeline(initial_date, final_date, project_id): - if initial_date or final_date or project_id: - timelines = Timeline.objects.all() - if initial_date: - timelines = timelines.filter(created__gte=initial_date) - if final_date: - timelines = timelines.filter(created__lt=final_date) - if project_id: - timelines = timelines.filter(project__id=project_id) - - timelines.delete() - - with patch('taiga.timeline.service._add_to_object_timeline', new=custom_add_to_object_timeline): - # Projects api wasn't a HistoryResourceMixin so we can't interate on the HistoryEntries in this case - projects = Project.objects.order_by("created_date") - history_entries = HistoryEntry.objects.order_by("created_at") - - if initial_date: - projects = projects.filter(created_date__gte=initial_date) - history_entries = history_entries.filter(created_at__gte=initial_date) - - if final_date: - projects = projects.filter(created_date__lt=final_date) - history_entries = history_entries.filter(created_at__lt=final_date) - - if project_id: - project = Project.objects.get(id=project_id) - epic_keys = ['epics.epic:%s'%(id) for id in project.epics.values_list("id", flat=True)] - us_keys = ['userstories.userstory:%s'%(id) for id in project.user_stories.values_list("id", - flat=True)] - tasks_keys = ['tasks.task:%s'%(id) for id in project.tasks.values_list("id", flat=True)] - issue_keys = ['issues.issue:%s'%(id) for id in project.issues.values_list("id", flat=True)] - wiki_keys = ['wiki.wikipage:%s'%(id) for id in project.wiki_pages.values_list("id", flat=True)] - keys = epic_keys + us_keys + tasks_keys + issue_keys + wiki_keys - - projects = projects.filter(id=project_id) - history_entries = history_entries.filter(key__in=keys) - - #Memberships - for membership in project.memberships.exclude(user=None).exclude(user=project.owner): - _push_to_timelines(project, membership.user, membership, "create", membership.created_at) - - for project in projects.iterator(): - print("Project:", project) - extra_data = { - "values_diff": {}, - "user": extract_user_info(project.owner), - } - _push_to_timelines(project, project.owner, project, "create", project.created_date, - extra_data=extra_data) - del extra_data - - for historyEntry in history_entries.iterator(): - print("History entry:", historyEntry.created_at) - try: - on_new_history_entry(None, historyEntry, None) - except ObjectDoesNotExist as e: - print("Ignoring") - - bulk_creator.flush() - class Command(BaseCommand): help = 'Regenerate project timeline' @@ -168,4 +60,4 @@ class Command(BaseCommand): if options["purge"] == True: Timeline.objects.all().delete() - generate_timeline(options["initial_date"], options["final_date"], options["project"]) + rebuild_timeline(options["initial_date"], options["final_date"], options["project"]) diff --git a/taiga/timeline/rebuilder.py b/taiga/timeline/rebuilder.py new file mode 100644 index 00000000..e8100fed --- /dev/null +++ b/taiga/timeline/rebuilder.py @@ -0,0 +1,137 @@ +# -*- coding: utf-8 -*- +# Copyright (C) 2014-2016 Andrey Antukh +# Copyright (C) 2014-2016 Jesús Espino +# Copyright (C) 2014-2016 David Barragán +# Copyright (C) 2014-2016 Alejandro Alonso +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +from django.contrib.contenttypes.models import ContentType +from django.core.exceptions import ObjectDoesNotExist +from django.db.models import Model +from django.test.utils import override_settings + +from taiga.projects.models import Project +from taiga.projects.history.models import HistoryEntry +from .models import Timeline +from .service import _get_impl_key_from_model, _timeline_impl_map, extract_user_info +from .signals import on_new_history_entry, _push_to_timelines + +from unittest.mock import patch + +import gc + + +class BulkCreator(object): + def __init__(self): + self.timeline_objects = [] + self.created = None + + def create_element(self, element): + self.timeline_objects.append(element) + if len(self.timeline_objects) > 999: + self.flush() + + def flush(self): + Timeline.objects.bulk_create(self.timeline_objects, batch_size=1000) + del self.timeline_objects + self.timeline_objects = [] + gc.collect() + +bulk_creator = BulkCreator() + + +def custom_add_to_object_timeline(obj:object, instance:object, event_type:str, created_datetime:object, + namespace:str="default", extra_data:dict={}): + assert isinstance(obj, Model), "obj must be a instance of Model" + assert isinstance(instance, Model), "instance must be a instance of Model" + event_type_key = _get_impl_key_from_model(instance.__class__, event_type) + impl = _timeline_impl_map.get(event_type_key, None) + + bulk_creator.create_element(Timeline( + content_object=obj, + namespace=namespace, + event_type=event_type_key, + project=instance.project, + data=impl(instance, extra_data=extra_data), + data_content_type=ContentType.objects.get_for_model(instance.__class__), + created=created_datetime, + )) + + +@override_settings(CELERY_ENABLED=False) +def rebuild_timeline(initial_date, final_date, project_id): + if initial_date or final_date or project_id: + timelines = Timeline.objects.all() + if initial_date: + timelines = timelines.filter(created__gte=initial_date) + if final_date: + timelines = timelines.filter(created__lt=final_date) + if project_id: + timelines = timelines.filter(project__id=project_id) + + timelines.delete() + + with patch('taiga.timeline.service._add_to_object_timeline', new=custom_add_to_object_timeline): + # Projects api wasn't a HistoryResourceMixin so we can't interate on the HistoryEntries in this case + projects = Project.objects.order_by("created_date") + history_entries = HistoryEntry.objects.order_by("created_at") + + if initial_date: + projects = projects.filter(created_date__gte=initial_date) + history_entries = history_entries.filter(created_at__gte=initial_date) + + if final_date: + projects = projects.filter(created_date__lt=final_date) + history_entries = history_entries.filter(created_at__lt=final_date) + + if project_id: + project = Project.objects.get(id=project_id) + epic_keys = ['epics.epic:%s'%(id) for id in project.epics.values_list("id", flat=True)] + us_keys = ['userstories.userstory:%s'%(id) for id in project.user_stories.values_list("id", + flat=True)] + tasks_keys = ['tasks.task:%s'%(id) for id in project.tasks.values_list("id", flat=True)] + issue_keys = ['issues.issue:%s'%(id) for id in project.issues.values_list("id", flat=True)] + wiki_keys = ['wiki.wikipage:%s'%(id) for id in project.wiki_pages.values_list("id", flat=True)] + keys = epic_keys + us_keys + tasks_keys + issue_keys + wiki_keys + + projects = projects.filter(id=project_id) + history_entries = history_entries.filter(key__in=keys) + + #Memberships + for membership in project.memberships.exclude(user=None).exclude(user=project.owner): + _push_to_timelines(project, membership.user, membership, "create", membership.created_at, refresh_totals=False) + + for project in projects.iterator(): + print("Project:", project) + extra_data = { + "values_diff": {}, + "user": extract_user_info(project.owner), + } + _push_to_timelines(project, project.owner, project, 'create', + project.created_date, extra_data=extra_data, + refresh_totals=False) + del extra_data + + for historyEntry in history_entries.iterator(): + print("History entry:", historyEntry.created_at) + try: + historyEntry.refresh_totals = False + on_new_history_entry(None, historyEntry, None) + except ObjectDoesNotExist as e: + print("Ignoring") + + for project in projects.iterator(): + project.refresh_totals() + + bulk_creator.flush() diff --git a/taiga/timeline/service.py b/taiga/timeline/service.py index 03ca3e96..07d742bf 100644 --- a/taiga/timeline/service.py +++ b/taiga/timeline/service.py @@ -93,7 +93,7 @@ def _push_to_timeline(objects, instance: object, event_type: str, created_dateti @app.task def push_to_timelines(project_id, user_id, obj_app_label, obj_model_name, obj_id, event_type, - created_datetime, extra_data={}): + created_datetime, extra_data={}, refresh_totals=True): ObjModel = apps.get_model(obj_app_label, obj_model_name) try: @@ -120,7 +120,8 @@ def push_to_timelines(project_id, user_id, obj_app_label, obj_model_name, obj_id namespace=build_project_namespace(project), extra_data=extra_data) - project.refresh_totals() + if refresh_totals: + project.refresh_totals() if hasattr(obj, "get_related_people"): related_people = obj.get_related_people() diff --git a/taiga/timeline/signals.py b/taiga/timeline/signals.py index 7f754b63..c5590c81 100644 --- a/taiga/timeline/signals.py +++ b/taiga/timeline/signals.py @@ -31,7 +31,7 @@ from taiga.timeline.service import (push_to_timelines, extract_user_info) -def _push_to_timelines(project, user, obj, event_type, created_datetime, extra_data={}): +def _push_to_timelines(project, user, obj, event_type, created_datetime, extra_data={}, refresh_totals=True): project_id = None if project is None else project.id ct = ContentType.objects.get_for_model(obj) @@ -43,7 +43,8 @@ def _push_to_timelines(project, user, obj, event_type, created_datetime, extra_d obj.id, event_type, created_datetime, - extra_data=extra_data)) + extra_data=extra_data, + refresh_totals=refresh_totals)) else: push_to_timelines(project_id, user.id, @@ -52,7 +53,8 @@ def _push_to_timelines(project, user, obj, event_type, created_datetime, extra_d obj.id, event_type, created_datetime, - extra_data=extra_data) + extra_data=extra_data, + refresh_totals=refresh_totals) def _clean_description_fields(values_diff): @@ -73,6 +75,8 @@ def on_new_history_entry(sender, instance, created, **kwargs): if instance.user["pk"] is None: return None + refresh_totals = getattr(instance, "refresh_totals", True) + model = history_services.get_model_from_key(instance.key) pk = history_services.get_pk_from_key(instance.key) obj = model.objects.get(pk=pk) @@ -105,7 +109,7 @@ def on_new_history_entry(sender, instance, created, **kwargs): extra_data["comment_edited"] = True created_datetime = instance.created_at - _push_to_timelines(project, user, obj, event_type, created_datetime, extra_data=extra_data) + _push_to_timelines(project, user, obj, event_type, created_datetime, extra_data=extra_data, refresh_totals=refresh_totals) def create_membership_push_to_timeline(sender, instance, created, **kwargs):