Merge pull request #887 from taigaio/timeline-rebuilder-as-a-service

Move the timeline rebuild to the timeline library

taiga/projects/management/commands/change_project_slug.py

@@ -23,7 +23,7 @@ from django.test.utils import override_settings
 from taiga.base.utils.slug import slugify_uniquely
 from taiga.projects.models import Project
 from taiga.projects.history.models import HistoryEntry
-from taiga.timeline.management.commands.rebuild_timeline import generate_timeline
+from taiga.timeline.rebuilder import rebuild_timeline
 
 
 class Command(BaseCommand):
@@ -58,4 +58,4 @@ class Command(BaseCommand):
 
         # Regenerate timeline
         self.stdout.write(self.style.SUCCESS("-> Regenerate timeline entries."))
-        generate_timeline(None, None, project.id)
+        rebuild_timeline(None, None, project.id)

taiga/timeline/management/commands/_rebuild_timeline_for_user_creation.py

@@ -1,97 +0,0 @@
-# -*- coding: utf-8 -*-
-# Copyright (C) 2014-2016 Andrey Antukh <niwi@niwi.nz>
-# Copyright (C) 2014-2016 Jesús Espino <jespinog@gmail.com>
-# Copyright (C) 2014-2016 David Barragán <bameda@dbarragan.com>
-# Copyright (C) 2014-2016 Alejandro Alonso <alejandro.alonso@kaleidos.net>
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-# Examples:
-# python manage.py rebuild_timeline_for_user_creation --settings=settings.local_timeline
-
-from django.contrib.auth import get_user_model
-from django.contrib.contenttypes.models import ContentType
-from django.core.management.base import BaseCommand
-from django.db.models import Model
-from django.test.utils import override_settings
-
-from taiga.timeline.service import _get_impl_key_from_model,
-from taiga.timeline.service import _timeline_impl_map,
-from taiga.timeline.service import extract_user_info)
-from taiga.timeline.models import Timeline
-from taiga.timeline.signals import _push_to_timelines
-
-from unittest.mock import patch
-
-import gc
-
-
-class BulkCreator(object):
-    def __init__(self):
-        self.timeline_objects = []
-        self.created = None
-
-    def create_element(self, element):
-        self.timeline_objects.append(element)
-        if len(self.timeline_objects) > 1000:
-            self.flush()
-
-    def flush(self):
-        Timeline.objects.bulk_create(self.timeline_objects, batch_size=1000)
-        del self.timeline_objects
-        self.timeline_objects = []
-        gc.collect()
-
-bulk_creator = BulkCreator()
-
-
-def custom_add_to_object_timeline(obj:object, instance:object, event_type:str, created_datetime:object,
-                                  namespace:str="default", extra_data:dict={}):
-    assert isinstance(obj, Model), "obj must be a instance of Model"
-    assert isinstance(instance, Model), "instance must be a instance of Model"
-    event_type_key = _get_impl_key_from_model(instance.__class__, event_type)
-    impl = _timeline_impl_map.get(event_type_key, None)
-
-    bulk_creator.create_element(Timeline(
-        content_object=obj,
-        namespace=namespace,
-        event_type=event_type_key,
-        project=None,
-        data=impl(instance, extra_data=extra_data),
-        data_content_type = ContentType.objects.get_for_model(instance.__class__),
-        created=created_datetime,
-    ))
-
-
-def generate_timeline():
-    with patch('taiga.timeline.service._add_to_object_timeline', new=custom_add_to_object_timeline):
-        # Users api wasn't a HistoryResourceMixin so we can't interate on the HistoryEntries in this case
-        users = get_user_model().objects.order_by("date_joined")
-        for user in users.iterator():
-            print("User:", user.date_joined)
-            extra_data = {
-                "values_diff": {},
-                "user": extract_user_info(user),
-            }
-            _push_to_timelines(None, user, user, "create", user.date_joined, extra_data=extra_data)
-            del extra_data
-
-    bulk_creator.flush()
-
-
-class Command(BaseCommand):
-    help = 'Regenerate project timeline'
-
-    @override_settings(DEBUG=False)
-    def handle(self, *args, **options):
-        generate_timeline()

taiga/timeline/management/commands/rebuild_timeline.py

@@ -21,122 +21,14 @@
 # python manage.py rebuild_timeline --settings=settings.local_timeline --purge
 # python manage.py rebuild_timeline --settings=settings.local_timeline --initial_date 2014-10-02
 
-from django.contrib.contenttypes.models import ContentType
-from django.core.exceptions import ObjectDoesNotExist
 from django.core.management.base import BaseCommand
-from django.db.models import Model
 from django.test.utils import override_settings
 
-from taiga.projects.models import Project
-from taiga.projects.history.models import HistoryEntry
 from taiga.timeline.models import Timeline
-from taiga.timeline.service import _get_impl_key_from_model,_timeline_impl_map, extract_user_info
+from taiga.timeline.rebuilder import rebuild_timeline
-from taiga.timeline.signals import on_new_history_entry, _push_to_timelines
 
-from unittest.mock import patch
 from optparse import make_option
 
-import gc
-
-
-class BulkCreator(object):
-    def __init__(self):
-        self.timeline_objects = []
-        self.created = None
-
-    def create_element(self, element):
-        self.timeline_objects.append(element)
-        if len(self.timeline_objects) > 999:
-            self.flush()
-
-    def flush(self):
-        Timeline.objects.bulk_create(self.timeline_objects, batch_size=1000)
-        del self.timeline_objects
-        self.timeline_objects = []
-        gc.collect()
-
-bulk_creator = BulkCreator()
-
-
-def custom_add_to_object_timeline(obj:object, instance:object, event_type:str, created_datetime:object,
-                                  namespace:str="default", extra_data:dict={}):
-    assert isinstance(obj, Model), "obj must be a instance of Model"
-    assert isinstance(instance, Model), "instance must be a instance of Model"
-    event_type_key = _get_impl_key_from_model(instance.__class__, event_type)
-    impl = _timeline_impl_map.get(event_type_key, None)
-
-    bulk_creator.create_element(Timeline(
-        content_object=obj,
-        namespace=namespace,
-        event_type=event_type_key,
-        project=instance.project,
-        data=impl(instance, extra_data=extra_data),
-        data_content_type=ContentType.objects.get_for_model(instance.__class__),
-        created=created_datetime,
-    ))
-
-
-def generate_timeline(initial_date, final_date, project_id):
-    if initial_date or final_date or project_id:
-        timelines = Timeline.objects.all()
-        if initial_date:
-            timelines = timelines.filter(created__gte=initial_date)
-        if final_date:
-            timelines = timelines.filter(created__lt=final_date)
-        if project_id:
-            timelines = timelines.filter(project__id=project_id)
-
-        timelines.delete()
-
-    with patch('taiga.timeline.service._add_to_object_timeline', new=custom_add_to_object_timeline):
-        # Projects api wasn't a HistoryResourceMixin so we can't interate on the HistoryEntries in this case
-        projects = Project.objects.order_by("created_date")
-        history_entries = HistoryEntry.objects.order_by("created_at")
-
-        if initial_date:
-            projects = projects.filter(created_date__gte=initial_date)
-            history_entries = history_entries.filter(created_at__gte=initial_date)
-
-        if final_date:
-            projects = projects.filter(created_date__lt=final_date)
-            history_entries = history_entries.filter(created_at__lt=final_date)
-
-        if project_id:
-            project = Project.objects.get(id=project_id)
-            epic_keys = ['epics.epic:%s'%(id) for id in project.epics.values_list("id", flat=True)]
-            us_keys = ['userstories.userstory:%s'%(id) for id in project.user_stories.values_list("id",
-                                                                                                  flat=True)]
-            tasks_keys = ['tasks.task:%s'%(id) for id in project.tasks.values_list("id", flat=True)]
-            issue_keys = ['issues.issue:%s'%(id) for id in project.issues.values_list("id", flat=True)]
-            wiki_keys = ['wiki.wikipage:%s'%(id) for id in project.wiki_pages.values_list("id", flat=True)]
-            keys = epic_keys + us_keys + tasks_keys + issue_keys + wiki_keys
-
-            projects = projects.filter(id=project_id)
-            history_entries = history_entries.filter(key__in=keys)
-
-            #Memberships
-            for membership in project.memberships.exclude(user=None).exclude(user=project.owner):
-                _push_to_timelines(project, membership.user, membership, "create", membership.created_at)
-
-        for project in projects.iterator():
-            print("Project:", project)
-            extra_data = {
-                "values_diff": {},
-                "user": extract_user_info(project.owner),
-            }
-            _push_to_timelines(project, project.owner, project, "create", project.created_date,
-                               extra_data=extra_data)
-            del extra_data
-
-        for historyEntry in history_entries.iterator():
-            print("History entry:", historyEntry.created_at)
-            try:
-                on_new_history_entry(None, historyEntry, None)
-            except ObjectDoesNotExist as e:
-                print("Ignoring")
-
-    bulk_creator.flush()
-
 
 class Command(BaseCommand):
     help = 'Regenerate project timeline'
@@ -168,4 +60,4 @@ class Command(BaseCommand):
         if options["purge"] == True:
             Timeline.objects.all().delete()
 
-        generate_timeline(options["initial_date"], options["final_date"], options["project"])
+        rebuild_timeline(options["initial_date"], options["final_date"], options["project"])

taiga/timeline/rebuilder.py

@@ -0,0 +1,137 @@
+# -*- coding: utf-8 -*-
+# Copyright (C) 2014-2016 Andrey Antukh <niwi@niwi.nz>
+# Copyright (C) 2014-2016 Jesús Espino <jespinog@gmail.com>
+# Copyright (C) 2014-2016 David Barragán <bameda@dbarragan.com>
+# Copyright (C) 2014-2016 Alejandro Alonso <alejandro.alonso@kaleidos.net>
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from django.contrib.contenttypes.models import ContentType
+from django.core.exceptions import ObjectDoesNotExist
+from django.db.models import Model
+from django.test.utils import override_settings
+
+from taiga.projects.models import Project
+from taiga.projects.history.models import HistoryEntry
+from .models import Timeline
+from .service import _get_impl_key_from_model, _timeline_impl_map, extract_user_info
+from .signals import on_new_history_entry, _push_to_timelines
+
+from unittest.mock import patch
+
+import gc
+
+
+class BulkCreator(object):
+    def __init__(self):
+        self.timeline_objects = []
+        self.created = None
+
+    def create_element(self, element):
+        self.timeline_objects.append(element)
+        if len(self.timeline_objects) > 999:
+            self.flush()
+
+    def flush(self):
+        Timeline.objects.bulk_create(self.timeline_objects, batch_size=1000)
+        del self.timeline_objects
+        self.timeline_objects = []
+        gc.collect()
+
+bulk_creator = BulkCreator()
+
+
+def custom_add_to_object_timeline(obj:object, instance:object, event_type:str, created_datetime:object,
+                                  namespace:str="default", extra_data:dict={}):
+    assert isinstance(obj, Model), "obj must be a instance of Model"
+    assert isinstance(instance, Model), "instance must be a instance of Model"
+    event_type_key = _get_impl_key_from_model(instance.__class__, event_type)
+    impl = _timeline_impl_map.get(event_type_key, None)
+
+    bulk_creator.create_element(Timeline(
+        content_object=obj,
+        namespace=namespace,
+        event_type=event_type_key,
+        project=instance.project,
+        data=impl(instance, extra_data=extra_data),
+        data_content_type=ContentType.objects.get_for_model(instance.__class__),
+        created=created_datetime,
+    ))
+
+
+@override_settings(CELERY_ENABLED=False)
+def rebuild_timeline(initial_date, final_date, project_id):
+    if initial_date or final_date or project_id:
+        timelines = Timeline.objects.all()
+        if initial_date:
+            timelines = timelines.filter(created__gte=initial_date)
+        if final_date:
+            timelines = timelines.filter(created__lt=final_date)
+        if project_id:
+            timelines = timelines.filter(project__id=project_id)
+
+        timelines.delete()
+
+    with patch('taiga.timeline.service._add_to_object_timeline', new=custom_add_to_object_timeline):
+        # Projects api wasn't a HistoryResourceMixin so we can't interate on the HistoryEntries in this case
+        projects = Project.objects.order_by("created_date")
+        history_entries = HistoryEntry.objects.order_by("created_at")
+
+        if initial_date:
+            projects = projects.filter(created_date__gte=initial_date)
+            history_entries = history_entries.filter(created_at__gte=initial_date)
+
+        if final_date:
+            projects = projects.filter(created_date__lt=final_date)
+            history_entries = history_entries.filter(created_at__lt=final_date)
+
+        if project_id:
+            project = Project.objects.get(id=project_id)
+            epic_keys = ['epics.epic:%s'%(id) for id in project.epics.values_list("id", flat=True)]
+            us_keys = ['userstories.userstory:%s'%(id) for id in project.user_stories.values_list("id",
+                                                                                                  flat=True)]
+            tasks_keys = ['tasks.task:%s'%(id) for id in project.tasks.values_list("id", flat=True)]
+            issue_keys = ['issues.issue:%s'%(id) for id in project.issues.values_list("id", flat=True)]
+            wiki_keys = ['wiki.wikipage:%s'%(id) for id in project.wiki_pages.values_list("id", flat=True)]
+            keys = epic_keys + us_keys + tasks_keys + issue_keys + wiki_keys
+
+            projects = projects.filter(id=project_id)
+            history_entries = history_entries.filter(key__in=keys)
+
+            #Memberships
+            for membership in project.memberships.exclude(user=None).exclude(user=project.owner):
+                _push_to_timelines(project, membership.user, membership, "create", membership.created_at, refresh_totals=False)
+
+        for project in projects.iterator():
+            print("Project:", project)
+            extra_data = {
+                "values_diff": {},
+                "user": extract_user_info(project.owner),
+            }
+            _push_to_timelines(project, project.owner, project, 'create',
+                               project.created_date, extra_data=extra_data,
+                               refresh_totals=False)
+            del extra_data
+
+        for historyEntry in history_entries.iterator():
+            print("History entry:", historyEntry.created_at)
+            try:
+                historyEntry.refresh_totals = False
+                on_new_history_entry(None, historyEntry, None)
+            except ObjectDoesNotExist as e:
+                print("Ignoring")
+
+        for project in projects.iterator():
+            project.refresh_totals()
+
+    bulk_creator.flush()

taiga/timeline/service.py

@@ -93,7 +93,7 @@ def _push_to_timeline(objects, instance: object, event_type: str, created_dateti
 
 @app.task
 def push_to_timelines(project_id, user_id, obj_app_label, obj_model_name, obj_id, event_type,
-                      created_datetime, extra_data={}):
+                      created_datetime, extra_data={}, refresh_totals=True):
 
     ObjModel = apps.get_model(obj_app_label, obj_model_name)
     try:
@@ -120,6 +120,7 @@ def push_to_timelines(project_id, user_id, obj_app_label, obj_model_name, obj_id
                           namespace=build_project_namespace(project),
                           extra_data=extra_data)
 
+        if refresh_totals:
             project.refresh_totals()
 
         if hasattr(obj, "get_related_people"):

taiga/timeline/signals.py

@@ -31,7 +31,7 @@ from taiga.timeline.service import (push_to_timelines,
                                     extract_user_info)
 
 
-def _push_to_timelines(project, user, obj, event_type, created_datetime, extra_data={}):
+def _push_to_timelines(project, user, obj, event_type, created_datetime, extra_data={}, refresh_totals=True):
     project_id = None if project is None else project.id
 
     ct = ContentType.objects.get_for_model(obj)
@@ -43,7 +43,8 @@ def _push_to_timelines(project, user, obj, event_type, created_datetime, extra_d
                                                              obj.id,
                                                              event_type,
                                                              created_datetime,
-                                                             extra_data=extra_data))
+                                                             extra_data=extra_data,
+                                                             refresh_totals=refresh_totals))
     else:
         push_to_timelines(project_id,
                           user.id,
@@ -52,7 +53,8 @@ def _push_to_timelines(project, user, obj, event_type, created_datetime, extra_d
                           obj.id,
                           event_type,
                           created_datetime,
-                          extra_data=extra_data)
+                          extra_data=extra_data,
+                          refresh_totals=refresh_totals)
 
 
 def _clean_description_fields(values_diff):
@@ -73,6 +75,8 @@ def on_new_history_entry(sender, instance, created, **kwargs):
     if instance.user["pk"] is None:
         return None
 
+    refresh_totals = getattr(instance, "refresh_totals", True)
+
     model = history_services.get_model_from_key(instance.key)
     pk = history_services.get_pk_from_key(instance.key)
     obj = model.objects.get(pk=pk)
@@ -105,7 +109,7 @@ def on_new_history_entry(sender, instance, created, **kwargs):
         extra_data["comment_edited"] = True
 
     created_datetime = instance.created_at
-    _push_to_timelines(project, user, obj, event_type, created_datetime, extra_data=extra_data)
+    _push_to_timelines(project, user, obj, event_type, created_datetime, extra_data=extra_data, refresh_totals=refresh_totals)
 
 
 def create_membership_push_to_timeline(sender, instance, created, **kwargs):