taiga
/
taiga-back
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Move the timeline rebuild to the timeline library

remotes/origin/issue/4795/notification_even_they_are_disabled
Jesús Espino 2016-11-24 11:35:28 +01:00
parent 5449aac84d
commit 4d3e65c603
6 changed files with 152 additions and 215 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
taiga/projects/management/commands/change_project_slug.py
View File

@ -23,7 +23,7 @@ from django.test.utils import override_settings
from taiga.base.utils.slug import slugify_uniquely
from taiga.projects.models import Project
from taiga.projects.history.models import HistoryEntry
from taiga.timeline.management.commands.rebuild_timeline import generate_timeline
from taiga.timeline.rebuilder import rebuild_timeline
class Command(BaseCommand):
@ -58,4 +58,4 @@ class Command(BaseCommand):
# Regenerate timeline
self.stdout.write(self.style.SUCCESS("-> Regenerate timeline entries."))
generate_timeline(None, None, project.id)
rebuild_timeline(None, None, project.id)

97
taiga/timeline/management/commands/_rebuild_timeline_for_user_creation.py
View File

@ -1,97 +0,0 @@
# -*- coding: utf-8 -*-
# Copyright (C) 2014-2016 Andrey Antukh <niwi@niwi.nz>
# Copyright (C) 2014-2016 Jesús Espino <jespinog@gmail.com>
# Copyright (C) 2014-2016 David Barragán <bameda@dbarragan.com>
# Copyright (C) 2014-2016 Alejandro Alonso <alejandro.alonso@kaleidos.net>
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
# Examples:
# python manage.py rebuild_timeline_for_user_creation --settings=settings.local_timeline
from django.contrib.auth import get_user_model
from django.contrib.contenttypes.models import ContentType
from django.core.management.base import BaseCommand
from django.db.models import Model
from django.test.utils import override_settings
from taiga.timeline.service import _get_impl_key_from_model,
from taiga.timeline.service import _timeline_impl_map,
from taiga.timeline.service import extract_user_info)
from taiga.timeline.models import Timeline
from taiga.timeline.signals import _push_to_timelines
from unittest.mock import patch
import gc
class BulkCreator(object):
def __init__(self):
self.timeline_objects = []
self.created = None
def create_element(self, element):
self.timeline_objects.append(element)
if len(self.timeline_objects) > 1000:
self.flush()
def flush(self):
Timeline.objects.bulk_create(self.timeline_objects, batch_size=1000)
del self.timeline_objects
self.timeline_objects = []
gc.collect()
bulk_creator = BulkCreator()
def custom_add_to_object_timeline(obj:object, instance:object, event_type:str, created_datetime:object,
namespace:str="default", extra_data:dict={}):
assert isinstance(obj, Model), "obj must be a instance of Model"
assert isinstance(instance, Model), "instance must be a instance of Model"
event_type_key = _get_impl_key_from_model(instance.__class__, event_type)
impl = _timeline_impl_map.get(event_type_key, None)
bulk_creator.create_element(Timeline(
content_object=obj,
namespace=namespace,
event_type=event_type_key,
project=None,
data=impl(instance, extra_data=extra_data),
data_content_type = ContentType.objects.get_for_model(instance.__class__),
created=created_datetime,
))
def generate_timeline():
with patch('taiga.timeline.service._add_to_object_timeline', new=custom_add_to_object_timeline):
# Users api wasn't a HistoryResourceMixin so we can't interate on the HistoryEntries in this case
users = get_user_model().objects.order_by("date_joined")
for user in users.iterator():
print("User:", user.date_joined)
extra_data = {
"values_diff": {},
"user": extract_user_info(user),
}
_push_to_timelines(None, user, user, "create", user.date_joined, extra_data=extra_data)
del extra_data
bulk_creator.flush()
class Command(BaseCommand):
help = 'Regenerate project timeline'
@override_settings(DEBUG=False)
def handle(self, *args, **options):
generate_timeline()

112
taiga/timeline/management/commands/rebuild_timeline.py
View File

@ -21,122 +21,14 @@
# python manage.py rebuild_timeline --settings=settings.local_timeline --purge
# python manage.py rebuild_timeline --settings=settings.local_timeline --initial_date 2014-10-02
from django.contrib.contenttypes.models import ContentType
from django.core.exceptions import ObjectDoesNotExist
from django.core.management.base import BaseCommand
from django.db.models import Model
from django.test.utils import override_settings
from taiga.projects.models import Project
from taiga.projects.history.models import HistoryEntry
from taiga.timeline.models import Timeline
from taiga.timeline.service import _get_impl_key_from_model,_timeline_impl_map, extract_user_info
from taiga.timeline.signals import on_new_history_entry, _push_to_timelines
from taiga.timeline.rebuilder import rebuild_timeline
from unittest.mock import patch
from optparse import make_option
import gc
class BulkCreator(object):
def __init__(self):
self.timeline_objects = []
self.created = None
def create_element(self, element):
self.timeline_objects.append(element)
if len(self.timeline_objects) > 999:
self.flush()
def flush(self):
Timeline.objects.bulk_create(self.timeline_objects, batch_size=1000)
del self.timeline_objects
self.timeline_objects = []
gc.collect()
bulk_creator = BulkCreator()
def custom_add_to_object_timeline(obj:object, instance:object, event_type:str, created_datetime:object,
namespace:str="default", extra_data:dict={}):
assert isinstance(obj, Model), "obj must be a instance of Model"
assert isinstance(instance, Model), "instance must be a instance of Model"
event_type_key = _get_impl_key_from_model(instance.__class__, event_type)
impl = _timeline_impl_map.get(event_type_key, None)
bulk_creator.create_element(Timeline(
content_object=obj,
namespace=namespace,
event_type=event_type_key,
project=instance.project,
data=impl(instance, extra_data=extra_data),
data_content_type=ContentType.objects.get_for_model(instance.__class__),
created=created_datetime,
))
def generate_timeline(initial_date, final_date, project_id):
if initial_date or final_date or project_id:
timelines = Timeline.objects.all()
if initial_date:
timelines = timelines.filter(created__gte=initial_date)
if final_date:
timelines = timelines.filter(created__lt=final_date)
if project_id:
timelines = timelines.filter(project__id=project_id)
timelines.delete()
with patch('taiga.timeline.service._add_to_object_timeline', new=custom_add_to_object_timeline):
# Projects api wasn't a HistoryResourceMixin so we can't interate on the HistoryEntries in this case
projects = Project.objects.order_by("created_date")
history_entries = HistoryEntry.objects.order_by("created_at")
if initial_date:
projects = projects.filter(created_date__gte=initial_date)
history_entries = history_entries.filter(created_at__gte=initial_date)
if final_date:
projects = projects.filter(created_date__lt=final_date)
history_entries = history_entries.filter(created_at__lt=final_date)
if project_id:
project = Project.objects.get(id=project_id)
epic_keys = ['epics.epic:%s'%(id) for id in project.epics.values_list("id", flat=True)]
us_keys = ['userstories.userstory:%s'%(id) for id in project.user_stories.values_list("id",
flat=True)]
tasks_keys = ['tasks.task:%s'%(id) for id in project.tasks.values_list("id", flat=True)]
issue_keys = ['issues.issue:%s'%(id) for id in project.issues.values_list("id", flat=True)]
wiki_keys = ['wiki.wikipage:%s'%(id) for id in project.wiki_pages.values_list("id", flat=True)]
keys = epic_keys + us_keys + tasks_keys + issue_keys + wiki_keys
projects = projects.filter(id=project_id)
history_entries = history_entries.filter(key__in=keys)
#Memberships
for membership in project.memberships.exclude(user=None).exclude(user=project.owner):
_push_to_timelines(project, membership.user, membership, "create", membership.created_at)
for project in projects.iterator():
print("Project:", project)
extra_data = {
"values_diff": {},
"user": extract_user_info(project.owner),
}
_push_to_timelines(project, project.owner, project, "create", project.created_date,
extra_data=extra_data)
del extra_data
for historyEntry in history_entries.iterator():
print("History entry:", historyEntry.created_at)
try:
on_new_history_entry(None, historyEntry, None)
except ObjectDoesNotExist as e:
print("Ignoring")
bulk_creator.flush()
class Command(BaseCommand):
help = 'Regenerate project timeline'
@ -168,4 +60,4 @@ class Command(BaseCommand):
if options["purge"] == True:
Timeline.objects.all().delete()
generate_timeline(options["initial_date"], options["final_date"], options["project"])
rebuild_timeline(options["initial_date"], options["final_date"], options["project"])

137
taiga/timeline/rebuilder.py Normal file
View File

@ -0,0 +1,137 @@
# -*- coding: utf-8 -*-
# Copyright (C) 2014-2016 Andrey Antukh <niwi@niwi.nz>
# Copyright (C) 2014-2016 Jesús Espino <jespinog@gmail.com>
# Copyright (C) 2014-2016 David Barragán <bameda@dbarragan.com>
# Copyright (C) 2014-2016 Alejandro Alonso <alejandro.alonso@kaleidos.net>
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
from django.contrib.contenttypes.models import ContentType
from django.core.exceptions import ObjectDoesNotExist
from django.db.models import Model
from django.test.utils import override_settings
from taiga.projects.models import Project
from taiga.projects.history.models import HistoryEntry
from .models import Timeline
from .service import _get_impl_key_from_model, _timeline_impl_map, extract_user_info
from .signals import on_new_history_entry, _push_to_timelines
from unittest.mock import patch
import gc
class BulkCreator(object):
def __init__(self):
self.timeline_objects = []
self.created = None
def create_element(self, element):
self.timeline_objects.append(element)
if len(self.timeline_objects) > 999:
self.flush()
def flush(self):
Timeline.objects.bulk_create(self.timeline_objects, batch_size=1000)
del self.timeline_objects
self.timeline_objects = []
gc.collect()
bulk_creator = BulkCreator()
def custom_add_to_object_timeline(obj:object, instance:object, event_type:str, created_datetime:object,
namespace:str="default", extra_data:dict={}):
assert isinstance(obj, Model), "obj must be a instance of Model"
assert isinstance(instance, Model), "instance must be a instance of Model"
event_type_key = _get_impl_key_from_model(instance.__class__, event_type)
impl = _timeline_impl_map.get(event_type_key, None)
bulk_creator.create_element(Timeline(
content_object=obj,
namespace=namespace,
event_type=event_type_key,
project=instance.project,
data=impl(instance, extra_data=extra_data),
data_content_type=ContentType.objects.get_for_model(instance.__class__),
created=created_datetime,
))
@override_settings(CELERY_ENABLED=False)
def rebuild_timeline(initial_date, final_date, project_id):
if initial_date or final_date or project_id:
timelines = Timeline.objects.all()
if initial_date:
timelines = timelines.filter(created__gte=initial_date)
if final_date:
timelines = timelines.filter(created__lt=final_date)
if project_id:
timelines = timelines.filter(project__id=project_id)
timelines.delete()
with patch('taiga.timeline.service._add_to_object_timeline', new=custom_add_to_object_timeline):
# Projects api wasn't a HistoryResourceMixin so we can't interate on the HistoryEntries in this case
projects = Project.objects.order_by("created_date")
history_entries = HistoryEntry.objects.order_by("created_at")
if initial_date:
projects = projects.filter(created_date__gte=initial_date)
history_entries = history_entries.filter(created_at__gte=initial_date)
if final_date:
projects = projects.filter(created_date__lt=final_date)
history_entries = history_entries.filter(created_at__lt=final_date)
if project_id:
project = Project.objects.get(id=project_id)
epic_keys = ['epics.epic:%s'%(id) for id in project.epics.values_list("id", flat=True)]
us_keys = ['userstories.userstory:%s'%(id) for id in project.user_stories.values_list("id",
flat=True)]
tasks_keys = ['tasks.task:%s'%(id) for id in project.tasks.values_list("id", flat=True)]
issue_keys = ['issues.issue:%s'%(id) for id in project.issues.values_list("id", flat=True)]
wiki_keys = ['wiki.wikipage:%s'%(id) for id in project.wiki_pages.values_list("id", flat=True)]
keys = epic_keys + us_keys + tasks_keys + issue_keys + wiki_keys
projects = projects.filter(id=project_id)
history_entries = history_entries.filter(key__in=keys)
#Memberships
for membership in project.memberships.exclude(user=None).exclude(user=project.owner):
_push_to_timelines(project, membership.user, membership, "create", membership.created_at, refresh_totals=False)
for project in projects.iterator():
print("Project:", project)
extra_data = {
"values_diff": {},
"user": extract_user_info(project.owner),
}
_push_to_timelines(project, project.owner, project, 'create',
project.created_date, extra_data=extra_data,
refresh_totals=False)
del extra_data
for historyEntry in history_entries.iterator():
print("History entry:", historyEntry.created_at)
try:
historyEntry.refresh_totals = False
on_new_history_entry(None, historyEntry, None)
except ObjectDoesNotExist as e:
print("Ignoring")
for project in projects.iterator():
project.refresh_totals()
bulk_creator.flush()

5
taiga/timeline/service.py
View File

@ -93,7 +93,7 @@ def _push_to_timeline(objects, instance: object, event_type: str, created_dateti
@app.task
def push_to_timelines(project_id, user_id, obj_app_label, obj_model_name, obj_id, event_type,
created_datetime, extra_data={}):
created_datetime, extra_data={}, refresh_totals=True):
ObjModel = apps.get_model(obj_app_label, obj_model_name)
try:
@ -120,7 +120,8 @@ def push_to_timelines(project_id, user_id, obj_app_label, obj_model_name, obj_id
namespace=build_project_namespace(project),
extra_data=extra_data)
project.refresh_totals()
if refresh_totals:
project.refresh_totals()
if hasattr(obj, "get_related_people"):
related_people = obj.get_related_people()

12
taiga/timeline/signals.py
View File

@ -31,7 +31,7 @@ from taiga.timeline.service import (push_to_timelines,
extract_user_info)
def _push_to_timelines(project, user, obj, event_type, created_datetime, extra_data={}):
def _push_to_timelines(project, user, obj, event_type, created_datetime, extra_data={}, refresh_totals=True):
project_id = None if project is None else project.id
ct = ContentType.objects.get_for_model(obj)
@ -43,7 +43,8 @@ def _push_to_timelines(project, user, obj, event_type, created_datetime, extra_d
obj.id,
event_type,
created_datetime,
extra_data=extra_data))
extra_data=extra_data,
refresh_totals=refresh_totals))
else:
push_to_timelines(project_id,
user.id,
@ -52,7 +53,8 @@ def _push_to_timelines(project, user, obj, event_type, created_datetime, extra_d
obj.id,
event_type,
created_datetime,
extra_data=extra_data)
extra_data=extra_data,
refresh_totals=refresh_totals)
def _clean_description_fields(values_diff):
@ -73,6 +75,8 @@ def on_new_history_entry(sender, instance, created, **kwargs):
if instance.user["pk"] is None:
return None
refresh_totals = getattr(instance, "refresh_totals", True)
model = history_services.get_model_from_key(instance.key)
pk = history_services.get_pk_from_key(instance.key)
obj = model.objects.get(pk=pk)
@ -105,7 +109,7 @@ def on_new_history_entry(sender, instance, created, **kwargs):
extra_data["comment_edited"] = True
created_datetime = instance.created_at
_push_to_timelines(project, user, obj, event_type, created_datetime, extra_data=extra_data)
_push_to_timelines(project, user, obj, event_type, created_datetime, extra_data=extra_data, refresh_totals=refresh_totals)
def create_membership_push_to_timeline(sender, instance, created, **kwargs):