taiga
/
taiga-back
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge pull request #760 from taigaio/export-performance 

Improve export performance
remotes/origin/issue/4795/notification_even_they_are_disabled
Alejandro 2016-06-21 11:51:10 +02:00 committed by GitHub
parent 9e9f2af52a 773ab63106
commit 089a864825
13 changed files with 722 additions and 388 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
taiga/export_import/api.py
View File

@ -18,6 +18,7 @@
import codecs
import uuid
import gzip
from django.utils.decorators import method_decorator
from django.utils.translation import ugettext as _
@ -64,16 +65,24 @@ class ProjectExporterViewSet(mixins.ImportThrottlingPolicyMixin, GenericViewSet)
project = get_object_or_404(self.get_queryset(), pk=pk)
self.check_permissions(request, 'export_project', project)
dump_format = request.QUERY_PARAMS.get("dump_format", None)
if settings.CELERY_ENABLED:
task = tasks.dump_project.delay(request.user, project)
tasks.delete_project_dump.apply_async((project.pk, project.slug, task.id),
task = tasks.dump_project.delay(request.user, project, dump_format)
tasks.delete_project_dump.apply_async((project.pk, project.slug, task.id, dump_format),
countdown=settings.EXPORTS_TTL)
return response.Accepted({"export_id": task.id})
path = "exports/{}/{}-{}.json".format(project.pk, project.slug, uuid.uuid4().hex)
storage_path = default_storage.path(path)
with default_storage.open(storage_path, mode="w") as outfile:
services.render_project(project, outfile)
if dump_format == "gzip":
path = "exports/{}/{}-{}.json.gz".format(project.pk, project.slug, uuid.uuid4().hex)
storage_path = default_storage.path(path)
with default_storage.open(storage_path, mode="wb") as outfile:
services.render_project(project, gzip.GzipFile(fileobj=outfile))
else:
path = "exports/{}/{}-{}.json".format(project.pk, project.slug, uuid.uuid4().hex)
storage_path = default_storage.path(path)
with default_storage.open(storage_path, mode="wb") as outfile:
services.render_project(project, outfile)
response_data = {
"url": default_storage.url(path)

19
taiga/export_import/management/commands/dump_project.py
View File

@ -22,6 +22,7 @@ from taiga.projects.models import Project
from taiga.export_import.services import render_project
import os
import gzip
class Command(BaseCommand):
@ -39,6 +40,13 @@ class Command(BaseCommand):
metavar="DIR",
help="Directory to save the json files. ('./' by default)")
parser.add_argument("-f", "--format",
action="store",
dest="format",
default="plain",
metavar="[plain|gzip]",
help="Format to the output file plain json or gzipped json. ('plain' by default)")
def handle(self, *args, **options):
dst_dir = options["dst_dir"]
@ -56,8 +64,13 @@ class Command(BaseCommand):
except Project.DoesNotExist:
raise CommandError("Project '{}' does not exist".format(project_slug))
dst_file = os.path.join(dst_dir, "{}.json".format(project_slug))
with open(dst_file, "w") as f:
render_project(project, f)
if options["format"] == "gzip":
dst_file = os.path.join(dst_dir, "{}.json.gz".format(project_slug))
with gzip.GzipFile(dst_file, "wb") as f:
render_project(project, f)
else:
dst_file = os.path.join(dst_dir, "{}.json".format(project_slug))
with open(dst_file, "wb") as f:
render_project(project, f)
print("-> Generate dump of project '{}' in '{}'".format(project.name, dst_file))

82
taiga/export_import/management/commands/dump_project_async.py Normal file
View File

@ -0,0 +1,82 @@
# -*- coding: utf-8 -*-
# Copyright (C) 2014-2016 Andrey Antukh <niwi@niwi.nz>
# Copyright (C) 2014-2016 Jesús Espino <jespinog@gmail.com>
# Copyright (C) 2014-2016 David Barragán <bameda@dbarragan.com>
# Copyright (C) 2014-2016 Alejandro Alonso <alejandro.alonso@kaleidos.net>
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
from django.core.management.base import BaseCommand, CommandError
from django.db.models import Q
from django.conf import settings
from taiga.projects.models import Project
from taiga.users.models import User
from taiga.permissions.services import is_project_admin
from taiga.export_import import tasks
class Command(BaseCommand):
help = "Export projects to a json file"
def add_arguments(self, parser):
parser.add_argument("project_slugs",
nargs="+",
help="<project_slug project_slug ...>")
parser.add_argument("-u", "--user",
action="store",
dest="user",
default="./",
metavar="DIR",
required=True,
help="Dump as user by email or username.")
parser.add_argument("-f", "--format",
action="store",
dest="format",
default="plain",
metavar="[plain|gzip]",
help="Format to the output file plain json or gzipped json. ('plain' by default)")
def handle(self, *args, **options):
username_or_email = options["user"]
dump_format = options["format"]
project_slugs = options["project_slugs"]
try:
user = User.objects.get(Q(username=username_or_email) | Q(email=username_or_email))
except Exception:
raise CommandError("Error loading user".format(username_or_email))
for project_slug in project_slugs:
try:
project = Project.objects.get(slug=project_slug)
except Project.DoesNotExist:
raise CommandError("Project '{}' does not exist".format(project_slug))
if not is_project_admin(user, project):
self.stderr.write(self.style.ERROR(
"ERROR: Not sending task because user '{}' doesn't have permissions to export '{}' project".format(
username_or_email,
project_slug
)
))
continue
task = tasks.dump_project.delay(user, project, dump_format)
tasks.delete_project_dump.apply_async(
(project.pk, project.slug, task.id, dump_format),
countdown=settings.EXPORTS_TTL
)
print("-> Sent task for dump of project '{}' as user {}".format(project.name, username_or_email))

45
taiga/export_import/serializers/__init__.py Normal file
View File

@ -0,0 +1,45 @@
# -*- coding: utf-8 -*-
# Copyright (C) 2014-2016 Andrey Antukh <niwi@niwi.nz>
# Copyright (C) 2014-2016 Jesús Espino <jespinog@gmail.com>
# Copyright (C) 2014-2016 David Barragán <bameda@dbarragan.com>
# Copyright (C) 2014-2016 Alejandro Alonso <alejandro.alonso@kaleidos.net>
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
from .serializers import PointsExportSerializer
from .serializers import UserStoryStatusExportSerializer
from .serializers import TaskStatusExportSerializer
from .serializers import IssueStatusExportSerializer
from .serializers import PriorityExportSerializer
from .serializers import SeverityExportSerializer
from .serializers import IssueTypeExportSerializer
from .serializers import RoleExportSerializer
from .serializers import UserStoryCustomAttributeExportSerializer
from .serializers import TaskCustomAttributeExportSerializer
from .serializers import IssueCustomAttributeExportSerializer
from .serializers import BaseCustomAttributesValuesExportSerializer
from .serializers import UserStoryCustomAttributesValuesExportSerializer
from .serializers import TaskCustomAttributesValuesExportSerializer
from .serializers import IssueCustomAttributesValuesExportSerializer
from .serializers import MembershipExportSerializer
from .serializers import RolePointsExportSerializer
from .serializers import MilestoneExportSerializer
from .serializers import TaskExportSerializer
from .serializers import UserStoryExportSerializer
from .serializers import IssueExportSerializer
from .serializers import WikiPageExportSerializer
from .serializers import WikiLinkExportSerializer
from .serializers import TimelineExportSerializer
from .serializers import ProjectExportSerializer
from .mixins import AttachmentExportSerializer
from .mixins import HistoryExportSerializer

42
taiga/export_import/serializers/cache.py Normal file
View File

@ -0,0 +1,42 @@
# -*- coding: utf-8 -*-
# Copyright (C) 2014-2016 Andrey Antukh <niwi@niwi.nz>
# Copyright (C) 2014-2016 Jesús Espino <jespinog@gmail.com>
# Copyright (C) 2014-2016 David Barragán <bameda@dbarragan.com>
# Copyright (C) 2014-2016 Alejandro Alonso <alejandro.alonso@kaleidos.net>
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
from taiga.users import models as users_models
_cache_user_by_pk = {}
_cache_user_by_email = {}
_custom_tasks_attributes_cache = {}
_custom_issues_attributes_cache = {}
_custom_userstories_attributes_cache = {}
def cached_get_user_by_pk(pk):
if pk not in _cache_user_by_pk:
try:
_cache_user_by_pk[pk] = users_models.User.objects.get(pk=pk)
except Exception:
_cache_user_by_pk[pk] = users_models.User.objects.get(pk=pk)
return _cache_user_by_pk[pk]
def cached_get_user_by_email(email):
if email not in _cache_user_by_email:
try:
_cache_user_by_email[email] = users_models.User.objects.get(email=email)
except Exception:
_cache_user_by_email[email] = users_models.User.objects.get(email=email)
return _cache_user_by_email[email]

266
taiga/export_import/serializers/fields.py Normal file
View File

@ -0,0 +1,266 @@
# -*- coding: utf-8 -*-
# Copyright (C) 2014-2016 Andrey Antukh <niwi@niwi.nz>
# Copyright (C) 2014-2016 Jesús Espino <jespinog@gmail.com>
# Copyright (C) 2014-2016 David Barragán <bameda@dbarragan.com>
# Copyright (C) 2014-2016 Alejandro Alonso <alejandro.alonso@kaleidos.net>
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import base64
import os
import copy
from collections import OrderedDict
from django.core.files.base import ContentFile
from django.core.exceptions import ObjectDoesNotExist
from django.core.exceptions import ValidationError
from django.utils.translation import ugettext as _
from django.contrib.contenttypes.models import ContentType
from taiga.base.api import serializers
from taiga.base.fields import JsonField
from taiga.mdrender.service import render as mdrender
from taiga.users import models as users_models
from .cache import cached_get_user_by_email, cached_get_user_by_pk
class FileField(serializers.WritableField):
read_only = False
def to_native(self, obj):
if not obj:
return None
data = base64.b64encode(obj.read()).decode('utf-8')
return OrderedDict([
("data", data),
("name", os.path.basename(obj.name)),
])
def from_native(self, data):
if not data:
return None
decoded_data = b''
# The original file was encoded by chunks but we don't really know its
# length or if it was multiple of 3 so we must iterate over all those chunks
# decoding them one by one
for decoding_chunk in data['data'].split("="):
# When encoding to base64 3 bytes are transformed into 4 bytes and
# the extra space of the block is filled with =
# We must ensure that the decoding chunk has a length multiple of 4 so
# we restore the stripped '='s adding appending them until the chunk has
# a length multiple of 4
decoding_chunk += "=" * (-len(decoding_chunk) % 4)
decoded_data += base64.b64decode(decoding_chunk+"=")
return ContentFile(decoded_data, name=data['name'])
class ContentTypeField(serializers.RelatedField):
read_only = False
def to_native(self, obj):
if obj:
return [obj.app_label, obj.model]
return None
def from_native(self, data):
try:
return ContentType.objects.get_by_natural_key(*data)
except Exception:
return None
class RelatedNoneSafeField(serializers.RelatedField):
def field_from_native(self, data, files, field_name, into):
if self.read_only:
return
try:
if self.many:
try:
# Form data
value = data.getlist(field_name)
if value == [''] or value == []:
raise KeyError
except AttributeError:
# Non-form data
value = data[field_name]
else:
value = data[field_name]
except KeyError:
if self.partial:
return
value = self.get_default_value()
key = self.source or field_name
if value in self.null_values:
if self.required:
raise ValidationError(self.error_messages['required'])
into[key] = None
elif self.many:
into[key] = [self.from_native(item) for item in value if self.from_native(item) is not None]
else:
into[key] = self.from_native(value)
class UserRelatedField(RelatedNoneSafeField):
read_only = False
def to_native(self, obj):
if obj:
return obj.email
return None
def from_native(self, data):
try:
return cached_get_user_by_email(data)
except users_models.User.DoesNotExist:
return None
class UserPkField(serializers.RelatedField):
read_only = False
def to_native(self, obj):
try:
user = cached_get_user_by_pk(obj)
return user.email
except users_models.User.DoesNotExist:
return None
def from_native(self, data):
try:
user = cached_get_user_by_email(data)
return user.pk
except users_models.User.DoesNotExist:
return None
class CommentField(serializers.WritableField):
read_only = False
def field_from_native(self, data, files, field_name, into):
super().field_from_native(data, files, field_name, into)
into["comment_html"] = mdrender(self.context['project'], data.get("comment", ""))
class ProjectRelatedField(serializers.RelatedField):
read_only = False
null_values = (None, "")
def __init__(self, slug_field, *args, **kwargs):
self.slug_field = slug_field
super().__init__(*args, **kwargs)
def to_native(self, obj):
if obj:
return getattr(obj, self.slug_field)
return None
def from_native(self, data):
try:
kwargs = {self.slug_field: data, "project": self.context['project']}
return self.queryset.get(**kwargs)
except ObjectDoesNotExist:
raise ValidationError(_("{}=\"{}\" not found in this project".format(self.slug_field, data)))
class HistoryUserField(JsonField):
def to_native(self, obj):
if obj is None or obj == {}:
return []
try:
user = cached_get_user_by_pk(obj['pk'])
except users_models.User.DoesNotExist:
user = None
return (UserRelatedField().to_native(user), obj['name'])
def from_native(self, data):
if data is None:
return {}
if len(data) < 2:
return {}
user = UserRelatedField().from_native(data[0])
if user:
pk = user.pk
else:
pk = None
return {"pk": pk, "name": data[1]}
class HistoryValuesField(JsonField):
def to_native(self, obj):
if obj is None:
return []
if "users" in obj:
obj['users'] = list(map(UserPkField().to_native, obj['users']))
return obj
def from_native(self, data):
if data is None:
return []
if "users" in data:
data['users'] = list(map(UserPkField().from_native, data['users']))
return data
class HistoryDiffField(JsonField):
def to_native(self, obj):
if obj is None:
return []
if "assigned_to" in obj:
obj['assigned_to'] = list(map(UserPkField().to_native, obj['assigned_to']))
return obj
def from_native(self, data):
if data is None:
return []
if "assigned_to" in data:
data['assigned_to'] = list(map(UserPkField().from_native, data['assigned_to']))
return data
class TimelineDataField(serializers.WritableField):
read_only = False
def to_native(self, data):
new_data = copy.deepcopy(data)
try:
user = cached_get_user_by_pk(new_data["user"]["id"])
new_data["user"]["email"] = user.email
del new_data["user"]["id"]
except Exception:
pass
return new_data
def from_native(self, data):
new_data = copy.deepcopy(data)
try:
user = cached_get_user_by_email(new_data["user"]["email"])
new_data["user"]["id"] = user.id
del new_data["user"]["email"]
except users_models.User.DoesNotExist:
pass
return new_data

141
taiga/export_import/serializers/mixins.py Normal file
View File

@ -0,0 +1,141 @@
# -*- coding: utf-8 -*-
# Copyright (C) 2014-2016 Andrey Antukh <niwi@niwi.nz>
# Copyright (C) 2014-2016 Jesús Espino <jespinog@gmail.com>
# Copyright (C) 2014-2016 David Barragán <bameda@dbarragan.com>
# Copyright (C) 2014-2016 Alejandro Alonso <alejandro.alonso@kaleidos.net>
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
from django.contrib.auth import get_user_model
from django.core.exceptions import ObjectDoesNotExist
from django.contrib.contenttypes.models import ContentType
from taiga.base.api import serializers
from taiga.projects.history import models as history_models
from taiga.projects.attachments import models as attachments_models
from taiga.projects.notifications import services as notifications_services
from taiga.projects.history import services as history_service
from .fields import (UserRelatedField, HistoryUserField, HistoryDiffField,
JsonField, HistoryValuesField, CommentField, FileField)
class HistoryExportSerializer(serializers.ModelSerializer):
user = HistoryUserField()
diff = HistoryDiffField(required=False)
snapshot = JsonField(required=False)
values = HistoryValuesField(required=False)
comment = CommentField(required=False)
delete_comment_date = serializers.DateTimeField(required=False)
delete_comment_user = HistoryUserField(required=False)
class Meta:
model = history_models.HistoryEntry
exclude = ("id", "comment_html", "key")
class HistoryExportSerializerMixin(serializers.ModelSerializer):
history = serializers.SerializerMethodField("get_history")
def get_history(self, obj):
history_qs = history_service.get_history_queryset_by_model_instance(obj,
types=(history_models.HistoryType.change, history_models.HistoryType.create,))
return HistoryExportSerializer(history_qs, many=True).data
class AttachmentExportSerializer(serializers.ModelSerializer):
owner = UserRelatedField(required=False)
attached_file = FileField()
modified_date = serializers.DateTimeField(required=False)
class Meta:
model = attachments_models.Attachment
exclude = ('id', 'content_type', 'object_id', 'project')
class AttachmentExportSerializerMixin(serializers.ModelSerializer):
attachments = serializers.SerializerMethodField("get_attachments")
def get_attachments(self, obj):
content_type = ContentType.objects.get_for_model(obj.__class__)
attachments_qs = attachments_models.Attachment.objects.filter(object_id=obj.pk,
content_type=content_type)
return AttachmentExportSerializer(attachments_qs, many=True).data
class CustomAttributesValuesExportSerializerMixin(serializers.ModelSerializer):
custom_attributes_values = serializers.SerializerMethodField("get_custom_attributes_values")
def custom_attributes_queryset(self, project):
raise NotImplementedError()
def get_custom_attributes_values(self, obj):
def _use_name_instead_id_as_key_in_custom_attributes_values(custom_attributes, values):
ret = {}
for attr in custom_attributes:
value = values.get(str(attr["id"]), None)
if value is not None:
ret[attr["name"]] = value
return ret
try:
values = obj.custom_attributes_values.attributes_values
custom_attributes = self.custom_attributes_queryset(obj.project)
return _use_name_instead_id_as_key_in_custom_attributes_values(custom_attributes, values)
except ObjectDoesNotExist:
return None
class WatcheableObjectModelSerializerMixin(serializers.ModelSerializer):
watchers = UserRelatedField(many=True, required=False)
def __init__(self, *args, **kwargs):
self._watchers_field = self.base_fields.pop("watchers", None)
super(WatcheableObjectModelSerializerMixin, self).__init__(*args, **kwargs)
"""
watchers is not a field from the model so we need to do some magic to make it work like a normal field
It's supposed to be represented as an email list but internally it's treated like notifications.Watched instances
"""
def restore_object(self, attrs, instance=None):
watcher_field = self.fields.pop("watchers", None)
instance = super(WatcheableObjectModelSerializerMixin, self).restore_object(attrs, instance)
self._watchers = self.init_data.get("watchers", [])
return instance
def save_watchers(self):
new_watcher_emails = set(self._watchers)
old_watcher_emails = set(self.object.get_watchers().values_list("email", flat=True))
adding_watcher_emails = list(new_watcher_emails.difference(old_watcher_emails))
removing_watcher_emails = list(old_watcher_emails.difference(new_watcher_emails))
User = get_user_model()
adding_users = User.objects.filter(email__in=adding_watcher_emails)
removing_users = User.objects.filter(email__in=removing_watcher_emails)
for user in adding_users:
notifications_services.add_watcher(self.object, user)
for user in removing_users:
notifications_services.remove_watcher(self.object, user)
self.object.watchers = [user.email for user in self.object.get_watchers()]
def to_native(self, obj):
ret = super(WatcheableObjectModelSerializerMixin, self).to_native(obj)
ret["watchers"] = [user.email for user in obj.get_watchers()]
return ret

377
taiga/export_import/serializers.py → taiga/export_import/serializers/serializers.py
View File

@ -16,25 +16,14 @@
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import base64
import copy
import os
from collections import OrderedDict
from django.apps import apps
from django.contrib.auth import get_user_model
from django.core.files.base import ContentFile
from django.core.exceptions import ObjectDoesNotExist
from django.core.exceptions import ValidationError
from django.core.exceptions import ObjectDoesNotExist
from django.utils.translation import ugettext as _
from django.contrib.contenttypes.models import ContentType
from taiga.base.api import serializers
from taiga.base.fields import JsonField, PgArrayField
from taiga.mdrender.service import render as mdrender
from taiga.projects import models as projects_models
from taiga.projects.custom_attributes import models as custom_attributes_models
from taiga.projects.userstories import models as userstories_models
@ -46,286 +35,19 @@ from taiga.projects.history import models as history_models
from taiga.projects.attachments import models as attachments_models
from taiga.timeline import models as timeline_models
from taiga.users import models as users_models
from taiga.projects.notifications import services as notifications_services
from taiga.projects.votes import services as votes_service
from taiga.projects.history import services as history_service
class FileField(serializers.WritableField):
read_only = False
def to_native(self, obj):
if not obj:
return None
data = base64.b64encode(obj.read()).decode('utf-8')
return OrderedDict([
("data", data),
("name", os.path.basename(obj.name)),
])
def from_native(self, data):
if not data:
return None
decoded_data = b''
# The original file was encoded by chunks but we don't really know its
# length or if it was multiple of 3 so we must iterate over all those chunks
# decoding them one by one
for decoding_chunk in data['data'].split("="):
# When encoding to base64 3 bytes are transformed into 4 bytes and
# the extra space of the block is filled with =
# We must ensure that the decoding chunk has a length multiple of 4 so
# we restore the stripped '='s adding appending them until the chunk has
# a length multiple of 4
decoding_chunk += "=" * (-len(decoding_chunk) % 4)
decoded_data += base64.b64decode(decoding_chunk+"=")
return ContentFile(decoded_data, name=data['name'])
class RelatedNoneSafeField(serializers.RelatedField):
def field_from_native(self, data, files, field_name, into):
if self.read_only:
return
try:
if self.many:
try:
# Form data
value = data.getlist(field_name)
if value == [''] or value == []:
raise KeyError
except AttributeError:
# Non-form data
value = data[field_name]
else:
value = data[field_name]
except KeyError:
if self.partial:
return
value = self.get_default_value()
key = self.source or field_name
if value in self.null_values:
if self.required:
raise ValidationError(self.error_messages['required'])
into[key] = None
elif self.many:
into[key] = [self.from_native(item) for item in value if self.from_native(item) is not None]
else:
into[key] = self.from_native(value)
class UserRelatedField(RelatedNoneSafeField):
read_only = False
def to_native(self, obj):
if obj:
return obj.email
return None
def from_native(self, data):
try:
return users_models.User.objects.get(email=data)
except users_models.User.DoesNotExist:
return None
class UserPkField(serializers.RelatedField):
read_only = False
def to_native(self, obj):
try:
user = users_models.User.objects.get(pk=obj)
return user.email
except users_models.User.DoesNotExist:
return None
def from_native(self, data):
try:
user = users_models.User.objects.get(email=data)
return user.pk
except users_models.User.DoesNotExist:
return None
class CommentField(serializers.WritableField):
read_only = False
def field_from_native(self, data, files, field_name, into):
super().field_from_native(data, files, field_name, into)
into["comment_html"] = mdrender(self.context['project'], data.get("comment", ""))
class ProjectRelatedField(serializers.RelatedField):
read_only = False
null_values = (None, "")
def __init__(self, slug_field, *args, **kwargs):
self.slug_field = slug_field
super().__init__(*args, **kwargs)
def to_native(self, obj):
if obj:
return getattr(obj, self.slug_field)
return None
def from_native(self, data):
try:
kwargs = {self.slug_field: data, "project": self.context['project']}
return self.queryset.get(**kwargs)
except ObjectDoesNotExist:
raise ValidationError(_("{}=\"{}\" not found in this project".format(self.slug_field, data)))
class HistoryUserField(JsonField):
def to_native(self, obj):
if obj is None or obj == {}:
return []
try:
user = users_models.User.objects.get(pk=obj['pk'])
except users_models.User.DoesNotExist:
user = None
return (UserRelatedField().to_native(user), obj['name'])
def from_native(self, data):
if data is None:
return {}
if len(data) < 2:
return {}
user = UserRelatedField().from_native(data[0])
if user:
pk = user.pk
else:
pk = None
return {"pk": pk, "name": data[1]}
class HistoryValuesField(JsonField):
def to_native(self, obj):
if obj is None:
return []
if "users" in obj:
obj['users'] = list(map(UserPkField().to_native, obj['users']))
return obj
def from_native(self, data):
if data is None:
return []
if "users" in data:
data['users'] = list(map(UserPkField().from_native, data['users']))
return data
class HistoryDiffField(JsonField):
def to_native(self, obj):
if obj is None:
return []
if "assigned_to" in obj:
obj['assigned_to'] = list(map(UserPkField().to_native, obj['assigned_to']))
return obj
def from_native(self, data):
if data is None:
return []
if "assigned_to" in data:
data['assigned_to'] = list(map(UserPkField().from_native, data['assigned_to']))
return data
class WatcheableObjectModelSerializer(serializers.ModelSerializer):
watchers = UserRelatedField(many=True, required=False)
def __init__(self, *args, **kwargs):
self._watchers_field = self.base_fields.pop("watchers", None)
super(WatcheableObjectModelSerializer, self).__init__(*args, **kwargs)
"""
watchers is not a field from the model so we need to do some magic to make it work like a normal field
It's supposed to be represented as an email list but internally it's treated like notifications.Watched instances
"""
def restore_object(self, attrs, instance=None):
watcher_field = self.fields.pop("watchers", None)
instance = super(WatcheableObjectModelSerializer, self).restore_object(attrs, instance)
self._watchers = self.init_data.get("watchers", [])
return instance
def save_watchers(self):
new_watcher_emails = set(self._watchers)
old_watcher_emails = set(self.object.get_watchers().values_list("email", flat=True))
adding_watcher_emails = list(new_watcher_emails.difference(old_watcher_emails))
removing_watcher_emails = list(old_watcher_emails.difference(new_watcher_emails))
User = get_user_model()
adding_users = User.objects.filter(email__in=adding_watcher_emails)
removing_users = User.objects.filter(email__in=removing_watcher_emails)
for user in adding_users:
notifications_services.add_watcher(self.object, user)
for user in removing_users:
notifications_services.remove_watcher(self.object, user)
self.object.watchers = [user.email for user in self.object.get_watchers()]
def to_native(self, obj):
ret = super(WatcheableObjectModelSerializer, self).to_native(obj)
ret["watchers"] = [user.email for user in obj.get_watchers()]
return ret
class HistoryExportSerializer(serializers.ModelSerializer):
user = HistoryUserField()
diff = HistoryDiffField(required=False)
snapshot = JsonField(required=False)
values = HistoryValuesField(required=False)
comment = CommentField(required=False)
delete_comment_date = serializers.DateTimeField(required=False)
delete_comment_user = HistoryUserField(required=False)
class Meta:
model = history_models.HistoryEntry
exclude = ("id", "comment_html", "key")
class HistoryExportSerializerMixin(serializers.ModelSerializer):
history = serializers.SerializerMethodField("get_history")
def get_history(self, obj):
history_qs = history_service.get_history_queryset_by_model_instance(obj,
types=(history_models.HistoryType.change, history_models.HistoryType.create,))
return HistoryExportSerializer(history_qs, many=True).data
class AttachmentExportSerializer(serializers.ModelSerializer):
owner = UserRelatedField(required=False)
attached_file = FileField()
modified_date = serializers.DateTimeField(required=False)
class Meta:
model = attachments_models.Attachment
exclude = ('id', 'content_type', 'object_id', 'project')
class AttachmentExportSerializerMixin(serializers.ModelSerializer):
attachments = serializers.SerializerMethodField("get_attachments")
def get_attachments(self, obj):
content_type = ContentType.objects.get_for_model(obj.__class__)
attachments_qs = attachments_models.Attachment.objects.filter(object_id=obj.pk,
content_type=content_type)
return AttachmentExportSerializer(attachments_qs, many=True).data
from .fields import (FileField, RelatedNoneSafeField, UserRelatedField,
UserPkField, CommentField, ProjectRelatedField,
HistoryUserField, HistoryValuesField, HistoryDiffField,
TimelineDataField, ContentTypeField)
from .mixins import (HistoryExportSerializerMixin,
AttachmentExportSerializerMixin,
CustomAttributesValuesExportSerializerMixin,
WatcheableObjectModelSerializerMixin)
from .cache import (_custom_tasks_attributes_cache,
_custom_userstories_attributes_cache,
_custom_issues_attributes_cache)
class PointsExportSerializer(serializers.ModelSerializer):
@ -402,31 +124,6 @@ class IssueCustomAttributeExportSerializer(serializers.ModelSerializer):
exclude = ('id', 'project')
class CustomAttributesValuesExportSerializerMixin(serializers.ModelSerializer):
custom_attributes_values = serializers.SerializerMethodField("get_custom_attributes_values")
def custom_attributes_queryset(self, project):
raise NotImplementedError()
def get_custom_attributes_values(self, obj):
def _use_name_instead_id_as_key_in_custom_attributes_values(custom_attributes, values):
ret = {}
for attr in custom_attributes:
value = values.get(str(attr["id"]), None)
if value is not None:
ret[attr["name"]] = value
return ret
try:
values = obj.custom_attributes_values.attributes_values
custom_attributes = self.custom_attributes_queryset(obj.project).values('id', 'name')
return _use_name_instead_id_as_key_in_custom_attributes_values(custom_attributes, values)
except ObjectDoesNotExist:
return None
class BaseCustomAttributesValuesExportSerializer(serializers.ModelSerializer):
attributes_values = JsonField(source="attributes_values",required=True)
_custom_attribute_model = None
@ -508,7 +205,7 @@ class RolePointsExportSerializer(serializers.ModelSerializer):
exclude = ('id', 'user_story')
class MilestoneExportSerializer(WatcheableObjectModelSerializer):
class MilestoneExportSerializer(WatcheableObjectModelSerializerMixin):
owner = UserRelatedField(required=False)
modified_date = serializers.DateTimeField(required=False)
estimated_start = serializers.DateField(required=False)
@ -537,7 +234,7 @@ class MilestoneExportSerializer(WatcheableObjectModelSerializer):
class TaskExportSerializer(CustomAttributesValuesExportSerializerMixin, HistoryExportSerializerMixin,
AttachmentExportSerializerMixin, WatcheableObjectModelSerializer):
AttachmentExportSerializerMixin, WatcheableObjectModelSerializerMixin):
owner = UserRelatedField(required=False)
status = ProjectRelatedField(slug_field="name")
user_story = ProjectRelatedField(slug_field="ref", required=False)
@ -550,11 +247,13 @@ class TaskExportSerializer(CustomAttributesValuesExportSerializerMixin, HistoryE
exclude = ('id', 'project')
def custom_attributes_queryset(self, project):
return project.taskcustomattributes.all()
if project.id not in _custom_tasks_attributes_cache:
_custom_tasks_attributes_cache[project.id] = list(project.taskcustomattributes.all().values('id', 'name'))
return _custom_tasks_attributes_cache[project.id]
class UserStoryExportSerializer(CustomAttributesValuesExportSerializerMixin, HistoryExportSerializerMixin,
AttachmentExportSerializerMixin, WatcheableObjectModelSerializer):
AttachmentExportSerializerMixin, WatcheableObjectModelSerializerMixin):
role_points = RolePointsExportSerializer(many=True, required=False)
owner = UserRelatedField(required=False)
assigned_to = UserRelatedField(required=False)
@ -568,11 +267,13 @@ class UserStoryExportSerializer(CustomAttributesValuesExportSerializerMixin, His
exclude = ('id', 'project', 'points', 'tasks')
def custom_attributes_queryset(self, project):
return project.userstorycustomattributes.all()
if project.id not in _custom_userstories_attributes_cache:
_custom_userstories_attributes_cache[project.id] = list(project.userstorycustomattributes.all().values('id', 'name'))
return _custom_userstories_attributes_cache[project.id]
class IssueExportSerializer(CustomAttributesValuesExportSerializerMixin, HistoryExportSerializerMixin,
AttachmentExportSerializerMixin, WatcheableObjectModelSerializer):
AttachmentExportSerializerMixin, WatcheableObjectModelSerializerMixin):
owner = UserRelatedField(required=False)
status = ProjectRelatedField(slug_field="name")
assigned_to = UserRelatedField(required=False)
@ -591,11 +292,13 @@ class IssueExportSerializer(CustomAttributesValuesExportSerializerMixin, History
return [x.email for x in votes_service.get_voters(obj)]
def custom_attributes_queryset(self, project):
return project.issuecustomattributes.all()
if project.id not in _custom_issues_attributes_cache:
_custom_issues_attributes_cache[project.id] = list(project.issuecustomattributes.all().values('id', 'name'))
return _custom_issues_attributes_cache[project.id]
class WikiPageExportSerializer(HistoryExportSerializerMixin, AttachmentExportSerializerMixin,
WatcheableObjectModelSerializer):
WatcheableObjectModelSerializerMixin):
owner = UserRelatedField(required=False)
last_modifier = UserRelatedField(required=False)
modified_date = serializers.DateTimeField(required=False)
@ -612,39 +315,15 @@ class WikiLinkExportSerializer(serializers.ModelSerializer):
class TimelineDataField(serializers.WritableField):
read_only = False
def to_native(self, data):
new_data = copy.deepcopy(data)
try:
user = users_models.User.objects.get(pk=new_data["user"]["id"])
new_data["user"]["email"] = user.email
del new_data["user"]["id"]
except users_models.User.DoesNotExist:
pass
return new_data
def from_native(self, data):
new_data = copy.deepcopy(data)
try:
user = users_models.User.objects.get(email=new_data["user"]["email"])
new_data["user"]["id"] = user.id
del new_data["user"]["email"]
except users_models.User.DoesNotExist:
pass
return new_data
class TimelineExportSerializer(serializers.ModelSerializer):
data = TimelineDataField()
data_content_type = ContentTypeField()
class Meta:
model = timeline_models.Timeline
exclude = ('id', 'project', 'namespace', 'object_id')
exclude = ('id', 'project', 'namespace', 'object_id', 'content_type')
class ProjectExportSerializer(WatcheableObjectModelSerializer):
class ProjectExportSerializer(WatcheableObjectModelSerializerMixin):
logo = FileField(required=False)
anon_permissions = PgArrayField(required=False)
public_permissions = PgArrayField(required=False)

41
taiga/export_import/services/render.py
View File

@ -34,13 +34,13 @@ from .. import serializers
def render_project(project, outfile, chunk_size = 8190):
serializer = serializers.ProjectExportSerializer(project)
outfile.write('{\n')
outfile.write(b'{\n')
first_field = True
for field_name in serializer.fields.keys():
# Avoid writing "," in the last element
if not first_field:
outfile.write(",\n")
outfile.write(b",\n")
else:
first_field = False
@ -50,7 +50,13 @@ def render_project(project, outfile, chunk_size = 8190):
# These four "special" fields hava attachments so we use them in a special way
if field_name in ["wiki_pages", "user_stories", "tasks", "issues"]:
value = get_component(project, field_name)
outfile.write('"{}": [\n'.format(field_name))
if field_name != "wiki_pages":
value = value.select_related('owner', 'status', 'milestone', 'project', 'assigned_to', 'custom_attributes_values')
if field_name == "issues":
value = value.select_related('severity', 'priority', 'type')
value = value.prefetch_related('history_entry', 'attachments')
outfile.write('"{}": [\n'.format(field_name).encode())
attachments_field = field.fields.pop("attachments", None)
if attachments_field:
@ -60,20 +66,20 @@ def render_project(project, outfile, chunk_size = 8190):
for item in value.iterator():
# Avoid writing "," in the last element
if not first_item:
outfile.write(",\n")
outfile.write(b",\n")
else:
first_item = False
dumped_value = json.dumps(field.to_native(item))
writing_value = dumped_value[:-1]+ ',\n "attachments": [\n'
outfile.write(writing_value)
outfile.write(writing_value.encode())
first_attachment = True
for attachment in item.attachments.iterator():
# Avoid writing "," in the last element
if not first_attachment:
outfile.write(",\n")
outfile.write(b",\n")
else:
first_attachment = False
@ -82,7 +88,7 @@ def render_project(project, outfile, chunk_size = 8190):
attached_file_serializer = attachment_serializer.fields.pop("attached_file")
dumped_value = json.dumps(attachment_serializer.data)
dumped_value = dumped_value[:-1] + ',\n "attached_file":{\n "data":"'
outfile.write(dumped_value)
outfile.write(dumped_value.encode())
# We write the attached_files by chunks so the memory used is not increased
attachment_file = attachment.attached_file
@ -93,33 +99,32 @@ def render_project(project, outfile, chunk_size = 8190):
if not bin_data:
break
b64_data = base64.b64encode(bin_data).decode('utf-8')
b64_data = base64.b64encode(bin_data)
outfile.write(b64_data)
outfile.write('", \n "name":"{}"}}\n}}'.format(
os.path.basename(attachment_file.name)))
os.path.basename(attachment_file.name)).encode())
outfile.write(']}')
outfile.write(b']}')
outfile.flush()
gc.collect()
outfile.write(']')
gc.collect()
outfile.write(b']')
else:
value = field.field_to_native(project, field_name)
outfile.write('"{}": {}'.format(field_name, json.dumps(value)))
outfile.write('"{}": {}'.format(field_name, json.dumps(value)).encode())
# Generate the timeline
outfile.write(',\n"timeline": [\n')
outfile.write(b',\n"timeline": [\n')
first_timeline = True
for timeline_item in get_project_timeline(project).iterator():
# Avoid writing "," in the last element
if not first_timeline:
outfile.write(",\n")
outfile.write(b",\n")
else:
first_timeline = False
dumped_value = json.dumps(serializers.TimelineExportSerializer(timeline_item).data)
outfile.write(dumped_value)
outfile.write(dumped_value.encode())
outfile.write(']}\n')
outfile.write(b']}\n')

3
taiga/export_import/services/store.py
View File

@ -577,6 +577,7 @@ def _store_timeline_entry(project, timeline):
serialized.object.project = project
serialized.object.namespace = build_project_namespace(project)
serialized.object.object_id = project.id
serialized.object.content_type = ContentType.objects.get_for_model(project.__class__)
serialized.object._importing = True
serialized.save()
return serialized
@ -725,7 +726,7 @@ def store_project_from_dict(data, owner=None):
except err.TaigaImportError:
# reraise known inport errors
raise
except:
except Exception:
# reise unknown errors as import error
raise err.TaigaImportError(_("unexpected error importing project"), project)

27
taiga/export_import/tasks.py
View File

@ -19,6 +19,7 @@
import datetime
import logging
import sys
import gzip
from django.core.files.storage import default_storage
from django.core.files.base import ContentFile
@ -41,14 +42,20 @@ import resource
@app.task(bind=True)
def dump_project(self, user, project):
path = "exports/{}/{}-{}.json".format(project.pk, project.slug, self.request.id)
storage_path = default_storage.path(path)
def dump_project(self, user, project, dump_format):
try:
if dump_format == "gzip":
path = "exports/{}/{}-{}.json.gz".format(project.pk, project.slug, self.request.id)
storage_path = default_storage.path(path)
with default_storage.open(storage_path, mode="wb") as outfile:
services.render_project(project, gzip.GzipFile(fileobj=outfile))
else:
path = "exports/{}/{}-{}.json".format(project.pk, project.slug, self.request.id)
storage_path = default_storage.path(path)
with default_storage.open(storage_path, mode="wb") as outfile:
services.render_project(project, outfile)
url = default_storage.url(path)
with default_storage.open(storage_path, mode="w") as outfile:
services.render_project(project, outfile)
except Exception:
# Error
@ -75,8 +82,12 @@ def dump_project(self, user, project):
@app.task
def delete_project_dump(project_id, project_slug, task_id):
default_storage.delete("exports/{}/{}-{}.json".format(project_id, project_slug, task_id))
def delete_project_dump(project_id, project_slug, task_id, dump_format):
if dump_format == "gzip":
path = "exports/{}/{}-{}.json.gz".format(project_id, project_slug, task_id)
else:
path = "exports/{}/{}-{}.json".format(project_id, project_slug, task_id)
default_storage.delete(path)
ADMIN_ERROR_LOAD_PROJECT_DUMP_MESSAGE = _("""

42
tests/integration/test_exporter_api.py
View File

@ -53,6 +53,24 @@ def test_valid_project_export_with_celery_disabled(client, settings):
assert response.status_code == 200
response_data = response.data
assert "url" in response_data
assert response_data["url"].endswith(".json")
def test_valid_project_export_with_celery_disabled_and_gzip(client, settings):
settings.CELERY_ENABLED = False
user = f.UserFactory.create()
project = f.ProjectFactory.create(owner=user)
f.MembershipFactory(project=project, user=user, is_admin=True)
client.login(user)
url = reverse("exporter-detail", args=[project.pk])
response = client.get(url+"?dump_format=gzip", content_type="application/json")
assert response.status_code == 200
response_data = response.data
assert "url" in response_data
assert response_data["url"].endswith(".gz")
def test_valid_project_export_with_celery_enabled(client, settings):
@ -72,7 +90,29 @@ def test_valid_project_export_with_celery_enabled(client, settings):
response_data = response.data
assert "export_id" in response_data
args = (project.id, project.slug, response_data["export_id"],)
args = (project.id, project.slug, response_data["export_id"], None)
kwargs = {"countdown": settings.EXPORTS_TTL}
delete_project_dump_mock.apply_async.assert_called_once_with(args, **kwargs)
def test_valid_project_export_with_celery_enabled_and_gzip(client, settings):
settings.CELERY_ENABLED = True
user = f.UserFactory.create()
project = f.ProjectFactory.create(owner=user)
f.MembershipFactory(project=project, user=user, is_admin=True)
client.login(user)
url = reverse("exporter-detail", args=[project.pk])
#delete_project_dump task should have been launched
with mock.patch('taiga.export_import.tasks.delete_project_dump') as delete_project_dump_mock:
response = client.get(url+"?dump_format=gzip", content_type="application/json")
assert response.status_code == 202
response_data = response.data
assert "export_id" in response_data
args = (project.id, project.slug, response_data["export_id"], "gzip")
kwargs = {"countdown": settings.EXPORTS_TTL}
delete_project_dump_mock.apply_async.assert_called_once_with(args, **kwargs)

4
tests/unit/test_export.py
View File

@ -28,7 +28,7 @@ pytestmark = pytest.mark.django_db
def test_export_issue_finish_date(client):
issue = f.IssueFactory.create(finished_date="2014-10-22")
output = io.StringIO()
output = io.BytesIO()
render_project(issue.project, output)
project_data = json.loads(output.getvalue())
finish_date = project_data["issues"][0]["finished_date"]
@ -37,7 +37,7 @@ def test_export_issue_finish_date(client):
def test_export_user_story_finish_date(client):
user_story = f.UserStoryFactory.create(finish_date="2014-10-22")
output = io.StringIO()
output = io.BytesIO()
render_project(user_story.project, output)
project_data = json.loads(output.getvalue())
finish_date = project_data["user_stories"][0]["finish_date"]