taiga
/
taiga-back
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge pull request #781 from taigaio/export-import-serializer-validator-separation 

Migrating export_import api to new serializers/validators
remotes/origin/issue/4795/notification_even_they_are_disabled
David Barragán Merino 2016-07-28 16:46:22 +02:00 committed by GitHub
parent 888162af63 1b6077c105
commit 340bc7670b
12 changed files with 1294 additions and 765 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
taiga/base/fields.py
View File

@ -18,7 +18,8 @@
from django.forms import widgets from django.forms import widgets
from django.utils.translation import ugettext as _ from django.utils.translation import ugettext as _
from taiga.base.api import serializers from taiga.base.api import serializers, ISO_8601
from taiga.base.api.settings import api_settings
import serpy import serpy
@ -128,4 +129,21 @@ class I18NJsonField(Field):
class FileField(Field): class FileField(Field):
def to_value(self, value): def to_value(self, value):
if value:
return value.name return value.name
return None
class DateTimeField(Field):
format = api_settings.DATETIME_FORMAT
def to_value(self, value):
if value is None or self.format is None:
return value
if self.format.lower() == ISO_8601:
ret = value.isoformat()
if ret.endswith("+00:00"):
ret = ret[:-6] + "Z"
return ret
return value.strftime(self.format)

57
taiga/export_import/api.py
View File

@ -44,11 +44,11 @@ from taiga.users import services as users_services
from . import exceptions as err from . import exceptions as err
from . import mixins from . import mixins
from . import permissions from . import permissions
from . import validators
from . import serializers from . import serializers
from . import services from . import services
from . import tasks from . import tasks
from . import throttling from . import throttling
from .renderers import ExportRenderer
from taiga.base.api.utils import get_object_or_404 from taiga.base.api.utils import get_object_or_404
@ -102,8 +102,7 @@ class ProjectImporterViewSet(mixins.ImportThrottlingPolicyMixin, CreateModelMixi
# Validate if the project can be imported # Validate if the project can be imported
is_private = data.get('is_private', False) is_private = data.get('is_private', False)
total_memberships = len([m for m in data.get("memberships", []) total_memberships = len([m for m in data.get("memberships", []) if m.get("email", None) != data["owner"]])
if m.get("email", None) != data["owner"]])
total_memberships = total_memberships + 1 # 1 is the owner total_memberships = total_memberships + 1 # 1 is the owner
(enough_slots, error_message) = users_services.has_available_slot_for_import_new_project( (enough_slots, error_message) = users_services.has_available_slot_for_import_new_project(
self.request.user, self.request.user,
@ -147,31 +146,31 @@ class ProjectImporterViewSet(mixins.ImportThrottlingPolicyMixin, CreateModelMixi
# Create project values choicess # Create project values choicess
if "points" in data: if "points" in data:
services.store.store_project_attributes_values(project_serialized.object, data, services.store.store_project_attributes_values(project_serialized.object, data,
"points", serializers.PointsExportSerializer) "points", validators.PointsExportValidator)
if "issue_types" in data: if "issue_types" in data:
services.store.store_project_attributes_values(project_serialized.object, data, services.store.store_project_attributes_values(project_serialized.object, data,
"issue_types", "issue_types",
serializers.IssueTypeExportSerializer) validators.IssueTypeExportValidator)
if "issue_statuses" in data: if "issue_statuses" in data:
services.store.store_project_attributes_values(project_serialized.object, data, services.store.store_project_attributes_values(project_serialized.object, data,
"issue_statuses", "issue_statuses",
serializers.IssueStatusExportSerializer,) validators.IssueStatusExportValidator,)
if "us_statuses" in data: if "us_statuses" in data:
services.store.store_project_attributes_values(project_serialized.object, data, services.store.store_project_attributes_values(project_serialized.object, data,
"us_statuses", "us_statuses",
serializers.UserStoryStatusExportSerializer,) validators.UserStoryStatusExportValidator,)
if "task_statuses" in data: if "task_statuses" in data:
services.store.store_project_attributes_values(project_serialized.object, data, services.store.store_project_attributes_values(project_serialized.object, data,
"task_statuses", "task_statuses",
serializers.TaskStatusExportSerializer) validators.TaskStatusExportValidator)
if "priorities" in data: if "priorities" in data:
services.store.store_project_attributes_values(project_serialized.object, data, services.store.store_project_attributes_values(project_serialized.object, data,
"priorities", "priorities",
serializers.PriorityExportSerializer) validators.PriorityExportValidator)
if "severities" in data: if "severities" in data:
services.store.store_project_attributes_values(project_serialized.object, data, services.store.store_project_attributes_values(project_serialized.object, data,
"severities", "severities",
serializers.SeverityExportSerializer) validators.SeverityExportValidator)
if ("points" in data or "issues_types" in data or if ("points" in data or "issues_types" in data or
"issues_statuses" in data or "us_statuses" in data or "issues_statuses" in data or "us_statuses" in data or
@ -183,17 +182,17 @@ class ProjectImporterViewSet(mixins.ImportThrottlingPolicyMixin, CreateModelMixi
if "userstorycustomattributes" in data: if "userstorycustomattributes" in data:
services.store.store_custom_attributes(project_serialized.object, data, services.store.store_custom_attributes(project_serialized.object, data,
"userstorycustomattributes", "userstorycustomattributes",
serializers.UserStoryCustomAttributeExportSerializer) validators.UserStoryCustomAttributeExportValidator)
if "taskcustomattributes" in data: if "taskcustomattributes" in data:
services.store.store_custom_attributes(project_serialized.object, data, services.store.store_custom_attributes(project_serialized.object, data,
"taskcustomattributes", "taskcustomattributes",
serializers.TaskCustomAttributeExportSerializer) validators.TaskCustomAttributeExportValidator)
if "issuecustomattributes" in data: if "issuecustomattributes" in data:
services.store.store_custom_attributes(project_serialized.object, data, services.store.store_custom_attributes(project_serialized.object, data,
"issuecustomattributes", "issuecustomattributes",
serializers.IssueCustomAttributeExportSerializer) validators.IssueCustomAttributeExportValidator)
# Is there any error? # Is there any error?
errors = services.store.get_errors() errors = services.store.get_errors()
@ -201,7 +200,7 @@ class ProjectImporterViewSet(mixins.ImportThrottlingPolicyMixin, CreateModelMixi
raise exc.BadRequest(errors) raise exc.BadRequest(errors)
# Importer process is OK # Importer process is OK
response_data = project_serialized.data response_data = serializers.ProjectExportSerializer(project_serialized.object).data
response_data['id'] = project_serialized.object.id response_data['id'] = project_serialized.object.id
headers = self.get_success_headers(response_data) headers = self.get_success_headers(response_data)
return response.Created(response_data, headers=headers) return response.Created(response_data, headers=headers)
@ -218,8 +217,9 @@ class ProjectImporterViewSet(mixins.ImportThrottlingPolicyMixin, CreateModelMixi
if errors: if errors:
raise exc.BadRequest(errors) raise exc.BadRequest(errors)
headers = self.get_success_headers(milestone.data) data = serializers.MilestoneExportSerializer(milestone.object).data
return response.Created(milestone.data, headers=headers) headers = self.get_success_headers(data)
return response.Created(data, headers=headers)
@detail_route(methods=['post']) @detail_route(methods=['post'])
@method_decorator(atomic) @method_decorator(atomic)
@ -233,8 +233,9 @@ class ProjectImporterViewSet(mixins.ImportThrottlingPolicyMixin, CreateModelMixi
if errors: if errors:
raise exc.BadRequest(errors) raise exc.BadRequest(errors)
headers = self.get_success_headers(us.data) data = serializers.UserStoryExportSerializer(us.object).data
return response.Created(us.data, headers=headers) headers = self.get_success_headers(data)
return response.Created(data, headers=headers)
@detail_route(methods=['post']) @detail_route(methods=['post'])
@method_decorator(atomic) @method_decorator(atomic)
@ -251,8 +252,9 @@ class ProjectImporterViewSet(mixins.ImportThrottlingPolicyMixin, CreateModelMixi
if errors: if errors:
raise exc.BadRequest(errors) raise exc.BadRequest(errors)
headers = self.get_success_headers(task.data) data = serializers.TaskExportSerializer(task.object).data
return response.Created(task.data, headers=headers) headers = self.get_success_headers(data)
return response.Created(data, headers=headers)
@detail_route(methods=['post']) @detail_route(methods=['post'])
@method_decorator(atomic) @method_decorator(atomic)
@ -269,8 +271,9 @@ class ProjectImporterViewSet(mixins.ImportThrottlingPolicyMixin, CreateModelMixi
if errors: if errors:
raise exc.BadRequest(errors) raise exc.BadRequest(errors)
headers = self.get_success_headers(issue.data) data = serializers.IssueExportSerializer(issue.object).data
return response.Created(issue.data, headers=headers) headers = self.get_success_headers(data)
return response.Created(data, headers=headers)
@detail_route(methods=['post']) @detail_route(methods=['post'])
@method_decorator(atomic) @method_decorator(atomic)
@ -284,8 +287,9 @@ class ProjectImporterViewSet(mixins.ImportThrottlingPolicyMixin, CreateModelMixi
if errors: if errors:
raise exc.BadRequest(errors) raise exc.BadRequest(errors)
headers = self.get_success_headers(wiki_page.data) data = serializers.WikiPageExportSerializer(wiki_page.object).data
return response.Created(wiki_page.data, headers=headers) headers = self.get_success_headers(data)
return response.Created(data, headers=headers)
@detail_route(methods=['post']) @detail_route(methods=['post'])
@method_decorator(atomic) @method_decorator(atomic)
@ -299,8 +303,9 @@ class ProjectImporterViewSet(mixins.ImportThrottlingPolicyMixin, CreateModelMixi
if errors: if errors:
raise exc.BadRequest(errors) raise exc.BadRequest(errors)
headers = self.get_success_headers(wiki_link.data) data = serializers.WikiLinkExportSerializer(wiki_link.object).data
return response.Created(wiki_link.data, headers=headers) headers = self.get_success_headers(data)
return response.Created(data, headers=headers)
@list_route(methods=["POST"]) @list_route(methods=["POST"])
@method_decorator(atomic) @method_decorator(atomic)

194
taiga/export_import/serializers/fields.py
View File

@ -21,24 +21,15 @@ import os
import copy import copy
from collections import OrderedDict from collections import OrderedDict
from django.core.files.base import ContentFile
from django.core.exceptions import ObjectDoesNotExist
from django.utils.translation import ugettext as _
from django.contrib.contenttypes.models import ContentType
from taiga.base.api import serializers from taiga.base.api import serializers
from taiga.base.exceptions import ValidationError from taiga.base.fields import Field
from taiga.base.fields import JsonField
from taiga.mdrender.service import render as mdrender
from taiga.users import models as users_models from taiga.users import models as users_models
from .cache import cached_get_user_by_email, cached_get_user_by_pk from .cache import cached_get_user_by_pk
class FileField(serializers.WritableField): class FileField(Field):
read_only = False def to_value(self, obj):
def to_native(self, obj):
if not obj: if not obj:
return None return None
@ -49,202 +40,74 @@ class FileField(serializers.WritableField):
("name", os.path.basename(obj.name)), ("name", os.path.basename(obj.name)),
]) ])
def from_native(self, data):
if not data:
return None
decoded_data = b'' class ContentTypeField(Field):
# The original file was encoded by chunks but we don't really know its def to_value(self, obj):
# length or if it was multiple of 3 so we must iterate over all those chunks
# decoding them one by one
for decoding_chunk in data['data'].split("="):
# When encoding to base64 3 bytes are transformed into 4 bytes and
# the extra space of the block is filled with =
# We must ensure that the decoding chunk has a length multiple of 4 so
# we restore the stripped '='s adding appending them until the chunk has
# a length multiple of 4
decoding_chunk += "=" * (-len(decoding_chunk) % 4)
decoded_data += base64.b64decode(decoding_chunk+"=")
return ContentFile(decoded_data, name=data['name'])
class ContentTypeField(serializers.RelatedField):
read_only = False
def to_native(self, obj):
if obj: if obj:
return [obj.app_label, obj.model] return [obj.app_label, obj.model]
return None return None
def from_native(self, data):
try:
return ContentType.objects.get_by_natural_key(*data)
except Exception:
return None
class UserRelatedField(Field):
class RelatedNoneSafeField(serializers.RelatedField): def to_value(self, obj):
def field_from_native(self, data, files, field_name, into):
if self.read_only:
return
try:
if self.many:
try:
# Form data
value = data.getlist(field_name)
if value == [''] or value == []:
raise KeyError
except AttributeError:
# Non-form data
value = data[field_name]
else:
value = data[field_name]
except KeyError:
if self.partial:
return
value = self.get_default_value()
key = self.source or field_name
if value in self.null_values:
if self.required:
raise ValidationError(self.error_messages['required'])
into[key] = None
elif self.many:
into[key] = [self.from_native(item) for item in value if self.from_native(item) is not None]
else:
into[key] = self.from_native(value)
class UserRelatedField(RelatedNoneSafeField):
read_only = False
def to_native(self, obj):
if obj: if obj:
return obj.email return obj.email
return None return None
def from_native(self, data):
try:
return cached_get_user_by_email(data)
except users_models.User.DoesNotExist:
return None
class UserPkField(Field):
class UserPkField(serializers.RelatedField): def to_value(self, obj):
read_only = False
def to_native(self, obj):
try: try:
user = cached_get_user_by_pk(obj) user = cached_get_user_by_pk(obj)
return user.email return user.email
except users_models.User.DoesNotExist: except users_models.User.DoesNotExist:
return None return None
def from_native(self, data):
try:
user = cached_get_user_by_email(data)
return user.pk
except users_models.User.DoesNotExist:
return None
class CommentField(serializers.WritableField):
read_only = False
def field_from_native(self, data, files, field_name, into):
super().field_from_native(data, files, field_name, into)
into["comment_html"] = mdrender(self.context['project'], data.get("comment", ""))
class ProjectRelatedField(serializers.RelatedField):
read_only = False
null_values = (None, "")
class SlugRelatedField(Field):
def __init__(self, slug_field, *args, **kwargs): def __init__(self, slug_field, *args, **kwargs):
self.slug_field = slug_field self.slug_field = slug_field
super().__init__(*args, **kwargs) super().__init__(*args, **kwargs)
def to_native(self, obj): def to_value(self, obj):
if obj: if obj:
return getattr(obj, self.slug_field) return getattr(obj, self.slug_field)
return None return None
def from_native(self, data):
try:
kwargs = {self.slug_field: data, "project": self.context['project']}
return self.queryset.get(**kwargs)
except ObjectDoesNotExist:
raise ValidationError(_("{}=\"{}\" not found in this project".format(self.slug_field, data)))
class HistoryUserField(Field):
class HistoryUserField(JsonField): def to_value(self, obj):
def to_native(self, obj):
if obj is None or obj == {}: if obj is None or obj == {}:
return [] return []
try: try:
user = cached_get_user_by_pk(obj['pk']) user = cached_get_user_by_pk(obj['pk'])
except users_models.User.DoesNotExist: except users_models.User.DoesNotExist:
user = None user = None
return (UserRelatedField().to_native(user), obj['name']) return (UserRelatedField().to_value(user), obj['name'])
def from_native(self, data):
if data is None:
return {}
if len(data) < 2:
return {}
user = UserRelatedField().from_native(data[0])
if user:
pk = user.pk
else:
pk = None
return {"pk": pk, "name": data[1]}
class HistoryValuesField(JsonField): class HistoryValuesField(Field):
def to_native(self, obj): def to_value(self, obj):
if obj is None: if obj is None:
return [] return []
if "users" in obj: if "users" in obj:
obj['users'] = list(map(UserPkField().to_native, obj['users'])) obj['users'] = list(map(UserPkField().to_value, obj['users']))
return obj return obj
def from_native(self, data):
if data is None:
return []
if "users" in data:
data['users'] = list(map(UserPkField().from_native, data['users']))
return data
class HistoryDiffField(Field):
class HistoryDiffField(JsonField): def to_value(self, obj):
def to_native(self, obj):
if obj is None: if obj is None:
return [] return []
if "assigned_to" in obj: if "assigned_to" in obj:
obj['assigned_to'] = list(map(UserPkField().to_native, obj['assigned_to'])) obj['assigned_to'] = list(map(UserPkField().to_value, obj['assigned_to']))
return obj return obj
def from_native(self, data):
if data is None:
return []
if "assigned_to" in data: class TimelineDataField(Field):
data['assigned_to'] = list(map(UserPkField().from_native, data['assigned_to'])) def to_value(self, data):
return data
class TimelineDataField(serializers.WritableField):
read_only = False
def to_native(self, data):
new_data = copy.deepcopy(data) new_data = copy.deepcopy(data)
try: try:
user = cached_get_user_by_pk(new_data["user"]["id"]) user = cached_get_user_by_pk(new_data["user"]["id"])
@ -253,14 +116,3 @@ class TimelineDataField(serializers.WritableField):
except Exception: except Exception:
pass pass
return new_data return new_data
def from_native(self, data):
new_data = copy.deepcopy(data)
try:
user = cached_get_user_by_email(new_data["user"]["email"])
new_data["user"]["id"] = user.id
del new_data["user"]["email"]
except users_models.User.DoesNotExist:
pass
return new_data

107
taiga/export_import/serializers/mixins.py
View File

@ -16,56 +16,62 @@
# You should have received a copy of the GNU Affero General Public License # You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>. # along with this program. If not, see <http://www.gnu.org/licenses/>.
from django.contrib.auth import get_user_model
from django.core.exceptions import ObjectDoesNotExist from django.core.exceptions import ObjectDoesNotExist
from django.contrib.contenttypes.models import ContentType from django.contrib.contenttypes.models import ContentType
from taiga.base.api import serializers from taiga.base.api import serializers
from taiga.base.fields import Field, MethodField, DateTimeField
from taiga.projects.history import models as history_models from taiga.projects.history import models as history_models
from taiga.projects.attachments import models as attachments_models from taiga.projects.attachments import models as attachments_models
from taiga.projects.notifications import services as notifications_services
from taiga.projects.history import services as history_service from taiga.projects.history import services as history_service
from .fields import (UserRelatedField, HistoryUserField, HistoryDiffField, from .fields import (UserRelatedField, HistoryUserField, HistoryDiffField,
JsonField, HistoryValuesField, CommentField, FileField) HistoryValuesField, FileField)
class HistoryExportSerializer(serializers.ModelSerializer): class HistoryExportSerializer(serializers.LightSerializer):
user = HistoryUserField() user = HistoryUserField()
diff = HistoryDiffField(required=False) diff = HistoryDiffField()
snapshot = JsonField(required=False) snapshot = Field()
values = HistoryValuesField(required=False) values = HistoryValuesField()
comment = CommentField(required=False) comment = Field()
delete_comment_date = serializers.DateTimeField(required=False) delete_comment_date = DateTimeField()
delete_comment_user = HistoryUserField(required=False) delete_comment_user = HistoryUserField()
comment_versions = Field()
class Meta: created_at = DateTimeField()
model = history_models.HistoryEntry edit_comment_date = DateTimeField()
exclude = ("id", "comment_html", "key") is_hidden = Field()
is_snapshot = Field()
type = Field()
class HistoryExportSerializerMixin(serializers.ModelSerializer): class HistoryExportSerializerMixin(serializers.LightSerializer):
history = serializers.SerializerMethodField("get_history") history = MethodField("get_history")
def get_history(self, obj): def get_history(self, obj):
history_qs = history_service.get_history_queryset_by_model_instance(obj, history_qs = history_service.get_history_queryset_by_model_instance(
types=(history_models.HistoryType.change, history_models.HistoryType.create,)) obj,
types=(history_models.HistoryType.change, history_models.HistoryType.create,)
)
return HistoryExportSerializer(history_qs, many=True).data return HistoryExportSerializer(history_qs, many=True).data
class AttachmentExportSerializer(serializers.ModelSerializer): class AttachmentExportSerializer(serializers.LightSerializer):
owner = UserRelatedField(required=False) owner = UserRelatedField()
attached_file = FileField() attached_file = FileField()
modified_date = serializers.DateTimeField(required=False) created_date = DateTimeField()
modified_date = DateTimeField()
class Meta: description = Field()
model = attachments_models.Attachment is_deprecated = Field()
exclude = ('id', 'content_type', 'object_id', 'project') name = Field()
order = Field()
sha1 = Field()
size = Field()
class AttachmentExportSerializerMixin(serializers.ModelSerializer): class AttachmentExportSerializerMixin(serializers.LightSerializer):
attachments = serializers.SerializerMethodField("get_attachments") attachments = MethodField()
def get_attachments(self, obj): def get_attachments(self, obj):
content_type = ContentType.objects.get_for_model(obj.__class__) content_type = ContentType.objects.get_for_model(obj.__class__)
@ -74,8 +80,8 @@ class AttachmentExportSerializerMixin(serializers.ModelSerializer):
return AttachmentExportSerializer(attachments_qs, many=True).data return AttachmentExportSerializer(attachments_qs, many=True).data
class CustomAttributesValuesExportSerializerMixin(serializers.ModelSerializer): class CustomAttributesValuesExportSerializerMixin(serializers.LightSerializer):
custom_attributes_values = serializers.SerializerMethodField("get_custom_attributes_values") custom_attributes_values = MethodField("get_custom_attributes_values")
def custom_attributes_queryset(self, project): def custom_attributes_queryset(self, project):
raise NotImplementedError() raise NotImplementedError()
@ -99,43 +105,8 @@ class CustomAttributesValuesExportSerializerMixin(serializers.ModelSerializer):
return None return None
class WatcheableObjectModelSerializerMixin(serializers.ModelSerializer): class WatcheableObjectLightSerializerMixin(serializers.LightSerializer):
watchers = UserRelatedField(many=True, required=False) watchers = MethodField()
def __init__(self, *args, **kwargs): def get_watchers(self, obj):
self._watchers_field = self.base_fields.pop("watchers", None) return [user.email for user in obj.get_watchers()]
super(WatcheableObjectModelSerializerMixin, self).__init__(*args, **kwargs)
"""
watchers is not a field from the model so we need to do some magic to make it work like a normal field
It's supposed to be represented as an email list but internally it's treated like notifications.Watched instances
"""
def restore_object(self, attrs, instance=None):
watcher_field = self.fields.pop("watchers", None)
instance = super(WatcheableObjectModelSerializerMixin, self).restore_object(attrs, instance)
self._watchers = self.init_data.get("watchers", [])
return instance
def save_watchers(self):
new_watcher_emails = set(self._watchers)
old_watcher_emails = set(self.object.get_watchers().values_list("email", flat=True))
adding_watcher_emails = list(new_watcher_emails.difference(old_watcher_emails))
removing_watcher_emails = list(old_watcher_emails.difference(new_watcher_emails))
User = get_user_model()
adding_users = User.objects.filter(email__in=adding_watcher_emails)
removing_users = User.objects.filter(email__in=removing_watcher_emails)
for user in adding_users:
notifications_services.add_watcher(self.object, user)
for user in removing_users:
notifications_services.remove_watcher(self.object, user)
self.object.watchers = [user.email for user in self.object.get_watchers()]
def to_native(self, obj):
ret = super(WatcheableObjectModelSerializerMixin, self).to_native(obj)
ret["watchers"] = [user.email for user in obj.get_watchers()]
return ret

512
taiga/export_import/serializers/serializers.py
View File

@ -16,231 +16,183 @@
# You should have received a copy of the GNU Affero General Public License # You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>. # along with this program. If not, see <http://www.gnu.org/licenses/>.
from django.utils.translation import ugettext as _
from taiga.base.api import serializers from taiga.base.api import serializers
from taiga.base.fields import JsonField, PgArrayField from taiga.base.fields import Field, DateTimeField, MethodField
from taiga.base.exceptions import ValidationError
from taiga.projects import models as projects_models
from taiga.projects.custom_attributes import models as custom_attributes_models
from taiga.projects.userstories import models as userstories_models
from taiga.projects.tasks import models as tasks_models
from taiga.projects.issues import models as issues_models
from taiga.projects.milestones import models as milestones_models
from taiga.projects.wiki import models as wiki_models
from taiga.timeline import models as timeline_models
from taiga.users import models as users_models
from taiga.projects.votes import services as votes_service from taiga.projects.votes import services as votes_service
from .fields import (FileField, UserRelatedField, from .fields import (FileField, UserRelatedField, TimelineDataField,
ProjectRelatedField, ContentTypeField, SlugRelatedField)
TimelineDataField, ContentTypeField)
from .mixins import (HistoryExportSerializerMixin, from .mixins import (HistoryExportSerializerMixin,
AttachmentExportSerializerMixin, AttachmentExportSerializerMixin,
CustomAttributesValuesExportSerializerMixin, CustomAttributesValuesExportSerializerMixin,
WatcheableObjectModelSerializerMixin) WatcheableObjectLightSerializerMixin)
from .cache import (_custom_tasks_attributes_cache, from .cache import (_custom_tasks_attributes_cache,
_custom_userstories_attributes_cache, _custom_userstories_attributes_cache,
_custom_issues_attributes_cache) _custom_issues_attributes_cache)
class PointsExportSerializer(serializers.ModelSerializer): class RelatedExportSerializer(serializers.LightSerializer):
class Meta: def to_value(self, value):
model = projects_models.Points if hasattr(value, 'all'):
exclude = ('id', 'project') return super().to_value(value.all())
return super().to_value(value)
class UserStoryStatusExportSerializer(serializers.ModelSerializer): class PointsExportSerializer(RelatedExportSerializer):
class Meta: name = Field()
model = projects_models.UserStoryStatus order = Field()
exclude = ('id', 'project') value = Field()
class TaskStatusExportSerializer(serializers.ModelSerializer): class UserStoryStatusExportSerializer(RelatedExportSerializer):
class Meta: name = Field()
model = projects_models.TaskStatus slug = Field()
exclude = ('id', 'project') order = Field()
is_closed = Field()
is_archived = Field()
color = Field()
wip_limit = Field()
class IssueStatusExportSerializer(serializers.ModelSerializer): class TaskStatusExportSerializer(RelatedExportSerializer):
class Meta: name = Field()
model = projects_models.IssueStatus slug = Field()
exclude = ('id', 'project') order = Field()
is_closed = Field()
color = Field()
class PriorityExportSerializer(serializers.ModelSerializer): class IssueStatusExportSerializer(RelatedExportSerializer):
class Meta: name = Field()
model = projects_models.Priority slug = Field()
exclude = ('id', 'project') order = Field()
is_closed = Field()
color = Field()
class SeverityExportSerializer(serializers.ModelSerializer): class PriorityExportSerializer(RelatedExportSerializer):
class Meta: name = Field()
model = projects_models.Severity order = Field()
exclude = ('id', 'project') color = Field()
class IssueTypeExportSerializer(serializers.ModelSerializer): class SeverityExportSerializer(RelatedExportSerializer):
class Meta: name = Field()
model = projects_models.IssueType order = Field()
exclude = ('id', 'project') color = Field()
class RoleExportSerializer(serializers.ModelSerializer): class IssueTypeExportSerializer(RelatedExportSerializer):
permissions = PgArrayField(required=False) name = Field()
order = Field()
class Meta: color = Field()
model = users_models.Role
exclude = ('id', 'project')
class UserStoryCustomAttributeExportSerializer(serializers.ModelSerializer): class RoleExportSerializer(RelatedExportSerializer):
modified_date = serializers.DateTimeField(required=False) name = Field()
slug = Field()
class Meta: order = Field()
model = custom_attributes_models.UserStoryCustomAttribute computable = Field()
exclude = ('id', 'project') permissions = Field()
class TaskCustomAttributeExportSerializer(serializers.ModelSerializer): class UserStoryCustomAttributeExportSerializer(RelatedExportSerializer):
modified_date = serializers.DateTimeField(required=False) name = Field()
description = Field()
class Meta: type = Field()
model = custom_attributes_models.TaskCustomAttribute order = Field()
exclude = ('id', 'project') created_date = DateTimeField()
modified_date = DateTimeField()
class IssueCustomAttributeExportSerializer(serializers.ModelSerializer): class TaskCustomAttributeExportSerializer(RelatedExportSerializer):
modified_date = serializers.DateTimeField(required=False) name = Field()
description = Field()
class Meta: type = Field()
model = custom_attributes_models.IssueCustomAttribute order = Field()
exclude = ('id', 'project') created_date = DateTimeField()
modified_date = DateTimeField()
class BaseCustomAttributesValuesExportSerializer(serializers.ModelSerializer): class IssueCustomAttributeExportSerializer(RelatedExportSerializer):
attributes_values = JsonField(source="attributes_values", required=True) name = Field()
_custom_attribute_model = None description = Field()
_container_field = None type = Field()
order = Field()
created_date = DateTimeField()
modified_date = DateTimeField()
class Meta:
exclude = ("id",)
def validate_attributes_values(self, attrs, source): class BaseCustomAttributesValuesExportSerializer(RelatedExportSerializer):
# values must be a dict attributes_values = Field(required=True)
data_values = attrs.get("attributes_values", None)
if self.object:
data_values = (data_values or self.object.attributes_values)
if type(data_values) is not dict:
raise ValidationError(_("Invalid content. It must be {\"key\": \"value\",...}"))
# Values keys must be in the container object project
data_container = attrs.get(self._container_field, None)
if data_container:
project_id = data_container.project_id
elif self.object:
project_id = getattr(self.object, self._container_field).project_id
else:
project_id = None
values_ids = list(data_values.keys())
qs = self._custom_attribute_model.objects.filter(project=project_id,
id__in=values_ids)
if qs.count() != len(values_ids):
raise ValidationError(_("It contain invalid custom fields."))
return attrs
class UserStoryCustomAttributesValuesExportSerializer(BaseCustomAttributesValuesExportSerializer): class UserStoryCustomAttributesValuesExportSerializer(BaseCustomAttributesValuesExportSerializer):
_custom_attribute_model = custom_attributes_models.UserStoryCustomAttribute user_story = Field(attr="user_story.id")
_container_model = "userstories.UserStory"
_container_field = "user_story"
class Meta(BaseCustomAttributesValuesExportSerializer.Meta):
model = custom_attributes_models.UserStoryCustomAttributesValues
class TaskCustomAttributesValuesExportSerializer(BaseCustomAttributesValuesExportSerializer): class TaskCustomAttributesValuesExportSerializer(BaseCustomAttributesValuesExportSerializer):
_custom_attribute_model = custom_attributes_models.TaskCustomAttribute task = Field(attr="task.id")
_container_field = "task"
class Meta(BaseCustomAttributesValuesExportSerializer.Meta):
model = custom_attributes_models.TaskCustomAttributesValues
class IssueCustomAttributesValuesExportSerializer(BaseCustomAttributesValuesExportSerializer): class IssueCustomAttributesValuesExportSerializer(BaseCustomAttributesValuesExportSerializer):
_custom_attribute_model = custom_attributes_models.IssueCustomAttribute issue = Field(attr="issue.id")
_container_field = "issue"
class Meta(BaseCustomAttributesValuesExportSerializer.Meta):
model = custom_attributes_models.IssueCustomAttributesValues
class MembershipExportSerializer(serializers.ModelSerializer): class MembershipExportSerializer(RelatedExportSerializer):
user = UserRelatedField(required=False) user = UserRelatedField()
role = ProjectRelatedField(slug_field="name") role = SlugRelatedField(slug_field="name")
invited_by = UserRelatedField(required=False) invited_by = UserRelatedField()
is_admin = Field()
class Meta: email = Field()
model = projects_models.Membership created_at = DateTimeField()
exclude = ('id', 'project', 'token') invitation_extra_text = Field()
user_order = Field()
def full_clean(self, instance):
return instance
class RolePointsExportSerializer(serializers.ModelSerializer): class RolePointsExportSerializer(RelatedExportSerializer):
role = ProjectRelatedField(slug_field="name") role = SlugRelatedField(slug_field="name")
points = ProjectRelatedField(slug_field="name") points = SlugRelatedField(slug_field="name")
class Meta:
model = userstories_models.RolePoints
exclude = ('id', 'user_story')
class MilestoneExportSerializer(WatcheableObjectModelSerializerMixin): class MilestoneExportSerializer(WatcheableObjectLightSerializerMixin, RelatedExportSerializer):
owner = UserRelatedField(required=False) name = Field()
modified_date = serializers.DateTimeField(required=False) owner = UserRelatedField()
estimated_start = serializers.DateField(required=False) created_date = DateTimeField()
estimated_finish = serializers.DateField(required=False) modified_date = DateTimeField()
estimated_start = Field()
def __init__(self, *args, **kwargs): estimated_finish = Field()
project = kwargs.pop('project', None) slug = Field()
super(MilestoneExportSerializer, self).__init__(*args, **kwargs) closed = Field()
if project: disponibility = Field()
self.project = project order = Field()
def validate_name(self, attrs, source):
"""
Check the milestone name is not duplicated in the project
"""
name = attrs[source]
qs = self.project.milestones.filter(name=name)
if qs.exists():
raise ValidationError(_("Name duplicated for the project"))
return attrs
class Meta:
model = milestones_models.Milestone
exclude = ('id', 'project')
class TaskExportSerializer(CustomAttributesValuesExportSerializerMixin, HistoryExportSerializerMixin, class TaskExportSerializer(CustomAttributesValuesExportSerializerMixin,
AttachmentExportSerializerMixin, WatcheableObjectModelSerializerMixin): HistoryExportSerializerMixin,
owner = UserRelatedField(required=False) AttachmentExportSerializerMixin,
status = ProjectRelatedField(slug_field="name") WatcheableObjectLightSerializerMixin,
user_story = ProjectRelatedField(slug_field="ref", required=False) RelatedExportSerializer):
milestone = ProjectRelatedField(slug_field="name", required=False) owner = UserRelatedField()
assigned_to = UserRelatedField(required=False) status = SlugRelatedField(slug_field="name")
modified_date = serializers.DateTimeField(required=False) user_story = SlugRelatedField(slug_field="ref")
milestone = SlugRelatedField(slug_field="name")
class Meta: assigned_to = UserRelatedField()
model = tasks_models.Task modified_date = DateTimeField()
exclude = ('id', 'project') created_date = DateTimeField()
finished_date = DateTimeField()
ref = Field()
subject = Field()
us_order = Field()
taskboard_order = Field()
description = Field()
is_iocaine = Field()
external_reference = Field()
version = Field()
blocked_note = Field()
is_blocked = Field()
tags = Field()
def custom_attributes_queryset(self, project): def custom_attributes_queryset(self, project):
if project.id not in _custom_tasks_attributes_cache: if project.id not in _custom_tasks_attributes_cache:
@ -248,19 +200,35 @@ class TaskExportSerializer(CustomAttributesValuesExportSerializerMixin, HistoryE
return _custom_tasks_attributes_cache[project.id] return _custom_tasks_attributes_cache[project.id]
class UserStoryExportSerializer(CustomAttributesValuesExportSerializerMixin, HistoryExportSerializerMixin, class UserStoryExportSerializer(CustomAttributesValuesExportSerializerMixin,
AttachmentExportSerializerMixin, WatcheableObjectModelSerializerMixin): HistoryExportSerializerMixin,
role_points = RolePointsExportSerializer(many=True, required=False) AttachmentExportSerializerMixin,
owner = UserRelatedField(required=False) WatcheableObjectLightSerializerMixin,
assigned_to = UserRelatedField(required=False) RelatedExportSerializer):
status = ProjectRelatedField(slug_field="name") role_points = RolePointsExportSerializer(many=True)
milestone = ProjectRelatedField(slug_field="name", required=False) owner = UserRelatedField()
modified_date = serializers.DateTimeField(required=False) assigned_to = UserRelatedField()
generated_from_issue = ProjectRelatedField(slug_field="ref", required=False) status = SlugRelatedField(slug_field="name")
milestone = SlugRelatedField(slug_field="name")
class Meta: modified_date = DateTimeField()
model = userstories_models.UserStory created_date = DateTimeField()
exclude = ('id', 'project', 'points', 'tasks') finish_date = DateTimeField()
generated_from_issue = SlugRelatedField(slug_field="ref")
ref = Field()
is_closed = Field()
backlog_order = Field()
sprint_order = Field()
kanban_order = Field()
subject = Field()
description = Field()
client_requirement = Field()
team_requirement = Field()
external_reference = Field()
tribe_gig = Field()
version = Field()
blocked_note = Field()
is_blocked = Field()
tags = Field()
def custom_attributes_queryset(self, project): def custom_attributes_queryset(self, project):
if project.id not in _custom_userstories_attributes_cache: if project.id not in _custom_userstories_attributes_cache:
@ -270,21 +238,31 @@ class UserStoryExportSerializer(CustomAttributesValuesExportSerializerMixin, His
return _custom_userstories_attributes_cache[project.id] return _custom_userstories_attributes_cache[project.id]
class IssueExportSerializer(CustomAttributesValuesExportSerializerMixin, HistoryExportSerializerMixin, class IssueExportSerializer(CustomAttributesValuesExportSerializerMixin,
AttachmentExportSerializerMixin, WatcheableObjectModelSerializerMixin): HistoryExportSerializerMixin,
owner = UserRelatedField(required=False) AttachmentExportSerializerMixin,
status = ProjectRelatedField(slug_field="name") WatcheableObjectLightSerializerMixin,
assigned_to = UserRelatedField(required=False) RelatedExportSerializer):
priority = ProjectRelatedField(slug_field="name") owner = UserRelatedField()
severity = ProjectRelatedField(slug_field="name") status = SlugRelatedField(slug_field="name")
type = ProjectRelatedField(slug_field="name") assigned_to = UserRelatedField()
milestone = ProjectRelatedField(slug_field="name", required=False) priority = SlugRelatedField(slug_field="name")
votes = serializers.SerializerMethodField("get_votes") severity = SlugRelatedField(slug_field="name")
modified_date = serializers.DateTimeField(required=False) type = SlugRelatedField(slug_field="name")
milestone = SlugRelatedField(slug_field="name")
votes = MethodField("get_votes")
modified_date = DateTimeField()
created_date = DateTimeField()
finished_date = DateTimeField()
class Meta: ref = Field()
model = issues_models.Issue subject = Field()
exclude = ('id', 'project') description = Field()
external_reference = Field()
version = Field()
blocked_note = Field()
is_blocked = Field()
tags = Field()
def get_votes(self, obj): def get_votes(self, obj):
return [x.email for x in votes_service.get_voters(obj)] return [x.email for x in votes_service.get_voters(obj)]
@ -295,65 +273,93 @@ class IssueExportSerializer(CustomAttributesValuesExportSerializerMixin, History
return _custom_issues_attributes_cache[project.id] return _custom_issues_attributes_cache[project.id]
class WikiPageExportSerializer(HistoryExportSerializerMixin, AttachmentExportSerializerMixin, class WikiPageExportSerializer(HistoryExportSerializerMixin,
WatcheableObjectModelSerializerMixin): AttachmentExportSerializerMixin,
owner = UserRelatedField(required=False) WatcheableObjectLightSerializerMixin,
last_modifier = UserRelatedField(required=False) RelatedExportSerializer):
modified_date = serializers.DateTimeField(required=False) slug = Field()
owner = UserRelatedField()
class Meta: last_modifier = UserRelatedField()
model = wiki_models.WikiPage modified_date = DateTimeField()
exclude = ('id', 'project') created_date = DateTimeField()
content = Field()
version = Field()
class WikiLinkExportSerializer(serializers.ModelSerializer): class WikiLinkExportSerializer(RelatedExportSerializer):
class Meta: title = Field()
model = wiki_models.WikiLink href = Field()
exclude = ('id', 'project') order = Field()
class TimelineExportSerializer(serializers.ModelSerializer): class TimelineExportSerializer(RelatedExportSerializer):
data = TimelineDataField() data = TimelineDataField()
data_content_type = ContentTypeField() data_content_type = ContentTypeField()
event_type = Field()
class Meta: created = DateTimeField()
model = timeline_models.Timeline
exclude = ('id', 'project', 'namespace', 'object_id', 'content_type')
class ProjectExportSerializer(WatcheableObjectModelSerializerMixin): class ProjectExportSerializer(WatcheableObjectLightSerializerMixin):
logo = FileField(required=False) name = Field()
anon_permissions = PgArrayField(required=False) slug = Field()
public_permissions = PgArrayField(required=False) description = Field()
modified_date = serializers.DateTimeField(required=False) created_date = DateTimeField()
roles = RoleExportSerializer(many=True, required=False) logo = FileField()
owner = UserRelatedField(required=False) total_milestones = Field()
memberships = MembershipExportSerializer(many=True, required=False) total_story_points = Field()
points = PointsExportSerializer(many=True, required=False) is_backlog_activated = Field()
us_statuses = UserStoryStatusExportSerializer(many=True, required=False) is_kanban_activated = Field()
task_statuses = TaskStatusExportSerializer(many=True, required=False) is_wiki_activated = Field()
issue_types = IssueTypeExportSerializer(many=True, required=False) is_issues_activated = Field()
issue_statuses = IssueStatusExportSerializer(many=True, required=False) videoconferences = Field()
priorities = PriorityExportSerializer(many=True, required=False) videoconferences_extra_data = Field()
severities = SeverityExportSerializer(many=True, required=False) creation_template = SlugRelatedField(slug_field="slug")
tags_colors = JsonField(required=False) is_private = Field()
default_points = serializers.SlugRelatedField(slug_field="name", required=False) is_featured = Field()
default_us_status = serializers.SlugRelatedField(slug_field="name", required=False) is_looking_for_people = Field()
default_task_status = serializers.SlugRelatedField(slug_field="name", required=False) looking_for_people_note = Field()
default_priority = serializers.SlugRelatedField(slug_field="name", required=False) userstories_csv_uuid = Field()
default_severity = serializers.SlugRelatedField(slug_field="name", required=False) tasks_csv_uuid = Field()
default_issue_status = serializers.SlugRelatedField(slug_field="name", required=False) issues_csv_uuid = Field()
default_issue_type = serializers.SlugRelatedField(slug_field="name", required=False) transfer_token = Field()
userstorycustomattributes = UserStoryCustomAttributeExportSerializer(many=True, required=False) blocked_code = Field()
taskcustomattributes = TaskCustomAttributeExportSerializer(many=True, required=False) totals_updated_datetime = DateTimeField()
issuecustomattributes = IssueCustomAttributeExportSerializer(many=True, required=False) total_fans = Field()
user_stories = UserStoryExportSerializer(many=True, required=False) total_fans_last_week = Field()
tasks = TaskExportSerializer(many=True, required=False) total_fans_last_month = Field()
milestones = MilestoneExportSerializer(many=True, required=False) total_fans_last_year = Field()
issues = IssueExportSerializer(many=True, required=False) total_activity = Field()
wiki_links = WikiLinkExportSerializer(many=True, required=False) total_activity_last_week = Field()
wiki_pages = WikiPageExportSerializer(many=True, required=False) total_activity_last_month = Field()
total_activity_last_year = Field()
class Meta: anon_permissions = Field()
model = projects_models.Project public_permissions = Field()
exclude = ('id', 'creation_template', 'members') modified_date = DateTimeField()
roles = RoleExportSerializer(many=True)
owner = UserRelatedField()
memberships = MembershipExportSerializer(many=True)
points = PointsExportSerializer(many=True)
us_statuses = UserStoryStatusExportSerializer(many=True)
task_statuses = TaskStatusExportSerializer(many=True)
issue_types = IssueTypeExportSerializer(many=True)
issue_statuses = IssueStatusExportSerializer(many=True)
priorities = PriorityExportSerializer(many=True)
severities = SeverityExportSerializer(many=True)
tags_colors = Field()
default_points = SlugRelatedField(slug_field="name")
default_us_status = SlugRelatedField(slug_field="name")
default_task_status = SlugRelatedField(slug_field="name")
default_priority = SlugRelatedField(slug_field="name")
default_severity = SlugRelatedField(slug_field="name")
default_issue_status = SlugRelatedField(slug_field="name")
default_issue_type = SlugRelatedField(slug_field="name")
userstorycustomattributes = UserStoryCustomAttributeExportSerializer(many=True)
taskcustomattributes = TaskCustomAttributeExportSerializer(many=True)
issuecustomattributes = IssueCustomAttributeExportSerializer(many=True)
user_stories = UserStoryExportSerializer(many=True)
tasks = TaskExportSerializer(many=True)
milestones = MilestoneExportSerializer(many=True)
issues = IssueExportSerializer(many=True)
wiki_links = WikiLinkExportSerializer(many=True)
wiki_pages = WikiPageExportSerializer(many=True)
tags = Field()

63
taiga/export_import/services/render.py
View File

@ -19,13 +19,10 @@
# This makes all code that import services works and # This makes all code that import services works and
# is not the baddest practice ;) # is not the baddest practice ;)
import base64
import gc import gc
import os
from django.core.files.storage import default_storage
from taiga.base.utils import json from taiga.base.utils import json
from taiga.base.fields import MethodField
from taiga.timeline.service import get_project_timeline from taiga.timeline.service import get_project_timeline
from taiga.base.api.fields import get_component from taiga.base.api.fields import get_component
@ -37,31 +34,29 @@ def render_project(project, outfile, chunk_size = 8190):
outfile.write(b'{\n') outfile.write(b'{\n')
first_field = True first_field = True
for field_name in serializer.fields.keys(): for field_name in serializer._field_map.keys():
# Avoid writing "," in the last element # Avoid writing "," in the last element
if not first_field: if not first_field:
outfile.write(b",\n") outfile.write(b",\n")
else: else:
first_field = False first_field = False
field = serializer.fields.get(field_name) field = serializer._field_map.get(field_name)
field.initialize(parent=serializer, field_name=field_name) # field.initialize(parent=serializer, field_name=field_name)
# These four "special" fields hava attachments so we use them in a special way # These four "special" fields hava attachments so we use them in a special way
if field_name in ["wiki_pages", "user_stories", "tasks", "issues"]: if field_name in ["wiki_pages", "user_stories", "tasks", "issues"]:
value = get_component(project, field_name) value = get_component(project, field_name)
if field_name != "wiki_pages": if field_name != "wiki_pages":
value = value.select_related('owner', 'status', 'milestone', 'project', 'assigned_to', 'custom_attributes_values') value = value.select_related('owner', 'status', 'milestone',
'project', 'assigned_to',
'custom_attributes_values')
if field_name == "issues": if field_name == "issues":
value = value.select_related('severity', 'priority', 'type') value = value.select_related('severity', 'priority', 'type')
value = value.prefetch_related('history_entry', 'attachments') value = value.prefetch_related('history_entry', 'attachments')
outfile.write('"{}": [\n'.format(field_name).encode()) outfile.write('"{}": [\n'.format(field_name).encode())
attachments_field = field.fields.pop("attachments", None)
if attachments_field:
attachments_field.initialize(parent=field, field_name="attachments")
first_item = True first_item = True
for item in value.iterator(): for item in value.iterator():
# Avoid writing "," in the last element # Avoid writing "," in the last element
@ -70,47 +65,18 @@ def render_project(project, outfile, chunk_size = 8190):
else: else:
first_item = False first_item = False
field.many = False
dumped_value = json.dumps(field.to_native(item)) dumped_value = json.dumps(field.to_value(item))
writing_value = dumped_value[:-1]+ ',\n "attachments": [\n'
outfile.write(writing_value.encode())
first_attachment = True
for attachment in item.attachments.iterator():
# Avoid writing "," in the last element
if not first_attachment:
outfile.write(b",\n")
else:
first_attachment = False
# Write all the data expect the serialized file
attachment_serializer = serializers.AttachmentExportSerializer(instance=attachment)
attached_file_serializer = attachment_serializer.fields.pop("attached_file")
dumped_value = json.dumps(attachment_serializer.data)
dumped_value = dumped_value[:-1] + ',\n "attached_file":{\n "data":"'
outfile.write(dumped_value.encode()) outfile.write(dumped_value.encode())
# We write the attached_files by chunks so the memory used is not increased
attachment_file = attachment.attached_file
if default_storage.exists(attachment_file.name):
with default_storage.open(attachment_file.name) as f:
while True:
bin_data = f.read(chunk_size)
if not bin_data:
break
b64_data = base64.b64encode(bin_data)
outfile.write(b64_data)
outfile.write('", \n "name":"{}"}}\n}}'.format(
os.path.basename(attachment_file.name)).encode())
outfile.write(b']}')
outfile.flush() outfile.flush()
gc.collect() gc.collect()
outfile.write(b']') outfile.write(b']')
else: else:
value = field.field_to_native(project, field_name) if isinstance(field, MethodField):
value = field.as_getter(field_name, serializers.ProjectExportSerializer)(serializer, project)
else:
attr = getattr(project, field_name)
value = field.to_value(attr)
outfile.write('"{}": {}'.format(field_name, json.dumps(value)).encode()) outfile.write('"{}": {}'.format(field_name, json.dumps(value)).encode())
# Generate the timeline # Generate the timeline
@ -127,4 +93,3 @@ def render_project(project, outfile, chunk_size = 8190):
outfile.write(dumped_value.encode()) outfile.write(dumped_value.encode())
outfile.write(b']}\n') outfile.write(b']}\n')

383
taiga/export_import/services/store.py
View File

@ -39,7 +39,7 @@ from taiga.timeline.service import build_project_namespace
from taiga.users import services as users_service from taiga.users import services as users_service
from .. import exceptions as err from .. import exceptions as err
from .. import serializers from .. import validators
######################################################################## ########################################################################
@ -90,13 +90,13 @@ def store_project(data):
if key not in excluded_fields: if key not in excluded_fields:
project_data[key] = value project_data[key] = value
serialized = serializers.ProjectExportSerializer(data=project_data) validator = validators.ProjectExportValidator(data=project_data)
if serialized.is_valid(): if validator.is_valid():
serialized.object._importing = True validator.object._importing = True
serialized.object.save() validator.object.save()
serialized.save_watchers() validator.save_watchers()
return serialized return validator
add_errors("project", serialized.errors) add_errors("project", validator.errors)
return None return None
@ -133,54 +133,55 @@ def _store_custom_attributes_values(obj, data_values, obj_field, serializer_clas
def _store_attachment(project, obj, attachment): def _store_attachment(project, obj, attachment):
serialized = serializers.AttachmentExportSerializer(data=attachment) validator = validators.AttachmentExportValidator(data=attachment)
if serialized.is_valid(): if validator.is_valid():
serialized.object.content_type = ContentType.objects.get_for_model(obj.__class__) validator.object.content_type = ContentType.objects.get_for_model(obj.__class__)
serialized.object.object_id = obj.id validator.object.object_id = obj.id
serialized.object.project = project validator.object.project = project
if serialized.object.owner is None: if validator.object.owner is None:
serialized.object.owner = serialized.object.project.owner validator.object.owner = validator.object.project.owner
serialized.object._importing = True validator.object._importing = True
serialized.object.size = serialized.object.attached_file.size validator.object.size = validator.object.attached_file.size
serialized.object.name = os.path.basename(serialized.object.attached_file.name) validator.object.name = os.path.basename(validator.object.attached_file.name)
serialized.save() validator.save()
return serialized return validator
add_errors("attachments", serialized.errors) add_errors("attachments", validator.errors)
return serialized return validator
def _store_history(project, obj, history): def _store_history(project, obj, history):
serialized = serializers.HistoryExportSerializer(data=history, context={"project": project}) validator = validators.HistoryExportValidator(data=history, context={"project": project})
if serialized.is_valid(): if validator.is_valid():
serialized.object.key = make_key_from_model_object(obj) validator.object.key = make_key_from_model_object(obj)
if serialized.object.diff is None: if validator.object.diff is None:
serialized.object.diff = [] validator.object.diff = []
serialized.object._importing = True validator.object.project_id = project.id
serialized.save() validator.object._importing = True
return serialized validator.save()
add_errors("history", serialized.errors) return validator
return serialized add_errors("history", validator.errors)
return validator
## ROLES ## ROLES
def _store_role(project, role): def _store_role(project, role):
serialized = serializers.RoleExportSerializer(data=role) validator = validators.RoleExportValidator(data=role)
if serialized.is_valid(): if validator.is_valid():
serialized.object.project = project validator.object.project = project
serialized.object._importing = True validator.object._importing = True
serialized.save() validator.save()
return serialized return validator
add_errors("roles", serialized.errors) add_errors("roles", validator.errors)
return None return None
def store_roles(project, data): def store_roles(project, data):
results = [] results = []
for role in data.get("roles", []): for role in data.get("roles", []):
serialized = _store_role(project, role) validator = _store_role(project, role)
if serialized: if validator:
results.append(serialized) results.append(validator)
return results return results
@ -188,17 +189,17 @@ def store_roles(project, data):
## MEMGERSHIPS ## MEMGERSHIPS
def _store_membership(project, membership): def _store_membership(project, membership):
serialized = serializers.MembershipExportSerializer(data=membership, context={"project": project}) validator = validators.MembershipExportValidator(data=membership, context={"project": project})
if serialized.is_valid(): if validator.is_valid():
serialized.object.project = project validator.object.project = project
serialized.object._importing = True validator.object._importing = True
serialized.object.token = str(uuid.uuid1()) validator.object.token = str(uuid.uuid1())
serialized.object.user = find_invited_user(serialized.object.email, validator.object.user = find_invited_user(validator.object.email,
default=serialized.object.user) default=validator.object.user)
serialized.save() validator.save()
return serialized return validator
add_errors("memberships", serialized.errors) add_errors("memberships", validator.errors)
return None return None
@ -212,13 +213,13 @@ def store_memberships(project, data):
## PROJECT ATTRIBUTES ## PROJECT ATTRIBUTES
def _store_project_attribute_value(project, data, field, serializer): def _store_project_attribute_value(project, data, field, serializer):
serialized = serializer(data=data) validator = serializer(data=data)
if serialized.is_valid(): if validator.is_valid():
serialized.object.project = project validator.object.project = project
serialized.object._importing = True validator.object._importing = True
serialized.save() validator.save()
return serialized.object return validator.object
add_errors(field, serialized.errors) add_errors(field, validator.errors)
return None return None
@ -253,13 +254,13 @@ def store_default_project_attributes_values(project, data):
## CUSTOM ATTRIBUTES ## CUSTOM ATTRIBUTES
def _store_custom_attribute(project, data, field, serializer): def _store_custom_attribute(project, data, field, serializer):
serialized = serializer(data=data) validator = serializer(data=data)
if serialized.is_valid(): if validator.is_valid():
serialized.object.project = project validator.object.project = project
serialized.object._importing = True validator.object._importing = True
serialized.save() validator.save()
return serialized.object return validator.object
add_errors(field, serialized.errors) add_errors(field, validator.errors)
return None return None
@ -273,19 +274,19 @@ def store_custom_attributes(project, data, field, serializer):
## MILESTONE ## MILESTONE
def store_milestone(project, milestone): def store_milestone(project, milestone):
serialized = serializers.MilestoneExportSerializer(data=milestone, project=project) validator = validators.MilestoneExportValidator(data=milestone, project=project)
if serialized.is_valid(): if validator.is_valid():
serialized.object.project = project validator.object.project = project
serialized.object._importing = True validator.object._importing = True
serialized.save() validator.save()
serialized.save_watchers() validator.save_watchers()
for task_without_us in milestone.get("tasks_without_us", []): for task_without_us in milestone.get("tasks_without_us", []):
task_without_us["user_story"] = None task_without_us["user_story"] = None
store_task(project, task_without_us) store_task(project, task_without_us)
return serialized return validator
add_errors("milestones", serialized.errors) add_errors("milestones", validator.errors)
return None return None
@ -300,20 +301,20 @@ def store_milestones(project, data):
## USER STORIES ## USER STORIES
def _store_role_point(project, us, role_point): def _store_role_point(project, us, role_point):
serialized = serializers.RolePointsExportSerializer(data=role_point, context={"project": project}) validator = validators.RolePointsExportValidator(data=role_point, context={"project": project})
if serialized.is_valid(): if validator.is_valid():
try: try:
existing_role_point = us.role_points.get(role=serialized.object.role) existing_role_point = us.role_points.get(role=validator.object.role)
existing_role_point.points = serialized.object.points existing_role_point.points = validator.object.points
existing_role_point.save() existing_role_point.save()
return existing_role_point return existing_role_point
except RolePoints.DoesNotExist: except RolePoints.DoesNotExist:
serialized.object.user_story = us validator.object.user_story = us
serialized.save() validator.save()
return serialized.object return validator.object
add_errors("role_points", serialized.errors) add_errors("role_points", validator.errors)
return None return None
def store_user_story(project, data): def store_user_story(project, data):
@ -322,51 +323,51 @@ def store_user_story(project, data):
us_data = {key: value for key, value in data.items() if key not in us_data = {key: value for key, value in data.items() if key not in
["role_points", "custom_attributes_values"]} ["role_points", "custom_attributes_values"]}
serialized = serializers.UserStoryExportSerializer(data=us_data, context={"project": project}) validator = validators.UserStoryExportValidator(data=us_data, context={"project": project})
if serialized.is_valid(): if validator.is_valid():
serialized.object.project = project validator.object.project = project
if serialized.object.owner is None: if validator.object.owner is None:
serialized.object.owner = serialized.object.project.owner validator.object.owner = validator.object.project.owner
serialized.object._importing = True validator.object._importing = True
serialized.object._not_notify = True validator.object._not_notify = True
serialized.save() validator.save()
serialized.save_watchers() validator.save_watchers()
if serialized.object.ref: if validator.object.ref:
sequence_name = refs.make_sequence_name(project) sequence_name = refs.make_sequence_name(project)
if not seq.exists(sequence_name): if not seq.exists(sequence_name):
seq.create(sequence_name) seq.create(sequence_name)
seq.set_max(sequence_name, serialized.object.ref) seq.set_max(sequence_name, validator.object.ref)
else: else:
serialized.object.ref, _ = refs.make_reference(serialized.object, project) validator.object.ref, _ = refs.make_reference(validator.object, project)
serialized.object.save() validator.object.save()
for us_attachment in data.get("attachments", []): for us_attachment in data.get("attachments", []):
_store_attachment(project, serialized.object, us_attachment) _store_attachment(project, validator.object, us_attachment)
for role_point in data.get("role_points", []): for role_point in data.get("role_points", []):
_store_role_point(project, serialized.object, role_point) _store_role_point(project, validator.object, role_point)
history_entries = data.get("history", []) history_entries = data.get("history", [])
for history in history_entries: for history in history_entries:
_store_history(project, serialized.object, history) _store_history(project, validator.object, history)
if not history_entries: if not history_entries:
take_snapshot(serialized.object, user=serialized.object.owner) take_snapshot(validator.object, user=validator.object.owner)
custom_attributes_values = data.get("custom_attributes_values", None) custom_attributes_values = data.get("custom_attributes_values", None)
if custom_attributes_values: if custom_attributes_values:
custom_attributes = serialized.object.project.userstorycustomattributes.all().values('id', 'name') custom_attributes = validator.object.project.userstorycustomattributes.all().values('id', 'name')
custom_attributes_values = _use_id_instead_name_as_key_in_custom_attributes_values( custom_attributes_values = _use_id_instead_name_as_key_in_custom_attributes_values(
custom_attributes, custom_attributes_values) custom_attributes, custom_attributes_values)
_store_custom_attributes_values(serialized.object, custom_attributes_values, _store_custom_attributes_values(validator.object, custom_attributes_values,
"user_story", serializers.UserStoryCustomAttributesValuesExportSerializer) "user_story", validators.UserStoryCustomAttributesValuesExportValidator)
return serialized return validator
add_errors("user_stories", serialized.errors) add_errors("user_stories", validator.errors)
return None return None
@ -384,47 +385,47 @@ def store_task(project, data):
if "status" not in data and project.default_task_status: if "status" not in data and project.default_task_status:
data["status"] = project.default_task_status.name data["status"] = project.default_task_status.name
serialized = serializers.TaskExportSerializer(data=data, context={"project": project}) validator = validators.TaskExportValidator(data=data, context={"project": project})
if serialized.is_valid(): if validator.is_valid():
serialized.object.project = project validator.object.project = project
if serialized.object.owner is None: if validator.object.owner is None:
serialized.object.owner = serialized.object.project.owner validator.object.owner = validator.object.project.owner
serialized.object._importing = True validator.object._importing = True
serialized.object._not_notify = True validator.object._not_notify = True
serialized.save() validator.save()
serialized.save_watchers() validator.save_watchers()
if serialized.object.ref: if validator.object.ref:
sequence_name = refs.make_sequence_name(project) sequence_name = refs.make_sequence_name(project)
if not seq.exists(sequence_name): if not seq.exists(sequence_name):
seq.create(sequence_name) seq.create(sequence_name)
seq.set_max(sequence_name, serialized.object.ref) seq.set_max(sequence_name, validator.object.ref)
else: else:
serialized.object.ref, _ = refs.make_reference(serialized.object, project) validator.object.ref, _ = refs.make_reference(validator.object, project)
serialized.object.save() validator.object.save()
for task_attachment in data.get("attachments", []): for task_attachment in data.get("attachments", []):
_store_attachment(project, serialized.object, task_attachment) _store_attachment(project, validator.object, task_attachment)
history_entries = data.get("history", []) history_entries = data.get("history", [])
for history in history_entries: for history in history_entries:
_store_history(project, serialized.object, history) _store_history(project, validator.object, history)
if not history_entries: if not history_entries:
take_snapshot(serialized.object, user=serialized.object.owner) take_snapshot(validator.object, user=validator.object.owner)
custom_attributes_values = data.get("custom_attributes_values", None) custom_attributes_values = data.get("custom_attributes_values", None)
if custom_attributes_values: if custom_attributes_values:
custom_attributes = serialized.object.project.taskcustomattributes.all().values('id', 'name') custom_attributes = validator.object.project.taskcustomattributes.all().values('id', 'name')
custom_attributes_values = _use_id_instead_name_as_key_in_custom_attributes_values( custom_attributes_values = _use_id_instead_name_as_key_in_custom_attributes_values(
custom_attributes, custom_attributes_values) custom_attributes, custom_attributes_values)
_store_custom_attributes_values(serialized.object, custom_attributes_values, _store_custom_attributes_values(validator.object, custom_attributes_values,
"task", serializers.TaskCustomAttributesValuesExportSerializer) "task", validators.TaskCustomAttributesValuesExportValidator)
return serialized return validator
add_errors("tasks", serialized.errors) add_errors("tasks", validator.errors)
return None return None
@ -439,7 +440,7 @@ def store_tasks(project, data):
## ISSUES ## ISSUES
def store_issue(project, data): def store_issue(project, data):
serialized = serializers.IssueExportSerializer(data=data, context={"project": project}) validator = validators.IssueExportValidator(data=data, context={"project": project})
if "type" not in data and project.default_issue_type: if "type" not in data and project.default_issue_type:
data["type"] = project.default_issue_type.name data["type"] = project.default_issue_type.name
@ -453,46 +454,46 @@ def store_issue(project, data):
if "severity" not in data and project.default_severity: if "severity" not in data and project.default_severity:
data["severity"] = project.default_severity.name data["severity"] = project.default_severity.name
if serialized.is_valid(): if validator.is_valid():
serialized.object.project = project validator.object.project = project
if serialized.object.owner is None: if validator.object.owner is None:
serialized.object.owner = serialized.object.project.owner validator.object.owner = validator.object.project.owner
serialized.object._importing = True validator.object._importing = True
serialized.object._not_notify = True validator.object._not_notify = True
serialized.save() validator.save()
serialized.save_watchers() validator.save_watchers()
if serialized.object.ref: if validator.object.ref:
sequence_name = refs.make_sequence_name(project) sequence_name = refs.make_sequence_name(project)
if not seq.exists(sequence_name): if not seq.exists(sequence_name):
seq.create(sequence_name) seq.create(sequence_name)
seq.set_max(sequence_name, serialized.object.ref) seq.set_max(sequence_name, validator.object.ref)
else: else:
serialized.object.ref, _ = refs.make_reference(serialized.object, project) validator.object.ref, _ = refs.make_reference(validator.object, project)
serialized.object.save() validator.object.save()
for attachment in data.get("attachments", []): for attachment in data.get("attachments", []):
_store_attachment(project, serialized.object, attachment) _store_attachment(project, validator.object, attachment)
history_entries = data.get("history", []) history_entries = data.get("history", [])
for history in history_entries: for history in history_entries:
_store_history(project, serialized.object, history) _store_history(project, validator.object, history)
if not history_entries: if not history_entries:
take_snapshot(serialized.object, user=serialized.object.owner) take_snapshot(validator.object, user=validator.object.owner)
custom_attributes_values = data.get("custom_attributes_values", None) custom_attributes_values = data.get("custom_attributes_values", None)
if custom_attributes_values: if custom_attributes_values:
custom_attributes = serialized.object.project.issuecustomattributes.all().values('id', 'name') custom_attributes = validator.object.project.issuecustomattributes.all().values('id', 'name')
custom_attributes_values = _use_id_instead_name_as_key_in_custom_attributes_values( custom_attributes_values = _use_id_instead_name_as_key_in_custom_attributes_values(
custom_attributes, custom_attributes_values) custom_attributes, custom_attributes_values)
_store_custom_attributes_values(serialized.object, custom_attributes_values, _store_custom_attributes_values(validator.object, custom_attributes_values,
"issue", serializers.IssueCustomAttributesValuesExportSerializer) "issue", validators.IssueCustomAttributesValuesExportValidator)
return serialized return validator
add_errors("issues", serialized.errors) add_errors("issues", validator.errors)
return None return None
@ -507,29 +508,29 @@ def store_issues(project, data):
def store_wiki_page(project, wiki_page): def store_wiki_page(project, wiki_page):
wiki_page["slug"] = slugify(unidecode(wiki_page.get("slug", ""))) wiki_page["slug"] = slugify(unidecode(wiki_page.get("slug", "")))
serialized = serializers.WikiPageExportSerializer(data=wiki_page) validator = validators.WikiPageExportValidator(data=wiki_page)
if serialized.is_valid(): if validator.is_valid():
serialized.object.project = project validator.object.project = project
if serialized.object.owner is None: if validator.object.owner is None:
serialized.object.owner = serialized.object.project.owner validator.object.owner = validator.object.project.owner
serialized.object._importing = True validator.object._importing = True
serialized.object._not_notify = True validator.object._not_notify = True
serialized.save() validator.save()
serialized.save_watchers() validator.save_watchers()
for attachment in wiki_page.get("attachments", []): for attachment in wiki_page.get("attachments", []):
_store_attachment(project, serialized.object, attachment) _store_attachment(project, validator.object, attachment)
history_entries = wiki_page.get("history", []) history_entries = wiki_page.get("history", [])
for history in history_entries: for history in history_entries:
_store_history(project, serialized.object, history) _store_history(project, validator.object, history)
if not history_entries: if not history_entries:
take_snapshot(serialized.object, user=serialized.object.owner) take_snapshot(validator.object, user=validator.object.owner)
return serialized return validator
add_errors("wiki_pages", serialized.errors) add_errors("wiki_pages", validator.errors)
return None return None
@ -543,14 +544,14 @@ def store_wiki_pages(project, data):
## WIKI LINKS ## WIKI LINKS
def store_wiki_link(project, wiki_link): def store_wiki_link(project, wiki_link):
serialized = serializers.WikiLinkExportSerializer(data=wiki_link) validator = validators.WikiLinkExportValidator(data=wiki_link)
if serialized.is_valid(): if validator.is_valid():
serialized.object.project = project validator.object.project = project
serialized.object._importing = True validator.object._importing = True
serialized.save() validator.save()
return serialized return validator
add_errors("wiki_links", serialized.errors) add_errors("wiki_links", validator.errors)
return None return None
@ -572,17 +573,17 @@ def store_tags_colors(project, data):
## TIMELINE ## TIMELINE
def _store_timeline_entry(project, timeline): def _store_timeline_entry(project, timeline):
serialized = serializers.TimelineExportSerializer(data=timeline, context={"project": project}) validator = validators.TimelineExportValidator(data=timeline, context={"project": project})
if serialized.is_valid(): if validator.is_valid():
serialized.object.project = project validator.object.project = project
serialized.object.namespace = build_project_namespace(project) validator.object.namespace = build_project_namespace(project)
serialized.object.object_id = project.id validator.object.object_id = project.id
serialized.object.content_type = ContentType.objects.get_for_model(project.__class__) validator.object.content_type = ContentType.objects.get_for_model(project.__class__)
serialized.object._importing = True validator.object._importing = True
serialized.save() validator.save()
return serialized return validator
add_errors("timeline", serialized.errors) add_errors("timeline", validator.errors)
return serialized return validator
def store_timeline_entries(project, data): def store_timeline_entries(project, data):
@ -617,13 +618,13 @@ def _validate_if_owner_have_enought_space_to_this_project(owner, data):
def _create_project_object(data): def _create_project_object(data):
# Create the project # Create the project
project_serialized = store_project(data) project_validator = store_project(data)
if not project_serialized: if not project_validator:
errors = get_errors(clear=True) errors = get_errors(clear=True)
raise err.TaigaImportError(_("error importing project data"), None, errors=errors) raise err.TaigaImportError(_("error importing project data"), None, errors=errors)
return project_serialized.object if project_serialized else None return project_validator.object if project_validator else None
def _create_membership_for_project_owner(project): def _create_membership_for_project_owner(project):
@ -654,13 +655,13 @@ def _populate_project_object(project, data):
check_if_there_is_some_error(_("error importing memberships"), project) check_if_there_is_some_error(_("error importing memberships"), project)
# Create project attributes values # Create project attributes values
store_project_attributes_values(project, data, "us_statuses", serializers.UserStoryStatusExportSerializer) store_project_attributes_values(project, data, "us_statuses", validators.UserStoryStatusExportValidator)
store_project_attributes_values(project, data, "points", serializers.PointsExportSerializer) store_project_attributes_values(project, data, "points", validators.PointsExportValidator)
store_project_attributes_values(project, data, "task_statuses", serializers.TaskStatusExportSerializer) store_project_attributes_values(project, data, "task_statuses", validators.TaskStatusExportValidator)
store_project_attributes_values(project, data, "issue_types", serializers.IssueTypeExportSerializer) store_project_attributes_values(project, data, "issue_types", validators.IssueTypeExportValidator)
store_project_attributes_values(project, data, "issue_statuses", serializers.IssueStatusExportSerializer) store_project_attributes_values(project, data, "issue_statuses", validators.IssueStatusExportValidator)
store_project_attributes_values(project, data, "priorities", serializers.PriorityExportSerializer) store_project_attributes_values(project, data, "priorities", validators.PriorityExportValidator)
store_project_attributes_values(project, data, "severities", serializers.SeverityExportSerializer) store_project_attributes_values(project, data, "severities", validators.SeverityExportValidator)
check_if_there_is_some_error(_("error importing lists of project attributes"), project) check_if_there_is_some_error(_("error importing lists of project attributes"), project)
# Create default values for project attributes # Create default values for project attributes
@ -669,11 +670,11 @@ def _populate_project_object(project, data):
# Create custom attributes # Create custom attributes
store_custom_attributes(project, data, "userstorycustomattributes", store_custom_attributes(project, data, "userstorycustomattributes",
serializers.UserStoryCustomAttributeExportSerializer) validators.UserStoryCustomAttributeExportValidator)
store_custom_attributes(project, data, "taskcustomattributes", store_custom_attributes(project, data, "taskcustomattributes",
serializers.TaskCustomAttributeExportSerializer) validators.TaskCustomAttributeExportValidator)
store_custom_attributes(project, data, "issuecustomattributes", store_custom_attributes(project, data, "issuecustomattributes",
serializers.IssueCustomAttributeExportSerializer) validators.IssueCustomAttributeExportValidator)
check_if_there_is_some_error(_("error importing custom attributes"), project) check_if_there_is_some_error(_("error importing custom attributes"), project)
# Create milestones # Create milestones

27
taiga/export_import/validators/__init__.py Normal file
View File

@ -0,0 +1,27 @@
from .validators import PointsExportValidator
from .validators import UserStoryStatusExportValidator
from .validators import TaskStatusExportValidator
from .validators import IssueStatusExportValidator
from .validators import PriorityExportValidator
from .validators import SeverityExportValidator
from .validators import IssueTypeExportValidator
from .validators import RoleExportValidator
from .validators import UserStoryCustomAttributeExportValidator
from .validators import TaskCustomAttributeExportValidator
from .validators import IssueCustomAttributeExportValidator
from .validators import BaseCustomAttributesValuesExportValidator
from .validators import UserStoryCustomAttributesValuesExportValidator
from .validators import TaskCustomAttributesValuesExportValidator
from .validators import IssueCustomAttributesValuesExportValidator
from .validators import MembershipExportValidator
from .validators import RolePointsExportValidator
from .validators import MilestoneExportValidator
from .validators import TaskExportValidator
from .validators import UserStoryExportValidator
from .validators import IssueExportValidator
from .validators import WikiPageExportValidator
from .validators import WikiLinkExportValidator
from .validators import TimelineExportValidator
from .validators import ProjectExportValidator
from .mixins import AttachmentExportValidator
from .mixins import HistoryExportValidator

42
taiga/export_import/validators/cache.py Normal file
View File

@ -0,0 +1,42 @@
# -*- coding: utf-8 -*-
# Copyright (C) 2014-2016 Andrey Antukh <niwi@niwi.nz>
# Copyright (C) 2014-2016 Jesús Espino <jespinog@gmail.com>
# Copyright (C) 2014-2016 David Barragán <bameda@dbarragan.com>
# Copyright (C) 2014-2016 Alejandro Alonso <alejandro.alonso@kaleidos.net>
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
from taiga.users import models as users_models
_cache_user_by_pk = {}
_cache_user_by_email = {}
_custom_tasks_attributes_cache = {}
_custom_issues_attributes_cache = {}
_custom_userstories_attributes_cache = {}
def cached_get_user_by_pk(pk):
if pk not in _cache_user_by_pk:
try:
_cache_user_by_pk[pk] = users_models.User.objects.get(pk=pk)
except Exception:
_cache_user_by_pk[pk] = users_models.User.objects.get(pk=pk)
return _cache_user_by_pk[pk]
def cached_get_user_by_email(email):
if email not in _cache_user_by_email:
try:
_cache_user_by_email[email] = users_models.User.objects.get(email=email)
except Exception:
_cache_user_by_email[email] = users_models.User.objects.get(email=email)
return _cache_user_by_email[email]

196
taiga/export_import/validators/fields.py Normal file
View File

@ -0,0 +1,196 @@
# -*- coding: utf-8 -*-
# Copyright (C) 2014-2016 Andrey Antukh <niwi@niwi.nz>
# Copyright (C) 2014-2016 Jesús Espino <jespinog@gmail.com>
# Copyright (C) 2014-2016 David Barragán <bameda@dbarragan.com>
# Copyright (C) 2014-2016 Alejandro Alonso <alejandro.alonso@kaleidos.net>
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import base64
import copy
from django.core.files.base import ContentFile
from django.core.exceptions import ObjectDoesNotExist
from django.utils.translation import ugettext as _
from django.contrib.contenttypes.models import ContentType
from taiga.base.api import serializers
from taiga.base.exceptions import ValidationError
from taiga.base.fields import JsonField
from taiga.mdrender.service import render as mdrender
from taiga.users import models as users_models
from .cache import cached_get_user_by_email
class FileField(serializers.WritableField):
read_only = False
def from_native(self, data):
if not data:
return None
decoded_data = b''
# The original file was encoded by chunks but we don't really know its
# length or if it was multiple of 3 so we must iterate over all those chunks
# decoding them one by one
for decoding_chunk in data['data'].split("="):
# When encoding to base64 3 bytes are transformed into 4 bytes and
# the extra space of the block is filled with =
# We must ensure that the decoding chunk has a length multiple of 4 so
# we restore the stripped '='s adding appending them until the chunk has
# a length multiple of 4
decoding_chunk += "=" * (-len(decoding_chunk) % 4)
decoded_data += base64.b64decode(decoding_chunk + "=")
return ContentFile(decoded_data, name=data['name'])
class ContentTypeField(serializers.RelatedField):
read_only = False
def from_native(self, data):
try:
return ContentType.objects.get_by_natural_key(*data)
except Exception:
return None
class RelatedNoneSafeField(serializers.RelatedField):
def field_from_native(self, data, files, field_name, into):
if self.read_only:
return
try:
if self.many:
try:
# Form data
value = data.getlist(field_name)
if value == [''] or value == []:
raise KeyError
except AttributeError:
# Non-form data
value = data[field_name]
else:
value = data[field_name]
except KeyError:
if self.partial:
return
value = self.get_default_value()
key = self.source or field_name
if value in self.null_values:
if self.required:
raise ValidationError(self.error_messages['required'])
into[key] = None
elif self.many:
into[key] = [self.from_native(item) for item in value if self.from_native(item) is not None]
else:
into[key] = self.from_native(value)
class UserRelatedField(RelatedNoneSafeField):
read_only = False
def from_native(self, data):
try:
return cached_get_user_by_email(data)
except users_models.User.DoesNotExist:
return None
class UserPkField(serializers.RelatedField):
read_only = False
def from_native(self, data):
try:
user = cached_get_user_by_email(data)
return user.pk
except users_models.User.DoesNotExist:
return None
class CommentField(serializers.WritableField):
read_only = False
def field_from_native(self, data, files, field_name, into):
super().field_from_native(data, files, field_name, into)
into["comment_html"] = mdrender(self.context['project'], data.get("comment", ""))
class ProjectRelatedField(serializers.RelatedField):
read_only = False
null_values = (None, "")
def __init__(self, slug_field, *args, **kwargs):
self.slug_field = slug_field
super().__init__(*args, **kwargs)
def from_native(self, data):
try:
kwargs = {self.slug_field: data, "project": self.context['project']}
return self.queryset.get(**kwargs)
except ObjectDoesNotExist:
raise ValidationError(_("{}=\"{}\" not found in this project".format(self.slug_field, data)))
class HistoryUserField(JsonField):
def from_native(self, data):
if data is None:
return {}
if len(data) < 2:
return {}
user = UserRelatedField().from_native(data[0])
if user:
pk = user.pk
else:
pk = None
return {"pk": pk, "name": data[1]}
class HistoryValuesField(JsonField):
def from_native(self, data):
if data is None:
return []
if "users" in data:
data['users'] = list(map(UserPkField().from_native, data['users']))
return data
class HistoryDiffField(JsonField):
def from_native(self, data):
if data is None:
return []
if "assigned_to" in data:
data['assigned_to'] = list(map(UserPkField().from_native, data['assigned_to']))
return data
class TimelineDataField(serializers.WritableField):
read_only = False
def from_native(self, data):
new_data = copy.deepcopy(data)
try:
user = cached_get_user_by_email(new_data["user"]["email"])
new_data["user"]["id"] = user.id
del new_data["user"]["email"]
except users_models.User.DoesNotExist:
pass
return new_data

97
taiga/export_import/validators/mixins.py Normal file
View File

@ -0,0 +1,97 @@
# -*- coding: utf-8 -*-
# Copyright (C) 2014-2016 Andrey Antukh <niwi@niwi.nz>
# Copyright (C) 2014-2016 Jesús Espino <jespinog@gmail.com>
# Copyright (C) 2014-2016 David Barragán <bameda@dbarragan.com>
# Copyright (C) 2014-2016 Alejandro Alonso <alejandro.alonso@kaleidos.net>
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
from django.contrib.auth import get_user_model
from django.core.exceptions import ObjectDoesNotExist
from django.contrib.contenttypes.models import ContentType
from taiga.base.api import serializers
from taiga.base.api import validators
from taiga.projects.history import models as history_models
from taiga.projects.attachments import models as attachments_models
from taiga.projects.notifications import services as notifications_services
from taiga.projects.history import services as history_service
from .fields import (UserRelatedField, HistoryUserField, HistoryDiffField,
JsonField, HistoryValuesField, CommentField, FileField)
class HistoryExportValidator(validators.ModelValidator):
user = HistoryUserField()
diff = HistoryDiffField(required=False)
snapshot = JsonField(required=False)
values = HistoryValuesField(required=False)
comment = CommentField(required=False)
delete_comment_date = serializers.DateTimeField(required=False)
delete_comment_user = HistoryUserField(required=False)
class Meta:
model = history_models.HistoryEntry
exclude = ("id", "comment_html", "key", "project")
class AttachmentExportValidator(validators.ModelValidator):
owner = UserRelatedField(required=False)
attached_file = FileField()
modified_date = serializers.DateTimeField(required=False)
class Meta:
model = attachments_models.Attachment
exclude = ('id', 'content_type', 'object_id', 'project')
class WatcheableObjectModelValidatorMixin(validators.ModelValidator):
watchers = UserRelatedField(many=True, required=False)
def __init__(self, *args, **kwargs):
self._watchers_field = self.base_fields.pop("watchers", None)
super(WatcheableObjectModelValidatorMixin, self).__init__(*args, **kwargs)
"""
watchers is not a field from the model so we need to do some magic to make it work like a normal field
It's supposed to be represented as an email list but internally it's treated like notifications.Watched instances
"""
def restore_object(self, attrs, instance=None):
self.fields.pop("watchers", None)
instance = super(WatcheableObjectModelValidatorMixin, self).restore_object(attrs, instance)
self._watchers = self.init_data.get("watchers", [])
return instance
def save_watchers(self):
new_watcher_emails = set(self._watchers)
old_watcher_emails = set(self.object.get_watchers().values_list("email", flat=True))
adding_watcher_emails = list(new_watcher_emails.difference(old_watcher_emails))
removing_watcher_emails = list(old_watcher_emails.difference(new_watcher_emails))
User = get_user_model()
adding_users = User.objects.filter(email__in=adding_watcher_emails)
removing_users = User.objects.filter(email__in=removing_watcher_emails)
for user in adding_users:
notifications_services.add_watcher(self.object, user)
for user in removing_users:
notifications_services.remove_watcher(self.object, user)
self.object.watchers = [user.email for user in self.object.get_watchers()]
def to_native(self, obj):
ret = super(WatcheableObjectModelValidatorMixin, self).to_native(obj)
ret["watchers"] = [user.email for user in obj.get_watchers()]
return ret

349
taiga/export_import/validators/validators.py Normal file
View File

@ -0,0 +1,349 @@
# -*- coding: utf-8 -*-
# Copyright (C) 2014-2016 Andrey Antukh <niwi@niwi.nz>
# Copyright (C) 2014-2016 Jesús Espino <jespinog@gmail.com>
# Copyright (C) 2014-2016 David Barragán <bameda@dbarragan.com>
# Copyright (C) 2014-2016 Alejandro Alonso <alejandro.alonso@kaleidos.net>
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
from django.utils.translation import ugettext as _
from taiga.base.api import serializers
from taiga.base.api import validators
from taiga.base.fields import JsonField, PgArrayField
from taiga.base.exceptions import ValidationError
from taiga.projects import models as projects_models
from taiga.projects.custom_attributes import models as custom_attributes_models
from taiga.projects.userstories import models as userstories_models
from taiga.projects.tasks import models as tasks_models
from taiga.projects.issues import models as issues_models
from taiga.projects.milestones import models as milestones_models
from taiga.projects.wiki import models as wiki_models
from taiga.timeline import models as timeline_models
from taiga.users import models as users_models
from .fields import (FileField, UserRelatedField,
ProjectRelatedField,
TimelineDataField, ContentTypeField)
from .mixins import WatcheableObjectModelValidatorMixin
from .cache import (_custom_tasks_attributes_cache,
_custom_userstories_attributes_cache,
_custom_issues_attributes_cache)
class PointsExportValidator(validators.ModelValidator):
class Meta:
model = projects_models.Points
exclude = ('id', 'project')
class UserStoryStatusExportValidator(validators.ModelValidator):
class Meta:
model = projects_models.UserStoryStatus
exclude = ('id', 'project')
class TaskStatusExportValidator(validators.ModelValidator):
class Meta:
model = projects_models.TaskStatus
exclude = ('id', 'project')
class IssueStatusExportValidator(validators.ModelValidator):
class Meta:
model = projects_models.IssueStatus
exclude = ('id', 'project')
class PriorityExportValidator(validators.ModelValidator):
class Meta:
model = projects_models.Priority
exclude = ('id', 'project')
class SeverityExportValidator(validators.ModelValidator):
class Meta:
model = projects_models.Severity
exclude = ('id', 'project')
class IssueTypeExportValidator(validators.ModelValidator):
class Meta:
model = projects_models.IssueType
exclude = ('id', 'project')
class RoleExportValidator(validators.ModelValidator):
permissions = PgArrayField(required=False)
class Meta:
model = users_models.Role
exclude = ('id', 'project')
class UserStoryCustomAttributeExportValidator(validators.ModelValidator):
modified_date = serializers.DateTimeField(required=False)
class Meta:
model = custom_attributes_models.UserStoryCustomAttribute
exclude = ('id', 'project')
class TaskCustomAttributeExportValidator(validators.ModelValidator):
modified_date = serializers.DateTimeField(required=False)
class Meta:
model = custom_attributes_models.TaskCustomAttribute
exclude = ('id', 'project')
class IssueCustomAttributeExportValidator(validators.ModelValidator):
modified_date = serializers.DateTimeField(required=False)
class Meta:
model = custom_attributes_models.IssueCustomAttribute
exclude = ('id', 'project')
class BaseCustomAttributesValuesExportValidator(validators.ModelValidator):
attributes_values = JsonField(source="attributes_values", required=True)
_custom_attribute_model = None
_container_field = None
class Meta:
exclude = ("id",)
def validate_attributes_values(self, attrs, source):
# values must be a dict
data_values = attrs.get("attributes_values", None)
if self.object:
data_values = (data_values or self.object.attributes_values)
if type(data_values) is not dict:
raise ValidationError(_("Invalid content. It must be {\"key\": \"value\",...}"))
# Values keys must be in the container object project
data_container = attrs.get(self._container_field, None)
if data_container:
project_id = data_container.project_id
elif self.object:
project_id = getattr(self.object, self._container_field).project_id
else:
project_id = None
values_ids = list(data_values.keys())
qs = self._custom_attribute_model.objects.filter(project=project_id,
id__in=values_ids)
if qs.count() != len(values_ids):
raise ValidationError(_("It contain invalid custom fields."))
return attrs
class UserStoryCustomAttributesValuesExportValidator(BaseCustomAttributesValuesExportValidator):
_custom_attribute_model = custom_attributes_models.UserStoryCustomAttribute
_container_model = "userstories.UserStory"
_container_field = "user_story"
class Meta(BaseCustomAttributesValuesExportValidator.Meta):
model = custom_attributes_models.UserStoryCustomAttributesValues
class TaskCustomAttributesValuesExportValidator(BaseCustomAttributesValuesExportValidator):
_custom_attribute_model = custom_attributes_models.TaskCustomAttribute
_container_field = "task"
class Meta(BaseCustomAttributesValuesExportValidator.Meta):
model = custom_attributes_models.TaskCustomAttributesValues
class IssueCustomAttributesValuesExportValidator(BaseCustomAttributesValuesExportValidator):
_custom_attribute_model = custom_attributes_models.IssueCustomAttribute
_container_field = "issue"
class Meta(BaseCustomAttributesValuesExportValidator.Meta):
model = custom_attributes_models.IssueCustomAttributesValues
class MembershipExportValidator(validators.ModelValidator):
user = UserRelatedField(required=False)
role = ProjectRelatedField(slug_field="name")
invited_by = UserRelatedField(required=False)
class Meta:
model = projects_models.Membership
exclude = ('id', 'project', 'token')
def full_clean(self, instance):
return instance
class RolePointsExportValidator(validators.ModelValidator):
role = ProjectRelatedField(slug_field="name")
points = ProjectRelatedField(slug_field="name")
class Meta:
model = userstories_models.RolePoints
exclude = ('id', 'user_story')
class MilestoneExportValidator(WatcheableObjectModelValidatorMixin):
owner = UserRelatedField(required=False)
modified_date = serializers.DateTimeField(required=False)
estimated_start = serializers.DateField(required=False)
estimated_finish = serializers.DateField(required=False)
def __init__(self, *args, **kwargs):
project = kwargs.pop('project', None)
super(MilestoneExportValidator, self).__init__(*args, **kwargs)
if project:
self.project = project
def validate_name(self, attrs, source):
"""
Check the milestone name is not duplicated in the project
"""
name = attrs[source]
qs = self.project.milestones.filter(name=name)
if qs.exists():
raise ValidationError(_("Name duplicated for the project"))
return attrs
class Meta:
model = milestones_models.Milestone
exclude = ('id', 'project')
class TaskExportValidator(WatcheableObjectModelValidatorMixin):
owner = UserRelatedField(required=False)
status = ProjectRelatedField(slug_field="name")
user_story = ProjectRelatedField(slug_field="ref", required=False)
milestone = ProjectRelatedField(slug_field="name", required=False)
assigned_to = UserRelatedField(required=False)
modified_date = serializers.DateTimeField(required=False)
class Meta:
model = tasks_models.Task
exclude = ('id', 'project')
def custom_attributes_queryset(self, project):
if project.id not in _custom_tasks_attributes_cache:
_custom_tasks_attributes_cache[project.id] = list(project.taskcustomattributes.all().values('id', 'name'))
return _custom_tasks_attributes_cache[project.id]
class UserStoryExportValidator(WatcheableObjectModelValidatorMixin):
role_points = RolePointsExportValidator(many=True, required=False)
owner = UserRelatedField(required=False)
assigned_to = UserRelatedField(required=False)
status = ProjectRelatedField(slug_field="name")
milestone = ProjectRelatedField(slug_field="name", required=False)
modified_date = serializers.DateTimeField(required=False)
generated_from_issue = ProjectRelatedField(slug_field="ref", required=False)
class Meta:
model = userstories_models.UserStory
exclude = ('id', 'project', 'points', 'tasks')
def custom_attributes_queryset(self, project):
if project.id not in _custom_userstories_attributes_cache:
_custom_userstories_attributes_cache[project.id] = list(
project.userstorycustomattributes.all().values('id', 'name')
)
return _custom_userstories_attributes_cache[project.id]
class IssueExportValidator(WatcheableObjectModelValidatorMixin):
owner = UserRelatedField(required=False)
status = ProjectRelatedField(slug_field="name")
assigned_to = UserRelatedField(required=False)
priority = ProjectRelatedField(slug_field="name")
severity = ProjectRelatedField(slug_field="name")
type = ProjectRelatedField(slug_field="name")
milestone = ProjectRelatedField(slug_field="name", required=False)
modified_date = serializers.DateTimeField(required=False)
class Meta:
model = issues_models.Issue
exclude = ('id', 'project')
def custom_attributes_queryset(self, project):
if project.id not in _custom_issues_attributes_cache:
_custom_issues_attributes_cache[project.id] = list(project.issuecustomattributes.all().values('id', 'name'))
return _custom_issues_attributes_cache[project.id]
class WikiPageExportValidator(WatcheableObjectModelValidatorMixin):
owner = UserRelatedField(required=False)
last_modifier = UserRelatedField(required=False)
modified_date = serializers.DateTimeField(required=False)
class Meta:
model = wiki_models.WikiPage
exclude = ('id', 'project')
class WikiLinkExportValidator(validators.ModelValidator):
class Meta:
model = wiki_models.WikiLink
exclude = ('id', 'project')
class TimelineExportValidator(validators.ModelValidator):
data = TimelineDataField()
data_content_type = ContentTypeField()
class Meta:
model = timeline_models.Timeline
exclude = ('id', 'project', 'namespace', 'object_id', 'content_type')
class ProjectExportValidator(WatcheableObjectModelValidatorMixin):
logo = FileField(required=False)
anon_permissions = PgArrayField(required=False)
public_permissions = PgArrayField(required=False)
modified_date = serializers.DateTimeField(required=False)
roles = RoleExportValidator(many=True, required=False)
owner = UserRelatedField(required=False)
memberships = MembershipExportValidator(many=True, required=False)
points = PointsExportValidator(many=True, required=False)
us_statuses = UserStoryStatusExportValidator(many=True, required=False)
task_statuses = TaskStatusExportValidator(many=True, required=False)
issue_types = IssueTypeExportValidator(many=True, required=False)
issue_statuses = IssueStatusExportValidator(many=True, required=False)
priorities = PriorityExportValidator(many=True, required=False)
severities = SeverityExportValidator(many=True, required=False)
tags_colors = JsonField(required=False)
creation_template = serializers.SlugRelatedField(slug_field="slug", required=False)
default_points = serializers.SlugRelatedField(slug_field="name", required=False)
default_us_status = serializers.SlugRelatedField(slug_field="name", required=False)
default_task_status = serializers.SlugRelatedField(slug_field="name", required=False)
default_priority = serializers.SlugRelatedField(slug_field="name", required=False)
default_severity = serializers.SlugRelatedField(slug_field="name", required=False)
default_issue_status = serializers.SlugRelatedField(slug_field="name", required=False)
default_issue_type = serializers.SlugRelatedField(slug_field="name", required=False)
userstorycustomattributes = UserStoryCustomAttributeExportValidator(many=True, required=False)
taskcustomattributes = TaskCustomAttributeExportValidator(many=True, required=False)
issuecustomattributes = IssueCustomAttributeExportValidator(many=True, required=False)
user_stories = UserStoryExportValidator(many=True, required=False)
tasks = TaskExportValidator(many=True, required=False)
milestones = MilestoneExportValidator(many=True, required=False)
issues = IssueExportValidator(many=True, required=False)
wiki_links = WikiLinkExportValidator(many=True, required=False)
wiki_pages = WikiPageExportValidator(many=True, required=False)
class Meta:
model = projects_models.Project
exclude = ('id', 'members')