diff --git a/taiga/base/fields.py b/taiga/base/fields.py index 30be6b60..3b19f15f 100644 --- a/taiga/base/fields.py +++ b/taiga/base/fields.py @@ -18,7 +18,8 @@ from django.forms import widgets from django.utils.translation import ugettext as _ -from taiga.base.api import serializers +from taiga.base.api import serializers, ISO_8601 +from taiga.base.api.settings import api_settings import serpy @@ -128,4 +129,21 @@ class I18NJsonField(Field): class FileField(Field): def to_value(self, value): - return value.name + if value: + return value.name + return None + + +class DateTimeField(Field): + format = api_settings.DATETIME_FORMAT + + def to_value(self, value): + if value is None or self.format is None: + return value + + if self.format.lower() == ISO_8601: + ret = value.isoformat() + if ret.endswith("+00:00"): + ret = ret[:-6] + "Z" + return ret + return value.strftime(self.format) diff --git a/taiga/export_import/api.py b/taiga/export_import/api.py index da2af132..75644365 100644 --- a/taiga/export_import/api.py +++ b/taiga/export_import/api.py @@ -44,11 +44,11 @@ from taiga.users import services as users_services from . import exceptions as err from . import mixins from . import permissions +from . import validators from . import serializers from . import services from . import tasks from . import throttling -from .renderers import ExportRenderer from taiga.base.api.utils import get_object_or_404 @@ -102,9 +102,8 @@ class ProjectImporterViewSet(mixins.ImportThrottlingPolicyMixin, CreateModelMixi # Validate if the project can be imported is_private = data.get('is_private', False) - total_memberships = len([m for m in data.get("memberships", []) - if m.get("email", None) != data["owner"]]) - total_memberships = total_memberships + 1 # 1 is the owner + total_memberships = len([m for m in data.get("memberships", []) if m.get("email", None) != data["owner"]]) + total_memberships = total_memberships + 1 # 1 is the owner (enough_slots, error_message) = users_services.has_available_slot_for_import_new_project( self.request.user, is_private, @@ -147,31 +146,31 @@ class ProjectImporterViewSet(mixins.ImportThrottlingPolicyMixin, CreateModelMixi # Create project values choicess if "points" in data: services.store.store_project_attributes_values(project_serialized.object, data, - "points", serializers.PointsExportSerializer) + "points", validators.PointsExportValidator) if "issue_types" in data: services.store.store_project_attributes_values(project_serialized.object, data, "issue_types", - serializers.IssueTypeExportSerializer) + validators.IssueTypeExportValidator) if "issue_statuses" in data: services.store.store_project_attributes_values(project_serialized.object, data, "issue_statuses", - serializers.IssueStatusExportSerializer,) + validators.IssueStatusExportValidator,) if "us_statuses" in data: services.store.store_project_attributes_values(project_serialized.object, data, "us_statuses", - serializers.UserStoryStatusExportSerializer,) + validators.UserStoryStatusExportValidator,) if "task_statuses" in data: services.store.store_project_attributes_values(project_serialized.object, data, "task_statuses", - serializers.TaskStatusExportSerializer) + validators.TaskStatusExportValidator) if "priorities" in data: services.store.store_project_attributes_values(project_serialized.object, data, "priorities", - serializers.PriorityExportSerializer) + validators.PriorityExportValidator) if "severities" in data: services.store.store_project_attributes_values(project_serialized.object, data, "severities", - serializers.SeverityExportSerializer) + validators.SeverityExportValidator) if ("points" in data or "issues_types" in data or "issues_statuses" in data or "us_statuses" in data or @@ -183,17 +182,17 @@ class ProjectImporterViewSet(mixins.ImportThrottlingPolicyMixin, CreateModelMixi if "userstorycustomattributes" in data: services.store.store_custom_attributes(project_serialized.object, data, "userstorycustomattributes", - serializers.UserStoryCustomAttributeExportSerializer) + validators.UserStoryCustomAttributeExportValidator) if "taskcustomattributes" in data: services.store.store_custom_attributes(project_serialized.object, data, "taskcustomattributes", - serializers.TaskCustomAttributeExportSerializer) + validators.TaskCustomAttributeExportValidator) if "issuecustomattributes" in data: services.store.store_custom_attributes(project_serialized.object, data, "issuecustomattributes", - serializers.IssueCustomAttributeExportSerializer) + validators.IssueCustomAttributeExportValidator) # Is there any error? errors = services.store.get_errors() @@ -201,7 +200,7 @@ class ProjectImporterViewSet(mixins.ImportThrottlingPolicyMixin, CreateModelMixi raise exc.BadRequest(errors) # Importer process is OK - response_data = project_serialized.data + response_data = serializers.ProjectExportSerializer(project_serialized.object).data response_data['id'] = project_serialized.object.id headers = self.get_success_headers(response_data) return response.Created(response_data, headers=headers) @@ -218,8 +217,9 @@ class ProjectImporterViewSet(mixins.ImportThrottlingPolicyMixin, CreateModelMixi if errors: raise exc.BadRequest(errors) - headers = self.get_success_headers(milestone.data) - return response.Created(milestone.data, headers=headers) + data = serializers.MilestoneExportSerializer(milestone.object).data + headers = self.get_success_headers(data) + return response.Created(data, headers=headers) @detail_route(methods=['post']) @method_decorator(atomic) @@ -233,8 +233,9 @@ class ProjectImporterViewSet(mixins.ImportThrottlingPolicyMixin, CreateModelMixi if errors: raise exc.BadRequest(errors) - headers = self.get_success_headers(us.data) - return response.Created(us.data, headers=headers) + data = serializers.UserStoryExportSerializer(us.object).data + headers = self.get_success_headers(data) + return response.Created(data, headers=headers) @detail_route(methods=['post']) @method_decorator(atomic) @@ -251,8 +252,9 @@ class ProjectImporterViewSet(mixins.ImportThrottlingPolicyMixin, CreateModelMixi if errors: raise exc.BadRequest(errors) - headers = self.get_success_headers(task.data) - return response.Created(task.data, headers=headers) + data = serializers.TaskExportSerializer(task.object).data + headers = self.get_success_headers(data) + return response.Created(data, headers=headers) @detail_route(methods=['post']) @method_decorator(atomic) @@ -269,8 +271,9 @@ class ProjectImporterViewSet(mixins.ImportThrottlingPolicyMixin, CreateModelMixi if errors: raise exc.BadRequest(errors) - headers = self.get_success_headers(issue.data) - return response.Created(issue.data, headers=headers) + data = serializers.IssueExportSerializer(issue.object).data + headers = self.get_success_headers(data) + return response.Created(data, headers=headers) @detail_route(methods=['post']) @method_decorator(atomic) @@ -284,8 +287,9 @@ class ProjectImporterViewSet(mixins.ImportThrottlingPolicyMixin, CreateModelMixi if errors: raise exc.BadRequest(errors) - headers = self.get_success_headers(wiki_page.data) - return response.Created(wiki_page.data, headers=headers) + data = serializers.WikiPageExportSerializer(wiki_page.object).data + headers = self.get_success_headers(data) + return response.Created(data, headers=headers) @detail_route(methods=['post']) @method_decorator(atomic) @@ -299,8 +303,9 @@ class ProjectImporterViewSet(mixins.ImportThrottlingPolicyMixin, CreateModelMixi if errors: raise exc.BadRequest(errors) - headers = self.get_success_headers(wiki_link.data) - return response.Created(wiki_link.data, headers=headers) + data = serializers.WikiLinkExportSerializer(wiki_link.object).data + headers = self.get_success_headers(data) + return response.Created(data, headers=headers) @list_route(methods=["POST"]) @method_decorator(atomic) diff --git a/taiga/export_import/serializers/fields.py b/taiga/export_import/serializers/fields.py index 9ed21a19..29ec85aa 100644 --- a/taiga/export_import/serializers/fields.py +++ b/taiga/export_import/serializers/fields.py @@ -21,24 +21,15 @@ import os import copy from collections import OrderedDict -from django.core.files.base import ContentFile -from django.core.exceptions import ObjectDoesNotExist -from django.utils.translation import ugettext as _ -from django.contrib.contenttypes.models import ContentType - from taiga.base.api import serializers -from taiga.base.exceptions import ValidationError -from taiga.base.fields import JsonField -from taiga.mdrender.service import render as mdrender +from taiga.base.fields import Field from taiga.users import models as users_models -from .cache import cached_get_user_by_email, cached_get_user_by_pk +from .cache import cached_get_user_by_pk -class FileField(serializers.WritableField): - read_only = False - - def to_native(self, obj): +class FileField(Field): + def to_value(self, obj): if not obj: return None @@ -49,202 +40,74 @@ class FileField(serializers.WritableField): ("name", os.path.basename(obj.name)), ]) - def from_native(self, data): - if not data: - return None - decoded_data = b'' - # The original file was encoded by chunks but we don't really know its - # length or if it was multiple of 3 so we must iterate over all those chunks - # decoding them one by one - for decoding_chunk in data['data'].split("="): - # When encoding to base64 3 bytes are transformed into 4 bytes and - # the extra space of the block is filled with = - # We must ensure that the decoding chunk has a length multiple of 4 so - # we restore the stripped '='s adding appending them until the chunk has - # a length multiple of 4 - decoding_chunk += "=" * (-len(decoding_chunk) % 4) - decoded_data += base64.b64decode(decoding_chunk+"=") - - return ContentFile(decoded_data, name=data['name']) - - -class ContentTypeField(serializers.RelatedField): - read_only = False - - def to_native(self, obj): +class ContentTypeField(Field): + def to_value(self, obj): if obj: return [obj.app_label, obj.model] return None - def from_native(self, data): - try: - return ContentType.objects.get_by_natural_key(*data) - except Exception: - return None - -class RelatedNoneSafeField(serializers.RelatedField): - def field_from_native(self, data, files, field_name, into): - if self.read_only: - return - - try: - if self.many: - try: - # Form data - value = data.getlist(field_name) - if value == [''] or value == []: - raise KeyError - except AttributeError: - # Non-form data - value = data[field_name] - else: - value = data[field_name] - except KeyError: - if self.partial: - return - value = self.get_default_value() - - key = self.source or field_name - if value in self.null_values: - if self.required: - raise ValidationError(self.error_messages['required']) - into[key] = None - elif self.many: - into[key] = [self.from_native(item) for item in value if self.from_native(item) is not None] - else: - into[key] = self.from_native(value) - - -class UserRelatedField(RelatedNoneSafeField): - read_only = False - - def to_native(self, obj): +class UserRelatedField(Field): + def to_value(self, obj): if obj: return obj.email return None - def from_native(self, data): - try: - return cached_get_user_by_email(data) - except users_models.User.DoesNotExist: - return None - -class UserPkField(serializers.RelatedField): - read_only = False - - def to_native(self, obj): +class UserPkField(Field): + def to_value(self, obj): try: user = cached_get_user_by_pk(obj) return user.email except users_models.User.DoesNotExist: return None - def from_native(self, data): - try: - user = cached_get_user_by_email(data) - return user.pk - except users_models.User.DoesNotExist: - return None - - -class CommentField(serializers.WritableField): - read_only = False - - def field_from_native(self, data, files, field_name, into): - super().field_from_native(data, files, field_name, into) - into["comment_html"] = mdrender(self.context['project'], data.get("comment", "")) - - -class ProjectRelatedField(serializers.RelatedField): - read_only = False - null_values = (None, "") +class SlugRelatedField(Field): def __init__(self, slug_field, *args, **kwargs): self.slug_field = slug_field super().__init__(*args, **kwargs) - def to_native(self, obj): + def to_value(self, obj): if obj: return getattr(obj, self.slug_field) return None - def from_native(self, data): - try: - kwargs = {self.slug_field: data, "project": self.context['project']} - return self.queryset.get(**kwargs) - except ObjectDoesNotExist: - raise ValidationError(_("{}=\"{}\" not found in this project".format(self.slug_field, data))) - -class HistoryUserField(JsonField): - def to_native(self, obj): +class HistoryUserField(Field): + def to_value(self, obj): if obj is None or obj == {}: return [] try: user = cached_get_user_by_pk(obj['pk']) except users_models.User.DoesNotExist: user = None - return (UserRelatedField().to_native(user), obj['name']) - - def from_native(self, data): - if data is None: - return {} - - if len(data) < 2: - return {} - - user = UserRelatedField().from_native(data[0]) - - if user: - pk = user.pk - else: - pk = None - - return {"pk": pk, "name": data[1]} + return (UserRelatedField().to_value(user), obj['name']) -class HistoryValuesField(JsonField): - def to_native(self, obj): +class HistoryValuesField(Field): + def to_value(self, obj): if obj is None: return [] if "users" in obj: - obj['users'] = list(map(UserPkField().to_native, obj['users'])) + obj['users'] = list(map(UserPkField().to_value, obj['users'])) return obj - def from_native(self, data): - if data is None: - return [] - if "users" in data: - data['users'] = list(map(UserPkField().from_native, data['users'])) - return data - -class HistoryDiffField(JsonField): - def to_native(self, obj): +class HistoryDiffField(Field): + def to_value(self, obj): if obj is None: return [] if "assigned_to" in obj: - obj['assigned_to'] = list(map(UserPkField().to_native, obj['assigned_to'])) + obj['assigned_to'] = list(map(UserPkField().to_value, obj['assigned_to'])) return obj - def from_native(self, data): - if data is None: - return [] - if "assigned_to" in data: - data['assigned_to'] = list(map(UserPkField().from_native, data['assigned_to'])) - return data - - -class TimelineDataField(serializers.WritableField): - read_only = False - - def to_native(self, data): +class TimelineDataField(Field): + def to_value(self, data): new_data = copy.deepcopy(data) try: user = cached_get_user_by_pk(new_data["user"]["id"]) @@ -253,14 +116,3 @@ class TimelineDataField(serializers.WritableField): except Exception: pass return new_data - - def from_native(self, data): - new_data = copy.deepcopy(data) - try: - user = cached_get_user_by_email(new_data["user"]["email"]) - new_data["user"]["id"] = user.id - del new_data["user"]["email"] - except users_models.User.DoesNotExist: - pass - - return new_data diff --git a/taiga/export_import/serializers/mixins.py b/taiga/export_import/serializers/mixins.py index 007649a2..3006500f 100644 --- a/taiga/export_import/serializers/mixins.py +++ b/taiga/export_import/serializers/mixins.py @@ -16,56 +16,62 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -from django.contrib.auth import get_user_model from django.core.exceptions import ObjectDoesNotExist from django.contrib.contenttypes.models import ContentType from taiga.base.api import serializers +from taiga.base.fields import Field, MethodField, DateTimeField from taiga.projects.history import models as history_models from taiga.projects.attachments import models as attachments_models -from taiga.projects.notifications import services as notifications_services from taiga.projects.history import services as history_service from .fields import (UserRelatedField, HistoryUserField, HistoryDiffField, - JsonField, HistoryValuesField, CommentField, FileField) + HistoryValuesField, FileField) -class HistoryExportSerializer(serializers.ModelSerializer): +class HistoryExportSerializer(serializers.LightSerializer): user = HistoryUserField() - diff = HistoryDiffField(required=False) - snapshot = JsonField(required=False) - values = HistoryValuesField(required=False) - comment = CommentField(required=False) - delete_comment_date = serializers.DateTimeField(required=False) - delete_comment_user = HistoryUserField(required=False) - - class Meta: - model = history_models.HistoryEntry - exclude = ("id", "comment_html", "key") + diff = HistoryDiffField() + snapshot = Field() + values = HistoryValuesField() + comment = Field() + delete_comment_date = DateTimeField() + delete_comment_user = HistoryUserField() + comment_versions = Field() + created_at = DateTimeField() + edit_comment_date = DateTimeField() + is_hidden = Field() + is_snapshot = Field() + type = Field() -class HistoryExportSerializerMixin(serializers.ModelSerializer): - history = serializers.SerializerMethodField("get_history") +class HistoryExportSerializerMixin(serializers.LightSerializer): + history = MethodField("get_history") def get_history(self, obj): - history_qs = history_service.get_history_queryset_by_model_instance(obj, - types=(history_models.HistoryType.change, history_models.HistoryType.create,)) + history_qs = history_service.get_history_queryset_by_model_instance( + obj, + types=(history_models.HistoryType.change, history_models.HistoryType.create,) + ) return HistoryExportSerializer(history_qs, many=True).data -class AttachmentExportSerializer(serializers.ModelSerializer): - owner = UserRelatedField(required=False) +class AttachmentExportSerializer(serializers.LightSerializer): + owner = UserRelatedField() attached_file = FileField() - modified_date = serializers.DateTimeField(required=False) - - class Meta: - model = attachments_models.Attachment - exclude = ('id', 'content_type', 'object_id', 'project') + created_date = DateTimeField() + modified_date = DateTimeField() + description = Field() + is_deprecated = Field() + name = Field() + order = Field() + sha1 = Field() + size = Field() -class AttachmentExportSerializerMixin(serializers.ModelSerializer): - attachments = serializers.SerializerMethodField("get_attachments") +class AttachmentExportSerializerMixin(serializers.LightSerializer): + attachments = MethodField() def get_attachments(self, obj): content_type = ContentType.objects.get_for_model(obj.__class__) @@ -74,8 +80,8 @@ class AttachmentExportSerializerMixin(serializers.ModelSerializer): return AttachmentExportSerializer(attachments_qs, many=True).data -class CustomAttributesValuesExportSerializerMixin(serializers.ModelSerializer): - custom_attributes_values = serializers.SerializerMethodField("get_custom_attributes_values") +class CustomAttributesValuesExportSerializerMixin(serializers.LightSerializer): + custom_attributes_values = MethodField("get_custom_attributes_values") def custom_attributes_queryset(self, project): raise NotImplementedError() @@ -85,13 +91,13 @@ class CustomAttributesValuesExportSerializerMixin(serializers.ModelSerializer): ret = {} for attr in custom_attributes: value = values.get(str(attr["id"]), None) - if value is not None: + if value is not None: ret[attr["name"]] = value return ret try: - values = obj.custom_attributes_values.attributes_values + values = obj.custom_attributes_values.attributes_values custom_attributes = self.custom_attributes_queryset(obj.project) return _use_name_instead_id_as_key_in_custom_attributes_values(custom_attributes, values) @@ -99,43 +105,8 @@ class CustomAttributesValuesExportSerializerMixin(serializers.ModelSerializer): return None -class WatcheableObjectModelSerializerMixin(serializers.ModelSerializer): - watchers = UserRelatedField(many=True, required=False) +class WatcheableObjectLightSerializerMixin(serializers.LightSerializer): + watchers = MethodField() - def __init__(self, *args, **kwargs): - self._watchers_field = self.base_fields.pop("watchers", None) - super(WatcheableObjectModelSerializerMixin, self).__init__(*args, **kwargs) - - """ - watchers is not a field from the model so we need to do some magic to make it work like a normal field - It's supposed to be represented as an email list but internally it's treated like notifications.Watched instances - """ - - def restore_object(self, attrs, instance=None): - watcher_field = self.fields.pop("watchers", None) - instance = super(WatcheableObjectModelSerializerMixin, self).restore_object(attrs, instance) - self._watchers = self.init_data.get("watchers", []) - return instance - - def save_watchers(self): - new_watcher_emails = set(self._watchers) - old_watcher_emails = set(self.object.get_watchers().values_list("email", flat=True)) - adding_watcher_emails = list(new_watcher_emails.difference(old_watcher_emails)) - removing_watcher_emails = list(old_watcher_emails.difference(new_watcher_emails)) - - User = get_user_model() - adding_users = User.objects.filter(email__in=adding_watcher_emails) - removing_users = User.objects.filter(email__in=removing_watcher_emails) - - for user in adding_users: - notifications_services.add_watcher(self.object, user) - - for user in removing_users: - notifications_services.remove_watcher(self.object, user) - - self.object.watchers = [user.email for user in self.object.get_watchers()] - - def to_native(self, obj): - ret = super(WatcheableObjectModelSerializerMixin, self).to_native(obj) - ret["watchers"] = [user.email for user in obj.get_watchers()] - return ret + def get_watchers(self, obj): + return [user.email for user in obj.get_watchers()] diff --git a/taiga/export_import/serializers/serializers.py b/taiga/export_import/serializers/serializers.py index 6a316b68..ff7e791c 100644 --- a/taiga/export_import/serializers/serializers.py +++ b/taiga/export_import/serializers/serializers.py @@ -16,231 +16,183 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -from django.utils.translation import ugettext as _ - from taiga.base.api import serializers -from taiga.base.fields import JsonField, PgArrayField -from taiga.base.exceptions import ValidationError +from taiga.base.fields import Field, DateTimeField, MethodField -from taiga.projects import models as projects_models -from taiga.projects.custom_attributes import models as custom_attributes_models -from taiga.projects.userstories import models as userstories_models -from taiga.projects.tasks import models as tasks_models -from taiga.projects.issues import models as issues_models -from taiga.projects.milestones import models as milestones_models -from taiga.projects.wiki import models as wiki_models -from taiga.timeline import models as timeline_models -from taiga.users import models as users_models from taiga.projects.votes import services as votes_service -from .fields import (FileField, UserRelatedField, - ProjectRelatedField, - TimelineDataField, ContentTypeField) +from .fields import (FileField, UserRelatedField, TimelineDataField, + ContentTypeField, SlugRelatedField) from .mixins import (HistoryExportSerializerMixin, AttachmentExportSerializerMixin, CustomAttributesValuesExportSerializerMixin, - WatcheableObjectModelSerializerMixin) + WatcheableObjectLightSerializerMixin) from .cache import (_custom_tasks_attributes_cache, _custom_userstories_attributes_cache, _custom_issues_attributes_cache) -class PointsExportSerializer(serializers.ModelSerializer): - class Meta: - model = projects_models.Points - exclude = ('id', 'project') +class RelatedExportSerializer(serializers.LightSerializer): + def to_value(self, value): + if hasattr(value, 'all'): + return super().to_value(value.all()) + return super().to_value(value) -class UserStoryStatusExportSerializer(serializers.ModelSerializer): - class Meta: - model = projects_models.UserStoryStatus - exclude = ('id', 'project') +class PointsExportSerializer(RelatedExportSerializer): + name = Field() + order = Field() + value = Field() -class TaskStatusExportSerializer(serializers.ModelSerializer): - class Meta: - model = projects_models.TaskStatus - exclude = ('id', 'project') +class UserStoryStatusExportSerializer(RelatedExportSerializer): + name = Field() + slug = Field() + order = Field() + is_closed = Field() + is_archived = Field() + color = Field() + wip_limit = Field() -class IssueStatusExportSerializer(serializers.ModelSerializer): - class Meta: - model = projects_models.IssueStatus - exclude = ('id', 'project') +class TaskStatusExportSerializer(RelatedExportSerializer): + name = Field() + slug = Field() + order = Field() + is_closed = Field() + color = Field() -class PriorityExportSerializer(serializers.ModelSerializer): - class Meta: - model = projects_models.Priority - exclude = ('id', 'project') +class IssueStatusExportSerializer(RelatedExportSerializer): + name = Field() + slug = Field() + order = Field() + is_closed = Field() + color = Field() -class SeverityExportSerializer(serializers.ModelSerializer): - class Meta: - model = projects_models.Severity - exclude = ('id', 'project') +class PriorityExportSerializer(RelatedExportSerializer): + name = Field() + order = Field() + color = Field() -class IssueTypeExportSerializer(serializers.ModelSerializer): - class Meta: - model = projects_models.IssueType - exclude = ('id', 'project') +class SeverityExportSerializer(RelatedExportSerializer): + name = Field() + order = Field() + color = Field() -class RoleExportSerializer(serializers.ModelSerializer): - permissions = PgArrayField(required=False) - - class Meta: - model = users_models.Role - exclude = ('id', 'project') +class IssueTypeExportSerializer(RelatedExportSerializer): + name = Field() + order = Field() + color = Field() -class UserStoryCustomAttributeExportSerializer(serializers.ModelSerializer): - modified_date = serializers.DateTimeField(required=False) - - class Meta: - model = custom_attributes_models.UserStoryCustomAttribute - exclude = ('id', 'project') +class RoleExportSerializer(RelatedExportSerializer): + name = Field() + slug = Field() + order = Field() + computable = Field() + permissions = Field() -class TaskCustomAttributeExportSerializer(serializers.ModelSerializer): - modified_date = serializers.DateTimeField(required=False) - - class Meta: - model = custom_attributes_models.TaskCustomAttribute - exclude = ('id', 'project') +class UserStoryCustomAttributeExportSerializer(RelatedExportSerializer): + name = Field() + description = Field() + type = Field() + order = Field() + created_date = DateTimeField() + modified_date = DateTimeField() -class IssueCustomAttributeExportSerializer(serializers.ModelSerializer): - modified_date = serializers.DateTimeField(required=False) - - class Meta: - model = custom_attributes_models.IssueCustomAttribute - exclude = ('id', 'project') +class TaskCustomAttributeExportSerializer(RelatedExportSerializer): + name = Field() + description = Field() + type = Field() + order = Field() + created_date = DateTimeField() + modified_date = DateTimeField() -class BaseCustomAttributesValuesExportSerializer(serializers.ModelSerializer): - attributes_values = JsonField(source="attributes_values", required=True) - _custom_attribute_model = None - _container_field = None +class IssueCustomAttributeExportSerializer(RelatedExportSerializer): + name = Field() + description = Field() + type = Field() + order = Field() + created_date = DateTimeField() + modified_date = DateTimeField() - class Meta: - exclude = ("id",) - def validate_attributes_values(self, attrs, source): - # values must be a dict - data_values = attrs.get("attributes_values", None) - if self.object: - data_values = (data_values or self.object.attributes_values) - - if type(data_values) is not dict: - raise ValidationError(_("Invalid content. It must be {\"key\": \"value\",...}")) - - # Values keys must be in the container object project - data_container = attrs.get(self._container_field, None) - if data_container: - project_id = data_container.project_id - elif self.object: - project_id = getattr(self.object, self._container_field).project_id - else: - project_id = None - - values_ids = list(data_values.keys()) - qs = self._custom_attribute_model.objects.filter(project=project_id, - id__in=values_ids) - if qs.count() != len(values_ids): - raise ValidationError(_("It contain invalid custom fields.")) - - return attrs +class BaseCustomAttributesValuesExportSerializer(RelatedExportSerializer): + attributes_values = Field(required=True) class UserStoryCustomAttributesValuesExportSerializer(BaseCustomAttributesValuesExportSerializer): - _custom_attribute_model = custom_attributes_models.UserStoryCustomAttribute - _container_model = "userstories.UserStory" - _container_field = "user_story" - - class Meta(BaseCustomAttributesValuesExportSerializer.Meta): - model = custom_attributes_models.UserStoryCustomAttributesValues + user_story = Field(attr="user_story.id") class TaskCustomAttributesValuesExportSerializer(BaseCustomAttributesValuesExportSerializer): - _custom_attribute_model = custom_attributes_models.TaskCustomAttribute - _container_field = "task" - - class Meta(BaseCustomAttributesValuesExportSerializer.Meta): - model = custom_attributes_models.TaskCustomAttributesValues + task = Field(attr="task.id") class IssueCustomAttributesValuesExportSerializer(BaseCustomAttributesValuesExportSerializer): - _custom_attribute_model = custom_attributes_models.IssueCustomAttribute - _container_field = "issue" - - class Meta(BaseCustomAttributesValuesExportSerializer.Meta): - model = custom_attributes_models.IssueCustomAttributesValues + issue = Field(attr="issue.id") -class MembershipExportSerializer(serializers.ModelSerializer): - user = UserRelatedField(required=False) - role = ProjectRelatedField(slug_field="name") - invited_by = UserRelatedField(required=False) - - class Meta: - model = projects_models.Membership - exclude = ('id', 'project', 'token') - - def full_clean(self, instance): - return instance +class MembershipExportSerializer(RelatedExportSerializer): + user = UserRelatedField() + role = SlugRelatedField(slug_field="name") + invited_by = UserRelatedField() + is_admin = Field() + email = Field() + created_at = DateTimeField() + invitation_extra_text = Field() + user_order = Field() -class RolePointsExportSerializer(serializers.ModelSerializer): - role = ProjectRelatedField(slug_field="name") - points = ProjectRelatedField(slug_field="name") - - class Meta: - model = userstories_models.RolePoints - exclude = ('id', 'user_story') +class RolePointsExportSerializer(RelatedExportSerializer): + role = SlugRelatedField(slug_field="name") + points = SlugRelatedField(slug_field="name") -class MilestoneExportSerializer(WatcheableObjectModelSerializerMixin): - owner = UserRelatedField(required=False) - modified_date = serializers.DateTimeField(required=False) - estimated_start = serializers.DateField(required=False) - estimated_finish = serializers.DateField(required=False) - - def __init__(self, *args, **kwargs): - project = kwargs.pop('project', None) - super(MilestoneExportSerializer, self).__init__(*args, **kwargs) - if project: - self.project = project - - def validate_name(self, attrs, source): - """ - Check the milestone name is not duplicated in the project - """ - name = attrs[source] - qs = self.project.milestones.filter(name=name) - if qs.exists(): - raise ValidationError(_("Name duplicated for the project")) - - return attrs - - class Meta: - model = milestones_models.Milestone - exclude = ('id', 'project') +class MilestoneExportSerializer(WatcheableObjectLightSerializerMixin, RelatedExportSerializer): + name = Field() + owner = UserRelatedField() + created_date = DateTimeField() + modified_date = DateTimeField() + estimated_start = Field() + estimated_finish = Field() + slug = Field() + closed = Field() + disponibility = Field() + order = Field() -class TaskExportSerializer(CustomAttributesValuesExportSerializerMixin, HistoryExportSerializerMixin, - AttachmentExportSerializerMixin, WatcheableObjectModelSerializerMixin): - owner = UserRelatedField(required=False) - status = ProjectRelatedField(slug_field="name") - user_story = ProjectRelatedField(slug_field="ref", required=False) - milestone = ProjectRelatedField(slug_field="name", required=False) - assigned_to = UserRelatedField(required=False) - modified_date = serializers.DateTimeField(required=False) - - class Meta: - model = tasks_models.Task - exclude = ('id', 'project') +class TaskExportSerializer(CustomAttributesValuesExportSerializerMixin, + HistoryExportSerializerMixin, + AttachmentExportSerializerMixin, + WatcheableObjectLightSerializerMixin, + RelatedExportSerializer): + owner = UserRelatedField() + status = SlugRelatedField(slug_field="name") + user_story = SlugRelatedField(slug_field="ref") + milestone = SlugRelatedField(slug_field="name") + assigned_to = UserRelatedField() + modified_date = DateTimeField() + created_date = DateTimeField() + finished_date = DateTimeField() + ref = Field() + subject = Field() + us_order = Field() + taskboard_order = Field() + description = Field() + is_iocaine = Field() + external_reference = Field() + version = Field() + blocked_note = Field() + is_blocked = Field() + tags = Field() def custom_attributes_queryset(self, project): if project.id not in _custom_tasks_attributes_cache: @@ -248,19 +200,35 @@ class TaskExportSerializer(CustomAttributesValuesExportSerializerMixin, HistoryE return _custom_tasks_attributes_cache[project.id] -class UserStoryExportSerializer(CustomAttributesValuesExportSerializerMixin, HistoryExportSerializerMixin, - AttachmentExportSerializerMixin, WatcheableObjectModelSerializerMixin): - role_points = RolePointsExportSerializer(many=True, required=False) - owner = UserRelatedField(required=False) - assigned_to = UserRelatedField(required=False) - status = ProjectRelatedField(slug_field="name") - milestone = ProjectRelatedField(slug_field="name", required=False) - modified_date = serializers.DateTimeField(required=False) - generated_from_issue = ProjectRelatedField(slug_field="ref", required=False) - - class Meta: - model = userstories_models.UserStory - exclude = ('id', 'project', 'points', 'tasks') +class UserStoryExportSerializer(CustomAttributesValuesExportSerializerMixin, + HistoryExportSerializerMixin, + AttachmentExportSerializerMixin, + WatcheableObjectLightSerializerMixin, + RelatedExportSerializer): + role_points = RolePointsExportSerializer(many=True) + owner = UserRelatedField() + assigned_to = UserRelatedField() + status = SlugRelatedField(slug_field="name") + milestone = SlugRelatedField(slug_field="name") + modified_date = DateTimeField() + created_date = DateTimeField() + finish_date = DateTimeField() + generated_from_issue = SlugRelatedField(slug_field="ref") + ref = Field() + is_closed = Field() + backlog_order = Field() + sprint_order = Field() + kanban_order = Field() + subject = Field() + description = Field() + client_requirement = Field() + team_requirement = Field() + external_reference = Field() + tribe_gig = Field() + version = Field() + blocked_note = Field() + is_blocked = Field() + tags = Field() def custom_attributes_queryset(self, project): if project.id not in _custom_userstories_attributes_cache: @@ -270,21 +238,31 @@ class UserStoryExportSerializer(CustomAttributesValuesExportSerializerMixin, His return _custom_userstories_attributes_cache[project.id] -class IssueExportSerializer(CustomAttributesValuesExportSerializerMixin, HistoryExportSerializerMixin, - AttachmentExportSerializerMixin, WatcheableObjectModelSerializerMixin): - owner = UserRelatedField(required=False) - status = ProjectRelatedField(slug_field="name") - assigned_to = UserRelatedField(required=False) - priority = ProjectRelatedField(slug_field="name") - severity = ProjectRelatedField(slug_field="name") - type = ProjectRelatedField(slug_field="name") - milestone = ProjectRelatedField(slug_field="name", required=False) - votes = serializers.SerializerMethodField("get_votes") - modified_date = serializers.DateTimeField(required=False) +class IssueExportSerializer(CustomAttributesValuesExportSerializerMixin, + HistoryExportSerializerMixin, + AttachmentExportSerializerMixin, + WatcheableObjectLightSerializerMixin, + RelatedExportSerializer): + owner = UserRelatedField() + status = SlugRelatedField(slug_field="name") + assigned_to = UserRelatedField() + priority = SlugRelatedField(slug_field="name") + severity = SlugRelatedField(slug_field="name") + type = SlugRelatedField(slug_field="name") + milestone = SlugRelatedField(slug_field="name") + votes = MethodField("get_votes") + modified_date = DateTimeField() + created_date = DateTimeField() + finished_date = DateTimeField() - class Meta: - model = issues_models.Issue - exclude = ('id', 'project') + ref = Field() + subject = Field() + description = Field() + external_reference = Field() + version = Field() + blocked_note = Field() + is_blocked = Field() + tags = Field() def get_votes(self, obj): return [x.email for x in votes_service.get_voters(obj)] @@ -295,65 +273,93 @@ class IssueExportSerializer(CustomAttributesValuesExportSerializerMixin, History return _custom_issues_attributes_cache[project.id] -class WikiPageExportSerializer(HistoryExportSerializerMixin, AttachmentExportSerializerMixin, - WatcheableObjectModelSerializerMixin): - owner = UserRelatedField(required=False) - last_modifier = UserRelatedField(required=False) - modified_date = serializers.DateTimeField(required=False) - - class Meta: - model = wiki_models.WikiPage - exclude = ('id', 'project') +class WikiPageExportSerializer(HistoryExportSerializerMixin, + AttachmentExportSerializerMixin, + WatcheableObjectLightSerializerMixin, + RelatedExportSerializer): + slug = Field() + owner = UserRelatedField() + last_modifier = UserRelatedField() + modified_date = DateTimeField() + created_date = DateTimeField() + content = Field() + version = Field() -class WikiLinkExportSerializer(serializers.ModelSerializer): - class Meta: - model = wiki_models.WikiLink - exclude = ('id', 'project') +class WikiLinkExportSerializer(RelatedExportSerializer): + title = Field() + href = Field() + order = Field() -class TimelineExportSerializer(serializers.ModelSerializer): +class TimelineExportSerializer(RelatedExportSerializer): data = TimelineDataField() data_content_type = ContentTypeField() - - class Meta: - model = timeline_models.Timeline - exclude = ('id', 'project', 'namespace', 'object_id', 'content_type') + event_type = Field() + created = DateTimeField() -class ProjectExportSerializer(WatcheableObjectModelSerializerMixin): - logo = FileField(required=False) - anon_permissions = PgArrayField(required=False) - public_permissions = PgArrayField(required=False) - modified_date = serializers.DateTimeField(required=False) - roles = RoleExportSerializer(many=True, required=False) - owner = UserRelatedField(required=False) - memberships = MembershipExportSerializer(many=True, required=False) - points = PointsExportSerializer(many=True, required=False) - us_statuses = UserStoryStatusExportSerializer(many=True, required=False) - task_statuses = TaskStatusExportSerializer(many=True, required=False) - issue_types = IssueTypeExportSerializer(many=True, required=False) - issue_statuses = IssueStatusExportSerializer(many=True, required=False) - priorities = PriorityExportSerializer(many=True, required=False) - severities = SeverityExportSerializer(many=True, required=False) - tags_colors = JsonField(required=False) - default_points = serializers.SlugRelatedField(slug_field="name", required=False) - default_us_status = serializers.SlugRelatedField(slug_field="name", required=False) - default_task_status = serializers.SlugRelatedField(slug_field="name", required=False) - default_priority = serializers.SlugRelatedField(slug_field="name", required=False) - default_severity = serializers.SlugRelatedField(slug_field="name", required=False) - default_issue_status = serializers.SlugRelatedField(slug_field="name", required=False) - default_issue_type = serializers.SlugRelatedField(slug_field="name", required=False) - userstorycustomattributes = UserStoryCustomAttributeExportSerializer(many=True, required=False) - taskcustomattributes = TaskCustomAttributeExportSerializer(many=True, required=False) - issuecustomattributes = IssueCustomAttributeExportSerializer(many=True, required=False) - user_stories = UserStoryExportSerializer(many=True, required=False) - tasks = TaskExportSerializer(many=True, required=False) - milestones = MilestoneExportSerializer(many=True, required=False) - issues = IssueExportSerializer(many=True, required=False) - wiki_links = WikiLinkExportSerializer(many=True, required=False) - wiki_pages = WikiPageExportSerializer(many=True, required=False) - - class Meta: - model = projects_models.Project - exclude = ('id', 'creation_template', 'members') +class ProjectExportSerializer(WatcheableObjectLightSerializerMixin): + name = Field() + slug = Field() + description = Field() + created_date = DateTimeField() + logo = FileField() + total_milestones = Field() + total_story_points = Field() + is_backlog_activated = Field() + is_kanban_activated = Field() + is_wiki_activated = Field() + is_issues_activated = Field() + videoconferences = Field() + videoconferences_extra_data = Field() + creation_template = SlugRelatedField(slug_field="slug") + is_private = Field() + is_featured = Field() + is_looking_for_people = Field() + looking_for_people_note = Field() + userstories_csv_uuid = Field() + tasks_csv_uuid = Field() + issues_csv_uuid = Field() + transfer_token = Field() + blocked_code = Field() + totals_updated_datetime = DateTimeField() + total_fans = Field() + total_fans_last_week = Field() + total_fans_last_month = Field() + total_fans_last_year = Field() + total_activity = Field() + total_activity_last_week = Field() + total_activity_last_month = Field() + total_activity_last_year = Field() + anon_permissions = Field() + public_permissions = Field() + modified_date = DateTimeField() + roles = RoleExportSerializer(many=True) + owner = UserRelatedField() + memberships = MembershipExportSerializer(many=True) + points = PointsExportSerializer(many=True) + us_statuses = UserStoryStatusExportSerializer(many=True) + task_statuses = TaskStatusExportSerializer(many=True) + issue_types = IssueTypeExportSerializer(many=True) + issue_statuses = IssueStatusExportSerializer(many=True) + priorities = PriorityExportSerializer(many=True) + severities = SeverityExportSerializer(many=True) + tags_colors = Field() + default_points = SlugRelatedField(slug_field="name") + default_us_status = SlugRelatedField(slug_field="name") + default_task_status = SlugRelatedField(slug_field="name") + default_priority = SlugRelatedField(slug_field="name") + default_severity = SlugRelatedField(slug_field="name") + default_issue_status = SlugRelatedField(slug_field="name") + default_issue_type = SlugRelatedField(slug_field="name") + userstorycustomattributes = UserStoryCustomAttributeExportSerializer(many=True) + taskcustomattributes = TaskCustomAttributeExportSerializer(many=True) + issuecustomattributes = IssueCustomAttributeExportSerializer(many=True) + user_stories = UserStoryExportSerializer(many=True) + tasks = TaskExportSerializer(many=True) + milestones = MilestoneExportSerializer(many=True) + issues = IssueExportSerializer(many=True) + wiki_links = WikiLinkExportSerializer(many=True) + wiki_pages = WikiPageExportSerializer(many=True) + tags = Field() diff --git a/taiga/export_import/services/render.py b/taiga/export_import/services/render.py index 923647a7..0b56f3f5 100644 --- a/taiga/export_import/services/render.py +++ b/taiga/export_import/services/render.py @@ -19,49 +19,44 @@ # This makes all code that import services works and # is not the baddest practice ;) -import base64 import gc -import os - -from django.core.files.storage import default_storage from taiga.base.utils import json +from taiga.base.fields import MethodField from taiga.timeline.service import get_project_timeline from taiga.base.api.fields import get_component from .. import serializers -def render_project(project, outfile, chunk_size = 8190): +def render_project(project, outfile, chunk_size=8190): serializer = serializers.ProjectExportSerializer(project) outfile.write(b'{\n') first_field = True - for field_name in serializer.fields.keys(): + for field_name in serializer._field_map.keys(): # Avoid writing "," in the last element if not first_field: outfile.write(b",\n") else: first_field = False - field = serializer.fields.get(field_name) - field.initialize(parent=serializer, field_name=field_name) + field = serializer._field_map.get(field_name) + # field.initialize(parent=serializer, field_name=field_name) # These four "special" fields hava attachments so we use them in a special way if field_name in ["wiki_pages", "user_stories", "tasks", "issues"]: value = get_component(project, field_name) if field_name != "wiki_pages": - value = value.select_related('owner', 'status', 'milestone', 'project', 'assigned_to', 'custom_attributes_values') + value = value.select_related('owner', 'status', 'milestone', + 'project', 'assigned_to', + 'custom_attributes_values') if field_name == "issues": value = value.select_related('severity', 'priority', 'type') value = value.prefetch_related('history_entry', 'attachments') outfile.write('"{}": [\n'.format(field_name).encode()) - attachments_field = field.fields.pop("attachments", None) - if attachments_field: - attachments_field.initialize(parent=field, field_name="attachments") - first_item = True for item in value.iterator(): # Avoid writing "," in the last element @@ -70,47 +65,18 @@ def render_project(project, outfile, chunk_size = 8190): else: first_item = False - - dumped_value = json.dumps(field.to_native(item)) - writing_value = dumped_value[:-1]+ ',\n "attachments": [\n' - outfile.write(writing_value.encode()) - - first_attachment = True - for attachment in item.attachments.iterator(): - # Avoid writing "," in the last element - if not first_attachment: - outfile.write(b",\n") - else: - first_attachment = False - - # Write all the data expect the serialized file - attachment_serializer = serializers.AttachmentExportSerializer(instance=attachment) - attached_file_serializer = attachment_serializer.fields.pop("attached_file") - dumped_value = json.dumps(attachment_serializer.data) - dumped_value = dumped_value[:-1] + ',\n "attached_file":{\n "data":"' - outfile.write(dumped_value.encode()) - - # We write the attached_files by chunks so the memory used is not increased - attachment_file = attachment.attached_file - if default_storage.exists(attachment_file.name): - with default_storage.open(attachment_file.name) as f: - while True: - bin_data = f.read(chunk_size) - if not bin_data: - break - - b64_data = base64.b64encode(bin_data) - outfile.write(b64_data) - - outfile.write('", \n "name":"{}"}}\n}}'.format( - os.path.basename(attachment_file.name)).encode()) - - outfile.write(b']}') + field.many = False + dumped_value = json.dumps(field.to_value(item)) + outfile.write(dumped_value.encode()) outfile.flush() gc.collect() outfile.write(b']') else: - value = field.field_to_native(project, field_name) + if isinstance(field, MethodField): + value = field.as_getter(field_name, serializers.ProjectExportSerializer)(serializer, project) + else: + attr = getattr(project, field_name) + value = field.to_value(attr) outfile.write('"{}": {}'.format(field_name, json.dumps(value)).encode()) # Generate the timeline @@ -127,4 +93,3 @@ def render_project(project, outfile, chunk_size = 8190): outfile.write(dumped_value.encode()) outfile.write(b']}\n') - diff --git a/taiga/export_import/services/store.py b/taiga/export_import/services/store.py index 5d71c445..9739bb1e 100644 --- a/taiga/export_import/services/store.py +++ b/taiga/export_import/services/store.py @@ -39,7 +39,7 @@ from taiga.timeline.service import build_project_namespace from taiga.users import services as users_service from .. import exceptions as err -from .. import serializers +from .. import validators ######################################################################## @@ -90,13 +90,13 @@ def store_project(data): if key not in excluded_fields: project_data[key] = value - serialized = serializers.ProjectExportSerializer(data=project_data) - if serialized.is_valid(): - serialized.object._importing = True - serialized.object.save() - serialized.save_watchers() - return serialized - add_errors("project", serialized.errors) + validator = validators.ProjectExportValidator(data=project_data) + if validator.is_valid(): + validator.object._importing = True + validator.object.save() + validator.save_watchers() + return validator + add_errors("project", validator.errors) return None @@ -133,54 +133,55 @@ def _store_custom_attributes_values(obj, data_values, obj_field, serializer_clas def _store_attachment(project, obj, attachment): - serialized = serializers.AttachmentExportSerializer(data=attachment) - if serialized.is_valid(): - serialized.object.content_type = ContentType.objects.get_for_model(obj.__class__) - serialized.object.object_id = obj.id - serialized.object.project = project - if serialized.object.owner is None: - serialized.object.owner = serialized.object.project.owner - serialized.object._importing = True - serialized.object.size = serialized.object.attached_file.size - serialized.object.name = os.path.basename(serialized.object.attached_file.name) - serialized.save() - return serialized - add_errors("attachments", serialized.errors) - return serialized + validator = validators.AttachmentExportValidator(data=attachment) + if validator.is_valid(): + validator.object.content_type = ContentType.objects.get_for_model(obj.__class__) + validator.object.object_id = obj.id + validator.object.project = project + if validator.object.owner is None: + validator.object.owner = validator.object.project.owner + validator.object._importing = True + validator.object.size = validator.object.attached_file.size + validator.object.name = os.path.basename(validator.object.attached_file.name) + validator.save() + return validator + add_errors("attachments", validator.errors) + return validator def _store_history(project, obj, history): - serialized = serializers.HistoryExportSerializer(data=history, context={"project": project}) - if serialized.is_valid(): - serialized.object.key = make_key_from_model_object(obj) - if serialized.object.diff is None: - serialized.object.diff = [] - serialized.object._importing = True - serialized.save() - return serialized - add_errors("history", serialized.errors) - return serialized + validator = validators.HistoryExportValidator(data=history, context={"project": project}) + if validator.is_valid(): + validator.object.key = make_key_from_model_object(obj) + if validator.object.diff is None: + validator.object.diff = [] + validator.object.project_id = project.id + validator.object._importing = True + validator.save() + return validator + add_errors("history", validator.errors) + return validator ## ROLES def _store_role(project, role): - serialized = serializers.RoleExportSerializer(data=role) - if serialized.is_valid(): - serialized.object.project = project - serialized.object._importing = True - serialized.save() - return serialized - add_errors("roles", serialized.errors) + validator = validators.RoleExportValidator(data=role) + if validator.is_valid(): + validator.object.project = project + validator.object._importing = True + validator.save() + return validator + add_errors("roles", validator.errors) return None def store_roles(project, data): results = [] for role in data.get("roles", []): - serialized = _store_role(project, role) - if serialized: - results.append(serialized) + validator = _store_role(project, role) + if validator: + results.append(validator) return results @@ -188,17 +189,17 @@ def store_roles(project, data): ## MEMGERSHIPS def _store_membership(project, membership): - serialized = serializers.MembershipExportSerializer(data=membership, context={"project": project}) - if serialized.is_valid(): - serialized.object.project = project - serialized.object._importing = True - serialized.object.token = str(uuid.uuid1()) - serialized.object.user = find_invited_user(serialized.object.email, - default=serialized.object.user) - serialized.save() - return serialized + validator = validators.MembershipExportValidator(data=membership, context={"project": project}) + if validator.is_valid(): + validator.object.project = project + validator.object._importing = True + validator.object.token = str(uuid.uuid1()) + validator.object.user = find_invited_user(validator.object.email, + default=validator.object.user) + validator.save() + return validator - add_errors("memberships", serialized.errors) + add_errors("memberships", validator.errors) return None @@ -212,13 +213,13 @@ def store_memberships(project, data): ## PROJECT ATTRIBUTES def _store_project_attribute_value(project, data, field, serializer): - serialized = serializer(data=data) - if serialized.is_valid(): - serialized.object.project = project - serialized.object._importing = True - serialized.save() - return serialized.object - add_errors(field, serialized.errors) + validator = serializer(data=data) + if validator.is_valid(): + validator.object.project = project + validator.object._importing = True + validator.save() + return validator.object + add_errors(field, validator.errors) return None @@ -253,13 +254,13 @@ def store_default_project_attributes_values(project, data): ## CUSTOM ATTRIBUTES def _store_custom_attribute(project, data, field, serializer): - serialized = serializer(data=data) - if serialized.is_valid(): - serialized.object.project = project - serialized.object._importing = True - serialized.save() - return serialized.object - add_errors(field, serialized.errors) + validator = serializer(data=data) + if validator.is_valid(): + validator.object.project = project + validator.object._importing = True + validator.save() + return validator.object + add_errors(field, validator.errors) return None @@ -273,19 +274,19 @@ def store_custom_attributes(project, data, field, serializer): ## MILESTONE def store_milestone(project, milestone): - serialized = serializers.MilestoneExportSerializer(data=milestone, project=project) - if serialized.is_valid(): - serialized.object.project = project - serialized.object._importing = True - serialized.save() - serialized.save_watchers() + validator = validators.MilestoneExportValidator(data=milestone, project=project) + if validator.is_valid(): + validator.object.project = project + validator.object._importing = True + validator.save() + validator.save_watchers() for task_without_us in milestone.get("tasks_without_us", []): task_without_us["user_story"] = None store_task(project, task_without_us) - return serialized + return validator - add_errors("milestones", serialized.errors) + add_errors("milestones", validator.errors) return None @@ -300,20 +301,20 @@ def store_milestones(project, data): ## USER STORIES def _store_role_point(project, us, role_point): - serialized = serializers.RolePointsExportSerializer(data=role_point, context={"project": project}) - if serialized.is_valid(): + validator = validators.RolePointsExportValidator(data=role_point, context={"project": project}) + if validator.is_valid(): try: - existing_role_point = us.role_points.get(role=serialized.object.role) - existing_role_point.points = serialized.object.points + existing_role_point = us.role_points.get(role=validator.object.role) + existing_role_point.points = validator.object.points existing_role_point.save() return existing_role_point except RolePoints.DoesNotExist: - serialized.object.user_story = us - serialized.save() - return serialized.object + validator.object.user_story = us + validator.save() + return validator.object - add_errors("role_points", serialized.errors) + add_errors("role_points", validator.errors) return None def store_user_story(project, data): @@ -322,51 +323,51 @@ def store_user_story(project, data): us_data = {key: value for key, value in data.items() if key not in ["role_points", "custom_attributes_values"]} - serialized = serializers.UserStoryExportSerializer(data=us_data, context={"project": project}) + validator = validators.UserStoryExportValidator(data=us_data, context={"project": project}) - if serialized.is_valid(): - serialized.object.project = project - if serialized.object.owner is None: - serialized.object.owner = serialized.object.project.owner - serialized.object._importing = True - serialized.object._not_notify = True + if validator.is_valid(): + validator.object.project = project + if validator.object.owner is None: + validator.object.owner = validator.object.project.owner + validator.object._importing = True + validator.object._not_notify = True - serialized.save() - serialized.save_watchers() + validator.save() + validator.save_watchers() - if serialized.object.ref: + if validator.object.ref: sequence_name = refs.make_sequence_name(project) if not seq.exists(sequence_name): seq.create(sequence_name) - seq.set_max(sequence_name, serialized.object.ref) + seq.set_max(sequence_name, validator.object.ref) else: - serialized.object.ref, _ = refs.make_reference(serialized.object, project) - serialized.object.save() + validator.object.ref, _ = refs.make_reference(validator.object, project) + validator.object.save() for us_attachment in data.get("attachments", []): - _store_attachment(project, serialized.object, us_attachment) + _store_attachment(project, validator.object, us_attachment) for role_point in data.get("role_points", []): - _store_role_point(project, serialized.object, role_point) + _store_role_point(project, validator.object, role_point) history_entries = data.get("history", []) for history in history_entries: - _store_history(project, serialized.object, history) + _store_history(project, validator.object, history) if not history_entries: - take_snapshot(serialized.object, user=serialized.object.owner) + take_snapshot(validator.object, user=validator.object.owner) custom_attributes_values = data.get("custom_attributes_values", None) if custom_attributes_values: - custom_attributes = serialized.object.project.userstorycustomattributes.all().values('id', 'name') + custom_attributes = validator.object.project.userstorycustomattributes.all().values('id', 'name') custom_attributes_values = _use_id_instead_name_as_key_in_custom_attributes_values( custom_attributes, custom_attributes_values) - _store_custom_attributes_values(serialized.object, custom_attributes_values, - "user_story", serializers.UserStoryCustomAttributesValuesExportSerializer) + _store_custom_attributes_values(validator.object, custom_attributes_values, + "user_story", validators.UserStoryCustomAttributesValuesExportValidator) - return serialized + return validator - add_errors("user_stories", serialized.errors) + add_errors("user_stories", validator.errors) return None @@ -384,47 +385,47 @@ def store_task(project, data): if "status" not in data and project.default_task_status: data["status"] = project.default_task_status.name - serialized = serializers.TaskExportSerializer(data=data, context={"project": project}) - if serialized.is_valid(): - serialized.object.project = project - if serialized.object.owner is None: - serialized.object.owner = serialized.object.project.owner - serialized.object._importing = True - serialized.object._not_notify = True + validator = validators.TaskExportValidator(data=data, context={"project": project}) + if validator.is_valid(): + validator.object.project = project + if validator.object.owner is None: + validator.object.owner = validator.object.project.owner + validator.object._importing = True + validator.object._not_notify = True - serialized.save() - serialized.save_watchers() + validator.save() + validator.save_watchers() - if serialized.object.ref: + if validator.object.ref: sequence_name = refs.make_sequence_name(project) if not seq.exists(sequence_name): seq.create(sequence_name) - seq.set_max(sequence_name, serialized.object.ref) + seq.set_max(sequence_name, validator.object.ref) else: - serialized.object.ref, _ = refs.make_reference(serialized.object, project) - serialized.object.save() + validator.object.ref, _ = refs.make_reference(validator.object, project) + validator.object.save() for task_attachment in data.get("attachments", []): - _store_attachment(project, serialized.object, task_attachment) + _store_attachment(project, validator.object, task_attachment) history_entries = data.get("history", []) for history in history_entries: - _store_history(project, serialized.object, history) + _store_history(project, validator.object, history) if not history_entries: - take_snapshot(serialized.object, user=serialized.object.owner) + take_snapshot(validator.object, user=validator.object.owner) custom_attributes_values = data.get("custom_attributes_values", None) if custom_attributes_values: - custom_attributes = serialized.object.project.taskcustomattributes.all().values('id', 'name') + custom_attributes = validator.object.project.taskcustomattributes.all().values('id', 'name') custom_attributes_values = _use_id_instead_name_as_key_in_custom_attributes_values( custom_attributes, custom_attributes_values) - _store_custom_attributes_values(serialized.object, custom_attributes_values, - "task", serializers.TaskCustomAttributesValuesExportSerializer) + _store_custom_attributes_values(validator.object, custom_attributes_values, + "task", validators.TaskCustomAttributesValuesExportValidator) - return serialized + return validator - add_errors("tasks", serialized.errors) + add_errors("tasks", validator.errors) return None @@ -439,7 +440,7 @@ def store_tasks(project, data): ## ISSUES def store_issue(project, data): - serialized = serializers.IssueExportSerializer(data=data, context={"project": project}) + validator = validators.IssueExportValidator(data=data, context={"project": project}) if "type" not in data and project.default_issue_type: data["type"] = project.default_issue_type.name @@ -453,46 +454,46 @@ def store_issue(project, data): if "severity" not in data and project.default_severity: data["severity"] = project.default_severity.name - if serialized.is_valid(): - serialized.object.project = project - if serialized.object.owner is None: - serialized.object.owner = serialized.object.project.owner - serialized.object._importing = True - serialized.object._not_notify = True + if validator.is_valid(): + validator.object.project = project + if validator.object.owner is None: + validator.object.owner = validator.object.project.owner + validator.object._importing = True + validator.object._not_notify = True - serialized.save() - serialized.save_watchers() + validator.save() + validator.save_watchers() - if serialized.object.ref: + if validator.object.ref: sequence_name = refs.make_sequence_name(project) if not seq.exists(sequence_name): seq.create(sequence_name) - seq.set_max(sequence_name, serialized.object.ref) + seq.set_max(sequence_name, validator.object.ref) else: - serialized.object.ref, _ = refs.make_reference(serialized.object, project) - serialized.object.save() + validator.object.ref, _ = refs.make_reference(validator.object, project) + validator.object.save() for attachment in data.get("attachments", []): - _store_attachment(project, serialized.object, attachment) + _store_attachment(project, validator.object, attachment) history_entries = data.get("history", []) for history in history_entries: - _store_history(project, serialized.object, history) + _store_history(project, validator.object, history) if not history_entries: - take_snapshot(serialized.object, user=serialized.object.owner) + take_snapshot(validator.object, user=validator.object.owner) custom_attributes_values = data.get("custom_attributes_values", None) if custom_attributes_values: - custom_attributes = serialized.object.project.issuecustomattributes.all().values('id', 'name') + custom_attributes = validator.object.project.issuecustomattributes.all().values('id', 'name') custom_attributes_values = _use_id_instead_name_as_key_in_custom_attributes_values( custom_attributes, custom_attributes_values) - _store_custom_attributes_values(serialized.object, custom_attributes_values, - "issue", serializers.IssueCustomAttributesValuesExportSerializer) + _store_custom_attributes_values(validator.object, custom_attributes_values, + "issue", validators.IssueCustomAttributesValuesExportValidator) - return serialized + return validator - add_errors("issues", serialized.errors) + add_errors("issues", validator.errors) return None @@ -507,29 +508,29 @@ def store_issues(project, data): def store_wiki_page(project, wiki_page): wiki_page["slug"] = slugify(unidecode(wiki_page.get("slug", ""))) - serialized = serializers.WikiPageExportSerializer(data=wiki_page) - if serialized.is_valid(): - serialized.object.project = project - if serialized.object.owner is None: - serialized.object.owner = serialized.object.project.owner - serialized.object._importing = True - serialized.object._not_notify = True - serialized.save() - serialized.save_watchers() + validator = validators.WikiPageExportValidator(data=wiki_page) + if validator.is_valid(): + validator.object.project = project + if validator.object.owner is None: + validator.object.owner = validator.object.project.owner + validator.object._importing = True + validator.object._not_notify = True + validator.save() + validator.save_watchers() for attachment in wiki_page.get("attachments", []): - _store_attachment(project, serialized.object, attachment) + _store_attachment(project, validator.object, attachment) history_entries = wiki_page.get("history", []) for history in history_entries: - _store_history(project, serialized.object, history) + _store_history(project, validator.object, history) if not history_entries: - take_snapshot(serialized.object, user=serialized.object.owner) + take_snapshot(validator.object, user=validator.object.owner) - return serialized + return validator - add_errors("wiki_pages", serialized.errors) + add_errors("wiki_pages", validator.errors) return None @@ -543,14 +544,14 @@ def store_wiki_pages(project, data): ## WIKI LINKS def store_wiki_link(project, wiki_link): - serialized = serializers.WikiLinkExportSerializer(data=wiki_link) - if serialized.is_valid(): - serialized.object.project = project - serialized.object._importing = True - serialized.save() - return serialized + validator = validators.WikiLinkExportValidator(data=wiki_link) + if validator.is_valid(): + validator.object.project = project + validator.object._importing = True + validator.save() + return validator - add_errors("wiki_links", serialized.errors) + add_errors("wiki_links", validator.errors) return None @@ -572,17 +573,17 @@ def store_tags_colors(project, data): ## TIMELINE def _store_timeline_entry(project, timeline): - serialized = serializers.TimelineExportSerializer(data=timeline, context={"project": project}) - if serialized.is_valid(): - serialized.object.project = project - serialized.object.namespace = build_project_namespace(project) - serialized.object.object_id = project.id - serialized.object.content_type = ContentType.objects.get_for_model(project.__class__) - serialized.object._importing = True - serialized.save() - return serialized - add_errors("timeline", serialized.errors) - return serialized + validator = validators.TimelineExportValidator(data=timeline, context={"project": project}) + if validator.is_valid(): + validator.object.project = project + validator.object.namespace = build_project_namespace(project) + validator.object.object_id = project.id + validator.object.content_type = ContentType.objects.get_for_model(project.__class__) + validator.object._importing = True + validator.save() + return validator + add_errors("timeline", validator.errors) + return validator def store_timeline_entries(project, data): @@ -617,13 +618,13 @@ def _validate_if_owner_have_enought_space_to_this_project(owner, data): def _create_project_object(data): # Create the project - project_serialized = store_project(data) + project_validator = store_project(data) - if not project_serialized: + if not project_validator: errors = get_errors(clear=True) raise err.TaigaImportError(_("error importing project data"), None, errors=errors) - return project_serialized.object if project_serialized else None + return project_validator.object if project_validator else None def _create_membership_for_project_owner(project): @@ -654,13 +655,13 @@ def _populate_project_object(project, data): check_if_there_is_some_error(_("error importing memberships"), project) # Create project attributes values - store_project_attributes_values(project, data, "us_statuses", serializers.UserStoryStatusExportSerializer) - store_project_attributes_values(project, data, "points", serializers.PointsExportSerializer) - store_project_attributes_values(project, data, "task_statuses", serializers.TaskStatusExportSerializer) - store_project_attributes_values(project, data, "issue_types", serializers.IssueTypeExportSerializer) - store_project_attributes_values(project, data, "issue_statuses", serializers.IssueStatusExportSerializer) - store_project_attributes_values(project, data, "priorities", serializers.PriorityExportSerializer) - store_project_attributes_values(project, data, "severities", serializers.SeverityExportSerializer) + store_project_attributes_values(project, data, "us_statuses", validators.UserStoryStatusExportValidator) + store_project_attributes_values(project, data, "points", validators.PointsExportValidator) + store_project_attributes_values(project, data, "task_statuses", validators.TaskStatusExportValidator) + store_project_attributes_values(project, data, "issue_types", validators.IssueTypeExportValidator) + store_project_attributes_values(project, data, "issue_statuses", validators.IssueStatusExportValidator) + store_project_attributes_values(project, data, "priorities", validators.PriorityExportValidator) + store_project_attributes_values(project, data, "severities", validators.SeverityExportValidator) check_if_there_is_some_error(_("error importing lists of project attributes"), project) # Create default values for project attributes @@ -669,11 +670,11 @@ def _populate_project_object(project, data): # Create custom attributes store_custom_attributes(project, data, "userstorycustomattributes", - serializers.UserStoryCustomAttributeExportSerializer) + validators.UserStoryCustomAttributeExportValidator) store_custom_attributes(project, data, "taskcustomattributes", - serializers.TaskCustomAttributeExportSerializer) + validators.TaskCustomAttributeExportValidator) store_custom_attributes(project, data, "issuecustomattributes", - serializers.IssueCustomAttributeExportSerializer) + validators.IssueCustomAttributeExportValidator) check_if_there_is_some_error(_("error importing custom attributes"), project) # Create milestones diff --git a/taiga/export_import/validators/__init__.py b/taiga/export_import/validators/__init__.py new file mode 100644 index 00000000..969a8d0c --- /dev/null +++ b/taiga/export_import/validators/__init__.py @@ -0,0 +1,27 @@ +from .validators import PointsExportValidator +from .validators import UserStoryStatusExportValidator +from .validators import TaskStatusExportValidator +from .validators import IssueStatusExportValidator +from .validators import PriorityExportValidator +from .validators import SeverityExportValidator +from .validators import IssueTypeExportValidator +from .validators import RoleExportValidator +from .validators import UserStoryCustomAttributeExportValidator +from .validators import TaskCustomAttributeExportValidator +from .validators import IssueCustomAttributeExportValidator +from .validators import BaseCustomAttributesValuesExportValidator +from .validators import UserStoryCustomAttributesValuesExportValidator +from .validators import TaskCustomAttributesValuesExportValidator +from .validators import IssueCustomAttributesValuesExportValidator +from .validators import MembershipExportValidator +from .validators import RolePointsExportValidator +from .validators import MilestoneExportValidator +from .validators import TaskExportValidator +from .validators import UserStoryExportValidator +from .validators import IssueExportValidator +from .validators import WikiPageExportValidator +from .validators import WikiLinkExportValidator +from .validators import TimelineExportValidator +from .validators import ProjectExportValidator +from .mixins import AttachmentExportValidator +from .mixins import HistoryExportValidator diff --git a/taiga/export_import/validators/cache.py b/taiga/export_import/validators/cache.py new file mode 100644 index 00000000..c4eb5bfa --- /dev/null +++ b/taiga/export_import/validators/cache.py @@ -0,0 +1,42 @@ +# -*- coding: utf-8 -*- +# Copyright (C) 2014-2016 Andrey Antukh +# Copyright (C) 2014-2016 Jesús Espino +# Copyright (C) 2014-2016 David Barragán +# Copyright (C) 2014-2016 Alejandro Alonso +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +from taiga.users import models as users_models + +_cache_user_by_pk = {} +_cache_user_by_email = {} +_custom_tasks_attributes_cache = {} +_custom_issues_attributes_cache = {} +_custom_userstories_attributes_cache = {} + + +def cached_get_user_by_pk(pk): + if pk not in _cache_user_by_pk: + try: + _cache_user_by_pk[pk] = users_models.User.objects.get(pk=pk) + except Exception: + _cache_user_by_pk[pk] = users_models.User.objects.get(pk=pk) + return _cache_user_by_pk[pk] + +def cached_get_user_by_email(email): + if email not in _cache_user_by_email: + try: + _cache_user_by_email[email] = users_models.User.objects.get(email=email) + except Exception: + _cache_user_by_email[email] = users_models.User.objects.get(email=email) + return _cache_user_by_email[email] diff --git a/taiga/export_import/validators/fields.py b/taiga/export_import/validators/fields.py new file mode 100644 index 00000000..e3d33c7a --- /dev/null +++ b/taiga/export_import/validators/fields.py @@ -0,0 +1,196 @@ +# -*- coding: utf-8 -*- +# Copyright (C) 2014-2016 Andrey Antukh +# Copyright (C) 2014-2016 Jesús Espino +# Copyright (C) 2014-2016 David Barragán +# Copyright (C) 2014-2016 Alejandro Alonso +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import base64 +import copy + +from django.core.files.base import ContentFile +from django.core.exceptions import ObjectDoesNotExist +from django.utils.translation import ugettext as _ +from django.contrib.contenttypes.models import ContentType + +from taiga.base.api import serializers +from taiga.base.exceptions import ValidationError +from taiga.base.fields import JsonField +from taiga.mdrender.service import render as mdrender +from taiga.users import models as users_models + +from .cache import cached_get_user_by_email + + +class FileField(serializers.WritableField): + read_only = False + + def from_native(self, data): + if not data: + return None + + decoded_data = b'' + # The original file was encoded by chunks but we don't really know its + # length or if it was multiple of 3 so we must iterate over all those chunks + # decoding them one by one + for decoding_chunk in data['data'].split("="): + # When encoding to base64 3 bytes are transformed into 4 bytes and + # the extra space of the block is filled with = + # We must ensure that the decoding chunk has a length multiple of 4 so + # we restore the stripped '='s adding appending them until the chunk has + # a length multiple of 4 + decoding_chunk += "=" * (-len(decoding_chunk) % 4) + decoded_data += base64.b64decode(decoding_chunk + "=") + + return ContentFile(decoded_data, name=data['name']) + + +class ContentTypeField(serializers.RelatedField): + read_only = False + + def from_native(self, data): + try: + return ContentType.objects.get_by_natural_key(*data) + except Exception: + return None + + +class RelatedNoneSafeField(serializers.RelatedField): + def field_from_native(self, data, files, field_name, into): + if self.read_only: + return + + try: + if self.many: + try: + # Form data + value = data.getlist(field_name) + if value == [''] or value == []: + raise KeyError + except AttributeError: + # Non-form data + value = data[field_name] + else: + value = data[field_name] + except KeyError: + if self.partial: + return + value = self.get_default_value() + + key = self.source or field_name + if value in self.null_values: + if self.required: + raise ValidationError(self.error_messages['required']) + into[key] = None + elif self.many: + into[key] = [self.from_native(item) for item in value if self.from_native(item) is not None] + else: + into[key] = self.from_native(value) + + +class UserRelatedField(RelatedNoneSafeField): + read_only = False + + def from_native(self, data): + try: + return cached_get_user_by_email(data) + except users_models.User.DoesNotExist: + return None + + +class UserPkField(serializers.RelatedField): + read_only = False + + def from_native(self, data): + try: + user = cached_get_user_by_email(data) + return user.pk + except users_models.User.DoesNotExist: + return None + + +class CommentField(serializers.WritableField): + read_only = False + + def field_from_native(self, data, files, field_name, into): + super().field_from_native(data, files, field_name, into) + into["comment_html"] = mdrender(self.context['project'], data.get("comment", "")) + + +class ProjectRelatedField(serializers.RelatedField): + read_only = False + null_values = (None, "") + + def __init__(self, slug_field, *args, **kwargs): + self.slug_field = slug_field + super().__init__(*args, **kwargs) + + def from_native(self, data): + try: + kwargs = {self.slug_field: data, "project": self.context['project']} + return self.queryset.get(**kwargs) + except ObjectDoesNotExist: + raise ValidationError(_("{}=\"{}\" not found in this project".format(self.slug_field, data))) + + +class HistoryUserField(JsonField): + def from_native(self, data): + if data is None: + return {} + + if len(data) < 2: + return {} + + user = UserRelatedField().from_native(data[0]) + + if user: + pk = user.pk + else: + pk = None + + return {"pk": pk, "name": data[1]} + + +class HistoryValuesField(JsonField): + def from_native(self, data): + if data is None: + return [] + if "users" in data: + data['users'] = list(map(UserPkField().from_native, data['users'])) + return data + + +class HistoryDiffField(JsonField): + def from_native(self, data): + if data is None: + return [] + + if "assigned_to" in data: + data['assigned_to'] = list(map(UserPkField().from_native, data['assigned_to'])) + return data + + +class TimelineDataField(serializers.WritableField): + read_only = False + + def from_native(self, data): + new_data = copy.deepcopy(data) + try: + user = cached_get_user_by_email(new_data["user"]["email"]) + new_data["user"]["id"] = user.id + del new_data["user"]["email"] + except users_models.User.DoesNotExist: + pass + + return new_data diff --git a/taiga/export_import/validators/mixins.py b/taiga/export_import/validators/mixins.py new file mode 100644 index 00000000..d07334b6 --- /dev/null +++ b/taiga/export_import/validators/mixins.py @@ -0,0 +1,97 @@ +# -*- coding: utf-8 -*- +# Copyright (C) 2014-2016 Andrey Antukh +# Copyright (C) 2014-2016 Jesús Espino +# Copyright (C) 2014-2016 David Barragán +# Copyright (C) 2014-2016 Alejandro Alonso +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +from django.contrib.auth import get_user_model +from django.core.exceptions import ObjectDoesNotExist +from django.contrib.contenttypes.models import ContentType + +from taiga.base.api import serializers +from taiga.base.api import validators +from taiga.projects.history import models as history_models +from taiga.projects.attachments import models as attachments_models +from taiga.projects.notifications import services as notifications_services +from taiga.projects.history import services as history_service + +from .fields import (UserRelatedField, HistoryUserField, HistoryDiffField, + JsonField, HistoryValuesField, CommentField, FileField) + + +class HistoryExportValidator(validators.ModelValidator): + user = HistoryUserField() + diff = HistoryDiffField(required=False) + snapshot = JsonField(required=False) + values = HistoryValuesField(required=False) + comment = CommentField(required=False) + delete_comment_date = serializers.DateTimeField(required=False) + delete_comment_user = HistoryUserField(required=False) + + class Meta: + model = history_models.HistoryEntry + exclude = ("id", "comment_html", "key", "project") + + +class AttachmentExportValidator(validators.ModelValidator): + owner = UserRelatedField(required=False) + attached_file = FileField() + modified_date = serializers.DateTimeField(required=False) + + class Meta: + model = attachments_models.Attachment + exclude = ('id', 'content_type', 'object_id', 'project') + + +class WatcheableObjectModelValidatorMixin(validators.ModelValidator): + watchers = UserRelatedField(many=True, required=False) + + def __init__(self, *args, **kwargs): + self._watchers_field = self.base_fields.pop("watchers", None) + super(WatcheableObjectModelValidatorMixin, self).__init__(*args, **kwargs) + + """ + watchers is not a field from the model so we need to do some magic to make it work like a normal field + It's supposed to be represented as an email list but internally it's treated like notifications.Watched instances + """ + + def restore_object(self, attrs, instance=None): + self.fields.pop("watchers", None) + instance = super(WatcheableObjectModelValidatorMixin, self).restore_object(attrs, instance) + self._watchers = self.init_data.get("watchers", []) + return instance + + def save_watchers(self): + new_watcher_emails = set(self._watchers) + old_watcher_emails = set(self.object.get_watchers().values_list("email", flat=True)) + adding_watcher_emails = list(new_watcher_emails.difference(old_watcher_emails)) + removing_watcher_emails = list(old_watcher_emails.difference(new_watcher_emails)) + + User = get_user_model() + adding_users = User.objects.filter(email__in=adding_watcher_emails) + removing_users = User.objects.filter(email__in=removing_watcher_emails) + + for user in adding_users: + notifications_services.add_watcher(self.object, user) + + for user in removing_users: + notifications_services.remove_watcher(self.object, user) + + self.object.watchers = [user.email for user in self.object.get_watchers()] + + def to_native(self, obj): + ret = super(WatcheableObjectModelValidatorMixin, self).to_native(obj) + ret["watchers"] = [user.email for user in obj.get_watchers()] + return ret diff --git a/taiga/export_import/validators/validators.py b/taiga/export_import/validators/validators.py new file mode 100644 index 00000000..818df0c3 --- /dev/null +++ b/taiga/export_import/validators/validators.py @@ -0,0 +1,349 @@ +# -*- coding: utf-8 -*- +# Copyright (C) 2014-2016 Andrey Antukh +# Copyright (C) 2014-2016 Jesús Espino +# Copyright (C) 2014-2016 David Barragán +# Copyright (C) 2014-2016 Alejandro Alonso +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +from django.utils.translation import ugettext as _ + +from taiga.base.api import serializers +from taiga.base.api import validators +from taiga.base.fields import JsonField, PgArrayField +from taiga.base.exceptions import ValidationError + +from taiga.projects import models as projects_models +from taiga.projects.custom_attributes import models as custom_attributes_models +from taiga.projects.userstories import models as userstories_models +from taiga.projects.tasks import models as tasks_models +from taiga.projects.issues import models as issues_models +from taiga.projects.milestones import models as milestones_models +from taiga.projects.wiki import models as wiki_models +from taiga.timeline import models as timeline_models +from taiga.users import models as users_models + +from .fields import (FileField, UserRelatedField, + ProjectRelatedField, + TimelineDataField, ContentTypeField) +from .mixins import WatcheableObjectModelValidatorMixin +from .cache import (_custom_tasks_attributes_cache, + _custom_userstories_attributes_cache, + _custom_issues_attributes_cache) + + +class PointsExportValidator(validators.ModelValidator): + class Meta: + model = projects_models.Points + exclude = ('id', 'project') + + +class UserStoryStatusExportValidator(validators.ModelValidator): + class Meta: + model = projects_models.UserStoryStatus + exclude = ('id', 'project') + + +class TaskStatusExportValidator(validators.ModelValidator): + class Meta: + model = projects_models.TaskStatus + exclude = ('id', 'project') + + +class IssueStatusExportValidator(validators.ModelValidator): + class Meta: + model = projects_models.IssueStatus + exclude = ('id', 'project') + + +class PriorityExportValidator(validators.ModelValidator): + class Meta: + model = projects_models.Priority + exclude = ('id', 'project') + + +class SeverityExportValidator(validators.ModelValidator): + class Meta: + model = projects_models.Severity + exclude = ('id', 'project') + + +class IssueTypeExportValidator(validators.ModelValidator): + class Meta: + model = projects_models.IssueType + exclude = ('id', 'project') + + +class RoleExportValidator(validators.ModelValidator): + permissions = PgArrayField(required=False) + + class Meta: + model = users_models.Role + exclude = ('id', 'project') + + +class UserStoryCustomAttributeExportValidator(validators.ModelValidator): + modified_date = serializers.DateTimeField(required=False) + + class Meta: + model = custom_attributes_models.UserStoryCustomAttribute + exclude = ('id', 'project') + + +class TaskCustomAttributeExportValidator(validators.ModelValidator): + modified_date = serializers.DateTimeField(required=False) + + class Meta: + model = custom_attributes_models.TaskCustomAttribute + exclude = ('id', 'project') + + +class IssueCustomAttributeExportValidator(validators.ModelValidator): + modified_date = serializers.DateTimeField(required=False) + + class Meta: + model = custom_attributes_models.IssueCustomAttribute + exclude = ('id', 'project') + + +class BaseCustomAttributesValuesExportValidator(validators.ModelValidator): + attributes_values = JsonField(source="attributes_values", required=True) + _custom_attribute_model = None + _container_field = None + + class Meta: + exclude = ("id",) + + def validate_attributes_values(self, attrs, source): + # values must be a dict + data_values = attrs.get("attributes_values", None) + if self.object: + data_values = (data_values or self.object.attributes_values) + + if type(data_values) is not dict: + raise ValidationError(_("Invalid content. It must be {\"key\": \"value\",...}")) + + # Values keys must be in the container object project + data_container = attrs.get(self._container_field, None) + if data_container: + project_id = data_container.project_id + elif self.object: + project_id = getattr(self.object, self._container_field).project_id + else: + project_id = None + + values_ids = list(data_values.keys()) + qs = self._custom_attribute_model.objects.filter(project=project_id, + id__in=values_ids) + if qs.count() != len(values_ids): + raise ValidationError(_("It contain invalid custom fields.")) + + return attrs + + +class UserStoryCustomAttributesValuesExportValidator(BaseCustomAttributesValuesExportValidator): + _custom_attribute_model = custom_attributes_models.UserStoryCustomAttribute + _container_model = "userstories.UserStory" + _container_field = "user_story" + + class Meta(BaseCustomAttributesValuesExportValidator.Meta): + model = custom_attributes_models.UserStoryCustomAttributesValues + + +class TaskCustomAttributesValuesExportValidator(BaseCustomAttributesValuesExportValidator): + _custom_attribute_model = custom_attributes_models.TaskCustomAttribute + _container_field = "task" + + class Meta(BaseCustomAttributesValuesExportValidator.Meta): + model = custom_attributes_models.TaskCustomAttributesValues + + +class IssueCustomAttributesValuesExportValidator(BaseCustomAttributesValuesExportValidator): + _custom_attribute_model = custom_attributes_models.IssueCustomAttribute + _container_field = "issue" + + class Meta(BaseCustomAttributesValuesExportValidator.Meta): + model = custom_attributes_models.IssueCustomAttributesValues + + +class MembershipExportValidator(validators.ModelValidator): + user = UserRelatedField(required=False) + role = ProjectRelatedField(slug_field="name") + invited_by = UserRelatedField(required=False) + + class Meta: + model = projects_models.Membership + exclude = ('id', 'project', 'token') + + def full_clean(self, instance): + return instance + + +class RolePointsExportValidator(validators.ModelValidator): + role = ProjectRelatedField(slug_field="name") + points = ProjectRelatedField(slug_field="name") + + class Meta: + model = userstories_models.RolePoints + exclude = ('id', 'user_story') + + +class MilestoneExportValidator(WatcheableObjectModelValidatorMixin): + owner = UserRelatedField(required=False) + modified_date = serializers.DateTimeField(required=False) + estimated_start = serializers.DateField(required=False) + estimated_finish = serializers.DateField(required=False) + + def __init__(self, *args, **kwargs): + project = kwargs.pop('project', None) + super(MilestoneExportValidator, self).__init__(*args, **kwargs) + if project: + self.project = project + + def validate_name(self, attrs, source): + """ + Check the milestone name is not duplicated in the project + """ + name = attrs[source] + qs = self.project.milestones.filter(name=name) + if qs.exists(): + raise ValidationError(_("Name duplicated for the project")) + + return attrs + + class Meta: + model = milestones_models.Milestone + exclude = ('id', 'project') + + +class TaskExportValidator(WatcheableObjectModelValidatorMixin): + owner = UserRelatedField(required=False) + status = ProjectRelatedField(slug_field="name") + user_story = ProjectRelatedField(slug_field="ref", required=False) + milestone = ProjectRelatedField(slug_field="name", required=False) + assigned_to = UserRelatedField(required=False) + modified_date = serializers.DateTimeField(required=False) + + class Meta: + model = tasks_models.Task + exclude = ('id', 'project') + + def custom_attributes_queryset(self, project): + if project.id not in _custom_tasks_attributes_cache: + _custom_tasks_attributes_cache[project.id] = list(project.taskcustomattributes.all().values('id', 'name')) + return _custom_tasks_attributes_cache[project.id] + + +class UserStoryExportValidator(WatcheableObjectModelValidatorMixin): + role_points = RolePointsExportValidator(many=True, required=False) + owner = UserRelatedField(required=False) + assigned_to = UserRelatedField(required=False) + status = ProjectRelatedField(slug_field="name") + milestone = ProjectRelatedField(slug_field="name", required=False) + modified_date = serializers.DateTimeField(required=False) + generated_from_issue = ProjectRelatedField(slug_field="ref", required=False) + + class Meta: + model = userstories_models.UserStory + exclude = ('id', 'project', 'points', 'tasks') + + def custom_attributes_queryset(self, project): + if project.id not in _custom_userstories_attributes_cache: + _custom_userstories_attributes_cache[project.id] = list( + project.userstorycustomattributes.all().values('id', 'name') + ) + return _custom_userstories_attributes_cache[project.id] + + +class IssueExportValidator(WatcheableObjectModelValidatorMixin): + owner = UserRelatedField(required=False) + status = ProjectRelatedField(slug_field="name") + assigned_to = UserRelatedField(required=False) + priority = ProjectRelatedField(slug_field="name") + severity = ProjectRelatedField(slug_field="name") + type = ProjectRelatedField(slug_field="name") + milestone = ProjectRelatedField(slug_field="name", required=False) + modified_date = serializers.DateTimeField(required=False) + + class Meta: + model = issues_models.Issue + exclude = ('id', 'project') + + def custom_attributes_queryset(self, project): + if project.id not in _custom_issues_attributes_cache: + _custom_issues_attributes_cache[project.id] = list(project.issuecustomattributes.all().values('id', 'name')) + return _custom_issues_attributes_cache[project.id] + + +class WikiPageExportValidator(WatcheableObjectModelValidatorMixin): + owner = UserRelatedField(required=False) + last_modifier = UserRelatedField(required=False) + modified_date = serializers.DateTimeField(required=False) + + class Meta: + model = wiki_models.WikiPage + exclude = ('id', 'project') + + +class WikiLinkExportValidator(validators.ModelValidator): + class Meta: + model = wiki_models.WikiLink + exclude = ('id', 'project') + + +class TimelineExportValidator(validators.ModelValidator): + data = TimelineDataField() + data_content_type = ContentTypeField() + + class Meta: + model = timeline_models.Timeline + exclude = ('id', 'project', 'namespace', 'object_id', 'content_type') + + +class ProjectExportValidator(WatcheableObjectModelValidatorMixin): + logo = FileField(required=False) + anon_permissions = PgArrayField(required=False) + public_permissions = PgArrayField(required=False) + modified_date = serializers.DateTimeField(required=False) + roles = RoleExportValidator(many=True, required=False) + owner = UserRelatedField(required=False) + memberships = MembershipExportValidator(many=True, required=False) + points = PointsExportValidator(many=True, required=False) + us_statuses = UserStoryStatusExportValidator(many=True, required=False) + task_statuses = TaskStatusExportValidator(many=True, required=False) + issue_types = IssueTypeExportValidator(many=True, required=False) + issue_statuses = IssueStatusExportValidator(many=True, required=False) + priorities = PriorityExportValidator(many=True, required=False) + severities = SeverityExportValidator(many=True, required=False) + tags_colors = JsonField(required=False) + creation_template = serializers.SlugRelatedField(slug_field="slug", required=False) + default_points = serializers.SlugRelatedField(slug_field="name", required=False) + default_us_status = serializers.SlugRelatedField(slug_field="name", required=False) + default_task_status = serializers.SlugRelatedField(slug_field="name", required=False) + default_priority = serializers.SlugRelatedField(slug_field="name", required=False) + default_severity = serializers.SlugRelatedField(slug_field="name", required=False) + default_issue_status = serializers.SlugRelatedField(slug_field="name", required=False) + default_issue_type = serializers.SlugRelatedField(slug_field="name", required=False) + userstorycustomattributes = UserStoryCustomAttributeExportValidator(many=True, required=False) + taskcustomattributes = TaskCustomAttributeExportValidator(many=True, required=False) + issuecustomattributes = IssueCustomAttributeExportValidator(many=True, required=False) + user_stories = UserStoryExportValidator(many=True, required=False) + tasks = TaskExportValidator(many=True, required=False) + milestones = MilestoneExportValidator(many=True, required=False) + issues = IssueExportValidator(many=True, required=False) + wiki_links = WikiLinkExportValidator(many=True, required=False) + wiki_pages = WikiPageExportValidator(many=True, required=False) + + class Meta: + model = projects_models.Project + exclude = ('id', 'members')