diff --git a/taiga/export_import/serializers.py b/taiga/export_import/serializers.py index 43acb5af..16877ea7 100644 --- a/taiga/export_import/serializers.py +++ b/taiga/export_import/serializers.py @@ -50,6 +50,28 @@ from taiga.projects.notifications import services as notifications_services from taiga.projects.votes import services as votes_service from taiga.projects.history import services as history_service +_cache_user_by_pk = {} +_cache_user_by_email = {} +_custom_tasks_attributes_cache = {} +_custom_issues_attributes_cache = {} +_custom_userstories_attributes_cache = {} + +def cached_get_user_by_pk(pk): + if pk not in _cache_user_by_pk: + try: + _cache_user_by_pk[pk] = users_models.User.objects.get(pk=pk) + except Exception: + _cache_user_by_pk[pk] = users_models.User.objects.get(pk=pk) + return _cache_user_by_pk[pk] + +def cached_get_user_by_email(email): + if email not in _cache_user_by_email: + try: + _cache_user_by_email[email] = users_models.User.objects.get(email=email) + except Exception: + _cache_user_by_email[email] = users_models.User.objects.get(email=email) + return _cache_user_by_email[email] + class FileField(serializers.WritableField): read_only = False @@ -128,7 +150,7 @@ class UserRelatedField(RelatedNoneSafeField): def from_native(self, data): try: - return users_models.User.objects.get(email=data) + return cached_get_user_by_email(data) except users_models.User.DoesNotExist: return None @@ -138,14 +160,14 @@ class UserPkField(serializers.RelatedField): def to_native(self, obj): try: - user = users_models.User.objects.get(pk=obj) + user = cached_get_user_by_pk(obj) return user.email except users_models.User.DoesNotExist: return None def from_native(self, data): try: - user = users_models.User.objects.get(email=data) + user = cached_get_user_by_email(data) return user.pk except users_models.User.DoesNotExist: return None @@ -185,7 +207,7 @@ class HistoryUserField(JsonField): if obj is None or obj == {}: return [] try: - user = users_models.User.objects.get(pk=obj['pk']) + user = cached_get_user_by_pk(obj['pk']) except users_models.User.DoesNotExist: user = None return (UserRelatedField().to_native(user), obj['name']) @@ -420,7 +442,7 @@ class CustomAttributesValuesExportSerializerMixin(serializers.ModelSerializer): try: values = obj.custom_attributes_values.attributes_values - custom_attributes = self.custom_attributes_queryset(obj.project).values('id', 'name') + custom_attributes = self.custom_attributes_queryset(obj.project) return _use_name_instead_id_as_key_in_custom_attributes_values(custom_attributes, values) except ObjectDoesNotExist: @@ -550,7 +572,9 @@ class TaskExportSerializer(CustomAttributesValuesExportSerializerMixin, HistoryE exclude = ('id', 'project') def custom_attributes_queryset(self, project): - return project.taskcustomattributes.all() + if project.id not in _custom_tasks_attributes_cache: + _custom_tasks_attributes_cache[project.id] = list(project.taskcustomattributes.all().values('id', 'name')) + return _custom_tasks_attributes_cache[project.id] class UserStoryExportSerializer(CustomAttributesValuesExportSerializerMixin, HistoryExportSerializerMixin, @@ -568,7 +592,9 @@ class UserStoryExportSerializer(CustomAttributesValuesExportSerializerMixin, His exclude = ('id', 'project', 'points', 'tasks') def custom_attributes_queryset(self, project): - return project.userstorycustomattributes.all() + if project.id not in _custom_userstories_attributes_cache: + _custom_userstories_attributes_cache[project.id] = list(project.userstorycustomattributes.all().values('id', 'name')) + return _custom_userstories_attributes_cache[project.id] class IssueExportSerializer(CustomAttributesValuesExportSerializerMixin, HistoryExportSerializerMixin, @@ -591,7 +617,9 @@ class IssueExportSerializer(CustomAttributesValuesExportSerializerMixin, History return [x.email for x in votes_service.get_voters(obj)] def custom_attributes_queryset(self, project): - return project.issuecustomattributes.all() + if project.id not in _custom_issues_attributes_cache: + _custom_issues_attributes_cache[project.id] = list(project.issuecustomattributes.all().values('id', 'name')) + return _custom_issues_attributes_cache[project.id] class WikiPageExportSerializer(HistoryExportSerializerMixin, AttachmentExportSerializerMixin, @@ -618,17 +646,17 @@ class TimelineDataField(serializers.WritableField): def to_native(self, data): new_data = copy.deepcopy(data) try: - user = users_models.User.objects.get(pk=new_data["user"]["id"]) + user = cached_get_user_by_pk(new_data["user"]["id"]) new_data["user"]["email"] = user.email del new_data["user"]["id"] - except users_models.User.DoesNotExist: + except Exception: pass return new_data def from_native(self, data): new_data = copy.deepcopy(data) try: - user = users_models.User.objects.get(email=new_data["user"]["email"]) + user = cached_get_user_by_email(new_data["user"]["email"]) new_data["user"]["id"] = user.id del new_data["user"]["email"] except users_models.User.DoesNotExist: diff --git a/taiga/export_import/services/render.py b/taiga/export_import/services/render.py index cc4f8edf..19015878 100644 --- a/taiga/export_import/services/render.py +++ b/taiga/export_import/services/render.py @@ -50,6 +50,12 @@ def render_project(project, outfile, chunk_size = 8190): # These four "special" fields hava attachments so we use them in a special way if field_name in ["wiki_pages", "user_stories", "tasks", "issues"]: value = get_component(project, field_name) + if field_name != "wiki_pages": + value = value.select_related('owner', 'status', 'milestone', 'project', 'assigned_to', 'custom_attributes_values') + if field_name == "issues": + value = value.select_related('severity', 'priority', 'type') + value = value.prefetch_related('history_entry', 'attachments') + outfile.write('"{}": [\n'.format(field_name)) attachments_field = field.fields.pop("attachments", None) @@ -101,9 +107,8 @@ def render_project(project, outfile, chunk_size = 8190): outfile.write(']}') outfile.flush() - gc.collect() + gc.collect() outfile.write(']') - else: value = field.field_to_native(project, field_name) outfile.write('"{}": {}'.format(field_name, json.dumps(value)))