taiga
/
taiga-back
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

[Backport] Improve performance on exports

remotes/origin/issue/4795/notification_even_they_are_disabled
Jesús Espino 2016-06-15 18:06:12 +02:00 committed by Alejandro Alonso
parent 0f7e25b870
commit ed0a650dc9
2 changed files with 46 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

50
taiga/export_import/serializers.py
View File

@ -50,6 +50,28 @@ from taiga.projects.notifications import services as notifications_services
from taiga.projects.votes import services as votes_service
from taiga.projects.history import services as history_service
_cache_user_by_pk = {}
_cache_user_by_email = {}
_custom_tasks_attributes_cache = {}
_custom_issues_attributes_cache = {}
_custom_userstories_attributes_cache = {}
def cached_get_user_by_pk(pk):
if pk not in _cache_user_by_pk:
try:
_cache_user_by_pk[pk] = users_models.User.objects.get(pk=pk)
except Exception:
_cache_user_by_pk[pk] = users_models.User.objects.get(pk=pk)
return _cache_user_by_pk[pk]
def cached_get_user_by_email(email):
if email not in _cache_user_by_email:
try:
_cache_user_by_email[email] = users_models.User.objects.get(email=email)
except Exception:
_cache_user_by_email[email] = users_models.User.objects.get(email=email)
return _cache_user_by_email[email]
class FileField(serializers.WritableField):
read_only = False
@ -128,7 +150,7 @@ class UserRelatedField(RelatedNoneSafeField):
def from_native(self, data):
try:
return users_models.User.objects.get(email=data)
return cached_get_user_by_email(data)
except users_models.User.DoesNotExist:
return None
@ -138,14 +160,14 @@ class UserPkField(serializers.RelatedField):
def to_native(self, obj):
try:
user = users_models.User.objects.get(pk=obj)
user = cached_get_user_by_pk(obj)
return user.email
except users_models.User.DoesNotExist:
return None
def from_native(self, data):
try:
user = users_models.User.objects.get(email=data)
user = cached_get_user_by_email(data)
return user.pk
except users_models.User.DoesNotExist:
return None
@ -185,7 +207,7 @@ class HistoryUserField(JsonField):
if obj is None or obj == {}:
return []
try:
user = users_models.User.objects.get(pk=obj['pk'])
user = cached_get_user_by_pk(obj['pk'])
except users_models.User.DoesNotExist:
user = None
return (UserRelatedField().to_native(user), obj['name'])
@ -420,7 +442,7 @@ class CustomAttributesValuesExportSerializerMixin(serializers.ModelSerializer):
try:
values = obj.custom_attributes_values.attributes_values
custom_attributes = self.custom_attributes_queryset(obj.project).values('id', 'name')
custom_attributes = self.custom_attributes_queryset(obj.project)
return _use_name_instead_id_as_key_in_custom_attributes_values(custom_attributes, values)
except ObjectDoesNotExist:
@ -550,7 +572,9 @@ class TaskExportSerializer(CustomAttributesValuesExportSerializerMixin, HistoryE
exclude = ('id', 'project')
def custom_attributes_queryset(self, project):
return project.taskcustomattributes.all()
if project.id not in _custom_tasks_attributes_cache:
_custom_tasks_attributes_cache[project.id] = list(project.taskcustomattributes.all().values('id', 'name'))
return _custom_tasks_attributes_cache[project.id]
class UserStoryExportSerializer(CustomAttributesValuesExportSerializerMixin, HistoryExportSerializerMixin,
@ -568,7 +592,9 @@ class UserStoryExportSerializer(CustomAttributesValuesExportSerializerMixin, His
exclude = ('id', 'project', 'points', 'tasks')
def custom_attributes_queryset(self, project):
return project.userstorycustomattributes.all()
if project.id not in _custom_userstories_attributes_cache:
_custom_userstories_attributes_cache[project.id] = list(project.userstorycustomattributes.all().values('id', 'name'))
return _custom_userstories_attributes_cache[project.id]
class IssueExportSerializer(CustomAttributesValuesExportSerializerMixin, HistoryExportSerializerMixin,
@ -591,7 +617,9 @@ class IssueExportSerializer(CustomAttributesValuesExportSerializerMixin, History
return [x.email for x in votes_service.get_voters(obj)]
def custom_attributes_queryset(self, project):
return project.issuecustomattributes.all()
if project.id not in _custom_issues_attributes_cache:
_custom_issues_attributes_cache[project.id] = list(project.issuecustomattributes.all().values('id', 'name'))
return _custom_issues_attributes_cache[project.id]
class WikiPageExportSerializer(HistoryExportSerializerMixin, AttachmentExportSerializerMixin,
@ -618,17 +646,17 @@ class TimelineDataField(serializers.WritableField):
def to_native(self, data):
new_data = copy.deepcopy(data)
try:
user = users_models.User.objects.get(pk=new_data["user"]["id"])
user = cached_get_user_by_pk(new_data["user"]["id"])
new_data["user"]["email"] = user.email
del new_data["user"]["id"]
except users_models.User.DoesNotExist:
except Exception:
pass
return new_data
def from_native(self, data):
new_data = copy.deepcopy(data)
try:
user = users_models.User.objects.get(email=new_data["user"]["email"])
user = cached_get_user_by_email(new_data["user"]["email"])
new_data["user"]["id"] = user.id
del new_data["user"]["email"]
except users_models.User.DoesNotExist:

9
taiga/export_import/services/render.py
View File

@ -50,6 +50,12 @@ def render_project(project, outfile, chunk_size = 8190):
# These four "special" fields hava attachments so we use them in a special way
if field_name in ["wiki_pages", "user_stories", "tasks", "issues"]:
value = get_component(project, field_name)
if field_name != "wiki_pages":
value = value.select_related('owner', 'status', 'milestone', 'project', 'assigned_to', 'custom_attributes_values')
if field_name == "issues":
value = value.select_related('severity', 'priority', 'type')
value = value.prefetch_related('history_entry', 'attachments')
outfile.write('"{}": [\n'.format(field_name))
attachments_field = field.fields.pop("attachments", None)
@ -101,9 +107,8 @@ def render_project(project, outfile, chunk_size = 8190):
outfile.write(']}')
outfile.flush()
gc.collect()
gc.collect()
outfile.write(']')
else:
value = field.field_to_native(project, field_name)
outfile.write('"{}": {}'.format(field_name, json.dumps(value)))