diff --git a/CHANGELOG.md b/CHANGELOG.md index bd6f2534..f625aa5c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,7 +17,8 @@ - Add endpoints to show the watchers list for issues, tasks and user stories. - Add headers to allow threading for notification emails about changes to issues, tasks, user stories, and wiki pages. (thanks to [@brett](https://github.com/brettp)). - Add externall apps: now Taiga can integrate with hundreds of applications and service. -- Improving searching system, now full text searchs are supported +- Improve searching system, now full text searchs are supported +- Improve export system, now is more efficient and prevents possible crashes with heavy projects. - i18n. - Add italian (it) translation. - Add polish (pl) translation. diff --git a/taiga/base/api/settings.py b/taiga/base/api/settings.py index 69440ec0..34dd9717 100644 --- a/taiga/base/api/settings.py +++ b/taiga/base/api/settings.py @@ -117,7 +117,7 @@ DEFAULTS = { "DATE_INPUT_FORMATS": ( ISO_8601, ), - "DATE_FORMAT": None, + "DATE_FORMAT": ISO_8601, "DATETIME_INPUT_FORMATS": ( ISO_8601, diff --git a/taiga/base/storage.py b/taiga/base/storage.py index acf93306..35e234df 100644 --- a/taiga/base/storage.py +++ b/taiga/base/storage.py @@ -18,7 +18,7 @@ from django.conf import settings from django.core.files import storage import django_sites as sites - +import os class FileSystemStorage(storage.FileSystemStorage): def __init__(self, *args, **kwargs): @@ -30,3 +30,33 @@ class FileSystemStorage(storage.FileSystemStorage): scheme = site.scheme and "{0}:".format(site.scheme) or "" self.base_url = url_tmpl.format(scheme=scheme, domain=site.domain, url=settings.MEDIA_URL) + + def open(self, name, mode='rb'): + """ + Let's create the needed directory structrue before opening the file + """ + + # Create any intermediate directories that do not exist. + # Note that there is a race between os.path.exists and os.makedirs: + # if os.makedirs fails with EEXIST, the directory was created + # concurrently, and we can continue normally. Refs #16082. + directory = os.path.dirname(name) + if not os.path.exists(directory): + try: + if self.directory_permissions_mode is not None: + # os.makedirs applies the global umask, so we reset it, + # for consistency with file_permissions_mode behavior. + old_umask = os.umask(0) + try: + os.makedirs(directory, self.directory_permissions_mode) + finally: + os.umask(old_umask) + else: + os.makedirs(directory) + except OSError as e: + if e.errno != errno.EEXIST: + raise + if not os.path.isdir(directory): + raise IOError("%s exists and is not a directory." % directory) + + return super().open(name, mode=mode) diff --git a/taiga/base/utils/json.py b/taiga/base/utils/json.py index 2f7ac2e5..40132b34 100644 --- a/taiga/base/utils/json.py +++ b/taiga/base/utils/json.py @@ -30,6 +30,8 @@ def loads(data): data = force_text(data) return json.loads(data) +load = json.load + # Some backward compatibility that should # be removed in near future. to_json = dumps diff --git a/taiga/export_import/api.py b/taiga/export_import/api.py index 48e8592c..6fabb96d 100644 --- a/taiga/export_import/api.py +++ b/taiga/export_import/api.py @@ -14,7 +14,6 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -import json import codecs import uuid @@ -26,6 +25,7 @@ from django.conf import settings from django.core.files.storage import default_storage from django.core.files.base import ContentFile +from taiga.base.utils import json from taiga.base.decorators import detail_route, list_route from taiga.base import exceptions as exc from taiga.base import response @@ -67,10 +67,10 @@ class ProjectExporterViewSet(mixins.ImportThrottlingPolicyMixin, GenericViewSet) return response.Accepted({"export_id": task.id}) path = "exports/{}/{}-{}.json".format(project.pk, project.slug, uuid.uuid4().hex) - content = ContentFile(ExportRenderer().render(service.project_to_dict(project), - renderer_context={"indent": 4}).decode('utf-8')) + storage_path = default_storage.path(path) + with default_storage.open(storage_path, mode="w") as outfile: + service.render_project(project, outfile) - default_storage.save(path, content) response_data = { "url": default_storage.url(path) } diff --git a/taiga/export_import/management/commands/dump_project.py b/taiga/export_import/management/commands/dump_project.py index 2ea0d7a3..9728d01c 100644 --- a/taiga/export_import/management/commands/dump_project.py +++ b/taiga/export_import/management/commands/dump_project.py @@ -18,7 +18,10 @@ from django.core.management.base import BaseCommand, CommandError from taiga.projects.models import Project from taiga.export_import.renderers import ExportRenderer -from taiga.export_import.service import project_to_dict +from taiga.export_import.service import render_project + + +import resource class Command(BaseCommand): @@ -34,6 +37,5 @@ class Command(BaseCommand): except Project.DoesNotExist: raise CommandError('Project "%s" does not exist' % project_slug) - data = project_to_dict(project) with open('%s.json'%(project_slug), 'w') as outfile: - self.renderer.render_to_file(data, outfile, renderer_context=self.renderer_context) + render_project(project, outfile) diff --git a/taiga/export_import/management/commands/load_dump.py b/taiga/export_import/management/commands/load_dump.py index 4566cf88..14016c6e 100644 --- a/taiga/export_import/management/commands/load_dump.py +++ b/taiga/export_import/management/commands/load_dump.py @@ -19,8 +19,7 @@ from django.db import transaction from django.db.models import signals from optparse import make_option -import json - +from taiga.base.utils import json from taiga.projects.models import Project from taiga.export_import.renderers import ExportRenderer from taiga.export_import.dump_service import dict_to_project, TaigaImportError diff --git a/taiga/export_import/serializers.py b/taiga/export_import/serializers.py index 9ea19705..02334510 100644 --- a/taiga/export_import/serializers.py +++ b/taiga/export_import/serializers.py @@ -494,6 +494,8 @@ class RolePointsExportSerializer(serializers.ModelSerializer): class MilestoneExportSerializer(WatcheableObjectModelSerializer): owner = UserRelatedField(required=False) modified_date = serializers.DateTimeField(required=False) + estimated_start = serializers.DateField(required=False) + estimated_finish = serializers.DateField(required=False) def __init__(self, *args, **kwargs): project = kwargs.pop('project', None) diff --git a/taiga/export_import/service.py b/taiga/export_import/service.py index 25b9be90..ef0b8a22 100644 --- a/taiga/export_import/service.py +++ b/taiga/export_import/service.py @@ -14,20 +14,28 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -import uuid +import base64 +import gc +import resource +import os import os.path as path +import uuid + from unidecode import unidecode from django.template.defaultfilters import slugify from django.contrib.contenttypes.models import ContentType from django.core.exceptions import ObjectDoesNotExist +from django.core.files.storage import default_storage +from taiga.base.utils import json from taiga.projects.history.services import make_key_from_model_object, take_snapshot from taiga.timeline.service import build_project_namespace from taiga.projects.references import sequences as seq from taiga.projects.references import models as refs from taiga.projects.userstories.models import RolePoints from taiga.projects.services import find_invited_user +from taiga.base.api.fields import get_component from . import serializers @@ -48,8 +56,81 @@ def add_errors(section, errors): _errors_log[section] = [errors] -def project_to_dict(project): - return serializers.ProjectExportSerializer(project).data +def render_project(project, outfile, chunk_size = 8192): + serializer = serializers.ProjectExportSerializer(project) + outfile.write('{\n') + + first_field = True + for field_name in serializer.fields.keys(): + # Avoid writing "," in the last element + if not first_field: + outfile.write(",\n") + else: + first_field = False + + field = serializer.fields.get(field_name) + field.initialize(parent=serializer, field_name=field_name) + + # These four "special" fields hava attachments so we use them in a special way + if field_name in ["wiki_pages", "user_stories", "tasks", "issues"]: + value = get_component(project, field_name) + outfile.write('"{}": [\n'.format(field_name)) + + attachments_field = field.fields.pop("attachments", None) + if attachments_field: + attachments_field.initialize(parent=field, field_name="attachments") + + first_item = True + for item in value.iterator(): + # Avoid writing "," in the last element + if not first_item: + outfile.write(",\n") + else: + first_item = False + + + dumped_value = json.dumps(field.to_native(item)) + writing_value = dumped_value[:-1]+ ',\n "attachments": [\n' + outfile.write(writing_value) + + first_attachment = True + for attachment in item.attachments.iterator(): + # Avoid writing "," in the last element + if not first_attachment: + outfile.write(",\n") + else: + first_attachment = False + + # Write all the data expect the serialized file + attachment_serializer = serializers.AttachmentExportSerializer(instance=attachment) + attached_file_serializer = attachment_serializer.fields.pop("attached_file") + dumped_value = json.dumps(attachment_serializer.data) + dumped_value = dumped_value[:-1] + ',\n "attached_file":{\n "data":"' + outfile.write(dumped_value) + + # We write the attached_files by chunks so the memory used is not increased + attachment_file = attachment.attached_file + with default_storage.open(attachment_file.name) as f: + while True: + bin_data = f.read(chunk_size) + if not bin_data: + break + + b64_data = base64.b64encode(bin_data).decode('utf-8') + outfile.write(b64_data) + + outfile.write('", \n "name":"{}"}}\n}}'.format(os.path.basename(attachment_file.name))) + + outfile.write(']}') + outfile.flush() + gc.collect() + outfile.write(']') + + else: + value = field.field_to_native(project, field_name) + outfile.write('"{}": {}'.format(field_name, json.dumps(value))) + + outfile.write('}\n') def store_project(data): diff --git a/taiga/export_import/tasks.py b/taiga/export_import/tasks.py index 1698b9c4..f833aef4 100644 --- a/taiga/export_import/tasks.py +++ b/taiga/export_import/tasks.py @@ -29,25 +29,26 @@ from djmail.template_mail import MagicMailBuilder, InlineCSSTemplateMail from taiga.celery import app -from .service import project_to_dict +from .service import render_project from .dump_service import dict_to_project from .renderers import ExportRenderer logger = logging.getLogger('taiga.export_import') +import resource + @app.task(bind=True) def dump_project(self, user, project): mbuilder = MagicMailBuilder(template_mail_cls=InlineCSSTemplateMail) path = "exports/{}/{}-{}.json".format(project.pk, project.slug, self.request.id) + storage_path = default_storage.path(path) try: - content = ExportRenderer().render(project_to_dict(project), renderer_context={"indent": 4}) - content = content.decode('utf-8') - content = ContentFile(content) - - default_storage.save(path, content) url = default_storage.url(path) + with default_storage.open(storage_path, mode="w") as outfile: + render_project(project, outfile) + except Exception: ctx = { "user": user, diff --git a/tests/integration/test_importer_api.py b/tests/integration/test_importer_api.py index a0f2219b..d8fe6f50 100644 --- a/tests/integration/test_importer_api.py +++ b/tests/integration/test_importer_api.py @@ -910,7 +910,6 @@ def test_valid_milestone_import(client): assert response.data["watchers"] == [user_watching.email] - def test_milestone_import_duplicated_milestone(client): user = f.UserFactory.create() project = f.ProjectFactory.create(owner=user) diff --git a/tests/unit/test_export.py b/tests/unit/test_export.py index 67de6eac..0a6ba9d6 100644 --- a/tests/unit/test_export.py +++ b/tests/unit/test_export.py @@ -15,21 +15,29 @@ # along with this program. If not, see . import pytest - +import io from .. import factories as f -from taiga.export_import.service import project_to_dict +from taiga.base.utils import json +from taiga.export_import.service import render_project pytestmark = pytest.mark.django_db def test_export_issue_finish_date(client): issue = f.IssueFactory.create(finished_date="2014-10-22") - finish_date = project_to_dict(issue.project)["issues"][0]["finished_date"] + output = io.StringIO() + render_project(issue.project, output) + print(output.getvalue()) + project_data = json.loads(output.getvalue()) + finish_date = project_data["issues"][0]["finished_date"] assert finish_date == "2014-10-22T00:00:00+0000" def test_export_user_story_finish_date(client): user_story = f.UserStoryFactory.create(finish_date="2014-10-22") - finish_date = project_to_dict(user_story.project)["user_stories"][0]["finish_date"] + output = io.StringIO() + render_project(user_story.project, output) + project_data = json.loads(output.getvalue()) + finish_date = project_data["user_stories"][0]["finish_date"] assert finish_date == "2014-10-22T00:00:00+0000"