diff --git a/requirements.txt b/requirements.txt index 7ff142ab..11efe393 100644 --- a/requirements.txt +++ b/requirements.txt @@ -17,6 +17,8 @@ Markdown==2.6.7 fn==0.4.3 diff-match-patch==20121119 requests==2.12.4 +requests-oauthlib==0.6.2 +webcolors==1.5 django-sr==0.0.4 easy-thumbnails==2.3 celery==3.1.24 @@ -35,3 +37,5 @@ netaddr==0.7.18 serpy==0.1.1 psd-tools==1.4 CairoSVG==2.0.1 +cryptography==1.7.1 +PyJWT==1.4.2 diff --git a/settings/common.py b/settings/common.py index b4c9e002..2e0cd3ad 100644 --- a/settings/common.py +++ b/settings/common.py @@ -318,6 +318,7 @@ INSTALLED_APPS = [ "taiga.hooks.bitbucket", "taiga.hooks.gogs", "taiga.webhooks", + "taiga.importers", "djmail", "django_jinja", @@ -561,6 +562,9 @@ MAX_PENDING_MEMBERSHIPS = 30 # Max number of unconfirmed memberships in a projec from .sr import * +TRELLO_API_KEY = "" +TRELLO_SECRET_KEY = "" + # NOTE: DON'T INSERT MORE SETTINGS AFTER THIS LINE TEST_RUNNER="django.test.runner.DiscoverRunner" diff --git a/taiga/front/urls.py b/taiga/front/urls.py index 77d53dab..5190d2f8 100644 --- a/taiga/front/urls.py +++ b/taiga/front/urls.py @@ -23,6 +23,8 @@ urls = { "login": "/login", "register": "/register", "forgot-password": "/forgot-password", + "new-project": "/project/new", + "new-project-import": "/project/new/import/{0}", "change-password": "/change-password/{0}", # user.token "change-email": "/change-email/{0}", # user.email_token diff --git a/taiga/importers/api.py b/taiga/importers/api.py new file mode 100644 index 00000000..c1e774cc --- /dev/null +++ b/taiga/importers/api.py @@ -0,0 +1,39 @@ +# -*- coding: utf-8 -*- +# Copyright (C) 2014-2016 Taiga Agile LLC +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +from taiga.base.api import viewsets +from taiga.base.decorators import list_route + + +class BaseImporterViewSet(viewsets.ViewSet): + @list_route(methods=["GET"]) + def list_users(self, request, *args, **kwargs): + raise NotImplementedError + + @list_route(methods=["GET"]) + def list_projects(self, request, *args, **kwargs): + raise NotImplementedError + + @list_route(methods=["POST"]) + def import_project(self, request, *args, **kwargs): + raise NotImplementedError + + @list_route(methods=["GET"]) + def auth_url(self, request, *args, **kwargs): + raise NotImplementedError + + @list_route(methods=["POST"]) + def authorize(self, request, *args, **kwargs): + raise NotImplementedError diff --git a/taiga/importers/management/__init__.py b/taiga/importers/management/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/taiga/importers/management/commands/__init__.py b/taiga/importers/management/commands/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/taiga/importers/management/commands/import_from_trello.py b/taiga/importers/management/commands/import_from_trello.py new file mode 100644 index 00000000..c04537fe --- /dev/null +++ b/taiga/importers/management/commands/import_from_trello.py @@ -0,0 +1,89 @@ +# -*- coding: utf-8 -*- +# Copyright (C) 2014-2016 Andrey Antukh +# Copyright (C) 2014-2016 Jesús Espino +# Copyright (C) 2014-2016 David Barragán +# Copyright (C) 2014-2016 Alejandro Alonso +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +from django.core.management.base import BaseCommand +from django.db.models import Q + +from taiga.importers.trello.importer import TrelloImporter +from taiga.users.models import User +from taiga.projects.services import projects as service + +import unittest.mock +import timeit + + +class Command(BaseCommand): + def add_arguments(self, parser): + parser.add_argument('--token', dest="token", type=str, + help='Auth token') + parser.add_argument('--project-id', dest="project_id", type=str, + help='Project ID or full name (ex: taigaio/taiga-back)') + parser.add_argument('--template', dest='template', default="kanban", + help='template to use: scrum or kanban (default kanban)') + parser.add_argument('--ask-for-users', dest='ask_for_users', const=True, + action="store_const", default=False, + help='Import closed data') + parser.add_argument('--closed-data', dest='closed_data', const=True, + action="store_const", default=False, + help='Import closed data') + parser.add_argument('--keep-external-reference', dest='keep_external_reference', const=True, + action="store_const", default=False, + help='Store external reference of imported data') + + def handle(self, *args, **options): + admin = User.objects.get(username="admin") + if options.get('token', None): + token = options.get('token') + else: + (oauth_token, oauth_token_secret, url) = TrelloImporter.get_auth_url() + print("Go to here and come with your token: {}".format(url)) + oauth_verifier = input("Code: ") + access_data = TrelloImporter.get_access_token(oauth_token, oauth_token_secret, oauth_verifier) + token = access_data['oauth_token'] + print("Access token: {}".format(token)) + importer = TrelloImporter(admin, token) + + if options.get('project_id', None): + project_id = options.get('project_id') + else: + print("Select the project to import:") + for project in importer.list_projects(): + print("- {}: {}".format(project['id'], project['name'])) + project_id = input("Project id: ") + + users_bindings = {} + if options.get('ask_for_users', None): + print("Add the username or email for next trello users:") + for user in importer.list_users(project_id): + while True: + username_or_email = input("{}: ".format(user['fullName'])) + if username_or_email == "": + break + try: + users_bindings[user['id']] = User.objects.get(Q(username=username_or_email) | Q(email=username_or_email)) + break + except User.DoesNotExist: + print("ERROR: Invalid username or email") + + options = { + "template": options.get('template'), + "import_closed_data": options.get("closed_data", False), + "users_bindings": users_bindings, + "keep_external_reference": options.get('keep_external_reference') + } + importer.import_project(project_id, options) diff --git a/taiga/importers/permissions.py b/taiga/importers/permissions.py new file mode 100644 index 00000000..1532989e --- /dev/null +++ b/taiga/importers/permissions.py @@ -0,0 +1,32 @@ +# -*- coding: utf-8 -*- +# Copyright (C) 2014-2016 Andrey Antukh +# Copyright (C) 2014-2016 Jesús Espino +# Copyright (C) 2014-2016 David Barragán +# Copyright (C) 2014-2016 Alejandro Alonso +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +from taiga.base.api.permissions import TaigaResourcePermission, AllowAny, IsAuthenticated +from taiga.base.api.permissions import IsSuperUser, HasProjectPerm, IsProjectAdmin + +from taiga.permissions.permissions import CommentAndOrUpdatePerm + + +class ImporterPermission(TaigaResourcePermission): + enought_perms = IsAuthenticated() + global_perms = None + auth_url_perms = IsAuthenticated() + authorize_perms = IsAuthenticated() + list_users_perms = IsAuthenticated() + list_projects_perms = IsAuthenticated() + import_project_perms = IsAuthenticated() diff --git a/taiga/importers/trello/api.py b/taiga/importers/trello/api.py new file mode 100644 index 00000000..33b9fe40 --- /dev/null +++ b/taiga/importers/trello/api.py @@ -0,0 +1,147 @@ +# -*- coding: utf-8 -*- +# Copyright (C) 2014-2016 Taiga Agile LLC +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import uuid + +from django.utils.translation import ugettext as _ +from django.conf import settings + +from taiga.base.api import viewsets +from taiga.base import response +from taiga.base import exceptions as exc +from taiga.base.decorators import list_route +from taiga.users.models import AuthData, User +from taiga.users.services import get_user_photo_url +from taiga.users.gravatar import get_user_gravatar_id +from taiga.projects.serializers import ProjectSerializer + +from .importer import TrelloImporter +from taiga.importers import permissions +from . import tasks + + +class TrelloImporterViewSet(viewsets.ViewSet): + permission_classes = (permissions.ImporterPermission,) + + @list_route(methods=["POST"]) + def list_users(self, request, *args, **kwargs): + self.check_permissions(request, "list_users", None) + + token = request.DATA.get('token', None) + project_id = request.DATA.get('project', None) + + if not project_id: + raise exc.WrongArguments(_("The project param is needed")) + + importer = TrelloImporter(request.user, token) + users = importer.list_users(project_id) + for user in users: + user['user'] = None + if not user['email']: + continue + + try: + taiga_user = User.objects.get(email=user['email']) + except User.DoesNotExist: + continue + + user['user'] = { + 'id': taiga_user.id, + 'full_name': taiga_user.get_full_name(), + 'gravatar_id': get_user_gravatar_id(taiga_user), + 'photo': get_user_photo_url(taiga_user), + } + return response.Ok(users) + + @list_route(methods=["POST"]) + def list_projects(self, request, *args, **kwargs): + self.check_permissions(request, "list_projects", None) + token = request.DATA.get('token', None) + importer = TrelloImporter(request.user, token) + projects = importer.list_projects() + return response.Ok(projects) + + @list_route(methods=["POST"]) + def import_project(self, request, *args, **kwargs): + self.check_permissions(request, "import_project", None) + + token = request.DATA.get('token', None) + project_id = request.DATA.get('project', None) + if not project_id: + raise exc.WrongArguments(_("The project param is needed")) + + options = { + "name": request.DATA.get('name', None), + "description": request.DATA.get('description', None), + "template": request.DATA.get('template', "kanban"), + "users_bindings": request.DATA.get("users_bindings", {}), + "keep_external_reference": request.DATA.get("keep_external_reference", False), + "is_private": request.DATA.get("is_private", False), + } + + if settings.CELERY_ENABLED: + task = tasks.import_project.delay(request.user.id, token, project_id, options) + return response.Accepted({"task_id": task.id}) + + importer = TrelloImporter(request.user, token) + project = importer.import_project(project_id, options) + project_data = { + "slug": project.slug, + "my_permissions": ["view_us"], + "is_backlog_activated": project.is_backlog_activated, + "is_kanban_activated": project.is_kanban_activated, + } + + return response.Ok(project_data) + + @list_route(methods=["GET"]) + def auth_url(self, request, *args, **kwargs): + self.check_permissions(request, "auth_url", None) + + (oauth_token, oauth_secret, url) = TrelloImporter.get_auth_url() + + (auth_data, created) = AuthData.objects.get_or_create( + user=request.user, + key="trello-oauth", + defaults={ + "value": uuid.uuid4().hex, + "extra": {}, + } + ) + auth_data.extra = { + "oauth_token": oauth_token, + "oauth_secret": oauth_secret, + } + auth_data.save() + + return response.Ok({"url": url}) + + @list_route(methods=["POST"]) + def authorize(self, request, *args, **kwargs): + self.check_permissions(request, "authorize", None) + + try: + oauth_data = request.user.auth_data.get(key="trello-oauth") + oauth_token = oauth_data.extra['oauth_token'] + oauth_secret = oauth_data.extra['oauth_secret'] + oauth_verifier = request.DATA.get('code') + oauth_data.delete() + trello_token = TrelloImporter.get_access_token(oauth_token, oauth_secret, oauth_verifier)['oauth_token'] + except Exception as e: + raise exc.WrongArguments(_("Invalid or expired auth token")) + + return response.Ok({ + "token": trello_token + }) diff --git a/taiga/importers/trello/importer.py b/taiga/importers/trello/importer.py new file mode 100644 index 00000000..e41063b8 --- /dev/null +++ b/taiga/importers/trello/importer.py @@ -0,0 +1,537 @@ +# -*- coding: utf-8 -*- +# Copyright (C) 2014-2016 Andrey Antukh +# Copyright (C) 2014-2016 Jesús Espino +# Copyright (C) 2014-2016 David Barragán +# Copyright (C) 2014-2016 Alejandro Alonso +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +from django.utils.translation import ugettext as _ + +from requests_oauthlib import OAuth1Session, OAuth1 +from django.conf import settings +from django.core.files.base import ContentFile +from django.contrib.contenttypes.models import ContentType +import requests +import webcolors + +from django.template.defaultfilters import slugify +from taiga.base import exceptions as exc +from taiga.projects.services import projects as projects_service +from taiga.projects.models import Project, ProjectTemplate, Membership +from taiga.projects.userstories.models import UserStory +from taiga.projects.tasks.models import Task +from taiga.projects.attachments.models import Attachment +from taiga.projects.history.services import (make_diff_from_dicts, + make_diff_values, + make_key_from_model_object, + get_typename_for_model_class, + FrozenDiff) +from taiga.projects.history.models import HistoryEntry +from taiga.projects.history.choices import HistoryType +from taiga.projects.custom_attributes.models import UserStoryCustomAttribute +from taiga.mdrender.service import render as mdrender +from taiga.timeline.rebuilder import rebuild_timeline +from taiga.timeline.models import Timeline +from taiga.front.templatetags.functions import resolve as resolve_front_url + +from taiga.base import exceptions + + +class TrelloClient: + def __init__(self, api_key, api_secret, token): + self.api_key = api_key + self.api_secret = api_secret + self.token = token + if self.token: + self.oauth = OAuth1( + client_key=self.api_key, + client_secret=self.api_secret, + resource_owner_key=self.token + ) + else: + self.oauth = None + + def get(self, uri_path, query_params=None): + headers = {'Accept': 'application/json'} + if query_params is None: + query_params = {} + + if uri_path[0] == '/': + uri_path = uri_path[1:] + url = 'https://api.trello.com/1/%s' % uri_path + + response = requests.get(url, params=query_params, headers=headers, auth=self.oauth) + + if response.status_code == 400: + raise exc.WrongArguments(_("Invalid Request: %s at %s") % (response.text, url)) + if response.status_code == 401: + raise exc.AuthenticationFailed(_("Unauthorized: %s at %s") % (response.text, url)) + if response.status_code == 403: + raise exc.PermissionDenied(_("Unauthorized: %s at %s") % (response.text, url)) + if response.status_code == 404: + raise exc.NotFound(_("Resource Unavailable: %s at %s") % (response.text, url)) + if response.status_code != 200: + raise exc.WrongArguments(_("Resource Unavailable: %s at %s") % (response.text, url)) + + return response.json() + + +class TrelloImporter: + def __init__(self, user, token): + self._user = user + self._cached_orgs = {} + self._client = TrelloClient( + api_key=settings.TRELLO_API_KEY, + api_secret=settings.TRELLO_SECRET_KEY, + token=token, + ) + + def list_projects(self): + projects_data = self._client.get("/members/me/boards", { + "fields": "id,name,desc,prefs,idOrganization", + "organization": "true", + "organization_fields": "prefs", + }) + projects = [] + for project in projects_data: + is_private = False + if project['prefs']['permissionLevel'] == "private": + is_private = True + + if project['prefs']['permissionLevel'] == "org": + if 'organization' not in project: + is_private = True + elif project['organization']['prefs']['permissionLevel'] == "private": + is_private = True + + projects.append({ + "id": project['id'], + "name": project['name'], + "description": project['desc'], + "is_private": is_private, + }) + return projects + + def list_users(self, project_id): + members = [] + for member in self._client.get("/board/{}/members/all".format(project_id), {"fields": "id"}): + user = self._client.get("/member/{}".format(member['id']), {"fields": "id,fullName,email"}) + members.append({ + "id": user['id'], + "full_name": user['fullName'], + "email": user['email'], + }) + return members + + def import_project(self, project_id, options): + data = self._client.get( + "/board/{}".format(project_id), + { + "fields": "name,desc", + "cards": "all", + "card_fields": "closed,labels,idList,desc,due,name,pos,dateLastActivity,idChecklists,idMembers,url", + "card_attachments": "true", + "labels": "all", + "labels_limit": "1000", + "lists": "all", + "list_fields": "closed,name,pos", + "members": "none", + "checklists": "all", + "checklist_fields": "name", + "organization": "true", + "organization_fields": "logoHash", + } + ) + + project = self._import_project_data(data, options) + self._import_user_stories_data(data, project, options) + self._cleanup(project, options) + Timeline.objects.filter(project=project).delete() + rebuild_timeline(None, None, project.id) + return project + + def _import_project_data(self, data, options): + board = data + labels = board['labels'] + statuses = board['lists'] + project_template = ProjectTemplate.objects.get(slug=options.get('template', "kanban")) + project_template.us_statuses = [] + counter = 0 + for us_status in statuses: + if counter == 0: + project_template.default_options["us_status"] = us_status['name'] + + counter += 1 + if us_status['name'] not in [s['name'] for s in project_template.us_statuses]: + project_template.us_statuses.append({ + "name": us_status['name'], + "slug": slugify(us_status['name']), + "is_closed": False, + "is_archived": True if us_status['closed'] else False, + "color": "#999999", + "wip_limit": None, + "order": us_status['pos'], + }) + + project_template.task_statuses = [] + project_template.task_statuses.append({ + "name": "Incomplete", + "slug": "incomplete", + "is_closed": False, + "color": "#ff8a84", + "order": 1, + }) + project_template.task_statuses.append({ + "name": "Complete", + "slug": "complete", + "is_closed": True, + "color": "#669900", + "order": 2, + }) + project_template.default_options["task_status"] = "Incomplete" + project_template.roles.append({ + "name": "Trello", + "slug": "trello", + "computable": False, + "permissions": project_template.roles[0]['permissions'], + "order": 70, + }) + + tags_colors = [] + for label in labels: + name = label['name'] + if not name: + name = label['color'] + name = name.lower() + color = self._ensure_hex_color(label['color']) + tags_colors.append([name, color]) + + project = Project( + name=options.get('name', None) or board['name'], + description=options.get('description', None) or board['desc'], + owner=self._user, + tags_colors=tags_colors, + creation_template=project_template, + is_private=options.get('is_private', False), + ) + (can_create, error_message) = projects_service.check_if_project_can_be_created_or_updated(project) + if not can_create: + raise exceptions.NotEnoughSlotsForProject(project.is_private, 1, error_message) + project.save() + + if board.get('organization', None): + trello_avatar_template = "https://trello-logos.s3.amazonaws.com/{}/170.png" + project_logo_url = trello_avatar_template.format(board['organization']['logoHash']) + data = requests.get(project_logo_url) + project.logo.save("logo.png", ContentFile(data.content), save=True) + + UserStoryCustomAttribute.objects.create( + name="Due", + description="Due date", + type="date", + order=1, + project=project + ) + for user in options.get('users_bindings', {}).values(): + Membership.objects.create( + user=user, + project=project, + role=project.get_roles().get(slug="trello"), + is_admin=False, + invited_by=self._user, + ) + return project + + def _import_user_stories_data(self, data, project, options): + users_bindings = options.get('users_bindings', {}) + statuses = {s['id']: s for s in data['lists']} + cards = data['cards'] + due_date_field = project.userstorycustomattributes.first() + + for card in cards: + if card['closed'] and not options.get("import_closed_data", False): + continue + if statuses[card['idList']]['closed'] and not options.get("import_closed_data", False): + continue + + tags = [] + for tag in card['labels']: + name = tag['name'] + if not name: + name = tag['color'] + name = name.lower() + tags.append(name) + + assigned_to = None + if len(card['idMembers']) > 0: + assigned_to = users_bindings.get(card['idMembers'][0], None) + + external_reference = None + if options.get('keep_external_reference', False): + external_reference = ["trello", card['url']] + + us = UserStory.objects.create( + project=project, + owner=self._user, + assigned_to=assigned_to, + status=project.us_statuses.get(name=statuses[card['idList']]['name']), + kanban_order=card['pos'], + sprint_order=card['pos'], + backlog_order=card['pos'], + subject=card['name'], + description=card['desc'], + tags=tags, + external_reference=external_reference + ) + + if len(card['idMembers']) > 1: + for watcher in card['idMembers'][1:]: + watcher_user = users_bindings.get(watcher, None) + if watcher_user: + us.add_watcher(watcher_user) + + if card['due']: + us.custom_attributes_values.attributes_values = {due_date_field.id: card['due']} + us.custom_attributes_values.save() + + UserStory.objects.filter(id=us.id).update( + modified_date=card['dateLastActivity'], + created_date=card['dateLastActivity'] + ) + self._import_attachments(us, card, options) + self._import_tasks(data, us, card) + self._import_actions(us, card, statuses, options) + + def _import_tasks(self, data, us, card): + checklists_by_id = {c['id']: c for c in data['checklists']} + for checklist_id in card['idChecklists']: + for item in checklists_by_id.get(checklist_id, {}).get('checkItems', []): + Task.objects.create( + subject=item['name'], + status=us.project.task_statuses.get(slug=item['state']), + project=us.project, + user_story=us + ) + + def _import_attachments(self, us, card, options): + users_bindings = options.get('users_bindings', {}) + for attachment in card['attachments']: + if attachment['bytes'] is None: + continue + data = requests.get(attachment['url']) + att = Attachment( + owner=users_bindings.get(attachment['idMember'], self._user), + project=us.project, + content_type=ContentType.objects.get_for_model(UserStory), + object_id=us.id, + name=attachment['name'], + size=attachment['bytes'], + created_date=attachment['date'], + is_deprecated=False, + ) + att.attached_file.save(attachment['name'], ContentFile(data.content), save=True) + + UserStory.objects.filter(id=us.id, created_date__gt=attachment['date']).update( + created_date=attachment['date'] + ) + + def _import_actions(self, us, card, statuses, options): + included_actions = [ + "addAttachmentToCard", "addMemberToCard", "commentCard", + "convertToCardFromCheckItem", "copyCommentCard", "createCard", + "deleteAttachmentFromCard", "deleteCard", "removeMemberFromCard", + "updateCard", + ] + + actions = self._client.get( + "/card/{}/actions".format(card['id']), + { + "filter": ",".join(included_actions), + "limit": "1000", + "memberCreator": "true", + "memberCreator_fields": "fullName", + } + ) + + while actions: + for action in actions: + self._import_action(us, action, statuses, options) + actions = self._client.get( + "/card/{}/actions".format(card['id']), + { + "filter": ",".join(included_actions), + "limit": "1000", + "since": "lastView", + "before": action['date'], + "memberCreator": "true", + "memberCreator_fields": "fullName", + } + ) + + def _import_action(self, us, action, statuses, options): + key = make_key_from_model_object(us) + typename = get_typename_for_model_class(UserStory) + action_data = self._transform_action_data(us, action, statuses, options) + if action_data is None: + return + + change_old = action_data['change_old'] + change_new = action_data['change_new'] + hist_type = action_data['hist_type'] + comment = action_data['comment'] + user = action_data['user'] + + diff = make_diff_from_dicts(change_old, change_new) + fdiff = FrozenDiff(key, diff, {}) + + entry = HistoryEntry.objects.create( + user=user, + project_id=us.project.id, + key=key, + type=hist_type, + snapshot=None, + diff=fdiff.diff, + values=make_diff_values(typename, fdiff), + comment=comment, + comment_html=mdrender(us.project, comment), + is_hidden=False, + is_snapshot=False, + ) + HistoryEntry.objects.filter(id=entry.id).update(created_at=action['date']) + return HistoryEntry.objects.get(id=entry.id) + + def _transform_action_data(self, us, action, statuses, options): + users_bindings = options.get('users_bindings', {}) + due_date_field = us.project.userstorycustomattributes.first() + + ignored_actions = ["addAttachmentToCard", "addMemberToCard", + "deleteAttachmentFromCard", "deleteCard", + "removeMemberFromCard"] + + if action['type'] in ignored_actions: + return None + + user = {"pk": None, "name": action.get('memberCreator', {}).get('fullName', None)} + taiga_user = users_bindings.get(action.get('memberCreator', {}).get('id', None), None) + if taiga_user: + user = {"pk": taiga_user.id, "name": taiga_user.get_full_name()} + + result = { + "change_old": {}, + "change_new": {}, + "hist_type": HistoryType.change, + "comment": "", + "user": user + } + + if action['type'] == "commentCard": + result['comment'] = str(action['data']['text']) + elif action['type'] == "convertToCardFromCheckItem": + UserStory.objects.filter(id=us.id, created_date__gt=action['date']).update( + created_date=action['date'], + owner=users_bindings.get(action["idMemberCreator"], self._user) + ) + result['hist_type'] = HistoryType.create + elif action['type'] == "copyCommentCard": + UserStory.objects.filter(id=us.id, created_date__gt=action['date']).update( + created_date=action['date'], + owner=users_bindings.get(action["idMemberCreator"], self._user) + ) + result['hist_type'] = HistoryType.create + elif action['type'] == "createCard": + UserStory.objects.filter(id=us.id, created_date__gt=action['date']).update( + created_date=action['date'], + owner=users_bindings.get(action["idMemberCreator"], self._user) + ) + result['hist_type'] = HistoryType.create + elif action['type'] == "updateCard": + if 'desc' in action['data']['old']: + result['change_old']["description"] = str(action['data']['old'].get('desc', '')) + result['change_new']["description"] = str(action['data']['card'].get('desc', '')) + result['change_old']["description_html"] = mdrender(us.project, str(action['data']['old'].get('desc', ''))) + result['change_new']["description_html"] = mdrender(us.project, str(action['data']['card'].get('desc', ''))) + if 'idList' in action['data']['old']: + old_status_name = statuses[action['data']['old']['idList']]['name'] + result['change_old']["status"] = us.project.us_statuses.get(name=old_status_name).id + new_status_name = statuses[action['data']['card']['idList']]['name'] + result['change_new']["status"] = us.project.us_statuses.get(name=new_status_name).id + if 'name' in action['data']['old']: + result['change_old']["subject"] = action['data']['old']['name'] + result['change_new']["subject"] = action['data']['card']['name'] + if 'due' in action['data']['old']: + result['change_old']["custom_attributes"] = [{ + "name": "Due", + "value": action['data']['old']['due'], + "id": due_date_field.id + }] + result['change_new']["custom_attributes"] = [{ + "name": "Due", + "value": action['data']['card']['due'], + "id": due_date_field.id + }] + + if result['change_old'] == {}: + return None + return result + + @classmethod + def get_auth_url(cls): + request_token_url = 'https://trello.com/1/OAuthGetRequestToken' + authorize_url = 'https://trello.com/1/OAuthAuthorizeToken' + return_url = resolve_front_url("new-project-import", "trello") + expiration = "1day" + scope = "read,write,account" + trello_key = settings.TRELLO_API_KEY + trello_secret = settings.TRELLO_SECRET_KEY + name = "Taiga" + + session = OAuth1Session(client_key=trello_key, client_secret=trello_secret) + response = session.fetch_request_token(request_token_url) + oauth_token, oauth_token_secret = response.get('oauth_token'), response.get('oauth_token_secret') + + return ( + oauth_token, + oauth_token_secret, + "{authorize_url}?oauth_token={oauth_token}&scope={scope}&expiration={expiration}&name={name}&return_url={return_url}".format( + authorize_url=authorize_url, + oauth_token=oauth_token, + expiration=expiration, + scope=scope, + name=name, + return_url=return_url, + ) + ) + + @classmethod + def get_access_token(cls, oauth_token, oauth_token_secret, oauth_verifier): + api_key = settings.TRELLO_API_KEY + api_secret = settings.TRELLO_SECRET_KEY + access_token_url = 'https://trello.com/1/OAuthGetAccessToken' + session = OAuth1Session(client_key=api_key, client_secret=api_secret, + resource_owner_key=oauth_token, resource_owner_secret=oauth_token_secret, + verifier=oauth_verifier) + access_token = session.fetch_access_token(access_token_url) + return access_token + + def _ensure_hex_color(self, color): + if color is None: + return None + try: + return webcolors.name_to_hex(color) + except ValueError: + return color + + def _cleanup(self, project, options): + if not options.get("import_closed_data", False): + project.us_statuses.filter(is_archived=True).delete() diff --git a/taiga/importers/trello/tasks.py b/taiga/importers/trello/tasks.py new file mode 100644 index 00000000..76f7f197 --- /dev/null +++ b/taiga/importers/trello/tasks.py @@ -0,0 +1,56 @@ +# -*- coding: utf-8 -*- +# Copyright (C) 2014-2016 Andrey Antukh +# Copyright (C) 2014-2016 Jesús Espino +# Copyright (C) 2014-2016 David Barragán +# Copyright (C) 2014-2016 Alejandro Alonso +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import logging +import sys + +from django.utils.translation import ugettext as _ + +from taiga.base.mails import mail_builder +from taiga.users.models import User +from taiga.celery import app +from .importer import TrelloImporter + +logger = logging.getLogger('taiga.importers.trello') + + +@app.task(bind=True) +def import_project(self, user_id, token, project_id, options): + user = User.object.get(id=user_id) + importer = TrelloImporter(user, token) + try: + project = importer.import_project(project_id, options) + except Exception as e: + # Error + ctx = { + "user": user, + "error_subject": _("Error importing trello project"), + "error_message": _("Error importing trello project"), + "project": project_id, + "exception": e + } + email = mail_builder.trello_import_error(admin, ctx) + email.send() + logger.error('Error importing trello project %s (by %s)', project_id, user, exc_info=sys.exc_info()) + else: + ctx = { + "project": project, + "user": user, + } + email = mail_builder.trello_import_success(user, ctx) + email.send() diff --git a/taiga/projects/epics/migrations/0005_epic_external_reference.py b/taiga/projects/epics/migrations/0005_epic_external_reference.py new file mode 100644 index 00000000..a1d10880 --- /dev/null +++ b/taiga/projects/epics/migrations/0005_epic_external_reference.py @@ -0,0 +1,21 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.9.2 on 2016-11-08 11:19 +from __future__ import unicode_literals + +import django.contrib.postgres.fields +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('epics', '0004_auto_20160928_0540'), + ] + + operations = [ + migrations.AddField( + model_name='epic', + name='external_reference', + field=django.contrib.postgres.fields.ArrayField(base_field=models.TextField(), blank=True, default=None, null=True, size=None, verbose_name='external reference'), + ), + ] diff --git a/taiga/projects/epics/models.py b/taiga/projects/epics/models.py index da0e4a3e..906e5f8d 100644 --- a/taiga/projects/epics/models.py +++ b/taiga/projects/epics/models.py @@ -18,6 +18,7 @@ from django.db import models from django.contrib.contenttypes.fields import GenericRelation +from django.contrib.postgres.fields import ArrayField from django.conf import settings from django.utils.translation import ugettext_lazy as _ from django.utils import timezone @@ -67,6 +68,8 @@ class Epic(OCCModelMixin, WatchedModelMixin, BlockedMixin, TaggedMixin, models.M user_stories = models.ManyToManyField("userstories.UserStory", related_name="epics", through='RelatedUserStory', verbose_name=_("user stories")) + external_reference = ArrayField(models.TextField(null=False, blank=False), + null=True, blank=True, default=None, verbose_name=_("external reference")) attachments = GenericRelation("attachments.Attachment") diff --git a/taiga/projects/references/models.py b/taiga/projects/references/models.py index 61097ecb..d6ad400b 100644 --- a/taiga/projects/references/models.py +++ b/taiga/projects/references/models.py @@ -68,6 +68,19 @@ def make_reference(instance, project, create=False): return refval, refinstance +def recalc_reference_counter(project): + seqname = make_sequence_name(project) + max_ref_us = project.user_stories.all().aggregate(max=models.Max('ref')) + max_ref_task = project.tasks.all().aggregate(max=models.Max('ref')) + max_ref_issue = project.issues.all().aggregate(max=models.Max('ref')) + max_references = list(filter(lambda x: x is not None, [max_ref_us['max'], max_ref_task['max'], max_ref_issue['max']])) + + max_value = 0 + if len(max_references) > 0: + max_value = max(max_references) + seq.set_max(seqname, max_value) + + def create_sequence(sender, instance, created, **kwargs): if not created: return diff --git a/taiga/projects/userstories/api.py b/taiga/projects/userstories/api.py index 3df5ddc9..c21ea7ae 100644 --- a/taiga/projects/userstories/api.py +++ b/taiga/projects/userstories/api.py @@ -106,6 +106,7 @@ class UserStoryViewSet(OCCResourceMixin, VotedResourceMixin, HistoryResourceMixi include_attachments = "include_attachments" in self.request.QUERY_PARAMS include_tasks = "include_tasks" in self.request.QUERY_PARAMS + epic_id = self.request.QUERY_PARAMS.get("epic", None) # We can be filtering by more than one epic so epic_id can consist # of different ids separete by comma. In that situation we will use diff --git a/taiga/routers.py b/taiga/routers.py index f59bea7e..528f7696 100644 --- a/taiga/routers.py +++ b/taiga/routers.py @@ -283,6 +283,11 @@ from taiga.external_apps.api import Application, ApplicationToken router.register(r"applications", Application, base_name="applications") router.register(r"application-tokens", ApplicationToken, base_name="application-tokens") +# Third party importers +from taiga.importers.trello.api import TrelloImporterViewSet + +router.register(r"importers/trello", TrelloImporterViewSet, base_name="importers-trello") + # Stats # - see taiga.stats.routers and taiga.stats.apps diff --git a/tests/integration/test_importers_trello_api.py b/tests/integration/test_importers_trello_api.py new file mode 100644 index 00000000..c83f5c57 --- /dev/null +++ b/tests/integration/test_importers_trello_api.py @@ -0,0 +1,244 @@ +# -*- coding: utf-8 -*- +# Copyright (C) 2014-2016 Andrey Antukh +# Copyright (C) 2014-2016 Jesús Espino +# Copyright (C) 2014-2016 David Barragán +# Copyright (C) 2014-2016 Alejandro Alonso +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import pytest +import json + +from unittest import mock + +from django.core.urlresolvers import reverse + +from .. import factories as f +from taiga.base.utils import json +from taiga.base import exceptions as exc + + +pytestmark = pytest.mark.django_db + + +def test_auth_url(client): + user = f.UserFactory.create() + client.login(user) + + url = reverse("importers-trello-auth-url") + + with mock.patch('taiga.importers.trello.importer.OAuth1Session') as OAuth1SessionMock: + session = mock.Mock() + session.fetch_request_token.return_value = {"oauth_token": "token", "oauth_token_secret": "token"} + OAuth1SessionMock.return_value = session + + response = client.get(url, content_type="application/json") + + assert response.status_code == 200 + assert 'url' in response.data + assert response.data['url'] == "https://trello.com/1/OAuthAuthorizeToken?oauth_token=token&scope=read,write,account&expiration=1day&name=Taiga&return_url=http://localhost:9001/project/new/import/trello" + + +def test_authorize(client): + user = f.UserFactory.create() + client.login(user) + + url = reverse("importers-trello-auth-url") + authorize_url = reverse("importers-trello-authorize") + + with mock.patch('taiga.importers.trello.importer.OAuth1Session') as OAuth1SessionMock: + session = mock.Mock() + session.fetch_request_token.return_value = {"oauth_token": "token", "oauth_token_secret": "token"} + session.fetch_access_token.return_value = {"oauth_token": "token", "oauth_token_secret": "token"} + OAuth1SessionMock.return_value = session + + client.get(url, content_type="application/json") + response = client.post(authorize_url, content_type="application/json", data=json.dumps({"oauth_verifier": "token"})) + + assert response.status_code == 200 + assert 'token' in response.data + assert response.data['token'] == "token" + +def test_authorize_without_token_and_secret(client): + user = f.UserFactory.create() + client.login(user) + + authorize_url = reverse("importers-trello-authorize") + + with mock.patch('taiga.importers.trello.importer.OAuth1Session') as OAuth1SessionMock: + session = mock.Mock() + session.fetch_access_token.return_value = {"oauth_token": "token", "oauth_token_secret": "token"} + OAuth1SessionMock.return_value = session + + response = client.post(authorize_url, content_type="application/json", data=json.dumps({"oauth_verifier": "token"})) + + assert response.status_code == 400 + assert 'token' not in response.data + + +def test_authorize_with_bad_verify(client): + user = f.UserFactory.create() + client.login(user) + + url = reverse("importers-trello-auth-url") + authorize_url = reverse("importers-trello-authorize") + + with mock.patch('taiga.importers.trello.importer.OAuth1Session') as OAuth1SessionMock: + session = mock.Mock() + session.fetch_request_token.return_value = {"oauth_token": "token", "oauth_token_secret": "token"} + session.fetch_access_token.side_effect = Exception("Bad Token") + OAuth1SessionMock.return_value = session + + client.get(url, content_type="application/json") + response = client.post(authorize_url, content_type="application/json", data=json.dumps({"oauth_verifier": "token"})) + + assert response.status_code == 400 + assert 'token' not in response.data + + +def test_import_trello_list_users(client, settings): + user = f.UserFactory.create() + client.login(user) + + url = reverse("importers-trello-list-users") + + with mock.patch('taiga.importers.trello.api.TrelloImporter') as TrelloImporterMock: + instance = mock.Mock() + instance.list_users.return_value = [ + {"id": 1, "fullName": "user1", "email": None}, + {"id": 2, "fullName": "user2", "email": None} + ] + TrelloImporterMock.return_value = instance + response = client.post(url, content_type="application/json", data=json.dumps({"token": "token", "project": 1})) + + assert response.status_code == 200 + assert response.data[0]["id"] == 1 + assert response.data[1]["id"] == 2 + + +def test_import_trello_list_users_without_project(client, settings): + user = f.UserFactory.create() + client.login(user) + + url = reverse("importers-trello-list-users") + + with mock.patch('taiga.importers.trello.api.TrelloImporter') as TrelloImporterMock: + instance = mock.Mock() + instance.list_users.return_value = [ + {"id": 1, "fullName": "user1", "email": None}, + {"id": 2, "fullName": "user2", "email": None} + ] + TrelloImporterMock.return_value = instance + response = client.post(url, content_type="application/json", data=json.dumps({"token": "token"})) + + assert response.status_code == 400 + + +def test_import_trello_list_users_with_problem_on_request(client, settings): + user = f.UserFactory.create() + client.login(user) + + url = reverse("importers-trello-list-users") + + with mock.patch('taiga.importers.trello.importer.TrelloClient') as TrelloClientMock: + instance = mock.Mock() + instance.get.side_effect = exc.WrongArguments("Invalid Request") + TrelloClientMock.return_value = instance + response = client.post(url, content_type="application/json", data=json.dumps({"token": "token", "project": 1})) + + assert response.status_code == 400 + + +def test_import_trello_list_projects(client, settings): + user = f.UserFactory.create() + client.login(user) + + url = reverse("importers-trello-list-projects") + + with mock.patch('taiga.importers.trello.api.TrelloImporter') as TrelloImporterMock: + instance = mock.Mock() + instance.list_projects.return_value = ["project1", "project2"] + TrelloImporterMock.return_value = instance + response = client.post(url, content_type="application/json", data=json.dumps({"token": "token"})) + + assert response.status_code == 200 + assert response.data[0] == "project1" + assert response.data[1] == "project2" + + +def test_import_trello_list_projects_with_problem_on_request(client, settings): + user = f.UserFactory.create() + client.login(user) + + url = reverse("importers-trello-list-projects") + + with mock.patch('taiga.importers.trello.importer.TrelloClient') as TrelloClientMock: + instance = mock.Mock() + instance.get.side_effect = exc.WrongArguments("Invalid Request") + TrelloClientMock.return_value = instance + response = client.post(url, content_type="application/json", data=json.dumps({"token": "token"})) + + assert response.status_code == 400 + + +def test_import_trello_project_without_project_id(client, settings): + settings.CELERY_ENABLED = True + + user = f.UserFactory.create() + client.login(user) + + url = reverse("importers-trello-import-project") + + with mock.patch('taiga.importers.trello.tasks.TrelloImporter') as TrelloImporterMock: + response = client.post(url, content_type="application/json", data=json.dumps({"token": "token"})) + + assert response.status_code == 400 + + +def test_import_trello_project_with_celery_enabled(client, settings): + settings.CELERY_ENABLED = True + + user = f.UserFactory.create() + project = f.ProjectFactory.create(slug="async-imported-project") + client.login(user) + + url = reverse("importers-trello-import-project") + + with mock.patch('taiga.importers.trello.tasks.TrelloImporter') as TrelloImporterMock: + instance = mock.Mock() + instance.import_project.return_value = project + TrelloImporterMock.return_value = instance + response = client.post(url, content_type="application/json", data=json.dumps({"token": "token", "project": 1})) + + assert response.status_code == 202 + assert "task_id" in response.data + + +def test_import_trello_project_with_celery_disabled(client, settings): + settings.CELERY_ENABLED = False + + user = f.UserFactory.create() + project = f.ProjectFactory.create(slug="imported-project") + client.login(user) + + url = reverse("importers-trello-import-project") + + with mock.patch('taiga.importers.trello.api.TrelloImporter') as TrelloImporterMock: + instance = mock.Mock() + instance.import_project.return_value = project + TrelloImporterMock.return_value = instance + response = client.post(url, content_type="application/json", data=json.dumps({"token": "token", "project": 1})) + + assert response.status_code == 200 + assert "slug" in response.data + assert response.data['slug'] == "imported-project"