Add trello importer

requirements.txt

@@ -17,6 +17,8 @@ Markdown==2.6.7
 fn==0.4.3
 diff-match-patch==20121119
 requests==2.12.4
+requests-oauthlib==0.6.2
+webcolors==1.5
 django-sr==0.0.4
 easy-thumbnails==2.3
 celery==3.1.24
@@ -35,3 +37,5 @@ netaddr==0.7.18
 serpy==0.1.1
 psd-tools==1.4
 CairoSVG==2.0.1
+cryptography==1.7.1
+PyJWT==1.4.2

settings/common.py

@@ -318,6 +318,7 @@ INSTALLED_APPS = [
     "taiga.hooks.bitbucket",
     "taiga.hooks.gogs",
     "taiga.webhooks",
+    "taiga.importers",
 
     "djmail",
     "django_jinja",
@@ -561,6 +562,9 @@ MAX_PENDING_MEMBERSHIPS = 30 # Max number of unconfirmed memberships in a projec
 from .sr import *
 
 
+TRELLO_API_KEY = ""
+TRELLO_SECRET_KEY = ""
+
 # NOTE: DON'T INSERT MORE SETTINGS AFTER THIS LINE
 TEST_RUNNER="django.test.runner.DiscoverRunner"
 

taiga/front/urls.py

@@ -23,6 +23,8 @@ urls = {
     "login": "/login",
     "register": "/register",
     "forgot-password": "/forgot-password",
+    "new-project": "/project/new",
+    "new-project-import": "/project/new/import/{0}",
 
     "change-password": "/change-password/{0}", # user.token
     "change-email": "/change-email/{0}", # user.email_token

taiga/importers/api.py

@@ -0,0 +1,39 @@
+# -*- coding: utf-8 -*-
+# Copyright (C) 2014-2016 Taiga Agile LLC <support@taiga.io>
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from taiga.base.api import viewsets
+from taiga.base.decorators import list_route
+
+
+class BaseImporterViewSet(viewsets.ViewSet):
+    @list_route(methods=["GET"])
+    def list_users(self, request, *args, **kwargs):
+        raise NotImplementedError
+
+    @list_route(methods=["GET"])
+    def list_projects(self, request, *args, **kwargs):
+        raise NotImplementedError
+
+    @list_route(methods=["POST"])
+    def import_project(self, request, *args, **kwargs):
+        raise NotImplementedError
+
+    @list_route(methods=["GET"])
+    def auth_url(self, request, *args, **kwargs):
+        raise NotImplementedError
+
+    @list_route(methods=["POST"])
+    def authorize(self, request, *args, **kwargs):
+        raise NotImplementedError

taiga/importers/management/commands/import_from_trello.py

@@ -0,0 +1,89 @@
+# -*- coding: utf-8 -*-
+# Copyright (C) 2014-2016 Andrey Antukh <niwi@niwi.nz>
+# Copyright (C) 2014-2016 Jesús Espino <jespinog@gmail.com>
+# Copyright (C) 2014-2016 David Barragán <bameda@dbarragan.com>
+# Copyright (C) 2014-2016 Alejandro Alonso <alejandro.alonso@kaleidos.net>
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from django.core.management.base import BaseCommand
+from django.db.models import Q
+
+from taiga.importers.trello.importer import TrelloImporter
+from taiga.users.models import User
+from taiga.projects.services import projects as service
+
+import unittest.mock
+import timeit
+
+
+class Command(BaseCommand):
+    def add_arguments(self, parser):
+        parser.add_argument('--token', dest="token", type=str,
+                            help='Auth token')
+        parser.add_argument('--project-id', dest="project_id", type=str,
+                            help='Project ID or full name (ex: taigaio/taiga-back)')
+        parser.add_argument('--template', dest='template', default="kanban",
+                            help='template to use: scrum or kanban (default kanban)')
+        parser.add_argument('--ask-for-users', dest='ask_for_users', const=True,
+                            action="store_const", default=False,
+                            help='Import closed data')
+        parser.add_argument('--closed-data', dest='closed_data', const=True,
+                            action="store_const", default=False,
+                            help='Import closed data')
+        parser.add_argument('--keep-external-reference', dest='keep_external_reference', const=True,
+                            action="store_const", default=False,
+                            help='Store external reference of imported data')
+
+    def handle(self, *args, **options):
+        admin = User.objects.get(username="admin")
+        if options.get('token', None):
+            token = options.get('token')
+        else:
+            (oauth_token, oauth_token_secret, url) = TrelloImporter.get_auth_url()
+            print("Go to here and come with your token: {}".format(url))
+            oauth_verifier = input("Code: ")
+            access_data = TrelloImporter.get_access_token(oauth_token, oauth_token_secret, oauth_verifier)
+            token = access_data['oauth_token']
+            print("Access token: {}".format(token))
+        importer = TrelloImporter(admin, token)
+
+        if options.get('project_id', None):
+            project_id = options.get('project_id')
+        else:
+            print("Select the project to import:")
+            for project in importer.list_projects():
+                print("- {}: {}".format(project['id'], project['name']))
+            project_id = input("Project id: ")
+
+        users_bindings = {}
+        if options.get('ask_for_users', None):
+            print("Add the username or email for next trello users:")
+            for user in importer.list_users(project_id):
+                while True:
+                    username_or_email = input("{}: ".format(user['fullName']))
+                    if username_or_email == "":
+                        break
+                    try:
+                        users_bindings[user['id']] = User.objects.get(Q(username=username_or_email) | Q(email=username_or_email))
+                        break
+                    except User.DoesNotExist:
+                        print("ERROR: Invalid username or email")
+
+        options = {
+            "template": options.get('template'),
+            "import_closed_data": options.get("closed_data", False),
+            "users_bindings": users_bindings,
+            "keep_external_reference": options.get('keep_external_reference')
+        }
+        importer.import_project(project_id, options)

taiga/importers/permissions.py

@@ -0,0 +1,32 @@
+# -*- coding: utf-8 -*-
+# Copyright (C) 2014-2016 Andrey Antukh <niwi@niwi.nz>
+# Copyright (C) 2014-2016 Jesús Espino <jespinog@gmail.com>
+# Copyright (C) 2014-2016 David Barragán <bameda@dbarragan.com>
+# Copyright (C) 2014-2016 Alejandro Alonso <alejandro.alonso@kaleidos.net>
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from taiga.base.api.permissions import TaigaResourcePermission, AllowAny, IsAuthenticated
+from taiga.base.api.permissions import IsSuperUser, HasProjectPerm, IsProjectAdmin
+
+from taiga.permissions.permissions import CommentAndOrUpdatePerm
+
+
+class ImporterPermission(TaigaResourcePermission):
+    enought_perms = IsAuthenticated()
+    global_perms = None
+    auth_url_perms = IsAuthenticated()
+    authorize_perms = IsAuthenticated()
+    list_users_perms = IsAuthenticated()
+    list_projects_perms = IsAuthenticated()
+    import_project_perms = IsAuthenticated()

taiga/importers/trello/api.py

@@ -0,0 +1,147 @@
+# -*- coding: utf-8 -*-
+# Copyright (C) 2014-2016 Taiga Agile LLC <support@taiga.io>
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import uuid
+
+from django.utils.translation import ugettext as _
+from django.conf import settings
+
+from taiga.base.api import viewsets
+from taiga.base import response
+from taiga.base import exceptions as exc
+from taiga.base.decorators import list_route
+from taiga.users.models import AuthData, User
+from taiga.users.services import get_user_photo_url
+from taiga.users.gravatar import get_user_gravatar_id
+from taiga.projects.serializers import ProjectSerializer
+
+from .importer import TrelloImporter
+from taiga.importers import permissions
+from . import tasks
+
+
+class TrelloImporterViewSet(viewsets.ViewSet):
+    permission_classes = (permissions.ImporterPermission,)
+
+    @list_route(methods=["POST"])
+    def list_users(self, request, *args, **kwargs):
+        self.check_permissions(request, "list_users", None)
+
+        token = request.DATA.get('token', None)
+        project_id = request.DATA.get('project', None)
+
+        if not project_id:
+            raise exc.WrongArguments(_("The project param is needed"))
+
+        importer = TrelloImporter(request.user, token)
+        users = importer.list_users(project_id)
+        for user in users:
+            user['user'] = None
+            if not user['email']:
+                continue
+
+            try:
+                taiga_user = User.objects.get(email=user['email'])
+            except User.DoesNotExist:
+                continue
+
+            user['user'] = {
+                'id': taiga_user.id,
+                'full_name': taiga_user.get_full_name(),
+                'gravatar_id': get_user_gravatar_id(taiga_user),
+                'photo': get_user_photo_url(taiga_user),
+            }
+        return response.Ok(users)
+
+    @list_route(methods=["POST"])
+    def list_projects(self, request, *args, **kwargs):
+        self.check_permissions(request, "list_projects", None)
+        token = request.DATA.get('token', None)
+        importer = TrelloImporter(request.user, token)
+        projects = importer.list_projects()
+        return response.Ok(projects)
+
+    @list_route(methods=["POST"])
+    def import_project(self, request, *args, **kwargs):
+        self.check_permissions(request, "import_project", None)
+
+        token = request.DATA.get('token', None)
+        project_id = request.DATA.get('project', None)
+        if not project_id:
+            raise exc.WrongArguments(_("The project param is needed"))
+
+        options = {
+            "name": request.DATA.get('name', None),
+            "description": request.DATA.get('description', None),
+            "template": request.DATA.get('template', "kanban"),
+            "users_bindings": request.DATA.get("users_bindings", {}),
+            "keep_external_reference": request.DATA.get("keep_external_reference", False),
+            "is_private": request.DATA.get("is_private", False),
+        }
+
+        if settings.CELERY_ENABLED:
+            task = tasks.import_project.delay(request.user.id, token, project_id, options)
+            return response.Accepted({"task_id": task.id})
+
+        importer = TrelloImporter(request.user, token)
+        project = importer.import_project(project_id, options)
+        project_data = {
+            "slug": project.slug,
+            "my_permissions": ["view_us"],
+            "is_backlog_activated": project.is_backlog_activated,
+            "is_kanban_activated": project.is_kanban_activated,
+        }
+
+        return response.Ok(project_data)
+
+    @list_route(methods=["GET"])
+    def auth_url(self, request, *args, **kwargs):
+        self.check_permissions(request, "auth_url", None)
+
+        (oauth_token, oauth_secret, url) = TrelloImporter.get_auth_url()
+
+        (auth_data, created) = AuthData.objects.get_or_create(
+            user=request.user,
+            key="trello-oauth",
+            defaults={
+                "value": uuid.uuid4().hex,
+                "extra": {},
+            }
+        )
+        auth_data.extra = {
+            "oauth_token": oauth_token,
+            "oauth_secret": oauth_secret,
+        }
+        auth_data.save()
+
+        return response.Ok({"url": url})
+
+    @list_route(methods=["POST"])
+    def authorize(self, request, *args, **kwargs):
+        self.check_permissions(request, "authorize", None)
+
+        try:
+            oauth_data = request.user.auth_data.get(key="trello-oauth")
+            oauth_token = oauth_data.extra['oauth_token']
+            oauth_secret = oauth_data.extra['oauth_secret']
+            oauth_verifier = request.DATA.get('code')
+            oauth_data.delete()
+            trello_token = TrelloImporter.get_access_token(oauth_token, oauth_secret, oauth_verifier)['oauth_token']
+        except Exception as e:
+            raise exc.WrongArguments(_("Invalid or expired auth token"))
+
+        return response.Ok({
+            "token": trello_token
+        })

taiga/importers/trello/importer.py

@@ -0,0 +1,537 @@
+# -*- coding: utf-8 -*-
+# Copyright (C) 2014-2016 Andrey Antukh <niwi@niwi.nz>
+# Copyright (C) 2014-2016 Jesús Espino <jespinog@gmail.com>
+# Copyright (C) 2014-2016 David Barragán <bameda@dbarragan.com>
+# Copyright (C) 2014-2016 Alejandro Alonso <alejandro.alonso@kaleidos.net>
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from django.utils.translation import ugettext as _
+
+from requests_oauthlib import OAuth1Session, OAuth1
+from django.conf import settings
+from django.core.files.base import ContentFile
+from django.contrib.contenttypes.models import ContentType
+import requests
+import webcolors
+
+from django.template.defaultfilters import slugify
+from taiga.base import exceptions as exc
+from taiga.projects.services import projects as projects_service
+from taiga.projects.models import Project, ProjectTemplate, Membership
+from taiga.projects.userstories.models import UserStory
+from taiga.projects.tasks.models import Task
+from taiga.projects.attachments.models import Attachment
+from taiga.projects.history.services import (make_diff_from_dicts,
+                                             make_diff_values,
+                                             make_key_from_model_object,
+                                             get_typename_for_model_class,
+                                             FrozenDiff)
+from taiga.projects.history.models import HistoryEntry
+from taiga.projects.history.choices import HistoryType
+from taiga.projects.custom_attributes.models import UserStoryCustomAttribute
+from taiga.mdrender.service import render as mdrender
+from taiga.timeline.rebuilder import rebuild_timeline
+from taiga.timeline.models import Timeline
+from taiga.front.templatetags.functions import resolve as resolve_front_url
+
+from taiga.base import exceptions
+
+
+class TrelloClient:
+    def __init__(self, api_key, api_secret, token):
+        self.api_key = api_key
+        self.api_secret = api_secret
+        self.token = token
+        if self.token:
+            self.oauth = OAuth1(
+                client_key=self.api_key,
+                client_secret=self.api_secret,
+                resource_owner_key=self.token
+            )
+        else:
+            self.oauth = None
+
+    def get(self, uri_path, query_params=None):
+        headers = {'Accept': 'application/json'}
+        if query_params is None:
+            query_params = {}
+
+        if uri_path[0] == '/':
+            uri_path = uri_path[1:]
+        url = 'https://api.trello.com/1/%s' % uri_path
+
+        response = requests.get(url, params=query_params, headers=headers, auth=self.oauth)
+
+        if response.status_code == 400:
+            raise exc.WrongArguments(_("Invalid Request: %s at %s") % (response.text, url))
+        if response.status_code == 401:
+            raise exc.AuthenticationFailed(_("Unauthorized: %s at %s") % (response.text, url))
+        if response.status_code == 403:
+            raise exc.PermissionDenied(_("Unauthorized: %s at %s") % (response.text, url))
+        if response.status_code == 404:
+            raise exc.NotFound(_("Resource Unavailable: %s at %s") % (response.text, url))
+        if response.status_code != 200:
+            raise exc.WrongArguments(_("Resource Unavailable: %s at %s") % (response.text, url))
+
+        return response.json()
+
+
+class TrelloImporter:
+    def __init__(self, user, token):
+        self._user = user
+        self._cached_orgs = {}
+        self._client = TrelloClient(
+            api_key=settings.TRELLO_API_KEY,
+            api_secret=settings.TRELLO_SECRET_KEY,
+            token=token,
+        )
+
+    def list_projects(self):
+        projects_data = self._client.get("/members/me/boards", {
+            "fields": "id,name,desc,prefs,idOrganization",
+            "organization": "true",
+            "organization_fields": "prefs",
+        })
+        projects = []
+        for project in projects_data:
+            is_private = False
+            if project['prefs']['permissionLevel'] == "private":
+                is_private = True
+
+            if project['prefs']['permissionLevel'] == "org":
+                if 'organization' not in project:
+                    is_private = True
+                elif project['organization']['prefs']['permissionLevel'] == "private":
+                    is_private = True
+
+            projects.append({
+                "id": project['id'],
+                "name": project['name'],
+                "description": project['desc'],
+                "is_private": is_private,
+            })
+        return projects
+
+    def list_users(self, project_id):
+        members = []
+        for member in self._client.get("/board/{}/members/all".format(project_id), {"fields": "id"}):
+            user = self._client.get("/member/{}".format(member['id']), {"fields": "id,fullName,email"})
+            members.append({
+                "id": user['id'],
+                "full_name": user['fullName'],
+                "email": user['email'],
+            })
+        return members
+
+    def import_project(self, project_id, options):
+        data = self._client.get(
+            "/board/{}".format(project_id),
+            {
+                "fields": "name,desc",
+                "cards": "all",
+                "card_fields": "closed,labels,idList,desc,due,name,pos,dateLastActivity,idChecklists,idMembers,url",
+                "card_attachments": "true",
+                "labels": "all",
+                "labels_limit": "1000",
+                "lists": "all",
+                "list_fields": "closed,name,pos",
+                "members": "none",
+                "checklists": "all",
+                "checklist_fields": "name",
+                "organization": "true",
+                "organization_fields": "logoHash",
+            }
+        )
+
+        project = self._import_project_data(data, options)
+        self._import_user_stories_data(data, project, options)
+        self._cleanup(project, options)
+        Timeline.objects.filter(project=project).delete()
+        rebuild_timeline(None, None, project.id)
+        return project
+
+    def _import_project_data(self, data, options):
+        board = data
+        labels = board['labels']
+        statuses = board['lists']
+        project_template = ProjectTemplate.objects.get(slug=options.get('template', "kanban"))
+        project_template.us_statuses = []
+        counter = 0
+        for us_status in statuses:
+            if counter == 0:
+                project_template.default_options["us_status"] = us_status['name']
+
+            counter += 1
+            if us_status['name'] not in [s['name'] for s in project_template.us_statuses]:
+                project_template.us_statuses.append({
+                    "name": us_status['name'],
+                    "slug": slugify(us_status['name']),
+                    "is_closed": False,
+                    "is_archived": True if us_status['closed'] else False,
+                    "color": "#999999",
+                    "wip_limit": None,
+                    "order": us_status['pos'],
+                })
+
+        project_template.task_statuses = []
+        project_template.task_statuses.append({
+            "name": "Incomplete",
+            "slug": "incomplete",
+            "is_closed": False,
+            "color": "#ff8a84",
+            "order": 1,
+        })
+        project_template.task_statuses.append({
+            "name": "Complete",
+            "slug": "complete",
+            "is_closed": True,
+            "color": "#669900",
+            "order": 2,
+        })
+        project_template.default_options["task_status"] = "Incomplete"
+        project_template.roles.append({
+            "name": "Trello",
+            "slug": "trello",
+            "computable": False,
+            "permissions": project_template.roles[0]['permissions'],
+            "order": 70,
+        })
+
+        tags_colors = []
+        for label in labels:
+            name = label['name']
+            if not name:
+                name = label['color']
+            name = name.lower()
+            color = self._ensure_hex_color(label['color'])
+            tags_colors.append([name, color])
+
+        project = Project(
+            name=options.get('name', None) or board['name'],
+            description=options.get('description', None) or board['desc'],
+            owner=self._user,
+            tags_colors=tags_colors,
+            creation_template=project_template,
+            is_private=options.get('is_private', False),
+        )
+        (can_create, error_message) = projects_service.check_if_project_can_be_created_or_updated(project)
+        if not can_create:
+            raise exceptions.NotEnoughSlotsForProject(project.is_private, 1, error_message)
+        project.save()
+
+        if board.get('organization', None):
+            trello_avatar_template = "https://trello-logos.s3.amazonaws.com/{}/170.png"
+            project_logo_url = trello_avatar_template.format(board['organization']['logoHash'])
+            data = requests.get(project_logo_url)
+            project.logo.save("logo.png", ContentFile(data.content), save=True)
+
+        UserStoryCustomAttribute.objects.create(
+            name="Due",
+            description="Due date",
+            type="date",
+            order=1,
+            project=project
+        )
+        for user in options.get('users_bindings', {}).values():
+            Membership.objects.create(
+                user=user,
+                project=project,
+                role=project.get_roles().get(slug="trello"),
+                is_admin=False,
+                invited_by=self._user,
+            )
+        return project
+
+    def _import_user_stories_data(self, data, project, options):
+        users_bindings = options.get('users_bindings', {})
+        statuses = {s['id']: s for s in data['lists']}
+        cards = data['cards']
+        due_date_field = project.userstorycustomattributes.first()
+
+        for card in cards:
+            if card['closed'] and not options.get("import_closed_data", False):
+                continue
+            if statuses[card['idList']]['closed'] and not options.get("import_closed_data", False):
+                continue
+
+            tags = []
+            for tag in card['labels']:
+                name = tag['name']
+                if not name:
+                    name = tag['color']
+                name = name.lower()
+                tags.append(name)
+
+            assigned_to = None
+            if len(card['idMembers']) > 0:
+                assigned_to = users_bindings.get(card['idMembers'][0], None)
+
+            external_reference = None
+            if options.get('keep_external_reference', False):
+                external_reference = ["trello", card['url']]
+
+            us = UserStory.objects.create(
+                project=project,
+                owner=self._user,
+                assigned_to=assigned_to,
+                status=project.us_statuses.get(name=statuses[card['idList']]['name']),
+                kanban_order=card['pos'],
+                sprint_order=card['pos'],
+                backlog_order=card['pos'],
+                subject=card['name'],
+                description=card['desc'],
+                tags=tags,
+                external_reference=external_reference
+            )
+
+            if len(card['idMembers']) > 1:
+                for watcher in card['idMembers'][1:]:
+                    watcher_user = users_bindings.get(watcher, None)
+                    if watcher_user:
+                        us.add_watcher(watcher_user)
+
+            if card['due']:
+                us.custom_attributes_values.attributes_values = {due_date_field.id: card['due']}
+                us.custom_attributes_values.save()
+
+            UserStory.objects.filter(id=us.id).update(
+                modified_date=card['dateLastActivity'],
+                created_date=card['dateLastActivity']
+            )
+            self._import_attachments(us, card, options)
+            self._import_tasks(data, us, card)
+            self._import_actions(us, card, statuses, options)
+
+    def _import_tasks(self, data, us, card):
+        checklists_by_id = {c['id']: c for c in data['checklists']}
+        for checklist_id in card['idChecklists']:
+            for item in checklists_by_id.get(checklist_id, {}).get('checkItems', []):
+                Task.objects.create(
+                    subject=item['name'],
+                    status=us.project.task_statuses.get(slug=item['state']),
+                    project=us.project,
+                    user_story=us
+                )
+
+    def _import_attachments(self, us, card, options):
+        users_bindings = options.get('users_bindings', {})
+        for attachment in card['attachments']:
+            if attachment['bytes'] is None:
+                continue
+            data = requests.get(attachment['url'])
+            att = Attachment(
+                owner=users_bindings.get(attachment['idMember'], self._user),
+                project=us.project,
+                content_type=ContentType.objects.get_for_model(UserStory),
+                object_id=us.id,
+                name=attachment['name'],
+                size=attachment['bytes'],
+                created_date=attachment['date'],
+                is_deprecated=False,
+            )
+            att.attached_file.save(attachment['name'], ContentFile(data.content), save=True)
+
+            UserStory.objects.filter(id=us.id, created_date__gt=attachment['date']).update(
+                created_date=attachment['date']
+            )
+
+    def _import_actions(self, us, card, statuses, options):
+        included_actions = [
+            "addAttachmentToCard", "addMemberToCard", "commentCard",
+            "convertToCardFromCheckItem", "copyCommentCard", "createCard",
+            "deleteAttachmentFromCard", "deleteCard", "removeMemberFromCard",
+            "updateCard",
+        ]
+
+        actions = self._client.get(
+            "/card/{}/actions".format(card['id']),
+            {
+                "filter": ",".join(included_actions),
+                "limit": "1000",
+                "memberCreator": "true",
+                "memberCreator_fields": "fullName",
+            }
+        )
+
+        while actions:
+            for action in actions:
+                self._import_action(us, action, statuses, options)
+            actions = self._client.get(
+                "/card/{}/actions".format(card['id']),
+                {
+                    "filter": ",".join(included_actions),
+                    "limit": "1000",
+                    "since": "lastView",
+                    "before": action['date'],
+                    "memberCreator": "true",
+                    "memberCreator_fields": "fullName",
+                }
+            )
+
+    def _import_action(self, us, action, statuses, options):
+        key = make_key_from_model_object(us)
+        typename = get_typename_for_model_class(UserStory)
+        action_data = self._transform_action_data(us, action, statuses, options)
+        if action_data is None:
+            return
+
+        change_old = action_data['change_old']
+        change_new = action_data['change_new']
+        hist_type = action_data['hist_type']
+        comment = action_data['comment']
+        user = action_data['user']
+
+        diff = make_diff_from_dicts(change_old, change_new)
+        fdiff = FrozenDiff(key, diff, {})
+
+        entry = HistoryEntry.objects.create(
+            user=user,
+            project_id=us.project.id,
+            key=key,
+            type=hist_type,
+            snapshot=None,
+            diff=fdiff.diff,
+            values=make_diff_values(typename, fdiff),
+            comment=comment,
+            comment_html=mdrender(us.project, comment),
+            is_hidden=False,
+            is_snapshot=False,
+        )
+        HistoryEntry.objects.filter(id=entry.id).update(created_at=action['date'])
+        return HistoryEntry.objects.get(id=entry.id)
+
+    def _transform_action_data(self, us, action, statuses, options):
+        users_bindings = options.get('users_bindings', {})
+        due_date_field = us.project.userstorycustomattributes.first()
+
+        ignored_actions = ["addAttachmentToCard", "addMemberToCard",
+                           "deleteAttachmentFromCard", "deleteCard",
+                           "removeMemberFromCard"]
+
+        if action['type'] in ignored_actions:
+            return None
+
+        user = {"pk": None, "name": action.get('memberCreator', {}).get('fullName', None)}
+        taiga_user = users_bindings.get(action.get('memberCreator', {}).get('id', None), None)
+        if taiga_user:
+            user = {"pk": taiga_user.id, "name": taiga_user.get_full_name()}
+
+        result = {
+            "change_old": {},
+            "change_new": {},
+            "hist_type": HistoryType.change,
+            "comment": "",
+            "user": user
+        }
+
+        if action['type'] == "commentCard":
+            result['comment'] = str(action['data']['text'])
+        elif action['type'] == "convertToCardFromCheckItem":
+            UserStory.objects.filter(id=us.id, created_date__gt=action['date']).update(
+                created_date=action['date'],
+                owner=users_bindings.get(action["idMemberCreator"], self._user)
+            )
+            result['hist_type'] = HistoryType.create
+        elif action['type'] == "copyCommentCard":
+            UserStory.objects.filter(id=us.id, created_date__gt=action['date']).update(
+                created_date=action['date'],
+                owner=users_bindings.get(action["idMemberCreator"], self._user)
+            )
+            result['hist_type'] = HistoryType.create
+        elif action['type'] == "createCard":
+            UserStory.objects.filter(id=us.id, created_date__gt=action['date']).update(
+                created_date=action['date'],
+                owner=users_bindings.get(action["idMemberCreator"], self._user)
+            )
+            result['hist_type'] = HistoryType.create
+        elif action['type'] == "updateCard":
+            if 'desc' in action['data']['old']:
+                result['change_old']["description"] = str(action['data']['old'].get('desc', ''))
+                result['change_new']["description"] = str(action['data']['card'].get('desc', ''))
+                result['change_old']["description_html"] = mdrender(us.project, str(action['data']['old'].get('desc', '')))
+                result['change_new']["description_html"] = mdrender(us.project, str(action['data']['card'].get('desc', '')))
+            if 'idList' in action['data']['old']:
+                old_status_name = statuses[action['data']['old']['idList']]['name']
+                result['change_old']["status"] = us.project.us_statuses.get(name=old_status_name).id
+                new_status_name = statuses[action['data']['card']['idList']]['name']
+                result['change_new']["status"] = us.project.us_statuses.get(name=new_status_name).id
+            if 'name' in action['data']['old']:
+                result['change_old']["subject"] = action['data']['old']['name']
+                result['change_new']["subject"] = action['data']['card']['name']
+            if 'due' in action['data']['old']:
+                result['change_old']["custom_attributes"] = [{
+                    "name": "Due",
+                    "value": action['data']['old']['due'],
+                    "id": due_date_field.id
+                }]
+                result['change_new']["custom_attributes"] = [{
+                    "name": "Due",
+                    "value": action['data']['card']['due'],
+                    "id": due_date_field.id
+                }]
+
+            if result['change_old'] == {}:
+                return None
+        return result
+
+    @classmethod
+    def get_auth_url(cls):
+        request_token_url = 'https://trello.com/1/OAuthGetRequestToken'
+        authorize_url = 'https://trello.com/1/OAuthAuthorizeToken'
+        return_url = resolve_front_url("new-project-import", "trello")
+        expiration = "1day"
+        scope = "read,write,account"
+        trello_key = settings.TRELLO_API_KEY
+        trello_secret = settings.TRELLO_SECRET_KEY
+        name = "Taiga"
+
+        session = OAuth1Session(client_key=trello_key, client_secret=trello_secret)
+        response = session.fetch_request_token(request_token_url)
+        oauth_token, oauth_token_secret = response.get('oauth_token'), response.get('oauth_token_secret')
+
+        return (
+            oauth_token,
+            oauth_token_secret,
+            "{authorize_url}?oauth_token={oauth_token}&scope={scope}&expiration={expiration}&name={name}&return_url={return_url}".format(
+                authorize_url=authorize_url,
+                oauth_token=oauth_token,
+                expiration=expiration,
+                scope=scope,
+                name=name,
+                return_url=return_url,
+            )
+        )
+
+    @classmethod
+    def get_access_token(cls, oauth_token, oauth_token_secret, oauth_verifier):
+        api_key = settings.TRELLO_API_KEY
+        api_secret = settings.TRELLO_SECRET_KEY
+        access_token_url = 'https://trello.com/1/OAuthGetAccessToken'
+        session = OAuth1Session(client_key=api_key, client_secret=api_secret,
+                                resource_owner_key=oauth_token, resource_owner_secret=oauth_token_secret,
+                                verifier=oauth_verifier)
+        access_token = session.fetch_access_token(access_token_url)
+        return access_token
+
+    def _ensure_hex_color(self, color):
+        if color is None:
+            return None
+        try:
+            return webcolors.name_to_hex(color)
+        except ValueError:
+            return color
+
+    def _cleanup(self, project, options):
+        if not options.get("import_closed_data", False):
+            project.us_statuses.filter(is_archived=True).delete()

taiga/importers/trello/tasks.py

@@ -0,0 +1,56 @@
+# -*- coding: utf-8 -*-
+# Copyright (C) 2014-2016 Andrey Antukh <niwi@niwi.nz>
+# Copyright (C) 2014-2016 Jesús Espino <jespinog@gmail.com>
+# Copyright (C) 2014-2016 David Barragán <bameda@dbarragan.com>
+# Copyright (C) 2014-2016 Alejandro Alonso <alejandro.alonso@kaleidos.net>
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import logging
+import sys
+
+from django.utils.translation import ugettext as _
+
+from taiga.base.mails import mail_builder
+from taiga.users.models import User
+from taiga.celery import app
+from .importer import TrelloImporter
+
+logger = logging.getLogger('taiga.importers.trello')
+
+
+@app.task(bind=True)
+def import_project(self, user_id, token, project_id, options):
+    user = User.object.get(id=user_id)
+    importer = TrelloImporter(user, token)
+    try:
+        project = importer.import_project(project_id, options)
+    except Exception as e:
+        # Error
+        ctx = {
+            "user": user,
+            "error_subject": _("Error importing trello project"),
+            "error_message": _("Error importing trello project"),
+            "project": project_id,
+            "exception": e
+        }
+        email = mail_builder.trello_import_error(admin, ctx)
+        email.send()
+        logger.error('Error importing trello project %s (by %s)', project_id, user, exc_info=sys.exc_info())
+    else:
+        ctx = {
+            "project": project,
+            "user": user,
+        }
+        email = mail_builder.trello_import_success(user, ctx)
+        email.send()

taiga/projects/epics/migrations/0005_epic_external_reference.py

@@ -0,0 +1,21 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.9.2 on 2016-11-08 11:19
+from __future__ import unicode_literals
+
+import django.contrib.postgres.fields
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('epics', '0004_auto_20160928_0540'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='epic',
+            name='external_reference',
+            field=django.contrib.postgres.fields.ArrayField(base_field=models.TextField(), blank=True, default=None, null=True, size=None, verbose_name='external reference'),
+        ),
+    ]

taiga/projects/epics/models.py

@@ -18,6 +18,7 @@
 
 from django.db import models
 from django.contrib.contenttypes.fields import GenericRelation
+from django.contrib.postgres.fields import ArrayField
 from django.conf import settings
 from django.utils.translation import ugettext_lazy as _
 from django.utils import timezone
@@ -67,6 +68,8 @@ class Epic(OCCModelMixin, WatchedModelMixin, BlockedMixin, TaggedMixin, models.M
     user_stories = models.ManyToManyField("userstories.UserStory", related_name="epics",
                                           through='RelatedUserStory',
                                           verbose_name=_("user stories"))
+    external_reference = ArrayField(models.TextField(null=False, blank=False),
+                                    null=True, blank=True, default=None, verbose_name=_("external reference"))
 
     attachments = GenericRelation("attachments.Attachment")
 

taiga/projects/references/models.py

@@ -68,6 +68,19 @@ def make_reference(instance, project, create=False):
     return refval, refinstance
 
 
+def recalc_reference_counter(project):
+    seqname = make_sequence_name(project)
+    max_ref_us = project.user_stories.all().aggregate(max=models.Max('ref'))
+    max_ref_task = project.tasks.all().aggregate(max=models.Max('ref'))
+    max_ref_issue = project.issues.all().aggregate(max=models.Max('ref'))
+    max_references = list(filter(lambda x: x is not None, [max_ref_us['max'], max_ref_task['max'], max_ref_issue['max']]))
+
+    max_value = 0
+    if len(max_references) > 0:
+        max_value = max(max_references)
+    seq.set_max(seqname, max_value)
+
+
 def create_sequence(sender, instance, created, **kwargs):
     if not created:
         return

taiga/projects/userstories/api.py

@@ -106,6 +106,7 @@ class UserStoryViewSet(OCCResourceMixin, VotedResourceMixin, HistoryResourceMixi
 
         include_attachments = "include_attachments" in self.request.QUERY_PARAMS
         include_tasks = "include_tasks" in self.request.QUERY_PARAMS
+
         epic_id = self.request.QUERY_PARAMS.get("epic", None)
         # We can be filtering by more than one epic so epic_id can consist
         # of different ids separete by comma. In that situation we will use

taiga/routers.py

@@ -283,6 +283,11 @@ from taiga.external_apps.api import Application, ApplicationToken
 router.register(r"applications", Application, base_name="applications")
 router.register(r"application-tokens", ApplicationToken, base_name="application-tokens")
 
+# Third party importers
+from taiga.importers.trello.api import TrelloImporterViewSet
+
+router.register(r"importers/trello", TrelloImporterViewSet, base_name="importers-trello")
+
 
 # Stats
 #   - see taiga.stats.routers and taiga.stats.apps

tests/integration/test_importers_trello_api.py

@@ -0,0 +1,244 @@
+# -*- coding: utf-8 -*-
+# Copyright (C) 2014-2016 Andrey Antukh <niwi@niwi.nz>
+# Copyright (C) 2014-2016 Jesús Espino <jespinog@gmail.com>
+# Copyright (C) 2014-2016 David Barragán <bameda@dbarragan.com>
+# Copyright (C) 2014-2016 Alejandro Alonso <alejandro.alonso@kaleidos.net>
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import pytest
+import json
+
+from unittest import mock
+
+from django.core.urlresolvers import reverse
+
+from .. import factories as f
+from taiga.base.utils import json
+from taiga.base import exceptions as exc
+
+
+pytestmark = pytest.mark.django_db
+
+
+def test_auth_url(client):
+    user = f.UserFactory.create()
+    client.login(user)
+
+    url = reverse("importers-trello-auth-url")
+
+    with mock.patch('taiga.importers.trello.importer.OAuth1Session') as OAuth1SessionMock:
+        session = mock.Mock()
+        session.fetch_request_token.return_value = {"oauth_token": "token", "oauth_token_secret": "token"}
+        OAuth1SessionMock.return_value = session
+
+        response = client.get(url, content_type="application/json")
+
+    assert response.status_code == 200
+    assert 'url' in response.data
+    assert response.data['url'] == "https://trello.com/1/OAuthAuthorizeToken?oauth_token=token&scope=read,write,account&expiration=1day&name=Taiga&return_url=http://localhost:9001/project/new/import/trello"
+
+
+def test_authorize(client):
+    user = f.UserFactory.create()
+    client.login(user)
+
+    url = reverse("importers-trello-auth-url")
+    authorize_url = reverse("importers-trello-authorize")
+
+    with mock.patch('taiga.importers.trello.importer.OAuth1Session') as OAuth1SessionMock:
+        session = mock.Mock()
+        session.fetch_request_token.return_value = {"oauth_token": "token", "oauth_token_secret": "token"}
+        session.fetch_access_token.return_value = {"oauth_token": "token", "oauth_token_secret": "token"}
+        OAuth1SessionMock.return_value = session
+
+        client.get(url, content_type="application/json")
+        response = client.post(authorize_url, content_type="application/json", data=json.dumps({"oauth_verifier": "token"}))
+
+    assert response.status_code == 200
+    assert 'token' in response.data
+    assert response.data['token'] == "token"
+
+def test_authorize_without_token_and_secret(client):
+    user = f.UserFactory.create()
+    client.login(user)
+
+    authorize_url = reverse("importers-trello-authorize")
+
+    with mock.patch('taiga.importers.trello.importer.OAuth1Session') as OAuth1SessionMock:
+        session = mock.Mock()
+        session.fetch_access_token.return_value = {"oauth_token": "token", "oauth_token_secret": "token"}
+        OAuth1SessionMock.return_value = session
+
+        response = client.post(authorize_url, content_type="application/json", data=json.dumps({"oauth_verifier": "token"}))
+
+    assert response.status_code == 400
+    assert 'token' not in response.data
+
+
+def test_authorize_with_bad_verify(client):
+    user = f.UserFactory.create()
+    client.login(user)
+
+    url = reverse("importers-trello-auth-url")
+    authorize_url = reverse("importers-trello-authorize")
+
+    with mock.patch('taiga.importers.trello.importer.OAuth1Session') as OAuth1SessionMock:
+        session = mock.Mock()
+        session.fetch_request_token.return_value = {"oauth_token": "token", "oauth_token_secret": "token"}
+        session.fetch_access_token.side_effect = Exception("Bad Token")
+        OAuth1SessionMock.return_value = session
+
+        client.get(url, content_type="application/json")
+        response = client.post(authorize_url, content_type="application/json", data=json.dumps({"oauth_verifier": "token"}))
+
+    assert response.status_code == 400
+    assert 'token' not in response.data
+
+
+def test_import_trello_list_users(client, settings):
+    user = f.UserFactory.create()
+    client.login(user)
+
+    url = reverse("importers-trello-list-users")
+
+    with mock.patch('taiga.importers.trello.api.TrelloImporter') as TrelloImporterMock:
+        instance = mock.Mock()
+        instance.list_users.return_value = [
+            {"id": 1, "fullName": "user1", "email": None},
+            {"id": 2, "fullName": "user2", "email": None}
+        ]
+        TrelloImporterMock.return_value = instance
+        response = client.post(url, content_type="application/json", data=json.dumps({"token": "token", "project": 1}))
+
+    assert response.status_code == 200
+    assert response.data[0]["id"] == 1
+    assert response.data[1]["id"] == 2
+
+
+def test_import_trello_list_users_without_project(client, settings):
+    user = f.UserFactory.create()
+    client.login(user)
+
+    url = reverse("importers-trello-list-users")
+
+    with mock.patch('taiga.importers.trello.api.TrelloImporter') as TrelloImporterMock:
+        instance = mock.Mock()
+        instance.list_users.return_value = [
+            {"id": 1, "fullName": "user1", "email": None},
+            {"id": 2, "fullName": "user2", "email": None}
+        ]
+        TrelloImporterMock.return_value = instance
+        response = client.post(url, content_type="application/json", data=json.dumps({"token": "token"}))
+
+    assert response.status_code == 400
+
+
+def test_import_trello_list_users_with_problem_on_request(client, settings):
+    user = f.UserFactory.create()
+    client.login(user)
+
+    url = reverse("importers-trello-list-users")
+
+    with mock.patch('taiga.importers.trello.importer.TrelloClient') as TrelloClientMock:
+        instance = mock.Mock()
+        instance.get.side_effect = exc.WrongArguments("Invalid Request")
+        TrelloClientMock.return_value = instance
+        response = client.post(url, content_type="application/json", data=json.dumps({"token": "token", "project": 1}))
+
+    assert response.status_code == 400
+
+
+def test_import_trello_list_projects(client, settings):
+    user = f.UserFactory.create()
+    client.login(user)
+
+    url = reverse("importers-trello-list-projects")
+
+    with mock.patch('taiga.importers.trello.api.TrelloImporter') as TrelloImporterMock:
+        instance = mock.Mock()
+        instance.list_projects.return_value = ["project1", "project2"]
+        TrelloImporterMock.return_value = instance
+        response = client.post(url, content_type="application/json", data=json.dumps({"token": "token"}))
+
+    assert response.status_code == 200
+    assert response.data[0] == "project1"
+    assert response.data[1] == "project2"
+
+
+def test_import_trello_list_projects_with_problem_on_request(client, settings):
+    user = f.UserFactory.create()
+    client.login(user)
+
+    url = reverse("importers-trello-list-projects")
+
+    with mock.patch('taiga.importers.trello.importer.TrelloClient') as TrelloClientMock:
+        instance = mock.Mock()
+        instance.get.side_effect = exc.WrongArguments("Invalid Request")
+        TrelloClientMock.return_value = instance
+        response = client.post(url, content_type="application/json", data=json.dumps({"token": "token"}))
+
+    assert response.status_code == 400
+
+
+def test_import_trello_project_without_project_id(client, settings):
+    settings.CELERY_ENABLED = True
+
+    user = f.UserFactory.create()
+    client.login(user)
+
+    url = reverse("importers-trello-import-project")
+
+    with mock.patch('taiga.importers.trello.tasks.TrelloImporter') as TrelloImporterMock:
+        response = client.post(url, content_type="application/json", data=json.dumps({"token": "token"}))
+
+    assert response.status_code == 400
+
+
+def test_import_trello_project_with_celery_enabled(client, settings):
+    settings.CELERY_ENABLED = True
+
+    user = f.UserFactory.create()
+    project = f.ProjectFactory.create(slug="async-imported-project")
+    client.login(user)
+
+    url = reverse("importers-trello-import-project")
+
+    with mock.patch('taiga.importers.trello.tasks.TrelloImporter') as TrelloImporterMock:
+        instance = mock.Mock()
+        instance.import_project.return_value = project
+        TrelloImporterMock.return_value = instance
+        response = client.post(url, content_type="application/json", data=json.dumps({"token": "token", "project": 1}))
+
+    assert response.status_code == 202
+    assert "task_id" in response.data
+
+
+def test_import_trello_project_with_celery_disabled(client, settings):
+    settings.CELERY_ENABLED = False
+
+    user = f.UserFactory.create()
+    project = f.ProjectFactory.create(slug="imported-project")
+    client.login(user)
+
+    url = reverse("importers-trello-import-project")
+
+    with mock.patch('taiga.importers.trello.api.TrelloImporter') as TrelloImporterMock:
+        instance = mock.Mock()
+        instance.import_project.return_value = project
+        TrelloImporterMock.return_value = instance
+        response = client.post(url, content_type="application/json", data=json.dumps({"token": "token", "project": 1}))
+
+    assert response.status_code == 200
+    assert "slug" in response.data
+    assert response.data['slug'] == "imported-project"