From cca421b14d22d0bb11c3a5e2d4adcfaa44aca01e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jes=C3=BAs=20Espino?= Date: Wed, 18 Jan 2017 09:40:05 +0100 Subject: [PATCH] Add github importer --- settings/common.py | 2 + taiga/importers/exceptions.py | 5 + taiga/importers/github/api.py | 131 ++++ taiga/importers/github/importer.py | 604 ++++++++++++++++++ taiga/importers/github/tasks.py | 56 ++ .../management/commands/import_from_github.py | 100 +++ taiga/routers.py | 2 + .../integration/test_importers_github_api.py | 228 +++++++ 8 files changed, 1128 insertions(+) create mode 100644 taiga/importers/exceptions.py create mode 100644 taiga/importers/github/api.py create mode 100644 taiga/importers/github/importer.py create mode 100644 taiga/importers/github/tasks.py create mode 100644 taiga/importers/management/commands/import_from_github.py create mode 100644 tests/integration/test_importers_github_api.py diff --git a/settings/common.py b/settings/common.py index 7a894969..9ce0e415 100644 --- a/settings/common.py +++ b/settings/common.py @@ -561,6 +561,8 @@ MAX_PENDING_MEMBERSHIPS = 30 # Max number of unconfirmed memberships in a projec from .sr import * +GITHUB_API_CLIENT_ID = "" +GITHUB_API_CLIENT_SECRET = "" TRELLO_API_KEY = "" TRELLO_SECRET_KEY = "" diff --git a/taiga/importers/exceptions.py b/taiga/importers/exceptions.py new file mode 100644 index 00000000..da430079 --- /dev/null +++ b/taiga/importers/exceptions.py @@ -0,0 +1,5 @@ +class InvalidAuthResult(Exception): + pass + +class FailedRequest(Exception): + pass diff --git a/taiga/importers/github/api.py b/taiga/importers/github/api.py new file mode 100644 index 00000000..e8991c0d --- /dev/null +++ b/taiga/importers/github/api.py @@ -0,0 +1,131 @@ +# -*- coding: utf-8 -*- +# Copyright (C) 2014-2016 Taiga Agile LLC +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +from django.utils.translation import ugettext as _ +from django.conf import settings + +from taiga.base.api import viewsets +from taiga.base import response +from taiga.base import exceptions as exc +from taiga.base.decorators import list_route +from taiga.users.models import AuthData, User +from taiga.users.services import get_user_photo_url +from taiga.users.gravatar import get_user_gravatar_id +from taiga.projects.serializers import ProjectSerializer + +from taiga.importers import permissions +from taiga.importers import exceptions +from .importer import GithubImporter +from . import tasks + + +class GithubImporterViewSet(viewsets.ViewSet): + permission_classes = (permissions.ImporterPermission,) + + @list_route(methods=["POST"]) + def list_users(self, request, *args, **kwargs): + self.check_permissions(request, "list_users", None) + + token = request.DATA.get('token', None) + project_id = request.DATA.get('project', None) + + if not project_id: + raise exc.WrongArguments(_("The project param is needed")) + + importer = GithubImporter(request.user, token) + users = importer.list_users(project_id) + for user in users: + if user['detected_user']: + user['user'] = { + 'id': user['detected_user'].id, + 'full_name': user['detected_user'].get_full_name(), + 'gravatar_id': get_user_gravatar_id(user['detected_user']), + 'photo': get_user_photo_url(user['detected_user']), + } + del(user['detected_user']) + return response.Ok(users) + + @list_route(methods=["POST"]) + def list_projects(self, request, *args, **kwargs): + self.check_permissions(request, "list_projects", None) + token = request.DATA.get('token', None) + importer = GithubImporter(request.user, token) + projects = importer.list_projects() + return response.Ok(projects) + + @list_route(methods=["POST"]) + def import_project(self, request, *args, **kwargs): + self.check_permissions(request, "import_project", None) + + token = request.DATA.get('token', None) + project_id = request.DATA.get('project', None) + if not project_id: + raise exc.WrongArguments(_("The project param is needed")) + + template = request.DATA.get('template', "scrum") + items_type = "user_stories" + if template == "issues": + items_type = "issues" + template = "scrum" + + options = { + "name": request.DATA.get('name', None), + "description": request.DATA.get('description', None), + "template": template, + "type": items_type, + "users_bindings": request.DATA.get("users_bindings", {}), + "keep_external_reference": request.DATA.get("keep_external_reference", False), + "is_private": request.DATA.get("is_private", False), + } + + if settings.CELERY_ENABLED: + task = tasks.import_project.delay(request.user.id, token, project_id, options) + return response.Accepted({"task_id": task.id}) + + importer = GithubImporter(request.user, token) + project = importer.import_project(project_id, options) + project_data = { + "slug": project.slug, + "my_permissions": ["view_us"], + "is_backlog_activated": project.is_backlog_activated, + "is_kanban_activated": project.is_kanban_activated, + } + + return response.Ok(project_data) + + @list_route(methods=["GET"]) + def auth_url(self, request, *args, **kwargs): + self.check_permissions(request, "auth_url", None) + callback_uri = request.QUERY_PARAMS.get('uri') + url = GithubImporter.get_auth_url(settings.GITHUB_API_CLIENT_ID, callback_uri) + return response.Ok({"url": url}) + + @list_route(methods=["POST"]) + def authorize(self, request, *args, **kwargs): + self.check_permissions(request, "authorize", None) + + code = request.DATA.get('code', None) + if code is None: + raise exc.BadRequest(_("Code param needed")) + + try: + token = GithubImporter.get_access_token(settings.GITHUB_API_CLIENT_ID, settings.GITHUB_API_CLIENT_SECRET, code) + return response.Ok({ + "token": token + }) + except exceptions.InvalidAuthResult: + raise exc.BadRequest(_("Invalid auth data")) + except exceptions.FailedRequest: + raise exc.BadRequest(_("Third party service failing")) diff --git a/taiga/importers/github/importer.py b/taiga/importers/github/importer.py new file mode 100644 index 00000000..b042feef --- /dev/null +++ b/taiga/importers/github/importer.py @@ -0,0 +1,604 @@ +import requests +from urllib.parse import parse_qsl +from django.core.files.base import ContentFile + +from taiga.projects.models import Project, ProjectTemplate, Membership +from taiga.projects.references.models import recalc_reference_counter +from taiga.projects.userstories.models import UserStory +from taiga.projects.issues.models import Issue +from taiga.projects.milestones.models import Milestone +from taiga.projects.history.services import take_snapshot +from taiga.projects.history.services import (make_diff_from_dicts, + make_diff_values, + make_key_from_model_object, + get_typename_for_model_class, + FrozenDiff) +from taiga.projects.history.models import HistoryEntry +from taiga.projects.history.choices import HistoryType +from taiga.timeline.rebuilder import rebuild_timeline +from taiga.timeline.models import Timeline +from taiga.users.models import User, AuthData + + +class GithubClient: + def __init__(self, token): + self.api_url = "https://api.github.com/{}" + self.token = token + + def get(self, uri_path, query_params=None): + headers = { + "Content-Type": "application/json", + "X-GitHub-Media-Type": "github.v3" + } + if self.token: + headers['Authorization'] = 'token {}'.format(self.token) + + if query_params is None: + query_params = {} + + if uri_path[0] == '/': + uri_path = uri_path[1:] + url = self.api_url.format(uri_path) + + response = requests.get(url, params=query_params, headers=headers) + + if response.status_code == 401: + raise Exception("Unauthorized: %s at %s" % (response.text, url), response) + if response.status_code != 200: + raise Exception("Resource Unavailable: %s at %s" % (response.text, url), response) + + return response.json() + + +class GithubImporter: + def __init__(self, user, token, import_closed_data=False): + self._import_closed_data = import_closed_data + self._user = user + self._client = GithubClient(token) + self._me = self._client.get("/user") + + def list_projects(self): + projects = [] + page = 1 + while True: + repos = self._client.get("/user/repos", { + "sort": "full_name", + "page": page, + "per_page": 100 + }) + page += 1 + + for repo in repos: + projects.append({ + "id": repo['full_name'], + "name": repo['full_name'], + "description": repo['description'], + "is_private": repo['private'], + }) + + if len(repos) < 100: + break + return projects + + def list_users(self, project_full_name): + collaborators = self._client.get("/repos/{}/collaborators".format(project_full_name)) + collaborators = [self._client.get("/users/{}".format(u['login'])) for u in collaborators] + return [{"id": u['id'], + "username": u['login'], + "full_name": u.get('name', u['login']), + "detected_user": self._get_user(u) } for u in collaborators] + + def _get_user(self, user, default=None): + if not user: + return default + + try: + return AuthData.objects.get(key="github", value=user['id']).user + except AuthData.DoesNotExist: + pass + + try: + return User.objects.get(email=user.get('email', "not-valid")) + except User.DoesNotExist: + pass + + return default + + def import_project(self, project_full_name, options={"keep_external_reference": False, "template": "kanban", "type": "user_stories"}): + repo = self._client.get('/repos/{}'.format(project_full_name)) + project = self._import_project_data(repo, options) + if options.get('type', None) == "user_stories": + self._import_user_stories_data(project, repo, options) + elif options.get('type', None) == "issues": + self._import_issues_data(project, repo, options) + self._import_comments(project, repo, options) + self._import_history(project, repo, options) + Timeline.objects.filter(project=project).delete() + rebuild_timeline(None, None, project.id) + recalc_reference_counter(project) + return project + + def _import_project_data(self, repo, options): + users_bindings = options.get('users_bindings', {}) + project_template = ProjectTemplate.objects.get(slug=options['template']) + + if options['type'] == "user_stories": + project_template.us_statuses = [] + project_template.us_statuses.append({ + "name": "Open", + "slug": "open", + "is_closed": False, + "is_archived": False, + "color": "#ff8a84", + "wip_limit": None, + "order": 1, + }) + project_template.us_statuses.append({ + "name": "Closed", + "slug": "closed", + "is_closed": True, + "is_archived": False, + "color": "#669900", + "wip_limit": None, + "order": 2, + }) + project_template.default_options["us_status"] = "Open" + elif options['type'] == "issues": + project_template.issue_statuses = [] + project_template.issue_statuses.append({ + "name": "Open", + "slug": "open", + "is_closed": False, + "color": "#ff8a84", + "order": 1, + }) + project_template.issue_statuses.append({ + "name": "Closed", + "slug": "closed", + "is_closed": True, + "color": "#669900", + "order": 2, + }) + project_template.default_options["issue_status"] = "Open" + + project_template.roles.append({ + "name": "Github", + "slug": "github", + "computable": False, + "permissions": project_template.roles[0]['permissions'], + "order": 70, + }) + + tags_colors = [] + for label in self._client.get("/repos/{}/labels".format(repo['full_name'])): + name = label['name'].lower() + color = "#{}".format(label['color']) + tags_colors.append([name, color]) + + project = Project.objects.create( + name=options.get('name', None) or repo['full_name'], + description=options.get('description', None) or repo['description'], + owner=self._user, + tags_colors=tags_colors, + creation_template=project_template, + is_private=options.get('is_private', False), + ) + + if 'organization' in repo and repo['organization'].get('avatar_url', None): + data = requests.get(repo['organization']['avatar_url']) + project.logo.save("logo.png", ContentFile(data.content), save=True) + + for user in self._client.get("/repos/{}/collaborators".format(repo['full_name'])): + taiga_user = users_bindings.get(user['id'], None) + if taiga_user is None or taiga_user == self._user: + continue + + Membership.objects.create( + user=taiga_user, + project=project, + role=project.get_roles().get(slug="github"), + is_admin=False, + invited_by=self._user, + ) + + for milestone in self._client.get("/repos/{}/milestones".format(repo['full_name'])): + taiga_milestone = Milestone.objects.create( + name=milestone['title'], + owner=users_bindings.get(milestone.get('creator', {}).get('id', None), self._user), + project=project, + estimated_start=milestone['created_at'][:10], + estimated_finish=milestone['due_on'][:10], + ) + Milestone.objects.filter(id=taiga_milestone.id).update( + created_date=milestone['created_at'], + modified_date=milestone['updated_at'], + ) + return project + + def _import_user_stories_data(self, project, repo, options): + users_bindings = options.get('users_bindings', {}) + + page = 1 + while True: + issues = self._client.get("/repos/{}/issues".format(repo['full_name']), { + "state": "all", + "sort": "created", + "direction": "asc", + "page": page, + "per_page": 100 + }) + page += 1 + for issue in issues: + tags = [] + for label in issue['labels']: + tags.append(label['name'].lower()) + + assigned_to = users_bindings.get(issue['assignee']['id'] if issue['assignee'] else None, None) + + external_reference = None + if options.get('keep_external_reference', False): + external_reference = ["github", issue['html_url']] + + us = UserStory.objects.create( + ref=issue['number'], + project=project, + owner=users_bindings.get(issue['user']['id'], self._user), + milestone=project.milestones.get(name=issue['milestone']['title']) if issue['milestone'] else None, + assigned_to=assigned_to, + status=project.us_statuses.get(slug=issue['state']), + kanban_order=issue['number'], + sprint_order=issue['number'], + backlog_order=issue['number'], + subject=issue['title'], + description=issue.get("body", "") or "", + tags=tags, + external_reference=external_reference, + modified_date=issue['updated_at'], + created_date=issue['created_at'], + ) + + assignees = issue.get('assignees', []) + if len(assignees) > 1: + for assignee in assignees: + if assignee['id'] != issue.get('assignee', {}).get('id', None): + assignee_user = users_bindings.get(assignee['id'], None) + if assignee_user is not None: + us.add_watcher(assignee_user) + + UserStory.objects.filter(id=us.id).update( + ref=issue['number'], + modified_date=issue['updated_at'], + created_date=issue['created_at'] + ) + + take_snapshot(us, comment="", user=None, delete=False) + + if len(issues) < 100: + break + + def _import_issues_data(self, project, repo, options): + users_bindings = options.get('users_bindings', {}) + + page = 1 + while True: + issues = self._client.get("/repos/{}/issues".format(repo['full_name']), { + "state": "all", + "sort": "created", + "direction": "asc", + "page": page, + "per_page": 100 + }) + page += 1 + for issue in issues: + tags = [] + for label in issue['labels']: + tags.append(label['name'].lower()) + + assigned_to = users_bindings.get(issue['assignee']['id'] if issue['assignee'] else None, None) + + external_reference = None + if options.get('keep_external_reference', False): + external_reference = ["github", issue['html_url']] + + taiga_issue = Issue.objects.create( + ref=issue['number'], + project=project, + owner=users_bindings.get(issue['user']['id'], self._user), + assigned_to=assigned_to, + status=project.issue_statuses.get(slug=issue['state']), + subject=issue['title'], + description=issue.get('body', "") or "", + tags=tags, + external_reference=external_reference, + modified_date=issue['updated_at'], + created_date=issue['created_at'], + ) + + assignees = issue.get('assignees', []) + if len(assignees) > 1: + for assignee in assignees: + if assignee['id'] != issue.get('assignee', {}).get('id', None): + assignee_user = users_bindings.get(assignee['id'], None) + if assignee_user is not None: + taiga_issue.add_watcher(assignee_user) + + Issue.objects.filter(id=taiga_issue.id).update( + ref=issue['number'], + modified_date=issue['updated_at'], + created_date=issue['created_at'] + ) + + take_snapshot(taiga_issue, comment="", user=None, delete=False) + + if len(issues) < 100: + break + + def _import_comments(self, project, repo, options): + users_bindings = options.get('users_bindings', {}) + + page = 1 + while True: + comments = self._client.get("/repos/{}/issues/comments".format(repo['full_name']), { + "page": page, + "per_page": 100 + }) + page += 1 + + for comment in comments: + issue_id = comment['issue_url'].split("/")[-1] + if options.get('type', None) == "user_stories": + obj = UserStory.objects.get(project=project, ref=issue_id) + elif options.get('type', None) == "issues": + obj = Issue.objects.get(project=project, ref=issue_id) + + snapshot = take_snapshot( + obj, + comment=comment['body'], + user=users_bindings.get(comment['user']['id'], User(full_name=comment['user'].get('name', None) or comment['user']['login'])), + delete=False + ) + HistoryEntry.objects.filter(id=snapshot.id).update(created_at=comment['created_at']) + + if len(comments) < 100: + break + + def _import_history(self, project, repo, options): + cumulative_data = {} + page = 1 + all_events = [] + while True: + events = self._client.get("/repos/{}/issues/events".format(repo['full_name']), { + "page": page, + "per_page": 100 + }) + page += 1 + all_events = all_events + events + + if len(events) < 100: + break + + for event in sorted(all_events, key=lambda x: x['id']): + if options.get('type', None) == "user_stories": + obj = UserStory.objects.get(project=project, ref=event['issue']['number']) + elif options.get('type', None) == "issues": + obj = Issue.objects.get(project=project, ref=event['issue']['number']) + + if event['issue']['number'] in cumulative_data: + obj_cumulative_data = cumulative_data[event['issue']['number']] + else: + obj_cumulative_data = { + "tags": set(), + "assigned_to": None, + "assigned_to_github_id": None, + "assigned_to_name": None, + "milestone": None, + } + cumulative_data[event['issue']['number']] = obj_cumulative_data + self._import_event(obj, event, options, obj_cumulative_data) + + def _import_event(self, obj, event, options, cumulative_data): + typename = get_typename_for_model_class(type(obj)) + key = make_key_from_model_object(obj) + event_data = self._transform_event_data(obj, event, options, cumulative_data) + if event_data is None: + return + + change_old = event_data['change_old'] + change_new = event_data['change_new'] + user = event_data['user'] + + diff = make_diff_from_dicts(change_old, change_new) + fdiff = FrozenDiff(key, diff, {}) + values = make_diff_values(typename, fdiff) + values.update(event_data['update_values']) + entry = HistoryEntry.objects.create( + user=user, + project_id=obj.project.id, + key=key, + type=HistoryType.change, + snapshot=None, + diff=fdiff.diff, + values=values, + comment="", + comment_html="", + is_hidden=False, + is_snapshot=False, + ) + HistoryEntry.objects.filter(id=entry.id).update(created_at=event['created_at']) + return HistoryEntry.objects.get(id=entry.id) + + def _transform_event_data(self, obj, event, options, cumulative_data): + users_bindings = options.get('users_bindings', {}) + + ignored_events = ["committed", "cross-referenced", "head_ref_deleted", + "head_ref_restored", "locked", "unlocked", "merged", + "referenced", "mentioned", "subscribed", + "unsubscribed"] + + if event['event'] in ignored_events: + return None + + user = {"pk": None, "name": event['actor'].get('name', event['actor']['login'])} + taiga_user = users_bindings.get(event['actor']['id'], None) if event['actor'] else None + if taiga_user: + user = {"pk": taiga_user.id, "name": taiga_user.get_full_name()} + + result = { + "change_old": {}, + "change_new": {}, + "user": user, + "update_values": {}, + } + + if event['event'] == "renamed": + result['change_old']["subject"] = event['rename']['from'] + result['change_new']["subject"] = event['rename']['to'] + elif event['event'] == "reopened": + if isinstance(obj, Issue): + result['change_old']["status"] = obj.project.issue_statuses.get(name='Closed').id + result['change_new']["status"] = obj.project.issue_statuses.get(name='Open').id + elif isinstance(obj, UserStory): + result['change_old']["status"] = obj.project.us_statuses.get(name='Closed').id + result['change_new']["status"] = obj.project.us_statuses.get(name='Open').id + elif event['event'] == "closed": + if isinstance(obj, Issue): + result['change_old']["status"] = obj.project.issue_statuses.get(name='Open').id + result['change_new']["status"] = obj.project.issue_statuses.get(name='Closed').id + elif isinstance(obj, UserStory): + result['change_old']["status"] = obj.project.us_statuses.get(name='Open').id + result['change_new']["status"] = obj.project.us_statuses.get(name='Closed').id + elif event['event'] == "assigned": + AssignedEventHandler(result, cumulative_data, users_bindings).handle(event) + elif event['event'] == "unassigned": + UnassignedEventHandler(result, cumulative_data, users_bindings).handle(event) + elif event['event'] == "demilestoned": + if isinstance(obj, UserStory): + try: + result['change_old']["milestone"] = obj.project.milestones.get(name=event['milestone']['title']).id + except Milestone.DoesNotExist: + result['change_old']["milestone"] = 0 + result['update_values'] = {"milestone": {"0": event['milestone']['title']}} + result['change_new']["milestone"] = None + cumulative_data['milestone'] = None + elif event['event'] == "milestoned": + if isinstance(obj, UserStory): + result['update_values']["milestone"] = {} + if cumulative_data['milestone'] is not None: + result['update_values']['milestone'][str(cumulative_data['milestone'])] = cumulative_data['milestone_name'] + result['change_old']["milestone"] = cumulative_data['milestone'] + try: + taiga_milestone = obj.project.milestones.get(name=event['milestone']['title']) + cumulative_data["milestone"] = taiga_milestone.id + cumulative_data['milestone_name'] = taiga_milestone.name + except Milestone.DoesNotExist: + if cumulative_data['milestone'] == 0: + cumulative_data['milestone'] = -1 + else: + cumulative_data['milestone'] = 0 + cumulative_data['milestone_name'] = event['milestone']['title'] + result['change_new']["milestone"] = cumulative_data['milestone'] + result['update_values']['milestone'][str(cumulative_data['milestone'])] = cumulative_data['milestone_name'] + elif event['event'] == "labeled": + result['change_old']["tags"] = list(cumulative_data['tags']) + cumulative_data['tags'].add(event['label']['name'].lower()) + result['change_new']["tags"] = list(cumulative_data['tags']) + elif event['event'] == "unlabeled": + result['change_old']["tags"] = list(cumulative_data['tags']) + if event['label']['name'].lower() in cumulative_data['tags']: + cumulative_data['tags'].remove(event['label']['name'].lower()) + result['change_new']["tags"] = list(cumulative_data['tags']) + + return result + + @classmethod + def get_auth_url(cls, client_id, callback_uri=None): + if callback_uri is None: + return "https://github.com/login/oauth/authorize?client_id={}&scope=user,repo".format(client_id) + return "https://github.com/login/oauth/authorize?client_id={}&scope=user,repo&redirect_uri={}".format(client_id, callback_uri) + + @classmethod + def get_access_token(cls, client_id, client_secret, code): + try: + result = requests.post("https://github.com/login/oauth/access_token", { + "client_id": client_id, + "client_secret": client_secret, + "code": code, + }) + except Exception: + raise FailedRequest() + + if result.status_code > 299: + raise InvalidAuthResult() + else: + return dict(parse_qsl(result.content))[b'access_token'].decode('utf-8') + + +class AssignedEventHandler: + def __init__(self, result, cumulative_data, users_bindings): + self.result = result + self.cumulative_data = cumulative_data + self.users_bindings = users_bindings + + def handle(self, event): + if self.cumulative_data['assigned_to_github_id'] is None: + self.result['update_values']["users"] = {} + self.generate_change_old(event) + self.generate_update_values_from_cumulative_data(event) + user = self.users_bindings.get(event['assignee']['id'], None) + self.generate_change_new(event, user) + self.update_cumulative_data(event, user) + self.generate_update_values_from_cumulative_data(event) + + def generate_change_old(self, event): + self.result['change_old']["assigned_to"] = self.cumulative_data['assigned_to'] + + def generate_update_values_from_cumulative_data(self, event): + if self.cumulative_data['assigned_to_name'] is not None: + self.result['update_values']["users"][str(self.cumulative_data['assigned_to'])] = self.cumulative_data['assigned_to_name'] + + def generate_change_new(self, event, user): + if user is None: + self.result['change_new']["assigned_to"] = 0 + else: + self.result['change_new']["assigned_to"] = user.id + + def update_cumulative_data(self, event, user): + self.cumulative_data['assigned_to_github_id'] = event['assignee']['id'] + if user is None: + self.cumulative_data['assigned_to'] = 0 + self.cumulative_data['assigned_to_name'] = event['assignee']['login'] + else: + self.cumulative_data['assigned_to'] = user.id + self.cumulative_data['assigned_to_name'] = user.get_full_name() + + +class UnassignedEventHandler: + def __init__(self, result, cumulative_data, users_bindings): + self.result = result + self.cumulative_data = cumulative_data + self.users_bindings = users_bindings + + def handle(self, event): + if self.cumulative_data['assigned_to_github_id'] == event['assignee']['id']: + self.result['update_values']["users"] = {} + + self.generate_change_old(event) + self.generate_update_values_from_cumulative_data(event) + self.generate_change_new(event) + self.update_cumulative_data(event) + self.generate_update_values_from_cumulative_data(event) + + def generate_change_old(self, event): + self.result['change_old']["assigned_to"] = self.cumulative_data['assigned_to'] + + def generate_update_values_from_cumulative_data(self, event): + if self.cumulative_data['assigned_to_name'] is not None: + self.result['update_values']["users"][str(self.cumulative_data['assigned_to'])] = self.cumulative_data['assigned_to_name'] + + def generate_change_new(self, event): + self.result['change_new']["assigned_to"] = None + + def update_cumulative_data(self, event): + self.cumulative_data['assigned_to_github_id'] = None + self.cumulative_data['assigned_to'] = None + self.cumulative_data['assigned_to_name'] = None diff --git a/taiga/importers/github/tasks.py b/taiga/importers/github/tasks.py new file mode 100644 index 00000000..f1069e9e --- /dev/null +++ b/taiga/importers/github/tasks.py @@ -0,0 +1,56 @@ +# -*- coding: utf-8 -*- +# Copyright (C) 2014-2016 Andrey Antukh +# Copyright (C) 2014-2016 Jesús Espino +# Copyright (C) 2014-2016 David Barragán +# Copyright (C) 2014-2016 Alejandro Alonso +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import logging +import sys + +from django.utils.translation import ugettext as _ + +from taiga.base.mails import mail_builder +from taiga.celery import app +from taiga.users.models import User +from .importer import GithubImporter + +logger = logging.getLogger('taiga.importers.github') + + +@app.task(bind=True) +def import_project(self, user_id, token, project_id, options): + user = User.object.get(id=user_id) + importer = GithubImporter(user, token) + try: + project = importer.import_project(project_id, options) + except Exception as e: + # Error + ctx = { + "user": user, + "error_subject": _("Error importing github project"), + "error_message": _("Error importing github project"), + "project": project_id, + "exception": e + } + email = mail_builder.github_import_error(admin, ctx) + email.send() + logger.error('Error importing github project %s (by %s)', project_id, user, exc_info=sys.exc_info()) + else: + ctx = { + "project": project, + "user": user, + } + email = mail_builder.github_import_success(user, ctx) + email.send() diff --git a/taiga/importers/management/commands/import_from_github.py b/taiga/importers/management/commands/import_from_github.py new file mode 100644 index 00000000..9c93a527 --- /dev/null +++ b/taiga/importers/management/commands/import_from_github.py @@ -0,0 +1,100 @@ +# -*- coding: utf-8 -*- +# Copyright (C) 2014-2016 Andrey Antukh +# Copyright (C) 2014-2016 Jesús Espino +# Copyright (C) 2014-2016 David Barragán +# Copyright (C) 2014-2016 Alejandro Alonso +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +from django.core.management.base import BaseCommand +from django.conf import settings +from django.db.models import Q + +from taiga.importers.github.importer import GithubImporter +from taiga.users.models import User, AuthData +from taiga.projects.services import projects as service + +import unittest.mock +import timeit + + +class Command(BaseCommand): + def add_arguments(self, parser): + parser.add_argument('--token', dest="token", type=str, + help='Auth token') + parser.add_argument('--project-id', dest="project_id", type=str, + help='Project ID or full name (ex: taigaio/taiga-back)') + parser.add_argument('--template', dest='template', default="kanban", + help='template to use: scrum or kanban (default kanban)') + parser.add_argument('--type', dest='type', default="user_stories", + help='type of object to use: user_stories or issues (default user_stories)') + parser.add_argument('--ask-for-users', dest='ask_for_users', const=True, + action="store_const", default=False, + help='Import closed data') + parser.add_argument('--keep-external-reference', dest='keep_external_reference', const=True, + action="store_const", default=False, + help='Store external reference of imported data') + + def handle(self, *args, **options): + admin = User.objects.get(username="admin") + + if options.get('token', None): + token = options.get('token') + else: + url = GithubImporter.get_auth_url(settings.GITHUB_API_CLIENT_ID) + print("Go to here and come with your code (in the redirected url): {}".format(url)) + code = input("Code: ") + access_data = GithubImporter.get_access_token(settings.GITHUB_API_CLIENT_ID, settings.GITHUB_API_CLIENT_SECRET, code) + token = access_data + + importer = GithubImporter(admin, token) + + if options.get('project_id', None): + project_id = options.get('project_id') + else: + print("Select the project to import:") + for project in importer.list_projects(): + print("- {}: {}".format(project['id'], project['name'])) + project_id = input("Project id: ") + + users_bindings = {} + if options.get('ask_for_users', None): + print("Add the username or email for next github users:") + + for user in importer.list_users(project_id): + while True: + if user['detected_user'] is not None: + print("User automatically detected: {} as {}".format(user['full_name'], user['detected_user'])) + users_bindings[user['id']] = user['detected_user'] + break + + if not options.get('ask_for_users', False): + break + + username_or_email = input("{}: ".format(user['full_name'] or user['username'])) + if username_or_email == "": + break + try: + users_bindings[user['id']] = User.objects.get(Q(username=username_or_email) | Q(email=username_or_email)) + break + except User.DoesNotExist: + print("ERROR: Invalid username or email") + + options = { + "template": options.get('template'), + "type": options.get('type'), + "users_bindings": users_bindings, + "keep_external_reference": options.get('keep_external_reference') + } + + importer.import_project(project_id, options) diff --git a/taiga/routers.py b/taiga/routers.py index 1546ef8b..961b7d19 100644 --- a/taiga/routers.py +++ b/taiga/routers.py @@ -286,9 +286,11 @@ router.register(r"application-tokens", ApplicationToken, base_name="application- # Third party importers from taiga.importers.trello.api import TrelloImporterViewSet from taiga.importers.jira.api import JiraImporterViewSet +from taiga.importers.github.api import GithubImporterViewSet router.register(r"importers/trello", TrelloImporterViewSet, base_name="importers-trello") router.register(r"importers/jira", JiraImporterViewSet, base_name="importers-jira") +router.register(r"importers/github", GithubImporterViewSet, base_name="importers-github") # Stats diff --git a/tests/integration/test_importers_github_api.py b/tests/integration/test_importers_github_api.py new file mode 100644 index 00000000..bda1c928 --- /dev/null +++ b/tests/integration/test_importers_github_api.py @@ -0,0 +1,228 @@ +# -*- coding: utf-8 -*- +# Copyright (C) 2014-2016 Andrey Antukh +# Copyright (C) 2014-2016 Jesús Espino +# Copyright (C) 2014-2016 David Barragán +# Copyright (C) 2014-2016 Alejandro Alonso +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import pytest +import json + +from unittest import mock + +from django.core.urlresolvers import reverse + +from .. import factories as f +from taiga.importers import exceptions +from taiga.base.utils import json +from taiga.base import exceptions as exc + + +pytestmark = pytest.mark.django_db + + +def test_auth_url(client): + user = f.UserFactory.create() + client.login(user) + + url = reverse("importers-github-auth-url")+"?uri=http://localhost:9001/project/new?from=github" + + response = client.get(url, content_type="application/json") + + assert response.status_code == 200 + assert 'url' in response.data + assert response.data['url'] == "https://github.com/login/oauth/authorize?client_id=&scope=user,repo&redirect_uri=http://localhost:9001/project/new?from=github" + +def test_authorize(client, settings): + user = f.UserFactory.create() + client.login(user) + + authorize_url = reverse("importers-github-authorize") + + with mock.patch('taiga.importers.github.api.GithubImporter') as GithubImporterMock: + GithubImporterMock.get_access_token.return_value = "token" + response = client.post(authorize_url, content_type="application/json", data=json.dumps({"code": "code"})) + assert GithubImporterMock.get_access_token.calledWith(settings.GITHUB_API_CLIENT_ID, settings.GITHUB_API_CLIENT_SECRET, "code") + + assert response.status_code == 200 + assert 'token' in response.data + assert response.data['token'] == "token" + +def test_authorize_without_code(client): + user = f.UserFactory.create() + client.login(user) + + authorize_url = reverse("importers-github-authorize") + + response = client.post(authorize_url, content_type="application/json", data=json.dumps({})) + + assert response.status_code == 400 + assert 'token' not in response.data + assert '_error_message' in response.data + assert response.data['_error_message'] == "Code param needed" + + +def test_authorize_with_bad_verify(client, settings): + user = f.UserFactory.create() + client.login(user) + + authorize_url = reverse("importers-github-authorize") + + with mock.patch('taiga.importers.github.api.GithubImporter') as GithubImporterMock: + GithubImporterMock.get_access_token.side_effect = exceptions.InvalidAuthResult() + response = client.post(authorize_url, content_type="application/json", data=json.dumps({"code": "bad"})) + assert GithubImporterMock.get_access_token.calledWith(settings.GITHUB_API_CLIENT_ID, settings.GITHUB_API_CLIENT_SECRET, "bad") + + assert response.status_code == 400 + assert 'token' not in response.data + assert '_error_message' in response.data + assert response.data['_error_message'] == "Invalid auth data" + + +def test_import_github_list_users(client): + user = f.UserFactory.create() + client.login(user) + + url = reverse("importers-github-list-users") + + with mock.patch('taiga.importers.github.api.GithubImporter') as GithubImporterMock: + instance = mock.Mock() + instance.list_users.return_value = [ + {"id": 1, "username": "user1", "full_name": "user1", "detected_user": None}, + {"id": 2, "username": "user2", "full_name": "user2", "detected_user": None} + ] + GithubImporterMock.return_value = instance + response = client.post(url, content_type="application/json", data=json.dumps({"token": "token", "project": 1})) + + assert response.status_code == 200 + assert response.data[0]["id"] == 1 + assert response.data[1]["id"] == 2 + + +def test_import_github_list_users_without_project(client): + user = f.UserFactory.create() + client.login(user) + + url = reverse("importers-github-list-users") + + with mock.patch('taiga.importers.github.api.GithubImporter') as GithubImporterMock: + instance = mock.Mock() + instance.list_users.return_value = [ + {"id": 1, "username": "user1", "full_name": "user1", "detected_user": None}, + {"id": 2, "username": "user2", "full_name": "user2", "detected_user": None} + ] + GithubImporterMock.return_value = instance + response = client.post(url, content_type="application/json", data=json.dumps({"token": "token"})) + + assert response.status_code == 400 + + +def test_import_github_list_users_with_problem_on_request(client): + user = f.UserFactory.create() + client.login(user) + + url = reverse("importers-github-list-users") + + with mock.patch('taiga.importers.github.importer.GithubClient') as GithubClientMock: + instance = mock.Mock() + instance.get.side_effect = exc.WrongArguments("Invalid Request") + GithubClientMock.return_value = instance + response = client.post(url, content_type="application/json", data=json.dumps({"token": "token", "project": 1})) + + assert response.status_code == 400 + + +def test_import_github_list_projects(client): + user = f.UserFactory.create() + client.login(user) + + url = reverse("importers-github-list-projects") + + with mock.patch('taiga.importers.github.api.GithubImporter') as GithubImporterMock: + instance = mock.Mock() + instance.list_projects.return_value = ["project1", "project2"] + GithubImporterMock.return_value = instance + response = client.post(url, content_type="application/json", data=json.dumps({"token": "token"})) + + assert response.status_code == 200 + assert response.data[0] == "project1" + assert response.data[1] == "project2" + + +def test_import_github_list_projects_with_problem_on_request(client): + user = f.UserFactory.create() + client.login(user) + + url = reverse("importers-github-list-projects") + + with mock.patch('taiga.importers.github.importer.GithubClient') as GithubClientMock: + instance = mock.Mock() + instance.get.side_effect = exc.WrongArguments("Invalid Request") + GithubClientMock.return_value = instance + response = client.post(url, content_type="application/json", data=json.dumps({"token": "token"})) + + assert response.status_code == 400 + + +def test_import_github_project_without_project_id(client, settings): + settings.CELERY_ENABLED = True + + user = f.UserFactory.create() + client.login(user) + + url = reverse("importers-github-import-project") + + with mock.patch('taiga.importers.github.tasks.GithubImporter') as GithubImporterMock: + response = client.post(url, content_type="application/json", data=json.dumps({"token": "token"})) + + assert response.status_code == 400 + + +def test_import_github_project_with_celery_enabled(client, settings): + settings.CELERY_ENABLED = True + + user = f.UserFactory.create() + project = f.ProjectFactory.create(slug="async-imported-project") + client.login(user) + + url = reverse("importers-github-import-project") + + with mock.patch('taiga.importers.github.tasks.GithubImporter') as GithubImporterMock: + instance = mock.Mock() + instance.import_project.return_value = project + GithubImporterMock.return_value = instance + response = client.post(url, content_type="application/json", data=json.dumps({"token": "token", "project": 1})) + + assert response.status_code == 202 + assert "task_id" in response.data + + +def test_import_github_project_with_celery_disabled(client, settings): + settings.CELERY_ENABLED = False + + user = f.UserFactory.create() + project = f.ProjectFactory.create(slug="imported-project") + client.login(user) + + url = reverse("importers-github-import-project") + + with mock.patch('taiga.importers.github.api.GithubImporter') as GithubImporterMock: + instance = mock.Mock() + instance.import_project.return_value = project + GithubImporterMock.return_value = instance + response = client.post(url, content_type="application/json", data=json.dumps({"token": "token", "project": 1})) + + assert response.status_code == 200 + assert "slug" in response.data + assert response.data['slug'] == "imported-project"