diff --git a/requirements.txt b/requirements.txt index 9769ddb3..b741b4f9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -26,6 +26,7 @@ redis==2.10.3 Unidecode==0.04.16 raven==5.1.1 bleach==1.4 +django-ipware==0.1.0 # Comment it if you are using python >= 3.4 enum34==1.0 diff --git a/settings/common.py b/settings/common.py index 0fcacf82..d57f14c3 100644 --- a/settings/common.py +++ b/settings/common.py @@ -196,6 +196,7 @@ INSTALLED_APPS = [ "taiga.feedback", "taiga.hooks.github", "taiga.hooks.gitlab", + "taiga.hooks.bitbucket", "rest_framework", "djmail", @@ -355,8 +356,10 @@ CHANGE_NOTIFICATIONS_MIN_INTERVAL = 0 #seconds PROJECT_MODULES_CONFIGURATORS = { "github": "taiga.hooks.github.services.get_or_generate_config", "gitlab": "taiga.hooks.gitlab.services.get_or_generate_config", + "bitbucket": "taiga.hooks.bitbucket.services.get_or_generate_config", } +BITBUCKET_VALID_ORIGIN_IPS = ["131.103.20.165", "131.103.20.166"] # NOTE: DON'T INSERT MORE SETTINGS AFTER THIS LINE TEST_RUNNER="django.test.runner.DiscoverRunner" diff --git a/taiga/hooks/bitbucket/__init__.py b/taiga/hooks/bitbucket/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/taiga/hooks/bitbucket/api.py b/taiga/hooks/bitbucket/api.py new file mode 100644 index 00000000..cd992036 --- /dev/null +++ b/taiga/hooks/bitbucket/api.py @@ -0,0 +1,96 @@ +# Copyright (C) 2014 Andrey Antukh +# Copyright (C) 2014 Jesús Espino +# Copyright (C) 2014 David Barragán +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +from rest_framework.response import Response +from django.utils.translation import ugettext_lazy as _ +from django.conf import settings + +from taiga.base.api.viewsets import GenericViewSet +from taiga.base import exceptions as exc +from taiga.base.utils import json +from taiga.projects.models import Project +from taiga.hooks.api import BaseWebhookApiViewSet + +from . import event_hooks +from ..exceptions import ActionSyntaxException + +from urllib.parse import parse_qs +from ipware.ip import get_real_ip + +class BitBucketViewSet(BaseWebhookApiViewSet): + event_hook_classes = { + "push": event_hooks.PushEventHook, + } + + def create(self, request, *args, **kwargs): + project = self._get_project(request) + if not project: + raise exc.BadRequest(_("The project doesn't exist")) + + if not self._validate_signature(project, request): + raise exc.BadRequest(_("Bad signature")) + + event_name = self._get_event_name(request) + + try: + body = parse_qs(request.body.decode("utf-8"), strict_parsing=True) + payload = body["payload"] + except (ValueError, KeyError): + raise exc.BadRequest(_("The payload is not a valid application/x-www-form-urlencoded")) + + event_hook_class = self.event_hook_classes.get(event_name, None) + if event_hook_class is not None: + event_hook = event_hook_class(project, payload) + try: + event_hook.process_event() + except ActionSyntaxException as e: + raise exc.BadRequest(e) + + return Response({}) + + def _validate_signature(self, project, request): + secret_key = request.GET.get("key", None) + + if secret_key is None: + return False + + if not hasattr(project, "modules_config"): + return False + + if project.modules_config.config is None: + return False + + project_secret = project.modules_config.config.get("bitbucket", {}).get("secret", "") + if not project_secret: + return False + + valid_origin_ips = project.modules_config.config.get("bitbucket", {}).get("valid_origin_ips", settings.BITBUCKET_VALID_ORIGIN_IPS) + origin_ip = get_real_ip(request) + if not origin_ip or not origin_ip in valid_origin_ips: + return False + + return project_secret == secret_key + + def _get_project(self, request): + project_id = request.GET.get("project", None) + try: + project = Project.objects.get(id=project_id) + return project + except Project.DoesNotExist: + return None + + def _get_event_name(self, request): + return "push" diff --git a/taiga/hooks/bitbucket/event_hooks.py b/taiga/hooks/bitbucket/event_hooks.py new file mode 100644 index 00000000..a149923d --- /dev/null +++ b/taiga/hooks/bitbucket/event_hooks.py @@ -0,0 +1,102 @@ +# Copyright (C) 2014 Andrey Antukh +# Copyright (C) 2014 Jesús Espino +# Copyright (C) 2014 David Barragán +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import re +import os + +from django.utils.translation import ugettext_lazy as _ + +from taiga.base import exceptions as exc +from taiga.projects.models import Project, IssueStatus, TaskStatus, UserStoryStatus +from taiga.projects.issues.models import Issue +from taiga.projects.tasks.models import Task +from taiga.projects.userstories.models import UserStory +from taiga.projects.history.services import take_snapshot +from taiga.projects.notifications.services import send_notifications +from taiga.hooks.event_hooks import BaseEventHook +from taiga.hooks.exceptions import ActionSyntaxException + +from .services import get_bitbucket_user + +import json + +class PushEventHook(BaseEventHook): + def process_event(self): + if self.payload is None: + return + + # In bitbucket the payload is a list! :( + for payload_element_text in self.payload: + try: + payload_element = json.loads(payload_element_text) + except ValueError: + raise exc.BadRequest(_("The payload is not valid")) + + commits = payload_element.get("commits", []) + for commit in commits: + message = commit.get("message", None) + self._process_message(message, None) + + def _process_message(self, message, bitbucket_user): + """ + The message we will be looking for seems like + TG-XX #yyyyyy + Where: + XX: is the ref for us, issue or task + yyyyyy: is the status slug we are setting + """ + if message is None: + return + + p = re.compile("tg-(\d+) +#([-\w]+)") + m = p.search(message.lower()) + if m: + ref = m.group(1) + status_slug = m.group(2) + self._change_status(ref, status_slug, bitbucket_user) + + def _change_status(self, ref, status_slug, bitbucket_user): + if Issue.objects.filter(project=self.project, ref=ref).exists(): + modelClass = Issue + statusClass = IssueStatus + elif Task.objects.filter(project=self.project, ref=ref).exists(): + modelClass = Task + statusClass = TaskStatus + elif UserStory.objects.filter(project=self.project, ref=ref).exists(): + modelClass = UserStory + statusClass = UserStoryStatus + else: + raise ActionSyntaxException(_("The referenced element doesn't exist")) + + element = modelClass.objects.get(project=self.project, ref=ref) + + try: + status = statusClass.objects.get(project=self.project, slug=status_slug) + except statusClass.DoesNotExist: + raise ActionSyntaxException(_("The status doesn't exist")) + + element.status = status + element.save() + + snapshot = take_snapshot(element, + comment="Status changed from BitBucket commit", + user=get_bitbucket_user(bitbucket_user)) + send_notifications(element, history=snapshot) + + +def replace_bitbucket_references(project_url, wiki_text): + template = "\g<1>[BitBucket#\g<2>]({}/issues/\g<2>)\g<3>".format(project_url) + return re.sub(r"(\s|^)#(\d+)(\s|$)", template, wiki_text, 0, re.M) diff --git a/taiga/hooks/bitbucket/migrations/0001_initial.py b/taiga/hooks/bitbucket/migrations/0001_initial.py new file mode 100644 index 00000000..372d93bb --- /dev/null +++ b/taiga/hooks/bitbucket/migrations/0001_initial.py @@ -0,0 +1,36 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import models, migrations +from django.core.files import File + +import uuid + +def create_github_system_user(apps, schema_editor): + # We get the model from the versioned app registry; + # if we directly import it, it'll be the wrong version + User = apps.get_model("users", "User") + db_alias = schema_editor.connection.alias + random_hash = uuid.uuid4().hex + user = User.objects.using(db_alias).create( + username="bitbucket-{}".format(random_hash), + email="bitbucket-{}@taiga.io".format(random_hash), + full_name="BitBucket", + is_active=False, + is_system=True, + bio="", + ) + f = open("taiga/hooks/bitbucket/migrations/logo.png", "rb") + user.photo.save("logo.png", File(f)) + user.save() + + +class Migration(migrations.Migration): + + dependencies = [ + ('users', '0006_auto_20141030_1132') + ] + + operations = [ + migrations.RunPython(create_github_system_user), + ] diff --git a/taiga/hooks/bitbucket/migrations/__init__.py b/taiga/hooks/bitbucket/migrations/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/taiga/hooks/bitbucket/migrations/logo.png b/taiga/hooks/bitbucket/migrations/logo.png new file mode 100644 index 00000000..fbc456a7 Binary files /dev/null and b/taiga/hooks/bitbucket/migrations/logo.png differ diff --git a/taiga/hooks/bitbucket/models.py b/taiga/hooks/bitbucket/models.py new file mode 100644 index 00000000..fca83d73 --- /dev/null +++ b/taiga/hooks/bitbucket/models.py @@ -0,0 +1 @@ +# This file is needed to load migrations diff --git a/taiga/hooks/bitbucket/services.py b/taiga/hooks/bitbucket/services.py new file mode 100644 index 00000000..bcb74f56 --- /dev/null +++ b/taiga/hooks/bitbucket/services.py @@ -0,0 +1,55 @@ +# Copyright (C) 2014 Andrey Antukh +# Copyright (C) 2014 Jesús Espino +# Copyright (C) 2014 David Barragán +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import uuid + +from django.core.urlresolvers import reverse +from django.conf import settings + +from taiga.users.models import User +from taiga.base.utils.urls import get_absolute_url + + +def get_or_generate_config(project): + config = project.modules_config.config + if config and "bitbucket" in config: + g_config = project.modules_config.config["bitbucket"] + else: + g_config = { + "secret": uuid.uuid4().hex, + "valid_origin_ips": settings.BITBUCKET_VALID_ORIGIN_IPS, + } + + url = reverse("bitbucket-hook-list") + url = get_absolute_url(url) + url = "%s?project=%s&key=%s"%(url, project.id, g_config["secret"]) + g_config["webhooks_url"] = url + return g_config + + +def get_bitbucket_user(user_email): + user = None + + if user_email: + try: + user = User.objects.get(email=user_email) + except User.DoesNotExist: + pass + + if user is None: + user = User.objects.get(is_system=True, username__startswith="bitbucket") + + return user diff --git a/taiga/routers.py b/taiga/routers.py index f5959d14..0b6ffba2 100644 --- a/taiga/routers.py +++ b/taiga/routers.py @@ -135,8 +135,13 @@ router.register(r"notify-policies", NotifyPolicyViewSet, base_name="notification from taiga.hooks.github.api import GitHubViewSet router.register(r"github-hook", GitHubViewSet, base_name="github-hook") +# Gitlab webhooks from taiga.hooks.gitlab.api import GitLabViewSet router.register(r"gitlab-hook", GitLabViewSet, base_name="gitlab-hook") +# Bitbucket webhooks +from taiga.hooks.bitbucket.api import BitBucketViewSet +router.register(r"bitbucket-hook", BitBucketViewSet, base_name="bitbucket-hook") + # feedback # - see taiga.feedback.routers and taiga.feedback.apps diff --git a/tests/integration/resources_permissions/test_users_resources.py b/tests/integration/resources_permissions/test_users_resources.py index 7e6db573..df172a1a 100644 --- a/tests/integration/resources_permissions/test_users_resources.py +++ b/tests/integration/resources_permissions/test_users_resources.py @@ -103,7 +103,7 @@ def test_user_list(client, data): response = client.get(url) users_data = json.loads(response.content.decode('utf-8')) - assert len(users_data) == 4 + assert len(users_data) == 6 assert response.status_code == 200 diff --git a/tests/integration/test_hooks_bitbucket.py b/tests/integration/test_hooks_bitbucket.py new file mode 100644 index 00000000..5ac6c88b --- /dev/null +++ b/tests/integration/test_hooks_bitbucket.py @@ -0,0 +1,233 @@ +import pytest +import json +import urllib + +from unittest import mock + +from django.core.urlresolvers import reverse +from django.core import mail +from django.conf import settings + +from taiga.hooks.bitbucket import event_hooks +from taiga.hooks.bitbucket.api import BitBucketViewSet +from taiga.hooks.exceptions import ActionSyntaxException +from taiga.projects.issues.models import Issue +from taiga.projects.tasks.models import Task +from taiga.projects.userstories.models import UserStory +from taiga.projects.models import Membership +from taiga.projects.history.services import get_history_queryset_by_model_instance, take_snapshot +from taiga.projects.notifications.choices import NotifyLevel +from taiga.projects.notifications.models import NotifyPolicy +from taiga.projects import services +from .. import factories as f + +pytestmark = pytest.mark.django_db + +def test_bad_signature(client): + project=f.ProjectFactory() + f.ProjectModulesConfigFactory(project=project, config={ + "bitbucket": { + "secret": "tpnIwJDz4e" + } + }) + + url = reverse("bitbucket-hook-list") + url = "{}?project={}&key={}".format(url, project.id, "badbadbad") + data = {} + response = client.post(url, urllib.parse.urlencode(data, True), content_type="application/x-www-form-urlencoded") + response_content = json.loads(response.content.decode("utf-8")) + assert response.status_code == 400 + assert "Bad signature" in response_content["_error_message"] + + +def test_ok_signature(client): + project=f.ProjectFactory() + f.ProjectModulesConfigFactory(project=project, config={ + "bitbucket": { + "secret": "tpnIwJDz4e" + } + }) + + url = reverse("bitbucket-hook-list") + url = "{}?project={}&key={}".format(url, project.id, "tpnIwJDz4e") + data = {'payload': ['{"commits": []}']} + response = client.post(url, + urllib.parse.urlencode(data, True), + content_type="application/x-www-form-urlencoded", + REMOTE_ADDR=settings.BITBUCKET_VALID_ORIGIN_IPS[0]) + assert response.status_code == 200 + + +def test_push_event_detected(client): + project=f.ProjectFactory() + url = reverse("bitbucket-hook-list") + url = "%s?project=%s"%(url, project.id) + data = {'payload': ['{"commits": [{"message": "test message"}]}']} + + BitBucketViewSet._validate_signature = mock.Mock(return_value=True) + + with mock.patch.object(event_hooks.PushEventHook, "process_event") as process_event_mock: + response = client.post(url, urllib.parse.urlencode(data, True), + content_type="application/x-www-form-urlencoded") + + assert process_event_mock.call_count == 1 + + assert response.status_code == 200 + + +def test_push_event_issue_processing(client): + creation_status = f.IssueStatusFactory() + role = f.RoleFactory(project=creation_status.project, permissions=["view_issues"]) + membership = f.MembershipFactory(project=creation_status.project, role=role, user=creation_status.project.owner) + new_status = f.IssueStatusFactory(project=creation_status.project) + issue = f.IssueFactory.create(status=creation_status, project=creation_status.project, owner=creation_status.project.owner) + payload = [ + '{"commits": [{"message": "test message test TG-%s #%s ok bye!"}]}'%(issue.ref, new_status.slug) + ] + mail.outbox = [] + ev_hook = event_hooks.PushEventHook(issue.project, payload) + ev_hook.process_event() + issue = Issue.objects.get(id=issue.id) + assert issue.status.id == new_status.id + assert len(mail.outbox) == 1 + + +def test_push_event_task_processing(client): + creation_status = f.TaskStatusFactory() + role = f.RoleFactory(project=creation_status.project, permissions=["view_tasks"]) + membership = f.MembershipFactory(project=creation_status.project, role=role, user=creation_status.project.owner) + new_status = f.TaskStatusFactory(project=creation_status.project) + task = f.TaskFactory.create(status=creation_status, project=creation_status.project, owner=creation_status.project.owner) + payload = [ + '{"commits": [{"message": "test message test TG-%s #%s ok bye!"}]}'%(task.ref, new_status.slug) + ] + mail.outbox = [] + ev_hook = event_hooks.PushEventHook(task.project, payload) + ev_hook.process_event() + task = Task.objects.get(id=task.id) + assert task.status.id == new_status.id + assert len(mail.outbox) == 1 + + +def test_push_event_user_story_processing(client): + creation_status = f.UserStoryStatusFactory() + role = f.RoleFactory(project=creation_status.project, permissions=["view_us"]) + membership = f.MembershipFactory(project=creation_status.project, role=role, user=creation_status.project.owner) + new_status = f.UserStoryStatusFactory(project=creation_status.project) + user_story = f.UserStoryFactory.create(status=creation_status, project=creation_status.project, owner=creation_status.project.owner) + payload = [ + '{"commits": [{"message": "test message test TG-%s #%s ok bye!"}]}'%(user_story.ref, new_status.slug) + ] + mail.outbox = [] + ev_hook = event_hooks.PushEventHook(user_story.project, payload) + ev_hook.process_event() + user_story = UserStory.objects.get(id=user_story.id) + assert user_story.status.id == new_status.id + assert len(mail.outbox) == 1 + + +def test_push_event_processing_case_insensitive(client): + creation_status = f.TaskStatusFactory() + role = f.RoleFactory(project=creation_status.project, permissions=["view_tasks"]) + membership = f.MembershipFactory(project=creation_status.project, role=role, user=creation_status.project.owner) + new_status = f.TaskStatusFactory(project=creation_status.project) + task = f.TaskFactory.create(status=creation_status, project=creation_status.project, owner=creation_status.project.owner) + payload = [ + '{"commits": [{"message": "test message test tg-%s #%s ok bye!"}]}'%(task.ref, new_status.slug.upper()) + ] + mail.outbox = [] + ev_hook = event_hooks.PushEventHook(task.project, payload) + ev_hook.process_event() + task = Task.objects.get(id=task.id) + assert task.status.id == new_status.id + assert len(mail.outbox) == 1 + + +def test_push_event_task_bad_processing_non_existing_ref(client): + issue_status = f.IssueStatusFactory() + payload = [ + '{"commits": [{"message": "test message test TG-6666666 #%s ok bye!"}]}'%(issue_status.slug) + ] + mail.outbox = [] + + ev_hook = event_hooks.PushEventHook(issue_status.project, payload) + with pytest.raises(ActionSyntaxException) as excinfo: + ev_hook.process_event() + + assert str(excinfo.value) == "The referenced element doesn't exist" + assert len(mail.outbox) == 0 + + +def test_push_event_us_bad_processing_non_existing_status(client): + user_story = f.UserStoryFactory.create() + payload = [ + '{"commits": [{"message": "test message test TG-%s #non-existing-slug ok bye!"}]}'%(user_story.ref) + ] + + mail.outbox = [] + + ev_hook = event_hooks.PushEventHook(user_story.project, payload) + with pytest.raises(ActionSyntaxException) as excinfo: + ev_hook.process_event() + + assert str(excinfo.value) == "The status doesn't exist" + assert len(mail.outbox) == 0 + + +def test_push_event_bad_processing_non_existing_status(client): + issue = f.IssueFactory.create() + payload = [ + '{"commits": [{"message": "test message test TG-%s #non-existing-slug ok bye!"}]}'%(issue.ref) + ] + mail.outbox = [] + + ev_hook = event_hooks.PushEventHook(issue.project, payload) + with pytest.raises(ActionSyntaxException) as excinfo: + ev_hook.process_event() + + assert str(excinfo.value) == "The status doesn't exist" + assert len(mail.outbox) == 0 + + +def test_api_get_project_modules(client): + project = f.create_project() + membership = f.MembershipFactory(project=project, user=project.owner, is_owner=True) + + url = reverse("projects-modules", args=(project.id,)) + + client.login(project.owner) + response = client.get(url) + assert response.status_code == 200 + content = json.loads(response.content.decode("utf-8")) + assert "bitbucket" in content + assert content["bitbucket"]["secret"] != "" + assert content["bitbucket"]["webhooks_url"] != "" + + +def test_api_patch_project_modules(client): + project = f.create_project() + membership = f.MembershipFactory(project=project, user=project.owner, is_owner=True) + + url = reverse("projects-modules", args=(project.id,)) + + client.login(project.owner) + data = { + "bitbucket": { + "secret": "test_secret", + "url": "test_url", + } + } + response = client.patch(url, json.dumps(data), content_type="application/json") + assert response.status_code == 204 + + config = services.get_modules_config(project).config + assert "bitbucket" in config + assert config["bitbucket"]["secret"] == "test_secret" + assert config["bitbucket"]["webhooks_url"] != "test_url" + +def test_replace_bitbucket_references(): + assert event_hooks.replace_bitbucket_references("project-url", "#2") == "[BitBucket#2](project-url/issues/2)" + assert event_hooks.replace_bitbucket_references("project-url", "#2 ") == "[BitBucket#2](project-url/issues/2) " + assert event_hooks.replace_bitbucket_references("project-url", " #2 ") == " [BitBucket#2](project-url/issues/2) " + assert event_hooks.replace_bitbucket_references("project-url", " #2") == " [BitBucket#2](project-url/issues/2)" + assert event_hooks.replace_bitbucket_references("project-url", "#test") == "#test"