Bitbucket webhooks for commits

requirements.txt

@@ -26,6 +26,7 @@ redis==2.10.3
 Unidecode==0.04.16
 raven==5.1.1
 bleach==1.4
+django-ipware==0.1.0
 
 # Comment it if you are using python >= 3.4
 enum34==1.0

settings/common.py

@@ -196,6 +196,7 @@ INSTALLED_APPS = [
     "taiga.feedback",
     "taiga.hooks.github",
     "taiga.hooks.gitlab",
+    "taiga.hooks.bitbucket",
 
     "rest_framework",
     "djmail",
@@ -355,8 +356,10 @@ CHANGE_NOTIFICATIONS_MIN_INTERVAL = 0 #seconds
 PROJECT_MODULES_CONFIGURATORS = {
     "github": "taiga.hooks.github.services.get_or_generate_config",
     "gitlab": "taiga.hooks.gitlab.services.get_or_generate_config",
+    "bitbucket": "taiga.hooks.bitbucket.services.get_or_generate_config",
 }
 
+BITBUCKET_VALID_ORIGIN_IPS = ["131.103.20.165", "131.103.20.166"]
 
 # NOTE: DON'T INSERT MORE SETTINGS AFTER THIS LINE
 TEST_RUNNER="django.test.runner.DiscoverRunner"

taiga/hooks/bitbucket/api.py

@@ -0,0 +1,96 @@
+# Copyright (C) 2014 Andrey Antukh <niwi@niwi.be>
+# Copyright (C) 2014 Jesús Espino <jespinog@gmail.com>
+# Copyright (C) 2014 David Barragán <bameda@dbarragan.com>
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from rest_framework.response import Response
+from django.utils.translation import ugettext_lazy as _
+from django.conf import settings
+
+from taiga.base.api.viewsets import GenericViewSet
+from taiga.base import exceptions as exc
+from taiga.base.utils import json
+from taiga.projects.models import Project
+from taiga.hooks.api import BaseWebhookApiViewSet
+
+from . import event_hooks
+from ..exceptions import ActionSyntaxException
+
+from urllib.parse import parse_qs
+from ipware.ip import get_real_ip
+
+class BitBucketViewSet(BaseWebhookApiViewSet):
+    event_hook_classes = {
+        "push": event_hooks.PushEventHook,
+    }
+
+    def create(self, request, *args, **kwargs):
+        project = self._get_project(request)
+        if not project:
+            raise exc.BadRequest(_("The project doesn't exist"))
+
+        if not self._validate_signature(project, request):
+            raise exc.BadRequest(_("Bad signature"))
+
+        event_name = self._get_event_name(request)
+
+        try:
+            body = parse_qs(request.body.decode("utf-8"), strict_parsing=True)
+            payload = body["payload"]
+        except (ValueError, KeyError):
+            raise exc.BadRequest(_("The payload is not a valid application/x-www-form-urlencoded"))
+
+        event_hook_class = self.event_hook_classes.get(event_name, None)
+        if event_hook_class is not None:
+            event_hook = event_hook_class(project, payload)
+            try:
+                event_hook.process_event()
+            except ActionSyntaxException as e:
+                raise exc.BadRequest(e)
+
+        return Response({})
+
+    def _validate_signature(self, project, request):
+        secret_key = request.GET.get("key", None)
+
+        if secret_key is None:
+            return False
+
+        if not hasattr(project, "modules_config"):
+            return False
+
+        if project.modules_config.config is None:
+            return False
+
+        project_secret = project.modules_config.config.get("bitbucket", {}).get("secret", "")
+        if not project_secret:
+            return False
+
+        valid_origin_ips = project.modules_config.config.get("bitbucket", {}).get("valid_origin_ips", settings.BITBUCKET_VALID_ORIGIN_IPS)
+        origin_ip = get_real_ip(request)
+        if not origin_ip or not origin_ip in valid_origin_ips:
+            return False
+
+        return project_secret == secret_key
+
+    def _get_project(self, request):
+        project_id = request.GET.get("project", None)
+        try:
+            project = Project.objects.get(id=project_id)
+            return project
+        except Project.DoesNotExist:
+            return None
+
+    def _get_event_name(self, request):
+        return "push"

taiga/hooks/bitbucket/event_hooks.py

@@ -0,0 +1,102 @@
+# Copyright (C) 2014 Andrey Antukh <niwi@niwi.be>
+# Copyright (C) 2014 Jesús Espino <jespinog@gmail.com>
+# Copyright (C) 2014 David Barragán <bameda@dbarragan.com>
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import re
+import os
+
+from django.utils.translation import ugettext_lazy as _
+
+from taiga.base import exceptions as exc
+from taiga.projects.models import Project, IssueStatus, TaskStatus, UserStoryStatus
+from taiga.projects.issues.models import Issue
+from taiga.projects.tasks.models import Task
+from taiga.projects.userstories.models import UserStory
+from taiga.projects.history.services import take_snapshot
+from taiga.projects.notifications.services import send_notifications
+from taiga.hooks.event_hooks import BaseEventHook
+from taiga.hooks.exceptions import ActionSyntaxException
+
+from .services import get_bitbucket_user
+
+import json
+
+class PushEventHook(BaseEventHook):
+    def process_event(self):
+        if self.payload is None:
+            return
+
+        # In bitbucket the payload is a list! :(
+        for payload_element_text in self.payload:
+            try:
+                payload_element = json.loads(payload_element_text)
+            except ValueError:
+                raise exc.BadRequest(_("The payload is not valid"))
+
+            commits = payload_element.get("commits", [])
+            for commit in commits:
+                message = commit.get("message", None)
+                self._process_message(message, None)
+
+    def _process_message(self, message, bitbucket_user):
+        """
+          The message we will be looking for seems like
+            TG-XX #yyyyyy
+          Where:
+            XX: is the ref for us, issue or task
+            yyyyyy: is the status slug we are setting
+        """
+        if message is None:
+            return
+
+        p = re.compile("tg-(\d+) +#([-\w]+)")
+        m = p.search(message.lower())
+        if m:
+            ref = m.group(1)
+            status_slug = m.group(2)
+            self._change_status(ref, status_slug, bitbucket_user)
+
+    def _change_status(self, ref, status_slug, bitbucket_user):
+        if Issue.objects.filter(project=self.project, ref=ref).exists():
+            modelClass = Issue
+            statusClass = IssueStatus
+        elif Task.objects.filter(project=self.project, ref=ref).exists():
+            modelClass = Task
+            statusClass = TaskStatus
+        elif UserStory.objects.filter(project=self.project, ref=ref).exists():
+            modelClass = UserStory
+            statusClass = UserStoryStatus
+        else:
+            raise ActionSyntaxException(_("The referenced element doesn't exist"))
+
+        element = modelClass.objects.get(project=self.project, ref=ref)
+
+        try:
+            status = statusClass.objects.get(project=self.project, slug=status_slug)
+        except statusClass.DoesNotExist:
+            raise ActionSyntaxException(_("The status doesn't exist"))
+
+        element.status = status
+        element.save()
+
+        snapshot = take_snapshot(element,
+                                 comment="Status changed from BitBucket commit",
+                                 user=get_bitbucket_user(bitbucket_user))
+        send_notifications(element, history=snapshot)
+
+
+def replace_bitbucket_references(project_url, wiki_text):
+    template = "\g<1>[BitBucket#\g<2>]({}/issues/\g<2>)\g<3>".format(project_url)
+    return re.sub(r"(\s|^)#(\d+)(\s|$)", template, wiki_text, 0, re.M)

taiga/hooks/bitbucket/migrations/0001_initial.py

@@ -0,0 +1,36 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import models, migrations
+from django.core.files import File
+
+import uuid
+
+def create_github_system_user(apps, schema_editor):
+    # We get the model from the versioned app registry;
+    # if we directly import it, it'll be the wrong version
+    User = apps.get_model("users", "User")
+    db_alias = schema_editor.connection.alias
+    random_hash = uuid.uuid4().hex
+    user = User.objects.using(db_alias).create(
+        username="bitbucket-{}".format(random_hash),
+        email="bitbucket-{}@taiga.io".format(random_hash),
+        full_name="BitBucket",
+        is_active=False,
+        is_system=True,
+        bio="",
+    )
+    f = open("taiga/hooks/bitbucket/migrations/logo.png", "rb")
+    user.photo.save("logo.png", File(f))
+    user.save()
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('users', '0006_auto_20141030_1132')
+    ]
+
+    operations = [
+        migrations.RunPython(create_github_system_user),
+    ]

taiga/hooks/bitbucket/models.py

@@ -0,0 +1 @@
+# This file is needed to load migrations

taiga/hooks/bitbucket/services.py

@@ -0,0 +1,55 @@
+# Copyright (C) 2014 Andrey Antukh <niwi@niwi.be>
+# Copyright (C) 2014 Jesús Espino <jespinog@gmail.com>
+# Copyright (C) 2014 David Barragán <bameda@dbarragan.com>
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import uuid
+
+from django.core.urlresolvers import reverse
+from django.conf import settings
+
+from taiga.users.models import User
+from taiga.base.utils.urls import get_absolute_url
+
+
+def get_or_generate_config(project):
+    config = project.modules_config.config
+    if config and "bitbucket" in config:
+        g_config = project.modules_config.config["bitbucket"]
+    else:
+        g_config = {
+            "secret": uuid.uuid4().hex,
+            "valid_origin_ips": settings.BITBUCKET_VALID_ORIGIN_IPS,
+        }
+
+    url = reverse("bitbucket-hook-list")
+    url = get_absolute_url(url)
+    url = "%s?project=%s&key=%s"%(url, project.id, g_config["secret"])
+    g_config["webhooks_url"] = url
+    return g_config
+
+
+def get_bitbucket_user(user_email):
+    user = None
+
+    if user_email:
+        try:
+            user = User.objects.get(email=user_email)
+        except User.DoesNotExist:
+            pass
+
+    if user is None:
+        user = User.objects.get(is_system=True, username__startswith="bitbucket")
+
+    return user

taiga/routers.py

@@ -135,8 +135,13 @@ router.register(r"notify-policies", NotifyPolicyViewSet, base_name="notification
 from taiga.hooks.github.api import GitHubViewSet
 router.register(r"github-hook", GitHubViewSet, base_name="github-hook")
 
+# Gitlab webhooks
 from taiga.hooks.gitlab.api import GitLabViewSet
 router.register(r"gitlab-hook", GitLabViewSet, base_name="gitlab-hook")
 
+# Bitbucket webhooks
+from taiga.hooks.bitbucket.api import BitBucketViewSet
+router.register(r"bitbucket-hook", BitBucketViewSet, base_name="bitbucket-hook")
+
 # feedback
 #   - see taiga.feedback.routers and taiga.feedback.apps

tests/integration/resources_permissions/test_users_resources.py

@@ -103,7 +103,7 @@ def test_user_list(client, data):
 
     response = client.get(url)
     users_data = json.loads(response.content.decode('utf-8'))
-    assert len(users_data) == 4
+    assert len(users_data) == 6
     assert response.status_code == 200
 
 

tests/integration/test_hooks_bitbucket.py

@@ -0,0 +1,233 @@
+import pytest
+import json
+import urllib
+
+from unittest import mock
+
+from django.core.urlresolvers import reverse
+from django.core import mail
+from django.conf import settings
+
+from taiga.hooks.bitbucket import event_hooks
+from taiga.hooks.bitbucket.api import BitBucketViewSet
+from taiga.hooks.exceptions import ActionSyntaxException
+from taiga.projects.issues.models import Issue
+from taiga.projects.tasks.models import Task
+from taiga.projects.userstories.models import UserStory
+from taiga.projects.models import Membership
+from taiga.projects.history.services import get_history_queryset_by_model_instance, take_snapshot
+from taiga.projects.notifications.choices import NotifyLevel
+from taiga.projects.notifications.models import NotifyPolicy
+from taiga.projects import services
+from .. import factories as f
+
+pytestmark = pytest.mark.django_db
+
+def test_bad_signature(client):
+    project=f.ProjectFactory()
+    f.ProjectModulesConfigFactory(project=project, config={
+        "bitbucket": {
+            "secret": "tpnIwJDz4e"
+        }
+    })
+
+    url = reverse("bitbucket-hook-list")
+    url = "{}?project={}&key={}".format(url, project.id, "badbadbad")
+    data = {}
+    response = client.post(url, urllib.parse.urlencode(data, True), content_type="application/x-www-form-urlencoded")
+    response_content = json.loads(response.content.decode("utf-8"))
+    assert response.status_code == 400
+    assert "Bad signature" in response_content["_error_message"]
+
+
+def test_ok_signature(client):
+    project=f.ProjectFactory()
+    f.ProjectModulesConfigFactory(project=project, config={
+        "bitbucket": {
+            "secret": "tpnIwJDz4e"
+        }
+    })
+
+    url = reverse("bitbucket-hook-list")
+    url = "{}?project={}&key={}".format(url, project.id, "tpnIwJDz4e")
+    data = {'payload': ['{"commits": []}']}
+    response = client.post(url,
+                           urllib.parse.urlencode(data, True),
+                           content_type="application/x-www-form-urlencoded",
+                           REMOTE_ADDR=settings.BITBUCKET_VALID_ORIGIN_IPS[0])
+    assert response.status_code == 200
+
+
+def test_push_event_detected(client):
+    project=f.ProjectFactory()
+    url = reverse("bitbucket-hook-list")
+    url = "%s?project=%s"%(url, project.id)
+    data = {'payload': ['{"commits": [{"message": "test message"}]}']}
+
+    BitBucketViewSet._validate_signature = mock.Mock(return_value=True)
+
+    with mock.patch.object(event_hooks.PushEventHook, "process_event") as process_event_mock:
+        response = client.post(url, urllib.parse.urlencode(data, True),
+            content_type="application/x-www-form-urlencoded")
+
+        assert process_event_mock.call_count == 1
+
+    assert response.status_code == 200
+
+
+def test_push_event_issue_processing(client):
+    creation_status = f.IssueStatusFactory()
+    role = f.RoleFactory(project=creation_status.project, permissions=["view_issues"])
+    membership = f.MembershipFactory(project=creation_status.project, role=role, user=creation_status.project.owner)
+    new_status = f.IssueStatusFactory(project=creation_status.project)
+    issue = f.IssueFactory.create(status=creation_status, project=creation_status.project, owner=creation_status.project.owner)
+    payload = [
+        '{"commits": [{"message": "test message   test   TG-%s    #%s   ok   bye!"}]}'%(issue.ref, new_status.slug)
+    ]
+    mail.outbox = []
+    ev_hook = event_hooks.PushEventHook(issue.project, payload)
+    ev_hook.process_event()
+    issue = Issue.objects.get(id=issue.id)
+    assert issue.status.id == new_status.id
+    assert len(mail.outbox) == 1
+
+
+def test_push_event_task_processing(client):
+    creation_status = f.TaskStatusFactory()
+    role = f.RoleFactory(project=creation_status.project, permissions=["view_tasks"])
+    membership = f.MembershipFactory(project=creation_status.project, role=role, user=creation_status.project.owner)
+    new_status = f.TaskStatusFactory(project=creation_status.project)
+    task = f.TaskFactory.create(status=creation_status, project=creation_status.project, owner=creation_status.project.owner)
+    payload = [
+        '{"commits": [{"message": "test message   test   TG-%s    #%s   ok   bye!"}]}'%(task.ref, new_status.slug)
+    ]
+    mail.outbox = []
+    ev_hook = event_hooks.PushEventHook(task.project, payload)
+    ev_hook.process_event()
+    task = Task.objects.get(id=task.id)
+    assert task.status.id == new_status.id
+    assert len(mail.outbox) == 1
+
+
+def test_push_event_user_story_processing(client):
+    creation_status = f.UserStoryStatusFactory()
+    role = f.RoleFactory(project=creation_status.project, permissions=["view_us"])
+    membership = f.MembershipFactory(project=creation_status.project, role=role, user=creation_status.project.owner)
+    new_status = f.UserStoryStatusFactory(project=creation_status.project)
+    user_story = f.UserStoryFactory.create(status=creation_status, project=creation_status.project, owner=creation_status.project.owner)
+    payload = [
+        '{"commits": [{"message": "test message   test   TG-%s    #%s   ok   bye!"}]}'%(user_story.ref, new_status.slug)
+    ]
+    mail.outbox = []
+    ev_hook = event_hooks.PushEventHook(user_story.project, payload)
+    ev_hook.process_event()
+    user_story = UserStory.objects.get(id=user_story.id)
+    assert user_story.status.id == new_status.id
+    assert len(mail.outbox) == 1
+
+
+def test_push_event_processing_case_insensitive(client):
+    creation_status = f.TaskStatusFactory()
+    role = f.RoleFactory(project=creation_status.project, permissions=["view_tasks"])
+    membership = f.MembershipFactory(project=creation_status.project, role=role, user=creation_status.project.owner)
+    new_status = f.TaskStatusFactory(project=creation_status.project)
+    task = f.TaskFactory.create(status=creation_status, project=creation_status.project, owner=creation_status.project.owner)
+    payload = [
+        '{"commits": [{"message": "test message   test   tg-%s    #%s   ok   bye!"}]}'%(task.ref, new_status.slug.upper())
+    ]
+    mail.outbox = []
+    ev_hook = event_hooks.PushEventHook(task.project, payload)
+    ev_hook.process_event()
+    task = Task.objects.get(id=task.id)
+    assert task.status.id == new_status.id
+    assert len(mail.outbox) == 1
+
+
+def test_push_event_task_bad_processing_non_existing_ref(client):
+    issue_status = f.IssueStatusFactory()
+    payload = [
+        '{"commits": [{"message": "test message   test   TG-6666666    #%s   ok   bye!"}]}'%(issue_status.slug)
+    ]
+    mail.outbox = []
+
+    ev_hook = event_hooks.PushEventHook(issue_status.project, payload)
+    with pytest.raises(ActionSyntaxException) as excinfo:
+        ev_hook.process_event()
+
+    assert str(excinfo.value) == "The referenced element doesn't exist"
+    assert len(mail.outbox) == 0
+
+
+def test_push_event_us_bad_processing_non_existing_status(client):
+    user_story = f.UserStoryFactory.create()
+    payload = [
+        '{"commits": [{"message": "test message   test   TG-%s    #non-existing-slug   ok   bye!"}]}'%(user_story.ref)
+    ]
+
+    mail.outbox = []
+
+    ev_hook = event_hooks.PushEventHook(user_story.project, payload)
+    with pytest.raises(ActionSyntaxException) as excinfo:
+        ev_hook.process_event()
+
+    assert str(excinfo.value) == "The status doesn't exist"
+    assert len(mail.outbox) == 0
+
+
+def test_push_event_bad_processing_non_existing_status(client):
+    issue = f.IssueFactory.create()
+    payload = [
+        '{"commits": [{"message": "test message   test   TG-%s    #non-existing-slug   ok   bye!"}]}'%(issue.ref)
+    ]
+    mail.outbox = []
+
+    ev_hook = event_hooks.PushEventHook(issue.project, payload)
+    with pytest.raises(ActionSyntaxException) as excinfo:
+        ev_hook.process_event()
+
+    assert str(excinfo.value) == "The status doesn't exist"
+    assert len(mail.outbox) == 0
+
+
+def test_api_get_project_modules(client):
+    project = f.create_project()
+    membership = f.MembershipFactory(project=project, user=project.owner, is_owner=True)
+
+    url = reverse("projects-modules", args=(project.id,))
+
+    client.login(project.owner)
+    response = client.get(url)
+    assert response.status_code == 200
+    content = json.loads(response.content.decode("utf-8"))
+    assert "bitbucket" in content
+    assert content["bitbucket"]["secret"] != ""
+    assert content["bitbucket"]["webhooks_url"] != ""
+
+
+def test_api_patch_project_modules(client):
+    project = f.create_project()
+    membership = f.MembershipFactory(project=project, user=project.owner, is_owner=True)
+
+    url = reverse("projects-modules", args=(project.id,))
+
+    client.login(project.owner)
+    data = {
+        "bitbucket": {
+            "secret": "test_secret",
+            "url": "test_url",
+        }
+    }
+    response = client.patch(url, json.dumps(data), content_type="application/json")
+    assert response.status_code == 204
+
+    config = services.get_modules_config(project).config
+    assert "bitbucket" in config
+    assert config["bitbucket"]["secret"] == "test_secret"
+    assert config["bitbucket"]["webhooks_url"] != "test_url"
+
+def test_replace_bitbucket_references():
+    assert event_hooks.replace_bitbucket_references("project-url", "#2") == "[BitBucket#2](project-url/issues/2)"
+    assert event_hooks.replace_bitbucket_references("project-url", "#2 ") == "[BitBucket#2](project-url/issues/2) "
+    assert event_hooks.replace_bitbucket_references("project-url", " #2 ") == " [BitBucket#2](project-url/issues/2) "
+    assert event_hooks.replace_bitbucket_references("project-url", " #2") == " [BitBucket#2](project-url/issues/2)"
+    assert event_hooks.replace_bitbucket_references("project-url", "#test") == "#test"