Added common throttle for all the API

settings/common.py

@@ -433,12 +433,13 @@ REST_FRAMEWORK = {
         "taiga.external_apps.auth_backends.Token",
     ),
     "DEFAULT_THROTTLE_CLASSES": (
-        "taiga.base.throttling.AnonRateThrottle",
+        "taiga.base.throttling.CommonThrottle",
-        "taiga.base.throttling.UserRateThrottle"
     ),
     "DEFAULT_THROTTLE_RATES": {
-        "anon": None,
+        "anon-write": None,
-        "user": None,
+        "user-write": None,
+        "anon-read": None,
+        "user-read": None,
         "import-mode": None,
         "import-dump-mode": "1/minute",
         "create-memberships": None,
@@ -446,6 +447,7 @@ REST_FRAMEWORK = {
         "register-success": None,
         "user-detail": None,
     },
+    "DEFAULT_THROTTLE_WHITELIST": [],
     "FILTER_BACKEND": "taiga.base.filters.FilterBackend",
     "EXCEPTION_HANDLER": "taiga.base.exceptions.exception_handler",
     "PAGINATE_BY": 30,

settings/testing.py

@@ -29,8 +29,10 @@ INSTALLED_APPS = INSTALLED_APPS + [
 ]
 
 REST_FRAMEWORK["DEFAULT_THROTTLE_RATES"] = {
-    "anon": None,
+    "anon-write": None,
-    "user": None,
+    "anon-read": None,
+    "user-write": None,
+    "user-read": None,
     "import-mode": None,
     "import-dump-mode": None,
     "create-memberships": None,

taiga/base/api/settings.py

@@ -106,6 +106,7 @@ DEFAULTS = {
         "user": None,
         "anon": None,
     },
+    "DEFAULT_THROTTLE_WHITELIST": [],
 
     # Pagination
     "PAGINATE_BY": None,

taiga/base/throttling.py

@@ -17,6 +17,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 from taiga.base.api import throttling
+from django.conf import settings
 
 
 class GlobalThrottlingMixin:
@@ -57,5 +58,135 @@ class UserRateThrottle(throttling.UserRateThrottle):
     scope = "user"
 
 
+class CommonThrottle(throttling.SimpleRateThrottle):
+    cache_format = "throtte_%(scope)s_%(rate)s_%(ident)s"
+
+    def __init__(self):
+        pass
+
+    def has_to_finalize(self, request, response, view):
+        return False
+
+    def allow_request(self, request, view):
+        scope = self.get_scope(request)
+        ident = self.get_ident(request)
+        rates = self.get_rates(scope)
+        if ident in settings.REST_FRAMEWORK['DEFAULT_THROTTLE_WHITELIST']:
+            return True
+
+        if rates is None or rates == []:
+            return True
+
+        now = self.timer()
+
+        waits = []
+        history_writes = []
+
+        for rate in rates:
+            rate_name = rate[0]
+            rate_num_requests = rate[1]
+            rate_duration = rate[2]
+
+            key = self.get_cache_key(ident, scope, rate_name)
+            history = self.cache.get(key, [])
+
+            while history and history[-1] <= now - rate_duration:
+                history.pop()
+
+            if len(history) >= rate_num_requests:
+                waits.append(self.wait_time(history, rate, now))
+
+            history_writes.append({
+                "key": key,
+                "history": history,
+                "rate_duration": rate_duration,
+            })
+
+        if waits:
+            self._wait = max(waits)
+            return False
+
+        for history_write in history_writes:
+            history_write['history'].insert(0, now)
+            self.cache.set(
+                history_write['key'],
+                history_write['history'],
+                history_write['rate_duration']
+            )
+        return True
+
+    def get_rates(self, scope):
+        try:
+            rates = self.THROTTLE_RATES[scope]
+        except KeyError:
+            msg = "No default throttle rate set for \"%s\" scope" % scope
+            raise ImproperlyConfigured(msg)
+
+        if rates is None:
+            return []
+        elif isinstance(rates, str):
+            return [self.parse_rate(rates)]
+        elif isinstance(rates, list):
+            return list(map(self.parse_rate, rates))
+        else:
+            msg = "No valid throttle rate set for \"%s\" scope" % scope
+            raise ImproperlyConfigured(msg)
+
+    def parse_rate(self, rate):
+        """
+        Given the request rate string, return a two tuple of:
+        <allowed number of requests>, <period of time in seconds>
+        """
+        if rate is None:
+            return None
+        num, period = rate.split("/")
+        num_requests = int(num)
+        duration = {"s": 1, "m": 60, "h": 3600, "d": 86400}[period[0]]
+        return (rate, num_requests, duration)
+
+    def get_scope(self, request):
+        if request.user.is_authenticated():
+            if request.method in ["POST", "PUT", "PATCH", "DELETE"]:
+                scope = "user-write"
+            else:
+                scope = "user-read"
+        else:
+            if request.method in ["POST", "PUT", "PATCH", "DELETE"]:
+                scope = "anon-write"
+            else:
+                scope = "anon-read"
+        return scope
+
+    def get_ident(self, request):
+        if request.user.is_authenticated():
+            ident = request.user.id
+        else:
+            ident = request.META.get("HTTP_X_FORWARDED_FOR")
+            if ident is None:
+                ident = request.META.get("REMOTE_ADDR")
+        return ident
+
+    def get_cache_key(self, ident, scope, rate):
+        return self.cache_format % { "scope": scope, "ident": ident, "rate": rate }
+
+    def wait_time(self, history, rate, now):
+        rate_num_requests = rate[1]
+        rate_duration = rate[2]
+
+        if history:
+            remaining_duration = rate_duration - (now - history[-1])
+        else:
+            remaining_duration = rate_duration
+
+        available_requests = rate_num_requests - len(history) + 1
+        if available_requests <= 0:
+            return remaining_duration
+
+        return remaining_duration / float(available_requests)
+
+    def wait(self):
+        return self._wait
+
+
 class SimpleRateThrottle(throttling.SimpleRateThrottle):
     pass

tests/integration/test_throwttling.py

@@ -29,20 +29,15 @@ from .. import factories as f
 pytestmark = pytest.mark.django_db
 
 
-anon_rate_path = "taiga.base.throttling.AnonRateThrottle.get_rate"
-user_rate_path = "taiga.base.throttling.UserRateThrottle.get_rate"
 import_rate_path = "taiga.export_import.throttling.ImportModeRateThrottle.get_rate"
 
 
 def test_anonimous_throttling_policy(client, settings):
     f.create_project()
     url = reverse("projects-list")
+    settings.REST_FRAMEWORK['DEFAULT_THROTTLE_RATES']['anon-read'] = "2/min"
 
-    with mock.patch(anon_rate_path) as anon_rate, \
+    with mock.patch(import_rate_path) as import_rate:
-            mock.patch(user_rate_path) as user_rate, \
-            mock.patch(import_rate_path) as import_rate:
-        anon_rate.return_value = "2/day"
-        user_rate.return_value = "4/day"
         import_rate.return_value = "7/day"
 
         cache.clear()
@@ -53,19 +48,19 @@ def test_anonimous_throttling_policy(client, settings):
         response = client.json.get(url)
         assert response.status_code == 429
 
+    settings.REST_FRAMEWORK['DEFAULT_THROTTLE_RATES']['anon-read'] = None
+    cache.clear()
+
 
 def test_user_throttling_policy(client, settings):
     project = f.create_project()
     f.MembershipFactory.create(project=project, user=project.owner, is_admin=True)
     url = reverse("projects-detail", kwargs={"pk": project.pk})
+    settings.REST_FRAMEWORK['DEFAULT_THROTTLE_RATES']['user-read'] = "4/min"
 
     client.login(project.owner)
 
-    with mock.patch(anon_rate_path) as anon_rate, \
+    with mock.patch(import_rate_path) as import_rate:
-            mock.patch(user_rate_path) as user_rate, \
-            mock.patch(import_rate_path) as import_rate:
-        anon_rate.return_value = "2/day"
-        user_rate.return_value = "4/day"
         import_rate.return_value = "7/day"
 
         cache.clear()
@@ -81,6 +76,8 @@ def test_user_throttling_policy(client, settings):
         assert response.status_code == 429
 
     client.logout()
+    settings.REST_FRAMEWORK['DEFAULT_THROTTLE_RATES']['user-read'] = None
+    cache.clear()
 
 
 def test_import_mode_throttling_policy(client, settings):
@@ -95,14 +92,12 @@ def test_import_mode_throttling_policy(client, settings):
     data = {
         "subject": "Test"
     }
+    settings.REST_FRAMEWORK['DEFAULT_THROTTLE_RATES']['anon-read'] = "2/min"
+    settings.REST_FRAMEWORK['DEFAULT_THROTTLE_RATES']['user-read'] = "4/min"
 
     client.login(project.owner)
 
-    with mock.patch(anon_rate_path) as anon_rate, \
+    with mock.patch(import_rate_path) as import_rate:
-            mock.patch(user_rate_path) as user_rate, \
-            mock.patch(import_rate_path) as import_rate:
-        anon_rate.return_value = "2/day"
-        user_rate.return_value = "4/day"
         import_rate.return_value = "7/day"
 
         cache.clear()
@@ -124,3 +119,7 @@ def test_import_mode_throttling_policy(client, settings):
         assert response.status_code == 429
 
     client.logout()
+
+    settings.REST_FRAMEWORK['DEFAULT_THROTTLE_RATES']['anon-read'] = None
+    settings.REST_FRAMEWORK['DEFAULT_THROTTLE_RATES']['user-read'] = None
+    cache.clear()

tests/unit/test_common_throttle.py

@@ -0,0 +1,265 @@
+# -*- coding: utf-8 -*-
+# Copyright (C) 2014-2017 Andrey Antukh <niwi@niwi.nz>
+# Copyright (C) 2014-2017 Jesús Espino <jespinog@gmail.com>
+# Copyright (C) 2014-2017 David Barragán <bameda@dbarragan.com>
+# Copyright (C) 2014-2017 Alejandro Alonso <alejandro.alonso@kaleidos.net>
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from django.test import RequestFactory
+from django.core.cache import cache
+from django.contrib.auth.models import AnonymousUser
+
+from taiga.base.throttling import CommonThrottle
+from taiga.users.models import User
+
+
+def test_user_no_write_throttling(settings, rf):
+    settings.REST_FRAMEWORK['DEFAULT_THROTTLE_RATES']['user-write'] = None
+    request = rf.post("/test")
+    request.user = User(id=1)
+    throttling = CommonThrottle()
+    for x in range(100):
+        assert throttling.allow_request(request, None)
+    cache.clear()
+    settings.REST_FRAMEWORK['DEFAULT_THROTTLE_RATES']['user-write'] = None
+
+def test_user_simple_write_throttling(settings, rf):
+    settings.REST_FRAMEWORK['DEFAULT_THROTTLE_RATES']['user-write'] = "1/min"
+    request = rf.post("/test")
+    request.user = User(id=1)
+    throttling = CommonThrottle()
+    assert throttling.allow_request(request, None)
+    assert throttling.allow_request(request, None) is False
+    for x in range(100):
+        assert throttling.allow_request(request, None) is False
+    cache.clear()
+    settings.REST_FRAMEWORK['DEFAULT_THROTTLE_RATES']['user-write'] = None
+
+def test_user_multi_write_first_small_throttling(settings, rf):
+    settings.REST_FRAMEWORK['DEFAULT_THROTTLE_RATES']['user-write'] = ["1/min", "10/min"]
+    request = rf.post("/test")
+    request.user = User(id=1)
+    throttling = CommonThrottle()
+    assert throttling.allow_request(request, None)
+    assert throttling.allow_request(request, None) is False
+    for x in range(100):
+        assert throttling.allow_request(request, None) is False
+    cache.clear()
+    settings.REST_FRAMEWORK['DEFAULT_THROTTLE_RATES']['user-write'] = None
+
+def test_user_multi_write_first_big_throttling(settings, rf):
+    settings.REST_FRAMEWORK['DEFAULT_THROTTLE_RATES']['user-write'] = ["10/min", "1/min"]
+    request = rf.post("/test")
+    request.user = User(id=1)
+    throttling = CommonThrottle()
+    assert throttling.allow_request(request, None)
+    assert throttling.allow_request(request, None) is False
+    for x in range(100):
+        assert throttling.allow_request(request, None) is False
+    cache.clear()
+    settings.REST_FRAMEWORK['DEFAULT_THROTTLE_RATES']['user-write'] = None
+
+def test_user_no_read_throttling(settings, rf):
+    settings.REST_FRAMEWORK['DEFAULT_THROTTLE_RATES']['user-read'] = None
+    request = rf.get("/test")
+    request.user = User(id=1)
+    throttling = CommonThrottle()
+    for x in range(100):
+        assert throttling.allow_request(request, None)
+    cache.clear()
+    settings.REST_FRAMEWORK['DEFAULT_THROTTLE_RATES']['user-read'] = None
+
+def test_user_simple_read_throttling(settings, rf):
+    settings.REST_FRAMEWORK['DEFAULT_THROTTLE_RATES']['user-read'] = "1/min"
+    request = rf.get("/test")
+    request.user = User(id=1)
+    throttling = CommonThrottle()
+    assert throttling.allow_request(request, None)
+    assert throttling.allow_request(request, None) is False
+    for x in range(100):
+        assert throttling.allow_request(request, None) is False
+    cache.clear()
+    settings.REST_FRAMEWORK['DEFAULT_THROTTLE_RATES']['user-read'] = None
+
+def test_user_multi_read_first_small_throttling(settings, rf):
+    settings.REST_FRAMEWORK['DEFAULT_THROTTLE_RATES']['user-read'] = ["1/min", "10/min"]
+    request = rf.get("/test")
+    request.user = User(id=1)
+    throttling = CommonThrottle()
+    assert throttling.allow_request(request, None)
+    assert throttling.allow_request(request, None) is False
+    for x in range(100):
+        assert throttling.allow_request(request, None) is False
+    cache.clear()
+    settings.REST_FRAMEWORK['DEFAULT_THROTTLE_RATES']['user-read'] = None
+
+def test_user_multi_read_first_big_throttling(settings, rf):
+    settings.REST_FRAMEWORK['DEFAULT_THROTTLE_RATES']['user-read'] = ["10/min", "1/min"]
+    request = rf.get("/test")
+    request.user = User(id=1)
+    throttling = CommonThrottle()
+    assert throttling.allow_request(request, None)
+    assert throttling.allow_request(request, None) is False
+    for x in range(100):
+        assert throttling.allow_request(request, None) is False
+    cache.clear()
+    settings.REST_FRAMEWORK['DEFAULT_THROTTLE_RATES']['user-read'] = None
+
+def test_whitelisted_user_throttling(settings, rf):
+    settings.REST_FRAMEWORK['DEFAULT_THROTTLE_RATES']['user-read'] = "1/min"
+    settings.REST_FRAMEWORK['DEFAULT_THROTTLE_WHITELIST'] = [1]
+    request = rf.get("/test")
+    request.user = User(id=1)
+    throttling = CommonThrottle()
+    assert throttling.allow_request(request, None)
+    assert throttling.allow_request(request, None)
+    cache.clear()
+    settings.REST_FRAMEWORK['DEFAULT_THROTTLE_RATES']['user-read'] = None
+    settings.REST_FRAMEWORK['DEFAULT_THROTTLE_WHITELIST'] = []
+
+def test_not_whitelisted_user_throttling(settings, rf):
+    settings.REST_FRAMEWORK['DEFAULT_THROTTLE_RATES']['user-read'] = "1/min"
+    settings.REST_FRAMEWORK['DEFAULT_THROTTLE_WHITELIST'] = [1]
+    request = rf.get("/test")
+    request.user = User(id=2)
+    throttling = CommonThrottle()
+    assert throttling.allow_request(request, None)
+    assert throttling.allow_request(request, None) is False
+    for x in range(100):
+        assert throttling.allow_request(request, None) is False
+    cache.clear()
+    settings.REST_FRAMEWORK['DEFAULT_THROTTLE_RATES']['user-read'] = None
+    settings.REST_FRAMEWORK['DEFAULT_THROTTLE_WHITELIST'] = []
+
+def test_anon_no_write_throttling(settings, rf):
+    settings.REST_FRAMEWORK['DEFAULT_THROTTLE_RATES']['anon-write'] = None
+    request = rf.post("/test")
+    request.user = AnonymousUser()
+    throttling = CommonThrottle()
+    for x in range(100):
+        assert throttling.allow_request(request, None)
+    cache.clear()
+    settings.REST_FRAMEWORK['DEFAULT_THROTTLE_RATES']['anon-write'] = None
+
+def test_anon_simple_write_throttling(settings, rf):
+    settings.REST_FRAMEWORK['DEFAULT_THROTTLE_RATES']['anon-write'] = "1/min"
+    request = rf.post("/test")
+    request.user = AnonymousUser()
+    throttling = CommonThrottle()
+    assert throttling.allow_request(request, None)
+    assert throttling.allow_request(request, None) is False
+    for x in range(100):
+        assert throttling.allow_request(request, None) is False
+    cache.clear()
+    settings.REST_FRAMEWORK['DEFAULT_THROTTLE_RATES']['anon-write'] = None
+
+def test_anon_multi_write_first_small_throttling(settings, rf):
+    settings.REST_FRAMEWORK['DEFAULT_THROTTLE_RATES']['anon-write'] = ["1/min", "10/min"]
+    request = rf.post("/test")
+    request.user = AnonymousUser()
+    throttling = CommonThrottle()
+    assert throttling.allow_request(request, None)
+    assert throttling.allow_request(request, None) is False
+    for x in range(100):
+        assert throttling.allow_request(request, None) is False
+    cache.clear()
+    settings.REST_FRAMEWORK['DEFAULT_THROTTLE_RATES']['anon-write'] = None
+
+def test_anon_multi_write_first_big_throttling(settings, rf):
+    settings.REST_FRAMEWORK['DEFAULT_THROTTLE_RATES']['anon-write'] = ["10/min", "1/min"]
+    request = rf.post("/test")
+    request.user = AnonymousUser()
+    throttling = CommonThrottle()
+    assert throttling.allow_request(request, None)
+    assert throttling.allow_request(request, None) is False
+    for x in range(100):
+        assert throttling.allow_request(request, None) is False
+    cache.clear()
+    settings.REST_FRAMEWORK['DEFAULT_THROTTLE_RATES']['anon-write'] = None
+
+def test_anon_no_read_throttling(settings, rf):
+    settings.REST_FRAMEWORK['DEFAULT_THROTTLE_RATES']['anon-read'] = None
+    request = rf.get("/test")
+    request.user = AnonymousUser()
+    throttling = CommonThrottle()
+    for x in range(100):
+        assert throttling.allow_request(request, None)
+    cache.clear()
+    settings.REST_FRAMEWORK['DEFAULT_THROTTLE_RATES']['anon-read'] = None
+
+def test_anon_simple_read_throttling(settings, rf):
+    settings.REST_FRAMEWORK['DEFAULT_THROTTLE_RATES']['anon-read'] = "1/min"
+    request = rf.get("/test")
+    request.user = AnonymousUser()
+    throttling = CommonThrottle()
+    assert throttling.allow_request(request, None)
+    assert throttling.allow_request(request, None) is False
+    for x in range(100):
+        assert throttling.allow_request(request, None) is False
+    cache.clear()
+    settings.REST_FRAMEWORK['DEFAULT_THROTTLE_RATES']['anon-read'] = None
+
+def test_anon_multi_read_first_small_throttling(settings, rf):
+    settings.REST_FRAMEWORK['DEFAULT_THROTTLE_RATES']['anon-read'] = ["1/min", "10/min"]
+    request = rf.get("/test")
+    request.user = AnonymousUser()
+    throttling = CommonThrottle()
+    assert throttling.allow_request(request, None)
+    assert throttling.allow_request(request, None) is False
+    for x in range(100):
+        assert throttling.allow_request(request, None) is False
+    cache.clear()
+    settings.REST_FRAMEWORK['DEFAULT_THROTTLE_RATES']['anon-read'] = None
+
+def test_anon_multi_read_first_big_throttling(settings, rf):
+    settings.REST_FRAMEWORK['DEFAULT_THROTTLE_RATES']['anon-read'] = ["10/min", "1/min"]
+    request = rf.get("/test")
+    request.user = AnonymousUser()
+    throttling = CommonThrottle()
+    assert throttling.allow_request(request, None)
+    assert throttling.allow_request(request, None) is False
+    for x in range(100):
+        assert throttling.allow_request(request, None) is False
+    cache.clear()
+    settings.REST_FRAMEWORK['DEFAULT_THROTTLE_RATES']['anon-read'] = None
+
+def test_whitelisted_anon_throttling(settings, rf):
+    settings.REST_FRAMEWORK['DEFAULT_THROTTLE_RATES']['anon-read'] = "1/min"
+    settings.REST_FRAMEWORK['DEFAULT_THROTTLE_WHITELIST'] = ["127.0.0.1"]
+    request = rf.get("/test")
+    request.user = AnonymousUser()
+    request.META["REMOTE_ADDR"] = "127.0.0.1"
+    throttling = CommonThrottle()
+    assert throttling.allow_request(request, None)
+    assert throttling.allow_request(request, None)
+    for x in range(100):
+        assert throttling.allow_request(request, None)
+    cache.clear()
+    settings.REST_FRAMEWORK['DEFAULT_THROTTLE_RATES']['anon-read'] = None
+    settings.REST_FRAMEWORK['DEFAULT_THROTTLE_WHITELIST'] = []
+
+def test_not_whitelisted_anon_throttling(settings, rf):
+    settings.REST_FRAMEWORK['DEFAULT_THROTTLE_RATES']['anon-read'] = "1/min"
+    settings.REST_FRAMEWORK['DEFAULT_THROTTLE_WHITELIST'] = ["127.0.0.1"]
+    request = rf.get("/test")
+    request.user = AnonymousUser()
+    request.META["REMOTE_ADDR"] = "127.0.0.2"
+    throttling = CommonThrottle()
+    assert throttling.allow_request(request, None)
+    assert throttling.allow_request(request, None) is False
+    for x in range(100):
+        assert throttling.allow_request(request, None) is False
+    cache.clear()
+    settings.REST_FRAMEWORK['DEFAULT_THROTTLE_RATES']['anon-read'] = None
+    settings.REST_FRAMEWORK['DEFAULT_THROTTLE_WHITELIST'] = []