taiga
/
taiga-back
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Added common throttle for all the API

remotes/origin/issue/4217/improving-mail-design
Jesús Espino 2017-02-28 16:56:07 +01:00 committed by David Barragán Merino
parent 7019de8f0d
commit 8dfcbe4006
6 changed files with 423 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
settings/common.py
View File

@ -433,12 +433,13 @@ REST_FRAMEWORK = {
"taiga.external_apps.auth_backends.Token",
),
"DEFAULT_THROTTLE_CLASSES": (
"taiga.base.throttling.AnonRateThrottle",
"taiga.base.throttling.UserRateThrottle"
"taiga.base.throttling.CommonThrottle",
),
"DEFAULT_THROTTLE_RATES": {
"anon": None,
"user": None,
"anon-write": None,
"user-write": None,
"anon-read": None,
"user-read": None,
"import-mode": None,
"import-dump-mode": "1/minute",
"create-memberships": None,
@ -446,6 +447,7 @@ REST_FRAMEWORK = {
"register-success": None,
"user-detail": None,
},
"DEFAULT_THROTTLE_WHITELIST": [],
"FILTER_BACKEND": "taiga.base.filters.FilterBackend",
"EXCEPTION_HANDLER": "taiga.base.exceptions.exception_handler",
"PAGINATE_BY": 30,

6
settings/testing.py
View File

@ -29,8 +29,10 @@ INSTALLED_APPS = INSTALLED_APPS + [
]
REST_FRAMEWORK["DEFAULT_THROTTLE_RATES"] = {
"anon": None,
"user": None,
"anon-write": None,
"anon-read": None,
"user-write": None,
"user-read": None,
"import-mode": None,
"import-dump-mode": None,
"create-memberships": None,

1
taiga/base/api/settings.py
View File

@ -106,6 +106,7 @@ DEFAULTS = {
"user": None,
"anon": None,
},
"DEFAULT_THROTTLE_WHITELIST": [],
# Pagination
"PAGINATE_BY": None,

131
taiga/base/throttling.py
View File

@ -17,6 +17,7 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
from taiga.base.api import throttling
from django.conf import settings
class GlobalThrottlingMixin:
@ -57,5 +58,135 @@ class UserRateThrottle(throttling.UserRateThrottle):
scope = "user"
class CommonThrottle(throttling.SimpleRateThrottle):
cache_format = "throtte_%(scope)s_%(rate)s_%(ident)s"
def __init__(self):
pass
def has_to_finalize(self, request, response, view):
return False
def allow_request(self, request, view):
scope = self.get_scope(request)
ident = self.get_ident(request)
rates = self.get_rates(scope)
if ident in settings.REST_FRAMEWORK['DEFAULT_THROTTLE_WHITELIST']:
return True
if rates is None or rates == []:
return True
now = self.timer()
waits = []
history_writes = []
for rate in rates:
rate_name = rate[0]
rate_num_requests = rate[1]
rate_duration = rate[2]
key = self.get_cache_key(ident, scope, rate_name)
history = self.cache.get(key, [])
while history and history[-1] <= now - rate_duration:
history.pop()
if len(history) >= rate_num_requests:
waits.append(self.wait_time(history, rate, now))
history_writes.append({
"key": key,
"history": history,
"rate_duration": rate_duration,
})
if waits:
self._wait = max(waits)
return False
for history_write in history_writes:
history_write['history'].insert(0, now)
self.cache.set(
history_write['key'],
history_write['history'],
history_write['rate_duration']
)
return True
def get_rates(self, scope):
try:
rates = self.THROTTLE_RATES[scope]
except KeyError:
msg = "No default throttle rate set for \"%s\" scope" % scope
raise ImproperlyConfigured(msg)
if rates is None:
return []
elif isinstance(rates, str):
return [self.parse_rate(rates)]
elif isinstance(rates, list):
return list(map(self.parse_rate, rates))
else:
msg = "No valid throttle rate set for \"%s\" scope" % scope
raise ImproperlyConfigured(msg)
def parse_rate(self, rate):
"""
Given the request rate string, return a two tuple of:
<allowed number of requests>, <period of time in seconds>
"""
if rate is None:
return None
num, period = rate.split("/")
num_requests = int(num)
duration = {"s": 1, "m": 60, "h": 3600, "d": 86400}[period[0]]
return (rate, num_requests, duration)
def get_scope(self, request):
if request.user.is_authenticated():
if request.method in ["POST", "PUT", "PATCH", "DELETE"]:
scope = "user-write"
else:
scope = "user-read"
else:
if request.method in ["POST", "PUT", "PATCH", "DELETE"]:
scope = "anon-write"
else:
scope = "anon-read"
return scope
def get_ident(self, request):
if request.user.is_authenticated():
ident = request.user.id
else:
ident = request.META.get("HTTP_X_FORWARDED_FOR")
if ident is None:
ident = request.META.get("REMOTE_ADDR")
return ident
def get_cache_key(self, ident, scope, rate):
return self.cache_format % { "scope": scope, "ident": ident, "rate": rate }
def wait_time(self, history, rate, now):
rate_num_requests = rate[1]
rate_duration = rate[2]
if history:
remaining_duration = rate_duration - (now - history[-1])
else:
remaining_duration = rate_duration
available_requests = rate_num_requests - len(history) + 1
if available_requests <= 0:
return remaining_duration
return remaining_duration / float(available_requests)
def wait(self):
return self._wait
class SimpleRateThrottle(throttling.SimpleRateThrottle):
pass

33
tests/integration/test_throwttling.py
View File

@ -29,20 +29,15 @@ from .. import factories as f
pytestmark = pytest.mark.django_db
anon_rate_path = "taiga.base.throttling.AnonRateThrottle.get_rate"
user_rate_path = "taiga.base.throttling.UserRateThrottle.get_rate"
import_rate_path = "taiga.export_import.throttling.ImportModeRateThrottle.get_rate"
def test_anonimous_throttling_policy(client, settings):
f.create_project()
url = reverse("projects-list")
settings.REST_FRAMEWORK['DEFAULT_THROTTLE_RATES']['anon-read'] = "2/min"
with mock.patch(anon_rate_path) as anon_rate, \
mock.patch(user_rate_path) as user_rate, \
mock.patch(import_rate_path) as import_rate:
anon_rate.return_value = "2/day"
user_rate.return_value = "4/day"
with mock.patch(import_rate_path) as import_rate:
import_rate.return_value = "7/day"
cache.clear()
@ -53,19 +48,19 @@ def test_anonimous_throttling_policy(client, settings):
response = client.json.get(url)
assert response.status_code == 429
settings.REST_FRAMEWORK['DEFAULT_THROTTLE_RATES']['anon-read'] = None
cache.clear()
def test_user_throttling_policy(client, settings):
project = f.create_project()
f.MembershipFactory.create(project=project, user=project.owner, is_admin=True)
url = reverse("projects-detail", kwargs={"pk": project.pk})
settings.REST_FRAMEWORK['DEFAULT_THROTTLE_RATES']['user-read'] = "4/min"
client.login(project.owner)
with mock.patch(anon_rate_path) as anon_rate, \
mock.patch(user_rate_path) as user_rate, \
mock.patch(import_rate_path) as import_rate:
anon_rate.return_value = "2/day"
user_rate.return_value = "4/day"
with mock.patch(import_rate_path) as import_rate:
import_rate.return_value = "7/day"
cache.clear()
@ -81,6 +76,8 @@ def test_user_throttling_policy(client, settings):
assert response.status_code == 429
client.logout()
settings.REST_FRAMEWORK['DEFAULT_THROTTLE_RATES']['user-read'] = None
cache.clear()
def test_import_mode_throttling_policy(client, settings):
@ -95,14 +92,12 @@ def test_import_mode_throttling_policy(client, settings):
data = {
"subject": "Test"
}
settings.REST_FRAMEWORK['DEFAULT_THROTTLE_RATES']['anon-read'] = "2/min"
settings.REST_FRAMEWORK['DEFAULT_THROTTLE_RATES']['user-read'] = "4/min"
client.login(project.owner)
with mock.patch(anon_rate_path) as anon_rate, \
mock.patch(user_rate_path) as user_rate, \
mock.patch(import_rate_path) as import_rate:
anon_rate.return_value = "2/day"
user_rate.return_value = "4/day"
with mock.patch(import_rate_path) as import_rate:
import_rate.return_value = "7/day"
cache.clear()
@ -124,3 +119,7 @@ def test_import_mode_throttling_policy(client, settings):
assert response.status_code == 429
client.logout()
settings.REST_FRAMEWORK['DEFAULT_THROTTLE_RATES']['anon-read'] = None
settings.REST_FRAMEWORK['DEFAULT_THROTTLE_RATES']['user-read'] = None
cache.clear()

265
tests/unit/test_common_throttle.py Normal file
View File

@ -0,0 +1,265 @@
# -*- coding: utf-8 -*-
# Copyright (C) 2014-2017 Andrey Antukh <niwi@niwi.nz>
# Copyright (C) 2014-2017 Jesús Espino <jespinog@gmail.com>
# Copyright (C) 2014-2017 David Barragán <bameda@dbarragan.com>
# Copyright (C) 2014-2017 Alejandro Alonso <alejandro.alonso@kaleidos.net>
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
from django.test import RequestFactory
from django.core.cache import cache
from django.contrib.auth.models import AnonymousUser
from taiga.base.throttling import CommonThrottle
from taiga.users.models import User
def test_user_no_write_throttling(settings, rf):
settings.REST_FRAMEWORK['DEFAULT_THROTTLE_RATES']['user-write'] = None
request = rf.post("/test")
request.user = User(id=1)
throttling = CommonThrottle()
for x in range(100):
assert throttling.allow_request(request, None)
cache.clear()
settings.REST_FRAMEWORK['DEFAULT_THROTTLE_RATES']['user-write'] = None
def test_user_simple_write_throttling(settings, rf):
settings.REST_FRAMEWORK['DEFAULT_THROTTLE_RATES']['user-write'] = "1/min"
request = rf.post("/test")
request.user = User(id=1)
throttling = CommonThrottle()
assert throttling.allow_request(request, None)
assert throttling.allow_request(request, None) is False
for x in range(100):
assert throttling.allow_request(request, None) is False
cache.clear()
settings.REST_FRAMEWORK['DEFAULT_THROTTLE_RATES']['user-write'] = None
def test_user_multi_write_first_small_throttling(settings, rf):
settings.REST_FRAMEWORK['DEFAULT_THROTTLE_RATES']['user-write'] = ["1/min", "10/min"]
request = rf.post("/test")
request.user = User(id=1)
throttling = CommonThrottle()
assert throttling.allow_request(request, None)
assert throttling.allow_request(request, None) is False
for x in range(100):
assert throttling.allow_request(request, None) is False
cache.clear()
settings.REST_FRAMEWORK['DEFAULT_THROTTLE_RATES']['user-write'] = None
def test_user_multi_write_first_big_throttling(settings, rf):
settings.REST_FRAMEWORK['DEFAULT_THROTTLE_RATES']['user-write'] = ["10/min", "1/min"]
request = rf.post("/test")
request.user = User(id=1)
throttling = CommonThrottle()
assert throttling.allow_request(request, None)
assert throttling.allow_request(request, None) is False
for x in range(100):
assert throttling.allow_request(request, None) is False
cache.clear()
settings.REST_FRAMEWORK['DEFAULT_THROTTLE_RATES']['user-write'] = None
def test_user_no_read_throttling(settings, rf):
settings.REST_FRAMEWORK['DEFAULT_THROTTLE_RATES']['user-read'] = None
request = rf.get("/test")
request.user = User(id=1)
throttling = CommonThrottle()
for x in range(100):
assert throttling.allow_request(request, None)
cache.clear()
settings.REST_FRAMEWORK['DEFAULT_THROTTLE_RATES']['user-read'] = None
def test_user_simple_read_throttling(settings, rf):
settings.REST_FRAMEWORK['DEFAULT_THROTTLE_RATES']['user-read'] = "1/min"
request = rf.get("/test")
request.user = User(id=1)
throttling = CommonThrottle()
assert throttling.allow_request(request, None)
assert throttling.allow_request(request, None) is False
for x in range(100):
assert throttling.allow_request(request, None) is False
cache.clear()
settings.REST_FRAMEWORK['DEFAULT_THROTTLE_RATES']['user-read'] = None
def test_user_multi_read_first_small_throttling(settings, rf):
settings.REST_FRAMEWORK['DEFAULT_THROTTLE_RATES']['user-read'] = ["1/min", "10/min"]
request = rf.get("/test")
request.user = User(id=1)
throttling = CommonThrottle()
assert throttling.allow_request(request, None)
assert throttling.allow_request(request, None) is False
for x in range(100):
assert throttling.allow_request(request, None) is False
cache.clear()
settings.REST_FRAMEWORK['DEFAULT_THROTTLE_RATES']['user-read'] = None
def test_user_multi_read_first_big_throttling(settings, rf):
settings.REST_FRAMEWORK['DEFAULT_THROTTLE_RATES']['user-read'] = ["10/min", "1/min"]
request = rf.get("/test")
request.user = User(id=1)
throttling = CommonThrottle()
assert throttling.allow_request(request, None)
assert throttling.allow_request(request, None) is False
for x in range(100):
assert throttling.allow_request(request, None) is False
cache.clear()
settings.REST_FRAMEWORK['DEFAULT_THROTTLE_RATES']['user-read'] = None
def test_whitelisted_user_throttling(settings, rf):
settings.REST_FRAMEWORK['DEFAULT_THROTTLE_RATES']['user-read'] = "1/min"
settings.REST_FRAMEWORK['DEFAULT_THROTTLE_WHITELIST'] = [1]
request = rf.get("/test")
request.user = User(id=1)
throttling = CommonThrottle()
assert throttling.allow_request(request, None)
assert throttling.allow_request(request, None)
cache.clear()
settings.REST_FRAMEWORK['DEFAULT_THROTTLE_RATES']['user-read'] = None
settings.REST_FRAMEWORK['DEFAULT_THROTTLE_WHITELIST'] = []
def test_not_whitelisted_user_throttling(settings, rf):
settings.REST_FRAMEWORK['DEFAULT_THROTTLE_RATES']['user-read'] = "1/min"
settings.REST_FRAMEWORK['DEFAULT_THROTTLE_WHITELIST'] = [1]
request = rf.get("/test")
request.user = User(id=2)
throttling = CommonThrottle()
assert throttling.allow_request(request, None)
assert throttling.allow_request(request, None) is False
for x in range(100):
assert throttling.allow_request(request, None) is False
cache.clear()
settings.REST_FRAMEWORK['DEFAULT_THROTTLE_RATES']['user-read'] = None
settings.REST_FRAMEWORK['DEFAULT_THROTTLE_WHITELIST'] = []
def test_anon_no_write_throttling(settings, rf):
settings.REST_FRAMEWORK['DEFAULT_THROTTLE_RATES']['anon-write'] = None
request = rf.post("/test")
request.user = AnonymousUser()
throttling = CommonThrottle()
for x in range(100):
assert throttling.allow_request(request, None)
cache.clear()
settings.REST_FRAMEWORK['DEFAULT_THROTTLE_RATES']['anon-write'] = None
def test_anon_simple_write_throttling(settings, rf):
settings.REST_FRAMEWORK['DEFAULT_THROTTLE_RATES']['anon-write'] = "1/min"
request = rf.post("/test")
request.user = AnonymousUser()
throttling = CommonThrottle()
assert throttling.allow_request(request, None)
assert throttling.allow_request(request, None) is False
for x in range(100):
assert throttling.allow_request(request, None) is False
cache.clear()
settings.REST_FRAMEWORK['DEFAULT_THROTTLE_RATES']['anon-write'] = None
def test_anon_multi_write_first_small_throttling(settings, rf):
settings.REST_FRAMEWORK['DEFAULT_THROTTLE_RATES']['anon-write'] = ["1/min", "10/min"]
request = rf.post("/test")
request.user = AnonymousUser()
throttling = CommonThrottle()
assert throttling.allow_request(request, None)
assert throttling.allow_request(request, None) is False
for x in range(100):
assert throttling.allow_request(request, None) is False
cache.clear()
settings.REST_FRAMEWORK['DEFAULT_THROTTLE_RATES']['anon-write'] = None
def test_anon_multi_write_first_big_throttling(settings, rf):
settings.REST_FRAMEWORK['DEFAULT_THROTTLE_RATES']['anon-write'] = ["10/min", "1/min"]
request = rf.post("/test")
request.user = AnonymousUser()
throttling = CommonThrottle()
assert throttling.allow_request(request, None)
assert throttling.allow_request(request, None) is False
for x in range(100):
assert throttling.allow_request(request, None) is False
cache.clear()
settings.REST_FRAMEWORK['DEFAULT_THROTTLE_RATES']['anon-write'] = None
def test_anon_no_read_throttling(settings, rf):
settings.REST_FRAMEWORK['DEFAULT_THROTTLE_RATES']['anon-read'] = None
request = rf.get("/test")
request.user = AnonymousUser()
throttling = CommonThrottle()
for x in range(100):
assert throttling.allow_request(request, None)
cache.clear()
settings.REST_FRAMEWORK['DEFAULT_THROTTLE_RATES']['anon-read'] = None
def test_anon_simple_read_throttling(settings, rf):
settings.REST_FRAMEWORK['DEFAULT_THROTTLE_RATES']['anon-read'] = "1/min"
request = rf.get("/test")
request.user = AnonymousUser()
throttling = CommonThrottle()
assert throttling.allow_request(request, None)
assert throttling.allow_request(request, None) is False
for x in range(100):
assert throttling.allow_request(request, None) is False
cache.clear()
settings.REST_FRAMEWORK['DEFAULT_THROTTLE_RATES']['anon-read'] = None
def test_anon_multi_read_first_small_throttling(settings, rf):
settings.REST_FRAMEWORK['DEFAULT_THROTTLE_RATES']['anon-read'] = ["1/min", "10/min"]
request = rf.get("/test")
request.user = AnonymousUser()
throttling = CommonThrottle()
assert throttling.allow_request(request, None)
assert throttling.allow_request(request, None) is False
for x in range(100):
assert throttling.allow_request(request, None) is False
cache.clear()
settings.REST_FRAMEWORK['DEFAULT_THROTTLE_RATES']['anon-read'] = None
def test_anon_multi_read_first_big_throttling(settings, rf):
settings.REST_FRAMEWORK['DEFAULT_THROTTLE_RATES']['anon-read'] = ["10/min", "1/min"]
request = rf.get("/test")
request.user = AnonymousUser()
throttling = CommonThrottle()
assert throttling.allow_request(request, None)
assert throttling.allow_request(request, None) is False
for x in range(100):
assert throttling.allow_request(request, None) is False
cache.clear()
settings.REST_FRAMEWORK['DEFAULT_THROTTLE_RATES']['anon-read'] = None
def test_whitelisted_anon_throttling(settings, rf):
settings.REST_FRAMEWORK['DEFAULT_THROTTLE_RATES']['anon-read'] = "1/min"
settings.REST_FRAMEWORK['DEFAULT_THROTTLE_WHITELIST'] = ["127.0.0.1"]
request = rf.get("/test")
request.user = AnonymousUser()
request.META["REMOTE_ADDR"] = "127.0.0.1"
throttling = CommonThrottle()
assert throttling.allow_request(request, None)
assert throttling.allow_request(request, None)
for x in range(100):
assert throttling.allow_request(request, None)
cache.clear()
settings.REST_FRAMEWORK['DEFAULT_THROTTLE_RATES']['anon-read'] = None
settings.REST_FRAMEWORK['DEFAULT_THROTTLE_WHITELIST'] = []
def test_not_whitelisted_anon_throttling(settings, rf):
settings.REST_FRAMEWORK['DEFAULT_THROTTLE_RATES']['anon-read'] = "1/min"
settings.REST_FRAMEWORK['DEFAULT_THROTTLE_WHITELIST'] = ["127.0.0.1"]
request = rf.get("/test")
request.user = AnonymousUser()
request.META["REMOTE_ADDR"] = "127.0.0.2"
throttling = CommonThrottle()
assert throttling.allow_request(request, None)
assert throttling.allow_request(request, None) is False
for x in range(100):
assert throttling.allow_request(request, None) is False
cache.clear()
settings.REST_FRAMEWORK['DEFAULT_THROTTLE_RATES']['anon-read'] = None
settings.REST_FRAMEWORK['DEFAULT_THROTTLE_WHITELIST'] = []