Merge branch 'master' of gitlab.com:allianceauth/allianceauth into v4.x

This commit is contained in:
Ariel Rin
2023-08-14 15:13:54 +10:00
107 changed files with 1250 additions and 370 deletions

View File

@@ -2,17 +2,30 @@ from django.urls import include
from django.contrib.auth.decorators import user_passes_test
from django.core.exceptions import PermissionDenied
from functools import wraps
from django.shortcuts import redirect
from typing import Callable, Iterable, Optional
from django.urls import include
from django.contrib import messages
from django.contrib.auth.decorators import login_required, user_passes_test
from django.core.exceptions import PermissionDenied
from django.shortcuts import redirect
from django.utils.translation import gettext_lazy as _
from django.contrib.auth.decorators import login_required
def user_has_main_character(user):
return bool(user.profile.main_character)
def decorate_url_patterns(urls, decorator):
def decorate_url_patterns(
urls, decorator: Callable, excluded_views: Optional[Iterable] = None
):
"""Decorate views given in url patterns except when they are explicitly excluded.
Args:
- urls: Django URL patterns
- decorator: Decorator to be added to each view
- exclude_views: Optional iterable of view names to be excluded
"""
url_list, app_name, namespace = include(urls)
def process_patterns(url_patterns):
@@ -22,6 +35,8 @@ def decorate_url_patterns(urls, decorator):
process_patterns(pattern.url_patterns)
else:
# this is a pattern
if excluded_views and pattern.lookup_str in excluded_views:
return
pattern.callback = decorator(pattern.callback)
process_patterns(url_list)

View File

@@ -1,7 +1,5 @@
from allianceauth.authentication import views
from django.urls import include
from django.urls import re_path
from django.urls import path
from django.urls import include, re_path, path
urlpatterns = [
path('activate/complete/', views.activation_complete, name='registration_activation_complete'),

0
allianceauth/authentication/managers.py Executable file → Normal file
View File

View File

@@ -0,0 +1,34 @@
# Generated by Django 4.0.10 on 2023-05-28 15:36
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("authentication", "0020_userprofile_language_userprofile_night_mode"),
]
operations = [
migrations.AlterField(
model_name="userprofile",
name="language",
field=models.CharField(
blank=True,
choices=[
("en", "English"),
("de", "German"),
("es", "Spanish"),
("zh-hans", "Chinese Simplified"),
("ru", "Russian"),
("ko", "Korean"),
("fr", "French"),
("ja", "Japanese"),
("it", "Italian"),
("uk", "Ukrainian"),
],
default="",
max_length=10,
verbose_name="Language",
),
),
]

29
allianceauth/authentication/models.py Executable file → Normal file
View File

@@ -63,6 +63,22 @@ class UserProfile(models.Model):
class Meta:
default_permissions = ('change',)
class Language(models.TextChoices):
"""
Choices for UserProfile.language
"""
ENGLISH = 'en', _('English')
GERMAN = 'de', _('German')
SPANISH = 'es', _('Spanish')
CHINESE = 'zh-hans', _('Chinese Simplified')
RUSSIAN = 'ru', _('Russian')
KOREAN = 'ko', _('Korean')
FRENCH = 'fr', _('French')
JAPANESE = 'ja', _('Japanese')
ITALIAN = 'it', _('Italian')
UKRAINIAN = 'uk', _('Ukrainian')
user = models.OneToOneField(
User,
related_name='profile',
@@ -76,20 +92,9 @@ class UserProfile(models.Model):
State,
on_delete=models.SET_DEFAULT,
default=get_guest_state_pk)
LANGUAGE_CHOICES = [
('en', _('English')),
('de', _('German')),
('es', _('Spanish')),
('zh-hans', _('Chinese Simplified')),
('ru', _('Russian')),
('ko', _('Korean')),
('fr', _('French')),
('ja', _('Japanese')),
('it', _('Italian')),
]
language = models.CharField(
_("Language"), max_length=10,
choices=LANGUAGE_CHOICES,
choices=Language.choices,
blank=True,
default='')
night_mode = models.BooleanField(

View File

@@ -1,35 +1,44 @@
from collections import namedtuple
"""Counters for Task Statistics."""
import datetime as dt
from typing import NamedTuple, Optional
from .event_series import EventSeries
from .helpers import ItemCounter
"""Global series for counting task events."""
# Global series for counting task events.
succeeded_tasks = EventSeries("SUCCEEDED_TASKS")
retried_tasks = EventSeries("RETRIED_TASKS")
failed_tasks = EventSeries("FAILED_TASKS")
running_tasks = ItemCounter("running_tasks")
_TaskCounts = namedtuple(
"_TaskCounts", ["succeeded", "retried", "failed", "total", "earliest_task", "hours"]
)
class _TaskCounts(NamedTuple):
succeeded: int
retried: int
failed: int
total: int
earliest_task: Optional[dt.datetime]
hours: int
running: int
def dashboard_results(hours: int) -> _TaskCounts:
"""Counts of all task events within the given timeframe."""
"""Counts of all task events within the given time frame."""
def earliest_if_exists(events: EventSeries, earliest: dt.datetime) -> list:
my_earliest = events.first_event(earliest=earliest)
return [my_earliest] if my_earliest else []
earliest = dt.datetime.utcnow() - dt.timedelta(hours=hours)
earliest_events = list()
earliest_events = []
succeeded_count = succeeded_tasks.count(earliest=earliest)
earliest_events += earliest_if_exists(succeeded_tasks, earliest)
retried_count = retried_tasks.count(earliest=earliest)
earliest_events += earliest_if_exists(retried_tasks, earliest)
failed_count = failed_tasks.count(earliest=earliest)
earliest_events += earliest_if_exists(failed_tasks, earliest)
running_count = running_tasks.value()
return _TaskCounts(
succeeded=succeeded_count,
retried=retried_count,
@@ -37,4 +46,5 @@ def dashboard_results(hours: int) -> _TaskCounts:
total=succeeded_count + retried_count + failed_count,
earliest_task=min(earliest_events) if earliest_events else None,
hours=hours,
running=running_count,
)

View File

@@ -1,61 +1,31 @@
"""Event series for Task Statistics."""
import datetime as dt
import logging
from typing import List, Optional
from pytz import utc
from redis import Redis, RedisError
from redis import Redis
from allianceauth.utils.cache import get_redis_client
from .helpers import get_redis_client_or_stub
logger = logging.getLogger(__name__)
class _RedisStub:
"""Stub of a Redis client.
It's purpose is to prevent EventSeries objects from trying to access Redis
when it is not available. e.g. when the Sphinx docs are rendered by readthedocs.org.
"""
def delete(self, *args, **kwargs):
pass
def incr(self, *args, **kwargs):
return 0
def zadd(self, *args, **kwargs):
pass
def zcount(self, *args, **kwargs):
pass
def zrangebyscore(self, *args, **kwargs):
pass
class EventSeries:
"""API for recording and analyzing a series of events."""
_ROOT_KEY = "ALLIANCEAUTH_EVENT_SERIES"
def __init__(self, key_id: str, redis: Redis = None) -> None:
self._redis = get_redis_client() if not redis else redis
try:
if not self._redis.ping():
raise RuntimeError()
except (AttributeError, RedisError, RuntimeError):
logger.exception(
"Failed to establish a connection with Redis. "
"This EventSeries object is disabled.",
)
self._redis = _RedisStub()
def __init__(self, key_id: str, redis: Optional[Redis] = None) -> None:
self._redis = get_redis_client_or_stub() if not redis else redis
self._key_id = str(key_id)
self.clear()
@property
def is_disabled(self):
"""True when this object is disabled, e.g. Redis was not available at startup."""
return isinstance(self._redis, _RedisStub)
return hasattr(self._redis, "IS_STUB")
@property
def _key_counter(self):
@@ -73,8 +43,8 @@ class EventSeries:
"""
if not event_time:
event_time = dt.datetime.utcnow()
id = self._redis.incr(self._key_counter)
self._redis.zadd(self._key_sorted_set, {id: event_time.timestamp()})
my_id = self._redis.incr(self._key_counter)
self._redis.zadd(self._key_sorted_set, {my_id: event_time.timestamp()})
def all(self) -> List[dt.datetime]:
"""List of all known events."""
@@ -95,15 +65,15 @@ class EventSeries:
self._redis.delete(self._key_counter)
def count(self, earliest: dt.datetime = None, latest: dt.datetime = None) -> int:
"""Count of events, can be restricted to given timeframe.
"""Count of events, can be restricted to given time frame.
Args:
- earliest: Date of first events to count(inclusive), or -infinite if not specified
- latest: Date of last events to count(inclusive), or +infinite if not specified
"""
min = "-inf" if not earliest else earliest.timestamp()
max = "+inf" if not latest else latest.timestamp()
return self._redis.zcount(self._key_sorted_set, min=min, max=max)
minimum = "-inf" if not earliest else earliest.timestamp()
maximum = "+inf" if not latest else latest.timestamp()
return self._redis.zcount(self._key_sorted_set, min=minimum, max=maximum)
def first_event(self, earliest: dt.datetime = None) -> Optional[dt.datetime]:
"""Date/Time of first event. Returns `None` if series has no events.
@@ -111,10 +81,10 @@ class EventSeries:
Args:
- earliest: Date of first events to count(inclusive), or any if not specified
"""
min = "-inf" if not earliest else earliest.timestamp()
minimum = "-inf" if not earliest else earliest.timestamp()
event = self._redis.zrangebyscore(
self._key_sorted_set,
min,
minimum,
"+inf",
withscores=True,
start=0,

View File

@@ -0,0 +1,108 @@
"""Helpers for Task Statistics."""
import logging
from typing import Optional
from redis import Redis, RedisError
from django.core.cache import cache
from allianceauth.utils.cache import get_redis_client
logger = logging.getLogger(__name__)
class _RedisStub:
"""Stub of a Redis client.
It's purpose is to prevent EventSeries objects from trying to access Redis
when it is not available. e.g. when the Sphinx docs are rendered by readthedocs.org.
"""
IS_STUB = True
def delete(self, *args, **kwargs):
pass
def incr(self, *args, **kwargs):
return 0
def zadd(self, *args, **kwargs):
pass
def zcount(self, *args, **kwargs):
pass
def zrangebyscore(self, *args, **kwargs):
pass
class ItemCounter:
"""A process safe item counter.
Args:
- name: Unique name for the counter
- minimum: Counter can not go below the minimum, when set
- redis: A Redis client. Will use AA's cache client by default
"""
CACHE_KEY_BASE = "allianceauth-item-counter"
DEFAULT_CACHE_TIMEOUT = 24 * 3600
def __init__(
self, name: str, minimum: Optional[int] = None, redis: Optional[Redis] = None
) -> None:
if not name:
raise ValueError("Must define a name")
self._name = str(name)
self._minimum = minimum
self._redis = get_redis_client_or_stub() if not redis else redis
@property
def _cache_key(self) -> str:
return f"{self.CACHE_KEY_BASE}-{self._name}"
def reset(self, init_value: int = 0):
"""Reset counter to initial value."""
with self._redis.lock(f"{self.CACHE_KEY_BASE}-reset"):
if self._minimum is not None and init_value < self._minimum:
raise ValueError("Can not reset below minimum")
cache.set(self._cache_key, init_value, self.DEFAULT_CACHE_TIMEOUT)
def incr(self, delta: int = 1):
"""Increment counter by delta."""
try:
cache.incr(self._cache_key, delta)
except ValueError:
pass
def decr(self, delta: int = 1):
"""Decrement counter by delta."""
with self._redis.lock(f"{self.CACHE_KEY_BASE}-decr"):
if self._minimum is not None and self.value() == self._minimum:
return
try:
cache.decr(self._cache_key, delta)
except ValueError:
pass
def value(self) -> Optional[int]:
"""Return current value or None if not yet initialized."""
return cache.get(self._cache_key)
def get_redis_client_or_stub() -> Redis:
"""Return AA's default cache client or a stub if Redis is not available."""
redis = get_redis_client()
try:
if not redis.ping():
raise RuntimeError()
except (AttributeError, RedisError, RuntimeError):
logger.exception(
"Failed to establish a connection with Redis. "
"This EventSeries object is disabled.",
)
return _RedisStub()
return redis

View File

@@ -1,14 +1,15 @@
"""Signals for Task Statistics."""
from celery.signals import (
task_failure,
task_internal_error,
task_retry,
task_success,
worker_ready
task_failure, task_internal_error, task_postrun, task_prerun, task_retry,
task_success, worker_ready,
)
from django.conf import settings
from .counters import failed_tasks, retried_tasks, succeeded_tasks
from .counters import (
failed_tasks, retried_tasks, running_tasks, succeeded_tasks,
)
def reset_counters():
@@ -16,9 +17,11 @@ def reset_counters():
succeeded_tasks.clear()
failed_tasks.clear()
retried_tasks.clear()
running_tasks.reset()
def is_enabled() -> bool:
"""Return True if task statistics are enabled, else return False."""
return not bool(
getattr(settings, "ALLIANCEAUTH_DASHBOARD_TASK_STATISTICS_DISABLED", False)
)
@@ -52,3 +55,15 @@ def record_task_failed(*args, **kwargs):
def record_task_internal_error(*args, **kwargs):
if is_enabled():
failed_tasks.add()
@task_prerun.connect
def record_task_prerun(*args, **kwargs):
if is_enabled():
running_tasks.incr()
@task_postrun.connect
def record_task_postrun(*args, **kwargs):
if is_enabled():
running_tasks.decr()

View File

@@ -8,25 +8,31 @@ from allianceauth.authentication.task_statistics.counters import (
succeeded_tasks,
retried_tasks,
failed_tasks,
running_tasks,
)
class TestDashboardResults(TestCase):
def test_should_return_counts_for_given_timeframe_only(self):
def test_should_return_counts_for_given_time_frame_only(self):
# given
earliest_task = now() - dt.timedelta(minutes=15)
succeeded_tasks.clear()
succeeded_tasks.add(now() - dt.timedelta(hours=1, seconds=1))
succeeded_tasks.add(earliest_task)
succeeded_tasks.add()
succeeded_tasks.add()
retried_tasks.clear()
retried_tasks.add(now() - dt.timedelta(hours=1, seconds=1))
retried_tasks.add(now() - dt.timedelta(seconds=30))
retried_tasks.add()
failed_tasks.clear()
failed_tasks.add(now() - dt.timedelta(hours=1, seconds=1))
failed_tasks.add()
running_tasks.reset(8)
# when
results = dashboard_results(hours=1)
# then
@@ -35,12 +41,14 @@ class TestDashboardResults(TestCase):
self.assertEqual(results.failed, 1)
self.assertEqual(results.total, 6)
self.assertEqual(results.earliest_task, earliest_task)
self.assertEqual(results.running, 8)
def test_should_work_with_no_data(self):
# given
succeeded_tasks.clear()
retried_tasks.clear()
failed_tasks.clear()
running_tasks.reset()
# when
results = dashboard_results(hours=1)
# then
@@ -49,3 +57,4 @@ class TestDashboardResults(TestCase):
self.assertEqual(results.failed, 0)
self.assertEqual(results.total, 0)
self.assertIsNone(results.earliest_task)
self.assertEqual(results.running, 0)

View File

@@ -1,48 +1,19 @@
import datetime as dt
from unittest.mock import patch
from pytz import utc
from redis import RedisError
from django.test import TestCase
from django.utils.timezone import now
from allianceauth.authentication.task_statistics.event_series import (
EventSeries,
_RedisStub,
)
from allianceauth.authentication.task_statistics.helpers import _RedisStub
MODULE_PATH = "allianceauth.authentication.task_statistics.event_series"
class TestEventSeries(TestCase):
def test_should_abort_without_redis_client(self):
# when
with patch(MODULE_PATH + ".get_redis_client") as mock:
mock.return_value = None
events = EventSeries("dummy")
# then
self.assertTrue(events._redis, _RedisStub)
self.assertTrue(events.is_disabled)
def test_should_disable_itself_if_redis_not_available_1(self):
# when
with patch(MODULE_PATH + ".get_redis_client") as mock_get_master_client:
mock_get_master_client.return_value.ping.side_effect = RedisError
events = EventSeries("dummy")
# then
self.assertIsInstance(events._redis, _RedisStub)
self.assertTrue(events.is_disabled)
def test_should_disable_itself_if_redis_not_available_2(self):
# when
with patch(MODULE_PATH + ".get_redis_client") as mock_get_master_client:
mock_get_master_client.return_value.ping.return_value = False
events = EventSeries("dummy")
# then
self.assertIsInstance(events._redis, _RedisStub)
self.assertTrue(events.is_disabled)
def test_should_add_event(self):
# given
events = EventSeries("dummy")
@@ -166,3 +137,15 @@ class TestEventSeries(TestCase):
results = events.all()
# then
self.assertEqual(len(results), 2)
def test_should_not_report_as_disabled_when_initialized_normally(self):
# given
events = EventSeries("dummy")
# when/then
self.assertFalse(events.is_disabled)
def test_should_report_as_disabled_when_initialized_with_redis_stub(self):
# given
events = EventSeries("dummy", redis=_RedisStub())
# when/then
self.assertTrue(events.is_disabled)

View File

@@ -0,0 +1,142 @@
from unittest import TestCase
from unittest.mock import patch
from redis import RedisError
from allianceauth.authentication.task_statistics.helpers import (
ItemCounter, _RedisStub, get_redis_client_or_stub,
)
MODULE_PATH = "allianceauth.authentication.task_statistics.helpers"
COUNTER_NAME = "test-counter"
class TestItemCounter(TestCase):
def test_can_create_counter(self):
# when
counter = ItemCounter(COUNTER_NAME)
# then
self.assertIsInstance(counter, ItemCounter)
def test_can_reset_counter_to_default(self):
# given
counter = ItemCounter(COUNTER_NAME)
# when
counter.reset()
# then
self.assertEqual(counter.value(), 0)
def test_can_reset_counter_to_custom_value(self):
# given
counter = ItemCounter(COUNTER_NAME)
# when
counter.reset(42)
# then
self.assertEqual(counter.value(), 42)
def test_can_increment_counter_by_default(self):
# given
counter = ItemCounter(COUNTER_NAME)
counter.reset(0)
# when
counter.incr()
# then
self.assertEqual(counter.value(), 1)
def test_can_increment_counter_by_custom_value(self):
# given
counter = ItemCounter(COUNTER_NAME)
counter.reset(0)
# when
counter.incr(8)
# then
self.assertEqual(counter.value(), 8)
def test_can_decrement_counter_by_default(self):
# given
counter = ItemCounter(COUNTER_NAME)
counter.reset(9)
# when
counter.decr()
# then
self.assertEqual(counter.value(), 8)
def test_can_decrement_counter_by_custom_value(self):
# given
counter = ItemCounter(COUNTER_NAME)
counter.reset(9)
# when
counter.decr(8)
# then
self.assertEqual(counter.value(), 1)
def test_can_decrement_counter_below_zero(self):
# given
counter = ItemCounter(COUNTER_NAME)
counter.reset(0)
# when
counter.decr(1)
# then
self.assertEqual(counter.value(), -1)
def test_can_not_decrement_counter_below_minimum(self):
# given
counter = ItemCounter(COUNTER_NAME, minimum=0)
counter.reset(0)
# when
counter.decr(1)
# then
self.assertEqual(counter.value(), 0)
def test_can_not_reset_counter_below_minimum(self):
# given
counter = ItemCounter(COUNTER_NAME, minimum=0)
# when/then
with self.assertRaises(ValueError):
counter.reset(-1)
def test_can_not_init_without_name(self):
# when/then
with self.assertRaises(ValueError):
ItemCounter(name="")
def test_can_ignore_invalid_values_when_incrementing(self):
# given
counter = ItemCounter(COUNTER_NAME)
counter.reset(0)
# when
with patch(MODULE_PATH + ".cache.incr") as m:
m.side_effect = ValueError
counter.incr()
# then
self.assertEqual(counter.value(), 0)
def test_can_ignore_invalid_values_when_decrementing(self):
# given
counter = ItemCounter(COUNTER_NAME)
counter.reset(1)
# when
with patch(MODULE_PATH + ".cache.decr") as m:
m.side_effect = ValueError
counter.decr()
# then
self.assertEqual(counter.value(), 1)
class TestGetRedisClient(TestCase):
def test_should_return_mock_if_redis_not_available_1(self):
# when
with patch(MODULE_PATH + ".get_redis_client") as mock_get_master_client:
mock_get_master_client.return_value.ping.side_effect = RedisError
result = get_redis_client_or_stub()
# then
self.assertIsInstance(result, _RedisStub)
def test_should_return_mock_if_redis_not_available_2(self):
# when
with patch(MODULE_PATH + ".get_redis_client") as mock_get_master_client:
mock_get_master_client.return_value.ping.return_value = False
result = get_redis_client_or_stub()
# then
self.assertIsInstance(result, _RedisStub)

View File

@@ -22,11 +22,12 @@ from allianceauth.eveonline.tasks import update_character
class TestTaskSignals(TestCase):
fixtures = ["disable_analytics"]
def test_should_record_successful_task(self):
# given
def setUp(self) -> None:
succeeded_tasks.clear()
retried_tasks.clear()
failed_tasks.clear()
def test_should_record_successful_task(self):
# when
with patch(
"allianceauth.eveonline.tasks.EveCharacter.objects.update_character"
@@ -39,10 +40,6 @@ class TestTaskSignals(TestCase):
self.assertEqual(failed_tasks.count(), 0)
def test_should_record_retried_task(self):
# given
succeeded_tasks.clear()
retried_tasks.clear()
failed_tasks.clear()
# when
with patch(
"allianceauth.eveonline.tasks.EveCharacter.objects.update_character"
@@ -55,10 +52,6 @@ class TestTaskSignals(TestCase):
self.assertEqual(retried_tasks.count(), 1)
def test_should_record_failed_task(self):
# given
succeeded_tasks.clear()
retried_tasks.clear()
failed_tasks.clear()
# when
with patch(
"allianceauth.eveonline.tasks.EveCharacter.objects.update_character"

View File

@@ -6,7 +6,7 @@
{% block page_title %}{% translate "Login" %}{% endblock %}
{% block middle_box_content %}
<a href="{% url 'auth_sso_login' %}{% if request.GET.next %}?next={{request.GET.next}}{%endif%}">
<a href="{% url 'auth_sso_login' %}{% if request.GET.next %}?next={{request.GET.next | urlencode}}{%endif%}">
<img class="img-responsive center-block" src="{% static 'allianceauth/authentication/img/sso/EVE_SSO_Login_Buttons_Large_Black.png' %}" alt="{% translate 'Login with Eve SSO' %}">
</a>
{% endblock %}

View File

@@ -4,16 +4,16 @@ from urllib import parse
from django.conf import settings
from django.contrib.auth.models import AnonymousUser
from django.http.response import HttpResponse
from django.shortcuts import reverse
from django.test import TestCase
from django.test.client import RequestFactory
from django.urls import reverse, URLPattern
from allianceauth.eveonline.models import EveCharacter
from allianceauth.tests.auth_utils import AuthUtils
from ..decorators import main_character_required
from ..models import CharacterOwnership
from ..decorators import decorate_url_patterns, main_character_required
from ..models import CharacterOwnership
MODULE_PATH = 'allianceauth.authentication'
@@ -66,3 +66,33 @@ class DecoratorTestCase(TestCase):
setattr(self.request, 'user', self.main_user)
response = self.dummy_view(self.request)
self.assertEqual(response.status_code, 200)
class TestDecorateUrlPatterns(TestCase):
def test_should_add_decorator_by_default(self):
# given
decorator = mock.MagicMock(name="decorator")
view = mock.MagicMock(name="view")
path = mock.MagicMock(spec=URLPattern, name="path")
path.callback = view
path.lookup_str = "my_lookup_str"
urls = [path]
urlconf_module = urls
# when
decorate_url_patterns(urlconf_module, decorator)
# then
self.assertEqual(path.callback, decorator(view))
def test_should_not_add_decorator_when_excluded(self):
# given
decorator = mock.MagicMock(name="decorator")
view = mock.MagicMock(name="view")
path = mock.MagicMock(spec=URLPattern, name="path")
path.callback = view
path.lookup_str = "my_lookup_str"
urls = [path]
urlconf_module = urls
# when
decorate_url_patterns(urlconf_module, decorator, excluded_views=["my_lookup_str"])
# then
self.assertEqual(path.callback, view)