Version Bump 2.11.1

Improve task statistics

See merge request allianceauth/allianceauth!1409

.gitignore

@@ -76,3 +76,4 @@ celerybeat-schedule
 .flake8
 .pylintrc
 Makefile
+.isort.cfg

allianceauth/__init__.py

@@ -1,7 +1,7 @@
 # This will make sure the app is always imported when
 # Django starts so that shared_task will use this app.
 
-__version__ = '2.10.1'
+__version__ = '2.11.1'
 __title__ = 'Alliance Auth'
 __url__ = 'https://gitlab.com/allianceauth/allianceauth'
 NAME = f'{__title__} v{__version__}'

allianceauth/authentication/admin.py

@@ -380,6 +380,7 @@ class UserAdmin(BaseUserAdmin):
         'username',
         'character_ownerships__character__character_name'
     )
+    readonly_fields = ('date_joined', 'last_login')
 
     def _characters(self, obj):
         character_ownerships = list(obj.character_ownerships.all())
@@ -425,10 +426,19 @@ class UserAdmin(BaseUserAdmin):
     def has_delete_permission(self, request, obj=None):
         return request.user.has_perm('auth.delete_user')
 
+    def get_object(self, *args , **kwargs):
+        obj = super().get_object(*args , **kwargs)
+        self.obj = obj  # storing current object for use in formfield_for_manytomany
+        return obj
+
     def formfield_for_manytomany(self, db_field, request, **kwargs):
-        """overriding this formfield to have sorted lists in the form"""
         if db_field.name == "groups":
-            kwargs["queryset"] = Group.objects.all().order_by(Lower('name'))
+            groups_qs = Group.objects.filter(authgroup__states__isnull=True)
+            obj_state = self.obj.profile.state
+            if obj_state:
+                matching_groups_qs = Group.objects.filter(authgroup__states=obj_state)
+                groups_qs = groups_qs | matching_groups_qs
+            kwargs["queryset"] = groups_qs.order_by(Lower('name'))
         return super().formfield_for_manytomany(db_field, request, **kwargs)
 
 

allianceauth/authentication/apps.py

@@ -3,10 +3,14 @@ from django.core.checks import register, Tags
 
 
 class AuthenticationConfig(AppConfig):
-    name = 'allianceauth.authentication'
-    label = 'authentication'
+    name = "allianceauth.authentication"
+    label = "authentication"
 
     def ready(self):
-        super().ready()
-        from allianceauth.authentication import checks, signals
+        from allianceauth.authentication import checks, signals  # noqa: F401
+        from allianceauth.authentication.task_statistics import (
+            signals as celery_signals,
+        )
+
         register(Tags.security)(checks.check_login_scopes_setting)
+        celery_signals.reset_counters()

allianceauth/authentication/task_statistics/counters.py

@@ -0,0 +1,40 @@
+from collections import namedtuple
+import datetime as dt
+
+from .event_series import EventSeries
+
+
+"""Global series for counting task events."""
+succeeded_tasks = EventSeries("SUCCEEDED_TASKS")
+retried_tasks = EventSeries("RETRIED_TASKS")
+failed_tasks = EventSeries("FAILED_TASKS")
+
+
+_TaskCounts = namedtuple(
+    "_TaskCounts", ["succeeded", "retried", "failed", "total", "earliest_task", "hours"]
+)
+
+
+def dashboard_results(hours: int) -> _TaskCounts:
+    """Counts of all task events within the given timeframe."""
+
+    def earliest_if_exists(events: EventSeries, earliest: dt.datetime) -> list:
+        my_earliest = events.first_event(earliest=earliest)
+        return [my_earliest] if my_earliest else []
+
+    earliest = dt.datetime.utcnow() - dt.timedelta(hours=hours)
+    earliest_events = list()
+    succeeded_count = succeeded_tasks.count(earliest=earliest)
+    earliest_events += earliest_if_exists(succeeded_tasks, earliest)
+    retried_count = retried_tasks.count(earliest=earliest)
+    earliest_events += earliest_if_exists(retried_tasks, earliest)
+    failed_count = failed_tasks.count(earliest=earliest)
+    earliest_events += earliest_if_exists(failed_tasks, earliest)
+    return _TaskCounts(
+        succeeded=succeeded_count,
+        retried=retried_count,
+        failed=failed_count,
+        total=succeeded_count + retried_count + failed_count,
+        earliest_task=min(earliest_events) if earliest_events else None,
+        hours=hours,
+    )

allianceauth/authentication/task_statistics/event_series.py

@@ -0,0 +1,95 @@
+import datetime as dt
+from typing import Optional, List
+
+from redis import Redis
+from pytz import utc
+
+from django.core.cache import cache
+
+
+class EventSeries:
+    """API for recording and analysing a series of events."""
+
+    _ROOT_KEY = "ALLIANCEAUTH_EVENT_SERIES"
+
+    def __init__(self, key_id: str, redis: Redis = None) -> None:
+        self._redis = cache.get_master_client() if not redis else redis
+        if not isinstance(self._redis, Redis):
+            raise TypeError(
+                "This class requires a Redis client, but none was provided "
+                "and the default Django cache backend is not Redis either."
+            )
+        self._key_id = str(key_id)
+        self.clear()
+
+    @property
+    def _key_counter(self):
+        return f"{self._ROOT_KEY}_{self._key_id}_COUNTER"
+
+    @property
+    def _key_sorted_set(self):
+        return f"{self._ROOT_KEY}_{self._key_id}_SORTED_SET"
+
+    def add(self, event_time: dt.datetime = None) -> None:
+        """Add event.
+
+        Args:
+        - event_time: timestamp of event. Will use current time if not specified.
+        """
+        if not event_time:
+            event_time = dt.datetime.utcnow()
+        id = self._redis.incr(self._key_counter)
+        self._redis.zadd(self._key_sorted_set, {id: event_time.timestamp()})
+
+    def all(self) -> List[dt.datetime]:
+        """List of all known events."""
+        return [
+            event[1]
+            for event in self._redis.zrangebyscore(
+                self._key_sorted_set,
+                "-inf",
+                "+inf",
+                withscores=True,
+                score_cast_func=self._cast_scores_to_dt,
+            )
+        ]
+
+    def clear(self) -> None:
+        """Clear all events."""
+        self._redis.delete(self._key_sorted_set)
+        self._redis.delete(self._key_counter)
+
+    def count(self, earliest: dt.datetime = None, latest: dt.datetime = None) -> int:
+        """Count of events, can be restricted to given timeframe.
+
+        Args:
+        - earliest: Date of first events to count(inclusive), or -infinite if not specified
+        - latest: Date of last events to count(inclusive), or +infinite if not specified
+        """
+        min = "-inf" if not earliest else earliest.timestamp()
+        max = "+inf" if not latest else latest.timestamp()
+        return self._redis.zcount(self._key_sorted_set, min=min, max=max)
+
+    def first_event(self, earliest: dt.datetime = None) -> Optional[dt.datetime]:
+        """Date/Time of first event. Returns `None` if series has no events.
+
+        Args:
+        - earliest: Date of first events to count(inclusive), or any if not specified
+        """
+        min = "-inf" if not earliest else earliest.timestamp()
+        event = self._redis.zrangebyscore(
+            self._key_sorted_set,
+            min,
+            "+inf",
+            withscores=True,
+            start=0,
+            num=1,
+            score_cast_func=self._cast_scores_to_dt,
+        )
+        if not event:
+            return None
+        return event[0][1]
+
+    @staticmethod
+    def _cast_scores_to_dt(score) -> dt.datetime:
+        return dt.datetime.fromtimestamp(float(score), tz=utc)

allianceauth/authentication/task_statistics/signals.py

@@ -0,0 +1,54 @@
+from celery.signals import (
+    task_failure,
+    task_internal_error,
+    task_retry,
+    task_success,
+    worker_ready
+)
+
+from django.conf import settings
+
+from .counters import failed_tasks, retried_tasks, succeeded_tasks
+
+
+def reset_counters():
+    """Reset all counters for the celery status."""
+    succeeded_tasks.clear()
+    failed_tasks.clear()
+    retried_tasks.clear()
+
+
+def is_enabled() -> bool:
+    return not bool(
+        getattr(settings, "ALLIANCEAUTH_DASHBOARD_TASK_STATISTICS_DISABLED", False)
+    )
+
+
+@worker_ready.connect
+def reset_counters_when_celery_restarted(*args, **kwargs):
+    if is_enabled():
+        reset_counters()
+
+
+@task_success.connect
+def record_task_succeeded(*args, **kwargs):
+    if is_enabled():
+        succeeded_tasks.add()
+
+
+@task_retry.connect
+def record_task_retried(*args, **kwargs):
+    if is_enabled():
+        retried_tasks.add()
+
+
+@task_failure.connect
+def record_task_failed(*args, **kwargs):
+    if is_enabled():
+        failed_tasks.add()
+
+
+@task_internal_error.connect
+def record_task_internal_error(*args, **kwargs):
+    if is_enabled():
+        failed_tasks.add()

allianceauth/authentication/task_statistics/tests/test_counters.py

@@ -0,0 +1,51 @@
+import datetime as dt
+
+from django.test import TestCase
+from django.utils.timezone import now
+
+from allianceauth.authentication.task_statistics.counters import (
+    dashboard_results,
+    succeeded_tasks,
+    retried_tasks,
+    failed_tasks,
+)
+
+
+class TestDashboardResults(TestCase):
+    def test_should_return_counts_for_given_timeframe_only(self):
+        # given
+        earliest_task = now() - dt.timedelta(minutes=15)
+        succeeded_tasks.clear()
+        succeeded_tasks.add(now() - dt.timedelta(hours=1, seconds=1))
+        succeeded_tasks.add(earliest_task)
+        succeeded_tasks.add()
+        succeeded_tasks.add()
+        retried_tasks.clear()
+        retried_tasks.add(now() - dt.timedelta(hours=1, seconds=1))
+        retried_tasks.add(now() - dt.timedelta(seconds=30))
+        retried_tasks.add()
+        failed_tasks.clear()
+        failed_tasks.add(now() - dt.timedelta(hours=1, seconds=1))
+        failed_tasks.add()
+        # when
+        results = dashboard_results(hours=1)
+        # then
+        self.assertEqual(results.succeeded, 3)
+        self.assertEqual(results.retried, 2)
+        self.assertEqual(results.failed, 1)
+        self.assertEqual(results.total, 6)
+        self.assertEqual(results.earliest_task, earliest_task)
+
+    def test_should_work_with_no_data(self):
+        # given
+        succeeded_tasks.clear()
+        retried_tasks.clear()
+        failed_tasks.clear()
+        # when
+        results = dashboard_results(hours=1)
+        # then
+        self.assertEqual(results.succeeded, 0)
+        self.assertEqual(results.retried, 0)
+        self.assertEqual(results.failed, 0)
+        self.assertEqual(results.total, 0)
+        self.assertIsNone(results.earliest_task)

allianceauth/authentication/task_statistics/tests/test_event_series.py

@@ -0,0 +1,133 @@
+import datetime as dt
+
+from pytz import utc
+from django.test import TestCase
+from django.utils.timezone import now
+
+from allianceauth.authentication.task_statistics.event_series import EventSeries
+
+
+class TestEventSeries(TestCase):
+    def test_should_add_event(self):
+        # given
+        events = EventSeries("dummy")
+        # when
+        events.add()
+        # then
+        result = events.all()
+        self.assertEqual(len(result), 1)
+        self.assertAlmostEqual(result[0], now(), delta=dt.timedelta(seconds=30))
+
+    def test_should_add_event_with_specified_time(self):
+        # given
+        events = EventSeries("dummy")
+        my_time = dt.datetime(2021, 11, 1, 12, 15, tzinfo=utc)
+        # when
+        events.add(my_time)
+        # then
+        result = events.all()
+        self.assertEqual(len(result), 1)
+        self.assertAlmostEqual(result[0], my_time, delta=dt.timedelta(seconds=30))
+
+    def test_should_count_events(self):
+        # given
+        events = EventSeries("dummy")
+        events.add()
+        events.add()
+        # when
+        result = events.count()
+        # then
+        self.assertEqual(result, 2)
+
+    def test_should_count_zero(self):
+        # given
+        events = EventSeries("dummy")
+        # when
+        result = events.count()
+        # then
+        self.assertEqual(result, 0)
+
+    def test_should_count_events_within_timeframe_1(self):
+        # given
+        events = EventSeries("dummy")
+        events.add(dt.datetime(2021, 12, 1, 12, 0, tzinfo=utc))
+        events.add(dt.datetime(2021, 12, 1, 12, 10, tzinfo=utc))
+        events.add(dt.datetime(2021, 12, 1, 12, 15, tzinfo=utc))
+        events.add(dt.datetime(2021, 12, 1, 12, 30, tzinfo=utc))
+        # when
+        result = events.count(
+            earliest=dt.datetime(2021, 12, 1, 12, 8, tzinfo=utc),
+            latest=dt.datetime(2021, 12, 1, 12, 17, tzinfo=utc),
+        )
+        # then
+        self.assertEqual(result, 2)
+
+    def test_should_count_events_within_timeframe_2(self):
+        # given
+        events = EventSeries("dummy")
+        events.add(dt.datetime(2021, 12, 1, 12, 0, tzinfo=utc))
+        events.add(dt.datetime(2021, 12, 1, 12, 10, tzinfo=utc))
+        events.add(dt.datetime(2021, 12, 1, 12, 15, tzinfo=utc))
+        events.add(dt.datetime(2021, 12, 1, 12, 30, tzinfo=utc))
+        # when
+        result = events.count(earliest=dt.datetime(2021, 12, 1, 12, 8))
+        # then
+        self.assertEqual(result, 3)
+
+    def test_should_count_events_within_timeframe_3(self):
+        # given
+        events = EventSeries("dummy")
+        events.add(dt.datetime(2021, 12, 1, 12, 0, tzinfo=utc))
+        events.add(dt.datetime(2021, 12, 1, 12, 10, tzinfo=utc))
+        events.add(dt.datetime(2021, 12, 1, 12, 15, tzinfo=utc))
+        events.add(dt.datetime(2021, 12, 1, 12, 30, tzinfo=utc))
+        # when
+        result = events.count(latest=dt.datetime(2021, 12, 1, 12, 12))
+        # then
+        self.assertEqual(result, 2)
+
+    def test_should_clear_events(self):
+        # given
+        events = EventSeries("dummy")
+        events.add()
+        events.add()
+        # when
+        events.clear()
+        # then
+        self.assertEqual(events.count(), 0)
+
+    def test_should_return_date_of_first_event(self):
+        # given
+        events = EventSeries("dummy")
+        events.add(dt.datetime(2021, 12, 1, 12, 0, tzinfo=utc))
+        events.add(dt.datetime(2021, 12, 1, 12, 10, tzinfo=utc))
+        events.add(dt.datetime(2021, 12, 1, 12, 15, tzinfo=utc))
+        events.add(dt.datetime(2021, 12, 1, 12, 30, tzinfo=utc))
+        # when
+        result = events.first_event()
+        # then
+        self.assertEqual(result, dt.datetime(2021, 12, 1, 12, 0, tzinfo=utc))
+
+    def test_should_return_date_of_first_event_with_range(self):
+        # given
+        events = EventSeries("dummy")
+        events.add(dt.datetime(2021, 12, 1, 12, 0, tzinfo=utc))
+        events.add(dt.datetime(2021, 12, 1, 12, 10, tzinfo=utc))
+        events.add(dt.datetime(2021, 12, 1, 12, 15, tzinfo=utc))
+        events.add(dt.datetime(2021, 12, 1, 12, 30, tzinfo=utc))
+        # when
+        result = events.first_event(
+            earliest=dt.datetime(2021, 12, 1, 12, 8, tzinfo=utc)
+        )
+        # then
+        self.assertEqual(result, dt.datetime(2021, 12, 1, 12, 10, tzinfo=utc))
+
+    def test_should_return_all_events(self):
+        # given
+        events = EventSeries("dummy")
+        events.add()
+        events.add()
+        # when
+        results = events.all()
+        # then
+        self.assertEqual(len(results), 2)

allianceauth/authentication/task_statistics/tests/test_signals.py

@@ -0,0 +1,93 @@
+from unittest.mock import patch
+
+from celery.exceptions import Retry
+
+from django.test import TestCase, override_settings
+
+from allianceauth.authentication.task_statistics.counters import (
+    failed_tasks,
+    retried_tasks,
+    succeeded_tasks,
+)
+from allianceauth.authentication.task_statistics.signals import (
+    reset_counters,
+    is_enabled,
+)
+from allianceauth.eveonline.tasks import update_character
+
+
+@override_settings(
+    CELERY_ALWAYS_EAGER=True,ALLIANCEAUTH_DASHBOARD_TASK_STATISTICS_DISABLED=False
+)
+class TestTaskSignals(TestCase):
+    fixtures = ["disable_analytics"]
+
+    def test_should_record_successful_task(self):
+        # given
+        succeeded_tasks.clear()
+        retried_tasks.clear()
+        failed_tasks.clear()
+        # when
+        with patch(
+            "allianceauth.eveonline.tasks.EveCharacter.objects.update_character"
+        ) as mock_update:
+            mock_update.return_value = None
+            update_character.delay(1)
+        # then
+        self.assertEqual(succeeded_tasks.count(), 1)
+        self.assertEqual(retried_tasks.count(), 0)
+        self.assertEqual(failed_tasks.count(), 0)
+
+    def test_should_record_retried_task(self):
+        # given
+        succeeded_tasks.clear()
+        retried_tasks.clear()
+        failed_tasks.clear()
+        # when
+        with patch(
+            "allianceauth.eveonline.tasks.EveCharacter.objects.update_character"
+        ) as mock_update:
+            mock_update.side_effect = Retry
+            update_character.delay(1)
+        # then
+        self.assertEqual(succeeded_tasks.count(), 0)
+        self.assertEqual(failed_tasks.count(), 0)
+        self.assertEqual(retried_tasks.count(), 1)
+
+    def test_should_record_failed_task(self):
+        # given
+        succeeded_tasks.clear()
+        retried_tasks.clear()
+        failed_tasks.clear()
+        # when
+        with patch(
+            "allianceauth.eveonline.tasks.EveCharacter.objects.update_character"
+        ) as mock_update:
+            mock_update.side_effect = RuntimeError
+            update_character.delay(1)
+        # then
+        self.assertEqual(succeeded_tasks.count(), 0)
+        self.assertEqual(retried_tasks.count(), 0)
+        self.assertEqual(failed_tasks.count(), 1)
+
+    def test_should_reset_counters(self):
+        # given
+        succeeded_tasks.add()
+        retried_tasks.add()
+        failed_tasks.add()
+        # when
+        reset_counters()
+        # then
+        self.assertEqual(succeeded_tasks.count(), 0)
+        self.assertEqual(retried_tasks.count(), 0)
+        self.assertEqual(failed_tasks.count(), 0)
+
+
+class TestIsEnabled(TestCase):
+    @override_settings(ALLIANCEAUTH_DASHBOARD_TASK_STATISTICS_DISABLED=False)
+    def test_enabled(self):
+        self.assertTrue(is_enabled())
+
+    @override_settings(ALLIANCEAUTH_DASHBOARD_TASK_STATISTICS_DISABLED=True)
+    def test_disabled(self):
+        self.assertFalse(is_enabled())

allianceauth/authentication/tests/test_admin.py

@@ -1,3 +1,4 @@
+from bs4 import BeautifulSoup
 from urllib.parse import quote
 from unittest.mock import patch, MagicMock
 
@@ -188,7 +189,7 @@ class TestCaseWithTestData(TestCase):
             corporation_id=5432,
             corporation_name="Xavier's School for Gifted Youngsters",
             corporation_ticker='MUTNT',
-            alliance_id = None,
+            alliance_id=None,
             faction_id=999,
             faction_name='The X-Men',
         )
@@ -206,6 +207,7 @@ class TestCaseWithTestData(TestCase):
         cls.user_4.profile.save()
         EveFactionInfo.objects.create(faction_id=999, faction_name='The X-Men')
 
+
 def make_generic_search_request(ModelClass: type, search_term: str):
     User.objects.create_superuser(
         username='superuser', password='secret', email='admin@example.com'
@@ -218,6 +220,7 @@ def make_generic_search_request(ModelClass: type, search_term: str):
 
 
 class TestCharacterOwnershipAdmin(TestCaseWithTestData):
+    fixtures = ["disable_analytics"]
 
     def setUp(self):
         self.modeladmin = CharacterOwnershipAdmin(
@@ -244,6 +247,7 @@ class TestCharacterOwnershipAdmin(TestCaseWithTestData):
 
 
 class TestOwnershipRecordAdmin(TestCaseWithTestData):
+    fixtures = ["disable_analytics"]
 
     def setUp(self):
         self.modeladmin = OwnershipRecordAdmin(
@@ -270,6 +274,7 @@ class TestOwnershipRecordAdmin(TestCaseWithTestData):
 
 
 class TestStateAdmin(TestCaseWithTestData):
+    fixtures = ["disable_analytics"]
 
     def setUp(self):
         self.modeladmin = StateAdmin(
@@ -299,6 +304,7 @@ class TestStateAdmin(TestCaseWithTestData):
 
 
 class TestUserAdmin(TestCaseWithTestData):
+    fixtures = ["disable_analytics"]
 
     def setUp(self):
         self.factory = RequestFactory()
@@ -344,7 +350,7 @@ class TestUserAdmin(TestCaseWithTestData):
         self.assertEqual(user_main_organization(self.user_3), expected)
 
     def test_user_main_organization_u4(self):
-        expected="Xavier's School for Gifted Youngsters<br>The X-Men"
+        expected = "Xavier's School for Gifted Youngsters<br>The X-Men"
         self.assertEqual(user_main_organization(self.user_4), expected)
 
     def test_characters_u1(self):
@@ -537,6 +543,42 @@ class TestUserAdmin(TestCaseWithTestData):
         self.assertEqual(response.status_code, expected)
 
 
+class TestUserAdminChangeForm(TestCase):
+    @classmethod
+    def setUpClass(cls) -> None:
+        super().setUpClass()
+        cls.modeladmin = UserAdmin(model=User, admin_site=AdminSite())
+
+    def test_should_show_groups_available_to_user_with_blue_state_only(self):
+        # given
+        superuser = User.objects.create_superuser("Super")
+        user = AuthUtils.create_user("Bruce Wayne")
+        character = AuthUtils.add_main_character_2(
+            user,
+            name="Bruce Wayne",
+            character_id=1001,
+            corp_id=2001,
+            corp_name="Wayne Technologies"
+        )
+        blue_state = State.objects.get(name="Blue")
+        blue_state.member_characters.add(character)
+        member_state = AuthUtils.get_member_state()
+        group_1 = Group.objects.create(name="Group 1")
+        group_2 = Group.objects.create(name="Group 2")
+        group_2.authgroup.states.add(blue_state)
+        group_3 = Group.objects.create(name="Group 3")
+        group_3.authgroup.states.add(member_state)
+        self.client.force_login(superuser)
+        # when
+        response = self.client.get(f"/admin/authentication/user/{user.pk}/change/")
+        # then
+        self.assertEqual(response.status_code, 200)
+        soup = BeautifulSoup(response.rendered_content, features="html.parser")
+        groups_select = soup.find("select", {"id": "id_groups"}).find_all('option')
+        group_ids = {int(option["value"]) for option in groups_select}
+        self.assertSetEqual(group_ids, {group_1.pk, group_2.pk})
+
+
 class TestMakeServicesHooksActions(TestCaseWithTestData):
 
     class MyServicesHookTypeA(ServicesHook):

allianceauth/authentication/tests/test_templatetags.py

@@ -55,7 +55,6 @@ TEST_VERSION = '2.6.5'
 
 
 class TestStatusOverviewTag(TestCase):
-
     @patch(MODULE_PATH + '.admin_status.__version__', TEST_VERSION)
     @patch(MODULE_PATH + '.admin_status._fetch_celery_queue_length')
     @patch(MODULE_PATH + '.admin_status._current_version_summary')
@@ -66,6 +65,7 @@ class TestStatusOverviewTag(TestCase):
         mock_current_version_info,
         mock_fetch_celery_queue_length
     ):
+        # given
         notifications = {
             'notifications': GITHUB_NOTIFICATION_ISSUES[:5]
         }
@@ -83,22 +83,20 @@ class TestStatusOverviewTag(TestCase):
         }
         mock_current_version_info.return_value = version_info
         mock_fetch_celery_queue_length.return_value = 3
-
+        # when
         result = status_overview()
-        expected = {
-            'notifications': GITHUB_NOTIFICATION_ISSUES[:5],
-            'latest_major': True,
-            'latest_minor': True,
-            'latest_patch': True,
-            'latest_beta': False,
-            'current_version': TEST_VERSION,
-            'latest_major_version': '2.4.5',
-            'latest_minor_version': '2.4.0',
-            'latest_patch_version': '2.4.5',
-            'latest_beta_version': '2.4.4a1',
-            'task_queue_length': 3,
-        }
-        self.assertEqual(result, expected)
+        # then
+        self.assertEqual(result["notifications"], GITHUB_NOTIFICATION_ISSUES[:5])
+        self.assertTrue(result["latest_major"])
+        self.assertTrue(result["latest_minor"])
+        self.assertTrue(result["latest_patch"])
+        self.assertFalse(result["latest_beta"])
+        self.assertEqual(result["current_version"], TEST_VERSION)
+        self.assertEqual(result["latest_major_version"], '2.4.5')
+        self.assertEqual(result["latest_minor_version"], '2.4.0')
+        self.assertEqual(result["latest_patch_version"], '2.4.5')
+        self.assertEqual(result["latest_beta_version"], '2.4.4a1')
+        self.assertEqual(result["task_queue_length"], 3)
 
 
 class TestNotifications(TestCase):

allianceauth/static/css/auth-base.css

@@ -95,6 +95,11 @@ ul.list-group.list-group-horizontal > li.list-group-item {
     .table-aa > tbody > tr:last-child {
         border-bottom: none;
     }
+
+    .task-status-progress-bar {
+        font-size: 15px!important;
+        line-height: normal!important;
+    }
 }
 
 /* highlight active menu items

allianceauth/templates/allianceauth/admin-status/celery_bar_partial.html

@@ -0,0 +1,11 @@
+{% load humanize %}
+
+<div
+    class="progress-bar progress-bar-{{ level }} task-status-progress-bar"
+    role="progressbar"
+    aria-valuenow="{% widthratio tasks_count tasks_total 100 %}"
+    aria-valuemin="0"
+    aria-valuemax="100"
+    style="width: {% widthratio tasks_count tasks_total 100 %}%;">
+    {% widthratio tasks_count tasks_total 100 %}%
+</div>

allianceauth/templates/allianceauth/admin-status/overview.html

@@ -1,4 +1,6 @@
 {% load i18n %}
+{% load humanize %}
+
 <div class="col-sm-12">
     <div class="row vertical-flexbox-row2">
         <div class="col-sm-6">
@@ -75,29 +77,25 @@
             <div class="panel panel-primary" style="height:50%;">
                 <div class="panel-heading text-center"><h3 class="panel-title">{% translate "Task Queue" %}</h3></div>
                 <div class="panel-body flex-center-horizontal">
-                    <div class="progress" style="height: 21px;">
-                        <div class="progress-bar
-                            {% if task_queue_length > 500 %}
-                                progress-bar-danger
-                            {% elif task_queue_length > 100 %}
-                                progress-bar-warning
-                            {% else %}
-                                progress-bar-success
-                            {% endif %}
-                            " role="progressbar" aria-valuenow="{% widthratio task_queue_length 500 100 %}"
-                                aria-valuemin="0" aria-valuemax="100"
-                                style="width: {% widthratio task_queue_length 500 100 %}%;">
-                        </div>
+                    <p>
+                    {% blocktranslate with total=tasks_total|intcomma latest=earliest_task|timesince|default:"?" %}
+                    Status of {{ total }} processed tasks • last {{ latest }}
+                    {% endblocktranslate %}
+                    </p>
+                    <div
+                        class="progress"
+                        style="height: 21px;"
+                        title="{{ tasks_succeeded|intcomma }} succeeded, {{ tasks_retried|intcomma }} retried, {{ tasks_failed|intcomma }} failed"
+                    >
+                        {% include "allianceauth/admin-status/celery_bar_partial.html" with label="suceeded" level="success" tasks_count=tasks_succeeded %}
+                        {% include "allianceauth/admin-status/celery_bar_partial.html" with label="retried" level="info" tasks_count=tasks_retried %}
+                        {% include "allianceauth/admin-status/celery_bar_partial.html" with label="failed" level="danger" tasks_count=tasks_failed %}
                     </div>
-                    {% if task_queue_length < 0 %}
-                        {% translate "Error retrieving task queue length" %}
-                    {% else %}
-                        {% blocktrans trimmed count tasks=task_queue_length %}
-                        {{ tasks }} task
-                        {% plural %}
-                        {{ tasks }} tasks
-                        {% endblocktrans %}
-                    {% endif %}
+                    <p>
+                        {% blocktranslate with queue_length=task_queue_length|default_if_none:"?"|intcomma %}
+                        {{ queue_length }} queued tasks
+                        {% endblocktranslate %}
+                    </p>
                 </div>
             </div>
         </div>

allianceauth/templatetags/admin_status.py

@@ -1,9 +1,11 @@
 import logging
+from typing import Optional
 
-import requests
 import amqp.exceptions
-from packaging.version import Version as Pep440Version, InvalidVersion
+import requests
 from celery.app import app_or_default
+from packaging.version import InvalidVersion
+from packaging.version import Version as Pep440Version
 
 from django import template
 from django.conf import settings
@@ -11,6 +13,7 @@ from django.core.cache import cache
 
 from allianceauth import __version__
 
+from ..authentication.task_statistics.counters import dashboard_results
 
 register = template.Library()
 
@@ -36,30 +39,51 @@ logger = logging.getLogger(__name__)
 @register.inclusion_tag('allianceauth/admin-status/overview.html')
 def status_overview() -> dict:
     response = {
-        'notifications': list(),
-        'current_version': __version__,
-        'task_queue_length': -1,
+        "notifications": list(),
+        "current_version": __version__,
+        "task_queue_length": None,
+        "tasks_succeeded": 0,
+        "tasks_retried": 0,
+        "tasks_failed": 0,
+        "tasks_total": 0,
+        "tasks_hours": 0,
+        "earliest_task": None
     }
     response.update(_current_notifications())
     response.update(_current_version_summary())
     response.update({'task_queue_length': _fetch_celery_queue_length()})
+    response.update(_celery_stats())
     return response
 
 
-def _fetch_celery_queue_length() -> int:
+def _celery_stats() -> dict:
+    hours = getattr(settings, "ALLIANCEAUTH_DASHBOARD_TASKS_MAX_HOURS", 24)
+    results = dashboard_results(hours=hours)
+    return {
+        "tasks_succeeded": results.succeeded,
+        "tasks_retried": results.retried,
+        "tasks_failed": results.failed,
+        "tasks_total": results.total,
+        "tasks_hours": results.hours,
+        "earliest_task": results.earliest_task
+    }
+
+
+def _fetch_celery_queue_length() -> Optional[int]:
     try:
         app = app_or_default(None)
         with app.connection_or_acquire() as conn:
-            return conn.default_channel.queue_declare(
+            result = conn.default_channel.queue_declare(
                 queue=getattr(settings, 'CELERY_DEFAULT_QUEUE', 'celery'),
                 passive=True
-            ).message_count
+            )
+            return result.message_count
     except amqp.exceptions.ChannelError:
         # Queue doesn't exist, probably empty
         return 0
     except Exception:
         logger.exception("Failed to get celery queue length")
-    return -1
+    return None
 
 
 def _current_notifications() -> dict:

allianceauth/tests/auth_utils.py

@@ -174,7 +174,7 @@ class AuthUtils:
         alliance_id=None,
         alliance_name='',
         disconnect_signals=False
-    ):
+    ) -> EveCharacter:
         """new version that works in all cases"""
         if disconnect_signals:
             cls.disconnect_signals()

docker/.env.example

@@ -1,7 +1,7 @@
 PROTOCOL=https://
 AUTH_SUBDOMAIN=%AUTH_SUBDOMAIN%
 DOMAIN=%DOMAIN%
-AA_DOCKER_TAG=registry.gitlab.com/allianceauth/allianceauth/auth:v2.10
+AA_DOCKER_TAG=registry.gitlab.com/allianceauth/allianceauth/auth:v2.11
 
 # Nginx Proxy Manager
 PROXY_HTTP_PORT=80

docker/Dockerfile

@@ -1,5 +1,5 @@
 FROM python:3.9-slim
-ARG AUTH_VERSION=2.10.0
+ARG AUTH_VERSION=2.11.1
 ARG AUTH_PACKAGE=allianceauth==${AUTH_VERSION}
 ENV VIRTUAL_ENV=/opt/venv
 ENV AUTH_USER=allianceauth
@@ -39,7 +39,6 @@ RUN allianceauth start myauth
 COPY /allianceauth/project_template/project_name/settings/local.py ${AUTH_HOME}/myauth/myauth/settings/local.py
 RUN allianceauth update myauth
 RUN mkdir -p ${STATIC_BASE}/myauth/static
-RUN python ${AUTH_HOME}/myauth/manage.py collectstatic --noinput
 COPY /docker/conf/supervisord.conf /etc/supervisor/conf.d/supervisord.conf
 RUN echo 'alias auth="python $AUTH_HOME/myauth/manage.py"' >> ~/.bashrc && \
     echo 'alias supervisord="supervisord -c /etc/supervisor/conf.d/supervisord.conf"' >> ~/.bashrc && \

docker/README.md

@@ -8,7 +8,7 @@ You should have the following available on the system you are using to set this
 
 ## Setup Guide
 
-1. run `bash <(curl -s https://gitlab.com/allianceauth/allianceauth/-/raw/v2.9.x/docker/scripts/download.sh)`. This will download all the files you need to install auth and place them in a directory named `aa-docker`. Feel free to rename/move this folder.
+1. run `bash <(curl -s https://gitlab.com/allianceauth/allianceauth/-/raw/v2.11.x/docker/scripts/download.sh)`. This will download all the files you need to install auth and place them in a directory named `aa-docker`. Feel free to rename/move this folder.
 1. run `./scripts/prepare-env.sh` to set up your environment
 1. (optional) Change `PROTOCOL` to `http://` if not using SSL in `.env`
 1. run `docker-compose --env-file=.env up -d` (NOTE: if this command hangs, follow the instructions [here](https://www.digitalocean.com/community/tutorials/how-to-setup-additional-entropy-for-cloud-servers-using-haveged))

docs/features/core/dashboard.md

@@ -7,3 +7,18 @@ The content of the dashboard is specific to the logged in user. It has a sidebar
 For admin users the dashboard shows additional technical information about the  AA instance.
 
 ![dashboard](/_static/images/features/core/dashboard/dashboard.png)
+
+## Settings
+
+Here is a list of available settings for the dashboard. They can be configured by adding them to your AA settings file (``local.py``).
+Note that all settings are optional and the app will use the documented default settings if they are not used.
+
+```eval_rst
++-----------------------------------------------------+-------------------------------------------------------------------------+-----------+
+| Name                                                | Description                                                             | Default   |
++=====================================================+=========================================================================+===========+
+| ``ALLIANCEAUTH_DASHBOARD_TASKS_MAX_HOURS``          | Statistics will be calculated for task events not older than max hours. | ``24``    |
++-----------------------------------------------------+-------------------------------------------------------------------------+-----------+
+| ``ALLIANCEAUTH_DASHBOARD_TASK_STATISTICS_DISABLED`` | Disables recording of task statistics. Used mainly in development.      | ``False`` |
++-----------------------------------------------------+-------------------------------------------------------------------------+-----------+
+```

tests/settings_all.py

@@ -154,3 +154,5 @@ PASSWORD_HASHERS = [
 ]
 
 LOGGING = None  # Comment out to enable logging for debugging
+
+ALLIANCEAUTH_DASHBOARD_TASK_STATISTICS_DISABLED = True  # disable for tests

tests/settings_core.py

@@ -27,3 +27,5 @@ PASSWORD_HASHERS = [
 ]
 
 LOGGING = None  # Comment out to enable logging for debugging
+
+ALLIANCEAUTH_DASHBOARD_TASK_STATISTICS_DISABLED = True  # disable for tests