diff --git a/allianceauth/__init__.py b/allianceauth/__init__.py index c846863b..b403718e 100644 --- a/allianceauth/__init__.py +++ b/allianceauth/__init__.py @@ -5,7 +5,7 @@ manage online service access. # This will make sure the app is always imported when # Django starts so that shared_task will use this app. -__version__ = '3.6.1' +__version__ = '3.7.0' __title__ = 'Alliance Auth' __url__ = 'https://gitlab.com/allianceauth/allianceauth' NAME = f'{__title__} v{__version__}' diff --git a/allianceauth/authentication/backends.py b/allianceauth/authentication/backends.py index 94ac066b..f7b2038f 100644 --- a/allianceauth/authentication/backends.py +++ b/allianceauth/authentication/backends.py @@ -65,7 +65,7 @@ class StateBackend(ModelBackend): # we've seen this character owner before. Re-attach to their old user account user = records[0].user if user.profile.main_character: - if ownership.user.profile.main_character.character_id != token.character_id: + if user.profile.main_character.character_id != token.character_id: ## this is an alt, enforce main only due to trust issues in SSO. if request: messages.error("Unable to authenticate with this Character, Please log in with the main character associated with this account. Then add this character from the dashboard.") diff --git a/allianceauth/authentication/core/__init__.py b/allianceauth/authentication/core/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/allianceauth/authentication/core/celery_workers.py b/allianceauth/authentication/core/celery_workers.py new file mode 100644 index 00000000..6da60924 --- /dev/null +++ b/allianceauth/authentication/core/celery_workers.py @@ -0,0 +1,48 @@ +"""API for interacting with celery workers.""" + +import itertools +import logging +from typing import Optional + +from amqp.exceptions import ChannelError +from celery import current_app + +from django.conf import settings + +logger = logging.getLogger(__name__) + + +def active_tasks_count() -> Optional[int]: + """Return count of currently active tasks + or None if celery workers are not online. + """ + inspect = current_app.control.inspect() + return _tasks_count(inspect.active()) + + +def _tasks_count(data: dict) -> Optional[int]: + """Return count of tasks in data from celery inspect API.""" + try: + tasks = itertools.chain(*data.values()) + except AttributeError: + return None + return len(list(tasks)) + + +def queued_tasks_count() -> Optional[int]: + """Return count of queued tasks. Return None if there was an error.""" + try: + with current_app.connection_or_acquire() as conn: + result = conn.default_channel.queue_declare( + queue=getattr(settings, "CELERY_DEFAULT_QUEUE", "celery"), passive=True + ) + return result.message_count + + except ChannelError: + # Queue doesn't exist, probably empty + return 0 + + except Exception: + logger.exception("Failed to get celery queue length") + + return None diff --git a/allianceauth/authentication/task_statistics/counters.py b/allianceauth/authentication/task_statistics/counters.py index 06e2af83..bdb6d034 100644 --- a/allianceauth/authentication/task_statistics/counters.py +++ b/allianceauth/authentication/task_statistics/counters.py @@ -4,13 +4,11 @@ import datetime as dt from typing import NamedTuple, Optional from .event_series import EventSeries -from .helpers import ItemCounter # Global series for counting task events. succeeded_tasks = EventSeries("SUCCEEDED_TASKS") retried_tasks = EventSeries("RETRIED_TASKS") failed_tasks = EventSeries("FAILED_TASKS") -running_tasks = ItemCounter("running_tasks") class _TaskCounts(NamedTuple): @@ -20,7 +18,6 @@ class _TaskCounts(NamedTuple): total: int earliest_task: Optional[dt.datetime] hours: int - running: int def dashboard_results(hours: int) -> _TaskCounts: @@ -38,7 +35,6 @@ def dashboard_results(hours: int) -> _TaskCounts: earliest_events += earliest_if_exists(retried_tasks, earliest) failed_count = failed_tasks.count(earliest=earliest) earliest_events += earliest_if_exists(failed_tasks, earliest) - running_count = running_tasks.value() return _TaskCounts( succeeded=succeeded_count, retried=retried_count, @@ -46,5 +42,4 @@ def dashboard_results(hours: int) -> _TaskCounts: total=succeeded_count + retried_count + failed_count, earliest_task=min(earliest_events) if earliest_events else None, hours=hours, - running=running_count, ) diff --git a/allianceauth/authentication/task_statistics/helpers.py b/allianceauth/authentication/task_statistics/helpers.py index b75fb39c..464cee8f 100644 --- a/allianceauth/authentication/task_statistics/helpers.py +++ b/allianceauth/authentication/task_statistics/helpers.py @@ -1,12 +1,9 @@ """Helpers for Task Statistics.""" import logging -from typing import Optional from redis import Redis, RedisError -from django.core.cache import cache - from allianceauth.utils.cache import get_redis_client logger = logging.getLogger(__name__) @@ -37,62 +34,6 @@ class _RedisStub: pass -class ItemCounter: - """A process safe item counter. - - Args: - - name: Unique name for the counter - - minimum: Counter can not go below the minimum, when set - - redis: A Redis client. Will use AA's cache client by default - """ - - CACHE_KEY_BASE = "allianceauth-item-counter" - DEFAULT_CACHE_TIMEOUT = 24 * 3600 - - def __init__( - self, name: str, minimum: Optional[int] = None, redis: Optional[Redis] = None - ) -> None: - if not name: - raise ValueError("Must define a name") - - self._name = str(name) - self._minimum = minimum - self._redis = get_redis_client_or_stub() if not redis else redis - - @property - def _cache_key(self) -> str: - return f"{self.CACHE_KEY_BASE}-{self._name}" - - def reset(self, init_value: int = 0): - """Reset counter to initial value.""" - with self._redis.lock(f"{self.CACHE_KEY_BASE}-reset"): - if self._minimum is not None and init_value < self._minimum: - raise ValueError("Can not reset below minimum") - - cache.set(self._cache_key, init_value, self.DEFAULT_CACHE_TIMEOUT) - - def incr(self, delta: int = 1): - """Increment counter by delta.""" - try: - cache.incr(self._cache_key, delta) - except ValueError: - pass - - def decr(self, delta: int = 1): - """Decrement counter by delta.""" - with self._redis.lock(f"{self.CACHE_KEY_BASE}-decr"): - if self._minimum is not None and self.value() == self._minimum: - return - try: - cache.decr(self._cache_key, delta) - except ValueError: - pass - - def value(self) -> Optional[int]: - """Return current value or None if not yet initialized.""" - return cache.get(self._cache_key) - - def get_redis_client_or_stub() -> Redis: """Return AA's default cache client or a stub if Redis is not available.""" redis = get_redis_client() diff --git a/allianceauth/authentication/task_statistics/signals.py b/allianceauth/authentication/task_statistics/signals.py index 17665d65..e9d7babc 100644 --- a/allianceauth/authentication/task_statistics/signals.py +++ b/allianceauth/authentication/task_statistics/signals.py @@ -1,15 +1,12 @@ """Signals for Task Statistics.""" from celery.signals import ( - task_failure, task_internal_error, task_postrun, task_prerun, task_retry, - task_success, worker_ready, + task_failure, task_internal_error, task_retry, task_success, worker_ready, ) from django.conf import settings -from .counters import ( - failed_tasks, retried_tasks, running_tasks, succeeded_tasks, -) +from .counters import failed_tasks, retried_tasks, succeeded_tasks def reset_counters(): @@ -17,7 +14,6 @@ def reset_counters(): succeeded_tasks.clear() failed_tasks.clear() retried_tasks.clear() - running_tasks.reset() def is_enabled() -> bool: @@ -55,15 +51,3 @@ def record_task_failed(*args, **kwargs): def record_task_internal_error(*args, **kwargs): if is_enabled(): failed_tasks.add() - - -@task_prerun.connect -def record_task_prerun(*args, **kwargs): - if is_enabled(): - running_tasks.incr() - - -@task_postrun.connect -def record_task_postrun(*args, **kwargs): - if is_enabled(): - running_tasks.decr() diff --git a/allianceauth/authentication/task_statistics/tests/test_counters.py b/allianceauth/authentication/task_statistics/tests/test_counters.py index 284f86ca..2d2555aa 100644 --- a/allianceauth/authentication/task_statistics/tests/test_counters.py +++ b/allianceauth/authentication/task_statistics/tests/test_counters.py @@ -4,11 +4,7 @@ from django.test import TestCase from django.utils.timezone import now from allianceauth.authentication.task_statistics.counters import ( - dashboard_results, - succeeded_tasks, - retried_tasks, - failed_tasks, - running_tasks, + dashboard_results, failed_tasks, retried_tasks, succeeded_tasks, ) @@ -32,7 +28,6 @@ class TestDashboardResults(TestCase): failed_tasks.add(now() - dt.timedelta(hours=1, seconds=1)) failed_tasks.add() - running_tasks.reset(8) # when results = dashboard_results(hours=1) # then @@ -41,14 +36,12 @@ class TestDashboardResults(TestCase): self.assertEqual(results.failed, 1) self.assertEqual(results.total, 6) self.assertEqual(results.earliest_task, earliest_task) - self.assertEqual(results.running, 8) def test_should_work_with_no_data(self): # given succeeded_tasks.clear() retried_tasks.clear() failed_tasks.clear() - running_tasks.reset() # when results = dashboard_results(hours=1) # then @@ -57,4 +50,3 @@ class TestDashboardResults(TestCase): self.assertEqual(results.failed, 0) self.assertEqual(results.total, 0) self.assertIsNone(results.earliest_task) - self.assertEqual(results.running, 0) diff --git a/allianceauth/authentication/task_statistics/tests/test_helpers.py b/allianceauth/authentication/task_statistics/tests/test_helpers.py index 757ae38e..51dae201 100644 --- a/allianceauth/authentication/task_statistics/tests/test_helpers.py +++ b/allianceauth/authentication/task_statistics/tests/test_helpers.py @@ -4,125 +4,11 @@ from unittest.mock import patch from redis import RedisError from allianceauth.authentication.task_statistics.helpers import ( - ItemCounter, _RedisStub, get_redis_client_or_stub, + _RedisStub, get_redis_client_or_stub, ) MODULE_PATH = "allianceauth.authentication.task_statistics.helpers" -COUNTER_NAME = "test-counter" - - -class TestItemCounter(TestCase): - def test_can_create_counter(self): - # when - counter = ItemCounter(COUNTER_NAME) - # then - self.assertIsInstance(counter, ItemCounter) - - def test_can_reset_counter_to_default(self): - # given - counter = ItemCounter(COUNTER_NAME) - # when - counter.reset() - # then - self.assertEqual(counter.value(), 0) - - def test_can_reset_counter_to_custom_value(self): - # given - counter = ItemCounter(COUNTER_NAME) - # when - counter.reset(42) - # then - self.assertEqual(counter.value(), 42) - - def test_can_increment_counter_by_default(self): - # given - counter = ItemCounter(COUNTER_NAME) - counter.reset(0) - # when - counter.incr() - # then - self.assertEqual(counter.value(), 1) - - def test_can_increment_counter_by_custom_value(self): - # given - counter = ItemCounter(COUNTER_NAME) - counter.reset(0) - # when - counter.incr(8) - # then - self.assertEqual(counter.value(), 8) - - def test_can_decrement_counter_by_default(self): - # given - counter = ItemCounter(COUNTER_NAME) - counter.reset(9) - # when - counter.decr() - # then - self.assertEqual(counter.value(), 8) - - def test_can_decrement_counter_by_custom_value(self): - # given - counter = ItemCounter(COUNTER_NAME) - counter.reset(9) - # when - counter.decr(8) - # then - self.assertEqual(counter.value(), 1) - - def test_can_decrement_counter_below_zero(self): - # given - counter = ItemCounter(COUNTER_NAME) - counter.reset(0) - # when - counter.decr(1) - # then - self.assertEqual(counter.value(), -1) - - def test_can_not_decrement_counter_below_minimum(self): - # given - counter = ItemCounter(COUNTER_NAME, minimum=0) - counter.reset(0) - # when - counter.decr(1) - # then - self.assertEqual(counter.value(), 0) - - def test_can_not_reset_counter_below_minimum(self): - # given - counter = ItemCounter(COUNTER_NAME, minimum=0) - # when/then - with self.assertRaises(ValueError): - counter.reset(-1) - - def test_can_not_init_without_name(self): - # when/then - with self.assertRaises(ValueError): - ItemCounter(name="") - - def test_can_ignore_invalid_values_when_incrementing(self): - # given - counter = ItemCounter(COUNTER_NAME) - counter.reset(0) - # when - with patch(MODULE_PATH + ".cache.incr") as m: - m.side_effect = ValueError - counter.incr() - # then - self.assertEqual(counter.value(), 0) - - def test_can_ignore_invalid_values_when_decrementing(self): - # given - counter = ItemCounter(COUNTER_NAME) - counter.reset(1) - # when - with patch(MODULE_PATH + ".cache.decr") as m: - m.side_effect = ValueError - counter.decr() - # then - self.assertEqual(counter.value(), 1) - class TestGetRedisClient(TestCase): def test_should_return_mock_if_redis_not_available_1(self): diff --git a/allianceauth/authentication/templates/public/lang_select.html b/allianceauth/authentication/templates/public/lang_select.html index 90a135d3..6556ad24 100644 --- a/allianceauth/authentication/templates/public/lang_select.html +++ b/allianceauth/authentication/templates/public/lang_select.html @@ -5,7 +5,7 @@