Merge branch 'seperate-out-celery-once' into 'master'

Remove dependecies from celery_once customizations

See merge request allianceauth/allianceauth!1491

README.md

@@ -63,7 +63,6 @@ Here is an example of the Alliance Auth web site with a mixture of Services, App
 - [Aaron Kable](https://gitlab.com/aaronkable/)
 - [Ariel Rin](https://gitlab.com/soratidus999/)
 - [Col Crunch](https://gitlab.com/colcrunch/)
-- [Erik Kalkoken](https://gitlab.com/ErikKalkoken/)
 - [Rounon Dax](https://gitlab.com/ppfeufer)
 - [snipereagle1](https://gitlab.com/mckernanin)
 
@@ -71,6 +70,7 @@ Here is an example of the Alliance Auth web site with a mixture of Services, App
 
 - [Adarnof](https://gitlab.com/adarnof/)
 - [Basraah](https://gitlab.com/basraah/)
+- [Erik Kalkoken](https://gitlab.com/ErikKalkoken/)
 
 ### Beta Testers / Bug Fixers
 

allianceauth/__init__.py

@@ -5,7 +5,7 @@ manage online service access.
 # This will make sure the app is always imported when
 # Django starts so that shared_task will use this app.
 
-__version__ = '4.11.1'
+__version__ = '4.11.2'
 __title__ = 'Alliance Auth'
 __title_useragent__ = 'AllianceAuth'
 __url__ = 'https://gitlab.com/allianceauth/allianceauth'

allianceauth/eveonline/tests/test_providers.py

@@ -727,7 +727,8 @@ class TestEveSwaggerProvider(TestCase):
         my_provider = EveSwaggerProvider()
         my_client = my_provider.client
         operation = my_client.Universe.get_universe_factions()
-        self.assertEqual(
-            operation.future.request.headers['User-Agent'],
-            f'AllianceAuth/{aa_version} (dummy@example.net; +{aa_url}) Django-ESI/{esi_version} (+{esi_url})'
-        )
+        expected_variants = {
+            f'AllianceAuth/{aa_version} (dummy@example.net; +{aa_url}) DjangoEsi/{esi_version} (+{esi_url})',  # Django-ESI 8.0.0
+            f'AllianceAuth/{aa_version} (dummy@example.net; +{aa_url}) Django-ESI/{esi_version} (+{esi_url})'  # Django-ESI 7.x, Py38 Py39
+        }
+        self.assertIn(operation.future.request.headers['User-Agent'], expected_variants)

allianceauth/project_template/project_name/celery.py

@@ -26,7 +26,7 @@ app.conf.task_default_priority = 5  # anything called with the task.delay() will
 app.conf.worker_prefetch_multiplier = 1  # only prefetch single tasks at a time on the workers so that prio tasks happen
 
 app.conf.ONCE = {
-    'backend': 'allianceauth.services.tasks.DjangoBackend',
+    'backend': 'allianceauth.services.celery_once.backends.DjangoBackend',
     'settings': {}
 }
 

allianceauth/services/celery_once/backends.py

@@ -0,0 +1,19 @@
+from celery_once import AlreadyQueued
+from django.core.cache import cache
+
+
+class DjangoBackend:
+    """Locking backend for celery once."""
+
+    def __init__(self, settings):
+        pass
+
+    @staticmethod
+    def raise_or_lock(key, timeout):
+        acquired = cache.add(key=key, value="lock", timeout=timeout)
+        if not acquired:
+            raise AlreadyQueued(int(cache.ttl(key)))
+
+    @staticmethod
+    def clear_lock(key):
+        return cache.delete(key)

allianceauth/services/celery_once/tasks.py

@@ -0,0 +1,8 @@
+from celery_once import QueueOnce as BaseTask
+
+
+class QueueOnce(BaseTask):
+    """QueueOnce class with custom defaults."""
+
+    once = BaseTask.once
+    once["graceful"] = True

allianceauth/services/celery_once/tests.py

@@ -0,0 +1,47 @@
+from celery_once import AlreadyQueued
+from django.core.cache import cache
+from django.test import TestCase
+
+from allianceauth.services.celery_once.backends import DjangoBackend
+
+
+class TestDjangoBackend(TestCase):
+    TEST_KEY = "my-django-backend-test-key"
+    TIMEOUT = 1800
+
+    def setUp(self) -> None:
+        cache.delete(self.TEST_KEY)
+        self.backend = DjangoBackend(dict())
+
+    def test_can_get_lock(self):
+        """
+        when lock can be acquired
+        then set it with timeout
+        """
+        self.backend.raise_or_lock(self.TEST_KEY, self.TIMEOUT)
+        self.assertIsNotNone(cache.get(self.TEST_KEY))
+        self.assertAlmostEqual(cache.ttl(self.TEST_KEY), self.TIMEOUT, delta=2)
+
+    def test_when_cant_get_lock_raise_exception(self):
+        """
+        when lock can bot be acquired
+        then raise AlreadyQueued exception with countdown
+        """
+        self.backend.raise_or_lock(self.TEST_KEY, self.TIMEOUT)
+
+        try:
+            self.backend.raise_or_lock(self.TEST_KEY, self.TIMEOUT)
+        except Exception as ex:
+            self.assertIsInstance(ex, AlreadyQueued)
+            self.assertAlmostEqual(ex.countdown, self.TIMEOUT, delta=2)
+
+    def test_can_clear_lock(self):
+        """
+        when a lock exists
+        then can get a new lock after clearing it
+        """
+        self.backend.raise_or_lock(self.TEST_KEY, self.TIMEOUT)
+
+        self.backend.clear_lock(self.TEST_KEY)
+        self.backend.raise_or_lock(self.TEST_KEY, self.TIMEOUT)
+        self.assertIsNotNone(cache.get(self.TEST_KEY))

allianceauth/services/tasks.py

@@ -3,33 +3,11 @@ import logging
 from celery import shared_task
 from django.contrib.auth.models import User
 from .hooks import ServicesHook
-from celery_once import QueueOnce as BaseTask, AlreadyQueued
-from django.core.cache import cache
-
+from .celery_once.tasks import QueueOnce  # noqa: F401 - for backwards compatibility
 
 logger = logging.getLogger(__name__)
 
 
-class QueueOnce(BaseTask):
-    once = BaseTask.once
-    once['graceful'] = True
-
-
-class DjangoBackend:
-    def __init__(self, settings):
-        pass
-
-    @staticmethod
-    def raise_or_lock(key, timeout):
-        acquired = cache.add(key=key, value="lock", timeout=timeout)
-        if not acquired:
-            raise AlreadyQueued(int(cache.ttl(key)))
-
-    @staticmethod
-    def clear_lock(key):
-        return cache.delete(key)
-
-
 @shared_task(bind=True)
 def validate_services(self, pk):
     user = User.objects.get(pk=pk)
@@ -38,7 +16,7 @@ def validate_services(self, pk):
     for svc in ServicesHook.get_services():
         try:
             svc.validate_user(user)
-        except:
+        except Exception:
             logger.exception(f'Exception running validate_user for services module {svc} on user {user}')
 
 

allianceauth/services/tests/test_tasks.py

@@ -1,15 +1,10 @@
 from unittest import mock
 
-from celery_once import AlreadyQueued
-
-from django.core.cache import cache
 from django.test import override_settings, TestCase
 
 from allianceauth.tests.auth_utils import AuthUtils
 from allianceauth.services.tasks import validate_services, update_groups_for_user
 
-from ..tasks import DjangoBackend
-
 
 @override_settings(CELERY_ALWAYS_EAGER=True, CELERY_EAGER_PROPAGATES_EXCEPTIONS=True)
 class ServicesTasksTestCase(TestCase):
@@ -46,46 +41,3 @@ class ServicesTasksTestCase(TestCase):
         self.assertTrue(svc.update_groups.called)
         args, _ = svc.update_groups.call_args
         self.assertEqual(self.member, args[0])  # Assert correct user
-
-
-class TestDjangoBackend(TestCase):
-
-    TEST_KEY = "my-django-backend-test-key"
-    TIMEOUT = 1800
-
-    def setUp(self) -> None:
-        cache.delete(self.TEST_KEY)
-        self.backend = DjangoBackend(dict())
-
-    def test_can_get_lock(self):
-        """
-        when lock can be acquired
-        then set it with timetout
-        """
-        self.backend.raise_or_lock(self.TEST_KEY, self.TIMEOUT)
-        self.assertIsNotNone(cache.get(self.TEST_KEY))
-        self.assertAlmostEqual(cache.ttl(self.TEST_KEY), self.TIMEOUT, delta=2)
-
-    def test_when_cant_get_lock_raise_exception(self):
-        """
-        when lock can bot be acquired
-        then raise AlreadyQueued exception with countdown
-        """
-        self.backend.raise_or_lock(self.TEST_KEY, self.TIMEOUT)
-
-        try:
-            self.backend.raise_or_lock(self.TEST_KEY, self.TIMEOUT)
-        except Exception as ex:
-            self.assertIsInstance(ex, AlreadyQueued)
-            self.assertAlmostEqual(ex.countdown, self.TIMEOUT, delta=2)
-
-    def test_can_clear_lock(self):
-        """
-        when a lock exists
-        then can get a new lock after clearing it
-        """
-        self.backend.raise_or_lock(self.TEST_KEY, self.TIMEOUT)
-
-        self.backend.clear_lock(self.TEST_KEY)
-        self.backend.raise_or_lock(self.TEST_KEY, self.TIMEOUT)
-        self.assertIsNotNone(cache.get(self.TEST_KEY))

docker/.env.example

@@ -1,7 +1,7 @@
 PROTOCOL=https://
 AUTH_SUBDOMAIN=%AUTH_SUBDOMAIN%
 DOMAIN=%DOMAIN%
-AA_DOCKER_TAG=registry.gitlab.com/allianceauth/allianceauth/auth:v4.11.1
+AA_DOCKER_TAG=registry.gitlab.com/allianceauth/allianceauth/auth:v4.11.2
 
 # Nginx Proxy Manager
 PROXY_HTTP_PORT=80

docker/Dockerfile

@@ -1,5 +1,5 @@
 FROM python:3.11-slim
-ARG AUTH_VERSION=v4.11.1
+ARG AUTH_VERSION=v4.11.2
 ARG AUTH_PACKAGE=allianceauth==${AUTH_VERSION}
 ENV AUTH_USER=allianceauth
 ENV AUTH_GROUP=allianceauth

tests/celery.py

@@ -26,7 +26,7 @@ app.conf.task_default_priority = 5  # anything called with the task.delay() will
 app.conf.worker_prefetch_multiplier = 1  # only prefetch single tasks at a time on the workers so that prio tasks happen
 
 app.conf.ONCE = {
-    'backend': 'allianceauth.services.tasks.DjangoBackend',
+    'backend': 'allianceauth.services.celery_once.backends.DjangoBackend',
     'settings': {}
 }