shift to custom Scheduler

allianceauth/apps.py

@@ -1,5 +1,4 @@
 from django.apps import AppConfig
-from celery.schedules import crontab
 
 
 class AllianceAuthConfig(AppConfig):
@@ -7,48 +6,3 @@ class AllianceAuthConfig(AppConfig):
 
     def ready(self) -> None:
         import allianceauth.checks # noqa
-        from django_celery_beat.models import CrontabSchedule, PeriodicTask
-        from allianceauth.crontab.cron import offset_cron
-
-        PeriodicTask.objects.update_or_create(
-            name='esi_cleanup_callbackredirect',
-            defaults={
-                'task': 'esi.tasks.cleanup_callbackredirect',
-                'crontab': CrontabSchedule.objects.get_or_create(minute='0', hour='0', day_of_week='*', day_of_month='*', month_of_year='*', timezone='UTC')[0],
-            },
-        )
-        PeriodicTask.objects.update_or_create(
-            name='esi_cleanup_token',
-            defaults={
-                'task': 'esi.tasks.cleanup_token',
-                'crontab': CrontabSchedule.objects.get_or_create(minute='0', hour='0', day_of_week='*', day_of_month='*', month_of_year='*', timezone='UTC')[0],
-            },
-        )
-
-        z = CrontabSchedule.from_schedule(offset_cron(crontab(minute='0', hour='*/6')))
-        PeriodicTask.objects.update_or_create(
-            name='run_model_update',
-            defaults={
-                'task': 'allianceauth.eveonline.tasks.run_model_update',
-                'crontab': CrontabSchedule.objects.get_or_create(  # Convert the offsetted cron into a DB object
-                    minute=z.minute, hour=z.hour, day_of_week=z.day_of_week, day_of_month=z.day_of_month, month_of_year=z.month_of_year, timezone=z.timezone)[0],
-            },
-        )
-
-        z = CrontabSchedule.from_schedule(offset_cron(crontab(minute='0', hour='*/4')))
-        PeriodicTask.objects.update_or_create(
-            name='check_all_character_ownership',
-            defaults={
-                'task': 'allianceauth.authentication.tasks.check_all_character_ownership',
-                'crontab': CrontabSchedule.objects.get_or_create(  # Convert the offsetted cron into a DB object
-                    minute=z.minute, hour=z.hour, day_of_week=z.day_of_week, day_of_month=z.day_of_month, month_of_year=z.month_of_year, timezone=z.timezone)[0],
-            },
-        )
-        PeriodicTask.objects.update_or_create(
-            name='analytics_daily_stats',
-            defaults={
-                'task': 'allianceauth.analytics.tasks.analytics_daily_stats',
-                'crontab': CrontabSchedule.objects.get_or_create(
-                    minute='0', hour='12', day_of_week='*', day_of_month='*', month_of_year='*', timezone='UTC')[0],
-            },
-        )

allianceauth/crontab/schedulers.py

@@ -0,0 +1,63 @@
+from django.core.exceptions import ObjectDoesNotExist
+from django_celery_beat.schedulers import (
+    DatabaseScheduler
+)
+from django_celery_beat.models import CrontabSchedule
+from django.db.utils import OperationalError, ProgrammingError
+
+from celery import schedules
+from celery.utils.log import get_logger
+
+from allianceauth.crontab.models import CronOffset
+from allianceauth.crontab.utils import offset_cron
+
+logger = get_logger(__name__)
+
+
+class OffsetDatabaseScheduler(DatabaseScheduler):
+    """
+    Customization of Django Celery Beat, Database Scheduler
+    Takes the Celery Schedule from local.py and applies our AA Framework Cron Offset, if apply_offset is true
+    Otherwise it passes it through as normal
+    """
+    def update_from_dict(self, mapping):
+        s = {}
+
+        try:
+            cron_offset = CronOffset.get_solo()
+        except (OperationalError, ProgrammingError, ObjectDoesNotExist) as exc:
+            # This is just incase we haven't migrated yet or something
+            logger.warning(
+                "OffsetDatabaseScheduler: Could not fetch CronOffset (%r). "
+                "Defering to DatabaseScheduler",
+                exc
+            )
+            return super().update_from_dict(mapping)
+
+        for name, entry_fields in mapping.items():
+            try:
+                apply_offset = entry_fields.pop("apply_offset", False)
+                entry = self.Entry.from_entry(name, app=self.app, **entry_fields)
+
+                if entry.model.enabled and apply_offset:
+                    schedule_obj = entry.schedule
+                    if isinstance(schedule_obj, schedules.crontab):
+                        offset_cs = CrontabSchedule.from_schedule(offset_cron(schedule_obj))
+                        offset_cs, created = CrontabSchedule.objects.get_or_create(
+                            minute=offset_cs.minute,
+                            hour=offset_cs.hour,
+                            day_of_month=offset_cs.day_of_month,
+                            month_of_year=offset_cs.month_of_year,
+                            day_of_week=offset_cs.day_of_week,
+                            timezone=offset_cs.timezone,
+                        )
+                        entry.model.crontab = offset_cs
+                        entry.model.save()
+                        logger.debug(f"Offset applied for '{name}' due to 'apply_offset' = True.")
+
+                s[name] = entry
+
+            except Exception as e:
+                logger.exception("Error updating schedule for %s: %r", name, e)
+
+        self.schedule.update(s)

allianceauth/crontab/tests/test_models.py

@@ -15,7 +15,6 @@ class CronOffsetModelTest(TestCase):
 
         # They should be the exact same object in memory
         self.assertEqual(offset1.pk, offset2.pk)
-        self.assertIs(offset1, offset2)
 
     def test_default_values_random(self):
         """

allianceauth/crontab/tests/test_utils.py

@@ -6,11 +6,12 @@ from django.test import TestCase
 from django.db import ProgrammingError
 from celery.schedules import crontab
 
-from allianceauth.crontab.cron import offset_cron
+from allianceauth.crontab.utils import offset_cron
 from allianceauth.crontab.models import CronOffset
 
 logger = logging.getLogger(__name__)
 
+
 class TestOffsetCron(TestCase):
 
     def test_offset_cron_normal(self):

allianceauth/crontab/utils.py

@@ -3,6 +3,7 @@ import logging
 from allianceauth.crontab.models import CronOffset
 from django.db import ProgrammingError
 
+
 logger = logging.getLogger(__name__)
 
 

allianceauth/project_template/project_name/settings/base.py

@@ -50,8 +50,32 @@ SECRET_KEY = "wow I'm a really bad default secret key"
 
 # Celery configuration
 BROKER_URL = 'redis://localhost:6379/0'
-CELERYBEAT_SCHEDULER = "django_celery_beat.schedulers.DatabaseScheduler"
-CELERYBEAT_SCHEDULE = {}
+CELERYBEAT_SCHEDULER = "allianceauth.crontab.schedulers.OffsetDatabaseScheduler"
+CELERYBEAT_SCHEDULE = {
+    'esi_cleanup_callbackredirect': {
+        'task': 'esi.tasks.cleanup_callbackredirect',
+        'schedule': crontab(minute='0', hour='*/4'),
+    },
+    'esi_cleanup_token': {
+        'task': 'esi.tasks.cleanup_token',
+        'schedule': crontab(minute='0', hour='0'),
+    },
+    'run_model_update': {
+        'task': 'allianceauth.eveonline.tasks.run_model_update',
+        'schedule': crontab(minute='0', hour="*/6"),
+        'apply_offset': True
+    },
+    'check_all_character_ownership': {
+        'task': 'allianceauth.authentication.tasks.check_all_character_ownership',
+        'schedule': crontab(minute='0', hour='*/4'),
+        'apply_offset': True
+    },
+    'analytics_daily_stats': {
+        'task': 'allianceauth.analytics.tasks.analytics_daily_stats',
+        'schedule': crontab(minute='0', hour='2'),
+    }
+}
+
 
 # Build paths inside the project like this: os.path.join(BASE_DIR, ...)
 PROJECT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))