document cron offsetting

allianceauth/apps.py

@@ -1,5 +1,4 @@
 from django.apps import AppConfig
-from celery.schedules import crontab
 
 
 class AllianceAuthConfig(AppConfig):
@@ -7,48 +6,3 @@ class AllianceAuthConfig(AppConfig):
 
     def ready(self) -> None:
         import allianceauth.checks # noqa
-        from django_celery_beat.models import CrontabSchedule, PeriodicTask
-        from allianceauth.crontab.cron import offset_cron
-
-        PeriodicTask.objects.update_or_create(
-            name='esi_cleanup_callbackredirect',
-            defaults={
-                'task': 'esi.tasks.cleanup_callbackredirect',
-                'crontab': CrontabSchedule.objects.get_or_create(minute='0', hour='0', day_of_week='*', day_of_month='*', month_of_year='*', timezone='UTC')[0],
-            },
-        )
-        PeriodicTask.objects.update_or_create(
-            name='esi_cleanup_token',
-            defaults={
-                'task': 'esi.tasks.cleanup_token',
-                'crontab': CrontabSchedule.objects.get_or_create(minute='0', hour='0', day_of_week='*', day_of_month='*', month_of_year='*', timezone='UTC')[0],
-            },
-        )
-
-        z = CrontabSchedule.from_schedule(offset_cron(crontab(minute='0', hour='*/6')))
-        PeriodicTask.objects.update_or_create(
-            name='run_model_update',
-            defaults={
-                'task': 'allianceauth.eveonline.tasks.run_model_update',
-                'crontab': CrontabSchedule.objects.get_or_create(  # Convert the offsetted cron into a DB object
-                    minute=z.minute, hour=z.hour, day_of_week=z.day_of_week, day_of_month=z.day_of_month, month_of_year=z.month_of_year, timezone=z.timezone)[0],
-            },
-        )
-
-        z = CrontabSchedule.from_schedule(offset_cron(crontab(minute='0', hour='*/4')))
-        PeriodicTask.objects.update_or_create(
-            name='check_all_character_ownership',
-            defaults={
-                'task': 'allianceauth.authentication.tasks.check_all_character_ownership',
-                'crontab': CrontabSchedule.objects.get_or_create(  # Convert the offsetted cron into a DB object
-                    minute=z.minute, hour=z.hour, day_of_week=z.day_of_week, day_of_month=z.day_of_month, month_of_year=z.month_of_year, timezone=z.timezone)[0],
-            },
-        )
-        PeriodicTask.objects.update_or_create(
-            name='analytics_daily_stats',
-            defaults={
-                'task': 'allianceauth.analytics.tasks.analytics_daily_stats',
-                'crontab': CrontabSchedule.objects.get_or_create(
-                    minute='0', hour='12', day_of_week='*', day_of_month='*', month_of_year='*', timezone='UTC')[0],
-            },
-        )

allianceauth/crontab/schedulers.py

@@ -0,0 +1,63 @@
+from django.core.exceptions import ObjectDoesNotExist
+from django_celery_beat.schedulers import (
+    DatabaseScheduler
+)
+from django_celery_beat.models import CrontabSchedule
+from django.db.utils import OperationalError, ProgrammingError
+
+from celery import schedules
+from celery.utils.log import get_logger
+
+from allianceauth.crontab.models import CronOffset
+from allianceauth.crontab.utils import offset_cron
+
+logger = get_logger(__name__)
+
+
+class OffsetDatabaseScheduler(DatabaseScheduler):
+    """
+    Customization of Django Celery Beat, Database Scheduler
+    Takes the Celery Schedule from local.py and applies our AA Framework Cron Offset, if apply_offset is true
+    Otherwise it passes it through as normal
+    """
+    def update_from_dict(self, mapping):
+        s = {}
+
+        try:
+            cron_offset = CronOffset.get_solo()
+        except (OperationalError, ProgrammingError, ObjectDoesNotExist) as exc:
+            # This is just incase we haven't migrated yet or something
+            logger.warning(
+                "OffsetDatabaseScheduler: Could not fetch CronOffset (%r). "
+                "Defering to DatabaseScheduler",
+                exc
+            )
+            return super().update_from_dict(mapping)
+
+        for name, entry_fields in mapping.items():
+            try:
+                apply_offset = entry_fields.pop("apply_offset", False)
+                entry = self.Entry.from_entry(name, app=self.app, **entry_fields)
+
+                if entry.model.enabled and apply_offset:
+                    schedule_obj = entry.schedule
+                    if isinstance(schedule_obj, schedules.crontab):
+                        offset_cs = CrontabSchedule.from_schedule(offset_cron(schedule_obj))
+                        offset_cs, created = CrontabSchedule.objects.get_or_create(
+                            minute=offset_cs.minute,
+                            hour=offset_cs.hour,
+                            day_of_month=offset_cs.day_of_month,
+                            month_of_year=offset_cs.month_of_year,
+                            day_of_week=offset_cs.day_of_week,
+                            timezone=offset_cs.timezone,
+                        )
+                        entry.model.crontab = offset_cs
+                        entry.model.save()
+                        logger.debug(f"Offset applied for '{name}' due to 'apply_offset' = True.")
+
+                s[name] = entry
+
+            except Exception as e:
+                logger.exception("Error updating schedule for %s: %r", name, e)
+
+        self.schedule.update(s)

allianceauth/crontab/tests/test_models.py

@@ -15,7 +15,6 @@ class CronOffsetModelTest(TestCase):
 
         # They should be the exact same object in memory
         self.assertEqual(offset1.pk, offset2.pk)
-        self.assertIs(offset1, offset2)
 
     def test_default_values_random(self):
         """

allianceauth/crontab/tests/test_utils.py

@@ -6,11 +6,12 @@ from django.test import TestCase
 from django.db import ProgrammingError
 from celery.schedules import crontab
 
-from allianceauth.crontab.cron import offset_cron
+from allianceauth.crontab.utils import offset_cron
 from allianceauth.crontab.models import CronOffset
 
 logger = logging.getLogger(__name__)
 
+
 class TestOffsetCron(TestCase):
 
     def test_offset_cron_normal(self):

allianceauth/crontab/utils.py

@@ -3,6 +3,7 @@ import logging
 from allianceauth.crontab.models import CronOffset
 from django.db import ProgrammingError
 
+
 logger = logging.getLogger(__name__)
 
 

allianceauth/project_template/project_name/settings/base.py

@@ -50,8 +50,32 @@ SECRET_KEY = "wow I'm a really bad default secret key"
 
 # Celery configuration
 BROKER_URL = 'redis://localhost:6379/0'
-CELERYBEAT_SCHEDULER = "django_celery_beat.schedulers.DatabaseScheduler"
+CELERYBEAT_SCHEDULER = "allianceauth.crontab.schedulers.OffsetDatabaseScheduler"
-CELERYBEAT_SCHEDULE = {}
+CELERYBEAT_SCHEDULE = {
+    'esi_cleanup_callbackredirect': {
+        'task': 'esi.tasks.cleanup_callbackredirect',
+        'schedule': crontab(minute='0', hour='*/4'),
+    },
+    'esi_cleanup_token': {
+        'task': 'esi.tasks.cleanup_token',
+        'schedule': crontab(minute='0', hour='0'),
+    },
+    'run_model_update': {
+        'task': 'allianceauth.eveonline.tasks.run_model_update',
+        'schedule': crontab(minute='0', hour="*/6"),
+        'apply_offset': True
+    },
+    'check_all_character_ownership': {
+        'task': 'allianceauth.authentication.tasks.check_all_character_ownership',
+        'schedule': crontab(minute='0', hour='*/4'),
+        'apply_offset': True
+    },
+    'analytics_daily_stats': {
+        'task': 'allianceauth.analytics.tasks.analytics_daily_stats',
+        'schedule': crontab(minute='0', hour='2'),
+    }
+}
+
 
 # Build paths inside the project like this: os.path.join(BASE_DIR, ...)
 PROJECT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))

docs/development/tech_docu/celery.md

@@ -123,6 +123,7 @@ Example setting:
 CELERYBEAT_SCHEDULE['structures_update_all_structures'] = {
     'task': 'structures.tasks.update_all_structures',
     'schedule': crontab(minute='*/30'),
+    'apply_offset': True,
 }
 ```
 
@@ -130,6 +131,7 @@ CELERYBEAT_SCHEDULE['structures_update_all_structures'] = {
 
 - `'task'`: Name of your task (full path)
 - `'schedule'`: Schedule definition (see Celery documentation on [Periodic Tasks](https://docs.celeryproject.org/en/latest/userguide/periodic-tasks.html) for details)
+- `'apply_offset'`: Boolean, Apply a Delay unique to the install, in order to reduce impact on ESI. See [Apply Offset](#apply-offset)
 
 ## How can I use priorities for tasks?
 
@@ -174,9 +176,54 @@ Large numbers of installs running the same crontab (ie. `0 * * * *`) can all sla
 
 Consider Artificially smoothing out your tasks with a few methods
 
-### Offset Crontabs
+### Apply Offset
 
-Avoid running your tasks on the hour or other nice neat human numbers, consider 23 minutes on the hour instead of at zero (`28 * * * *`)
+`allianceauth.crontab` contains a series of Offsets stored in the DB that are both static for an install, but random across all AA installs.
+
+This enables us to spread our load on ESI (or other external resources) across a greater window, making it unlikely that two installs will hit ESI at the same time.
+
+Tasks defined in local.py, can have `'apply_offset': True` added to their Task Definition
+
+```python
+CELERYBEAT_SCHEDULE['taskname'] = {
+    'task': 'module.tasks.task',
+    'schedule': crontab(minute='*/30'),
+    'apply_offset': True,
+}
+```
+
+Tasks added to directly to Django Celery Beat Models (Using a Management Task etc) can pass their Cron Schedule through offset_cron(crontab)
+
+```{eval-rst}
+.. automodule:: allianceauth.crontab.utils
+    :members:
+    :undoc-members:
+```
+
+```python
+from django_celery_beat.models import CrontabSchedule, PeriodicTask
+from celery.schedules import crontab
+
+schedule = CrontabSchedule.from_schedule(offset_cron(crontab(minute='0', hour='0')))
+
+schedule, created = CrontabSchedule.objects.get_or_create(
+    minute=schedule.minute,
+    hour=schedule.hour,
+    day_of_month=schedule.day_of_month,
+    month_of_year=schedule.month_of_year,
+    day_of_week=schedule.day_of_week,
+    timezone=schedule.timezone,
+)
+
+PeriodicTask.objects.update_or_create(
+    task='module.tasks.task',
+    defaults={
+        'crontab': schedule,
+        'name': 'task name',
+        'enabled': True
+    }
+)
+```
 
 ### Subset Tasks