From 6f2f39d7fa9c5d8999c276e793778968f7284c25 Mon Sep 17 00:00:00 2001 From: Joel Falknau Date: Mon, 30 Dec 2024 18:12:11 +1000 Subject: [PATCH] shift to custom Scheduler --- allianceauth/apps.py | 46 -------------- allianceauth/crontab/schedulers.py | 63 +++++++++++++++++++ allianceauth/crontab/tests/test_models.py | 1 - .../tests/{test_cron.py => test_utils.py} | 3 +- allianceauth/crontab/{cron.py => utils.py} | 1 + .../project_name/settings/base.py | 28 ++++++++- 6 files changed, 92 insertions(+), 50 deletions(-) create mode 100644 allianceauth/crontab/schedulers.py rename allianceauth/crontab/tests/{test_cron.py => test_utils.py} (98%) rename allianceauth/crontab/{cron.py => utils.py} (99%) diff --git a/allianceauth/apps.py b/allianceauth/apps.py index c2397c9b..1191fed9 100644 --- a/allianceauth/apps.py +++ b/allianceauth/apps.py @@ -1,5 +1,4 @@ from django.apps import AppConfig -from celery.schedules import crontab class AllianceAuthConfig(AppConfig): @@ -7,48 +6,3 @@ class AllianceAuthConfig(AppConfig): def ready(self) -> None: import allianceauth.checks # noqa - from django_celery_beat.models import CrontabSchedule, PeriodicTask - from allianceauth.crontab.cron import offset_cron - - PeriodicTask.objects.update_or_create( - name='esi_cleanup_callbackredirect', - defaults={ - 'task': 'esi.tasks.cleanup_callbackredirect', - 'crontab': CrontabSchedule.objects.get_or_create(minute='0', hour='0', day_of_week='*', day_of_month='*', month_of_year='*', timezone='UTC')[0], - }, - ) - PeriodicTask.objects.update_or_create( - name='esi_cleanup_token', - defaults={ - 'task': 'esi.tasks.cleanup_token', - 'crontab': CrontabSchedule.objects.get_or_create(minute='0', hour='0', day_of_week='*', day_of_month='*', month_of_year='*', timezone='UTC')[0], - }, - ) - - z = CrontabSchedule.from_schedule(offset_cron(crontab(minute='0', hour='*/6'))) - PeriodicTask.objects.update_or_create( - name='run_model_update', - defaults={ - 'task': 'allianceauth.eveonline.tasks.run_model_update', - 'crontab': CrontabSchedule.objects.get_or_create( # Convert the offsetted cron into a DB object - minute=z.minute, hour=z.hour, day_of_week=z.day_of_week, day_of_month=z.day_of_month, month_of_year=z.month_of_year, timezone=z.timezone)[0], - }, - ) - - z = CrontabSchedule.from_schedule(offset_cron(crontab(minute='0', hour='*/4'))) - PeriodicTask.objects.update_or_create( - name='check_all_character_ownership', - defaults={ - 'task': 'allianceauth.authentication.tasks.check_all_character_ownership', - 'crontab': CrontabSchedule.objects.get_or_create( # Convert the offsetted cron into a DB object - minute=z.minute, hour=z.hour, day_of_week=z.day_of_week, day_of_month=z.day_of_month, month_of_year=z.month_of_year, timezone=z.timezone)[0], - }, - ) - PeriodicTask.objects.update_or_create( - name='analytics_daily_stats', - defaults={ - 'task': 'allianceauth.analytics.tasks.analytics_daily_stats', - 'crontab': CrontabSchedule.objects.get_or_create( - minute='0', hour='12', day_of_week='*', day_of_month='*', month_of_year='*', timezone='UTC')[0], - }, - ) diff --git a/allianceauth/crontab/schedulers.py b/allianceauth/crontab/schedulers.py new file mode 100644 index 00000000..0b36eead --- /dev/null +++ b/allianceauth/crontab/schedulers.py @@ -0,0 +1,63 @@ +from django.core.exceptions import ObjectDoesNotExist +from django_celery_beat.schedulers import ( + DatabaseScheduler +) +from django_celery_beat.models import CrontabSchedule +from django.db.utils import OperationalError, ProgrammingError + +from celery import schedules +from celery.utils.log import get_logger + +from allianceauth.crontab.models import CronOffset +from allianceauth.crontab.utils import offset_cron + +logger = get_logger(__name__) + + +class OffsetDatabaseScheduler(DatabaseScheduler): + """ + Customization of Django Celery Beat, Database Scheduler + Takes the Celery Schedule from local.py and applies our AA Framework Cron Offset, if apply_offset is true + Otherwise it passes it through as normal + """ + def update_from_dict(self, mapping): + s = {} + + try: + cron_offset = CronOffset.get_solo() + except (OperationalError, ProgrammingError, ObjectDoesNotExist) as exc: + # This is just incase we haven't migrated yet or something + logger.warning( + "OffsetDatabaseScheduler: Could not fetch CronOffset (%r). " + "Defering to DatabaseScheduler", + exc + ) + return super().update_from_dict(mapping) + + for name, entry_fields in mapping.items(): + try: + apply_offset = entry_fields.pop("apply_offset", False) + entry = self.Entry.from_entry(name, app=self.app, **entry_fields) + + if entry.model.enabled and apply_offset: + schedule_obj = entry.schedule + if isinstance(schedule_obj, schedules.crontab): + offset_cs = CrontabSchedule.from_schedule(offset_cron(schedule_obj)) + offset_cs, created = CrontabSchedule.objects.get_or_create( + minute=offset_cs.minute, + hour=offset_cs.hour, + day_of_month=offset_cs.day_of_month, + month_of_year=offset_cs.month_of_year, + day_of_week=offset_cs.day_of_week, + timezone=offset_cs.timezone, + ) + entry.model.crontab = offset_cs + entry.model.save() + logger.debug(f"Offset applied for '{name}' due to 'apply_offset' = True.") + + s[name] = entry + + except Exception as e: + logger.exception("Error updating schedule for %s: %r", name, e) + + self.schedule.update(s) diff --git a/allianceauth/crontab/tests/test_models.py b/allianceauth/crontab/tests/test_models.py index f5f2aa50..cb92e667 100644 --- a/allianceauth/crontab/tests/test_models.py +++ b/allianceauth/crontab/tests/test_models.py @@ -15,7 +15,6 @@ class CronOffsetModelTest(TestCase): # They should be the exact same object in memory self.assertEqual(offset1.pk, offset2.pk) - self.assertIs(offset1, offset2) def test_default_values_random(self): """ diff --git a/allianceauth/crontab/tests/test_cron.py b/allianceauth/crontab/tests/test_utils.py similarity index 98% rename from allianceauth/crontab/tests/test_cron.py rename to allianceauth/crontab/tests/test_utils.py index c9a2a970..48592c99 100644 --- a/allianceauth/crontab/tests/test_cron.py +++ b/allianceauth/crontab/tests/test_utils.py @@ -6,11 +6,12 @@ from django.test import TestCase from django.db import ProgrammingError from celery.schedules import crontab -from allianceauth.crontab.cron import offset_cron +from allianceauth.crontab.utils import offset_cron from allianceauth.crontab.models import CronOffset logger = logging.getLogger(__name__) + class TestOffsetCron(TestCase): def test_offset_cron_normal(self): diff --git a/allianceauth/crontab/cron.py b/allianceauth/crontab/utils.py similarity index 99% rename from allianceauth/crontab/cron.py rename to allianceauth/crontab/utils.py index bbeecddd..150664ca 100644 --- a/allianceauth/crontab/cron.py +++ b/allianceauth/crontab/utils.py @@ -3,6 +3,7 @@ import logging from allianceauth.crontab.models import CronOffset from django.db import ProgrammingError + logger = logging.getLogger(__name__) diff --git a/allianceauth/project_template/project_name/settings/base.py b/allianceauth/project_template/project_name/settings/base.py index bfe6e8ee..19bdbe77 100644 --- a/allianceauth/project_template/project_name/settings/base.py +++ b/allianceauth/project_template/project_name/settings/base.py @@ -50,8 +50,32 @@ SECRET_KEY = "wow I'm a really bad default secret key" # Celery configuration BROKER_URL = 'redis://localhost:6379/0' -CELERYBEAT_SCHEDULER = "django_celery_beat.schedulers.DatabaseScheduler" -CELERYBEAT_SCHEDULE = {} +CELERYBEAT_SCHEDULER = "allianceauth.crontab.schedulers.OffsetDatabaseScheduler" +CELERYBEAT_SCHEDULE = { + 'esi_cleanup_callbackredirect': { + 'task': 'esi.tasks.cleanup_callbackredirect', + 'schedule': crontab(minute='0', hour='*/4'), + }, + 'esi_cleanup_token': { + 'task': 'esi.tasks.cleanup_token', + 'schedule': crontab(minute='0', hour='0'), + }, + 'run_model_update': { + 'task': 'allianceauth.eveonline.tasks.run_model_update', + 'schedule': crontab(minute='0', hour="*/6"), + 'apply_offset': True + }, + 'check_all_character_ownership': { + 'task': 'allianceauth.authentication.tasks.check_all_character_ownership', + 'schedule': crontab(minute='0', hour='*/4'), + 'apply_offset': True + }, + 'analytics_daily_stats': { + 'task': 'allianceauth.analytics.tasks.analytics_daily_stats', + 'schedule': crontab(minute='0', hour='2'), + } +} + # Build paths inside the project like this: os.path.join(BASE_DIR, ...) PROJECT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))