From c6890dd2c61fd20ecf1ebc585c626f5b272ed0de Mon Sep 17 00:00:00 2001 From: Joel Falknau Date: Wed, 4 Dec 2024 18:01:01 +1000 Subject: [PATCH] Spread esi tasks over 10 minutes --- allianceauth/eveonline/tasks.py | 42 ++++++++++++++++++++------------- 1 file changed, 26 insertions(+), 16 deletions(-) diff --git a/allianceauth/eveonline/tasks.py b/allianceauth/eveonline/tasks.py index ce2b4219..657e63f1 100644 --- a/allianceauth/eveonline/tasks.py +++ b/allianceauth/eveonline/tasks.py @@ -1,4 +1,5 @@ import logging +from random import randint from celery import shared_task @@ -9,7 +10,8 @@ from . import providers logger = logging.getLogger(__name__) TASK_PRIORITY = 7 -CHUNK_SIZE = 500 +CHARACTER_AFFILIATION_CHUNK_SIZE = 500 +EVEONLINE_TASK_JITTER = 600 def chunks(lst, n): @@ -19,13 +21,13 @@ def chunks(lst, n): @shared_task -def update_corp(corp_id): +def update_corp(corp_id: int) -> None: """Update given corporation from ESI""" EveCorporationInfo.objects.update_corporation(corp_id) @shared_task -def update_alliance(alliance_id): +def update_alliance(alliance_id: int) -> None: """Update given alliance from ESI""" EveAllianceInfo.objects.update_alliance(alliance_id).populate_alliance() @@ -37,23 +39,30 @@ def update_character(character_id: int) -> None: @shared_task -def run_model_update(): +def run_model_update() -> None: """Update all alliances, corporations and characters from ESI""" - #update existing corp models + # Queue update tasks for Known Corporation Models for corp in EveCorporationInfo.objects.all().values('corporation_id'): - update_corp.apply_async(args=[corp['corporation_id']], priority=TASK_PRIORITY) + update_corp.apply_async( + args=[corp['corporation_id']], + priority=TASK_PRIORITY, + countdown=randint(1, EVEONLINE_TASK_JITTER)) - # update existing alliance models + # Queue update tasks for Known Alliance Models for alliance in EveAllianceInfo.objects.all().values('alliance_id'): - update_alliance.apply_async(args=[alliance['alliance_id']], priority=TASK_PRIORITY) + update_alliance.apply_async( + args=[alliance['alliance_id']], + priority=TASK_PRIORITY, + countdown=randint(1, EVEONLINE_TASK_JITTER)) - # update existing character models + # Queue update tasks for Known Character Models character_ids = EveCharacter.objects.all().values_list('character_id', flat=True) - for character_ids_chunk in chunks(character_ids, CHUNK_SIZE): + for character_ids_chunk in chunks(character_ids, CHARACTER_AFFILIATION_CHUNK_SIZE): update_character_chunk.apply_async( - args=[character_ids_chunk], priority=TASK_PRIORITY - ) + args=[character_ids_chunk], + priority=TASK_PRIORITY, + countdown=randint(1, EVEONLINE_TASK_JITTER)) @shared_task @@ -68,8 +77,9 @@ def update_character_chunk(character_ids_chunk: list): logger.info("Failed to bulk update characters. Attempting single updates") for character_id in character_ids_chunk: update_character.apply_async( - args=[character_id], priority=TASK_PRIORITY - ) + args=[character_id], + priority=TASK_PRIORITY, + countdown=randint(1, EVEONLINE_TASK_JITTER)) return affiliations = { @@ -107,5 +117,5 @@ def update_character_chunk(character_ids_chunk: list): if corp_changed or alliance_changed or name_changed: update_character.apply_async( - args=[character.get('character_id')], priority=TASK_PRIORITY - ) + args=[character.get('character_id')], + priority=TASK_PRIORITY)