Mathis/allianceauth
1
0
Fork 0
mirror of https://gitlab.com/allianceauth/allianceauth.git synced 2025-07-09 12:30:15 +02:00
Code Issues Packages Projects Releases Wiki Activity

Spread esi tasks over 10 minutes

Browse Source
This commit is contained in:
Joel Falknau 2024-12-04 18:01:01 +10:00
parent 164cd4fbb2
commit c6890dd2c6
No known key found for this signature in database
1 changed files with 26 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

42
allianceauth/eveonline/tasks.py
View File

@ -1,4 +1,5 @@
import logging import logging
from random import randint
from celery import shared_task from celery import shared_task
@ -9,7 +10,8 @@ from . import providers
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
TASK_PRIORITY = 7 TASK_PRIORITY = 7
CHUNK_SIZE = 500 CHARACTER_AFFILIATION_CHUNK_SIZE = 500
EVEONLINE_TASK_JITTER = 600
def chunks(lst, n): def chunks(lst, n):
@ -19,13 +21,13 @@ def chunks(lst, n):
@shared_task @shared_task
def update_corp(corp_id): def update_corp(corp_id: int) -> None:
"""Update given corporation from ESI""" """Update given corporation from ESI"""
EveCorporationInfo.objects.update_corporation(corp_id) EveCorporationInfo.objects.update_corporation(corp_id)
@shared_task @shared_task
def update_alliance(alliance_id): def update_alliance(alliance_id: int) -> None:
"""Update given alliance from ESI""" """Update given alliance from ESI"""
EveAllianceInfo.objects.update_alliance(alliance_id).populate_alliance() EveAllianceInfo.objects.update_alliance(alliance_id).populate_alliance()
@ -37,23 +39,30 @@ def update_character(character_id: int) -> None:
@shared_task @shared_task
def run_model_update(): def run_model_update() -> None:
"""Update all alliances, corporations and characters from ESI""" """Update all alliances, corporations and characters from ESI"""
#update existing corp models # Queue update tasks for Known Corporation Models
for corp in EveCorporationInfo.objects.all().values('corporation_id'): for corp in EveCorporationInfo.objects.all().values('corporation_id'):
update_corp.apply_async(args=[corp['corporation_id']], priority=TASK_PRIORITY) update_corp.apply_async(
args=[corp['corporation_id']],
priority=TASK_PRIORITY,
countdown=randint(1, EVEONLINE_TASK_JITTER))
# update existing alliance models # Queue update tasks for Known Alliance Models
for alliance in EveAllianceInfo.objects.all().values('alliance_id'): for alliance in EveAllianceInfo.objects.all().values('alliance_id'):
update_alliance.apply_async(args=[alliance['alliance_id']], priority=TASK_PRIORITY) update_alliance.apply_async(
args=[alliance['alliance_id']],
priority=TASK_PRIORITY,
countdown=randint(1, EVEONLINE_TASK_JITTER))
# update existing character models # Queue update tasks for Known Character Models
character_ids = EveCharacter.objects.all().values_list('character_id', flat=True) character_ids = EveCharacter.objects.all().values_list('character_id', flat=True)
for character_ids_chunk in chunks(character_ids, CHUNK_SIZE): for character_ids_chunk in chunks(character_ids, CHARACTER_AFFILIATION_CHUNK_SIZE):
update_character_chunk.apply_async( update_character_chunk.apply_async(
args=[character_ids_chunk], priority=TASK_PRIORITY args=[character_ids_chunk],
) priority=TASK_PRIORITY,
countdown=randint(1, EVEONLINE_TASK_JITTER))
@shared_task @shared_task
@ -68,8 +77,9 @@ def update_character_chunk(character_ids_chunk: list):
logger.info("Failed to bulk update characters. Attempting single updates") logger.info("Failed to bulk update characters. Attempting single updates")
for character_id in character_ids_chunk: for character_id in character_ids_chunk:
update_character.apply_async( update_character.apply_async(
args=[character_id], priority=TASK_PRIORITY args=[character_id],
) priority=TASK_PRIORITY,
countdown=randint(1, EVEONLINE_TASK_JITTER))
return return
affiliations = { affiliations = {
@ -107,5 +117,5 @@ def update_character_chunk(character_ids_chunk: list):
if corp_changed or alliance_changed or name_changed: if corp_changed or alliance_changed or name_changed:
update_character.apply_async( update_character.apply_async(
args=[character.get('character_id')], priority=TASK_PRIORITY args=[character.get('character_id')],
) priority=TASK_PRIORITY)