Mathis/allianceauth
1
0
Fork 0
mirror of https://gitlab.com/allianceauth/allianceauth.git synced 2026-02-04 06:06:19 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'randomdelay' into 'master'

Browse Source 
Spread esi tasks over 10 minutes

See merge request allianceauth/allianceauth!1666
This commit is contained in:
Ariel Rin
2024-12-09 23:56:13 +00:00
parent 2a894cd62c 8489f204dd
commit 795a7e006f
3 changed files with 72 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files

42
allianceauth/eveonline/tasks.py
View File

@@ -1,4 +1,5 @@
import logging
from random import randint
from celery import shared_task
@@ -9,7 +10,8 @@ from . import providers
logger = logging.getLogger(__name__)
TASK_PRIORITY = 7
CHUNK_SIZE = 500
CHARACTER_AFFILIATION_CHUNK_SIZE = 500
EVEONLINE_TASK_JITTER = 600
def chunks(lst, n):
@@ -19,13 +21,13 @@ def chunks(lst, n):
@shared_task
def update_corp(corp_id):
def update_corp(corp_id: int) -> None:
"""Update given corporation from ESI"""
EveCorporationInfo.objects.update_corporation(corp_id)
@shared_task
def update_alliance(alliance_id):
def update_alliance(alliance_id: int) -> None:
"""Update given alliance from ESI"""
EveAllianceInfo.objects.update_alliance(alliance_id).populate_alliance()
@@ -37,23 +39,30 @@ def update_character(character_id: int) -> None:
@shared_task
def run_model_update():
def run_model_update() -> None:
"""Update all alliances, corporations and characters from ESI"""
#update existing corp models
# Queue update tasks for Known Corporation Models
for corp in EveCorporationInfo.objects.all().values('corporation_id'):
update_corp.apply_async(args=[corp['corporation_id']], priority=TASK_PRIORITY)
update_corp.apply_async(
args=[corp['corporation_id']],
priority=TASK_PRIORITY,
countdown=randint(1, EVEONLINE_TASK_JITTER))
# update existing alliance models
# Queue update tasks for Known Alliance Models
for alliance in EveAllianceInfo.objects.all().values('alliance_id'):
update_alliance.apply_async(args=[alliance['alliance_id']], priority=TASK_PRIORITY)
update_alliance.apply_async(
args=[alliance['alliance_id']],
priority=TASK_PRIORITY,
countdown=randint(1, EVEONLINE_TASK_JITTER))
# update existing character models
# Queue update tasks for Known Character Models
character_ids = EveCharacter.objects.all().values_list('character_id', flat=True)
for character_ids_chunk in chunks(character_ids, CHUNK_SIZE):
for character_ids_chunk in chunks(character_ids, CHARACTER_AFFILIATION_CHUNK_SIZE):
update_character_chunk.apply_async(
args=[character_ids_chunk], priority=TASK_PRIORITY
)
args=[character_ids_chunk],
priority=TASK_PRIORITY,
countdown=randint(1, EVEONLINE_TASK_JITTER))
@shared_task
@@ -68,8 +77,9 @@ def update_character_chunk(character_ids_chunk: list):
logger.info("Failed to bulk update characters. Attempting single updates")
for character_id in character_ids_chunk:
update_character.apply_async(
args=[character_id], priority=TASK_PRIORITY
)
args=[character_id],
priority=TASK_PRIORITY,
countdown=randint(1, EVEONLINE_TASK_JITTER))
return
affiliations = {
@@ -107,5 +117,5 @@ def update_character_chunk(character_ids_chunk: list):
if corp_changed or alliance_changed or name_changed:
update_character.apply_async(
args=[character.get('character_id')], priority=TASK_PRIORITY
)
args=[character.get('character_id')],
priority=TASK_PRIORITY)

4
allianceauth/eveonline/tests/test_tasks.py
View File

@@ -84,7 +84,7 @@ class TestUpdateTasks(TestCase):
@override_settings(CELERY_ALWAYS_EAGER=True)
@patch('allianceauth.eveonline.providers.esi_client_factory')
@patch('allianceauth.eveonline.tasks.providers')
@patch('allianceauth.eveonline.tasks.CHUNK_SIZE', 2)
@patch('allianceauth.eveonline.tasks.CHARACTER_AFFILIATION_CHUNK_SIZE', 2)
class TestRunModelUpdate(TransactionTestCase):
def test_should_run_updates(self, mock_providers, mock_esi_client_factory):
# given
@@ -139,7 +139,7 @@ class TestRunModelUpdate(TransactionTestCase):
@patch('allianceauth.eveonline.tasks.update_character', wraps=update_character)
@patch('allianceauth.eveonline.providers.esi_client_factory')
@patch('allianceauth.eveonline.tasks.providers')
@patch('allianceauth.eveonline.tasks.CHUNK_SIZE', 2)
@patch('allianceauth.eveonline.tasks.CHARACTER_AFFILIATION_CHUNK_SIZE', 2)
class TestUpdateCharacterChunk(TestCase):
@staticmethod
def _updated_character_ids(spy_update_character) -> set: