From 1d20a3029f95b938aa831b7ec2b74493a09f436f Mon Sep 17 00:00:00 2001 From: Erik Kalkoken Date: Thu, 20 Aug 2020 03:14:40 +0000 Subject: [PATCH] Only update characters if they have changed corp or alliance by bulk calling affiliations before calling character tasks. --- allianceauth/eveonline/tasks.py | 71 +++++++- allianceauth/eveonline/tests/test_tasks.py | 193 ++++++++++++++++++--- 2 files changed, 236 insertions(+), 28 deletions(-) diff --git a/allianceauth/eveonline/tasks.py b/allianceauth/eveonline/tasks.py index 29cb0feb..5f3f4d42 100644 --- a/allianceauth/eveonline/tasks.py +++ b/allianceauth/eveonline/tasks.py @@ -5,35 +5,96 @@ from .models import EveAllianceInfo from .models import EveCharacter from .models import EveCorporationInfo +from . import providers + logger = logging.getLogger(__name__) TASK_PRIORITY = 7 +CHUNK_SIZE = 500 + + +def chunks(lst, n): + """Yield successive n-sized chunks from lst.""" + for i in range(0, len(lst), n): + yield lst[i:i + n] + @shared_task def update_corp(corp_id): + """Update given corporation from ESI""" EveCorporationInfo.objects.update_corporation(corp_id) @shared_task def update_alliance(alliance_id): + """Update given alliance from ESI""" EveAllianceInfo.objects.update_alliance(alliance_id).populate_alliance() @shared_task def update_character(character_id): + """Update given character from ESI""" EveCharacter.objects.update_character(character_id) @shared_task def run_model_update(): + """Update all alliances, corporations and characters from ESI""" + # update existing corp models for corp in EveCorporationInfo.objects.all().values('corporation_id'): - update_corp.apply_async(args=[corp['corporation_id']], priority=TASK_PRIORITY) + update_corp.apply_async( + args=[corp['corporation_id']], priority=TASK_PRIORITY + ) # update existing alliance models for alliance in EveAllianceInfo.objects.all().values('alliance_id'): - update_alliance.apply_async(args=[alliance['alliance_id']], priority=TASK_PRIORITY) + update_alliance.apply_async( + args=[alliance['alliance_id']], priority=TASK_PRIORITY + ) - #update existing character models - for character in EveCharacter.objects.all().values('character_id'): - update_character.apply_async(args=[character['character_id']], priority=TASK_PRIORITY) + # update existing character models + character_ids = EveCharacter.objects.all().values_list('character_id', flat=True) + for character_ids_chunk in chunks(character_ids, CHUNK_SIZE): + affiliations_raw = providers.provider.client.Character\ + .post_characters_affiliation(characters=character_ids_chunk).result() + character_names = providers.provider.client.Universe\ + .post_universe_names(ids=character_ids_chunk).result() + + affiliations = { + affiliation.get('character_id'): affiliation + for affiliation in affiliations_raw + } + # add character names to affiliations + for character in character_names: + character_id = character.get('id') + if character_id in affiliations: + affiliations[character_id]['name'] = character.get('name') + + # fetch current characters + characters = EveCharacter.objects.filter(character_id__in=character_ids_chunk)\ + .values('character_id', 'corporation_id', 'alliance_id', 'character_name') + + for character in characters: + character_id = character.get('character_id') + if character_id in affiliations: + affiliation = affiliations[character_id] + + corp_changed = ( + character.get('corporation_id') != affiliation.get('corporation_id') + ) + + alliance_id = character.get('alliance_id') + if not alliance_id: + alliance_id = None + alliance_changed = alliance_id != affiliation.get('alliance_id') + + name_changed = False + fetched_name = affiliation.get('name', False) + if fetched_name: + name_changed = character.get('character_name') != fetched_name + + if corp_changed or alliance_changed or name_changed: + update_character.apply_async( + args=[character.get('character_id')], priority=TASK_PRIORITY + ) diff --git a/allianceauth/eveonline/tests/test_tasks.py b/allianceauth/eveonline/tests/test_tasks.py index e6803639..00a010a3 100644 --- a/allianceauth/eveonline/tests/test_tasks.py +++ b/allianceauth/eveonline/tests/test_tasks.py @@ -1,4 +1,4 @@ -from unittest.mock import patch +from unittest.mock import patch, Mock from django.test import TestCase @@ -44,55 +44,202 @@ class TestTasks(TestCase): mock_EveCharacter.objects.update_character.call_args[0][0], 42 ) - @patch('allianceauth.eveonline.tasks.update_character') - @patch('allianceauth.eveonline.tasks.update_alliance') - @patch('allianceauth.eveonline.tasks.update_corp') - def test_run_model_update( - self, - mock_update_corp, - mock_update_alliance, - mock_update_character, - ): + +@patch('allianceauth.eveonline.tasks.update_character') +@patch('allianceauth.eveonline.tasks.update_alliance') +@patch('allianceauth.eveonline.tasks.update_corp') +@patch('allianceauth.eveonline.providers.provider') +@patch('allianceauth.eveonline.tasks.CHUNK_SIZE', 2) +class TestRunModelUpdate(TestCase): + + @classmethod + def setUpClass(cls): + super().setUpClass() EveCorporationInfo.objects.all().delete() EveAllianceInfo.objects.all().delete() EveCharacter.objects.all().delete() - + EveCorporationInfo.objects.create( corporation_id=2345, corporation_name='corp.name', - corporation_ticker='corp.ticker', + corporation_ticker='c.c.t', member_count=10, alliance=None, ) EveAllianceInfo.objects.create( alliance_id=3456, alliance_name='alliance.name', - alliance_ticker='alliance.ticker', - executor_corp_id='78910', + alliance_ticker='a.t', + executor_corp_id=5, + ) + EveCharacter.objects.create( + character_id=1, + character_name='character.name1', + corporation_id=2345, + corporation_name='character.corp.name', + corporation_ticker='c.c.t', # max 5 chars + alliance_id=None ) EveCharacter.objects.create( - character_id=1234, - character_name='character.name', - corporation_id=2345, + character_id=2, + character_name='character.name2', + corporation_id=9876, corporation_name='character.corp.name', corporation_ticker='c.c.t', # max 5 chars alliance_id=3456, alliance_name='character.alliance.name', ) + EveCharacter.objects.create( + character_id=3, + character_name='character.name3', + corporation_id=9876, + corporation_name='character.corp.name', + corporation_ticker='c.c.t', # max 5 chars + alliance_id=3456, + alliance_name='character.alliance.name', + ) + EveCharacter.objects.create( + character_id=4, + character_name='character.name4', + corporation_id=9876, + corporation_name='character.corp.name', + corporation_ticker='c.c.t', # max 5 chars + alliance_id=3456, + alliance_name='character.alliance.name', + ) + """ + EveCharacter.objects.create( + character_id=5, + character_name='character.name5', + corporation_id=9876, + corporation_name='character.corp.name', + corporation_ticker='c.c.t', # max 5 chars + alliance_id=3456, + alliance_name='character.alliance.name', + ) + """ + + def setUp(self): + self.affiliations = [ + {'character_id': 1, 'corporation_id': 5}, + {'character_id': 2, 'corporation_id': 9876, 'alliance_id': 3456}, + {'character_id': 3, 'corporation_id': 9876, 'alliance_id': 7456}, + {'character_id': 4, 'corporation_id': 9876, 'alliance_id': 3456} + ] + self.names = [ + {'id': 1, 'name': 'character.name1'}, + {'id': 2, 'name': 'character.name2'}, + {'id': 3, 'name': 'character.name3'}, + {'id': 4, 'name': 'character.name4_new'} + ] + def test_normal_run( + self, + mock_provider, + mock_update_corp, + mock_update_alliance, + mock_update_character, + ): + def get_affiliations(characters: list): + response = [x for x in self.affiliations if x['character_id'] in characters] + mock_operator = Mock(**{'result.return_value': response}) + return mock_operator + + def get_names(ids: list): + response = [x for x in self.names if x['id'] in ids] + mock_operator = Mock(**{'result.return_value': response}) + return mock_operator + + mock_provider.client.Character.post_characters_affiliation.side_effect \ + = get_affiliations + + mock_provider.client.Universe.post_universe_names.side_effect = get_names + run_model_update() + + self.assertEqual( + mock_provider.client.Character.post_characters_affiliation.call_count, 2 + ) + self.assertEqual( + mock_provider.client.Universe.post_universe_names.call_count, 2 + ) + # character 1 has changed corp + # character 2 no change + # character 3 has changed alliance + # character 4 has changed name self.assertEqual(mock_update_corp.apply_async.call_count, 1) self.assertEqual( int(mock_update_corp.apply_async.call_args[1]['args'][0]), 2345 ) - self.assertEqual(mock_update_alliance.apply_async.call_count, 1) self.assertEqual( int(mock_update_alliance.apply_async.call_args[1]['args'][0]), 3456 - ) + ) + characters_updated = { + x[1]['args'][0] for x in mock_update_character.apply_async.call_args_list + } + excepted = {1, 3, 4} + self.assertSetEqual(characters_updated, excepted) + + def test_ignore_character_not_in_affiliations( + self, + mock_provider, + mock_update_corp, + mock_update_alliance, + mock_update_character, + ): + def get_affiliations(characters: list): + response = [x for x in self.affiliations if x['character_id'] in characters] + mock_operator = Mock(**{'result.return_value': response}) + return mock_operator + + def get_names(ids: list): + response = [x for x in self.names if x['id'] in ids] + mock_operator = Mock(**{'result.return_value': response}) + return mock_operator + + del self.affiliations[0] + + mock_provider.client.Character.post_characters_affiliation.side_effect \ + = get_affiliations + + mock_provider.client.Universe.post_universe_names.side_effect = get_names - self.assertEqual(mock_update_character.apply_async.call_count, 1) - self.assertEqual( - int(mock_update_character.apply_async.call_args[1]['args'][0]), 1234 - ) + run_model_update() + characters_updated = { + x[1]['args'][0] for x in mock_update_character.apply_async.call_args_list + } + excepted = {3, 4} + self.assertSetEqual(characters_updated, excepted) + + def test_ignore_character_not_in_names( + self, + mock_provider, + mock_update_corp, + mock_update_alliance, + mock_update_character, + ): + def get_affiliations(characters: list): + response = [x for x in self.affiliations if x['character_id'] in characters] + mock_operator = Mock(**{'result.return_value': response}) + return mock_operator + + def get_names(ids: list): + response = [x for x in self.names if x['id'] in ids] + mock_operator = Mock(**{'result.return_value': response}) + return mock_operator + + del self.names[3] + + mock_provider.client.Character.post_characters_affiliation.side_effect \ + = get_affiliations + + mock_provider.client.Universe.post_universe_names.side_effect = get_names + + run_model_update() + characters_updated = { + x[1]['args'][0] for x in mock_update_character.apply_async.call_args_list + } + excepted = {1, 3} + self.assertSetEqual(characters_updated, excepted)