Only update characters if they have changed corp or alliance by bulk calling affiliations before calling character tasks.

This commit is contained in:
Erik Kalkoken 2020-08-20 03:14:40 +00:00 committed by Ariel Rin
parent 9ea55fa51f
commit 1d20a3029f
2 changed files with 236 additions and 28 deletions

View File

@ -5,35 +5,96 @@ from .models import EveAllianceInfo
from .models import EveCharacter
from .models import EveCorporationInfo
from . import providers
logger = logging.getLogger(__name__)
TASK_PRIORITY = 7
CHUNK_SIZE = 500
def chunks(lst, n):
"""Yield successive n-sized chunks from lst."""
for i in range(0, len(lst), n):
yield lst[i:i + n]
@shared_task
def update_corp(corp_id):
"""Update given corporation from ESI"""
EveCorporationInfo.objects.update_corporation(corp_id)
@shared_task
def update_alliance(alliance_id):
"""Update given alliance from ESI"""
EveAllianceInfo.objects.update_alliance(alliance_id).populate_alliance()
@shared_task
def update_character(character_id):
"""Update given character from ESI"""
EveCharacter.objects.update_character(character_id)
@shared_task
def run_model_update():
"""Update all alliances, corporations and characters from ESI"""
# update existing corp models
for corp in EveCorporationInfo.objects.all().values('corporation_id'):
update_corp.apply_async(args=[corp['corporation_id']], priority=TASK_PRIORITY)
update_corp.apply_async(
args=[corp['corporation_id']], priority=TASK_PRIORITY
)
# update existing alliance models
for alliance in EveAllianceInfo.objects.all().values('alliance_id'):
update_alliance.apply_async(args=[alliance['alliance_id']], priority=TASK_PRIORITY)
update_alliance.apply_async(
args=[alliance['alliance_id']], priority=TASK_PRIORITY
)
#update existing character models
for character in EveCharacter.objects.all().values('character_id'):
update_character.apply_async(args=[character['character_id']], priority=TASK_PRIORITY)
# update existing character models
character_ids = EveCharacter.objects.all().values_list('character_id', flat=True)
for character_ids_chunk in chunks(character_ids, CHUNK_SIZE):
affiliations_raw = providers.provider.client.Character\
.post_characters_affiliation(characters=character_ids_chunk).result()
character_names = providers.provider.client.Universe\
.post_universe_names(ids=character_ids_chunk).result()
affiliations = {
affiliation.get('character_id'): affiliation
for affiliation in affiliations_raw
}
# add character names to affiliations
for character in character_names:
character_id = character.get('id')
if character_id in affiliations:
affiliations[character_id]['name'] = character.get('name')
# fetch current characters
characters = EveCharacter.objects.filter(character_id__in=character_ids_chunk)\
.values('character_id', 'corporation_id', 'alliance_id', 'character_name')
for character in characters:
character_id = character.get('character_id')
if character_id in affiliations:
affiliation = affiliations[character_id]
corp_changed = (
character.get('corporation_id') != affiliation.get('corporation_id')
)
alliance_id = character.get('alliance_id')
if not alliance_id:
alliance_id = None
alliance_changed = alliance_id != affiliation.get('alliance_id')
name_changed = False
fetched_name = affiliation.get('name', False)
if fetched_name:
name_changed = character.get('character_name') != fetched_name
if corp_changed or alliance_changed or name_changed:
update_character.apply_async(
args=[character.get('character_id')], priority=TASK_PRIORITY
)

View File

@ -1,4 +1,4 @@
from unittest.mock import patch
from unittest.mock import patch, Mock
from django.test import TestCase
@ -44,55 +44,202 @@ class TestTasks(TestCase):
mock_EveCharacter.objects.update_character.call_args[0][0], 42
)
@patch('allianceauth.eveonline.tasks.update_character')
@patch('allianceauth.eveonline.tasks.update_alliance')
@patch('allianceauth.eveonline.tasks.update_corp')
def test_run_model_update(
self,
mock_update_corp,
mock_update_alliance,
mock_update_character,
):
@patch('allianceauth.eveonline.tasks.update_character')
@patch('allianceauth.eveonline.tasks.update_alliance')
@patch('allianceauth.eveonline.tasks.update_corp')
@patch('allianceauth.eveonline.providers.provider')
@patch('allianceauth.eveonline.tasks.CHUNK_SIZE', 2)
class TestRunModelUpdate(TestCase):
@classmethod
def setUpClass(cls):
super().setUpClass()
EveCorporationInfo.objects.all().delete()
EveAllianceInfo.objects.all().delete()
EveCharacter.objects.all().delete()
EveCorporationInfo.objects.create(
corporation_id=2345,
corporation_name='corp.name',
corporation_ticker='corp.ticker',
corporation_ticker='c.c.t',
member_count=10,
alliance=None,
)
EveAllianceInfo.objects.create(
alliance_id=3456,
alliance_name='alliance.name',
alliance_ticker='alliance.ticker',
executor_corp_id='78910',
alliance_ticker='a.t',
executor_corp_id=5,
)
EveCharacter.objects.create(
character_id=1,
character_name='character.name1',
corporation_id=2345,
corporation_name='character.corp.name',
corporation_ticker='c.c.t', # max 5 chars
alliance_id=None
)
EveCharacter.objects.create(
character_id=1234,
character_name='character.name',
corporation_id=2345,
character_id=2,
character_name='character.name2',
corporation_id=9876,
corporation_name='character.corp.name',
corporation_ticker='c.c.t', # max 5 chars
alliance_id=3456,
alliance_name='character.alliance.name',
)
EveCharacter.objects.create(
character_id=3,
character_name='character.name3',
corporation_id=9876,
corporation_name='character.corp.name',
corporation_ticker='c.c.t', # max 5 chars
alliance_id=3456,
alliance_name='character.alliance.name',
)
EveCharacter.objects.create(
character_id=4,
character_name='character.name4',
corporation_id=9876,
corporation_name='character.corp.name',
corporation_ticker='c.c.t', # max 5 chars
alliance_id=3456,
alliance_name='character.alliance.name',
)
"""
EveCharacter.objects.create(
character_id=5,
character_name='character.name5',
corporation_id=9876,
corporation_name='character.corp.name',
corporation_ticker='c.c.t', # max 5 chars
alliance_id=3456,
alliance_name='character.alliance.name',
)
"""
def setUp(self):
self.affiliations = [
{'character_id': 1, 'corporation_id': 5},
{'character_id': 2, 'corporation_id': 9876, 'alliance_id': 3456},
{'character_id': 3, 'corporation_id': 9876, 'alliance_id': 7456},
{'character_id': 4, 'corporation_id': 9876, 'alliance_id': 3456}
]
self.names = [
{'id': 1, 'name': 'character.name1'},
{'id': 2, 'name': 'character.name2'},
{'id': 3, 'name': 'character.name3'},
{'id': 4, 'name': 'character.name4_new'}
]
def test_normal_run(
self,
mock_provider,
mock_update_corp,
mock_update_alliance,
mock_update_character,
):
def get_affiliations(characters: list):
response = [x for x in self.affiliations if x['character_id'] in characters]
mock_operator = Mock(**{'result.return_value': response})
return mock_operator
def get_names(ids: list):
response = [x for x in self.names if x['id'] in ids]
mock_operator = Mock(**{'result.return_value': response})
return mock_operator
mock_provider.client.Character.post_characters_affiliation.side_effect \
= get_affiliations
mock_provider.client.Universe.post_universe_names.side_effect = get_names
run_model_update()
self.assertEqual(
mock_provider.client.Character.post_characters_affiliation.call_count, 2
)
self.assertEqual(
mock_provider.client.Universe.post_universe_names.call_count, 2
)
# character 1 has changed corp
# character 2 no change
# character 3 has changed alliance
# character 4 has changed name
self.assertEqual(mock_update_corp.apply_async.call_count, 1)
self.assertEqual(
int(mock_update_corp.apply_async.call_args[1]['args'][0]), 2345
)
self.assertEqual(mock_update_alliance.apply_async.call_count, 1)
self.assertEqual(
int(mock_update_alliance.apply_async.call_args[1]['args'][0]), 3456
)
)
characters_updated = {
x[1]['args'][0] for x in mock_update_character.apply_async.call_args_list
}
excepted = {1, 3, 4}
self.assertSetEqual(characters_updated, excepted)
def test_ignore_character_not_in_affiliations(
self,
mock_provider,
mock_update_corp,
mock_update_alliance,
mock_update_character,
):
def get_affiliations(characters: list):
response = [x for x in self.affiliations if x['character_id'] in characters]
mock_operator = Mock(**{'result.return_value': response})
return mock_operator
def get_names(ids: list):
response = [x for x in self.names if x['id'] in ids]
mock_operator = Mock(**{'result.return_value': response})
return mock_operator
del self.affiliations[0]
mock_provider.client.Character.post_characters_affiliation.side_effect \
= get_affiliations
mock_provider.client.Universe.post_universe_names.side_effect = get_names
self.assertEqual(mock_update_character.apply_async.call_count, 1)
self.assertEqual(
int(mock_update_character.apply_async.call_args[1]['args'][0]), 1234
)
run_model_update()
characters_updated = {
x[1]['args'][0] for x in mock_update_character.apply_async.call_args_list
}
excepted = {3, 4}
self.assertSetEqual(characters_updated, excepted)
def test_ignore_character_not_in_names(
self,
mock_provider,
mock_update_corp,
mock_update_alliance,
mock_update_character,
):
def get_affiliations(characters: list):
response = [x for x in self.affiliations if x['character_id'] in characters]
mock_operator = Mock(**{'result.return_value': response})
return mock_operator
def get_names(ids: list):
response = [x for x in self.names if x['id'] in ids]
mock_operator = Mock(**{'result.return_value': response})
return mock_operator
del self.names[3]
mock_provider.client.Character.post_characters_affiliation.side_effect \
= get_affiliations
mock_provider.client.Universe.post_universe_names.side_effect = get_names
run_model_update()
characters_updated = {
x[1]['args'][0] for x in mock_update_character.apply_async.call_args_list
}
excepted = {1, 3}
self.assertSetEqual(characters_updated, excepted)