mirror of
https://gitlab.com/allianceauth/allianceauth.git
synced 2025-07-10 13:00:16 +02:00
Only update characters if they have changed corp or alliance by bulk calling affiliations before calling character tasks.
This commit is contained in:
Erik Kalkoken
Ariel Rin
parent
9ea55fa51f
commit
1d20a3029f
@ -5,35 +5,96 @@ from .models import EveAllianceInfo
|
||||
from .models import EveCharacter
|
||||
from .models import EveCorporationInfo
|
||||
|
||||
from . import providers
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
TASK_PRIORITY = 7
|
||||
CHUNK_SIZE = 500
|
||||
|
||||
|
||||
def chunks(lst, n):
|
||||
"""Yield successive n-sized chunks from lst."""
|
||||
for i in range(0, len(lst), n):
|
||||
yield lst[i:i + n]
|
||||
|
||||
|
||||
@shared_task
|
||||
def update_corp(corp_id):
|
||||
"""Update given corporation from ESI"""
|
||||
EveCorporationInfo.objects.update_corporation(corp_id)
|
||||
|
||||
|
||||
@shared_task
|
||||
def update_alliance(alliance_id):
|
||||
"""Update given alliance from ESI"""
|
||||
EveAllianceInfo.objects.update_alliance(alliance_id).populate_alliance()
|
||||
|
||||
|
||||
@shared_task
|
||||
def update_character(character_id):
|
||||
"""Update given character from ESI"""
|
||||
EveCharacter.objects.update_character(character_id)
|
||||
|
||||
|
||||
@shared_task
|
||||
def run_model_update():
|
||||
"""Update all alliances, corporations and characters from ESI"""
|
||||
|
||||
# update existing corp models
|
||||
for corp in EveCorporationInfo.objects.all().values('corporation_id'):
|
||||
update_corp.apply_async(args=[corp['corporation_id']], priority=TASK_PRIORITY)
|
||||
update_corp.apply_async(
|
||||
args=[corp['corporation_id']], priority=TASK_PRIORITY
|
||||
)
|
||||
|
||||
# update existing alliance models
|
||||
for alliance in EveAllianceInfo.objects.all().values('alliance_id'):
|
||||
update_alliance.apply_async(args=[alliance['alliance_id']], priority=TASK_PRIORITY)
|
||||
update_alliance.apply_async(
|
||||
args=[alliance['alliance_id']], priority=TASK_PRIORITY
|
||||
)
|
||||
|
||||
#update existing character models
|
||||
for character in EveCharacter.objects.all().values('character_id'):
|
||||
update_character.apply_async(args=[character['character_id']], priority=TASK_PRIORITY)
|
||||
# update existing character models
|
||||
character_ids = EveCharacter.objects.all().values_list('character_id', flat=True)
|
||||
for character_ids_chunk in chunks(character_ids, CHUNK_SIZE):
|
||||
affiliations_raw = providers.provider.client.Character\
|
||||
.post_characters_affiliation(characters=character_ids_chunk).result()
|
||||
character_names = providers.provider.client.Universe\
|
||||
.post_universe_names(ids=character_ids_chunk).result()
|
||||
|
||||
affiliations = {
|
||||
affiliation.get('character_id'): affiliation
|
||||
for affiliation in affiliations_raw
|
||||
}
|
||||
# add character names to affiliations
|
||||
for character in character_names:
|
||||
character_id = character.get('id')
|
||||
if character_id in affiliations:
|
||||
affiliations[character_id]['name'] = character.get('name')
|
||||
|
||||
# fetch current characters
|
||||
characters = EveCharacter.objects.filter(character_id__in=character_ids_chunk)\
|
||||
.values('character_id', 'corporation_id', 'alliance_id', 'character_name')
|
||||
|
||||
for character in characters:
|
||||
character_id = character.get('character_id')
|
||||
if character_id in affiliations:
|
||||
affiliation = affiliations[character_id]
|
||||
|
||||
corp_changed = (
|
||||
character.get('corporation_id') != affiliation.get('corporation_id')
|
||||
)
|
||||
|
||||
alliance_id = character.get('alliance_id')
|
||||
if not alliance_id:
|
||||
alliance_id = None
|
||||
alliance_changed = alliance_id != affiliation.get('alliance_id')
|
||||
|
||||
name_changed = False
|
||||
fetched_name = affiliation.get('name', False)
|
||||
if fetched_name:
|
||||
name_changed = character.get('character_name') != fetched_name
|
||||
|
||||
if corp_changed or alliance_changed or name_changed:
|
||||
update_character.apply_async(
|
||||
args=[character.get('character_id')], priority=TASK_PRIORITY
|
||||
)
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
from unittest.mock import patch
|
||||
from unittest.mock import patch, Mock
|
||||
|
||||
from django.test import TestCase
|
||||
|
||||
@ -44,55 +44,202 @@ class TestTasks(TestCase):
|
||||
mock_EveCharacter.objects.update_character.call_args[0][0], 42
|
||||
)
|
||||
|
||||
@patch('allianceauth.eveonline.tasks.update_character')
|
||||
@patch('allianceauth.eveonline.tasks.update_alliance')
|
||||
@patch('allianceauth.eveonline.tasks.update_corp')
|
||||
def test_run_model_update(
|
||||
self,
|
||||
mock_update_corp,
|
||||
mock_update_alliance,
|
||||
mock_update_character,
|
||||
):
|
||||
|
||||
@patch('allianceauth.eveonline.tasks.update_character')
|
||||
@patch('allianceauth.eveonline.tasks.update_alliance')
|
||||
@patch('allianceauth.eveonline.tasks.update_corp')
|
||||
@patch('allianceauth.eveonline.providers.provider')
|
||||
@patch('allianceauth.eveonline.tasks.CHUNK_SIZE', 2)
|
||||
class TestRunModelUpdate(TestCase):
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
super().setUpClass()
|
||||
EveCorporationInfo.objects.all().delete()
|
||||
EveAllianceInfo.objects.all().delete()
|
||||
EveCharacter.objects.all().delete()
|
||||
|
||||
|
||||
EveCorporationInfo.objects.create(
|
||||
corporation_id=2345,
|
||||
corporation_name='corp.name',
|
||||
corporation_ticker='corp.ticker',
|
||||
corporation_ticker='c.c.t',
|
||||
member_count=10,
|
||||
alliance=None,
|
||||
)
|
||||
EveAllianceInfo.objects.create(
|
||||
alliance_id=3456,
|
||||
alliance_name='alliance.name',
|
||||
alliance_ticker='alliance.ticker',
|
||||
executor_corp_id='78910',
|
||||
alliance_ticker='a.t',
|
||||
executor_corp_id=5,
|
||||
)
|
||||
EveCharacter.objects.create(
|
||||
character_id=1,
|
||||
character_name='character.name1',
|
||||
corporation_id=2345,
|
||||
corporation_name='character.corp.name',
|
||||
corporation_ticker='c.c.t', # max 5 chars
|
||||
alliance_id=None
|
||||
)
|
||||
EveCharacter.objects.create(
|
||||
character_id=1234,
|
||||
character_name='character.name',
|
||||
corporation_id=2345,
|
||||
character_id=2,
|
||||
character_name='character.name2',
|
||||
corporation_id=9876,
|
||||
corporation_name='character.corp.name',
|
||||
corporation_ticker='c.c.t', # max 5 chars
|
||||
alliance_id=3456,
|
||||
alliance_name='character.alliance.name',
|
||||
)
|
||||
EveCharacter.objects.create(
|
||||
character_id=3,
|
||||
character_name='character.name3',
|
||||
corporation_id=9876,
|
||||
corporation_name='character.corp.name',
|
||||
corporation_ticker='c.c.t', # max 5 chars
|
||||
alliance_id=3456,
|
||||
alliance_name='character.alliance.name',
|
||||
)
|
||||
EveCharacter.objects.create(
|
||||
character_id=4,
|
||||
character_name='character.name4',
|
||||
corporation_id=9876,
|
||||
corporation_name='character.corp.name',
|
||||
corporation_ticker='c.c.t', # max 5 chars
|
||||
alliance_id=3456,
|
||||
alliance_name='character.alliance.name',
|
||||
)
|
||||
"""
|
||||
EveCharacter.objects.create(
|
||||
character_id=5,
|
||||
character_name='character.name5',
|
||||
corporation_id=9876,
|
||||
corporation_name='character.corp.name',
|
||||
corporation_ticker='c.c.t', # max 5 chars
|
||||
alliance_id=3456,
|
||||
alliance_name='character.alliance.name',
|
||||
)
|
||||
"""
|
||||
|
||||
def setUp(self):
|
||||
self.affiliations = [
|
||||
{'character_id': 1, 'corporation_id': 5},
|
||||
{'character_id': 2, 'corporation_id': 9876, 'alliance_id': 3456},
|
||||
{'character_id': 3, 'corporation_id': 9876, 'alliance_id': 7456},
|
||||
{'character_id': 4, 'corporation_id': 9876, 'alliance_id': 3456}
|
||||
]
|
||||
self.names = [
|
||||
{'id': 1, 'name': 'character.name1'},
|
||||
{'id': 2, 'name': 'character.name2'},
|
||||
{'id': 3, 'name': 'character.name3'},
|
||||
{'id': 4, 'name': 'character.name4_new'}
|
||||
]
|
||||
|
||||
def test_normal_run(
|
||||
self,
|
||||
mock_provider,
|
||||
mock_update_corp,
|
||||
mock_update_alliance,
|
||||
mock_update_character,
|
||||
):
|
||||
def get_affiliations(characters: list):
|
||||
response = [x for x in self.affiliations if x['character_id'] in characters]
|
||||
mock_operator = Mock(**{'result.return_value': response})
|
||||
return mock_operator
|
||||
|
||||
def get_names(ids: list):
|
||||
response = [x for x in self.names if x['id'] in ids]
|
||||
mock_operator = Mock(**{'result.return_value': response})
|
||||
return mock_operator
|
||||
|
||||
mock_provider.client.Character.post_characters_affiliation.side_effect \
|
||||
= get_affiliations
|
||||
|
||||
mock_provider.client.Universe.post_universe_names.side_effect = get_names
|
||||
|
||||
run_model_update()
|
||||
|
||||
self.assertEqual(
|
||||
mock_provider.client.Character.post_characters_affiliation.call_count, 2
|
||||
)
|
||||
self.assertEqual(
|
||||
mock_provider.client.Universe.post_universe_names.call_count, 2
|
||||
)
|
||||
|
||||
# character 1 has changed corp
|
||||
# character 2 no change
|
||||
# character 3 has changed alliance
|
||||
# character 4 has changed name
|
||||
self.assertEqual(mock_update_corp.apply_async.call_count, 1)
|
||||
self.assertEqual(
|
||||
int(mock_update_corp.apply_async.call_args[1]['args'][0]), 2345
|
||||
)
|
||||
|
||||
self.assertEqual(mock_update_alliance.apply_async.call_count, 1)
|
||||
self.assertEqual(
|
||||
int(mock_update_alliance.apply_async.call_args[1]['args'][0]), 3456
|
||||
)
|
||||
)
|
||||
characters_updated = {
|
||||
x[1]['args'][0] for x in mock_update_character.apply_async.call_args_list
|
||||
}
|
||||
excepted = {1, 3, 4}
|
||||
self.assertSetEqual(characters_updated, excepted)
|
||||
|
||||
def test_ignore_character_not_in_affiliations(
|
||||
self,
|
||||
mock_provider,
|
||||
mock_update_corp,
|
||||
mock_update_alliance,
|
||||
mock_update_character,
|
||||
):
|
||||
def get_affiliations(characters: list):
|
||||
response = [x for x in self.affiliations if x['character_id'] in characters]
|
||||
mock_operator = Mock(**{'result.return_value': response})
|
||||
return mock_operator
|
||||
|
||||
def get_names(ids: list):
|
||||
response = [x for x in self.names if x['id'] in ids]
|
||||
mock_operator = Mock(**{'result.return_value': response})
|
||||
return mock_operator
|
||||
|
||||
del self.affiliations[0]
|
||||
|
||||
mock_provider.client.Character.post_characters_affiliation.side_effect \
|
||||
= get_affiliations
|
||||
|
||||
mock_provider.client.Universe.post_universe_names.side_effect = get_names
|
||||
|
||||
self.assertEqual(mock_update_character.apply_async.call_count, 1)
|
||||
self.assertEqual(
|
||||
int(mock_update_character.apply_async.call_args[1]['args'][0]), 1234
|
||||
)
|
||||
run_model_update()
|
||||
characters_updated = {
|
||||
x[1]['args'][0] for x in mock_update_character.apply_async.call_args_list
|
||||
}
|
||||
excepted = {3, 4}
|
||||
self.assertSetEqual(characters_updated, excepted)
|
||||
|
||||
def test_ignore_character_not_in_names(
|
||||
self,
|
||||
mock_provider,
|
||||
mock_update_corp,
|
||||
mock_update_alliance,
|
||||
mock_update_character,
|
||||
):
|
||||
def get_affiliations(characters: list):
|
||||
response = [x for x in self.affiliations if x['character_id'] in characters]
|
||||
mock_operator = Mock(**{'result.return_value': response})
|
||||
return mock_operator
|
||||
|
||||
def get_names(ids: list):
|
||||
response = [x for x in self.names if x['id'] in ids]
|
||||
mock_operator = Mock(**{'result.return_value': response})
|
||||
return mock_operator
|
||||
|
||||
del self.names[3]
|
||||
|
||||
mock_provider.client.Character.post_characters_affiliation.side_effect \
|
||||
= get_affiliations
|
||||
|
||||
mock_provider.client.Universe.post_universe_names.side_effect = get_names
|
||||
|
||||
run_model_update()
|
||||
characters_updated = {
|
||||
x[1]['args'][0] for x in mock_update_character.apply_async.call_args_list
|
||||
}
|
||||
excepted = {1, 3}
|
||||
self.assertSetEqual(characters_updated, excepted)
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user