mirror of
https://gitlab.com/allianceauth/allianceauth.git
synced 2025-07-10 13:00:16 +02:00
Only update characters if they have changed corp or alliance by bulk calling affiliations before calling character tasks.
This commit is contained in:
parent
9ea55fa51f
commit
1d20a3029f
@ -5,35 +5,96 @@ from .models import EveAllianceInfo
|
|||||||
from .models import EveCharacter
|
from .models import EveCharacter
|
||||||
from .models import EveCorporationInfo
|
from .models import EveCorporationInfo
|
||||||
|
|
||||||
|
from . import providers
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
TASK_PRIORITY = 7
|
TASK_PRIORITY = 7
|
||||||
|
CHUNK_SIZE = 500
|
||||||
|
|
||||||
|
|
||||||
|
def chunks(lst, n):
|
||||||
|
"""Yield successive n-sized chunks from lst."""
|
||||||
|
for i in range(0, len(lst), n):
|
||||||
|
yield lst[i:i + n]
|
||||||
|
|
||||||
|
|
||||||
@shared_task
|
@shared_task
|
||||||
def update_corp(corp_id):
|
def update_corp(corp_id):
|
||||||
|
"""Update given corporation from ESI"""
|
||||||
EveCorporationInfo.objects.update_corporation(corp_id)
|
EveCorporationInfo.objects.update_corporation(corp_id)
|
||||||
|
|
||||||
|
|
||||||
@shared_task
|
@shared_task
|
||||||
def update_alliance(alliance_id):
|
def update_alliance(alliance_id):
|
||||||
|
"""Update given alliance from ESI"""
|
||||||
EveAllianceInfo.objects.update_alliance(alliance_id).populate_alliance()
|
EveAllianceInfo.objects.update_alliance(alliance_id).populate_alliance()
|
||||||
|
|
||||||
|
|
||||||
@shared_task
|
@shared_task
|
||||||
def update_character(character_id):
|
def update_character(character_id):
|
||||||
|
"""Update given character from ESI"""
|
||||||
EveCharacter.objects.update_character(character_id)
|
EveCharacter.objects.update_character(character_id)
|
||||||
|
|
||||||
|
|
||||||
@shared_task
|
@shared_task
|
||||||
def run_model_update():
|
def run_model_update():
|
||||||
|
"""Update all alliances, corporations and characters from ESI"""
|
||||||
|
|
||||||
# update existing corp models
|
# update existing corp models
|
||||||
for corp in EveCorporationInfo.objects.all().values('corporation_id'):
|
for corp in EveCorporationInfo.objects.all().values('corporation_id'):
|
||||||
update_corp.apply_async(args=[corp['corporation_id']], priority=TASK_PRIORITY)
|
update_corp.apply_async(
|
||||||
|
args=[corp['corporation_id']], priority=TASK_PRIORITY
|
||||||
|
)
|
||||||
|
|
||||||
# update existing alliance models
|
# update existing alliance models
|
||||||
for alliance in EveAllianceInfo.objects.all().values('alliance_id'):
|
for alliance in EveAllianceInfo.objects.all().values('alliance_id'):
|
||||||
update_alliance.apply_async(args=[alliance['alliance_id']], priority=TASK_PRIORITY)
|
update_alliance.apply_async(
|
||||||
|
args=[alliance['alliance_id']], priority=TASK_PRIORITY
|
||||||
|
)
|
||||||
|
|
||||||
#update existing character models
|
# update existing character models
|
||||||
for character in EveCharacter.objects.all().values('character_id'):
|
character_ids = EveCharacter.objects.all().values_list('character_id', flat=True)
|
||||||
update_character.apply_async(args=[character['character_id']], priority=TASK_PRIORITY)
|
for character_ids_chunk in chunks(character_ids, CHUNK_SIZE):
|
||||||
|
affiliations_raw = providers.provider.client.Character\
|
||||||
|
.post_characters_affiliation(characters=character_ids_chunk).result()
|
||||||
|
character_names = providers.provider.client.Universe\
|
||||||
|
.post_universe_names(ids=character_ids_chunk).result()
|
||||||
|
|
||||||
|
affiliations = {
|
||||||
|
affiliation.get('character_id'): affiliation
|
||||||
|
for affiliation in affiliations_raw
|
||||||
|
}
|
||||||
|
# add character names to affiliations
|
||||||
|
for character in character_names:
|
||||||
|
character_id = character.get('id')
|
||||||
|
if character_id in affiliations:
|
||||||
|
affiliations[character_id]['name'] = character.get('name')
|
||||||
|
|
||||||
|
# fetch current characters
|
||||||
|
characters = EveCharacter.objects.filter(character_id__in=character_ids_chunk)\
|
||||||
|
.values('character_id', 'corporation_id', 'alliance_id', 'character_name')
|
||||||
|
|
||||||
|
for character in characters:
|
||||||
|
character_id = character.get('character_id')
|
||||||
|
if character_id in affiliations:
|
||||||
|
affiliation = affiliations[character_id]
|
||||||
|
|
||||||
|
corp_changed = (
|
||||||
|
character.get('corporation_id') != affiliation.get('corporation_id')
|
||||||
|
)
|
||||||
|
|
||||||
|
alliance_id = character.get('alliance_id')
|
||||||
|
if not alliance_id:
|
||||||
|
alliance_id = None
|
||||||
|
alliance_changed = alliance_id != affiliation.get('alliance_id')
|
||||||
|
|
||||||
|
name_changed = False
|
||||||
|
fetched_name = affiliation.get('name', False)
|
||||||
|
if fetched_name:
|
||||||
|
name_changed = character.get('character_name') != fetched_name
|
||||||
|
|
||||||
|
if corp_changed or alliance_changed or name_changed:
|
||||||
|
update_character.apply_async(
|
||||||
|
args=[character.get('character_id')], priority=TASK_PRIORITY
|
||||||
|
)
|
||||||
|
@ -1,4 +1,4 @@
|
|||||||
from unittest.mock import patch
|
from unittest.mock import patch, Mock
|
||||||
|
|
||||||
from django.test import TestCase
|
from django.test import TestCase
|
||||||
|
|
||||||
@ -44,55 +44,202 @@ class TestTasks(TestCase):
|
|||||||
mock_EveCharacter.objects.update_character.call_args[0][0], 42
|
mock_EveCharacter.objects.update_character.call_args[0][0], 42
|
||||||
)
|
)
|
||||||
|
|
||||||
@patch('allianceauth.eveonline.tasks.update_character')
|
|
||||||
@patch('allianceauth.eveonline.tasks.update_alliance')
|
@patch('allianceauth.eveonline.tasks.update_character')
|
||||||
@patch('allianceauth.eveonline.tasks.update_corp')
|
@patch('allianceauth.eveonline.tasks.update_alliance')
|
||||||
def test_run_model_update(
|
@patch('allianceauth.eveonline.tasks.update_corp')
|
||||||
self,
|
@patch('allianceauth.eveonline.providers.provider')
|
||||||
mock_update_corp,
|
@patch('allianceauth.eveonline.tasks.CHUNK_SIZE', 2)
|
||||||
mock_update_alliance,
|
class TestRunModelUpdate(TestCase):
|
||||||
mock_update_character,
|
|
||||||
):
|
@classmethod
|
||||||
|
def setUpClass(cls):
|
||||||
|
super().setUpClass()
|
||||||
EveCorporationInfo.objects.all().delete()
|
EveCorporationInfo.objects.all().delete()
|
||||||
EveAllianceInfo.objects.all().delete()
|
EveAllianceInfo.objects.all().delete()
|
||||||
EveCharacter.objects.all().delete()
|
EveCharacter.objects.all().delete()
|
||||||
|
|
||||||
EveCorporationInfo.objects.create(
|
EveCorporationInfo.objects.create(
|
||||||
corporation_id=2345,
|
corporation_id=2345,
|
||||||
corporation_name='corp.name',
|
corporation_name='corp.name',
|
||||||
corporation_ticker='corp.ticker',
|
corporation_ticker='c.c.t',
|
||||||
member_count=10,
|
member_count=10,
|
||||||
alliance=None,
|
alliance=None,
|
||||||
)
|
)
|
||||||
EveAllianceInfo.objects.create(
|
EveAllianceInfo.objects.create(
|
||||||
alliance_id=3456,
|
alliance_id=3456,
|
||||||
alliance_name='alliance.name',
|
alliance_name='alliance.name',
|
||||||
alliance_ticker='alliance.ticker',
|
alliance_ticker='a.t',
|
||||||
executor_corp_id='78910',
|
executor_corp_id=5,
|
||||||
|
)
|
||||||
|
EveCharacter.objects.create(
|
||||||
|
character_id=1,
|
||||||
|
character_name='character.name1',
|
||||||
|
corporation_id=2345,
|
||||||
|
corporation_name='character.corp.name',
|
||||||
|
corporation_ticker='c.c.t', # max 5 chars
|
||||||
|
alliance_id=None
|
||||||
)
|
)
|
||||||
EveCharacter.objects.create(
|
EveCharacter.objects.create(
|
||||||
character_id=1234,
|
character_id=2,
|
||||||
character_name='character.name',
|
character_name='character.name2',
|
||||||
corporation_id=2345,
|
corporation_id=9876,
|
||||||
corporation_name='character.corp.name',
|
corporation_name='character.corp.name',
|
||||||
corporation_ticker='c.c.t', # max 5 chars
|
corporation_ticker='c.c.t', # max 5 chars
|
||||||
alliance_id=3456,
|
alliance_id=3456,
|
||||||
alliance_name='character.alliance.name',
|
alliance_name='character.alliance.name',
|
||||||
)
|
)
|
||||||
|
EveCharacter.objects.create(
|
||||||
|
character_id=3,
|
||||||
|
character_name='character.name3',
|
||||||
|
corporation_id=9876,
|
||||||
|
corporation_name='character.corp.name',
|
||||||
|
corporation_ticker='c.c.t', # max 5 chars
|
||||||
|
alliance_id=3456,
|
||||||
|
alliance_name='character.alliance.name',
|
||||||
|
)
|
||||||
|
EveCharacter.objects.create(
|
||||||
|
character_id=4,
|
||||||
|
character_name='character.name4',
|
||||||
|
corporation_id=9876,
|
||||||
|
corporation_name='character.corp.name',
|
||||||
|
corporation_ticker='c.c.t', # max 5 chars
|
||||||
|
alliance_id=3456,
|
||||||
|
alliance_name='character.alliance.name',
|
||||||
|
)
|
||||||
|
"""
|
||||||
|
EveCharacter.objects.create(
|
||||||
|
character_id=5,
|
||||||
|
character_name='character.name5',
|
||||||
|
corporation_id=9876,
|
||||||
|
corporation_name='character.corp.name',
|
||||||
|
corporation_ticker='c.c.t', # max 5 chars
|
||||||
|
alliance_id=3456,
|
||||||
|
alliance_name='character.alliance.name',
|
||||||
|
)
|
||||||
|
"""
|
||||||
|
|
||||||
|
def setUp(self):
|
||||||
|
self.affiliations = [
|
||||||
|
{'character_id': 1, 'corporation_id': 5},
|
||||||
|
{'character_id': 2, 'corporation_id': 9876, 'alliance_id': 3456},
|
||||||
|
{'character_id': 3, 'corporation_id': 9876, 'alliance_id': 7456},
|
||||||
|
{'character_id': 4, 'corporation_id': 9876, 'alliance_id': 3456}
|
||||||
|
]
|
||||||
|
self.names = [
|
||||||
|
{'id': 1, 'name': 'character.name1'},
|
||||||
|
{'id': 2, 'name': 'character.name2'},
|
||||||
|
{'id': 3, 'name': 'character.name3'},
|
||||||
|
{'id': 4, 'name': 'character.name4_new'}
|
||||||
|
]
|
||||||
|
|
||||||
|
def test_normal_run(
|
||||||
|
self,
|
||||||
|
mock_provider,
|
||||||
|
mock_update_corp,
|
||||||
|
mock_update_alliance,
|
||||||
|
mock_update_character,
|
||||||
|
):
|
||||||
|
def get_affiliations(characters: list):
|
||||||
|
response = [x for x in self.affiliations if x['character_id'] in characters]
|
||||||
|
mock_operator = Mock(**{'result.return_value': response})
|
||||||
|
return mock_operator
|
||||||
|
|
||||||
|
def get_names(ids: list):
|
||||||
|
response = [x for x in self.names if x['id'] in ids]
|
||||||
|
mock_operator = Mock(**{'result.return_value': response})
|
||||||
|
return mock_operator
|
||||||
|
|
||||||
|
mock_provider.client.Character.post_characters_affiliation.side_effect \
|
||||||
|
= get_affiliations
|
||||||
|
|
||||||
|
mock_provider.client.Universe.post_universe_names.side_effect = get_names
|
||||||
|
|
||||||
run_model_update()
|
run_model_update()
|
||||||
|
|
||||||
|
self.assertEqual(
|
||||||
|
mock_provider.client.Character.post_characters_affiliation.call_count, 2
|
||||||
|
)
|
||||||
|
self.assertEqual(
|
||||||
|
mock_provider.client.Universe.post_universe_names.call_count, 2
|
||||||
|
)
|
||||||
|
|
||||||
|
# character 1 has changed corp
|
||||||
|
# character 2 no change
|
||||||
|
# character 3 has changed alliance
|
||||||
|
# character 4 has changed name
|
||||||
self.assertEqual(mock_update_corp.apply_async.call_count, 1)
|
self.assertEqual(mock_update_corp.apply_async.call_count, 1)
|
||||||
self.assertEqual(
|
self.assertEqual(
|
||||||
int(mock_update_corp.apply_async.call_args[1]['args'][0]), 2345
|
int(mock_update_corp.apply_async.call_args[1]['args'][0]), 2345
|
||||||
)
|
)
|
||||||
|
|
||||||
self.assertEqual(mock_update_alliance.apply_async.call_count, 1)
|
self.assertEqual(mock_update_alliance.apply_async.call_count, 1)
|
||||||
self.assertEqual(
|
self.assertEqual(
|
||||||
int(mock_update_alliance.apply_async.call_args[1]['args'][0]), 3456
|
int(mock_update_alliance.apply_async.call_args[1]['args'][0]), 3456
|
||||||
)
|
)
|
||||||
|
characters_updated = {
|
||||||
|
x[1]['args'][0] for x in mock_update_character.apply_async.call_args_list
|
||||||
|
}
|
||||||
|
excepted = {1, 3, 4}
|
||||||
|
self.assertSetEqual(characters_updated, excepted)
|
||||||
|
|
||||||
|
def test_ignore_character_not_in_affiliations(
|
||||||
|
self,
|
||||||
|
mock_provider,
|
||||||
|
mock_update_corp,
|
||||||
|
mock_update_alliance,
|
||||||
|
mock_update_character,
|
||||||
|
):
|
||||||
|
def get_affiliations(characters: list):
|
||||||
|
response = [x for x in self.affiliations if x['character_id'] in characters]
|
||||||
|
mock_operator = Mock(**{'result.return_value': response})
|
||||||
|
return mock_operator
|
||||||
|
|
||||||
|
def get_names(ids: list):
|
||||||
|
response = [x for x in self.names if x['id'] in ids]
|
||||||
|
mock_operator = Mock(**{'result.return_value': response})
|
||||||
|
return mock_operator
|
||||||
|
|
||||||
|
del self.affiliations[0]
|
||||||
|
|
||||||
|
mock_provider.client.Character.post_characters_affiliation.side_effect \
|
||||||
|
= get_affiliations
|
||||||
|
|
||||||
|
mock_provider.client.Universe.post_universe_names.side_effect = get_names
|
||||||
|
|
||||||
self.assertEqual(mock_update_character.apply_async.call_count, 1)
|
run_model_update()
|
||||||
self.assertEqual(
|
characters_updated = {
|
||||||
int(mock_update_character.apply_async.call_args[1]['args'][0]), 1234
|
x[1]['args'][0] for x in mock_update_character.apply_async.call_args_list
|
||||||
)
|
}
|
||||||
|
excepted = {3, 4}
|
||||||
|
self.assertSetEqual(characters_updated, excepted)
|
||||||
|
|
||||||
|
def test_ignore_character_not_in_names(
|
||||||
|
self,
|
||||||
|
mock_provider,
|
||||||
|
mock_update_corp,
|
||||||
|
mock_update_alliance,
|
||||||
|
mock_update_character,
|
||||||
|
):
|
||||||
|
def get_affiliations(characters: list):
|
||||||
|
response = [x for x in self.affiliations if x['character_id'] in characters]
|
||||||
|
mock_operator = Mock(**{'result.return_value': response})
|
||||||
|
return mock_operator
|
||||||
|
|
||||||
|
def get_names(ids: list):
|
||||||
|
response = [x for x in self.names if x['id'] in ids]
|
||||||
|
mock_operator = Mock(**{'result.return_value': response})
|
||||||
|
return mock_operator
|
||||||
|
|
||||||
|
del self.names[3]
|
||||||
|
|
||||||
|
mock_provider.client.Character.post_characters_affiliation.side_effect \
|
||||||
|
= get_affiliations
|
||||||
|
|
||||||
|
mock_provider.client.Universe.post_universe_names.side_effect = get_names
|
||||||
|
|
||||||
|
run_model_update()
|
||||||
|
characters_updated = {
|
||||||
|
x[1]['args'][0] for x in mock_update_character.apply_async.call_args_list
|
||||||
|
}
|
||||||
|
excepted = {1, 3}
|
||||||
|
self.assertSetEqual(characters_updated, excepted)
|
||||||
|
Loading…
x
Reference in New Issue
Block a user