Only update characters if they have changed corp or alliance by bulk calling affiliations before calling character tasks.

allianceauth/eveonline/tasks.py

@@ -5,35 +5,96 @@ from .models import EveAllianceInfo
 from .models import EveCharacter
 from .models import EveCorporationInfo
 
+from . import providers
+
 logger = logging.getLogger(__name__)
 
 TASK_PRIORITY = 7
+CHUNK_SIZE = 500
+
+
+def chunks(lst, n):
+    """Yield successive n-sized chunks from lst."""
+    for i in range(0, len(lst), n):
+        yield lst[i:i + n]
+
 
 @shared_task
 def update_corp(corp_id):
+    """Update given corporation from ESI"""
     EveCorporationInfo.objects.update_corporation(corp_id)
 
 
 @shared_task
 def update_alliance(alliance_id):
+    """Update given alliance from ESI"""
     EveAllianceInfo.objects.update_alliance(alliance_id).populate_alliance()
 
 
 @shared_task
 def update_character(character_id):
+    """Update given character from ESI"""
     EveCharacter.objects.update_character(character_id)
 
 
 @shared_task
 def run_model_update():
+    """Update all alliances, corporations and characters from ESI"""
+    
     # update existing corp models
     for corp in EveCorporationInfo.objects.all().values('corporation_id'):
-        update_corp.apply_async(args=[corp['corporation_id']], priority=TASK_PRIORITY)
+        update_corp.apply_async(
+            args=[corp['corporation_id']], priority=TASK_PRIORITY
+        )
 
     # update existing alliance models
     for alliance in EveAllianceInfo.objects.all().values('alliance_id'):
-        update_alliance.apply_async(args=[alliance['alliance_id']], priority=TASK_PRIORITY)
+        update_alliance.apply_async(
+            args=[alliance['alliance_id']], priority=TASK_PRIORITY
+        )
 
-    #update existing character models
+    # update existing character models
-    for character in EveCharacter.objects.all().values('character_id'):
+    character_ids = EveCharacter.objects.all().values_list('character_id', flat=True)    
-        update_character.apply_async(args=[character['character_id']], priority=TASK_PRIORITY)
+    for character_ids_chunk in chunks(character_ids, CHUNK_SIZE):
+        affiliations_raw = providers.provider.client.Character\
+            .post_characters_affiliation(characters=character_ids_chunk).result()
+        character_names = providers.provider.client.Universe\
+            .post_universe_names(ids=character_ids_chunk).result()
+
+        affiliations = {
+            affiliation.get('character_id'): affiliation 
+            for affiliation in affiliations_raw
+        }
+        # add character names to affiliations
+        for character in character_names:            
+            character_id = character.get('id')
+            if character_id in affiliations:
+                affiliations[character_id]['name'] = character.get('name')
+
+        # fetch current characters
+        characters = EveCharacter.objects.filter(character_id__in=character_ids_chunk)\
+            .values('character_id', 'corporation_id', 'alliance_id', 'character_name')  
+        
+        for character in characters:
+            character_id = character.get('character_id')
+            if character_id in affiliations:
+                affiliation = affiliations[character_id]
+                
+                corp_changed = (
+                    character.get('corporation_id') != affiliation.get('corporation_id')
+                )
+                
+                alliance_id = character.get('alliance_id')
+                if not alliance_id:
+                    alliance_id = None
+                alliance_changed = alliance_id != affiliation.get('alliance_id')
+                
+                name_changed = False
+                fetched_name = affiliation.get('name', False)
+                if fetched_name:
+                    name_changed = character.get('character_name') != fetched_name
+                
+                if corp_changed or alliance_changed or name_changed:  
+                    update_character.apply_async(
+                        args=[character.get('character_id')], priority=TASK_PRIORITY
+                    )

allianceauth/eveonline/tests/test_tasks.py

@@ -1,4 +1,4 @@
-from unittest.mock import patch
+from unittest.mock import patch, Mock
 
 from django.test import TestCase
 
@@ -44,55 +44,202 @@ class TestTasks(TestCase):
             mock_EveCharacter.objects.update_character.call_args[0][0], 42
         )
 
-    @patch('allianceauth.eveonline.tasks.update_character')
+
-    @patch('allianceauth.eveonline.tasks.update_alliance')
+@patch('allianceauth.eveonline.tasks.update_character')
-    @patch('allianceauth.eveonline.tasks.update_corp')
+@patch('allianceauth.eveonline.tasks.update_alliance')
-    def test_run_model_update(
+@patch('allianceauth.eveonline.tasks.update_corp')
-        self,
+@patch('allianceauth.eveonline.providers.provider')
-        mock_update_corp,
+@patch('allianceauth.eveonline.tasks.CHUNK_SIZE', 2)
-        mock_update_alliance,
+class TestRunModelUpdate(TestCase):
-        mock_update_character,
+
-    ):
+    @classmethod
+    def setUpClass(cls):
+        super().setUpClass()
         EveCorporationInfo.objects.all().delete()
         EveAllianceInfo.objects.all().delete()
         EveCharacter.objects.all().delete()
-
+        
         EveCorporationInfo.objects.create(
             corporation_id=2345,
             corporation_name='corp.name',
-            corporation_ticker='corp.ticker',
+            corporation_ticker='c.c.t',
             member_count=10,
             alliance=None,
         )
         EveAllianceInfo.objects.create(
             alliance_id=3456,
             alliance_name='alliance.name',
-            alliance_ticker='alliance.ticker',
+            alliance_ticker='a.t',
-            executor_corp_id='78910',
+            executor_corp_id=5,
+        )       
+        EveCharacter.objects.create(
+            character_id=1,
+            character_name='character.name1',
+            corporation_id=2345,
+            corporation_name='character.corp.name',
+            corporation_ticker='c.c.t',  # max 5 chars 
+            alliance_id=None
         )
         EveCharacter.objects.create(
-            character_id=1234,
+            character_id=2,
-            character_name='character.name',
+            character_name='character.name2',
-            corporation_id=2345,
+            corporation_id=9876,
             corporation_name='character.corp.name',
             corporation_ticker='c.c.t',  # max 5 chars 
             alliance_id=3456,
             alliance_name='character.alliance.name',
         )
+        EveCharacter.objects.create(
+            character_id=3,
+            character_name='character.name3',
+            corporation_id=9876,
+            corporation_name='character.corp.name',
+            corporation_ticker='c.c.t',  # max 5 chars 
+            alliance_id=3456,
+            alliance_name='character.alliance.name',
+        )
+        EveCharacter.objects.create(
+            character_id=4,
+            character_name='character.name4',
+            corporation_id=9876,
+            corporation_name='character.corp.name',
+            corporation_ticker='c.c.t',  # max 5 chars 
+            alliance_id=3456,
+            alliance_name='character.alliance.name',
+        )
+        """
+        EveCharacter.objects.create(
+            character_id=5,
+            character_name='character.name5',
+            corporation_id=9876,
+            corporation_name='character.corp.name',
+            corporation_ticker='c.c.t',  # max 5 chars 
+            alliance_id=3456,
+            alliance_name='character.alliance.name',
+        )
+        """      
+        
+    def setUp(self):
+        self.affiliations = [
+            {'character_id': 1, 'corporation_id': 5},
+            {'character_id': 2, 'corporation_id': 9876, 'alliance_id': 3456},
+            {'character_id': 3, 'corporation_id': 9876, 'alliance_id': 7456},
+            {'character_id': 4, 'corporation_id': 9876, 'alliance_id': 3456}
+        ]
+        self.names = [
+            {'id': 1, 'name': 'character.name1'},
+            {'id': 2, 'name': 'character.name2'},
+            {'id': 3, 'name': 'character.name3'},
+            {'id': 4, 'name': 'character.name4_new'}
+        ]
 
+    def test_normal_run(
+        self,
+        mock_provider,
+        mock_update_corp,
+        mock_update_alliance,
+        mock_update_character,
+    ):
+        def get_affiliations(characters: list):
+            response = [x for x in self.affiliations if x['character_id'] in characters]
+            mock_operator = Mock(**{'result.return_value': response})
+            return mock_operator
+
+        def get_names(ids: list):
+            response = [x for x in self.names if x['id'] in ids]
+            mock_operator = Mock(**{'result.return_value': response})
+            return mock_operator
+
+        mock_provider.client.Character.post_characters_affiliation.side_effect \
+            = get_affiliations
+       
+        mock_provider.client.Universe.post_universe_names.side_effect = get_names
+        
         run_model_update()
+        
+        self.assertEqual(
+            mock_provider.client.Character.post_characters_affiliation.call_count, 2
+        )        
+        self.assertEqual(
+            mock_provider.client.Universe.post_universe_names.call_count, 2
+        )
 
+        # character 1 has changed corp
+        # character 2 no change
+        # character 3 has changed alliance
+        # character 4 has changed name
         self.assertEqual(mock_update_corp.apply_async.call_count, 1)
         self.assertEqual(
             int(mock_update_corp.apply_async.call_args[1]['args'][0]), 2345
         )
-        
         self.assertEqual(mock_update_alliance.apply_async.call_count, 1)
         self.assertEqual(
             int(mock_update_alliance.apply_async.call_args[1]['args'][0]), 3456
-        )
+        )        
+        characters_updated = {
+            x[1]['args'][0] for x in mock_update_character.apply_async.call_args_list
+        }
+        excepted = {1, 3, 4}
+        self.assertSetEqual(characters_updated, excepted)
+
+    def test_ignore_character_not_in_affiliations(
+        self,
+        mock_provider,
+        mock_update_corp,
+        mock_update_alliance,
+        mock_update_character,
+    ):
+        def get_affiliations(characters: list):
+            response = [x for x in self.affiliations if x['character_id'] in characters]
+            mock_operator = Mock(**{'result.return_value': response})
+            return mock_operator
+
+        def get_names(ids: list):
+            response = [x for x in self.names if x['id'] in ids]
+            mock_operator = Mock(**{'result.return_value': response})
+            return mock_operator
+
+        del self.affiliations[0]
+
+        mock_provider.client.Character.post_characters_affiliation.side_effect \
+            = get_affiliations
+       
+        mock_provider.client.Universe.post_universe_names.side_effect = get_names
         
-        self.assertEqual(mock_update_character.apply_async.call_count, 1)
+        run_model_update()
-        self.assertEqual(
+        characters_updated = {
-            int(mock_update_character.apply_async.call_args[1]['args'][0]), 1234
+            x[1]['args'][0] for x in mock_update_character.apply_async.call_args_list
-        )
+        }
+        excepted = {3, 4}
+        self.assertSetEqual(characters_updated, excepted)
+
+    def test_ignore_character_not_in_names(
+        self,
+        mock_provider,
+        mock_update_corp,
+        mock_update_alliance,
+        mock_update_character,
+    ):
+        def get_affiliations(characters: list):
+            response = [x for x in self.affiliations if x['character_id'] in characters]
+            mock_operator = Mock(**{'result.return_value': response})
+            return mock_operator
+
+        def get_names(ids: list):
+            response = [x for x in self.names if x['id'] in ids]
+            mock_operator = Mock(**{'result.return_value': response})
+            return mock_operator
+
+        del self.names[3]
+
+        mock_provider.client.Character.post_characters_affiliation.side_effect \
+            = get_affiliations
+       
+        mock_provider.client.Universe.post_universe_names.side_effect = get_names
+        
+        run_model_update()
+        characters_updated = {
+            x[1]['args'][0] for x in mock_update_character.apply_async.call_args_list
+        }
+        excepted = {1, 3}
+        self.assertSetEqual(characters_updated, excepted)