Merge branch 'master' of https://gitlab.com/allianceauth/allianceauth into django4

allianceauth/__init__.py

@@ -1,7 +1,7 @@
 # This will make sure the app is always imported when
 # Django starts so that shared_task will use this app.
 
-__version__ = '2.9.3'
+__version__ = '2.9.4'
 __title__ = 'Alliance Auth'
 __url__ = 'https://gitlab.com/allianceauth/allianceauth'
 NAME = f'{__title__} v{__version__}'

allianceauth/analytics/middleware.py

@@ -1,5 +1,6 @@
 from bs4 import BeautifulSoup
 
+from django.conf import settings
 from django.utils.deprecation import MiddlewareMixin
 from .models import AnalyticsTokens, AnalyticsIdentifier
 from .tasks import send_ga_tracking_web_view
@@ -10,6 +11,8 @@ import re
 class AnalyticsMiddleware(MiddlewareMixin):
     def process_response(self, request, response):
         """Django Middleware: Process Page Views and creates Analytics Celery Tasks"""
+        if getattr(settings, "ANALYTICS_DISABLED", False):
+            return response
         analyticstokens = AnalyticsTokens.objects.all()
         client_id = AnalyticsIdentifier.objects.get(id=1).identifier.hex
         try:

allianceauth/analytics/migrations/0004_auto_20211015_0502.py

@@ -1,11 +1,11 @@
 # Generated by Django 3.1.13 on 2021-10-15 05:02
 
+from django.core.exceptions import ObjectDoesNotExist
 from django.db import migrations
 
 
 def modify_aa_team_token_add_page_ignore_paths(apps, schema_editor):
-    # We can't import the Person model directly as it may be a newer
+    # Add /admin/ and /user_notifications_count/ path to ignore
-    # version than this migration expects. We use the historical version.
 
     AnalyticsPath = apps.get_model('analytics', 'AnalyticsPath')
     admin = AnalyticsPath.objects.create(ignore_path=r"^\/admin\/.*")
@@ -17,8 +17,19 @@ def modify_aa_team_token_add_page_ignore_paths(apps, schema_editor):
 
 
 def undo_modify_aa_team_token_add_page_ignore_paths(apps, schema_editor):
-    # nothing should need to migrate away here?
+    #
-    return True
+    AnalyticsPath = apps.get_model('analytics', 'AnalyticsPath')
+    Tokens = apps.get_model('analytics', 'AnalyticsTokens')
+
+    token = Tokens.objects.get(token="UA-186249766-2")
+    try:
+        admin = AnalyticsPath.objects.get(ignore_path=r"^\/admin\/.*", analyticstokens=token)
+        user_notifications_count = AnalyticsPath.objects.get(ignore_path=r"^\/user_notifications_count\/.*", analyticstokens=token)
+        admin.delete()
+        user_notifications_count.delete()
+    except ObjectDoesNotExist:
+        # Its fine if it doesnt exist, we just dont want them building up when re-migrating
+        pass
 
 
 class Migration(migrations.Migration):

allianceauth/analytics/migrations/0006_more_ignore_paths.py

@@ -0,0 +1,40 @@
+# Generated by Django 3.2.8 on 2021-10-19 01:47
+
+from django.core.exceptions import ObjectDoesNotExist
+from django.db import migrations
+
+
+def modify_aa_team_token_add_page_ignore_paths(apps, schema_editor):
+    # Add the /account/activate path to ignore
+
+    AnalyticsPath = apps.get_model('analytics', 'AnalyticsPath')
+    account_activate = AnalyticsPath.objects.create(ignore_path=r"^\/account\/activate\/.*")
+
+    Tokens = apps.get_model('analytics', 'AnalyticsTokens')
+    token = Tokens.objects.get(token="UA-186249766-2")
+    token.ignore_paths.add(account_activate)
+
+
+def undo_modify_aa_team_token_add_page_ignore_paths(apps, schema_editor):
+    #
+    AnalyticsPath = apps.get_model('analytics', 'AnalyticsPath')
+    Tokens = apps.get_model('analytics', 'AnalyticsTokens')
+
+    token = Tokens.objects.get(token="UA-186249766-2")
+
+    try:
+        account_activate = AnalyticsPath.objects.get(ignore_path=r"^\/account\/activate\/.*", analyticstokens=token)
+        account_activate.delete()
+    except ObjectDoesNotExist:
+        # Its fine if it doesnt exist, we just dont want them building up when re-migrating
+        pass
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('analytics', '0005_alter_analyticspath_ignore_path'),
+    ]
+
+    operations = [
+        migrations.RunPython(modify_aa_team_token_add_page_ignore_paths, undo_modify_aa_team_token_add_page_ignore_paths)
+    ]

allianceauth/analytics/signals.py

@@ -1,7 +1,8 @@
-from allianceauth.analytics.tasks import analytics_event
-from celery.signals import task_failure, task_success
-
 import logging
+from celery.signals import task_failure, task_success
+from django.conf import settings
+from allianceauth.analytics.tasks import analytics_event
+
 logger = logging.getLogger(__name__)
 
 
@@ -11,6 +12,8 @@ def process_failure_signal(
                         sender, task_id, signal,
                         args, kwargs, einfo, **kw):
     logger.debug("Celery task_failure signal %s" % sender.__class__.__name__)
+    if getattr(settings, "ANALYTICS_DISABLED", False):
+        return
 
     category = sender.__module__
 
@@ -30,6 +33,8 @@ def process_failure_signal(
 @task_success.connect
 def celery_success_signal(sender, result=None, **kw):
     logger.debug("Celery task_success signal %s" % sender.__class__.__name__)
+    if getattr(settings, "ANALYTICS_DISABLED", False):
+        return
 
     category = sender.__module__
 

allianceauth/analytics/tasks.py

@@ -21,8 +21,8 @@ if getattr(settings, "ANALYTICS_ENABLE_DEBUG", False) and settings.DEBUG:
     # Force sending of analytics data during in a debug/test environemt
     # Usefull for developers working on this feature.
     logger.warning(
-                "You have 'ANALYTICS_ENABLE_DEBUG' Enabled! "
+        "You have 'ANALYTICS_ENABLE_DEBUG' Enabled! "
-                "This debug instance will send analytics data!")
+        "This debug instance will send analytics data!")
     DEBUG_URL = COLLECTION_URL
 
 ANALYTICS_URL = COLLECTION_URL
@@ -40,13 +40,12 @@ def analytics_event(category: str,
     Send a Google Analytics Event for each token stored
     Includes check for if its enabled/disabled
 
-    Parameters
+    Args:
-    -------
+        `category` (str): Celery Namespace
-    `category` (str): Celery Namespace
+        `action` (str): Task Name
-    `action` (str): Task Name
+        `label` (str): Optional, Task Success/Exception
-    `label` (str): Optional, Task Success/Exception
+        `value` (int): Optional, If bulk, Query size, can be a binary True/False
-    `value` (int): Optional, If bulk, Query size, can be a binary True/False
+        `event_type` (str): Optional, Celery or Stats only, Default to Celery
-    `event_type` (str): Optional, Celery or Stats only, Default to Celery
     """
     analyticstokens = AnalyticsTokens.objects.all()
     client_id = AnalyticsIdentifier.objects.get(id=1).identifier.hex
@@ -60,20 +59,21 @@ def analytics_event(category: str,
 
         if allowed is True:
             tracking_id = token.token
-            send_ga_tracking_celery_event.s(tracking_id=tracking_id,
+            send_ga_tracking_celery_event.s(
-                                            client_id=client_id,
+                tracking_id=tracking_id,
-                                            category=category,
+                client_id=client_id,
-                                            action=action,
+                category=category,
-                                            label=label,
+                action=action,
-                                            value=value).\
+                label=label,
-                apply_async(priority=9)
+                value=value).apply_async(priority=9)
 
 
 @shared_task()
 def analytics_daily_stats():
     """Celery Task: Do not call directly
 
-    Gathers a series of daily statistics and sends analytics events containing them"""
+    Gathers a series of daily statistics and sends analytics events containing them
+    """
     users = install_stat_users()
     tokens = install_stat_tokens()
     addons = install_stat_addons()

allianceauth/analytics/tests/test_integration.py

@@ -0,0 +1,108 @@
+from unittest.mock import patch
+from urllib.parse import parse_qs
+
+import requests_mock
+
+from django.test import TestCase, override_settings
+
+from allianceauth.analytics.tasks import ANALYTICS_URL
+from allianceauth.eveonline.tasks import update_character
+from allianceauth.tests.auth_utils import AuthUtils
+
+
+@override_settings(CELERY_ALWAYS_EAGER=True)
+@requests_mock.mock()
+class TestAnalyticsForViews(TestCase):
+    @override_settings(ANALYTICS_DISABLED=False)
+    def test_should_run_analytics(self, requests_mocker):
+        # given
+        requests_mocker.post(ANALYTICS_URL)
+        user = AuthUtils.create_user("Bruce Wayne")
+        self.client.force_login(user)
+        # when
+        response = self.client.get("/dashboard/")
+        # then
+        self.assertEqual(response.status_code, 200)
+        self.assertTrue(requests_mocker.called)
+
+    @override_settings(ANALYTICS_DISABLED=True)
+    def test_should_not_run_analytics(self, requests_mocker):
+        # given
+        requests_mocker.post(ANALYTICS_URL)
+        user = AuthUtils.create_user("Bruce Wayne")
+        self.client.force_login(user)
+        # when
+        response = self.client.get("/dashboard/")
+        # then
+        self.assertEqual(response.status_code, 200)
+        self.assertFalse(requests_mocker.called)
+
+
+@override_settings(CELERY_ALWAYS_EAGER=True)
+@requests_mock.mock()
+class TestAnalyticsForTasks(TestCase):
+    @override_settings(ANALYTICS_DISABLED=False)
+    @patch("allianceauth.eveonline.models.EveCharacter.objects.update_character")
+    def test_should_run_analytics_for_successful_task(
+        self, requests_mocker, mock_update_character
+    ):
+        # given
+        requests_mocker.post(ANALYTICS_URL)
+        user = AuthUtils.create_user("Bruce Wayne")
+        character = AuthUtils.add_main_character_2(user, "Bruce Wayne", 1001)
+        # when
+        update_character.delay(character.character_id)
+        # then
+        self.assertTrue(mock_update_character.called)
+        self.assertTrue(requests_mocker.called)
+        payload = parse_qs(requests_mocker.last_request.text)
+        self.assertListEqual(payload["el"], ["Success"])
+
+    @override_settings(ANALYTICS_DISABLED=True)
+    @patch("allianceauth.eveonline.models.EveCharacter.objects.update_character")
+    def test_should_not_run_analytics_for_successful_task(
+        self, requests_mocker, mock_update_character
+    ):
+        # given
+        requests_mocker.post(ANALYTICS_URL)
+        user = AuthUtils.create_user("Bruce Wayne")
+        character = AuthUtils.add_main_character_2(user, "Bruce Wayne", 1001)
+        # when
+        update_character.delay(character.character_id)
+        # then
+        self.assertTrue(mock_update_character.called)
+        self.assertFalse(requests_mocker.called)
+
+    @override_settings(ANALYTICS_DISABLED=False)
+    @patch("allianceauth.eveonline.models.EveCharacter.objects.update_character")
+    def test_should_run_analytics_for_failed_task(
+        self, requests_mocker, mock_update_character
+    ):
+        # given
+        requests_mocker.post(ANALYTICS_URL)
+        mock_update_character.side_effect = RuntimeError
+        user = AuthUtils.create_user("Bruce Wayne")
+        character = AuthUtils.add_main_character_2(user, "Bruce Wayne", 1001)
+        # when
+        update_character.delay(character.character_id)
+        # then
+        self.assertTrue(mock_update_character.called)
+        self.assertTrue(requests_mocker.called)
+        payload = parse_qs(requests_mocker.last_request.text)
+        self.assertNotEqual(payload["el"], ["Success"])
+
+    @override_settings(ANALYTICS_DISABLED=True)
+    @patch("allianceauth.eveonline.models.EveCharacter.objects.update_character")
+    def test_should_not_run_analytics_for_failed_task(
+        self, requests_mocker, mock_update_character
+    ):
+        # given
+        requests_mocker.post(ANALYTICS_URL)
+        mock_update_character.side_effect = RuntimeError
+        user = AuthUtils.create_user("Bruce Wayne")
+        character = AuthUtils.add_main_character_2(user, "Bruce Wayne", 1001)
+        # when
+        update_character.delay(character.character_id)
+        # then
+        self.assertTrue(mock_update_character.called)
+        self.assertFalse(requests_mocker.called)

allianceauth/corputils/templates/corputils/corpstats.html

@@ -193,6 +193,8 @@
             "columnDefs": [
                 { "sortable": false, "targets": [1] },
             ],
+            "stateSave": true,
+            "stateDuration": 0
         });
         $('#table-members').DataTable({
             "columnDefs": [
@@ -200,6 +202,8 @@
                 { "sortable": false, "targets": [0, 2] },
             ],
             "order": [[ 1, "asc" ]],
+            "stateSave": true,
+            "stateDuration": 0
         });
         $('#table-unregistered').DataTable({
             "columnDefs": [
@@ -207,6 +211,8 @@
                 { "sortable": false, "targets": [0, 2] },
             ],
             "order": [[ 1, "asc" ]],
+            "stateSave": true,
+            "stateDuration": 0
         });
 
     });

allianceauth/corputils/templates/corputils/search.html

@@ -43,6 +43,9 @@
 {% endblock %}
 {% block extra_script %}
     $(document).ready(function(){
-        $('#table-search').DataTable();
+        $('#table-search').DataTable({
+            "stateSave": true,
+            "stateDuration": 0
+        });
     });
 {% endblock %}

allianceauth/eveonline/models.py

@@ -1,13 +1,27 @@
-from django.db import models
+import logging
 from typing import Union
 
-from .managers import EveCharacterManager, EveCharacterProviderManager
+from django.core.exceptions import ObjectDoesNotExist
-from .managers import EveCorporationManager, EveCorporationProviderManager
+from django.db import models
-from .managers import EveAllianceManager, EveAllianceProviderManager
+from esi.models import Token
+
+from allianceauth.notifications import notify
+
 from . import providers
 from .evelinks import eveimageserver
+from .managers import (
+    EveAllianceManager,
+    EveAllianceProviderManager,
+    EveCharacterManager,
+    EveCharacterProviderManager,
+    EveCorporationManager,
+    EveCorporationProviderManager,
+)
+
+logger = logging.getLogger(__name__)
 
 _DEFAULT_IMAGE_SIZE = 32
+DOOMHEIM_CORPORATION_ID = 1000001
 
 
 class EveFactionInfo(models.Model):
@@ -68,13 +82,12 @@ class EveAllianceInfo(models.Model):
         for corp_id in alliance.corp_ids:
             if not EveCorporationInfo.objects.filter(corporation_id=corp_id).exists():
                 EveCorporationInfo.objects.create_corporation(corp_id)
-        EveCorporationInfo.objects.filter(
+        EveCorporationInfo.objects.filter(corporation_id__in=alliance.corp_ids).update(
-            corporation_id__in=alliance.corp_ids).update(alliance=self
+            alliance=self
         )
-        EveCorporationInfo.objects\
+        EveCorporationInfo.objects.filter(alliance=self).exclude(
-            .filter(alliance=self)\
+            corporation_id__in=alliance.corp_ids
-            .exclude(corporation_id__in=alliance.corp_ids)\
+        ).update(alliance=None)
-            .update(alliance=None)
 
     def update_alliance(self, alliance: providers.Alliance = None):
         if alliance is None:
@@ -182,6 +195,7 @@ class EveCorporationInfo(models.Model):
 
 
 class EveCharacter(models.Model):
+    """Character in Eve Online"""
     character_id = models.PositiveIntegerField(unique=True)
     character_name = models.CharField(max_length=254, unique=True)
     corporation_id = models.PositiveIntegerField()
@@ -198,12 +212,20 @@ class EveCharacter(models.Model):
 
     class Meta:
         indexes = [
-                    models.Index(fields=['corporation_id',]),
+            models.Index(fields=['corporation_id',]),
-                    models.Index(fields=['alliance_id',]),
+            models.Index(fields=['alliance_id',]),
-                    models.Index(fields=['corporation_name',]),
+            models.Index(fields=['corporation_name',]),
-                    models.Index(fields=['alliance_name',]),
+            models.Index(fields=['alliance_name',]),
-                    models.Index(fields=['faction_id',]),
+            models.Index(fields=['faction_id',]),
-                  ]
+        ]
+
+    def __str__(self):
+        return self.character_name
+
+    @property
+    def is_biomassed(self) -> bool:
+        """Whether this character is dead or not."""
+        return self.corporation_id == DOOMHEIM_CORPORATION_ID
 
     @property
     def alliance(self) -> Union[EveAllianceInfo, None]:
@@ -249,10 +271,36 @@ class EveCharacter(models.Model):
         self.faction_id = character.faction.id
         self.faction_name = character.faction.name
         self.save()
+        if self.is_biomassed:
+            self._remove_tokens_of_biomassed_character()
         return self
 
-    def __str__(self):
+    def _remove_tokens_of_biomassed_character(self) -> None:
-        return self.character_name
+        """Remove tokens of this biomassed character."""
+        try:
+            user = self.character_ownership.user
+        except ObjectDoesNotExist:
+            return
+        tokens_to_delete = Token.objects.filter(character_id=self.character_id)
+        tokens_count = tokens_to_delete.count()
+        if not tokens_count:
+            return
+        tokens_to_delete.delete()
+        logger.info(
+            "%d tokens from user %s for biomassed character %s [id:%s] deleted.",
+            tokens_count,
+            user,
+            self,
+            self.character_id,
+        )
+        notify(
+            user=user,
+            title=f"Character {self} biomassed",
+            message=(
+                f"Your former character {self} has been biomassed "
+                "and has been removed from the list of your alts."
+            )
+        )
 
     @staticmethod
     def generic_portrait_url(
@@ -336,7 +384,6 @@ class EveCharacter(models.Model):
         """image URL for alliance of this character or empty string"""
         return self.alliance_logo_url(256)
 
-
     def faction_logo_url(self, size=_DEFAULT_IMAGE_SIZE) -> str:
         """image URL for alliance of this character or empty string"""
         if self.faction_id:

allianceauth/eveonline/providers.py

@@ -170,7 +170,7 @@ class EveProvider:
         """
         :return: an ItemType object for the given ID
         """
-        raise NotImplemented()
+        raise NotImplementedError()
 
 
 class EveSwaggerProvider(EveProvider):
@@ -207,7 +207,8 @@ class EveSwaggerProvider(EveProvider):
     def __str__(self):
         return 'esi'
 
-    def get_alliance(self, alliance_id):
+    def get_alliance(self, alliance_id: int) -> Alliance:
+        """Fetch alliance from ESI."""
         try:
             data = self.client.Alliance.get_alliances_alliance_id(alliance_id=alliance_id).result()
             corps = self.client.Alliance.get_alliances_alliance_id_corporations(alliance_id=alliance_id).result()
@@ -223,7 +224,8 @@ class EveSwaggerProvider(EveProvider):
         except HTTPNotFound:
             raise ObjectNotFound(alliance_id, 'alliance')
 
-    def get_corp(self, corp_id):
+    def get_corp(self, corp_id: int) -> Corporation:
+        """Fetch corporation from ESI."""
         try:
             data = self.client.Corporation.get_corporations_corporation_id(corporation_id=corp_id).result()
             model = Corporation(
@@ -239,29 +241,43 @@ class EveSwaggerProvider(EveProvider):
         except HTTPNotFound:
             raise ObjectNotFound(corp_id, 'corporation')
 
-    def get_character(self, character_id):
+    def get_character(self, character_id: int) -> Character:
+        """Fetch character from ESI."""
         try:
-            data = self.client.Character.get_characters_character_id(character_id=character_id).result()
+            character_name = self._fetch_character_name(character_id)
             affiliation = self.client.Character.post_characters_affiliation(characters=[character_id]).result()[0]
-
             model = Character(
                 id=character_id,
-                name=data['name'],
+                name=character_name,
                 corp_id=affiliation['corporation_id'],
                 alliance_id=affiliation['alliance_id'] if 'alliance_id' in affiliation else None,
                 faction_id=affiliation['faction_id'] if 'faction_id' in affiliation else None,
             )
             return model
-        except (HTTPNotFound, HTTPUnprocessableEntity):
+        except (HTTPNotFound, HTTPUnprocessableEntity, ObjectNotFound):
             raise ObjectNotFound(character_id, 'character')
 
+    def _fetch_character_name(self, character_id: int) -> str:
+        """Fetch character name from ESI."""
+        data = self.client.Universe.post_universe_names(ids=[character_id]).result()
+        character = data.pop() if data else None
+        if (
+            not character
+            or character["category"] != "character"
+            or character["id"] != character_id
+        ):
+            raise ObjectNotFound(character_id, 'character')
+        return character["name"]
+
     def get_all_factions(self):
+        """Fetch all factions from ESI."""
         if not self._faction_list:
             self._faction_list = self.client.Universe.get_universe_factions().result()
         return self._faction_list
 
-    def get_faction(self, faction_id):
+    def get_faction(self, faction_id: int):
-        faction_id=int(faction_id)
+        """Fetch faction from ESI."""
+        faction_id = int(faction_id)
         try:
             if not self._faction_list:
                 _ = self.get_all_factions()
@@ -273,7 +289,8 @@ class EveSwaggerProvider(EveProvider):
         except (HTTPNotFound, HTTPUnprocessableEntity, KeyError):
             raise ObjectNotFound(faction_id, 'faction')
 
-    def get_itemtype(self, type_id):
+    def get_itemtype(self, type_id: int) -> ItemType:
+        """Fetch inventory item from ESI."""
         try:
             data = self.client.Universe.get_universe_types_type_id(type_id=type_id).result()
             return ItemType(id=type_id, name=data['name'])

allianceauth/eveonline/tasks.py

@@ -1,12 +1,11 @@
 import logging
 
 from celery import shared_task
-from .models import EveAllianceInfo
-from .models import EveCharacter
-from .models import EveCorporationInfo
 
+from .models import EveAllianceInfo, EveCharacter, EveCorporationInfo
 from . import providers
 
+
 logger = logging.getLogger(__name__)
 
 TASK_PRIORITY = 7
@@ -32,8 +31,8 @@ def update_alliance(alliance_id):
 
 
 @shared_task
-def update_character(character_id):
+def update_character(character_id: int) -> None:
-    """Update given character from ESI"""
+    """Update given character from ESI."""
     EveCharacter.objects.update_character(character_id)
 
 
@@ -65,17 +64,17 @@ def update_character_chunk(character_ids_chunk: list):
             .post_characters_affiliation(characters=character_ids_chunk).result()
         character_names = providers.provider.client.Universe\
             .post_universe_names(ids=character_ids_chunk).result()
-    except:
+    except OSError:
         logger.info("Failed to bulk update characters. Attempting single updates")
         for character_id in character_ids_chunk:
             update_character.apply_async(
-                        args=[character_id], priority=TASK_PRIORITY
+                args=[character_id], priority=TASK_PRIORITY
-                    )
+            )
         return
 
     affiliations = {
-            affiliation.get('character_id'): affiliation
+        affiliation.get('character_id'): affiliation
-            for affiliation in affiliations_raw
+        for affiliation in affiliations_raw
     }
     # add character names to affiliations
     for character in character_names:
@@ -108,5 +107,5 @@ def update_character_chunk(character_ids_chunk: list):
 
             if corp_changed or alliance_changed or name_changed:
                 update_character.apply_async(
-                        args=[character.get('character_id')], priority=TASK_PRIORITY
+                    args=[character.get('character_id')], priority=TASK_PRIORITY
-                    )
+                )

allianceauth/eveonline/tests/esi_client_stub.py

@@ -0,0 +1,168 @@
+from bravado.exception import HTTPNotFound
+
+
+class BravadoResponseStub:
+    """Stub for IncomingResponse in bravado, e.g. for HTTPError exceptions"""
+
+    def __init__(
+        self, status_code, reason="", text="", headers=None, raw_bytes=None
+    ) -> None:
+        self.reason = reason
+        self.status_code = status_code
+        self.text = text
+        self.headers = headers if headers else dict()
+        self.raw_bytes = raw_bytes
+
+    def __str__(self):
+        return f"{self.status_code} {self.reason}"
+
+
+class BravadoOperationStub:
+    """Stub to simulate the operation object return from bravado via django-esi"""
+
+    class RequestConfig:
+        def __init__(self, also_return_response):
+            self.also_return_response = also_return_response
+
+    class ResponseStub:
+        def __init__(self, headers):
+            self.headers = headers
+
+    def __init__(self, data, headers: dict = None, also_return_response: bool = False):
+        self._data = data
+        self._headers = headers if headers else {"x-pages": 1}
+        self.request_config = BravadoOperationStub.RequestConfig(also_return_response)
+
+    def result(self, **kwargs):
+        if self.request_config.also_return_response:
+            return [self._data, self.ResponseStub(self._headers)]
+        else:
+            return self._data
+
+    def results(self, **kwargs):
+        return self.result(**kwargs)
+
+
+class EsiClientStub:
+    """Stub for an ESI client."""
+    class Alliance:
+        @staticmethod
+        def get_alliances_alliance_id(alliance_id):
+            data = {
+                3001: {
+                    "name": "Wayne Enterprises",
+                    "ticker": "WYE",
+                    "executor_corporation_id": 2001
+                }
+            }
+            try:
+                return BravadoOperationStub(data[int(alliance_id)])
+            except KeyError:
+                response = BravadoResponseStub(
+                    404, f"Alliance with ID {alliance_id} not found"
+                )
+                raise HTTPNotFound(response)
+
+        @staticmethod
+        def get_alliances_alliance_id_corporations(alliance_id):
+            data = [2001, 2002, 2003]
+            return BravadoOperationStub(data)
+
+    class Character:
+        @staticmethod
+        def get_characters_character_id(character_id):
+            data = {
+                1001: {
+                    "corporation_id": 2001,
+                    "name": "Bruce Wayne",
+                },
+                1002: {
+                    "corporation_id": 2001,
+                    "name": "Peter Parker",
+                },
+                1011: {
+                    "corporation_id": 2011,
+                    "name": "Lex Luthor",
+                }
+            }
+            try:
+                return BravadoOperationStub(data[int(character_id)])
+            except KeyError:
+                response = BravadoResponseStub(
+                    404, f"Character with ID {character_id} not found"
+                )
+                raise HTTPNotFound(response)
+
+        @staticmethod
+        def post_characters_affiliation(characters: list):
+            data = [
+                {'character_id': 1001, 'corporation_id': 2001, 'alliance_id': 3001},
+                {'character_id': 1002, 'corporation_id': 2001, 'alliance_id': 3001},
+                {'character_id': 1011, 'corporation_id': 2011},
+                {'character_id': 1666, 'corporation_id': 1000001},
+            ]
+            return BravadoOperationStub(
+                [x for x in data if x['character_id'] in characters]
+            )
+
+    class Corporation:
+        @staticmethod
+        def get_corporations_corporation_id(corporation_id):
+            data = {
+                2001: {
+                    "ceo_id": 1091,
+                    "member_count": 10,
+                    "name": "Wayne Technologies",
+                    "ticker": "WTE",
+                    "alliance_id": 3001
+                },
+                2002: {
+                    "ceo_id": 1092,
+                    "member_count": 10,
+                    "name": "Wayne Food",
+                    "ticker": "WFO",
+                    "alliance_id": 3001
+                },
+                2003: {
+                    "ceo_id": 1093,
+                    "member_count": 10,
+                    "name": "Wayne Energy",
+                    "ticker": "WEG",
+                    "alliance_id": 3001
+                },
+                2011: {
+                    "ceo_id": 1,
+                    "member_count": 3,
+                    "name": "LexCorp",
+                    "ticker": "LC",
+                },
+                1000001: {
+                    "ceo_id": 3000001,
+                    "creator_id": 1,
+                    "description": "The internal corporation used for characters in graveyard.",
+                    "member_count": 6329026,
+                    "name": "Doomheim",
+                    "ticker": "666",
+                }
+            }
+            try:
+                return BravadoOperationStub(data[int(corporation_id)])
+            except KeyError:
+                response = BravadoResponseStub(
+                    404, f"Corporation with ID {corporation_id} not found"
+                )
+                raise HTTPNotFound(response)
+
+    class Universe:
+        @staticmethod
+        def post_universe_names(ids: list):
+            data = [
+                {"category": "character", "id": 1001, "name": "Bruce Wayne"},
+                {"category": "character", "id": 1002, "name": "Peter Parker"},
+                {"category": "character", "id": 1011, "name": "Lex Luthor"},
+                {"category": "character", "id": 1666, "name": "Hal Jordan"},
+                {"category": "corporation", "id": 2001, "name": "Wayne Technologies"},
+                {"category": "corporation","id": 2002, "name": "Wayne Food"},
+                {"category": "corporation","id": 1000001, "name": "Doomheim"},
+            ]
+            return BravadoOperationStub([x for x in data if x['id'] in ids])

allianceauth/eveonline/tests/test_models.py

@@ -1,12 +1,15 @@
 from unittest.mock import Mock, patch
 
+from django.core.exceptions import ObjectDoesNotExist
 from django.test import TestCase
+from esi.models import Token
+
+from allianceauth.tests.auth_utils import AuthUtils
 
-from ..models import (
-    EveCharacter, EveCorporationInfo, EveAllianceInfo, EveFactionInfo
-)
-from ..providers import Alliance, Corporation, Character
 from ..evelinks import eveimageserver
+from ..models import EveAllianceInfo, EveCharacter, EveCorporationInfo, EveFactionInfo
+from ..providers import Alliance, Character, Corporation
+from .esi_client_stub import EsiClientStub
 
 
 class EveCharacterTestCase(TestCase):
@@ -402,8 +405,8 @@ class EveAllianceTestCase(TestCase):
         my_alliance.save()
         my_alliance.populate_alliance()
 
-        for corporation in EveCorporationInfo.objects\
+        for corporation in (
-            .filter(corporation_id__in=[2001, 2002]
+            EveCorporationInfo.objects.filter(corporation_id__in=[2001, 2002])
         ):
             self.assertEqual(corporation.alliance, my_alliance)
 
@@ -587,3 +590,98 @@ class EveCorporationTestCase(TestCase):
             self.my_corp.logo_url_256,
             'https://images.evetech.net/corporations/2001/logo?size=256'
         )
+
+
+@patch('allianceauth.eveonline.providers.esi_client_factory')
+@patch("allianceauth.eveonline.models.notify")
+class TestCharacterUpdate(TestCase):
+    def test_should_update_normal_character(self, mock_notify, mock_esi_client_factory):
+        # given
+        mock_esi_client_factory.return_value = EsiClientStub()
+        my_character = EveCharacter.objects.create(
+            character_id=1001,
+            character_name="not my name",
+            corporation_id=2002,
+            corporation_name="Wayne Food",
+            corporation_ticker="WYF",
+            alliance_id=None
+        )
+        # when
+        my_character.update_character()
+        # then
+        my_character.refresh_from_db()
+        self.assertEqual(my_character.character_name, "Bruce Wayne")
+        self.assertEqual(my_character.corporation_id, 2001)
+        self.assertEqual(my_character.corporation_name, "Wayne Technologies")
+        self.assertEqual(my_character.corporation_ticker, "WTE")
+        self.assertEqual(my_character.alliance_id, 3001)
+        self.assertEqual(my_character.alliance_name, "Wayne Enterprises")
+        self.assertEqual(my_character.alliance_ticker, "WYE")
+        self.assertFalse(mock_notify.called)
+
+    def test_should_update_dead_character_with_owner(
+        self, mock_notify, mock_esi_client_factory
+    ):
+        # given
+        mock_esi_client_factory.return_value = EsiClientStub()
+        character_1666 = EveCharacter.objects.create(
+            character_id=1666,
+            character_name="Hal Jordan",
+            corporation_id=2002,
+            corporation_name="Wayne Food",
+            corporation_ticker="WYF",
+            alliance_id=None
+        )
+        user = AuthUtils.create_user("Bruce Wayne")
+        token_1666 = Token.objects.create(
+            user=user,
+            character_id=character_1666.character_id,
+            character_name=character_1666.character_name,
+            character_owner_hash="ABC123-1666",
+        )
+        character_1001 = EveCharacter.objects.create(
+            character_id=1001,
+            character_name="Bruce Wayne",
+            corporation_id=2001,
+            corporation_name="Wayne Technologies",
+            corporation_ticker="WYT",
+            alliance_id=None
+        )
+        token_1001 = Token.objects.create(
+            user=user,
+            character_id=character_1001.character_id,
+            character_name=character_1001.character_name,
+            character_owner_hash="ABC123-1001",
+        )
+        # when
+        character_1666.update_character()
+        # then
+        character_1666.refresh_from_db()
+        self.assertTrue(character_1666.is_biomassed)
+        self.assertNotIn(token_1666, user.token_set.all())
+        self.assertIn(token_1001, user.token_set.all())
+        with self.assertRaises(ObjectDoesNotExist):
+            self.assertTrue(character_1666.character_ownership)
+        user.profile.refresh_from_db()
+        self.assertIsNone(user.profile.main_character)
+        self.assertTrue(mock_notify.called)
+
+    def test_should_handle_dead_character_without_owner(
+        self, mock_notify, mock_esi_client_factory
+    ):
+        # given
+        mock_esi_client_factory.return_value = EsiClientStub()
+        character_1666 = EveCharacter.objects.create(
+            character_id=1666,
+            character_name="Hal Jordan",
+            corporation_id=1011,
+            corporation_name="LexCorp",
+            corporation_ticker='LC',
+            alliance_id=None
+        )
+        # when
+        character_1666.update_character()
+        # then
+        character_1666.refresh_from_db()
+        self.assertTrue(character_1666.is_biomassed)
+        self.assertFalse(mock_notify.called)

allianceauth/eveonline/tests/test_providers.py

@@ -7,6 +7,7 @@ from jsonschema.exceptions import RefResolutionError
 from django.test import TestCase
 
 from . import set_logger
+from .esi_client_stub import EsiClientStub
 from ..providers import (
     ObjectNotFound,
     Entity,
@@ -632,13 +633,7 @@ class TestEveSwaggerProvider(TestCase):
 
     @patch(MODULE_PATH + '.esi_client_factory')
     def test_get_character(self, mock_esi_client_factory):
-        mock_esi_client_factory.return_value \
+        mock_esi_client_factory.return_value = EsiClientStub()
-            .Character.get_characters_character_id \
-            = TestEveSwaggerProvider.esi_get_characters_character_id
-        mock_esi_client_factory.return_value \
-            .Character.post_characters_affiliation \
-            = TestEveSwaggerProvider.esi_post_characters_affiliation
-
         my_provider = EveSwaggerProvider()
 
         # character with alliance
@@ -649,8 +644,8 @@ class TestEveSwaggerProvider(TestCase):
         self.assertEqual(my_character.alliance_id, 3001)
 
         # character wo/ alliance
-        my_character = my_provider.get_character(1002)
+        my_character = my_provider.get_character(1011)
-        self.assertEqual(my_character.id, 1002)
+        self.assertEqual(my_character.id, 1011)
         self.assertEqual(my_character.alliance_id, None)
 
         # character not found

allianceauth/eveonline/tests/test_tasks.py

@@ -1,245 +1,271 @@
-from unittest.mock import patch, Mock
+from unittest.mock import patch
 
-from django.test import TestCase
+from django.test import TestCase, TransactionTestCase, override_settings
 
-from ..models import EveCharacter, EveCorporationInfo, EveAllianceInfo
+from ..models import EveAllianceInfo, EveCharacter, EveCorporationInfo
 from ..tasks import (
+    run_model_update,
     update_alliance,
-    update_corp,
     update_character,
-    run_model_update
+    update_character_chunk,
+    update_corp,
 )
+from .esi_client_stub import EsiClientStub
 
 
-class TestTasks(TestCase):
+@patch('allianceauth.eveonline.providers.esi_client_factory')
-
+class TestUpdateTasks(TestCase):
-    @patch('allianceauth.eveonline.tasks.EveCorporationInfo')
+    def test_should_update_alliance(self, mock_esi_client_factory):
-    def test_update_corp(self, mock_EveCorporationInfo):
+        # given
-        update_corp(42)
+        mock_esi_client_factory.return_value = EsiClientStub()
-        self.assertEqual(
+        my_alliance = EveAllianceInfo.objects.create(
-            mock_EveCorporationInfo.objects.update_corporation.call_count, 1
+            alliance_id=3001,
-        )
+            alliance_name="Wayne Enterprises",
-        self.assertEqual(
+            alliance_ticker="WYE",
-            mock_EveCorporationInfo.objects.update_corporation.call_args[0][0], 42
+            executor_corp_id=2003
         )
+        # when
+        update_alliance(my_alliance.alliance_id)
+        # then
+        my_alliance.refresh_from_db()
+        self.assertEqual(my_alliance.executor_corp_id, 2001)
 
-    @patch('allianceauth.eveonline.tasks.EveAllianceInfo')
+    def test_should_update_character(self, mock_esi_client_factory):
-    def test_update_alliance(self, mock_EveAllianceInfo):
+        # given
-        update_alliance(42)
+        mock_esi_client_factory.return_value = EsiClientStub()
-        self.assertEqual(
+        my_character = EveCharacter.objects.create(
-            mock_EveAllianceInfo.objects.update_alliance.call_args[0][0], 42
+            character_id=1001,
-        )
+            character_name="Bruce Wayne",
-        self.assertEqual(
+            corporation_id=2002,
-            mock_EveAllianceInfo.objects
+            corporation_name="Wayne Food",
-            .update_alliance.return_value.populate_alliance.call_count, 1
+            corporation_ticker="WYF",
+            alliance_id=None
         )
+        # when
+        update_character(my_character.character_id)
+        # then
+        my_character.refresh_from_db()
+        self.assertEqual(my_character.corporation_id, 2001)
 
-    @patch('allianceauth.eveonline.tasks.EveCharacter')
+    def test_should_update_corp(self, mock_esi_client_factory):
-    def test_update_character(self, mock_EveCharacter):
+        # given
-        update_character(42)
+        mock_esi_client_factory.return_value = EsiClientStub()
-        self.assertEqual(
+        EveAllianceInfo.objects.create(
-            mock_EveCharacter.objects.update_character.call_count, 1
+            alliance_id=3001,
+            alliance_name="Wayne Enterprises",
+            alliance_ticker="WYE",
+            executor_corp_id=2003
         )
-        self.assertEqual(
+        my_corporation = EveCorporationInfo.objects.create(
-            mock_EveCharacter.objects.update_character.call_args[0][0], 42
+            corporation_id=2003,
+            corporation_name="Wayne Food",
+            corporation_ticker="WFO",
+            member_count=1,
+            alliance=None,
+            ceo_id=1999
         )
+        # when
+        update_corp(my_corporation.corporation_id)
+        # then
+        my_corporation.refresh_from_db()
+        self.assertEqual(my_corporation.alliance.alliance_id, 3001)
+
+    # @patch('allianceauth.eveonline.tasks.EveCharacter')
+    # def test_update_character(self, mock_EveCharacter):
+    #     update_character(42)
+    #     self.assertEqual(
+    #         mock_EveCharacter.objects.update_character.call_count, 1
+    #     )
+    #     self.assertEqual(
+    #         mock_EveCharacter.objects.update_character.call_args[0][0], 42
+    #     )
 
 
-@patch('allianceauth.eveonline.tasks.update_character')
+@override_settings(CELERY_ALWAYS_EAGER=True)
-@patch('allianceauth.eveonline.tasks.update_alliance')
+@patch('allianceauth.eveonline.providers.esi_client_factory')
-@patch('allianceauth.eveonline.tasks.update_corp')
+@patch('allianceauth.eveonline.tasks.providers')
-@patch('allianceauth.eveonline.providers.provider')
 @patch('allianceauth.eveonline.tasks.CHUNK_SIZE', 2)
-class TestRunModelUpdate(TestCase):
+class TestRunModelUpdate(TransactionTestCase):
-
+    def test_should_run_updates(self, mock_providers, mock_esi_client_factory):
-    @classmethod
+        # given
-    def setUpClass(cls):
+        mock_providers.provider.client = EsiClientStub()
-        super().setUpClass()
+        mock_esi_client_factory.return_value = EsiClientStub()
-        EveCorporationInfo.objects.all().delete()
-        EveAllianceInfo.objects.all().delete()
-        EveCharacter.objects.all().delete()
-
         EveCorporationInfo.objects.create(
-            corporation_id=2345,
+            corporation_id=2001,
-            corporation_name='corp.name',
+            corporation_name="Wayne Technologies",
-            corporation_ticker='c.c.t',
+            corporation_ticker="WTE",
             member_count=10,
             alliance=None,
         )
-        EveAllianceInfo.objects.create(
+        alliance_3001 = EveAllianceInfo.objects.create(
-            alliance_id=3456,
+            alliance_id=3001,
-            alliance_name='alliance.name',
+            alliance_name="Wayne Enterprises",
-            alliance_ticker='a.t',
+            alliance_ticker="WYE",
-            executor_corp_id=5,
+            executor_corp_id=2003
         )
-        EveCharacter.objects.create(
+        corporation_2003 = EveCorporationInfo.objects.create(
-            character_id=1,
+            corporation_id=2003,
-            character_name='character.name1',
+            corporation_name="Wayne Energy",
-            corporation_id=2345,
+            corporation_ticker="WEG",
-            corporation_name='character.corp.name',
+            member_count=99,
-            corporation_ticker='c.c.t',  # max 5 chars
+            alliance=None,
+        )
+        character_1001 = EveCharacter.objects.create(
+            character_id=1001,
+            character_name="Bruce Wayne",
+            corporation_id=2002,
+            corporation_name="Wayne Food",
+            corporation_ticker="WYF",
             alliance_id=None
         )
-        EveCharacter.objects.create(
+        # when
-            character_id=2,
-            character_name='character.name2',
-            corporation_id=9876,
-            corporation_name='character.corp.name',
-            corporation_ticker='c.c.t',  # max 5 chars
-            alliance_id=3456,
-            alliance_name='character.alliance.name',
-        )
-        EveCharacter.objects.create(
-            character_id=3,
-            character_name='character.name3',
-            corporation_id=9876,
-            corporation_name='character.corp.name',
-            corporation_ticker='c.c.t',  # max 5 chars
-            alliance_id=3456,
-            alliance_name='character.alliance.name',
-        )
-        EveCharacter.objects.create(
-            character_id=4,
-            character_name='character.name4',
-            corporation_id=9876,
-            corporation_name='character.corp.name',
-            corporation_ticker='c.c.t',  # max 5 chars
-            alliance_id=3456,
-            alliance_name='character.alliance.name',
-        )
-        """
-        EveCharacter.objects.create(
-            character_id=5,
-            character_name='character.name5',
-            corporation_id=9876,
-            corporation_name='character.corp.name',
-            corporation_ticker='c.c.t',  # max 5 chars
-            alliance_id=3456,
-            alliance_name='character.alliance.name',
-        )
-        """
-
-    def setUp(self):
-        self.affiliations = [
-            {'character_id': 1, 'corporation_id': 5},
-            {'character_id': 2, 'corporation_id': 9876, 'alliance_id': 3456},
-            {'character_id': 3, 'corporation_id': 9876, 'alliance_id': 7456},
-            {'character_id': 4, 'corporation_id': 9876, 'alliance_id': 3456}
-        ]
-        self.names = [
-            {'id': 1, 'name': 'character.name1'},
-            {'id': 2, 'name': 'character.name2'},
-            {'id': 3, 'name': 'character.name3'},
-            {'id': 4, 'name': 'character.name4_new'}
-        ]
-
-    def test_normal_run(
-        self,
-        mock_provider,
-        mock_update_corp,
-        mock_update_alliance,
-        mock_update_character,
-    ):
-        def get_affiliations(characters: list):
-            response = [x for x in self.affiliations if x['character_id'] in characters]
-            mock_operator = Mock(**{'result.return_value': response})
-            return mock_operator
-
-        def get_names(ids: list):
-            response = [x for x in self.names if x['id'] in ids]
-            mock_operator = Mock(**{'result.return_value': response})
-            return mock_operator
-
-        mock_provider.client.Character.post_characters_affiliation.side_effect \
-            = get_affiliations
-
-        mock_provider.client.Universe.post_universe_names.side_effect = get_names
-
         run_model_update()
-
+        # then
+        character_1001.refresh_from_db()
         self.assertEqual(
-            mock_provider.client.Character.post_characters_affiliation.call_count, 2
+            character_1001.corporation_id, 2001  # char has new corp
         )
+        corporation_2003.refresh_from_db()
         self.assertEqual(
-            mock_provider.client.Universe.post_universe_names.call_count, 2
+            corporation_2003.alliance.alliance_id, 3001  # corp has new alliance
+        )
+        alliance_3001.refresh_from_db()
+        self.assertEqual(
+            alliance_3001.executor_corp_id, 2001  # alliance has been updated
         )
 
-        # character 1 has changed corp
+
-        # character 2 no change
+@override_settings(CELERY_ALWAYS_EAGER=True)
-        # character 3 has changed alliance
+@patch('allianceauth.eveonline.tasks.update_character', wraps=update_character)
-        # character 4 has changed name
+@patch('allianceauth.eveonline.providers.esi_client_factory')
-        self.assertEqual(mock_update_corp.apply_async.call_count, 1)
+@patch('allianceauth.eveonline.tasks.providers')
-        self.assertEqual(
+@patch('allianceauth.eveonline.tasks.CHUNK_SIZE', 2)
-            int(mock_update_corp.apply_async.call_args[1]['args'][0]), 2345
+class TestUpdateCharacterChunk(TestCase):
-        )
+    @staticmethod
-        self.assertEqual(mock_update_alliance.apply_async.call_count, 1)
+    def _updated_character_ids(spy_update_character) -> set:
-        self.assertEqual(
+        """Character IDs passed to update_character task for update."""
-            int(mock_update_alliance.apply_async.call_args[1]['args'][0]), 3456
+        return {
-        )
+            x[1]["args"][0] for x in spy_update_character.apply_async.call_args_list
-        characters_updated = {
-            x[1]['args'][0] for x in mock_update_character.apply_async.call_args_list
         }
-        excepted = {1, 3, 4}
-        self.assertSetEqual(characters_updated, excepted)
 
-    def test_ignore_character_not_in_affiliations(
+    def test_should_update_corp_change(
-        self,
+        self, mock_providers, mock_esi_client_factory, spy_update_character
-        mock_provider,
-        mock_update_corp,
-        mock_update_alliance,
-        mock_update_character,
     ):
-        def get_affiliations(characters: list):
+        # given
-            response = [x for x in self.affiliations if x['character_id'] in characters]
+        mock_providers.provider.client = EsiClientStub()
-            mock_operator = Mock(**{'result.return_value': response})
+        mock_esi_client_factory.return_value = EsiClientStub()
-            return mock_operator
+        character_1001 = EveCharacter.objects.create(
+            character_id=1001,
+            character_name="Bruce Wayne",
+            corporation_id=2003,
+            corporation_name="Wayne Energy",
+            corporation_ticker="WEG",
+            alliance_id=3001,
+            alliance_name="Wayne Enterprises",
+            alliance_ticker="WYE",
+        )
+        character_1002 = EveCharacter.objects.create(
+            character_id=1002,
+            character_name="Peter Parker",
+            corporation_id=2001,
+            corporation_name="Wayne Technologies",
+            corporation_ticker="WTE",
+            alliance_id=3001,
+            alliance_name="Wayne Enterprises",
+            alliance_ticker="WYE",
+        )
+        # when
+        update_character_chunk([
+            character_1001.character_id, character_1002.character_id
+        ])
+        # then
+        character_1001.refresh_from_db()
+        self.assertEqual(character_1001.corporation_id, 2001)
+        self.assertSetEqual(self._updated_character_ids(spy_update_character), {1001})
 
-        def get_names(ids: list):
+    def test_should_update_name_change(
-            response = [x for x in self.names if x['id'] in ids]
+        self, mock_providers, mock_esi_client_factory, spy_update_character
-            mock_operator = Mock(**{'result.return_value': response})
-            return mock_operator
-
-        del self.affiliations[0]
-
-        mock_provider.client.Character.post_characters_affiliation.side_effect \
-            = get_affiliations
-
-        mock_provider.client.Universe.post_universe_names.side_effect = get_names
-
-        run_model_update()
-        characters_updated = {
-            x[1]['args'][0] for x in mock_update_character.apply_async.call_args_list
-        }
-        excepted = {3, 4}
-        self.assertSetEqual(characters_updated, excepted)
-
-    def test_ignore_character_not_in_names(
-        self,
-        mock_provider,
-        mock_update_corp,
-        mock_update_alliance,
-        mock_update_character,
     ):
-        def get_affiliations(characters: list):
+        # given
-            response = [x for x in self.affiliations if x['character_id'] in characters]
+        mock_providers.provider.client = EsiClientStub()
-            mock_operator = Mock(**{'result.return_value': response})
+        mock_esi_client_factory.return_value = EsiClientStub()
-            return mock_operator
+        character_1001 = EveCharacter.objects.create(
+            character_id=1001,
+            character_name="Batman",
+            corporation_id=2001,
+            corporation_name="Wayne Technologies",
+            corporation_ticker="WTE",
+            alliance_id=3001,
+            alliance_name="Wayne Technologies",
+            alliance_ticker="WYT",
+        )
+        # when
+        update_character_chunk([character_1001.character_id])
+        # then
+        character_1001.refresh_from_db()
+        self.assertEqual(character_1001.character_name, "Bruce Wayne")
+        self.assertSetEqual(self._updated_character_ids(spy_update_character), {1001})
 
-        def get_names(ids: list):
+    def test_should_update_alliance_change(
-            response = [x for x in self.names if x['id'] in ids]
+        self, mock_providers, mock_esi_client_factory, spy_update_character
-            mock_operator = Mock(**{'result.return_value': response})
+    ):
-            return mock_operator
+        # given
+        mock_providers.provider.client = EsiClientStub()
+        mock_esi_client_factory.return_value = EsiClientStub()
+        character_1001 = EveCharacter.objects.create(
+            character_id=1001,
+            character_name="Bruce Wayne",
+            corporation_id=2001,
+            corporation_name="Wayne Technologies",
+            corporation_ticker="WTE",
+            alliance_id=None,
+        )
+        # when
+        update_character_chunk([character_1001.character_id])
+        # then
+        character_1001.refresh_from_db()
+        self.assertEqual(character_1001.alliance_id, 3001)
+        self.assertSetEqual(self._updated_character_ids(spy_update_character), {1001})
 
-        del self.names[3]
+    def test_should_not_update_when_not_changed(
+        self, mock_providers, mock_esi_client_factory, spy_update_character
+    ):
+        # given
+        mock_providers.provider.client = EsiClientStub()
+        mock_esi_client_factory.return_value = EsiClientStub()
+        character_1001 = EveCharacter.objects.create(
+            character_id=1001,
+            character_name="Bruce Wayne",
+            corporation_id=2001,
+            corporation_name="Wayne Technologies",
+            corporation_ticker="WTE",
+            alliance_id=3001,
+            alliance_name="Wayne Technologies",
+            alliance_ticker="WYT",
+        )
+        # when
+        update_character_chunk([character_1001.character_id])
+        # then
+        self.assertSetEqual(self._updated_character_ids(spy_update_character), set())
 
-        mock_provider.client.Character.post_characters_affiliation.side_effect \
+    def test_should_fall_back_to_single_updates_when_bulk_update_failed(
-            = get_affiliations
+        self, mock_providers, mock_esi_client_factory, spy_update_character
-
+    ):
-        mock_provider.client.Universe.post_universe_names.side_effect = get_names
+        # given
-
+        mock_providers.provider.client.Character.post_characters_affiliation\
-        run_model_update()
+            .side_effect = OSError
-        characters_updated = {
+        mock_esi_client_factory.return_value = EsiClientStub()
-            x[1]['args'][0] for x in mock_update_character.apply_async.call_args_list
+        character_1001 = EveCharacter.objects.create(
-        }
+            character_id=1001,
-        excepted = {1, 3}
+            character_name="Bruce Wayne",
-        self.assertSetEqual(characters_updated, excepted)
+            corporation_id=2001,
+            corporation_name="Wayne Technologies",
+            corporation_ticker="WTE",
+            alliance_id=3001,
+            alliance_name="Wayne Technologies",
+            alliance_ticker="WYT",
+        )
+        # when
+        update_character_chunk([character_1001.character_id])
+        # then
+        self.assertSetEqual(self._updated_character_ids(spy_update_character), {1001})

allianceauth/groupmanagement/templates/groupmanagement/audit.html

@@ -127,6 +127,8 @@
                 ],
                 bootstrap: true
             },
+            "stateSave": true,
+            "stateDuration": 0
         });
     });
 {% endblock %}

allianceauth/groupmanagement/templates/groupmanagement/groupmembers.html

@@ -104,7 +104,9 @@
                     "sortable": false,
                     "targets": [2]
                 },
-            ]
+            ],
+            "stateSave": true,
+            "stateDuration": 0
         });
     });
 {% endblock %}

allianceauth/permissions_tool/templates/permissions_tool/audit.html

@@ -73,6 +73,8 @@
                 ],
                 bootstrap: true
             },
+            "stateSave": true,
+            "stateDuration": 0,
             drawCallback: function ( settings ) {
                 let api = this.api();
                 let rows = api.rows( {page:'current'} ).nodes();

allianceauth/permissions_tool/templates/permissions_tool/overview.html

@@ -106,8 +106,10 @@
                         idx: 1
                     }
                 ],
-                bootstrap: true
+                bootstrap: true,
             },
+            "stateSave": true,
+            "stateDuration": 0,
             drawCallback: function ( settings ) {
                 let api = this.api();
                 let rows = api.rows( {page:'current'} ).nodes();

allianceauth/services/modules/teamspeak3/admin.py

@@ -1,7 +1,8 @@
 from django.contrib import admin
-
+from django.contrib.auth.models import Group
-from .models import AuthTS, Teamspeak3User, StateGroup
+from .models import AuthTS, Teamspeak3User, StateGroup, TSgroup
 from ...admin import ServicesUserAdmin
+from allianceauth.groupmanagement.models import ReservedGroupName
 
 
 @admin.register(Teamspeak3User)
@@ -25,6 +26,16 @@ class AuthTSgroupAdmin(admin.ModelAdmin):
     fields = ('auth_group', 'ts_group')
     filter_horizontal = ('ts_group',)
 
+    def formfield_for_foreignkey(self, db_field, request, **kwargs):
+        if db_field.name == 'auth_group':
+            kwargs['queryset'] = Group.objects.exclude(name__in=ReservedGroupName.objects.values_list('name', flat=True))
+        return super().formfield_for_foreignkey(db_field, request, **kwargs)
+
+    def formfield_for_manytomany(self, db_field, request, **kwargs):
+        if db_field.name == 'ts_group':
+            kwargs['queryset'] = TSgroup.objects.exclude(ts_group_name__in=ReservedGroupName.objects.values_list('name', flat=True))
+        return super().formfield_for_manytomany(db_field, request, **kwargs)
+
     def _ts_group(self, obj):
         return [x for x in obj.ts_group.all().order_by('ts_group_id')]
 

allianceauth/services/modules/teamspeak3/manager.py

@@ -4,6 +4,7 @@ from django.conf import settings
 
 from .util.ts3 import TS3Server, TeamspeakError
 from .models import TSgroup
+from allianceauth.groupmanagement.models import ReservedGroupName
 
 logger = logging.getLogger(__name__)
 
@@ -156,32 +157,25 @@ class Teamspeak3Manager:
             logger.info(f"Removed user id {uid} from group id {groupid} on TS3 server.")
 
     def _sync_ts_group_db(self):
-        logger.debug("_sync_ts_group_db function called.")
         try:
             remote_groups = self._group_list()
-            local_groups = TSgroup.objects.all()
+            managed_groups = {g:int(remote_groups[g]) for g in remote_groups if g in set(remote_groups.keys()) - set(ReservedGroupName.objects.values_list('name', flat=True))}
-            logger.debug("Comparing remote groups to TSgroup objects: %s" % local_groups)
+            remove = TSgroup.objects.exclude(ts_group_id__in=managed_groups.values())
-            for key in remote_groups:
+
-                logger.debug(f"Typecasting remote_group value at position {key} to int: {remote_groups[key]}")
+            if remove:
-                remote_groups[key] = int(remote_groups[key])
+                logger.debug(f"Deleting {remove.count()} TSgroup models: not found on server, or reserved name.")
+                remove.delete()
+
+            add = {g:managed_groups[g] for g in managed_groups if managed_groups[g] in set(managed_groups.values()) - set(TSgroup.objects.values_list("ts_group_id", flat=True))}
+            if add:
+                logger.debug(f"Adding {len(add)} new TSgroup models.")
+                models = [TSgroup(ts_group_name=name, ts_group_id=add[name]) for name in add]
+                TSgroup.objects.bulk_create(models)
 
-            for group in local_groups:
-                logger.debug("Checking local group %s" % group)
-                if group.ts_group_id not in remote_groups.values():
-                    logger.debug(
-                        f"Local group id {group.ts_group_id} not found on server. Deleting model {group}")
-                    TSgroup.objects.filter(ts_group_id=group.ts_group_id).delete()
-            for key in remote_groups:
-                g = TSgroup(ts_group_id=remote_groups[key], ts_group_name=key)
-                q = TSgroup.objects.filter(ts_group_id=g.ts_group_id)
-                if not q:
-                    logger.debug("Local group does not exist for TS group {}. Creating TSgroup model {}".format(
-                        remote_groups[key], g))
-                    g.save()
         except TeamspeakError as e:
-            logger.error("Error occured while syncing TS group db: %s" % str(e))
+            logger.error(f"Error occurred while syncing TS group db: {str(e)}")
-        except:
+        except Exception:
-            logger.exception("An unhandled exception has occured while syncing TS groups.")
+            logger.exception(f"An unhandled exception has occurred while syncing TS groups.")
 
     def add_user(self, user, fmt_name):
         username_clean = self.__santatize_username(fmt_name[:30])
@@ -240,7 +234,7 @@ class Teamspeak3Manager:
                 logger.exception(f"Failed to delete user id {uid} from TS3 - received response {ret}")
                 return False
         else:
-            logger.warn("User with id %s not found on TS3 server. Assuming succesful deletion." % uid)
+            logger.warning("User with id %s not found on TS3 server. Assuming succesful deletion." % uid)
             return True
 
     def check_user_exists(self, uid):
@@ -270,7 +264,8 @@ class Teamspeak3Manager:
                     addgroups.append(ts_groups[ts_group_key])
             for user_ts_group_key in user_ts_groups:
                 if user_ts_groups[user_ts_group_key] not in ts_groups.values():
-                    remgroups.append(user_ts_groups[user_ts_group_key])
+                    if not ReservedGroupName.objects.filter(name=user_ts_group_key).exists():
+                        remgroups.append(user_ts_groups[user_ts_group_key])
 
             for g in addgroups:
                 logger.info(f"Adding Teamspeak user {userid} into group {g}")

allianceauth/services/modules/teamspeak3/tests.py

@@ -5,16 +5,18 @@ from django import urls
 from django.contrib.auth.models import User, Group, Permission
 from django.core.exceptions import ObjectDoesNotExist
 from django.db.models import signals
+from django.contrib.admin import AdminSite
 
 from allianceauth.tests.auth_utils import AuthUtils
 from .auth_hooks import Teamspeak3Service
 from .models import Teamspeak3User, AuthTS, TSgroup, StateGroup
 from .tasks import Teamspeak3Tasks
 from .signals import m2m_changed_authts_group, post_save_authts, post_delete_authts
+from .admin import AuthTSgroupAdmin
 
 from .manager import Teamspeak3Manager
 from .util.ts3 import TeamspeakError
-from allianceauth.authentication.models import State
+from allianceauth.groupmanagement.models import ReservedGroupName
 
 MODULE_PATH = 'allianceauth.services.modules.teamspeak3'
 DEFAULT_AUTH_GROUP = 'Member'
@@ -315,6 +317,9 @@ class Teamspeak3SignalsTestCase(TestCase):
 
 
 class Teamspeak3ManagerTestCase(TestCase):
+    @classmethod
+    def setUpTestData(cls):
+        cls.reserved = ReservedGroupName.objects.create(name='reserved', reason='tests', created_by='Bob, praise be!')
 
     @staticmethod
     def my_side_effect(*args, **kwargs):
@@ -334,8 +339,135 @@ class Teamspeak3ManagerTestCase(TestCase):
         manager._server = server
 
         # create test data
-        user = User.objects.create_user("dummy")
+        user = AuthUtils.create_user("dummy")
-        user.profile.state = State.objects.filter(name="Member").first()
+        AuthUtils.assign_state(user, AuthUtils.get_member_state())
 
         # perform test
         manager.add_user(user, "Dummy User")
+
+    @mock.patch.object(Teamspeak3Manager, '_get_userid')
+    @mock.patch.object(Teamspeak3Manager, '_user_group_list')
+    @mock.patch.object(Teamspeak3Manager, '_add_user_to_group')
+    @mock.patch.object(Teamspeak3Manager, '_remove_user_from_group')
+    def test_update_groups_add(self, remove, add, groups, userid):
+        """Add to one group"""
+        userid.return_value = 1
+        groups.return_value = {'test': 1}
+
+        Teamspeak3Manager().update_groups(1, {'test': 1, 'dummy': 2})
+        self.assertEqual(add.call_count, 1)
+        self.assertEqual(remove.call_count, 0)
+        self.assertEqual(add.call_args[0][1], 2)
+
+    @mock.patch.object(Teamspeak3Manager, '_get_userid')
+    @mock.patch.object(Teamspeak3Manager, '_user_group_list')
+    @mock.patch.object(Teamspeak3Manager, '_add_user_to_group')
+    @mock.patch.object(Teamspeak3Manager, '_remove_user_from_group')
+    def test_update_groups_remove(self, remove, add, groups, userid):
+        """Remove from one group"""
+        userid.return_value = 1
+        groups.return_value = {'test': '1', 'dummy': '2'}
+
+        Teamspeak3Manager().update_groups(1, {'test': 1})
+        self.assertEqual(add.call_count, 0)
+        self.assertEqual(remove.call_count, 1)
+        self.assertEqual(remove.call_args[0][1], 2)
+
+    @mock.patch.object(Teamspeak3Manager, '_get_userid')
+    @mock.patch.object(Teamspeak3Manager, '_user_group_list')
+    @mock.patch.object(Teamspeak3Manager, '_add_user_to_group')
+    @mock.patch.object(Teamspeak3Manager, '_remove_user_from_group')
+    def test_update_groups_remove_reserved(self, remove, add, groups, userid):
+        """Remove from one group, but do not touch reserved group"""
+        userid.return_value = 1
+        groups.return_value = {'test': 1, 'dummy': 2, self.reserved.name: 3}
+
+        Teamspeak3Manager().update_groups(1, {'test': 1})
+        self.assertEqual(add.call_count, 0)
+        self.assertEqual(remove.call_count, 1)
+        self.assertEqual(remove.call_args[0][1], 2)
+
+    @mock.patch.object(Teamspeak3Manager, '_group_list')
+    def test_sync_group_db_create(self, group_list):
+        """Populate the list of all TSgroups"""
+        group_list.return_value = {'allowed':'1', 'also allowed':'2'}
+        Teamspeak3Manager()._sync_ts_group_db()
+        self.assertEqual(TSgroup.objects.all().count(), 2)
+
+    @mock.patch.object(Teamspeak3Manager, '_group_list')
+    def test_sync_group_db_delete(self, group_list):
+        """Populate the list of all TSgroups, and delete one which no longer exists"""
+        TSgroup.objects.create(ts_group_name='deleted', ts_group_id=3)
+        group_list.return_value = {'allowed': '1'}
+        Teamspeak3Manager()._sync_ts_group_db()
+        self.assertEqual(TSgroup.objects.all().count(), 1)
+        self.assertFalse(TSgroup.objects.filter(ts_group_name='deleted').exists())
+
+    @mock.patch.object(Teamspeak3Manager, '_group_list')
+    def test_sync_group_db_dont_create_reserved(self, group_list):
+        """Populate the list of all TSgroups, ignoring a reserved group name"""
+        group_list.return_value = {'allowed': '1', 'reserved': '4'}
+        Teamspeak3Manager()._sync_ts_group_db()
+        self.assertEqual(TSgroup.objects.all().count(), 1)
+        self.assertFalse(TSgroup.objects.filter(ts_group_name='reserved').exists())
+
+    @mock.patch.object(Teamspeak3Manager, '_group_list')
+    def test_sync_group_db_delete_reserved(self, group_list):
+        """Populate the list of all TSgroups, deleting the TSgroup model for one which has become reserved"""
+        TSgroup.objects.create(ts_group_name='reserved', ts_group_id=4)
+        group_list.return_value = {'allowed': '1', 'reserved': '4'}
+        Teamspeak3Manager()._sync_ts_group_db()
+        self.assertEqual(TSgroup.objects.all().count(), 1)
+        self.assertFalse(TSgroup.objects.filter(ts_group_name='reserved').exists())
+
+    @mock.patch.object(Teamspeak3Manager, '_group_list')
+    def test_sync_group_db_partial_addition(self, group_list):
+        """Some TSgroups already exist in database, add new ones"""
+        TSgroup.objects.create(ts_group_name='allowed', ts_group_id=1)
+        group_list.return_value = {'allowed': '1', 'also allowed': '2'}
+        Teamspeak3Manager()._sync_ts_group_db()
+        self.assertEqual(TSgroup.objects.all().count(), 2)
+
+    @mock.patch.object(Teamspeak3Manager, '_group_list')
+    def test_sync_group_db_partial_removal(self, group_list):
+        """One TSgroup has been deleted on server, so remove its model"""
+        TSgroup.objects.create(ts_group_name='allowed', ts_group_id=1)
+        TSgroup.objects.create(ts_group_name='also allowed', ts_group_id=2)
+        group_list.return_value = {'allowed': '1'}
+        Teamspeak3Manager()._sync_ts_group_db()
+        self.assertEqual(TSgroup.objects.all().count(), 1)
+
+
+class MockRequest:
+    pass
+
+
+class MockSuperUser:
+    def has_perm(self, perm, obj=None):
+        return True
+
+
+request = MockRequest()
+request.user = MockSuperUser()
+
+
+class Teamspeak3AdminTestCase(TestCase):
+    @classmethod
+    def setUpTestData(cls):
+        cls.site = AdminSite()
+        cls.admin = AuthTSgroupAdmin(AuthTS, cls.site)
+        cls.group = Group.objects.create(name='test')
+        cls.ts_group = TSgroup.objects.create(ts_group_name='test')
+
+    def test_field_queryset_no_reserved_names(self):
+        """Ensure all groups are listed when no reserved names"""
+        form = self.admin.get_form(request)
+        self.assertQuerysetEqual(form.base_fields['auth_group']._get_queryset(), Group.objects.all())
+        self.assertQuerysetEqual(form.base_fields['ts_group']._get_queryset(), TSgroup.objects.all())
+
+    def test_field_queryset_reserved_names(self):
+        """Ensure reserved group names are filtered out"""
+        ReservedGroupName.objects.bulk_create([ReservedGroupName(name='test', reason='tests', created_by='Bob')])
+        form = self.admin.get_form(request)
+        self.assertQuerysetEqual(form.base_fields['auth_group']._get_queryset(), Group.objects.none())
+        self.assertQuerysetEqual(form.base_fields['ts_group']._get_queryset(), TSgroup.objects.none())

allianceauth/srp/templates/srp/data.html

@@ -267,7 +267,9 @@ ESC to cancel{% endblocktrans %}"id="blah"></i></th>
                     "targets": [4, 5],
                     "type": "num"
                 }
-            ]
+            ],
+            "stateSave": true,
+            "stateDuration": 0
         });
 
         // tooltip

allianceauth/static/js/eve-time.js

@@ -1,58 +1,20 @@
 $(document).ready(function () {
     'use strict';
 
-    /**
-     * check time
-     * @param i
-     * @returns {string}
-     */
-    let checkTime = function (i) {
-        if (i < 10) {
-            i = '0' + i;
-        }
-
-        return i;
-    };
-
     /**
      * render a JS clock for Eve Time
      * @param element
-     * @param utcOffset
      */
-    let renderClock = function (element, utcOffset) {
+    const renderClock = function (element) {
-        let today = new Date();
+        const datetimeNow = new Date();
-        let h = today.getUTCHours();
+        const h = String(datetimeNow.getUTCHours()).padStart(2, '0');
-        let m = today.getUTCMinutes();
+        const m = String(datetimeNow.getUTCMinutes()).padStart(2, '0');
-
-        h = h + utcOffset;
-
-        if (h > 24) {
-            h = h - 24;
-        }
-
-        if (h < 0) {
-            h = h + 24;
-        }
-
-        h = checkTime(h);
-        m = checkTime(m);
 
         element.html(h + ':' + m);
-
-        setTimeout(function () {
-            renderClock(element, 0);
-        }, 500);
     };
 
-    /**
+    // Start the Eve time clock in the top menu bar
-     * functions that need to be executed on load
+    setInterval(function () {
-     */
+        renderClock($('.eve-time-wrapper .eve-time-clock'));
-    let init = function () {
+    }, 500);
-        renderClock($('.eve-time-wrapper .eve-time-clock'), 0);
-    };
-
-    /**
-     * start the show
-     */
-    init();
 });

docker/docker-compose.yml

@@ -52,7 +52,7 @@ services:
       - auth_mysql
 
   grafana:
-    image: grafana/grafana-oss:8.2
+    image: grafana/grafana-oss:8.3.2
     restart: always
     depends_on:
       - auth_mysql

docs/conf.py

@@ -42,6 +42,7 @@ from recommonmark.transform import AutoStructify
 extensions = [
     'sphinx_rtd_theme',
     'sphinx.ext.autodoc',
+    'sphinx.ext.napoleon',
     'recommonmark',
 ]
 

docs/features/core/analytics.md

@@ -10,6 +10,12 @@ To Opt-Out, modify our pre-loaded token using the Admin dashboard */admin/analyt
 
 Each of the three features Daily Stats, Celery Events and Page Views can be enabled/Disabled independently.
 
+Alternatively, you can fully opt out of analytics with the following optional setting:
+
+```python
+ANALYTICS_DISABLED = True
+```
+
 ![Analytics Tokens](/_static/images/features/core/analytics/tokens.png)
 
 ## What
@@ -58,6 +64,8 @@ This data is stored in a Team Google Analytics Dashboard. The Maintainers all ha
 
 ### Analytics Event
 
+```eval_rst
 .. automodule:: allianceauth.analytics.tasks
     :members: analytics_event
     :undoc-members:
+```

docs/features/core/groups.md

@@ -48,7 +48,7 @@ When using Alliance Auth to manage external services like Discord, Auth will aut
 
 ```eval_rst
 .. note::
-    While this feature can help to avoid naming conflicts with groups on external services, the respective service component in Alliance Auth also needs to be build in such a way that it knows how to prevent these conflicts. Currently only the Discord service has this ability.
+    While this feature can help to avoid naming conflicts with groups on external services, the respective service component in Alliance Auth also needs to be build in such a way that it knows how to prevent these conflicts. Currently only the Discord and Teamspeak3 services have this ability.
 ```
 
 ## Managing groups

docs/features/services/teamspeak3.md

@@ -160,7 +160,7 @@ This error generally means teamspeak returned an error message that went unhandl
 
 This most commonly happens when your teamspeak server is externally hosted. You need to add the auth server IP to the teamspeak serverquery whitelist. This varies by provider.
 
-If you have SSH access to the server hosting it, you need to locate the teamspeak server folder and add the auth server IP on a new line in  `server_query_whitelist.txt`
+If you have SSH access to the server hosting it, you need to locate the teamspeak server folder and add the auth server IP on a new line in  `query_ip_allowlist.txt` (named `query_ip_whitelist.txt` on older teamspeak versions).
 
 ### `520 invalid loginname or password`