diff --git a/allianceauth/__init__.py b/allianceauth/__init__.py index 8d04fd9d..7146f256 100644 --- a/allianceauth/__init__.py +++ b/allianceauth/__init__.py @@ -1,7 +1,7 @@ # This will make sure the app is always imported when # Django starts so that shared_task will use this app. -__version__ = '2.9.3' +__version__ = '2.9.4' __title__ = 'Alliance Auth' __url__ = 'https://gitlab.com/allianceauth/allianceauth' NAME = f'{__title__} v{__version__}' diff --git a/allianceauth/analytics/middleware.py b/allianceauth/analytics/middleware.py index ea636b6c..4dd93685 100644 --- a/allianceauth/analytics/middleware.py +++ b/allianceauth/analytics/middleware.py @@ -1,5 +1,6 @@ from bs4 import BeautifulSoup +from django.conf import settings from django.utils.deprecation import MiddlewareMixin from .models import AnalyticsTokens, AnalyticsIdentifier from .tasks import send_ga_tracking_web_view @@ -10,6 +11,8 @@ import re class AnalyticsMiddleware(MiddlewareMixin): def process_response(self, request, response): """Django Middleware: Process Page Views and creates Analytics Celery Tasks""" + if getattr(settings, "ANALYTICS_DISABLED", False): + return response analyticstokens = AnalyticsTokens.objects.all() client_id = AnalyticsIdentifier.objects.get(id=1).identifier.hex try: diff --git a/allianceauth/analytics/migrations/0004_auto_20211015_0502.py b/allianceauth/analytics/migrations/0004_auto_20211015_0502.py index c9da3708..2cf2dc67 100644 --- a/allianceauth/analytics/migrations/0004_auto_20211015_0502.py +++ b/allianceauth/analytics/migrations/0004_auto_20211015_0502.py @@ -1,11 +1,11 @@ # Generated by Django 3.1.13 on 2021-10-15 05:02 +from django.core.exceptions import ObjectDoesNotExist from django.db import migrations def modify_aa_team_token_add_page_ignore_paths(apps, schema_editor): - # We can't import the Person model directly as it may be a newer - # version than this migration expects. We use the historical version. + # Add /admin/ and /user_notifications_count/ path to ignore AnalyticsPath = apps.get_model('analytics', 'AnalyticsPath') admin = AnalyticsPath.objects.create(ignore_path=r"^\/admin\/.*") @@ -17,8 +17,19 @@ def modify_aa_team_token_add_page_ignore_paths(apps, schema_editor): def undo_modify_aa_team_token_add_page_ignore_paths(apps, schema_editor): - # nothing should need to migrate away here? - return True + # + AnalyticsPath = apps.get_model('analytics', 'AnalyticsPath') + Tokens = apps.get_model('analytics', 'AnalyticsTokens') + + token = Tokens.objects.get(token="UA-186249766-2") + try: + admin = AnalyticsPath.objects.get(ignore_path=r"^\/admin\/.*", analyticstokens=token) + user_notifications_count = AnalyticsPath.objects.get(ignore_path=r"^\/user_notifications_count\/.*", analyticstokens=token) + admin.delete() + user_notifications_count.delete() + except ObjectDoesNotExist: + # Its fine if it doesnt exist, we just dont want them building up when re-migrating + pass class Migration(migrations.Migration): diff --git a/allianceauth/analytics/migrations/0006_more_ignore_paths.py b/allianceauth/analytics/migrations/0006_more_ignore_paths.py new file mode 100644 index 00000000..392b7f9e --- /dev/null +++ b/allianceauth/analytics/migrations/0006_more_ignore_paths.py @@ -0,0 +1,40 @@ +# Generated by Django 3.2.8 on 2021-10-19 01:47 + +from django.core.exceptions import ObjectDoesNotExist +from django.db import migrations + + +def modify_aa_team_token_add_page_ignore_paths(apps, schema_editor): + # Add the /account/activate path to ignore + + AnalyticsPath = apps.get_model('analytics', 'AnalyticsPath') + account_activate = AnalyticsPath.objects.create(ignore_path=r"^\/account\/activate\/.*") + + Tokens = apps.get_model('analytics', 'AnalyticsTokens') + token = Tokens.objects.get(token="UA-186249766-2") + token.ignore_paths.add(account_activate) + + +def undo_modify_aa_team_token_add_page_ignore_paths(apps, schema_editor): + # + AnalyticsPath = apps.get_model('analytics', 'AnalyticsPath') + Tokens = apps.get_model('analytics', 'AnalyticsTokens') + + token = Tokens.objects.get(token="UA-186249766-2") + + try: + account_activate = AnalyticsPath.objects.get(ignore_path=r"^\/account\/activate\/.*", analyticstokens=token) + account_activate.delete() + except ObjectDoesNotExist: + # Its fine if it doesnt exist, we just dont want them building up when re-migrating + pass + +class Migration(migrations.Migration): + + dependencies = [ + ('analytics', '0005_alter_analyticspath_ignore_path'), + ] + + operations = [ + migrations.RunPython(modify_aa_team_token_add_page_ignore_paths, undo_modify_aa_team_token_add_page_ignore_paths) + ] diff --git a/allianceauth/analytics/signals.py b/allianceauth/analytics/signals.py index b3943220..91565c5e 100644 --- a/allianceauth/analytics/signals.py +++ b/allianceauth/analytics/signals.py @@ -1,7 +1,8 @@ -from allianceauth.analytics.tasks import analytics_event -from celery.signals import task_failure, task_success - import logging +from celery.signals import task_failure, task_success +from django.conf import settings +from allianceauth.analytics.tasks import analytics_event + logger = logging.getLogger(__name__) @@ -11,6 +12,8 @@ def process_failure_signal( sender, task_id, signal, args, kwargs, einfo, **kw): logger.debug("Celery task_failure signal %s" % sender.__class__.__name__) + if getattr(settings, "ANALYTICS_DISABLED", False): + return category = sender.__module__ @@ -30,6 +33,8 @@ def process_failure_signal( @task_success.connect def celery_success_signal(sender, result=None, **kw): logger.debug("Celery task_success signal %s" % sender.__class__.__name__) + if getattr(settings, "ANALYTICS_DISABLED", False): + return category = sender.__module__ diff --git a/allianceauth/analytics/tasks.py b/allianceauth/analytics/tasks.py index 7fe22477..34826e21 100644 --- a/allianceauth/analytics/tasks.py +++ b/allianceauth/analytics/tasks.py @@ -21,8 +21,8 @@ if getattr(settings, "ANALYTICS_ENABLE_DEBUG", False) and settings.DEBUG: # Force sending of analytics data during in a debug/test environemt # Usefull for developers working on this feature. logger.warning( - "You have 'ANALYTICS_ENABLE_DEBUG' Enabled! " - "This debug instance will send analytics data!") + "You have 'ANALYTICS_ENABLE_DEBUG' Enabled! " + "This debug instance will send analytics data!") DEBUG_URL = COLLECTION_URL ANALYTICS_URL = COLLECTION_URL @@ -40,13 +40,12 @@ def analytics_event(category: str, Send a Google Analytics Event for each token stored Includes check for if its enabled/disabled - Parameters - ------- - `category` (str): Celery Namespace - `action` (str): Task Name - `label` (str): Optional, Task Success/Exception - `value` (int): Optional, If bulk, Query size, can be a binary True/False - `event_type` (str): Optional, Celery or Stats only, Default to Celery + Args: + `category` (str): Celery Namespace + `action` (str): Task Name + `label` (str): Optional, Task Success/Exception + `value` (int): Optional, If bulk, Query size, can be a binary True/False + `event_type` (str): Optional, Celery or Stats only, Default to Celery """ analyticstokens = AnalyticsTokens.objects.all() client_id = AnalyticsIdentifier.objects.get(id=1).identifier.hex @@ -60,20 +59,21 @@ def analytics_event(category: str, if allowed is True: tracking_id = token.token - send_ga_tracking_celery_event.s(tracking_id=tracking_id, - client_id=client_id, - category=category, - action=action, - label=label, - value=value).\ - apply_async(priority=9) + send_ga_tracking_celery_event.s( + tracking_id=tracking_id, + client_id=client_id, + category=category, + action=action, + label=label, + value=value).apply_async(priority=9) @shared_task() def analytics_daily_stats(): """Celery Task: Do not call directly - Gathers a series of daily statistics and sends analytics events containing them""" + Gathers a series of daily statistics and sends analytics events containing them + """ users = install_stat_users() tokens = install_stat_tokens() addons = install_stat_addons() diff --git a/allianceauth/analytics/tests/test_integration.py b/allianceauth/analytics/tests/test_integration.py new file mode 100644 index 00000000..0fbe445b --- /dev/null +++ b/allianceauth/analytics/tests/test_integration.py @@ -0,0 +1,108 @@ +from unittest.mock import patch +from urllib.parse import parse_qs + +import requests_mock + +from django.test import TestCase, override_settings + +from allianceauth.analytics.tasks import ANALYTICS_URL +from allianceauth.eveonline.tasks import update_character +from allianceauth.tests.auth_utils import AuthUtils + + +@override_settings(CELERY_ALWAYS_EAGER=True) +@requests_mock.mock() +class TestAnalyticsForViews(TestCase): + @override_settings(ANALYTICS_DISABLED=False) + def test_should_run_analytics(self, requests_mocker): + # given + requests_mocker.post(ANALYTICS_URL) + user = AuthUtils.create_user("Bruce Wayne") + self.client.force_login(user) + # when + response = self.client.get("/dashboard/") + # then + self.assertEqual(response.status_code, 200) + self.assertTrue(requests_mocker.called) + + @override_settings(ANALYTICS_DISABLED=True) + def test_should_not_run_analytics(self, requests_mocker): + # given + requests_mocker.post(ANALYTICS_URL) + user = AuthUtils.create_user("Bruce Wayne") + self.client.force_login(user) + # when + response = self.client.get("/dashboard/") + # then + self.assertEqual(response.status_code, 200) + self.assertFalse(requests_mocker.called) + + +@override_settings(CELERY_ALWAYS_EAGER=True) +@requests_mock.mock() +class TestAnalyticsForTasks(TestCase): + @override_settings(ANALYTICS_DISABLED=False) + @patch("allianceauth.eveonline.models.EveCharacter.objects.update_character") + def test_should_run_analytics_for_successful_task( + self, requests_mocker, mock_update_character + ): + # given + requests_mocker.post(ANALYTICS_URL) + user = AuthUtils.create_user("Bruce Wayne") + character = AuthUtils.add_main_character_2(user, "Bruce Wayne", 1001) + # when + update_character.delay(character.character_id) + # then + self.assertTrue(mock_update_character.called) + self.assertTrue(requests_mocker.called) + payload = parse_qs(requests_mocker.last_request.text) + self.assertListEqual(payload["el"], ["Success"]) + + @override_settings(ANALYTICS_DISABLED=True) + @patch("allianceauth.eveonline.models.EveCharacter.objects.update_character") + def test_should_not_run_analytics_for_successful_task( + self, requests_mocker, mock_update_character + ): + # given + requests_mocker.post(ANALYTICS_URL) + user = AuthUtils.create_user("Bruce Wayne") + character = AuthUtils.add_main_character_2(user, "Bruce Wayne", 1001) + # when + update_character.delay(character.character_id) + # then + self.assertTrue(mock_update_character.called) + self.assertFalse(requests_mocker.called) + + @override_settings(ANALYTICS_DISABLED=False) + @patch("allianceauth.eveonline.models.EveCharacter.objects.update_character") + def test_should_run_analytics_for_failed_task( + self, requests_mocker, mock_update_character + ): + # given + requests_mocker.post(ANALYTICS_URL) + mock_update_character.side_effect = RuntimeError + user = AuthUtils.create_user("Bruce Wayne") + character = AuthUtils.add_main_character_2(user, "Bruce Wayne", 1001) + # when + update_character.delay(character.character_id) + # then + self.assertTrue(mock_update_character.called) + self.assertTrue(requests_mocker.called) + payload = parse_qs(requests_mocker.last_request.text) + self.assertNotEqual(payload["el"], ["Success"]) + + @override_settings(ANALYTICS_DISABLED=True) + @patch("allianceauth.eveonline.models.EveCharacter.objects.update_character") + def test_should_not_run_analytics_for_failed_task( + self, requests_mocker, mock_update_character + ): + # given + requests_mocker.post(ANALYTICS_URL) + mock_update_character.side_effect = RuntimeError + user = AuthUtils.create_user("Bruce Wayne") + character = AuthUtils.add_main_character_2(user, "Bruce Wayne", 1001) + # when + update_character.delay(character.character_id) + # then + self.assertTrue(mock_update_character.called) + self.assertFalse(requests_mocker.called) diff --git a/allianceauth/corputils/templates/corputils/corpstats.html b/allianceauth/corputils/templates/corputils/corpstats.html index df5a8527..5ab9de1a 100644 --- a/allianceauth/corputils/templates/corputils/corpstats.html +++ b/allianceauth/corputils/templates/corputils/corpstats.html @@ -193,6 +193,8 @@ "columnDefs": [ { "sortable": false, "targets": [1] }, ], + "stateSave": true, + "stateDuration": 0 }); $('#table-members').DataTable({ "columnDefs": [ @@ -200,6 +202,8 @@ { "sortable": false, "targets": [0, 2] }, ], "order": [[ 1, "asc" ]], + "stateSave": true, + "stateDuration": 0 }); $('#table-unregistered').DataTable({ "columnDefs": [ @@ -207,6 +211,8 @@ { "sortable": false, "targets": [0, 2] }, ], "order": [[ 1, "asc" ]], + "stateSave": true, + "stateDuration": 0 }); }); diff --git a/allianceauth/corputils/templates/corputils/search.html b/allianceauth/corputils/templates/corputils/search.html index 502f748a..e8e80c1d 100644 --- a/allianceauth/corputils/templates/corputils/search.html +++ b/allianceauth/corputils/templates/corputils/search.html @@ -43,6 +43,9 @@ {% endblock %} {% block extra_script %} $(document).ready(function(){ - $('#table-search').DataTable(); + $('#table-search').DataTable({ + "stateSave": true, + "stateDuration": 0 + }); }); {% endblock %} diff --git a/allianceauth/eveonline/models.py b/allianceauth/eveonline/models.py index 390c00b4..1a7f5c39 100644 --- a/allianceauth/eveonline/models.py +++ b/allianceauth/eveonline/models.py @@ -1,13 +1,27 @@ -from django.db import models +import logging from typing import Union -from .managers import EveCharacterManager, EveCharacterProviderManager -from .managers import EveCorporationManager, EveCorporationProviderManager -from .managers import EveAllianceManager, EveAllianceProviderManager +from django.core.exceptions import ObjectDoesNotExist +from django.db import models +from esi.models import Token + +from allianceauth.notifications import notify + from . import providers from .evelinks import eveimageserver +from .managers import ( + EveAllianceManager, + EveAllianceProviderManager, + EveCharacterManager, + EveCharacterProviderManager, + EveCorporationManager, + EveCorporationProviderManager, +) + +logger = logging.getLogger(__name__) _DEFAULT_IMAGE_SIZE = 32 +DOOMHEIM_CORPORATION_ID = 1000001 class EveFactionInfo(models.Model): @@ -68,13 +82,12 @@ class EveAllianceInfo(models.Model): for corp_id in alliance.corp_ids: if not EveCorporationInfo.objects.filter(corporation_id=corp_id).exists(): EveCorporationInfo.objects.create_corporation(corp_id) - EveCorporationInfo.objects.filter( - corporation_id__in=alliance.corp_ids).update(alliance=self + EveCorporationInfo.objects.filter(corporation_id__in=alliance.corp_ids).update( + alliance=self ) - EveCorporationInfo.objects\ - .filter(alliance=self)\ - .exclude(corporation_id__in=alliance.corp_ids)\ - .update(alliance=None) + EveCorporationInfo.objects.filter(alliance=self).exclude( + corporation_id__in=alliance.corp_ids + ).update(alliance=None) def update_alliance(self, alliance: providers.Alliance = None): if alliance is None: @@ -182,6 +195,7 @@ class EveCorporationInfo(models.Model): class EveCharacter(models.Model): + """Character in Eve Online""" character_id = models.PositiveIntegerField(unique=True) character_name = models.CharField(max_length=254, unique=True) corporation_id = models.PositiveIntegerField() @@ -198,12 +212,20 @@ class EveCharacter(models.Model): class Meta: indexes = [ - models.Index(fields=['corporation_id',]), - models.Index(fields=['alliance_id',]), - models.Index(fields=['corporation_name',]), - models.Index(fields=['alliance_name',]), - models.Index(fields=['faction_id',]), - ] + models.Index(fields=['corporation_id',]), + models.Index(fields=['alliance_id',]), + models.Index(fields=['corporation_name',]), + models.Index(fields=['alliance_name',]), + models.Index(fields=['faction_id',]), + ] + + def __str__(self): + return self.character_name + + @property + def is_biomassed(self) -> bool: + """Whether this character is dead or not.""" + return self.corporation_id == DOOMHEIM_CORPORATION_ID @property def alliance(self) -> Union[EveAllianceInfo, None]: @@ -249,10 +271,36 @@ class EveCharacter(models.Model): self.faction_id = character.faction.id self.faction_name = character.faction.name self.save() + if self.is_biomassed: + self._remove_tokens_of_biomassed_character() return self - def __str__(self): - return self.character_name + def _remove_tokens_of_biomassed_character(self) -> None: + """Remove tokens of this biomassed character.""" + try: + user = self.character_ownership.user + except ObjectDoesNotExist: + return + tokens_to_delete = Token.objects.filter(character_id=self.character_id) + tokens_count = tokens_to_delete.count() + if not tokens_count: + return + tokens_to_delete.delete() + logger.info( + "%d tokens from user %s for biomassed character %s [id:%s] deleted.", + tokens_count, + user, + self, + self.character_id, + ) + notify( + user=user, + title=f"Character {self} biomassed", + message=( + f"Your former character {self} has been biomassed " + "and has been removed from the list of your alts." + ) + ) @staticmethod def generic_portrait_url( @@ -336,7 +384,6 @@ class EveCharacter(models.Model): """image URL for alliance of this character or empty string""" return self.alliance_logo_url(256) - def faction_logo_url(self, size=_DEFAULT_IMAGE_SIZE) -> str: """image URL for alliance of this character or empty string""" if self.faction_id: diff --git a/allianceauth/eveonline/providers.py b/allianceauth/eveonline/providers.py index f399613e..fb1749dd 100644 --- a/allianceauth/eveonline/providers.py +++ b/allianceauth/eveonline/providers.py @@ -170,7 +170,7 @@ class EveProvider: """ :return: an ItemType object for the given ID """ - raise NotImplemented() + raise NotImplementedError() class EveSwaggerProvider(EveProvider): @@ -207,7 +207,8 @@ class EveSwaggerProvider(EveProvider): def __str__(self): return 'esi' - def get_alliance(self, alliance_id): + def get_alliance(self, alliance_id: int) -> Alliance: + """Fetch alliance from ESI.""" try: data = self.client.Alliance.get_alliances_alliance_id(alliance_id=alliance_id).result() corps = self.client.Alliance.get_alliances_alliance_id_corporations(alliance_id=alliance_id).result() @@ -223,7 +224,8 @@ class EveSwaggerProvider(EveProvider): except HTTPNotFound: raise ObjectNotFound(alliance_id, 'alliance') - def get_corp(self, corp_id): + def get_corp(self, corp_id: int) -> Corporation: + """Fetch corporation from ESI.""" try: data = self.client.Corporation.get_corporations_corporation_id(corporation_id=corp_id).result() model = Corporation( @@ -239,29 +241,43 @@ class EveSwaggerProvider(EveProvider): except HTTPNotFound: raise ObjectNotFound(corp_id, 'corporation') - def get_character(self, character_id): + def get_character(self, character_id: int) -> Character: + """Fetch character from ESI.""" try: - data = self.client.Character.get_characters_character_id(character_id=character_id).result() + character_name = self._fetch_character_name(character_id) affiliation = self.client.Character.post_characters_affiliation(characters=[character_id]).result()[0] - model = Character( id=character_id, - name=data['name'], + name=character_name, corp_id=affiliation['corporation_id'], alliance_id=affiliation['alliance_id'] if 'alliance_id' in affiliation else None, faction_id=affiliation['faction_id'] if 'faction_id' in affiliation else None, ) return model - except (HTTPNotFound, HTTPUnprocessableEntity): + except (HTTPNotFound, HTTPUnprocessableEntity, ObjectNotFound): raise ObjectNotFound(character_id, 'character') + def _fetch_character_name(self, character_id: int) -> str: + """Fetch character name from ESI.""" + data = self.client.Universe.post_universe_names(ids=[character_id]).result() + character = data.pop() if data else None + if ( + not character + or character["category"] != "character" + or character["id"] != character_id + ): + raise ObjectNotFound(character_id, 'character') + return character["name"] + def get_all_factions(self): + """Fetch all factions from ESI.""" if not self._faction_list: self._faction_list = self.client.Universe.get_universe_factions().result() return self._faction_list - def get_faction(self, faction_id): - faction_id=int(faction_id) + def get_faction(self, faction_id: int): + """Fetch faction from ESI.""" + faction_id = int(faction_id) try: if not self._faction_list: _ = self.get_all_factions() @@ -273,7 +289,8 @@ class EveSwaggerProvider(EveProvider): except (HTTPNotFound, HTTPUnprocessableEntity, KeyError): raise ObjectNotFound(faction_id, 'faction') - def get_itemtype(self, type_id): + def get_itemtype(self, type_id: int) -> ItemType: + """Fetch inventory item from ESI.""" try: data = self.client.Universe.get_universe_types_type_id(type_id=type_id).result() return ItemType(id=type_id, name=data['name']) diff --git a/allianceauth/eveonline/tasks.py b/allianceauth/eveonline/tasks.py index 13825f9b..a6b11a24 100644 --- a/allianceauth/eveonline/tasks.py +++ b/allianceauth/eveonline/tasks.py @@ -1,12 +1,11 @@ import logging from celery import shared_task -from .models import EveAllianceInfo -from .models import EveCharacter -from .models import EveCorporationInfo +from .models import EveAllianceInfo, EveCharacter, EveCorporationInfo from . import providers + logger = logging.getLogger(__name__) TASK_PRIORITY = 7 @@ -32,8 +31,8 @@ def update_alliance(alliance_id): @shared_task -def update_character(character_id): - """Update given character from ESI""" +def update_character(character_id: int) -> None: + """Update given character from ESI.""" EveCharacter.objects.update_character(character_id) @@ -65,17 +64,17 @@ def update_character_chunk(character_ids_chunk: list): .post_characters_affiliation(characters=character_ids_chunk).result() character_names = providers.provider.client.Universe\ .post_universe_names(ids=character_ids_chunk).result() - except: + except OSError: logger.info("Failed to bulk update characters. Attempting single updates") for character_id in character_ids_chunk: update_character.apply_async( - args=[character_id], priority=TASK_PRIORITY - ) + args=[character_id], priority=TASK_PRIORITY + ) return affiliations = { - affiliation.get('character_id'): affiliation - for affiliation in affiliations_raw + affiliation.get('character_id'): affiliation + for affiliation in affiliations_raw } # add character names to affiliations for character in character_names: @@ -108,5 +107,5 @@ def update_character_chunk(character_ids_chunk: list): if corp_changed or alliance_changed or name_changed: update_character.apply_async( - args=[character.get('character_id')], priority=TASK_PRIORITY - ) + args=[character.get('character_id')], priority=TASK_PRIORITY + ) diff --git a/allianceauth/eveonline/tests/esi_client_stub.py b/allianceauth/eveonline/tests/esi_client_stub.py new file mode 100644 index 00000000..1eadfbd3 --- /dev/null +++ b/allianceauth/eveonline/tests/esi_client_stub.py @@ -0,0 +1,168 @@ +from bravado.exception import HTTPNotFound + + +class BravadoResponseStub: + """Stub for IncomingResponse in bravado, e.g. for HTTPError exceptions""" + + def __init__( + self, status_code, reason="", text="", headers=None, raw_bytes=None + ) -> None: + self.reason = reason + self.status_code = status_code + self.text = text + self.headers = headers if headers else dict() + self.raw_bytes = raw_bytes + + def __str__(self): + return f"{self.status_code} {self.reason}" + + +class BravadoOperationStub: + """Stub to simulate the operation object return from bravado via django-esi""" + + class RequestConfig: + def __init__(self, also_return_response): + self.also_return_response = also_return_response + + class ResponseStub: + def __init__(self, headers): + self.headers = headers + + def __init__(self, data, headers: dict = None, also_return_response: bool = False): + self._data = data + self._headers = headers if headers else {"x-pages": 1} + self.request_config = BravadoOperationStub.RequestConfig(also_return_response) + + def result(self, **kwargs): + if self.request_config.also_return_response: + return [self._data, self.ResponseStub(self._headers)] + else: + return self._data + + def results(self, **kwargs): + return self.result(**kwargs) + + +class EsiClientStub: + """Stub for an ESI client.""" + class Alliance: + @staticmethod + def get_alliances_alliance_id(alliance_id): + data = { + 3001: { + "name": "Wayne Enterprises", + "ticker": "WYE", + "executor_corporation_id": 2001 + } + } + try: + return BravadoOperationStub(data[int(alliance_id)]) + except KeyError: + response = BravadoResponseStub( + 404, f"Alliance with ID {alliance_id} not found" + ) + raise HTTPNotFound(response) + + @staticmethod + def get_alliances_alliance_id_corporations(alliance_id): + data = [2001, 2002, 2003] + return BravadoOperationStub(data) + + class Character: + @staticmethod + def get_characters_character_id(character_id): + data = { + 1001: { + "corporation_id": 2001, + "name": "Bruce Wayne", + }, + 1002: { + "corporation_id": 2001, + "name": "Peter Parker", + }, + 1011: { + "corporation_id": 2011, + "name": "Lex Luthor", + } + } + try: + return BravadoOperationStub(data[int(character_id)]) + except KeyError: + response = BravadoResponseStub( + 404, f"Character with ID {character_id} not found" + ) + raise HTTPNotFound(response) + + @staticmethod + def post_characters_affiliation(characters: list): + data = [ + {'character_id': 1001, 'corporation_id': 2001, 'alliance_id': 3001}, + {'character_id': 1002, 'corporation_id': 2001, 'alliance_id': 3001}, + {'character_id': 1011, 'corporation_id': 2011}, + {'character_id': 1666, 'corporation_id': 1000001}, + ] + return BravadoOperationStub( + [x for x in data if x['character_id'] in characters] + ) + + class Corporation: + @staticmethod + def get_corporations_corporation_id(corporation_id): + data = { + 2001: { + "ceo_id": 1091, + "member_count": 10, + "name": "Wayne Technologies", + "ticker": "WTE", + "alliance_id": 3001 + }, + 2002: { + "ceo_id": 1092, + "member_count": 10, + "name": "Wayne Food", + "ticker": "WFO", + "alliance_id": 3001 + }, + 2003: { + "ceo_id": 1093, + "member_count": 10, + "name": "Wayne Energy", + "ticker": "WEG", + "alliance_id": 3001 + }, + 2011: { + "ceo_id": 1, + "member_count": 3, + "name": "LexCorp", + "ticker": "LC", + }, + 1000001: { + "ceo_id": 3000001, + "creator_id": 1, + "description": "The internal corporation used for characters in graveyard.", + "member_count": 6329026, + "name": "Doomheim", + "ticker": "666", + } + } + try: + return BravadoOperationStub(data[int(corporation_id)]) + except KeyError: + response = BravadoResponseStub( + 404, f"Corporation with ID {corporation_id} not found" + ) + raise HTTPNotFound(response) + + class Universe: + @staticmethod + def post_universe_names(ids: list): + data = [ + {"category": "character", "id": 1001, "name": "Bruce Wayne"}, + {"category": "character", "id": 1002, "name": "Peter Parker"}, + {"category": "character", "id": 1011, "name": "Lex Luthor"}, + {"category": "character", "id": 1666, "name": "Hal Jordan"}, + {"category": "corporation", "id": 2001, "name": "Wayne Technologies"}, + {"category": "corporation","id": 2002, "name": "Wayne Food"}, + {"category": "corporation","id": 1000001, "name": "Doomheim"}, + ] + return BravadoOperationStub([x for x in data if x['id'] in ids]) diff --git a/allianceauth/eveonline/tests/test_models.py b/allianceauth/eveonline/tests/test_models.py index fc716c00..94c9614a 100644 --- a/allianceauth/eveonline/tests/test_models.py +++ b/allianceauth/eveonline/tests/test_models.py @@ -1,12 +1,15 @@ from unittest.mock import Mock, patch +from django.core.exceptions import ObjectDoesNotExist from django.test import TestCase +from esi.models import Token + +from allianceauth.tests.auth_utils import AuthUtils -from ..models import ( - EveCharacter, EveCorporationInfo, EveAllianceInfo, EveFactionInfo -) -from ..providers import Alliance, Corporation, Character from ..evelinks import eveimageserver +from ..models import EveAllianceInfo, EveCharacter, EveCorporationInfo, EveFactionInfo +from ..providers import Alliance, Character, Corporation +from .esi_client_stub import EsiClientStub class EveCharacterTestCase(TestCase): @@ -402,8 +405,8 @@ class EveAllianceTestCase(TestCase): my_alliance.save() my_alliance.populate_alliance() - for corporation in EveCorporationInfo.objects\ - .filter(corporation_id__in=[2001, 2002] + for corporation in ( + EveCorporationInfo.objects.filter(corporation_id__in=[2001, 2002]) ): self.assertEqual(corporation.alliance, my_alliance) @@ -587,3 +590,98 @@ class EveCorporationTestCase(TestCase): self.my_corp.logo_url_256, 'https://images.evetech.net/corporations/2001/logo?size=256' ) + + +@patch('allianceauth.eveonline.providers.esi_client_factory') +@patch("allianceauth.eveonline.models.notify") +class TestCharacterUpdate(TestCase): + def test_should_update_normal_character(self, mock_notify, mock_esi_client_factory): + # given + mock_esi_client_factory.return_value = EsiClientStub() + my_character = EveCharacter.objects.create( + character_id=1001, + character_name="not my name", + corporation_id=2002, + corporation_name="Wayne Food", + corporation_ticker="WYF", + alliance_id=None + ) + # when + my_character.update_character() + # then + my_character.refresh_from_db() + self.assertEqual(my_character.character_name, "Bruce Wayne") + self.assertEqual(my_character.corporation_id, 2001) + self.assertEqual(my_character.corporation_name, "Wayne Technologies") + self.assertEqual(my_character.corporation_ticker, "WTE") + self.assertEqual(my_character.alliance_id, 3001) + self.assertEqual(my_character.alliance_name, "Wayne Enterprises") + self.assertEqual(my_character.alliance_ticker, "WYE") + self.assertFalse(mock_notify.called) + + def test_should_update_dead_character_with_owner( + self, mock_notify, mock_esi_client_factory + ): + # given + mock_esi_client_factory.return_value = EsiClientStub() + character_1666 = EveCharacter.objects.create( + character_id=1666, + character_name="Hal Jordan", + corporation_id=2002, + corporation_name="Wayne Food", + corporation_ticker="WYF", + alliance_id=None + ) + user = AuthUtils.create_user("Bruce Wayne") + token_1666 = Token.objects.create( + user=user, + character_id=character_1666.character_id, + character_name=character_1666.character_name, + character_owner_hash="ABC123-1666", + ) + character_1001 = EveCharacter.objects.create( + character_id=1001, + character_name="Bruce Wayne", + corporation_id=2001, + corporation_name="Wayne Technologies", + corporation_ticker="WYT", + alliance_id=None + ) + token_1001 = Token.objects.create( + user=user, + character_id=character_1001.character_id, + character_name=character_1001.character_name, + character_owner_hash="ABC123-1001", + ) + # when + character_1666.update_character() + # then + character_1666.refresh_from_db() + self.assertTrue(character_1666.is_biomassed) + self.assertNotIn(token_1666, user.token_set.all()) + self.assertIn(token_1001, user.token_set.all()) + with self.assertRaises(ObjectDoesNotExist): + self.assertTrue(character_1666.character_ownership) + user.profile.refresh_from_db() + self.assertIsNone(user.profile.main_character) + self.assertTrue(mock_notify.called) + + def test_should_handle_dead_character_without_owner( + self, mock_notify, mock_esi_client_factory + ): + # given + mock_esi_client_factory.return_value = EsiClientStub() + character_1666 = EveCharacter.objects.create( + character_id=1666, + character_name="Hal Jordan", + corporation_id=1011, + corporation_name="LexCorp", + corporation_ticker='LC', + alliance_id=None + ) + # when + character_1666.update_character() + # then + character_1666.refresh_from_db() + self.assertTrue(character_1666.is_biomassed) + self.assertFalse(mock_notify.called) diff --git a/allianceauth/eveonline/tests/test_providers.py b/allianceauth/eveonline/tests/test_providers.py index 59e9dfe6..95f4f069 100644 --- a/allianceauth/eveonline/tests/test_providers.py +++ b/allianceauth/eveonline/tests/test_providers.py @@ -7,6 +7,7 @@ from jsonschema.exceptions import RefResolutionError from django.test import TestCase from . import set_logger +from .esi_client_stub import EsiClientStub from ..providers import ( ObjectNotFound, Entity, @@ -632,13 +633,7 @@ class TestEveSwaggerProvider(TestCase): @patch(MODULE_PATH + '.esi_client_factory') def test_get_character(self, mock_esi_client_factory): - mock_esi_client_factory.return_value \ - .Character.get_characters_character_id \ - = TestEveSwaggerProvider.esi_get_characters_character_id - mock_esi_client_factory.return_value \ - .Character.post_characters_affiliation \ - = TestEveSwaggerProvider.esi_post_characters_affiliation - + mock_esi_client_factory.return_value = EsiClientStub() my_provider = EveSwaggerProvider() # character with alliance @@ -649,8 +644,8 @@ class TestEveSwaggerProvider(TestCase): self.assertEqual(my_character.alliance_id, 3001) # character wo/ alliance - my_character = my_provider.get_character(1002) - self.assertEqual(my_character.id, 1002) + my_character = my_provider.get_character(1011) + self.assertEqual(my_character.id, 1011) self.assertEqual(my_character.alliance_id, None) # character not found diff --git a/allianceauth/eveonline/tests/test_tasks.py b/allianceauth/eveonline/tests/test_tasks.py index 4bf40671..78054e8f 100644 --- a/allianceauth/eveonline/tests/test_tasks.py +++ b/allianceauth/eveonline/tests/test_tasks.py @@ -1,245 +1,271 @@ -from unittest.mock import patch, Mock +from unittest.mock import patch -from django.test import TestCase +from django.test import TestCase, TransactionTestCase, override_settings -from ..models import EveCharacter, EveCorporationInfo, EveAllianceInfo +from ..models import EveAllianceInfo, EveCharacter, EveCorporationInfo from ..tasks import ( + run_model_update, update_alliance, - update_corp, update_character, - run_model_update + update_character_chunk, + update_corp, ) +from .esi_client_stub import EsiClientStub -class TestTasks(TestCase): - - @patch('allianceauth.eveonline.tasks.EveCorporationInfo') - def test_update_corp(self, mock_EveCorporationInfo): - update_corp(42) - self.assertEqual( - mock_EveCorporationInfo.objects.update_corporation.call_count, 1 - ) - self.assertEqual( - mock_EveCorporationInfo.objects.update_corporation.call_args[0][0], 42 +@patch('allianceauth.eveonline.providers.esi_client_factory') +class TestUpdateTasks(TestCase): + def test_should_update_alliance(self, mock_esi_client_factory): + # given + mock_esi_client_factory.return_value = EsiClientStub() + my_alliance = EveAllianceInfo.objects.create( + alliance_id=3001, + alliance_name="Wayne Enterprises", + alliance_ticker="WYE", + executor_corp_id=2003 ) + # when + update_alliance(my_alliance.alliance_id) + # then + my_alliance.refresh_from_db() + self.assertEqual(my_alliance.executor_corp_id, 2001) - @patch('allianceauth.eveonline.tasks.EveAllianceInfo') - def test_update_alliance(self, mock_EveAllianceInfo): - update_alliance(42) - self.assertEqual( - mock_EveAllianceInfo.objects.update_alliance.call_args[0][0], 42 - ) - self.assertEqual( - mock_EveAllianceInfo.objects - .update_alliance.return_value.populate_alliance.call_count, 1 + def test_should_update_character(self, mock_esi_client_factory): + # given + mock_esi_client_factory.return_value = EsiClientStub() + my_character = EveCharacter.objects.create( + character_id=1001, + character_name="Bruce Wayne", + corporation_id=2002, + corporation_name="Wayne Food", + corporation_ticker="WYF", + alliance_id=None ) + # when + update_character(my_character.character_id) + # then + my_character.refresh_from_db() + self.assertEqual(my_character.corporation_id, 2001) - @patch('allianceauth.eveonline.tasks.EveCharacter') - def test_update_character(self, mock_EveCharacter): - update_character(42) - self.assertEqual( - mock_EveCharacter.objects.update_character.call_count, 1 + def test_should_update_corp(self, mock_esi_client_factory): + # given + mock_esi_client_factory.return_value = EsiClientStub() + EveAllianceInfo.objects.create( + alliance_id=3001, + alliance_name="Wayne Enterprises", + alliance_ticker="WYE", + executor_corp_id=2003 ) - self.assertEqual( - mock_EveCharacter.objects.update_character.call_args[0][0], 42 + my_corporation = EveCorporationInfo.objects.create( + corporation_id=2003, + corporation_name="Wayne Food", + corporation_ticker="WFO", + member_count=1, + alliance=None, + ceo_id=1999 ) + # when + update_corp(my_corporation.corporation_id) + # then + my_corporation.refresh_from_db() + self.assertEqual(my_corporation.alliance.alliance_id, 3001) + + # @patch('allianceauth.eveonline.tasks.EveCharacter') + # def test_update_character(self, mock_EveCharacter): + # update_character(42) + # self.assertEqual( + # mock_EveCharacter.objects.update_character.call_count, 1 + # ) + # self.assertEqual( + # mock_EveCharacter.objects.update_character.call_args[0][0], 42 + # ) -@patch('allianceauth.eveonline.tasks.update_character') -@patch('allianceauth.eveonline.tasks.update_alliance') -@patch('allianceauth.eveonline.tasks.update_corp') -@patch('allianceauth.eveonline.providers.provider') +@override_settings(CELERY_ALWAYS_EAGER=True) +@patch('allianceauth.eveonline.providers.esi_client_factory') +@patch('allianceauth.eveonline.tasks.providers') @patch('allianceauth.eveonline.tasks.CHUNK_SIZE', 2) -class TestRunModelUpdate(TestCase): - - @classmethod - def setUpClass(cls): - super().setUpClass() - EveCorporationInfo.objects.all().delete() - EveAllianceInfo.objects.all().delete() - EveCharacter.objects.all().delete() - +class TestRunModelUpdate(TransactionTestCase): + def test_should_run_updates(self, mock_providers, mock_esi_client_factory): + # given + mock_providers.provider.client = EsiClientStub() + mock_esi_client_factory.return_value = EsiClientStub() EveCorporationInfo.objects.create( - corporation_id=2345, - corporation_name='corp.name', - corporation_ticker='c.c.t', + corporation_id=2001, + corporation_name="Wayne Technologies", + corporation_ticker="WTE", member_count=10, alliance=None, ) - EveAllianceInfo.objects.create( - alliance_id=3456, - alliance_name='alliance.name', - alliance_ticker='a.t', - executor_corp_id=5, + alliance_3001 = EveAllianceInfo.objects.create( + alliance_id=3001, + alliance_name="Wayne Enterprises", + alliance_ticker="WYE", + executor_corp_id=2003 ) - EveCharacter.objects.create( - character_id=1, - character_name='character.name1', - corporation_id=2345, - corporation_name='character.corp.name', - corporation_ticker='c.c.t', # max 5 chars + corporation_2003 = EveCorporationInfo.objects.create( + corporation_id=2003, + corporation_name="Wayne Energy", + corporation_ticker="WEG", + member_count=99, + alliance=None, + ) + character_1001 = EveCharacter.objects.create( + character_id=1001, + character_name="Bruce Wayne", + corporation_id=2002, + corporation_name="Wayne Food", + corporation_ticker="WYF", alliance_id=None ) - EveCharacter.objects.create( - character_id=2, - character_name='character.name2', - corporation_id=9876, - corporation_name='character.corp.name', - corporation_ticker='c.c.t', # max 5 chars - alliance_id=3456, - alliance_name='character.alliance.name', - ) - EveCharacter.objects.create( - character_id=3, - character_name='character.name3', - corporation_id=9876, - corporation_name='character.corp.name', - corporation_ticker='c.c.t', # max 5 chars - alliance_id=3456, - alliance_name='character.alliance.name', - ) - EveCharacter.objects.create( - character_id=4, - character_name='character.name4', - corporation_id=9876, - corporation_name='character.corp.name', - corporation_ticker='c.c.t', # max 5 chars - alliance_id=3456, - alliance_name='character.alliance.name', - ) - """ - EveCharacter.objects.create( - character_id=5, - character_name='character.name5', - corporation_id=9876, - corporation_name='character.corp.name', - corporation_ticker='c.c.t', # max 5 chars - alliance_id=3456, - alliance_name='character.alliance.name', - ) - """ - - def setUp(self): - self.affiliations = [ - {'character_id': 1, 'corporation_id': 5}, - {'character_id': 2, 'corporation_id': 9876, 'alliance_id': 3456}, - {'character_id': 3, 'corporation_id': 9876, 'alliance_id': 7456}, - {'character_id': 4, 'corporation_id': 9876, 'alliance_id': 3456} - ] - self.names = [ - {'id': 1, 'name': 'character.name1'}, - {'id': 2, 'name': 'character.name2'}, - {'id': 3, 'name': 'character.name3'}, - {'id': 4, 'name': 'character.name4_new'} - ] - - def test_normal_run( - self, - mock_provider, - mock_update_corp, - mock_update_alliance, - mock_update_character, - ): - def get_affiliations(characters: list): - response = [x for x in self.affiliations if x['character_id'] in characters] - mock_operator = Mock(**{'result.return_value': response}) - return mock_operator - - def get_names(ids: list): - response = [x for x in self.names if x['id'] in ids] - mock_operator = Mock(**{'result.return_value': response}) - return mock_operator - - mock_provider.client.Character.post_characters_affiliation.side_effect \ - = get_affiliations - - mock_provider.client.Universe.post_universe_names.side_effect = get_names - + # when run_model_update() - + # then + character_1001.refresh_from_db() self.assertEqual( - mock_provider.client.Character.post_characters_affiliation.call_count, 2 + character_1001.corporation_id, 2001 # char has new corp ) + corporation_2003.refresh_from_db() self.assertEqual( - mock_provider.client.Universe.post_universe_names.call_count, 2 + corporation_2003.alliance.alliance_id, 3001 # corp has new alliance + ) + alliance_3001.refresh_from_db() + self.assertEqual( + alliance_3001.executor_corp_id, 2001 # alliance has been updated ) - # character 1 has changed corp - # character 2 no change - # character 3 has changed alliance - # character 4 has changed name - self.assertEqual(mock_update_corp.apply_async.call_count, 1) - self.assertEqual( - int(mock_update_corp.apply_async.call_args[1]['args'][0]), 2345 - ) - self.assertEqual(mock_update_alliance.apply_async.call_count, 1) - self.assertEqual( - int(mock_update_alliance.apply_async.call_args[1]['args'][0]), 3456 - ) - characters_updated = { - x[1]['args'][0] for x in mock_update_character.apply_async.call_args_list + +@override_settings(CELERY_ALWAYS_EAGER=True) +@patch('allianceauth.eveonline.tasks.update_character', wraps=update_character) +@patch('allianceauth.eveonline.providers.esi_client_factory') +@patch('allianceauth.eveonline.tasks.providers') +@patch('allianceauth.eveonline.tasks.CHUNK_SIZE', 2) +class TestUpdateCharacterChunk(TestCase): + @staticmethod + def _updated_character_ids(spy_update_character) -> set: + """Character IDs passed to update_character task for update.""" + return { + x[1]["args"][0] for x in spy_update_character.apply_async.call_args_list } - excepted = {1, 3, 4} - self.assertSetEqual(characters_updated, excepted) - def test_ignore_character_not_in_affiliations( - self, - mock_provider, - mock_update_corp, - mock_update_alliance, - mock_update_character, + def test_should_update_corp_change( + self, mock_providers, mock_esi_client_factory, spy_update_character ): - def get_affiliations(characters: list): - response = [x for x in self.affiliations if x['character_id'] in characters] - mock_operator = Mock(**{'result.return_value': response}) - return mock_operator + # given + mock_providers.provider.client = EsiClientStub() + mock_esi_client_factory.return_value = EsiClientStub() + character_1001 = EveCharacter.objects.create( + character_id=1001, + character_name="Bruce Wayne", + corporation_id=2003, + corporation_name="Wayne Energy", + corporation_ticker="WEG", + alliance_id=3001, + alliance_name="Wayne Enterprises", + alliance_ticker="WYE", + ) + character_1002 = EveCharacter.objects.create( + character_id=1002, + character_name="Peter Parker", + corporation_id=2001, + corporation_name="Wayne Technologies", + corporation_ticker="WTE", + alliance_id=3001, + alliance_name="Wayne Enterprises", + alliance_ticker="WYE", + ) + # when + update_character_chunk([ + character_1001.character_id, character_1002.character_id + ]) + # then + character_1001.refresh_from_db() + self.assertEqual(character_1001.corporation_id, 2001) + self.assertSetEqual(self._updated_character_ids(spy_update_character), {1001}) - def get_names(ids: list): - response = [x for x in self.names if x['id'] in ids] - mock_operator = Mock(**{'result.return_value': response}) - return mock_operator - - del self.affiliations[0] - - mock_provider.client.Character.post_characters_affiliation.side_effect \ - = get_affiliations - - mock_provider.client.Universe.post_universe_names.side_effect = get_names - - run_model_update() - characters_updated = { - x[1]['args'][0] for x in mock_update_character.apply_async.call_args_list - } - excepted = {3, 4} - self.assertSetEqual(characters_updated, excepted) - - def test_ignore_character_not_in_names( - self, - mock_provider, - mock_update_corp, - mock_update_alliance, - mock_update_character, + def test_should_update_name_change( + self, mock_providers, mock_esi_client_factory, spy_update_character ): - def get_affiliations(characters: list): - response = [x for x in self.affiliations if x['character_id'] in characters] - mock_operator = Mock(**{'result.return_value': response}) - return mock_operator + # given + mock_providers.provider.client = EsiClientStub() + mock_esi_client_factory.return_value = EsiClientStub() + character_1001 = EveCharacter.objects.create( + character_id=1001, + character_name="Batman", + corporation_id=2001, + corporation_name="Wayne Technologies", + corporation_ticker="WTE", + alliance_id=3001, + alliance_name="Wayne Technologies", + alliance_ticker="WYT", + ) + # when + update_character_chunk([character_1001.character_id]) + # then + character_1001.refresh_from_db() + self.assertEqual(character_1001.character_name, "Bruce Wayne") + self.assertSetEqual(self._updated_character_ids(spy_update_character), {1001}) - def get_names(ids: list): - response = [x for x in self.names if x['id'] in ids] - mock_operator = Mock(**{'result.return_value': response}) - return mock_operator + def test_should_update_alliance_change( + self, mock_providers, mock_esi_client_factory, spy_update_character + ): + # given + mock_providers.provider.client = EsiClientStub() + mock_esi_client_factory.return_value = EsiClientStub() + character_1001 = EveCharacter.objects.create( + character_id=1001, + character_name="Bruce Wayne", + corporation_id=2001, + corporation_name="Wayne Technologies", + corporation_ticker="WTE", + alliance_id=None, + ) + # when + update_character_chunk([character_1001.character_id]) + # then + character_1001.refresh_from_db() + self.assertEqual(character_1001.alliance_id, 3001) + self.assertSetEqual(self._updated_character_ids(spy_update_character), {1001}) - del self.names[3] + def test_should_not_update_when_not_changed( + self, mock_providers, mock_esi_client_factory, spy_update_character + ): + # given + mock_providers.provider.client = EsiClientStub() + mock_esi_client_factory.return_value = EsiClientStub() + character_1001 = EveCharacter.objects.create( + character_id=1001, + character_name="Bruce Wayne", + corporation_id=2001, + corporation_name="Wayne Technologies", + corporation_ticker="WTE", + alliance_id=3001, + alliance_name="Wayne Technologies", + alliance_ticker="WYT", + ) + # when + update_character_chunk([character_1001.character_id]) + # then + self.assertSetEqual(self._updated_character_ids(spy_update_character), set()) - mock_provider.client.Character.post_characters_affiliation.side_effect \ - = get_affiliations - - mock_provider.client.Universe.post_universe_names.side_effect = get_names - - run_model_update() - characters_updated = { - x[1]['args'][0] for x in mock_update_character.apply_async.call_args_list - } - excepted = {1, 3} - self.assertSetEqual(characters_updated, excepted) + def test_should_fall_back_to_single_updates_when_bulk_update_failed( + self, mock_providers, mock_esi_client_factory, spy_update_character + ): + # given + mock_providers.provider.client.Character.post_characters_affiliation\ + .side_effect = OSError + mock_esi_client_factory.return_value = EsiClientStub() + character_1001 = EveCharacter.objects.create( + character_id=1001, + character_name="Bruce Wayne", + corporation_id=2001, + corporation_name="Wayne Technologies", + corporation_ticker="WTE", + alliance_id=3001, + alliance_name="Wayne Technologies", + alliance_ticker="WYT", + ) + # when + update_character_chunk([character_1001.character_id]) + # then + self.assertSetEqual(self._updated_character_ids(spy_update_character), {1001}) diff --git a/allianceauth/groupmanagement/templates/groupmanagement/audit.html b/allianceauth/groupmanagement/templates/groupmanagement/audit.html index 7f1dc73b..4c1759de 100644 --- a/allianceauth/groupmanagement/templates/groupmanagement/audit.html +++ b/allianceauth/groupmanagement/templates/groupmanagement/audit.html @@ -127,6 +127,8 @@ ], bootstrap: true }, + "stateSave": true, + "stateDuration": 0 }); }); {% endblock %} diff --git a/allianceauth/groupmanagement/templates/groupmanagement/groupmembers.html b/allianceauth/groupmanagement/templates/groupmanagement/groupmembers.html index 1ce7e3fd..ab6be2b2 100644 --- a/allianceauth/groupmanagement/templates/groupmanagement/groupmembers.html +++ b/allianceauth/groupmanagement/templates/groupmanagement/groupmembers.html @@ -104,7 +104,9 @@ "sortable": false, "targets": [2] }, - ] + ], + "stateSave": true, + "stateDuration": 0 }); }); {% endblock %} diff --git a/allianceauth/permissions_tool/templates/permissions_tool/audit.html b/allianceauth/permissions_tool/templates/permissions_tool/audit.html index f168e74d..d04a5ed0 100644 --- a/allianceauth/permissions_tool/templates/permissions_tool/audit.html +++ b/allianceauth/permissions_tool/templates/permissions_tool/audit.html @@ -73,6 +73,8 @@ ], bootstrap: true }, + "stateSave": true, + "stateDuration": 0, drawCallback: function ( settings ) { let api = this.api(); let rows = api.rows( {page:'current'} ).nodes(); diff --git a/allianceauth/permissions_tool/templates/permissions_tool/overview.html b/allianceauth/permissions_tool/templates/permissions_tool/overview.html index 637e53a4..05bcfb80 100644 --- a/allianceauth/permissions_tool/templates/permissions_tool/overview.html +++ b/allianceauth/permissions_tool/templates/permissions_tool/overview.html @@ -106,8 +106,10 @@ idx: 1 } ], - bootstrap: true + bootstrap: true, }, + "stateSave": true, + "stateDuration": 0, drawCallback: function ( settings ) { let api = this.api(); let rows = api.rows( {page:'current'} ).nodes(); diff --git a/allianceauth/services/modules/teamspeak3/admin.py b/allianceauth/services/modules/teamspeak3/admin.py index a8b614d8..dbba1cb1 100644 --- a/allianceauth/services/modules/teamspeak3/admin.py +++ b/allianceauth/services/modules/teamspeak3/admin.py @@ -1,7 +1,8 @@ from django.contrib import admin - -from .models import AuthTS, Teamspeak3User, StateGroup +from django.contrib.auth.models import Group +from .models import AuthTS, Teamspeak3User, StateGroup, TSgroup from ...admin import ServicesUserAdmin +from allianceauth.groupmanagement.models import ReservedGroupName @admin.register(Teamspeak3User) @@ -25,6 +26,16 @@ class AuthTSgroupAdmin(admin.ModelAdmin): fields = ('auth_group', 'ts_group') filter_horizontal = ('ts_group',) + def formfield_for_foreignkey(self, db_field, request, **kwargs): + if db_field.name == 'auth_group': + kwargs['queryset'] = Group.objects.exclude(name__in=ReservedGroupName.objects.values_list('name', flat=True)) + return super().formfield_for_foreignkey(db_field, request, **kwargs) + + def formfield_for_manytomany(self, db_field, request, **kwargs): + if db_field.name == 'ts_group': + kwargs['queryset'] = TSgroup.objects.exclude(ts_group_name__in=ReservedGroupName.objects.values_list('name', flat=True)) + return super().formfield_for_manytomany(db_field, request, **kwargs) + def _ts_group(self, obj): return [x for x in obj.ts_group.all().order_by('ts_group_id')] diff --git a/allianceauth/services/modules/teamspeak3/manager.py b/allianceauth/services/modules/teamspeak3/manager.py index d05f910d..519a8a47 100755 --- a/allianceauth/services/modules/teamspeak3/manager.py +++ b/allianceauth/services/modules/teamspeak3/manager.py @@ -4,6 +4,7 @@ from django.conf import settings from .util.ts3 import TS3Server, TeamspeakError from .models import TSgroup +from allianceauth.groupmanagement.models import ReservedGroupName logger = logging.getLogger(__name__) @@ -156,32 +157,25 @@ class Teamspeak3Manager: logger.info(f"Removed user id {uid} from group id {groupid} on TS3 server.") def _sync_ts_group_db(self): - logger.debug("_sync_ts_group_db function called.") try: remote_groups = self._group_list() - local_groups = TSgroup.objects.all() - logger.debug("Comparing remote groups to TSgroup objects: %s" % local_groups) - for key in remote_groups: - logger.debug(f"Typecasting remote_group value at position {key} to int: {remote_groups[key]}") - remote_groups[key] = int(remote_groups[key]) + managed_groups = {g:int(remote_groups[g]) for g in remote_groups if g in set(remote_groups.keys()) - set(ReservedGroupName.objects.values_list('name', flat=True))} + remove = TSgroup.objects.exclude(ts_group_id__in=managed_groups.values()) + + if remove: + logger.debug(f"Deleting {remove.count()} TSgroup models: not found on server, or reserved name.") + remove.delete() + + add = {g:managed_groups[g] for g in managed_groups if managed_groups[g] in set(managed_groups.values()) - set(TSgroup.objects.values_list("ts_group_id", flat=True))} + if add: + logger.debug(f"Adding {len(add)} new TSgroup models.") + models = [TSgroup(ts_group_name=name, ts_group_id=add[name]) for name in add] + TSgroup.objects.bulk_create(models) - for group in local_groups: - logger.debug("Checking local group %s" % group) - if group.ts_group_id not in remote_groups.values(): - logger.debug( - f"Local group id {group.ts_group_id} not found on server. Deleting model {group}") - TSgroup.objects.filter(ts_group_id=group.ts_group_id).delete() - for key in remote_groups: - g = TSgroup(ts_group_id=remote_groups[key], ts_group_name=key) - q = TSgroup.objects.filter(ts_group_id=g.ts_group_id) - if not q: - logger.debug("Local group does not exist for TS group {}. Creating TSgroup model {}".format( - remote_groups[key], g)) - g.save() except TeamspeakError as e: - logger.error("Error occured while syncing TS group db: %s" % str(e)) - except: - logger.exception("An unhandled exception has occured while syncing TS groups.") + logger.error(f"Error occurred while syncing TS group db: {str(e)}") + except Exception: + logger.exception(f"An unhandled exception has occurred while syncing TS groups.") def add_user(self, user, fmt_name): username_clean = self.__santatize_username(fmt_name[:30]) @@ -240,7 +234,7 @@ class Teamspeak3Manager: logger.exception(f"Failed to delete user id {uid} from TS3 - received response {ret}") return False else: - logger.warn("User with id %s not found on TS3 server. Assuming succesful deletion." % uid) + logger.warning("User with id %s not found on TS3 server. Assuming succesful deletion." % uid) return True def check_user_exists(self, uid): @@ -270,7 +264,8 @@ class Teamspeak3Manager: addgroups.append(ts_groups[ts_group_key]) for user_ts_group_key in user_ts_groups: if user_ts_groups[user_ts_group_key] not in ts_groups.values(): - remgroups.append(user_ts_groups[user_ts_group_key]) + if not ReservedGroupName.objects.filter(name=user_ts_group_key).exists(): + remgroups.append(user_ts_groups[user_ts_group_key]) for g in addgroups: logger.info(f"Adding Teamspeak user {userid} into group {g}") diff --git a/allianceauth/services/modules/teamspeak3/tests.py b/allianceauth/services/modules/teamspeak3/tests.py index 1cb923ed..2354bf98 100644 --- a/allianceauth/services/modules/teamspeak3/tests.py +++ b/allianceauth/services/modules/teamspeak3/tests.py @@ -5,16 +5,18 @@ from django import urls from django.contrib.auth.models import User, Group, Permission from django.core.exceptions import ObjectDoesNotExist from django.db.models import signals +from django.contrib.admin import AdminSite from allianceauth.tests.auth_utils import AuthUtils from .auth_hooks import Teamspeak3Service from .models import Teamspeak3User, AuthTS, TSgroup, StateGroup from .tasks import Teamspeak3Tasks from .signals import m2m_changed_authts_group, post_save_authts, post_delete_authts +from .admin import AuthTSgroupAdmin from .manager import Teamspeak3Manager from .util.ts3 import TeamspeakError -from allianceauth.authentication.models import State +from allianceauth.groupmanagement.models import ReservedGroupName MODULE_PATH = 'allianceauth.services.modules.teamspeak3' DEFAULT_AUTH_GROUP = 'Member' @@ -315,6 +317,9 @@ class Teamspeak3SignalsTestCase(TestCase): class Teamspeak3ManagerTestCase(TestCase): + @classmethod + def setUpTestData(cls): + cls.reserved = ReservedGroupName.objects.create(name='reserved', reason='tests', created_by='Bob, praise be!') @staticmethod def my_side_effect(*args, **kwargs): @@ -334,8 +339,135 @@ class Teamspeak3ManagerTestCase(TestCase): manager._server = server # create test data - user = User.objects.create_user("dummy") - user.profile.state = State.objects.filter(name="Member").first() + user = AuthUtils.create_user("dummy") + AuthUtils.assign_state(user, AuthUtils.get_member_state()) # perform test manager.add_user(user, "Dummy User") + + @mock.patch.object(Teamspeak3Manager, '_get_userid') + @mock.patch.object(Teamspeak3Manager, '_user_group_list') + @mock.patch.object(Teamspeak3Manager, '_add_user_to_group') + @mock.patch.object(Teamspeak3Manager, '_remove_user_from_group') + def test_update_groups_add(self, remove, add, groups, userid): + """Add to one group""" + userid.return_value = 1 + groups.return_value = {'test': 1} + + Teamspeak3Manager().update_groups(1, {'test': 1, 'dummy': 2}) + self.assertEqual(add.call_count, 1) + self.assertEqual(remove.call_count, 0) + self.assertEqual(add.call_args[0][1], 2) + + @mock.patch.object(Teamspeak3Manager, '_get_userid') + @mock.patch.object(Teamspeak3Manager, '_user_group_list') + @mock.patch.object(Teamspeak3Manager, '_add_user_to_group') + @mock.patch.object(Teamspeak3Manager, '_remove_user_from_group') + def test_update_groups_remove(self, remove, add, groups, userid): + """Remove from one group""" + userid.return_value = 1 + groups.return_value = {'test': '1', 'dummy': '2'} + + Teamspeak3Manager().update_groups(1, {'test': 1}) + self.assertEqual(add.call_count, 0) + self.assertEqual(remove.call_count, 1) + self.assertEqual(remove.call_args[0][1], 2) + + @mock.patch.object(Teamspeak3Manager, '_get_userid') + @mock.patch.object(Teamspeak3Manager, '_user_group_list') + @mock.patch.object(Teamspeak3Manager, '_add_user_to_group') + @mock.patch.object(Teamspeak3Manager, '_remove_user_from_group') + def test_update_groups_remove_reserved(self, remove, add, groups, userid): + """Remove from one group, but do not touch reserved group""" + userid.return_value = 1 + groups.return_value = {'test': 1, 'dummy': 2, self.reserved.name: 3} + + Teamspeak3Manager().update_groups(1, {'test': 1}) + self.assertEqual(add.call_count, 0) + self.assertEqual(remove.call_count, 1) + self.assertEqual(remove.call_args[0][1], 2) + + @mock.patch.object(Teamspeak3Manager, '_group_list') + def test_sync_group_db_create(self, group_list): + """Populate the list of all TSgroups""" + group_list.return_value = {'allowed':'1', 'also allowed':'2'} + Teamspeak3Manager()._sync_ts_group_db() + self.assertEqual(TSgroup.objects.all().count(), 2) + + @mock.patch.object(Teamspeak3Manager, '_group_list') + def test_sync_group_db_delete(self, group_list): + """Populate the list of all TSgroups, and delete one which no longer exists""" + TSgroup.objects.create(ts_group_name='deleted', ts_group_id=3) + group_list.return_value = {'allowed': '1'} + Teamspeak3Manager()._sync_ts_group_db() + self.assertEqual(TSgroup.objects.all().count(), 1) + self.assertFalse(TSgroup.objects.filter(ts_group_name='deleted').exists()) + + @mock.patch.object(Teamspeak3Manager, '_group_list') + def test_sync_group_db_dont_create_reserved(self, group_list): + """Populate the list of all TSgroups, ignoring a reserved group name""" + group_list.return_value = {'allowed': '1', 'reserved': '4'} + Teamspeak3Manager()._sync_ts_group_db() + self.assertEqual(TSgroup.objects.all().count(), 1) + self.assertFalse(TSgroup.objects.filter(ts_group_name='reserved').exists()) + + @mock.patch.object(Teamspeak3Manager, '_group_list') + def test_sync_group_db_delete_reserved(self, group_list): + """Populate the list of all TSgroups, deleting the TSgroup model for one which has become reserved""" + TSgroup.objects.create(ts_group_name='reserved', ts_group_id=4) + group_list.return_value = {'allowed': '1', 'reserved': '4'} + Teamspeak3Manager()._sync_ts_group_db() + self.assertEqual(TSgroup.objects.all().count(), 1) + self.assertFalse(TSgroup.objects.filter(ts_group_name='reserved').exists()) + + @mock.patch.object(Teamspeak3Manager, '_group_list') + def test_sync_group_db_partial_addition(self, group_list): + """Some TSgroups already exist in database, add new ones""" + TSgroup.objects.create(ts_group_name='allowed', ts_group_id=1) + group_list.return_value = {'allowed': '1', 'also allowed': '2'} + Teamspeak3Manager()._sync_ts_group_db() + self.assertEqual(TSgroup.objects.all().count(), 2) + + @mock.patch.object(Teamspeak3Manager, '_group_list') + def test_sync_group_db_partial_removal(self, group_list): + """One TSgroup has been deleted on server, so remove its model""" + TSgroup.objects.create(ts_group_name='allowed', ts_group_id=1) + TSgroup.objects.create(ts_group_name='also allowed', ts_group_id=2) + group_list.return_value = {'allowed': '1'} + Teamspeak3Manager()._sync_ts_group_db() + self.assertEqual(TSgroup.objects.all().count(), 1) + + +class MockRequest: + pass + + +class MockSuperUser: + def has_perm(self, perm, obj=None): + return True + + +request = MockRequest() +request.user = MockSuperUser() + + +class Teamspeak3AdminTestCase(TestCase): + @classmethod + def setUpTestData(cls): + cls.site = AdminSite() + cls.admin = AuthTSgroupAdmin(AuthTS, cls.site) + cls.group = Group.objects.create(name='test') + cls.ts_group = TSgroup.objects.create(ts_group_name='test') + + def test_field_queryset_no_reserved_names(self): + """Ensure all groups are listed when no reserved names""" + form = self.admin.get_form(request) + self.assertQuerysetEqual(form.base_fields['auth_group']._get_queryset(), Group.objects.all()) + self.assertQuerysetEqual(form.base_fields['ts_group']._get_queryset(), TSgroup.objects.all()) + + def test_field_queryset_reserved_names(self): + """Ensure reserved group names are filtered out""" + ReservedGroupName.objects.bulk_create([ReservedGroupName(name='test', reason='tests', created_by='Bob')]) + form = self.admin.get_form(request) + self.assertQuerysetEqual(form.base_fields['auth_group']._get_queryset(), Group.objects.none()) + self.assertQuerysetEqual(form.base_fields['ts_group']._get_queryset(), TSgroup.objects.none()) diff --git a/allianceauth/srp/templates/srp/data.html b/allianceauth/srp/templates/srp/data.html index 3710c33f..d4f1e278 100644 --- a/allianceauth/srp/templates/srp/data.html +++ b/allianceauth/srp/templates/srp/data.html @@ -267,7 +267,9 @@ ESC to cancel{% endblocktrans %}"id="blah"> "targets": [4, 5], "type": "num" } - ] + ], + "stateSave": true, + "stateDuration": 0 }); // tooltip diff --git a/allianceauth/static/js/eve-time.js b/allianceauth/static/js/eve-time.js index ba5656fd..1cbe36d2 100644 --- a/allianceauth/static/js/eve-time.js +++ b/allianceauth/static/js/eve-time.js @@ -1,58 +1,20 @@ $(document).ready(function () { 'use strict'; - /** - * check time - * @param i - * @returns {string} - */ - let checkTime = function (i) { - if (i < 10) { - i = '0' + i; - } - - return i; - }; - /** * render a JS clock for Eve Time * @param element - * @param utcOffset */ - let renderClock = function (element, utcOffset) { - let today = new Date(); - let h = today.getUTCHours(); - let m = today.getUTCMinutes(); - - h = h + utcOffset; - - if (h > 24) { - h = h - 24; - } - - if (h < 0) { - h = h + 24; - } - - h = checkTime(h); - m = checkTime(m); + const renderClock = function (element) { + const datetimeNow = new Date(); + const h = String(datetimeNow.getUTCHours()).padStart(2, '0'); + const m = String(datetimeNow.getUTCMinutes()).padStart(2, '0'); element.html(h + ':' + m); - - setTimeout(function () { - renderClock(element, 0); - }, 500); }; - /** - * functions that need to be executed on load - */ - let init = function () { - renderClock($('.eve-time-wrapper .eve-time-clock'), 0); - }; - - /** - * start the show - */ - init(); + // Start the Eve time clock in the top menu bar + setInterval(function () { + renderClock($('.eve-time-wrapper .eve-time-clock')); + }, 500); }); diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 495dc0bc..63a4044f 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -52,7 +52,7 @@ services: - auth_mysql grafana: - image: grafana/grafana-oss:8.2 + image: grafana/grafana-oss:8.3.2 restart: always depends_on: - auth_mysql diff --git a/docs/conf.py b/docs/conf.py index 682a0653..de813c7e 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -42,6 +42,7 @@ from recommonmark.transform import AutoStructify extensions = [ 'sphinx_rtd_theme', 'sphinx.ext.autodoc', + 'sphinx.ext.napoleon', 'recommonmark', ] diff --git a/docs/features/core/analytics.md b/docs/features/core/analytics.md index 7b750ada..0ee03f8a 100644 --- a/docs/features/core/analytics.md +++ b/docs/features/core/analytics.md @@ -10,6 +10,12 @@ To Opt-Out, modify our pre-loaded token using the Admin dashboard */admin/analyt Each of the three features Daily Stats, Celery Events and Page Views can be enabled/Disabled independently. +Alternatively, you can fully opt out of analytics with the following optional setting: + +```python +ANALYTICS_DISABLED = True +``` + ![Analytics Tokens](/_static/images/features/core/analytics/tokens.png) ## What @@ -58,6 +64,8 @@ This data is stored in a Team Google Analytics Dashboard. The Maintainers all ha ### Analytics Event +```eval_rst .. automodule:: allianceauth.analytics.tasks :members: analytics_event :undoc-members: +``` diff --git a/docs/features/core/groups.md b/docs/features/core/groups.md index 0d75d327..2c15cb91 100644 --- a/docs/features/core/groups.md +++ b/docs/features/core/groups.md @@ -48,7 +48,7 @@ When using Alliance Auth to manage external services like Discord, Auth will aut ```eval_rst .. note:: - While this feature can help to avoid naming conflicts with groups on external services, the respective service component in Alliance Auth also needs to be build in such a way that it knows how to prevent these conflicts. Currently only the Discord service has this ability. + While this feature can help to avoid naming conflicts with groups on external services, the respective service component in Alliance Auth also needs to be build in such a way that it knows how to prevent these conflicts. Currently only the Discord and Teamspeak3 services have this ability. ``` ## Managing groups diff --git a/docs/features/services/teamspeak3.md b/docs/features/services/teamspeak3.md index 80c7dad3..507ab1bc 100644 --- a/docs/features/services/teamspeak3.md +++ b/docs/features/services/teamspeak3.md @@ -160,7 +160,7 @@ This error generally means teamspeak returned an error message that went unhandl This most commonly happens when your teamspeak server is externally hosted. You need to add the auth server IP to the teamspeak serverquery whitelist. This varies by provider. -If you have SSH access to the server hosting it, you need to locate the teamspeak server folder and add the auth server IP on a new line in `server_query_whitelist.txt` +If you have SSH access to the server hosting it, you need to locate the teamspeak server folder and add the auth server IP on a new line in `query_ip_allowlist.txt` (named `query_ip_whitelist.txt` on older teamspeak versions). ### `520 invalid loginname or password`