From dfe62db8ee765b3dc964704a6e90747a73533a80 Mon Sep 17 00:00:00 2001 From: Ariel Rin Date: Sat, 27 Nov 2021 23:02:33 +1000 Subject: [PATCH 01/11] add datatables savestate feature --- allianceauth/corputils/templates/corputils/corpstats.html | 6 ++++++ allianceauth/corputils/templates/corputils/search.html | 5 ++++- .../groupmanagement/templates/groupmanagement/audit.html | 2 ++ .../templates/groupmanagement/groupmembers.html | 4 +++- .../permissions_tool/templates/permissions_tool/audit.html | 2 ++ .../templates/permissions_tool/overview.html | 4 +++- allianceauth/srp/templates/srp/data.html | 4 +++- 7 files changed, 23 insertions(+), 4 deletions(-) diff --git a/allianceauth/corputils/templates/corputils/corpstats.html b/allianceauth/corputils/templates/corputils/corpstats.html index df5a8527..5ab9de1a 100644 --- a/allianceauth/corputils/templates/corputils/corpstats.html +++ b/allianceauth/corputils/templates/corputils/corpstats.html @@ -193,6 +193,8 @@ "columnDefs": [ { "sortable": false, "targets": [1] }, ], + "stateSave": true, + "stateDuration": 0 }); $('#table-members').DataTable({ "columnDefs": [ @@ -200,6 +202,8 @@ { "sortable": false, "targets": [0, 2] }, ], "order": [[ 1, "asc" ]], + "stateSave": true, + "stateDuration": 0 }); $('#table-unregistered').DataTable({ "columnDefs": [ @@ -207,6 +211,8 @@ { "sortable": false, "targets": [0, 2] }, ], "order": [[ 1, "asc" ]], + "stateSave": true, + "stateDuration": 0 }); }); diff --git a/allianceauth/corputils/templates/corputils/search.html b/allianceauth/corputils/templates/corputils/search.html index 502f748a..e8e80c1d 100644 --- a/allianceauth/corputils/templates/corputils/search.html +++ b/allianceauth/corputils/templates/corputils/search.html @@ -43,6 +43,9 @@ {% endblock %} {% block extra_script %} $(document).ready(function(){ - $('#table-search').DataTable(); + $('#table-search').DataTable({ + "stateSave": true, + "stateDuration": 0 + }); }); {% endblock %} diff --git a/allianceauth/groupmanagement/templates/groupmanagement/audit.html b/allianceauth/groupmanagement/templates/groupmanagement/audit.html index 7f1dc73b..4c1759de 100644 --- a/allianceauth/groupmanagement/templates/groupmanagement/audit.html +++ b/allianceauth/groupmanagement/templates/groupmanagement/audit.html @@ -127,6 +127,8 @@ ], bootstrap: true }, + "stateSave": true, + "stateDuration": 0 }); }); {% endblock %} diff --git a/allianceauth/groupmanagement/templates/groupmanagement/groupmembers.html b/allianceauth/groupmanagement/templates/groupmanagement/groupmembers.html index 1ce7e3fd..ab6be2b2 100644 --- a/allianceauth/groupmanagement/templates/groupmanagement/groupmembers.html +++ b/allianceauth/groupmanagement/templates/groupmanagement/groupmembers.html @@ -104,7 +104,9 @@ "sortable": false, "targets": [2] }, - ] + ], + "stateSave": true, + "stateDuration": 0 }); }); {% endblock %} diff --git a/allianceauth/permissions_tool/templates/permissions_tool/audit.html b/allianceauth/permissions_tool/templates/permissions_tool/audit.html index f168e74d..d04a5ed0 100644 --- a/allianceauth/permissions_tool/templates/permissions_tool/audit.html +++ b/allianceauth/permissions_tool/templates/permissions_tool/audit.html @@ -73,6 +73,8 @@ ], bootstrap: true }, + "stateSave": true, + "stateDuration": 0, drawCallback: function ( settings ) { let api = this.api(); let rows = api.rows( {page:'current'} ).nodes(); diff --git a/allianceauth/permissions_tool/templates/permissions_tool/overview.html b/allianceauth/permissions_tool/templates/permissions_tool/overview.html index 637e53a4..05bcfb80 100644 --- a/allianceauth/permissions_tool/templates/permissions_tool/overview.html +++ b/allianceauth/permissions_tool/templates/permissions_tool/overview.html @@ -106,8 +106,10 @@ idx: 1 } ], - bootstrap: true + bootstrap: true, }, + "stateSave": true, + "stateDuration": 0, drawCallback: function ( settings ) { let api = this.api(); let rows = api.rows( {page:'current'} ).nodes(); diff --git a/allianceauth/srp/templates/srp/data.html b/allianceauth/srp/templates/srp/data.html index 3710c33f..d4f1e278 100644 --- a/allianceauth/srp/templates/srp/data.html +++ b/allianceauth/srp/templates/srp/data.html @@ -267,7 +267,9 @@ ESC to cancel{% endblocktrans %}"id="blah"> "targets": [4, 5], "type": "num" } - ] + ], + "stateSave": true, + "stateDuration": 0 }); // tooltip From d11832913d1563c62cc285e4ac1ad4d105265963 Mon Sep 17 00:00:00 2001 From: Adarnof Date: Tue, 30 Nov 2021 23:41:26 -0500 Subject: [PATCH 02/11] Implement reserved group names in Teamspeak3 service module. Closes #1302 --- .../services/modules/teamspeak3/manager.py | 4 +- .../services/modules/teamspeak3/tests.py | 49 +++++++++++++++++++ docs/features/core/groups.md | 2 +- 3 files changed, 53 insertions(+), 2 deletions(-) diff --git a/allianceauth/services/modules/teamspeak3/manager.py b/allianceauth/services/modules/teamspeak3/manager.py index d05f910d..9f7eea6c 100755 --- a/allianceauth/services/modules/teamspeak3/manager.py +++ b/allianceauth/services/modules/teamspeak3/manager.py @@ -4,6 +4,7 @@ from django.conf import settings from .util.ts3 import TS3Server, TeamspeakError from .models import TSgroup +from allianceauth.groupmanagement.models import ReservedGroupName logger = logging.getLogger(__name__) @@ -270,7 +271,8 @@ class Teamspeak3Manager: addgroups.append(ts_groups[ts_group_key]) for user_ts_group_key in user_ts_groups: if user_ts_groups[user_ts_group_key] not in ts_groups.values(): - remgroups.append(user_ts_groups[user_ts_group_key]) + if not ReservedGroupName.objects.filter(name=user_ts_group_key).exists(): + remgroups.append(user_ts_groups[user_ts_group_key]) for g in addgroups: logger.info(f"Adding Teamspeak user {userid} into group {g}") diff --git a/allianceauth/services/modules/teamspeak3/tests.py b/allianceauth/services/modules/teamspeak3/tests.py index 1cb923ed..4abc68e4 100644 --- a/allianceauth/services/modules/teamspeak3/tests.py +++ b/allianceauth/services/modules/teamspeak3/tests.py @@ -15,6 +15,7 @@ from .signals import m2m_changed_authts_group, post_save_authts, post_delete_aut from .manager import Teamspeak3Manager from .util.ts3 import TeamspeakError from allianceauth.authentication.models import State +from allianceauth.groupmanagement.models import ReservedGroupName MODULE_PATH = 'allianceauth.services.modules.teamspeak3' DEFAULT_AUTH_GROUP = 'Member' @@ -316,6 +317,9 @@ class Teamspeak3SignalsTestCase(TestCase): class Teamspeak3ManagerTestCase(TestCase): + def setUp(self): + self.reserved = ReservedGroupName.objects.create(name='reserved', reason='tests', created_by='Bob, praise be!') + @staticmethod def my_side_effect(*args, **kwargs): raise TeamspeakError(1) @@ -339,3 +343,48 @@ class Teamspeak3ManagerTestCase(TestCase): # perform test manager.add_user(user, "Dummy User") + + @mock.patch.object(Teamspeak3Manager, '_get_userid') + @mock.patch.object(Teamspeak3Manager, '_user_group_list') + @mock.patch.object(Teamspeak3Manager, '_add_user_to_group') + @mock.patch.object(Teamspeak3Manager, '_remove_user_from_group') + @mock.patch.object(Teamspeak3Manager, 'server') + def test_update_groups_add(self, server, remove, add, groups, userid): + """Add to one group""" + userid.return_value = 1 + groups.return_value = {'test': 1} + + Teamspeak3Manager().update_groups(1, {'test': 1, 'dummy': 2}) + self.assertEqual(add.call_count, 1) + self.assertEqual(remove.call_count, 0) + self.assertEqual(add.call_args[0][1], 2) + + @mock.patch.object(Teamspeak3Manager, '_get_userid') + @mock.patch.object(Teamspeak3Manager, '_user_group_list') + @mock.patch.object(Teamspeak3Manager, '_add_user_to_group') + @mock.patch.object(Teamspeak3Manager, '_remove_user_from_group') + @mock.patch.object(Teamspeak3Manager, 'server') + def test_update_groups_remove(self, server, remove, add, groups, userid): + """Remove from one group""" + userid.return_value = 1 + groups.return_value = {'test': 1, 'dummy': 2} + + Teamspeak3Manager().update_groups(1, {'test': 1}) + self.assertEqual(add.call_count, 0) + self.assertEqual(remove.call_count, 1) + self.assertEqual(remove.call_args[0][1], 2) + + @mock.patch.object(Teamspeak3Manager, '_get_userid') + @mock.patch.object(Teamspeak3Manager, '_user_group_list') + @mock.patch.object(Teamspeak3Manager, '_add_user_to_group') + @mock.patch.object(Teamspeak3Manager, '_remove_user_from_group') + @mock.patch.object(Teamspeak3Manager, 'server') + def test_update_groups_remove_reserved(self, server, remove, add, groups, userid): + """Remove from one group, but do not touch reserved group""" + userid.return_value = 1 + groups.return_value = {'test': 1, 'dummy': 2, self.reserved.name: 3} + + Teamspeak3Manager().update_groups(1, {'test': 1}) + self.assertEqual(add.call_count, 0) + self.assertEqual(remove.call_count, 1) + self.assertEqual(remove.call_args[0][1], 2) diff --git a/docs/features/core/groups.md b/docs/features/core/groups.md index 25fd16c2..8530a62a 100644 --- a/docs/features/core/groups.md +++ b/docs/features/core/groups.md @@ -48,7 +48,7 @@ When using Alliance Auth to manage external services like Discord, Auth will aut ```eval_rst .. note:: - While this feature can help to avoid naming conflicts with groups on external services, the respective service component in Alliance Auth also needs to be build in such a way that it knows how to prevent these conflicts. Currently only the Discord service has this ability. + While this feature can help to avoid naming conflicts with groups on external services, the respective service component in Alliance Auth also needs to be build in such a way that it knows how to prevent these conflicts. Currently only the Discord and Teamspeak3 services have this ability. ``` ## Managing groups From 72740b9e4da7b736a2bd64e33864c90f3627110a Mon Sep 17 00:00:00 2001 From: Adarnof Date: Wed, 8 Dec 2021 23:41:10 -0500 Subject: [PATCH 03/11] Prevent assignment of reserved groups to AuthTSgroup mappings. Implemented in TS group updates to prevent their creation / delete once reserved, and the admin site for when a reserved group name is created but before the TS group sync occurs. --- .../services/modules/teamspeak3/admin.py | 15 +++- .../services/modules/teamspeak3/manager.py | 35 +++----- .../services/modules/teamspeak3/tests.py | 90 ++++++++++++++++--- 3 files changed, 105 insertions(+), 35 deletions(-) diff --git a/allianceauth/services/modules/teamspeak3/admin.py b/allianceauth/services/modules/teamspeak3/admin.py index a8b614d8..dbba1cb1 100644 --- a/allianceauth/services/modules/teamspeak3/admin.py +++ b/allianceauth/services/modules/teamspeak3/admin.py @@ -1,7 +1,8 @@ from django.contrib import admin - -from .models import AuthTS, Teamspeak3User, StateGroup +from django.contrib.auth.models import Group +from .models import AuthTS, Teamspeak3User, StateGroup, TSgroup from ...admin import ServicesUserAdmin +from allianceauth.groupmanagement.models import ReservedGroupName @admin.register(Teamspeak3User) @@ -25,6 +26,16 @@ class AuthTSgroupAdmin(admin.ModelAdmin): fields = ('auth_group', 'ts_group') filter_horizontal = ('ts_group',) + def formfield_for_foreignkey(self, db_field, request, **kwargs): + if db_field.name == 'auth_group': + kwargs['queryset'] = Group.objects.exclude(name__in=ReservedGroupName.objects.values_list('name', flat=True)) + return super().formfield_for_foreignkey(db_field, request, **kwargs) + + def formfield_for_manytomany(self, db_field, request, **kwargs): + if db_field.name == 'ts_group': + kwargs['queryset'] = TSgroup.objects.exclude(ts_group_name__in=ReservedGroupName.objects.values_list('name', flat=True)) + return super().formfield_for_manytomany(db_field, request, **kwargs) + def _ts_group(self, obj): return [x for x in obj.ts_group.all().order_by('ts_group_id')] diff --git a/allianceauth/services/modules/teamspeak3/manager.py b/allianceauth/services/modules/teamspeak3/manager.py index 9f7eea6c..9cad7cae 100755 --- a/allianceauth/services/modules/teamspeak3/manager.py +++ b/allianceauth/services/modules/teamspeak3/manager.py @@ -157,32 +157,25 @@ class Teamspeak3Manager: logger.info(f"Removed user id {uid} from group id {groupid} on TS3 server.") def _sync_ts_group_db(self): - logger.debug("_sync_ts_group_db function called.") try: remote_groups = self._group_list() - local_groups = TSgroup.objects.all() - logger.debug("Comparing remote groups to TSgroup objects: %s" % local_groups) - for key in remote_groups: - logger.debug(f"Typecasting remote_group value at position {key} to int: {remote_groups[key]}") - remote_groups[key] = int(remote_groups[key]) + managed_groups = {g:remote_groups[g] for g in remote_groups if g in set(remote_groups.keys()) - set(ReservedGroupName.objects.values_list('name', flat=True))} + remove = TSgroup.objects.exclude(ts_group_id__in=managed_groups.values()) + + if remove: + logger.debug(f"Deleting {remove.count()} TSgroup models: not found on server, or reserved name.") + remove.delete() + + add = {g:managed_groups[g] for g in managed_groups if managed_groups[g] in set(managed_groups.values()) - set(TSgroup.objects.values_list("ts_group_id", flat=True))} + if add: + logger.debug(f"Adding {len(add)} new TSgroup models.") + models = [TSgroup(ts_group_name=name, ts_group_id=add[name]) for name in add] + TSgroup.objects.bulk_create(models) - for group in local_groups: - logger.debug("Checking local group %s" % group) - if group.ts_group_id not in remote_groups.values(): - logger.debug( - f"Local group id {group.ts_group_id} not found on server. Deleting model {group}") - TSgroup.objects.filter(ts_group_id=group.ts_group_id).delete() - for key in remote_groups: - g = TSgroup(ts_group_id=remote_groups[key], ts_group_name=key) - q = TSgroup.objects.filter(ts_group_id=g.ts_group_id) - if not q: - logger.debug("Local group does not exist for TS group {}. Creating TSgroup model {}".format( - remote_groups[key], g)) - g.save() except TeamspeakError as e: - logger.error("Error occured while syncing TS group db: %s" % str(e)) + logger.error(f"Error occurred while syncing TS group db: {str(e)}") except: - logger.exception("An unhandled exception has occured while syncing TS groups.") + logger.exception("An unhandled exception has occurred while syncing TS groups.") def add_user(self, user, fmt_name): username_clean = self.__santatize_username(fmt_name[:30]) diff --git a/allianceauth/services/modules/teamspeak3/tests.py b/allianceauth/services/modules/teamspeak3/tests.py index 4abc68e4..bfed1906 100644 --- a/allianceauth/services/modules/teamspeak3/tests.py +++ b/allianceauth/services/modules/teamspeak3/tests.py @@ -5,16 +5,17 @@ from django import urls from django.contrib.auth.models import User, Group, Permission from django.core.exceptions import ObjectDoesNotExist from django.db.models import signals +from django.contrib.admin import AdminSite from allianceauth.tests.auth_utils import AuthUtils from .auth_hooks import Teamspeak3Service from .models import Teamspeak3User, AuthTS, TSgroup, StateGroup from .tasks import Teamspeak3Tasks from .signals import m2m_changed_authts_group, post_save_authts, post_delete_authts +from .admin import AuthTSgroupAdmin from .manager import Teamspeak3Manager from .util.ts3 import TeamspeakError -from allianceauth.authentication.models import State from allianceauth.groupmanagement.models import ReservedGroupName MODULE_PATH = 'allianceauth.services.modules.teamspeak3' @@ -316,9 +317,9 @@ class Teamspeak3SignalsTestCase(TestCase): class Teamspeak3ManagerTestCase(TestCase): - - def setUp(self): - self.reserved = ReservedGroupName.objects.create(name='reserved', reason='tests', created_by='Bob, praise be!') + @classmethod + def setUpTestData(cls): + cls.reserved = ReservedGroupName.objects.create(name='reserved', reason='tests', created_by='Bob, praise be!') @staticmethod def my_side_effect(*args, **kwargs): @@ -338,8 +339,8 @@ class Teamspeak3ManagerTestCase(TestCase): manager._server = server # create test data - user = User.objects.create_user("dummy") - user.profile.state = State.objects.filter(name="Member").first() + user = AuthUtils.create_user("dummy") + AuthUtils.assign_state(user, AuthUtils.get_member_state()) # perform test manager.add_user(user, "Dummy User") @@ -348,8 +349,7 @@ class Teamspeak3ManagerTestCase(TestCase): @mock.patch.object(Teamspeak3Manager, '_user_group_list') @mock.patch.object(Teamspeak3Manager, '_add_user_to_group') @mock.patch.object(Teamspeak3Manager, '_remove_user_from_group') - @mock.patch.object(Teamspeak3Manager, 'server') - def test_update_groups_add(self, server, remove, add, groups, userid): + def test_update_groups_add(self, remove, add, groups, userid): """Add to one group""" userid.return_value = 1 groups.return_value = {'test': 1} @@ -363,8 +363,7 @@ class Teamspeak3ManagerTestCase(TestCase): @mock.patch.object(Teamspeak3Manager, '_user_group_list') @mock.patch.object(Teamspeak3Manager, '_add_user_to_group') @mock.patch.object(Teamspeak3Manager, '_remove_user_from_group') - @mock.patch.object(Teamspeak3Manager, 'server') - def test_update_groups_remove(self, server, remove, add, groups, userid): + def test_update_groups_remove(self, remove, add, groups, userid): """Remove from one group""" userid.return_value = 1 groups.return_value = {'test': 1, 'dummy': 2} @@ -378,8 +377,7 @@ class Teamspeak3ManagerTestCase(TestCase): @mock.patch.object(Teamspeak3Manager, '_user_group_list') @mock.patch.object(Teamspeak3Manager, '_add_user_to_group') @mock.patch.object(Teamspeak3Manager, '_remove_user_from_group') - @mock.patch.object(Teamspeak3Manager, 'server') - def test_update_groups_remove_reserved(self, server, remove, add, groups, userid): + def test_update_groups_remove_reserved(self, remove, add, groups, userid): """Remove from one group, but do not touch reserved group""" userid.return_value = 1 groups.return_value = {'test': 1, 'dummy': 2, self.reserved.name: 3} @@ -388,3 +386,71 @@ class Teamspeak3ManagerTestCase(TestCase): self.assertEqual(add.call_count, 0) self.assertEqual(remove.call_count, 1) self.assertEqual(remove.call_args[0][1], 2) + + @mock.patch.object(Teamspeak3Manager, '_group_list') + def test_sync_group_db_create(self, group_list): + """Populate the list of all TSgroups""" + group_list.return_value = {'allowed':1, 'also allowed': 2} + Teamspeak3Manager()._sync_ts_group_db() + self.assertEqual(TSgroup.objects.all().count(), 2) + + @mock.patch.object(Teamspeak3Manager, '_group_list') + def test_sync_group_db_delete(self, group_list): + """Populate the list of all TSgroups, and delete one which no longer exists""" + TSgroup.objects.create(ts_group_name='deleted', ts_group_id=3) + group_list.return_value = {'allowed': 1, 'also allowed': 2} + Teamspeak3Manager()._sync_ts_group_db() + self.assertEqual(TSgroup.objects.all().count(), 2) + self.assertFalse(TSgroup.objects.filter(ts_group_name='deleted').exists()) + + @mock.patch.object(Teamspeak3Manager, '_group_list') + def test_sync_group_db_dont_create_reserved(self, group_list): + """Populate the list of all TSgroups, ignoring a reserved group name""" + group_list.return_value = {'allowed': 1, 'reserved': 4} + Teamspeak3Manager()._sync_ts_group_db() + self.assertEqual(TSgroup.objects.all().count(), 1) + self.assertFalse(TSgroup.objects.filter(ts_group_name='reserved').exists()) + + @mock.patch.object(Teamspeak3Manager, '_group_list') + def test_sync_group_db_delete_reserved(self, group_list): + """Populate the list of all TSgroups, deleting the TSgroup model for one which has become reserved""" + TSgroup.objects.create(ts_group_name='reserved', ts_group_id=4) + group_list.return_value = {'allowed': 1, 'reserved': 4} + Teamspeak3Manager()._sync_ts_group_db() + self.assertEqual(TSgroup.objects.all().count(), 1) + self.assertFalse(TSgroup.objects.filter(ts_group_name='reserved').exists()) + + +class MockRequest: + pass + + +class MockSuperUser: + def has_perm(self, perm, obj=None): + return True + + +request = MockRequest() +request.user = MockSuperUser() + + +class Teamspeak3AdminTestCase(TestCase): + @classmethod + def setUpTestData(cls): + cls.site = AdminSite() + cls.admin = AuthTSgroupAdmin(AuthTS, cls.site) + cls.group = Group.objects.create(name='test') + cls.ts_group = TSgroup.objects.create(ts_group_name='test') + + def test_field_queryset_no_reserved_names(self): + """Ensure all groups are listed when no reserved names""" + form = self.admin.get_form(request) + self.assertEqual(form.base_fields['auth_group']._get_queryset().count(), 1) + self.assertEqual(form.base_fields['ts_group']._get_queryset().count(), 1) + + def test_field_queryset_reserved_names(self): + """Ensure reserved group names are filtered out""" + ReservedGroupName.objects.bulk_create([ReservedGroupName(name='test', reason='tests', created_by='Bob')]) + form = self.admin.get_form(request) + self.assertEqual(form.base_fields['auth_group']._get_queryset().count(), 0) + self.assertEqual(form.base_fields['ts_group']._get_queryset().count(), 0) From 6688f735653763bd0bfa4fd539502091c76c6a4b Mon Sep 17 00:00:00 2001 From: Adarnof Date: Wed, 15 Dec 2021 23:54:53 -0500 Subject: [PATCH 04/11] Use integer teamspeak group IDs when filtering. --- .../services/modules/teamspeak3/manager.py | 8 ++-- .../services/modules/teamspeak3/tests.py | 37 ++++++++++++++----- 2 files changed, 31 insertions(+), 14 deletions(-) diff --git a/allianceauth/services/modules/teamspeak3/manager.py b/allianceauth/services/modules/teamspeak3/manager.py index 9cad7cae..519a8a47 100755 --- a/allianceauth/services/modules/teamspeak3/manager.py +++ b/allianceauth/services/modules/teamspeak3/manager.py @@ -159,7 +159,7 @@ class Teamspeak3Manager: def _sync_ts_group_db(self): try: remote_groups = self._group_list() - managed_groups = {g:remote_groups[g] for g in remote_groups if g in set(remote_groups.keys()) - set(ReservedGroupName.objects.values_list('name', flat=True))} + managed_groups = {g:int(remote_groups[g]) for g in remote_groups if g in set(remote_groups.keys()) - set(ReservedGroupName.objects.values_list('name', flat=True))} remove = TSgroup.objects.exclude(ts_group_id__in=managed_groups.values()) if remove: @@ -174,8 +174,8 @@ class Teamspeak3Manager: except TeamspeakError as e: logger.error(f"Error occurred while syncing TS group db: {str(e)}") - except: - logger.exception("An unhandled exception has occurred while syncing TS groups.") + except Exception: + logger.exception(f"An unhandled exception has occurred while syncing TS groups.") def add_user(self, user, fmt_name): username_clean = self.__santatize_username(fmt_name[:30]) @@ -234,7 +234,7 @@ class Teamspeak3Manager: logger.exception(f"Failed to delete user id {uid} from TS3 - received response {ret}") return False else: - logger.warn("User with id %s not found on TS3 server. Assuming succesful deletion." % uid) + logger.warning("User with id %s not found on TS3 server. Assuming succesful deletion." % uid) return True def check_user_exists(self, uid): diff --git a/allianceauth/services/modules/teamspeak3/tests.py b/allianceauth/services/modules/teamspeak3/tests.py index bfed1906..2354bf98 100644 --- a/allianceauth/services/modules/teamspeak3/tests.py +++ b/allianceauth/services/modules/teamspeak3/tests.py @@ -366,7 +366,7 @@ class Teamspeak3ManagerTestCase(TestCase): def test_update_groups_remove(self, remove, add, groups, userid): """Remove from one group""" userid.return_value = 1 - groups.return_value = {'test': 1, 'dummy': 2} + groups.return_value = {'test': '1', 'dummy': '2'} Teamspeak3Manager().update_groups(1, {'test': 1}) self.assertEqual(add.call_count, 0) @@ -390,7 +390,7 @@ class Teamspeak3ManagerTestCase(TestCase): @mock.patch.object(Teamspeak3Manager, '_group_list') def test_sync_group_db_create(self, group_list): """Populate the list of all TSgroups""" - group_list.return_value = {'allowed':1, 'also allowed': 2} + group_list.return_value = {'allowed':'1', 'also allowed':'2'} Teamspeak3Manager()._sync_ts_group_db() self.assertEqual(TSgroup.objects.all().count(), 2) @@ -398,15 +398,15 @@ class Teamspeak3ManagerTestCase(TestCase): def test_sync_group_db_delete(self, group_list): """Populate the list of all TSgroups, and delete one which no longer exists""" TSgroup.objects.create(ts_group_name='deleted', ts_group_id=3) - group_list.return_value = {'allowed': 1, 'also allowed': 2} + group_list.return_value = {'allowed': '1'} Teamspeak3Manager()._sync_ts_group_db() - self.assertEqual(TSgroup.objects.all().count(), 2) + self.assertEqual(TSgroup.objects.all().count(), 1) self.assertFalse(TSgroup.objects.filter(ts_group_name='deleted').exists()) @mock.patch.object(Teamspeak3Manager, '_group_list') def test_sync_group_db_dont_create_reserved(self, group_list): """Populate the list of all TSgroups, ignoring a reserved group name""" - group_list.return_value = {'allowed': 1, 'reserved': 4} + group_list.return_value = {'allowed': '1', 'reserved': '4'} Teamspeak3Manager()._sync_ts_group_db() self.assertEqual(TSgroup.objects.all().count(), 1) self.assertFalse(TSgroup.objects.filter(ts_group_name='reserved').exists()) @@ -415,11 +415,28 @@ class Teamspeak3ManagerTestCase(TestCase): def test_sync_group_db_delete_reserved(self, group_list): """Populate the list of all TSgroups, deleting the TSgroup model for one which has become reserved""" TSgroup.objects.create(ts_group_name='reserved', ts_group_id=4) - group_list.return_value = {'allowed': 1, 'reserved': 4} + group_list.return_value = {'allowed': '1', 'reserved': '4'} Teamspeak3Manager()._sync_ts_group_db() self.assertEqual(TSgroup.objects.all().count(), 1) self.assertFalse(TSgroup.objects.filter(ts_group_name='reserved').exists()) + @mock.patch.object(Teamspeak3Manager, '_group_list') + def test_sync_group_db_partial_addition(self, group_list): + """Some TSgroups already exist in database, add new ones""" + TSgroup.objects.create(ts_group_name='allowed', ts_group_id=1) + group_list.return_value = {'allowed': '1', 'also allowed': '2'} + Teamspeak3Manager()._sync_ts_group_db() + self.assertEqual(TSgroup.objects.all().count(), 2) + + @mock.patch.object(Teamspeak3Manager, '_group_list') + def test_sync_group_db_partial_removal(self, group_list): + """One TSgroup has been deleted on server, so remove its model""" + TSgroup.objects.create(ts_group_name='allowed', ts_group_id=1) + TSgroup.objects.create(ts_group_name='also allowed', ts_group_id=2) + group_list.return_value = {'allowed': '1'} + Teamspeak3Manager()._sync_ts_group_db() + self.assertEqual(TSgroup.objects.all().count(), 1) + class MockRequest: pass @@ -445,12 +462,12 @@ class Teamspeak3AdminTestCase(TestCase): def test_field_queryset_no_reserved_names(self): """Ensure all groups are listed when no reserved names""" form = self.admin.get_form(request) - self.assertEqual(form.base_fields['auth_group']._get_queryset().count(), 1) - self.assertEqual(form.base_fields['ts_group']._get_queryset().count(), 1) + self.assertQuerysetEqual(form.base_fields['auth_group']._get_queryset(), Group.objects.all()) + self.assertQuerysetEqual(form.base_fields['ts_group']._get_queryset(), TSgroup.objects.all()) def test_field_queryset_reserved_names(self): """Ensure reserved group names are filtered out""" ReservedGroupName.objects.bulk_create([ReservedGroupName(name='test', reason='tests', created_by='Bob')]) form = self.admin.get_form(request) - self.assertEqual(form.base_fields['auth_group']._get_queryset().count(), 0) - self.assertEqual(form.base_fields['ts_group']._get_queryset().count(), 0) + self.assertQuerysetEqual(form.base_fields['auth_group']._get_queryset(), Group.objects.none()) + self.assertQuerysetEqual(form.base_fields['ts_group']._get_queryset(), TSgroup.objects.none()) From 8de2c3bfcb0c6240f3c19f6b43f0b86c27b6e70e Mon Sep 17 00:00:00 2001 From: Adarnof Date: Thu, 16 Dec 2021 22:23:15 -0500 Subject: [PATCH 05/11] Update name of serverquery IP file changed in TS3 v3.13.0 Changelog indicates old filenames are still accepted, but newly installed servers come with the new file names. Closes #1298 --- docs/features/services/teamspeak3.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/features/services/teamspeak3.md b/docs/features/services/teamspeak3.md index 80c7dad3..507ab1bc 100644 --- a/docs/features/services/teamspeak3.md +++ b/docs/features/services/teamspeak3.md @@ -160,7 +160,7 @@ This error generally means teamspeak returned an error message that went unhandl This most commonly happens when your teamspeak server is externally hosted. You need to add the auth server IP to the teamspeak serverquery whitelist. This varies by provider. -If you have SSH access to the server hosting it, you need to locate the teamspeak server folder and add the auth server IP on a new line in `server_query_whitelist.txt` +If you have SSH access to the server hosting it, you need to locate the teamspeak server folder and add the auth server IP on a new line in `query_ip_allowlist.txt` (named `query_ip_whitelist.txt` on older teamspeak versions). ### `520 invalid loginname or password` From ea8958ccc3664d0a498d89d3670991be11fd5b89 Mon Sep 17 00:00:00 2001 From: Ariel Rin Date: Tue, 28 Dec 2021 21:56:46 +1000 Subject: [PATCH 06/11] Version Bump v2.9.4 --- allianceauth/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/allianceauth/__init__.py b/allianceauth/__init__.py index 8d04fd9d..7146f256 100644 --- a/allianceauth/__init__.py +++ b/allianceauth/__init__.py @@ -1,7 +1,7 @@ # This will make sure the app is always imported when # Django starts so that shared_task will use this app. -__version__ = '2.9.3' +__version__ = '2.9.4' __title__ = 'Alliance Auth' __url__ = 'https://gitlab.com/allianceauth/allianceauth' NAME = f'{__title__} v{__version__}' From 827291dda42374ad72634d8ddb0794355d1fce86 Mon Sep 17 00:00:00 2001 From: Kevin McKernan Date: Fri, 7 Jan 2022 10:48:50 -0700 Subject: [PATCH 07/11] fix grafana image again, thanks grafana for not tagging your new images properly --- docker/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 495dc0bc..63a4044f 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -52,7 +52,7 @@ services: - auth_mysql grafana: - image: grafana/grafana-oss:8.2 + image: grafana/grafana-oss:8.3.2 restart: always depends_on: - auth_mysql From e39a3c072b9794e6ede398bf3350bd7c5eaf182b Mon Sep 17 00:00:00 2001 From: Peter Pfeufer Date: Thu, 27 Jan 2022 04:35:15 +0000 Subject: [PATCH 08/11] Evetime js update --- allianceauth/static/js/eve-time.js | 54 +++++------------------------- 1 file changed, 8 insertions(+), 46 deletions(-) diff --git a/allianceauth/static/js/eve-time.js b/allianceauth/static/js/eve-time.js index ba5656fd..1cbe36d2 100644 --- a/allianceauth/static/js/eve-time.js +++ b/allianceauth/static/js/eve-time.js @@ -1,58 +1,20 @@ $(document).ready(function () { 'use strict'; - /** - * check time - * @param i - * @returns {string} - */ - let checkTime = function (i) { - if (i < 10) { - i = '0' + i; - } - - return i; - }; - /** * render a JS clock for Eve Time * @param element - * @param utcOffset */ - let renderClock = function (element, utcOffset) { - let today = new Date(); - let h = today.getUTCHours(); - let m = today.getUTCMinutes(); - - h = h + utcOffset; - - if (h > 24) { - h = h - 24; - } - - if (h < 0) { - h = h + 24; - } - - h = checkTime(h); - m = checkTime(m); + const renderClock = function (element) { + const datetimeNow = new Date(); + const h = String(datetimeNow.getUTCHours()).padStart(2, '0'); + const m = String(datetimeNow.getUTCMinutes()).padStart(2, '0'); element.html(h + ':' + m); - - setTimeout(function () { - renderClock(element, 0); - }, 500); }; - /** - * functions that need to be executed on load - */ - let init = function () { - renderClock($('.eve-time-wrapper .eve-time-clock'), 0); - }; - - /** - * start the show - */ - init(); + // Start the Eve time clock in the top menu bar + setInterval(function () { + renderClock($('.eve-time-wrapper .eve-time-clock')); + }, 500); }); From f348b1a34c7cfa138302a47b0d6d80e39f99444c Mon Sep 17 00:00:00 2001 From: Erik Kalkoken Date: Thu, 27 Jan 2022 05:02:57 +0000 Subject: [PATCH 09/11] Fix: Can not update biomassed characters --- allianceauth/eveonline/models.py | 85 +++- allianceauth/eveonline/providers.py | 39 +- allianceauth/eveonline/tasks.py | 23 +- .../eveonline/tests/esi_client_stub.py | 168 +++++++ allianceauth/eveonline/tests/test_models.py | 110 ++++- .../eveonline/tests/test_providers.py | 13 +- allianceauth/eveonline/tests/test_tasks.py | 440 ++++++++++-------- 7 files changed, 614 insertions(+), 264 deletions(-) create mode 100644 allianceauth/eveonline/tests/esi_client_stub.py diff --git a/allianceauth/eveonline/models.py b/allianceauth/eveonline/models.py index 390c00b4..1a7f5c39 100644 --- a/allianceauth/eveonline/models.py +++ b/allianceauth/eveonline/models.py @@ -1,13 +1,27 @@ -from django.db import models +import logging from typing import Union -from .managers import EveCharacterManager, EveCharacterProviderManager -from .managers import EveCorporationManager, EveCorporationProviderManager -from .managers import EveAllianceManager, EveAllianceProviderManager +from django.core.exceptions import ObjectDoesNotExist +from django.db import models +from esi.models import Token + +from allianceauth.notifications import notify + from . import providers from .evelinks import eveimageserver +from .managers import ( + EveAllianceManager, + EveAllianceProviderManager, + EveCharacterManager, + EveCharacterProviderManager, + EveCorporationManager, + EveCorporationProviderManager, +) + +logger = logging.getLogger(__name__) _DEFAULT_IMAGE_SIZE = 32 +DOOMHEIM_CORPORATION_ID = 1000001 class EveFactionInfo(models.Model): @@ -68,13 +82,12 @@ class EveAllianceInfo(models.Model): for corp_id in alliance.corp_ids: if not EveCorporationInfo.objects.filter(corporation_id=corp_id).exists(): EveCorporationInfo.objects.create_corporation(corp_id) - EveCorporationInfo.objects.filter( - corporation_id__in=alliance.corp_ids).update(alliance=self + EveCorporationInfo.objects.filter(corporation_id__in=alliance.corp_ids).update( + alliance=self ) - EveCorporationInfo.objects\ - .filter(alliance=self)\ - .exclude(corporation_id__in=alliance.corp_ids)\ - .update(alliance=None) + EveCorporationInfo.objects.filter(alliance=self).exclude( + corporation_id__in=alliance.corp_ids + ).update(alliance=None) def update_alliance(self, alliance: providers.Alliance = None): if alliance is None: @@ -182,6 +195,7 @@ class EveCorporationInfo(models.Model): class EveCharacter(models.Model): + """Character in Eve Online""" character_id = models.PositiveIntegerField(unique=True) character_name = models.CharField(max_length=254, unique=True) corporation_id = models.PositiveIntegerField() @@ -198,12 +212,20 @@ class EveCharacter(models.Model): class Meta: indexes = [ - models.Index(fields=['corporation_id',]), - models.Index(fields=['alliance_id',]), - models.Index(fields=['corporation_name',]), - models.Index(fields=['alliance_name',]), - models.Index(fields=['faction_id',]), - ] + models.Index(fields=['corporation_id',]), + models.Index(fields=['alliance_id',]), + models.Index(fields=['corporation_name',]), + models.Index(fields=['alliance_name',]), + models.Index(fields=['faction_id',]), + ] + + def __str__(self): + return self.character_name + + @property + def is_biomassed(self) -> bool: + """Whether this character is dead or not.""" + return self.corporation_id == DOOMHEIM_CORPORATION_ID @property def alliance(self) -> Union[EveAllianceInfo, None]: @@ -249,10 +271,36 @@ class EveCharacter(models.Model): self.faction_id = character.faction.id self.faction_name = character.faction.name self.save() + if self.is_biomassed: + self._remove_tokens_of_biomassed_character() return self - def __str__(self): - return self.character_name + def _remove_tokens_of_biomassed_character(self) -> None: + """Remove tokens of this biomassed character.""" + try: + user = self.character_ownership.user + except ObjectDoesNotExist: + return + tokens_to_delete = Token.objects.filter(character_id=self.character_id) + tokens_count = tokens_to_delete.count() + if not tokens_count: + return + tokens_to_delete.delete() + logger.info( + "%d tokens from user %s for biomassed character %s [id:%s] deleted.", + tokens_count, + user, + self, + self.character_id, + ) + notify( + user=user, + title=f"Character {self} biomassed", + message=( + f"Your former character {self} has been biomassed " + "and has been removed from the list of your alts." + ) + ) @staticmethod def generic_portrait_url( @@ -336,7 +384,6 @@ class EveCharacter(models.Model): """image URL for alliance of this character or empty string""" return self.alliance_logo_url(256) - def faction_logo_url(self, size=_DEFAULT_IMAGE_SIZE) -> str: """image URL for alliance of this character or empty string""" if self.faction_id: diff --git a/allianceauth/eveonline/providers.py b/allianceauth/eveonline/providers.py index f399613e..fb1749dd 100644 --- a/allianceauth/eveonline/providers.py +++ b/allianceauth/eveonline/providers.py @@ -170,7 +170,7 @@ class EveProvider: """ :return: an ItemType object for the given ID """ - raise NotImplemented() + raise NotImplementedError() class EveSwaggerProvider(EveProvider): @@ -207,7 +207,8 @@ class EveSwaggerProvider(EveProvider): def __str__(self): return 'esi' - def get_alliance(self, alliance_id): + def get_alliance(self, alliance_id: int) -> Alliance: + """Fetch alliance from ESI.""" try: data = self.client.Alliance.get_alliances_alliance_id(alliance_id=alliance_id).result() corps = self.client.Alliance.get_alliances_alliance_id_corporations(alliance_id=alliance_id).result() @@ -223,7 +224,8 @@ class EveSwaggerProvider(EveProvider): except HTTPNotFound: raise ObjectNotFound(alliance_id, 'alliance') - def get_corp(self, corp_id): + def get_corp(self, corp_id: int) -> Corporation: + """Fetch corporation from ESI.""" try: data = self.client.Corporation.get_corporations_corporation_id(corporation_id=corp_id).result() model = Corporation( @@ -239,29 +241,43 @@ class EveSwaggerProvider(EveProvider): except HTTPNotFound: raise ObjectNotFound(corp_id, 'corporation') - def get_character(self, character_id): + def get_character(self, character_id: int) -> Character: + """Fetch character from ESI.""" try: - data = self.client.Character.get_characters_character_id(character_id=character_id).result() + character_name = self._fetch_character_name(character_id) affiliation = self.client.Character.post_characters_affiliation(characters=[character_id]).result()[0] - model = Character( id=character_id, - name=data['name'], + name=character_name, corp_id=affiliation['corporation_id'], alliance_id=affiliation['alliance_id'] if 'alliance_id' in affiliation else None, faction_id=affiliation['faction_id'] if 'faction_id' in affiliation else None, ) return model - except (HTTPNotFound, HTTPUnprocessableEntity): + except (HTTPNotFound, HTTPUnprocessableEntity, ObjectNotFound): raise ObjectNotFound(character_id, 'character') + def _fetch_character_name(self, character_id: int) -> str: + """Fetch character name from ESI.""" + data = self.client.Universe.post_universe_names(ids=[character_id]).result() + character = data.pop() if data else None + if ( + not character + or character["category"] != "character" + or character["id"] != character_id + ): + raise ObjectNotFound(character_id, 'character') + return character["name"] + def get_all_factions(self): + """Fetch all factions from ESI.""" if not self._faction_list: self._faction_list = self.client.Universe.get_universe_factions().result() return self._faction_list - def get_faction(self, faction_id): - faction_id=int(faction_id) + def get_faction(self, faction_id: int): + """Fetch faction from ESI.""" + faction_id = int(faction_id) try: if not self._faction_list: _ = self.get_all_factions() @@ -273,7 +289,8 @@ class EveSwaggerProvider(EveProvider): except (HTTPNotFound, HTTPUnprocessableEntity, KeyError): raise ObjectNotFound(faction_id, 'faction') - def get_itemtype(self, type_id): + def get_itemtype(self, type_id: int) -> ItemType: + """Fetch inventory item from ESI.""" try: data = self.client.Universe.get_universe_types_type_id(type_id=type_id).result() return ItemType(id=type_id, name=data['name']) diff --git a/allianceauth/eveonline/tasks.py b/allianceauth/eveonline/tasks.py index 13825f9b..a6b11a24 100644 --- a/allianceauth/eveonline/tasks.py +++ b/allianceauth/eveonline/tasks.py @@ -1,12 +1,11 @@ import logging from celery import shared_task -from .models import EveAllianceInfo -from .models import EveCharacter -from .models import EveCorporationInfo +from .models import EveAllianceInfo, EveCharacter, EveCorporationInfo from . import providers + logger = logging.getLogger(__name__) TASK_PRIORITY = 7 @@ -32,8 +31,8 @@ def update_alliance(alliance_id): @shared_task -def update_character(character_id): - """Update given character from ESI""" +def update_character(character_id: int) -> None: + """Update given character from ESI.""" EveCharacter.objects.update_character(character_id) @@ -65,17 +64,17 @@ def update_character_chunk(character_ids_chunk: list): .post_characters_affiliation(characters=character_ids_chunk).result() character_names = providers.provider.client.Universe\ .post_universe_names(ids=character_ids_chunk).result() - except: + except OSError: logger.info("Failed to bulk update characters. Attempting single updates") for character_id in character_ids_chunk: update_character.apply_async( - args=[character_id], priority=TASK_PRIORITY - ) + args=[character_id], priority=TASK_PRIORITY + ) return affiliations = { - affiliation.get('character_id'): affiliation - for affiliation in affiliations_raw + affiliation.get('character_id'): affiliation + for affiliation in affiliations_raw } # add character names to affiliations for character in character_names: @@ -108,5 +107,5 @@ def update_character_chunk(character_ids_chunk: list): if corp_changed or alliance_changed or name_changed: update_character.apply_async( - args=[character.get('character_id')], priority=TASK_PRIORITY - ) + args=[character.get('character_id')], priority=TASK_PRIORITY + ) diff --git a/allianceauth/eveonline/tests/esi_client_stub.py b/allianceauth/eveonline/tests/esi_client_stub.py new file mode 100644 index 00000000..1eadfbd3 --- /dev/null +++ b/allianceauth/eveonline/tests/esi_client_stub.py @@ -0,0 +1,168 @@ +from bravado.exception import HTTPNotFound + + +class BravadoResponseStub: + """Stub for IncomingResponse in bravado, e.g. for HTTPError exceptions""" + + def __init__( + self, status_code, reason="", text="", headers=None, raw_bytes=None + ) -> None: + self.reason = reason + self.status_code = status_code + self.text = text + self.headers = headers if headers else dict() + self.raw_bytes = raw_bytes + + def __str__(self): + return f"{self.status_code} {self.reason}" + + +class BravadoOperationStub: + """Stub to simulate the operation object return from bravado via django-esi""" + + class RequestConfig: + def __init__(self, also_return_response): + self.also_return_response = also_return_response + + class ResponseStub: + def __init__(self, headers): + self.headers = headers + + def __init__(self, data, headers: dict = None, also_return_response: bool = False): + self._data = data + self._headers = headers if headers else {"x-pages": 1} + self.request_config = BravadoOperationStub.RequestConfig(also_return_response) + + def result(self, **kwargs): + if self.request_config.also_return_response: + return [self._data, self.ResponseStub(self._headers)] + else: + return self._data + + def results(self, **kwargs): + return self.result(**kwargs) + + +class EsiClientStub: + """Stub for an ESI client.""" + class Alliance: + @staticmethod + def get_alliances_alliance_id(alliance_id): + data = { + 3001: { + "name": "Wayne Enterprises", + "ticker": "WYE", + "executor_corporation_id": 2001 + } + } + try: + return BravadoOperationStub(data[int(alliance_id)]) + except KeyError: + response = BravadoResponseStub( + 404, f"Alliance with ID {alliance_id} not found" + ) + raise HTTPNotFound(response) + + @staticmethod + def get_alliances_alliance_id_corporations(alliance_id): + data = [2001, 2002, 2003] + return BravadoOperationStub(data) + + class Character: + @staticmethod + def get_characters_character_id(character_id): + data = { + 1001: { + "corporation_id": 2001, + "name": "Bruce Wayne", + }, + 1002: { + "corporation_id": 2001, + "name": "Peter Parker", + }, + 1011: { + "corporation_id": 2011, + "name": "Lex Luthor", + } + } + try: + return BravadoOperationStub(data[int(character_id)]) + except KeyError: + response = BravadoResponseStub( + 404, f"Character with ID {character_id} not found" + ) + raise HTTPNotFound(response) + + @staticmethod + def post_characters_affiliation(characters: list): + data = [ + {'character_id': 1001, 'corporation_id': 2001, 'alliance_id': 3001}, + {'character_id': 1002, 'corporation_id': 2001, 'alliance_id': 3001}, + {'character_id': 1011, 'corporation_id': 2011}, + {'character_id': 1666, 'corporation_id': 1000001}, + ] + return BravadoOperationStub( + [x for x in data if x['character_id'] in characters] + ) + + class Corporation: + @staticmethod + def get_corporations_corporation_id(corporation_id): + data = { + 2001: { + "ceo_id": 1091, + "member_count": 10, + "name": "Wayne Technologies", + "ticker": "WTE", + "alliance_id": 3001 + }, + 2002: { + "ceo_id": 1092, + "member_count": 10, + "name": "Wayne Food", + "ticker": "WFO", + "alliance_id": 3001 + }, + 2003: { + "ceo_id": 1093, + "member_count": 10, + "name": "Wayne Energy", + "ticker": "WEG", + "alliance_id": 3001 + }, + 2011: { + "ceo_id": 1, + "member_count": 3, + "name": "LexCorp", + "ticker": "LC", + }, + 1000001: { + "ceo_id": 3000001, + "creator_id": 1, + "description": "The internal corporation used for characters in graveyard.", + "member_count": 6329026, + "name": "Doomheim", + "ticker": "666", + } + } + try: + return BravadoOperationStub(data[int(corporation_id)]) + except KeyError: + response = BravadoResponseStub( + 404, f"Corporation with ID {corporation_id} not found" + ) + raise HTTPNotFound(response) + + class Universe: + @staticmethod + def post_universe_names(ids: list): + data = [ + {"category": "character", "id": 1001, "name": "Bruce Wayne"}, + {"category": "character", "id": 1002, "name": "Peter Parker"}, + {"category": "character", "id": 1011, "name": "Lex Luthor"}, + {"category": "character", "id": 1666, "name": "Hal Jordan"}, + {"category": "corporation", "id": 2001, "name": "Wayne Technologies"}, + {"category": "corporation","id": 2002, "name": "Wayne Food"}, + {"category": "corporation","id": 1000001, "name": "Doomheim"}, + ] + return BravadoOperationStub([x for x in data if x['id'] in ids]) diff --git a/allianceauth/eveonline/tests/test_models.py b/allianceauth/eveonline/tests/test_models.py index fc716c00..94c9614a 100644 --- a/allianceauth/eveonline/tests/test_models.py +++ b/allianceauth/eveonline/tests/test_models.py @@ -1,12 +1,15 @@ from unittest.mock import Mock, patch +from django.core.exceptions import ObjectDoesNotExist from django.test import TestCase +from esi.models import Token + +from allianceauth.tests.auth_utils import AuthUtils -from ..models import ( - EveCharacter, EveCorporationInfo, EveAllianceInfo, EveFactionInfo -) -from ..providers import Alliance, Corporation, Character from ..evelinks import eveimageserver +from ..models import EveAllianceInfo, EveCharacter, EveCorporationInfo, EveFactionInfo +from ..providers import Alliance, Character, Corporation +from .esi_client_stub import EsiClientStub class EveCharacterTestCase(TestCase): @@ -402,8 +405,8 @@ class EveAllianceTestCase(TestCase): my_alliance.save() my_alliance.populate_alliance() - for corporation in EveCorporationInfo.objects\ - .filter(corporation_id__in=[2001, 2002] + for corporation in ( + EveCorporationInfo.objects.filter(corporation_id__in=[2001, 2002]) ): self.assertEqual(corporation.alliance, my_alliance) @@ -587,3 +590,98 @@ class EveCorporationTestCase(TestCase): self.my_corp.logo_url_256, 'https://images.evetech.net/corporations/2001/logo?size=256' ) + + +@patch('allianceauth.eveonline.providers.esi_client_factory') +@patch("allianceauth.eveonline.models.notify") +class TestCharacterUpdate(TestCase): + def test_should_update_normal_character(self, mock_notify, mock_esi_client_factory): + # given + mock_esi_client_factory.return_value = EsiClientStub() + my_character = EveCharacter.objects.create( + character_id=1001, + character_name="not my name", + corporation_id=2002, + corporation_name="Wayne Food", + corporation_ticker="WYF", + alliance_id=None + ) + # when + my_character.update_character() + # then + my_character.refresh_from_db() + self.assertEqual(my_character.character_name, "Bruce Wayne") + self.assertEqual(my_character.corporation_id, 2001) + self.assertEqual(my_character.corporation_name, "Wayne Technologies") + self.assertEqual(my_character.corporation_ticker, "WTE") + self.assertEqual(my_character.alliance_id, 3001) + self.assertEqual(my_character.alliance_name, "Wayne Enterprises") + self.assertEqual(my_character.alliance_ticker, "WYE") + self.assertFalse(mock_notify.called) + + def test_should_update_dead_character_with_owner( + self, mock_notify, mock_esi_client_factory + ): + # given + mock_esi_client_factory.return_value = EsiClientStub() + character_1666 = EveCharacter.objects.create( + character_id=1666, + character_name="Hal Jordan", + corporation_id=2002, + corporation_name="Wayne Food", + corporation_ticker="WYF", + alliance_id=None + ) + user = AuthUtils.create_user("Bruce Wayne") + token_1666 = Token.objects.create( + user=user, + character_id=character_1666.character_id, + character_name=character_1666.character_name, + character_owner_hash="ABC123-1666", + ) + character_1001 = EveCharacter.objects.create( + character_id=1001, + character_name="Bruce Wayne", + corporation_id=2001, + corporation_name="Wayne Technologies", + corporation_ticker="WYT", + alliance_id=None + ) + token_1001 = Token.objects.create( + user=user, + character_id=character_1001.character_id, + character_name=character_1001.character_name, + character_owner_hash="ABC123-1001", + ) + # when + character_1666.update_character() + # then + character_1666.refresh_from_db() + self.assertTrue(character_1666.is_biomassed) + self.assertNotIn(token_1666, user.token_set.all()) + self.assertIn(token_1001, user.token_set.all()) + with self.assertRaises(ObjectDoesNotExist): + self.assertTrue(character_1666.character_ownership) + user.profile.refresh_from_db() + self.assertIsNone(user.profile.main_character) + self.assertTrue(mock_notify.called) + + def test_should_handle_dead_character_without_owner( + self, mock_notify, mock_esi_client_factory + ): + # given + mock_esi_client_factory.return_value = EsiClientStub() + character_1666 = EveCharacter.objects.create( + character_id=1666, + character_name="Hal Jordan", + corporation_id=1011, + corporation_name="LexCorp", + corporation_ticker='LC', + alliance_id=None + ) + # when + character_1666.update_character() + # then + character_1666.refresh_from_db() + self.assertTrue(character_1666.is_biomassed) + self.assertFalse(mock_notify.called) diff --git a/allianceauth/eveonline/tests/test_providers.py b/allianceauth/eveonline/tests/test_providers.py index 59e9dfe6..95f4f069 100644 --- a/allianceauth/eveonline/tests/test_providers.py +++ b/allianceauth/eveonline/tests/test_providers.py @@ -7,6 +7,7 @@ from jsonschema.exceptions import RefResolutionError from django.test import TestCase from . import set_logger +from .esi_client_stub import EsiClientStub from ..providers import ( ObjectNotFound, Entity, @@ -632,13 +633,7 @@ class TestEveSwaggerProvider(TestCase): @patch(MODULE_PATH + '.esi_client_factory') def test_get_character(self, mock_esi_client_factory): - mock_esi_client_factory.return_value \ - .Character.get_characters_character_id \ - = TestEveSwaggerProvider.esi_get_characters_character_id - mock_esi_client_factory.return_value \ - .Character.post_characters_affiliation \ - = TestEveSwaggerProvider.esi_post_characters_affiliation - + mock_esi_client_factory.return_value = EsiClientStub() my_provider = EveSwaggerProvider() # character with alliance @@ -649,8 +644,8 @@ class TestEveSwaggerProvider(TestCase): self.assertEqual(my_character.alliance_id, 3001) # character wo/ alliance - my_character = my_provider.get_character(1002) - self.assertEqual(my_character.id, 1002) + my_character = my_provider.get_character(1011) + self.assertEqual(my_character.id, 1011) self.assertEqual(my_character.alliance_id, None) # character not found diff --git a/allianceauth/eveonline/tests/test_tasks.py b/allianceauth/eveonline/tests/test_tasks.py index 4bf40671..78054e8f 100644 --- a/allianceauth/eveonline/tests/test_tasks.py +++ b/allianceauth/eveonline/tests/test_tasks.py @@ -1,245 +1,271 @@ -from unittest.mock import patch, Mock +from unittest.mock import patch -from django.test import TestCase +from django.test import TestCase, TransactionTestCase, override_settings -from ..models import EveCharacter, EveCorporationInfo, EveAllianceInfo +from ..models import EveAllianceInfo, EveCharacter, EveCorporationInfo from ..tasks import ( + run_model_update, update_alliance, - update_corp, update_character, - run_model_update + update_character_chunk, + update_corp, ) +from .esi_client_stub import EsiClientStub -class TestTasks(TestCase): - - @patch('allianceauth.eveonline.tasks.EveCorporationInfo') - def test_update_corp(self, mock_EveCorporationInfo): - update_corp(42) - self.assertEqual( - mock_EveCorporationInfo.objects.update_corporation.call_count, 1 - ) - self.assertEqual( - mock_EveCorporationInfo.objects.update_corporation.call_args[0][0], 42 +@patch('allianceauth.eveonline.providers.esi_client_factory') +class TestUpdateTasks(TestCase): + def test_should_update_alliance(self, mock_esi_client_factory): + # given + mock_esi_client_factory.return_value = EsiClientStub() + my_alliance = EveAllianceInfo.objects.create( + alliance_id=3001, + alliance_name="Wayne Enterprises", + alliance_ticker="WYE", + executor_corp_id=2003 ) + # when + update_alliance(my_alliance.alliance_id) + # then + my_alliance.refresh_from_db() + self.assertEqual(my_alliance.executor_corp_id, 2001) - @patch('allianceauth.eveonline.tasks.EveAllianceInfo') - def test_update_alliance(self, mock_EveAllianceInfo): - update_alliance(42) - self.assertEqual( - mock_EveAllianceInfo.objects.update_alliance.call_args[0][0], 42 - ) - self.assertEqual( - mock_EveAllianceInfo.objects - .update_alliance.return_value.populate_alliance.call_count, 1 + def test_should_update_character(self, mock_esi_client_factory): + # given + mock_esi_client_factory.return_value = EsiClientStub() + my_character = EveCharacter.objects.create( + character_id=1001, + character_name="Bruce Wayne", + corporation_id=2002, + corporation_name="Wayne Food", + corporation_ticker="WYF", + alliance_id=None ) + # when + update_character(my_character.character_id) + # then + my_character.refresh_from_db() + self.assertEqual(my_character.corporation_id, 2001) - @patch('allianceauth.eveonline.tasks.EveCharacter') - def test_update_character(self, mock_EveCharacter): - update_character(42) - self.assertEqual( - mock_EveCharacter.objects.update_character.call_count, 1 + def test_should_update_corp(self, mock_esi_client_factory): + # given + mock_esi_client_factory.return_value = EsiClientStub() + EveAllianceInfo.objects.create( + alliance_id=3001, + alliance_name="Wayne Enterprises", + alliance_ticker="WYE", + executor_corp_id=2003 ) - self.assertEqual( - mock_EveCharacter.objects.update_character.call_args[0][0], 42 + my_corporation = EveCorporationInfo.objects.create( + corporation_id=2003, + corporation_name="Wayne Food", + corporation_ticker="WFO", + member_count=1, + alliance=None, + ceo_id=1999 ) + # when + update_corp(my_corporation.corporation_id) + # then + my_corporation.refresh_from_db() + self.assertEqual(my_corporation.alliance.alliance_id, 3001) + + # @patch('allianceauth.eveonline.tasks.EveCharacter') + # def test_update_character(self, mock_EveCharacter): + # update_character(42) + # self.assertEqual( + # mock_EveCharacter.objects.update_character.call_count, 1 + # ) + # self.assertEqual( + # mock_EveCharacter.objects.update_character.call_args[0][0], 42 + # ) -@patch('allianceauth.eveonline.tasks.update_character') -@patch('allianceauth.eveonline.tasks.update_alliance') -@patch('allianceauth.eveonline.tasks.update_corp') -@patch('allianceauth.eveonline.providers.provider') +@override_settings(CELERY_ALWAYS_EAGER=True) +@patch('allianceauth.eveonline.providers.esi_client_factory') +@patch('allianceauth.eveonline.tasks.providers') @patch('allianceauth.eveonline.tasks.CHUNK_SIZE', 2) -class TestRunModelUpdate(TestCase): - - @classmethod - def setUpClass(cls): - super().setUpClass() - EveCorporationInfo.objects.all().delete() - EveAllianceInfo.objects.all().delete() - EveCharacter.objects.all().delete() - +class TestRunModelUpdate(TransactionTestCase): + def test_should_run_updates(self, mock_providers, mock_esi_client_factory): + # given + mock_providers.provider.client = EsiClientStub() + mock_esi_client_factory.return_value = EsiClientStub() EveCorporationInfo.objects.create( - corporation_id=2345, - corporation_name='corp.name', - corporation_ticker='c.c.t', + corporation_id=2001, + corporation_name="Wayne Technologies", + corporation_ticker="WTE", member_count=10, alliance=None, ) - EveAllianceInfo.objects.create( - alliance_id=3456, - alliance_name='alliance.name', - alliance_ticker='a.t', - executor_corp_id=5, + alliance_3001 = EveAllianceInfo.objects.create( + alliance_id=3001, + alliance_name="Wayne Enterprises", + alliance_ticker="WYE", + executor_corp_id=2003 ) - EveCharacter.objects.create( - character_id=1, - character_name='character.name1', - corporation_id=2345, - corporation_name='character.corp.name', - corporation_ticker='c.c.t', # max 5 chars + corporation_2003 = EveCorporationInfo.objects.create( + corporation_id=2003, + corporation_name="Wayne Energy", + corporation_ticker="WEG", + member_count=99, + alliance=None, + ) + character_1001 = EveCharacter.objects.create( + character_id=1001, + character_name="Bruce Wayne", + corporation_id=2002, + corporation_name="Wayne Food", + corporation_ticker="WYF", alliance_id=None ) - EveCharacter.objects.create( - character_id=2, - character_name='character.name2', - corporation_id=9876, - corporation_name='character.corp.name', - corporation_ticker='c.c.t', # max 5 chars - alliance_id=3456, - alliance_name='character.alliance.name', - ) - EveCharacter.objects.create( - character_id=3, - character_name='character.name3', - corporation_id=9876, - corporation_name='character.corp.name', - corporation_ticker='c.c.t', # max 5 chars - alliance_id=3456, - alliance_name='character.alliance.name', - ) - EveCharacter.objects.create( - character_id=4, - character_name='character.name4', - corporation_id=9876, - corporation_name='character.corp.name', - corporation_ticker='c.c.t', # max 5 chars - alliance_id=3456, - alliance_name='character.alliance.name', - ) - """ - EveCharacter.objects.create( - character_id=5, - character_name='character.name5', - corporation_id=9876, - corporation_name='character.corp.name', - corporation_ticker='c.c.t', # max 5 chars - alliance_id=3456, - alliance_name='character.alliance.name', - ) - """ - - def setUp(self): - self.affiliations = [ - {'character_id': 1, 'corporation_id': 5}, - {'character_id': 2, 'corporation_id': 9876, 'alliance_id': 3456}, - {'character_id': 3, 'corporation_id': 9876, 'alliance_id': 7456}, - {'character_id': 4, 'corporation_id': 9876, 'alliance_id': 3456} - ] - self.names = [ - {'id': 1, 'name': 'character.name1'}, - {'id': 2, 'name': 'character.name2'}, - {'id': 3, 'name': 'character.name3'}, - {'id': 4, 'name': 'character.name4_new'} - ] - - def test_normal_run( - self, - mock_provider, - mock_update_corp, - mock_update_alliance, - mock_update_character, - ): - def get_affiliations(characters: list): - response = [x for x in self.affiliations if x['character_id'] in characters] - mock_operator = Mock(**{'result.return_value': response}) - return mock_operator - - def get_names(ids: list): - response = [x for x in self.names if x['id'] in ids] - mock_operator = Mock(**{'result.return_value': response}) - return mock_operator - - mock_provider.client.Character.post_characters_affiliation.side_effect \ - = get_affiliations - - mock_provider.client.Universe.post_universe_names.side_effect = get_names - + # when run_model_update() - + # then + character_1001.refresh_from_db() self.assertEqual( - mock_provider.client.Character.post_characters_affiliation.call_count, 2 + character_1001.corporation_id, 2001 # char has new corp ) + corporation_2003.refresh_from_db() self.assertEqual( - mock_provider.client.Universe.post_universe_names.call_count, 2 + corporation_2003.alliance.alliance_id, 3001 # corp has new alliance + ) + alliance_3001.refresh_from_db() + self.assertEqual( + alliance_3001.executor_corp_id, 2001 # alliance has been updated ) - # character 1 has changed corp - # character 2 no change - # character 3 has changed alliance - # character 4 has changed name - self.assertEqual(mock_update_corp.apply_async.call_count, 1) - self.assertEqual( - int(mock_update_corp.apply_async.call_args[1]['args'][0]), 2345 - ) - self.assertEqual(mock_update_alliance.apply_async.call_count, 1) - self.assertEqual( - int(mock_update_alliance.apply_async.call_args[1]['args'][0]), 3456 - ) - characters_updated = { - x[1]['args'][0] for x in mock_update_character.apply_async.call_args_list + +@override_settings(CELERY_ALWAYS_EAGER=True) +@patch('allianceauth.eveonline.tasks.update_character', wraps=update_character) +@patch('allianceauth.eveonline.providers.esi_client_factory') +@patch('allianceauth.eveonline.tasks.providers') +@patch('allianceauth.eveonline.tasks.CHUNK_SIZE', 2) +class TestUpdateCharacterChunk(TestCase): + @staticmethod + def _updated_character_ids(spy_update_character) -> set: + """Character IDs passed to update_character task for update.""" + return { + x[1]["args"][0] for x in spy_update_character.apply_async.call_args_list } - excepted = {1, 3, 4} - self.assertSetEqual(characters_updated, excepted) - def test_ignore_character_not_in_affiliations( - self, - mock_provider, - mock_update_corp, - mock_update_alliance, - mock_update_character, + def test_should_update_corp_change( + self, mock_providers, mock_esi_client_factory, spy_update_character ): - def get_affiliations(characters: list): - response = [x for x in self.affiliations if x['character_id'] in characters] - mock_operator = Mock(**{'result.return_value': response}) - return mock_operator + # given + mock_providers.provider.client = EsiClientStub() + mock_esi_client_factory.return_value = EsiClientStub() + character_1001 = EveCharacter.objects.create( + character_id=1001, + character_name="Bruce Wayne", + corporation_id=2003, + corporation_name="Wayne Energy", + corporation_ticker="WEG", + alliance_id=3001, + alliance_name="Wayne Enterprises", + alliance_ticker="WYE", + ) + character_1002 = EveCharacter.objects.create( + character_id=1002, + character_name="Peter Parker", + corporation_id=2001, + corporation_name="Wayne Technologies", + corporation_ticker="WTE", + alliance_id=3001, + alliance_name="Wayne Enterprises", + alliance_ticker="WYE", + ) + # when + update_character_chunk([ + character_1001.character_id, character_1002.character_id + ]) + # then + character_1001.refresh_from_db() + self.assertEqual(character_1001.corporation_id, 2001) + self.assertSetEqual(self._updated_character_ids(spy_update_character), {1001}) - def get_names(ids: list): - response = [x for x in self.names if x['id'] in ids] - mock_operator = Mock(**{'result.return_value': response}) - return mock_operator - - del self.affiliations[0] - - mock_provider.client.Character.post_characters_affiliation.side_effect \ - = get_affiliations - - mock_provider.client.Universe.post_universe_names.side_effect = get_names - - run_model_update() - characters_updated = { - x[1]['args'][0] for x in mock_update_character.apply_async.call_args_list - } - excepted = {3, 4} - self.assertSetEqual(characters_updated, excepted) - - def test_ignore_character_not_in_names( - self, - mock_provider, - mock_update_corp, - mock_update_alliance, - mock_update_character, + def test_should_update_name_change( + self, mock_providers, mock_esi_client_factory, spy_update_character ): - def get_affiliations(characters: list): - response = [x for x in self.affiliations if x['character_id'] in characters] - mock_operator = Mock(**{'result.return_value': response}) - return mock_operator + # given + mock_providers.provider.client = EsiClientStub() + mock_esi_client_factory.return_value = EsiClientStub() + character_1001 = EveCharacter.objects.create( + character_id=1001, + character_name="Batman", + corporation_id=2001, + corporation_name="Wayne Technologies", + corporation_ticker="WTE", + alliance_id=3001, + alliance_name="Wayne Technologies", + alliance_ticker="WYT", + ) + # when + update_character_chunk([character_1001.character_id]) + # then + character_1001.refresh_from_db() + self.assertEqual(character_1001.character_name, "Bruce Wayne") + self.assertSetEqual(self._updated_character_ids(spy_update_character), {1001}) - def get_names(ids: list): - response = [x for x in self.names if x['id'] in ids] - mock_operator = Mock(**{'result.return_value': response}) - return mock_operator + def test_should_update_alliance_change( + self, mock_providers, mock_esi_client_factory, spy_update_character + ): + # given + mock_providers.provider.client = EsiClientStub() + mock_esi_client_factory.return_value = EsiClientStub() + character_1001 = EveCharacter.objects.create( + character_id=1001, + character_name="Bruce Wayne", + corporation_id=2001, + corporation_name="Wayne Technologies", + corporation_ticker="WTE", + alliance_id=None, + ) + # when + update_character_chunk([character_1001.character_id]) + # then + character_1001.refresh_from_db() + self.assertEqual(character_1001.alliance_id, 3001) + self.assertSetEqual(self._updated_character_ids(spy_update_character), {1001}) - del self.names[3] + def test_should_not_update_when_not_changed( + self, mock_providers, mock_esi_client_factory, spy_update_character + ): + # given + mock_providers.provider.client = EsiClientStub() + mock_esi_client_factory.return_value = EsiClientStub() + character_1001 = EveCharacter.objects.create( + character_id=1001, + character_name="Bruce Wayne", + corporation_id=2001, + corporation_name="Wayne Technologies", + corporation_ticker="WTE", + alliance_id=3001, + alliance_name="Wayne Technologies", + alliance_ticker="WYT", + ) + # when + update_character_chunk([character_1001.character_id]) + # then + self.assertSetEqual(self._updated_character_ids(spy_update_character), set()) - mock_provider.client.Character.post_characters_affiliation.side_effect \ - = get_affiliations - - mock_provider.client.Universe.post_universe_names.side_effect = get_names - - run_model_update() - characters_updated = { - x[1]['args'][0] for x in mock_update_character.apply_async.call_args_list - } - excepted = {1, 3} - self.assertSetEqual(characters_updated, excepted) + def test_should_fall_back_to_single_updates_when_bulk_update_failed( + self, mock_providers, mock_esi_client_factory, spy_update_character + ): + # given + mock_providers.provider.client.Character.post_characters_affiliation\ + .side_effect = OSError + mock_esi_client_factory.return_value = EsiClientStub() + character_1001 = EveCharacter.objects.create( + character_id=1001, + character_name="Bruce Wayne", + corporation_id=2001, + corporation_name="Wayne Technologies", + corporation_ticker="WTE", + alliance_id=3001, + alliance_name="Wayne Technologies", + alliance_ticker="WYT", + ) + # when + update_character_chunk([character_1001.character_id]) + # then + self.assertSetEqual(self._updated_character_ids(spy_update_character), {1001}) From ba39318313c81592e6d5ad5b4ac030c7d18fa2e3 Mon Sep 17 00:00:00 2001 From: Erik Kalkoken Date: Thu, 27 Jan 2022 05:14:11 +0000 Subject: [PATCH 10/11] Add setting to disable analytics --- allianceauth/analytics/middleware.py | 3 + allianceauth/analytics/signals.py | 11 +- allianceauth/analytics/tasks.py | 16 +-- .../analytics/tests/test_integration.py | 108 ++++++++++++++++++ docs/conf.py | 1 + docs/features/core/analytics.md | 8 ++ 6 files changed, 136 insertions(+), 11 deletions(-) create mode 100644 allianceauth/analytics/tests/test_integration.py diff --git a/allianceauth/analytics/middleware.py b/allianceauth/analytics/middleware.py index ea636b6c..4dd93685 100644 --- a/allianceauth/analytics/middleware.py +++ b/allianceauth/analytics/middleware.py @@ -1,5 +1,6 @@ from bs4 import BeautifulSoup +from django.conf import settings from django.utils.deprecation import MiddlewareMixin from .models import AnalyticsTokens, AnalyticsIdentifier from .tasks import send_ga_tracking_web_view @@ -10,6 +11,8 @@ import re class AnalyticsMiddleware(MiddlewareMixin): def process_response(self, request, response): """Django Middleware: Process Page Views and creates Analytics Celery Tasks""" + if getattr(settings, "ANALYTICS_DISABLED", False): + return response analyticstokens = AnalyticsTokens.objects.all() client_id = AnalyticsIdentifier.objects.get(id=1).identifier.hex try: diff --git a/allianceauth/analytics/signals.py b/allianceauth/analytics/signals.py index b3943220..91565c5e 100644 --- a/allianceauth/analytics/signals.py +++ b/allianceauth/analytics/signals.py @@ -1,7 +1,8 @@ -from allianceauth.analytics.tasks import analytics_event -from celery.signals import task_failure, task_success - import logging +from celery.signals import task_failure, task_success +from django.conf import settings +from allianceauth.analytics.tasks import analytics_event + logger = logging.getLogger(__name__) @@ -11,6 +12,8 @@ def process_failure_signal( sender, task_id, signal, args, kwargs, einfo, **kw): logger.debug("Celery task_failure signal %s" % sender.__class__.__name__) + if getattr(settings, "ANALYTICS_DISABLED", False): + return category = sender.__module__ @@ -30,6 +33,8 @@ def process_failure_signal( @task_success.connect def celery_success_signal(sender, result=None, **kw): logger.debug("Celery task_success signal %s" % sender.__class__.__name__) + if getattr(settings, "ANALYTICS_DISABLED", False): + return category = sender.__module__ diff --git a/allianceauth/analytics/tasks.py b/allianceauth/analytics/tasks.py index 7fe22477..c8708b75 100644 --- a/allianceauth/analytics/tasks.py +++ b/allianceauth/analytics/tasks.py @@ -40,13 +40,12 @@ def analytics_event(category: str, Send a Google Analytics Event for each token stored Includes check for if its enabled/disabled - Parameters - ------- - `category` (str): Celery Namespace - `action` (str): Task Name - `label` (str): Optional, Task Success/Exception - `value` (int): Optional, If bulk, Query size, can be a binary True/False - `event_type` (str): Optional, Celery or Stats only, Default to Celery + Args: + `category` (str): Celery Namespace + `action` (str): Task Name + `label` (str): Optional, Task Success/Exception + `value` (int): Optional, If bulk, Query size, can be a binary True/False + `event_type` (str): Optional, Celery or Stats only, Default to Celery """ analyticstokens = AnalyticsTokens.objects.all() client_id = AnalyticsIdentifier.objects.get(id=1).identifier.hex @@ -73,7 +72,8 @@ def analytics_event(category: str, def analytics_daily_stats(): """Celery Task: Do not call directly - Gathers a series of daily statistics and sends analytics events containing them""" + Gathers a series of daily statistics and sends analytics events containing them + """ users = install_stat_users() tokens = install_stat_tokens() addons = install_stat_addons() diff --git a/allianceauth/analytics/tests/test_integration.py b/allianceauth/analytics/tests/test_integration.py new file mode 100644 index 00000000..0fbe445b --- /dev/null +++ b/allianceauth/analytics/tests/test_integration.py @@ -0,0 +1,108 @@ +from unittest.mock import patch +from urllib.parse import parse_qs + +import requests_mock + +from django.test import TestCase, override_settings + +from allianceauth.analytics.tasks import ANALYTICS_URL +from allianceauth.eveonline.tasks import update_character +from allianceauth.tests.auth_utils import AuthUtils + + +@override_settings(CELERY_ALWAYS_EAGER=True) +@requests_mock.mock() +class TestAnalyticsForViews(TestCase): + @override_settings(ANALYTICS_DISABLED=False) + def test_should_run_analytics(self, requests_mocker): + # given + requests_mocker.post(ANALYTICS_URL) + user = AuthUtils.create_user("Bruce Wayne") + self.client.force_login(user) + # when + response = self.client.get("/dashboard/") + # then + self.assertEqual(response.status_code, 200) + self.assertTrue(requests_mocker.called) + + @override_settings(ANALYTICS_DISABLED=True) + def test_should_not_run_analytics(self, requests_mocker): + # given + requests_mocker.post(ANALYTICS_URL) + user = AuthUtils.create_user("Bruce Wayne") + self.client.force_login(user) + # when + response = self.client.get("/dashboard/") + # then + self.assertEqual(response.status_code, 200) + self.assertFalse(requests_mocker.called) + + +@override_settings(CELERY_ALWAYS_EAGER=True) +@requests_mock.mock() +class TestAnalyticsForTasks(TestCase): + @override_settings(ANALYTICS_DISABLED=False) + @patch("allianceauth.eveonline.models.EveCharacter.objects.update_character") + def test_should_run_analytics_for_successful_task( + self, requests_mocker, mock_update_character + ): + # given + requests_mocker.post(ANALYTICS_URL) + user = AuthUtils.create_user("Bruce Wayne") + character = AuthUtils.add_main_character_2(user, "Bruce Wayne", 1001) + # when + update_character.delay(character.character_id) + # then + self.assertTrue(mock_update_character.called) + self.assertTrue(requests_mocker.called) + payload = parse_qs(requests_mocker.last_request.text) + self.assertListEqual(payload["el"], ["Success"]) + + @override_settings(ANALYTICS_DISABLED=True) + @patch("allianceauth.eveonline.models.EveCharacter.objects.update_character") + def test_should_not_run_analytics_for_successful_task( + self, requests_mocker, mock_update_character + ): + # given + requests_mocker.post(ANALYTICS_URL) + user = AuthUtils.create_user("Bruce Wayne") + character = AuthUtils.add_main_character_2(user, "Bruce Wayne", 1001) + # when + update_character.delay(character.character_id) + # then + self.assertTrue(mock_update_character.called) + self.assertFalse(requests_mocker.called) + + @override_settings(ANALYTICS_DISABLED=False) + @patch("allianceauth.eveonline.models.EveCharacter.objects.update_character") + def test_should_run_analytics_for_failed_task( + self, requests_mocker, mock_update_character + ): + # given + requests_mocker.post(ANALYTICS_URL) + mock_update_character.side_effect = RuntimeError + user = AuthUtils.create_user("Bruce Wayne") + character = AuthUtils.add_main_character_2(user, "Bruce Wayne", 1001) + # when + update_character.delay(character.character_id) + # then + self.assertTrue(mock_update_character.called) + self.assertTrue(requests_mocker.called) + payload = parse_qs(requests_mocker.last_request.text) + self.assertNotEqual(payload["el"], ["Success"]) + + @override_settings(ANALYTICS_DISABLED=True) + @patch("allianceauth.eveonline.models.EveCharacter.objects.update_character") + def test_should_not_run_analytics_for_failed_task( + self, requests_mocker, mock_update_character + ): + # given + requests_mocker.post(ANALYTICS_URL) + mock_update_character.side_effect = RuntimeError + user = AuthUtils.create_user("Bruce Wayne") + character = AuthUtils.add_main_character_2(user, "Bruce Wayne", 1001) + # when + update_character.delay(character.character_id) + # then + self.assertTrue(mock_update_character.called) + self.assertFalse(requests_mocker.called) diff --git a/docs/conf.py b/docs/conf.py index 682a0653..de813c7e 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -42,6 +42,7 @@ from recommonmark.transform import AutoStructify extensions = [ 'sphinx_rtd_theme', 'sphinx.ext.autodoc', + 'sphinx.ext.napoleon', 'recommonmark', ] diff --git a/docs/features/core/analytics.md b/docs/features/core/analytics.md index 7b750ada..0ee03f8a 100644 --- a/docs/features/core/analytics.md +++ b/docs/features/core/analytics.md @@ -10,6 +10,12 @@ To Opt-Out, modify our pre-loaded token using the Admin dashboard */admin/analyt Each of the three features Daily Stats, Celery Events and Page Views can be enabled/Disabled independently. +Alternatively, you can fully opt out of analytics with the following optional setting: + +```python +ANALYTICS_DISABLED = True +``` + ![Analytics Tokens](/_static/images/features/core/analytics/tokens.png) ## What @@ -58,6 +64,8 @@ This data is stored in a Team Google Analytics Dashboard. The Maintainers all ha ### Analytics Event +```eval_rst .. automodule:: allianceauth.analytics.tasks :members: analytics_event :undoc-members: +``` From 13e88492f1bb0c0b679456b37aca3fea6ff41cae Mon Sep 17 00:00:00 2001 From: Ariel Rin Date: Mon, 31 Jan 2022 09:23:43 +0000 Subject: [PATCH 11/11] Analytics - Extra Ignore Path --- .../migrations/0004_auto_20211015_0502.py | 19 +++++++-- .../migrations/0006_more_ignore_paths.py | 40 +++++++++++++++++++ allianceauth/analytics/tasks.py | 18 ++++----- 3 files changed, 64 insertions(+), 13 deletions(-) create mode 100644 allianceauth/analytics/migrations/0006_more_ignore_paths.py diff --git a/allianceauth/analytics/migrations/0004_auto_20211015_0502.py b/allianceauth/analytics/migrations/0004_auto_20211015_0502.py index c9da3708..2cf2dc67 100644 --- a/allianceauth/analytics/migrations/0004_auto_20211015_0502.py +++ b/allianceauth/analytics/migrations/0004_auto_20211015_0502.py @@ -1,11 +1,11 @@ # Generated by Django 3.1.13 on 2021-10-15 05:02 +from django.core.exceptions import ObjectDoesNotExist from django.db import migrations def modify_aa_team_token_add_page_ignore_paths(apps, schema_editor): - # We can't import the Person model directly as it may be a newer - # version than this migration expects. We use the historical version. + # Add /admin/ and /user_notifications_count/ path to ignore AnalyticsPath = apps.get_model('analytics', 'AnalyticsPath') admin = AnalyticsPath.objects.create(ignore_path=r"^\/admin\/.*") @@ -17,8 +17,19 @@ def modify_aa_team_token_add_page_ignore_paths(apps, schema_editor): def undo_modify_aa_team_token_add_page_ignore_paths(apps, schema_editor): - # nothing should need to migrate away here? - return True + # + AnalyticsPath = apps.get_model('analytics', 'AnalyticsPath') + Tokens = apps.get_model('analytics', 'AnalyticsTokens') + + token = Tokens.objects.get(token="UA-186249766-2") + try: + admin = AnalyticsPath.objects.get(ignore_path=r"^\/admin\/.*", analyticstokens=token) + user_notifications_count = AnalyticsPath.objects.get(ignore_path=r"^\/user_notifications_count\/.*", analyticstokens=token) + admin.delete() + user_notifications_count.delete() + except ObjectDoesNotExist: + # Its fine if it doesnt exist, we just dont want them building up when re-migrating + pass class Migration(migrations.Migration): diff --git a/allianceauth/analytics/migrations/0006_more_ignore_paths.py b/allianceauth/analytics/migrations/0006_more_ignore_paths.py new file mode 100644 index 00000000..392b7f9e --- /dev/null +++ b/allianceauth/analytics/migrations/0006_more_ignore_paths.py @@ -0,0 +1,40 @@ +# Generated by Django 3.2.8 on 2021-10-19 01:47 + +from django.core.exceptions import ObjectDoesNotExist +from django.db import migrations + + +def modify_aa_team_token_add_page_ignore_paths(apps, schema_editor): + # Add the /account/activate path to ignore + + AnalyticsPath = apps.get_model('analytics', 'AnalyticsPath') + account_activate = AnalyticsPath.objects.create(ignore_path=r"^\/account\/activate\/.*") + + Tokens = apps.get_model('analytics', 'AnalyticsTokens') + token = Tokens.objects.get(token="UA-186249766-2") + token.ignore_paths.add(account_activate) + + +def undo_modify_aa_team_token_add_page_ignore_paths(apps, schema_editor): + # + AnalyticsPath = apps.get_model('analytics', 'AnalyticsPath') + Tokens = apps.get_model('analytics', 'AnalyticsTokens') + + token = Tokens.objects.get(token="UA-186249766-2") + + try: + account_activate = AnalyticsPath.objects.get(ignore_path=r"^\/account\/activate\/.*", analyticstokens=token) + account_activate.delete() + except ObjectDoesNotExist: + # Its fine if it doesnt exist, we just dont want them building up when re-migrating + pass + +class Migration(migrations.Migration): + + dependencies = [ + ('analytics', '0005_alter_analyticspath_ignore_path'), + ] + + operations = [ + migrations.RunPython(modify_aa_team_token_add_page_ignore_paths, undo_modify_aa_team_token_add_page_ignore_paths) + ] diff --git a/allianceauth/analytics/tasks.py b/allianceauth/analytics/tasks.py index c8708b75..34826e21 100644 --- a/allianceauth/analytics/tasks.py +++ b/allianceauth/analytics/tasks.py @@ -21,8 +21,8 @@ if getattr(settings, "ANALYTICS_ENABLE_DEBUG", False) and settings.DEBUG: # Force sending of analytics data during in a debug/test environemt # Usefull for developers working on this feature. logger.warning( - "You have 'ANALYTICS_ENABLE_DEBUG' Enabled! " - "This debug instance will send analytics data!") + "You have 'ANALYTICS_ENABLE_DEBUG' Enabled! " + "This debug instance will send analytics data!") DEBUG_URL = COLLECTION_URL ANALYTICS_URL = COLLECTION_URL @@ -59,13 +59,13 @@ def analytics_event(category: str, if allowed is True: tracking_id = token.token - send_ga_tracking_celery_event.s(tracking_id=tracking_id, - client_id=client_id, - category=category, - action=action, - label=label, - value=value).\ - apply_async(priority=9) + send_ga_tracking_celery_event.s( + tracking_id=tracking_id, + client_id=client_id, + category=category, + action=action, + label=label, + value=value).apply_async(priority=9) @shared_task()