Version Bump 2.7.5

discord.com replaces discordapp.com

See merge request allianceauth/allianceauth!1248

allianceauth/__init__.py

@@ -1,7 +1,7 @@
 # This will make sure the app is always imported when
 # Django starts so that shared_task will use this app.
 
-__version__ = '2.7.4'
+__version__ = '2.7.5'
 __title__ = 'Alliance Auth'
 __url__ = 'https://gitlab.com/allianceauth/allianceauth'
 NAME = '%s v%s' % (__title__, __version__)

allianceauth/eveonline/tasks.py

@@ -5,35 +5,96 @@ from .models import EveAllianceInfo
 from .models import EveCharacter
 from .models import EveCorporationInfo
 
+from . import providers
+
 logger = logging.getLogger(__name__)
 
 TASK_PRIORITY = 7
+CHUNK_SIZE = 500
+
+
+def chunks(lst, n):
+    """Yield successive n-sized chunks from lst."""
+    for i in range(0, len(lst), n):
+        yield lst[i:i + n]
+
 
 @shared_task
 def update_corp(corp_id):
+    """Update given corporation from ESI"""
     EveCorporationInfo.objects.update_corporation(corp_id)
 
 
 @shared_task
 def update_alliance(alliance_id):
+    """Update given alliance from ESI"""
     EveAllianceInfo.objects.update_alliance(alliance_id).populate_alliance()
 
 
 @shared_task
 def update_character(character_id):
+    """Update given character from ESI"""
     EveCharacter.objects.update_character(character_id)
 
 
 @shared_task
 def run_model_update():
+    """Update all alliances, corporations and characters from ESI"""
+    
     # update existing corp models
     for corp in EveCorporationInfo.objects.all().values('corporation_id'):
-        update_corp.apply_async(args=[corp['corporation_id']], priority=TASK_PRIORITY)
+        update_corp.apply_async(
+            args=[corp['corporation_id']], priority=TASK_PRIORITY
+        )
 
     # update existing alliance models
     for alliance in EveAllianceInfo.objects.all().values('alliance_id'):
-        update_alliance.apply_async(args=[alliance['alliance_id']], priority=TASK_PRIORITY)
+        update_alliance.apply_async(
+            args=[alliance['alliance_id']], priority=TASK_PRIORITY
+        )
 
-    #update existing character models
-    for character in EveCharacter.objects.all().values('character_id'):
-        update_character.apply_async(args=[character['character_id']], priority=TASK_PRIORITY)
+    # update existing character models
+    character_ids = EveCharacter.objects.all().values_list('character_id', flat=True)    
+    for character_ids_chunk in chunks(character_ids, CHUNK_SIZE):
+        affiliations_raw = providers.provider.client.Character\
+            .post_characters_affiliation(characters=character_ids_chunk).result()
+        character_names = providers.provider.client.Universe\
+            .post_universe_names(ids=character_ids_chunk).result()
+
+        affiliations = {
+            affiliation.get('character_id'): affiliation 
+            for affiliation in affiliations_raw
+        }
+        # add character names to affiliations
+        for character in character_names:            
+            character_id = character.get('id')
+            if character_id in affiliations:
+                affiliations[character_id]['name'] = character.get('name')
+
+        # fetch current characters
+        characters = EveCharacter.objects.filter(character_id__in=character_ids_chunk)\
+            .values('character_id', 'corporation_id', 'alliance_id', 'character_name')  
+        
+        for character in characters:
+            character_id = character.get('character_id')
+            if character_id in affiliations:
+                affiliation = affiliations[character_id]
+                
+                corp_changed = (
+                    character.get('corporation_id') != affiliation.get('corporation_id')
+                )
+                
+                alliance_id = character.get('alliance_id')
+                if not alliance_id:
+                    alliance_id = None
+                alliance_changed = alliance_id != affiliation.get('alliance_id')
+                
+                name_changed = False
+                fetched_name = affiliation.get('name', False)
+                if fetched_name:
+                    name_changed = character.get('character_name') != fetched_name
+                
+                if corp_changed or alliance_changed or name_changed:  
+                    update_character.apply_async(
+                        args=[character.get('character_id')], priority=TASK_PRIORITY
+                    )

allianceauth/eveonline/tests/test_tasks.py

@@ -1,4 +1,4 @@
-from unittest.mock import patch
+from unittest.mock import patch, Mock
 
 from django.test import TestCase
 
@@ -44,55 +44,202 @@ class TestTasks(TestCase):
             mock_EveCharacter.objects.update_character.call_args[0][0], 42
         )
 
-    @patch('allianceauth.eveonline.tasks.update_character')
-    @patch('allianceauth.eveonline.tasks.update_alliance')
-    @patch('allianceauth.eveonline.tasks.update_corp')
-    def test_run_model_update(
-        self,
-        mock_update_corp,
-        mock_update_alliance,
-        mock_update_character,
-    ):
+
+@patch('allianceauth.eveonline.tasks.update_character')
+@patch('allianceauth.eveonline.tasks.update_alliance')
+@patch('allianceauth.eveonline.tasks.update_corp')
+@patch('allianceauth.eveonline.providers.provider')
+@patch('allianceauth.eveonline.tasks.CHUNK_SIZE', 2)
+class TestRunModelUpdate(TestCase):
+
+    @classmethod
+    def setUpClass(cls):
+        super().setUpClass()
         EveCorporationInfo.objects.all().delete()
         EveAllianceInfo.objects.all().delete()
         EveCharacter.objects.all().delete()
-
+        
         EveCorporationInfo.objects.create(
             corporation_id=2345,
             corporation_name='corp.name',
-            corporation_ticker='corp.ticker',
+            corporation_ticker='c.c.t',
             member_count=10,
             alliance=None,
         )
         EveAllianceInfo.objects.create(
             alliance_id=3456,
             alliance_name='alliance.name',
-            alliance_ticker='alliance.ticker',
-            executor_corp_id='78910',
+            alliance_ticker='a.t',
+            executor_corp_id=5,
+        )       
+        EveCharacter.objects.create(
+            character_id=1,
+            character_name='character.name1',
+            corporation_id=2345,
+            corporation_name='character.corp.name',
+            corporation_ticker='c.c.t',  # max 5 chars 
+            alliance_id=None
         )
         EveCharacter.objects.create(
-            character_id=1234,
-            character_name='character.name',
-            corporation_id=2345,
+            character_id=2,
+            character_name='character.name2',
+            corporation_id=9876,
             corporation_name='character.corp.name',
             corporation_ticker='c.c.t',  # max 5 chars 
             alliance_id=3456,
             alliance_name='character.alliance.name',
         )
+        EveCharacter.objects.create(
+            character_id=3,
+            character_name='character.name3',
+            corporation_id=9876,
+            corporation_name='character.corp.name',
+            corporation_ticker='c.c.t',  # max 5 chars 
+            alliance_id=3456,
+            alliance_name='character.alliance.name',
+        )
+        EveCharacter.objects.create(
+            character_id=4,
+            character_name='character.name4',
+            corporation_id=9876,
+            corporation_name='character.corp.name',
+            corporation_ticker='c.c.t',  # max 5 chars 
+            alliance_id=3456,
+            alliance_name='character.alliance.name',
+        )
+        """
+        EveCharacter.objects.create(
+            character_id=5,
+            character_name='character.name5',
+            corporation_id=9876,
+            corporation_name='character.corp.name',
+            corporation_ticker='c.c.t',  # max 5 chars 
+            alliance_id=3456,
+            alliance_name='character.alliance.name',
+        )
+        """      
+        
+    def setUp(self):
+        self.affiliations = [
+            {'character_id': 1, 'corporation_id': 5},
+            {'character_id': 2, 'corporation_id': 9876, 'alliance_id': 3456},
+            {'character_id': 3, 'corporation_id': 9876, 'alliance_id': 7456},
+            {'character_id': 4, 'corporation_id': 9876, 'alliance_id': 3456}
+        ]
+        self.names = [
+            {'id': 1, 'name': 'character.name1'},
+            {'id': 2, 'name': 'character.name2'},
+            {'id': 3, 'name': 'character.name3'},
+            {'id': 4, 'name': 'character.name4_new'}
+        ]
 
+    def test_normal_run(
+        self,
+        mock_provider,
+        mock_update_corp,
+        mock_update_alliance,
+        mock_update_character,
+    ):
+        def get_affiliations(characters: list):
+            response = [x for x in self.affiliations if x['character_id'] in characters]
+            mock_operator = Mock(**{'result.return_value': response})
+            return mock_operator
+
+        def get_names(ids: list):
+            response = [x for x in self.names if x['id'] in ids]
+            mock_operator = Mock(**{'result.return_value': response})
+            return mock_operator
+
+        mock_provider.client.Character.post_characters_affiliation.side_effect \
+            = get_affiliations
+       
+        mock_provider.client.Universe.post_universe_names.side_effect = get_names
+        
         run_model_update()
+        
+        self.assertEqual(
+            mock_provider.client.Character.post_characters_affiliation.call_count, 2
+        )        
+        self.assertEqual(
+            mock_provider.client.Universe.post_universe_names.call_count, 2
+        )
 
+        # character 1 has changed corp
+        # character 2 no change
+        # character 3 has changed alliance
+        # character 4 has changed name
         self.assertEqual(mock_update_corp.apply_async.call_count, 1)
         self.assertEqual(
             int(mock_update_corp.apply_async.call_args[1]['args'][0]), 2345
         )
-        
         self.assertEqual(mock_update_alliance.apply_async.call_count, 1)
         self.assertEqual(
             int(mock_update_alliance.apply_async.call_args[1]['args'][0]), 3456
-        )
+        )        
+        characters_updated = {
+            x[1]['args'][0] for x in mock_update_character.apply_async.call_args_list
+        }
+        excepted = {1, 3, 4}
+        self.assertSetEqual(characters_updated, excepted)
+
+    def test_ignore_character_not_in_affiliations(
+        self,
+        mock_provider,
+        mock_update_corp,
+        mock_update_alliance,
+        mock_update_character,
+    ):
+        def get_affiliations(characters: list):
+            response = [x for x in self.affiliations if x['character_id'] in characters]
+            mock_operator = Mock(**{'result.return_value': response})
+            return mock_operator
+
+        def get_names(ids: list):
+            response = [x for x in self.names if x['id'] in ids]
+            mock_operator = Mock(**{'result.return_value': response})
+            return mock_operator
+
+        del self.affiliations[0]
+
+        mock_provider.client.Character.post_characters_affiliation.side_effect \
+            = get_affiliations
+       
+        mock_provider.client.Universe.post_universe_names.side_effect = get_names
         
-        self.assertEqual(mock_update_character.apply_async.call_count, 1)
-        self.assertEqual(
-            int(mock_update_character.apply_async.call_args[1]['args'][0]), 1234
-        )
+        run_model_update()
+        characters_updated = {
+            x[1]['args'][0] for x in mock_update_character.apply_async.call_args_list
+        }
+        excepted = {3, 4}
+        self.assertSetEqual(characters_updated, excepted)
+
+    def test_ignore_character_not_in_names(
+        self,
+        mock_provider,
+        mock_update_corp,
+        mock_update_alliance,
+        mock_update_character,
+    ):
+        def get_affiliations(characters: list):
+            response = [x for x in self.affiliations if x['character_id'] in characters]
+            mock_operator = Mock(**{'result.return_value': response})
+            return mock_operator
+
+        def get_names(ids: list):
+            response = [x for x in self.names if x['id'] in ids]
+            mock_operator = Mock(**{'result.return_value': response})
+            return mock_operator
+
+        del self.names[3]
+
+        mock_provider.client.Character.post_characters_affiliation.side_effect \
+            = get_affiliations
+       
+        mock_provider.client.Universe.post_universe_names.side_effect = get_names
+        
+        run_model_update()
+        characters_updated = {
+            x[1]['args'][0] for x in mock_update_character.apply_async.call_args_list
+        }
+        excepted = {1, 3}
+        self.assertSetEqual(characters_updated, excepted)

allianceauth/groupmanagement/templates/groupmanagement/audit.html

@@ -33,7 +33,7 @@
                             <tbody>
                                 {% for entry in entries %}
                                     <tr>
-                                        <td class="text-center">{{ entry.date|date:"Y-M-d H:i" }}</td>
+                                        <td class="text-center">{{ entry.date|date:"Y-M-d, H:i" }}</td>
                                         <td class="text-center">{{ entry.requestor }}</td>
                                         <td class="text-center">{{ entry.req_char }}</td>
                                         <td class="text-center">{{ entry.req_char.corporation_name }}</td>
@@ -66,6 +66,7 @@
 
 {% block extra_javascript %}
     {% include 'bundles/datatables-js.html' %}
+    {% include 'bundles/moment-js.html' %}
     <script type="text/javascript" src="{% static 'js/filterDropDown/filterDropDown.min.js' %}"></script>
 {% endblock %}
 
@@ -74,7 +75,26 @@
 {% endblock %}
 
 {% block extra_script %}
+
+    $.fn.dataTable.moment = function ( format, locale ) {
+        var types = $.fn.dataTable.ext.type;
+    
+        // Add type detection
+        types.detect.unshift( function ( d ) {
+            return moment( d, format, locale, true ).isValid() ?
+                'moment-'+format :
+                null;
+        } );
+    
+        // Add sorting method - use an integer for the sorting
+        types.order[ 'moment-'+format+'-pre' ] = function ( d ) {
+            return moment( d, format, locale, true ).unix();
+        };
+    };
+
     $(document).ready(function(){
+        $.fn.dataTable.moment( 'YYYY-MMM-D, HH:mm' );
+
         $('#log-entries').DataTable({
             order: [[ 0, 'desc' ], [ 1, 'asc' ] ],
             filterDropDown:

allianceauth/services/modules/discord/discord_client/app_settings.py

@@ -3,7 +3,7 @@ from ..utils import clean_setting
 
 # Base URL for all API calls. Must end with /.
 DISCORD_API_BASE_URL = clean_setting(
-    'DISCORD_API_BASE_URL', 'https://discordapp.com/api/'
+    'DISCORD_API_BASE_URL', 'https://discord.com/api/'
 )
 
 # Low level connecttimeout for requests to the Discord API in seconds
@@ -18,12 +18,12 @@ DISCORD_API_TIMEOUT_READ = clean_setting(
 
 # Base authorization URL for Discord Oauth
 DISCORD_OAUTH_BASE_URL = clean_setting(
-    'DISCORD_OAUTH_BASE_URL', 'https://discordapp.com/api/oauth2/authorize'
+    'DISCORD_OAUTH_BASE_URL', 'https://discord.com/api/oauth2/authorize'
 )
 
 # Base authorization URL for Discord Oauth
 DISCORD_OAUTH_TOKEN_URL = clean_setting(
-    'DISCORD_OAUTH_TOKEN_URL', 'https://discordapp.com/api/oauth2/token'
+    'DISCORD_OAUTH_TOKEN_URL', 'https://discord.com/api/oauth2/token'
 )
 
 # How long the Discord guild names retrieved from the server are 

allianceauth/services/modules/discord/discord_client/tests/test_client.py

@@ -33,7 +33,7 @@ logger = set_logger_to_file(
 )
 
 MODULE_PATH = 'allianceauth.services.modules.discord.discord_client.client'
-API_BASE_URL = 'https://discordapp.com/api/'
+API_BASE_URL = 'https://discord.com/api/'
 
 TEST_RETRY_AFTER = 3000
 

allianceauth/services/tasks.py

@@ -5,7 +5,6 @@ from django.contrib.auth.models import User
 from .hooks import ServicesHook
 from celery_once import QueueOnce as BaseTask, AlreadyQueued
 from django.core.cache import cache
-from time import time
 
 
 logger = logging.getLogger(__name__)
@@ -22,14 +21,9 @@ class DjangoBackend:
 
     @staticmethod
     def raise_or_lock(key, timeout):
-        now = int(time())
-        result = cache.get(key)
-        if result:
-            remaining = int(result) - now
-            if remaining > 0:
-                raise AlreadyQueued(remaining)
-        else:
-            cache.set(key, now + timeout, timeout)
+        acquired = cache.add(key=key, value="lock", timeout=timeout)
+        if not acquired:            
+            raise AlreadyQueued(int(cache.ttl(key)))
 
     @staticmethod
     def clear_lock(key):

allianceauth/services/tests/test_tasks.py

@@ -1,11 +1,15 @@
 from unittest import mock
 
+from celery_once import AlreadyQueued
+
+from django.core.cache import cache
 from django.test import TestCase
 
 from allianceauth.tests.auth_utils import AuthUtils
-
 from allianceauth.services.tasks import validate_services
 
+from ..tasks import DjangoBackend
+
 
 class ServicesTasksTestCase(TestCase):
     def setUp(self):
@@ -24,3 +28,46 @@ class ServicesTasksTestCase(TestCase):
         self.assertTrue(svc.validate_user.called)
         args, kwargs = svc.validate_user.call_args
         self.assertEqual(self.member, args[0])  # Assert correct user is passed to service hook function
+
+
+class TestDjangoBackend(TestCase):
+
+    TEST_KEY = "my-django-backend-test-key"
+    TIMEOUT = 1800
+    
+    def setUp(self) -> None:
+        cache.delete(self.TEST_KEY)
+        self.backend = DjangoBackend(dict())
+    
+    def test_can_get_lock(self):
+        """
+        when lock can be acquired 
+        then set it with timetout
+        """
+        self.backend.raise_or_lock(self.TEST_KEY, self.TIMEOUT)
+        self.assertIsNotNone(cache.get(self.TEST_KEY))
+        self.assertAlmostEqual(cache.ttl(self.TEST_KEY), self.TIMEOUT, delta=2)
+        
+    def test_when_cant_get_lock_raise_exception(self):
+        """
+        when lock can bot be acquired 
+        then raise AlreadyQueued exception with countdown
+        """
+        self.backend.raise_or_lock(self.TEST_KEY, self.TIMEOUT)
+
+        try: 
+            self.backend.raise_or_lock(self.TEST_KEY, self.TIMEOUT)
+        except Exception as ex:
+            self.assertIsInstance(ex, AlreadyQueued)
+            self.assertAlmostEqual(ex.countdown, self.TIMEOUT, delta=2)
+
+    def test_can_clear_lock(self):
+        """
+        when a lock exists
+        then can get a new lock after clearing it 
+        """
+        self.backend.raise_or_lock(self.TEST_KEY, self.TIMEOUT)
+
+        self.backend.clear_lock(self.TEST_KEY)
+        self.backend.raise_or_lock(self.TEST_KEY, self.TIMEOUT)
+        self.assertIsNotNone(cache.get(self.TEST_KEY))

docs/features/services/discord.md

@@ -37,11 +37,11 @@ CELERYBEAT_SCHEDULE['discord.update_all_usernames'] = {
 
 ### Creating a Server
 
-Navigate to the [Discord site](https://discordapp.com/) and register an account, or log in if you have one already.
+Navigate to the [Discord site](https://discord.com/) and register an account, or log in if you have one already.
 
 On the left side of the screen you’ll see a circle with a plus sign. This is the button to create a new server. Go ahead and do that, naming it something obvious.
 
-Now retrieve the server ID [following this procedure.](https://support.discordapp.com/hc/en-us/articles/206346498-Where-can-I-find-my-User-Server-Message-ID-)
+Now retrieve the server ID [following this procedure.](https://support.discord.com/hc/en-us/articles/206346498-Where-can-I-find-my-User-Server-Message-ID-)
 
 Update your auth project's settings file, inputting the server ID as `DISCORD_GUILD_ID`
 
@@ -52,7 +52,7 @@ Update your auth project's settings file, inputting the server ID as `DISCORD_GU
 
 ### Registering an Application
 
-Navigate to the [Discord Developers site.](https://discordapp.com/developers/applications/me) Press the plus sign to create a new application.
+Navigate to the [Discord Developers site.](https://discord.com/developers/applications/me) Press the plus sign to create a new application.
 
 Give it a name and description relating to your auth site. Add a redirect to `https://example.com/discord/callback/`, substituting your domain. Press Create Application.
 
@@ -76,7 +76,7 @@ Once created, navigate to the services page of your Alliance Auth install as the
 
 This adds a new user to your Discord server with a `BOT` tag, and a new role with the same name as your Discord application. Don't touch either of these. If for some reason the bot loses permissions or is removed from the server, click this button again.
 
-To manage roles, this bot role must be at the top of the hierarchy. Edit your Discord server, roles, and click and drag the role with the same name as your application to the top of the list. This role must stay at the top of the list for the bot to work.  Finally, the owner of the bot account must enable 2 Factor Authentication (this is required from Discord for kicking and modifying member roles).  If you are unsure what 2FA is or how to set it up, refer to [this support page](https://support.discordapp.com/hc/en-us/articles/219576828).  It is also recommended to force 2FA on your server (this forces any admins or moderators to have 2fa enabled to perform similar functions on discord).
+To manage roles, this bot role must be at the top of the hierarchy. Edit your Discord server, roles, and click and drag the role with the same name as your application to the top of the list. This role must stay at the top of the list for the bot to work.  Finally, the owner of the bot account must enable 2 Factor Authentication (this is required from Discord for kicking and modifying member roles).  If you are unsure what 2FA is or how to set it up, refer to [this support page](https://support.discord.com/hc/en-us/articles/219576828).  It is also recommended to force 2FA on your server (this forces any admins or moderators to have 2fa enabled to perform similar functions on discord).
 
 Note that the bot will never appear online as it does not participate in chat channels.
 

docs/features/services/mumble.md

@@ -216,6 +216,48 @@ On a freshly installed mumble server only your superuser has the right to config
 - user: `SuperUser`
 - password: *what you defined when configuring your mumble server*
 
+## Optimizing a Mumble Server
+
+The needs and available resources will vary between Alliance Auth installations. Consider yours when applying these settings.
+
+### Bandwidth
+
+<https://wiki.mumble.info/wiki/Murmur.ini#bandwidth>
+This is likely the most important setting for scaling a Mumble install, The default maximum Bandwidth is 72000bps Per User. Reducing this value will cause your clients to automatically scale back their bandwidth transmitted, while causing a reduction in voice quality. A value thats still high may cause robotic voices or users with bad connections to drop due entirely due to network load.
+
+Please tune this value to your individual needs, the below scale may provide a rough starting point.
+72000 - Superior voice quality - Less than 50 users.
+54000 - No noticeable reduction in quality - 50+ Users or many channels with active audio.
+36000 - Mild reduction in quality - 100+ Users
+30000 - Noticeable reduction in quality but not function - 250+ Users
+
+### Forcing Opus
+
+<https://wiki.mumble.info/wiki/Murmur.ini#opusthreshold>
+A Mumble server by default, will fall back to the older CELT codec as soon as a single user connects with an old client. This will significantly reduce your audio quality and likely place higher load on your server. We _highly_ reccommend setting this to Zero, to force OPUS to be used at all times. Be aware any users with Mumble clients prior to 1.2.4 (From 2013...) Will not hear any audio.
+
+`opusthreshold=0`
+
+### AutoBan and Rate Limiting
+
+<https://wiki.mumble.info/wiki/Murmur.ini#autobanAttempts.2C_autobanTimeframe_and_autobanTime>
+The AutoBan feature has some sensible settings by default, You may wish to tune these if your users keep locking themselves out by opening two clients by mistake, or if you are receiving unwanted attention
+
+<https://wiki.mumble.info/wiki/Murmur.ini#messagelimit_and_messageburst>
+This too, is set to a sensible configuration by default. Take note on upgrading older installs, as this may actually be set too restrictively and will rate-limit your admins accidentally, take note of the configuration in <https://github.com/mumble-voip/mumble/blob/master/scripts/murmur.ini#L156>
+
+### "Suggest" Options
+
+There is no way to force your users to update their clients or use Push to Talk, but these options will throw an error into their Mumble Client.
+
+<https://wiki.mumble.info/wiki/Murmur.ini#Miscellany>
+
+We suggest using Mumble 1.3.0+ for your server and Clients, you can tune this to the latest Patch version.
+`suggestVersion=1.3.0`
+
+If Push to Talk is to your tastes, configure the suggestion as follows
+`suggestPushToTalk=true`
+
 ## General notes
 
 ### Setting a server password
@@ -238,6 +280,7 @@ Save the file and restart your mumble server afterwards.
 service mumble-server restart
 ```
 
-From now on, only registerd member can join your mumble server. Now if you still want to allow guests to join you have 2 options.
+From now on, only registered member can join your mumble server. Now if you still want to allow guests to join you have 2 options.
+
 - Allow the "Guest" state to activate the Mumble service in your Auth instance
-- Use [Mumble temporary links](https://github.com/pvyParts/allianceauth-mumble-temp)
+- Use [Mumble temporary links](https://github.com/pvyParts/allianceauth-mumble-temp)