Version Bump 4.5.0

Executableflag

See merge request allianceauth/allianceauth!1667

allianceauth/__init__.py

@@ -5,7 +5,7 @@ manage online service access.
 # This will make sure the app is always imported when
 # Django starts so that shared_task will use this app.
 
-__version__ = '4.4.2'
+__version__ = '4.5.0'
 __title__ = 'Alliance Auth'
 __url__ = 'https://gitlab.com/allianceauth/allianceauth'
 NAME = f'{__title__} v{__version__}'

allianceauth/eveonline/tasks.py

@@ -1,4 +1,5 @@
 import logging
+from random import randint
 
 from celery import shared_task
 
@@ -9,7 +10,8 @@ from . import providers
 logger = logging.getLogger(__name__)
 
 TASK_PRIORITY = 7
-CHUNK_SIZE = 500
+CHARACTER_AFFILIATION_CHUNK_SIZE = 500
+EVEONLINE_TASK_JITTER = 600
 
 
 def chunks(lst, n):
@@ -19,13 +21,13 @@ def chunks(lst, n):
 
 
 @shared_task
-def update_corp(corp_id):
+def update_corp(corp_id: int) -> None:
     """Update given corporation from ESI"""
     EveCorporationInfo.objects.update_corporation(corp_id)
 
 
 @shared_task
-def update_alliance(alliance_id):
+def update_alliance(alliance_id: int) -> None:
     """Update given alliance from ESI"""
     EveAllianceInfo.objects.update_alliance(alliance_id).populate_alliance()
 
@@ -37,23 +39,30 @@ def update_character(character_id: int) -> None:
 
 
 @shared_task
-def run_model_update():
+def run_model_update() -> None:
     """Update all alliances, corporations and characters from ESI"""
 
-    #update existing corp models
+    # Queue update tasks for Known Corporation Models
     for corp in EveCorporationInfo.objects.all().values('corporation_id'):
-        update_corp.apply_async(args=[corp['corporation_id']], priority=TASK_PRIORITY)
+        update_corp.apply_async(
+            args=[corp['corporation_id']],
+            priority=TASK_PRIORITY,
+            countdown=randint(1, EVEONLINE_TASK_JITTER))
 
-    # update existing alliance models
+    # Queue update tasks for Known Alliance Models
     for alliance in EveAllianceInfo.objects.all().values('alliance_id'):
-        update_alliance.apply_async(args=[alliance['alliance_id']], priority=TASK_PRIORITY)
+        update_alliance.apply_async(
+            args=[alliance['alliance_id']],
+            priority=TASK_PRIORITY,
+            countdown=randint(1, EVEONLINE_TASK_JITTER))
 
-    # update existing character models
+    # Queue update tasks for Known Character Models
     character_ids = EveCharacter.objects.all().values_list('character_id', flat=True)
-    for character_ids_chunk in chunks(character_ids, CHUNK_SIZE):
+    for character_ids_chunk in chunks(character_ids, CHARACTER_AFFILIATION_CHUNK_SIZE):
         update_character_chunk.apply_async(
-            args=[character_ids_chunk], priority=TASK_PRIORITY
-        )
+            args=[character_ids_chunk],
+            priority=TASK_PRIORITY,
+            countdown=randint(1, EVEONLINE_TASK_JITTER))
 
 
 @shared_task
@@ -68,8 +77,9 @@ def update_character_chunk(character_ids_chunk: list):
         logger.info("Failed to bulk update characters. Attempting single updates")
         for character_id in character_ids_chunk:
             update_character.apply_async(
-                args=[character_id], priority=TASK_PRIORITY
-            )
+                args=[character_id],
+                priority=TASK_PRIORITY,
+                countdown=randint(1, EVEONLINE_TASK_JITTER))
         return
 
     affiliations = {
@@ -107,5 +117,5 @@ def update_character_chunk(character_ids_chunk: list):
 
             if corp_changed or alliance_changed or name_changed:
                 update_character.apply_async(
-                    args=[character.get('character_id')], priority=TASK_PRIORITY
-                )
+                    args=[character.get('character_id')],
+                    priority=TASK_PRIORITY)

allianceauth/eveonline/tests/test_tasks.py

@@ -84,7 +84,7 @@ class TestUpdateTasks(TestCase):
 @override_settings(CELERY_ALWAYS_EAGER=True)
 @patch('allianceauth.eveonline.providers.esi_client_factory')
 @patch('allianceauth.eveonline.tasks.providers')
-@patch('allianceauth.eveonline.tasks.CHUNK_SIZE', 2)
+@patch('allianceauth.eveonline.tasks.CHARACTER_AFFILIATION_CHUNK_SIZE', 2)
 class TestRunModelUpdate(TransactionTestCase):
     def test_should_run_updates(self, mock_providers, mock_esi_client_factory):
         # given
@@ -139,7 +139,7 @@ class TestRunModelUpdate(TransactionTestCase):
 @patch('allianceauth.eveonline.tasks.update_character', wraps=update_character)
 @patch('allianceauth.eveonline.providers.esi_client_factory')
 @patch('allianceauth.eveonline.tasks.providers')
-@patch('allianceauth.eveonline.tasks.CHUNK_SIZE', 2)
+@patch('allianceauth.eveonline.tasks.CHARACTER_AFFILIATION_CHUNK_SIZE', 2)
 class TestUpdateCharacterChunk(TestCase):
     @staticmethod
     def _updated_character_ids(spy_update_character) -> set:

allianceauth/framework/static/allianceauth/framework/css/auth-framework.css

@@ -13,6 +13,15 @@
     }
 }
 
+/* Image overflow fix
+------------------------------------------------------------------------------------- */
+@media all {
+    img {
+        max-width: 100%;
+        height: auto;
+    }
+}
+
 /* Side Navigation
 ------------------------------------------------------------------------------------- */
 @media all {

allianceauth/groupmanagement/templates/groupmanagement/groups.html

@@ -56,7 +56,7 @@
                                     {% endif %}
                                 {% endfor %}
                             {% endif %}
-                            {% if g.group.authgroup.group_leaders.all.count %}
+                            {% if g.group.authgroup.group_leader_groups.all.count %}
                                 {% for group in g.group.authgroup.group_leader_groups.all %}
                                     <span class="my-1 me-1 badge bg-secondary">{{group.name}}</span>
                                 {% endfor %}

allianceauth/theme/hooks.py

@@ -8,14 +8,16 @@ class ThemeHook:
     """
 
     def __init__(self,
-                name: str,
-                description: str,
-                css: List[dict],
-                js: List[dict],
-                css_template: Optional[str] = None,
-                js_template: Optional[str] = None,
-                html_tags: Optional[Union[dict, str]] = None,
-                header_padding: Optional[str] = "4em"):
+        name: str,
+        description: str,
+        css: List[dict],
+        js: List[dict],
+        css_template: Optional[str] = None,
+        js_template: Optional[str] = None,
+        js_type: Optional[str] = None,
+        html_tags: Optional[Union[dict, str]] = None,
+        header_padding: Optional[str] = "4em"
+    ):
         """
         :param name: Theme python name
         :type name: str
@@ -29,11 +31,14 @@ class ThemeHook:
         :type css_template: Optional[str], optional
         :param js_template: _description_, defaults to None
         :type js_template: Optional[str], optional
+        :param js_type: The type of the JS (e.g.: 'module'), defaults to None
+        :type js_type: Optional[str], optional
         :param html_tags: Attributes added to the `<html>` tag, defaults to None
         :type html_tags: Optional[dict|str], optional
         :param header_padding: Top padding, defaults to "4em"
         :type header_padding: Optional[str], optional
         """
+
         self.name = name
         self.description = description
 
@@ -45,11 +50,15 @@ class ThemeHook:
         self.css_template = css_template
         self.js_template = js_template
 
+        # Define the JS type (e.g.: 'module')
+        self.js_type = js_type
+
         self.html_tags = (
             " ".join([f"{key}={value}" for key, value in html_tags.items()])
             if isinstance(html_tags, dict)
             else html_tags
         )
         self.header_padding = header_padding
+
     def get_name(self):
         return f"{self.__class__.__module__}.{self.__class__.__name__}"

allianceauth/theme/templates/theme/theme_imports_js.html

@@ -4,7 +4,13 @@
     {% include theme.js_template %}
 {% else %}
     {% for x in theme.js %}
-        <script src="{{ x.url }}" integrity="{{ x.integrity }}" crossorigin="anonymous" referrerpolicy="no-referrer"></script>
+        <script
+            {% if x.js_type %}type="{{ x.js_type }}"{% endif %}
+            src="{{ x.url }}"
+            integrity="{{ x.integrity }}"
+            crossorigin="anonymous"
+            referrerpolicy="no-referrer"
+        ></script>
     {% endfor %}
 {% endif %}
 <!-- allianceauth.theme.{{ theme.name }} JS Ends-->

docker/.env.example

@@ -1,7 +1,7 @@
 PROTOCOL=https://
 AUTH_SUBDOMAIN=%AUTH_SUBDOMAIN%
 DOMAIN=%DOMAIN%
-AA_DOCKER_TAG=registry.gitlab.com/allianceauth/allianceauth/auth:v4.4.2
+AA_DOCKER_TAG=registry.gitlab.com/allianceauth/allianceauth/auth:v4.5.0
 
 # Nginx Proxy Manager
 PROXY_HTTP_PORT=80

docker/Dockerfile

@@ -1,5 +1,5 @@
 FROM python:3.11-slim
-ARG AUTH_VERSION=v4.4.2
+ARG AUTH_VERSION=v4.5.0
 ARG AUTH_PACKAGE=allianceauth==${AUTH_VERSION}
 ENV AUTH_USER=allianceauth
 ENV AUTH_GROUP=allianceauth

docker/conf/aa_mariadb.cnf

@@ -0,0 +1,6 @@
+[mariadb]
+# Provided as an Example
+# AA Doesnt use Aria or MyISAM, So these are worth Considering
+
+# aria_pagecache_buffer_size = 16M
+# key_buffer_size = 16M

docker/docker-compose.yml

@@ -49,6 +49,7 @@ services:
     volumes:
       - ./mysql-data:/var/lib/mysql
       - ./setup.sql:/docker-entrypoint-initdb.d/setup.sql
+      - ./conf/aa_mariadb.cnf:/etc/mysql/conf.d/aa_mariadb.cnf
     environment:
       - MYSQL_ROOT_PASSWORD=${AA_DB_ROOT_PASSWORD?err}
       - MARIADB_MYSQL_LOCALHOST_USER=1
@@ -83,7 +84,7 @@ services:
       - "redis-data:/data"
       - ./conf/redis_healthcheck.sh:/usr/local/bin/redis_healthcheck.sh
     healthcheck:
-      test: ["CMD", "bash", "/usr/local/bin/redis_healthcheck.sh"]
+      test: ["CMD", "/usr/local/bin/redis_healthcheck.sh"]
     logging:
       driver: "json-file"
       options:

docker/scripts/download.sh

@@ -1,4 +1,6 @@
 #!/bin/bash
 git clone https://gitlab.com/allianceauth/allianceauth.git aa-git
 cp -R aa-git/docker ./aa-docker
+chmod +x aa-docker/conf/memory_check.sh
+chmod +x aa-docker/conf/redis_healthcheck.sh
 rm -rf aa-git

docker/setup.base.sql

@@ -1,6 +1,6 @@
 CREATE USER 'aauth'@'%' IDENTIFIED BY 'authpass';
 CREATE USER 'grafana'@'%' IDENTIFIED BY 'grafanapass';
-CREATE DATABASE alliance_auth CHARACTER SET utf8mb4;
+CREATE DATABASE alliance_auth CHARACTER SET utf8mb4 DEFAULT COLLATE utf8mb4_unicode_ci;
 GRANT ALL PRIVILEGES ON alliance_auth.* TO 'aauth'@'%';
 GRANT
 SELECT,

docs/development/tech_docu/celery.md

@@ -40,10 +40,10 @@ Please use the following approach to ensure your tasks are working properly with
 Here is an example implementation of a task:
 
 ```python
-import logging
+from allianceauth.services.hooks import get_extension_logger
 from celery import shared_task
 
-logger = logging.getLogger(__name__)
+logger = get_extension_logger(__name__)
 
 
 @shared_task
@@ -80,10 +80,10 @@ However, many long-running tasks consist of several smaller processes that need
 Example implementation for a celery chain:
 
 ```python
-import logging
+from allianceauth.services.hooks import get_extension_logger
 from celery import shared_task, chain
 
-logger = logging.getLogger(__name__)
+logger = get_extension_logger(__name__)
 
 
 @shared_task
@@ -96,7 +96,7 @@ def long_runner():
     my_tasks = list()
     for _ in range(10):
         task_signature = example.si()
-        my_task.append(task_signature)
+        my_tasks.append(task_signature)
 
     chain(my_tasks).delay()
 ```
@@ -168,6 +168,49 @@ example.apply_async(priority=3)
 For defining a priority to tasks, you cannot use the convenient shortcut ``delay()``, but instead need to start a task with ``apply_async()``, which also requires you to pass parameters to your task function differently. Please check out the `official docs <https://docs.celeryproject.org/en/stable/reference/celery.app.task.html#celery.app.task.Task.apply_async>`_ for details.
 :::
 
+## Rate-Limiting and Smoothing of Task Execution
+
+Large numbers of installs running the same crontab (ie. `0 * * * *`) can all slam an external service at the same time.
+
+Consider Artificially smoothing out your tasks with a few methods
+
+### Offset Crontabs
+
+Avoid running your tasks on the hour or other nice neat human numbers, consider 23 minutes on the hour instead of at zero (`28 * * * *`)
+
+### Subset Tasks
+
+Slice your tasks needed up into more manageable chunks and run them more often. 1/10th of your tasks run 10x more often will return the same end result with less peak loads on external services and your task queue.
+
+### Celery ETA/Countdown
+
+Scatter your tasks across a larger window using <https://docs.celeryq.dev/en/latest/userguide/calling.html#eta-and-countdown>
+
+This example will queue up tasks across the next 10 minutes, trickling them into your workers (and the external service)
+
+```python
+for corp in EveCorporationInfo.objects.all().values('corporation_id'):
+    update_corp.apply_async(args=[corp['corporation_id']], priority=TASK_PRIORITY)
+    update_corp.apply_async(
+        args=[corp['corporation_id']],
+        priority=TASK_PRIORITY,
+        countdown=randint(1, 600))
+```
+
+### Celery Rate Limits
+
+Celery Rate Limits come with a small catch, its _per worker_, you may have to be either very conservative or have these configurable by the end user if they varied their worker count.
+
+<https://docs.celeryq.dev/en/latest/userguide/tasks.html#Task.rate_limit>
+
+This example of 10 Tasks per Minute will result in ~100 tasks per minute at 10 Workers
+
+```python
+@shared_task(rate_limit="10/m")
+def update_charactercorporationhistory(character_id: int) -> None:
+    """Update CharacterCorporationHistory models from ESI"""
+```
+
 ## What special features should I be aware of?
 
 Every Alliance Auth installation will come with a couple of special celery related features "out-of-the-box" that you can make use of in your apps.
@@ -192,6 +235,6 @@ You can use it like so:
 
 Please see the [official documentation](https://pypi.org/project/celery_once/) of celery-once for details.
 
-### task priorities
+### Task Priorities
 
 Alliance Auth is using task priorities to enable priority-based scheduling of task execution. Please see [How can I use priorities for tasks?](#how-can-i-use-priorities-for-tasks) for details.

docs/installation-containerized/docker.md

@@ -65,7 +65,7 @@ Using a custom docker image is the preferred approach, as it gives you the stabi
     * e.g.
 
     ```docker
-      x-allianceauth-base: &allianceauth-base
+    x-allianceauth-base: &allianceauth-base
       # image: ${AA_DOCKER_TAG?err}
       build:
         context: .

docs/installation/allianceauth.md

@@ -246,7 +246,7 @@ and create them as follows, replacing `PASSWORD` with an actual secure password:
 
 ```sql
 CREATE USER 'allianceserver'@'localhost' IDENTIFIED BY 'PASSWORD';
-CREATE DATABASE alliance_auth CHARACTER SET utf8mb4;
+CREATE DATABASE alliance_auth CHARACTER SET utf8mb4 DEFAULT COLLATE utf8mb4_unicode_ci;
 GRANT ALL PRIVILEGES ON alliance_auth . * TO 'allianceserver'@'localhost';
 ```