Compare commits

..

No commits in common. "7033406ba6b526eb16bcb1b22b1a35756bdf1246" and "164cd4fbb2690b93a47413dbebae860306af0c94" have entirely different histories.

17 changed files with 41 additions and 127 deletions

View File

@ -5,7 +5,7 @@ manage online service access.
# This will make sure the app is always imported when
# Django starts so that shared_task will use this app.
__version__ = '4.5.0'
__version__ = '4.4.2'
__title__ = 'Alliance Auth'
__url__ = 'https://gitlab.com/allianceauth/allianceauth'
NAME = f'{__title__} v{__version__}'

View File

@ -1,5 +1,4 @@
import logging
from random import randint
from celery import shared_task
@ -10,8 +9,7 @@ from . import providers
logger = logging.getLogger(__name__)
TASK_PRIORITY = 7
CHARACTER_AFFILIATION_CHUNK_SIZE = 500
EVEONLINE_TASK_JITTER = 600
CHUNK_SIZE = 500
def chunks(lst, n):
@ -21,13 +19,13 @@ def chunks(lst, n):
@shared_task
def update_corp(corp_id: int) -> None:
def update_corp(corp_id):
"""Update given corporation from ESI"""
EveCorporationInfo.objects.update_corporation(corp_id)
@shared_task
def update_alliance(alliance_id: int) -> None:
def update_alliance(alliance_id):
"""Update given alliance from ESI"""
EveAllianceInfo.objects.update_alliance(alliance_id).populate_alliance()
@ -39,30 +37,23 @@ def update_character(character_id: int) -> None:
@shared_task
def run_model_update() -> None:
def run_model_update():
"""Update all alliances, corporations and characters from ESI"""
# Queue update tasks for Known Corporation Models
#update existing corp models
for corp in EveCorporationInfo.objects.all().values('corporation_id'):
update_corp.apply_async(
args=[corp['corporation_id']],
priority=TASK_PRIORITY,
countdown=randint(1, EVEONLINE_TASK_JITTER))
update_corp.apply_async(args=[corp['corporation_id']], priority=TASK_PRIORITY)
# Queue update tasks for Known Alliance Models
# update existing alliance models
for alliance in EveAllianceInfo.objects.all().values('alliance_id'):
update_alliance.apply_async(
args=[alliance['alliance_id']],
priority=TASK_PRIORITY,
countdown=randint(1, EVEONLINE_TASK_JITTER))
update_alliance.apply_async(args=[alliance['alliance_id']], priority=TASK_PRIORITY)
# Queue update tasks for Known Character Models
# update existing character models
character_ids = EveCharacter.objects.all().values_list('character_id', flat=True)
for character_ids_chunk in chunks(character_ids, CHARACTER_AFFILIATION_CHUNK_SIZE):
for character_ids_chunk in chunks(character_ids, CHUNK_SIZE):
update_character_chunk.apply_async(
args=[character_ids_chunk],
priority=TASK_PRIORITY,
countdown=randint(1, EVEONLINE_TASK_JITTER))
args=[character_ids_chunk], priority=TASK_PRIORITY
)
@shared_task
@ -77,9 +68,8 @@ def update_character_chunk(character_ids_chunk: list):
logger.info("Failed to bulk update characters. Attempting single updates")
for character_id in character_ids_chunk:
update_character.apply_async(
args=[character_id],
priority=TASK_PRIORITY,
countdown=randint(1, EVEONLINE_TASK_JITTER))
args=[character_id], priority=TASK_PRIORITY
)
return
affiliations = {
@ -117,5 +107,5 @@ def update_character_chunk(character_ids_chunk: list):
if corp_changed or alliance_changed or name_changed:
update_character.apply_async(
args=[character.get('character_id')],
priority=TASK_PRIORITY)
args=[character.get('character_id')], priority=TASK_PRIORITY
)

View File

@ -84,7 +84,7 @@ class TestUpdateTasks(TestCase):
@override_settings(CELERY_ALWAYS_EAGER=True)
@patch('allianceauth.eveonline.providers.esi_client_factory')
@patch('allianceauth.eveonline.tasks.providers')
@patch('allianceauth.eveonline.tasks.CHARACTER_AFFILIATION_CHUNK_SIZE', 2)
@patch('allianceauth.eveonline.tasks.CHUNK_SIZE', 2)
class TestRunModelUpdate(TransactionTestCase):
def test_should_run_updates(self, mock_providers, mock_esi_client_factory):
# given
@ -139,7 +139,7 @@ class TestRunModelUpdate(TransactionTestCase):
@patch('allianceauth.eveonline.tasks.update_character', wraps=update_character)
@patch('allianceauth.eveonline.providers.esi_client_factory')
@patch('allianceauth.eveonline.tasks.providers')
@patch('allianceauth.eveonline.tasks.CHARACTER_AFFILIATION_CHUNK_SIZE', 2)
@patch('allianceauth.eveonline.tasks.CHUNK_SIZE', 2)
class TestUpdateCharacterChunk(TestCase):
@staticmethod
def _updated_character_ids(spy_update_character) -> set:

View File

@ -13,15 +13,6 @@
}
}
/* Image overflow fix
------------------------------------------------------------------------------------- */
@media all {
img {
max-width: 100%;
height: auto;
}
}
/* Side Navigation
------------------------------------------------------------------------------------- */
@media all {

View File

@ -56,7 +56,7 @@
{% endif %}
{% endfor %}
{% endif %}
{% if g.group.authgroup.group_leader_groups.all.count %}
{% if g.group.authgroup.group_leaders.all.count %}
{% for group in g.group.authgroup.group_leader_groups.all %}
<span class="my-1 me-1 badge bg-secondary">{{group.name}}</span>
{% endfor %}

View File

@ -8,16 +8,14 @@ class ThemeHook:
"""
def __init__(self,
name: str,
description: str,
css: List[dict],
js: List[dict],
css_template: Optional[str] = None,
js_template: Optional[str] = None,
js_type: Optional[str] = None,
html_tags: Optional[Union[dict, str]] = None,
header_padding: Optional[str] = "4em"
):
name: str,
description: str,
css: List[dict],
js: List[dict],
css_template: Optional[str] = None,
js_template: Optional[str] = None,
html_tags: Optional[Union[dict, str]] = None,
header_padding: Optional[str] = "4em"):
"""
:param name: Theme python name
:type name: str
@ -31,14 +29,11 @@ class ThemeHook:
:type css_template: Optional[str], optional
:param js_template: _description_, defaults to None
:type js_template: Optional[str], optional
:param js_type: The type of the JS (e.g.: 'module'), defaults to None
:type js_type: Optional[str], optional
:param html_tags: Attributes added to the `<html>` tag, defaults to None
:type html_tags: Optional[dict|str], optional
:param header_padding: Top padding, defaults to "4em"
:type header_padding: Optional[str], optional
"""
self.name = name
self.description = description
@ -50,15 +45,11 @@ class ThemeHook:
self.css_template = css_template
self.js_template = js_template
# Define the JS type (e.g.: 'module')
self.js_type = js_type
self.html_tags = (
" ".join([f"{key}={value}" for key, value in html_tags.items()])
if isinstance(html_tags, dict)
else html_tags
)
self.header_padding = header_padding
def get_name(self):
return f"{self.__class__.__module__}.{self.__class__.__name__}"

View File

@ -4,13 +4,7 @@
{% include theme.js_template %}
{% else %}
{% for x in theme.js %}
<script
{% if x.js_type %}type="{{ x.js_type }}"{% endif %}
src="{{ x.url }}"
integrity="{{ x.integrity }}"
crossorigin="anonymous"
referrerpolicy="no-referrer"
></script>
<script src="{{ x.url }}" integrity="{{ x.integrity }}" crossorigin="anonymous" referrerpolicy="no-referrer"></script>
{% endfor %}
{% endif %}
<!-- allianceauth.theme.{{ theme.name }} JS Ends-->

View File

@ -1,7 +1,7 @@
PROTOCOL=https://
AUTH_SUBDOMAIN=%AUTH_SUBDOMAIN%
DOMAIN=%DOMAIN%
AA_DOCKER_TAG=registry.gitlab.com/allianceauth/allianceauth/auth:v4.5.0
AA_DOCKER_TAG=registry.gitlab.com/allianceauth/allianceauth/auth:v4.4.2
# Nginx Proxy Manager
PROXY_HTTP_PORT=80

View File

@ -1,5 +1,5 @@
FROM python:3.11-slim
ARG AUTH_VERSION=v4.5.0
ARG AUTH_VERSION=v4.4.2
ARG AUTH_PACKAGE=allianceauth==${AUTH_VERSION}
ENV AUTH_USER=allianceauth
ENV AUTH_GROUP=allianceauth

View File

@ -1,6 +0,0 @@
[mariadb]
# Provided as an Example
# AA Doesnt use Aria or MyISAM, So these are worth Considering
# aria_pagecache_buffer_size = 16M
# key_buffer_size = 16M

0
docker/conf/redis_healthcheck.sh Executable file → Normal file
View File

View File

@ -49,7 +49,6 @@ services:
volumes:
- ./mysql-data:/var/lib/mysql
- ./setup.sql:/docker-entrypoint-initdb.d/setup.sql
- ./conf/aa_mariadb.cnf:/etc/mysql/conf.d/aa_mariadb.cnf
environment:
- MYSQL_ROOT_PASSWORD=${AA_DB_ROOT_PASSWORD?err}
- MARIADB_MYSQL_LOCALHOST_USER=1
@ -84,7 +83,7 @@ services:
- "redis-data:/data"
- ./conf/redis_healthcheck.sh:/usr/local/bin/redis_healthcheck.sh
healthcheck:
test: ["CMD", "/usr/local/bin/redis_healthcheck.sh"]
test: ["CMD", "bash", "/usr/local/bin/redis_healthcheck.sh"]
logging:
driver: "json-file"
options:

View File

@ -1,6 +1,4 @@
#!/bin/bash
git clone https://gitlab.com/allianceauth/allianceauth.git aa-git
cp -R aa-git/docker ./aa-docker
chmod +x aa-docker/conf/memory_check.sh
chmod +x aa-docker/conf/redis_healthcheck.sh
rm -rf aa-git

View File

@ -1,6 +1,6 @@
CREATE USER 'aauth'@'%' IDENTIFIED BY 'authpass';
CREATE USER 'grafana'@'%' IDENTIFIED BY 'grafanapass';
CREATE DATABASE alliance_auth CHARACTER SET utf8mb4 DEFAULT COLLATE utf8mb4_unicode_ci;
CREATE DATABASE alliance_auth CHARACTER SET utf8mb4;
GRANT ALL PRIVILEGES ON alliance_auth.* TO 'aauth'@'%';
GRANT
SELECT,

View File

@ -40,10 +40,10 @@ Please use the following approach to ensure your tasks are working properly with
Here is an example implementation of a task:
```python
from allianceauth.services.hooks import get_extension_logger
import logging
from celery import shared_task
logger = get_extension_logger(__name__)
logger = logging.getLogger(__name__)
@shared_task
@ -80,10 +80,10 @@ However, many long-running tasks consist of several smaller processes that need
Example implementation for a celery chain:
```python
from allianceauth.services.hooks import get_extension_logger
import logging
from celery import shared_task, chain
logger = get_extension_logger(__name__)
logger = logging.getLogger(__name__)
@shared_task
@ -96,7 +96,7 @@ def long_runner():
my_tasks = list()
for _ in range(10):
task_signature = example.si()
my_tasks.append(task_signature)
my_task.append(task_signature)
chain(my_tasks).delay()
```
@ -168,49 +168,6 @@ example.apply_async(priority=3)
For defining a priority to tasks, you cannot use the convenient shortcut ``delay()``, but instead need to start a task with ``apply_async()``, which also requires you to pass parameters to your task function differently. Please check out the `official docs <https://docs.celeryproject.org/en/stable/reference/celery.app.task.html#celery.app.task.Task.apply_async>`_ for details.
:::
## Rate-Limiting and Smoothing of Task Execution
Large numbers of installs running the same crontab (ie. `0 * * * *`) can all slam an external service at the same time.
Consider Artificially smoothing out your tasks with a few methods
### Offset Crontabs
Avoid running your tasks on the hour or other nice neat human numbers, consider 23 minutes on the hour instead of at zero (`28 * * * *`)
### Subset Tasks
Slice your tasks needed up into more manageable chunks and run them more often. 1/10th of your tasks run 10x more often will return the same end result with less peak loads on external services and your task queue.
### Celery ETA/Countdown
Scatter your tasks across a larger window using <https://docs.celeryq.dev/en/latest/userguide/calling.html#eta-and-countdown>
This example will queue up tasks across the next 10 minutes, trickling them into your workers (and the external service)
```python
for corp in EveCorporationInfo.objects.all().values('corporation_id'):
update_corp.apply_async(args=[corp['corporation_id']], priority=TASK_PRIORITY)
update_corp.apply_async(
args=[corp['corporation_id']],
priority=TASK_PRIORITY,
countdown=randint(1, 600))
```
### Celery Rate Limits
Celery Rate Limits come with a small catch, its _per worker_, you may have to be either very conservative or have these configurable by the end user if they varied their worker count.
<https://docs.celeryq.dev/en/latest/userguide/tasks.html#Task.rate_limit>
This example of 10 Tasks per Minute will result in ~100 tasks per minute at 10 Workers
```python
@shared_task(rate_limit="10/m")
def update_charactercorporationhistory(character_id: int) -> None:
"""Update CharacterCorporationHistory models from ESI"""
```
## What special features should I be aware of?
Every Alliance Auth installation will come with a couple of special celery related features "out-of-the-box" that you can make use of in your apps.
@ -235,6 +192,6 @@ You can use it like so:
Please see the [official documentation](https://pypi.org/project/celery_once/) of celery-once for details.
### Task Priorities
### task priorities
Alliance Auth is using task priorities to enable priority-based scheduling of task execution. Please see [How can I use priorities for tasks?](#how-can-i-use-priorities-for-tasks) for details.

View File

@ -65,7 +65,7 @@ Using a custom docker image is the preferred approach, as it gives you the stabi
* e.g.
```docker
x-allianceauth-base: &allianceauth-base
x-allianceauth-base: &allianceauth-base
# image: ${AA_DOCKER_TAG?err}
build:
context: .

View File

@ -246,7 +246,7 @@ and create them as follows, replacing `PASSWORD` with an actual secure password:
```sql
CREATE USER 'allianceserver'@'localhost' IDENTIFIED BY 'PASSWORD';
CREATE DATABASE alliance_auth CHARACTER SET utf8mb4 DEFAULT COLLATE utf8mb4_unicode_ci;
CREATE DATABASE alliance_auth CHARACTER SET utf8mb4;
GRANT ALL PRIVILEGES ON alliance_auth . * TO 'allianceserver'@'localhost';
```