Merge branch 'servicesworker' into 'v5.x'

Singlethread Services Worker

See merge request allianceauth/allianceauth!1698

.pre-commit-config.yaml

@@ -24,20 +24,14 @@ exclude: |
   )
 
 repos:
-  - repo: https://github.com/astral-sh/ruff-pre-commit
+  # Code Upgrades
-    rev: v0.11.4
-    hooks:
-      # Run the linter, and only the linter
-      - id: ruff
-
   - repo: https://github.com/adamchainz/django-upgrade
-    rev: 1.24.0
+    rev: 1.25.0
     hooks:
       - id: django-upgrade
-        args: [--target-version=5.1]
+        args: [--target-version=5.2]
-
+  - repo: https://github.com/asottile/pyupgrade
-  - repo: https://github.com/asottile/pyupgrade # Ruff doesnt get everything.
+    rev: v3.20.0
-    rev: v3.19.1
     hooks:
       - id: pyupgrade
         args: [--py310-plus]
@@ -59,7 +53,7 @@ repos:
       - id: detect-private-key
       - id: check-case-conflict
       # Python checks
-#
+#      - id: check-docstring-first
       - id: debug-statements
 #      - id: requirements-txt-fixer
       - id: fix-encoding-pragma
@@ -77,21 +71,22 @@ repos:
     hooks:
       - id: editorconfig-checker
   - repo: https://github.com/igorshubovych/markdownlint-cli
-    rev: v0.44.0
+    rev: v0.45.0
     hooks:
       - id: markdownlint
         language: node
         args:
           - --disable=MD013
+
   # Infrastructure
   - repo: https://github.com/tox-dev/pyproject-fmt
-    rev: v2.5.1
+    rev: v2.6.0
     hooks:
       - id: pyproject-fmt
         args:
           - --indent=4
         additional_dependencies:
-          - tox==4.24.1 # https://github.com/tox-dev/tox/releases/latest
+          - tox==4.26.0 # https://github.com/tox-dev/tox/releases/latest
   - repo: https://github.com/tox-dev/tox-ini-fmt
     rev: 1.5.0
     hooks:

allianceauth/__init__.py

@@ -5,7 +5,7 @@ manage online service access.
 # This will make sure the app is always imported when
 # Django starts so that shared_task will use this app.
 
-__version__ = '5.0.0a1'
+__version__ = '5.0.0a3'
 __title__ = 'AllianceAuth'
 __url__ = 'https://gitlab.com/allianceauth/allianceauth'
 NAME = f'{__title__} v{__version__}'

allianceauth/eveonline/providers.py

@@ -1,6 +1,7 @@
 import logging
 import os
 
+from bravado.client import SwaggerClient
 from bravado.exception import HTTPError, HTTPNotFound, HTTPUnprocessableEntity
 from jsonschema.exceptions import RefResolutionError
 
@@ -8,7 +9,7 @@ from django.conf import settings
 
 from esi.clients import esi_client_factory
 
-from allianceauth import __version__
+from allianceauth import __version__, __title__, __url__
 from allianceauth.utils.django import StartupCommand
 
 SWAGGER_SPEC_PATH = os.path.join(os.path.dirname(
@@ -175,7 +176,11 @@ class EveProvider:
 
 
 class EveSwaggerProvider(EveProvider):
-    def __init__(self, token=None, adapter=None):
+    def __init__(self, token=None, adapter=None) -> None:
+        self._token = token
+        self.adapter = adapter or self
+        self._faction_list = None  # what are the odds this will change? could cache forever!
+
         if settings.DEBUG or StartupCommand().is_management_command:
             self._client = None
             logger.info('ESI client will be loaded on-demand')
@@ -183,9 +188,10 @@ class EveSwaggerProvider(EveProvider):
             logger.info('Loading ESI client')
             try:
                 self._client = esi_client_factory(
-                    token=token,
+                    token=self._token,
-                    spec_file=SWAGGER_SPEC_PATH,
+                    ua_appname=__title__,
-                    app_info_text=f"allianceauth v{__version__}"
+                    ua_version=__version__,
+                    ua_url=__url__,
                 )
             except (HTTPError, RefResolutionError):
                 logger.exception(
@@ -194,15 +200,14 @@ class EveSwaggerProvider(EveProvider):
                 )
                 self._client = None
 
-        self._token = token
-        self.adapter = adapter or self
-        self._faction_list = None  # what are the odds this will change? could cache forever!
-
     @property
-    def client(self):
+    def client(self) -> SwaggerClient:
         if self._client is None:
             self._client = esi_client_factory(
-                token=self._token, spec_file=SWAGGER_SPEC_PATH, app_info_text=("allianceauth v" + __version__)
+                token=self._token,
+                ua_appname=__title__,
+                ua_version=__version__,
+                ua_url=__url__,
             )
         return self._client
 

allianceauth/eveonline/tests/test_providers.py

@@ -676,16 +676,6 @@ class TestEveSwaggerProvider(TestCase):
         self.assertTrue(mock_esi_client_factory.called)
         self.assertIsNotNone(my_provider._client)
 
-    @patch(MODULE_PATH + '.SWAGGER_SPEC_PATH', SWAGGER_OLD_SPEC_PATH)
-    @patch(MODULE_PATH + '.settings.DEBUG', False)
-    @patch('socket.socket')
-    def test_create_client_on_normal_startup_w_old_swagger_spec(
-            self, mock_socket
-    ):
-        mock_socket.side_effect = Exception('Network blocked for testing')
-        my_provider = EveSwaggerProvider()
-        self.assertIsNone(my_provider._client)
-
     @patch(MODULE_PATH + '.settings.DEBUG', True)
     @patch(MODULE_PATH + '.esi_client_factory')
     def test_dont_create_client_on_debug_startup(self, mock_esi_client_factory):
@@ -722,6 +712,6 @@ class TestEveSwaggerProvider(TestCase):
         my_provider = EveSwaggerProvider()
         my_client = my_provider.client
         operation = my_client.Universe.get_universe_factions()
-        self.assertEqual(
+        self.assertIn(
-            operation.future.request.headers['User-Agent'], 'allianceauth v1.0.0 dummy@example.net'
+            'AllianceAuth/1.0.0 (dummy@example.net; +https://gitlab.com/allianceauth/allianceauth)', operation.future.request.headers['User-Agent']
         )

allianceauth/project_template/project_name/celery.py

@@ -31,6 +31,13 @@ app.conf.ONCE = {
     'settings': {}
 }
 
+app.conf.task_routes = {
+    # Some AA Services are sensitive to threaded tasks
+    # Utilize a single threaded worker to process these tasks
+    # Discord: Multithreads can cause duplicate role creation.
+    "discord.*": {"queue": "services"},
+}
+
 # Load task modules from all registered Django app configs.
 app.autodiscover_tasks(lambda: settings.INSTALLED_APPS)
 

allianceauth/project_template/project_name/settings/base.py

@@ -57,9 +57,10 @@ CELERYBEAT_SCHEDULE = {
         'task': 'esi.tasks.cleanup_callbackredirect',
         'schedule': crontab(minute='0', hour='*/4'),
     },
-    'esi_cleanup_token': {
+    'esi_cleanup_token': {  # 1/48th * 1hr = 48Hr/2Day Refresh Cycles.
-        'task': 'esi.tasks.cleanup_token',
+        'task': 'esi.tasks.cleanup_token_subset',
-        'schedule': crontab(minute='0', hour='0'),
+        'schedule': crontab(minute="0", hour="*"),
+        'apply_offset': True
     },
     'run_model_update': {
         'task': 'allianceauth.eveonline.tasks.run_model_update',

allianceauth/project_template/supervisor.conf

@@ -24,6 +24,21 @@ stopwaitsecs = 600
 killasgroup=true
 priority=998
 
+[program:worker_services]
+command={{ celery }} -A {{ project_name }} worker --pool=threads --concurrency=1 -Q services -n %(program_name)s_%(process_num)02d
+directory={{ project_directory }}
+user=allianceserver
+numprocs=1
+process_name=%(program_name)s_%(process_num)02d
+stdout_logfile={{ project_directory }}/log/worker_services.log
+stderr_logfile={{ project_directory }}/log/worker_services.log
+autostart=true
+autorestart=true
+startsecs=10
+stopwaitsecs = 600
+killasgroup=true
+priority=998
+
 {% if gunicorn %}
 [program:gunicorn]
 user = allianceserver
@@ -38,12 +53,12 @@ stopsignal=INT
 {% endif %}
 
 [eventlistener:memmon]
-command={{ memmon }} -p worker_00=256MB -p gunicorn=256MB
+command={{ memmon }} -p worker_00=256MB -p worker_services_00=256MB -p gunicorn=256MB
 directory={{ project_directory }}
 events=TICK_60
 stdout_logfile={{ project_directory }}/log/memmon.log
 stderr_logfile={{ project_directory }}/log/memmon.log
 
 [group:{{ project_name }}]
-programs=beat,worker{% if gunicorn %},gunicorn{% endif %}
+programs=beat,worker,worker_services{% if gunicorn %},gunicorn{% endif %}
 priority=999

allianceauth/srp/providers.py

@@ -2,7 +2,7 @@ import os
 
 from esi.clients import EsiClientProvider
 
-from allianceauth import __version__
+from allianceauth import __version__, __title__, __url__
 
 SWAGGER_SPEC = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'swagger.json')
 
@@ -12,8 +12,8 @@ get_killmails_killmail_id_killmail_hash
 get_universe_types_type_id
 """
 
-
 esi = EsiClientProvider(
-    spec_file=SWAGGER_SPEC,
+    ua_appname=__title__,
-    app_info_text=("allianceauth v" + __version__)
+    ua_version=__version__,
+    ua_url=__url__,
 )

docker/.env.example

@@ -1,7 +1,7 @@
 PROTOCOL=https://
 AUTH_SUBDOMAIN=%AUTH_SUBDOMAIN%
 DOMAIN=%DOMAIN%
-AA_DOCKER_TAG=registry.gitlab.com/allianceauth/allianceauth/auth:v5.0.0a1
+AA_DOCKER_TAG=registry.gitlab.com/allianceauth/allianceauth/auth:v5.0.0a3
 
 # Nginx Proxy Manager
 PROXY_HTTP_PORT=80

docker/Dockerfile

@@ -1,5 +1,5 @@
 FROM python:3.12-slim
-ARG AUTH_VERSION=v5.0.0a1
+ARG AUTH_VERSION=v5.0.0a3
 ARG AUTH_PACKAGE=allianceauth==${AUTH_VERSION}
 ENV AUTH_USER=allianceauth
 ENV AUTH_GROUP=allianceauth

docker/conf/celery.py

@@ -35,6 +35,13 @@ app.conf.ONCE = {
     'settings': {}
 }
 
+app.conf.task_routes = {
+    # Some AA Services are sensitive to threaded tasks
+    # Utilize a single threaded worker to process these tasks
+    # Discord: Multithreads can cause duplicate role creation.
+    "discord.*": {"queue": "services"},
+}
+
 # Load task modules from all registered Django app configs.
 app.autodiscover_tasks(lambda: settings.INSTALLED_APPS)
 

docker/docker-compose.yml

@@ -103,6 +103,10 @@ services:
     deploy:
       replicas: 2
 
+  allianceauth_worker_services:
+    <<: [*allianceauth-base, *allianceauth-health-checks]
+    entrypoint: ["celery", "-A", "myauth", "worker", "--pool=threads", "--concurrency=1", "-Q" , "services" , "-n", "worker_services_%n"]
+
   grafana:
     image: grafana/grafana-oss:latest
     restart: always

docs/development/tech_docu/celery.md

@@ -266,14 +266,14 @@ Every Alliance Auth installation will come with a couple of special celery relat
 
 Celery-once is a celery extension "that allows you to prevent multiple execution and queuing of celery tasks". What that means is that you can ensure that only one instance of a celery task runs at any given time. This can be useful, for example, if you do not want multiple instances of your task to talk to the same external service at the same time.
 
-We use a custom backend for celery_once in Alliance Auth defined [here](https://gitlab.com/allianceauth/allianceauth/-/blob/master/allianceauth/services/tasks.py#L14)
+We use a custom backend for celery_once in Alliance Auth defined in [allianceauth.services.tasks](https://gitlab.com/allianceauth/allianceauth/-/blob/master/allianceauth/services/tasks.py#L14)
 You can import it for use like so:
 
 ```python
 from allianceauth.services.tasks import QueueOnce
 ```
 
-An example of Alliance Auth's use within the `@sharedtask` decorator, can be seen [here](https://gitlab.com/allianceauth/allianceauth/-/blob/master/allianceauth/services/modules/discord/tasks.py#L62) in the discord module
+An example of Alliance Auth's use within the `@sharedtask` decorator, can be seen in [allianceauth.services.modules.discord.tasks](https://gitlab.com/allianceauth/allianceauth/-/blob/master/allianceauth/services/modules/discord/tasks.py#L62) in the discord module
 You can use it like so:
 
 ```python

docs/installation-containerized/docker.md

@@ -17,7 +17,7 @@ If at any point `docker compose` does not work, but `docker-compose` does, you h
 1. run `bash <(curl -s https://gitlab.com/allianceauth/allianceauth/-/raw/master/docker/scripts/download.sh)`. This will download all the files you need to install Alliance Auth and place them in a directory named `aa-docker`. Feel free to rename/move this folder.
 1. run `./scripts/prepare-env.sh` to set up your environment
 1. (optional) Change `PROTOCOL` to `http://` if not using SSL in `.env`
-1. run `docker compose --env-file=.env up -d` (NOTE: if this command hangs, follow the instructions [here](https://www.digitalocean.com/community/tutorials/how-to-setup-additional-entropy-for-cloud-servers-using-haveged))
+1. run `docker compose --env-file=.env up -d` (NOTE: if this command hangs, follow the instructions on [How to Setup Entropy](https://www.digitalocean.com/community/tutorials/how-to-setup-additional-entropy-for-cloud-servers-using-haveged))
 1. run `docker compose exec allianceauth_gunicorn bash` to open up a terminal inside an auth container
 1. run `auth migrate`
 1. run `auth collectstatic`

docs/installation/gunicorn.md

@@ -69,7 +69,7 @@ Whatever you decide to use, remember it because we'll need it when configuring y
 
 ##### Number of workers
 
-By default, Gunicorn will spawn only one worker. The number you set this to will depend on your own server environment, how many visitors you have etc. Gunicorn suggests `(2 x $num_cores) + 1` for the number of workers. So, for example, if you have 2 cores, you want 2 x 2 + 1 = 5 workers. See [here](https://docs.gunicorn.org/en/stable/design.html#how-many-workers) for the official discussion on this topic.
+By default, Gunicorn will spawn only one worker. The number you set this to will depend on your own server environment, how many visitors you have etc. Gunicorn suggests `(2 x $num_cores) + 1` for the number of workers. So, for example, if you have 2 cores, you want 2 x 2 + 1 = 5 workers. See [How Mnay Workers](https://docs.gunicorn.org/en/stable/design.html#how-many-workers) for the official discussion on this topic.
 
 Change it by adding `--workers=5` to the command.
 

docs/maintenance/tuning/web.md

@@ -6,7 +6,7 @@
 
 The default installation will have 3 workers configured for Gunicorn. This will be fine on most systems, but if your system as more than one core than you might want to increase the number of workers to get better response times. Note that more workers will also need more RAM though.
 
-The number you set this to will depend on your own server environment, how many visitors you have etc. Gunicorn suggests `(2 x $num_cores) + 1` for the number of workers. So for example, if you have 2 cores, you want 2 x 2 + 1 = 5 workers. See [here](https://docs.gunicorn.org/en/stable/design.html#how-many-workers) for the official discussion on this topic.
+The number you set this to will depend on your own server environment, how many visitors you have etc. Gunicorn suggests `(2 x $num_cores) + 1` for the number of workers. So for example, if you have 2 cores, you want 2 x 2 + 1 = 5 workers. See [How Many Workers](https://docs.gunicorn.org/en/stable/design.html#how-many-workers) for the official discussion on this topic.
 
 ::::{tabs}
 :::{group-tab} Ubuntu 2204, 2404

pyproject.toml

@@ -20,7 +20,7 @@ classifiers = [
     "Environment :: Web Environment",
     "Framework :: Celery",
     "Framework :: Django",
-    "Framework :: Django :: 5.1",
+    "Framework :: Django :: 5.2",
     "Intended Audience :: Developers",
     "License :: OSI Approved :: GNU General Public License v2 (GPLv2)",
     "Operating System :: POSIX :: Linux",
@@ -42,13 +42,13 @@ dependencies = [
     "beautifulsoup4",
     "celery>=5.5,<6",
     "celery-once>=3.0.1",
-    "django>=5.1,<5.2",
+    "django>=5.2,<6",
     "django-bootstrap-form",
     "django-bootstrap5>=23.3",
-    "django-celery-beat>=2.7",
+    "django-celery-beat>=2.8",
-    "django-esi>=5",
+    "django-esi>=7.0.0b1",
     "django-redis>=5.4",
-    "django-registration>=5.1,<6",
+    "django-registration>=5.2,<6",
     "django-solo",
     "django-sortedm2m",
     "django-sri",
@@ -87,50 +87,6 @@ scripts.allianceauth = "allianceauth.bin.allianceauth:main"
 [tool.flit.module]
 name = "allianceauth"
 
-[tool.ruff]
-line-length = 119
-format.line-ending = "lf"
-lint.select = [
-    "B", # flake8-bugbear
-    "C", # pylint convention
-    # "D", # pydocstyle, Want to turn these on, but our docstrings are lightly used
-    "D300",   # use triple double-quotes in docstrings PEP 257
-    "DJ",     # flake8-django
-    "DOC",    # pylintdoc
-    "E",      # pycodestyle error
-    "F",      # pyflakes (flake8 base)
-    "G010",   # logging-warn, Warn on using logging.warn
-    "I",      # isort
-    "PGH005", # pygrep-hooks, python-check-mock-method
-    "RUF100", # basically yesqa
-    "UP",     # pyupgrade, will target requires-python
-    "W",      # pycodestyle Warning
-]
-lint.ignore = [
-    "E501", # Line too long, WIP across repo.
-]
-
-lint.per-file-ignores = { "*local.py" = [ "F405", "F403" ] }
-
-lint.isort.combine-as-imports = true # profile=django
-lint.isort.section-order = [
-    "future",
-    "standard-library",
-    "third-party",
-    "DJANGO",
-    "ESI",
-    "first-party",
-    "local-folder",
-]
-lint.isort.sections."DJANGO" = [
-    "django",
-    "django_redis",
-    "django_registration",
-]
-lint.isort.sections."ESI" = [
-    "esi",
-]
-
 [tool.isort]
 profile = "django"
 sections = [