Merge branch 'fix/missing-celery-setting' into 'master'

fix missing setting in celery

See merge request allianceauth/allianceauth!1636

.gitlab-ci.yml

@@ -25,7 +25,7 @@ before_script:
 pre-commit-check:
   <<: *only-default
   stage: pre-commit
-  image: python:3.11-bullseye
+  image: python:3.11-bookworm
   # variables:
   #   PRE_COMMIT_HOME: ${CI_PROJECT_DIR}/.cache/pre-commit
   # cache:
@@ -53,7 +53,7 @@ secret_detection:
 
 test-3.8-core:
   <<: *only-default
-  image: python:3.8-bullseye
+  image: python:3.8-bookworm
   script:
     - tox -e py38-core
   artifacts:
@@ -65,7 +65,7 @@ test-3.8-core:
 
 test-3.9-core:
   <<: *only-default
-  image: python:3.9-bullseye
+  image: python:3.9-bookworm
   script:
     - tox -e py39-core
   artifacts:
@@ -77,7 +77,7 @@ test-3.9-core:
 
 test-3.10-core:
   <<: *only-default
-  image: python:3.10-bullseye
+  image: python:3.10-bookworm
   script:
     - tox -e py310-core
   artifacts:
@@ -89,7 +89,7 @@ test-3.10-core:
 
 test-3.11-core:
   <<: *only-default
-  image: python:3.11-bullseye
+  image: python:3.11-bookworm
   script:
     - tox -e py311-core
   artifacts:
@@ -101,7 +101,7 @@ test-3.11-core:
 
 test-3.12-core:
   <<: *only-default
-  image: python:3.12-rc-bullseye
+  image: python:3.12-bookworm
   script:
     - tox -e py312-core
   artifacts:
@@ -113,7 +113,7 @@ test-3.12-core:
 
 test-3.8-all:
   <<: *only-default
-  image: python:3.8-bullseye
+  image: python:3.8-bookworm
   script:
     - tox -e py38-all
   artifacts:
@@ -125,7 +125,7 @@ test-3.8-all:
 
 test-3.9-all:
   <<: *only-default
-  image: python:3.9-bullseye
+  image: python:3.9-bookworm
   script:
     - tox -e py39-all
   artifacts:
@@ -137,7 +137,7 @@ test-3.9-all:
 
 test-3.10-all:
   <<: *only-default
-  image: python:3.10-bullseye
+  image: python:3.10-bookworm
   script:
     - tox -e py310-all
   artifacts:
@@ -149,7 +149,7 @@ test-3.10-all:
 
 test-3.11-all:
   <<: *only-default
-  image: python:3.11-bullseye
+  image: python:3.11-bookworm
   script:
     - tox -e py311-all
   artifacts:
@@ -162,7 +162,7 @@ test-3.11-all:
 
 test-3.12-all:
   <<: *only-default
-  image: python:3.12-rc-bullseye
+  image: python:3.12-bookworm
   script:
     - tox -e py312-all
   artifacts:
@@ -174,7 +174,7 @@ test-3.12-all:
 
 build-test:
   stage: test
-  image: python:3.11-bullseye
+  image: python:3.11-bookworm
 
   before_script:
     - python -m pip install --upgrade pip
@@ -193,13 +193,13 @@ build-test:
 
 test-docs:
   <<: *only-default
-  image: python:3.11-bullseye
+  image: python:3.11-bookworm
   script:
     - tox -e docs
 
 deploy_production:
   stage: deploy
-  image: python:3.11-bullseye
+  image: python:3.11-bookworm
 
   before_script:
     - python -m pip install --upgrade pip

allianceauth/__init__.py

@@ -5,7 +5,7 @@ manage online service access.
 # This will make sure the app is always imported when
 # Django starts so that shared_task will use this app.
 
-__version__ = '4.1.0'
+__version__ = '4.2.1'
 __title__ = 'Alliance Auth'
 __url__ = 'https://gitlab.com/allianceauth/allianceauth'
 NAME = f'{__title__} v{__version__}'

allianceauth/analytics/tasks.py

@@ -101,11 +101,38 @@ def analytics_daily_stats():
                     event_type='Stats')
 
     for appconfig in apps.get_app_configs():
-        analytics_event(namespace='allianceauth.analytics',
-                        task='send_extension_stats',
-                        label=appconfig.label,
-                        value=1,
-                        event_type='Stats')
+        if appconfig.label in [
+            "django_celery_beat",
+            "bootstrapform",
+            "messages",
+            "sessions",
+            "auth",
+            "staticfiles",
+            "users",
+            "addons",
+            "admin",
+            "humanize",
+            "contenttypes",
+            "sortedm2m",
+            "django_bootstrap5",
+            "tokens",
+            "authentication",
+            "services",
+            "framework",
+            "notifications"
+            "eveonline",
+            "navhelper",
+            "analytics",
+            "menu",
+            "theme"
+        ]:
+            pass
+        else:
+            analytics_event(namespace='allianceauth.analytics',
+                            task='send_extension_stats',
+                            label=appconfig.label,
+                            value=1,
+                            event_type='Stats')
 
 
 @shared_task()

allianceauth/apps.py

@@ -5,26 +5,5 @@ from django.core.checks import Warning, Error, register
 class AllianceAuthConfig(AppConfig):
     name = 'allianceauth'
 
-
-@register()
-def check_settings(app_configs, **kwargs):
-    from django.conf import settings
-
-    errors = []
-    if hasattr(settings, "SITE_URL"):
-        if settings.SITE_URL[-1] == "/":
-            errors.append(Warning(
-                "'SITE_URL' Has a trailing slash. This may lead to incorrect links being generated by Auth."))
-    else:
-        errors.append(Error(
-            "No 'SITE_URL' found is settings. This may lead to incorrect links being generated by Auth or Errors in 3rd party modules."))
-    if hasattr(settings, "CSRF_TRUSTED_ORIGINS"):
-        if hasattr(settings, "SITE_URL"):
-            if settings.SITE_URL not in settings.CSRF_TRUSTED_ORIGINS:
-                errors.append(Warning(
-                    "'SITE_URL' not found in 'CSRF_TRUSTED_ORIGINS'. Auth may not load pages correctly until this is rectified."))
-    else:
-        errors.append(Error(
-            "No 'CSRF_TRUSTED_ORIGINS' found is settings, Auth may not load pages correctly until this is rectified"))
-
-    return errors
+    def ready(self) -> None:
+        import allianceauth.checks # noqa

allianceauth/checks.py

@@ -0,0 +1,179 @@
+from typing import List
+from django import db
+from django.core.checks import CheckMessage, Error, register, Warning
+from allianceauth.utils.cache import get_redis_client
+from django.utils import timezone
+from packaging.version import InvalidVersion, Version as Pep440Version
+from celery import current_app
+from django.conf import settings
+from sqlite3.dbapi2 import sqlite_version_info
+
+"""
+A = System Packages
+B = Configuration
+"""
+
+
+@register()
+def django_settings(app_configs, **kwargs) -> List[CheckMessage]:
+    errors: List[CheckMessage] = []
+    if hasattr(settings, "SITE_URL"):
+        if settings.SITE_URL[-1] == "/":
+            errors.append(Warning("'SITE_URL' Has a trailing slash. This may lead to incorrect links being generated by Auth.", hint="", id="allianceauth.checks.B005"))
+    else:
+        errors.append(Error("No 'SITE_URL' found is settings. This may lead to incorrect links being generated by Auth or Errors in 3rd party modules.", hint="", id="allianceauth.checks.B006"))
+
+    if hasattr(settings, "CSRF_TRUSTED_ORIGINS") and hasattr(settings, "SITE_URL"):
+        if settings.SITE_URL not in settings.CSRF_TRUSTED_ORIGINS:
+            errors.append(Warning("'SITE_URL' not found in 'CSRF_TRUSTED_ORIGINS'. Auth may not load pages correctly until this is rectified.", hint="", id="allianceauth.checks.B007"))
+    else:
+        errors.append(Error("No 'CSRF_TRUSTED_ORIGINS' found is settings, Auth may not load pages correctly until this is rectified", hint="", id="allianceauth.checks.B008"))
+
+    return errors
+
+
+@register()
+def system_package_redis(app_configs, **kwargs) -> List[CheckMessage]:
+    errors: List[CheckMessage] = []
+    try:
+        redis_version = Pep440Version(get_redis_client().info()['redis_version'])
+    except InvalidVersion:
+        errors.append(Warning("Unable to confirm Redis Version"))
+        return errors
+
+    if redis_version.major == 7 and redis_version.minor == 2 and timezone.now() > timezone.datetime(year=2025, month=8, day=31, tzinfo=timezone.utc):
+        errors.append(Error(f"Redis {redis_version.public} in Security Support only, Updating Suggested", hint="https://redis.io/docs/latest/operate/oss_and_stack/install/install-redis/install-redis-on-linux/", id="allianceauth.checks.A001"))
+    elif redis_version.major == 7 and redis_version.minor == 0:
+        errors.append(Warning(f"Redis {redis_version.public} in Security Support only, Updating Suggested", hint="https://redis.io/docs/latest/operate/oss_and_stack/install/install-redis/install-redis-on-linux/", id="allianceauth.checks.A002"))
+    elif redis_version.major == 6 and redis_version.minor == 2:
+        errors.append(Warning(f"Redis {redis_version.public} in Security Support only, Updating Suggested", hint="https://redis.io/docs/latest/operate/oss_and_stack/install/install-redis/install-redis-on-linux/", id="allianceauth.checks.A018"))
+    elif redis_version.major in [6, 5]:
+        errors.append(Error(f"Redis {redis_version.public} EOL", hint="https://redis.io/docs/latest/operate/oss_and_stack/install/install-redis/install-redis-on-linux/", id="allianceauth.checks.A003"))
+
+    return errors
+
+
+@register()
+def system_package_mysql(app_configs, **kwargs) -> List[CheckMessage]:
+    errors: List[CheckMessage] = []
+
+    for connection in db.connections.all():
+        if connection.vendor == "mysql":
+            try:
+                mysql_version = Pep440Version(".".join(str(i) for i in connection.mysql_version))
+            except InvalidVersion:
+                errors.append(Warning("Unable to confirm MySQL Version"))
+                return errors
+
+            # MySQL 8
+            if mysql_version.major == 8 and mysql_version.minor == 4 and timezone.now() > timezone.datetime(year=2032, month=4, day=30, tzinfo=timezone.utc):
+                errors.append(Error(f"MySQL {mysql_version.public} EOL", hint="https://dev.mysql.com/doc/mysql-apt-repo-quick-guide/en/", id="allianceauth.checks.A004"))
+            elif mysql_version.major == 8 and mysql_version.minor == 3:
+                errors.append(Warning(f"MySQL {mysql_version.public} Non LTS", hint="https://dev.mysql.com/doc/mysql-apt-repo-quick-guide/en/", id="allianceauth.checks.A005"))
+            elif mysql_version.major == 8 and mysql_version.minor == 2:
+                errors.append(Warning(f"MySQL {mysql_version.public} Non LTS", hint="https://dev.mysql.com/doc/mysql-apt-repo-quick-guide/en/", id="allianceauth.checks.A006"))
+            elif mysql_version.major == 8 and mysql_version.minor == 1:
+                errors.append(Error(f"MySQL {mysql_version.public} EOL", hint="https://dev.mysql.com/doc/mysql-apt-repo-quick-guide/en/", id="allianceauth.checks.A007"))
+            elif mysql_version.major == 8 and mysql_version.minor == 0 and timezone.now() > timezone.datetime(year=2026, month=4, day=30, tzinfo=timezone.utc):
+                errors.append(Error(f"MySQL {mysql_version.public} EOL", hint="https://dev.mysql.com/doc/mysql-apt-repo-quick-guide/en/", id="allianceauth.checks.A008"))
+            elif mysql_version.major < 8:  # This will also catch Mariadb 5.x
+                errors.append(Error(f"MySQL or MariaDB {mysql_version.public} EOL", hint="https://dev.mysql.com/doc/mysql-apt-repo-quick-guide/en/", id="allianceauth.checks.A009"))
+    return errors
+
+
+@register()
+def system_package_mariadb(app_configs, **kwargs) -> List[CheckMessage]:
+    errors: List[CheckMessage] = []
+
+    for connection in db.connections.all():
+        if connection.vendor == "mysql":  # Still to find a way to determine MySQL vs MariaDB
+            try:
+                mariadb_version = Pep440Version(".".join(str(i) for i in connection.mysql_version))
+            except InvalidVersion:
+                errors.append(Warning("Unable to confirm MariaDB Version"))
+                return errors
+
+            # MariaDB 11
+            if mariadb_version.major == 11 and mariadb_version.minor == 4 and timezone.now() > timezone.datetime(year=2029, month=5, day=19, tzinfo=timezone.utc):
+                errors.append(Error(f"MariaDB {mariadb_version.public} EOL", hint="https://mariadb.org/download/?t=repo-config", id="allianceauth.checks.A010"))
+            elif mariadb_version.major == 11 and mariadb_version.minor == 2:
+                errors.append(Warning(f"MariaDB {mariadb_version.public} Non LTS", hint="https://mariadb.org/download/?t=repo-config", id="allianceauth.checks.A018"))
+                if timezone.now() > timezone.datetime(year=2024, month=11, day=21, tzinfo=timezone.utc):
+                    errors.append(Error(f"MariaDB {mariadb_version.public}  EOL", hint="https://mariadb.org/download/?t=repo-config", id="allianceauth.checks.A011"))
+            elif mariadb_version.major == 11 and mariadb_version.minor == 1:
+                errors.append(Warning(f"MariaDB {mariadb_version.public} Non LTS", hint="https://mariadb.org/download/?t=repo-config", id="allianceauth.checks.A019"))
+                if timezone.now() > timezone.datetime(year=2024, month=8, day=21, tzinfo=timezone.utc):
+                    errors.append(Error(f"MariaDB {mariadb_version.public} EOL", hint="https://mariadb.org/download/?t=repo-config", id="allianceauth.checks.A012"))
+            elif mariadb_version.major == 11 and mariadb_version.minor in [0, 3]:  # Demote versions down here once EOL
+                errors.append(Error(f"MariaDB {mariadb_version.public} EOL", hint="https://mariadb.org/download/?t=repo-config.", id="allianceauth.checks.A013"))
+
+            # MariaDB 10
+            elif mariadb_version.major == 10 and mariadb_version.minor == 11 and timezone.now() > timezone.datetime(year=2028, month=2, day=10, tzinfo=timezone.utc):
+                errors.append(Error(f"MariaDB {mariadb_version.public} EOL", hint="https://mariadb.org/download/?t=repo-config.", id="allianceauth.checks.A014"))
+            elif mariadb_version.major == 10 and mariadb_version.minor == 6 and timezone.now() > timezone.datetime(year=2026, month=7, day=6, tzinfo=timezone.utc):
+                errors.append(Error(f"MariaDB {mariadb_version.public} EOL", hint="https://mariadb.org/download/?t=repo-config", id="allianceauth.checks.A0015"))
+            elif mariadb_version.major == 10 and mariadb_version.minor == 5 and timezone.now() > timezone.datetime(year=2025, month=6, day=24, tzinfo=timezone.utc):
+                errors.append(Error(f"MariaDB {mariadb_version.public} EOL", hint="https://mariadb.org/download/?t=repo-config", id="allianceauth.checks.A016"))
+            elif mariadb_version.major == 10 and mariadb_version.minor in [0, 1, 2, 3, 4, 7, 9, 10]:  # Demote versions down here once EOL
+                errors.append(Error(f"MariaDB {mariadb_version.public} EOL", hint="https://mariadb.org/download/?t=repo-config", id="allianceauth.checks.A017"))
+
+    return errors
+
+
+@register()
+def system_package_sqlite(app_configs, **kwargs) -> List[CheckMessage]:
+    errors: List[CheckMessage] = []
+    for connection in db.connections.all():
+        if connection.vendor == "sqlite":
+            try:
+                sqlite_version = Pep440Version(".".join(str(i) for i in sqlite_version_info))
+            except InvalidVersion:
+                errors.append(Warning("Unable to confirm SQLite Version"))
+                return errors
+            if sqlite_version.major == 3 and sqlite_version.minor < 27:
+                errors.append(Error(f"SQLite {sqlite_version.public} Unsupported by Django", hint="https://pkgs.org/download/sqlite3", id="allianceauth.checks.A020"))
+    return errors
+
+
+@register()
+def sql_settings(app_configs, **kwargs) -> List[CheckMessage]:
+    errors: List[CheckMessage] = []
+    for connection in db.connections.all():
+        if connection.vendor == "mysql":
+            try:
+                if connection.settings_dict["OPTIONS"]["charset"] != "utf8mb4":
+                    errors.append(Error(f"SQL Charset is not set to utf8mb4 DB:{connection.alias}", hint="https://gitlab.com/allianceauth/allianceauth/-/commit/89be2456fb2d741b86417e889da9b6129525bec8", id="allianceauth.checks.B001"))
+            except KeyError:
+                errors.append(Error(f"SQL Charset is not set to utf8mb4 DB:{connection.alias}", hint="https://gitlab.com/allianceauth/allianceauth/-/commit/89be2456fb2d741b86417e889da9b6129525bec8", id="allianceauth.checks.B001"))
+
+            # This hasn't actually been set on AA yet
+            # try:
+            #     if connection.settings_dict["OPTIONS"]["collation"] != "utf8mb4_unicode_ci":
+            #         errors.append(Error(f"SQL Collation is not set to utf8mb4_unicode_ci DB:{connection.alias}", hint="https://gitlab.com/allianceauth/allianceauth/-/commit/89be2456fb2d741b86417e889da9b6129525bec8", id="allianceauth.checks.B001"))
+            # except KeyError:
+            #     errors.append(Error(f"SQL Collation is not set to utf8mb4_unicode_ci DB:{connection.alias}", hint="https://gitlab.com/allianceauth/allianceauth/-/commit/89be2456fb2d741b86417e889da9b6129525bec8", id="allianceauth.checks.B001"))
+
+        # if connection.vendor == "sqlite":
+
+    return errors
+
+
+@register()
+def celery_settings(app_configs, **kwargs) -> List[CheckMessage]:
+    errors: List[CheckMessage] = []
+    if current_app.conf.broker_transport_options != {'priority_steps': [0, 1, 2, 3, 4, 5, 6, 7, 8, 9], 'queue_order_strategy': 'priority'}:
+        errors.append(Error("Celery Priorities are not set correctly", hint="https://gitlab.com/allianceauth/allianceauth/-/commit/8861ec0a61790eca0261f1adc1cc04ca5f243cbc", id="allianceauth.checks.B003"))
+
+    if current_app.conf.broker_connection_retry_on_startup != True:
+        errors.append(Error("Celery broker_connection_retry_on_startup not set correctly", hint="https://gitlab.com/allianceauth/allianceauth/-/commit/380c41400b535447839e5552df2410af35a75280", id="allianceauth.checks.B004"))
+    return errors
+
+
+# IDEAS
+
+# Any other celery things weve manually changed over the years
+# I'd be happy to add Community App checks, old versions the owners dont want to support etc.
+
+
+# Check Default Collation on DB
+# Check Charset Collation on all tables

docker/.env.example

@@ -1,7 +1,7 @@
 PROTOCOL=https://
 AUTH_SUBDOMAIN=%AUTH_SUBDOMAIN%
 DOMAIN=%DOMAIN%
-AA_DOCKER_TAG=registry.gitlab.com/allianceauth/allianceauth/auth:v4.1.0
+AA_DOCKER_TAG=registry.gitlab.com/allianceauth/allianceauth/auth:v4.2.1
 
 # Nginx Proxy Manager
 PROXY_HTTP_PORT=80

docker/Dockerfile

@@ -1,5 +1,5 @@
 FROM python:3.11-slim
-ARG AUTH_VERSION=v4.1.0
+ARG AUTH_VERSION=v4.2.1
 ARG AUTH_PACKAGE=allianceauth==${AUTH_VERSION}
 ENV AUTH_USER=allianceauth
 ENV AUTH_GROUP=allianceauth

docker/conf/celery.py

@@ -9,6 +9,10 @@ from django.conf import settings  # noqa
 
 app = Celery('myauth')
 
+# Automatically try to establish the connection to the AMQP broker on
+# Celery startup if it is unavailable.
+app.conf.broker_connection_retry_on_startup = True
+
 # Using a string here means the worker don't have to serialize
 # the configuration object to child processes.
 app.config_from_object('django.conf:settings')

pyproject.toml

@@ -18,6 +18,7 @@ authors = [
 requires-python = ">=3.8"
 classifiers = [
     "Environment :: Web Environment",
+    "Framework :: Celery",
     "Framework :: Django",
     "Framework :: Django :: 4.2",
     "Intended Audience :: Developers",