Merge branch 'servicesworker' into 'v5.x'

Singlethread Services Worker

See merge request allianceauth/allianceauth!1698

allianceauth/project_template/project_name/celery.py

@@ -31,6 +31,13 @@ app.conf.ONCE = {
     'settings': {}
 }
 
+app.conf.task_routes = {
+    # Some AA Services are sensitive to threaded tasks
+    # Utilize a single threaded worker to process these tasks
+    # Discord: Multithreads can cause duplicate role creation.
+    "discord.*": {"queue": "services"},
+}
+
 # Load task modules from all registered Django app configs.
 app.autodiscover_tasks(lambda: settings.INSTALLED_APPS)
 

allianceauth/project_template/supervisor.conf

@@ -24,6 +24,21 @@ stopwaitsecs = 600
 killasgroup=true
 priority=998
 
+[program:worker_services]
+command={{ celery }} -A {{ project_name }} worker --pool=threads --concurrency=1 -Q services -n %(program_name)s_%(process_num)02d
+directory={{ project_directory }}
+user=allianceserver
+numprocs=1
+process_name=%(program_name)s_%(process_num)02d
+stdout_logfile={{ project_directory }}/log/worker_services.log
+stderr_logfile={{ project_directory }}/log/worker_services.log
+autostart=true
+autorestart=true
+startsecs=10
+stopwaitsecs = 600
+killasgroup=true
+priority=998
+
 {% if gunicorn %}
 [program:gunicorn]
 user = allianceserver
@@ -38,12 +53,12 @@ stopsignal=INT
 {% endif %}
 
 [eventlistener:memmon]
-command={{ memmon }} -p worker_00=256MB -p gunicorn=256MB
+command={{ memmon }} -p worker_00=256MB -p worker_services_00=256MB -p gunicorn=256MB
 directory={{ project_directory }}
 events=TICK_60
 stdout_logfile={{ project_directory }}/log/memmon.log
 stderr_logfile={{ project_directory }}/log/memmon.log
 
 [group:{{ project_name }}]
-programs=beat,worker{% if gunicorn %},gunicorn{% endif %}
+programs=beat,worker,worker_services{% if gunicorn %},gunicorn{% endif %}
 priority=999

docker/conf/celery.py

@@ -35,6 +35,13 @@ app.conf.ONCE = {
     'settings': {}
 }
 
+app.conf.task_routes = {
+    # Some AA Services are sensitive to threaded tasks
+    # Utilize a single threaded worker to process these tasks
+    # Discord: Multithreads can cause duplicate role creation.
+    "discord.*": {"queue": "services"},
+}
+
 # Load task modules from all registered Django app configs.
 app.autodiscover_tasks(lambda: settings.INSTALLED_APPS)
 

docker/docker-compose.yml

@@ -103,6 +103,10 @@ services:
     deploy:
       replicas: 2
 
+  allianceauth_worker_services:
+    <<: [*allianceauth-base, *allianceauth-health-checks]
+    entrypoint: ["celery", "-A", "myauth", "worker", "--pool=threads", "--concurrency=1", "-Q" , "services" , "-n", "worker_services_%n"]
+
   grafana:
     image: grafana/grafana-oss:latest
     restart: always