Merge branch 'servicesworker' into 'v5.x'

Singlethread Services Worker

See merge request allianceauth/allianceauth!1698

allianceauth/project_template/project_name/celery.py

@@ -31,6 +31,13 @@ app.conf.ONCE = {
     'settings': {}
 }
 
+app.conf.task_routes = {
+    # Some AA Services are sensitive to threaded tasks
+    # Utilize a single threaded worker to process these tasks
+    # Discord: Multithreads can cause duplicate role creation.
+    "discord.*": {"queue": "services"},
+}
+
 # Load task modules from all registered Django app configs.
 app.autodiscover_tasks(lambda: settings.INSTALLED_APPS)
 

allianceauth/project_template/supervisor.conf

@@ -24,6 +24,21 @@ stopwaitsecs = 600
 killasgroup = true
 priority = 998
 
+[program:worker_services]
+command={{ celery }} -A {{ project_name }} worker --pool=threads --concurrency=1 -Q services -n %(program_name)s_%(process_num)02d
+directory={{ project_directory }}
+user=allianceserver
+numprocs=1
+process_name=%(program_name)s_%(process_num)02d
+stdout_logfile={{ project_directory }}/log/worker_services.log
+stderr_logfile={{ project_directory }}/log/worker_services.log
+autostart=true
+autorestart=true
+startsecs=10
+stopwaitsecs = 600
+killasgroup=true
+priority=998
+
 {% if gunicorn %}
 [program:gunicorn]
 user = allianceserver
@@ -37,12 +52,12 @@ stopsignal = INT
 {% endif %}
 
 [eventlistener:memmon]
-command = {{ memmon }} -p worker_00=256MB -p gunicorn=256MB
+command={{ memmon }} -p worker_00=256MB -p worker_services_00=256MB -p gunicorn=256MB
-directory = {{ project_directory }}
+directory={{ project_directory }}
-events = TICK_60
+events=TICK_60
-stdout_logfile = {{ project_directory }}/log/memmon.log
+stdout_logfile={{ project_directory }}/log/memmon.log
-stderr_logfile = {{ project_directory }}/log/memmon.log
+stderr_logfile={{ project_directory }}/log/memmon.log
 
 [group:{{ project_name }}]
-programs = beat,worker{% if gunicorn %},gunicorn{% endif %}
+programs=beat,worker,worker_services{% if gunicorn %},gunicorn{% endif %}
 priority = 999

docker/conf/celery.py

@@ -35,6 +35,13 @@ app.conf.ONCE = {
     'settings': {}
 }
 
+app.conf.task_routes = {
+    # Some AA Services are sensitive to threaded tasks
+    # Utilize a single threaded worker to process these tasks
+    # Discord: Multithreads can cause duplicate role creation.
+    "discord.*": {"queue": "services"},
+}
+
 # Load task modules from all registered Django app configs.
 app.autodiscover_tasks(lambda: settings.INSTALLED_APPS)
 

docker/docker-compose.yml

@@ -103,6 +103,10 @@ services:
     deploy:
       replicas: 2
 
+  allianceauth_worker_services:
+    <<: [*allianceauth-base, *allianceauth-health-checks]
+    entrypoint: ["celery", "-A", "myauth", "worker", "--pool=threads", "--concurrency=1", "-Q" , "services" , "-n", "worker_services_%n"]
+
   grafana:
     image: grafana/grafana-oss:latest
     restart: always