Replace django-celery with base Celery (#791)

Update celery tasks to new style & remove djcelery
Vanilla celery + django-celery-beat take over the role of djcelery. Task schedules are consolidated into settings instead of residing in code.
Update docs and example supervisor configs.

alliance_auth/settings.py.example

@@ -12,17 +12,12 @@ https://docs.djangoproject.com/en/1.10/ref/settings/
 
 import os
 
-import djcelery
-
 from django.contrib import messages
 from celery.schedules import crontab
 
-djcelery.setup_loader()
-
 # Celery configuration
 BROKER_URL = 'redis://localhost:6379/0'
-CELERYBEAT_SCHEDULER = "djcelery.schedulers.DatabaseScheduler"
-CELERYBEAT_SCHEDULE = dict()
+CELERYBEAT_SCHEDULER = "django_celery_beat.schedulers.DatabaseScheduler"
 
 # Build paths inside the project like this: os.path.join(BASE_DIR, ...)
 BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
@@ -50,7 +45,7 @@ INSTALLED_APPS = [
     'django.contrib.messages',
     'django.contrib.staticfiles',
     'django.contrib.humanize',
-    'djcelery',
+    'django_celery_beat',
     'bootstrapform',
     'authentication',
     'services',
@@ -237,6 +232,25 @@ NOCAPTCHA = True
 ##
 #####################################################
 
+
+#########################
+# CELERY SCHEDULED TASKS
+#########################
+CELERYBEAT_SCHEDULE = {
+    'run_api_refresh': {
+        'task': 'eveonline.tasks.run_api_refresh',
+        'schedule': crontab(minute=0, hour="*/3"),
+    },
+    'run_corp_update': {
+        'task': 'eveonline.tasks.run_corp_update',
+        'schedule': crontab(minute=0, hour="*/2"),
+    },
+    'update_all_corpstats': {
+        'task': 'corputils.tasks.update_all_corpstats',
+        'schedule': crontab(minute=0, hour="*/6"),
+    },
+}
+
 #################
 # EMAIL SETTINGS
 #################

alliance_auth/tests/test_settings.py

@@ -4,14 +4,10 @@ Alliance Auth Test Suite Django settings.
 
 import os
 
-import djcelery
-
 from django.contrib import messages
 
 import alliance_auth
 
-djcelery.setup_loader()
-
 # Use nose to run all tests
 TEST_RUNNER = 'django_nose.NoseTestSuiteRunner'
 
@@ -40,7 +36,7 @@ INSTALLED_APPS = [
     'django.contrib.messages',
     'django.contrib.staticfiles',
     'django.contrib.humanize',
-    'djcelery',
+    'django_celery_beat',
     'bootstrapform',
     'authentication',
     'services',

corputils/tasks.py

@@ -1,15 +1,14 @@
 from corputils.models import CorpStats
-from celery.task import task, periodic_task
-from celery.task.schedules import crontab
+from alliance_auth.celeryapp import app
 
 
-@task
+@app.task
 def update_corpstats(pk):
     cs = CorpStats.objects.get(pk=pk)
     cs.update()
 
 
-@periodic_task(run_every=crontab(minute=0, hour="*/6"))
+@app.task
 def update_all_corpstats():
     for cs in CorpStats.objects.all():
         update_corpstats.delay(cs.pk)

docs/installation/auth/quickstart.md

@@ -10,5 +10,5 @@ The big goal of AllianceAuth is the automation of group membership, so we’ll n
 
 To start the background processes to sync groups and check api keys, issue these commands:
 
-    screen -dm bash -c 'python manage.py celeryd'
-    screen -dm bash -c 'python manage.py celerybeat'
+    screen -dm bash -c 'celery -A alliance_auth worker'
+    screen -dm bash -c 'celery -A alliance_auth beat'

docs/installation/auth/supervisor.md

@@ -20,7 +20,7 @@ CentOS:
 
 ## Configuration
 
-Auth provides example config files for the celery workers (celeryd), the periodic task scheduler (celerybeat), and the mumble authenticator. All of these are available in `thirdparty/Supervisor`.
+Auth provides example config files for the celery workers, the periodic task scheduler (celery beat), and the mumble authenticator. All of these are available in `thirdparty/Supervisor`.
 
 For most users, all you have to do is copy the config files to `/etc/supervisor/conf.d` then restart the service. Copy `auth-celerybeat.conf` and `auth-celeryd.conf` for the celery workers, and `auth-mumble.conf` for the mumble authenticator. For all three just use a wildcard:
 
@@ -41,15 +41,15 @@ To ensure the processes are working, check their status:
     sudo supervisorctl status
 
 Processes will be `STARTING`, `RUNNING`, or `ERROR`. If an error has occurred, check their log files:
- - celeryd: `log/worker.log`
- - celerybeat: `log/beat.log`
+ - celery workers: `log/worker.log`
+ - celery beat: `log/beat.log`
  - authenticator: `log/authenticator.log`
 
 ## Customizing Config Files
 
 The only real customization needed is if running in a virtual environment. The python path will have to be changed in order to start in the venv.
 
-Edit the config files and find the line saying `command`. Replace `python` with `/path/to/venv/python`. This can be relative to the `directory` specified in the config file.
+Edit the config files and find the line saying `command`. Replace `python` with `/path/to/venv/bin/python`. For Celery replace `celery` with `/path/to/venv/bin/celery`. This can be relative to the `directory` specified in the config file.
 
 Note that for config changes to be loaded, the supervisor service must be restarted.
 

docs/installation/services/teamspeak3.md

@@ -84,7 +84,7 @@ To enable advanced permissions, on your client go to the `Tools` menu, `Applicat
 ### TS group models not populating on admin site
 The method which populates these runs every 30 minutes. To populate manually, start a celery shell:
 
-    python manage.py celery shell
+    celery -A alliance_auth shell
 
 And execute the update:
 

docs/maintenance/troubleshooting.md

@@ -19,7 +19,7 @@ Either you need to `sudo` that command, or it's a missing dependency. Check [the
 
 ### I'm getting an error 500 trying to connect to the website on a new install
 
-Read the apache error log: `sudo nano /var/log/apache2/error.log`
+Read the apache error log: `sudo less /var/log/apache2/error.log`. Press Shift+G to go to the end of the file.
 
 If it talks about failing to import something, google its name and install it.
 
@@ -36,13 +36,9 @@ Make sure the background processes are running: `ps aux | grep celery` should re
 If that doesn't do it, try clearing the worker queue. First kill all celery processes as described above, then do the following:
 
     redis-cli FLUSHALL
-    python manage.py celeryd --purge
+    celery -A alliance_auth worker --purge
 
-Press control+C once.
-
-    python manage.py celeryd --discard
-
-Press control+C once.
+Press Control+C once.
 
 Now start celery again with [these background process commands.](../installation/auth/quickstart.md)
 

eveonline/tasks.py

@@ -1,10 +1,8 @@
 from __future__ import unicode_literals
 from django.conf import settings
-from celery.task import periodic_task
 from django.contrib.auth.models import User
 from notifications import notify
 from celery import task
-from celery.task.schedules import crontab
 from authentication.models import AuthServicesInfo
 from eveonline.managers import EveManager
 from eveonline.models import EveApiKeyPair
@@ -17,10 +15,12 @@ from authentication.tasks import set_state
 import logging
 import evelink
 
+from alliance_auth.celeryapp import app
+
 logger = logging.getLogger(__name__)
 
 
-@task
+@app.task
 def refresh_api(api):
     logger.debug('Running update on api key %s' % api.api_id)
     still_valid = True
@@ -70,12 +70,12 @@ def refresh_api(api):
                    level="danger")
 
 
-@task
+@app.task
 def refresh_user_apis(user):
     logger.debug('Refreshing all APIs belonging to user %s' % user)
     apis = EveApiKeyPair.objects.filter(user=user)
     for x in apis:
-        refresh_api(x)
+        refresh_api.apply(args=(x,))
     # Check our main character
     auth = AuthServicesInfo.objects.get(user=user)
     if auth.main_char_id:
@@ -91,7 +91,7 @@ def refresh_user_apis(user):
     set_state(user)
 
 
-@periodic_task(run_every=crontab(minute=0, hour="*/3"))
+@app.task
 def run_api_refresh():
     if not EveApiManager.check_if_api_server_online():
         logger.warn("Aborted scheduled API key refresh: API server unreachable")
@@ -101,18 +101,18 @@ def run_api_refresh():
         refresh_user_apis.delay(u)
 
 
-@task
+@app.task
 def update_corp(id, is_blue=None):
     EveManager.update_corporation(id, is_blue=is_blue)
 
 
-@task
+@app.task
 def update_alliance(id, is_blue=None):
     EveManager.update_alliance(id, is_blue=is_blue)
     EveManager.populate_alliance(id)
 
 
-@periodic_task(run_every=crontab(minute=0, hour="*/2"))
+@app.task
 def run_corp_update():
     if not EveApiManager.check_if_api_server_online():
         logger.warn("Aborted updating corp and alliance models: API server unreachable")
@@ -123,7 +123,7 @@ def run_corp_update():
         is_blue = True if corp_id in settings.STR_BLUE_CORP_IDS else False
         try:
             if EveCorporationInfo.objects.filter(corporation_id=corp_id).exists():
-                update_corp(corp_id, is_blue=is_blue)
+                update_corp.apply(args=(corp_id,), kwargs={'is_blue': is_blue})
             else:
                 EveManager.create_corporation(corp_id, is_blue=is_blue)
         except ObjectNotFound:

eveonline/views.py

@@ -164,7 +164,7 @@ def user_refresh_api(request, api_id):
     if EveApiKeyPair.objects.filter(api_id=api_id).exists():
         api_key_pair = EveApiKeyPair.objects.get(api_id=api_id)
         if api_key_pair.user == request.user:
-            refresh_api(api_key_pair)
+            refresh_api.apply(args=(api_key_pair,))
             messages.success(request, _('Refreshed API key %(apiid)s') % {"apiid": api_id})
             set_state(request.user)
         else:

requirements.txt

@@ -9,6 +9,7 @@ slugify
 requests-oauthlib
 sleekxmpp
 redis
+celery>=4.0.2
 
 # Django Stuff #
 django>=1.10,<2.0
@@ -17,10 +18,7 @@ django-navhelper
 django-bootstrap-pagination
 django-redis>=4.4
 django-recaptcha
-
-# awating release for fix to celery/django-celery#447
-# django-celery
-git+https://github.com/celery/django-celery
+django-celery-beat
 
 # awating pyghassen/openfire-restapi #1 to fix installation issues
 git+https://github.com/adarnof/openfire-restapi

run_alliance_corp_update.py

@@ -1,5 +1,5 @@
 from __future__ import unicode_literals
 from eveonline.tasks import run_corp_update
 
-run_corp_update()
+run_corp_update.apply()
 quit()

thirdparty/Supervisor/auth-celerybeat.conf

@@ -1,5 +1,5 @@
 [program:auth-celerybeat]
-command=python manage.py celerybeat
+command=celery -A alliance_auth beat
 directory=/home/allianceserver/allianceauth
 user=allianceserver
 stdout_logfile=/home/allianceserver/allianceauth/log/beat.log

thirdparty/Supervisor/auth-celeryd.conf

@@ -1,5 +1,5 @@
 [program:auth-celeryd]
-command=python manage.py celeryd
+command=celery -A alliance_auth worker
 directory=/home/allianceserver/allianceauth
 user=allianceserver
 numprocs=1