diff --git a/alliance_auth/settings.py.example b/alliance_auth/settings.py.example index 3454c46b..60eff347 100644 --- a/alliance_auth/settings.py.example +++ b/alliance_auth/settings.py.example @@ -12,17 +12,12 @@ https://docs.djangoproject.com/en/1.10/ref/settings/ import os -import djcelery - from django.contrib import messages from celery.schedules import crontab -djcelery.setup_loader() - # Celery configuration BROKER_URL = 'redis://localhost:6379/0' -CELERYBEAT_SCHEDULER = "djcelery.schedulers.DatabaseScheduler" -CELERYBEAT_SCHEDULE = dict() +CELERYBEAT_SCHEDULER = "django_celery_beat.schedulers.DatabaseScheduler" # Build paths inside the project like this: os.path.join(BASE_DIR, ...) BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) @@ -50,7 +45,7 @@ INSTALLED_APPS = [ 'django.contrib.messages', 'django.contrib.staticfiles', 'django.contrib.humanize', - 'djcelery', + 'django_celery_beat', 'bootstrapform', 'authentication', 'services', @@ -237,6 +232,25 @@ NOCAPTCHA = True ## ##################################################### + +######################### +# CELERY SCHEDULED TASKS +######################### +CELERYBEAT_SCHEDULE = { + 'run_api_refresh': { + 'task': 'eveonline.tasks.run_api_refresh', + 'schedule': crontab(minute=0, hour="*/3"), + }, + 'run_corp_update': { + 'task': 'eveonline.tasks.run_corp_update', + 'schedule': crontab(minute=0, hour="*/2"), + }, + 'update_all_corpstats': { + 'task': 'corputils.tasks.update_all_corpstats', + 'schedule': crontab(minute=0, hour="*/6"), + }, +} + ################# # EMAIL SETTINGS ################# diff --git a/alliance_auth/tests/test_settings.py b/alliance_auth/tests/test_settings.py index d4720472..5c3c6b3a 100644 --- a/alliance_auth/tests/test_settings.py +++ b/alliance_auth/tests/test_settings.py @@ -4,14 +4,10 @@ Alliance Auth Test Suite Django settings. import os -import djcelery - from django.contrib import messages import alliance_auth -djcelery.setup_loader() - # Use nose to run all tests TEST_RUNNER = 'django_nose.NoseTestSuiteRunner' @@ -40,7 +36,7 @@ INSTALLED_APPS = [ 'django.contrib.messages', 'django.contrib.staticfiles', 'django.contrib.humanize', - 'djcelery', + 'django_celery_beat', 'bootstrapform', 'authentication', 'services', diff --git a/corputils/tasks.py b/corputils/tasks.py index 2dbb7197..8aa63ef1 100644 --- a/corputils/tasks.py +++ b/corputils/tasks.py @@ -1,15 +1,14 @@ from corputils.models import CorpStats -from celery.task import task, periodic_task -from celery.task.schedules import crontab +from alliance_auth.celeryapp import app -@task +@app.task def update_corpstats(pk): cs = CorpStats.objects.get(pk=pk) cs.update() -@periodic_task(run_every=crontab(minute=0, hour="*/6")) +@app.task def update_all_corpstats(): for cs in CorpStats.objects.all(): update_corpstats.delay(cs.pk) diff --git a/docs/installation/auth/quickstart.md b/docs/installation/auth/quickstart.md index 2d20cce6..ca409485 100644 --- a/docs/installation/auth/quickstart.md +++ b/docs/installation/auth/quickstart.md @@ -10,5 +10,5 @@ The big goal of AllianceAuth is the automation of group membership, so we’ll n To start the background processes to sync groups and check api keys, issue these commands: - screen -dm bash -c 'python manage.py celeryd' - screen -dm bash -c 'python manage.py celerybeat' + screen -dm bash -c 'celery -A alliance_auth worker' + screen -dm bash -c 'celery -A alliance_auth beat' diff --git a/docs/installation/auth/supervisor.md b/docs/installation/auth/supervisor.md index 8d89d14e..6e71ec8a 100644 --- a/docs/installation/auth/supervisor.md +++ b/docs/installation/auth/supervisor.md @@ -20,7 +20,7 @@ CentOS: ## Configuration -Auth provides example config files for the celery workers (celeryd), the periodic task scheduler (celerybeat), and the mumble authenticator. All of these are available in `thirdparty/Supervisor`. +Auth provides example config files for the celery workers, the periodic task scheduler (celery beat), and the mumble authenticator. All of these are available in `thirdparty/Supervisor`. For most users, all you have to do is copy the config files to `/etc/supervisor/conf.d` then restart the service. Copy `auth-celerybeat.conf` and `auth-celeryd.conf` for the celery workers, and `auth-mumble.conf` for the mumble authenticator. For all three just use a wildcard: @@ -41,15 +41,15 @@ To ensure the processes are working, check their status: sudo supervisorctl status Processes will be `STARTING`, `RUNNING`, or `ERROR`. If an error has occurred, check their log files: - - celeryd: `log/worker.log` - - celerybeat: `log/beat.log` + - celery workers: `log/worker.log` + - celery beat: `log/beat.log` - authenticator: `log/authenticator.log` ## Customizing Config Files The only real customization needed is if running in a virtual environment. The python path will have to be changed in order to start in the venv. -Edit the config files and find the line saying `command`. Replace `python` with `/path/to/venv/python`. This can be relative to the `directory` specified in the config file. +Edit the config files and find the line saying `command`. Replace `python` with `/path/to/venv/bin/python`. For Celery replace `celery` with `/path/to/venv/bin/celery`. This can be relative to the `directory` specified in the config file. Note that for config changes to be loaded, the supervisor service must be restarted. diff --git a/docs/installation/services/teamspeak3.md b/docs/installation/services/teamspeak3.md index 8ff38838..b98928fa 100644 --- a/docs/installation/services/teamspeak3.md +++ b/docs/installation/services/teamspeak3.md @@ -84,7 +84,7 @@ To enable advanced permissions, on your client go to the `Tools` menu, `Applicat ### TS group models not populating on admin site The method which populates these runs every 30 minutes. To populate manually, start a celery shell: - python manage.py celery shell + celery -A alliance_auth shell And execute the update: diff --git a/docs/maintenance/troubleshooting.md b/docs/maintenance/troubleshooting.md index 6cd94466..1330932a 100644 --- a/docs/maintenance/troubleshooting.md +++ b/docs/maintenance/troubleshooting.md @@ -19,7 +19,7 @@ Either you need to `sudo` that command, or it's a missing dependency. Check [the ### I'm getting an error 500 trying to connect to the website on a new install -Read the apache error log: `sudo nano /var/log/apache2/error.log` +Read the apache error log: `sudo less /var/log/apache2/error.log`. Press Shift+G to go to the end of the file. If it talks about failing to import something, google its name and install it. @@ -36,13 +36,9 @@ Make sure the background processes are running: `ps aux | grep celery` should re If that doesn't do it, try clearing the worker queue. First kill all celery processes as described above, then do the following: redis-cli FLUSHALL - python manage.py celeryd --purge + celery -A alliance_auth worker --purge -Press control+C once. - - python manage.py celeryd --discard - -Press control+C once. +Press Control+C once. Now start celery again with [these background process commands.](../installation/auth/quickstart.md) diff --git a/eveonline/tasks.py b/eveonline/tasks.py index d15ab692..bce37a33 100644 --- a/eveonline/tasks.py +++ b/eveonline/tasks.py @@ -1,10 +1,8 @@ from __future__ import unicode_literals from django.conf import settings -from celery.task import periodic_task from django.contrib.auth.models import User from notifications import notify from celery import task -from celery.task.schedules import crontab from authentication.models import AuthServicesInfo from eveonline.managers import EveManager from eveonline.models import EveApiKeyPair @@ -17,10 +15,12 @@ from authentication.tasks import set_state import logging import evelink +from alliance_auth.celeryapp import app + logger = logging.getLogger(__name__) -@task +@app.task def refresh_api(api): logger.debug('Running update on api key %s' % api.api_id) still_valid = True @@ -70,12 +70,12 @@ def refresh_api(api): level="danger") -@task +@app.task def refresh_user_apis(user): logger.debug('Refreshing all APIs belonging to user %s' % user) apis = EveApiKeyPair.objects.filter(user=user) for x in apis: - refresh_api(x) + refresh_api.apply(args=(x,)) # Check our main character auth = AuthServicesInfo.objects.get(user=user) if auth.main_char_id: @@ -91,7 +91,7 @@ def refresh_user_apis(user): set_state(user) -@periodic_task(run_every=crontab(minute=0, hour="*/3")) +@app.task def run_api_refresh(): if not EveApiManager.check_if_api_server_online(): logger.warn("Aborted scheduled API key refresh: API server unreachable") @@ -101,18 +101,18 @@ def run_api_refresh(): refresh_user_apis.delay(u) -@task +@app.task def update_corp(id, is_blue=None): EveManager.update_corporation(id, is_blue=is_blue) -@task +@app.task def update_alliance(id, is_blue=None): EveManager.update_alliance(id, is_blue=is_blue) EveManager.populate_alliance(id) -@periodic_task(run_every=crontab(minute=0, hour="*/2")) +@app.task def run_corp_update(): if not EveApiManager.check_if_api_server_online(): logger.warn("Aborted updating corp and alliance models: API server unreachable") @@ -123,7 +123,7 @@ def run_corp_update(): is_blue = True if corp_id in settings.STR_BLUE_CORP_IDS else False try: if EveCorporationInfo.objects.filter(corporation_id=corp_id).exists(): - update_corp(corp_id, is_blue=is_blue) + update_corp.apply(args=(corp_id,), kwargs={'is_blue': is_blue}) else: EveManager.create_corporation(corp_id, is_blue=is_blue) except ObjectNotFound: diff --git a/eveonline/views.py b/eveonline/views.py index 6a0610a9..b4f46e4d 100755 --- a/eveonline/views.py +++ b/eveonline/views.py @@ -164,7 +164,7 @@ def user_refresh_api(request, api_id): if EveApiKeyPair.objects.filter(api_id=api_id).exists(): api_key_pair = EveApiKeyPair.objects.get(api_id=api_id) if api_key_pair.user == request.user: - refresh_api(api_key_pair) + refresh_api.apply(args=(api_key_pair,)) messages.success(request, _('Refreshed API key %(apiid)s') % {"apiid": api_id}) set_state(request.user) else: diff --git a/requirements.txt b/requirements.txt index 14fbe899..a5028544 100755 --- a/requirements.txt +++ b/requirements.txt @@ -9,6 +9,7 @@ slugify requests-oauthlib sleekxmpp redis +celery>=4.0.2 # Django Stuff # django>=1.10,<2.0 @@ -17,10 +18,7 @@ django-navhelper django-bootstrap-pagination django-redis>=4.4 django-recaptcha - -# awating release for fix to celery/django-celery#447 -# django-celery -git+https://github.com/celery/django-celery +django-celery-beat # awating pyghassen/openfire-restapi #1 to fix installation issues git+https://github.com/adarnof/openfire-restapi diff --git a/run_alliance_corp_update.py b/run_alliance_corp_update.py index 938961d9..db3d5967 100644 --- a/run_alliance_corp_update.py +++ b/run_alliance_corp_update.py @@ -1,5 +1,5 @@ from __future__ import unicode_literals from eveonline.tasks import run_corp_update -run_corp_update() +run_corp_update.apply() quit() diff --git a/thirdparty/Supervisor/auth-celerybeat.conf b/thirdparty/Supervisor/auth-celerybeat.conf index 7832f22b..527fd7e2 100644 --- a/thirdparty/Supervisor/auth-celerybeat.conf +++ b/thirdparty/Supervisor/auth-celerybeat.conf @@ -1,5 +1,5 @@ [program:auth-celerybeat] -command=python manage.py celerybeat +command=celery -A alliance_auth beat directory=/home/allianceserver/allianceauth user=allianceserver stdout_logfile=/home/allianceserver/allianceauth/log/beat.log diff --git a/thirdparty/Supervisor/auth-celeryd.conf b/thirdparty/Supervisor/auth-celeryd.conf index e00f37ec..033cdc88 100644 --- a/thirdparty/Supervisor/auth-celeryd.conf +++ b/thirdparty/Supervisor/auth-celeryd.conf @@ -1,5 +1,5 @@ [program:auth-celeryd] -command=python manage.py celeryd +command=celery -A alliance_auth worker directory=/home/allianceserver/allianceauth user=allianceserver numprocs=1