V4.x Docker Refactoring and Docs

.gitlab-ci.yml

@@ -243,10 +243,10 @@ deploy_production:
 
 build-image:
   before_script: []
-  image: docker:20.10.10
+  image: docker:24.0
   stage: docker
   services:
-    - docker:20.10.10-dind
+    - docker:24.0-dind
   script: |
     CURRENT_DATE=$(echo $CI_COMMIT_TIMESTAMP | head -c 10 | tr -d -)
     IMAGE_TAG=$CI_REGISTRY_IMAGE/auth:$CURRENT_DATE-$CI_COMMIT_SHORT_SHA
@@ -256,12 +256,10 @@ build-image:
     LATEST_TAG=$CI_REGISTRY_IMAGE/auth:latest
 
     docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
-    docker build . -t $IMAGE_TAG -f docker/Dockerfile --build-arg AUTH_VERSION=$(echo $CI_COMMIT_TAG | cut -c 2-)
-    docker tag $IMAGE_TAG $CURRENT_TAG
-    docker tag $IMAGE_TAG $MINOR_TAG
-    docker tag $IMAGE_TAG $MAJOR_TAG
-    docker tag $IMAGE_TAG $LATEST_TAG
-    docker image push --all-tags $CI_REGISTRY_IMAGE/auth
+    docker run --privileged --rm tonistiigi/binfmt --uninstall qemu-*
+    docker run --privileged --rm tonistiigi/binfmt --install all
+    docker buildx create --use --name new-builder
+    docker buildx build . -t $IMAGE_TAG -t $MINOR_TAG -t $MAJOR_TAG -t $LATEST_TAG -f docker/Dockerfile --platform linux/amd64,linux/arm64 --push --build-arg AUTH_VERSION=$(echo $CI_COMMIT_TAG | cut -c 2-)
   rules:
     - if: $CI_COMMIT_TAG
       when: delayed
@@ -269,17 +267,19 @@ build-image:
 
 build-image-dev:
   before_script: []
-  image: docker:20.10.10
+  image: docker:24.0
   stage: docker
   services:
-    - docker:20.10.10-dind
+    - docker:24.0-dind
   script: |
     CURRENT_DATE=$(echo $CI_COMMIT_TIMESTAMP | head -c 10 | tr -d -)
     IMAGE_TAG=$CI_REGISTRY_IMAGE/auth:$CURRENT_DATE-$CI_COMMIT_BRANCH-$CI_COMMIT_SHORT_SHA
 
     docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
-    docker build . -t $IMAGE_TAG -f docker/Dockerfile --build-arg AUTH_PACKAGE=git+https://gitlab.com/allianceauth/allianceauth@$CI_COMMIT_BRANCH
-    docker push $IMAGE_TAG
+    docker run --privileged --rm tonistiigi/binfmt --uninstall qemu-*
+    docker run --privileged --rm tonistiigi/binfmt --install all
+    docker buildx create --use --name new-builder
+    docker buildx build . -t $IMAGE_TAG -f docker/Dockerfile --platform linux/amd64,linux/arm64 --push --build-arg AUTH_PACKAGE=git+https://gitlab.com/allianceauth/allianceauth@$CI_COMMIT_BRANCH
   rules:
     - if: '$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME == ""'
       when: manual
@@ -288,17 +288,19 @@ build-image-dev:
 
 build-image-mr:
   before_script: []
-  image: docker:20.10.10
+  image: docker:24.0
   stage: docker
   services:
-    - docker:20.10.10-dind
+    - docker:24.0-dind
   script: |
     CURRENT_DATE=$(echo $CI_COMMIT_TIMESTAMP | head -c 10 | tr -d -)
     IMAGE_TAG=$CI_REGISTRY_IMAGE/auth:$CURRENT_DATE-$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME-$CI_COMMIT_SHORT_SHA
 
     docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
-    docker build . -t $IMAGE_TAG -f docker/Dockerfile --build-arg AUTH_PACKAGE=git+$CI_MERGE_REQUEST_SOURCE_PROJECT_URL@$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME
-    docker push $IMAGE_TAG
+    docker run --privileged --rm tonistiigi/binfmt --uninstall qemu-*
+    docker run --privileged --rm tonistiigi/binfmt --install all
+    docker buildx create --use --name new-builder
+    docker buildx build . -t $IMAGE_TAG -f docker/Dockerfile --platform linux/amd64,linux/arm64 --push --build-arg AUTH_PACKAGE=git+$CI_MERGE_REQUEST_SOURCE_PROJECT_URL@$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME
   rules:
     - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
       when: manual

docker/Dockerfile

@@ -1,7 +1,6 @@
 FROM python:3.11-slim
 ARG AUTH_VERSION=v4.0.0a1
 ARG AUTH_PACKAGE=allianceauth==${AUTH_VERSION}
-ENV VIRTUAL_ENV=/opt/venv
 ENV AUTH_USER=allianceauth
 ENV AUTH_GROUP=allianceauth
 ENV AUTH_USERGROUP=${AUTH_USER}:${AUTH_GROUP}
@@ -12,37 +11,31 @@ ENV AUTH_HOME=/home/allianceauth
 SHELL ["/bin/bash", "-c"]
 RUN groupadd -g 61000 ${AUTH_GROUP}
 RUN useradd -g 61000 -l -M -s /bin/false -u 61000 ${AUTH_USER}
-RUN mkdir -p ${VIRTUAL_ENV} \
-    && chown ${AUTH_USERGROUP} ${VIRTUAL_ENV} \
-    && mkdir -p ${STATIC_BASE} \
+RUN mkdir -p ${STATIC_BASE} \
     && chown ${AUTH_USERGROUP} ${STATIC_BASE} \
     && mkdir -p ${AUTH_HOME} \
     && chown ${AUTH_USERGROUP} ${AUTH_HOME}
 
 # Install build dependencies
 RUN apt-get update && apt-get upgrade -y && apt-get install -y \
-    libmariadb-dev gcc supervisor git htop
-
-# Switch to non-root user
-USER ${AUTH_USER}
-RUN python3 -m venv $VIRTUAL_ENV
-ENV PATH="$VIRTUAL_ENV/bin:$PATH"
-WORKDIR ${AUTH_HOME}
+    libmariadb-dev gcc git
 
 # Install python dependencies
 RUN pip install --upgrade pip
 RUN pip install wheel gunicorn
 RUN pip install ${AUTH_PACKAGE}
 
+# Switch to non-root user
+USER ${AUTH_USER}
+WORKDIR ${AUTH_HOME}
+
 # Initialize auth
 RUN allianceauth start myauth
 COPY /allianceauth/project_template/project_name/settings/local.py ${AUTH_HOME}/myauth/myauth/settings/local.py
 RUN allianceauth update myauth
 RUN mkdir -p ${STATIC_BASE}/myauth/static
-COPY /docker/conf/supervisord.conf /etc/supervisor/conf.d/supervisord.conf
+
 RUN echo 'alias auth="python $AUTH_HOME/myauth/manage.py"' >> ~/.bashrc && \
-    echo 'alias supervisord="supervisord -c /etc/supervisor/conf.d/supervisord.conf"' >> ~/.bashrc && \
     source ~/.bashrc
 
-EXPOSE 8000
-CMD ["/usr/bin/supervisord", "-c", "/etc/supervisor/conf.d/supervisord.conf"]
+ENTRYPOINT ["sh", "-c"]

docker/conf/celery.py

@@ -0,0 +1,33 @@
+import os
+from celery import Celery
+from celery.app import trace
+
+# set the default Django settings module for the 'celery' program.
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'myauth.settings.local')
+
+from django.conf import settings  # noqa
+
+app = Celery('myauth')
+
+# Using a string here means the worker don't have to serialize
+# the configuration object to child processes.
+app.config_from_object('django.conf:settings')
+
+# setup priorities ( 0 Highest, 9 Lowest )
+app.conf.broker_transport_options = {
+    'priority_steps': list(range(10)),  # setup que to have 10 steps
+    'queue_order_strategy': 'priority',  # setup que to use prio sorting
+}
+app.conf.task_default_priority = 5  # anything called with the task.delay() will be given normal priority (5)
+app.conf.worker_prefetch_multiplier = 1  # only prefetch single tasks at a time on the workers so that prio tasks happen
+
+app.conf.ONCE = {
+    'backend': 'allianceauth.services.tasks.DjangoBackend',
+    'settings': {}
+}
+
+# Load task modules from all registered Django app configs.
+app.autodiscover_tasks(lambda: settings.INSTALLED_APPS)
+
+# Remove result from default log message on task success
+trace.LOG_SUCCESS = "Task %(name)s[%(id)s] succeeded in %(runtime)ss"

docker/conf/memory_check.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+max_mem=$1
+cur_mem=$(</sys/fs/cgroup/memory.current)
+health_file="/tmp/health.stat"
+if [ -f "$health_file" ]; then
+    echo "$health_file exists."
+else
+    echo "$health_file does not exist. Creating"
+    echo 0 > "$health_file"
+fi
+health=$(<$health_file)
+echo "Testing Mem: $cur_mem / $max_mem"
+if [[ max_mem -gt cur_mem ]]
+then
+    echo 0 > "$health_file"
+    echo "All Ok"
+    exit 0
+else
+    new_val=$((1+$health))
+    echo "Un-healthy! Check #$new_val"
+    echo $new_val > "$health_file"
+    if (($new_val > 3)); then
+        echo "Starting a restart of this the container..."
+        kill -SIGTERM 1
+    fi
+    exit 1
+fi

docker/conf/nginx.conf

@@ -11,7 +11,7 @@ server {
     }
 
     location / {
-        proxy_pass http://allianceauth:8000;
+        proxy_pass http://allianceauth_gunicorn:8000;
         proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
         proxy_set_header Host $host;
         proxy_set_header  X-Real-IP   $remote_addr;

docker/conf/supervisord.conf

@@ -1,56 +0,0 @@
-[supervisord]
-nodaemon=true
-user=allianceauth
-
-[program:beat]
-command=/opt/venv/bin/celery -A myauth beat
-directory=/home/allianceauth/myauth
-user=allianceauth
-stdout_logfile=/dev/stdout
-stderr_logfile=/dev/stderr
-stdout_logfile_maxbytes=0
-stderr_logfile_maxbytes=0
-autostart=true
-autorestart=true
-startsecs=10
-priority=998
-stdout_events_enabled=true
-stderr_events_enabled=true
-
-[program:worker]
-command=/opt/venv/bin/celery -A myauth worker -l INFO --max-tasks-per-child=250
-directory=/home/allianceauth/myauth
-user=allianceauth
-stdout_logfile=/dev/stdout
-stderr_logfile=/dev/stderr
-stdout_logfile_maxbytes=0
-stderr_logfile_maxbytes=0
-numprocs=1
-autostart=true
-autorestart=true
-startsecs=10
-stopwaitsecs = 600
-killasgroup=true
-priority=998
-stdout_events_enabled=true
-stderr_events_enabled=true
-
-[program:gunicorn]
-user=allianceauth
-directory=/home/allianceauth/myauth
-command=/opt/venv/bin/gunicorn myauth.wsgi --bind :8000 --workers=3 --timeout 120
-stdout_logfile=/dev/stdout
-stderr_logfile=/dev/stderr
-stdout_logfile_maxbytes=0
-stderr_logfile_maxbytes=0
-stdout_events_enabled=true
-stderr_events_enabled=true
-autostart=true
-autorestart=true
-stopsignal=INT
-
-[group:myauth]
-programs=beat,worker,gunicorn
-priority=999
-
-[supervisorctl]

docker/conf/urls.py

@@ -0,0 +1,11 @@
+from allianceauth import urls
+from django.urls import include, path
+
+urlpatterns = [
+    path('', include(urls)),
+]
+
+handler500 = 'allianceauth.views.Generic500Redirect'
+handler404 = 'allianceauth.views.Generic404Redirect'
+handler403 = 'allianceauth.views.Generic403Redirect'
+handler400 = 'allianceauth.views.Generic400Redirect'

docker/docker-compose.yml

@@ -1,8 +1,45 @@
 version: '3.8'
 
+x-allianceauth-base: &allianceauth-base
+  image: ${AA_DOCKER_TAG?err}
+  # build:
+  #   context: .
+  #   dockerfile: custom.dockerfile
+  #   args:
+  #     AA_DOCKER_TAG: ${AA_DOCKER_TAG?err}
+  restart: always
+  env_file:
+    - ./.env
+  volumes:
+    - ./conf/local.py:/home/allianceauth/myauth/myauth/settings/local.py
+    - ./conf/celery.py:/home/allianceauth/myauth/myauth/celery.py
+    - ./conf/urls.py:/home/allianceauth/myauth/myauth/urls.py
+    - ./conf/memory_check.sh:/memory_check.sh
+    - ./templates:/home/allianceauth/myauth/myauth/templates/
+    - static-volume:/var/www/myauth/static
+  depends_on:
+    - redis
+    - auth_mysql
+  working_dir: /home/allianceauth/myauth/
+  stop_grace_period: 10m
+
+x-allianceauth-health-check: &allianceauth-health-checks
+  healthcheck:
+    test: [
+      "CMD",
+      "/memory_check.sh",
+      "500000000"
+    ]
+    interval: 60s
+    timeout: 10s
+    retries: 3
+    start_period: 5m
+  labels:
+    - "autoheal=true"
+
 services:
   auth_mysql:
-    image: mysql:8.0
+    image: mariadb:10.11
     command: [mysqld, --character-set-server=utf8mb4, --collation-server=utf8mb4_unicode_ci, --default-authentication-plugin=mysql_native_password]
     volumes:
       - ./mysql-data:/var/lib/mysql
@@ -23,7 +60,7 @@ services:
       - ./conf/nginx.conf:/etc/nginx/conf.d/default.conf
       - static-volume:/var/www/myauth/static
     depends_on:
-      - allianceauth
+      - allianceauth_gunicorn
 
   redis:
     image: redis:7.0
@@ -32,24 +69,45 @@ services:
     volumes:
       - "redis-data:/data"
 
-  allianceauth:
-    image: ${AA_DOCKER_TAG?err}
-    # build:
-    #   context: .
-    #   dockerfile: custom.dockerfile
-    #   args:
-    #     AA_DOCKER_TAG: ${AA_DOCKER_TAG?err}
-    restart: always
-    env_file:
-      - ./.env
-    volumes:
-      - ./conf/local.py:/home/allianceauth/myauth/myauth/settings/local.py
-      - ./templates:/home/allianceauth/myauth/myauth/templates/
-      - ./conf/supervisord.conf:/etc/supervisor/conf.d/supervisord.conf
-      - static-volume:/var/www/myauth/static
-    depends_on:
-      - redis
-      - auth_mysql
+  allianceauth_gunicorn:
+    ports:
+      - 8000:8000
+    container_name: allianceauth_gunicorn
+    <<: [*allianceauth-base]
+    entrypoint: [
+      "/opt/venv/bin/gunicorn",
+      "myauth.wsgi",
+      "--bind=0.0.0.0:8000",
+      "--workers=3",
+      "--timeout=120",
+      "--max-requests=500",
+      "--max-requests-jitter=50"
+    ]
+
+  allianceauth_beat:
+    container_name: auth_worker_beat
+    <<: [*allianceauth-base]
+    entrypoint: [
+      "/opt/venv/bin/celery",
+      "-A",
+      "myauth",
+      "beat"
+    ]
+
+  allianceauth_worker:
+    <<: [*allianceauth-base, *allianceauth-health-checks]
+    entrypoint: [
+      "/opt/venv/bin/celery",
+      "-A",
+      "myauth",
+      "worker",
+      "--pool=threads",
+      "--concurrency=5",
+      "-n",
+      "worker_%n"
+    ]
+    deploy:
+      replicas: 2
 
   grafana:
     image: grafana/grafana-oss:9.5.2
@@ -58,6 +116,7 @@ services:
       - auth_mysql
     volumes:
       - grafana-data:/var/lib/grafana
+
   proxy:
     image: 'jc21/nginx-proxy-manager:latest'
     restart: always
@@ -74,6 +133,7 @@ services:
     volumes:
       - proxy-data:/data
       - proxy-le:/etc/letsencrypt
+
   proxy-db:
     image: 'jc21/mariadb-aria:latest'
     restart: always

docs/index.md

@@ -13,6 +13,7 @@ Welcome to the official documentation for **Alliance Auth**!
     :caption: Contents
 
     installation/index
+    installation containerized/index
     features/index
     maintenance/index
     support/index

docs/installation containerized/docker.md

@@ -0,0 +1,101 @@
+# Installation -- Docker
+
+## Prerequisites
+
+You should have the following available on the system you are using to set this up:
+
+* Docker - <https://docs.docker.com/get-docker/>
+* git
+* curl
+
+## Setup Guide
+
+1. run `bash <(curl -s https://gitlab.com/allianceauth/allianceauth/-/raw/master/docker/scripts/download.sh)`. This will download all the files you need to install Alliance Auth and place them in a directory named `aa-docker`. Feel free to rename/move this folder.
+1. run `./scripts/prepare-env.sh` to set up your environment
+1. (optional) Change `PROTOCOL` to `http://` if not using SSL in `.env`
+1. run `docker-compose --env-file=.env up -d` (NOTE: if this command hangs, follow the instructions [here](https://www.digitalocean.com/community/tutorials/how-to-setup-additional-entropy-for-cloud-servers-using-haveged))
+1. run `docker-compose exec allianceauth bash` to open up a terminal inside your auth container
+1. run `auth migrate`
+1. run `auth collectstatic`
+1. run `auth createsuperuser`
+1. visit <http://yourdomain:81> to set up nginx proxy manager (NOTE: if this doesn't work, the machine likely has a firewall. You'll want to open up ports 80,443, and 81. [Instructions for ufw](https://www.digitalocean.com/community/tutorials/ufw-essentials-common-firewall-rules-and-commands))
+1. login with user `admin@example.com` and password `changeme`, then update your password as requested
+1. click on "Proxy Hosts"
+1. click "Add Proxy Host", with the following settings for auth. The example uses `auth.localhost` for the domain, but you'll want to use whatever address you have auth configured on
+  ![nginx-host](/_static/images/installation/dokcer/nginx-host.png)
+1. click "Add Proxy Host", with the following settings for grafana. The example uses `grafana.localhost` for the domain
+  ![grafana-host](/_static/images/installation/dokcer/grafana-host.png))
+
+Congrats! You should now see auth running at <http://auth.yourdomain> and grafana at <http://grafana.yourdomain>!
+
+## SSL Guide
+
+Unless you're running auth locally in docker for testing, you should be using SSL.
+Thankfully, setting up SSL in nginx Proxy Manager takes about three clicks.
+
+1. Edit your existing proxy host, and go to the SSL tab. Select "Request a new SSL Certificate" from the drop down.
+1. Now, enable "Force SSL" and "HTTP/2 Support". (NOTE: Do not enable HSTS unless you know what you're doing. This will force your domains to only work with SSL enabled, and is cached extremely hard in browsers. )
+  ![proxy-manager-ssl](/_static/images/installation/dokcer/proxy-manager-ssl.png)
+1. (optional) select "Use a DNS Challenge". This is not a required option, but it is recommended if you use a supported DNS provider. You'll then be asked for an API key for the provider you choose. If you use Cloudflare, you'll probably have issues getting SSL certs unless you use a DNS Challenge.
+1. The email address here will be used to notify you if there are issues renewing your certificates.
+1. Repeat for any other services, like grafana.
+
+That's it! You should now be able to access your auth install at <https://auth.yourdomain>
+
+## Adding extra packages
+
+There are a handful of ways to add packages:
+
+* Running `pip install` in the container
+* Modifying the container's initial command to install packages
+* Building a custom Docker image (recommended, and less scary than it sounds!)
+
+### Using a custom docker image
+
+Using a custom docker image is the preferred approach, as it gives you the stability of packages only changing when you tell them to, along with packages not having to be downloaded every time your container restarts
+
+1. Add each additional package that you want to install to a single line in `conf/requirements.txt`. It is recommended, but not required, that you include a version number as well. This will keep your packages from magically updating. You can lookup packages on <https://package.wiki>, and copy everything after `pip install` from the top of the page to use the most recent version. It should look something like `allianceauth-signal-pings==0.0.7`. Every entry in this file should be on a separate line
+1. In `docker-compose.yml`, comment out the `image` line under `allianceauth` (line 36... ish) and uncomment the `build` section
+1. run `docker-compose --env-file=.env up -d`, your custom container will be built, and auth will have your new packages. Make sure to follow the package's instructions on config values that go in `local.py`
+1. run `docker-compose exec allianceauth bash` to open up a terminal inside your auth container
+1. run `allianceauth update myauth`
+1. run `auth migrate`
+1. run `auth collectstatic`
+
+_NOTE: It is recommended that you put any secret values (API keys, database credentials, etc) in an environment variable instead of hardcoding them into `local.py`. This gives you the ability to track your config in git without committing passwords. To do this, just add it to your `.env` file, and then reference in `local.py` with `os.environ.get("SECRET_NAME")`_
+
+## Updating Auth
+
+### Base Image
+
+Whether you're using a custom image or not, the version of auth is dictated by $AA_DOCKER_TAG in your `.env` file.
+
+1. To update to a new version of auth, update the version number at the end (or replace the whole value with the tag in the release notes).
+1. run `docker-compose pull`
+1. run `docker-compose --env-file=.env up -d`
+1. run `docker-compose exec allianceauth bash` to open up a terminal inside your auth container
+1. run `allianceauth update myauth`
+1. run `auth migrate`
+1. run `auth collectstatic`
+
+_NOTE: If you specify a version of allianceauth in your `requirements.txt` in a custom image it will override the version from the base image. Not recommended unless you know what you're doing_
+
+### Custom Packages
+
+1. Update the versions in your `requirements.txt` file
+1. Run `docker-compose build`
+1. Run `docker-compose --env-file=.env up -d`
+
+## Notes
+
+### Apple M1 Support
+
+If you want to run locally on an M1 powered Apple device, you'll need to add `platform: linux/x86_64` under each container in `docker-compose.yml` as the auth container is not compiled for ARM (other containers may work without this, but it's known to work if added to all containers).
+
+Example:
+
+```yaml
+  redis:
+    platform: linux/x86_64
+    image: redis:6.2
+```

docs/installation containerized/index.md

@@ -0,0 +1,17 @@
+# Installation
+
+This document describes how to install **Alliance Auth** using various Containerization techniques.
+
+If you would like to install on Bare Metal instead see :doc:`/installation/index` instead
+
+```eval_rst
+.. note::
+    There are additional installation steps for activating services and apps that come with **Alliance Auth**. Please see the page for the respective service or apps in chapter :doc:`/features/index` for details.
+```
+
+```eval_rst
+.. toctree::
+    :maxdepth: 1
+
+    docker
+```