Merge branch 'refresh-task-queue' into 'master'

[CHANGE] Frequently update task queue section on dashboard

See merge request allianceauth/allianceauth!1752

allianceauth/authentication/tests/test_views.py

@@ -9,6 +9,7 @@ from allianceauth.tests.auth_utils import AuthUtils
 from allianceauth.authentication.constants import ESI_ERROR_MESSAGE_OVERRIDES
 
 MODULE_PATH = "allianceauth.authentication.views"
+TEMPLATETAGS_PATH = "allianceauth.templatetags.admin_status"
 
 
 def jsonresponse_to_dict(response) -> dict:
@@ -17,6 +18,7 @@ def jsonresponse_to_dict(response) -> dict:
 
 @patch(MODULE_PATH + ".queued_tasks_count")
 @patch(MODULE_PATH + ".active_tasks_count")
+@patch(MODULE_PATH + "._celery_stats")
 class TestRunningTasksCount(TestCase):
     @classmethod
     def setUpClass(cls) -> None:
@@ -26,36 +28,64 @@ class TestRunningTasksCount(TestCase):
         cls.user.is_superuser = True
         cls.user.save()
 
-    def test_should_return_data(
+    def test_should_return_data(self, mock_celery_stats, mock_tasks_queued, mock_tasks_running):
-        self, mock_active_tasks_count, mock_queued_tasks_count
-    ):
         # given
-        mock_active_tasks_count.return_value = 2
+        mock_tasks_running.return_value = 2
-        mock_queued_tasks_count.return_value = 3
+        mock_tasks_queued.return_value = 3
+        mock_celery_stats.return_value = {
+            "tasks_succeeded": 5,
+            "tasks_retried": 1,
+            "tasks_failed": 4,
+            "tasks_total": 11,
+            "tasks_hours": 24,
+            "earliest_task": "2025-08-14T22:47:54.853Z",
+        }
+
         request = self.factory.get("/")
         request.user = self.user
+
         # when
         response = task_counts(request)
+
         # then
         self.assertEqual(response.status_code, 200)
         self.assertDictEqual(
-            jsonresponse_to_dict(response), {
+            jsonresponse_to_dict(response),
-                "tasks_running": 2, "tasks_queued": 3}
+            {
+                "tasks_succeeded": 5,
+                "tasks_retried": 1,
+                "tasks_failed": 4,
+                "tasks_total": 11,
+                "tasks_hours": 24,
+                "earliest_task": "2025-08-14T22:47:54.853Z",
+                "tasks_running": 3,
+                "tasks_queued": 2,
+            }
         )
 
-    def test_su_only(
+    def test_su_only(self, mock_celery_stats, mock_tasks_queued, mock_tasks_running):
-        self, mock_active_tasks_count, mock_queued_tasks_count
-    ):
         self.user.is_superuser = False
         self.user.save()
         self.user.refresh_from_db()
+
         # given
-        mock_active_tasks_count.return_value = 2
+        mock_tasks_running.return_value = 2
-        mock_queued_tasks_count.return_value = 3
+        mock_tasks_queued.return_value = 3
+        mock_celery_stats.return_value = {
+            "tasks_succeeded": 5,
+            "tasks_retried": 1,
+            "tasks_failed": 4,
+            "tasks_total": 11,
+            "tasks_hours": 24,
+            "earliest_task": "2025-08-14T22:47:54.853Z",
+        }
+
         request = self.factory.get("/")
         request.user = self.user
+
         # when
         response = task_counts(request)
+
         # then
         self.assertEqual(response.status_code, 302)
 

allianceauth/authentication/views.py

@@ -27,6 +27,7 @@ from allianceauth.hooks import get_hooks
 
 from .constants import ESI_ERROR_MESSAGE_OVERRIDES
 from .core.celery_workers import active_tasks_count, queued_tasks_count
+from allianceauth.templatetags.admin_status import _celery_stats
 from .forms import RegistrationForm
 from .models import CharacterOwnership
 
@@ -370,10 +371,10 @@ def registration_closed(request):
 @user_passes_test(lambda u: u.is_superuser)
 def task_counts(request) -> JsonResponse:
     """Return task counts as JSON for an AJAX call."""
-    data = {
+    data = _celery_stats()
-        "tasks_running": active_tasks_count(),
+    data.update(
-        "tasks_queued": queued_tasks_count()
+        {"tasks_running": active_tasks_count(), "tasks_queued": queued_tasks_count()}
-    }
+    )
     return JsonResponse(data)
 
 

allianceauth/templates/allianceauth/admin-status/celery_bar_partial.html

@@ -2,6 +2,7 @@
 {% load admin_status %}
 
 <div
+    id="celery-progress-bar-{{ label }}"
     class="progress-bar text-bg-{{ level }} task-status-progress-bar"
     role="progressbar"
     aria-valuenow="{% decimal_widthratio tasks_count tasks_total 100 %}"
@@ -9,5 +10,5 @@
     aria-valuemax="100"
     style="width: {% decimal_widthratio tasks_count tasks_total 100 %}%;"
 >
-    <span>{% widthratio tasks_count tasks_total 100 %}%</span>
+    <span id="celery-progress-bar-{{ label }}-progress">{% widthratio tasks_count tasks_total 100 %}%</span>
 </div>

allianceauth/templates/allianceauth/admin-status/overview.html

@@ -1,6 +1,27 @@
 {% load i18n %}
 {% load humanize %}
 
+{% get_current_language as LANGUAGE_CODE %}
+
+{% comment %}
+Some translations used in the HTML and JavaScript code below.
+We define them here so that they can be used in the JavaScript code as well with
+the escapejs filter without having to redefine them later.
+{% endcomment %}
+{% translate "second" as l10nSecondSingular %}
+{% translate "seconds" as l10nSecondPlural %}
+{% translate "minute" as l10nMinuteSingular %}
+{% translate "minutes" as l10nMinutePlural %}
+{% translate "hour" as l10nHourSingular %}
+{% translate "hours" as l10nHourPlural %}
+{% translate "N/A" as l10nNA %}
+{% translate "ERROR" as l10nError %}
+{% translate "running" as l10nRunning %}
+{% translate "queued" as l10nQueued %}
+{% translate "succeeded" as l10nSucceeded %}
+{% translate "retried" as l10nRetried %}
+{% translate "failed" as l10nFailed %}
+
 {% if debug %}
     <div id="aa-dashboard-panel-debug" class="col-12 mb-3">
         <div class="card text-bg-warning">
@@ -111,23 +132,27 @@
                 <div>
                     <p>
                         {% blocktranslate with total=tasks_total|intcomma latest=earliest_task|timesince|default:"?" %}
-                            Status of {{ total }} processed tasks • last {{ latest }}
+                            Status of <span id="total-task-count">?</span> processed tasks • last <span id="celery-uptime">?</span>
                         {% endblocktranslate %}
                     </p>
 
                     <div
-                        class="progress"
+                        id="celery-tasks-progress-bar"
+                        class="progress mb-2"
                         style="height: 21px;"
-                        title="{{ tasks_succeeded|intcomma }} succeeded, {{ tasks_retried|intcomma }} retried, {{ tasks_failed|intcomma }} failed"
+                        title="? {{ l10nSucceeded }}, ? {{ l10nRetried }}, ? {{ l10nFailed }}"
                     >
-                        {% include "allianceauth/admin-status/celery_bar_partial.html" with label="suceeded" level="success" tasks_count=tasks_succeeded %}
+                        {% include "allianceauth/admin-status/celery_bar_partial.html" with label="succeeded" level="success" tasks_count=0 %}
-                        {% include "allianceauth/admin-status/celery_bar_partial.html" with label="retried" level="info" tasks_count=tasks_retried %}
+                        {% include "allianceauth/admin-status/celery_bar_partial.html" with label="retried" level="info" tasks_count=0 %}
-                        {% include "allianceauth/admin-status/celery_bar_partial.html" with label="failed" level="danger" tasks_count=tasks_failed %}
+                        {% include "allianceauth/admin-status/celery_bar_partial.html" with label="failed" level="danger" tasks_count=0 %}
                     </div>
 
                     <p>
-                        <span id="task-counts">?</span> {% translate 'running' %} |
+                        <span id="running-task-count">?</span> {{ l10nRunning }} |
-                        <span id="queued-tasks-count">?</span> {% translate 'queued' %}
+                        <span id="queued-tasks-count">?</span> {{ l10nQueued }} |
+                        <span id="succeeded-tasks-count">?</span> {{ l10nSucceeded }} |
+                        <span id="retried-tasks-count">?</span> {{ l10nRetried }} |
+                        <span id="failed-tasks-count">?</span> {{ l10nFailed }}
                     </p>
                 </div>
             </div>
@@ -136,24 +161,206 @@
 </div>
 
 <script>
-    const elemRunning = document.getElementById('task-counts');
+    const elements = {
-    const elemQueued = document.getElementById('queued-tasks-count');
+        total: document.getElementById('total-task-count'),
-
+        uptime: document.getElementById('celery-uptime'),
-    fetchGet({url: '{% url "authentication:task_counts" %}'})
+        running: document.getElementById('running-task-count'),
-        .then((data) => {
+        queued: document.getElementById('queued-tasks-count'),
-            const running = data.tasks_running;
+        succeeded: document.getElementById('succeeded-tasks-count'),
-            const queued = data.tasks_queued;
+        retried: document.getElementById('retried-tasks-count'),
-
+        failed: document.getElementById('failed-tasks-count')
-            const updateTaskCount = (element, value) => {
-                element.textContent = value == null ? 'N/A' : value.toLocaleString();
     };
 
-            updateTaskCount(elemRunning, running);
+    /**
-            updateTaskCount(elemQueued, queued);
+     * Fetches the task queue status and updates the UI elements accordingly.
+     * It retrieves the total number of tasks, running tasks, queued tasks,
+     * succeeded tasks, retried tasks, and failed tasks, and updates the
+     * corresponding HTML elements with the fetched data.
+     * It also updates the progress bars for succeeded, retried, and failed tasks.
+     * The function is called immediately and then every 30 seconds to keep the
+     * task queue status up to date.
+     */
+    const updateTaskCount = () => {
+        fetchGet({url: '{% url "authentication:task_counts" %}'})
+            .then((data) => {
+                const numberL10nFormat = new Intl.NumberFormat('{{ LANGUAGE_CODE }}');
+                const elemProgressBar = document.getElementById('celery-tasks-progress-bar');
+                const progressElements = {
+                    succeeded: {
+                        bar: document.getElementById('celery-progress-bar-succeeded'),
+                        text: document.getElementById('celery-progress-bar-succeeded-progress')
+                    },
+                    retried: {
+                        bar: document.getElementById('celery-progress-bar-retried'),
+                        text: document.getElementById('celery-progress-bar-retried-progress')
+                    },
+                    failed: {
+                        bar: document.getElementById('celery-progress-bar-failed'),
+                        text: document.getElementById('celery-progress-bar-failed-progress')
+                    }
+                };
+
+                // Assign progress data from the fetched data to variables
+                const {
+                    earliest_task: earliestTask,
+                    tasks_total: tasksTotal,
+                    tasks_running: tasksRunning,
+                    tasks_queued: tasksQueued,
+                    tasks_succeeded: tasksSucceeded,
+                    tasks_retried: tasksRetried,
+                    tasks_failed: tasksFailed
+                } = data;
+
+                /**
+                 * Updates the text content of the specified HTML element with the given value.
+                 * If the value is null, it sets the text to 'N/A'.
+                 * Otherwise, it formats the number using the locale-specific format.
+                 *
+                 * @param {HTMLElement} element The HTML element to update.
+                 * @param {number|null} value The value to set in the element.
+                 */
+                const updateTaskCount = (element, value) => {
+                    element.textContent = value == null ? '{{ l10nNA|escapejs }}' : numberL10nFormat.format(value);
+                };
+
+                /**
+                 * Calculates the time since the given timestamp and returns a formatted string.
+                 * If the timestamp is null or undefined, it returns 'N/A'.
+                 * The returned string is in the format of "X hours, Y minutes" or "X minutes, Y seconds".
+                 *
+                 * @param {string|null} timestamp The timestamp to calculate the time since.
+                 * @returns {string} A formatted string representing the time since the timestamp.
+                 */
+                const timeSince = (timestamp) => {
+                    if (!timestamp) {
+                        return '{{ l10nNA|escapejs }}';
+                    }
+
+                    const diffSecs = Math.floor((Date.now() - new Date(timestamp)) / 1000);
+
+                    if (diffSecs >= 3600) {
+                        const hours = Math.floor(diffSecs / 3600);
+                        const minutes = Math.floor((diffSecs % 3600) / 60);
+
+                        if (minutes > 0) {
+                            const hourText = hours === 1 ? '{{ l10nHourSingular|escapejs }}' : '{{ l10nHourPlural|escapejs }}';
+                            const minuteText = minutes === 1 ? '{{ l10nMinuteSingular|escapejs }}' : '{{ l10nMinutePlural|escapejs }}';
+
+                            return `${hours} ${hourText}, ${minutes} ${minuteText}`;
+                        }
+
+                        const hourText = hours === 1 ? '{{ l10nHourSingular|escapejs }}' : '{{ l10nHourPlural|escapejs }}';
+
+                        return `${hours} ${hourText}`;
+                    }
+
+                    const units = [
+                        [
+                            60,
+                            '{{ l10nMinuteSingular|escapejs }}',
+                            '{{ l10nMinutePlural|escapejs }}'
+                        ],
+                        [
+                            1,
+                            '{{ l10nSecondSingular|escapejs }}',
+                            '{{ l10nSecondPlural|escapejs }}'
+                        ]
+                    ];
+
+                    for (const [seconds, singular, plural] of units) {
+                        const value = Math.floor(diffSecs / seconds);
+
+                        if (value > 0) {
+                            return `${value} ${value > 1 ? plural : singular}`;
+                        }
+                    }
+
+                    return '0 {{ l10nSecondPlural|escapejs }}';
+                };
+
+                /**
+                 * Updates the progress bar element and its text content based on the given value and total.
+                 * It calculates the percentage of completion and updates the aria attributes and styles accordingly.
+                 *
+                 * @param {HTMLElement} element The progress bar element to update.
+                 * @param {HTMLElement} textElement The text element to update with the percentage.
+                 * @param {number} value The current value to set in the progress bar.
+                 * @param {number} total The total value for calculating the percentage.
+                 */
+                const updateProgressBar = (element, textElement, value, total) => {
+                    const percentage = total ? (value / total) * 100 : 0;
+
+                    element.setAttribute('aria-valuenow', percentage.toString());
+                    textElement.textContent = `${numberL10nFormat.format(percentage.toFixed(0))}%`;
+                    element.style.width = `${percentage}%`;
+                };
+
+                // Update task counts
+                [
+                    [elements.total, tasksTotal],
+                    [elements.running, tasksRunning],
+                    [elements.queued, tasksQueued],
+                    [elements.succeeded, tasksSucceeded],
+                    [elements.retried, tasksRetried],
+                    [elements.failed, tasksFailed]
+                ].forEach(([element, value]) => {
+                    updateTaskCount(element, value);
+                });
+
+                // Update uptime
+                elements.uptime.textContent = timeSince(earliestTask);
+
+                // Update progress bar title
+                const [
+                    titleTextSucceeded,
+                    titleTextRetried,
+                    titleTextFailed
+                ] = [
+                    [tasksSucceeded, '{{ l10nSucceeded|escapejs }}'],
+                    [tasksRetried, '{{ l10nRetried|escapejs }}'],
+                    [tasksFailed, '{{ l10nFailed|escapejs }}']
+                ].map(([count, label]) => {
+                    return `${numberL10nFormat.format(count)} ${label}`;
+                });
+
+                // Set the title attribute for the progress bar
+                elemProgressBar.setAttribute(
+                    'title',
+                    `${titleTextSucceeded}, ${titleTextRetried}, ${titleTextFailed}`
+                );
+
+                // Update progress bars
+                [
+                    tasksSucceeded,
+                    tasksRetried,
+                    tasksFailed
+                ].forEach((count, index) => {
+                    const type = ['succeeded', 'retried', 'failed'][index];
+
+                    updateProgressBar(
+                        progressElements[type].bar,
+                        progressElements[type].text,
+                        count,
+                        tasksTotal
+                    );
+                });
             })
             .catch((error) => {
                 console.error('Error fetching task queue:', error);
 
-            [elemRunning, elemQueued].forEach(elem => elem.textContent = 'ERROR');
+                // If there is an error fetching the task queue, set all elements to 'ERROR'
+                [
+                    elements.running,
+                    elements.queued,
+                    elements.succeeded,
+                    elements.retried,
+                    elements.failed
+                ].forEach((elem) => {
+                    elem.textContent = '{{ l10nError|escapejs }}';
                 });
+            });
+    };
+
+    updateTaskCount();
+    setInterval(updateTaskCount, 30000);
 </script>