From 97f603c13802ea68972d6389804ab890b5f3601b Mon Sep 17 00:00:00 2001 From: T'rahk Rokym Date: Sun, 27 Apr 2025 12:33:02 +0200 Subject: [PATCH] Move all the code in an application and stores Announcements in the database so they can be marked as closed --- allianceauth/admin_status/__init__.py | 0 allianceauth/admin_status/admin.py | 19 + allianceauth/admin_status/apps.py | 6 + allianceauth/admin_status/hooks.py | 207 +++++++++++ allianceauth/admin_status/managers.py | 57 +++ .../admin_status/migrations/0001_initial.py | 33 ++ .../admin_status/migrations/__init__.py | 0 allianceauth/admin_status/models.py | 45 +++ .../admin-status/celery_bar_partial.html | 0 .../templates}/admin-status/esi_check.html | 0 .../templates}/admin-status/include.html | 0 .../templates}/admin-status/overview.html | 80 ++--- .../admin_status/templatetags/__init__.py | 0 .../admin_status/templatetags/admin_status.py | 181 ++++++++++ allianceauth/admin_status/tests/__init__.py | 0 .../admin_status/tests/test_managers.py | 75 ++++ .../tests/test_templatetags.py | 70 +++- allianceauth/authentication/views.py | 4 +- allianceauth/services/hooks.py | 2 +- allianceauth/templatetags/admin_status.py | 326 ------------------ .../custom/app-announcement-hooks.md | 5 +- 21 files changed, 706 insertions(+), 404 deletions(-) create mode 100644 allianceauth/admin_status/__init__.py create mode 100644 allianceauth/admin_status/admin.py create mode 100644 allianceauth/admin_status/apps.py create mode 100644 allianceauth/admin_status/hooks.py create mode 100644 allianceauth/admin_status/managers.py create mode 100644 allianceauth/admin_status/migrations/0001_initial.py create mode 100644 allianceauth/admin_status/migrations/__init__.py create mode 100644 allianceauth/admin_status/models.py rename allianceauth/{templates/allianceauth => admin_status/templates}/admin-status/celery_bar_partial.html (100%) rename allianceauth/{templates/allianceauth => admin_status/templates}/admin-status/esi_check.html (100%) rename allianceauth/{templates/allianceauth => admin_status/templates}/admin-status/include.html (100%) rename allianceauth/{templates/allianceauth => admin_status/templates}/admin-status/overview.html (69%) create mode 100644 allianceauth/admin_status/templatetags/__init__.py create mode 100644 allianceauth/admin_status/templatetags/admin_status.py create mode 100644 allianceauth/admin_status/tests/__init__.py create mode 100644 allianceauth/admin_status/tests/test_managers.py rename allianceauth/{authentication => admin_status}/tests/test_templatetags.py (80%) delete mode 100644 allianceauth/templatetags/admin_status.py diff --git a/allianceauth/admin_status/__init__.py b/allianceauth/admin_status/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/allianceauth/admin_status/admin.py b/allianceauth/admin_status/admin.py new file mode 100644 index 00000000..a6a10970 --- /dev/null +++ b/allianceauth/admin_status/admin.py @@ -0,0 +1,19 @@ +"""Admin site for admin status applicaton""" +from django.contrib import admin + +from allianceauth.admin_status.models import ApplicationAnnouncement + + +@admin.register(ApplicationAnnouncement) +class ApplicationAnnouncementAdmin(admin.ModelAdmin): + list_display = ["application_name", "announcement_number", "announcement_text", "hide_announcement"] + list_filter = ["hide_announcement"] + ordering = ["application_name", "announcement_number"] + readonly_fields = ["application_name", "announcement_number", "announcement_text", "announcement_url"] + fields = ["application_name", "announcement_number", "announcement_text", "announcement_url", "hide_announcement"] + + def has_add_permission(self, request): + return False + + def has_delete_permission(self, request, obj=None): + return False diff --git a/allianceauth/admin_status/apps.py b/allianceauth/admin_status/apps.py new file mode 100644 index 00000000..35f5e6dd --- /dev/null +++ b/allianceauth/admin_status/apps.py @@ -0,0 +1,6 @@ +from django.apps import AppConfig + + +class AdminStatusApplication(AppConfig): + name = 'allianceauth.admin_status' + label = 'admin_status' diff --git a/allianceauth/admin_status/hooks.py b/allianceauth/admin_status/hooks.py new file mode 100644 index 00000000..fc5ca081 --- /dev/null +++ b/allianceauth/admin_status/hooks.py @@ -0,0 +1,207 @@ +import hashlib +import logging +from dataclasses import dataclass +from enum import Enum +from urllib.parse import quote_plus + +import requests + +from django.core.cache import cache + +from allianceauth.hooks import get_hooks, register + +logger = logging.getLogger(__name__) + +# timeout for all requests +REQUESTS_TIMEOUT = 5 # 5 seconds +# max pages to be fetched from gitlab +MAX_PAGES = 50 +# Cache time +NOTIFICATION_CACHE_TIME = 300 # 5 minutes + + +@dataclass +class Announcement: + """ + Dataclass storing all data for an announcement to be sent arround + """ + application_name: str + announcement_url: str + announcement_number: int + announcement_text: str + + @classmethod + def build_from_gitlab_issue_dict(cls, application_name: str, gitlab_issue: dict) -> "Announcement": + """Builds the announcement from the JSON dict of a GitLab issue""" + return Announcement(application_name, gitlab_issue["web_url"], gitlab_issue["iid"], gitlab_issue["title"]) + + @classmethod + def build_from_github_issue_dict(cls, application_name: str, github_issue: dict) -> "Announcement": + """Builds the announcement from the JSON dict of a GitHub issue""" + return Announcement(application_name, github_issue["html_url"], github_issue["number"], github_issue["title"]) + + def get_hash(self): + """Get a hash of the Announcement for comparison""" + name = f"{self.application_name}.{self.announcement_number}" + hash_value = hashlib.sha256(name.encode("utf-8")).hexdigest() + return hash_value + + +@dataclass +class AppAnnouncementHook: + """ + A hook for an application to send GitHub/GitLab issues as announcements on the dashboard + + Args: + - app_name: The name of your application + - repository_namespace: The namespace of the remote repository of your application source code. + It should look like `/`. + - repository_kind: Enumeration to determine if your repository is a GitHub or GitLab repository. + - label: The label applied to issues that should be seen as announcements, case-sensitive. + Default value: `announcement` + """ + class Service(Enum): + """Simple enumeration to determine which api should be called to access issues""" + GITLAB = "gitlab" + GITHUB = "github" + + app_name: str + repository_namespace: str + repository_kind: Service + label: str = "announcement" + + + def get_announcement_list(self) -> list[Announcement]: + """ + Checks the application repository to find issues with the `Announcement` tag and return their title and link to + be displayed. + """ + logger.debug("Getting announcement list for the app %s", self.app_name) + match self.repository_kind: + case AppAnnouncementHook.Service.GITHUB: + announcement_list = self._get_github_announcement_list() + case AppAnnouncementHook.Service.GITLAB: + announcement_list = self._get_gitlab_announcement_list() + case _: + announcement_list = [] + + logger.debug("Announcements for app %s: %s", self.app_name, announcement_list) + return announcement_list + + def _get_github_announcement_list(self) -> list[Announcement]: + """ + Return the issue list for a GitHub repository + Will filter if the `pull_request` attribute is present + """ + raw_list = _fetch_list_from_github( + f"https://api.github.com/repos/{self.repository_namespace}/issues" + f"?labels={self.label}" + ) + return [Announcement.build_from_github_issue_dict(self.app_name, github_issue) for github_issue in raw_list] + + def _get_gitlab_announcement_list(self) -> list[Announcement]: + """Return the issues list for a GitLab repository""" + raw_list = _fetch_list_from_gitlab( + f"https://gitlab.com/api/v4/projects/{quote_plus(self.repository_namespace)}/issues" + f"?labels={self.label}&state=opened") + return [Announcement.build_from_gitlab_issue_dict(self.app_name, gitlab_issue) for gitlab_issue in raw_list] + +@register("app_announcement_hook") +def alliance_auth_announcements_hook(): + return AppAnnouncementHook("AllianceAuth", "allianceauth/allianceauth", AppAnnouncementHook.Service.GITLAB) + +def get_all_applications_announcements() -> list[Announcement]: + """ + Retrieve all known application announcements and returns them + """ + application_notifications = [] + + hooks = [fn() for fn in get_hooks("app_announcement_hook")] + for hook in hooks: + logger.debug(hook) + try: + application_notifications.extend(cache.get_or_set( + f"{hook.app_name}_notification_issues", + hook.get_announcement_list, + NOTIFICATION_CACHE_TIME, + )) + except requests.HTTPError: + logger.warning("Error when getting %s notifications", hook, exc_info=True) + + logger.debug(application_notifications) + if application_notifications: + application_notifications = application_notifications[:10] + + return application_notifications + + +def _fetch_list_from_gitlab(url: str, max_pages: int = MAX_PAGES) -> list: + """returns a list from the GitLab API. Supports paging""" + result = [] + + for page in range(1, max_pages + 1): + try: + request = requests.get( + url, params={'page': page}, timeout=REQUESTS_TIMEOUT + ) + request.raise_for_status() + except requests.exceptions.RequestException as e: + error_str = str(e) + + logger.warning( + f'Unable to fetch from GitLab API. Error: {error_str}', + exc_info=True, + ) + + return result + + result += request.json() + + if 'x-total-pages' in request.headers: + try: + total_pages = int(request.headers['x-total-pages']) + except ValueError: + total_pages = None + else: + total_pages = None + + if not total_pages or page >= total_pages: + break + + return result + +def _fetch_list_from_github(url: str, max_pages: int = MAX_PAGES) -> list: + """returns a list from the GitHub API. Supports paging""" + + result = [] + for page in range(1, max_pages+1): + try: + request = requests.get( + url, + params={'page': page}, + headers={ + "Accept": "application/vnd.github+json", + "X-GitHub-Api-Version": "2022-11-28" + }, + timeout=REQUESTS_TIMEOUT, + ) + request.raise_for_status() + except requests.exceptions.RequestException as e: + error_str = str(e) + + logger.warning( + f'Unable to fetch from GitHub API. Error: {error_str}', + exc_info=True, + ) + + return result + + result += request.json() + logger.debug(request.json()) + + # https://docs.github.com/en/rest/using-the-rest-api/using-pagination-in-the-rest-api?apiVersion=2022-11-28 + # See Example creating a pagination method + if not ('link' in request.headers and 'rel=\"next\"' in request.headers['link']): + break + + return result diff --git a/allianceauth/admin_status/managers.py b/allianceauth/admin_status/managers.py new file mode 100644 index 00000000..f445fad2 --- /dev/null +++ b/allianceauth/admin_status/managers.py @@ -0,0 +1,57 @@ +from typing import TYPE_CHECKING + +from django.db import models + +from allianceauth.admin_status.hooks import ( + Announcement, + get_all_applications_announcements, +) +from allianceauth.services.hooks import get_extension_logger + +if TYPE_CHECKING: + from .models import ApplicationAnnouncement + +logger = get_extension_logger(__name__) + +class ApplicationAnnouncementManager(models.Manager): + + def sync_and_return(self): + """ + Checks all hooks if new notifications need to be created. + Return all notification objects after + """ + logger.info("Syncing announcements") + current_announcements = get_all_applications_announcements() + self._delete_obsolete_announcements(current_announcements) + self._store_new_announcements(current_announcements) + + return self.all() + + def _delete_obsolete_announcements(self, current_announcements: list[Announcement]): + """Deletes all announcements stored in the database that aren't retrieved anymore""" + hashes = [announcement.get_hash() for announcement in current_announcements] + self.exclude(announcement_hash__in=hashes).delete() + + def _store_new_announcements(self, current_announcements: list[Announcement]): + """Stores a new database object for new application announcements""" + + for current_announcement in current_announcements: + try: + announcement = self.get(announcement_hash=current_announcement.get_hash()) + except self.model.DoesNotExist: + self.create_from_announcement(current_announcement) + else: + # if exists update the text only + if announcement.announcement_text != current_announcement.announcement_text: + announcement.announcement_text = current_announcement.announcement_text + announcement.save() + + def create_from_announcement(self, announcement: Announcement) -> "ApplicationAnnouncement": + """Creates from the Announcement dataclass""" + return self.create( + application_name=announcement.application_name, + announcement_number=announcement.announcement_number, + announcement_text=announcement.announcement_text, + announcement_url=announcement.announcement_url, + announcement_hash=announcement.get_hash(), + ) diff --git a/allianceauth/admin_status/migrations/0001_initial.py b/allianceauth/admin_status/migrations/0001_initial.py new file mode 100644 index 00000000..f7bcdc26 --- /dev/null +++ b/allianceauth/admin_status/migrations/0001_initial.py @@ -0,0 +1,33 @@ +# Generated by Django 5.1.9 on 2025-05-18 15:43 + +import django.db.models.manager +from django.db import migrations, models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ] + + operations = [ + migrations.CreateModel( + name='ApplicationAnnouncement', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('application_name', models.CharField(help_text='Name of the application that issued the announcement', max_length=50)), + ('announcement_number', models.IntegerField(help_text='Issue number on the notification source')), + ('announcement_text', models.TextField(help_text='Issue title text displayed on the dashboard', max_length=300)), + ('announcement_url', models.TextField(max_length=200)), + ('announcement_hash', models.CharField(default=None, editable=False, help_text='hash of an announcement. Must be nullable for unique comparison.', max_length=64, null=True, unique=True)), + ('hide_announcement', models.BooleanField(default=False, help_text='Set to true if the announcement should not be displayed on the dashboard')), + ], + options={ + 'constraints': [models.UniqueConstraint(fields=('application_name', 'announcement_number'), name='functional_pk_applicationissuenumber')], + }, + managers=[ + ('object', django.db.models.manager.Manager()), + ], + ), + ] diff --git a/allianceauth/admin_status/migrations/__init__.py b/allianceauth/admin_status/migrations/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/allianceauth/admin_status/models.py b/allianceauth/admin_status/models.py new file mode 100644 index 00000000..7b048378 --- /dev/null +++ b/allianceauth/admin_status/models.py @@ -0,0 +1,45 @@ +from django.db import models +from django.utils.translation import gettext_lazy as _ + +from allianceauth.admin_status.managers import ApplicationAnnouncementManager + + +class ApplicationAnnouncement(models.Model): + """ + Announcement originating from an application + """ + object = ApplicationAnnouncementManager() + + application_name = models.CharField(max_length=50, help_text=_("Name of the application that issued the announcement")) + announcement_number = models.IntegerField(help_text=_("Issue number on the notification source")) + announcement_text = models.TextField(max_length=300, help_text=_("Issue title text displayed on the dashboard")) + announcement_url = models.TextField(max_length=200) + + announcement_hash = models.CharField( + max_length=64, + default=None, + unique=True, + editable=False, + help_text="hash of an announcement." + ) + + hide_announcement = models.BooleanField( + default=False, + help_text=_("Set to true if the announcement should not be displayed on the dashboard") + ) + + class Meta: + # Should be updated to a composite key when the switch to Django 5.2 is made + # https://docs.djangoproject.com/en/5.2/topics/composite-primary-key/ + constraints = [ + models.UniqueConstraint( + fields=["application_name", "announcement_number"], name="functional_pk_applicationissuenumber" + ) + ] + + def __str__(self): + return f"{self.application_name} announcement #{self.announcement_number}" + + def is_hidden(self) -> bool: + """Function in case rules are made in the future to force hide/force show some announcements""" + return self.hide_announcement diff --git a/allianceauth/templates/allianceauth/admin-status/celery_bar_partial.html b/allianceauth/admin_status/templates/admin-status/celery_bar_partial.html similarity index 100% rename from allianceauth/templates/allianceauth/admin-status/celery_bar_partial.html rename to allianceauth/admin_status/templates/admin-status/celery_bar_partial.html diff --git a/allianceauth/templates/allianceauth/admin-status/esi_check.html b/allianceauth/admin_status/templates/admin-status/esi_check.html similarity index 100% rename from allianceauth/templates/allianceauth/admin-status/esi_check.html rename to allianceauth/admin_status/templates/admin-status/esi_check.html diff --git a/allianceauth/templates/allianceauth/admin-status/include.html b/allianceauth/admin_status/templates/admin-status/include.html similarity index 100% rename from allianceauth/templates/allianceauth/admin-status/include.html rename to allianceauth/admin_status/templates/admin-status/include.html diff --git a/allianceauth/templates/allianceauth/admin-status/overview.html b/allianceauth/admin_status/templates/admin-status/overview.html similarity index 69% rename from allianceauth/templates/allianceauth/admin-status/overview.html rename to allianceauth/admin_status/templates/admin-status/overview.html index 867322b6..448a58e5 100644 --- a/allianceauth/templates/allianceauth/admin-status/overview.html +++ b/allianceauth/admin_status/templates/admin-status/overview.html @@ -2,65 +2,21 @@ {% load humanize %} {% if notifications %} -
+
- {% translate "Alliance Auth Notifications" as widget_title %} + {% translate "AllianceAuth and 3rd party Applications Notifications" as widget_title %} {% include "framework/dashboard/widget-title.html" with title=widget_title %}
    {% for notif in notifications %} -
  • - {% if notif.state == 'opened' %} - {% translate "Open" %} - {% else %} - {% translate "Closed" %} - {% endif %} - #{{ notif.iid }} {{ notif.title }} -
    • - {% empty %} - - {% endfor %} -
- -
- - - - {% translate 'Powered by GitLab' %} - - - - - - {% translate 'Support Discord' %} - - -
-
-
-
-
-{% endif %} - -{% if application_notifications %} -
-
-
- {% translate "Application Notifications" as widget_title %} - {% include "framework/dashboard/widget-title.html" with title=widget_title %} - -
- {# TODO maybe add some disclaimer that those are managed by application devs? #} @@ -131,9 +87,9 @@ style="height: 21px;" title="{{ tasks_succeeded|intcomma }} succeeded, {{ tasks_retried|intcomma }} retried, {{ tasks_failed|intcomma }} failed" > - {% include "allianceauth/admin-status/celery_bar_partial.html" with label="suceeded" level="success" tasks_count=tasks_succeeded %} - {% include "allianceauth/admin-status/celery_bar_partial.html" with label="retried" level="info" tasks_count=tasks_retried %} - {% include "allianceauth/admin-status/celery_bar_partial.html" with label="failed" level="danger" tasks_count=tasks_failed %} + {% include "admin-status/celery_bar_partial.html" with label="suceeded" level="success" tasks_count=tasks_succeeded %} + {% include "admin-status/celery_bar_partial.html" with label="retried" level="info" tasks_count=tasks_retried %} + {% include "admin-status/celery_bar_partial.html" with label="failed" level="danger" tasks_count=tasks_failed %}

@@ -142,6 +98,20 @@

+
+ + + + {% translate 'Powered by GitLab' %} + + + + + + {% translate 'Support Discord' %} + + +
diff --git a/allianceauth/admin_status/templatetags/__init__.py b/allianceauth/admin_status/templatetags/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/allianceauth/admin_status/templatetags/admin_status.py b/allianceauth/admin_status/templatetags/admin_status.py new file mode 100644 index 00000000..af1f8f48 --- /dev/null +++ b/allianceauth/admin_status/templatetags/admin_status.py @@ -0,0 +1,181 @@ +import logging + +import requests +from packaging.version import InvalidVersion, Version as Pep440Version + +from django import template +from django.conf import settings +from django.core.cache import cache + +from allianceauth import __version__ +from allianceauth.admin_status.models import ApplicationAnnouncement +from allianceauth.authentication.task_statistics.counters import ( + dashboard_results, +) + +register = template.Library() + +# cache timers +TAG_CACHE_TIME = 3600 # 1 hours +NOTIFICATION_CACHE_TIME = 300 # 5 minutes +# timeout for all requests +REQUESTS_TIMEOUT = 5 # 5 seconds +# max pages to be fetched from gitlab +MAX_PAGES = 50 + +GITLAB_AUTH_REPOSITORY_TAGS_URL = ( + 'https://gitlab.com/api/v4/projects/allianceauth%2Fallianceauth/repository/tags' +) + +logger = logging.getLogger(__name__) + + +@register.simple_tag() +def decimal_widthratio(this_value, max_value, max_width) -> str: + if max_value == 0: + return str(0) + + return str(round(this_value / max_value * max_width, 2)) + + +@register.inclusion_tag('admin-status/overview.html') +def status_overview() -> dict: + response = { + "notifications": [], + "current_version": __version__, + "tasks_succeeded": 0, + "tasks_retried": 0, + "tasks_failed": 0, + "tasks_total": 0, + "tasks_hours": 0, + "earliest_task": None, + } + response.update(_current_notifications()) + response.update(_current_version_summary()) + response.update(_celery_stats()) + return response + + +def _celery_stats() -> dict: + hours = getattr(settings, "ALLIANCEAUTH_DASHBOARD_TASKS_MAX_HOURS", 24) + results = dashboard_results(hours=hours) + return { + "tasks_succeeded": results.succeeded, + "tasks_retried": results.retried, + "tasks_failed": results.failed, + "tasks_total": results.total, + "tasks_hours": results.hours, + "earliest_task": results.earliest_task, + } + + +def _current_notifications() -> dict: + """returns announcements from AllianceAuth and third party applications""" + + application_notifications = ApplicationAnnouncement.object.sync_and_return() + + response = { + 'notifications': application_notifications, + } + return response + +def _current_version_summary() -> dict: + """returns the current version info""" + try: + tags = cache.get_or_set( + 'git_release_tags', _fetch_tags_from_gitlab, TAG_CACHE_TIME + ) + except requests.HTTPError: + logger.warning('Error while getting gitlab release tags', exc_info=True) + return {} + + if not tags: + return {} + + ( + latest_patch_version, + latest_beta_version + ) = _latests_versions(tags) + current_version = Pep440Version(__version__) + + has_latest_patch = \ + current_version >= latest_patch_version if latest_patch_version else False + has_current_beta = \ + current_version <= latest_beta_version \ + and latest_patch_version <= latest_beta_version \ + if latest_beta_version else False + + response = { + 'latest_patch': has_latest_patch, + 'latest_beta': has_current_beta, + 'current_version': str(current_version), + 'latest_patch_version': str(latest_patch_version), + 'latest_beta_version': str(latest_beta_version) + } + return response + + +def _fetch_tags_from_gitlab(): + return _fetch_list_from_gitlab(GITLAB_AUTH_REPOSITORY_TAGS_URL) + + +def _latests_versions(tags: list) -> tuple: + """returns latests version from given tags list + + Non-compliant tags will be ignored + """ + versions = [] + betas = [] + for tag in tags: + try: + version = Pep440Version(tag.get('name')) + except InvalidVersion: + pass + else: + if version.is_prerelease or version.is_devrelease: + betas.append(version) + else: + versions.append(version) + + latest_patch_version = max(versions) + latest_beta_version = max(betas) + return ( + latest_patch_version, + latest_beta_version + ) + + +def _fetch_list_from_gitlab(url: str, max_pages: int = MAX_PAGES) -> list: + """returns a list from the GitLab API. Supports paging""" + result = [] + + for page in range(1, max_pages + 1): + try: + request = requests.get( + url, params={'page': page}, timeout=REQUESTS_TIMEOUT + ) + request.raise_for_status() + except requests.exceptions.RequestException as e: + error_str = str(e) + + logger.warning( + f'Unable to fetch from GitLab API. Error: {error_str}', + exc_info=True, + ) + + return result + + result += request.json() + + if 'x-total-pages' in request.headers: + try: + total_pages = int(request.headers['x-total-pages']) + except ValueError: + total_pages = None + else: + total_pages = None + + if not total_pages or page >= total_pages: + break + + return result diff --git a/allianceauth/admin_status/tests/__init__.py b/allianceauth/admin_status/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/allianceauth/admin_status/tests/test_managers.py b/allianceauth/admin_status/tests/test_managers.py new file mode 100644 index 00000000..9735e00d --- /dev/null +++ b/allianceauth/admin_status/tests/test_managers.py @@ -0,0 +1,75 @@ +from unittest.mock import patch + +from allianceauth.admin_status.hooks import Announcement +from allianceauth.admin_status.models import ApplicationAnnouncement +from allianceauth.utils.testing import NoSocketsTestCase + +MODULE_PATH = 'allianceauth.admin_status.managers' + +DEFAULT_ANNOUNCEMENTS = [ + Announcement( + application_name="Test GitHub Application", + announcement_number=1, + announcement_text="GitHub issue", + announcement_url="https://github.com/r0kym/test/issues/1", + ), + Announcement( + application_name="Test Gitlab Application", + announcement_number=1, + announcement_text="GitLab issue", + announcement_url="https://gitlab.com/r0kym/allianceauth-example-plugin/-/issues/1", + ) +] + +class TestSyncManager(NoSocketsTestCase): + + def setUp(self): + ApplicationAnnouncement.object.create( + application_name="Test GitHub Application", + announcement_number=1, + announcement_text="GitHub issue", + announcement_url="https://github.com/r0kym/test/issues/1", + announcement_hash="9dbedb9c47529bb43cfecb704768a35d085b145930e13cced981623e5f162a85", + ) + ApplicationAnnouncement.object.create( + application_name="Test Gitlab Application", + announcement_number=1, + announcement_text="GitLab issue", + announcement_url="https://gitlab.com/r0kym/allianceauth-example-plugin/-/issues/1", + announcement_hash="8955a9c12a1cfa9e1776662bdaf111147b84e35c79f24bfb758e35333a18b1bd", + ) + + @patch(MODULE_PATH + '.get_all_applications_announcements') + def test_announcements_stay_as_is(self, all_announcements_mocker): + # given + announcement_ids = set(ApplicationAnnouncement.object.values_list("id", flat=True)) + all_announcements_mocker.return_value = DEFAULT_ANNOUNCEMENTS + # when + ApplicationAnnouncement.object.sync_and_return() + # then + self.assertEqual(ApplicationAnnouncement.object.count(), 2) + self.assertEqual(set(ApplicationAnnouncement.object.values_list("id", flat=True)), announcement_ids) + + @patch(MODULE_PATH + '.get_all_applications_announcements') + def test_announcement_add(self, all_announcements_mocker): + # given + returned_announcements = DEFAULT_ANNOUNCEMENTS + [Announcement(application_name="Test Application", announcement_number=1, announcement_text="New test announcement", announcement_url="https://example.com")] + all_announcements_mocker.return_value = returned_announcements + # when + ApplicationAnnouncement.object.sync_and_return() + # then + self.assertEqual(ApplicationAnnouncement.object.count(), 3) + self.assertTrue(ApplicationAnnouncement.object.filter(application_name="Test Application", announcement_number=1, announcement_text="New test announcement", announcement_url="https://example.com")) + + @patch(MODULE_PATH + '.get_all_applications_announcements') + def test_announcement_remove(self, all_announcements_mocker): + # given + all_announcements_mocker.return_value = DEFAULT_ANNOUNCEMENTS + ApplicationAnnouncement.object.sync_and_return() + self.assertEqual(ApplicationAnnouncement.object.count(), 2) + all_announcements_mocker.return_value = DEFAULT_ANNOUNCEMENTS[:1] + # when + ApplicationAnnouncement.object.sync_and_return() + # then + self.assertEqual(ApplicationAnnouncement.object.count(), 1) + self.assertTrue(ApplicationAnnouncement.object.filter(application_name="Test GitHub Application").exists()) diff --git a/allianceauth/authentication/tests/test_templatetags.py b/allianceauth/admin_status/tests/test_templatetags.py similarity index 80% rename from allianceauth/authentication/tests/test_templatetags.py rename to allianceauth/admin_status/tests/test_templatetags.py index 7f42f830..9f5d6234 100644 --- a/allianceauth/authentication/tests/test_templatetags.py +++ b/allianceauth/admin_status/tests/test_templatetags.py @@ -8,23 +8,61 @@ from packaging.version import Version as Pep440Version from django.core.cache import cache from django.test import TestCase -from allianceauth.templatetags.admin_status import ( +from allianceauth.admin_status.models import ApplicationAnnouncement +from allianceauth.admin_status.templatetags.admin_status import ( _current_notifications, _current_version_summary, _fetch_list_from_gitlab, - _fetch_notification_issues_from_gitlab, _latests_versions, status_overview, ) -MODULE_PATH = 'allianceauth.templatetags' +MODULE_PATH = 'allianceauth.admin_status.templatetags' def create_tags_list(tag_names: list): return [{'name': str(tag_name)} for tag_name in tag_names] +def get_app_announcement_as_dict(app_announcement: ApplicationAnnouncement) -> dict: + """Transforms an app announcement object in a dict easy to compare""" + return { + "application_name": app_announcement.application_name, + "announcement_number": app_announcement.announcement_number, + "announcement_text": app_announcement.announcement_text, + "announcement_url": app_announcement.announcement_url, + } + GITHUB_TAGS = create_tags_list(['v2.4.6a1', 'v2.4.5', 'v2.4.0', 'v2.0.0', 'v1.1.1']) +STORED_NOTIFICATIONS = [ + ApplicationAnnouncement( + application_name="Test GitHub Application", + announcement_number=1, + announcement_text="GitHub issue", + announcement_url="https://github.com/r0kym/test/issues/1", + announcement_hash="hash1", + ), + ApplicationAnnouncement( + application_name="Test Gitlab Application", + announcement_number=1, + announcement_text="GitLab issue", + announcement_url="https://gitlab.com/r0kym/allianceauth-example-plugin/-/issues/1", + announcement_hash="hash2", + ), +] +ANNOUNCEMENT_DICT = [ + { + "application_name": "Test GitHub Application", + "announcement_number": 1, + "announcement_text": "GitHub issue", + "announcement_url": "https://github.com/r0kym/test/issues/1", + }, { + "application_name": "Test Gitlab Application", + "announcement_number": 1, + "announcement_text": "GitLab issue", + "announcement_url": "https://gitlab.com/r0kym/allianceauth-example-plugin/-/issues/1", + } +] GITHUB_NOTIFICATION_ISSUES = [ { 'id': 1, @@ -52,6 +90,10 @@ GITHUB_NOTIFICATION_ISSUES = [ }, ] TEST_VERSION = '2.6.5' +GITLAB_AUTH_ANNOUNCEMENT_ISSUES_URL = ( + 'https://gitlab.com/api/v4/projects/allianceauth%2Fallianceauth/issues' + '?labels=announcement&state=opened' +) class TestStatusOverviewTag(TestCase): @@ -107,18 +149,19 @@ class TestNotifications(TestCase): ) requests_mocker.get(url, json=GITHUB_NOTIFICATION_ISSUES) # when - result = _fetch_notification_issues_from_gitlab() + result = _fetch_list_from_gitlab(GITLAB_AUTH_ANNOUNCEMENT_ISSUES_URL, 10) # then self.assertEqual(result, GITHUB_NOTIFICATION_ISSUES) - @patch(MODULE_PATH + '.admin_status.cache') - def test_current_notifications_normal(self, mock_cache): + @patch(MODULE_PATH + '.admin_status.ApplicationAnnouncement') + def test_current_notifications_normal(self, mock_application_announcement): # given - mock_cache.get_or_set.return_value = GITHUB_NOTIFICATION_ISSUES + mock_application_announcement.object.sync_and_return.return_value = STORED_NOTIFICATIONS # when result = _current_notifications() # then - self.assertEqual(result['notifications'], GITHUB_NOTIFICATION_ISSUES[:5]) + for notification in result["notifications"]: + self.assertIn(get_app_announcement_as_dict(notification), ANNOUNCEMENT_DICT) @requests_mock.mock() def test_current_notifications_failed(self, requests_mocker): @@ -131,16 +174,7 @@ class TestNotifications(TestCase): # when result = _current_notifications() # then - self.assertEqual(result['notifications'], []) - - @patch(MODULE_PATH + '.admin_status.cache') - def test_current_notifications_is_none(self, mock_cache): - # given - mock_cache.get_or_set.return_value = None - # when - result = _current_notifications() - # then - self.assertEqual(result['notifications'], []) + self.assertEqual(list(result['notifications']), []) class TestCeleryQueueLength(TestCase): diff --git a/allianceauth/authentication/views.py b/allianceauth/authentication/views.py index b5983a27..63c2d6c7 100644 --- a/allianceauth/authentication/views.py +++ b/allianceauth/authentication/views.py @@ -74,14 +74,14 @@ def dashboard_characters(request): def dashboard_admin(request): if request.user.is_superuser: - return render_to_string('allianceauth/admin-status/include.html', request=request) + return render_to_string('admin-status/include.html', request=request) else: return "" def dashboard_esi_check(request): if request.user.is_superuser: - return render_to_string('allianceauth/admin-status/esi_check.html', request=request) + return render_to_string('admin-status/esi_check.html', request=request) else: return "" diff --git a/allianceauth/services/hooks.py b/allianceauth/services/hooks.py index 75ed30dc..b52527e0 100644 --- a/allianceauth/services/hooks.py +++ b/allianceauth/services/hooks.py @@ -8,7 +8,7 @@ from django.utils.functional import cached_property from allianceauth.hooks import get_hooks from allianceauth.menu.hooks import MenuItemHook -from allianceauth.templatetags.admin_status import AppAnnouncementHook +from allianceauth.admin_status.hooks import AppAnnouncementHook from .models import NameFormatConfig diff --git a/allianceauth/templatetags/admin_status.py b/allianceauth/templatetags/admin_status.py deleted file mode 100644 index 6768a9a7..00000000 --- a/allianceauth/templatetags/admin_status.py +++ /dev/null @@ -1,326 +0,0 @@ -import logging -from dataclasses import dataclass -from enum import Enum -from urllib.parse import quote_plus - -import requests -from packaging.version import InvalidVersion, Version as Pep440Version - -from django import template -from django.conf import settings -from django.core.cache import cache - -from allianceauth import __version__ -from allianceauth.authentication.task_statistics.counters import ( - dashboard_results, -) -from allianceauth.hooks import get_hooks - -register = template.Library() - -# cache timers -TAG_CACHE_TIME = 3600 # 1 hours -NOTIFICATION_CACHE_TIME = 300 # 5 minutes -# timeout for all requests -REQUESTS_TIMEOUT = 5 # 5 seconds -# max pages to be fetched from gitlab -MAX_PAGES = 50 - -GITLAB_AUTH_REPOSITORY_TAGS_URL = ( - 'https://gitlab.com/api/v4/projects/allianceauth%2Fallianceauth/repository/tags' -) -GITLAB_AUTH_ANNOUNCEMENT_ISSUES_URL = ( - 'https://gitlab.com/api/v4/projects/allianceauth%2Fallianceauth/issues' - '?labels=announcement&state=opened' -) - -logger = logging.getLogger(__name__) - -@dataclass -class AppAnnouncementHook: - """ - A hook for an application to send GitHub/GitLab issues as announcements on the dashboard - - Args: - - app_name: The name of your application - - repository_namespace: The namespace of the remote repository of your application source code. - It should look like `/`. - - repository_kind: Enumeration to determine if your repository is a GitHub or GitLab repository. - - label: The label applied to issues that should be seen as announcements, case-sensitive. - Default value: `announcement` - """ - class RepositoryKind(Enum): - """Simple enumeration to determine which api should be called to access issues""" - GITLAB = "gitlab" - GITHUB = "github" - - app_name: str - repository_namespace: str - repository_kind: RepositoryKind - label: str = "announcement" - - - def get_announcement_list(self) -> list: - """ - Checks the application repository to find issues with the `Announcement` tag and return their title and link to - be displayed. - """ - match self.repository_kind: - case AppAnnouncementHook.RepositoryKind.GITHUB: - announcement_list = self._get_github_announcement_list() - case AppAnnouncementHook.RepositoryKind.GITLAB: - announcement_list = self._get_gitlab_announcement_list() - case _: - return [] - - for announcement in announcement_list: - announcement["app_name"] = self.app_name - - return announcement_list - - - def _get_github_announcement_list(self) -> list: - """ - Return the issue list for a GitHub repository - Will filter if the `pull_request` attribute is present - """ - raw_list = _fetch_list_from_github( - f"https://api.github.com/repos/{self.repository_namespace}/issues" - f"?labels={self.label}" - ) - # Translates GitHub attributes to GitLab and filters out pull requests - clean_list = [] - for element in raw_list: - if not element.get("pull_request"): - element["web_url"] = element["html_url"] - element["iid"] = element["number"] - clean_list.append(element) - return clean_list - - def _get_gitlab_announcement_list(self) -> list: - """Return the issues list for a GitLab repository""" - return _fetch_list_from_gitlab( - f"https://gitlab.com/api/v4/projects/{quote_plus(self.repository_namespace)}/issues" - f"?labels={self.label}&state=opened") - -@register.simple_tag() -def decimal_widthratio(this_value, max_value, max_width) -> str: - if max_value == 0: - return str(0) - - return str(round(this_value / max_value * max_width, 2)) - - -@register.inclusion_tag('allianceauth/admin-status/overview.html') -def status_overview() -> dict: - response = { - "notifications": [], - "current_version": __version__, - "tasks_succeeded": 0, - "tasks_retried": 0, - "tasks_failed": 0, - "tasks_total": 0, - "tasks_hours": 0, - "earliest_task": None, - } - response.update(_current_notifications()) - response.update(_current_version_summary()) - response.update(_celery_stats()) - return response - - -def _celery_stats() -> dict: - hours = getattr(settings, "ALLIANCEAUTH_DASHBOARD_TASKS_MAX_HOURS", 24) - results = dashboard_results(hours=hours) - return { - "tasks_succeeded": results.succeeded, - "tasks_retried": results.retried, - "tasks_failed": results.failed, - "tasks_total": results.total, - "tasks_hours": results.hours, - "earliest_task": results.earliest_task, - } - - -def _current_notifications() -> dict: - """returns the newest 5 announcement issues""" - try: - notifications = cache.get_or_set( - 'gitlab_notification_issues', - _fetch_notification_issues_from_gitlab, - NOTIFICATION_CACHE_TIME - ) - except requests.HTTPError: - logger.warning('Error while getting gitlab notifications', exc_info=True) - top_notifications = [] - else: - if notifications: - top_notifications = notifications[:5] - else: - top_notifications = [] - - app_notifications = [] - hooks = [fn() for fn in get_hooks("app_announcement_hook")] - for hook in hooks: - logger.debug(hook) - try: - app_notifications.extend(cache.get_or_set( - f"{hook.app_name}_notification_issues", - hook.get_announcement_list, - NOTIFICATION_CACHE_TIME, - )) - except requests.HTTPError: - logger.warning("Error when getting %s notifications", hook, exc_info=True) - - if app_notifications: - logger.debug(app_notifications) - application_notifications = app_notifications[:10] - else: - application_notifications = [] - - response = { - 'notifications': top_notifications, - 'application_notifications': application_notifications, - } - return response - - -def _fetch_notification_issues_from_gitlab() -> list: - return _fetch_list_from_gitlab(GITLAB_AUTH_ANNOUNCEMENT_ISSUES_URL, max_pages=10) - - -def _current_version_summary() -> dict: - """returns the current version info""" - try: - tags = cache.get_or_set( - 'git_release_tags', _fetch_tags_from_gitlab, TAG_CACHE_TIME - ) - except requests.HTTPError: - logger.warning('Error while getting gitlab release tags', exc_info=True) - return {} - - if not tags: - return {} - - ( - latest_patch_version, - latest_beta_version - ) = _latests_versions(tags) - current_version = Pep440Version(__version__) - - has_latest_patch = \ - current_version >= latest_patch_version if latest_patch_version else False - has_current_beta = \ - current_version <= latest_beta_version \ - and latest_patch_version <= latest_beta_version \ - if latest_beta_version else False - - response = { - 'latest_patch': has_latest_patch, - 'latest_beta': has_current_beta, - 'current_version': str(current_version), - 'latest_patch_version': str(latest_patch_version), - 'latest_beta_version': str(latest_beta_version) - } - return response - - -def _fetch_tags_from_gitlab(): - return _fetch_list_from_gitlab(GITLAB_AUTH_REPOSITORY_TAGS_URL) - - -def _latests_versions(tags: list) -> tuple: - """returns latests version from given tags list - - Non-compliant tags will be ignored - """ - versions = [] - betas = [] - for tag in tags: - try: - version = Pep440Version(tag.get('name')) - except InvalidVersion: - pass - else: - if version.is_prerelease or version.is_devrelease: - betas.append(version) - else: - versions.append(version) - - latest_patch_version = max(versions) - latest_beta_version = max(betas) - return ( - latest_patch_version, - latest_beta_version - ) - - -def _fetch_list_from_gitlab(url: str, max_pages: int = MAX_PAGES) -> list: - """returns a list from the GitLab API. Supports paging""" - result = [] - - for page in range(1, max_pages + 1): - try: - request = requests.get( - url, params={'page': page}, timeout=REQUESTS_TIMEOUT - ) - request.raise_for_status() - except requests.exceptions.RequestException as e: - error_str = str(e) - - logger.warning( - f'Unable to fetch from GitLab API. Error: {error_str}', - exc_info=True, - ) - - return result - - result += request.json() - - if 'x-total-pages' in request.headers: - try: - total_pages = int(request.headers['x-total-pages']) - except ValueError: - total_pages = None - else: - total_pages = None - - if not total_pages or page >= total_pages: - break - - return result - -def _fetch_list_from_github(url: str, max_pages: int = MAX_PAGES) -> list: - """returns a list from the GitHub API. Supports paging""" - - result = [] - for page in range(1, max_pages+1): - try: - request = requests.get( - url, - params={'page': page}, - headers={ - "Accept": "application/vnd.github+json", - "X-GitHub-Api-Version": "2022-11-28" - } - ) - request.raise_for_status() - except requests.exceptions.RequestException as e: - error_str = str(e) - - logger.warning( - f'Unable to fetch from GitHub API. Error: {error_str}', - exc_info=True, - ) - - return result - - result += request.json() - logger.debug(request.json()) - - # https://docs.github.com/en/rest/using-the-rest-api/using-pagination-in-the-rest-api?apiVersion=2022-11-28 - # See Example creating a pagination method - if not ('link' in request.headers and 'rel=\"next\"' in request.headers['link']): - break - - return result diff --git a/docs/development/custom/app-announcement-hooks.md b/docs/development/custom/app-announcement-hooks.md index 03650a87..594a1d90 100644 --- a/docs/development/custom/app-announcement-hooks.md +++ b/docs/development/custom/app-announcement-hooks.md @@ -9,11 +9,12 @@ To register an AppAnnouncementHook class, you would do the following: ```python from allianceauth import hooks -from allianceauth.services.hooks import AppAnnouncementHook +from allianceauth.admin_status.hooks import AppAnnouncementHook + @hooks.register('app_announcement_hook') def announcement_hook(): - return AppAnnouncementHook("Your app name", "USERNAME/REPOSITORY_NAME", AppAnnouncementHook.RepositoryKind.GITLAB) + return AppAnnouncementHook("Your app name", "USERNAME/REPOSITORY_NAME", AppAnnouncementHook.Service.GITLAB) ``` ```{eval-rst}