Merge branch 'datatables-framework' into 'master'

Draft: Basic framework for datatables server rendering

See merge request allianceauth/allianceauth!1785

allianceauth/framework/datatables.py

@@ -0,0 +1,143 @@
+from collections import defaultdict
+import re
+
+from django.db.models import Model, Q
+from django.http import HttpRequest, JsonResponse
+from django.template import Context, Template
+from django.template.loader import render_to_string
+from django.views import View
+
+from allianceauth.services.hooks import get_extension_logger
+
+
+logger = get_extension_logger(__name__)
+
+class DataTablesView(View):
+
+    model: Model = None
+    columns: list[tuple] = []
+
+    COL_SEARCH_REGEX = r"columns\[(?P<id>[0-9]{1,})\]\[search\]\[value\]"
+    COL_SEARCHABLE_REGEX = r"columns\[(?P<id>[0-9]{1,})\]\[searchable\]"
+    COL_SEARCHABLE_AS_REGEX_REGEX = r"columns\[(?P<id>[0-9]{1,})\]\[search\]\[regex\]"
+    COL_ORDERABLE_REGEX = r"columns\[(?P<id>[0-9]{1,})\]\[orderable\]"
+
+    search_regex: re.Pattern = None
+    searchable_regex: re.Pattern = None
+    searchable_as_regex_regex: re.Pattern = None
+    order_regex: re.Pattern = None
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        self.search_regex = re.compile(self.COL_SEARCH_REGEX)
+        self.searchable_regex = re.compile(self.COL_SEARCHABLE_REGEX)
+        self.searchable_as_regex_regex = re.compile(self.COL_SEARCHABLE_AS_REGEX_REGEX)
+        self.order_regex = re.compile(self.COL_ORDERABLE_REGEX)
+
+    def get_model_qs(self, request: HttpRequest):
+        return self.model.objects
+
+    def get_param(self, request: HttpRequest, key: str, cast=str, default=""):
+        return cast(request.GET.get(key, default))
+
+    def filter_qs(self, search_string: str, column_conf: dict):
+        # Search
+        filter_q = Q()
+        for id, c in column_conf.items():
+            _c = self.columns[id][0]
+            if c["searchable"] and len(_c) > 0:
+                if len(c["search"]) and len(_c):
+                    if c["regex"]:
+                        filter_q |= Q(**{f'{_c}__iregex': c["search"]})
+                    else:
+                        filter_q |= Q(**{f'{_c}__icontains': c["search"]})
+                if len(search_string) > 0:
+                    filter_q |= Q(**{f'{_c}__icontains': search_string})
+        return filter_q
+
+    def get_columns_config(self, get: dict):
+        columns = defaultdict(
+            lambda: {
+                "searchable": False,
+                "orderable": False,
+                "regex": False,
+                "search": ""
+            }
+        )
+        for c in get:
+            _r = self.search_regex.findall(c)
+            if _r:
+                columns[int(_r[0])]["search"] = get[c]
+            _r_s = self.searchable_regex.findall(c)
+            if _r_s:
+                columns[int(_r_s[0])]["searchable"] = get[c] == "true"
+            _r_s_r = self.searchable_as_regex_regex.findall(c)
+            if _r_s_r:
+                columns[int(_r_s_r[0])]["regex"] = get[c] == "true"
+            _r_o = self.order_regex.findall(c)
+            if _r_o:
+                columns[int(_r_o[0])]["orderable"] = get[c] == "true"
+        return columns
+
+    def order_str(self, order_col, order_dir, column_conf: dict):
+        order = ""
+        _o = column_conf[order_col]
+        if _o["orderable"]:
+            if order_dir == 'desc':
+                order = '-' + self.columns[order_col][0]
+            else:
+                order = self.columns[order_col][0]
+        return order
+
+    def render_template(self, request: HttpRequest, template: str, ctx: dict):
+        if "{{" in template:
+            _template = Template(template)
+            return _template.render(Context(ctx))
+        else:
+            return render_to_string(
+                template,
+                ctx,
+                request
+            )
+
+    def get(self, request: HttpRequest, *args, **kwargs):
+        # Get all our Params out of GET
+        length = self.get_param(request, "length", int)
+        start = self.get_param(request, "start", int)
+        search_string = self.get_param(request, "search[value]")
+        order_col = self.get_param(request, "order[0][column]", int, 0)
+        order_dir = self.get_param(request, "order[0][dir]")
+        draw = self.get_param(request, "draw", int)
+
+        limit = start + length
+
+        column_conf = self.get_columns_config(request.GET)
+
+        # Searches
+        filter_q = Q() | self.filter_qs(search_string, column_conf)
+
+        # Build response rows
+        items = []
+        qs = self.get_model_qs(request).filter(filter_q).order_by()
+
+        # Apply ordering
+        order = self.order_str(order_col, order_dir, column_conf)
+        if order != "":
+            qs = qs.order_by(order)
+
+        # build output
+        for row in qs[start:limit]:
+            ctx = {"row": row}
+            row = []
+            for t in self.columns:
+                row.append(self.render_template(request, t[1], ctx))
+            items.append(row)
+
+        # Build our output dict
+        datatables_data = {}
+        datatables_data['draw'] = draw
+        datatables_data['recordsTotal'] = self.get_model_qs(request).all().count()
+        datatables_data['recordsFiltered'] = qs.count()
+        datatables_data['data'] = items
+
+        return JsonResponse(datatables_data)

allianceauth/framework/tests/test_datatables.py

@@ -0,0 +1,160 @@
+"""
+Test sentinel user
+"""
+
+import json
+import re
+
+# Django
+from allianceauth.tests.auth_utils import AuthUtils
+from django.test import RequestFactory, TestCase
+from django.http import HttpRequest
+# Alliance Auth
+from allianceauth.framework.datatables import DataTablesView
+from allianceauth.eveonline.models import EveCharacter
+
+class TestView(DataTablesView):
+    model=EveCharacter
+    columns = [
+        ("", "{{ row.character_id }}"),
+        ("character_name", "{{ row.character_name }}"),
+        ("corporation_name", "{{ row.corporation_name }}"),
+        ("alliance_name", "{{ row.alliance_name }}"),
+    ]
+
+class TestDataTables(TestCase):
+    """
+    Tests for get_main_character_from_user
+    """
+
+    def setUp(self):
+        self.get_params = {
+            'draw': '1',
+            'columns[0][data]': '0',
+            'columns[0][name]': '',
+            'columns[0][searchable]': 'false',
+            'columns[0][orderable]': 'false',
+            'columns[0][search][value]': '',
+            'columns[0][search][regex]': 'false',
+            'columns[1][data]': '1',
+            'columns[1][name]': '',
+            'columns[1][searchable]': 'true',
+            'columns[1][orderable]': 'true',
+            'columns[1][search][value]': '',
+            'columns[1][search][regex]': 'false',
+            'columns[2][data]': '2',
+            'columns[2][name]': '',
+            'columns[2][searchable]': 'true',
+            'columns[2][orderable]': 'false',
+            'columns[2][search][value]': '',
+            'columns[2][search][regex]': 'false',
+            'columns[3][data]': '3',
+            'columns[3][name]': '',
+            'columns[3][searchable]': 'true',
+            'columns[3][orderable]': 'true',
+            'columns[3][search][value]': '',
+            'columns[3][search][regex]': 'false',
+            'order[0][column]': '1',
+            'order[0][dir]': 'asc',
+            'start': '0',
+            'length': '10',
+            'search[value]': '',
+            'search[regex]': 'false',
+            '_': '123456789'
+        }
+
+
+    @classmethod
+    def setUpClass(cls) -> None:
+        """
+        Set up eve models
+        """
+
+        super().setUpClass()
+        cls.factory = RequestFactory()
+
+        cls.user = AuthUtils.create_user("bruce_wayne")
+        cls.user.is_superuser = True
+        cls.user.save()
+
+        EveCharacter.objects.all().delete()
+        for i in range(1,21):
+            EveCharacter.objects.create(
+                character_id=1000+i,
+                character_name=f"{1000+i} - Test Character",
+                corporation_id=2000+i,
+                corporation_name=f"{2000+i} - Test Corporation",
+                alliance_id=3000+i,
+                alliance_name=f"{3000+i} - Test Alliance",
+            )
+
+
+    def test_view_default(self):
+
+        self.client.force_login(self.user)
+        request = self.factory.get('/fake-url/', data=self.get_params)
+        response = TestView()
+        response.setup(request)
+        data = json.loads(response.get(request).content)["data"]
+        self.assertEqual(data[0][0], "1001")
+        self.assertEqual(data[9][0], "1010")
+
+    def test_view_reverse_sort(self):
+        self.get_params["order[0][dir]"] = "desc"
+        self.client.force_login(self.user)
+        request = self.factory.get('/fake-url/', data=self.get_params)
+        response = TestView()
+        response.setup(request)
+        data = json.loads(response.get(request).content)["data"]
+        self.assertEqual(data[0][0], "1020")
+        self.assertEqual(data[9][0], "1011")
+
+    def test_view_non_sortable_sort(self):
+        self.get_params["order[0][dir]"] = "desc"
+        self.get_params["order[0][column]"] = "0"
+        self.client.force_login(self.user)
+        request = self.factory.get('/fake-url/', data=self.get_params)
+        response = TestView()
+        response.setup(request)
+        data = json.loads(response.get(request).content)["data"]
+        self.assertEqual(data[0][0], "1001")
+        self.assertEqual(data[9][0], "1010")
+
+    def test_view_20_rows(self):
+        self.get_params["length"] = "20"
+        self.client.force_login(self.user)
+        request = self.factory.get('/fake-url/', data=self.get_params)
+        response = TestView()
+        response.setup(request)
+        data = json.loads(response.get(request).content)["data"]
+        self.assertEqual(data[0][0], "1001")
+        self.assertEqual(data[19][0], "1020")
+
+    def test_view_global_search(self):
+        self.get_params["search[value]"] = "1020"
+        self.client.force_login(self.user)
+        request = self.factory.get('/fake-url/', data=self.get_params)
+        response = TestView()
+        response.setup(request)
+        data = json.loads(response.get(request).content)["data"]
+        self.assertEqual(len(data), 1)
+        self.assertEqual(data[0][0], "1020")
+
+    def test_view_col_1_search(self):
+        self.get_params["columns[1][search][value]"] = "1020"
+        self.client.force_login(self.user)
+        request = self.factory.get('/fake-url/', data=self.get_params)
+        response = TestView()
+        response.setup(request)
+        data = json.loads(response.get(request).content)["data"]
+        self.assertEqual(len(data), 1)
+        self.assertEqual(data[0][0], "1020")
+
+    def test_view_col_1_search_empty(self):
+        self.get_params["columns[1][search][value]"] = "zzz"
+        self.client.force_login(self.user)
+        request = self.factory.get('/fake-url/', data=self.get_params)
+        response = TestView()
+        response.setup(request)
+        data = json.loads(response.get(request).content)["data"]
+        self.assertEqual(len(data), 0)

docs/development/custom/framework/datatables.md

@@ -0,0 +1,179 @@
+# DataTables Server Side Rendering
+
+The `allianceauth.framework.datatables.DataTablesView` module provides a simple class based view to
+implement simple server side filtering ordering and searching of DataTables.
+
+This is intended to make the life of our community apps developer a little
+easier, so they don't have to reinvent the wheel.
+
+## Usage
+
+To use this view is as easy as defining your stub templates, and fields and adding the view to the `urls.py`
+
+Given the `EveCharacter` Model as our model of choice we would define our stubs like so
+
+## Add our Templates
+
+### template/appname/stubs/icon.html
+
+```django
+{% load evelinks %}
+{% character_portrait_url row 32 %}
+```
+
+### template/appname/stubs/name.html
+
+```django
+{{ row.character_name }} <span class="text-small">({{ row.character_ownership.user.username }})</span>
+```
+
+### template/appname/stubs/corp.html
+
+```django
+{{ row.corporation_name }}
+```
+
+### template/appname/list.html
+
+```django
+{% extends "allianceauth/base-bs5.html" %}
+{% load i18n %}
+{% block page_title %}
+    {% translate "App Name" %}
+{% endblock page_title %}
+{% block content %}
+    <table class="table table-striped w-100" id="table">
+        <!-- Normal Header Rows -->
+        <thead>
+            <tr>
+                <th></th>
+                <th>{% translate "Name" %}</th>
+                <th>{% translate "Corporation" %}</th>
+                <th>{% translate "Alliance" %}</th>
+            </tr>
+        </thead>
+        <!-- Put the footer in the header so it doesn't trigger the sorting on searching -->
+        <!-- only footers with text will get given an input -->
+        <!-- This is just an option, you do it however you like -->
+        <tfoot style="display: table-header-group;">
+            <tr>
+                <th></th>
+                <th>{% translate "Name" %}</th>
+                <th>{% translate "Corporation" %}</th>
+                <th>{% translate "Alliance" %}</th>
+            </tr>
+        </tfoot>
+        <!-- Empty tbody -->
+        <tbody>
+        </tbody>
+    </table>
+{% endblock content %}
+{% block extra_css %}
+    {% include 'bundles/datatables-css-bs5.html' %}
+{% endblock %}
+{% block extra_javascript %}
+    {% include 'bundles/datatables-js-bs5.html' %}
+    <script>
+        $(document).ready(function() {
+            $('#table').DataTable({
+                serverSide: true,
+                ajax: '{% url "appname:table" %}',
+                // This is for the column searching
+                initComplete: function () {
+                    this.api()
+                    .columns()
+                    .every(function () {
+                        let column = this;
+                        let title = column.footer().textContent;
+                        if (title != ""){
+                            // Create input element
+                            let input = document.createElement('input');
+                            input.classList.add("w-100")
+                            input.placeholder = title;
+                            column.footer().replaceChildren(input);
+
+                            // Event listener for user input
+                            input.addEventListener('keyup', () => {
+                                if (column.search() !== this.value) {
+                                    column.search(input.value).draw();
+                                }
+                            });
+                        }
+                    });
+                },
+                columnDefs: [
+                    { "searchable": false, "targets": [0] },
+                    { "sortable": false, "targets": [0] },
+                ],
+                order: [
+                    [1, "asc"]
+                ],
+                pageLength: 10,
+                responsive : true
+            });
+        });
+    </script>
+{% endblock extra_javascript %}
+
+```
+
+## Add our Views
+
+Then we can setup out view in our `appname/views.py` file.
+
+```python
+from django.shortcuts import render
+# Alliance Auth
+from allianceauth.framework.datatables import DataTablesView
+from allianceauth.eveonline.models import EveCharacter
+
+## Datatables server side view
+class EveCharacterTable(DataTablesView):
+    model = EveCharacter
+
+    # Define the columns as a tuple.
+    # String for field name for filtering and ordering
+    # String for the render template
+    # Templates can be a html file or template language directly in the list below
+    columns = [
+        # ("field_for_queries_or_sort", template: str)
+        ("", "appname/stubs/icon.html"),
+        ("character_name", "appname/stubs/name.html"),
+        ("corporation_name", "appname/stubs/corp.html"),
+        ("alliance_name", "{{ row.alliance_name }} {{ row.alliance_id }}"),
+    ]
+
+    # if you need to do some prefetch or pre-filtering you can overide this function
+    def get_model_qs(self, request: HttpRequest):
+        qs = self.model.objects
+        if not request.user.is_superuser:
+            # eg only show unlinked characters to non-superusers
+            # just an example
+            # filtering here will prevent people searching things that may not be visible to them
+            qs = qs.filter(character_ownership__isnull=True)
+        # maybe some character ownership select related for performance?
+        return qs.select_related("character_ownership", "character_ownership__user")
+
+## Main Page View
+def show_table(request):
+    return render("appname/list.html")
+```
+
+## Add our Urls
+
+### appname/urls.py
+
+```python
+from django.urls import path
+
+from . import views
+
+app_name = 'appname'
+
+urlpatterns = [
+    path("list/", views.EveCharacterTable.as_view(), name='eve_character_table'),
+    path("tables/data_table", views.show_table, name='table')
+]
+```
+
+and you are done.

tox.ini

@@ -7,7 +7,7 @@ envlist = py{38,39,310,311,312}-{all,core}, docs
 [testenv]
 setenv =
     all: DJANGO_SETTINGS_MODULE = tests.settings_all
-    core: DJANGO_SETTINGS_MODULE = tests.settings_core
+    ; core: DJANGO_SETTINGS_MODULE = tests.settings_core
 basepython =
     py38: python3.8
     py39: python3.9
@@ -18,10 +18,10 @@ deps=
     coverage
 install_command = pip install -e ".[test]" -U {opts} {packages}
 commands =
-    all: coverage run runtests.py -v 2 --debug-mode
+    all: coverage run runtests.py allianceauth.framework.tests.test_datatables -v 2 --debug-mode
-    core: coverage run runtests.py allianceauth.authentication.tests.test_app_settings -v 2 --debug-mode
+    ; core: coverage run runtests.py allianceauth.authentication.tests.test_app_settings -v 2 --debug-mode
-    all: coverage report -m
+    ; all: coverage report -m
-    all: coverage xml
+    ; all: coverage xml
 
 [testenv:docs]
 description = invoke sphinx-build to build the HTML docs