Merge branch 'datatables-framework' into 'master'

Draft: Basic framework for datatables server rendering

See merge request allianceauth/allianceauth!1785

allianceauth/framework/datatables.py

@@ -1,3 +1,4 @@
+from collections import defaultdict
 import re
 
 from django.db.models import Model, Q
@@ -12,54 +13,80 @@ from allianceauth.services.hooks import get_extension_logger
 logger = get_extension_logger(__name__)
 
 class DataTablesView(View):
-    """
-    Basic DataTables server side table rendering
-    """
+
     model: Model = None
-    templates: list[str] = []
     columns: list[tuple] = []
 
+    COL_SEARCH_REGEX = r"columns\[(?P<id>[0-9]{1,})\]\[search\]\[value\]"
+    COL_SEARCHABLE_REGEX = r"columns\[(?P<id>[0-9]{1,})\]\[searchable\]"
+    COL_SEARCHABLE_AS_REGEX_REGEX = r"columns\[(?P<id>[0-9]{1,})\]\[search\]\[regex\]"
+    COL_ORDERABLE_REGEX = r"columns\[(?P<id>[0-9]{1,})\]\[orderable\]"
+
+    search_regex: re.Pattern = None
+    searchable_regex: re.Pattern = None
+    searchable_as_regex_regex: re.Pattern = None
+    order_regex: re.Pattern = None
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        self.search_regex = re.compile(self.COL_SEARCH_REGEX)
+        self.searchable_regex = re.compile(self.COL_SEARCHABLE_REGEX)
+        self.searchable_as_regex_regex = re.compile(self.COL_SEARCHABLE_AS_REGEX_REGEX)
+        self.order_regex = re.compile(self.COL_ORDERABLE_REGEX)
+
     def get_model_qs(self, request: HttpRequest):
         return self.model.objects
 
     def get_param(self, request: HttpRequest, key: str, cast=str, default=""):
         return cast(request.GET.get(key, default))
 
-    def filter_qs(self, search_string: str):
-        # Global Search
+    def filter_qs(self, search_string: str, column_conf: dict):
+        # Search
         filter_q = Q()
-        if len(search_string) > 0:
-            val = search_string
-            for x in self.columns:
-                if x[0]:
-                    # Apply search to all Columns
-                    filter_q |= Q(**{f'{x[1]}__icontains': val})
+        for id, c in column_conf.items():
+            _c = self.columns[id][0]
+            if c["searchable"] and len(_c) > 0:
+                if len(c["search"]) and len(_c):
+                    if c["regex"]:
+                        filter_q |= Q(**{f'{_c}__iregex': c["search"]})
+                    else:
+                        filter_q |= Q(**{f'{_c}__icontains': c["search"]})
+                if len(search_string) > 0:
+                    filter_q |= Q(**{f'{_c}__icontains': search_string})
         return filter_q
 
-    def filter_col_qs(self, get: dict):
-        # Row Search
-        # TODO check if we have a regex search or not
-        # TODO check if searchable or not
-        col_id_regex = r"columns\[(?P<id>[0-9]{1,})\]\[search\]\[value\]"
-        regex = re.compile(col_id_regex)
-        filter_q = Q()
+    def get_columns_config(self, get: dict):
+        columns = defaultdict(
+            lambda: {
+                "searchable": False,
+                "orderable": False,
+                "regex": False,
+                "search": ""
+            }
+        )
         for c in get:
-            _r = regex.findall(c)
+            _r = self.search_regex.findall(c)
             if _r:
-                _c = self.columns[int(_r[0])]
-                if _c[0]:
-                    if len(get[c]):
-                        filter_q |= Q(**{f'{_c[1]}__iregex': get[c]})
-        return filter_q
+                columns[int(_r[0])]["search"] = get[c]
+            _r_s = self.searchable_regex.findall(c)
+            if _r_s:
+                columns[int(_r_s[0])]["searchable"] = get[c] == "true"
+            _r_s_r = self.searchable_as_regex_regex.findall(c)
+            if _r_s_r:
+                columns[int(_r_s_r[0])]["regex"] = get[c] == "true"
+            _r_o = self.order_regex.findall(c)
+            if _r_o:
+                columns[int(_r_o[0])]["orderable"] = get[c] == "true"
+        return columns
 
-    def order_str(self, order_col, order_dir):
+    def order_str(self, order_col, order_dir, column_conf: dict):
         order = ""
-        _o = self.columns[order_col]
-        if _o[0]:
+        _o = column_conf[order_col]
+        if _o["orderable"]:
             if order_dir == 'desc':
-                order = '-' + _o[1]
+                order = '-' + self.columns[order_col][0]
             else:
-                order = _o[1]
+                order = self.columns[order_col][0]
         return order
 
     def render_template(self, request: HttpRequest, template: str, ctx: dict):
@@ -74,7 +101,6 @@ class DataTablesView(View):
             )
 
     def get(self, request: HttpRequest, *args, **kwargs):
-
         # Get all our Params out of GET
         length = self.get_param(request, "length", int)
         start = self.get_param(request, "start", int)
@@ -85,15 +111,17 @@ class DataTablesView(View):
 
         limit = start + length
 
+        column_conf = self.get_columns_config(request.GET)
+
         # Searches
-        filter_q = Q() | self.filter_qs(search_string) | self.filter_col_qs(request.GET)
+        filter_q = Q() | self.filter_qs(search_string, column_conf)
 
         # Build response rows
         items = []
         qs = self.get_model_qs(request).filter(filter_q).order_by()
 
         # Apply ordering
-        order = self.order_str(order_col, order_dir)
+        order = self.order_str(order_col, order_dir, column_conf)
         if order != "":
             qs = qs.order_by(order)
 
@@ -101,8 +129,8 @@ class DataTablesView(View):
         for row in qs[start:limit]:
             ctx = {"row": row}
             row = []
-            for t in self.templates:
-                row.append(self.render_template(request, t, ctx))
+            for t in self.columns:
+                row.append(self.render_template(request, t[1], ctx))
             items.append(row)
 
         # Build our output dict

allianceauth/framework/tests/test_datatables.py

@@ -0,0 +1,160 @@
+"""
+Test sentinel user
+"""
+
+import json
+import re
+
+# Django
+from allianceauth.tests.auth_utils import AuthUtils
+from django.test import RequestFactory, TestCase
+from django.http import HttpRequest
+# Alliance Auth
+from allianceauth.framework.datatables import DataTablesView
+from allianceauth.eveonline.models import EveCharacter
+
+class TestView(DataTablesView):
+    model=EveCharacter
+    columns = [
+        ("", "{{ row.character_id }}"),
+        ("character_name", "{{ row.character_name }}"),
+        ("corporation_name", "{{ row.corporation_name }}"),
+        ("alliance_name", "{{ row.alliance_name }}"),
+    ]
+
+class TestDataTables(TestCase):
+    """
+    Tests for get_main_character_from_user
+    """
+
+    def setUp(self):
+        self.get_params = {
+            'draw': '1',
+            'columns[0][data]': '0',
+            'columns[0][name]': '',
+            'columns[0][searchable]': 'false',
+            'columns[0][orderable]': 'false',
+            'columns[0][search][value]': '',
+            'columns[0][search][regex]': 'false',
+            'columns[1][data]': '1',
+            'columns[1][name]': '',
+            'columns[1][searchable]': 'true',
+            'columns[1][orderable]': 'true',
+            'columns[1][search][value]': '',
+            'columns[1][search][regex]': 'false',
+            'columns[2][data]': '2',
+            'columns[2][name]': '',
+            'columns[2][searchable]': 'true',
+            'columns[2][orderable]': 'false',
+            'columns[2][search][value]': '',
+            'columns[2][search][regex]': 'false',
+            'columns[3][data]': '3',
+            'columns[3][name]': '',
+            'columns[3][searchable]': 'true',
+            'columns[3][orderable]': 'true',
+            'columns[3][search][value]': '',
+            'columns[3][search][regex]': 'false',
+            'order[0][column]': '1',
+            'order[0][dir]': 'asc',
+            'start': '0',
+            'length': '10',
+            'search[value]': '',
+            'search[regex]': 'false',
+            '_': '123456789'
+        }
+
+
+    @classmethod
+    def setUpClass(cls) -> None:
+        """
+        Set up eve models
+        """
+
+        super().setUpClass()
+        cls.factory = RequestFactory()
+
+        cls.user = AuthUtils.create_user("bruce_wayne")
+        cls.user.is_superuser = True
+        cls.user.save()
+
+        EveCharacter.objects.all().delete()
+        for i in range(1,21):
+            EveCharacter.objects.create(
+                character_id=1000+i,
+                character_name=f"{1000+i} - Test Character",
+                corporation_id=2000+i,
+                corporation_name=f"{2000+i} - Test Corporation",
+                alliance_id=3000+i,
+                alliance_name=f"{3000+i} - Test Alliance",
+            )
+
+
+    def test_view_default(self):
+
+        self.client.force_login(self.user)
+        request = self.factory.get('/fake-url/', data=self.get_params)
+        response = TestView()
+        response.setup(request)
+        data = json.loads(response.get(request).content)["data"]
+        self.assertEqual(data[0][0], "1001")
+        self.assertEqual(data[9][0], "1010")
+
+    def test_view_reverse_sort(self):
+        self.get_params["order[0][dir]"] = "desc"
+        self.client.force_login(self.user)
+        request = self.factory.get('/fake-url/', data=self.get_params)
+        response = TestView()
+        response.setup(request)
+        data = json.loads(response.get(request).content)["data"]
+        self.assertEqual(data[0][0], "1020")
+        self.assertEqual(data[9][0], "1011")
+
+    def test_view_non_sortable_sort(self):
+        self.get_params["order[0][dir]"] = "desc"
+        self.get_params["order[0][column]"] = "0"
+        self.client.force_login(self.user)
+        request = self.factory.get('/fake-url/', data=self.get_params)
+        response = TestView()
+        response.setup(request)
+        data = json.loads(response.get(request).content)["data"]
+        self.assertEqual(data[0][0], "1001")
+        self.assertEqual(data[9][0], "1010")
+
+    def test_view_20_rows(self):
+        self.get_params["length"] = "20"
+        self.client.force_login(self.user)
+        request = self.factory.get('/fake-url/', data=self.get_params)
+        response = TestView()
+        response.setup(request)
+        data = json.loads(response.get(request).content)["data"]
+        self.assertEqual(data[0][0], "1001")
+        self.assertEqual(data[19][0], "1020")
+
+    def test_view_global_search(self):
+        self.get_params["search[value]"] = "1020"
+        self.client.force_login(self.user)
+        request = self.factory.get('/fake-url/', data=self.get_params)
+        response = TestView()
+        response.setup(request)
+        data = json.loads(response.get(request).content)["data"]
+        self.assertEqual(len(data), 1)
+        self.assertEqual(data[0][0], "1020")
+
+    def test_view_col_1_search(self):
+        self.get_params["columns[1][search][value]"] = "1020"
+        self.client.force_login(self.user)
+        request = self.factory.get('/fake-url/', data=self.get_params)
+        response = TestView()
+        response.setup(request)
+        data = json.loads(response.get(request).content)["data"]
+        self.assertEqual(len(data), 1)
+        self.assertEqual(data[0][0], "1020")
+
+    def test_view_col_1_search_empty(self):
+        self.get_params["columns[1][search][value]"] = "zzz"
+        self.client.force_login(self.user)
+        request = self.factory.get('/fake-url/', data=self.get_params)
+        response = TestView()
+        response.setup(request)
+        data = json.loads(response.get(request).content)["data"]
+        self.assertEqual(len(data), 0)

docs/development/custom/framework/datatables.md

@@ -16,26 +16,26 @@ Given the `EveCharacter` Model as our model of choice we would define our stubs
 
 ### template/appname/stubs/icon.html
 
-```html
+```django
 {% load evelinks %}
 {% character_portrait_url row 32 %}
 ```
 
 ### template/appname/stubs/name.html
 
-```html
-{{ row.character_name }} <span class="text-small">({{row.character_ownership.user.username}})</span>
+```django
+{{ row.character_name }} <span class="text-small">({{ row.character_ownership.user.username }})</span>
 ```
 
 ### template/appname/stubs/corp.html
 
-```html
+```django
 {{ row.corporation_name }}
 ```
 
 ### template/appname/list.html
 
-```html
+```django
 {% extends "allianceauth/base-bs5.html" %}
 {% load i18n %}
 {% block page_title %}
@@ -77,7 +77,7 @@ Given the `EveCharacter` Model as our model of choice we would define our stubs
         $(document).ready(function() {
             $('#table').DataTable({
                 serverSide: true,
-                ajax: '/appname/tables/data_table',
+                ajax: '{% url "appname:table" %}',
                 // This is for the column searching
                 initComplete: function () {
                     this.api()
@@ -130,21 +130,17 @@ from allianceauth.eveonline.models import EveCharacter
 ## Datatables server side view
 class EveCharacterTable(DataTablesView):
     model = EveCharacter
-    # Templates can be a html file or template language directly in the list below
-    templates = [
-        "appname/stubs/icon.html",
-        "appname/stubs/name.html",
-        "appname/stubs/corp.html",
-        "{{ row.alliance_name }} {{ row.alliance_id }}"
-    ]
 
-    # Define the columns for filtering and ordering
+    # Define the columns as a tuple.
+    # String for field name for filtering and ordering
+    # String for the render template
+    # Templates can be a html file or template language directly in the list below
     columns = [
-        # (can_sort: bool, "field_for_queries_or_sort")
-        (False, ""),
-        (True, "character_name"),
-        (True, "corporation_name"),
-        (True, "alliance_name"),
+        # ("field_for_queries_or_sort", template: str)
+        ("", "appname/stubs/icon.html"),
+        ("character_name", "appname/stubs/name.html"),
+        ("corporation_name", "appname/stubs/corp.html"),
+        ("alliance_name", "{{ row.alliance_name }} {{ row.alliance_id }}"),
     ]
 
     # if you need to do some prefetch or pre-filtering you can overide this function

tox.ini

@@ -7,7 +7,7 @@ envlist = py{38,39,310,311,312}-{all,core}, docs
 [testenv]
 setenv =
     all: DJANGO_SETTINGS_MODULE = tests.settings_all
-    core: DJANGO_SETTINGS_MODULE = tests.settings_core
+    ; core: DJANGO_SETTINGS_MODULE = tests.settings_core
 basepython =
     py38: python3.8
     py39: python3.9
@@ -18,10 +18,10 @@ deps=
     coverage
 install_command = pip install -e ".[test]" -U {opts} {packages}
 commands =
-    all: coverage run runtests.py -v 2 --debug-mode
-    core: coverage run runtests.py allianceauth.authentication.tests.test_app_settings -v 2 --debug-mode
-    all: coverage report -m
-    all: coverage xml
+    all: coverage run runtests.py allianceauth.framework.tests.test_datatables -v 2 --debug-mode
+    ; core: coverage run runtests.py allianceauth.authentication.tests.test_app_settings -v 2 --debug-mode
+    ; all: coverage report -m
+    ; all: coverage xml
 
 [testenv:docs]
 description = invoke sphinx-build to build the HTML docs