Merge branch 'datatables-framework' into 'master'

Draft: Basic framework for datatables server rendering

See merge request allianceauth/allianceauth!1785

allianceauth/framework/datatables.py

@@ -1,5 +1,6 @@
 from collections import defaultdict
 import re
+from typing import List
 
 from django.db.models import Model, Q
 from django.http import HttpRequest, JsonResponse
@@ -12,81 +13,82 @@ from allianceauth.services.hooks import get_extension_logger
 
 logger = get_extension_logger(__name__)
 
+
+class nested_param_dict(dict):
+    """
+    Helper to create infinite depth default dicts for setting from params
+    """
+    def __setitem__(self, item, value):
+        if "." in item:
+            head, path = item.split(".", 1)
+            try:
+                head = int(head)
+            except ValueError:
+                pass
+            obj = self.setdefault(head, nested_param_dict())
+            obj[path] = value
+        else:
+            super().__setitem__(item, value)
+
+
+def defaultdict_to_dict(d):
+    """
+    Helper to convert default dict back to dict
+    """
+    if isinstance(d, defaultdict):
+        d = {k: defaultdict_to_dict(v) for k, v in d.items()}
+    return d
+
+
 class DataTablesView(View):
 
     model: Model = None
     columns: list[tuple] = []
 
-    COL_SEARCH_REGEX = r"columns\[(?P<id>[0-9]{1,})\]\[search\]\[value\]"
-    COL_SEARCHABLE_REGEX = r"columns\[(?P<id>[0-9]{1,})\]\[searchable\]"
-    COL_SEARCHABLE_AS_REGEX_REGEX = r"columns\[(?P<id>[0-9]{1,})\]\[search\]\[regex\]"
-    COL_ORDERABLE_REGEX = r"columns\[(?P<id>[0-9]{1,})\]\[orderable\]"
-
-    search_regex: re.Pattern = None
-    searchable_regex: re.Pattern = None
-    searchable_as_regex_regex: re.Pattern = None
-    order_regex: re.Pattern = None
-
-    def __init__(self, **kwargs):
-        super().__init__(**kwargs)
-        self.search_regex = re.compile(self.COL_SEARCH_REGEX)
-        self.searchable_regex = re.compile(self.COL_SEARCHABLE_REGEX)
-        self.searchable_as_regex_regex = re.compile(self.COL_SEARCHABLE_AS_REGEX_REGEX)
-        self.order_regex = re.compile(self.COL_ORDERABLE_REGEX)
-
-    def get_model_qs(self, request: HttpRequest):
+    def get_model_qs(self, request: HttpRequest, *args, **kwargs):
         return self.model.objects
 
-    def get_param(self, request: HttpRequest, key: str, cast=str, default=""):
-        return cast(request.GET.get(key, default))
-
-    def filter_qs(self, search_string: str, column_conf: dict):
+    def filter_qs(self, table_conf: dict):
         # Search
-        filter_q = Q()
-        for id, c in column_conf.items():
-            _c = self.columns[id][0]
+        filter_qs = Q()
+        for id, c in table_conf["columns"].items():
+            _c = self.columns[int(id)][0]
             if c["searchable"] and len(_c) > 0:
-                if len(c["search"]) and len(_c):
-                    if c["regex"]:
-                        filter_q |= Q(**{f'{_c}__iregex': c["search"]})
+                _sv = str(c["search"]["value"])
+                if len(_sv) > 0:
+                    if c["search"]["regex"]:
+                        filter_qs |= Q(**{f'{_c}__iregex': _sv})
                     else:
-                        filter_q |= Q(**{f'{_c}__icontains': c["search"]})
-                if len(search_string) > 0:
-                    filter_q |= Q(**{f'{_c}__icontains': search_string})
-        return filter_q
+                        filter_qs |= Q(**{f'{_c}__icontains': _sv})
+                _gsv = str(table_conf["search"]["value"])
+                if len(_gsv) > 0:
+                    filter_qs |= Q(**{f'{_c}__icontains': _gsv})
+        return filter_qs
 
-    def get_columns_config(self, get: dict):
-        columns = defaultdict(
-            lambda: {
-                "searchable": False,
-                "orderable": False,
-                "regex": False,
-                "search": ""
-            }
-        )
-        for c in get:
-            _r = self.search_regex.findall(c)
-            if _r:
-                columns[int(_r[0])]["search"] = get[c]
-            _r_s = self.searchable_regex.findall(c)
-            if _r_s:
-                columns[int(_r_s[0])]["searchable"] = get[c] == "true"
-            _r_s_r = self.searchable_as_regex_regex.findall(c)
-            if _r_s_r:
-                columns[int(_r_s_r[0])]["regex"] = get[c] == "true"
-            _r_o = self.order_regex.findall(c)
-            if _r_o:
-                columns[int(_r_o[0])]["orderable"] = get[c] == "true"
-        return columns
-
-    def order_str(self, order_col, order_dir, column_conf: dict):
-        order = ""
-        _o = column_conf[order_col]
-        if _o["orderable"]:
-            if order_dir == 'desc':
-                order = '-' + self.columns[order_col][0]
+    def get_table_config(self, get: dict):
+        _cols = nested_param_dict()
+        for c, v in get.items():
+            _keys = [_k for _k in c.replace("]", "").split("[")]
+            _v = v
+            if v in ["true", "false"]:
+                _v = _v == "true"
             else:
-                order = self.columns[order_col][0]
+                try:
+                    _v = int(_v)
+                except ValueError:
+                    pass # not an integer
+            _cols[".".join(_keys)] = _v
+        return defaultdict_to_dict(_cols)
+
+    def get_order(self, table_conf: dict):
+        order = []
+        for oc, od in table_conf["order"].items():
+            _c = table_conf["columns"][od["column"]]
+            if _c["orderable"]:
+                if od["dir"] == "desc":
+                    order.append("-" + self.columns[int(od["column"])][0])
+                else:
+                    order.append(self.columns[int(od["column"])][0])
         return order
 
     def render_template(self, request: HttpRequest, template: str, ctx: dict):
@@ -101,29 +103,24 @@ class DataTablesView(View):
             )
 
     def get(self, request: HttpRequest, *args, **kwargs):
-        # Get all our Params out of GET
-        length = self.get_param(request, "length", int)
-        start = self.get_param(request, "start", int)
-        search_string = self.get_param(request, "search[value]")
-        order_col = self.get_param(request, "order[0][column]", int, 0)
-        order_dir = self.get_param(request, "order[0][dir]")
-        draw = self.get_param(request, "draw", int)
-
+        table_conf = self.get_table_config(request.GET)
+        draw = int(table_conf["draw"])
+        start = int(table_conf["start"])
+        length = int(table_conf["length"])
         limit = start + length
 
-        column_conf = self.get_columns_config(request.GET)
-
-        # Searches
-        filter_q = Q() | self.filter_qs(search_string, column_conf)
 
         # Build response rows
         items = []
-        qs = self.get_model_qs(request).filter(filter_q).order_by()
-
-        # Apply ordering
-        order = self.order_str(order_col, order_dir, column_conf)
-        if order != "":
-            qs = qs.order_by(order)
+        qs = self.get_model_qs(
+            request,
+            *args,
+            **kwargs
+        ).filter(
+            self.filter_qs(table_conf)
+        ).order_by(
+            *self.get_order(table_conf)
+        )
 
         # build output
         for row in qs[start:limit]:
@@ -136,7 +133,7 @@ class DataTablesView(View):
         # Build our output dict
         datatables_data = {}
         datatables_data['draw'] = draw
-        datatables_data['recordsTotal'] = self.get_model_qs(request).all().count()
+        datatables_data['recordsTotal'] = self.get_model_qs(request, *args, **kwargs).all().count()
         datatables_data['recordsFiltered'] = qs.count()
         datatables_data['data'] = items
 

allianceauth/framework/tests/test_datatables.py

@@ -23,10 +23,6 @@ class TestView(DataTablesView):
     ]
 
 class TestDataTables(TestCase):
-    """
-    Tests for get_main_character_from_user
-    """
-
     def setUp(self):
         self.get_params = {
             'draw': '1',
@@ -90,7 +86,6 @@ class TestDataTables(TestCase):
 
 
     def test_view_default(self):
-
         self.client.force_login(self.user)
         request = self.factory.get('/fake-url/', data=self.get_params)
         response = TestView()

docs/development/custom/framework/datatables.md

@@ -121,6 +121,24 @@ Given the `EveCharacter` Model as our model of choice we would define our stubs
 
 Then we can setup out view in our `appname/views.py` file.
 
+### Columns definition
+
+The `columns` must be defined as a 2 part tuple
+
+- Part 1 is the database field that will be used for filtering and ordering. If this is a foreign key you need to point to a field that is compatible with `__icontains` like `charField` or `textField`. It can be `None`/`False`/`""` if no ordering for filtering is required for this row.
+  - Examples for the EveCharacter Model:
+    - `character_name`
+    - `character_ownership__user__username`
+    - `character_ownership__user__profile__main_character__character_name`
+- Part 2 is a string that is used to the render the column for each row. This can be a html stub or a string containing django style template language.
+  - Examples for the EveCharacter Model
+    - `{{ row.character_name }}`
+    - `{{ row.character_ownership.user.username }}`
+    - `{{ row.character_ownership.user.profile.main_character.character_name }}`
+    - `appname/stubs/character_img.html`
+
+### appname/views.py
+
 ```python
 from django.shortcuts import render
 # Alliance Auth