Merge branch 'datatables-framework' into 'master'

Draft: Basic framework for datatables server rendering

See merge request allianceauth/allianceauth!1785

allianceauth/framework/datatables.py

@@ -1,6 +1,5 @@
 from collections import defaultdict
 import re
-from typing import List
 
 from django.db.models import Model, Q
 from django.http import HttpRequest, JsonResponse
@@ -13,82 +12,81 @@ from allianceauth.services.hooks import get_extension_logger
 
 logger = get_extension_logger(__name__)
 
-
-class nested_param_dict(dict):
-    """
-    Helper to create infinite depth default dicts for setting from params
-    """
-    def __setitem__(self, item, value):
-        if "." in item:
-            head, path = item.split(".", 1)
-            try:
-                head = int(head)
-            except ValueError:
-                pass
-            obj = self.setdefault(head, nested_param_dict())
-            obj[path] = value
-        else:
-            super().__setitem__(item, value)
-
-
-def defaultdict_to_dict(d):
-    """
-    Helper to convert default dict back to dict
-    """
-    if isinstance(d, defaultdict):
-        d = {k: defaultdict_to_dict(v) for k, v in d.items()}
-    return d
-
-
 class DataTablesView(View):
 
     model: Model = None
     columns: list[tuple] = []
 
-    def get_model_qs(self, request: HttpRequest, *args, **kwargs):
+    COL_SEARCH_REGEX = r"columns\[(?P<id>[0-9]{1,})\]\[search\]\[value\]"
+    COL_SEARCHABLE_REGEX = r"columns\[(?P<id>[0-9]{1,})\]\[searchable\]"
+    COL_SEARCHABLE_AS_REGEX_REGEX = r"columns\[(?P<id>[0-9]{1,})\]\[search\]\[regex\]"
+    COL_ORDERABLE_REGEX = r"columns\[(?P<id>[0-9]{1,})\]\[orderable\]"
+
+    search_regex: re.Pattern = None
+    searchable_regex: re.Pattern = None
+    searchable_as_regex_regex: re.Pattern = None
+    order_regex: re.Pattern = None
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        self.search_regex = re.compile(self.COL_SEARCH_REGEX)
+        self.searchable_regex = re.compile(self.COL_SEARCHABLE_REGEX)
+        self.searchable_as_regex_regex = re.compile(self.COL_SEARCHABLE_AS_REGEX_REGEX)
+        self.order_regex = re.compile(self.COL_ORDERABLE_REGEX)
+
+    def get_model_qs(self, request: HttpRequest):
         return self.model.objects
 
-    def filter_qs(self, table_conf: dict):
+    def get_param(self, request: HttpRequest, key: str, cast=str, default=""):
+        return cast(request.GET.get(key, default))
+
+    def filter_qs(self, search_string: str, column_conf: dict):
         # Search
-        filter_qs = Q()
+        filter_q = Q()
-        for id, c in table_conf["columns"].items():
+        for id, c in column_conf.items():
-            _c = self.columns[int(id)][0]
+            _c = self.columns[id][0]
             if c["searchable"] and len(_c) > 0:
-                _sv = str(c["search"]["value"])
+                if len(c["search"]) and len(_c):
-                if len(_sv) > 0:
+                    if c["regex"]:
-                    if c["search"]["regex"]:
+                        filter_q |= Q(**{f'{_c}__iregex': c["search"]})
-                        filter_qs |= Q(**{f'{_c}__iregex': _sv})
                     else:
-                        filter_qs |= Q(**{f'{_c}__icontains': _sv})
+                        filter_q |= Q(**{f'{_c}__icontains': c["search"]})
-                _gsv = str(table_conf["search"]["value"])
+                if len(search_string) > 0:
-                if len(_gsv) > 0:
+                    filter_q |= Q(**{f'{_c}__icontains': search_string})
-                    filter_qs |= Q(**{f'{_c}__icontains': _gsv})
+        return filter_q
-        return filter_qs
 
-    def get_table_config(self, get: dict):
+    def get_columns_config(self, get: dict):
-        _cols = nested_param_dict()
+        columns = defaultdict(
-        for c, v in get.items():
+            lambda: {
-            _keys = [_k for _k in c.replace("]", "").split("[")]
+                "searchable": False,
-            _v = v
+                "orderable": False,
-            if v in ["true", "false"]:
+                "regex": False,
-                _v = _v == "true"
+                "search": ""
+            }
+        )
+        for c in get:
+            _r = self.search_regex.findall(c)
+            if _r:
+                columns[int(_r[0])]["search"] = get[c]
+            _r_s = self.searchable_regex.findall(c)
+            if _r_s:
+                columns[int(_r_s[0])]["searchable"] = get[c] == "true"
+            _r_s_r = self.searchable_as_regex_regex.findall(c)
+            if _r_s_r:
+                columns[int(_r_s_r[0])]["regex"] = get[c] == "true"
+            _r_o = self.order_regex.findall(c)
+            if _r_o:
+                columns[int(_r_o[0])]["orderable"] = get[c] == "true"
+        return columns
+
+    def order_str(self, order_col, order_dir, column_conf: dict):
+        order = ""
+        _o = column_conf[order_col]
+        if _o["orderable"]:
+            if order_dir == 'desc':
+                order = '-' + self.columns[order_col][0]
             else:
-                try:
+                order = self.columns[order_col][0]
-                    _v = int(_v)
-                except ValueError:
-                    pass # not an integer
-            _cols[".".join(_keys)] = _v
-        return defaultdict_to_dict(_cols)
-
-    def get_order(self, table_conf: dict):
-        order = []
-        for oc, od in table_conf["order"].items():
-            _c = table_conf["columns"][od["column"]]
-            if _c["orderable"]:
-                if od["dir"] == "desc":
-                    order.append("-" + self.columns[int(od["column"])][0])
-                else:
-                    order.append(self.columns[int(od["column"])][0])
         return order
 
     def render_template(self, request: HttpRequest, template: str, ctx: dict):
@@ -103,24 +101,29 @@ class DataTablesView(View):
             )
 
     def get(self, request: HttpRequest, *args, **kwargs):
-        table_conf = self.get_table_config(request.GET)
+        # Get all our Params out of GET
-        draw = int(table_conf["draw"])
+        length = self.get_param(request, "length", int)
-        start = int(table_conf["start"])
+        start = self.get_param(request, "start", int)
-        length = int(table_conf["length"])
+        search_string = self.get_param(request, "search[value]")
+        order_col = self.get_param(request, "order[0][column]", int, 0)
+        order_dir = self.get_param(request, "order[0][dir]")
+        draw = self.get_param(request, "draw", int)
+
         limit = start + length
 
+        column_conf = self.get_columns_config(request.GET)
+
+        # Searches
+        filter_q = Q() | self.filter_qs(search_string, column_conf)
 
         # Build response rows
         items = []
-        qs = self.get_model_qs(
+        qs = self.get_model_qs(request).filter(filter_q).order_by()
-            request,
+
-            *args,
+        # Apply ordering
-            **kwargs
+        order = self.order_str(order_col, order_dir, column_conf)
-        ).filter(
+        if order != "":
-            self.filter_qs(table_conf)
+            qs = qs.order_by(order)
-        ).order_by(
-            *self.get_order(table_conf)
-        )
 
         # build output
         for row in qs[start:limit]:
@@ -133,7 +136,7 @@ class DataTablesView(View):
         # Build our output dict
         datatables_data = {}
         datatables_data['draw'] = draw
-        datatables_data['recordsTotal'] = self.get_model_qs(request, *args, **kwargs).all().count()
+        datatables_data['recordsTotal'] = self.get_model_qs(request).all().count()
         datatables_data['recordsFiltered'] = qs.count()
         datatables_data['data'] = items
 

allianceauth/framework/tests/test_datatables.py

@@ -23,6 +23,10 @@ class TestView(DataTablesView):
     ]
 
 class TestDataTables(TestCase):
+    """
+    Tests for get_main_character_from_user
+    """
+
     def setUp(self):
         self.get_params = {
             'draw': '1',
@@ -86,6 +90,7 @@ class TestDataTables(TestCase):
 
 
     def test_view_default(self):
+
         self.client.force_login(self.user)
         request = self.factory.get('/fake-url/', data=self.get_params)
         response = TestView()

docs/development/custom/framework/datatables.md

@@ -121,24 +121,6 @@ Given the `EveCharacter` Model as our model of choice we would define our stubs
 
 Then we can setup out view in our `appname/views.py` file.
 
-### Columns definition
-
-The `columns` must be defined as a 2 part tuple
-
-- Part 1 is the database field that will be used for filtering and ordering. If this is a foreign key you need to point to a field that is compatible with `__icontains` like `charField` or `textField`. It can be `None`/`False`/`""` if no ordering for filtering is required for this row.
-  - Examples for the EveCharacter Model:
-    - `character_name`
-    - `character_ownership__user__username`
-    - `character_ownership__user__profile__main_character__character_name`
-- Part 2 is a string that is used to the render the column for each row. This can be a html stub or a string containing django style template language.
-  - Examples for the EveCharacter Model
-    - `{{ row.character_name }}`
-    - `{{ row.character_ownership.user.username }}`
-    - `{{ row.character_ownership.user.profile.main_character.character_name }}`
-    - `appname/stubs/character_img.html`
-
-### appname/views.py
-
 ```python
 from django.shortcuts import render
 # Alliance Auth