[Feature] Basic framework for datatables server rendering

This commit is contained in:
Aaron Kable
2026-01-20 04:38:22 +00:00
committed by Ariel Rin
parent 99c65d2a5d
commit 543e4169e4
4 changed files with 676 additions and 0 deletions

View File

@@ -0,0 +1,216 @@
from collections import defaultdict
import re
from typing import List
from django.db.models import Model, Q
from django.http import HttpRequest, JsonResponse
from django.template import Context, Template
from django.template.loader import render_to_string
from django.views import View
from allianceauth.services.hooks import get_extension_logger
logger = get_extension_logger(__name__)
class nested_param_dict(dict):
"""
Helper to create infinite depth default dicts for setting from params
"""
def __setitem__(self, item, value):
if "." in item:
head, path = item.split(".", 1)
try:
head = int(head)
except ValueError:
pass
obj = self.setdefault(head, nested_param_dict())
obj[path] = value
else:
super().__setitem__(item, value)
def defaultdict_to_dict(d):
"""
Helper to convert default dict back to dict
"""
if isinstance(d, defaultdict):
d = {k: defaultdict_to_dict(v) for k, v in d.items()}
return d
class DataTablesView(View):
model: Model = None
columns: List[tuple] = []
def get_model_qs(self, request: HttpRequest, *args, **kwargs):
return self.model.objects
def filter_qs(self, table_conf: dict):
# Search
filter_qs = Q()
for id, c in table_conf["columns"].items():
_c = self.columns[int(id)][0]
if c.get("searchable", False) and len(_c) > 0:
if c.get("columnControl", False):
_sv = str(c["columnControl"]["search"]["value"])
"""contains, equal, ends, starts, empty"""
_logic = str(c["columnControl"]["search"]["logic"])
"""text, date, num"""
_type = str(c["columnControl"]["search"]["type"])
if _type == "text":
if _logic == "empty":
filter_qs &= Q(**{f'{_c}': ""})
elif len(_sv) > 0:
if _logic == "contains":
filter_qs &= Q(**{f'{_c}__icontains': _sv})
elif _logic == "starts":
filter_qs &= Q(**{f'{_c}__istartswith': _sv})
elif _logic == "ends":
filter_qs &= Q(**{f'{_c}__iendswith': _sv})
elif _logic == "equal":
filter_qs &= Q(**{f'{_c}': _sv})
elif _type == "num":
if _logic == "empty":
filter_qs &= Q(**{f'{_c}__isnull': True})
elif len(_sv) > 0:
try:
if _logic == "greater":
filter_qs &= Q(**{f'{_c}__gt': float(_sv)})
elif _logic == "less":
filter_qs &= Q(**{f'{_c}__lt': float(_sv)})
elif _logic == "greaterOrEqual":
filter_qs &= Q(**{f'{_c}__gte': float(_sv)})
elif _logic == "lessOrEqual":
filter_qs &= Q(**{f'{_c}__lte': float(_sv)})
elif _logic == "equal":
filter_qs &= Q(**{f'{_c}': float(_sv)})
except ValueError:
pass
else:
_sv = str(c["search"]["value"])
if len(_sv) > 0:
if c["search"]["regex"]:
filter_qs |= Q(**{f'{_c}__iregex': _sv})
else:
filter_qs |= Q(**{f'{_c}__icontains': _sv})
_gsv = str(table_conf["search"]["value"])
if len(_gsv) > 0:
filter_qs |= Q(**{f'{_c}__icontains': _gsv})
return filter_qs
def except_qs(self, table_conf: dict):
# Search
except_qs = Q()
for id, c in table_conf["columns"].items():
_c = self.columns[int(id)][0]
if c.get("searchable", False) and len(_c) > 0:
if c.get("columnControl", False):
_sv = str(c["columnControl"]["search"]["value"])
"""notContains, notEqual, notEmpty"""
_logic = str(c["columnControl"]["search"]["logic"])
"""text, date, num"""
_type = str(c["columnControl"]["search"]["type"])
if _type == "text":
if _logic == "notEmpty":
except_qs |= Q(**{f'{_c}': ""})
elif len(_sv) > 0:
if _logic == "notContains":
except_qs |= Q(**{f'{_c}__icontains': _sv})
elif _logic == "notEqual":
except_qs |= Q(**{f'{_c}': _sv})
elif _type == "num":
if _logic == "notEmpty":
except_qs |= Q(**{f'{_c}__isnull': False})
elif len(_sv) > 0:
if _logic == "notEqual":
try:
except_qs |= Q(**{f'{_c}': float(_sv)})
except ValueError:
pass
return except_qs
def get_table_config(self, get: dict):
_cols = nested_param_dict()
for c, v in get.items():
_keys = [_k for _k in c.replace("]", "").split("[")]
_v = v
if v in ["true", "false"]:
_v = _v == "true"
else:
try:
_v = int(_v)
except ValueError:
pass # not an integer
_cols[".".join(_keys)] = _v
return defaultdict_to_dict(_cols)
def get_order(self, table_conf: dict):
order = []
for oc, od in table_conf.get("order", {}).items():
_c = table_conf["columns"][od["column"]]
if _c["orderable"]:
if od["dir"] == "desc":
order.append("-" + self.columns[int(od["column"])][0])
else:
order.append(self.columns[int(od["column"])][0])
return order
def render_template(self, request: HttpRequest, template: str, ctx: dict):
if "{{" in template:
_template = Template(template)
return _template.render(Context(ctx))
else:
return render_to_string(
template,
ctx,
request
)
def get(self, request: HttpRequest, *args, **kwargs):
table_conf = self.get_table_config(request.GET)
draw = int(table_conf["draw"])
start = int(table_conf["start"])
length = int(table_conf["length"])
if length <= 0:
logger.warning(
"Using no pagination is not recommended for server side rendered datatables"
)
limit = start + length
# Build response rows
items = []
qs = self.get_model_qs(
request,
*args,
**kwargs
).filter(
self.filter_qs(table_conf)
).exclude(
self.except_qs(table_conf)
).order_by(
*self.get_order(table_conf)
)
# build output
if length > 0:
qs = qs[start:limit]
for row in qs:
ctx = {"row": row}
row = []
for t in self.columns:
row.append(self.render_template(request, t[1], ctx))
items.append(row)
# Build our output dict
datatables_data = {}
datatables_data['draw'] = draw
datatables_data['recordsTotal'] = self.get_model_qs(request, *args, **kwargs).all().count()
datatables_data['recordsFiltered'] = qs.count()
datatables_data['data'] = items
return JsonResponse(datatables_data)

View File

@@ -0,0 +1,278 @@
"""
Test sentinel user
"""
import json
import re
# Django
from allianceauth.tests.auth_utils import AuthUtils
from django.test import RequestFactory, TestCase
from django.http import HttpRequest
# Alliance Auth
from allianceauth.framework.datatables import DataTablesView
from allianceauth.eveonline.models import EveCharacter
class TestView(DataTablesView):
model=EveCharacter
columns = [
("", "{{ row.character_id }}"),
("character_name", "{{ row.character_name }}"),
("corporation_name", "{{ row.corporation_name }}"),
("alliance_name", "{{ row.alliance_name }}"),
]
class TestDataTables(TestCase):
def setUp(self):
self.get_params = {
'draw': '1',
'columns[0][data]': '0',
'columns[0][name]': '',
'columns[0][searchable]': 'false',
'columns[0][orderable]': 'false',
'columns[0][search][value]': '',
'columns[0][search][regex]': 'false',
'columns[1][data]': '1',
'columns[1][name]': '',
'columns[1][searchable]': 'true',
'columns[1][orderable]': 'true',
'columns[1][search][value]': '',
'columns[1][search][regex]': 'false',
'columns[2][data]': '2',
'columns[2][name]': '',
'columns[2][searchable]': 'true',
'columns[2][orderable]': 'false',
'columns[2][search][value]': '',
'columns[2][search][regex]': 'false',
'columns[3][data]': '3',
'columns[3][name]': '',
'columns[3][searchable]': 'true',
'columns[3][orderable]': 'true',
'columns[3][search][value]': '',
'columns[3][search][regex]': 'false',
'order[0][column]': '1',
'order[0][dir]': 'asc',
'start': '0',
'length': '10',
'search[value]': '',
'search[regex]': 'false',
'_': '123456789'
}
@classmethod
def setUpClass(cls) -> None:
"""
Set up eve models
"""
super().setUpClass()
cls.factory = RequestFactory()
cls.user = AuthUtils.create_user("bruce_wayne")
cls.user.is_superuser = True
cls.user.save()
EveCharacter.objects.all().delete()
for i in range(1,16):
EveCharacter.objects.create(
character_id=1000+i,
character_name=f"{1000+i} - Test Character - {1000+i}",
corporation_id=2000+i,
corporation_name=f"{2000+i} - Test Corporation",
)
for i in range(16,21):
EveCharacter.objects.create(
character_id=1000+i,
character_name=f"{1000+i} - Test Character - {1000+i}",
corporation_id=2000+i,
corporation_name=f"{2000+i} - Test Corporation",
alliance_id=3000+i,
alliance_name=f"{3000+i} - Test Alliance",
)
def test_view_default(self):
self.client.force_login(self.user)
request = self.factory.get('/fake-url/', data=self.get_params)
response = TestView()
response.setup(request)
data = json.loads(response.get(request).content)["data"]
self.assertEqual(data[0][0], "1001")
self.assertEqual(data[9][0], "1010")
def test_view_reverse_sort(self):
self.get_params["order[0][dir]"] = "desc"
self.client.force_login(self.user)
request = self.factory.get('/fake-url/', data=self.get_params)
response = TestView()
response.setup(request)
data = json.loads(response.get(request).content)["data"]
self.assertEqual(data[0][0], "1020")
self.assertEqual(data[9][0], "1011")
def test_view_no_sort(self):
self.get_params.pop("order[0][column]")
self.get_params.pop("order[0][dir]")
self.client.force_login(self.user)
request = self.factory.get('/fake-url/', data=self.get_params)
response = TestView()
response.setup(request)
data = json.loads(response.get(request).content)["data"]
self.assertEqual(data[0][0], "1001")
self.assertEqual(data[9][0], "1010")
def test_view_non_sortable_sort(self):
self.get_params["order[0][dir]"] = "desc"
self.get_params["order[0][column]"] = "0"
self.client.force_login(self.user)
request = self.factory.get('/fake-url/', data=self.get_params)
response = TestView()
response.setup(request)
data = json.loads(response.get(request).content)["data"]
self.assertEqual(data[0][0], "1001")
self.assertEqual(data[9][0], "1010")
def test_view_20_rows(self):
self.get_params["length"] = "20"
self.client.force_login(self.user)
request = self.factory.get('/fake-url/', data=self.get_params)
response = TestView()
response.setup(request)
data = json.loads(response.get(request).content)["data"]
self.assertEqual(data[0][0], "1001")
self.assertEqual(data[19][0], "1020")
def test_view_global_search(self):
self.get_params["search[value]"] = "1020"
self.client.force_login(self.user)
request = self.factory.get('/fake-url/', data=self.get_params)
response = TestView()
response.setup(request)
data = json.loads(response.get(request).content)["data"]
self.assertEqual(len(data), 1)
self.assertEqual(data[0][0], "1020")
def test_view_col_1_search(self):
self.get_params["columns[1][search][value]"] = "1020"
self.client.force_login(self.user)
request = self.factory.get('/fake-url/', data=self.get_params)
response = TestView()
response.setup(request)
data = json.loads(response.get(request).content)["data"]
self.assertEqual(len(data), 1)
self.assertEqual(data[0][0], "1020")
def test_view_col_1_search_empty(self):
self.get_params["columns[1][search][value]"] = "zzz"
self.client.force_login(self.user)
request = self.factory.get('/fake-url/', data=self.get_params)
response = TestView()
response.setup(request)
data = json.loads(response.get(request).content)["data"]
self.assertEqual(len(data), 0)
def test_view_cc_3_search_empty(self):
self.get_params["columns[3][columnControl][search][value]"] = ""
self.get_params["columns[3][columnControl][search][logic]"] = "empty"
self.get_params["columns[3][columnControl][search][type]"] = "text"
self.get_params["length"] = "20"
self.client.force_login(self.user)
request = self.factory.get('/fake-url/', data=self.get_params)
response = TestView()
response.setup(request)
data = json.loads(response.get(request).content)["data"]
self.assertEqual(len(data), 15)
def test_view_cc_3_search_not_empty(self):
self.get_params["columns[3][columnControl][search][value]"] = ""
self.get_params["columns[3][columnControl][search][logic]"] = "notEmpty"
self.get_params["columns[3][columnControl][search][type]"] = "text"
self.client.force_login(self.user)
request = self.factory.get('/fake-url/', data=self.get_params)
response = TestView()
response.setup(request)
data = json.loads(response.get(request).content)["data"]
self.assertEqual(len(data), 5)
def test_view_cc_1_search_ends_with(self):
self.get_params["columns[1][columnControl][search][value]"] = "9"
self.get_params["columns[1][columnControl][search][logic]"] = "ends"
self.get_params["columns[1][columnControl][search][type]"] = "text"
self.client.force_login(self.user)
request = self.factory.get('/fake-url/', data=self.get_params)
response = TestView()
response.setup(request)
data = json.loads(response.get(request).content)["data"]
self.assertEqual(len(data), 2)
def test_view_cc_1_search_starts_with(self):
self.get_params["columns[1][columnControl][search][value]"] = "1009"
self.get_params["columns[1][columnControl][search][logic]"] = "starts"
self.get_params["columns[1][columnControl][search][type]"] = "text"
self.client.force_login(self.user)
request = self.factory.get('/fake-url/', data=self.get_params)
response = TestView()
response.setup(request)
data = json.loads(response.get(request).content)["data"]
self.assertEqual(len(data), 1)
def test_view_cc_1_search_not_contains(self):
self.get_params["columns[1][columnControl][search][value]"] = "100"
self.get_params["columns[1][columnControl][search][logic]"] = "notContains"
self.get_params["columns[1][columnControl][search][type]"] = "text"
self.get_params["length"] = "20"
self.client.force_login(self.user)
request = self.factory.get('/fake-url/', data=self.get_params)
response = TestView()
response.setup(request)
data = json.loads(response.get(request).content)["data"]
self.assertEqual(len(data), 11)
def test_view_cc_1_search_contains(self):
self.get_params["columns[1][columnControl][search][value]"] = "100"
self.get_params["columns[1][columnControl][search][logic]"] = "contains"
self.get_params["columns[1][columnControl][search][type]"] = "text"
self.get_params["length"] = "20"
self.client.force_login(self.user)
request = self.factory.get('/fake-url/', data=self.get_params)
response = TestView()
response.setup(request)
data = json.loads(response.get(request).content)["data"]
self.assertEqual(len(data), 9)
def test_view_cc_1_search_equal(self):
self.get_params["columns[1][columnControl][search][value]"] = "1001 - Test Character - 1001"
self.get_params["columns[1][columnControl][search][logic]"] = "equal"
self.get_params["columns[1][columnControl][search][type]"] = "text"
self.get_params["length"] = "20"
self.client.force_login(self.user)
request = self.factory.get('/fake-url/', data=self.get_params)
response = TestView()
response.setup(request)
data = json.loads(response.get(request).content)["data"]
self.assertEqual(len(data), 1)
def test_view_cc_1_search_not_equal(self):
self.get_params["columns[1][columnControl][search][value]"] = "1001 - Test Character - 1001"
self.get_params["columns[1][columnControl][search][logic]"] = "notEqual"
self.get_params["columns[1][columnControl][search][type]"] = "text"
self.get_params["length"] = "20"
self.client.force_login(self.user)
request = self.factory.get('/fake-url/', data=self.get_params)
response = TestView()
response.setup(request)
data = json.loads(response.get(request).content)["data"]
self.assertEqual(len(data), 19)
def test_view_cc_no_pagination(self):
self.get_params["length"] = "-1"
self.client.force_login(self.user)
request = self.factory.get('/fake-url/', data=self.get_params)
response = TestView()
response.setup(request)
data = json.loads(response.get(request).content)["data"]
self.assertEqual(len(data), 20)

View File

@@ -13,6 +13,7 @@ The Alliance Auth framework is split into several submodules, each of which is d
framework/api
framework/css
framework/datatables
framework/js
framework/templates
framework/svg-sprite

View File

@@ -0,0 +1,181 @@
# DataTables Server Side Rendering
The `allianceauth.framework.datatables.DataTablesView` module provides a simple class based view to
implement simple server side filtering ordering and searching of DataTables.
This is intended to make the life of our community apps developer a little
easier, so they don't have to reinvent the wheel.
## Usage
To use this view is as easy as defining your stub templates, and fields and adding the view to the `urls.py`
Given the `EveCharacter` Model as our model of choice we would define our stubs like so
## Add our Templates
### template/appname/stubs/icon.html
```django
{% load evelinks %}
{% character_portrait_url row 32 %}
```
### template/appname/stubs/name.html
```django
{{ row.character_name }} <span class="text-small">({{ row.character_ownership.user.username }})</span>
```
### template/appname/stubs/corp.html
```django
{{ row.corporation_name }}
```
### template/appname/list.html
```django
{% extends "allianceauth/base-bs5.html" %}
{% load i18n %}
{% block page_title %}
{% translate "App Name" %}
{% endblock page_title %}
{% block content %}
<table class="table table-striped w-100" id="table">
<!-- Normal Header Rows -->
<thead>
<tr>
<th></th>
<th>{% translate "Name" %}</th>
<th>{% translate "Corporation" %}</th>
<th>{% translate "Alliance" %}</th>
</tr>
</thead>
</table>
{% endblock content %}
{% block extra_css %}
<link rel="stylesheet" href="https://cdn.datatables.net/2.3.6/css/dataTables.bootstrap5.css" />
<link href="https://cdn.datatables.net/columncontrol/1.2.0/css/columnControl.bootstrap5.min.css" rel="stylesheet">
{% endblock %}
{% block extra_javascript %}
<script src="https://cdn.datatables.net/2.3.6/js/dataTables.js"></script>
<script src="https://cdn.datatables.net/2.3.6/js/dataTables.bootstrap5.js"></script>
<script src="https://cdn.datatables.net/columncontrol/1.2.0/js/dataTables.columnControl.min.js"></script>
<script>
$(document).ready(function() {
$('#table').DataTable({
serverSide: true,
ajax: '{% url "appname:table" %}',
columnDefs: [
{
targets: [0],
columnControl: [],
sortable: false,
searchable: false
},
{
targets: [1,2,3],
columnControl: [
{
target: 0,
content: []
},
{
target: 1,
content: ['search']
}
],
}
],
order: [
[1, "asc"]
],
pageLength: 10,
responsive : true
});
});
</script>
{% endblock extra_javascript %}
```
## Add our Views
Then we can setup out view in our `appname/views.py` file.
### Columns definition
The `columns` must be defined as a 2 part tuple
- Part 1 is the database field that will be used for filtering and ordering. If this is a foreign key you need to point to a field that is compatible with `__icontains` like `charField` or `textField`. It can be `None`/`False`/`""` if no ordering for filtering is required for this row.
- Examples for the EveCharacter Model:
- `character_name`
- `character_ownership__user__username`
- `character_ownership__user__profile__main_character__character_name`
- Part 2 is a string that is used to the render the column for each row. This can be a html stub or a string containing django style template language.
- Examples for the EveCharacter Model
- `{{ row.character_name }}`
- `{{ row.character_ownership.user.username }}`
- `{{ row.character_ownership.user.profile.main_character.character_name }}`
- `appname/stubs/character_img.html`
### appname/views.py
```python
from django.shortcuts import render
# Alliance Auth
from allianceauth.framework.datatables import DataTablesView
from allianceauth.eveonline.models import EveCharacter
## Datatables server side view
class EveCharacterTable(DataTablesView):
model = EveCharacter
# Define the columns as a tuple.
# String for field name for filtering and ordering
# String for the render template
# Templates can be a html file or template language directly in the list below
columns = [
# ("field_for_queries_or_sort", template: str)
("", "appname/stubs/icon.html"),
("character_name", "appname/stubs/name.html"),
("corporation_name", "appname/stubs/corp.html"),
("alliance_name", "{{ row.alliance_name }} {{ row.alliance_id }}"),
]
# if you need to do some prefetch or pre-filtering you can overide this function
def get_model_qs(self, request: HttpRequest):
qs = self.model.objects
if not request.user.is_superuser:
# eg only show unlinked characters to non-superusers
# just an example
# filtering here will prevent people searching things that may not be visible to them
qs = qs.filter(character_ownership__isnull=True)
# maybe some character ownership select related for performance?
return qs.select_related("character_ownership", "character_ownership__user")
## Main Page View
def show_table(request):
return render("appname/list.html")
```
## Add our Urls
### appname/urls.py
```python
from django.urls import path
from . import views
app_name = 'appname'
urlpatterns = [
path("list/", views.EveCharacterTable.as_view(), name='eve_character_table'),
path("tables/data_table", views.show_table, name='table')
]
```
and you are done.