Mathis/allianceauth
1
0
Fork 0
mirror of https://gitlab.com/allianceauth/allianceauth.git synced 2025-07-13 14:30:17 +02:00
Code Issues Packages Projects Releases Wiki Activity

Use Django's cache framework for service group names (#857)

Browse Source 
Use django-redis-cache backend for locking get_or_set
No longer require group-related tasks to be locked to one simultaneous execution.
Remove legacy service group cache models.

Truncate Discord nicknames to 32 characters
Correct Discourse group name extension using only valid leading characters.
Prevent name slicing from ending with illegal character

Closes #801
Closes #847
Closes #835
Closes #852
This commit is contained in:
Adarnof 2017-09-11 20:42:13 -04:00 committed by GitHub
parent 27c9b09116
commit 8987cf2199
10 changed files with 144 additions and 173 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
alliance_auth/settings.py.example
View File

@ -211,10 +211,10 @@ MESSAGE_TAGS = {
CACHES = {
"default": {
"BACKEND": "django_redis.cache.RedisCache",
"LOCATION": "redis://127.0.0.1:6379/1",
"BACKEND": "redis_cache.RedisCache",
"LOCATION": "localhost:6379",
"OPTIONS": {
"CLIENT_CLASS": "django_redis.client.DefaultClient",
"DB": 1,
}
}
}

2
requirements.txt
View File

@ -16,7 +16,7 @@ django>=1.10,<2.0
django-bootstrap-form
django-navhelper
django-bootstrap-pagination
django-redis>=4.4
django-redis-cache>=1.7.1
django-recaptcha
django-celery-beat

5
services/admin.py
View File

@ -1,5 +0,0 @@
from __future__ import unicode_literals
from django.contrib import admin
from services.models import GroupCache
admin.site.register(GroupCache)

18
services/migrations/0003_delete_groupcache.py Normal file
View File

@ -0,0 +1,18 @@
# -*- coding: utf-8 -*-
# Generated by Django 1.10.5 on 2017-09-02 06:07
from __future__ import unicode_literals
from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
('services', '0002_auto_20161016_0135'),
]
operations = [
migrations.DeleteModel(
name='GroupCache',
),
]

18
services/models.py
View File

@ -1,18 +0,0 @@
from __future__ import unicode_literals
from django.utils.encoding import python_2_unicode_compatible
from django.db import models
@python_2_unicode_compatible
class GroupCache(models.Model):
SERVICE_CHOICES = (
("discourse", "discourse"),
("discord", "discord"),
)
created = models.DateTimeField(auto_now_add=True)
groups = models.TextField(default={})
service = models.CharField(max_length=254, choices=SERVICE_CHOICES, unique=True)
def __str__(self):
return self.service

76
services/modules/discord/manager.py
View File

@ -3,14 +3,13 @@ import requests
import json
import re
from django.conf import settings
from services.models import GroupCache
from requests_oauthlib import OAuth2Session
from functools import wraps
import logging
import datetime
import time
from django.utils import timezone
from django.core.cache import cache
from hashlib import md5
logger = logging.getLogger(__name__)
@ -20,9 +19,13 @@ EVE_IMAGE_SERVER = "https://image.eveonline.com"
AUTH_URL = "https://discordapp.com/api/oauth2/authorize"
TOKEN_URL = "https://discordapp.com/api/oauth2/token"
# needs administrator, since Discord can't get their permissions system to work
# was kick members, manage roles, manage nicknames
#BOT_PERMISSIONS = 0x00000002 + 0x10000000 + 0x08000000
"""
Previously all we asked for was permission to kick members, manage roles, and manage nicknames.
Users have reported weird unauthorized errors we don't understand. So now we ask for full server admin.
It's almost fixed the problem.
"""
# kick members, manage roles, manage nicknames
# BOT_PERMISSIONS = 0x00000002 + 0x10000000 + 0x08000000
BOT_PERMISSIONS = 0x00000008
# get user ID, accept invite
@ -31,7 +34,7 @@ SCOPES = [
'guilds.join',
]
GROUP_CACHE_MAX_AGE = datetime.timedelta(minutes=30)
GROUP_CACHE_MAX_AGE = int(getattr(settings, 'DISCORD_GROUP_CACHE_MAX_AGE', 2 * 60 * 60)) # 2 hours default
class DiscordApiException(Exception):
@ -150,10 +153,14 @@ class DiscordOAuthManager:
def __init__(self):
pass
@staticmethod
def _sanitize_name(name):
return re.sub('[^\w.-]', '', name)[:32]
@staticmethod
def _sanitize_groupname(name):
name = name.strip(' _')
return re.sub('[^\w.-]', '', name)
return DiscordOAuthManager._sanitize_name(name)
@staticmethod
def generate_bot_add_url():
@ -198,8 +205,9 @@ class DiscordOAuthManager:
@staticmethod
def update_nickname(user_id, nickname):
try:
nickname = DiscordOAuthManager._sanitize_name(nickname)
custom_headers = {'content-type': 'application/json', 'authorization': 'Bot ' + settings.DISCORD_BOT_TOKEN}
data = {'nick': nickname, }
data = {'nick': nickname}
path = DISCORD_URL + "/guilds/" + str(settings.DISCORD_GUILD_ID) + "/members/" + str(user_id)
r = requests.patch(path, headers=custom_headers, json=data)
logger.debug("Got status code %s after setting nickname for Discord user ID %s (%s)" % (
@ -230,7 +238,7 @@ class DiscordOAuthManager:
return False
@staticmethod
def __get_groups():
def _get_groups():
custom_headers = {'accept': 'application/json', 'authorization': 'Bot ' + settings.DISCORD_BOT_TOKEN}
path = DISCORD_URL + "/guilds/" + str(settings.DISCORD_GUILD_ID) + "/roles"
r = requests.get(path, headers=custom_headers)
@ -239,41 +247,20 @@ class DiscordOAuthManager:
return r.json()
@staticmethod
def __update_group_cache():
GroupCache.objects.filter(service="discord").delete()
cache = GroupCache.objects.create(service="discord")
cache.groups = json.dumps(DiscordOAuthManager.__get_groups())
cache.save()
return cache
def _generate_cache_role_key(name):
return 'DISCORD_ROLE_NAME__%s' % md5(str(name).encode('utf-8')).hexdigest()
@staticmethod
def __get_group_cache():
if not GroupCache.objects.filter(service="discord").exists():
DiscordOAuthManager.__update_group_cache()
cache = GroupCache.objects.get(service="discord")
age = timezone.now() - cache.created
if age > GROUP_CACHE_MAX_AGE:
logger.debug("Group cache has expired. Triggering update.")
cache = DiscordOAuthManager.__update_group_cache()
return json.loads(cache.groups)
def _group_name_to_id(name):
name = DiscordOAuthManager._sanitize_groupname(name)
@staticmethod
def __group_name_to_id(name):
cache = DiscordOAuthManager.__get_group_cache()
for g in cache:
if g['name'] == name:
return g['id']
logger.debug("Group %s not found on Discord. Creating" % name)
DiscordOAuthManager.__create_group(name)
return DiscordOAuthManager.__group_name_to_id(name)
@staticmethod
def __group_id_to_name(id):
cache = DiscordOAuthManager.__get_group_cache()
for g in cache:
if g['id'] == id:
return g['name']
raise KeyError("Group ID %s not found on Discord" % id)
def get_or_make_role():
groups = DiscordOAuthManager._get_groups()
for g in groups:
if g['name'] == name:
return g['id']
return DiscordOAuthManager._create_group(name)['id']
return cache.get_or_set(DiscordOAuthManager._generate_cache_role_key(name), get_or_make_role, GROUP_CACHE_MAX_AGE)
@staticmethod
def __generate_role():
@ -300,16 +287,15 @@ class DiscordOAuthManager:
return r.json()
@staticmethod
def __create_group(name):
def _create_group(name):
role = DiscordOAuthManager.__generate_role()
DiscordOAuthManager.__edit_role(role['id'], name)
DiscordOAuthManager.__update_group_cache()
return DiscordOAuthManager.__edit_role(role['id'], name)
@staticmethod
@api_backoff
def update_groups(user_id, groups):
custom_headers = {'content-type': 'application/json', 'authorization': 'Bot ' + settings.DISCORD_BOT_TOKEN}
group_ids = [DiscordOAuthManager.__group_name_to_id(DiscordOAuthManager._sanitize_groupname(g)) for g in groups]
group_ids = [DiscordOAuthManager._group_name_to_id(DiscordOAuthManager._sanitize_groupname(g)) for g in groups]
path = DISCORD_URL + "/guilds/" + str(settings.DISCORD_GUILD_ID) + "/members/" + str(user_id)
data = {'roles': group_ids}
r = requests.patch(path, headers=custom_headers, json=data)

1
services/modules/discord/tasks.py
View File

@ -10,7 +10,6 @@ from django.core.exceptions import ObjectDoesNotExist
from eveonline.managers import EveManager
from notifications import notify
from services.modules.discord.manager import DiscordOAuthManager, DiscordApiBackoff
from services.tasks import only_one
from .models import DiscordUser
logger = logging.getLogger(__name__)

6
services/modules/discord/tests.py
View File

@ -351,7 +351,7 @@ class DiscordManagerTestCase(TestCase):
# Assert
self.assertTrue(result)
@mock.patch(MODULE_PATH + '.manager.DiscordOAuthManager._DiscordOAuthManager__get_group_cache')
@mock.patch(MODULE_PATH + '.manager.DiscordOAuthManager._get_groups')
@requests_mock.Mocker()
def test_update_groups(self, group_cache, m):
from . import manager
@ -385,7 +385,7 @@ class DiscordManagerTestCase(TestCase):
self.assertNotIn(444, history['roles'], 'The group id 444 must NOT be added to the request')
@mock.patch(MODULE_PATH + '.manager.cache')
@mock.patch(MODULE_PATH + '.manager.DiscordOAuthManager._DiscordOAuthManager__get_group_cache')
@mock.patch(MODULE_PATH + '.manager.DiscordOAuthManager._get_groups')
@requests_mock.Mocker()
def test_update_groups_backoff(self, group_cache, djcache, m):
from . import manager
@ -420,7 +420,7 @@ class DiscordManagerTestCase(TestCase):
self.assertTrue(datetime.datetime.strptime(args[1], manager.cache_time_format) > datetime.datetime.now())
@mock.patch(MODULE_PATH + '.manager.cache')
@mock.patch(MODULE_PATH + '.manager.DiscordOAuthManager._DiscordOAuthManager__get_group_cache')
@mock.patch(MODULE_PATH + '.manager.DiscordOAuthManager._get_groups')
@requests_mock.Mocker()
def test_update_groups_global_backoff(self, group_cache, djcache, m):
from . import manager

183
services/modules/discourse/manager.py
View File

@ -1,17 +1,15 @@
from __future__ import unicode_literals
import logging
import requests
import random
import string
import datetime
import json
import re
from django.conf import settings
from django.utils import timezone
from services.models import GroupCache
from django.core.cache import cache
from hashlib import md5
logger = logging.getLogger(__name__)
GROUP_CACHE_MAX_AGE = int(getattr(settings, 'DISCOURSE_GROUP_CACHE_MAX_AGE', 2 * 60 * 60)) # default 2 hours
class DiscourseError(Exception):
def __init__(self, endpoint, errors):
@ -21,12 +19,13 @@ class DiscourseError(Exception):
def __str__(self):
return "API execution failed.\nErrors: %s\nEndpoint: %s" % (self.errors, self.endpoint)
# not exhaustive, only the ones we need
ENDPOINTS = {
'groups': {
'list': {
'path': "/admin/groups.json",
'method': requests.get,
'method': 'get',
'args': {
'required': [],
'optional': [],
@ -34,7 +33,7 @@ ENDPOINTS = {
},
'create': {
'path': "/admin/groups",
'method': requests.post,
'method': 'post',
'args': {
'required': ['name'],
'optional': ['visible'],
@ -42,7 +41,7 @@ ENDPOINTS = {
},
'add_user': {
'path': "/admin/groups/%s/members.json",
'method': requests.put,
'method': 'put',
'args': {
'required': ['usernames'],
'optional': [],
@ -50,7 +49,7 @@ ENDPOINTS = {
},
'remove_user': {
'path': "/admin/groups/%s/members.json",
'method': requests.delete,
'method': 'delete',
'args': {
'required': ['username'],
'optional': [],
@ -58,7 +57,7 @@ ENDPOINTS = {
},
'delete': {
'path': "/admin/groups/%s.json",
'method': requests.delete,
'method': 'delete',
'args': {
'required': [],
'optional': [],
@ -68,7 +67,7 @@ ENDPOINTS = {
'users': {
'create': {
'path': "/users",
'method': requests.post,
'method': 'post',
'args': {
'required': ['name', 'email', 'password', 'username'],
'optional': ['active'],
@ -76,7 +75,7 @@ ENDPOINTS = {
},
'update': {
'path': "/users/%s.json",
'method': requests.put,
'method': 'put',
'args': {
'required': ['params'],
'optional': [],
@ -84,7 +83,7 @@ ENDPOINTS = {
},
'get': {
'path': "/users/%s.json",
'method': requests.get,
'method': 'get',
'args': {
'required': [],
'optional': [],
@ -92,7 +91,7 @@ ENDPOINTS = {
},
'activate': {
'path': "/admin/users/%s/activate",
'method': requests.put,
'method': 'put',
'args': {
'required': [],
'optional': [],
@ -100,7 +99,7 @@ ENDPOINTS = {
},
'set_email': {
'path': "/users/%s/preferences/email",
'method': requests.put,
'method': 'put',
'args': {
'required': ['email'],
'optional': [],
@ -108,7 +107,7 @@ ENDPOINTS = {
},
'suspend': {
'path': "/admin/users/%s/suspend",
'method': requests.put,
'method': 'put',
'args': {
'required': ['duration', 'reason'],
'optional': [],
@ -116,7 +115,7 @@ ENDPOINTS = {
},
'unsuspend': {
'path': "/admin/users/%s/unsuspend",
'method': requests.put,
'method': 'put',
'args': {
'required': [],
'optional': [],
@ -124,7 +123,7 @@ ENDPOINTS = {
},
'logout': {
'path': "/admin/users/%s/log_out",
'method': requests.post,
'method': 'post',
'args': {
'required': [],
'optional': [],
@ -132,7 +131,7 @@ ENDPOINTS = {
},
'external': {
'path': "/users/by-external/%s.json",
'method': requests.get,
'method': 'get',
'args': {
'required': [],
'optional': [],
@ -146,8 +145,7 @@ class DiscourseManager:
def __init__(self):
pass
GROUP_CACHE_MAX_AGE = datetime.timedelta(minutes=30)
REVOKED_EMAIL = 'revoked@' + settings.DOMAIN
REVOKED_EMAIL = 'revoked@localhost'
SUSPEND_DAYS = 99999
SUSPEND_REASON = "Disabled by auth."
@ -171,7 +169,8 @@ class DiscourseManager:
for arg in kwargs:
if arg not in endpoint['args']['required'] and arg not in endpoint['args']['optional'] and not silent:
logger.warn("Received unrecognized kwarg %s for endpoint %s" % (arg, endpoint))
r = endpoint['method'](settings.DISCOURSE_URL + endpoint['parsed_url'], params=params, json=data)
r = getattr(requests, endpoint['method'])(settings.DISCOURSE_URL + endpoint['parsed_url'], params=params,
json=data)
try:
if 'errors' in r.json() and not silent:
logger.error("Discourse execution failed.\nEndpoint: %s\nErrors: %s" % (endpoint, r.json()['errors']))
@ -190,67 +189,59 @@ class DiscourseManager:
return out
@staticmethod
def __generate_random_pass():
return ''.join([random.choice(string.ascii_letters + string.digits) for n in range(16)])
@staticmethod
def __get_groups():
def _get_groups():
endpoint = ENDPOINTS['groups']['list']
data = DiscourseManager.__exc(endpoint)
return [g for g in data if not g['automatic']]
@staticmethod
def __update_group_cache():
GroupCache.objects.filter(service="discourse").delete()
cache = GroupCache.objects.create(service="discourse")
cache.groups = json.dumps(DiscourseManager.__get_groups())
cache.save()
return cache
@staticmethod
def __get_group_cache():
if not GroupCache.objects.filter(service="discourse").exists():
DiscourseManager.__update_group_cache()
cache = GroupCache.objects.get(service="discourse")
age = timezone.now() - cache.created
if age > DiscourseManager.GROUP_CACHE_MAX_AGE:
logger.debug("Group cache has expired. Triggering update.")
cache = DiscourseManager.__update_group_cache()
return json.loads(cache.groups)
@staticmethod
def __create_group(name):
def _create_group(name):
endpoint = ENDPOINTS['groups']['create']
DiscourseManager.__exc(endpoint, name=name[:20], visible=True)
DiscourseManager.__update_group_cache()
return DiscourseManager.__exc(endpoint, name=name[:20], visible=True)['basic_group']
@staticmethod
def _generate_cache_group_name_key(name):
return 'DISCOURSE_GROUP_NAME__%s' % md5(name.encode('utf-8')).hexdigest()
@staticmethod
def _generate_cache_group_id_key(g_id):
return 'DISCOURSE_GROUP_ID__%s' % g_id
@staticmethod
def __group_name_to_id(name):
cache = DiscourseManager.__get_group_cache()
for g in cache:
if g['name'] == name[0:20]:
return g['id']
logger.debug("Group %s not found on Discourse. Creating" % name)
DiscourseManager.__create_group(name)
return DiscourseManager.__group_name_to_id(name)
name = DiscourseManager._sanitize_groupname(name)
def get_or_create_group():
groups = DiscourseManager._get_groups()
for g in groups:
if g['name'] == name:
return g['id']
return DiscourseManager._create_group(name)['id']
return cache.get_or_set(DiscourseManager._generate_cache_group_name_key(name), get_or_create_group,
GROUP_CACHE_MAX_AGE)
@staticmethod
def __group_id_to_name(id):
cache = DiscourseManager.__get_group_cache()
for g in cache:
if g['id'] == id:
return g['name']
raise KeyError("Group ID %s not found on Discourse" % id)
def __group_id_to_name(g_id):
def get_group_name():
groups = DiscourseManager._get_groups()
for g in groups:
if g['id'] == g_id:
return g['name']
raise KeyError("Group ID %s not found on Discourse" % g_id)
return cache.get_or_set(DiscourseManager._generate_cache_group_id_key(g_id), get_group_name,
GROUP_CACHE_MAX_AGE)
@staticmethod
def __add_user_to_group(id, username):
def __add_user_to_group(g_id, username):
endpoint = ENDPOINTS['groups']['add_user']
DiscourseManager.__exc(endpoint, id, usernames=[username])
DiscourseManager.__exc(endpoint, g_id, usernames=[username])
@staticmethod
def __remove_user_from_group(id, username):
def __remove_user_from_group(g_id, username):
endpoint = ENDPOINTS['groups']['remove_user']
DiscourseManager.__exc(endpoint, id, username=username)
DiscourseManager.__exc(endpoint, g_id, username=username)
@staticmethod
def __generate_group_dict(names):
@ -269,10 +260,6 @@ class DiscourseManager:
data = DiscourseManager.__get_user(name, silent=silent)
return data['user']['id']
@staticmethod
def __user_id_to_name(id):
raise NotImplementedError
@staticmethod
def __get_user(username, silent=False):
endpoint = ENDPOINTS['users']['get']
@ -281,14 +268,14 @@ class DiscourseManager:
@staticmethod
def __activate_user(username):
endpoint = ENDPOINTS['users']['activate']
id = DiscourseManager.__user_name_to_id(username)
DiscourseManager.__exc(endpoint, id)
u_id = DiscourseManager.__user_name_to_id(username)
DiscourseManager.__exc(endpoint, u_id)
@staticmethod
def __update_user(username, **kwargs):
endpoint = ENDPOINTS['users']['update']
id = DiscourseManager.__user_name_to_id(username)
DiscourseManager.__exc(endpoint, id, params=kwargs)
u_id = DiscourseManager.__user_name_to_id(username)
DiscourseManager.__exc(endpoint, u_id, params=kwargs)
@staticmethod
def __create_user(username, email, password):
@ -300,21 +287,21 @@ class DiscourseManager:
try:
DiscourseManager.__user_name_to_id(username, silent=True)
return True
except:
except DiscourseError:
return False
@staticmethod
def __suspend_user(username):
id = DiscourseManager.__user_name_to_id(username)
u_id = DiscourseManager.__user_name_to_id(username)
endpoint = ENDPOINTS['users']['suspend']
return DiscourseManager.__exc(endpoint, id, duration=DiscourseManager.SUSPEND_DAYS,
return DiscourseManager.__exc(endpoint, u_id, duration=DiscourseManager.SUSPEND_DAYS,
reason=DiscourseManager.SUSPEND_REASON)
@staticmethod
def __unsuspend(username):
id = DiscourseManager.__user_name_to_id(username)
u_id = DiscourseManager.__user_name_to_id(username)
endpoint = ENDPOINTS['users']['unsuspend']
return DiscourseManager.__exc(endpoint, id)
return DiscourseManager.__exc(endpoint, u_id)
@staticmethod
def __set_email(username, email):
@ -322,47 +309,53 @@ class DiscourseManager:
return DiscourseManager.__exc(endpoint, username, email=email)
@staticmethod
def __logout(id):
def __logout(u_id):
endpoint = ENDPOINTS['users']['logout']
return DiscourseManager.__exc(endpoint, id)
return DiscourseManager.__exc(endpoint, u_id)
@staticmethod
def __get_user_by_external(id):
def __get_user_by_external(u_id):
endpoint = ENDPOINTS['users']['external']
return DiscourseManager.__exc(endpoint, id)
return DiscourseManager.__exc(endpoint, u_id)
@staticmethod
def __user_id_by_external_id(id):
data = DiscourseManager.__get_user_by_external(id)
def __user_id_by_external_id(u_id):
data = DiscourseManager.__get_user_by_external(u_id)
return data['user']['id']
@staticmethod
def _sanitize_name(name):
name = name.replace(' ', '_')
name = name.replace("'", '')
name = name.lstrip(' _')
name = name[:20]
name = name.rstrip(' _')
return name
@staticmethod
def _sanitize_username(username):
sanitized = username.replace(" ", "_")
sanitized = sanitized.strip(' _')
sanitized = sanitized.replace("'", "")
return sanitized
return DiscourseManager._sanitize_name(username)
@staticmethod
def _sanitize_groupname(name):
name = name.strip(' _')
name = re.sub('[^\w]', '', name)
name = DiscourseManager._sanitize_name(name)
if len(name) < 3:
name = name + "".join('_' for i in range(3-len(name)))
return name[:20]
name = "Group " + name
return name
@staticmethod
def update_groups(user):
groups = []
for g in user.groups.all():
groups.append(DiscourseManager._sanitize_groupname(str(g)[:20]))
groups.append(DiscourseManager._sanitize_groupname(str(g)))
logger.debug("Updating discourse user %s groups to %s" % (user, groups))
group_dict = DiscourseManager.__generate_group_dict(groups)
inv_group_dict = {v: k for k, v in group_dict.items()}
username = DiscourseManager.__get_user_by_external(user.pk)['user']['username']
user_groups = DiscourseManager.__get_user_groups(username)
add_groups = [group_dict[x] for x in group_dict if not group_dict[x] in user_groups]
rem_groups = [x for x in user_groups if not x in inv_group_dict]
rem_groups = [x for x in user_groups if x not in inv_group_dict]
if add_groups or rem_groups:
logger.info(
"Updating discourse user %s groups: adding %s, removing %s" % (username, add_groups, rem_groups))

2
services/modules/discourse/tasks.py
View File

@ -5,8 +5,6 @@ from django.contrib.auth.models import User
from django.core.exceptions import ObjectDoesNotExist
from notifications import notify
from services.tasks import only_one
from .manager import DiscourseManager
from .models import DiscourseUser