From 55cbbadc2b08f2de5ede84b5d1a526f92a607174 Mon Sep 17 00:00:00 2001 From: Adarnof Date: Fri, 13 Jan 2017 20:56:41 -0500 Subject: [PATCH] Cache EVE API objects using django-redis to speed repeated queries. (#638) --- alliance_auth/settings.py.example | 14 ++- eveonline/providers.py | 140 ++++++++++++++++++++++++++++-- eveonline/views.py | 21 ++--- requirements.txt | 1 + 4 files changed, 156 insertions(+), 20 deletions(-) diff --git a/alliance_auth/settings.py.example b/alliance_auth/settings.py.example index ede008a7..2f2a1ba7 100644 --- a/alliance_auth/settings.py.example +++ b/alliance_auth/settings.py.example @@ -185,6 +185,16 @@ MESSAGE_TAGS = { messages.ERROR: 'danger' } +CACHES = { + "default": { + "BACKEND": "django_redis.cache.RedisCache", + "LOCATION": "redis://127.0.0.1:6379/1", + "OPTIONS": { + "CLIENT_CLASS": "django_redis.client.DefaultClient", + } + } +} + ##################################################### ## ## Auth configuration starts here @@ -259,7 +269,6 @@ BLUE_ALLIANCE_GROUPS = 'True' == os.environ.get('AA_BLUE_ALLIANCE_GROUPS', 'Fals # ENABLE_AUTH_IPS4 - Enable IPS4 support in the auth for auth'd members # ENABLE_AUTH_SMF - Enable SMF forum support in the auth for auth'd members # ENABLE_AUTH_MARKET = Enable Alliance Market support in auth for auth'd members -# ENABLE_AUTH_PATHFINDER = Enable Alliance Pathfinder suppor in auth for auth'd members # ENABLE_AUTH_XENFORO = Enable XenForo forums support in the auth for auth'd members ######################### ENABLE_AUTH_FORUM = 'True' == os.environ.get('AA_ENABLE_AUTH_FORUM', 'False') @@ -286,7 +295,6 @@ ENABLE_AUTH_XENFORO = 'True' == os.environ.get('AA_ENABLE_AUTH_XENFORO', 'False' # ENABLE_BLUE_IPS4 - Enable IPS4 forum support in the auth for blues # ENABLE_BLUE_SMF - Enable SMF forum support in the auth for blues # ENABLE_BLUE_MARKET - Enable Alliance Market in the auth for blues -# ENABLE_BLUE_PATHFINDER = Enable Pathfinder support in the auth for blues # ENABLE_BLUE_XENFORO = Enable XenForo forum support in the auth for blue ##################### ENABLE_BLUE_FORUM = 'True' == os.environ.get('AA_ENABLE_BLUE_FORUM', 'False') @@ -352,7 +360,7 @@ API_SSO_VALIDATION = 'True' == os.environ.get('AA_API_SSO_VALIDATION', 'False') # EVEONLINE_CORP_PROVIDER - Name of default data source for getting eve corporation data # EVEONLINE_ALLIANCE_PROVIDER - Name of default data source for getting eve alliance data # -# Available soruces are 'esi' and 'xml' +# Available sources are 'esi' and 'xml' ####################### EVEONLINE_CHARACTER_PROVIDER = os.environ.get('AA_EVEONLINE_CHARACTER_PROVIDER', 'esi') EVEONLINE_CORP_PROVIDER = os.environ.get('AA_EVEONLINE_CORP_PROVIDER', 'esi') diff --git a/eveonline/providers.py b/eveonline/providers.py index ce408e08..258eb766 100644 --- a/eveonline/providers.py +++ b/eveonline/providers.py @@ -1,8 +1,17 @@ from django.utils.encoding import python_2_unicode_compatible from esi.clients import esi_client_factory from django.conf import settings +from django.core.cache import cache +import json from bravado.exception import HTTPNotFound, HTTPUnprocessableEntity import evelink +import logging + +logger = logging.getLogger(__name__) + +# optional setting to control cached object lifespan +OBJ_CACHE_DURATION = int(getattr(settings, 'EVEONLINE_OBJ_CACHE_DURATION', 300)) + @python_2_unicode_compatible class ObjectNotFound(Exception): @@ -32,6 +41,16 @@ class Entity(object): def __eq__(self, other): return self.id == other.id + def serialize(self): + return { + 'id': self.id, + 'name': self.name, + } + + @classmethod + def from_dict(cls, dict): + return cls(dict['id'], dict['name']) + class Corporation(Entity): def __init__(self, provider, id, name, ticker, ceo_id, members, alliance_id): @@ -58,6 +77,28 @@ class Corporation(Entity): self._ceo = self.provider.get_character(self.ceo_id) return self._ceo + def serialize(self): + return { + 'id': self.id, + 'name': self.name, + 'ticker': self.ticker, + 'ceo_id': self.ceo_id, + 'members': self.members, + 'alliance_id': self.alliance_id + } + + @classmethod + def from_dict(cls, dict): + return cls( + None, + dict['id'], + dict['name'], + dict['ticker'], + dict['ceo_id'], + dict['members'], + dict['alliance_id'], + ) + class Alliance(Entity): def __init__(self, provider, id, name, ticker, corp_ids, executor_corp_id): @@ -70,7 +111,7 @@ class Alliance(Entity): def corp(self, id): assert id in self.corp_ids - if not id in self._corps: + if id not in self._corps: self._corps[id] = self.provider.get_corp(id) self._corps[id]._alliance = self return self._corps[id] @@ -83,6 +124,26 @@ class Alliance(Entity): def executor_corp(self): return self.corp(self.executor_corp_id) + def serialize(self): + return { + 'id': self.id, + 'name': self.name, + 'ticker': self.ticker, + 'corp_ids': self.corp_ids, + 'executor_corp_id': self.executor_corp_id, + } + + @classmethod + def from_dict(cls, dict): + return cls( + None, + dict['id'], + dict['name'], + dict['ticker'], + dict['corp_ids'], + dict['executor_corp_id'], + ) + class Character(Entity): def __init__(self, provider, id, name, corp_id, alliance_id): @@ -96,7 +157,7 @@ class Character(Entity): @property def corp(self): if not self._corp: - self._corp = self.provider.get_corp(self.corp_id) + self._corp = self.provider.get_corp(self.corp_id) return self._corp @property @@ -105,8 +166,26 @@ class Character(Entity): return self.corp.alliance return Entity(None, None) + def serialize(self): + return { + 'id': self.id, + 'name': self.name, + 'corp_id': self.corp_id, + 'alliance_id': self.alliance_id, + } -class EveProvider: + @classmethod + def from_dict(cls, dict): + return cls( + None, + dict['id'], + dict['name'], + dict['corp_id'], + dict['alliance_id'], + ) + + +class EveProvider(object): def get_alliance(self, alliance_id): """ :return: an Alliance object for the given ID @@ -255,6 +334,7 @@ class EveAdapter(EveProvider): """ Redirects queries to appropriate data source. """ + def __init__(self, char_provider, corp_provider, alliance_provider): self.char_provider = char_provider self.corp_provider = corp_provider @@ -264,19 +344,65 @@ class EveAdapter(EveProvider): self.alliance_provider.adapter = self def __repr__(self): - return "<{} (char:{}, corp:{}, alliance:{})>".format(self.__class__.__name__, str(self.char_provider), str(self.corp_provider), str(self.alliance_provider)) + return "<{} (char:{}, corp:{}, alliance:{})>".format(self.__class__.__name__, str(self.char_provider), + str(self.corp_provider), str(self.alliance_provider)) + + @staticmethod + def _get_from_cache(obj_class, id): + data = cache.get('%s__%s' % (obj_class.__name__.lower(), id)) + if data: + obj = obj_class.from_dict(json.loads(data)) + logger.debug('Got from cache: %s' % obj.__repr__()) + return obj + else: + return None + + @staticmethod + def _cache(obj): + logger.debug('Caching: %s ' % obj.__repr__()) + cache.set('%s__%s' % (obj.__class__.__name__.lower(), obj.id), json.dumps(obj.serialize()), + int(OBJ_CACHE_DURATION)) def get_character(self, id): - return self.char_provider.get_character(id) + obj = self._get_from_cache(Character, id) + if obj: + obj.provider = self + else: + obj = self._get_character(id) + self._cache(obj) + return obj def get_corp(self, id): - return self.corp_provider.get_corp(id) + obj = self._get_from_cache(Corporation, id) + if obj: + obj.provider = self + else: + obj = self._get_corp(id) + self._cache(obj) + return obj def get_alliance(self, id): + obj = self._get_from_cache(Alliance, id) + if obj: + obj.provider = self + else: + obj = self._get_alliance(id) + self._cache(obj) + return obj + + def _get_character(self, id): + return self.char_provider.get_character(id) + + def _get_corp(self, id): + return self.corp_provider.get_corp(id) + + def _get_alliance(self, id): return self.alliance_provider.get_alliance(id) -def eve_adapter_factory(character_source=settings.EVEONLINE_CHARACTER_PROVIDER, corp_source=settings.EVEONLINE_CORP_PROVIDER, alliance_source=settings.EVEONLINE_ALLIANCE_PROVIDER, api_key=None, token=None): +def eve_adapter_factory(character_source=settings.EVEONLINE_CHARACTER_PROVIDER, + corp_source=settings.EVEONLINE_CORP_PROVIDER, + alliance_source=settings.EVEONLINE_ALLIANCE_PROVIDER, api_key=None, token=None): sources = [character_source, corp_source, alliance_source] providers = [] diff --git a/eveonline/views.py b/eveonline/views.py index 3b6acffb..e490b257 100755 --- a/eveonline/views.py +++ b/eveonline/views.py @@ -2,7 +2,6 @@ from __future__ import unicode_literals from django.shortcuts import render, redirect, get_object_or_404 from django.contrib.auth.decorators import login_required from django.contrib import messages - from eveonline.forms import UpdateKeyForm from eveonline.managers import EveManager from authentication.managers import AuthServicesInfoManager @@ -11,7 +10,6 @@ from eveonline.models import EveApiKeyPair, EveCharacter from authentication.models import AuthServicesInfo from authentication.tasks import set_state from eveonline.tasks import refresh_api - from esi.decorators import token_required from django.conf import settings import logging @@ -30,7 +28,7 @@ def add_api_key(request): api_key=form.cleaned_data['api_key']).exists(): # allow orphaned keys to proceed to SSO validation upon re-entry api_key = EveApiKeyPair.objects.get(api_id=form.cleaned_data['api_id'], - api_key=form.cleaned_data['api_key']) + api_key=form.cleaned_data['api_key']) elif EveApiKeyPair.objects.filter(api_id=form.cleaned_data['api_id']).exists(): logger.warn('API %s re-added with different vcode.' % form.cleaned_data['api_id']) EveApiKeyPair.objects.filter(api_id=form.cleaned_data['api_id']).delete() @@ -47,7 +45,8 @@ def add_api_key(request): owner = request.user # Grab characters associated with the key pair characters = EveManager.get_characters_from_api(api_key) - [EveManager.create_character_obj(c, owner, api_key.api_id) for c in characters if not EveCharacter.objects.filter(character_id=c.id).exists()] + [EveManager.create_character_obj(c, owner, api_key.api_id) for c in characters if + not EveCharacter.objects.filter(character_id=c.id).exists()] logger.info("Successfully processed api add form for user %s" % request.user) if not settings.API_SSO_VALIDATION: messages.success(request, 'Added API key %s to your account.' % form.cleaned_data['api_id']) @@ -57,7 +56,7 @@ def add_api_key(request): return redirect("auth_dashboard") else: logger.debug('Requesting SSO validation of API %s by user %s' % (api_key.api_id, request.user)) - return render(request, 'registered/apisso.html', context={'api':api_key}) + return render(request, 'registered/apisso.html', context={'api': api_key}) else: logger.debug("Form invalid: returning to form.") else: @@ -93,8 +92,9 @@ def api_sso_validate(request, token, api_id): return redirect('auth_characters') return redirect('auth_dashboard') else: - messages.warning(request, '%s not found on API %s. Please SSO as a character on the API.' % (token.character_name, api.api_id)) - return render(request, 'registered/apisso.html', context={'api':api}) + messages.warning(request, '%s not found on API %s. Please SSO as a character on the API.' % ( + token.character_name, api.api_id)) + return render(request, 'registered/apisso.html', context={'api': api}) @login_required @@ -110,7 +110,7 @@ def dashboard_view(request): api_chars.append({ 'id': api.api_id, 'sso_verified': api.sso_verified if sso_validation else True, - 'characters': EveManager.get_characters_by_api_id(api.api_id), + 'characters': EveCharacter.objects.filter(api_id=api.api_id), }) context = { @@ -139,7 +139,7 @@ def api_key_removal(request, api_id): @login_required def characters_view(request): logger.debug("characters_view called by user %s" % request.user) - render_items = {'characters': EveManager.get_characters_by_owner_id(request.user.id), + render_items = {'characters': EveCharacter.objects.filter(user=request.user), 'authinfo': AuthServicesInfo.objects.get(user=request.user)} return render(request, 'registered/characters.html', context=render_items) @@ -147,7 +147,8 @@ def characters_view(request): @login_required def main_character_change(request, char_id): logger.debug("main_character_change called by user %s for character id %s" % (request.user, char_id)) - if EveManager.check_if_character_owned_by_user(char_id, request.user): + if EveCharacter.objects.filter(character_id=char_id).exists() and EveCharacter.objects.get( + character_id=char_id).user == request.user: AuthServicesInfoManager.update_main_char_id(char_id, request.user) messages.success(request, 'Changed main character ID to %s' % char_id) set_state(request.user) diff --git a/requirements.txt b/requirements.txt index 3e6ce386..4814eff0 100755 --- a/requirements.txt +++ b/requirements.txt @@ -16,6 +16,7 @@ django>=1.10,<2.0 django-bootstrap-form django-navhelper django-bootstrap-pagination +django-redis>=4.4 # awating release for fix to celery/django-celery#447 # django-celery