Merge branch 'master' of https://github.com/Adarnof/allianceauth into custom_user

alliance_auth/settings.py.example

@@ -35,6 +35,7 @@ INSTALLED_APPS = [
     'optimer',
     'corputils',
     'fleetactivitytracking',
+    'fleetup',
     'notifications',
     'esi',
     'permissions_tool',
@@ -580,6 +581,10 @@ LOGGING = {
             'handlers': ['log_file', 'console', 'notifications'],
             'level': 'ERROR',
         },
+        'fleetup': {
+            'handlers': ['log_file', 'console', 'notifications'],
+            'level': 'DEBUG',
+        },
         'util': {
             'handlers': ['log_file', 'console', 'notifications'],
             'level': 'DEBUG',

alliance_auth/tests/test_settings.py

@@ -14,6 +14,7 @@ TEST_RUNNER = 'django_nose.NoseTestSuiteRunner'
 NOSE_ARGS = [
     #'--with-coverage',
     #'--cover-package=',
+    #'--exe',  # If your tests need this to be found/run, check they py files are not chmodded +x
 ]
 
 # Celery configuration
@@ -48,6 +49,7 @@ INSTALLED_APPS = [
     'optimer',
     'corputils',
     'fleetactivitytracking',
+    'fleetup',
     'notifications',
     'esi',
     'permissions_tool',
@@ -383,12 +385,12 @@ TEAMSPEAK3_PUBLIC_URL = os.environ.get('AA_TEAMSPEAK3_PUBLIC_URL', 'example.com'
 # DISCORD_CALLBACK_URL - oauth callback url
 # DISCORD_SYNC_NAMES - enable to force discord nicknames to be set to eve char name (bot needs Manage Nicknames permission)
 ######################################
-DISCORD_GUILD_ID = os.environ.get('AA_DISCORD_GUILD_ID', '')
+DISCORD_GUILD_ID = os.environ.get('AA_DISCORD_GUILD_ID', '0118999')
-DISCORD_BOT_TOKEN = os.environ.get('AA_DISCORD_BOT_TOKEN', '')
+DISCORD_BOT_TOKEN = os.environ.get('AA_DISCORD_BOT_TOKEN', 'bottoken')
-DISCORD_INVITE_CODE = os.environ.get('AA_DISCORD_INVITE_CODE', '')
+DISCORD_INVITE_CODE = os.environ.get('AA_DISCORD_INVITE_CODE', 'invitecode')
-DISCORD_APP_ID = os.environ.get('AA_DISCORD_APP_ID', '')
+DISCORD_APP_ID = os.environ.get('AA_DISCORD_APP_ID', 'appid')
-DISCORD_APP_SECRET = os.environ.get('AA_DISCORD_APP_SECRET', '')
+DISCORD_APP_SECRET = os.environ.get('AA_DISCORD_APP_SECRET', 'secret')
-DISCORD_CALLBACK_URL = os.environ.get('AA_DISCORD_CALLBACK_URL', 'http://example.com/discord_callback')
+DISCORD_CALLBACK_URL = os.environ.get('AA_DISCORD_CALLBACK_URL', 'http://example.com/discord/callback')
 DISCORD_SYNC_NAMES = 'True' == os.environ.get('AA_DISCORD_SYNC_NAMES', 'False')
 
 ######################################

alliance_auth/urls.py

@@ -11,7 +11,7 @@ import groupmanagement.views
 import optimer.views
 import timerboard.views
 import fleetactivitytracking.views
-import fleetup.views
+import fleetup.urls
 import srp.views
 import notifications.views
 import hrapplications.views
@@ -70,12 +70,7 @@ urlpatterns = [
 urlpatterns += i18n_patterns(
 
     # Fleetup
-    url(r'^fleetup/$', fleetup.views.fleetup_view, name='auth_fleetup_view'),
+    url(r'^fleetup/', include(fleetup.urls.urlpatterns)),
-    url(r'^fleetup/fittings/$', fleetup.views.fleetup_fittings, name='auth_fleetup_fittings'),
-    url(r'^fleetup/fittings/(?P<fittingnumber>[0-9]+)/$', fleetup.views.fleetup_fitting, name='auth_fleetup_fitting'),
-    url(r'^fleetup/doctrines/$', fleetup.views.fleetup_doctrines, name='auth_fleetup_doctrines'),
-    url(r'^fleetup/characters/$', fleetup.views.fleetup_characters, name='auth_fleetup_characters'),
-    url(r'^fleetup/doctrines/(?P<doctrinenumber>[0-9]+)/$', fleetup.views.fleetup_doctrine, name='auth_fleetup_doctrine'),
 
     # Corputils
     url(_(r'^corpstats/'), include(corputils.urls, namespace='corputils')),

fleetup/__init__.py

@@ -1 +1,2 @@
 from __future__ import unicode_literals
+default_app_config = 'fleetup.apps.FleetupConfig'

fleetup/admin.py

@@ -1 +0,0 @@
-from __future__ import unicode_literals

fleetup/forms.py

@@ -1 +0,0 @@
-from __future__ import unicode_literals

fleetup/managers.py

@@ -1,50 +1,94 @@
 from __future__ import unicode_literals
 from django.conf import settings
+from django.core.cache import cache
 from datetime import datetime
 
 import logging
 import requests
-import json
+import hashlib
 
 logger = logging.getLogger(__name__)
 
-appkey = settings.FLEETUP_APP_KEY
-userid = settings.FLEETUP_USER_ID
-apiid = settings.FLEETUP_API_ID
-groupid = settings.FLEETUP_GROUP_ID
-
 
 class FleetUpManager:
+    APP_KEY = settings.FLEETUP_APP_KEY
+    USER_ID = settings.FLEETUP_USER_ID
+    API_ID = settings.FLEETUP_API_ID
+    GROUP_ID = settings.FLEETUP_GROUP_ID
+    BASE_URL = "http://api.fleet-up.com/Api.svc/{}/{}/{}".format(APP_KEY, USER_ID, API_ID)
+
     def __init__(self):
         pass
 
-    @staticmethod
+    @classmethod
-    def get_fleetup_members():
+    def _request_cache_key(cls, url):
-        url = "http://api.fleet-up.com/Api.svc/" + str(appkey) + "/" + str(userid) + "/" + str(
+        h = hashlib.sha1()
-            apiid) + "/GroupCharacters/" + str(groupid) + ""
+        h.update(url.encode('utf-8'))
+        return 'FLEETUP_ENDPOINT_' + h.hexdigest()
+
+    @classmethod
+    def _cache_until_seconds(cls, cache_until_json):
+        # Format comes in like "/Date(1493896236163)/"
         try:
-            jsondata = requests.get(url).content
+            epoch_ms = int(cache_until_json[6:-2])
-            fmembers = json.loads(jsondata.decode())
+            cache_delta = datetime.fromtimestamp(epoch_ms/1000) - datetime.now()
+            cache_delta_seconds = cache_delta.total_seconds()
+            if cache_delta_seconds < 0:
+                return 0
+            elif cache_delta_seconds > 3600:
+                return 3600
+            else:
+                return cache_delta_seconds
+        except TypeError:
+            logger.debug("Couldn't convert CachedUntil time, defaulting to 600 seconds")
+        return 600
+
+    @classmethod
+    def get_endpoint(cls, url):
+        try:
+            cache_key = cls._request_cache_key(url)
+            cached = cache.get(cache_key)
+            if cached:
+                return cached
+
+            r = requests.get(url)
+            r.raise_for_status()
+
+            json = r.json()
+
+            if json['Success']:
+                cache.set(cache_key, json, cls._cache_until_seconds(json['CachedUntilUTC']))
+            return json
+        except requests.exceptions.ConnectionError:
+            logger.warn("Can't connect to Fleet-Up API, is it offline?!")
+        except requests.HTTPError:
+            logger.exception("Error accessing Fleetup API")
+        return None
+
+    @classmethod
+    def get_fleetup_members(cls):
+        url = "{}/GroupCharacters/{}".format(cls.BASE_URL, cls.GROUP_ID)
+        try:
+            fmembers = cls.get_endpoint(url)
+            if not fmembers:
+                return None
             return {row["UserId"]: {"user_id": row["UserId"],
                                     "char_name": row["EveCharName"],
                                     "char_id": row["EveCharId"],
                                     "corporation": row["Corporation"]} for row in fmembers["Data"]}
-        except requests.exceptions.ConnectionError:
-            logger.warn("Can't connect to Fleet-Up API, is it offline?!")
         except (ValueError, UnicodeDecodeError, TypeError):
             logger.debug("No fleetup members retrieved.")
         return {}
 
-    @staticmethod
+    @classmethod
-    def get_fleetup_operations():
+    def get_fleetup_operations(cls):
-        url = "http://api.fleet-up.com/Api.svc/" + str(appkey) + "/" + str(userid) + "/" + str(
+        url = "{}/Operations/{}".format(cls.BASE_URL, cls.GROUP_ID)
-            apiid) + "/Operations/" + str(groupid) + ""
+        foperations = cls.get_endpoint(url)
-        try:
+        if foperations is None:
-            jsondata = requests.get(url).content
+            return None
-            foperations = json.loads(jsondata.decode())
         return {row["StartString"]: {"subject": row["Subject"],
-                                         "start": (datetime.strptime(row["StartString"], "%Y-%m-%d %H:%M:%S")),
+                                     "start": datetime.strptime(row["StartString"], "%Y-%m-%d %H:%M:%S"),
-                                         "end": (datetime.strptime(row["EndString"], "%Y-%m-%d %H:%M:%S")),
+                                     "end": datetime.strptime(row["EndString"], "%Y-%m-%d %H:%M:%S"),
                                      "operation_id": row["OperationId"],
                                      "location": row["Location"],
                                      "location_info": row["LocationInfo"],
@@ -52,19 +96,13 @@ class FleetUpManager:
                                      "url": row["Url"],
                                      "doctrine": row["Doctrines"],
                                      "organizer": row["Organizer"]} for row in foperations["Data"]}
-        except requests.exceptions.ConnectionError:
-            logger.warn("Can't connect to Fleet-Up API, is it offline?!")
-        except (ValueError, UnicodeDecodeError):
-            logger.debug("No fleetup operations retrieved.")
-        return {}
 
-    @staticmethod
+    @classmethod
-    def get_fleetup_timers():
+    def get_fleetup_timers(cls):
-        url = "http://api.fleet-up.com/Api.svc/" + str(appkey) + "/" + str(userid) + "/" + str(
+        url = "{}/Timers/{}".format(cls.BASE_URL, cls.GROUP_ID)
-            apiid) + "/Timers/" + str(groupid) + ""
+        ftimers = cls.get_endpoint(url)
-        try:
+        if not ftimers:
-            jsondata = requests.get(url).content
+            return None
-            ftimers = json.loads(jsondata.decode())
         return {row["ExpiresString"]: {"solarsystem": row["SolarSystem"],
                                        "planet": row["Planet"],
                                        "moon": row["Moon"],
@@ -73,47 +111,30 @@ class FleetUpManager:
                                        "timer_type": row["TimerType"],
                                        "expires": (datetime.strptime(row["ExpiresString"], "%Y-%m-%d %H:%M:%S")),
                                        "notes": row["Notes"]} for row in ftimers["Data"]}
-        except requests.exceptions.ConnectionError:
-            logger.warn("Can't connect to Fleet-Up API, is it offline?!")
-        except (ValueError, UnicodeDecodeError, TypeError):
-            logger.debug("No fleetup timers retrieved.")
         return {}
 
-    @staticmethod
+    @classmethod
-    def get_fleetup_doctrines():
+    def get_fleetup_doctrines(cls):
-        url = "http://api.fleet-up.com/Api.svc/" + str(appkey) + "/" + str(userid) + "/" + str(
+        url = "{}/Doctrines/{}".format(cls.BASE_URL, cls.GROUP_ID)
-            apiid) + "/Doctrines/" + str(groupid) + ""
+        fdoctrines = cls.get_endpoint(url)
-        try:
+        if not fdoctrines:
-            jsondata = requests.get(url).content
+            return None
-            fdoctrines = json.loads(jsondata.decode())
         return {"fleetup_doctrines": fdoctrines["Data"]}
-        except requests.exceptions.ConnectionError:
-            logger.warn("Can't connect to Fleet-Up API, is it offline?!")
-        except (ValueError, UnicodeDecodeError):
-            logger.debug("No fleetup doctrines retrieved.")
-        return {"fleetup_doctrines": []}
 
-    @staticmethod
+    @classmethod
-    def get_fleetup_doctrine(doctrinenumber):
+    def get_fleetup_doctrine(cls, doctrinenumber):
-        url = "http://api.fleet-up.com/Api.svc/" + str(appkey) + "/" + str(userid) + "/" + str(
+        url = "{}/DoctrineFittings/{}".format(cls.BASE_URL, doctrinenumber)
-            apiid) + "/DoctrineFittings/%s" % doctrinenumber
+        fdoctrine = cls.get_endpoint(url)
-        try:
+        if not fdoctrine:
-            jsondata = requests.get(url).content
+            return None
-            fdoctrine = json.loads(jsondata.decode())
         return {"fitting_doctrine": fdoctrine}
-        except requests.exceptions.ConnectionError:
-            logger.warn("Can't connect to Fleet-Up API, is it offline?!")
-        except (ValueError, UnicodeDecodeError):
-            logger.warn("Fleetup doctrine number %s not found" % doctrinenumber)
-        return {"fitting_doctrine": {}}
 
-    @staticmethod
+    @classmethod
-    def get_fleetup_fittings():
+    def get_fleetup_fittings(cls):
-        url = "http://api.fleet-up.com/Api.svc/" + str(appkey) + "/" + str(userid) + "/" + str(
+        url = "{}/Fittings/{}".format(cls.BASE_URL, cls.GROUP_ID)
-            apiid) + "/Fittings/" + str(groupid) + ""
+        ffittings = cls.get_endpoint(url)
-        try:
+        if not ffittings:
-            jsondata = requests.get(url).content
+            return None
-            ffittings = json.loads(jsondata.decode())
         return {row["FittingId"]: {"fitting_id": row["FittingId"],
                                    "name": row["Name"],
                                    "icon_id": row["EveTypeId"],
@@ -125,54 +146,39 @@ class FleetUpManager:
                                    "last_update": (
                                    datetime.strptime(row["LastUpdatedString"], "%Y-%m-%d %H:%M:%S"))} for row in
                 ffittings["Data"]}
-        except requests.exceptions.ConnectionError:
-            logger.warn("Can't connect to Fleet-Up API, is it offline?!")
-        except (ValueError, UnicodeDecodeError, TypeError):
-            logger.debug("No fleetup fittings retrieved.")
-        return {}
 
-    @staticmethod
+    @classmethod
-    def get_fleetup_fitting(fittingnumber):
+    def get_fleetup_fitting(cls, fittingnumber):
-        url = "http://api.fleet-up.com/Api.svc/" + str(appkey) + "/" + str(userid) + "/" + str(
+        url = "{}/Fitting/{}".format(cls.BASE_URL, fittingnumber)
-            apiid) + "/Fitting/%s" % fittingnumber
         try:
-            jsondata = requests.get(url).content
+            ffitting = cls.get_endpoint(url)
-            ffitting = json.loads(jsondata.decode())
+            if not ffitting:
+                return None
             return {"fitting_data": ffitting["Data"]}
-        except requests.exceptions.ConnectionError:
-            logger.warn("Can't connect to Fleet-Up API, is it offline?!")
-        except (ValueError, UnicodeDecodeError):
-            logger.warn("Fleetup fitting number %s not found" % fittingnumber)
         except KeyError:
             logger.warn("Failed to retrieve fleetup fitting number %s" % fittingnumber)
         return {"fitting_data": {}}
 
-    @staticmethod
+    @classmethod
-    def get_fleetup_doctrineid(fittingnumber):
+    def get_fleetup_doctrineid(cls, fittingnumber):
-        url = "http://api.fleet-up.com/Api.svc/" + str(appkey) + "/" + str(userid) + "/" + str(
+        url = "{}/Fitting/{}".format(cls.BASE_URL, fittingnumber)
-            apiid) + "/Fitting/%s" % fittingnumber
         try:
-            jsondata = requests.get(url).content
+            fdoctrineid = cls.get_endpoint(url)
-            fdoctrineid = json.loads(jsondata.decode())
+            if not fdoctrineid:
+                return None
             return fdoctrineid['Data']['Doctrines'][0]['DoctrineId']
-        except requests.exceptions.ConnectionError:
-            logger.warn("Can't connect to Fleet-Up API, is it offline?!")
-        except (ValueError, UnicodeDecodeError):
-            logger.warn("Fleetup doctrine number not found for fitting number %s" % fittingnumber)
         except (KeyError, IndexError):
             logger.debug("Fleetup fitting number %s not in a doctrine." % fittingnumber)
-        return None
+        return {}
 
-    @staticmethod
+    @classmethod
-    def get_fleetup_fitting_eft(fittingnumber):
+    def get_fleetup_fitting_eft(cls, fittingnumber):
-        url = "http://api.fleet-up.com/Api.svc/" + str(appkey) + "/" + str(userid) + "/" + str(
+        url = "{}/Fitting/{}/eft".format(cls.BASE_URL, fittingnumber)
-            apiid) + "/Fitting/%s/eft" % fittingnumber
         try:
-            jsondata = requests.get(url).content
+            ffittingeft = cls.get_endpoint(url)
-            ffittingeft = json.loads(jsondata.decode())
+            if not ffittingeft:
+                return None
             return {"fitting_eft": ffittingeft["Data"]["FittingData"]}
-        except requests.exceptions.ConnectionError:
+        except KeyError:
-            logger.warn("Can't connect to Fleet-Up API, is it offline?!")
-        except (ValueError, UnicodeDecodeError):
             logger.warn("Fleetup fitting eft not found for fitting number %s" % fittingnumber)
         return {"fitting_eft": {}}

fleetup/models.py

@@ -1 +0,0 @@
-from __future__ import unicode_literals

fleetup/templates/fleetup/characters.html

@@ -9,30 +9,7 @@
 {% block content %}
 <div class="col-lg-12">
 {% if perms.auth.corp_stats %}
-    <nav class="navbar navbar-default">
+    {% include "fleetup/menu.html" %}
-    <div class="container-fluid">
-        <div class="navbar-header">
-            <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#bs-example-navbar-collapse-1" aria-expanded="false">
-                <span class="sr-only">{% trans "Toggle navigation" %}</span>
-                <span class="icon-bar"></span>
-                <span class="icon-bar"></span>
-                <span class="icon-bar"></span>
-            </button>
-            <a class="navbar-brand" href="#">Fleet-Up</a>
-        </div>
-        <div class="collapse navbar-collapse" id="bs-example-navbar-collapse-1">
-            <ul class="nav navbar-nav">
-                <li><a href="{% url 'auth_fleetup_view' %}">{% trans "Ops and Timers" %}</a></li>
-                <li><a href="{% url 'auth_fleetup_doctrines' %}">{% trans "Doctrines" %}</a></li>
-                <li><a href="{% url 'auth_fleetup_fittings' %}">{% trans "Fittings" %}</a></li>
-                
-                <li class="active"><a href="{% url 'auth_fleetup_characters' %}">{% trans "Characters" %} <span class="sr-only">(current)</span></a></li>
-                
-                <li></li>
-            </ul>
-        </div>
-    </div>
-    </nav>
     <div class="panel panel-default">
         <div class="panel-heading">
             <h3 class="panel-title">{% trans "Characters registered on Fleet-Up.com" %}</h3>

fleetup/templates/fleetup/doctrine.html

@@ -8,30 +8,7 @@
 
 {% block content %}
 <div class="col-lg-12">
-<nav class="navbar navbar-default">
+    {% include "fleetup/menu.html" %}
-    <div class="container-fluid">
-        <div class="navbar-header">
-            <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#bs-example-navbar-collapse-1" aria-expanded="false">
-                <span class="sr-only">{% trans "Toggle navigation" %}</span>
-                <span class="icon-bar"></span>
-                <span class="icon-bar"></span>
-                <span class="icon-bar"></span>
-            </button>
-            <a class="navbar-brand" href="#">Fleet-Up</a>
-        </div>
-        <div class="collapse navbar-collapse" id="bs-example-navbar-collapse-1">
-            <ul class="nav navbar-nav">
-                <li><a href="{% url 'auth_fleetup_view' %}">{% trans "Ops and Timers" %}</a></li>
-                <li class="active"><a href="{% url 'auth_fleetup_doctrines' %}">{% trans "Doctrines" %} <span class="sr-only">(current)</span></a></li>
-                <li><a href="{% url 'auth_fleetup_fittings' %}">{% trans "Fittings" %}</a></li>
-                {% if perms.auth.corp_stats %}
-                <li><a href="{% url 'auth_fleetup_characters' %}">{% trans "Characters" %}</a></li>
-                {% endif %}
-                <li></li>
-            </ul>
-        </div>
-    </div>
-</nav>
     <div class="panel">
         {% for a, j in doctrine.items %}
         {% regroup j.Data|dictsort:"Role" by Role as role_list %}

fleetup/templates/fleetup/doctrinesview.html

@@ -8,30 +8,7 @@
 
 {% block content %}
 <div class="col-lg-12">
-<nav class="navbar navbar-default">
+    {% include "fleetup/menu.html" %}
-    <div class="container-fluid">
-        <div class="navbar-header">
-            <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#bs-example-navbar-collapse-1" aria-expanded="false">
-                <span class="sr-only">{% trans "Toggle navigation" %}</span>
-                <span class="icon-bar"></span>
-                <span class="icon-bar"></span>
-                <span class="icon-bar"></span>
-            </button>
-            <a class="navbar-brand" href="#">Fleet-Up</a>
-        </div>
-        <div class="collapse navbar-collapse" id="bs-example-navbar-collapse-1">
-            <ul class="nav navbar-nav">
-                <li><a href="{% url 'auth_fleetup_view' %}">{% trans "Ops and Timers" %}</a></li>
-                <li class="active"><a href="{% url 'auth_fleetup_doctrines' %}">{% trans "Doctrines" %} <span class="sr-only">(current)</span></a></li>
-                <li><a href="{% url 'auth_fleetup_fittings' %}">{% trans "Fittings" %}</a></li>
-                {% if perms.auth.corp_stats %}
-                <li><a href="{% url 'auth_fleetup_characters' %}">{% trans "Characters" %}</a></li>
-                {% endif %}
-                <li></li>
-            </ul>
-        </div>
-    </div>
-</nav>
     <div class="panel">
         {% if doctrines_list %}
         {% for a, j in doctrines_list.items %}

fleetup/templates/fleetup/fitting.html

@@ -8,30 +8,7 @@
 
 {% block content %}
 <div class="col-lg-12">
-<nav class="navbar navbar-default">
+    {% include "fleetup/menu.html" %}
-    <div class="container-fluid">
-        <div class="navbar-header">
-            <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#bs-example-navbar-collapse-1" aria-expanded="false">
-                <span class="sr-only">{% trans "Toggle navigation" %}</span>
-                <span class="icon-bar"></span>
-                <span class="icon-bar"></span>
-                <span class="icon-bar"></span>
-            </button>
-            <a class="navbar-brand" href="#">{% trans "Fleet-Up" %}</a>
-        </div>
-        <div class="collapse navbar-collapse" id="bs-example-navbar-collapse-1">
-            <ul class="nav navbar-nav">
-                <li><a href="{% url 'auth_fleetup_view' %}">{% trans "Ops and Timers" %}</a></li>
-                <li><a href="{% url 'auth_fleetup_doctrines' %}">{% trans "Doctrines" %}</a></li>
-                <li class="active"><a href="{% url 'auth_fleetup_fittings' %}">{% trans "Fittings" %} <span class="sr-only">(current)</span></a></li>
-                {% if perms.auth.corp_stats %}
-                <li><a href="{% url 'auth_fleetup_characters' %}">{% trans "Characters" %}</a></li>
-                {% endif %}
-                <li></li>
-            </ul>
-        </div>
-    </div>
-</nav>
     <div class="tab-content">
         <div id="fit" class="tab-pane fade in active">
             <div class="col-lg-3">
@@ -56,8 +33,8 @@
                                 </div>
                             </div>
                         </div>
-                        <div class="col-lg-8 col-lg-offset-7">
+                        <div class="pull-right">
-                            <a class="btn btn-primary" href="/fleetup/doctrines/{{ doctrin.DoctrineId }}/">{% trans "See doctrine" %}</a>
+                            <a class="btn btn-primary" href="{% url 'auth_fleetup_doctrine' doctrin.DoctrineId %}">{% trans "See doctrine" %}</a>
                         </div>
                         {% endfor %}
                     </div>
@@ -140,7 +117,7 @@
                     <div class="panel-body">
                     {% for data in fitting_eft.items %}
                     {% autoescape off %}
-                    <pre>{{ fitting_eft.fitting_eft }}</pre>
+                    <textarea class="form-control" rows="25" spellcheck="false" onclick="this.focus();this.select()" readonly>{{ fitting_eft.fitting_eft }}</textarea>
                     {% endautoescape %}
                     {% endfor %}
                     </div>

fleetup/templates/fleetup/fittingsview.html

@@ -8,30 +8,7 @@
 
 {% block content %}
 <div class="col-lg-12">
-<nav class="navbar navbar-default">
+{% include "fleetup/menu.html" %}
-    <div class="container-fluid">
-        <div class="navbar-header">
-            <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#bs-example-navbar-collapse-1" aria-expanded="false">
-                <span class="sr-only">{% trans "Toggle navigation" %}</span>
-                <span class="icon-bar"></span>
-                <span class="icon-bar"></span>
-                <span class="icon-bar"></span>
-            </button>
-            <a class="navbar-brand" href="#">{% trans "Fleet-Up" %}</a>
-        </div>
-        <div class="collapse navbar-collapse" id="bs-example-navbar-collapse-1">
-            <ul class="nav navbar-nav">
-                <li><a href="{% url 'auth_fleetup_view' %}">{% trans "Ops and Timers" %}</a></li>
-                <li><a href="{% url 'auth_fleetup_doctrines' %}">{% trans "Doctrines" %}</a></li>
-                <li class="active"><a href="{% url 'auth_fleetup_fittings' %}">{% trans "Fittings" %} <span class="sr-only">(current)</span></a></li>
-                {% if perms.auth.corp_stats %}
-                <li><a href="{% url 'auth_fleetup_characters' %}">{% trans "Characters" %}</a></li>
-                {% endif %}
-                <li></li>
-            </ul>
-        </div>
-    </div>
-</nav>
 <div class="panel">
     {% if fitting_list %}
         <table class="table table-condensed table-hover table-striped">

fleetup/templates/fleetup/index.html

@@ -8,30 +8,7 @@
 
 {% block content %}
 <div class="col-lg-12">
-<nav class="navbar navbar-default">
+{% include "fleetup/menu.html" %}
-    <div class="container-fluid">
-        <div class="navbar-header">
-            <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#bs-example-navbar-collapse-1" aria-expanded="false">
-                <span class="sr-only">{% trans "Toggle navigation" %}</span>
-                <span class="icon-bar"></span>
-                <span class="icon-bar"></span>
-                <span class="icon-bar"></span>
-            </button>
-            <a class="navbar-brand" href="#">{% trans "Fleet-Up" %}</a>
-        </div>
-        <div class="collapse navbar-collapse" id="bs-example-navbar-collapse-1">
-            <ul class="nav navbar-nav">
-                <li class="active"><a href="#">{% trans "Ops and Timers" %} <span class="sr-only">(current)</span></a></li>
-                <li><a href="{% url 'auth_fleetup_doctrines' %}">{% trans "Doctrines" %}</a></li>
-                <li><a href="{% url 'auth_fleetup_fittings' %}">{% trans "Fittings" %}</a></li>
-                {% if perms.auth.human_resources %}
-                <li><a href="{% url 'auth_fleetup_characters' %}">{% trans "Characters" %}</a></li>
-                {% endif %}
-                <li></li>
-            </ul>
-        </div>
-    </div>
-</nav>
 <div class="panel">
     <ul class="nav nav-tabs">
         <li class="active"><a data-toggle="tab" href="#operations">{% trans "Operations" %}</a></li>

fleetup/templates/fleetup/menu.html

@@ -0,0 +1,26 @@
+{% load i18n %}
+{% load navactive %}
+
+<nav class="navbar navbar-default">
+    <div class="container-fluid">
+        <div class="navbar-header">
+            <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#bs-example-navbar-collapse-1" aria-expanded="false">
+                <span class="sr-only">{% trans "Toggle navigation" %}</span>
+                <span class="icon-bar"></span>
+                <span class="icon-bar"></span>
+                <span class="icon-bar"></span>
+            </button>
+            <a class="navbar-brand" href="#">Fleet-Up</a>
+        </div>
+        <div class="collapse navbar-collapse" id="bs-example-navbar-collapse-1">
+            <ul class="nav navbar-nav">
+                <li class="{% navactive request 'auth_fleetup_view' %}"><a href="{% url 'auth_fleetup_view' %}">{% trans "Ops and Timers" %}</a></li>
+                <li class="{% navactive request 'auth_fleetup_doctrines auth_fleetup_doctrine' %}"><a href="{% url 'auth_fleetup_doctrines' %}">{% trans "Doctrines" %}</a></li>
+                <li class="{% navactive request 'auth_fleetup_fittings auth_fleetup_fitting' %}"><a href="{% url 'auth_fleetup_fittings' %}">{% trans "Fittings" %}</a></li>
+                {% if perms.auth.corp_stats %}
+                <li class="{% navactive request 'auth_fleetup_characters' %}"><a href="{% url 'auth_fleetup_characters' %}">{% trans "Characters" %}</a></li>
+                {% endif %}
+            </ul>
+        </div>
+    </div>
+</nav>

fleetup/tests/test_manager.py

@@ -0,0 +1,509 @@
+from __future__ import unicode_literals
+
+try:
+    # Py3
+    from unittest import mock
+except ImportError:
+    # Py2
+    import mock
+
+import requests_mock
+import json
+import datetime
+
+from django.test import TestCase
+
+from fleetup.managers import FleetUpManager
+
+
+class FleetupManagerTestCase(TestCase):
+    def setUp(self):
+        pass
+
+    def test__request_cache_key(self):
+
+        cache_key = FleetUpManager._request_cache_key('testurl')
+
+        self.assertEqual('FLEETUP_ENDPOINT_a39562b6ef5b858220be13d2adb61d3f10cf8d61',
+                         cache_key)
+
+    @mock.patch('fleetup.managers.cache')
+    @requests_mock.Mocker()
+    def test_get_endpoint(self, cache, m):
+        url = "http://example.com/test/endpoint/"
+        json_data = {'data': "123456", 'CachedUntilUTC': '/Date(1493896236163)/', 'Success': True}
+        m.register_uri('GET', url,
+                       text=json.dumps(json_data))
+
+        cache.get.return_value = None  # No cached value
+
+        # Act
+        result = FleetUpManager.get_endpoint(url)
+
+        # Assert
+        self.assertTrue(cache.get.called)
+        self.assertTrue(cache.set.called)
+        args, kwargs = cache.set.call_args
+        self.assertDictEqual(json_data, args[1])
+
+        self.assertDictEqual(json_data, result)
+
+    @mock.patch('fleetup.managers.cache')
+    @requests_mock.Mocker()
+    def test_get_endpoint_error(self, cache, m):
+        url = "http://example.com/test/endpoint/"
+        json_data = {'data': [], 'Success': False}
+        m.register_uri('GET', url,
+                       text=json.dumps(json_data),
+                       status_code=400)
+
+        cache.get.return_value = None  # No cached value
+
+        # Act
+        result = FleetUpManager.get_endpoint(url)
+
+        # Assert
+        self.assertTrue(cache.get.called)
+        self.assertFalse(cache.set.called)
+        self.assertIsNone(result)
+
+    @mock.patch('fleetup.managers.FleetUpManager.get_endpoint')
+    def test_get_fleetup_members(self, get_endpoint):
+
+        get_endpoint.return_value = {"Data": [
+            {
+                'UserId': 1234,
+                'EveCharName': 'test_name',
+                'EveCharId': 5678,
+                'Corporation': 'test_corporation',
+            }
+        ]}
+
+        # Act
+        result = FleetUpManager.get_fleetup_members()
+
+        # Asset
+        self.assertTrue(get_endpoint.called)
+        args, kwargs = get_endpoint.call_args
+        self.assertEqual(args[0],
+                         FleetUpManager.BASE_URL + '/GroupCharacters/' +
+                         FleetUpManager.GROUP_ID)
+        expected_result = {
+            1234: {
+                'user_id': 1234,
+                'char_name': 'test_name',
+                'char_id': 5678,
+                'corporation': 'test_corporation',
+            }
+        }
+        self.assertDictEqual(expected_result, result)
+
+        # Test None response
+        # Arrange
+        get_endpoint.return_value = None
+
+        # Act
+        result = FleetUpManager.get_fleetup_members()
+
+        # Assert
+        self.assertIsNone(result)
+
+        # Test Empty response
+        # Arrange
+        get_endpoint.return_value = {'Data': []}
+
+        # Act
+        result = FleetUpManager.get_fleetup_members()
+
+        # Assert
+        self.assertDictEqual({}, result)
+
+    @mock.patch('fleetup.managers.FleetUpManager.get_endpoint')
+    def test_get_fleetup_operations(self, get_endpoint):
+
+        get_endpoint.return_value = {"Data": [
+            {
+                'Subject': 'test_operation',
+                'StartString': '2017-05-06 11:11:11',
+                'EndString': '2017-05-06 12:12:12',
+                'OperationId': 1234,
+                'Location': 'Jita',
+                'LocationInfo': '4-4',
+                'Details': 'This is a test operation',
+                'Url': 'http://example.com/1234',
+                'Doctrines': 'Foxcats',
+                'Organizer': 'Example FC'
+            }
+        ]}
+
+        # Act
+        result = FleetUpManager.get_fleetup_operations()
+        self.maxDiff = None
+        # Asset
+        self.assertTrue(get_endpoint.called)
+        args, kwargs = get_endpoint.call_args
+        self.assertEqual(args[0],
+                         FleetUpManager.BASE_URL + '/Operations/' +
+                         FleetUpManager.GROUP_ID)
+        expected_result = {
+            '2017-05-06 11:11:11': {
+                'subject': 'test_operation',
+                'start': datetime.datetime(2017, 5, 6, 11, 11, 11),
+                'end': datetime.datetime(2017, 5, 6, 12, 12, 12),
+                'operation_id': 1234,
+                'location': 'Jita',
+                'location_info': '4-4',
+                'details': 'This is a test operation',
+                'url': 'http://example.com/1234',
+                'doctrine': 'Foxcats',
+                'organizer': 'Example FC'
+            }
+        }
+        self.assertDictEqual(expected_result, result)
+
+        # Test None response
+        # Arrange
+        get_endpoint.return_value = None
+
+        # Act
+        result = FleetUpManager.get_fleetup_operations()
+
+        # Assert
+        self.assertIsNone(result)
+
+        # Test Empty response
+        # Arrange
+        get_endpoint.return_value = {'Data': []}
+
+        # Act
+        result = FleetUpManager.get_fleetup_operations()
+
+        # Assert
+        self.assertDictEqual({}, result)
+
+    @mock.patch('fleetup.managers.FleetUpManager.get_endpoint')
+    def test_get_fleetup_timers(self, get_endpoint):
+
+        get_endpoint.return_value = {"Data": [
+            {
+                'ExpiresString': '2017-05-06 11:11:11',
+                'SolarSystem': 'Jita',
+                'Planet': '4',
+                'Moon': '4',
+                'Owner': 'Caldari Navy',
+                'Type': 'Caldari Station',
+                'TimerType': 'Armor',
+                'Notes': 'Burn Jita?'
+            }
+        ]}
+
+        # Act
+        result = FleetUpManager.get_fleetup_timers()
+
+        # Asset
+        self.assertTrue(get_endpoint.called)
+        args, kwargs = get_endpoint.call_args
+        self.assertEqual(args[0],
+                         FleetUpManager.BASE_URL + '/Timers/' +
+                         FleetUpManager.GROUP_ID)
+        expected_result = {
+            '2017-05-06 11:11:11': {
+                'expires': datetime.datetime(2017, 5, 6, 11, 11, 11),
+                'solarsystem': 'Jita',
+                'planet': '4',
+                'moon': '4',
+                'owner': 'Caldari Navy',
+                'type': 'Caldari Station',
+                'timer_type': 'Armor',
+                'notes': 'Burn Jita?'
+            }
+        }
+        self.assertDictEqual(expected_result, result)
+
+        # Test None response
+        # Arrange
+        get_endpoint.return_value = None
+
+        # Act
+        result = FleetUpManager.get_fleetup_timers()
+
+        # Assert
+        self.assertIsNone(result)
+
+        # Test Empty response
+        # Arrange
+        get_endpoint.return_value = {'Data': []}
+
+        # Act
+        result = FleetUpManager.get_fleetup_timers()
+
+        # Assert
+        self.assertDictEqual({}, result)
+
+    @mock.patch('fleetup.managers.FleetUpManager.get_endpoint')
+    def test_get_fleetup_doctrines(self, get_endpoint):
+
+        get_endpoint.return_value = {"Data": [
+            {
+                'TestData': True
+            }
+        ]}
+
+        # Act
+        result = FleetUpManager.get_fleetup_doctrines()
+
+        # Asset
+        self.assertTrue(get_endpoint.called)
+        args, kwargs = get_endpoint.call_args
+        self.assertEqual(args[0],
+                         FleetUpManager.BASE_URL + '/Doctrines/' +
+                         FleetUpManager.GROUP_ID)
+        expected_result = {
+            'fleetup_doctrines': [{
+                'TestData': True
+            }]
+        }
+        self.assertDictEqual(expected_result, result)
+
+        # Test None response
+        # Arrange
+        get_endpoint.return_value = None
+
+        # Act
+        result = FleetUpManager.get_fleetup_doctrines()
+
+        # Assert
+        self.assertIsNone(result)
+
+        # Test Empty response
+        # Arrange
+        get_endpoint.return_value = {'Data': []}
+
+        # Act
+        result = FleetUpManager.get_fleetup_doctrines()
+
+        # Assert
+        self.assertDictEqual({"fleetup_doctrines": []}, result)
+
+    @mock.patch('fleetup.managers.FleetUpManager.get_endpoint')
+    def test_get_fleetup_doctrine(self, get_endpoint):
+
+        get_endpoint.return_value = {"Data": [
+            {
+                'TestData': True
+            }
+        ]}
+
+        # Act
+        result = FleetUpManager.get_fleetup_doctrine(1234)
+
+        # Asset
+        self.assertTrue(get_endpoint.called)
+        args, kwargs = get_endpoint.call_args
+        self.assertEqual(args[0],
+                         FleetUpManager.BASE_URL + '/DoctrineFittings/1234')
+        expected_result = {
+            'fitting_doctrine': {'Data': [{
+                'TestData': True
+            }]}
+        }
+        self.assertDictEqual(expected_result, result)
+
+        # Test None response
+        # Arrange
+        get_endpoint.return_value = None
+
+        # Act
+        result = FleetUpManager.get_fleetup_doctrine(1234)
+
+        # Assert
+        self.assertIsNone(result)
+
+        # Test Empty response
+        # Arrange
+        get_endpoint.return_value = {'Data': []}
+
+        # Act
+        result = FleetUpManager.get_fleetup_doctrine(1234)
+
+        # Assert
+        self.assertDictEqual({"fitting_doctrine": {'Data': []}}, result)
+
+    @mock.patch('fleetup.managers.FleetUpManager.get_endpoint')
+    def test_get_fleetup_fittings(self, get_endpoint):
+
+        get_endpoint.return_value = {"Data": [
+            {
+                'FittingId': 1234,
+                'Name': 'Foxcat',
+                'EveTypeId': 17726,
+                'HullType': 'Battleship',
+                'ShipType': 'Apocalypse Navy Issue',
+                'EstPrice': 500000000,
+                'Faction': 'Amarr',
+                'Categories': ["Armor", "Laser"],
+                'LastUpdatedString': '2017-05-06 11:11:11',
+            }
+        ]}
+
+        # Act
+        result = FleetUpManager.get_fleetup_fittings()
+
+        # Asset
+        self.assertTrue(get_endpoint.called)
+        expected_result = {
+            1234: {
+                'fitting_id': 1234,
+                'name': 'Foxcat',
+                'icon_id': 17726,
+                'hull': 'Battleship',
+                'shiptype': 'Apocalypse Navy Issue',
+                'estimated': 500000000,
+                'faction': 'Amarr',
+                'categories': ["Armor", "Laser"],
+                'last_update': datetime.datetime(2017, 5, 6, 11, 11, 11)
+            }
+        }
+        self.assertDictEqual(expected_result, result)
+
+        # Test None response
+        # Arrange
+        get_endpoint.return_value = None
+
+        # Act
+        result = FleetUpManager.get_fleetup_fittings()
+
+        # Assert
+        self.assertIsNone(result)
+
+        # Test Empty response
+        # Arrange
+        get_endpoint.return_value = {'Data': []}
+
+        # Act
+        result = FleetUpManager.get_fleetup_fittings()
+
+        # Assert
+        self.assertDictEqual({}, result)
+
+    @mock.patch('fleetup.managers.FleetUpManager.get_endpoint')
+    def test_get_fleetup_fitting(self, get_endpoint):
+
+        get_endpoint.return_value = {"Data":
+            {
+                'FittingData': [{}]
+            }
+        }
+
+        # Act
+        result = FleetUpManager.get_fleetup_fitting(1234)
+
+        # Asset
+        self.assertTrue(get_endpoint.called)
+        args, kwargs = get_endpoint.call_args
+        self.assertEqual(args[0], FleetUpManager.BASE_URL + '/Fitting/1234')
+        expected_result = {
+            'fitting_data': {
+                'FittingData': [{}]
+            }
+        }
+        self.assertDictEqual(expected_result, result)
+
+        # Test None response
+        # Arrange
+        get_endpoint.return_value = None
+
+        # Act
+        result = FleetUpManager.get_fleetup_fitting(1234)
+
+        # Assert
+        self.assertIsNone(result)
+
+        # Test Empty response
+        # Arrange
+        get_endpoint.return_value = {'Data': {}}
+
+        # Act
+        result = FleetUpManager.get_fleetup_fitting(1234)
+
+        # Assert
+        self.assertDictEqual({"fitting_data": {}}, result)
+
+    @mock.patch('fleetup.managers.FleetUpManager.get_endpoint')
+    def test_get_fleetup_doctrineid(self, get_endpoint):
+
+        get_endpoint.return_value = {
+            "Data": {
+                'Doctrines': [{'DoctrineId': 4567}]
+            }
+        }
+
+        # Act
+        result = FleetUpManager.get_fleetup_doctrineid(1234)
+
+        # Asset
+        self.assertTrue(get_endpoint.called)
+        args, kwargs = get_endpoint.call_args
+        self.assertEqual(args[0], FleetUpManager.BASE_URL + '/Fitting/1234')
+
+        self.assertEqual(4567, result)
+
+        # Test None response
+        # Arrange
+        get_endpoint.return_value = None
+
+        # Act
+        result = FleetUpManager.get_fleetup_doctrineid(1234)
+
+        # Assert
+        self.assertIsNone(result)
+
+        # Test Empty response
+        # Arrange
+        get_endpoint.return_value = {'Data': {}}
+
+        # Act
+        result = FleetUpManager.get_fleetup_doctrineid(1234)
+
+        # Assert
+        self.assertDictEqual({}, result)
+
+    @mock.patch('fleetup.managers.FleetUpManager.get_endpoint')
+    def test_get_fleetup_fitting_eft(self, get_endpoint):
+
+        get_endpoint.return_value = {
+            "Data": {
+                'FittingData': '[Apocalypse Navy Issue, Foxcat]'
+            }
+        }
+
+        # Act
+        result = FleetUpManager.get_fleetup_fitting_eft(1234)
+
+        # Asset
+        self.assertTrue(get_endpoint.called)
+        args, kwargs = get_endpoint.call_args
+        self.assertEqual(args[0], FleetUpManager.BASE_URL + '/Fitting/1234/eft')
+
+        self.assertDictEqual({"fitting_eft": '[Apocalypse Navy Issue, Foxcat]'},
+                             result)
+
+        # Test None response
+        # Arrange
+        get_endpoint.return_value = None
+
+        # Act
+        result = FleetUpManager.get_fleetup_fitting_eft(1234)
+
+        # Assert
+        self.assertIsNone(result)
+
+        # Test Empty response
+        # Arrange
+        get_endpoint.return_value = {'Data': {}}
+
+        # Act
+        result = FleetUpManager.get_fleetup_fitting_eft(1234)
+
+        # Assert
+        self.assertDictEqual({"fitting_eft": {}}, result)

fleetup/urls.py

@@ -0,0 +1,13 @@
+from __future__ import unicode_literals
+from django.conf.urls import url
+
+from . import views
+
+urlpatterns = [
+    url(r'^$', views.fleetup_view, name='auth_fleetup_view'),
+    url(r'^fittings/$', views.fleetup_fittings, name='auth_fleetup_fittings'),
+    url(r'^fittings/(?P<fittingnumber>[0-9]+)/$', views.fleetup_fitting, name='auth_fleetup_fitting'),
+    url(r'^doctrines/$', views.fleetup_doctrines, name='auth_fleetup_doctrines'),
+    url(r'^characters/$', views.fleetup_characters, name='auth_fleetup_characters'),
+    url(r'^doctrines/(?P<doctrinenumber>[0-9]+)/$', views.fleetup_doctrine, name='auth_fleetup_doctrine'),
+]

fleetup/views.py

@@ -4,6 +4,8 @@ from django.shortcuts import render
 from django.contrib.auth.decorators import login_required
 from django.contrib.auth.decorators import permission_required
 from django.template.defaulttags import register
+from django.contrib import messages
+from django.utils.translation import ugettext_lazy as _
 from fleetup.managers import FleetUpManager
 
 import logging
@@ -22,14 +24,20 @@ def fleetup_view(request):
     logger.debug("fleetup_view called by user %s" % request.user)
 
     operations_list = FleetUpManager.get_fleetup_operations()
+    if operations_list is None:
+        messages.add_message(request, messages.ERROR, _("Failed to get operations list, contact your administrator"))
+        operations_list = {}
     timers_list = FleetUpManager.get_fleetup_timers()
+    if timers_list is None:
+        messages.add_message(request, messages.ERROR, _("Failed to get timers list, contact your administrator"))
+        timers_list = {}
     now = datetime.datetime.now().strftime('%H:%M:%S')
 
     context = {"timers_list": sorted(timers_list.items()),
                "operations_list": sorted(operations_list.items()),
                "now": now}
 
-    return render(request, 'registered/fleetup.html', context=context)
+    return render(request, 'fleetup/index.html', context=context)
 
 
 @login_required
@@ -39,10 +47,13 @@ def fleetup_characters(request):
     logger.debug("fleetup_characters called by user %s" % request.user)
 
     member_list = FleetUpManager.get_fleetup_members()
+    if member_list is None:
+        messages.add_message(request, messages.ERROR, _("Failed to get member list, contact your administrator"))
+        member_list = {}
 
     context = {"member_list": sorted(member_list.items())}
 
-    return render(request, 'registered/fleetupcharacters.html', context=context)
+    return render(request, 'fleetup/characters.html', context=context)
 
 
 @login_required
@@ -50,8 +61,13 @@ def fleetup_characters(request):
 def fleetup_fittings(request):
     logger.debug("fleetup_fittings called by user %s" % request.user)
     fitting_list = FleetUpManager.get_fleetup_fittings()
+
+    if fitting_list is None:
+        messages.add_message(request, messages.ERROR, _("Failed to get fitting list, contact your administrator"))
+        fitting_list = {}
+
     context = {"fitting_list": sorted(fitting_list.items())}
-    return render(request, 'registered/fleetupfittingsview.html', context=context)
+    return render(request, 'fleetup/fittingsview.html', context=context)
 
 
 @login_required
@@ -62,10 +78,15 @@ def fleetup_fitting(request, fittingnumber):
     fitting_data = FleetUpManager.get_fleetup_fitting(fittingnumber)
     doctrinenumber = FleetUpManager.get_fleetup_doctrineid(fittingnumber)
     doctrines_list = FleetUpManager.get_fleetup_doctrine(doctrinenumber)
+
+    if fitting_eft is None or fitting_data is None or doctrinenumber is None:
+        messages.add_message(request, messages.ERROR, _("There was an error getting some of the data for this fitting. "
+                                                        "Contact your administrator"))
+
     context = {"fitting_eft": fitting_eft,
                "fitting_data": fitting_data,
                "doctrines_list": doctrines_list}
-    return render(request, 'registered/fleetupfitting.html', context=context)
+    return render(request, 'fleetup/fitting.html', context=context)
 
 
 @login_required
@@ -73,8 +94,11 @@ def fleetup_fitting(request, fittingnumber):
 def fleetup_doctrines(request):
     logger.debug("fleetup_doctrines called by user %s" % request.user)
     doctrines_list = FleetUpManager.get_fleetup_doctrines()
+    if doctrines_list is None:
+        messages.add_message(request, messages.ERROR, _("Failed to get doctrines list, contact your administrator"))
+
     context = {"doctrines_list": doctrines_list}
-    return render(request, 'registered/fleetupdoctrinesview.html', context=context)
+    return render(request, 'fleetup/doctrinesview.html', context=context)
 
 
 @login_required
@@ -82,5 +106,7 @@ def fleetup_doctrines(request):
 def fleetup_doctrine(request, doctrinenumber):
     logger.debug("fleetup_doctrine called by user %s" % request.user)
     doctrine = FleetUpManager.get_fleetup_doctrine(doctrinenumber)
+    if doctrine is None:
+        messages.add_message(request, messages.ERROR, _("Failed to get doctine, contact your administrator"))
     context = {"doctrine": doctrine}
-    return render(request, 'registered/fleetupdoctrine.html', context=context)
+    return render(request, 'fleetup/doctrine.html', context=context)

services/modules/discord/manager.py

@@ -5,9 +5,12 @@ import re
 from django.conf import settings
 from services.models import GroupCache
 from requests_oauthlib import OAuth2Session
+from functools import wraps
 import logging
 import datetime
+import time
 from django.utils import timezone
+from django.core.cache import cache
 
 logger = logging.getLogger(__name__)
 
@@ -31,6 +34,118 @@ SCOPES = [
 GROUP_CACHE_MAX_AGE = datetime.timedelta(minutes=30)
 
 
+class DiscordApiException(Exception):
+    def __init__(self):
+        super(Exception, self).__init__()
+
+
+class DiscordApiTooBusy(DiscordApiException):
+    def __init__(self):
+        super(DiscordApiException, self).__init__()
+        self.message = "The Discord API is too busy to process this request now, please try again later."
+
+
+class DiscordApiBackoff(DiscordApiException):
+    def __init__(self, retry_after, global_ratelimit):
+        super(DiscordApiException, self).__init__()
+        self.retry_after = retry_after
+        self.global_ratelimit = global_ratelimit
+
+
+cache_time_format = '%Y-%m-%d %H:%M:%S'
+
+
+def api_backoff(func):
+    """
+    Decorator, Handles HTTP 429 "Too Many Requests" messages from the Discord API
+    If blocking=True is specified, this function will block and retry
+    the function up to max_retries=n times, or 3 if retries is not specified.
+    If the API call still recieves a backoff timer this function will raise
+    a <DiscordApiTooBusy> exception.
+    If the caller chooses blocking=False, the decorator will raise a DiscordApiBackoff
+    exception and the caller can choose to retry after the given timespan available in
+    the retry_after property in seconds.
+    """
+
+    class PerformBackoff(Exception):
+        def __init__(self, retry_after, retry_datetime, global_ratelimit):
+            super(Exception, self).__init__()
+            self.retry_after = int(retry_after)
+            self.retry_datetime = retry_datetime
+            self.global_ratelimit = global_ratelimit
+
+    @wraps(func)
+    def decorated(*args, **kwargs):
+        blocking = kwargs.get('blocking', False)
+        retries = kwargs.get('max_retries', 3)
+
+        # Strip our parameters
+        if 'max_retries' in kwargs:
+            del kwargs['max_retries']
+        if 'blocking' in kwargs:
+            del kwargs['blocking']
+
+        cache_key = 'DISCORD_BACKOFF_' + func.__name__
+        cache_global_key = 'DISCORD_BACKOFF_GLOBAL'
+
+        while retries > 0:
+            try:
+                try:
+                    # Check global backoff first, then route backoff
+                    existing_global_backoff = cache.get(cache_global_key)
+                    existing_backoff = existing_global_backoff or cache.get(cache_key)
+                    if existing_backoff:
+                        backoff_timer = datetime.datetime.strptime(existing_backoff, cache_time_format)
+                        if backoff_timer > datetime.datetime.utcnow():
+                            backoff_seconds = (backoff_timer - datetime.datetime.utcnow()).total_seconds()
+                            logger.debug("Still under backoff for {} seconds, backing off" % backoff_seconds)
+                            # Still under backoff
+                            raise PerformBackoff(
+                                retry_after=backoff_seconds,
+                                retry_datetime=backoff_timer,
+                                global_ratelimit=bool(existing_global_backoff)
+                            )
+                    logger.debug("Calling API calling function")
+                    func(*args, **kwargs)
+                    break
+                except requests.HTTPError as e:
+                    if e.response.status_code == 429:
+                        if 'Retry-After' in e.response.headers:
+                            retry_after = e.response.headers['Retry-After']
+                        else:
+                            # Pick some random time
+                            retry_after = 5
+
+                        logger.info("Received backoff from API of %s seconds, handling" % retry_after)
+                        # Store value in redis
+                        backoff_until = (datetime.datetime.utcnow() +
+                                         datetime.timedelta(seconds=int(retry_after)))
+                        global_backoff = bool(e.response.headers.get('X-RateLimit-Global', False))
+                        if global_backoff:
+                            logger.info("Global backoff!!")
+                            cache.set(cache_global_key, backoff_until.strftime(cache_time_format), retry_after)
+                        else:
+                            cache.set(cache_key, backoff_until.strftime(cache_time_format), retry_after)
+                        raise PerformBackoff(retry_after=retry_after, retry_datetime=backoff_until,
+                                             global_ratelimit=global_backoff)
+                    else:
+                        # Not 429, re-raise
+                        raise e
+            except PerformBackoff as bo:
+                # Sleep if we're blocking
+                if blocking:
+                    logger.info("Blocking Back off from API calls for %s seconds" % bo.retry_after)
+                    time.sleep(10 if bo.retry_after > 10 else bo.retry_after)
+                else:
+                    # Otherwise raise exception and let caller handle the backoff
+                    raise DiscordApiBackoff(retry_after=bo.retry_after, global_ratelimit=bo.global_ratelimit)
+            finally:
+                retries -= 1
+        if retries == 0:
+            raise DiscordApiTooBusy()
+    return decorated
+
+
 class DiscordOAuthManager:
     def __init__(self):
         pass
@@ -191,6 +306,7 @@ class DiscordOAuthManager:
         DiscordOAuthManager.__update_group_cache()
 
     @staticmethod
+    @api_backoff
     def update_groups(user_id, groups):
         custom_headers = {'content-type': 'application/json', 'authorization': 'Bot ' + settings.DISCORD_BOT_TOKEN}
         group_ids = [DiscordOAuthManager.__group_name_to_id(DiscordOAuthManager._sanitize_groupname(g)) for g in groups]
@@ -199,3 +315,4 @@ class DiscordOAuthManager:
         r = requests.patch(path, headers=custom_headers, json=data)
         logger.debug("Received status code %s after setting user roles" % r.status_code)
         r.raise_for_status()
+

services/modules/discord/tasks.py

@@ -8,7 +8,7 @@ from django.contrib.auth.models import User
 from django.core.exceptions import ObjectDoesNotExist
 
 from notifications import notify
-from services.modules.discord.manager import DiscordOAuthManager
+from services.modules.discord.manager import DiscordOAuthManager, DiscordApiBackoff
 from services.tasks import only_one
 from .models import DiscordUser
 
@@ -74,6 +74,10 @@ class DiscordTasks:
             logger.debug("Updating user %s discord groups to %s" % (user, groups))
             try:
                 DiscordOAuthManager.update_groups(user.discord.uid, groups)
+            except DiscordApiBackoff as bo:
+                logger.info("Discord group sync API back off for %s, "
+                            "retrying in %s seconds" % (user, bo.retry_after))
+                raise task_self.retry(countdown=bo.retry_after)
             except Exception as e:
                 if task_self:
                     logger.exception("Discord group sync failed for %s, retrying in 10 mins" % user)

services/modules/discord/tests.py

@@ -16,6 +16,10 @@ from alliance_auth.tests.auth_utils import AuthUtils
 from .auth_hooks import DiscordService
 from .models import DiscordUser
 from .tasks import DiscordTasks
+from .manager import DiscordOAuthManager
+
+import requests_mock
+import datetime
 
 MODULE_PATH = 'services.modules.discord'
 DEFAULT_AUTH_GROUP = 'Member'
@@ -191,3 +195,254 @@ class DiscordViewsTestCase(TestCase):
         self.assertRedirects(response, expected_url='/en/services/', target_status_code=200)
         with self.assertRaises(ObjectDoesNotExist):
             discord_user = User.objects.get(pk=self.member.pk).discord
+
+
+class DiscordManagerTestCase(TestCase):
+    def setUp(self):
+        pass
+
+    def test__sanitize_groupname(self):
+        test_group_name = ' Group Name_Test_'
+        group_name = DiscordOAuthManager._sanitize_groupname(test_group_name)
+
+        self.assertEqual(group_name, 'GroupName_Test')
+
+    def test_generate_Bot_add_url(self):
+        from . import manager
+        bot_add_url = DiscordOAuthManager.generate_bot_add_url()
+
+        auth_url = manager.AUTH_URL
+        real_bot_add_url = '{}?client_id=appid&scope=bot&permissions={}'.format(auth_url, manager.BOT_PERMISSIONS)
+        self.assertEqual(bot_add_url, real_bot_add_url)
+
+    def test_generate_oauth_redirect_url(self):
+        from . import manager
+        import urllib
+        import sys
+        oauth_url = DiscordOAuthManager.generate_oauth_redirect_url()
+
+        self.assertIn(manager.AUTH_URL, oauth_url)
+        self.assertIn('+'.join(manager.SCOPES), oauth_url)
+        self.assertIn(settings.DISCORD_APP_ID, oauth_url)
+        if sys.version_info[0] < 3:
+            # Py2
+            self.assertIn(urllib.quote_plus(settings.DISCORD_CALLBACK_URL), oauth_url)
+        else: # Py3
+            self.assertIn(urllib.parse.quote_plus(settings.DISCORD_CALLBACK_URL), oauth_url)
+
+    @mock.patch(MODULE_PATH + '.manager.OAuth2Session')
+    def test__process_callback_code(self, oauth):
+        from . import manager
+        instance = oauth.return_value
+        instance.fetch_token.return_value = {'access_token': 'mywonderfultoken'}
+
+        token = DiscordOAuthManager._process_callback_code('12345')
+
+        self.assertTrue(oauth.called)
+        args, kwargs = oauth.call_args
+        self.assertEqual(args[0], settings.DISCORD_APP_ID)
+        self.assertEqual(kwargs['redirect_uri'], settings.DISCORD_CALLBACK_URL)
+        self.assertTrue(instance.fetch_token.called)
+        args, kwargs = instance.fetch_token.call_args
+        self.assertEqual(args[0], manager.TOKEN_URL)
+        self.assertEqual(kwargs['client_secret'], settings.DISCORD_APP_SECRET)
+        self.assertEqual(kwargs['code'], '12345')
+        self.assertEqual(token['access_token'], 'mywonderfultoken')
+
+    @mock.patch(MODULE_PATH + '.manager.DiscordOAuthManager._process_callback_code')
+    @requests_mock.Mocker()
+    def test_add_user(self, oauth_token, m):
+        from . import manager
+        import json
+
+        # Arrange
+        oauth_token.return_value = {'access_token': 'accesstoken'}
+
+        headers = {'accept': 'application/json', 'authorization': 'Bearer accesstoken'}
+
+        m.register_uri('POST',
+                       manager.DISCORD_URL + '/invites/'+str(settings.DISCORD_INVITE_CODE),
+                       request_headers=headers,
+                       text='{}')
+
+        m.register_uri('GET',
+                       manager.DISCORD_URL + "/users/@me",
+                       request_headers=headers,
+                       text=json.dumps({'id': "123456"}))
+
+        # Act
+        return_value = DiscordOAuthManager.add_user('abcdef')
+
+        # Assert
+        self.assertEqual(return_value, '123456')
+        self.assertEqual(m.call_count, 2)
+
+    @requests_mock.Mocker()
+    def test_delete_user(self, m):
+        from . import manager
+        import json
+
+        # Arrange
+        headers = {'accept': 'application/json', 'authorization': 'Bot ' + settings.DISCORD_BOT_TOKEN}
+
+        user_id = 12345
+        request_url = '{}/guilds/{}/members/{}'.format(manager.DISCORD_URL, settings.DISCORD_GUILD_ID, user_id)
+        m.register_uri('DELETE',
+                       request_url,
+                       request_headers=headers,
+                       text=json.dumps({}))
+
+        # Act
+        result = DiscordOAuthManager.delete_user(user_id)
+
+        # Assert
+        self.assertTrue(result)
+
+        ###
+        # Test 404 (already deleted)
+        # Arrange
+        m.register_uri('DELETE',
+                       request_url,
+                       request_headers=headers,
+                       status_code=404)
+
+        # Act
+        result = DiscordOAuthManager.delete_user(user_id)
+
+        # Assert
+        self.assertTrue(result)
+
+        ###
+        # Test 500 (some random API error)
+        # Arrange
+        m.register_uri('DELETE',
+                       request_url,
+                       request_headers=headers,
+                       status_code=500)
+
+        # Act
+        result = DiscordOAuthManager.delete_user(user_id)
+
+        # Assert
+        self.assertFalse(result)
+
+    @requests_mock.Mocker()
+    def test_update_nickname(self, m):
+        from . import manager
+        import json
+        # Arrange
+        headers = {'content-type': 'application/json', 'authorization': 'Bot ' + settings.DISCORD_BOT_TOKEN}
+
+        user_id = 12345
+        request_url = '{}/guilds/{}/members/{}'.format(manager.DISCORD_URL, settings.DISCORD_GUILD_ID, user_id)
+        m.patch(request_url,
+                request_headers=headers)
+
+        # Act
+        result = DiscordOAuthManager.update_nickname(user_id, 'somenick')
+
+        # Assert
+        self.assertTrue(result)
+
+    @mock.patch(MODULE_PATH + '.manager.DiscordOAuthManager._DiscordOAuthManager__get_group_cache')
+    @requests_mock.Mocker()
+    def test_update_groups(self, group_cache, m):
+        from . import manager
+        import json
+
+        # Arrange
+        groups = ['Member', 'Blue', 'Special Group']
+
+        group_cache.return_value = [{'id': 111, 'name': 'Member'},
+                                    {'id': 222, 'name': 'Blue'},
+                                    {'id': 333, 'name': 'SpecialGroup'},
+                                    {'id': 444, 'name': 'NotYourGroup'}]
+
+        headers = {'content-type': 'application/json', 'authorization': 'Bot ' + settings.DISCORD_BOT_TOKEN}
+        user_id = 12345
+        request_url = '{}/guilds/{}/members/{}'.format(manager.DISCORD_URL, settings.DISCORD_GUILD_ID, user_id)
+
+        m.patch(request_url,
+                request_headers=headers)
+
+        # Act
+        DiscordOAuthManager.update_groups(user_id, groups)
+
+        # Assert
+        self.assertEqual(len(m.request_history), 1, 'Must be one HTTP call made')
+        history = json.loads(m.request_history[0].text)
+        self.assertIn('roles', history, "'The request must send JSON object with the 'roles' key")
+        self.assertIn(111, history['roles'], 'The group id 111 must be added to the request')
+        self.assertIn(222, history['roles'], 'The group id 222 must be added to the request')
+        self.assertIn(333, history['roles'], 'The group id 333 must be added to the request')
+        self.assertNotIn(444, history['roles'], 'The group id 444 must NOT be added to the request')
+
+    @mock.patch(MODULE_PATH + '.manager.cache')
+    @mock.patch(MODULE_PATH + '.manager.DiscordOAuthManager._DiscordOAuthManager__get_group_cache')
+    @requests_mock.Mocker()
+    def test_update_groups_backoff(self, group_cache, djcache, m):
+        from . import manager
+
+        # Arrange
+        groups = ['Member']
+        group_cache.return_value = [{'id': 111, 'name': 'Member'}]
+
+        headers = {'content-type': 'application/json', 'authorization': 'Bot ' + settings.DISCORD_BOT_TOKEN}
+        user_id = 12345
+        request_url = '{}/guilds/{}/members/{}'.format(manager.DISCORD_URL, settings.DISCORD_GUILD_ID, user_id)
+
+        djcache.get.return_value = None  # No existing backoffs in cache
+
+        m.patch(request_url,
+                request_headers=headers,
+                headers={'Retry-After': '200'},
+                status_code=429)
+
+        # Act & Assert
+        with self.assertRaises(manager.DiscordApiBackoff) as bo:
+            try:
+                DiscordOAuthManager.update_groups(user_id, groups, blocking=False)
+            except manager.DiscordApiBackoff as bo:
+                self.assertEqual(bo.retry_after, 200, 'Retry-After time must be equal to Retry-After set in header')
+                self.assertFalse(bo.global_ratelimit, 'global_ratelimit must be False')
+                raise bo
+
+        self.assertTrue(djcache.set.called)
+        args, kwargs = djcache.set.call_args
+        self.assertEqual(args[0], 'DISCORD_BACKOFF_update_groups')
+        self.assertTrue(datetime.datetime.strptime(args[1], manager.cache_time_format) > datetime.datetime.now())
+
+    @mock.patch(MODULE_PATH + '.manager.cache')
+    @mock.patch(MODULE_PATH + '.manager.DiscordOAuthManager._DiscordOAuthManager__get_group_cache')
+    @requests_mock.Mocker()
+    def test_update_groups_global_backoff(self, group_cache, djcache, m):
+        from . import manager
+
+        # Arrange
+        groups = ['Member']
+        group_cache.return_value = [{'id': 111, 'name': 'Member'}]
+
+        headers = {'content-type': 'application/json', 'authorization': 'Bot ' + settings.DISCORD_BOT_TOKEN}
+        user_id = 12345
+        request_url = '{}/guilds/{}/members/{}'.format(manager.DISCORD_URL, settings.DISCORD_GUILD_ID, user_id)
+
+        djcache.get.return_value = None  # No existing backoffs in cache
+
+        m.patch(request_url,
+                request_headers=headers,
+                headers={'Retry-After': '200', 'X-RateLimit-Global': 'true'},
+                status_code=429)
+
+        # Act & Assert
+        with self.assertRaises(manager.DiscordApiBackoff) as bo:
+            try:
+                DiscordOAuthManager.update_groups(user_id, groups, blocking=False)
+            except manager.DiscordApiBackoff as bo:
+                self.assertEqual(bo.retry_after, 200, 'Retry-After time must be equal to Retry-After set in header')
+                self.assertTrue(bo.global_ratelimit, 'global_ratelimit must be True')
+                raise bo
+
+        self.assertTrue(djcache.set.called)
+        args, kwargs = djcache.set.call_args
+        self.assertEqual(args[0], 'DISCORD_BACKOFF_GLOBAL')
+        self.assertTrue(datetime.datetime.strptime(args[1], manager.cache_time_format) > datetime.datetime.now())

testing-requirements.txt

@@ -7,3 +7,4 @@ nose>=1.3.7
 django-nose>=1.4.4
 coverage>=4.3.1
 coveralls>=1.1
+requests-mock>=1.2.0