Prevent checking out v2 when trying to install v1

(cherry picked from commit 77c93ed96b)

.idea/allianceauth.iml

@@ -1,22 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<module type="PYTHON_MODULE" version="4">
-  <component name="FacetManager">
-    <facet type="django" name="Django">
-      <configuration>
-        <option name="rootFolder" value="$MODULE_DIR$" />
-        <option name="settingsModule" value="alliance_auth/settings.py.example" />
-        <option name="manageScript" value="manage.py" />
-        <option name="environment" value="&lt;map/&gt;" />
-        <option name="commandsToSkip" value="" />
-      </configuration>
-    </facet>
-  </component>
-  <component name="NewModuleRootManager">
-    <content url="file://$MODULE_DIR$" />
-    <orderEntry type="jdk" jdkName="Python 2.7.11 virtualenv at ~/1.6" jdkType="Python SDK" />
-    <orderEntry type="sourceFolder" forTests="false" />
-  </component>
-  <component name="TemplatesService">
-    <option name="TEMPLATE_CONFIGURATION" value="Django" />
-  </component>
-</module>

.idea/dataSources.ids

@@ -1,12 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<component name="dataSourceStorage">
-  <data-source source="LOCAL" name="Django default" uuid="3eb61453-647a-4832-8320-f3561f039abc">
-    <database-info product="" version="" jdbc-version="" driver-name="" driver-version=""/>
-  </data-source>
-  <data-source source="LOCAL" name="Django phpbb3" uuid="2de247c2-1951-4e74-8276-6a1c89c396fa">
-    <database-info product="" version="" jdbc-version="" driver-name="" driver-version=""/>
-  </data-source>
-  <data-source source="LOCAL" name="Django mumble" uuid="9963e5ca-7f2f-4dd3-9175-bc7102dfd48c">
-    <database-info product="" version="" jdbc-version="" driver-name="" driver-version=""/>
-  </data-source>
-</component>

.idea/dataSources.xml

@@ -1,20 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project version="4">
-  <component name="DataSourceManagerImpl" format="xml" multifile-model="true">
-    <data-source source="LOCAL" name="Django default" uuid="3eb61453-647a-4832-8320-f3561f039abc">
-      <driver-ref>mysql</driver-ref>
-      <jdbc-driver>com.mysql.jdbc.Driver</jdbc-driver>
-      <jdbc-url>jdbc:mysql://127.0.0.1:3306/alliance_auth</jdbc-url>
-    </data-source>
-    <data-source source="LOCAL" name="Django phpbb3" uuid="2de247c2-1951-4e74-8276-6a1c89c396fa">
-      <driver-ref>mysql</driver-ref>
-      <jdbc-driver>com.mysql.jdbc.Driver</jdbc-driver>
-      <jdbc-url>jdbc:mysql://127.0.0.1:3306/alliance_forum</jdbc-url>
-    </data-source>
-    <data-source source="LOCAL" name="Django mumble" uuid="9963e5ca-7f2f-4dd3-9175-bc7102dfd48c">
-      <driver-ref>mysql</driver-ref>
-      <jdbc-driver>com.mysql.jdbc.Driver</jdbc-driver>
-      <jdbc-url>jdbc:mysql://127.0.0.1:3306/alliance_mumble</jdbc-url>
-    </data-source>
-  </component>
-</project>

.idea/encodings.xml

@@ -1,5 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project version="4">
-  <component name="Encoding" useUTFGuessing="true" native2AsciiForPropertiesFiles="false" />
-</project>
-

.idea/inspectionProfiles/Project_Default.xml

@@ -1,11 +0,0 @@
-<component name="InspectionProjectProfileManager">
-  <profile version="1.0" is_locked="false">
-    <option name="myName" value="Project Default" />
-    <option name="myLocal" value="false" />
-    <inspection_tool class="SpellCheckingInspection" enabled="false" level="TYPO" enabled_by_default="false">
-      <option name="processCode" value="true" />
-      <option name="processLiterals" value="true" />
-      <option name="processComments" value="true" />
-    </inspection_tool>
-  </profile>
-</component>

.idea/inspectionProfiles/profiles_settings.xml

@@ -1,7 +0,0 @@
-<component name="InspectionProjectProfileManager">
-  <settings>
-    <option name="PROJECT_PROFILE" />
-    <option name="USE_PROJECT_PROFILE" value="false" />
-    <version value="1.0" />
-  </settings>
-</component>

.idea/misc.xml

@@ -1,7 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project version="4">
-  <component name="ProjectRootManager" version="2" project-jdk-name="Python 2.7.11 virtualenv at ~/1.6" project-jdk-type="Python SDK" />
-  <component name="PythonCompatibilityInspectionAdvertiser">
-    <option name="version" value="1" />
-  </component>
-</project>

.idea/modules.xml

@@ -1,9 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project version="4">
-  <component name="ProjectModuleManager">
-    <modules>
-      <module fileurl="file://$PROJECT_DIR$/.idea/allianceauth.iml" filepath="$PROJECT_DIR$/.idea/allianceauth.iml" />
-    </modules>
-  </component>
-</project>
-

.idea/scopes/scope_settings.xml

@@ -1,5 +0,0 @@
-<component name="DependencyValidationManager">
-  <state>
-    <option name="SKIP_IMPORT_STATEMENTS" value="false" />
-  </state>
-</component>

.idea/vcs.xml

@@ -1,7 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project version="4">
-  <component name="VcsDirectoryMappings">
-    <mapping directory="$PROJECT_DIR$" vcs="Git" />
-  </component>
-</project>
-

README.md

@@ -2,7 +2,7 @@ Alliance Auth
 ============
 
 [![Join the chat at https://gitter.im/R4stl1n/allianceauth](https://badges.gitter.im/R4stl1n/allianceauth.svg)](https://gitter.im/R4stl1n/allianceauth?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
-[![Documentation Status](https://readthedocs.org/projects/allianceauth/badge/?version=latest)](http://allianceauth.readthedocs.io/en/latest/?badge=latest)
+[![Documentation Status](https://readthedocs.org/projects/allianceauth/badge/?version=latest)](http://allianceauth.readthedocs.io/?badge=latest)
 [![Build Status](https://travis-ci.org/allianceauth/allianceauth.svg?branch=master)](https://travis-ci.org/allianceauth/allianceauth)
 [![Coverage Status](https://coveralls.io/repos/github/allianceauth/allianceauth/badge.svg?branch=master)](https://coveralls.io/github/allianceauth/allianceauth?branch=master)
 
@@ -10,6 +10,8 @@ Alliance Auth
 EVE service auth to help corps, alliances, and coalitions manage services.
 Built for "The 99 Percent" open for anyone to use.
 
+[Read the docs here.](http://allianceauth.rtfd.io)
+
 Special Permissions In Admin:
 
     auth | user | group_management ( Access to add members to groups within the alliance )

alliance_auth/__init__.py

@@ -4,5 +4,5 @@ from __future__ import absolute_import, unicode_literals
 # Django starts so that shared_task will use this app.
 from .celeryapp import app as celery_app  # noqa
 
-__version__ = '1.15.0'
+__version__ = '1.15.7'
 NAME = 'Alliance Auth v%s' % __version__

alliance_auth/settings.py.example

@@ -1,3 +1,4 @@
+# -*- coding: UTF-8 -*-
 """
 Django settings for alliance_auth project.
 
@@ -12,17 +13,12 @@ https://docs.djangoproject.com/en/1.10/ref/settings/
 
 import os
 
-import djcelery
-
 from django.contrib import messages
 from celery.schedules import crontab
 
-djcelery.setup_loader()
-
 # Celery configuration
 BROKER_URL = 'redis://localhost:6379/0'
-CELERYBEAT_SCHEDULER = "djcelery.schedulers.DatabaseScheduler"
-CELERYBEAT_SCHEDULE = dict()
+CELERYBEAT_SCHEDULER = "django_celery_beat.schedulers.DatabaseScheduler"
 
 # Build paths inside the project like this: os.path.join(BASE_DIR, ...)
 BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
@@ -50,7 +46,7 @@ INSTALLED_APPS = [
     'django.contrib.messages',
     'django.contrib.staticfiles',
     'django.contrib.humanize',
-    'djcelery',
+    'django_celery_beat',
     'bootstrapform',
     'authentication',
     'services',
@@ -62,11 +58,13 @@ INSTALLED_APPS = [
     'optimer',
     'corputils',
     'fleetactivitytracking',
+    'fleetup',
     'notifications',
     'esi',
     'permissions_tool',
     'geelweb.django.navhelper',
     'bootstrap_pagination',
+    'captcha',
 
     # Services
     'services.modules.mumble',
@@ -213,20 +211,48 @@ MESSAGE_TAGS = {
 
 CACHES = {
     "default": {
-        "BACKEND": "django_redis.cache.RedisCache",
-        "LOCATION": "redis://127.0.0.1:6379/1",
+        "BACKEND": "redis_cache.RedisCache",
+        "LOCATION": "localhost:6379",
         "OPTIONS": {
-            "CLIENT_CLASS": "django_redis.client.DefaultClient",
+            "DB": 1,
         }
     }
 }
 
+# Google Recaptcha
+
+CAPTCHA_ENABLED = False
+
+RECAPTCHA_PUBLIC_KEY = 'MyRecaptchaKey'
+RECAPTCHA_PRIVATE_KEY = 'MyRecaptchaPrivateKey'
+
+NOCAPTCHA = True
+
 #####################################################
 ##
 ## Auth configuration starts here
 ##
 #####################################################
 
+
+#########################
+# CELERY SCHEDULED TASKS
+#########################
+CELERYBEAT_SCHEDULE = {
+    'run_api_refresh': {
+        'task': 'eveonline.tasks.run_api_refresh',
+        'schedule': crontab(minute=0, hour="*/3"),
+    },
+    'run_corp_update': {
+        'task': 'eveonline.tasks.run_corp_update',
+        'schedule': crontab(minute=0, hour="*/2"),
+    },
+    'update_all_corpstats': {
+        'task': 'corputils.tasks.update_all_corpstats',
+        'schedule': crontab(minute=0, hour="*/6"),
+    },
+}
+
 #################
 # EMAIL SETTINGS
 #################
@@ -416,11 +442,10 @@ BROADCAST_SERVICE_NAME = os.environ.get('AA_BROADCAST_SERVICE_NAME', "broadcast"
 ######################################
 # Mumble Configuration
 ######################################
-# MUMBLE_URL - Mumble server url
-# MUMBLE_SERVER_ID - Mumble server id
+# MUMBLE_URL - Mumble server host
+#               Do not include leading http:// or mumble://
 ######################################
-MUMBLE_URL = os.environ.get('AA_MUMBLE_URL', "https://example.com")
-MUMBLE_SERVER_ID = int(os.environ.get('AA_MUMBLE_SERVER_ID', '1'))
+MUMBLE_URL = os.environ.get('AA_MUMBLE_URL', "example.com")
 
 ######################################
 # PHPBB3 Configuration
@@ -457,7 +482,6 @@ TEAMSPEAK3_PUBLIC_URL = os.environ.get('AA_TEAMSPEAK3_PUBLIC_URL', 'example.com'
 ######################################
 # DISCORD_GUILD_ID - ID of the guild to manage
 # DISCORD_BOT_TOKEN - oauth token of the app bot user
-# DISCORD_INVITE_CODE - invite code to the server
 # DISCORD_APP_ID - oauth app client ID
 # DISCORD_APP_SECRET - oauth app secret
 # DISCORD_CALLBACK_URL - oauth callback url
@@ -465,7 +489,6 @@ TEAMSPEAK3_PUBLIC_URL = os.environ.get('AA_TEAMSPEAK3_PUBLIC_URL', 'example.com'
 ######################################
 DISCORD_GUILD_ID = os.environ.get('AA_DISCORD_GUILD_ID', '')
 DISCORD_BOT_TOKEN = os.environ.get('AA_DISCORD_BOT_TOKEN', '')
-DISCORD_INVITE_CODE = os.environ.get('AA_DISCORD_INVITE_CODE', '')
 DISCORD_APP_ID = os.environ.get('AA_DISCORD_APP_ID', '')
 DISCORD_APP_SECRET = os.environ.get('AA_DISCORD_APP_SECRET', '')
 DISCORD_CALLBACK_URL = os.environ.get('AA_DISCORD_CALLBACK_URL', 'http://example.com/discord/callback')
@@ -638,6 +661,10 @@ LOGGING = {
             'handlers': ['log_file', 'console', 'notifications'],
             'level': 'ERROR',
         },
+        'fleetup': {
+            'handlers': ['log_file', 'console', 'notifications'],
+            'level': 'DEBUG',
+        },
         'util': {
             'handlers': ['log_file', 'console', 'notifications'],
             'level': 'DEBUG',

alliance_auth/tests/test_settings.py

@@ -4,20 +4,17 @@ Alliance Auth Test Suite Django settings.
 
 import os
 
-import djcelery
-
 from django.contrib import messages
 
 import alliance_auth
 
-djcelery.setup_loader()
-
 # Use nose to run all tests
 TEST_RUNNER = 'django_nose.NoseTestSuiteRunner'
 
 NOSE_ARGS = [
     #'--with-coverage',
     #'--cover-package=',
+    #'--exe',  # If your tests need this to be found/run, check they py files are not chmodded +x
 ]
 
 # Celery configuration
@@ -40,7 +37,7 @@ INSTALLED_APPS = [
     'django.contrib.messages',
     'django.contrib.staticfiles',
     'django.contrib.humanize',
-    'djcelery',
+    'django_celery_beat',
     'bootstrapform',
     'authentication',
     'services',
@@ -52,6 +49,7 @@ INSTALLED_APPS = [
     'optimer',
     'corputils',
     'fleetactivitytracking',
+    'fleetup',
     'notifications',
     'esi',
     'permissions_tool',
@@ -400,12 +398,12 @@ TEAMSPEAK3_PUBLIC_URL = os.environ.get('AA_TEAMSPEAK3_PUBLIC_URL', 'example.com'
 # DISCORD_CALLBACK_URL - oauth callback url
 # DISCORD_SYNC_NAMES - enable to force discord nicknames to be set to eve char name (bot needs Manage Nicknames permission)
 ######################################
-DISCORD_GUILD_ID = os.environ.get('AA_DISCORD_GUILD_ID', '')
-DISCORD_BOT_TOKEN = os.environ.get('AA_DISCORD_BOT_TOKEN', '')
-DISCORD_INVITE_CODE = os.environ.get('AA_DISCORD_INVITE_CODE', '')
-DISCORD_APP_ID = os.environ.get('AA_DISCORD_APP_ID', '')
-DISCORD_APP_SECRET = os.environ.get('AA_DISCORD_APP_SECRET', '')
-DISCORD_CALLBACK_URL = os.environ.get('AA_DISCORD_CALLBACK_URL', 'http://example.com/discord_callback')
+DISCORD_GUILD_ID = os.environ.get('AA_DISCORD_GUILD_ID', '0118999')
+DISCORD_BOT_TOKEN = os.environ.get('AA_DISCORD_BOT_TOKEN', 'bottoken')
+DISCORD_INVITE_CODE = os.environ.get('AA_DISCORD_INVITE_CODE', 'invitecode')
+DISCORD_APP_ID = os.environ.get('AA_DISCORD_APP_ID', 'appid')
+DISCORD_APP_SECRET = os.environ.get('AA_DISCORD_APP_SECRET', 'secret')
+DISCORD_CALLBACK_URL = os.environ.get('AA_DISCORD_CALLBACK_URL', 'http://example.com/discord/callback')
 DISCORD_SYNC_NAMES = 'True' == os.environ.get('AA_DISCORD_SYNC_NAMES', 'False')
 
 ######################################

alliance_auth/urls.py

@@ -11,7 +11,7 @@ import groupmanagement.views
 import optimer.views
 import timerboard.views
 import fleetactivitytracking.views
-import fleetup.views
+import fleetup.urls
 import srp.views
 import notifications.views
 import hrapplications.views
@@ -55,11 +55,14 @@ urlpatterns = [
         name='auth_srp_fleet_mark_completed'),
     url(r'^srp_fleet_mark_uncompleted/(\w+)', srp.views.srp_fleet_mark_uncompleted,
         name='auth_srp_fleet_mark_uncompleted'),
-    url(r'^srp_request_remove/(\w+)', srp.views.srp_request_remove,
+    url(r'^srp_request_remove/', srp.views.srp_request_remove,
         name="auth_srp_request_remove"),
-    url(r'srp_request_approve/(\w+)', srp.views.srp_request_approve,
+    url(r'srp_request_approve/', srp.views.srp_request_approve,
         name='auth_srp_request_approve'),
-    url(r'srp_request_reject/(\w+)', srp.views.srp_request_reject, name='auth_srp_request_reject'),
+    url(r'srp_request_reject/', srp.views.srp_request_reject, 
+        name='auth_srp_request_reject'),
+    url(_(r'srp_request_amount_update/(\w+)'), srp.views.srp_request_update_amount,
+        name="auth_srp_request_update_amount"),
 
     # Notifications
     url(r'^remove_notifications/(\w+)/$', notifications.views.remove_notification, name='auth_remove_notification'),
@@ -72,12 +75,7 @@ urlpatterns = [
 urlpatterns += i18n_patterns(
 
     # Fleetup
-    url(r'^fleetup/$', fleetup.views.fleetup_view, name='auth_fleetup_view'),
-    url(r'^fleetup/fittings/$', fleetup.views.fleetup_fittings, name='auth_fleetup_fittings'),
-    url(r'^fleetup/fittings/(?P<fittingnumber>[0-9]+)/$', fleetup.views.fleetup_fitting, name='auth_fleetup_fitting'),
-    url(r'^fleetup/doctrines/$', fleetup.views.fleetup_doctrines, name='auth_fleetup_doctrines'),
-    url(r'^fleetup/characters/$', fleetup.views.fleetup_characters, name='auth_fleetup_characters'),
-    url(r'^fleetup/doctrines/(?P<doctrinenumber>[0-9]+)/$', fleetup.views.fleetup_doctrine, name='auth_fleetup_doctrine'),
+    url(r'^fleetup/', include(fleetup.urls.urlpatterns)),
 
     # Authentication
     url(_(r'^login_user/'), authentication.views.login_user, name='auth_login_user'),
@@ -179,8 +177,6 @@ urlpatterns += i18n_patterns(
     url(_(r'^srp_fleet_add_view/$'), srp.views.srp_fleet_add_view, name='auth_srp_fleet_add_view'),
     url(_(r'^srp_fleet_edit/(\w+)$'), srp.views.srp_fleet_edit_view, name='auth_srp_fleet_edit_view'),
     url(_(r'^srp_request/(\w+)'), srp.views.srp_request_view, name='auth_srp_request_view'),
-    url(_(r'srp_request_amount_update/(\w+)'), srp.views.srp_request_update_amount_view,
-        name="auth_srp_request_update_amount_view"),
 
     # Tools
     url(_(r'^tool/fleet_formatter_tool/$'), services.views.fleet_formatter_view,
@@ -193,6 +189,9 @@ urlpatterns += i18n_patterns(
     # FleetActivityTracking (FAT)
     url(r'^fat/$', fleetactivitytracking.views.fatlink_view, name='auth_fatlink_view'),
     url(r'^fat/statistics/$', fleetactivitytracking.views.fatlink_statistics_view, name='auth_fatlink_view_statistics'),
+    url(r'^fat/statistics/corp/(\w+)$', fleetactivitytracking.views.fatlink_statistics_corp_view, name='auth_fatlink_view_statistics_corp'),
+    url(r'^fat/statistics/corp/(?P<corpid>\w+)/(?P<year>[0-9]+)/(?P<month>[0-9]+)/', fleetactivitytracking.views.fatlink_statistics_corp_view,
+        name='auth_fatlink_view_statistics_corp_month'),
     url(r'^fat/statistics/(?P<year>[0-9]+)/(?P<month>[0-9]+)/$', fleetactivitytracking.views.fatlink_statistics_view,
         name='auth_fatlink_view_statistics_month'),
     url(r'^fat/user/statistics/$', fleetactivitytracking.views.fatlink_personal_statistics_view,

authentication/forms.py

@@ -2,6 +2,7 @@ from __future__ import unicode_literals
 from django import forms
 from django.utils.translation import ugettext_lazy as _
 from django.contrib.auth.models import User
+from django.conf import settings
 import re
 
 
@@ -9,6 +10,10 @@ class LoginForm(forms.Form):
     username = forms.CharField(label=_('Username'), max_length=32, required=True)
     password = forms.CharField(label=_('Password'), widget=forms.PasswordInput())
 
+    if getattr(settings, 'CAPTCHA_ENABLED', False):
+        from captcha.fields import ReCaptchaField
+        captcha = ReCaptchaField()
+
 
 class RegistrationForm(forms.Form):
     username = forms.CharField(label=_('Username'), max_length=30, required=True)
@@ -17,16 +22,16 @@ class RegistrationForm(forms.Form):
     email = forms.CharField(label=_('Email'), max_length=254, required=True)
     email_again = forms.CharField(label=_('Email Again'), max_length=254, required=True)
 
+    if getattr(settings, 'CAPTCHA_ENABLED', False):
+        from captcha.fields import ReCaptchaField
+        captcha = ReCaptchaField()
+
     def clean(self):
         if ' ' in self.cleaned_data['username']:
             raise forms.ValidationError('Username cannot contain a space')
 
-        # We attempt to get the user object if we succeed we know email as been used
-        try:
-            User.objects.get(email=self.cleaned_data['email'])
+        if User.objects.filter(email=self.cleaned_data['email']).count() >= 1:
             raise forms.ValidationError('Email as already been used')
-        except User.DoesNotExist:
-            pass
 
         if not re.match("^\w+$", self.cleaned_data['username']):
             raise forms.ValidationError('Username contains illegal characters')

corputils/models.py

@@ -11,6 +11,10 @@ from corputils.managers import CorpStatsManager
 from operator import attrgetter
 import json
 import logging
+import os
+
+
+SWAGGER_SPEC_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'swagger.json')
 
 logger = logging.getLogger(__name__)
 
@@ -41,7 +45,7 @@ class CorpStats(models.Model):
 
     def update(self):
         try:
-            c = self.token.get_esi_client(Character='v4', Corporation='v2')
+            c = self.token.get_esi_client(spec_file=SWAGGER_SPEC_PATH)
             assert c.Character.get_characters_character_id(character_id=self.token.character_id).result()[
                        'corporation_id'] == int(self.corp.corporation_id)
             members = c.Corporation.get_corporations_corporation_id_members(
@@ -163,7 +167,10 @@ class CorpStats(models.Model):
 
     def get_member_objects(self, user):
         show_apis = self.show_apis(user)
-        return sorted([CorpStats.MemberObject(id, name, show_apis=show_apis) for id, name in self.members.items()], key=attrgetter('main_user', 'character_name'))
+        member_list = [CorpStats.MemberObject(id, name, show_apis=show_apis) for id, name in self.members.items()]
+        outlist = sorted([m for m in member_list if m.main_user], key=attrgetter('main_user', 'character_name'))
+        outlist = outlist + sorted([m for m in member_list if not m.main_user], key=attrgetter('character_name'))
+        return outlist
 
     def can_update(self, user):
         return user.is_superuser or user == self.token.user

corputils/tasks.py

@@ -1,15 +1,14 @@
 from corputils.models import CorpStats
-from celery.task import task, periodic_task
-from celery.task.schedules import crontab
+from alliance_auth.celeryapp import app
 
 
-@task
+@app.task
 def update_corpstats(pk):
     cs = CorpStats.objects.get(pk=pk)
     cs.update()
 
 
-@periodic_task(run_every=crontab(minute=0, hour="*/6"))
+@app.task
 def update_all_corpstats():
     for cs in CorpStats.objects.all():
         update_corpstats.delay(cs.pk)

corputils/templates/corputils/corpstats.html

@@ -25,7 +25,7 @@
                 </div>
                 <div class="row">
                     <div class="col-lg-12">
-                        <b>{% trans "API Index: " %}</b> {{ corpstats.total_users }} Main Characters
+                        <b>{% trans "API Index: " %}</b> {{ corpstats.total_users }} Main Character{{ corpstats.total_users|pluralize }}
                         <div class="progress">
                             <div class="progress-bar" role="progressbar" aria-valuenow="{{ corpstats.registered_members }}" aria-valuemin="0" aria-valuemax="{{ corpstats.total_members }}" style="width: {% widthratio corpstats.registered_members corpstats.total_members 100 %}%;">
                                 {{ corpstats.registered_members }}/{{ corpstats.total_members }}

corputils/views.py

@@ -12,6 +12,9 @@ from eveonline.models import EveCharacter, EveCorporationInfo
 from corputils.models import CorpStats
 from esi.decorators import token_required
 from bravado.exception import HTTPError
+import os
+
+SWAGGER_SPEC_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'swagger.json')
 
 MEMBERS_PER_PAGE = int(getattr(settings, 'CORPSTATS_MEMBERS_PER_PAGE', 20))
 
@@ -41,12 +44,14 @@ def corpstats_add(request, token):
         if EveCharacter.objects.filter(character_id=token.character_id).exists():
             corp_id = EveCharacter.objects.get(character_id=token.character_id).corporation_id
         else:
-            corp_id = \
-                token.get_esi_client(Character='v4').Character.get_characters_character_id(character_id=token.character_id).result()[
-                    'corporation_id']
+            corp_id = token.get_esi_client(spec_file=SWAGGER_SPEC_PATH).Character.get_characters_character_id(
+                character_id=token.character_id).result()['corporation_id']
         corp = EveCorporationInfo.objects.get(corporation_id=corp_id)
         cs = CorpStats.objects.create(token=token, corp=corp)
-        cs.update()
+        try:
+            cs.update()
+        except HTTPError as e:
+            messages.error(request, str(e))
         assert cs.pk  # ensure update was successful
         if CorpStats.objects.filter(pk=cs.pk).visible_to(request.user).exists():
             return redirect('corputils:view_corp', corp_id=corp.corporation_id)

docs/features/corpstats.md

@@ -6,17 +6,17 @@ This module is used to check the registration status of corp members and to dete
 
 Upon initial install, nothing will be visible. For every corp, a model will have to be created before data can be viewed.
 
-![nothing is visible](http://i.imgur.com/va3DyT6.png)
+![nothing is visible](/_static/images/features/corpstats/blank_header.png)
 
 If you are a superuser, the add button will be immediate visible to you. If not, your user account requires the `add_corpstats` permission.
 
 Corp Stats requires an EVE SSO token to access data from the EVE Swagger Interface. Upon pressing the Add button, you will be prompted to authenticated. Please select the character who is in the corp you want data for.
 
-![authorize from the EVE site](http://i.imgur.com/OnyoOAZ.png)
+![authorize from the EVE site](/_static/images/features/corpstats/eve_sso_authorization.png)
 
 You will return to auth where you are asked to select a token with the green arrow button. If you want to use a different character, press the `LOG IN with EVE Online` button.
 
-![select an SSO token to create with](http://i.imgur.com/KdA0XH0.png)
+![select an SSO token to create with](/_static/images/features/corpstats/select_sso_token.png)
 
 If this works (and you have permission to view the Corp Stats you just created) you'll be returned to a view of the Corp Stats.
 If it fails an error message will be displayed.
@@ -25,7 +25,7 @@ If it fails an error message will be displayed.
 
 ### Navigation Bar
 
-![navigation bar](http://i.imgur.com/2l9gbml.png)
+![navigation bar](/_static/images/features/corpstats/navbar.png)
 
 This bar contains a dropdown menu of all available corps. If the user has the `add_corpstats` permission, a button to add a Corp Stats will be shown.
 
@@ -33,13 +33,13 @@ On the right of this bar is a search field. Press enter to search. It checks all
 
 ### API Index
 
-![API Index](http://i.imgur.com/P1U2WJ2.png)
+![API Index](/_static/images/features/corpstats/api_index.png)
 
 This is a visual indication of the number of registered characters.
 
 ### Last Update
 
-![last update and update button](http://i.imgur.com/yHbueGK.png)
+![last update and update button](/_static/images/features/corpstats/last_update.png)
 
 Corp Stats do not automatically update. They update once upon creation for initial data, and whenever someone presses the update button.
 
@@ -47,7 +47,7 @@ Only superusers and the creator of the Corp Stat can update it.
 
 ### Member List
 
-![member list](http://i.imgur.com/udEVoSh.png)
+![member list](/_static/images/features/corpstats/member_list.png)
 
 The list contains all characters in the corp. Red backgrounds means they are not registered in auth. If registered, and the user has the required permission to view APIs, a link to JackKnife will be present.
 A link to zKillboard is present for all characters.
@@ -55,11 +55,11 @@ If registered, the character will also have a main character, main corporation,
 
 This view is paginated: use the navigation arrows to view more pages (sorted alphabetically by character name), or search for a specific character.
 
-![pagination buttons](http://i.imgur.com/otcPGsU.png)
+![pagination buttons](/_static/images/features/corpstats/pagination.png)
 
 ## Search View
 
-![search results](http://i.imgur.com/7wf0Q2C.png)
+![search results](/_static/images/features/corpstats/search_view.png)
 
 This view is essentially the same as the Corp Stats page, but not specific to a single corp.
 The search query is visible in the search box.

docs/installation/auth/apache.md

@@ -5,6 +5,8 @@ AllianceAuth gets served using a Web Server Gateway Interface (WSGI) script. Thi
 
 In the interest of ~~laziness~~ time-efficiency, scroll down for example configs. Use these, changing the ServerName to your domain name.
 
+If you're using a small VPS to host services with very limited memory resources, consider using NGINX with [Gunicorn](gunicorn.md). Even if you would like to use Apache, Gunicorn may give you lower memory usage over mod_wsgi.
+
 ### Required Parameters for AllianceAuth Core
 
 The AllianceAuth core requires the following parameters to be set:
@@ -52,6 +54,31 @@ You can supply your own SSL certificates if you so desire. The alternative is ru
 
 ## Sample Config Files
 
+### Minimally functional config
+
+```
+<VirtualHost *:80>
+        ServerName example.com
+        ServerAdmin webmaster@localhost
+ 
+        DocumentRoot /var/www
+ 
+        WSGIDaemonProcess allianceauth python-path=/home/allianceserver/allianceauth
+        WSGIProcessGroup allianceauth
+        WSGIScriptAlias / /home/allianceserver/allianceauth/alliance_auth/wsgi.py
+ 
+        Alias /static/ /home/allianceserver/allianceauth/static/
+ 
+        <Directory /home/allianceserver/allianceauth/>
+                Require all granted
+        </Directory>
+
+        <Directory /var/www/>
+                Require all granted
+        </Directory>        
+</VirtualHost>
+```
+
 ### Own SSL Cert
  - Apache 2.4 or newer:
    - [000-default.conf](http://pastebin.com/3LLzyNmV)

docs/installation/auth/centos.md

@@ -57,7 +57,7 @@ Enter the folder by issuing `cd allianceauth`
 
 Ensure you're on the latest version with the following:
 
-    git tag | sort -n | tail -1 | xargs git checkout
+    git checkout v1.15.7
 
 Python package dependencies can be installed from the requirements file:
 

docs/installation/auth/dependencies.md

@@ -25,6 +25,17 @@ Required for displaying web content
     apache2 libapache2-mod-php5 libapache2-mod-wsgi
 
 ### PHP
+
+```eval_rst
+.. note::
+    If you are not planing to install either phpBB, smf, evernus alliance market, etc do not install these modules.
+```
+
+```eval_rst
+.. important::
+    If you are not planing to use SeAT; php7.0 is the minimum required.
+```
+
 Required for phpBB, smf, evernus alliance market, etc
 
     php5 php5-gd php5-mysqlnd php5-curl php5-gd php5-intl php-pear php5-imagick php5-imap php5-mcrypt php5-memcache php5-ming php5-ps php5-pspell php5-recode php5-snmp php5-sqlite php5-tidy php5-xmlrpc php5-xsl

docs/installation/auth/gunicorn.md

@@ -0,0 +1,121 @@
+# Gunicorn
+
+[Gunicorn](http://gunicorn.org) is a Python WSGI HTTP Server for UNIX. The Gunicorn server is light on server resources, and fairly speedy.
+
+If you find Apache's `mod_wsgi` to be a headache or want to use NGINX (or some other webserver), then Gunicorn could be for you. There are a number of other WSGI server options out there and this documentation should be enough for you to piece together how to get them working with your environment.
+
+Check out the full [Gunicorn docs](http://docs.gunicorn.org/en/latest/index.html).
+
+## Setting up Gunicorn
+
+```eval_rst
+.. note::
+    If you're using a virtual environment (and I would encourage you to do so when hosting Alliance Auth), activate it now. `source /path/to/venv/bin/activate`.
+```
+
+Install Gunicorn using pip, `pip install gunicorn`.
+
+In your `allianceauth` base directory, try running `gunicorn --bind 0.0.0.0:8000 alliance_auth.wsgi`. You should be able to browse to http://yourserver:8000 and see your Alliance Auth installation running. Images and styling will be missing, but dont worry, your web server will provide them.
+
+Once you validate its running, you can kill the process with Ctrl+C and continue.
+
+## Running Gunicorn with Supervisor
+
+You should use [Supervisor](supervisor.md) to keep all of Alliance Auth components running (instead of using screen). You don't _have to_ but we will be using it to start and run Gunicorn so you might as well.
+
+### Sample Supervisor config
+You'll want to edit `/etc/supervisor/conf.d/aauth_gunicorn.conf` (or whatever you want to call the config file)
+```
+[program:aauth-gunicorn]
+user = www-data
+directory=/home/allianceserver/allianceauth/
+command=gunicorn alliance_auth.wsgi --workers=3 --timeout 120
+autostart=true
+autorestart=true
+stopsignal=INT
+```
+
+- `[program:aauth-gunicorn]` - Change aauth-gunicorn to whatever you wish to call your process in Supervisor.
+- `user = www-data` - Change to whatever user you wish Gunicorn to run as. You could even set this as allianceserver if you wished. I'll leave the question security of that up to you.
+- `directory=/home/allianceserver/allianceauth/` - Needs to be the path to your Alliance Auth install.
+- `command=gunicorn alliance_auth.wsgi --workers=3 --timeout 120` - Running Gunicorn and the options to launch with. This is where you have some decisions to make, we'll continue below.
+
+#### Gunicorn Arguments
+
+See the [Commonly Used Arguments](http://docs.gunicorn.org/en/latest/run.html#commonly-used-arguments) or [Full list of settings](http://docs.gunicorn.org/en/stable/settings.html) for more information.
+
+##### Where to bind Gunicorn to?
+What address are you going to use to reference it? By default, without a bind parameter, Gunicorn will bind to `127.0.0.1:8000`. This might be fine for your application. If it clashes with another application running on that port you will need to change it. I would suggest using UNIX sockets too, if you can.
+ 
+For UNIX sockets add `--bind=unix:/run/allianceauth.sock` (or to a path you wish to use). Remember that your web server will need to be able to access this socket file.
+ 
+For a TCP address add `--bind=127.0.0.1:8001` (or to the address/port you wish to use, but I would strongly advise against binding it to an external address).
+ 
+Whatever you decide to use, remember it because we'll need it when configuring your webserver.
+
+##### Number of workers
+By default Gunicorn will spawn only one worker. The number you set this to will depend on your own server environment, how many visitors you have etc. Gunicorn suggests between 2-4 workers per core. Really you could probably get away with 2-4 in total for most installs.
+
+Change it by adding `--workers=2` to the command.
+
+##### Running with a virtual environment
+If you're running with a virtual environment, you'll need to add the path to the `command=` config line.
+
+e.g. `command=/path/to/venv/bin/gunicorn alliance_auth.wsgi`
+
+### Starting via Supervisor
+
+Once you have your configuration all sorted, you will need to reload your supervisor config `sudo service supervisor reload` and then you can start the Gunicorn server via `sudo supervisorctl start aauth-gunicorn` (or whatever you renamed it to). You should see something like the following `aauth-gunicorn: started`. If you get some other message, you'll need to consult the Supervisor log files, usually found in `/var/log/supervisor/`.
+
+
+## Configuring your webserver
+
+### NGINX
+To your server config add:
+
+```
+location / {
+    proxy_pass              http://127.0.0.1:8000;
+    proxy_read_timeout      90;
+    proxy_redirect          http://127.0.0.1:8000/ http://$host/;
+    proxy_set_header        Host $host;
+    proxy_set_header        X-Forwarded-For $proxy_add_x_forwarded_for;
+    proxy_set_header        X-Real-IP $remote_addr;
+    proxy_set_header        X-Forwarded-Proto $scheme;
+}
+```
+
+Set `proxy_pass` and `proxy_redirect` to the address you set under `--bind=`. Set the second part of `proxy_redirect` to the URL you're hosting services on. Tell NGINX to reload your config, job done. Enjoy your lower memory usage and better performance!
+
+If PHP is stopping you moving to NGINX, check out php-fpm as a way to run your PHP applications.
+
+### Apache
+If you were using mod_wsgi before, make a backup of your old config first and then strip out all of the mod_wsgi config from your Apache VirtualHost first config.
+
+Your config will need something along these lines:
+```
+ProxyPreserveHost On
+<Location />
+    SSLRequireSSL
+    ProxyPass http://127.0.0.1:8000/
+    ProxyPassReverse http://127.0.0.1:8000/
+    RequestHeader set X-FORWARDED-PROTOCOL ssl
+    RequestHeader set X-FORWARDED-SSL on
+</Location>
+```
+
+Set `ProxyPass` and `ProxyPassReverse` addresses to your `--bind=` address set earlier. 
+
+You will need to enable some Apache mods. `sudo a2enmod http_proxy` should take care of the dependencies.
+
+Restart Apache and you should be done.
+
+### Other web servers
+
+Any web server capable of proxy passing should be able to sit in front of Gunicorn. Consult their documentation armed with your `--bind=` address and you should be able to find how to do it relatively easy.
+
+
+## Restarting Gunicorn
+In the past when you made changes you restarted the entire Apache server. This is no longer required. When you update or make configuration changes that ask you to restart Apache, instead you can just restart Gunicorn:
+
+`sudo supervisorctl restart aauth-gunicorn`, or the service name you chose for it.

docs/installation/auth/index.md

@@ -7,7 +7,9 @@
     ubuntu
     centos
     settings
+    nginx
     apache
+    gunicorn
     cloudflare
     supervisor
     quickstart

docs/installation/auth/nginx.md

@@ -0,0 +1,93 @@
+# NGINX
+
+## Overivew
+
+Nginx (engine x) is a HTTP server known for its high performance, stability, simple configuration, and low resource consumption. Unlike traditional servers (i.e. Apache), Nginx doesn't rely on threads to serve requests, rather using an asynchronous event driven approach which permits predictable resource usage and performance under load.
+
+If you're trying to cram Alliance Auth into a very small VPS of say, 1-2GB or less, then Nginx will be considerably friendlier to your resources compared to Apache.
+
+You can read more about NGINX on the [NGINX wiki](https://www.nginx.com/resources/wiki/).
+
+## Coming from Apache
+
+If you're converting from Apache, here are some things to consider.
+
+Nginx is lightweight for a reason. It doesn't try to do everything internally and instead concentrates on just being a good HTTP server. This means that, unlike Apache, it wont automatically run PHP scripts via mod_php and doesn't have an internal WSGI server like mod_wsgi. That doesn't mean that it can't, just that it relies on external processes to run these instead. This might be good or bad depending on your outlook. It's good because it allows you to segment your applications, restarting Alliance Auth wont impact your PHP applications. On the other hand it means more config and more management of services. For some people it will be worth it, for others losing the centralised nature of Apache may not be worth it.
+
+```eval_rst
++-----------+----------------------------------------+
+| Apache    | Nginx Replacement                      | 
++===========+========================================+
+| mod_php   | php5-fpm or php7-fpm (PHP FastCGI)     | 
++-----------+----------------------------------------+
+| mod_wsgi  | Gunicorn or other external WSGI server |
++-----------+----------------------------------------+
+
+```
+
+Your .htaccess files wont work. Nginx has a separate way of managing access to folders via the server config. Everything you can do with htaccess files you can do with Nginx config. [Read more on the Nginx wiki](https://www.nginx.com/resources/wiki/start/topics/examples/likeapache-htaccess/)
+
+## Setting up Nginx
+
+Install Nginx via your preferred package manager or other method. If you need help just search, there are plenty of guides on installing Nginx out there.
+
+You will need to have [Gunicorn](gunicorn.md) or some other WSGI server setup for hosting Alliance Auth.
+
+Create a config file in `/etc/nginx/sites-available` call it `alliance-auth.conf` or whatever your preferred name is and copy the basic config in. Make whatever changes you feel are necessary.
+
+Create a symbolic link to enable the site `sudo ln -s /etc/nginx/sites-available/alliance-auth.conf /etc/nginx/sites-enabled/` and then reload Nginx for the config to take effect, `sudo service nginx reload` for Ubuntu.
+
+### Basic config
+
+```
+server {
+    listen 80;
+    server_name example.com;
+
+    location = /favicon.ico { access_log off; log_not_found off; }
+
+    location /static/ {
+        alias /home/allianceserver/allianceauth/static/;
+        autoindex off;
+    }
+
+    # Gunicorn config goes below
+    location / {
+        include proxy_params;
+        proxy_pass http://127.0.0.1:8000;
+    }
+}
+```
+
+#### Adding TLS/SSL
+
+With [Let's Encrypt](https://letsencrypt.org/) offering free SSL certificates, there's no good reason to not run HTTPS anymore.
+
+Your config will need a few additions once you've got your certificate.
+
+```
+    listen 443 ssl http2; # Replace listen 80; with this
+
+    ssl_certificate           /path/to/your/cert.crt;
+    ssl_certificate_key       /path/to/your/cert.key;
+
+    ssl on;
+    ssl_session_cache  builtin:1000  shared:SSL:10m;
+    ssl_protocols  TLSv1 TLSv1.1 TLSv1.2;
+    ssl_ciphers EECDH+ECDSA+AESGCM:EECDH+aRSA+AESGCM:EECDH+ECDSA+SHA384:EECDH+ECDSA+SHA256:EECDH+aRSA+SHA384:EECDH+aRSA+SHA256:EECDH+aRSA+RC4:EECDH:EDH+aRSA:RC4:!aNULL:!eNULL:!LOW:!3DES:!MD5:!EXP:!PSK:!SRP:!DSS;
+    ssl_prefer_server_ciphers on;
+```
+
+If you want to redirect all your non-SSL visitors to your secure site, below your main configs `server` block, add the following:
+
+```
+server {
+    listen 80;
+    server_name example.com;
+
+    # Redirect all HTTP requests to HTTPS with a 301 Moved Permanently response.
+    return 301 https://$host$request_uri;
+}
+```
+
+If you have trouble with the `ssl_ciphers` listed here or some other part of the SSL config, try getting the values from [Mozilla's SSL Config Generator](https://mozilla.github.io/server-side-tls/ssl-config-generator/).

docs/installation/auth/quickstart.md

@@ -10,5 +10,5 @@ The big goal of AllianceAuth is the automation of group membership, so we’ll n
 
 To start the background processes to sync groups and check api keys, issue these commands:
 
-    screen -dm bash -c 'python manage.py celeryd'
-    screen -dm bash -c 'python manage.py celerybeat'
+    screen -dm bash -c 'celery -A alliance_auth worker'
+    screen -dm bash -c 'celery -A alliance_auth beat'

docs/installation/auth/settings.md

@@ -68,6 +68,7 @@ If using Openfire, the following need to be set in accordance with the [install
  - [BROADCAST_USER](#broadcast-user)
  - [BROADCAST_USER_PASSWORD](#broadcast-user-password)
  - [BROADCAST_SERVICE_NAME](#broadcast-service-name)
+ - [BROADCAST_IGNORE_INVALID_CERT](#broadcast-ignore-invalid-cert)
 
 ### Mumble
 If using Mumble, the following needs to be set to the address of the mumble server:
@@ -136,6 +137,13 @@ Fittings and operations can be imported from Fleet-Up. Define the following to d
  - [FLEETUP_API_ID](#fleetup-api-id)
  - [FLEETUP_GROUP_ID](#fleetup-group-id)
 
+### CAPTCHA
+To help prevent bots from registering and brute forcing the login. Get the reCaptcha keys from [here](https://www.google.com/recaptcha/intro/index.html)
+ - [CAPTCHA_ENABLED](#captcha_enabled)
+ - [RECAPTCHA_PUBLIC_KEY](#recaptcha_public_key)
+ - [RECAPTCHA_PRIVATE_KEY](#recaptcha_private_key)
+ - [NOCAPTCHA](#nocaptcha)
+
 # Description of Settings
 ## Django
 ### SECRET_KEY
@@ -150,6 +158,14 @@ List of databases available. Contains the Django database, and may include servi
 Friendly name of the local language.
 ### TIME_ZONE
 Friendly name of the local timezone.
+### CAPTCHA_ENABLED
+Enable Google reCaptcha
+### RECAPTCHA_PUBLIC_KEY
+Google reCaptcha public key
+### RECAPTCHA_PRIVATE_KEY
+Google reCaptcha private key
+### NOCAPTCHA
+Enable New No Captcha reCaptcha
 ### STATIC_URL
 Absolute URL to serve static files from.
 ### STATIC_ROOT
@@ -199,52 +215,6 @@ If `True`, add members to groups with their alliance name, prefixed with `Allian
 If `True`, add blues to groups with their corp name, prefixed with `Corp_`
 ### BLUE_ALLIANCE_GROUPS
 If `True`, add blues to groups with their alliance name, prefixed with `Alliance_`
-## Alliance Service Setup
-### ENABLE_AUTH_FORUM
-Allow members of the owning corp or alliance to generate accounts on a Phpbb3 install.
-### ENABLE_AUTH_JABBER
-Allow members of the owning corp or alliance to generate accounts on an Openfire install.
-### ENABLE_AUTH_MUMBLE
-Allow members of the owning corp or alliance to generate accounts on a Mumble install.
-### ENABLE_AUTH_IPBOARD
-Allow members of the owning corp or alliance to generate accounts on an IPBoard install.
-### ENABLE_AUTH_TEAMSPEAK3
-Allow members of the owning corp or alliance to generate accounts on a Teamspeak3 install.
-### ENABLE_AUTH_DISCORD
-Allow members of the owning corp or alliance to link accounts to a Discord server.
-### ENABLE_AUTH_DISCOURSE
-Allow members of the owning corp or alliance to generate accounts on a Discourse install
-### ENABLE_AUTH_IPS4
-Allow members of the owning corp or alliance to generate accounts on a IPSuite4 install.
-### ENABLE_AUTH_SMF
-Allow members of the owning corp or alliance to generate accounts on a SMF install.
-### ENABLE_AUTH_MARKET
-Allow members of the owning corp or alliance to generate accounts on an alliance market install.
-### ENABLE_AUTH_XENFORO
-Allow members of the owning corp or alliance to generate accounts on a XenForo install.
-## Blue Service Setup
-### ENABLE_BLUE_FORUM
-Allow blues of the owning corp or alliance to generate accounts on a Phpbb3 install.
-### ENABLE_BLUE_JABBER
-Allow blues of the owning corp or alliance to generate accounts on an Openfire install.
-### ENABLE_BLUE_MUMBLE
-Allow blues of the owning corp or alliance to generate accounts on a Mumble install.
-### ENABLE_BLUE_IPBOARD
-Allow blues of the owning corp or alliance to generate accounts on an IPBoard install.
-### ENABLE_BLUE_TEAMSPEAK3
-Allow blues of the owning corp or alliance to generate accounts on a Teamspeak3 install.
-### ENABLE_BLUE_DISCORD
-Allow blues of the owning corp or alliance to link accounts to a Discord server.
-### ENABLE_BLUE_DISCOURSE
-Allow blues of the owning corp or alliance to generate accounts on a Discourse install.
-### ENABLE_BLUE_IPS4
-Allow blues of the owning corp or alliance to generate accounts on an IPSuite4 install.
-### ENABLE_BLUE_SMF
-Allow blues of the owning corp or alliance to generate accounts on a SMF install.
-### ENABLE_BLUE_MARKET
-Allow blues of the owning corp or alliance to generate accounts on an alliance market install.
-### ENABLE_BLUE_XENFORO
-Allow blues of the owning corp or alliance to generate accounts on a XenForo install.
 ## Tenant Configuration
 Characters of any corp or alliance with their ID here will be treated as a member.
 ### CORP_IDS

docs/installation/auth/supervisor.md

@@ -15,14 +15,14 @@ Ubuntu:
 CentOS:
 
     sudo yum install supervisor
-    sudo systemctl enable supervisor.service
-    sudo systemctl start supervisor.service
+    sudo systemctl enable supervisord.service
+    sudo systemctl start supervisord.service
 
 ## Configuration
 
-Auth provides example config files for the celery workers (celeryd), the periodic task scheduler (celerybeat), and the mumble authenticator. All of these are available in `thirdparty/Supervisor`.
+Auth provides example config files for the celery workers, the periodic task scheduler (celery beat), and the mumble authenticator. All of these are available in `thirdparty/Supervisor`.
 
-For most users, all you have to do is copy the config files to `/etc/supervisor/conf.d` then restart the service. Copy `auth-celerybeat.conf` and `auth-celeryd.conf` for the celery workers, and `auth-mumble.conf` for the mumble authenticator. For all three just use a wildcard:
+For most users, all you have to do is copy the config files to `/etc/supervisor/conf.d` then restart the service. Copy `auth.conf` for the celery workers, and `auth-mumble.conf` for the mumble authenticator. For all three just use a wildcard:
 
     sudo cp thirdparty/Supervisor/* /etc/supervisor/conf.d
 
@@ -41,15 +41,33 @@ To ensure the processes are working, check their status:
     sudo supervisorctl status
 
 Processes will be `STARTING`, `RUNNING`, or `ERROR`. If an error has occurred, check their log files:
- - celeryd: `log/worker.log`
- - celerybeat: `log/beat.log`
+ - celery workers: `log/worker.log`
+ - celery beat: `log/beat.log`
  - authenticator: `log/authenticator.log`
+ 
+## Restarting Processes
+
+To restart the celery group:
+
+    sudo supervisorctl restart auth:*
+
+To restart just celerybeat:
+
+    sudo supervisorctl restart auth:celerybeat
+
+To restart just celeryd:
+
+    sudo supervisorctl restart auth:celeryd
+
+To restart just mumble authenticator:
+
+    sudo supervisorctl restart auth-mumble
 
 ## Customizing Config Files
 
 The only real customization needed is if running in a virtual environment. The python path will have to be changed in order to start in the venv.
 
-Edit the config files and find the line saying `command`. Replace `python` with `/path/to/venv/python`. This can be relative to the `directory` specified in the config file.
+Edit the config files and find the line saying `command`. Replace `python` with `/path/to/venv/bin/python`. For Celery replace `celery` with `/path/to/venv/bin/celery`. This can be relative to the `directory` specified in the config file.
 
 Note that for config changes to be loaded, the supervisor service must be restarted.
 
@@ -60,4 +78,4 @@ Most often this is caused by a permissions issue on the allianceauth directory (
 
 ### Workers are using old settings
 
-Every time the codebase is updated or settings file changed, workers will have to be restarted. Easiest way is to restart the supervisor service (see configuration above for commands)
+Every time the codebase is updated or settings file changed, workers will have to be restarted. Easiest way is to restart the supervisor service (see configuration above for commands)

docs/installation/auth/ubuntu.md

@@ -46,7 +46,7 @@ Enter the folder by issuing `cd allianceauth`
 
 Ensure you're on the latest version with the following:
 
-    git tag | sort -n | tail -1 | xargs git checkout
+    git checkout v1.15.7
 
 Python package dependencies can be installed from the requirements file:
 

docs/installation/services/discord.md

@@ -21,13 +21,6 @@ with a server ID of `120631096835571712`
 
 Update settings.py, inputting the server ID as `DISCORD_GUILD_ID`
 
-### Generating an Invite
-Still on the Discord site, in your new server, an invite needs to be generated for users to join. If you with for users to  initially join a different channel than `#general`, create it and follow the steps below, substituting this channel for `#general`.
-
-On the left bar under the Text Channels heading, hover over `#general` on the right site. There are two icons, a box with an arrow and a gear. Press the box, then on the bottom left select Advanced Settings. Set the expiration to never, and no limit on uses. Press generate.
-
-This returns a code that looks like `https://discord.gg/0fmA8MyXV6qt7XAZ`. The part after the last slash, `0fmA8MyXV6qt7XAZ`, is the invite code. Update settings.py, inputting this invite code as `DISCORD_INVITE_CODE`
-
 ### Registering an Application
 
 Navigate to the [Discord Developers site.](https://discordapp.com/developers/applications/me) Press the plus sign to create a new application.

docs/installation/services/discourse.md

@@ -65,6 +65,11 @@ Now build:
     sudo ./launcher bootstrap app
     sudo ./launcher start app
 
+#### Errors:
+in case you run into not enough RAM for the docker bootstraping you might want to consider using `./discourse-setup` command. It will start bootstraping and is going to create the `/containers/app.yml` which you can edit.
+Note: every time you change something in the `app.yml` you must bootstrap again which will take between *2-8 minutes* and is accomplished by `./launcher rebuild app`.
+
+***
 ## Apache config
 
 Discourse must run on its own subdomain - it can't handle routing behind an alias like '/forums'. To do so, make a new apache config:
@@ -81,9 +86,59 @@ And enter the following, changing the port if you used a different number:
 
 Now enable proxies and restart apache:
 
+    sudo a2ensite discourse
     sudo a2enmod proxy_http
     sudo service apache2 reload
 
+### Setting up SSL
+
+It is 2017 and there is no reason why you should not setup a SSL certificate and enforce https. You may want to consider certbot with Let's encrypt: https://www.digitalocean.com/community/tutorials/how-to-secure-apache-with-let-s-encrypt-on-ubuntu-16-04
+
+    sudo certbot --apache -d example.com
+
+now adapt the apache configuration:
+
+    sudo nano /etc/apache2/sites-enabled/discourse.conf
+
+and adapt it followlingly:
+
+    <VirtualHost *:80>
+        ServerName discourse.example.com
+        RewriteEngine on
+        RewriteCond %{SERVER_NAME} =discourse.example.com
+        RewriteRule ^ https://%{SERVER_NAME}%{REQUEST_URI} [END,NE,R=permanent]
+    </VirtualHost>
+
+Then adapt change the ssl-config file:
+
+    sudo nano /etc/apache2/sites-enabled/discourse-le-ssl.conf
+
+and adapt it followlingly:
+
+    <IfModule mod_ssl.c>
+    <VirtualHost *:443>
+      ServerName discourse.example.com
+      ProxyPass / http://127.0.0.1:7890/
+      ProxyPassReverse / http://127.0.0.1:7890/
+      ProxyPreserveHost On
+      RequestHeader set X-FORWARDED-PROTOCOL https
+      RequestHeader set X-FORWARDED-SSL on
+      SSLCertificateFile /etc/letsencrypt/live/discourse.example.com/fullchain.pem
+      SSLCertificateKeyFile /etc/letsencrypt/live/discourse.example.com/privkey.pem
+      Include /etc/letsencrypt/options-ssl-apache.conf
+    </VirtualHost>
+    </IfModule>
+
+make sure that `a2enmod headers` is enabled and run:
+
+      sudo service apache2 restart
+
+Now you are all set-up and can even enforce https in discourse settings.
+
+
+
+
+
 ## Configure API
 
 ### Generate admin account
@@ -108,6 +163,7 @@ Scroll down to the Discourse section and set the following:
  - `DISCOURSE_API_USERNAME`: the username of the admin account you generated the API key with
  - `DISCOURSE_API_KEY`: the key you just generated
 
+***
 ### Configure SSO
 
 Navigate to `discourse.example.com` and log in. Back to the admin site, scroll down to find SSO settings and set the following:
@@ -120,6 +176,7 @@ Save, now change settings.py and add the following:
 
 ### Enable for your members
 
-Set either or both of `ENABLE_AUTH_DISCOURSE` and `ENABLE_BLUE_DISCOURSE` in settings.py for your members to gain access. Save and exit with control+o, enter, control+x.
+Assign discourse permissions for each auth-group that should have access to discourse.
+You might want to setup Read/Write/Delete rights per Auth group in discourse as you can limit which categories shall be accessablie per auth-group.
 
 ## Done

docs/installation/services/jacknife.md

@@ -68,4 +68,4 @@ Enter your database password and press Check. If all the boxes come back green p
 
 ## Update Auth Settings
 
-Edit your aut settings file (`nano ~/allianceauth/alliance_auth/settings.py`) and replace `API_KEY_AUDIT_URL` with either `jacknife.example.com/?usid={api_id}&apik={vcode}` or `example.com/jacknife/?usid={api_id}&apik={vcode}` depending on your apache choice.
+Edit your aut settings file (`nano ~/allianceauth/alliance_auth/settings.py`) and replace `API_KEY_AUDIT_URL` with either `http://jacknife.example.com/?usid={api_id}&apik={vcode}` or `http://example.com/jacknife/?usid={api_id}&apik={vcode}` depending on your apache choice.

docs/installation/services/openfire.md

@@ -82,6 +82,8 @@ Navigate to the `Server` tab, `Server Manager` subtab, and select `System Proper
  - Name: `plugin.broadcast.allowedUsers`
    - Value: `broadcast@example.com`, replacing the domain name with yours
    - Do not encrypt this property value
+   
+If you have troubles getting broadcasts to work, you can try setting the optional (you will need to add it) `BROADCAST_IGNORE_INVALID_CERT` setting to `True`. This will allow invalid certificates to be used when connecting to the Openfire server to send a broadcast.
 
 ### Group Chat
 Channels are available which function like a chat room. Access can be controlled either by password or ACL (not unlike mumble).

docs/installation/services/teamspeak3.md

@@ -84,7 +84,7 @@ To enable advanced permissions, on your client go to the `Tools` menu, `Applicat
 ### TS group models not populating on admin site
 The method which populates these runs every 30 minutes. To populate manually, start a celery shell:
 
-    python manage.py celery shell
+    celery -A alliance_auth shell
 
 And execute the update:
 

docs/maintenance/troubleshooting.md

@@ -19,7 +19,7 @@ Either you need to `sudo` that command, or it's a missing dependency. Check [the
 
 ### I'm getting an error 500 trying to connect to the website on a new install
 
-Read the apache error log: `sudo nano /var/log/apache2/error.log`
+Read the apache error log: `sudo less /var/log/apache2/error.log`. Press Shift+G to go to the end of the file.
 
 If it talks about failing to import something, google its name and install it.
 
@@ -36,13 +36,9 @@ Make sure the background processes are running: `ps aux | grep celery` should re
 If that doesn't do it, try clearing the worker queue. First kill all celery processes as described above, then do the following:
 
     redis-cli FLUSHALL
-    python manage.py celeryd --purge
+    celery -A alliance_auth worker --purge
 
-Press control+C once.
-
-    python manage.py celeryd --discard
-
-Press control+C once.
+Press Control+C once.
 
 Now start celery again with [these background process commands.](../installation/auth/quickstart.md)
 

eveonline/providers.py

@@ -6,6 +6,9 @@ import json
 from bravado.exception import HTTPNotFound, HTTPUnprocessableEntity
 import evelink
 import logging
+import os
+
+SWAGGER_SPEC_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'swagger.json')
 
 logger = logging.getLogger(__name__)
 
@@ -228,7 +231,7 @@ class EveProvider(object):
 @python_2_unicode_compatible
 class EveSwaggerProvider(EveProvider):
     def __init__(self, token=None, adapter=None):
-        self.client = esi_client_factory(token=token, Alliance='v1', Character='v4', Corporation='v2', Universe='v2')
+        self.client = esi_client_factory(token=token, spec_file=SWAGGER_SPEC_PATH)
         self.adapter = adapter or self
 
     def __str__(self):
@@ -244,7 +247,7 @@ class EveSwaggerProvider(EveProvider):
                 data['alliance_name'],
                 data['ticker'],
                 corps,
-                data['executor_corporation_id'],
+                data['executor_corp'],
             )
             return model
         except HTTPNotFound:
@@ -264,7 +267,7 @@ class EveSwaggerProvider(EveProvider):
             )
             return model
         except HTTPNotFound:
-            raise ObjectNotFound(id, 'corporation')
+            raise ObjectNotFound(corp_id, 'corporation')
 
     def get_character(self, character_id):
         try:
@@ -330,9 +333,9 @@ class EveXmlProvider(EveProvider):
             self.adapter,
             id,
             corpinfo['name'],
+            corpinfo['ticker'],
             corpinfo['ceo']['id'],
             corpinfo['members']['current'],
-            corpinfo['ticker'],
             corpinfo['alliance']['id'] if corpinfo['alliance'] else None,
         )
         return model

eveonline/tasks.py

@@ -1,10 +1,8 @@
 from __future__ import unicode_literals
 from django.conf import settings
-from celery.task import periodic_task
 from django.contrib.auth.models import User
 from notifications import notify
 from celery import task
-from celery.task.schedules import crontab
 from authentication.models import AuthServicesInfo
 from eveonline.managers import EveManager
 from eveonline.models import EveApiKeyPair
@@ -17,10 +15,12 @@ from authentication.tasks import set_state
 import logging
 import evelink
 
+from alliance_auth.celeryapp import app
+
 logger = logging.getLogger(__name__)
 
 
-@task
+@app.task
 def refresh_api(api):
     logger.debug('Running update on api key %s' % api.api_id)
     still_valid = True
@@ -36,7 +36,7 @@ def refresh_api(api):
                 EveManager.create_character_obj(c, api.user, api.api_id)
         current_chars = EveCharacter.objects.filter(api_id=api.api_id)
         for c in current_chars:
-            if not int(c.character_id) in [c.id for c in characters]:
+            if not int(c.character_id) in [d.id for d in characters]:
                 logger.info("Character %s no longer found on API ID %s" % (c, api.api_id))
                 c.delete()
     except evelink.api.APIError as e:
@@ -60,7 +60,7 @@ def refresh_api(api):
                    api.api_id, e.required_mask, e.api_mask), level="danger")
         still_valid = False
     except EveApiManager.ApiServerUnreachableError as e:
-        logger.warn("Error updating API %s\n%s" % (api.api_id, str(e)))
+        logger.warning("Error updating API %s\n%s" % (api.api_id, str(e)))
     finally:
         if not still_valid:
             EveManager.delete_characters_by_api_id(api.api_id, api.user.id)
@@ -70,12 +70,13 @@ def refresh_api(api):
                    level="danger")
 
 
-@task
-def refresh_user_apis(user):
+@app.task
+def refresh_user_apis(pk):
+    user = User.objects.get(pk=pk)
     logger.debug('Refreshing all APIs belonging to user %s' % user)
     apis = EveApiKeyPair.objects.filter(user=user)
     for x in apis:
-        refresh_api(x)
+        refresh_api.apply(args=(x,))
     # Check our main character
     auth = AuthServicesInfo.objects.get(user=user)
     if auth.main_char_id:
@@ -91,28 +92,28 @@ def refresh_user_apis(user):
     set_state(user)
 
 
-@periodic_task(run_every=crontab(minute=0, hour="*/3"))
+@app.task
 def run_api_refresh():
     if not EveApiManager.check_if_api_server_online():
         logger.warn("Aborted scheduled API key refresh: API server unreachable")
         return
 
     for u in User.objects.all():
-        refresh_user_apis.delay(u)
+        refresh_user_apis.delay(u.pk)
 
 
-@task
+@app.task
 def update_corp(id, is_blue=None):
     EveManager.update_corporation(id, is_blue=is_blue)
 
 
-@task
+@app.task
 def update_alliance(id, is_blue=None):
     EveManager.update_alliance(id, is_blue=is_blue)
     EveManager.populate_alliance(id)
 
 
-@periodic_task(run_every=crontab(minute=0, hour="*/2"))
+@app.task
 def run_corp_update():
     if not EveApiManager.check_if_api_server_online():
         logger.warn("Aborted updating corp and alliance models: API server unreachable")
@@ -123,7 +124,7 @@ def run_corp_update():
         is_blue = True if corp_id in settings.STR_BLUE_CORP_IDS else False
         try:
             if EveCorporationInfo.objects.filter(corporation_id=corp_id).exists():
-                update_corp(corp_id, is_blue=is_blue)
+                update_corp.apply(args=(corp_id,), kwargs={'is_blue': is_blue})
             else:
                 EveManager.create_corporation(corp_id, is_blue=is_blue)
         except ObjectNotFound:

eveonline/tests.py

@@ -1 +0,0 @@
-# Create your tests here.

eveonline/tests/test_tasks.py

@@ -0,0 +1,125 @@
+from __future__ import unicode_literals
+
+try:
+    # Py3
+    from unittest import mock
+except ImportError:
+    # Py2
+    import mock
+
+from django.test import TestCase
+
+from alliance_auth.tests.auth_utils import AuthUtils
+from eveonline.providers import Character, Alliance, Corporation
+from eveonline.managers import EveManager
+from eveonline import tasks
+from eveonline.models import EveApiKeyPair, EveCharacter
+from services.managers.eve_api_manager import EveApiManager
+
+MODULE_PATH = 'eveonline.tasks'
+
+
+class EveOnlineTasksTestCase(TestCase):
+    def setUp(self):
+        self.user = AuthUtils.create_member('joebloggs')
+        self.api_key = EveApiKeyPair.objects.create(api_id='0118999',
+                                                    api_key='hunter2',
+                                                    user=self.user,
+                                                    sso_verified=True)
+
+    @mock.patch(MODULE_PATH + '.EveApiManager.validate_api')
+    @mock.patch(MODULE_PATH + '.EveManager.get_characters_from_api')
+    def test_refresh_api_characters(self, get_characters_from_api, validate_api):
+        # Arrange
+        provider = mock.MagicMock()
+
+        provider.get_alliance.return_value = Alliance(provider, 22222, 'Test Alliance', 'TEST', [11111], 11111)
+        provider.get_corp.return_value = Corporation(provider, 11111, 'Test Corp', 'HERP', 12345, [12345, 23456], 22222)
+
+        mock_api_data = [
+            Character(provider, 12345, 'testchar1', 11111, 22222),
+            Character(provider, 23456, 'Will beAdded', 11111, 22222)
+        ]
+
+        get_characters_from_api.return_value = mock_api_data
+        validate_api.return_value = True
+
+        EveManager.create_character_obj(mock_api_data[0], self.user, '0118999')
+        EveManager.create_character_obj(Character(provider, 34567, 'deletedcharacter', 11111, 22222),
+                                        self.user, '0118999')
+
+        # Act
+        tasks.refresh_api(self.api_key)
+
+        # Assert
+        self.assertTrue(EveCharacter.objects.filter(character_id='12345').exists())
+        self.assertTrue(EveCharacter.objects.filter(character_id='23456').exists())
+        self.assertFalse(EveCharacter.objects.filter(character_id='34567').exists())
+
+        args, kwargs = validate_api.call_args
+        self.assertEqual(args[0], self.api_key.api_id)
+        self.assertEqual(args[1], self.api_key.api_key)
+        self.assertEqual(args[2], self.api_key.user)
+
+    @mock.patch(MODULE_PATH + '.EveApiManager.validate_api')
+    @mock.patch(MODULE_PATH + '.EveManager')
+    def test_refresh_api_evelink_exception(self, evemanager, validate_api):
+        import evelink
+
+        validate_api.side_effect = evelink.api.APIError()
+
+        tasks.refresh_api(self.api_key)
+
+        self.assertTrue(validate_api.called)
+        self.assertFalse(evemanager.get_characters_from_api.called)
+        self.assertFalse(evemanager.delete_characters_by_api_id.called)
+        self.assertFalse(evemanager.delete_api_key_pair.called)
+
+    @mock.patch(MODULE_PATH + '.EveApiManager.validate_api')
+    @mock.patch(MODULE_PATH + '.EveManager')
+    def test_refresh_api_invalid(self, evemanager, validate_api):
+        validate_api.side_effect = EveApiManager.ApiInvalidError(self.api_key.api_id)
+
+        tasks.refresh_api(self.api_key)
+
+        self.assertTrue(validate_api.called)
+        self.assertFalse(evemanager.get_characters_from_api.called)
+        self.assertTrue(evemanager.delete_characters_by_api_id.called)
+        self.assertTrue(evemanager.delete_api_key_pair.called)
+
+    @mock.patch(MODULE_PATH + '.EveApiManager.validate_api')
+    @mock.patch(MODULE_PATH + '.EveManager')
+    def test_refresh_api_accountvalidationerror(self, evemanager, validate_api):
+        validate_api.side_effect = EveApiManager.ApiAccountValidationError(self.api_key.api_id)
+
+        tasks.refresh_api(self.api_key)
+
+        self.assertTrue(validate_api.called)
+        self.assertFalse(evemanager.get_characters_from_api.called)
+        self.assertTrue(evemanager.delete_characters_by_api_id.called)
+        self.assertTrue(evemanager.delete_api_key_pair.called)
+
+    @mock.patch(MODULE_PATH + '.EveApiManager.validate_api')
+    @mock.patch(MODULE_PATH + '.EveManager')
+    def test_refresh_api_maskvalidationerror(self, evemanager, validate_api):
+        validate_api.side_effect = EveApiManager.ApiMaskValidationError('12345', '1111', self.api_key.api_id)
+
+        tasks.refresh_api(self.api_key)
+
+        self.assertTrue(validate_api.called)
+        self.assertFalse(evemanager.get_characters_from_api.called)
+        self.assertTrue(evemanager.delete_characters_by_api_id.called)
+        self.assertTrue(evemanager.delete_api_key_pair.called)
+
+    @mock.patch(MODULE_PATH + '.EveApiManager.validate_api')
+    @mock.patch(MODULE_PATH + '.EveManager')
+    def test_refresh_api_invalid(self, evemanager, validate_api):
+        validate_api.side_effect = EveApiManager.ApiServerUnreachableError(self.api_key.api_id)
+
+        tasks.refresh_api(self.api_key)
+
+        self.assertTrue(validate_api.called)
+        self.assertFalse(evemanager.get_characters_from_api.called)
+        # Lets hope we never see that again
+        self.assertFalse(evemanager.delete_characters_by_api_id.called)
+        self.assertFalse(evemanager.delete_api_key_pair.called)

eveonline/views.py

@@ -131,7 +131,7 @@ def api_key_removal(request, api_id):
     messages.success(request, _('Deleted API key %(apiid)s') % {"apiid": api_id})
     logger.info("Succesfully processed api delete request by user %s for api %s" % (request.user, api_id))
     if not EveCharacter.objects.filter(character_id=authinfo.main_char_id).exists():
-        authinfo.main_char_id = None
+        authinfo.main_char_id = ''
         authinfo.save()
         set_state(request.user)
     return redirect("auth_dashboard")
@@ -164,7 +164,7 @@ def user_refresh_api(request, api_id):
     if EveApiKeyPair.objects.filter(api_id=api_id).exists():
         api_key_pair = EveApiKeyPair.objects.get(api_id=api_id)
         if api_key_pair.user == request.user:
-            refresh_api(api_key_pair)
+            refresh_api.apply(args=(api_key_pair,))
             messages.success(request, _('Refreshed API key %(apiid)s') % {"apiid": api_id})
             set_state(request.user)
         else:

fleetactivitytracking/models.py

@@ -36,5 +36,5 @@ class Fat(models.Model):
         unique_together = (('character', 'fatlink'),)
 
     def __str__(self):
-        output = "Fat-link for %s" % self.character.character_name
-        return output.encode('utf-8')
+        return "Fat-link for %s" % self.character.character_name
+

fleetactivitytracking/templates/fleetactivitytracking/characternotexisting.html

@@ -2,6 +2,8 @@
 {% load i18n %}
 {% block title %}Fleet Participation{% endblock %}
 {% block page_title %}{% trans "Fleet Participation" %}{% endblock %}
+
+{% block content %}
 <div class="col-lg-12">
     <h1 class="page-header text-center">{% trans "Character not found!" %}</h1>
     <div class="col-lg-12 container" id="example">

fleetactivitytracking/templates/fleetactivitytracking/fatlinkformatter.html

@@ -6,8 +6,6 @@
 {% block title %}Alliance Auth - Fatlink Create{% endblock %}
 
 {% block page_title %}{% trans "Create Fatlink" %}{% endblock page_title %}
-{% block extra_css %}
-    <link href="{% static 'css/jquery.datetimepicker.css' %}" rel="stylesheet" type="text/css">{% endblock extra_css %}
 
 {% block content %}
     <div class="col-lg-12">

fleetactivitytracking/templates/fleetactivitytracking/fatlinkmodify.html

@@ -37,7 +37,7 @@
                             <td class="text-center">{{ fat.user }}</td>
                             <td class="text-center">{{ fat.character.character_name }}</td>
                             {%  if fat.station != "No Station" %}
-                                <td class="text-center">{% blocktrans %}Docked in {{ fat.system }}{% endblocktrans %}</td>
+                                <td class="text-center">{% blocktrans %}Docked in {% endblocktrans %}{{ fat.system }}</td>
                             {% else %}
                                 <td class="text-center">{{ fat.system }}</td>
                             {% endif %}

fleetactivitytracking/templates/fleetactivitytracking/fatlinkstatisticscorpview.html

@@ -0,0 +1,48 @@
+{% extends "public/base.html" %}
+{% load bootstrap %}
+{% load staticfiles %}
+{% load i18n %}
+
+{% block title %}Alliance Auth{% endblock %}
+{% block page_title %}{% trans "Fatlink Corp Statistics" %}{% endblock page_title %}
+
+{% block content %}
+    <div class="col-lg-12">
+        <h1 class="page-header text-center">{% blocktrans %}Participation data statistics for {{ month }}, {{ year }}{% endblocktrans %}
+            <div class="text-right">
+                <a href="{% url 'auth_fatlink_view_statistics_corp_month' corpid previous_month|date:"Y" previous_month|date:"m" %}" class="btn btn-info">{% trans "Previous month" %}</a>
+                {%  if next_month %}
+                <a href="{% url 'auth_fatlink_view_statistics_corp_month' corpid next_month|date:"Y" next_month|date:"m" %}" class="btn btn-info">{% trans "Next month" %}</a>
+                {%  endif %}
+            </div>
+        </h1>
+        {% if fatStats %}
+        <table class="table table-responsive">
+            <tr>
+                <th class="col-md-1"></th>
+                <th class="col-md-2 text-center">{% trans "Main Character" %}</th>
+                <th class="col-md-2 text-center">{% trans "Characters" %}</th>
+                <th class="col-md-2 text-center">{% trans "Fats" %}</th>
+                <th class="col-md-2 text-center">{% trans "Average fats" %}
+                  <i class="glyphicon glyphicon-question-sign" rel="tooltip" title="Fats ÷ Characters"></i>
+                </th>
+            </tr>
+            {% for memberStat in fatStats %}
+            <tr>
+                <td>
+                    <img src="https://image.eveonline.com/Character/{{ memberStat.mainchid }}_32.jpg" class="ra-avatar img-responsive">
+                </td>
+                <td class="text-center">{{ memberStat.mainchar.character_name }}</td>
+                <td class="text-center">{{ memberStat.n_chars }}</td>
+                <td class="text-center">{{ memberStat.n_fats }}</td>
+                <td class="text-center">{{ memberStat.avg_fat }}</td>
+            </tr>
+            {% endfor %}
+        </table>
+        {% endif %}
+    </div>
+{% endblock content %}
+{% block extra_script %}
+$(document).ready(function(){
+  $("[rel=tooltip]").tooltip();
+{% endblock extra_script %}

fleetactivitytracking/templates/fleetactivitytracking/fatlinkstatisticsview.html

@@ -24,14 +24,16 @@
                 <th class="col-md-5 text-center">{% trans "Corp" %}</th>
                 <th class="col-md-2 text-center">{% trans "Members" %}</th>
                 <th class="col-md-2 text-center">{% trans "Fats" %}</th>
-                <th class="col-md-2 text-center">{% trans "Average fats" %}</th>
+                <th class="col-md-2 text-center">{% trans "Average fats" %}
+                  <i class="glyphicon glyphicon-question-sign" rel="tooltip" title="Fats ÷ Characters"></i>
+                </th>
             </tr>
             {% for corpStat in fatStats %}
             <tr>
                 <td>
                     <img src="https://image.eveonline.com/Corporation/{{ corpStat.corp.corporation_id }}_32.png" class="ra-avatar img-responsive">
                 </td>
-                <td class="text-center">[{{ corpStat.corp.corporation_ticker }}]</td>
+                <td class="text-center"><a href="{% url 'auth_fatlink_view_statistics_corp' corpStat.corp.corporation_id %}">[{{ corpStat.corp.corporation_ticker }}]</td>
                 <td class="text-center">{{ corpStat.corp.corporation_name }}</td>
                 <td class="text-center">{{ corpStat.corp.member_count }}</td>
                 <td class="text-center">{{ corpStat.n_fats }}</td>
@@ -42,3 +44,7 @@
         {% endif %}
     </div>
 {% endblock content %}
+{% block extra_script %}
+$(document).ready(function(){
+  $("[rel=tooltip]").tooltip();
+{% endblock extra_script %}

fleetactivitytracking/templates/fleetactivitytracking/fatlinkview.html

@@ -36,7 +36,7 @@
                 <td class="text-center">{{ fat.fatlink.name }}</td>
                 <td class="text-center">{{ fat.character.character_name }}</td>
                 {%  if fat.station != "No Station" %}
-                <td class="text-center">{% blocktrans %}Docked in {{ fat.system }}{% endblocktrans %}</td>
+                <td class="text-center">{% blocktrans %}Docked in {% endblocktrans %}{{ fat.system }}</td>
                 {% else %}
                 <td class="text-center">{{ fat.system }}</td>
                 {% endif %}

fleetactivitytracking/views.py

@@ -9,21 +9,22 @@ from django.utils.translation import ugettext_lazy as _
 from django.utils import timezone
 from django.contrib import messages
 from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger
+from django.db.models import Q
 from eveonline.models import EveCharacter
 from eveonline.models import EveCorporationInfo
 from eveonline.managers import EveManager
+from authentication.models import AuthServicesInfo
 from fleetactivitytracking.forms import FatlinkForm
 from fleetactivitytracking.models import Fatlink, Fat
-
 from esi.decorators import token_required
-
 from slugify import slugify
-
 import string
 import random
 import datetime
-
 import logging
+import os
+
+SWAGGER_SPEC_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'swagger.json')
 
 logger = logging.getLogger(__name__)
 
@@ -51,8 +52,34 @@ class CorpStat(object):
             fatlink__fatdatetime__gte=start_of_month).filter(fatlink__fatdatetime__lte=start_of_next_month).count()
         self.blue = self.corp.is_blue
 
+    @property
     def avg_fat(self):
-        return "%.2f" % (float(self.n_fats) / float(self.corp.member_count))
+        try:
+            return "%.2f" % (float(self.n_fats) / float(self.corp.member_count))
+        except ZeroDivisionError:
+            return "%.2f" % 0
+
+
+class MemberStat(object):
+    def __init__(self, member, start_of_month, start_of_next_month, mainchid=None):
+        if mainchid:
+            self.mainchid = mainchid
+        else:
+            self.mainchid = AuthServicesInfo.objects.get(user_id=member['user_id']).main_char_id
+        self.mainchar = EveCharacter.objects.get(character_id=self.mainchid)
+        nchars = 0
+        for alliance_id in settings.STR_ALLIANCE_IDS:
+            nchars += EveCharacter.objects.filter(user_id=member['user_id']).filter(alliance_id=alliance_id).count()
+        self.n_chars = nchars
+        self.n_fats = Fat.objects.filter(user_id=member['user_id']).filter(
+            fatlink__fatdatetime__gte=start_of_month).filter(fatlink__fatdatetime__lte=start_of_next_month).count()
+
+    @property
+    def avg_fat(self):
+        try:
+            return "%.2f" % (float(self.n_fats) / float(self.n_chars))
+        except ZeroDivisionError:
+            return "%.2f" % 0
 
 
 def first_day_of_next_month(year, month):
@@ -88,6 +115,41 @@ def fatlink_view(request):
     return render(request, 'fleetactivitytracking/fatlinkview.html', context=context)
 
 
+@login_required
+@permission_required('auth.fleetactivitytracking_statistics')
+def fatlink_statistics_corp_view(request, corpid, year=None, month=None):
+    if year is None:
+        year = datetime.date.today().year
+    if month is None:
+        month = datetime.date.today().month
+
+    year = int(year)
+    month = int(month)
+    start_of_month = datetime.datetime(year, month, 1)
+    start_of_next_month = first_day_of_next_month(year, month)
+    start_of_previous_month = first_day_of_previous_month(year, month)
+    fat_stats = {}
+    corp_members = EveCharacter.objects.filter(corporation_id=corpid).values('user_id').distinct()
+
+    for member in corp_members:
+        try:
+            fat_stats[member['user_id']] = MemberStat(member, start_of_month, start_of_next_month)
+        except ObjectDoesNotExist:
+            continue
+
+    # collect and sort stats
+    stat_list = [fat_stats[x] for x in fat_stats]
+    stat_list.sort(key=lambda stat: stat.mainchar.character_name)
+    stat_list.sort(key=lambda stat: (stat.n_fats, stat.avg_fat), reverse=True)
+    
+    context = {'fatStats': stat_list, 'month': start_of_month.strftime("%B"), 'year': year,
+           'previous_month': start_of_previous_month, 'corpid': corpid}
+    if datetime.datetime.now() > start_of_next_month:
+        context.update({'next_month': start_of_next_month})
+
+    return render(request, 'fleetactivitytracking/fatlinkstatisticscorpview.html', context=context)
+
+
 @login_required
 @permission_required('auth.fleetactivitytracking_statistics')
 def fatlink_statistics_view(request, year=datetime.date.today().year, month=datetime.date.today().month):
@@ -100,33 +162,28 @@ def fatlink_statistics_view(request, year=datetime.date.today().year, month=date
     fat_stats = {}
 
     # get FAT stats for member corps
-    for corp_id in settings.STR_CORP_IDS:
-        fat_stats[corp_id] = CorpStat(corp_id, start_of_month, start_of_next_month)
-    for alliance_id in settings.STR_ALLIANCE_IDS:
-        alliance_corps = EveCorporationInfo.objects.filter(alliance__alliance_id=alliance_id)
-        for corp in alliance_corps:
-            fat_stats[corp.corporation_id] = CorpStat(corp.corporation_id, start_of_month, start_of_next_month)
+    query = Q(corporation_id__in=settings.STR_CORP_IDS) | Q(alliance__alliance_id__in=settings.STR_ALLIANCE_IDS)
+    for corp in EveCorporationInfo.objects.filter(query).distinct():
+        fat_stats[corp.corporation_id] = CorpStat(corp.corporation_id, start_of_month, start_of_next_month)
 
     # get FAT stats for corps not in alliance
     fats_in_span = Fat.objects.filter(fatlink__fatdatetime__gte=start_of_month).filter(
         fatlink__fatdatetime__lt=start_of_next_month).exclude(character__corporation_id__in=fat_stats)
 
-    for fat in fats_in_span:
-        if fat.character.corporation_id not in fat_stats:
+    for fat in fats_in_span.exclude(character__corporation_id__in=fat_stats):
+        if EveCorporationInfo.objects.filter(corporation_id=fat.character.corporation_id).exists():
             fat_stats[fat.character.corporation_id] = CorpStat(fat.character.corporation_id, start_of_month,
                                                                start_of_next_month)
 
     # collect and sort stats
     stat_list = [fat_stats[x] for x in fat_stats]
     stat_list.sort(key=lambda stat: stat.corp.corporation_name)
-    stat_list.sort(key=lambda stat: (stat.n_fats, stat.n_fats / stat.corp.member_count), reverse=True)
+    stat_list.sort(key=lambda stat: (stat.n_fats, stat.avg_fat), reverse=True)
 
+    context = {'fatStats': stat_list, 'month': start_of_month.strftime("%B"), 'year': year,
+           'previous_month': start_of_previous_month}
     if datetime.datetime.now() > start_of_next_month:
-        context = {'fatStats': stat_list, 'month': start_of_month.strftime("%B"), 'year': year,
-                   'previous_month': start_of_previous_month, 'next_month': start_of_next_month}
-    else:
-        context = {'fatStats': stat_list, 'month': start_of_month.strftime("%B"), 'year': year,
-                   'previous_month': start_of_previous_month}
+        context.update({'next_month': start_of_next_month})
 
     return render(request, 'fleetactivitytracking/fatlinkstatisticsview.html', context=context)
 
@@ -206,19 +263,19 @@ def click_fatlink_view(request, token, hash, fatname):
 
             if character:
                 # get data
-                c = token.get_esi_client(Location='v1', Universe='v2')
+                c = token.get_esi_client(spec_file=SWAGGER_SPEC_PATH)
                 location = c.Location.get_characters_character_id_location(character_id=token.character_id).result()
                 ship = c.Location.get_characters_character_id_ship(character_id=token.character_id).result()
                 location['solar_system_name'] = \
                     c.Universe.get_universe_systems_system_id(system_id=location['solar_system_id']).result()[
                         'name']
-                if location['structure_id']:
+                if location['station_id']:
+                    location['station_name'] = \
+                        c.Universe.get_universe_stations_station_id(station_id=location['station_id']).result()['name']
+                elif location['structure_id']:
                     location['station_name'] = \
                         c.Universe.get_universe_structures_structure_id(structure_id=location['structure_id']).result()[
                             'name']
-                elif location['station_id']:
-                    location['station_name'] = \
-                        c.Universe.get_universe_stations_station_id(station_id=location['station_id']).result()['name']
                 else:
                     location['station_name'] = "No Station"
                 ship['ship_type_name'] = EveManager.get_itemtype(ship['ship_type_id']).name

fleetup/__init__.py

@@ -1 +1,2 @@
 from __future__ import unicode_literals
+default_app_config = 'fleetup.apps.FleetupConfig'

fleetup/admin.py

@@ -1 +0,0 @@
-from __future__ import unicode_literals

fleetup/forms.py

@@ -1 +0,0 @@
-from __future__ import unicode_literals

fleetup/managers.py

@@ -1,178 +1,190 @@
 from __future__ import unicode_literals
 from django.conf import settings
+from django.core.cache import cache
+from django.utils import timezone
 from datetime import datetime
 
 import logging
 import requests
-import json
+import hashlib
 
 logger = logging.getLogger(__name__)
 
-appkey = settings.FLEETUP_APP_KEY
-userid = settings.FLEETUP_USER_ID
-apiid = settings.FLEETUP_API_ID
-groupid = settings.FLEETUP_GROUP_ID
-
 
 class FleetUpManager:
+    APP_KEY = settings.FLEETUP_APP_KEY
+    USER_ID = settings.FLEETUP_USER_ID
+    API_ID = settings.FLEETUP_API_ID
+    GROUP_ID = settings.FLEETUP_GROUP_ID
+    BASE_URL = "http://api.fleet-up.com/Api.svc/{}/{}/{}".format(APP_KEY, USER_ID, API_ID)
+
+    TZ = timezone.utc
+
     def __init__(self):
         pass
 
-    @staticmethod
-    def get_fleetup_members():
-        url = "http://api.fleet-up.com/Api.svc/" + str(appkey) + "/" + str(userid) + "/" + str(
-            apiid) + "/GroupCharacters/" + str(groupid) + ""
+    @classmethod
+    def _request_cache_key(cls, url):
+        h = hashlib.sha1()
+        h.update(url.encode('utf-8'))
+        return 'FLEETUP_ENDPOINT_' + h.hexdigest()
+
+    @classmethod
+    def _cache_until_seconds(cls, cache_until_json):
+        # Format comes in like "/Date(1493896236163)/"
         try:
-            jsondata = requests.get(url).content
-            fmembers = json.loads(jsondata.decode())
+            epoch_ms = int(cache_until_json[6:-2])
+            cache_delta = datetime.fromtimestamp(epoch_ms/1000) - datetime.now()
+            cache_delta_seconds = cache_delta.total_seconds()
+            if cache_delta_seconds < 0:
+                return 0
+            elif cache_delta_seconds > 3600:
+                return 3600
+            else:
+                return cache_delta_seconds
+        except TypeError:
+            logger.debug("Couldn't convert CachedUntil time, defaulting to 600 seconds")
+        return 600
+
+    @classmethod
+    def get_endpoint(cls, url):
+        try:
+            cache_key = cls._request_cache_key(url)
+            cached = cache.get(cache_key)
+            if cached:
+                return cached
+
+            r = requests.get(url)
+            r.raise_for_status()
+
+            json = r.json()
+
+            if json['Success']:
+                cache.set(cache_key, json, cls._cache_until_seconds(json['CachedUntilUTC']))
+            return json
+        except requests.exceptions.ConnectionError:
+            logger.warning("Can't connect to Fleet-Up API, is it offline?!")
+        except requests.HTTPError:
+            logger.exception("Error accessing Fleetup API")
+        return None
+
+    @classmethod
+    def get_fleetup_members(cls):
+        url = "{}/GroupCharacters/{}".format(cls.BASE_URL, cls.GROUP_ID)
+        try:
+            fmembers = cls.get_endpoint(url)
+            if not fmembers:
+                return None
             return {row["UserId"]: {"user_id": row["UserId"],
                                     "char_name": row["EveCharName"],
                                     "char_id": row["EveCharId"],
                                     "corporation": row["Corporation"]} for row in fmembers["Data"]}
-        except requests.exceptions.ConnectionError:
-            logger.warn("Can't connect to Fleet-Up API, is it offline?!")
         except (ValueError, UnicodeDecodeError, TypeError):
             logger.debug("No fleetup members retrieved.")
         return {}
 
-    @staticmethod
-    def get_fleetup_operations():
-        url = "http://api.fleet-up.com/Api.svc/" + str(appkey) + "/" + str(userid) + "/" + str(
-            apiid) + "/Operations/" + str(groupid) + ""
-        try:
-            jsondata = requests.get(url).content
-            foperations = json.loads(jsondata.decode())
-            return {row["StartString"]: {"subject": row["Subject"],
-                                         "start": (datetime.strptime(row["StartString"], "%Y-%m-%d %H:%M:%S")),
-                                         "end": (datetime.strptime(row["EndString"], "%Y-%m-%d %H:%M:%S")),
-                                         "operation_id": row["OperationId"],
-                                         "location": row["Location"],
-                                         "location_info": row["LocationInfo"],
-                                         "details": row["Details"],
-                                         "url": row["Url"],
-                                         "doctrine": row["Doctrines"],
-                                         "organizer": row["Organizer"]} for row in foperations["Data"]}
-        except requests.exceptions.ConnectionError:
-            logger.warn("Can't connect to Fleet-Up API, is it offline?!")
-        except (ValueError, UnicodeDecodeError):
-            logger.debug("No fleetup operations retrieved.")
-        return {}
+    @classmethod
+    def get_fleetup_operations(cls):
+        url = "{}/Operations/{}".format(cls.BASE_URL, cls.GROUP_ID)
+        foperations = cls.get_endpoint(url)
+        if foperations is None:
+            return None
+        return {row["StartString"]: {"subject": row["Subject"],
+                                     "start": timezone.make_aware(
+                                         datetime.strptime(row["StartString"], "%Y-%m-%d %H:%M:%S"), cls.TZ),
+                                     "end": timezone.make_aware(
+                                         datetime.strptime(row["EndString"], "%Y-%m-%d %H:%M:%S"), cls.TZ),
+                                     "operation_id": row["OperationId"],
+                                     "location": row["Location"],
+                                     "location_info": row["LocationInfo"],
+                                     "details": row["Details"],
+                                     "url": row["Url"],
+                                     "doctrine": row["Doctrines"],
+                                     "organizer": row["Organizer"]} for row in foperations["Data"]}
 
-    @staticmethod
-    def get_fleetup_timers():
-        url = "http://api.fleet-up.com/Api.svc/" + str(appkey) + "/" + str(userid) + "/" + str(
-            apiid) + "/Timers/" + str(groupid) + ""
-        try:
-            jsondata = requests.get(url).content
-            ftimers = json.loads(jsondata.decode())
-            return {row["ExpiresString"]: {"solarsystem": row["SolarSystem"],
-                                           "planet": row["Planet"],
-                                           "moon": row["Moon"],
-                                           "owner": row["Owner"],
-                                           "type": row["Type"],
-                                           "timer_type": row["TimerType"],
-                                           "expires": (datetime.strptime(row["ExpiresString"], "%Y-%m-%d %H:%M:%S")),
-                                           "notes": row["Notes"]} for row in ftimers["Data"]}
-        except requests.exceptions.ConnectionError:
-            logger.warn("Can't connect to Fleet-Up API, is it offline?!")
-        except (ValueError, UnicodeDecodeError, TypeError):
-            logger.debug("No fleetup timers retrieved.")
-        return {}
+    @classmethod
+    def get_fleetup_timers(cls):
+        url = "{}/Timers/{}".format(cls.BASE_URL, cls.GROUP_ID)
+        ftimers = cls.get_endpoint(url)
+        if not ftimers:
+            return None
+        return {row["ExpiresString"]: {"solarsystem": row["SolarSystem"],
+                                       "planet": row["Planet"],
+                                       "moon": row["Moon"],
+                                       "owner": row["Owner"],
+                                       "type": row["Type"],
+                                       "timer_type": row["TimerType"],
+                                       "expires": timezone.make_aware(
+                                           datetime.strptime(row["ExpiresString"], "%Y-%m-%d %H:%M:%S"), cls.TZ),
+                                       "notes": row["Notes"]} for row in ftimers["Data"]}
 
-    @staticmethod
-    def get_fleetup_doctrines():
-        url = "http://api.fleet-up.com/Api.svc/" + str(appkey) + "/" + str(userid) + "/" + str(
-            apiid) + "/Doctrines/" + str(groupid) + ""
-        try:
-            jsondata = requests.get(url).content
-            fdoctrines = json.loads(jsondata.decode())
-            return {"fleetup_doctrines": fdoctrines["Data"]}
-        except requests.exceptions.ConnectionError:
-            logger.warn("Can't connect to Fleet-Up API, is it offline?!")
-        except (ValueError, UnicodeDecodeError):
-            logger.debug("No fleetup doctrines retrieved.")
-        return {"fleetup_doctrines": []}
+    @classmethod
+    def get_fleetup_doctrines(cls):
+        url = "{}/Doctrines/{}".format(cls.BASE_URL, cls.GROUP_ID)
+        fdoctrines = cls.get_endpoint(url)
+        if not fdoctrines:
+            return None
+        return {"fleetup_doctrines": fdoctrines["Data"]}
 
-    @staticmethod
-    def get_fleetup_doctrine(doctrinenumber):
-        url = "http://api.fleet-up.com/Api.svc/" + str(appkey) + "/" + str(userid) + "/" + str(
-            apiid) + "/DoctrineFittings/%s" % doctrinenumber
-        try:
-            jsondata = requests.get(url).content
-            fdoctrine = json.loads(jsondata.decode())
-            return {"fitting_doctrine": fdoctrine}
-        except requests.exceptions.ConnectionError:
-            logger.warn("Can't connect to Fleet-Up API, is it offline?!")
-        except (ValueError, UnicodeDecodeError):
-            logger.warn("Fleetup doctrine number %s not found" % doctrinenumber)
-        return {"fitting_doctrine": {}}
+    @classmethod
+    def get_fleetup_doctrine(cls, doctrinenumber):
+        url = "{}/DoctrineFittings/{}".format(cls.BASE_URL, doctrinenumber)
+        fdoctrine = cls.get_endpoint(url)
+        if not fdoctrine:
+            return None
+        return {"fitting_doctrine": fdoctrine}
 
-    @staticmethod
-    def get_fleetup_fittings():
-        url = "http://api.fleet-up.com/Api.svc/" + str(appkey) + "/" + str(userid) + "/" + str(
-            apiid) + "/Fittings/" + str(groupid) + ""
-        try:
-            jsondata = requests.get(url).content
-            ffittings = json.loads(jsondata.decode())
-            return {row["FittingId"]: {"fitting_id": row["FittingId"],
-                                       "name": row["Name"],
-                                       "icon_id": row["EveTypeId"],
-                                       "hull": row["HullType"],
-                                       "shiptype": row["ShipType"],
-                                       "estimated": row["EstPrice"],
-                                       "faction": row["Faction"],
-                                       "categories": row["Categories"],
-                                       "last_update": (
-                                       datetime.strptime(row["LastUpdatedString"], "%Y-%m-%d %H:%M:%S"))} for row in
-                    ffittings["Data"]}
-        except requests.exceptions.ConnectionError:
-            logger.warn("Can't connect to Fleet-Up API, is it offline?!")
-        except (ValueError, UnicodeDecodeError, TypeError):
-            logger.debug("No fleetup fittings retrieved.")
-        return {}
+    @classmethod
+    def get_fleetup_fittings(cls):
+        url = "{}/Fittings/{}".format(cls.BASE_URL, cls.GROUP_ID)
+        ffittings = cls.get_endpoint(url)
+        if not ffittings:
+            return None
+        return {row["FittingId"]: {"fitting_id": row["FittingId"],
+                                   "name": row["Name"],
+                                   "icon_id": row["EveTypeId"],
+                                   "hull": row["HullType"],
+                                   "shiptype": row["ShipType"],
+                                   "estimated": row["EstPrice"],
+                                   "faction": row["Faction"],
+                                   "categories": row["Categories"],
+                                   "last_update":
+                                       timezone.make_aware(
+                                           datetime.strptime(row["LastUpdatedString"], "%Y-%m-%d %H:%M:%S"), cls.TZ)}
+                for row in ffittings["Data"]}
 
-    @staticmethod
-    def get_fleetup_fitting(fittingnumber):
-        url = "http://api.fleet-up.com/Api.svc/" + str(appkey) + "/" + str(userid) + "/" + str(
-            apiid) + "/Fitting/%s" % fittingnumber
+    @classmethod
+    def get_fleetup_fitting(cls, fittingnumber):
+        url = "{}/Fitting/{}".format(cls.BASE_URL, fittingnumber)
         try:
-            jsondata = requests.get(url).content
-            ffitting = json.loads(jsondata.decode())
+            ffitting = cls.get_endpoint(url)
+            if not ffitting:
+                return None
             return {"fitting_data": ffitting["Data"]}
-        except requests.exceptions.ConnectionError:
-            logger.warn("Can't connect to Fleet-Up API, is it offline?!")
-        except (ValueError, UnicodeDecodeError):
-            logger.warn("Fleetup fitting number %s not found" % fittingnumber)
         except KeyError:
-            logger.warn("Failed to retrieve fleetup fitting number %s" % fittingnumber)
+            logger.warning("Failed to retrieve fleetup fitting number %s" % fittingnumber)
         return {"fitting_data": {}}
 
-    @staticmethod
-    def get_fleetup_doctrineid(fittingnumber):
-        url = "http://api.fleet-up.com/Api.svc/" + str(appkey) + "/" + str(userid) + "/" + str(
-            apiid) + "/Fitting/%s" % fittingnumber
+    @classmethod
+    def get_fleetup_doctrineid(cls, fittingnumber):
+        url = "{}/Fitting/{}".format(cls.BASE_URL, fittingnumber)
         try:
-            jsondata = requests.get(url).content
-            fdoctrineid = json.loads(jsondata.decode())
+            fdoctrineid = cls.get_endpoint(url)
+            if not fdoctrineid:
+                return None
             return fdoctrineid['Data']['Doctrines'][0]['DoctrineId']
-        except requests.exceptions.ConnectionError:
-            logger.warn("Can't connect to Fleet-Up API, is it offline?!")
-        except (ValueError, UnicodeDecodeError):
-            logger.warn("Fleetup doctrine number not found for fitting number %s" % fittingnumber)
         except (KeyError, IndexError):
             logger.debug("Fleetup fitting number %s not in a doctrine." % fittingnumber)
-        return None
+        return {}
 
-    @staticmethod
-    def get_fleetup_fitting_eft(fittingnumber):
-        url = "http://api.fleet-up.com/Api.svc/" + str(appkey) + "/" + str(userid) + "/" + str(
-            apiid) + "/Fitting/%s/eft" % fittingnumber
+    @classmethod
+    def get_fleetup_fitting_eft(cls, fittingnumber):
+        url = "{}/Fitting/{}/eft".format(cls.BASE_URL, fittingnumber)
         try:
-            jsondata = requests.get(url).content
-            ffittingeft = json.loads(jsondata.decode())
+            ffittingeft = cls.get_endpoint(url)
+            if not ffittingeft:
+                return None
             return {"fitting_eft": ffittingeft["Data"]["FittingData"]}
-        except requests.exceptions.ConnectionError:
-            logger.warn("Can't connect to Fleet-Up API, is it offline?!")
-        except (ValueError, UnicodeDecodeError):
-            logger.warn("Fleetup fitting eft not found for fitting number %s" % fittingnumber)
+        except KeyError:
+            logger.warning("Fleetup fitting eft not found for fitting number %s" % fittingnumber)
         return {"fitting_eft": {}}

fleetup/models.py

@@ -1 +0,0 @@
-from __future__ import unicode_literals

fleetup/templates/fleetup/characters.html

@@ -9,30 +9,7 @@
 {% block content %}
 <div class="col-lg-12">
 {% if perms.auth.corp_stats %}
-    <nav class="navbar navbar-default">
-    <div class="container-fluid">
-        <div class="navbar-header">
-            <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#bs-example-navbar-collapse-1" aria-expanded="false">
-                <span class="sr-only">{% trans "Toggle navigation" %}</span>
-                <span class="icon-bar"></span>
-                <span class="icon-bar"></span>
-                <span class="icon-bar"></span>
-            </button>
-            <a class="navbar-brand" href="#">Fleet-Up</a>
-        </div>
-        <div class="collapse navbar-collapse" id="bs-example-navbar-collapse-1">
-            <ul class="nav navbar-nav">
-                <li><a href="{% url 'auth_fleetup_view' %}">{% trans "Ops and Timers" %}</a></li>
-                <li><a href="{% url 'auth_fleetup_doctrines' %}">{% trans "Doctrines" %}</a></li>
-                <li><a href="{% url 'auth_fleetup_fittings' %}">{% trans "Fittings" %}</a></li>
-                
-                <li class="active"><a href="{% url 'auth_fleetup_characters' %}">{% trans "Characters" %} <span class="sr-only">(current)</span></a></li>
-                
-                <li></li>
-            </ul>
-        </div>
-    </div>
-    </nav>
+    {% include "fleetup/menu.html" %}
     <div class="panel panel-default">
         <div class="panel-heading">
             <h3 class="panel-title">{% trans "Characters registered on Fleet-Up.com" %}</h3>

fleetup/templates/fleetup/doctrine.html

@@ -8,30 +8,7 @@
 
 {% block content %}
 <div class="col-lg-12">
-<nav class="navbar navbar-default">
-    <div class="container-fluid">
-        <div class="navbar-header">
-            <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#bs-example-navbar-collapse-1" aria-expanded="false">
-                <span class="sr-only">{% trans "Toggle navigation" %}</span>
-                <span class="icon-bar"></span>
-                <span class="icon-bar"></span>
-                <span class="icon-bar"></span>
-            </button>
-            <a class="navbar-brand" href="#">Fleet-Up</a>
-        </div>
-        <div class="collapse navbar-collapse" id="bs-example-navbar-collapse-1">
-            <ul class="nav navbar-nav">
-                <li><a href="{% url 'auth_fleetup_view' %}">{% trans "Ops and Timers" %}</a></li>
-                <li class="active"><a href="{% url 'auth_fleetup_doctrines' %}">{% trans "Doctrines" %} <span class="sr-only">(current)</span></a></li>
-                <li><a href="{% url 'auth_fleetup_fittings' %}">{% trans "Fittings" %}</a></li>
-                {% if perms.auth.corp_stats %}
-                <li><a href="{% url 'auth_fleetup_characters' %}">{% trans "Characters" %}</a></li>
-                {% endif %}
-                <li></li>
-            </ul>
-        </div>
-    </div>
-</nav>
+    {% include "fleetup/menu.html" %}
     <div class="panel">
         {% for a, j in doctrine.items %}
         {% regroup j.Data|dictsort:"Role" by Role as role_list %}

fleetup/templates/fleetup/doctrinesview.html

@@ -8,30 +8,7 @@
 
 {% block content %}
 <div class="col-lg-12">
-<nav class="navbar navbar-default">
-    <div class="container-fluid">
-        <div class="navbar-header">
-            <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#bs-example-navbar-collapse-1" aria-expanded="false">
-                <span class="sr-only">{% trans "Toggle navigation" %}</span>
-                <span class="icon-bar"></span>
-                <span class="icon-bar"></span>
-                <span class="icon-bar"></span>
-            </button>
-            <a class="navbar-brand" href="#">Fleet-Up</a>
-        </div>
-        <div class="collapse navbar-collapse" id="bs-example-navbar-collapse-1">
-            <ul class="nav navbar-nav">
-                <li><a href="{% url 'auth_fleetup_view' %}">{% trans "Ops and Timers" %}</a></li>
-                <li class="active"><a href="{% url 'auth_fleetup_doctrines' %}">{% trans "Doctrines" %} <span class="sr-only">(current)</span></a></li>
-                <li><a href="{% url 'auth_fleetup_fittings' %}">{% trans "Fittings" %}</a></li>
-                {% if perms.auth.corp_stats %}
-                <li><a href="{% url 'auth_fleetup_characters' %}">{% trans "Characters" %}</a></li>
-                {% endif %}
-                <li></li>
-            </ul>
-        </div>
-    </div>
-</nav>
+    {% include "fleetup/menu.html" %}
     <div class="panel">
         {% if doctrines_list %}
         {% for a, j in doctrines_list.items %}

fleetup/templates/fleetup/fitting.html

@@ -8,30 +8,7 @@
 
 {% block content %}
 <div class="col-lg-12">
-<nav class="navbar navbar-default">
-    <div class="container-fluid">
-        <div class="navbar-header">
-            <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#bs-example-navbar-collapse-1" aria-expanded="false">
-                <span class="sr-only">{% trans "Toggle navigation" %}</span>
-                <span class="icon-bar"></span>
-                <span class="icon-bar"></span>
-                <span class="icon-bar"></span>
-            </button>
-            <a class="navbar-brand" href="#">{% trans "Fleet-Up" %}</a>
-        </div>
-        <div class="collapse navbar-collapse" id="bs-example-navbar-collapse-1">
-            <ul class="nav navbar-nav">
-                <li><a href="{% url 'auth_fleetup_view' %}">{% trans "Ops and Timers" %}</a></li>
-                <li><a href="{% url 'auth_fleetup_doctrines' %}">{% trans "Doctrines" %}</a></li>
-                <li class="active"><a href="{% url 'auth_fleetup_fittings' %}">{% trans "Fittings" %} <span class="sr-only">(current)</span></a></li>
-                {% if perms.auth.corp_stats %}
-                <li><a href="{% url 'auth_fleetup_characters' %}">{% trans "Characters" %}</a></li>
-                {% endif %}
-                <li></li>
-            </ul>
-        </div>
-    </div>
-</nav>
+    {% include "fleetup/menu.html" %}
     <div class="tab-content">
         <div id="fit" class="tab-pane fade in active">
             <div class="col-lg-3">
@@ -56,8 +33,8 @@
                                 </div>
                             </div>
                         </div>
-                        <div class="col-lg-8 col-lg-offset-7">
-                            <a class="btn btn-primary" href="/fleetup/doctrines/{{ doctrin.DoctrineId }}/">{% trans "See doctrine" %}</a>
+                        <div class="pull-right">
+                            <a class="btn btn-primary" href="{% url 'auth_fleetup_doctrine' doctrin.DoctrineId %}">{% trans "See doctrine" %}</a>
                         </div>
                         {% endfor %}
                     </div>
@@ -140,7 +117,7 @@
                     <div class="panel-body">
                     {% for data in fitting_eft.items %}
                     {% autoescape off %}
-                    <pre>{{ fitting_eft.fitting_eft }}</pre>
+                    <textarea class="form-control" rows="25" spellcheck="false" onclick="this.focus();this.select()" readonly>{{ fitting_eft.fitting_eft }}</textarea>
                     {% endautoescape %}
                     {% endfor %}
                     </div>

fleetup/templates/fleetup/fittingsview.html

@@ -8,30 +8,7 @@
 
 {% block content %}
 <div class="col-lg-12">
-<nav class="navbar navbar-default">
-    <div class="container-fluid">
-        <div class="navbar-header">
-            <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#bs-example-navbar-collapse-1" aria-expanded="false">
-                <span class="sr-only">{% trans "Toggle navigation" %}</span>
-                <span class="icon-bar"></span>
-                <span class="icon-bar"></span>
-                <span class="icon-bar"></span>
-            </button>
-            <a class="navbar-brand" href="#">{% trans "Fleet-Up" %}</a>
-        </div>
-        <div class="collapse navbar-collapse" id="bs-example-navbar-collapse-1">
-            <ul class="nav navbar-nav">
-                <li><a href="{% url 'auth_fleetup_view' %}">{% trans "Ops and Timers" %}</a></li>
-                <li><a href="{% url 'auth_fleetup_doctrines' %}">{% trans "Doctrines" %}</a></li>
-                <li class="active"><a href="{% url 'auth_fleetup_fittings' %}">{% trans "Fittings" %} <span class="sr-only">(current)</span></a></li>
-                {% if perms.auth.corp_stats %}
-                <li><a href="{% url 'auth_fleetup_characters' %}">{% trans "Characters" %}</a></li>
-                {% endif %}
-                <li></li>
-            </ul>
-        </div>
-    </div>
-</nav>
+{% include "fleetup/menu.html" %}
 <div class="panel">
     {% if fitting_list %}
         <table class="table table-condensed table-hover table-striped">

fleetup/templates/fleetup/index.html

@@ -8,30 +8,7 @@
 
 {% block content %}
 <div class="col-lg-12">
-<nav class="navbar navbar-default">
-    <div class="container-fluid">
-        <div class="navbar-header">
-            <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#bs-example-navbar-collapse-1" aria-expanded="false">
-                <span class="sr-only">{% trans "Toggle navigation" %}</span>
-                <span class="icon-bar"></span>
-                <span class="icon-bar"></span>
-                <span class="icon-bar"></span>
-            </button>
-            <a class="navbar-brand" href="#">{% trans "Fleet-Up" %}</a>
-        </div>
-        <div class="collapse navbar-collapse" id="bs-example-navbar-collapse-1">
-            <ul class="nav navbar-nav">
-                <li class="active"><a href="#">{% trans "Ops and Timers" %} <span class="sr-only">(current)</span></a></li>
-                <li><a href="{% url 'auth_fleetup_doctrines' %}">{% trans "Doctrines" %}</a></li>
-                <li><a href="{% url 'auth_fleetup_fittings' %}">{% trans "Fittings" %}</a></li>
-                {% if perms.auth.human_resources %}
-                <li><a href="{% url 'auth_fleetup_characters' %}">{% trans "Characters" %}</a></li>
-                {% endif %}
-                <li></li>
-            </ul>
-        </div>
-    </div>
-</nav>
+{% include "fleetup/menu.html" %}
 <div class="panel">
     <ul class="nav nav-tabs">
         <li class="active"><a data-toggle="tab" href="#operations">{% trans "Operations" %}</a></li>

fleetup/templates/fleetup/menu.html

@@ -0,0 +1,26 @@
+{% load i18n %}
+{% load navactive %}
+
+<nav class="navbar navbar-default">
+    <div class="container-fluid">
+        <div class="navbar-header">
+            <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#bs-example-navbar-collapse-1" aria-expanded="false">
+                <span class="sr-only">{% trans "Toggle navigation" %}</span>
+                <span class="icon-bar"></span>
+                <span class="icon-bar"></span>
+                <span class="icon-bar"></span>
+            </button>
+            <a class="navbar-brand" href="#">Fleet-Up</a>
+        </div>
+        <div class="collapse navbar-collapse" id="bs-example-navbar-collapse-1">
+            <ul class="nav navbar-nav">
+                <li class="{% navactive request 'auth_fleetup_view' %}"><a href="{% url 'auth_fleetup_view' %}">{% trans "Ops and Timers" %}</a></li>
+                <li class="{% navactive request 'auth_fleetup_doctrines auth_fleetup_doctrine' %}"><a href="{% url 'auth_fleetup_doctrines' %}">{% trans "Doctrines" %}</a></li>
+                <li class="{% navactive request 'auth_fleetup_fittings auth_fleetup_fitting' %}"><a href="{% url 'auth_fleetup_fittings' %}">{% trans "Fittings" %}</a></li>
+                {% if perms.auth.corp_stats %}
+                <li class="{% navactive request 'auth_fleetup_characters' %}"><a href="{% url 'auth_fleetup_characters' %}">{% trans "Characters" %}</a></li>
+                {% endif %}
+            </ul>
+        </div>
+    </div>
+</nav>

fleetup/tests/test_manager.py

@@ -0,0 +1,510 @@
+from __future__ import unicode_literals
+
+try:
+    # Py3
+    from unittest import mock
+except ImportError:
+    # Py2
+    import mock
+
+import requests_mock
+import json
+import datetime
+
+from django.test import TestCase
+from django.utils.timezone import make_aware, utc
+
+from fleetup.managers import FleetUpManager
+
+
+class FleetupManagerTestCase(TestCase):
+    def setUp(self):
+        pass
+
+    def test__request_cache_key(self):
+
+        cache_key = FleetUpManager._request_cache_key('testurl')
+
+        self.assertEqual('FLEETUP_ENDPOINT_a39562b6ef5b858220be13d2adb61d3f10cf8d61',
+                         cache_key)
+
+    @mock.patch('fleetup.managers.cache')
+    @requests_mock.Mocker()
+    def test_get_endpoint(self, cache, m):
+        url = "http://example.com/test/endpoint/"
+        json_data = {'data': "123456", 'CachedUntilUTC': '/Date(1493896236163)/', 'Success': True}
+        m.register_uri('GET', url,
+                       text=json.dumps(json_data))
+
+        cache.get.return_value = None  # No cached value
+
+        # Act
+        result = FleetUpManager.get_endpoint(url)
+
+        # Assert
+        self.assertTrue(cache.get.called)
+        self.assertTrue(cache.set.called)
+        args, kwargs = cache.set.call_args
+        self.assertDictEqual(json_data, args[1])
+
+        self.assertDictEqual(json_data, result)
+
+    @mock.patch('fleetup.managers.cache')
+    @requests_mock.Mocker()
+    def test_get_endpoint_error(self, cache, m):
+        url = "http://example.com/test/endpoint/"
+        json_data = {'data': [], 'Success': False}
+        m.register_uri('GET', url,
+                       text=json.dumps(json_data),
+                       status_code=400)
+
+        cache.get.return_value = None  # No cached value
+
+        # Act
+        result = FleetUpManager.get_endpoint(url)
+
+        # Assert
+        self.assertTrue(cache.get.called)
+        self.assertFalse(cache.set.called)
+        self.assertIsNone(result)
+
+    @mock.patch('fleetup.managers.FleetUpManager.get_endpoint')
+    def test_get_fleetup_members(self, get_endpoint):
+
+        get_endpoint.return_value = {"Data": [
+            {
+                'UserId': 1234,
+                'EveCharName': 'test_name',
+                'EveCharId': 5678,
+                'Corporation': 'test_corporation',
+            }
+        ]}
+
+        # Act
+        result = FleetUpManager.get_fleetup_members()
+
+        # Asset
+        self.assertTrue(get_endpoint.called)
+        args, kwargs = get_endpoint.call_args
+        self.assertEqual(args[0],
+                         FleetUpManager.BASE_URL + '/GroupCharacters/' +
+                         FleetUpManager.GROUP_ID)
+        expected_result = {
+            1234: {
+                'user_id': 1234,
+                'char_name': 'test_name',
+                'char_id': 5678,
+                'corporation': 'test_corporation',
+            }
+        }
+        self.assertDictEqual(expected_result, result)
+
+        # Test None response
+        # Arrange
+        get_endpoint.return_value = None
+
+        # Act
+        result = FleetUpManager.get_fleetup_members()
+
+        # Assert
+        self.assertIsNone(result)
+
+        # Test Empty response
+        # Arrange
+        get_endpoint.return_value = {'Data': []}
+
+        # Act
+        result = FleetUpManager.get_fleetup_members()
+
+        # Assert
+        self.assertDictEqual({}, result)
+
+    @mock.patch('fleetup.managers.FleetUpManager.get_endpoint')
+    def test_get_fleetup_operations(self, get_endpoint):
+
+        get_endpoint.return_value = {"Data": [
+            {
+                'Subject': 'test_operation',
+                'StartString': '2017-05-06 11:11:11',
+                'EndString': '2017-05-06 12:12:12',
+                'OperationId': 1234,
+                'Location': 'Jita',
+                'LocationInfo': '4-4',
+                'Details': 'This is a test operation',
+                'Url': 'http://example.com/1234',
+                'Doctrines': 'Foxcats',
+                'Organizer': 'Example FC'
+            }
+        ]}
+
+        # Act
+        result = FleetUpManager.get_fleetup_operations()
+        self.maxDiff = None
+        # Asset
+        self.assertTrue(get_endpoint.called)
+        args, kwargs = get_endpoint.call_args
+        self.assertEqual(args[0],
+                         FleetUpManager.BASE_URL + '/Operations/' +
+                         FleetUpManager.GROUP_ID)
+        expected_result = {
+            '2017-05-06 11:11:11': {
+                'subject': 'test_operation',
+                'start': make_aware(datetime.datetime(2017, 5, 6, 11, 11, 11), utc),
+                'end': make_aware(datetime.datetime(2017, 5, 6, 12, 12, 12), utc),
+                'operation_id': 1234,
+                'location': 'Jita',
+                'location_info': '4-4',
+                'details': 'This is a test operation',
+                'url': 'http://example.com/1234',
+                'doctrine': 'Foxcats',
+                'organizer': 'Example FC'
+            }
+        }
+        self.assertDictEqual(expected_result, result)
+
+        # Test None response
+        # Arrange
+        get_endpoint.return_value = None
+
+        # Act
+        result = FleetUpManager.get_fleetup_operations()
+
+        # Assert
+        self.assertIsNone(result)
+
+        # Test Empty response
+        # Arrange
+        get_endpoint.return_value = {'Data': []}
+
+        # Act
+        result = FleetUpManager.get_fleetup_operations()
+
+        # Assert
+        self.assertDictEqual({}, result)
+
+    @mock.patch('fleetup.managers.FleetUpManager.get_endpoint')
+    def test_get_fleetup_timers(self, get_endpoint):
+
+        get_endpoint.return_value = {"Data": [
+            {
+                'ExpiresString': '2017-05-06 11:11:11',
+                'SolarSystem': 'Jita',
+                'Planet': '4',
+                'Moon': '4',
+                'Owner': 'Caldari Navy',
+                'Type': 'Caldari Station',
+                'TimerType': 'Armor',
+                'Notes': 'Burn Jita?'
+            }
+        ]}
+
+        # Act
+        result = FleetUpManager.get_fleetup_timers()
+
+        # Asset
+        self.assertTrue(get_endpoint.called)
+        args, kwargs = get_endpoint.call_args
+        self.assertEqual(args[0],
+                         FleetUpManager.BASE_URL + '/Timers/' +
+                         FleetUpManager.GROUP_ID)
+        expected_result = {
+            '2017-05-06 11:11:11': {
+                'expires': make_aware(datetime.datetime(2017, 5, 6, 11, 11, 11), utc),
+                'solarsystem': 'Jita',
+                'planet': '4',
+                'moon': '4',
+                'owner': 'Caldari Navy',
+                'type': 'Caldari Station',
+                'timer_type': 'Armor',
+                'notes': 'Burn Jita?'
+            }
+        }
+        self.assertDictEqual(expected_result, result)
+
+        # Test None response
+        # Arrange
+        get_endpoint.return_value = None
+
+        # Act
+        result = FleetUpManager.get_fleetup_timers()
+
+        # Assert
+        self.assertIsNone(result)
+
+        # Test Empty response
+        # Arrange
+        get_endpoint.return_value = {'Data': []}
+
+        # Act
+        result = FleetUpManager.get_fleetup_timers()
+
+        # Assert
+        self.assertDictEqual({}, result)
+
+    @mock.patch('fleetup.managers.FleetUpManager.get_endpoint')
+    def test_get_fleetup_doctrines(self, get_endpoint):
+
+        get_endpoint.return_value = {"Data": [
+            {
+                'TestData': True
+            }
+        ]}
+
+        # Act
+        result = FleetUpManager.get_fleetup_doctrines()
+
+        # Asset
+        self.assertTrue(get_endpoint.called)
+        args, kwargs = get_endpoint.call_args
+        self.assertEqual(args[0],
+                         FleetUpManager.BASE_URL + '/Doctrines/' +
+                         FleetUpManager.GROUP_ID)
+        expected_result = {
+            'fleetup_doctrines': [{
+                'TestData': True
+            }]
+        }
+        self.assertDictEqual(expected_result, result)
+
+        # Test None response
+        # Arrange
+        get_endpoint.return_value = None
+
+        # Act
+        result = FleetUpManager.get_fleetup_doctrines()
+
+        # Assert
+        self.assertIsNone(result)
+
+        # Test Empty response
+        # Arrange
+        get_endpoint.return_value = {'Data': []}
+
+        # Act
+        result = FleetUpManager.get_fleetup_doctrines()
+
+        # Assert
+        self.assertDictEqual({"fleetup_doctrines": []}, result)
+
+    @mock.patch('fleetup.managers.FleetUpManager.get_endpoint')
+    def test_get_fleetup_doctrine(self, get_endpoint):
+
+        get_endpoint.return_value = {"Data": [
+            {
+                'TestData': True
+            }
+        ]}
+
+        # Act
+        result = FleetUpManager.get_fleetup_doctrine(1234)
+
+        # Asset
+        self.assertTrue(get_endpoint.called)
+        args, kwargs = get_endpoint.call_args
+        self.assertEqual(args[0],
+                         FleetUpManager.BASE_URL + '/DoctrineFittings/1234')
+        expected_result = {
+            'fitting_doctrine': {'Data': [{
+                'TestData': True
+            }]}
+        }
+        self.assertDictEqual(expected_result, result)
+
+        # Test None response
+        # Arrange
+        get_endpoint.return_value = None
+
+        # Act
+        result = FleetUpManager.get_fleetup_doctrine(1234)
+
+        # Assert
+        self.assertIsNone(result)
+
+        # Test Empty response
+        # Arrange
+        get_endpoint.return_value = {'Data': []}
+
+        # Act
+        result = FleetUpManager.get_fleetup_doctrine(1234)
+
+        # Assert
+        self.assertDictEqual({"fitting_doctrine": {'Data': []}}, result)
+
+    @mock.patch('fleetup.managers.FleetUpManager.get_endpoint')
+    def test_get_fleetup_fittings(self, get_endpoint):
+
+        get_endpoint.return_value = {"Data": [
+            {
+                'FittingId': 1234,
+                'Name': 'Foxcat',
+                'EveTypeId': 17726,
+                'HullType': 'Battleship',
+                'ShipType': 'Apocalypse Navy Issue',
+                'EstPrice': 500000000,
+                'Faction': 'Amarr',
+                'Categories': ["Armor", "Laser"],
+                'LastUpdatedString': '2017-05-06 11:11:11',
+            }
+        ]}
+
+        # Act
+        result = FleetUpManager.get_fleetup_fittings()
+
+        # Asset
+        self.assertTrue(get_endpoint.called)
+        expected_result = {
+            1234: {
+                'fitting_id': 1234,
+                'name': 'Foxcat',
+                'icon_id': 17726,
+                'hull': 'Battleship',
+                'shiptype': 'Apocalypse Navy Issue',
+                'estimated': 500000000,
+                'faction': 'Amarr',
+                'categories': ["Armor", "Laser"],
+                'last_update': make_aware(datetime.datetime(2017, 5, 6, 11, 11, 11), utc)
+            }
+        }
+        self.assertDictEqual(expected_result, result)
+
+        # Test None response
+        # Arrange
+        get_endpoint.return_value = None
+
+        # Act
+        result = FleetUpManager.get_fleetup_fittings()
+
+        # Assert
+        self.assertIsNone(result)
+
+        # Test Empty response
+        # Arrange
+        get_endpoint.return_value = {'Data': []}
+
+        # Act
+        result = FleetUpManager.get_fleetup_fittings()
+
+        # Assert
+        self.assertDictEqual({}, result)
+
+    @mock.patch('fleetup.managers.FleetUpManager.get_endpoint')
+    def test_get_fleetup_fitting(self, get_endpoint):
+
+        get_endpoint.return_value = {"Data":
+            {
+                'FittingData': [{}]
+            }
+        }
+
+        # Act
+        result = FleetUpManager.get_fleetup_fitting(1234)
+
+        # Asset
+        self.assertTrue(get_endpoint.called)
+        args, kwargs = get_endpoint.call_args
+        self.assertEqual(args[0], FleetUpManager.BASE_URL + '/Fitting/1234')
+        expected_result = {
+            'fitting_data': {
+                'FittingData': [{}]
+            }
+        }
+        self.assertDictEqual(expected_result, result)
+
+        # Test None response
+        # Arrange
+        get_endpoint.return_value = None
+
+        # Act
+        result = FleetUpManager.get_fleetup_fitting(1234)
+
+        # Assert
+        self.assertIsNone(result)
+
+        # Test Empty response
+        # Arrange
+        get_endpoint.return_value = {'Data': {}}
+
+        # Act
+        result = FleetUpManager.get_fleetup_fitting(1234)
+
+        # Assert
+        self.assertDictEqual({"fitting_data": {}}, result)
+
+    @mock.patch('fleetup.managers.FleetUpManager.get_endpoint')
+    def test_get_fleetup_doctrineid(self, get_endpoint):
+
+        get_endpoint.return_value = {
+            "Data": {
+                'Doctrines': [{'DoctrineId': 4567}]
+            }
+        }
+
+        # Act
+        result = FleetUpManager.get_fleetup_doctrineid(1234)
+
+        # Asset
+        self.assertTrue(get_endpoint.called)
+        args, kwargs = get_endpoint.call_args
+        self.assertEqual(args[0], FleetUpManager.BASE_URL + '/Fitting/1234')
+
+        self.assertEqual(4567, result)
+
+        # Test None response
+        # Arrange
+        get_endpoint.return_value = None
+
+        # Act
+        result = FleetUpManager.get_fleetup_doctrineid(1234)
+
+        # Assert
+        self.assertIsNone(result)
+
+        # Test Empty response
+        # Arrange
+        get_endpoint.return_value = {'Data': {}}
+
+        # Act
+        result = FleetUpManager.get_fleetup_doctrineid(1234)
+
+        # Assert
+        self.assertDictEqual({}, result)
+
+    @mock.patch('fleetup.managers.FleetUpManager.get_endpoint')
+    def test_get_fleetup_fitting_eft(self, get_endpoint):
+
+        get_endpoint.return_value = {
+            "Data": {
+                'FittingData': '[Apocalypse Navy Issue, Foxcat]'
+            }
+        }
+
+        # Act
+        result = FleetUpManager.get_fleetup_fitting_eft(1234)
+
+        # Asset
+        self.assertTrue(get_endpoint.called)
+        args, kwargs = get_endpoint.call_args
+        self.assertEqual(args[0], FleetUpManager.BASE_URL + '/Fitting/1234/eft')
+
+        self.assertDictEqual({"fitting_eft": '[Apocalypse Navy Issue, Foxcat]'},
+                             result)
+
+        # Test None response
+        # Arrange
+        get_endpoint.return_value = None
+
+        # Act
+        result = FleetUpManager.get_fleetup_fitting_eft(1234)
+
+        # Assert
+        self.assertIsNone(result)
+
+        # Test Empty response
+        # Arrange
+        get_endpoint.return_value = {'Data': {}}
+
+        # Act
+        result = FleetUpManager.get_fleetup_fitting_eft(1234)
+
+        # Assert
+        self.assertDictEqual({"fitting_eft": {}}, result)

fleetup/urls.py

@@ -0,0 +1,13 @@
+from __future__ import unicode_literals
+from django.conf.urls import url
+
+from . import views
+
+urlpatterns = [
+    url(r'^$', views.fleetup_view, name='auth_fleetup_view'),
+    url(r'^fittings/$', views.fleetup_fittings, name='auth_fleetup_fittings'),
+    url(r'^fittings/(?P<fittingnumber>[0-9]+)/$', views.fleetup_fitting, name='auth_fleetup_fitting'),
+    url(r'^doctrines/$', views.fleetup_doctrines, name='auth_fleetup_doctrines'),
+    url(r'^characters/$', views.fleetup_characters, name='auth_fleetup_characters'),
+    url(r'^doctrines/(?P<doctrinenumber>[0-9]+)/$', views.fleetup_doctrine, name='auth_fleetup_doctrine'),
+]

fleetup/views.py

@@ -4,6 +4,8 @@ from django.shortcuts import render
 from django.contrib.auth.decorators import login_required
 from django.contrib.auth.decorators import permission_required
 from django.template.defaulttags import register
+from django.contrib import messages
+from django.utils.translation import ugettext_lazy as _
 from fleetup.managers import FleetUpManager
 from authentication.decorators import members_and_blues
 
@@ -23,14 +25,20 @@ def fleetup_view(request):
     logger.debug("fleetup_view called by user %s" % request.user)
 
     operations_list = FleetUpManager.get_fleetup_operations()
+    if operations_list is None:
+        messages.add_message(request, messages.ERROR, _("Failed to get operations list, contact your administrator"))
+        operations_list = {}
     timers_list = FleetUpManager.get_fleetup_timers()
+    if timers_list is None:
+        messages.add_message(request, messages.ERROR, _("Failed to get timers list, contact your administrator"))
+        timers_list = {}
     now = datetime.datetime.now().strftime('%H:%M:%S')
 
     context = {"timers_list": sorted(timers_list.items()),
                "operations_list": sorted(operations_list.items()),
                "now": now}
 
-    return render(request, 'registered/fleetup.html', context=context)
+    return render(request, 'fleetup/index.html', context=context)
 
 
 @login_required
@@ -39,10 +47,13 @@ def fleetup_characters(request):
     logger.debug("fleetup_characters called by user %s" % request.user)
 
     member_list = FleetUpManager.get_fleetup_members()
+    if member_list is None:
+        messages.add_message(request, messages.ERROR, _("Failed to get member list, contact your administrator"))
+        member_list = {}
 
     context = {"member_list": sorted(member_list.items())}
 
-    return render(request, 'registered/fleetupcharacters.html', context=context)
+    return render(request, 'fleetup/characters.html', context=context)
 
 
 @login_required
@@ -50,8 +61,13 @@ def fleetup_characters(request):
 def fleetup_fittings(request):
     logger.debug("fleetup_fittings called by user %s" % request.user)
     fitting_list = FleetUpManager.get_fleetup_fittings()
+
+    if fitting_list is None:
+        messages.add_message(request, messages.ERROR, _("Failed to get fitting list, contact your administrator"))
+        fitting_list = {}
+
     context = {"fitting_list": sorted(fitting_list.items())}
-    return render(request, 'registered/fleetupfittingsview.html', context=context)
+    return render(request, 'fleetup/fittingsview.html', context=context)
 
 
 @login_required
@@ -62,10 +78,15 @@ def fleetup_fitting(request, fittingnumber):
     fitting_data = FleetUpManager.get_fleetup_fitting(fittingnumber)
     doctrinenumber = FleetUpManager.get_fleetup_doctrineid(fittingnumber)
     doctrines_list = FleetUpManager.get_fleetup_doctrine(doctrinenumber)
+
+    if fitting_eft is None or fitting_data is None or doctrinenumber is None:
+        messages.add_message(request, messages.ERROR, _("There was an error getting some of the data for this fitting. "
+                                                        "Contact your administrator"))
+
     context = {"fitting_eft": fitting_eft,
                "fitting_data": fitting_data,
                "doctrines_list": doctrines_list}
-    return render(request, 'registered/fleetupfitting.html', context=context)
+    return render(request, 'fleetup/fitting.html', context=context)
 
 
 @login_required
@@ -73,8 +94,11 @@ def fleetup_fitting(request, fittingnumber):
 def fleetup_doctrines(request):
     logger.debug("fleetup_doctrines called by user %s" % request.user)
     doctrines_list = FleetUpManager.get_fleetup_doctrines()
+    if doctrines_list is None:
+        messages.add_message(request, messages.ERROR, _("Failed to get doctrines list, contact your administrator"))
+
     context = {"doctrines_list": doctrines_list}
-    return render(request, 'registered/fleetupdoctrinesview.html', context=context)
+    return render(request, 'fleetup/doctrinesview.html', context=context)
 
 
 @login_required
@@ -82,5 +106,7 @@ def fleetup_doctrines(request):
 def fleetup_doctrine(request, doctrinenumber):
     logger.debug("fleetup_doctrine called by user %s" % request.user)
     doctrine = FleetUpManager.get_fleetup_doctrine(doctrinenumber)
+    if doctrine is None:
+        messages.add_message(request, messages.ERROR, _("Failed to get doctine, contact your administrator"))
     context = {"doctrine": doctrine}
-    return render(request, 'registered/fleetupdoctrine.html', context=context)
+    return render(request, 'fleetup/doctrine.html', context=context)

hrapplications/admin.py

@@ -6,9 +6,22 @@ from hrapplications.models import ApplicationQuestion
 from hrapplications.models import ApplicationForm
 from hrapplications.models import ApplicationResponse
 from hrapplications.models import ApplicationComment
+from hrapplications.models import ApplicationChoice
+
+class ChoiceInline(admin.TabularInline):
+    model = ApplicationChoice
+    extra = 0
+    verbose_name_plural = 'Choices (optional)'
+    verbose_name= 'Choice'
+
+class QuestionAdmin(admin.ModelAdmin):
+    fieldsets = [
+            (None,      {'fields': ['title', 'help_text', 'multi_select']}),
+            ]
+    inlines = [ChoiceInline]
 
 admin.site.register(Application)
 admin.site.register(ApplicationComment)
-admin.site.register(ApplicationQuestion)
+admin.site.register(ApplicationQuestion, QuestionAdmin)
 admin.site.register(ApplicationForm)
 admin.site.register(ApplicationResponse)

hrapplications/migrations/0002_choices_for_questions.py

@@ -0,0 +1,33 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.4 on 2017-08-23 19:46
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('hrapplications', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='ApplicationChoice',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('choice_text', models.CharField(max_length=200, verbose_name='Choice')),
+            ],
+        ),
+        migrations.AlterField(
+            model_name='applicationquestion',
+            name='title',
+            field=models.CharField(max_length=254, verbose_name='Question'),
+        ),
+        migrations.AddField(
+            model_name='applicationchoice',
+            name='question',
+            field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='choices', to='hrapplications.ApplicationQuestion'),
+        ),
+    ]

hrapplications/migrations/0003_applicationquestion_multi_select.py

@@ -0,0 +1,20 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.10 on 2017-10-20 13:51
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('hrapplications', '0002_choices_for_questions'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='applicationquestion',
+            name='multi_select',
+            field=models.BooleanField(default=False),
+        ),
+    ]

hrapplications/models.py

@@ -11,13 +11,22 @@ from authentication.models import AuthServicesInfo
 
 @python_2_unicode_compatible
 class ApplicationQuestion(models.Model):
-    title = models.CharField(max_length=254)
+    title = models.CharField(max_length=254, verbose_name='Question')
     help_text = models.CharField(max_length=254, blank=True, null=True)
+    multi_select = models.BooleanField(default=False)
 
     def __str__(self):
         return "Question: " + self.title
 
 
+@python_2_unicode_compatible
+class ApplicationChoice(models.Model):
+    question = models.ForeignKey(ApplicationQuestion,on_delete=models.CASCADE,related_name="choices")
+    choice_text = models.CharField(max_length=200, verbose_name='Choice')
+
+    def __str__(self):
+        return self.choice_text
+
 @python_2_unicode_compatible
 class ApplicationForm(models.Model):
     questions = models.ManyToManyField(ApplicationQuestion)

hrapplications/views.py

@@ -39,13 +39,14 @@ def hr_application_management_view(request):
         except EveCharacter.DoesNotExist:
             pass
     if request.user.is_superuser:
-        corp_applications = Application.objects.filter(approved=None)
-        finished_corp_applications = Application.objects.exclude(approved=None)
+        corp_applications = Application.objects.filter(approved=None).order_by('-created')
+        finished_corp_applications = Application.objects.exclude(approved=None).order_by('-created')
     elif request.user.has_perm('auth.human_resources') and main_char:
         if ApplicationForm.objects.filter(corp__corporation_id=main_char.corporation_id).exists():
             app_form = ApplicationForm.objects.get(corp__corporation_id=main_char.corporation_id)
-            corp_applications = Application.objects.filter(form=app_form).filter(approved=None)
-            finished_corp_applications = Application.objects.filter(form=app_form).filter(approved__in=[True, False])
+            corp_applications = Application.objects.filter(form=app_form).filter(approved=None).sorted_by('-created')
+            finished_corp_applications = Application.objects.filter(form=app_form).filter(
+                approved__in=[True, False]).order_by('-created')
     logger.debug("Retrieved %s personal, %s corp applications for %s" % (
         len(request.user.applications.all()), len(corp_applications), request.user))
     context = {
@@ -71,8 +72,8 @@ def hr_application_create_view(request, form_id=None):
                 application.save()
                 for question in app_form.questions.all():
                     response = ApplicationResponse(question=question, application=application)
-                    response.answer = request.POST.get(str(question.pk),
-                                                       "Failed to retrieve answer provided by applicant.")
+                    response.answer = "\n".join(request.POST.getlist(str(question.pk),
+                                                       ""))
                     response.save()
                 logger.info("%s created %s" % (request.user, application))
             return redirect('auth_hrapplications_view')

optimer/form.py

@@ -6,9 +6,7 @@ from django.utils.translation import ugettext_lazy as _
 class opForm(forms.Form):
     doctrine = forms.CharField(max_length=254, required=True, label=_('Doctrine'))
     system = forms.CharField(max_length=254, required=True, label=_("System"))
-    location = forms.CharField(max_length=254, required=True, label=_("Location"))
     start = forms.DateTimeField(required=True, label=_("Start Time"))
     duration = forms.CharField(max_length=254, required=True, label=_("Duration"))
     operation_name = forms.CharField(max_length=254, required=True, label=_("Operation Name"))
     fc = forms.CharField(max_length=254, required=True, label=_("Fleet Commander"))
-    details = forms.CharField(max_length=254, required=False, label=_("Extra Details"))

optimer/migrations/0002_auto_20170413_0442.py

@@ -0,0 +1,23 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11 on 2017-04-13 04:42
+from __future__ import unicode_literals
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('optimer', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='optimer',
+            name='details',
+        ),
+        migrations.RemoveField(
+            model_name='optimer',
+            name='location',
+        ),
+    ]

optimer/models.py

@@ -13,15 +13,12 @@ class optimer(models.Model):
 
     doctrine = models.CharField(max_length=254, default="")
     system = models.CharField(max_length=254, default="")
-    location = models.CharField(max_length=254, default="")
     start = models.DateTimeField(default=datetime.now)
     duration = models.CharField(max_length=25, default="")
     operation_name = models.CharField(max_length=254, default="")
     fc = models.CharField(max_length=254, default="")
-    details = models.CharField(max_length=254, default="")
     post_time = models.DateTimeField(default=timezone.now)
     eve_character = models.ForeignKey(EveCharacter)
 
     def __str__(self):
-        output = self.operation_name
-        return output.encode('utf-8')
+        return self.operation_name

optimer/views.py

@@ -22,7 +22,11 @@ logger = logging.getLogger(__name__)
 @permission_required('auth.optimer_view')
 def optimer_view(request):
     logger.debug("optimer_view called by user %s" % request.user)
-    render_items = {'optimer': optimer.objects.all(), }
+    render_items = {'optimer': optimer.objects.all(),
+                    'future_timers': optimer.objects.all().filter(
+                        start__gte=timezone.now()),
+                    'past_timers': optimer.objects.all().filter(
+                        start__lt=timezone.now()).order_by('-start')}
 
     return render(request, 'registered/operationmanagement.html', context=render_items)
 
@@ -44,12 +48,10 @@ def add_optimer_view(request):
             op = optimer()
             op.doctrine = form.cleaned_data['doctrine']
             op.system = form.cleaned_data['system']
-            op.location = form.cleaned_data['location']
             op.start = form.cleaned_data['start']
             op.duration = form.cleaned_data['duration']
             op.operation_name = form.cleaned_data['operation_name']
             op.fc = form.cleaned_data['fc']
-            op.details = form.cleaned_data['details']
             op.create_time = post_time
             op.eve_character = character
             op.save()
@@ -93,12 +95,10 @@ def edit_optimer(request, optimer_id):
             character = EveManager.get_character_by_id(auth_info.main_char_id)
             op.doctrine = form.cleaned_data['doctrine']
             op.system = form.cleaned_data['system']
-            op.location = form.cleaned_data['location']
             op.start = form.cleaned_data['start']
             op.duration = form.cleaned_data['duration']
             op.operation_name = form.cleaned_data['operation_name']
             op.fc = form.cleaned_data['fc']
-            op.details = form.cleaned_data['details']
             op.eve_character = character
             logger.info("User %s updating optimer id %s " % (request.user, optimer_id))
             op.save()
@@ -108,12 +108,10 @@ def edit_optimer(request, optimer_id):
         data = {
             'doctrine': op.doctrine,
             'system': op.system,
-            'location': op.location,
             'start': op.start,
             'duration': op.duration,
             'operation_name': op.operation_name,
             'fc': op.fc,
-            'details': op.details,
         }
         form = opForm(initial=data)
     return render(request, 'registered/optimerupdate.html', context={'form': form})

requirements.txt

@@ -5,25 +5,22 @@ dnspython
 passlib
 requests>=2.9.1
 bcrypt
-slugify
+python-slugify>=1.2
 requests-oauthlib
 sleekxmpp
 redis
+celery>=4.0.2
 
 # Django Stuff #
 django>=1.10,<2.0
 django-bootstrap-form
 django-navhelper
 django-bootstrap-pagination
-django-redis>=4.4
-
-# awating release for fix to celery/django-celery#447
-# django-celery
-git+https://github.com/celery/django-celery
-
-git+git://github.com/nikdoof/python-ts3.git
+django-redis-cache>=1.7.1
+django-recaptcha
+django-celery-beat
 
 # awating pyghassen/openfire-restapi #1 to fix installation issues
 git+https://github.com/adarnof/openfire-restapi
 
-git+https://github.com/adarnof/adarnauth-esi
+adarnauth-esi>=1.4.1,<2.0

run_alliance_corp_update.py

@@ -1,5 +1,5 @@
 from __future__ import unicode_literals
 from eveonline.tasks import run_corp_update
 
-run_corp_update()
+run_corp_update.apply()
 quit()

services/admin.py

@@ -1,5 +0,0 @@
-from __future__ import unicode_literals
-from django.contrib import admin
-from services.models import GroupCache
-
-admin.site.register(GroupCache)

services/context_processors.py

@@ -15,6 +15,7 @@ def auth_settings(request):
         'IPS4_URL': settings.IPS4_URL,
         'SMF_URL': settings.SMF_URL,
         'MARKET_URL': settings.MARKET_URL,
+        'SEAT_URL': settings.SEAT_URL,
         'EXTERNAL_MEDIA_URL': settings.EXTERNAL_MEDIA_URL,
         'CURRENT_UTC_TIME': timezone.now(),
         'BLUE_API_MASK': settings.BLUE_API_MASK,

services/managers/srp_manager.py

@@ -27,10 +27,11 @@ class srpManager:
         r = requests.get(url, headers=headers)
         result = r.json()[0]
         if result:
-            ship_type = result['victim']['shipTypeID']
-            logger.debug("Ship type for kill ID %s is determined to be %s" % (kill_id, ship_type))
+            ship_type = result['victim']['ship_type_id']
+            logger.debug("Ship type for kill ID %s is %s" % (kill_id, ship_type))
             ship_value = result['zkb']['totalValue']
-            logger.debug("total loss value for kill id %s is %s" % (kill_id, ship_value))
-            return ship_type, ship_value
+            logger.debug("Total loss value for kill id %s is %s" % (kill_id, ship_value))
+            victim_id = result['victim']['character_id']
+            return ship_type, ship_value, victim_id
         else:
             raise ValueError("Invalid Kill ID")

services/migrations/0003_delete_groupcache.py

@@ -0,0 +1,18 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.10.5 on 2017-09-02 06:07
+from __future__ import unicode_literals
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('services', '0002_auto_20161016_0135'),
+    ]
+
+    operations = [
+        migrations.DeleteModel(
+            name='GroupCache',
+        ),
+    ]

services/models.py

@@ -1,18 +0,0 @@
-from __future__ import unicode_literals
-from django.utils.encoding import python_2_unicode_compatible
-from django.db import models
-
-
-@python_2_unicode_compatible
-class GroupCache(models.Model):
-    SERVICE_CHOICES = (
-        ("discourse", "discourse"),
-        ("discord", "discord"),
-    )
-
-    created = models.DateTimeField(auto_now_add=True)
-    groups = models.TextField(default={})
-    service = models.CharField(max_length=254, choices=SERVICE_CHOICES, unique=True)
-
-    def __str__(self):
-        return self.service

services/modules/discord/manager.py

@@ -1,13 +1,15 @@
 from __future__ import unicode_literals
-import requests
-import json
 import re
+import requests
+import math
 from django.conf import settings
-from services.models import GroupCache
 from requests_oauthlib import OAuth2Session
+from functools import wraps
 import logging
 import datetime
-from django.utils import timezone
+import time
+from django.core.cache import cache
+from hashlib import md5
 
 logger = logging.getLogger(__name__)
 
@@ -17,9 +19,13 @@ EVE_IMAGE_SERVER = "https://image.eveonline.com"
 AUTH_URL = "https://discordapp.com/api/oauth2/authorize"
 TOKEN_URL = "https://discordapp.com/api/oauth2/token"
 
-# needs administrator, since Discord can't get their permissions system to work
-# was kick members, manage roles, manage nicknames
-#BOT_PERMISSIONS = 0x00000002 + 0x10000000 + 0x08000000
+"""
+Previously all we asked for was permission to kick members, manage roles, and manage nicknames.
+Users have reported weird unauthorized errors we don't understand. So now we ask for full server admin.
+It's almost fixed the problem.
+"""
+# kick members, manage roles, manage nicknames, create instant invite
+# BOT_PERMISSIONS = 0x00000002 + 0x10000000 + 0x08000000 + 0x00000001
 BOT_PERMISSIONS = 0x00000008
 
 # get user ID, accept invite
@@ -28,7 +34,126 @@ SCOPES = [
     'guilds.join',
 ]
 
-GROUP_CACHE_MAX_AGE = datetime.timedelta(minutes=30)
+GROUP_CACHE_MAX_AGE = int(getattr(settings, 'DISCORD_GROUP_CACHE_MAX_AGE', 2 * 60 * 60))  # 2 hours default
+
+
+class DiscordApiException(Exception):
+    def __init__(self):
+        super(Exception, self).__init__()
+
+
+class DiscordApiTooBusy(DiscordApiException):
+    def __init__(self):
+        super(DiscordApiException, self).__init__()
+        self.message = "The Discord API is too busy to process this request now, please try again later."
+
+
+class DiscordApiBackoff(DiscordApiException):
+    def __init__(self, retry_after, global_ratelimit):
+        """
+        :param retry_after: int time to retry after in milliseconds
+        :param global_ratelimit: bool Is the API under a global backoff
+        """
+        super(DiscordApiException, self).__init__()
+        self.retry_after = retry_after
+        self.global_ratelimit = global_ratelimit
+
+    @property
+    def retry_after_seconds(self):
+        return math.ceil(self.retry_after / 1000)
+
+
+cache_time_format = '%Y-%m-%d %H:%M:%S.%f'
+
+
+def api_backoff(func):
+    """
+    Decorator, Handles HTTP 429 "Too Many Requests" messages from the Discord API
+    If blocking=True is specified, this function will block and retry
+    the function up to max_retries=n times, or 3 if retries is not specified.
+    If the API call still recieves a backoff timer this function will raise
+    a <DiscordApiTooBusy> exception.
+    If the caller chooses blocking=False, the decorator will raise a DiscordApiBackoff
+    exception and the caller can choose to retry after the given timespan available in
+    the retry_after property in seconds.
+    """
+
+    class PerformBackoff(Exception):
+        def __init__(self, retry_after, retry_datetime, global_ratelimit):
+            super(Exception, self).__init__()
+            self.retry_after = int(retry_after)
+            self.retry_datetime = retry_datetime
+            self.global_ratelimit = global_ratelimit
+
+    @wraps(func)
+    def decorated(*args, **kwargs):
+        blocking = kwargs.get('blocking', False)
+        retries = kwargs.get('max_retries', 3)
+
+        # Strip our parameters
+        if 'max_retries' in kwargs:
+            del kwargs['max_retries']
+        if 'blocking' in kwargs:
+            del kwargs['blocking']
+
+        cache_key = 'DISCORD_BACKOFF_' + func.__name__
+        cache_global_key = 'DISCORD_BACKOFF_GLOBAL'
+
+        while retries > 0:
+            try:
+                try:
+                    # Check global backoff first, then route backoff
+                    existing_global_backoff = cache.get(cache_global_key)
+                    existing_backoff = existing_global_backoff or cache.get(cache_key)
+                    if existing_backoff:
+                        backoff_timer = datetime.datetime.strptime(existing_backoff, cache_time_format)
+                        if backoff_timer > datetime.datetime.utcnow():
+                            backoff_seconds = (backoff_timer - datetime.datetime.utcnow()).total_seconds()
+                            logger.debug("Still under backoff for {} seconds, backing off" % backoff_seconds)
+                            # Still under backoff
+                            raise PerformBackoff(
+                                retry_after=backoff_seconds,
+                                retry_datetime=backoff_timer,
+                                global_ratelimit=bool(existing_global_backoff)
+                            )
+                    logger.debug("Calling API calling function")
+                    return func(*args, **kwargs)
+                except requests.HTTPError as e:
+                    if e.response.status_code == 429:
+                        try:
+                            retry_after = int(e.response.headers['Retry-After'])
+                        except (TypeError, KeyError):
+                            # Pick some random time
+                            retry_after = 5000
+
+                        logger.info("Received backoff from API of %s seconds, handling" % retry_after)
+                        # Store value in redis
+                        backoff_until = (datetime.datetime.utcnow() +
+                                         datetime.timedelta(milliseconds=retry_after))
+                        global_backoff = bool(e.response.headers.get('X-RateLimit-Global', False))
+                        if global_backoff:
+                            logger.info("Global backoff!!")
+                            cache.set(cache_global_key, backoff_until.strftime(cache_time_format), retry_after)
+                        else:
+                            cache.set(cache_key, backoff_until.strftime(cache_time_format), retry_after)
+                        raise PerformBackoff(retry_after=retry_after, retry_datetime=backoff_until,
+                                             global_ratelimit=global_backoff)
+                    else:
+                        # Not 429, re-raise
+                        raise e
+            except PerformBackoff as bo:
+                # Sleep if we're blocking
+                if blocking:
+                    logger.info("Blocking Back off from API calls for %s seconds" % bo.retry_after)
+                    time.sleep((10 if bo.retry_after > 10 else bo.retry_after) / 1000)
+                else:
+                    # Otherwise raise exception and let caller handle the backoff
+                    raise DiscordApiBackoff(retry_after=bo.retry_after, global_ratelimit=bo.global_ratelimit)
+            finally:
+                retries -= 1
+        if retries == 0:
+            raise DiscordApiTooBusy()
+    return decorated
 
 
 class DiscordOAuthManager:
@@ -36,9 +161,12 @@ class DiscordOAuthManager:
         pass
 
     @staticmethod
-    def _sanitize_groupname(name):
-        name = name.strip(' _')
-        return re.sub('[^\w.-]', '', name)
+    def _sanitize_name(name):
+        return name[:32]
+
+    @staticmethod
+    def _sanitize_group_name(name):
+        return name[:100]
 
     @staticmethod
     def generate_bot_add_url():
@@ -57,23 +185,33 @@ class DiscordOAuthManager:
         return token
 
     @staticmethod
-    def add_user(code):
+    def add_user(code, groups, nickname=None):
         try:
             token = DiscordOAuthManager._process_callback_code(code)['access_token']
             logger.debug("Received token from OAuth")
 
             custom_headers = {'accept': 'application/json', 'authorization': 'Bearer ' + token}
-            path = DISCORD_URL + "/invites/" + str(settings.DISCORD_INVITE_CODE)
-            r = requests.post(path, headers=custom_headers)
-            logger.debug("Got status code %s after accepting Discord invite" % r.status_code)
-            r.raise_for_status()
-
             path = DISCORD_URL + "/users/@me"
             r = requests.get(path, headers=custom_headers)
             logger.debug("Got status code %s after retrieving Discord profile" % r.status_code)
             r.raise_for_status()
 
             user_id = r.json()['id']
+
+            path = DISCORD_URL + "/guilds/" + str(settings.DISCORD_GUILD_ID) + "/members/" + str(user_id)
+            group_ids = [DiscordOAuthManager._group_name_to_id(DiscordOAuthManager._sanitize_group_name(g)) for g in
+                         groups]
+            data = {
+                'roles': group_ids,
+                'access_token': token,
+            }
+            if nickname:
+                data['nick'] = nickname
+            custom_headers['authorization'] = 'Bot ' + settings.DISCORD_BOT_TOKEN
+            r = requests.put(path, headers=custom_headers, json=data)
+            logger.debug("Got status code %s after joining Discord server" % r.status_code)
+            r.raise_for_status()
+
             logger.info("Added Discord user ID %s to server." % user_id)
             return user_id
         except:
@@ -81,10 +219,12 @@ class DiscordOAuthManager:
             return None
 
     @staticmethod
+    @api_backoff
     def update_nickname(user_id, nickname):
         try:
+            nickname = DiscordOAuthManager._sanitize_name(nickname)
             custom_headers = {'content-type': 'application/json', 'authorization': 'Bot ' + settings.DISCORD_BOT_TOKEN}
-            data = {'nick': nickname, }
+            data = {'nick': nickname}
             path = DISCORD_URL + "/guilds/" + str(settings.DISCORD_GUILD_ID) + "/members/" + str(user_id)
             r = requests.patch(path, headers=custom_headers, json=data)
             logger.debug("Got status code %s after setting nickname for Discord user ID %s (%s)" % (
@@ -115,7 +255,7 @@ class DiscordOAuthManager:
             return False
 
     @staticmethod
-    def __get_groups():
+    def _get_groups():
         custom_headers = {'accept': 'application/json', 'authorization': 'Bot ' + settings.DISCORD_BOT_TOKEN}
         path = DISCORD_URL + "/guilds/" + str(settings.DISCORD_GUILD_ID) + "/roles"
         r = requests.get(path, headers=custom_headers)
@@ -124,76 +264,50 @@ class DiscordOAuthManager:
         return r.json()
 
     @staticmethod
-    def __update_group_cache():
-        GroupCache.objects.filter(service="discord").delete()
-        cache = GroupCache.objects.create(service="discord")
-        cache.groups = json.dumps(DiscordOAuthManager.__get_groups())
-        cache.save()
-        return cache
+    def _generate_cache_role_key(name):
+        return 'DISCORD_ROLE_NAME__%s' % md5(str(name).encode('utf-8')).hexdigest()
 
     @staticmethod
-    def __get_group_cache():
-        if not GroupCache.objects.filter(service="discord").exists():
-            DiscordOAuthManager.__update_group_cache()
-        cache = GroupCache.objects.get(service="discord")
-        age = timezone.now() - cache.created
-        if age > GROUP_CACHE_MAX_AGE:
-            logger.debug("Group cache has expired. Triggering update.")
-            cache = DiscordOAuthManager.__update_group_cache()
-        return json.loads(cache.groups)
+    def _group_name_to_id(name):
+        name = DiscordOAuthManager._sanitize_group_name(name)
+
+        def get_or_make_role():
+            groups = DiscordOAuthManager._get_groups()
+            for g in groups:
+                if g['name'] == name:
+                    return g['id']
+            return DiscordOAuthManager._create_group(name)['id']
+        return cache.get_or_set(DiscordOAuthManager._generate_cache_role_key(name), get_or_make_role, GROUP_CACHE_MAX_AGE)
 
     @staticmethod
-    def __group_name_to_id(name):
-        cache = DiscordOAuthManager.__get_group_cache()
-        for g in cache:
-            if g['name'] == name:
-                return g['id']
-        logger.debug("Group %s not found on Discord. Creating" % name)
-        DiscordOAuthManager.__create_group(name)
-        return DiscordOAuthManager.__group_name_to_id(name)
-
-    @staticmethod
-    def __group_id_to_name(id):
-        cache = DiscordOAuthManager.__get_group_cache()
-        for g in cache:
-            if g['id'] == id:
-                return g['name']
-        raise KeyError("Group ID %s not found on Discord" % id)
-
-    @staticmethod
-    def __generate_role():
+    def __generate_role(name, **kwargs):
         custom_headers = {'accept': 'application/json', 'authorization': 'Bot ' + settings.DISCORD_BOT_TOKEN}
         path = DISCORD_URL + "/guilds/" + str(settings.DISCORD_GUILD_ID) + "/roles"
-        r = requests.post(path, headers=custom_headers)
+        data = {'name': name}
+        data.update(kwargs)
+        r = requests.post(path, headers=custom_headers, json=data)
         logger.debug("Received status code %s after generating new role." % r.status_code)
         r.raise_for_status()
         return r.json()
 
     @staticmethod
-    def __edit_role(role_id, name, color=0, hoist=True, permissions=36785152):
+    def __edit_role(role_id, **kwargs):
         custom_headers = {'content-type': 'application/json', 'authorization': 'Bot ' + settings.DISCORD_BOT_TOKEN}
-        data = {
-            'color': color,
-            'hoist': hoist,
-            'name': name,
-            'permissions': permissions,
-        }
         path = DISCORD_URL + "/guilds/" + str(settings.DISCORD_GUILD_ID) + "/roles/" + str(role_id)
-        r = requests.patch(path, headers=custom_headers, data=json.dumps(data))
+        r = requests.patch(path, headers=custom_headers, json=kwargs)
         logger.debug("Received status code %s after editing role id %s" % (r.status_code, role_id))
         r.raise_for_status()
         return r.json()
 
     @staticmethod
-    def __create_group(name):
-        role = DiscordOAuthManager.__generate_role()
-        DiscordOAuthManager.__edit_role(role['id'], name)
-        DiscordOAuthManager.__update_group_cache()
+    def _create_group(name):
+        return DiscordOAuthManager.__generate_role(name)
 
     @staticmethod
+    @api_backoff
     def update_groups(user_id, groups):
         custom_headers = {'content-type': 'application/json', 'authorization': 'Bot ' + settings.DISCORD_BOT_TOKEN}
-        group_ids = [DiscordOAuthManager.__group_name_to_id(DiscordOAuthManager._sanitize_groupname(g)) for g in groups]
+        group_ids = [DiscordOAuthManager._group_name_to_id(DiscordOAuthManager._sanitize_group_name(g)) for g in groups]
         path = DISCORD_URL + "/guilds/" + str(settings.DISCORD_GUILD_ID) + "/members/" + str(user_id)
         data = {'roles': group_ids}
         r = requests.patch(path, headers=custom_headers, json=data)

services/modules/discord/tasks.py

@@ -6,11 +6,10 @@ from alliance_auth.celeryapp import app
 from django.conf import settings
 from django.contrib.auth.models import User
 from django.core.exceptions import ObjectDoesNotExist
-
+from requests.exceptions import HTTPError
 from eveonline.managers import EveManager
 from notifications import notify
-from services.modules.discord.manager import DiscordOAuthManager
-from services.tasks import only_one
+from services.modules.discord.manager import DiscordOAuthManager, DiscordApiBackoff
 from .models import DiscordUser
 
 logger = logging.getLogger(__name__)
@@ -22,15 +21,16 @@ class DiscordTasks:
 
     @classmethod
     def add_user(cls, user, code):
-        user_id = DiscordOAuthManager.add_user(code)
+        groups = DiscordTasks.get_groups(user)
+        nickname = None
+        if settings.DISCORD_SYNC_NAMES:
+            nickname = DiscordTasks.get_nickname(user)
+        user_id = DiscordOAuthManager.add_user(code, groups, nickname=nickname)
         if user_id:
             discord_user = DiscordUser()
             discord_user.user = user
             discord_user.uid = user_id
             discord_user.save()
-            if settings.DISCORD_SYNC_NAMES:
-                cls.update_nickname.delay(user.pk)
-            cls.update_groups.delay(user.pk)
             return True
         return False
 
@@ -65,15 +65,23 @@ class DiscordTasks:
         user = User.objects.get(pk=pk)
         logger.debug("Updating discord groups for user %s" % user)
         if DiscordTasks.has_account(user):
-            groups = []
-            for group in user.groups.all():
-                groups.append(str(group.name))
-            if len(groups) == 0:
-                logger.debug("No syncgroups found for user. Adding empty group.")
-                groups.append('empty')
+            groups = DiscordTasks.get_groups(user)
             logger.debug("Updating user %s discord groups to %s" % (user, groups))
             try:
                 DiscordOAuthManager.update_groups(user.discord.uid, groups)
+            except DiscordApiBackoff as bo:
+                logger.info("Discord group sync API back off for %s, "
+                            "retrying in %s seconds" % (user, bo.retry_after_seconds))
+                raise task_self.retry(countdown=bo.retry_after_seconds)
+            except HTTPError as e:
+                if e.response.status_code == 404:
+                    try:
+                        if e.response.json()['code'] == 10007:
+                            # user has left the server
+                            DiscordTasks.delete_user(user)
+                            return
+                    finally:
+                        raise e
             except Exception as e:
                 if task_self:
                     logger.exception("Discord group sync failed for %s, retrying in 10 mins" % user)
@@ -94,18 +102,22 @@ class DiscordTasks:
 
     @staticmethod
     @app.task(bind=True, name='discord.update_nickname')
-    def update_nickname(self, pk):
+    def update_nickname(task_self, pk):
         user = User.objects.get(pk=pk)
         logger.debug("Updating discord nickname for user %s" % user)
         if DiscordTasks.has_account(user):
-            character = EveManager.get_main_character(user)
-            logger.debug("Updating user %s discord nickname to %s" % (user, character.character_name))
+            name = DiscordTasks.get_nickname(user)
+            logger.debug("Updating user %s discord nickname to %s" % (user, name))
             try:
-                DiscordOAuthManager.update_nickname(user.discord.uid, character.character_name)
+                DiscordOAuthManager.update_nickname(user.discord.uid, name)
+            except DiscordApiBackoff as bo:
+                logger.info("Discord nickname update API back off for %s, "
+                            "retrying in %s seconds" % (user, bo.retry_after_seconds))
+                raise task_self.retry(countdown=bo.retry_after_seconds)
             except Exception as e:
-                if self:
+                if task_self:
                     logger.exception("Discord nickname sync failed for %s, retrying in 10 mins" % user)
-                    raise self.retry(countdown=60 * 10)
+                    raise task_self.retry(countdown=60 * 10)
                 else:
                     # Rethrow
                     raise e
@@ -118,8 +130,16 @@ class DiscordTasks:
     def update_all_nicknames():
         logger.debug("Updating ALL discord nicknames")
         for discord_user in DiscordUser.objects.exclude(uid__exact=''):
-            DiscordTasks.update_nickname.delay(discord_user.user.user_id)
+            DiscordTasks.update_nickname.delay(discord_user.user.pk)
 
     @classmethod
     def disable(cls):
         DiscordUser.objects.all().delete()
+
+    @staticmethod
+    def get_nickname(user):
+        return EveManager.get_main_character(user).character_name
+
+    @staticmethod
+    def get_groups(user):
+        return [g.name for g in user.groups.all()]

services/modules/discord/templates/registered/discord_service_ctrl.html

@@ -7,14 +7,14 @@
     <td class="text-center"><a href="https://discordapp.com/channels/{{ DISCORD_SERVER_ID }}/{{ DISCORD_SERVER_ID}}">https://discordapp.com</a></td>
     <td class="text-center">
         {% if not discord_uid %}
-            <a href="{% url 'auth_activate_discord' %}" class="btn btn-warning">
+            <a href="{% url 'auth_activate_discord' %}" title="Activate" class="btn btn-warning">
                 <span class="glyphicon glyphicon-ok"></span>
             </a>
         {% else %}
-            <a href="{% url 'auth_reset_discord' %}" class="btn btn-primary">
+            <a href="{% url 'auth_reset_discord' %}" title="Reset" class="btn btn-primary">
                 <span class="glyphicon glyphicon-refresh"></span>
             </a>
-            <a href="{% url 'auth_deactivate_discord' %}" class="btn btn-danger">
+            <a href="{% url 'auth_deactivate_discord' %}" title="Deactivate" class="btn btn-danger">
                 <span class="glyphicon glyphicon-remove"></span>
             </a>
         {% endif %}

services/modules/discord/tests.py

@@ -17,6 +17,10 @@ from alliance_auth.tests.auth_utils import AuthUtils
 from .auth_hooks import DiscordService
 from .models import DiscordUser
 from .tasks import DiscordTasks
+from .manager import DiscordOAuthManager
+
+import requests_mock
+import datetime
 
 MODULE_PATH = 'services.modules.discord'
 
@@ -198,3 +202,256 @@ class DiscordViewsTestCase(TestCase):
         self.assertRedirects(response, expected_url='/en/services/', target_status_code=200)
         with self.assertRaises(ObjectDoesNotExist):
             discord_user = User.objects.get(pk=self.member.pk).discord
+
+
+class DiscordManagerTestCase(TestCase):
+    def setUp(self):
+        pass
+
+    def test__sanitize_group_name(self):
+        test_group_name = str(10**103)
+        group_name = DiscordOAuthManager._sanitize_group_name(test_group_name)
+
+        self.assertEqual(group_name, test_group_name[:100])
+
+    def test_generate_Bot_add_url(self):
+        from . import manager
+        bot_add_url = DiscordOAuthManager.generate_bot_add_url()
+
+        auth_url = manager.AUTH_URL
+        real_bot_add_url = '{}?client_id=appid&scope=bot&permissions={}'.format(auth_url, manager.BOT_PERMISSIONS)
+        self.assertEqual(bot_add_url, real_bot_add_url)
+
+    def test_generate_oauth_redirect_url(self):
+        from . import manager
+        import urllib
+        import sys
+        oauth_url = DiscordOAuthManager.generate_oauth_redirect_url()
+
+        self.assertIn(manager.AUTH_URL, oauth_url)
+        self.assertIn('+'.join(manager.SCOPES), oauth_url)
+        self.assertIn(settings.DISCORD_APP_ID, oauth_url)
+        if sys.version_info[0] < 3:
+            # Py2
+            self.assertIn(urllib.quote_plus(settings.DISCORD_CALLBACK_URL), oauth_url)
+        else: # Py3
+            self.assertIn(urllib.parse.quote_plus(settings.DISCORD_CALLBACK_URL), oauth_url)
+
+    @mock.patch(MODULE_PATH + '.manager.OAuth2Session')
+    def test__process_callback_code(self, oauth):
+        from . import manager
+        instance = oauth.return_value
+        instance.fetch_token.return_value = {'access_token': 'mywonderfultoken'}
+
+        token = DiscordOAuthManager._process_callback_code('12345')
+
+        self.assertTrue(oauth.called)
+        args, kwargs = oauth.call_args
+        self.assertEqual(args[0], settings.DISCORD_APP_ID)
+        self.assertEqual(kwargs['redirect_uri'], settings.DISCORD_CALLBACK_URL)
+        self.assertTrue(instance.fetch_token.called)
+        args, kwargs = instance.fetch_token.call_args
+        self.assertEqual(args[0], manager.TOKEN_URL)
+        self.assertEqual(kwargs['client_secret'], settings.DISCORD_APP_SECRET)
+        self.assertEqual(kwargs['code'], '12345')
+        self.assertEqual(token['access_token'], 'mywonderfultoken')
+
+    @mock.patch(MODULE_PATH + '.manager.DiscordOAuthManager._process_callback_code')
+    @requests_mock.Mocker()
+    def test_add_user(self, oauth_token, m):
+        from . import manager
+        import json
+
+        # Arrange
+        oauth_token.return_value = {'access_token': 'accesstoken'}
+
+        headers = {'accept': 'application/json', 'authorization': 'Bearer accesstoken'}
+
+        m.register_uri('GET',
+                       manager.DISCORD_URL + "/users/@me",
+                       request_headers=headers,
+                       text=json.dumps({'id': "123456"}))
+
+        headers = {'accept': 'application/json', 'authorization': 'Bot ' + settings.DISCORD_BOT_TOKEN}
+
+        m.register_uri('PUT',
+                       manager.DISCORD_URL + '/guilds/' + str(settings.DISCORD_GUILD_ID) + '/members/123456',
+                       request_headers=headers,
+                       text='{}')
+
+        # Act
+        return_value = DiscordOAuthManager.add_user('abcdef', [])
+
+        # Assert
+        self.assertEqual(return_value, '123456')
+        self.assertEqual(m.call_count, 2)
+
+    @requests_mock.Mocker()
+    def test_delete_user(self, m):
+        from . import manager
+        import json
+
+        # Arrange
+        headers = {'accept': 'application/json', 'authorization': 'Bot ' + settings.DISCORD_BOT_TOKEN}
+
+        user_id = 12345
+        request_url = '{}/guilds/{}/members/{}'.format(manager.DISCORD_URL, settings.DISCORD_GUILD_ID, user_id)
+        m.register_uri('DELETE',
+                       request_url,
+                       request_headers=headers,
+                       text=json.dumps({}))
+
+        # Act
+        result = DiscordOAuthManager.delete_user(user_id)
+
+        # Assert
+        self.assertTrue(result)
+
+        ###
+        # Test 404 (already deleted)
+        # Arrange
+        m.register_uri('DELETE',
+                       request_url,
+                       request_headers=headers,
+                       status_code=404)
+
+        # Act
+        result = DiscordOAuthManager.delete_user(user_id)
+
+        # Assert
+        self.assertTrue(result)
+
+        ###
+        # Test 500 (some random API error)
+        # Arrange
+        m.register_uri('DELETE',
+                       request_url,
+                       request_headers=headers,
+                       status_code=500)
+
+        # Act
+        result = DiscordOAuthManager.delete_user(user_id)
+
+        # Assert
+        self.assertFalse(result)
+
+    @requests_mock.Mocker()
+    def test_update_nickname(self, m):
+        from . import manager
+        import json
+        # Arrange
+        headers = {'content-type': 'application/json', 'authorization': 'Bot ' + settings.DISCORD_BOT_TOKEN}
+
+        user_id = 12345
+        request_url = '{}/guilds/{}/members/{}'.format(manager.DISCORD_URL, settings.DISCORD_GUILD_ID, user_id)
+        m.patch(request_url,
+                request_headers=headers)
+
+        # Act
+        result = DiscordOAuthManager.update_nickname(user_id, 'somenick')
+
+        # Assert
+        self.assertTrue(result)
+
+    @mock.patch(MODULE_PATH + '.manager.DiscordOAuthManager._get_groups')
+    @requests_mock.Mocker()
+    def test_update_groups(self, group_cache, m):
+        from . import manager
+        import json
+
+        # Arrange
+        groups = ['Member', 'Blue', 'SpecialGroup']
+
+        group_cache.return_value = [{'id': 111, 'name': 'Member'},
+                                    {'id': 222, 'name': 'Blue'},
+                                    {'id': 333, 'name': 'SpecialGroup'},
+                                    {'id': 444, 'name': 'NotYourGroup'}]
+
+        headers = {'content-type': 'application/json', 'authorization': 'Bot ' + settings.DISCORD_BOT_TOKEN}
+        user_id = 12345
+        request_url = '{}/guilds/{}/members/{}'.format(manager.DISCORD_URL, settings.DISCORD_GUILD_ID, user_id)
+
+        m.patch(request_url,
+                request_headers=headers)
+
+        # Act
+        DiscordOAuthManager.update_groups(user_id, groups)
+
+        # Assert
+        self.assertEqual(len(m.request_history), 1, 'Must be one HTTP call made')
+        history = json.loads(m.request_history[0].text)
+        self.assertIn('roles', history, "'The request must send JSON object with the 'roles' key")
+        self.assertIn(111, history['roles'], 'The group id 111 must be added to the request')
+        self.assertIn(222, history['roles'], 'The group id 222 must be added to the request')
+        self.assertIn(333, history['roles'], 'The group id 333 must be added to the request')
+        self.assertNotIn(444, history['roles'], 'The group id 444 must NOT be added to the request')
+
+    @mock.patch(MODULE_PATH + '.manager.cache')
+    @mock.patch(MODULE_PATH + '.manager.DiscordOAuthManager._get_groups')
+    @requests_mock.Mocker()
+    def test_update_groups_backoff(self, group_cache, djcache, m):
+        from . import manager
+
+        # Arrange
+        groups = ['Member']
+        group_cache.return_value = [{'id': 111, 'name': 'Member'}]
+
+        headers = {'content-type': 'application/json', 'authorization': 'Bot ' + settings.DISCORD_BOT_TOKEN}
+        user_id = 12345
+        request_url = '{}/guilds/{}/members/{}'.format(manager.DISCORD_URL, settings.DISCORD_GUILD_ID, user_id)
+
+        djcache.get.return_value = None  # No existing backoffs in cache
+
+        m.patch(request_url,
+                request_headers=headers,
+                headers={'Retry-After': '200000'},
+                status_code=429)
+
+        # Act & Assert
+        with self.assertRaises(manager.DiscordApiBackoff) as bo:
+            try:
+                DiscordOAuthManager.update_groups(user_id, groups, blocking=False)
+            except manager.DiscordApiBackoff as bo:
+                self.assertEqual(bo.retry_after, 200000, 'Retry-After time must be equal to Retry-After set in header')
+                self.assertFalse(bo.global_ratelimit, 'global_ratelimit must be False')
+                raise bo
+
+        self.assertTrue(djcache.set.called)
+        args, kwargs = djcache.set.call_args
+        self.assertEqual(args[0], 'DISCORD_BACKOFF_update_groups')
+        self.assertTrue(datetime.datetime.strptime(args[1], manager.cache_time_format) > datetime.datetime.now())
+
+    @mock.patch(MODULE_PATH + '.manager.cache')
+    @mock.patch(MODULE_PATH + '.manager.DiscordOAuthManager._get_groups')
+    @requests_mock.Mocker()
+    def test_update_groups_global_backoff(self, group_cache, djcache, m):
+        from . import manager
+
+        # Arrange
+        groups = ['Member']
+        group_cache.return_value = [{'id': 111, 'name': 'Member'}]
+
+        headers = {'content-type': 'application/json', 'authorization': 'Bot ' + settings.DISCORD_BOT_TOKEN}
+        user_id = 12345
+        request_url = '{}/guilds/{}/members/{}'.format(manager.DISCORD_URL, settings.DISCORD_GUILD_ID, user_id)
+
+        djcache.get.return_value = None  # No existing backoffs in cache
+
+        m.patch(request_url,
+                request_headers=headers,
+                headers={'Retry-After': '200000', 'X-RateLimit-Global': 'true'},
+                status_code=429)
+
+        # Act & Assert
+        with self.assertRaises(manager.DiscordApiBackoff) as bo:
+            try:
+                DiscordOAuthManager.update_groups(user_id, groups, blocking=False)
+            except manager.DiscordApiBackoff as bo:
+                self.assertEqual(bo.retry_after, 200000, 'Retry-After time must be equal to Retry-After set in header')
+                self.assertTrue(bo.global_ratelimit, 'global_ratelimit must be True')
+                raise bo
+
+        self.assertTrue(djcache.set.called)
+        args, kwargs = djcache.set.call_args
+        self.assertEqual(args[0], 'DISCORD_BACKOFF_GLOBAL')
+        self.assertTrue(datetime.datetime.strptime(args[1], manager.cache_time_format) > datetime.datetime.now())

services/modules/discourse/manager.py

@@ -1,17 +1,15 @@
 from __future__ import unicode_literals
 import logging
 import requests
-import random
-import string
-import datetime
-import json
 import re
 from django.conf import settings
-from django.utils import timezone
-from services.models import GroupCache
+from django.core.cache import cache
+from hashlib import md5
 
 logger = logging.getLogger(__name__)
 
+GROUP_CACHE_MAX_AGE = int(getattr(settings, 'DISCOURSE_GROUP_CACHE_MAX_AGE', 2 * 60 * 60))  # default 2 hours
+
 
 class DiscourseError(Exception):
     def __init__(self, endpoint, errors):
@@ -21,12 +19,13 @@ class DiscourseError(Exception):
     def __str__(self):
         return "API execution failed.\nErrors: %s\nEndpoint: %s" % (self.errors, self.endpoint)
 
+
 # not exhaustive, only the ones we need
 ENDPOINTS = {
     'groups': {
         'list': {
             'path': "/admin/groups.json",
-            'method': requests.get,
+            'method': 'get',
             'args': {
                 'required': [],
                 'optional': [],
@@ -34,7 +33,7 @@ ENDPOINTS = {
         },
         'create': {
             'path': "/admin/groups",
-            'method': requests.post,
+            'method': 'post',
             'args': {
                 'required': ['name'],
                 'optional': ['visible'],
@@ -42,7 +41,7 @@ ENDPOINTS = {
         },
         'add_user': {
             'path': "/admin/groups/%s/members.json",
-            'method': requests.put,
+            'method': 'put',
             'args': {
                 'required': ['usernames'],
                 'optional': [],
@@ -50,7 +49,7 @@ ENDPOINTS = {
         },
         'remove_user': {
             'path': "/admin/groups/%s/members.json",
-            'method': requests.delete,
+            'method': 'delete',
             'args': {
                 'required': ['username'],
                 'optional': [],
@@ -58,7 +57,7 @@ ENDPOINTS = {
         },
         'delete': {
             'path': "/admin/groups/%s.json",
-            'method': requests.delete,
+            'method': 'delete',
             'args': {
                 'required': [],
                 'optional': [],
@@ -68,7 +67,7 @@ ENDPOINTS = {
     'users': {
         'create': {
             'path': "/users",
-            'method': requests.post,
+            'method': 'post',
             'args': {
                 'required': ['name', 'email', 'password', 'username'],
                 'optional': ['active'],
@@ -76,7 +75,7 @@ ENDPOINTS = {
         },
         'update': {
             'path': "/users/%s.json",
-            'method': requests.put,
+            'method': 'put',
             'args': {
                 'required': ['params'],
                 'optional': [],
@@ -84,7 +83,7 @@ ENDPOINTS = {
         },
         'get': {
             'path': "/users/%s.json",
-            'method': requests.get,
+            'method': 'get',
             'args': {
                 'required': [],
                 'optional': [],
@@ -92,7 +91,7 @@ ENDPOINTS = {
         },
         'activate': {
             'path': "/admin/users/%s/activate",
-            'method': requests.put,
+            'method': 'put',
             'args': {
                 'required': [],
                 'optional': [],
@@ -100,7 +99,7 @@ ENDPOINTS = {
         },
         'set_email': {
             'path': "/users/%s/preferences/email",
-            'method': requests.put,
+            'method': 'put',
             'args': {
                 'required': ['email'],
                 'optional': [],
@@ -108,7 +107,7 @@ ENDPOINTS = {
         },
         'suspend': {
             'path': "/admin/users/%s/suspend",
-            'method': requests.put,
+            'method': 'put',
             'args': {
                 'required': ['duration', 'reason'],
                 'optional': [],
@@ -116,7 +115,7 @@ ENDPOINTS = {
         },
         'unsuspend': {
             'path': "/admin/users/%s/unsuspend",
-            'method': requests.put,
+            'method': 'put',
             'args': {
                 'required': [],
                 'optional': [],
@@ -124,7 +123,7 @@ ENDPOINTS = {
         },
         'logout': {
             'path': "/admin/users/%s/log_out",
-            'method': requests.post,
+            'method': 'post',
             'args': {
                 'required': [],
                 'optional': [],
@@ -132,7 +131,7 @@ ENDPOINTS = {
         },
         'external': {
             'path': "/users/by-external/%s.json",
-            'method': requests.get,
+            'method': 'get',
             'args': {
                 'required': [],
                 'optional': [],
@@ -146,8 +145,7 @@ class DiscourseManager:
     def __init__(self):
         pass
 
-    GROUP_CACHE_MAX_AGE = datetime.timedelta(minutes=30)
-    REVOKED_EMAIL = 'revoked@' + settings.DOMAIN
+    REVOKED_EMAIL = 'revoked@localhost'
     SUSPEND_DAYS = 99999
     SUSPEND_REASON = "Disabled by auth."
 
@@ -171,7 +169,8 @@ class DiscourseManager:
         for arg in kwargs:
             if arg not in endpoint['args']['required'] and arg not in endpoint['args']['optional'] and not silent:
                 logger.warn("Received unrecognized kwarg %s for endpoint %s" % (arg, endpoint))
-        r = endpoint['method'](settings.DISCOURSE_URL + endpoint['parsed_url'], params=params, json=data)
+        r = getattr(requests, endpoint['method'])(settings.DISCOURSE_URL + endpoint['parsed_url'], params=params,
+                                                  json=data)
         try:
             if 'errors' in r.json() and not silent:
                 logger.error("Discourse execution failed.\nEndpoint: %s\nErrors: %s" % (endpoint, r.json()['errors']))
@@ -190,67 +189,59 @@ class DiscourseManager:
         return out
 
     @staticmethod
-    def __generate_random_pass():
-        return ''.join([random.choice(string.ascii_letters + string.digits) for n in range(16)])
-
-    @staticmethod
-    def __get_groups():
+    def _get_groups():
         endpoint = ENDPOINTS['groups']['list']
         data = DiscourseManager.__exc(endpoint)
         return [g for g in data if not g['automatic']]
 
     @staticmethod
-    def __update_group_cache():
-        GroupCache.objects.filter(service="discourse").delete()
-        cache = GroupCache.objects.create(service="discourse")
-        cache.groups = json.dumps(DiscourseManager.__get_groups())
-        cache.save()
-        return cache
-
-    @staticmethod
-    def __get_group_cache():
-        if not GroupCache.objects.filter(service="discourse").exists():
-            DiscourseManager.__update_group_cache()
-        cache = GroupCache.objects.get(service="discourse")
-        age = timezone.now() - cache.created
-        if age > DiscourseManager.GROUP_CACHE_MAX_AGE:
-            logger.debug("Group cache has expired. Triggering update.")
-            cache = DiscourseManager.__update_group_cache()
-        return json.loads(cache.groups)
-
-    @staticmethod
-    def __create_group(name):
+    def _create_group(name):
         endpoint = ENDPOINTS['groups']['create']
-        DiscourseManager.__exc(endpoint, name=name[:20], visible=True)
-        DiscourseManager.__update_group_cache()
+        return DiscourseManager.__exc(endpoint, name=name[:20], visible=True)['basic_group']
+
+    @staticmethod
+    def _generate_cache_group_name_key(name):
+        return 'DISCOURSE_GROUP_NAME__%s' % md5(name.encode('utf-8')).hexdigest()
+
+    @staticmethod
+    def _generate_cache_group_id_key(g_id):
+        return 'DISCOURSE_GROUP_ID__%s' % g_id
 
     @staticmethod
     def __group_name_to_id(name):
-        cache = DiscourseManager.__get_group_cache()
-        for g in cache:
-            if g['name'] == name[0:20]:
-                return g['id']
-        logger.debug("Group %s not found on Discourse. Creating" % name)
-        DiscourseManager.__create_group(name)
-        return DiscourseManager.__group_name_to_id(name)
+        name = DiscourseManager._sanitize_groupname(name)
+
+        def get_or_create_group():
+            groups = DiscourseManager._get_groups()
+            for g in groups:
+                if g['name'].lower() == name.lower():
+                    return g['id']
+            return DiscourseManager._create_group(name)['id']
+
+        return cache.get_or_set(DiscourseManager._generate_cache_group_name_key(name), get_or_create_group,
+                                GROUP_CACHE_MAX_AGE)
 
     @staticmethod
-    def __group_id_to_name(id):
-        cache = DiscourseManager.__get_group_cache()
-        for g in cache:
-            if g['id'] == id:
-                return g['name']
-        raise KeyError("Group ID %s not found on Discourse" % id)
+    def __group_id_to_name(g_id):
+        def get_group_name():
+            groups = DiscourseManager._get_groups()
+            for g in groups:
+                if g['id'] == g_id:
+                    return g['name']
+            raise KeyError("Group ID %s not found on Discourse" % g_id)
+
+        return cache.get_or_set(DiscourseManager._generate_cache_group_id_key(g_id), get_group_name,
+                                GROUP_CACHE_MAX_AGE)
 
     @staticmethod
-    def __add_user_to_group(id, username):
+    def __add_user_to_group(g_id, username):
         endpoint = ENDPOINTS['groups']['add_user']
-        DiscourseManager.__exc(endpoint, id, usernames=[username])
+        DiscourseManager.__exc(endpoint, g_id, usernames=[username])
 
     @staticmethod
-    def __remove_user_from_group(id, username):
+    def __remove_user_from_group(g_id, username):
         endpoint = ENDPOINTS['groups']['remove_user']
-        DiscourseManager.__exc(endpoint, id, username=username)
+        DiscourseManager.__exc(endpoint, g_id, username=username)
 
     @staticmethod
     def __generate_group_dict(names):
@@ -269,10 +260,6 @@ class DiscourseManager:
         data = DiscourseManager.__get_user(name, silent=silent)
         return data['user']['id']
 
-    @staticmethod
-    def __user_id_to_name(id):
-        raise NotImplementedError
-
     @staticmethod
     def __get_user(username, silent=False):
         endpoint = ENDPOINTS['users']['get']
@@ -281,14 +268,14 @@ class DiscourseManager:
     @staticmethod
     def __activate_user(username):
         endpoint = ENDPOINTS['users']['activate']
-        id = DiscourseManager.__user_name_to_id(username)
-        DiscourseManager.__exc(endpoint, id)
+        u_id = DiscourseManager.__user_name_to_id(username)
+        DiscourseManager.__exc(endpoint, u_id)
 
     @staticmethod
     def __update_user(username, **kwargs):
         endpoint = ENDPOINTS['users']['update']
-        id = DiscourseManager.__user_name_to_id(username)
-        DiscourseManager.__exc(endpoint, id, params=kwargs)
+        u_id = DiscourseManager.__user_name_to_id(username)
+        DiscourseManager.__exc(endpoint, u_id, params=kwargs)
 
     @staticmethod
     def __create_user(username, email, password):
@@ -300,21 +287,21 @@ class DiscourseManager:
         try:
             DiscourseManager.__user_name_to_id(username, silent=True)
             return True
-        except:
+        except DiscourseError:
             return False
 
     @staticmethod
     def __suspend_user(username):
-        id = DiscourseManager.__user_name_to_id(username)
+        u_id = DiscourseManager.__user_name_to_id(username)
         endpoint = ENDPOINTS['users']['suspend']
-        return DiscourseManager.__exc(endpoint, id, duration=DiscourseManager.SUSPEND_DAYS,
+        return DiscourseManager.__exc(endpoint, u_id, duration=DiscourseManager.SUSPEND_DAYS,
                                       reason=DiscourseManager.SUSPEND_REASON)
 
     @staticmethod
     def __unsuspend(username):
-        id = DiscourseManager.__user_name_to_id(username)
+        u_id = DiscourseManager.__user_name_to_id(username)
         endpoint = ENDPOINTS['users']['unsuspend']
-        return DiscourseManager.__exc(endpoint, id)
+        return DiscourseManager.__exc(endpoint, u_id)
 
     @staticmethod
     def __set_email(username, email):
@@ -322,47 +309,53 @@ class DiscourseManager:
         return DiscourseManager.__exc(endpoint, username, email=email)
 
     @staticmethod
-    def __logout(id):
+    def __logout(u_id):
         endpoint = ENDPOINTS['users']['logout']
-        return DiscourseManager.__exc(endpoint, id)
+        return DiscourseManager.__exc(endpoint, u_id)
 
     @staticmethod
-    def __get_user_by_external(id):
+    def __get_user_by_external(u_id):
         endpoint = ENDPOINTS['users']['external']
-        return DiscourseManager.__exc(endpoint, id)
+        return DiscourseManager.__exc(endpoint, u_id)
 
     @staticmethod
-    def __user_id_by_external_id(id):
-        data = DiscourseManager.__get_user_by_external(id)
+    def __user_id_by_external_id(u_id):
+        data = DiscourseManager.__get_user_by_external(u_id)
         return data['user']['id']
 
+    @staticmethod
+    def _sanitize_name(name):
+        name = name.replace(' ', '_')
+        name = name.replace("'", '')
+        name = name.lstrip(' _')
+        name = name[:20]
+        name = name.rstrip(' _')
+        return name
+
     @staticmethod
     def _sanitize_username(username):
-        sanitized = username.replace(" ", "_")
-        sanitized = sanitized.strip(' _')
-        sanitized = sanitized.replace("'", "")
-        return sanitized
+        return DiscourseManager._sanitize_name(username)
 
     @staticmethod
     def _sanitize_groupname(name):
-        name = name.strip(' _')
         name = re.sub('[^\w]', '', name)
+        name = DiscourseManager._sanitize_name(name)
         if len(name) < 3:
-            name = name + "".join('_' for i in range(3-len(name)))
-        return name[:20]
+            name = "Group " + name
+        return name
 
     @staticmethod
     def update_groups(user):
         groups = []
         for g in user.groups.all():
-            groups.append(DiscourseManager._sanitize_groupname(str(g)[:20]))
+            groups.append(DiscourseManager._sanitize_groupname(str(g)))
         logger.debug("Updating discourse user %s groups to %s" % (user, groups))
         group_dict = DiscourseManager.__generate_group_dict(groups)
         inv_group_dict = {v: k for k, v in group_dict.items()}
         username = DiscourseManager.__get_user_by_external(user.pk)['user']['username']
         user_groups = DiscourseManager.__get_user_groups(username)
         add_groups = [group_dict[x] for x in group_dict if not group_dict[x] in user_groups]
-        rem_groups = [x for x in user_groups if not x in inv_group_dict]
+        rem_groups = [x for x in user_groups if x not in inv_group_dict]
         if add_groups or rem_groups:
             logger.info(
                 "Updating discourse user %s groups: adding %s, removing %s" % (username, add_groups, rem_groups))
@@ -376,6 +369,5 @@ class DiscourseManager:
         logger.debug("Disabling user %s Discourse access." % user)
         d_user = DiscourseManager.__get_user_by_external(user.pk)
         DiscourseManager.__logout(d_user['user']['id'])
-        DiscourseManager.__suspend_user(d_user['user']['username'])
         logger.info("Disabled user %s Discourse access." % user)
         return True