N+1 query fixes

allianceauth/fleetactivitytracking/views.py

@@ -5,14 +5,12 @@ import random
 import string
 
 from allianceauth.authentication.models import CharacterOwnership
-from django.conf import settings
 from django.contrib import messages
 from django.contrib.auth.decorators import login_required
 from django.contrib.auth.decorators import permission_required
 from django.contrib.auth.models import User
 from django.core.exceptions import ValidationError, ObjectDoesNotExist
-from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger
+from django.shortcuts import render, redirect, get_object_or_404, Http404
-from django.shortcuts import render, redirect, get_object_or_404
 from django.utils import timezone
 from django.utils.translation import ugettext_lazy as _
 from esi.decorators import token_required
@@ -83,9 +81,9 @@ def fatlink_view(request):
     user = request.user
     logger.debug("fatlink_view called by user %s" % request.user)
 
-    latest_fats = Fat.objects.filter(user=user).order_by('-id')[:5]
+    latest_fats = Fat.objects.select_related('character', 'fatlink').filter(user=user).order_by('-id')[:5]
     if user.has_perm('auth.fleetactivitytracking'):
-        latest_links = Fatlink.objects.all().order_by('-id')[:5]
+        latest_links = Fatlink.objects.select_related('creator').all().order_by('-id')[:5]
         context = {'user': user, 'fats': latest_fats, 'fatlinks': latest_links}
 
     else:
@@ -173,7 +171,7 @@ def fatlink_personal_statistics_view(request, year=datetime.date.today().year):
     user = request.user
     logger.debug("fatlink_personal_statistics_view called by user %s" % request.user)
 
-    personal_fats = Fat.objects.filter(user=user).order_by('id')
+    personal_fats = Fat.objects.select_related('fatlink').filter(user=user).order_by('id')
 
     monthlystats = [0 for month in range(1, 13)]
 
@@ -207,8 +205,8 @@ def fatlink_monthly_personal_statistics_view(request, year, month, char_id=None)
         user = request.user
     logger.debug("Personal monthly statistics view for user %s called by %s" % (user, request.user))
 
-    personal_fats = Fat.objects.filter(user=user).filter(fatlink__fatdatetime__gte=start_of_month).filter(
+    personal_fats = Fat.objects.filter(user=user)\
-        fatlink__fatdatetime__lt=start_of_next_month)
+        .filter(fatlink__fatdatetime__gte=start_of_month).filter(fatlink__fatdatetime__lt=start_of_next_month)
 
     ship_statistics = dict()
     n_fats = 0
@@ -330,7 +328,10 @@ def modify_fatlink_view(request, hash=""):
     if not hash:
         return redirect('fatlink:view')
 
-    fatlink = Fatlink.objects.filter(hash=hash)[0]
+    try:
+        fatlink = Fatlink.objects.get(hash=hash)
+    except Fatlink.DoesNotExist:
+        raise Http404
 
     if request.GET.get('removechar', None):
         character_id = request.GET.get('removechar')
@@ -344,7 +345,8 @@ def modify_fatlink_view(request, hash=""):
         fatlink.delete()
         return redirect('fatlink:view')
 
-    registered_fats = Fat.objects.filter(fatlink=fatlink).order_by('character__character_name')
+    registered_fats = Fat.objects.select_related('character', 'fatlink', 'user')\
+        .filter(fatlink=fatlink).order_by('character__character_name')
 
     context = {'fatlink': fatlink, 'registered_fats': registered_fats}
 

allianceauth/groupmanagement/managers.py

@@ -7,11 +7,11 @@ class GroupManager:
 
     @staticmethod
     def get_joinable_groups():
-        return Group.objects.exclude(authgroup__internal=True)
+        return Group.objects.select_related('authgroup').exclude(authgroup__internal=True)
 
     @staticmethod
     def get_group_leaders_groups(user):
-        return Group.objects.filter(authgroup__group_leaders__in=[user])
+        return Group.objects.select_related('authgroup').filter(authgroup__group_leaders__in=[user])
 
     @staticmethod
     def joinable_group(group):

allianceauth/groupmanagement/views.py

@@ -24,12 +24,13 @@ def group_management(request):
     acceptrequests = []
     leaverequests = []
 
+    base_group_query = GroupRequest.objects.select_related('user', 'group')
     if GroupManager.has_management_permission(request.user):
         # Full access
-        group_requests = GroupRequest.objects.all()
+        group_requests = base_group_query.all()
     else:
         # Group specific leader
-        group_requests = GroupRequest.objects.filter(group__authgroup__group_leaders__in=[request.user])
+        group_requests = base_group_query.filter(group__authgroup__group_leaders__in=[request.user])
 
     for grouprequest in group_requests:
         if grouprequest.leave_request:
@@ -83,7 +84,7 @@ def group_membership_list(request, group_id):
 
     members = list()
 
-    for member in group.user_set.all().order_by('username'):
+    for member in group.user_set.select_related('profile').all().order_by('username'):
 
         members.append({
             'user': member,

allianceauth/hrapplications/views.py

@@ -3,7 +3,8 @@ import logging
 from django.contrib.auth.decorators import login_required
 from django.contrib.auth.decorators import permission_required
 from django.contrib.auth.decorators import user_passes_test
-from django.shortcuts import render, get_object_or_404, redirect
+from django.shortcuts import render, get_object_or_404, redirect, Http404
+from django.db.models import Q
 from .models import Application
 from .models import ApplicationComment
 from .models import ApplicationForm
@@ -26,14 +27,16 @@ def hr_application_management_view(request):
     corp_applications = []
     finished_corp_applications = []
     main_char = request.user.profile.main_character
+
+    base_app_query = Application.objects.select_related('user', 'form', 'form__corp')
     if request.user.is_superuser:
-        corp_applications = Application.objects.filter(approved=None)
+        corp_applications = base_app_query.filter(approved=None)
-        finished_corp_applications = Application.objects.exclude(approved=None)
+        finished_corp_applications = base_app_query.exclude(approved=None)
     elif request.user.has_perm('auth.human_resources') and main_char:
         if ApplicationForm.objects.filter(corp__corporation_id=main_char.corporation_id).exists():
             app_form = ApplicationForm.objects.get(corp__corporation_id=main_char.corporation_id)
-            corp_applications = Application.objects.filter(form=app_form).filter(approved=None)
+            corp_applications = base_app_query.filter(form=app_form).filter(approved=None)
-            finished_corp_applications = Application.objects.filter(form=app_form).filter(approved__in=[True, False])
+            finished_corp_applications = base_app_query.filter(form=app_form).filter(approved__in=[True, False])
     logger.debug("Retrieved %s personal, %s corp applications for %s" % (
         len(request.user.applications.all()), len(corp_applications), request.user))
     context = {
@@ -46,7 +49,6 @@ def hr_application_management_view(request):
     return render(request, 'hrapplications/management.html', context=context)
 
 
-
 @login_required
 @user_passes_test(create_application_test)
 def hr_application_create_view(request, form_id=None):
@@ -114,7 +116,10 @@ def hr_application_personal_removal(request, app_id):
 @permission_required('auth.human_resources')
 def hr_application_view(request, app_id):
     logger.debug("hr_application_view called by user %s for app id %s" % (request.user, app_id))
-    app = get_object_or_404(Application, pk=app_id)
+    try:
+        app = Application.objects.prefetch_related('responses', 'comments', 'comments__user').get(pk=app_id)
+    except Application.DoesNotExist:
+        raise Http404
     if request.method == 'POST':
         if request.user.has_perm('hrapplications.add_applicationcomment'):
             form = HRApplicationCommentForm(request.POST)
@@ -135,9 +140,9 @@ def hr_application_view(request, app_id):
         form = HRApplicationCommentForm()
     context = {
         'app': app,
-        'responses': ApplicationResponse.objects.filter(application=app),
+        'responses': app.responses.all(),
         'buttons': True,
-        'comments': ApplicationComment.objects.filter(application=app),
+        'comments': app.comments.all(),
         'comment_form': form,
     }
     return render(request, 'hrapplications/view.html', context=context)
@@ -200,35 +205,24 @@ def hr_application_search(request):
             searchstring = form.cleaned_data['search_string'].lower()
             applications = set([])
             logger.debug("Searching for application with character name %s for user %s" % (searchstring, request.user))
-            app_list = []
+            app_list = Application.objects.all()
-            if request.user.is_superuser:
+            if not request.user.is_superuser:
-                app_list = Application.objects.all()
-            else:
                 try:
-                    app_list = Application.objects.filter(form__corp__corporation_id=request.user.profile.main_character.corporation_id)
+                    app_list = app_list.filter(
+                        form__corp__corporation_id=request.user.profile.main_character.corporation_id)
                 except AttributeError:
                     logger.warn(
                         "User %s missing main character model: unable to filter applications to search" % request.user)
-            for application in app_list:
+
-                if application.main_character:
+            applications = app_list.filter(
-                    if searchstring in application.main_character.character_name.lower():
+                Q(user__profile__main_character__character_name__icontains=searchstring) |
-                        applications.add(application)
+                Q(user__profile__main_character__corporation_name__icontains=searchstring) |
-                    if searchstring in application.main_character.corporation_name.lower():
+                Q(user__profile__main_character__alliance_name__icontains=searchstring) |
-                        applications.add(application)
+                Q(user__character_ownerships__character__character_name__icontains=searchstring) |
-                    if application.main_character.alliance_name \
+                Q(user__character_ownerships__character__corporation_name__icontains=searchstring) |
-                            and searchstring in application.main_character.alliance_name.lower():
+                Q(user__character_ownerships__character__alliance_name__icontains=searchstring) |
-                        applications.add(application)
+                Q(user__username__icontains=searchstring)
-                for character in application.characters:
+            )
-                    if searchstring in character.character_name.lower():
-                        applications.add(application)
-                    if searchstring in character.corporation_name.lower():
-                        applications.add(application)
-                    if character.alliance_name and searchstring in character.alliance_name.lower():
-                        applications.add(application)
-                if searchstring in application.user.username.lower():
-                    applications.add(application)
-            logger.info("Found %s Applications for user %s matching search string %s" % (
-                len(applications), request.user, searchstring))
 
             context = {'applications': applications, 'search_form': HRApplicationSearchForm()}
 

allianceauth/optimer/views.py

@@ -18,10 +18,11 @@ logger = logging.getLogger(__name__)
 @permission_required('auth.optimer_view')
 def optimer_view(request):
     logger.debug("optimer_view called by user %s" % request.user)
-    render_items = {'optimer': OpTimer.objects.all(),
+    base_query = OpTimer.objects.select_related('eve_character')
-                    'future_timers': OpTimer.objects.all().filter(
+    render_items = {'optimer': base_query.all(),
+                    'future_timers': base_query.filter(
                         start__gte=timezone.now()),
-                    'past_timers': OpTimer.objects.all().filter(
+                    'past_timers': base_query.filter(
                         start__lt=timezone.now()).order_by('-start')}
 
     return render(request, 'optimer/management.html', context=render_items)

allianceauth/permissions_tool/views.py

@@ -3,9 +3,8 @@ import logging
 from django.contrib.auth.decorators import login_required, permission_required
 from django.contrib.auth.models import Permission, User
 from django.db.models import Count
-from django.shortcuts import render, get_object_or_404
+from django.shortcuts import render, Http404
 
-from allianceauth.authentication.models import UserProfile
 
 logger = logging.getLogger(__name__)
 
@@ -14,25 +13,28 @@ logger = logging.getLogger(__name__)
 @permission_required('permissions_tool.audit_permissions')
 def permissions_overview(request):
     logger.debug("permissions_overview called by user %s" % request.user)
-    perms = Permission.objects.all()
+    perms = Permission.objects.select_related('content_type').all()\
+        .annotate(Count('user', distinct=True))\
+        .annotate(Count('group', distinct=True)) \
+        .annotate(Count('group__user', distinct=True)) \
+        .annotate(Count('state', distinct=True))\
+        .annotate(Count('state__userprofile', distinct=True))
 
     get_all = True if request.GET.get('all', 'no') == 'yes' else False
 
     context = {'permissions': []}
     for perm in perms:
         this_perm = {
-            'users': perm.user_set.all().count(),
+            'users': perm.user__count,
-            'groups': perm.group_set.all().count(),
+            'groups': perm.group__count,
-            'states': perm.state_set.all().count(),
+            'group_users': perm.group__user__count,
-            'permission': perm
+            'states': perm.state__count,
+            'state_users': perm.state__userprofile__count,
+            'permission': perm,
         }
 
-        if get_all or this_perm['users'] > 0 or this_perm['groups'] > 0 or this_perm['states'] > 0:
+        if get_all or sum([this_perm['users'], this_perm['groups'], this_perm['states']]) > 0:
             # Only add if we're getting everything or one of the objects has this permission
-            # Add group_users separately to improve performance
-            this_perm['group_users'] = sum(group.user_count for group in
-                                           perm.group_set.annotate(user_count=Count('user')))
-            this_perm['state_users'] = UserProfile.objects.filter(state__in=perm.state_set.all()).count()
             context['permissions'].append(this_perm)
 
     return render(request, 'permissions_tool/overview.html', context=context)
@@ -42,10 +44,13 @@ def permissions_overview(request):
 @permission_required('permissions_tool.audit_permissions')
 def permissions_audit(request, app_label, model, codename):
     logger.debug("permissions_audit called by user {} on {}:{}:{}".format(request.user, app_label, model, codename))
-    perm = get_object_or_404(Permission,
+    try:
-                             content_type__app_label=app_label,
+        perm = Permission.objects\
-                             content_type__model=model,
+            .prefetch_related('group_set', 'user_set', 'state_set',
-                             codename=codename)
+                              'state_set__userprofile_set', 'group_set__user_set', 'state_set__userprofile_set__user')\
+            .get(content_type__app_label=app_label, content_type__model=model, codename=codename)
+    except Permission.DoesNotExist:
+        raise Http404
 
     context = {'permission': {
         'permission': perm,
@@ -53,7 +58,7 @@ def permissions_audit(request, app_label, model, codename):
         'groups': perm.group_set.all(),
         'states': perm.state_set.all(),
         'group_users': [group.user_set.all() for group in perm.group_set.all()],
-        'state_users': User.objects.filter(profile__state__in=perm.state_set.all()),
+        'state_users': [state.userprofile_set.all() for state in perm.state_set.all()],
         }
     }
 

allianceauth/srp/urls.py

@@ -7,7 +7,7 @@ app_name = 'srp'
 urlpatterns = [
     # SRP URLS
     url(r'^$', views.srp_management, name='management'),
-    url(r'^all/$', views.srp_management_all, name='all'),
+    url(r'^all/$', views.srp_management, {'all': True}, name='all'),
     url(r'^(\w+)/view$', views.srp_fleet_view, name='fleet'),
     url(r'^add/$', views.srp_fleet_add_view, name='add'),
     url(r'^(\w+)/edit$', views.srp_fleet_edit_view, name='edit'),

allianceauth/srp/views.py

@@ -5,10 +5,11 @@ from django.contrib import messages
 from django.contrib.auth.decorators import login_required
 from django.contrib.auth.decorators import permission_required
 from django.contrib.humanize.templatetags.humanize import intcomma
-from django.http import JsonResponse
+from django.http import JsonResponse, Http404
 from django.shortcuts import render, redirect, get_object_or_404
 from django.utils import timezone
 from django.utils.translation import ugettext_lazy as _
+from django.db.models import Sum
 from allianceauth.eveonline.providers import provider
 from allianceauth.notifications import notify
 from .form import SrpFleetMainForm
@@ -31,31 +32,27 @@ def random_string(string_length=10):
 
 @login_required
 @permission_required('srp.access_srp')
-def srp_management(request):
+def srp_management(request, all=False):
     logger.debug("srp_management called by user %s" % request.user)
-    fleets = SrpFleetMain.objects.filter(fleet_srp_status="")
+    fleets = SrpFleetMain.objects.select_related('fleet_commander').prefetch_related('srpuserrequest_set').all()
-    totalcost = sum([int(fleet.total_cost) for fleet in fleets])
+    if not all:
+        fleets = fleets.filter(fleet_srp_status="")
+    else:
+        logger.debug("Returning all SRP requests")
+    totalcost = fleets.aggregate(total_cost=Sum('srpuserrequest__srp_total_amount')).get('total_cost', 0)
     context = {"srpfleets": fleets, "totalcost": totalcost}
     return render(request, 'srp/management.html', context=context)
 
-
-@login_required
-@permission_required('srp.access_srp')
-def srp_management_all(request):
-    logger.debug("srp_management_all called by user %s" % request.user)
-    fleets = SrpFleetMain.objects.all()
-    totalcost = sum([int(fleet.total_cost) for fleet in fleets])
-    context = {"srpfleets": SrpFleetMain.objects.all(), "totalcost": totalcost}
-    return render(request, 'srp/management.html', context=context)
-
-
 @login_required
 @permission_required('srp.access_srp')
 def srp_fleet_view(request, fleet_id):
     logger.debug("srp_fleet_view called by user %s for fleet id %s" % (request.user, fleet_id))
-    fleet_main = get_object_or_404(SrpFleetMain, id=fleet_id)
+    try:
+        fleet_main = SrpFleetMain.objects.get(id=fleet_id)
+    except SrpFleetMain.DoesNotExist:
+        raise Http404
     context = {"fleet_id": fleet_id, "fleet_status": fleet_main.fleet_srp_status,
-               "srpfleetrequests": fleet_main.srpuserrequest_set.order_by('srp_ship_name'),
+               "srpfleetrequests": fleet_main.srpuserrequest_set.select_related('character').order_by('srp_ship_name'),
                "totalcost": fleet_main.total_cost}
 
     return render(request, 'srp/data.html', context=context)

allianceauth/timerboard/views.py

@@ -32,12 +32,12 @@ class TimerView(BaseTimerView):
             corp = char.corporation
         else:
             corp = None
-
+        base_query = Timer.objects.select_related('eve_character')
         render_items = {
-            'timers': Timer.objects.filter(corp_timer=False),
+            'timers': base_query.filter(corp_timer=False),
-            'corp_timers': Timer.objects.filter(corp_timer=True, eve_corp=corp),
+            'corp_timers': base_query.filter(corp_timer=True, eve_corp=corp),
-            'future_timers': Timer.objects.filter(corp_timer=False, eve_time__gte=timezone.now()),
+            'future_timers': base_query.filter(corp_timer=False, eve_time__gte=timezone.now()),
-            'past_timers': Timer.objects.filter(corp_timer=False, eve_time__lt=timezone.now()),
+            'past_timers': base_query.filter(corp_timer=False, eve_time__lt=timezone.now()),
         }
 
         return render(request, self.template_name, context=render_items)