Merge branch 'use-django-3.2' into 'v2.9.x'

Bumped minimum Django version to 3.2.7

See merge request allianceauth/allianceauth!1336

.editorconfig

@@ -0,0 +1,24 @@
+# http://editorconfig.org
+root = true
+
+[*]
+indent_style = space
+indent_size = 4
+tab_width = 4
+end_of_line = lf
+charset = utf-8
+trim_trailing_whitespace = true
+insert_final_newline = true
+
+[*.{yaml,yml,less}]
+indent_size = 2
+
+[*.md]
+indent_size = 2
+
+# Makefiles always use tabs for indentation
+[Makefile]
+indent_style = tab
+
+[*.bat]
+indent_style = tab

.gitlab-ci.yml

@@ -1,53 +1,106 @@
 stages:
+- pre-commit
+- gitlab
 - test
 - deploy
 
+include:
+- template: Dependency-Scanning.gitlab-ci.yml
+- template: Security/SAST.gitlab-ci.yml
+
 before_script:
   - apt-get update && apt-get install redis-server -y
   - redis-server --daemonize yes
-- redis-cli ping
   - python -V
   - pip install wheel tox
 
-test-3.6-core:
+pre-commit-check:
+  stage: pre-commit
   image: python:3.6-buster
+  variables:
+    PRE_COMMIT_HOME: ${CI_PROJECT_DIR}/.cache/pre-commit
+  cache:
+    paths:
+      - ${PRE_COMMIT_HOME}
   script:
-  - tox -e py36-core
+    - pip install pre-commit
+    - pre-commit run --all-files
+
+sast:
+  stage: gitlab
+  before_script: []
+
+dependency_scanning:
+  stage: gitlab
+  before_script:
+  - apt-get update && apt-get install redis-server libmariadb-dev -y
+  - redis-server --daemonize yes
+  - python -V
+  - pip install wheel tox
 
 test-3.7-core:
-  image: python:3.7-buster
+  image: python:3.7-bullseye
   script:
   - tox -e py37-core
+  artifacts:
+    when: always
+    reports:
+      cobertura: coverage.xml
 
 test-3.8-core:
-  image: python:3.8-buster
+  image: python:3.8-bullseye
   script:
   - tox -e py38-core
+  artifacts:
+    when: always
+    reports:
+      cobertura: coverage.xml
 
-test-3.6-all:
+test-3.9-core:
-  image: python:3.6-buster
+  image: python:3.9-bullseye
   script:
-  - tox -e py36-all
+  - tox -e py39-core
+  artifacts:
+    when: always
+    reports:
+      cobertura: coverage.xml
 
 test-3.7-all:
-  image: python:3.7-buster
+  image: python:3.7-bullseye
   script:
   - tox -e py37-all
+  artifacts:
+    when: always
+    reports:
+      cobertura: coverage.xml
 
 test-3.8-all:
-  image: python:3.8-buster
+  image: python:3.8-bullseye
   script:
   - tox -e py38-all
+  artifacts:
+    when: always
+    reports:
+      cobertura: coverage.xml
+
+test-3.9-all:
+  image: python:3.9-bullseye
+  script:
+  - tox -e py39-all
+  artifacts:
+    when: always
+    reports:
+      cobertura: coverage.xml
 
 deploy_production:
   stage: deploy
-  image: python:3.6-stretch
+  image: python:3.9-bullseye
 
   before_script:
-    - pip install twine
+    - pip install twine wheel
 
   script:
-    - python setup.py sdist
+    - python setup.py sdist bdist_wheel
     - twine upload dist/*
 
   rules:

.pre-commit-config.yaml

@@ -0,0 +1,28 @@
+# Apply to all files without committing:
+#   pre-commit run --all-files
+# Update this file:
+#   pre-commit autoupdate
+
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.0.1
+    hooks:
+      - id: check-case-conflict
+      - id: check-json
+      - id: check-xml
+      - id: check-yaml
+      - id: fix-byte-order-marker
+      - id: trailing-whitespace
+        exclude: (\.min\.css|\.min\.js|\.mo|\.po|swagger\.json)$
+      - id: end-of-file-fixer
+        exclude: (\.min\.css|\.min\.js|\.mo|\.po|swagger\.json)$
+      - id: mixed-line-ending
+        args: [ '--fix=lf' ]
+      - id: fix-encoding-pragma
+        args: [ '--remove' ]
+
+  - repo: https://github.com/editorconfig-checker/editorconfig-checker.python
+    rev: 2.3.5
+    hooks:
+      - id: editorconfig-checker
+        exclude: ^(LICENSE|allianceauth\/static\/css\/themes\/bootstrap-locals.less|allianceauth\/eveonline\/swagger.json|(.*.po)|(.*.mo))

.readthedocs.yml

@@ -20,8 +20,4 @@ formats: all
 python:
   version: 3.7
   install:
-      - method: pip
+    - requirements: docs/requirements.txt
-        path: .        
-        extra_requirements:
-           - testing
-   system_packages: true

LICENSE

@@ -337,4 +337,3 @@ proprietary programs.  If your program is a subroutine library, you may
 consider it more useful to permit linking proprietary applications with the
 library.  If this is what you want to do, use the GNU Lesser General
 Public License instead of this License.
-

README.md

@@ -36,7 +36,7 @@ Main features:
 
 - Can be easily extended with additional services and apps. Many are provided by the community and can be found here: [Community Creations](https://gitlab.com/allianceauth/community-creations)
 
-- Chinese :cn:, English :us:, German :de: and Spanish :es: localization
+- English :flag_gb:, Chinese :flag_cn:, German :flag_de:, Spanish :flag_es:, Korean :flag_kr: and Russian :flag_ru: localization
 
 For further details about AA - including an installation guide and a full list of included services and plugin apps - please see the [official documentation](http://allianceauth.rtfd.io).
 

allianceauth/__init__.py

@@ -1,7 +1,7 @@
 # This will make sure the app is always imported when
 # Django starts so that shared_task will use this app.
 
-__version__ = '2.7.3'
+__version__ = '2.9.0b1'
 __title__ = 'Alliance Auth'
 __url__ = 'https://gitlab.com/allianceauth/allianceauth'
 NAME = '%s v%s' % (__title__, __version__)

allianceauth/analytics/__init__.py

@@ -0,0 +1 @@
+default_app_config = 'allianceauth.analytics.apps.AnalyticsConfig'

allianceauth/analytics/admin.py

@@ -0,0 +1,21 @@
+from django.contrib import admin
+
+from .models import AnalyticsIdentifier, AnalyticsPath, AnalyticsTokens
+
+
+@admin.register(AnalyticsIdentifier)
+class AnalyticsIdentifierAdmin(admin.ModelAdmin):
+    search_fields = ['identifier', ]
+    list_display = ('identifier',)
+
+
+@admin.register(AnalyticsTokens)
+class AnalyticsTokensAdmin(admin.ModelAdmin):
+    search_fields = ['name', ]
+    list_display = ('name', 'type',)
+
+
+@admin.register(AnalyticsPath)
+class AnalyticsPathAdmin(admin.ModelAdmin):
+    search_fields = ['ignore_path', ]
+    list_display = ('ignore_path',)

allianceauth/analytics/apps.py

@@ -0,0 +1,9 @@
+from django.apps import AppConfig
+
+
+class AnalyticsConfig(AppConfig):
+    name = 'allianceauth.analytics'
+    label = 'analytics'
+
+    def ready(self):
+        import allianceauth.analytics.signals

allianceauth/analytics/fixtures/disable_analytics.json

@@ -0,0 +1,21 @@
+[
+    {
+    "model": "analytics.AnalyticsTokens",
+    "pk": 1,
+    "fields": {
+        "name": "AA Team Public Google Analytics (Universal)",
+        "type": "GA-V4",
+        "token": "UA-186249766-2",
+        "send_page_views": "False",
+        "send_celery_tasks": "False",
+        "send_stats": "False"
+        }
+    },
+    {
+        "model": "analytics.AnalyticsIdentifier",
+        "pk": 1,
+        "fields": {
+            "identifier": "ab33e241fbf042b6aa77c7655a768af7"
+        }
+    }
+]

allianceauth/analytics/middleware.py

@@ -0,0 +1,41 @@
+from bs4 import BeautifulSoup
+
+from django.utils.deprecation import MiddlewareMixin
+from .models import AnalyticsTokens, AnalyticsIdentifier
+from .tasks import send_ga_tracking_web_view
+
+
+class AnalyticsMiddleware(MiddlewareMixin):
+    def process_response(self, request, response):
+        """Django Middleware: Process Page Views and creates Analytics Celery Tasks"""
+        analyticstokens = AnalyticsTokens.objects.all()
+        client_id = AnalyticsIdentifier.objects.get(id=1).identifier.hex
+        try:
+            title = BeautifulSoup(
+                response.content, "html.parser").html.head.title.text
+        except AttributeError:
+            title = ''
+        for token in analyticstokens:
+            # Check if Page View Sending is Disabled
+            if token.send_page_views is False:
+                continue
+            # Check Exclusions
+            if request.path in token.ignore_paths.all():
+                continue
+
+            tracking_id = token.token
+            locale = request.LANGUAGE_CODE
+            path = request.path
+            try:
+                useragent = request.headers["User-Agent"]
+            except KeyError:
+                useragent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36"
+
+            send_ga_tracking_web_view.s(tracking_id=tracking_id,
+                                        client_id=client_id,
+                                        page=path,
+                                        title=title,
+                                        locale=locale,
+                                        useragent=useragent).\
+                apply_async(priority=9)
+        return response

allianceauth/analytics/migrations/0001_initial.py

@@ -0,0 +1,42 @@
+# Generated by Django 3.1.4 on 2020-12-30 13:11
+
+from django.db import migrations, models
+import uuid
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='AnalyticsIdentifier',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('identifier', models.UUIDField(default=uuid.uuid4, editable=False)),
+            ],
+        ),
+        migrations.CreateModel(
+            name='AnalyticsPath',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('ignore_path', models.CharField(default='/example/', max_length=254)),
+            ],
+        ),
+        migrations.CreateModel(
+            name='AnalyticsTokens',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('name', models.CharField(max_length=254)),
+                ('type', models.CharField(choices=[('GA-U', 'Google Analytics Universal'), ('GA-V4', 'Google Analytics V4')], max_length=254)),
+                ('token', models.CharField(max_length=254)),
+                ('send_page_views', models.BooleanField(default=False)),
+                ('send_celery_tasks', models.BooleanField(default=False)),
+                ('send_stats', models.BooleanField(default=False)),
+                ('ignore_paths', models.ManyToManyField(blank=True, to='analytics.AnalyticsPath')),
+            ],
+        ),
+    ]

allianceauth/analytics/migrations/0002_add_AA_Team_Token.py

@@ -0,0 +1,34 @@
+# Generated by Django 3.1.4 on 2020-12-30 08:53
+
+from django.db import migrations
+
+
+def add_aa_team_token(apps, schema_editor):
+    # We can't import the Person model directly as it may be a newer
+    # version than this migration expects. We use the historical version.
+    Tokens = apps.get_model('analytics', 'AnalyticsTokens')
+    token = Tokens()
+    token.type = 'GA-U'
+    token.token = 'UA-186249766-2'
+    token.send_page_views = True
+    token.send_celery_tasks = True
+    token.send_stats = True
+    token.name = 'AA Team Public Google Analytics (Universal)'
+    token.save()
+
+
+def remove_aa_team_token(apps, schema_editor):
+    # Have to define some code to remove this identifier
+    # In case of migration rollback?
+    Tokens = apps.get_model('analytics', 'AnalyticsTokens')
+    token = Tokens.objects.filter(token="UA-186249766-2").delete()
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('analytics', '0001_initial'),
+        ]
+
+    operations = [migrations.RunPython(add_aa_team_token, remove_aa_team_token)
+                    ]

allianceauth/analytics/migrations/0003_Generate_Identifier.py

@@ -0,0 +1,30 @@
+# Generated by Django 3.1.4 on 2020-12-30 08:53
+
+from uuid import uuid4
+from django.db import migrations
+
+
+def generate_identifier(apps, schema_editor):
+    # We can't import the Person model directly as it may be a newer
+    # version than this migration expects. We use the historical version.
+    Identifier = apps.get_model('analytics', 'AnalyticsIdentifier')
+    identifier = Identifier()
+    identifier.id = 1
+    identifier.save()
+
+
+def zero_identifier(apps, schema_editor):
+    # Have to define some code to remove this identifier
+    # In case of migration rollback?
+    Identifier = apps.get_model('analytics', 'AnalyticsIdentifier')
+    Identifier.objects.filter(id=1).delete()
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('analytics', '0002_add_AA_Team_Token'),
+    ]
+
+    operations = [migrations.RunPython(generate_identifier, zero_identifier)
+                    ]

allianceauth/analytics/models.py

@@ -0,0 +1,38 @@
+from django.db import models
+from django.core.exceptions import ValidationError
+from django.utils.translation import gettext_lazy as _
+
+from uuid import uuid4
+
+
+class AnalyticsIdentifier(models.Model):
+
+    identifier = models.UUIDField(default=uuid4,
+                                  editable=False)
+
+    def save(self, *args, **kwargs):
+        if not self.pk and AnalyticsIdentifier.objects.exists():
+            # Force a single object
+            raise ValidationError('There is can be only one \
+                                   AnalyticsIdentifier instance')
+        self.pk = self.id = 1 # If this happens to be deleted and recreated, force it to be 1
+        return super(AnalyticsIdentifier, self).save(*args, **kwargs)
+
+
+class AnalyticsPath(models.Model):
+    ignore_path = models.CharField(max_length=254, default="/example/")
+
+
+class AnalyticsTokens(models.Model):
+
+    class Analytics_Type(models.TextChoices):
+        GA_U = 'GA-U', _('Google Analytics Universal')
+        GA_V4 = 'GA-V4', _('Google Analytics V4')
+
+    name = models.CharField(max_length=254)
+    type = models.CharField(max_length=254, choices=Analytics_Type.choices)
+    token = models.CharField(max_length=254, blank=False)
+    send_page_views = models.BooleanField(default=False)
+    send_celery_tasks = models.BooleanField(default=False)
+    send_stats = models.BooleanField(default=False)
+    ignore_paths = models.ManyToManyField(AnalyticsPath, blank=True)

allianceauth/analytics/signals.py

@@ -0,0 +1,50 @@
+from allianceauth.analytics.tasks import analytics_event
+from celery.signals import task_failure, task_success
+
+import logging
+logger = logging.getLogger(__name__)
+
+
+@task_failure.connect
+def process_failure_signal(
+                        exception, traceback,
+                        sender, task_id, signal,
+                        args, kwargs, einfo, **kw):
+    logger.debug("Celery task_failure signal %s" % sender.__class__.__name__)
+
+    category = sender.__module__
+
+    if 'allianceauth.analytics' not in category:
+        if category.endswith(".tasks"):
+            category = category[:-6]
+
+        action = sender.__name__
+
+        label = f"{exception.__class__.__name__}: {str(exception)}"
+
+        analytics_event(category=category,
+                        action=action,
+                        label=label)
+
+
+@task_success.connect
+def celery_success_signal(sender, result=None, **kw):
+    logger.debug("Celery task_success signal %s" % sender.__class__.__name__)
+
+    category = sender.__module__
+
+    if 'allianceauth.analytics' not in category:
+        if category.endswith(".tasks"):
+            category = category[:-6]
+
+        action = sender.__name__
+        label = "Success"
+
+        value = 0
+        if isinstance(result, int):
+            value = result
+
+        analytics_event(category=category,
+                        action=action,
+                        label=label,
+                        value=value)

allianceauth/analytics/tasks.py

@@ -0,0 +1,207 @@
+import requests
+import logging
+from django.conf import settings
+from django.apps import apps
+from celery import shared_task
+from allianceauth import __version__
+from .models import AnalyticsTokens, AnalyticsIdentifier
+from .utils import (
+    install_stat_addons,
+    install_stat_tokens,
+    install_stat_users)
+
+logger = logging.getLogger(__name__)
+
+BASE_URL = "https://www.google-analytics.com/"
+
+DEBUG_URL = f"{BASE_URL}debug/collect"
+COLLECTION_URL = f"{BASE_URL}collect"
+
+if getattr(settings, "ANALYTICS_ENABLE_DEBUG", False) and settings.DEBUG:
+    # Force sending of analytics data during in a debug/test environemt
+    # Usefull for developers working on this feature.
+    logger.warning(
+                "You have 'ANALYTICS_ENABLE_DEBUG' Enabled! "
+                "This debug instance will send analytics data!")
+    DEBUG_URL = COLLECTION_URL
+
+ANALYTICS_URL = COLLECTION_URL
+
+if settings.DEBUG is True:
+    ANALYTICS_URL = DEBUG_URL
+
+
+def analytics_event(category: str,
+                    action: str,
+                    label: str,
+                    value: int = 0,
+                    event_type: str = 'Celery'):
+    """
+    Send a Google Analytics Event for each token stored
+    Includes check for if its enabled/disabled
+
+    Parameters
+    -------
+    `category` (str): Celery Namespace
+    `action` (str): Task Name
+    `label` (str): Optional, Task Success/Exception
+    `value` (int): Optional, If bulk, Query size, can be a binary True/False
+    `event_type` (str): Optional, Celery or Stats only, Default to Celery
+    """
+    analyticstokens = AnalyticsTokens.objects.all()
+    client_id = AnalyticsIdentifier.objects.get(id=1).identifier.hex
+    for token in analyticstokens:
+        if event_type == 'Celery':
+            allowed = token.send_celery_tasks
+        elif event_type == 'Stats':
+            allowed = token.send_stats
+        else:
+            allowed = False
+
+        if allowed is True:
+            tracking_id = token.token
+            send_ga_tracking_celery_event.s(tracking_id=tracking_id,
+                                            client_id=client_id,
+                                            category=category,
+                                            action=action,
+                                            label=label,
+                                            value=value).\
+                apply_async(priority=9)
+
+
+@shared_task()
+def analytics_daily_stats():
+    """Celery Task: Do not call directly
+
+    Gathers a series of daily statistics and sends analytics events containing them"""
+    users = install_stat_users()
+    tokens = install_stat_tokens()
+    addons = install_stat_addons()
+    logger.debug("Running Daily Analytics Upload")
+
+    analytics_event(category='allianceauth.analytics',
+                    action='send_install_stats',
+                    label='existence',
+                    value=1,
+                    event_type='Stats')
+    analytics_event(category='allianceauth.analytics',
+                    action='send_install_stats',
+                    label='users',
+                    value=users,
+                    event_type='Stats')
+    analytics_event(category='allianceauth.analytics',
+                    action='send_install_stats',
+                    label='tokens',
+                    value=tokens,
+                    event_type='Stats')
+    analytics_event(category='allianceauth.analytics',
+                    action='send_install_stats',
+                    label='addons',
+                    value=addons,
+                    event_type='Stats')
+
+    for appconfig in apps.get_app_configs():
+        analytics_event(category='allianceauth.analytics',
+                        action='send_extension_stats',
+                        label=appconfig.label,
+                        value=1,
+                        event_type='Stats')
+
+
+@shared_task()
+def send_ga_tracking_web_view(
+                            tracking_id: str,
+                            client_id: str,
+                            page: str,
+                            title: str,
+                            locale: str,
+                            useragent: str) -> requests.Response:
+
+    """Celery Task: Do not call directly
+
+    Sends Page View events to GA, Called only via analytics.middleware
+
+    Parameters
+    ----------
+    `tracking_id` (str): Unique Server Identifier
+    `client_id` (str): GA Token
+    `page` (str): Page Path
+    `title` (str): Page Title
+    `locale` (str): Browser Language
+    `useragent` (str): Browser UserAgent
+
+    Returns
+    -------
+    requests.Reponse Object
+    """
+    headers = {"User-Agent": useragent}
+
+    payload = {
+            'v': '1',
+            'tid': tracking_id,
+            'cid': client_id,
+            't': 'pageview',
+            'dp': page,
+            'dt': title,
+            'ul': locale,
+            'ua': useragent,
+            'aip': 1,
+            'an': "allianceauth",
+            'av': __version__
+            }
+
+    response = requests.post(
+                        ANALYTICS_URL, data=payload,
+                        timeout=5, headers=headers)
+    logger.debug(f"Analytics Page View HTTP{response.status_code}")
+    return response
+
+
+@shared_task()
+def send_ga_tracking_celery_event(
+                                tracking_id: str,
+                                client_id: str,
+                                category: str,
+                                action: str,
+                                label: str,
+                                value: int) -> requests.Response:
+    """Celery Task: Do not call directly
+
+    Sends Page View events to GA, Called only via analytics.middleware
+
+    Parameters
+    ----------
+    `tracking_id` (str): Unique Server Identifier
+    `client_id` (str): GA Token
+    `category` (str): Celery Namespace
+    `action` (str): Task Name
+    `label` (str): Optional, Task Success/Exception
+    `value` (int): Optional, If bulk, Query size, can be a binary True/False
+
+    Returns
+    -------
+    requests.Reponse Object
+    """
+
+    headers = {
+        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36"}
+
+    payload = {
+            'v': '1',
+            'tid': tracking_id,
+            'cid': client_id,
+            't': 'event',
+            'ec': category,
+            'ea': action,
+            'el': label,
+            'ev': value,
+            'aip': 1,
+            'an': "allianceauth",
+            'av': __version__
+            }
+
+    response = requests.post(
+                        ANALYTICS_URL, data=payload,
+                        timeout=5, headers=headers)
+    logger.debug(f"Analytics Celery/Stats Event HTTP{response.status_code}")
+    return response

allianceauth/analytics/tests/test_middleware.py

@@ -0,0 +1,23 @@
+from allianceauth.analytics.middleware import AnalyticsMiddleware
+from unittest.mock import Mock
+
+from django.test.testcases import TestCase
+
+
+class TestAnalyticsMiddleware(TestCase):
+
+    def setUp(self):
+        self.middleware = AnalyticsMiddleware()
+        self.request = Mock()
+        self.request.headers = {
+                "User-Agent": "AUTOMATED TEST"
+            }
+        self.request.path = '/testURL/'
+        self.request.session = {}
+        self.request.LANGUAGE_CODE = 'en'
+        self.response = Mock()
+        self.response.content = 'hello world'
+
+    def test_middleware(self):
+        response = self.middleware.process_response(self.request, self.response)
+        self.assertEqual(self.response, response)

allianceauth/analytics/tests/test_models.py

@@ -0,0 +1,26 @@
+from allianceauth.analytics.models import AnalyticsIdentifier
+from django.core.exceptions import ValidationError
+
+from django.test.testcases import TestCase
+
+from uuid import UUID, uuid4
+
+
+# Identifiers
+uuid_1 = "ab33e241fbf042b6aa77c7655a768af7"
+uuid_2 = "7aa6bd70701f44729af5e3095ff4b55c"
+
+
+class TestAnalyticsIdentifier(TestCase):
+
+    def test_identifier_random(self):
+        self.assertNotEqual(AnalyticsIdentifier.objects.get(), uuid4)
+
+    def test_identifier_singular(self):
+        AnalyticsIdentifier.objects.all().delete()
+        AnalyticsIdentifier.objects.create(identifier=uuid_1)
+        # Yeah i have multiple asserts here, they all do the same thing
+        with self.assertRaises(ValidationError):
+            AnalyticsIdentifier.objects.create(identifier=uuid_2)
+        self.assertEqual(AnalyticsIdentifier.objects.count(), 1)
+        self.assertEqual(AnalyticsIdentifier.objects.get(pk=1).identifier, UUID(uuid_1))

allianceauth/analytics/tests/test_tasks.py

@@ -0,0 +1,119 @@
+from allianceauth.analytics.tasks import (
+    analytics_event,
+    send_ga_tracking_celery_event,
+    send_ga_tracking_web_view)
+from django.test.testcases import TestCase
+
+
+class TestAnalyticsTasks(TestCase):
+    def test_analytics_event(self):
+        analytics_event(
+                        category='allianceauth.analytics',
+                        action='send_tests',
+                        label='test',
+                        value=1,
+                        event_type='Stats')
+
+    def test_send_ga_tracking_web_view_sent(self):
+        # This test sends if the event SENDS to google
+        # Not if it was successful
+        tracking_id = 'UA-186249766-2'
+        client_id = 'ab33e241fbf042b6aa77c7655a768af7'
+        page = '/index/'
+        title = 'Hello World'
+        locale = 'en'
+        useragent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36"
+        response = send_ga_tracking_web_view(
+                                            tracking_id,
+                                            client_id,
+                                            page,
+                                            title,
+                                            locale,
+                                            useragent)
+        self.assertEqual(response.status_code, 200)
+
+    def test_send_ga_tracking_web_view_success(self):
+        tracking_id = 'UA-186249766-2'
+        client_id = 'ab33e241fbf042b6aa77c7655a768af7'
+        page = '/index/'
+        title = 'Hello World'
+        locale = 'en'
+        useragent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36"
+        json_response = send_ga_tracking_web_view(
+                                                tracking_id,
+                                                client_id,
+                                                page,
+                                                title,
+                                                locale,
+                                                useragent).json()
+        self.assertTrue(json_response["hitParsingResult"][0]["valid"])
+
+    def test_send_ga_tracking_web_view_invalid_token(self):
+        tracking_id = 'UA-IntentionallyBadTrackingID-2'
+        client_id = 'ab33e241fbf042b6aa77c7655a768af7'
+        page = '/index/'
+        title = 'Hello World'
+        locale = 'en'
+        useragent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36"
+        json_response = send_ga_tracking_web_view(
+                                            tracking_id,
+                                            client_id,
+                                            page,
+                                            title,
+                                            locale,
+                                            useragent).json()
+        self.assertFalse(json_response["hitParsingResult"][0]["valid"])
+        self.assertEqual(json_response["hitParsingResult"][0]["parserMessage"][1]["description"], "The value provided for parameter 'tid' is invalid. Please see http://goo.gl/a8d4RP#tid for details.")
+
+        # [{'valid': False, 'parserMessage': [{'messageType': 'INFO', 'description': 'IP Address from this hit was anonymized to 1.132.110.0.', 'messageCode': 'VALUE_MODIFIED'}, {'messageType': 'ERROR', 'description': "The value provided for parameter 'tid' is invalid. Please see http://goo.gl/a8d4RP#tid for details.", 'messageCode': 'VALUE_INVALID', 'parameter': 'tid'}], 'hit': '/debug/collect?v=1&tid=UA-IntentionallyBadTrackingID-2&cid=ab33e241fbf042b6aa77c7655a768af7&t=pageview&dp=/index/&dt=Hello World&ul=en&ua=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36&aip=1&an=allianceauth&av=2.9.0a2'}]
+
+    def test_send_ga_tracking_celery_event_sent(self):
+        tracking_id = 'UA-186249766-2'
+        client_id = 'ab33e241fbf042b6aa77c7655a768af7'
+        category = 'test'
+        action = 'test'
+        label = 'test'
+        value = '1'
+        response = send_ga_tracking_celery_event(
+                                                tracking_id,
+                                                client_id,
+                                                category,
+                                                action,
+                                                label,
+                                                value)
+        self.assertEqual(response.status_code, 200)
+
+    def test_send_ga_tracking_celery_event_success(self):
+        tracking_id = 'UA-186249766-2'
+        client_id = 'ab33e241fbf042b6aa77c7655a768af7'
+        category = 'test'
+        action = 'test'
+        label = 'test'
+        value = '1'
+        json_response = send_ga_tracking_celery_event(
+                                                    tracking_id,
+                                                    client_id,
+                                                    category,
+                                                    action,
+                                                    label,
+                                                    value).json()
+        self.assertTrue(json_response["hitParsingResult"][0]["valid"])
+
+    def test_send_ga_tracking_celery_event_invalid_token(self):
+        tracking_id = 'UA-IntentionallyBadTrackingID-2'
+        client_id = 'ab33e241fbf042b6aa77c7655a768af7'
+        category = 'test'
+        action = 'test'
+        label = 'test'
+        value = '1'
+        json_response = send_ga_tracking_celery_event(
+                                                    tracking_id,
+                                                    client_id,
+                                                    category,
+                                                    action,
+                                                    label,
+                                                    value).json()
+        self.assertFalse(json_response["hitParsingResult"][0]["valid"])
+        self.assertEqual(json_response["hitParsingResult"][0]["parserMessage"][1]["description"], "The value provided for parameter 'tid' is invalid. Please see http://goo.gl/a8d4RP#tid for details.")
+
+        # [{'valid': False, 'parserMessage': [{'messageType': 'INFO', 'description': 'IP Address from this hit was anonymized to 1.132.110.0.', 'messageCode': 'VALUE_MODIFIED'}, {'messageType': 'ERROR', 'description': "The value provided for parameter 'tid' is invalid. Please see http://goo.gl/a8d4RP#tid for details.", 'messageCode': 'VALUE_INVALID', 'parameter': 'tid'}], 'hit': '/debug/collect?v=1&tid=UA-IntentionallyBadTrackingID-2&cid=ab33e241fbf042b6aa77c7655a768af7&t=pageview&dp=/index/&dt=Hello World&ul=en&ua=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36&aip=1&an=allianceauth&av=2.9.0a2'}]

allianceauth/analytics/tests/test_utils.py

@@ -0,0 +1,55 @@
+from django.apps import apps
+from allianceauth.authentication.models import User
+from esi.models import Token
+from allianceauth.analytics.utils import install_stat_users, install_stat_tokens, install_stat_addons
+
+from django.test.testcases import TestCase
+
+
+def create_testdata():
+    User.objects.all().delete()
+    User.objects.create_user(
+        'user_1'
+        'abc@example.com',
+        'password'
+    )
+    User.objects.create_user(
+        'user_2'
+        'abc@example.com',
+        'password'
+    )
+    #Token.objects.all().delete()
+    #Token.objects.create(
+    #            character_id=101,
+    #            character_name='character1',
+    #            access_token='my_access_token'
+    #)
+    #Token.objects.create(
+    #            character_id=102,
+    #            character_name='character2',
+    #            access_token='my_access_token'
+    #)
+
+
+class TestAnalyticsUtils(TestCase):
+
+    def test_install_stat_users(self):
+        create_testdata()
+        expected = 2
+
+        users = install_stat_users()
+        self.assertEqual(users, expected)
+
+    #def test_install_stat_tokens(self):
+    #    create_testdata()
+    #    expected = 2
+    #
+    #   tokens = install_stat_tokens()
+    #   self.assertEqual(tokens, expected)
+
+    def test_install_stat_addons(self):
+        # this test does what its testing...
+        # but helpful for existing as a sanity check
+        expected = len(list(apps.get_app_configs()))
+        addons = install_stat_addons()
+        self.assertEqual(addons, expected)

allianceauth/analytics/utils.py

@@ -0,0 +1,36 @@
+from django.apps import apps
+from allianceauth.authentication.models import User
+from esi.models import Token
+
+
+def install_stat_users() -> int:
+    """Count and Return the number of User accounts
+
+    Returns
+    -------
+    int
+        The Number of User objects"""
+    users = User.objects.count()
+    return users
+
+
+def install_stat_tokens() -> int:
+    """Count and Return the number of ESI Tokens Stored
+
+    Returns
+    -------
+    int
+        The Number of Token Objects"""
+    tokens = Token.objects.count()
+    return tokens
+
+
+def install_stat_addons() -> int:
+    """Count and Return the number of Django Applications Installed
+
+    Returns
+    -------
+    int
+        The Number of Installed Apps"""
+    addons = len(list(apps.get_app_configs()))
+    return addons

allianceauth/authentication/admin.py

@@ -1,10 +1,8 @@
-from django.conf import settings
-
 from django.contrib import admin
 from django.contrib.auth.admin import UserAdmin as BaseUserAdmin
 from django.contrib.auth.models import User as BaseUser, \
     Permission as BasePermission, Group
-from django.db.models import Q, F
+from django.db.models import Count, Q
 from allianceauth.services.hooks import ServicesHook
 from django.db.models.signals import pre_save, post_save, pre_delete, \
     post_delete, m2m_changed
@@ -24,11 +22,6 @@ from allianceauth.eveonline.tasks import update_character
 from .app_settings import AUTHENTICATION_ADMIN_USERS_MAX_GROUPS, \
     AUTHENTICATION_ADMIN_USERS_MAX_CHARS
 
-if 'allianceauth.eveonline.autogroups' in settings.INSTALLED_APPS:
-    _has_auto_groups = True    
-else:
-    _has_auto_groups = False
-
 
 def make_service_hooks_update_groups_action(service):
     """
@@ -92,7 +85,6 @@ class UserProfileInline(admin.StackedInline):
                 query |= Q(userprofile__isnull=True)
             else:
                 query |= Q(character_ownership__user=obj)
-        qs = EveCharacter.objects.filter(query)
         formset = super().get_formset(request, obj=obj, **kwargs)
 
         def get_kwargs(self, index):
@@ -100,7 +92,7 @@ class UserProfileInline(admin.StackedInline):
         formset.get_form_kwargs = get_kwargs
         return formset
 
-    def has_add_permission(self, request):
+    def has_add_permission(self, request, obj=None):
         return False
 
     def has_delete_permission(self, request, obj=None):
@@ -121,6 +113,8 @@ def user_profile_pic(obj):
         )
     else:
         return None
+
+
 user_profile_pic.short_description = ''
 
 
@@ -152,6 +146,7 @@ def user_username(obj):
             user_obj.username,
         )
 
+
 user_username.short_description = 'user / main'
 user_username.admin_order_field = 'username'
 
@@ -168,7 +163,8 @@ def user_main_organization(obj):
     else:
         corporation = user_obj.profile.main_character.corporation_name
         if user_obj.profile.main_character.alliance_id:
-            result = format_html('{}<br>{}',
+            result = format_html(
+                '{}<br>{}',
                 corporation,
                 user_obj.profile.main_character.alliance_name
             )
@@ -176,6 +172,7 @@ def user_main_organization(obj):
             result = corporation
     return result
 
+
 user_main_organization.short_description = 'Corporation / Alliance (Main)'
 user_main_organization.admin_order_field = \
     'profile__main_character__corporation_name'
@@ -205,13 +202,13 @@ class MainCorporationsFilter(admin.SimpleListFilter):
             return qs.all()
         else:
             if qs.model == User:
-                return qs\
+                return qs.filter(
-                    .filter(profile__main_character__corporation_id=\
+                    profile__main_character__corporation_id=self.value()
-                        self.value())
+                )
             else:
-                return qs\
+                return qs.filter(
-                    .filter(user__profile__main_character__corporation_id=\
+                    user__profile__main_character__corporation_id=self.value()
-                        self.value())
+                )
 
 
 class MainAllianceFilter(admin.SimpleListFilter):
@@ -239,12 +236,11 @@ class MainAllianceFilter(admin.SimpleListFilter):
             return qs.all()
         else:
             if qs.model == User:
-                return qs\
+                return qs.filter(profile__main_character__alliance_id=self.value())
-                    .filter(profile__main_character__alliance_id=self.value())                
             else:
-                return qs\
+                return qs.filter(
-                    .filter(user__profile__main_character__alliance_id=\
+                    user__profile__main_character__alliance_id=self.value()
-                        self.value())
+                )
 
 
 def update_main_character_model(modeladmin, request, queryset):
@@ -259,6 +255,7 @@ def update_main_character_model(modeladmin, request, queryset):
         'Update from ESI started for {} characters'.format(tasks_count)
     )
 
+
 update_main_character_model.short_description = \
     'Update main character model from ESI'
 
@@ -267,7 +264,6 @@ class UserAdmin(BaseUserAdmin):
     """Extending Django's UserAdmin model
 
     Behavior of groups and characters columns can be configured via settings
-
     """
 
     class Media:
@@ -275,24 +271,9 @@ class UserAdmin(BaseUserAdmin):
             "all": ("authentication/css/admin.css",)
         }
 
-    class RealGroupsFilter(admin.SimpleListFilter):
+    def get_queryset(self, request):
-        """Custom filter to get groups w/o Autogroups"""
+        qs = super().get_queryset(request)
-        title = 'group'
+        return qs.prefetch_related("character_ownerships__character", "groups")
-        parameter_name = 'group_id__exact'
-
-        def lookups(self, request, model_admin):
-            qs = Group.objects.all().order_by(Lower('name'))
-            if _has_auto_groups:
-                qs = qs\
-                    .filter(managedalliancegroup__isnull=True)\
-                    .filter(managedcorpgroup__isnull=True)                
-            return tuple([(x.pk, x.name) for x in qs])
-
-        def queryset(self, request, queryset):
-            if self.value() is None:
-                return queryset.all()
-            else:    
-                return queryset.filter(groups__pk=self.value())
 
     def get_actions(self, request):
         actions = super(BaseUserAdmin, self).get_actions(request)
@@ -341,11 +322,9 @@ class UserAdmin(BaseUserAdmin):
         return result
 
     inlines = BaseUserAdmin.inlines + [UserProfileInline]
-
     ordering = ('username', )
-    list_select_related = True
+    list_select_related = ('profile__state', 'profile__main_character')
     show_full_result_count = True
-    
     list_display = (
         user_profile_pic,
         user_username,
@@ -358,10 +337,9 @@ class UserAdmin(BaseUserAdmin):
         '_role'
     )
     list_display_links = None
-
     list_filter = (
         'profile__state',
-        RealGroupsFilter,        
+        'groups',
         MainCorporationsFilter,
         MainAllianceFilter,
         'is_active',
@@ -375,21 +353,15 @@ class UserAdmin(BaseUserAdmin):
     )
 
     def _characters(self, obj):
-        my_characters = [
+        character_ownerships = list(obj.character_ownerships.all())
-            x.character.character_name 
+        characters = [obj.character.character_name for obj in character_ownerships]
-            for x in CharacterOwnership.objects\
-                .filter(user=obj)\
-                .order_by('character__character_name')\
-                .select_related()
-        ]
         return self._list_2_html_w_tooltips(
-            my_characters, 
+            sorted(characters),
             AUTHENTICATION_ADMIN_USERS_MAX_CHARS
         )
 
     _characters.short_description = 'characters'
 
-
     def _state(self, obj):
         return obj.profile.state.name
 
@@ -397,19 +369,9 @@ class UserAdmin(BaseUserAdmin):
     _state.admin_order_field = 'profile__state'
 
     def _groups(self, obj):
-        if not _has_auto_groups:
+        my_groups = sorted([group.name for group in list(obj.groups.all())])
-            my_groups = [x.name for x in obj.groups.order_by('name')]
-        else:
-            my_groups = [
-                x.name for x in obj.groups\
-                    .filter(managedalliancegroup__isnull=True)\
-                    .filter(managedcorpgroup__isnull=True)\
-                    .order_by('name')
-            ]
-        
         return self._list_2_html_w_tooltips(
-            my_groups, 
+            my_groups, AUTHENTICATION_ADMIN_USERS_MAX_GROUPS
-            AUTHENTICATION_ADMIN_USERS_MAX_GROUPS
         )
 
     _groups.short_description = 'groups'
@@ -446,9 +408,14 @@ class StateAdmin(admin.ModelAdmin):
     list_select_related = True
     list_display = ('name', 'priority', '_user_count')
 
+    def get_queryset(self, request):
+        qs = super().get_queryset(request)
+        return qs.annotate(user_count=Count("userprofile__id"))
+
     def _user_count(self, obj):
-        return obj.userprofile_set.all().count()
+        return obj.user_count
     _user_count.short_description = 'Users'
+    _user_count.admin_order_field = 'user_count'
 
     fieldsets = (
         (None, {
@@ -481,6 +448,8 @@ class StateAdmin(admin.ModelAdmin):
         elif db_field.name == "member_alliances":
             kwargs["queryset"] = EveAllianceInfo.objects.all()\
                 .order_by(Lower('alliance_name'))
+        elif db_field.name == "permissions":
+            kwargs["queryset"] = Permission.objects.select_related("content_type").all()
         return super().formfield_for_manytomany(db_field, request, **kwargs)
 
     def has_delete_permission(self, request, obj=None):
@@ -504,7 +473,8 @@ class BaseOwnershipAdmin(admin.ModelAdmin):
             "all": ("authentication/css/admin.css",)
         }
 
-    list_select_related = True
+    list_select_related = (
+        'user__profile__state', 'user__profile__main_character', 'character')
     list_display = (
         user_profile_pic,
         user_username,
@@ -542,6 +512,7 @@ class CharacterOwnershipAdmin(BaseOwnershipAdmin):
 class PermissionAdmin(admin.ModelAdmin):
     actions = None
     readonly_fields = [field.name for field in BasePermission._meta.fields]
+    search_fields = ('codename', )
     list_display = ('admin_name', 'name', 'codename', 'content_type')
     list_filter = ('content_type__app_label',)
 
@@ -549,7 +520,7 @@ class PermissionAdmin(admin.ModelAdmin):
     def admin_name(obj):
         return str(obj)
 
-    def has_add_permission(self, request):
+    def has_add_permission(self, request, obj=None):
         return False
 
     def has_delete_permission(self, request, obj=None):

allianceauth/authentication/app_settings.py

@@ -43,4 +43,3 @@ AUTHENTICATION_ADMIN_USERS_MAX_GROUPS = \
 
 AUTHENTICATION_ADMIN_USERS_MAX_CHARS = \
     _clean_setting('AUTHENTICATION_ADMIN_USERS_MAX_CHARS', 5)
-

allianceauth/authentication/forms.py

@@ -1,6 +1,8 @@
 from django import forms
 from django.utils.translation import ugettext_lazy as _
-
+from allianceauth.authentication.models import User
-
 class RegistrationForm(forms.Form):
     email = forms.EmailField(label=_('Email'), max_length=254, required=True)
+
+    class _meta:
+        model = User

allianceauth/authentication/hmac_urls.py

@@ -10,5 +10,5 @@ urlpatterns = [
     url(r'^register/$', views.RegistrationView.as_view(), name='registration_register'),
     url(r'^register/complete/$', views.registration_complete, name='registration_complete'),
     url(r'^register/closed/$', views.registration_closed, name='registration_disallowed'),
-    url(r'', include('registration.auth_urls')),
+    url(r'', include('django.contrib.auth.urls')),
 ]

allianceauth/authentication/managers.py

@@ -23,8 +23,7 @@ class CharacterOwnershipManager(Manager):
     def create_by_token(self, token):
         if not EveCharacter.objects.filter(character_id=token.character_id).exists():
             EveCharacter.objects.create_character(token.character_id)
-        return self.create(character=EveCharacter.objects.get(character_id=token.character_id), user=token.user,
+        return self.create(character=EveCharacter.objects.get(character_id=token.character_id), user=token.user, owner_hash=token.character_owner_hash)
-                           owner_hash=token.character_owner_hash)
 
 
 class StateQuerySet(QuerySet):

allianceauth/authentication/migrations/0001_initial.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Generated by Django 1.10.1 on 2016-09-05 21:38
 from __future__ import unicode_literals
 

allianceauth/authentication/migrations/0002_auto_20160907_1914.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Generated by Django 1.10.1 on 2016-09-07 19:14
 from __future__ import unicode_literals
 

allianceauth/authentication/migrations/0003_authservicesinfo_state.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Generated by Django 1.10.1 on 2016-09-09 20:29
 from __future__ import unicode_literals
 

allianceauth/authentication/migrations/0004_create_permissions.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Generated by Django 1.10.1 on 2016-09-09 23:19
 from __future__ import unicode_literals
 

allianceauth/authentication/migrations/0005_delete_perms.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Generated by Django 1.10.1 on 2016-09-09 23:11
 from __future__ import unicode_literals
 

allianceauth/authentication/migrations/0006_auto_20160910_0542.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Generated by Django 1.10.1 on 2016-09-10 05:42
 from __future__ import unicode_literals
 

allianceauth/authentication/migrations/0007_remove_authservicesinfo_is_blue.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Generated by Django 1.10.1 on 2016-09-10 21:50
 from __future__ import unicode_literals
 

allianceauth/authentication/migrations/0008_set_state.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Generated by Django 1.10.1 on 2016-09-12 13:04
 from __future__ import unicode_literals
 

allianceauth/authentication/migrations/0009_auto_20161021_0228.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Generated by Django 1.10.2 on 2016-10-21 02:28
 from __future__ import unicode_literals
 

allianceauth/authentication/migrations/0010_only_one_authservicesinfo.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Generated by Django 1.10.1 on 2017-01-07 06:47
 from __future__ import unicode_literals
 

allianceauth/authentication/migrations/0011_authservicesinfo_user_onetoonefield.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Generated by Django 1.10.1 on 2017-01-07 07:11
 from __future__ import unicode_literals
 

allianceauth/authentication/migrations/0012_remove_add_delete_authservicesinfo_permissions.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Generated by Django 1.10.5 on 2017-01-12 00:59
 from __future__ import unicode_literals
 

allianceauth/authentication/migrations/0013_service_modules.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Generated by Django 1.10.2 on 2016-12-11 23:14
 from __future__ import unicode_literals
 

allianceauth/authentication/migrations/0014_fleetup_permission.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Generated by Django 1.10.1 on 2016-09-09 23:19
 from __future__ import unicode_literals
 

allianceauth/authentication/migrations/0015_user_profiles.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Generated by Django 1.10.5 on 2017-03-22 23:09
 from __future__ import unicode_literals
 
@@ -171,8 +170,7 @@ def recreate_authservicesinfo(apps, schema_editor):
 
     # repopulate main characters
     for profile in UserProfile.objects.exclude(main_character__isnull=True).select_related('user', 'main_character'):
-        AuthServicesInfo.objects.update_or_create(user=profile.user,
+        AuthServicesInfo.objects.update_or_create(user=profile.user, defaults={'main_char_id': profile.main_character.character_id})
-                                                  defaults={'main_char_id': profile.main_character.character_id})
 
     # repopulate states we understand
     for profile in UserProfile.objects.exclude(state__name='Guest').filter(

allianceauth/authentication/migrations/0017_remove_fleetup_permission.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 from __future__ import unicode_literals
 
 from django.db import migrations

allianceauth/authentication/models.py

@@ -14,15 +14,11 @@ logger = logging.getLogger(__name__)
 class State(models.Model):
     name = models.CharField(max_length=20, unique=True)
     permissions = models.ManyToManyField(Permission, blank=True)
-    priority = models.IntegerField(unique=True,
+    priority = models.IntegerField(unique=True, help_text="Users get assigned the state with the highest priority available to them.")
-                                   help_text="Users get assigned the state with the highest priority available to them.")
 
-    member_characters = models.ManyToManyField(EveCharacter, blank=True,
+    member_characters = models.ManyToManyField(EveCharacter, blank=True, help_text="Characters to which this state is available.")
-                                               help_text="Characters to which this state is available.")
+    member_corporations = models.ManyToManyField(EveCorporationInfo, blank=True, help_text="Corporations to whose members this state is available.")
-    member_corporations = models.ManyToManyField(EveCorporationInfo, blank=True,
+    member_alliances = models.ManyToManyField(EveAllianceInfo, blank=True, help_text="Alliances to whose members this state is available.")
-                                                 help_text="Corporations to whose members this state is available.")
-    member_alliances = models.ManyToManyField(EveAllianceInfo, blank=True,
-                                              help_text="Alliances to whose members this state is available.")
     public = models.BooleanField(default=False, help_text="Make this state available to any character.")
 
     objects = StateManager()
@@ -81,6 +77,11 @@ class UserProfile(models.Model):
                     'info'
                 )
                 from allianceauth.authentication.signals import state_changed
+
+                # We need to ensure we get up to date perms here as they will have just changed.
+                # Clear all attribute caches and reload the model that will get passed to the signals!
+                self.refresh_from_db()
+
                 state_changed.send(
                     sender=self.__class__, user=self.user, state=self.state
                 )

allianceauth/authentication/signals.py

@@ -75,8 +75,7 @@ def create_required_models(sender, instance, created, *args, **kwargs):
 @receiver(post_save, sender=Token)
 def record_character_ownership(sender, instance, created, *args, **kwargs):
     if created:
-        logger.debug('New token for {0} character {1} saved. Evaluating ownership.'.format(instance.user,
+        logger.debug('New token for {0} character {1} saved. Evaluating ownership.'.format(instance.user, instance.character_name))
-                                                                                           instance.character_name))
         if instance.user:
             query = Q(owner_hash=instance.character_owner_hash) & Q(user=instance.user)
         else:
@@ -85,18 +84,14 @@ def record_character_ownership(sender, instance, created, *args, **kwargs):
         CharacterOwnership.objects.filter(character__character_id=instance.character_id).exclude(query).delete()
         # create character if needed
         if EveCharacter.objects.filter(character_id=instance.character_id).exists() is False:
-            logger.debug('Token is for a new character. Creating model for {0} ({1})'.format(instance.character_name,
+            logger.debug('Token is for a new character. Creating model for {0} ({1})'.format(instance.character_name, instance.character_id))
-                                                                                             instance.character_id))
             EveCharacter.objects.create_character(instance.character_id)
         char = EveCharacter.objects.get(character_id=instance.character_id)
         # check if we need to create ownership
         if instance.user and not CharacterOwnership.objects.filter(
                 character__character_id=instance.character_id).exists():
-            logger.debug("Character {0} is not yet owned. Assigning ownership to {1}".format(instance.character_name,
+            logger.debug("Character {0} is not yet owned. Assigning ownership to {1}".format(instance.character_name, instance.user))
-                                                                                             instance.user))
+            CharacterOwnership.objects.update_or_create(character=char, defaults={'owner_hash': instance.character_owner_hash, 'user': instance.user})
-            CharacterOwnership.objects.update_or_create(character=char,
-                                                        defaults={'owner_hash': instance.character_owner_hash,
-                                                                  'user': instance.user})
 
 
 @receiver(pre_delete, sender=CharacterOwnership)

allianceauth/authentication/templates/authentication/dashboard.html

@@ -1,5 +1,5 @@
 {% extends "allianceauth/base.html" %}
-{% load staticfiles %}
+{% load static %}
 {% load i18n %}
 
 {% block page_title %}{% trans "Dashboard" %}{% endblock %}

allianceauth/authentication/templates/public/register.html

@@ -1,7 +1,7 @@
-{% load staticfiles %}
+{% extends 'public/base.html' %}
+{% load static %}
 {% load bootstrap %}
 {% load i18n %}
-{% extends 'public/base.html' %}
 {% block page_title %}Registration{% endblock %}
 {% block extra_include %}
     {% include 'bundles/bootstrap-css.html' %}

allianceauth/authentication/tests/__init__.py

@@ -10,6 +10,7 @@ def get_admin_change_view_url(obj: object) -> str:
         args=(obj.pk,)
     )
 
+
 def get_admin_search_url(ModelClass: type) -> str:
     """returns URL to search URL for model of given object"""
     return '{}{}/'.format(

allianceauth/authentication/tests/test_admin.py

@@ -1,10 +1,8 @@
 from urllib.parse import quote
 from unittest.mock import patch, MagicMock
 
-from django.conf import settings
-from django.contrib import admin
 from django.contrib.admin.sites import AdminSite
-from django.contrib.auth.models import User as BaseUser, Group
+from django.contrib.auth.models import Group
 from django.test import TestCase, RequestFactory, Client
 
 from allianceauth.authentication.models import (
@@ -19,7 +17,6 @@ from allianceauth.tests.auth_utils import AuthUtils
 from ..admin import (
     BaseUserAdmin,
     CharacterOwnershipAdmin,
-    PermissionAdmin,
     StateAdmin,
     MainCorporationsFilter,
     MainAllianceFilter,
@@ -35,11 +32,6 @@ from ..admin import (
 )
 from . import get_admin_change_view_url, get_admin_search_url
 
-if 'allianceauth.eveonline.autogroups' in settings.INSTALLED_APPS:
-    _has_auto_groups = True
-    from allianceauth.eveonline.autogroups.models import AutogroupsConfig
-else:
-    _has_auto_groups = False
 
 MODULE_PATH = 'allianceauth.authentication.admin'
 
@@ -48,6 +40,7 @@ class MockRequest(object):
     def __init__(self, user=None):
         self.user = user
 
+
 class TestCaseWithTestData(TestCase):
 
     @classmethod
@@ -279,6 +272,7 @@ class TestStateAdmin(TestCaseWithTestData):
         expected = 200
         self.assertEqual(response.status_code, expected)
 
+
 class TestUserAdmin(TestCaseWithTestData):
 
     def setUp(self):
@@ -288,23 +282,11 @@ class TestUserAdmin(TestCaseWithTestData):
         )
         self.character_1 = self.user_1.character_ownerships.first().character
 
-    def _create_autogroups(self):
-        """create autogroups for corps and alliances"""
-        if _has_auto_groups:
-            autogroups_config = AutogroupsConfig(
-                corp_groups = True,
-                alliance_groups = True
-            )
-            autogroups_config.save()
-            for state in State.objects.all():
-                autogroups_config.states.add(state)        
-            autogroups_config.update_corp_group_membership(self.user_1)        
-    
-    # column rendering
-
     def test_user_profile_pic_u1(self):
-        expected = ('<img src="https://images.evetech.net/characters/1001/'
+        expected = (
-            'portrait?size=32" class="img-circle">')        
+            '<img src="https://images.evetech.net/characters/1001/'
+            'portrait?size=32" class="img-circle">'
+        )
         self.assertEqual(user_profile_pic(self.user_1), expected)
 
     def test_user_profile_pic_u3(self):
@@ -352,36 +334,16 @@ class TestUserAdmin(TestCaseWithTestData):
         self.assertEqual(result, expected)
 
     def test_groups_u1(self):
-        self._create_autogroups()
         expected = 'Group 1'
         result = self.modeladmin._groups(self.user_1)
         self.assertEqual(result, expected)
 
     def test_groups_u2(self):
-        self._create_autogroups()
         expected = 'Group 2'
         result = self.modeladmin._groups(self.user_2)
         self.assertEqual(result, expected)
 
     def test_groups_u3(self):
-        self._create_autogroups()
-        result = self.modeladmin._groups(self.user_3)
-        self.assertIsNone(result)
-
-    @patch(MODULE_PATH + '._has_auto_groups', False)
-    def test_groups_u1_no_autogroups(self):        
-        expected = 'Group 1'
-        result = self.modeladmin._groups(self.user_1)
-        self.assertEqual(result, expected)
-
-    @patch(MODULE_PATH + '._has_auto_groups', False)
-    def test_groups_u2_no_autogroups(self):        
-        expected = 'Group 2'
-        result = self.modeladmin._groups(self.user_2)
-        self.assertEqual(result, expected)
-    
-    @patch(MODULE_PATH + '._has_auto_groups', False)
-    def test_groups_u3_no_autogroups(self):                
         result = self.modeladmin._groups(self.user_3)
         self.assertIsNone(result)
 
@@ -413,8 +375,10 @@ class TestUserAdmin(TestCaseWithTestData):
 
     def test_list_2_html_w_tooltips_w_cutoff(self):
         items = ['one', 'two', 'three']
-        expected = ('<span data-tooltip="one, two, three" '
+        expected = (
-            'class="tooltip">one, two, (...)</span>')
+            '<span data-tooltip="one, two, three" '
+            'class="tooltip">one, two, (...)</span>'
+        )
         result = self.modeladmin._list_2_html_w_tooltips(items, 2)
         self.assertEqual(expected, result)
 
@@ -440,62 +404,6 @@ class TestUserAdmin(TestCaseWithTestData):
 
     # filters
 
-    def test_filter_real_groups_with_autogroups(self):
-        
-        class UserAdminTest(BaseUserAdmin): 
-            list_filter = (UserAdmin.RealGroupsFilter,)
-                
-        self._create_autogroups()        
-        my_modeladmin = UserAdminTest(User, AdminSite())
-
-        # Make sure the lookups are correct
-        request = self.factory.get('/')
-        request.user = self.user_1
-        changelist = my_modeladmin.get_changelist_instance(request)
-        filters = changelist.get_filters(request)
-        filterspec = filters[0][0]
-        expected = [
-            (self.group_1.pk, self.group_1.name), 
-            (self.group_2.pk, self.group_2.name),
-        ]
-        self.assertEqual(filterspec.lookup_choices, expected)
-
-        # Make sure the correct queryset is returned
-        request = self.factory.get('/', {'group_id__exact': self.group_1.pk})
-        request.user = self.user_1                
-        changelist = my_modeladmin.get_changelist_instance(request)
-        queryset = changelist.get_queryset(request)
-        expected = User.objects.filter(groups__in=[self.group_1])
-        self.assertSetEqual(set(queryset), set(expected))
-
-    @patch(MODULE_PATH + '._has_auto_groups', False)
-    def test_filter_real_groups_no_autogroups(self):
-        
-        class UserAdminTest(BaseUserAdmin): 
-            list_filter = (UserAdmin.RealGroupsFilter,)
-                        
-        my_modeladmin = UserAdminTest(User, AdminSite())
-
-        # Make sure the lookups are correct
-        request = self.factory.get('/')
-        request.user = self.user_1
-        changelist = my_modeladmin.get_changelist_instance(request)
-        filters = changelist.get_filters(request)
-        filterspec = filters[0][0]
-        expected = [
-            (self.group_1.pk, self.group_1.name), 
-            (self.group_2.pk, self.group_2.name),
-        ]
-        self.assertEqual(filterspec.lookup_choices, expected)
-
-        # Make sure the correct queryset is returned
-        request = self.factory.get('/', {'group_id__exact': self.group_1.pk})
-        request.user = self.user_1                
-        changelist = my_modeladmin.get_changelist_instance(request)
-        queryset = changelist.get_queryset(request)
-        expected = User.objects.filter(groups__in=[self.group_1])
-        self.assertSetEqual(set(queryset), set(expected))
-
     def test_filter_main_corporations(self):
 
         class UserAdminTest(BaseUserAdmin):
@@ -603,7 +511,6 @@ class TestMakeServicesHooksActions(TestCaseWithTestData):
         def sync_nicknames_bulk(self, user):
             pass
 
-
     def test_service_has_update_groups_only(self):
         service = self.MyServicesHookTypeA()
         mock_service = MagicMock(spec=service)

allianceauth/authentication/tests/test_templatetags.py

@@ -1,10 +1,10 @@
 from math import ceil
 from unittest.mock import patch
 
-from requests import RequestException
 import requests_mock
 from packaging.version import Version as Pep440Version
 
+from django.core.cache import cache
 from django.test import TestCase
 
 from allianceauth.templatetags.admin_status import (
@@ -13,7 +13,6 @@ from allianceauth.templatetags.admin_status import (
     _current_notifications,
     _current_version_summary,
     _fetch_notification_issues_from_gitlab,
-    _fetch_tags_from_gitlab,
     _latests_versions
 )
 
@@ -84,8 +83,7 @@ class TestStatusOverviewTag(TestCase):
         mock_current_version_info.return_value = version_info
         mock_fetch_celery_queue_length.return_value = 3
 
-        context = {}
+        result = status_overview()
-        result = status_overview(context)
         expected = {
             'notifications': GITHUB_NOTIFICATION_ISSUES[:5],
             'latest_major': True,
@@ -104,28 +102,51 @@ class TestStatusOverviewTag(TestCase):
 
 class TestNotifications(TestCase):
 
+    def setUp(self) -> None:
+        cache.clear()
+
     @requests_mock.mock()
     def test_fetch_notification_issues_from_gitlab(self, requests_mocker):
+        # given
         url = (
             'https://gitlab.com/api/v4/projects/allianceauth%2Fallianceauth/issues'
             '?labels=announcement'
         )
         requests_mocker.get(url, json=GITHUB_NOTIFICATION_ISSUES)
+        # when
         result = _fetch_notification_issues_from_gitlab()
+        # then
         self.assertEqual(result, GITHUB_NOTIFICATION_ISSUES)
 
     @patch(MODULE_PATH + '.admin_status.cache')
     def test_current_notifications_normal(self, mock_cache):
+        # given
         mock_cache.get_or_set.return_value = GITHUB_NOTIFICATION_ISSUES
-
+        # when
         result = _current_notifications()
+        # then
         self.assertEqual(result['notifications'], GITHUB_NOTIFICATION_ISSUES[:5])
 
-    @patch(MODULE_PATH + '.admin_status.cache')
+    @requests_mock.mock()
-    def test_current_notifications_failed(self, mock_cache):
+    def test_current_notifications_failed(self, requests_mocker):
-        mock_cache.get_or_set.side_effect = RequestException
+        # given
-
+        url = (
+            'https://gitlab.com/api/v4/projects/allianceauth%2Fallianceauth/issues'
+            '?labels=announcement'
+        )
+        requests_mocker.get(url, status_code=404)
+        # when
         result = _current_notifications()
+        # then
+        self.assertEqual(result['notifications'], list())
+
+    @patch(MODULE_PATH + '.admin_status.cache')
+    def test_current_notifications_is_none(self, mock_cache):
+        # given
+        mock_cache.get_or_set.return_value = None
+        # when
+        result = _current_notifications()
+        # then
         self.assertEqual(result['notifications'], list())
 
 
@@ -137,12 +158,17 @@ class TestCeleryQueueLength(TestCase):
 
 class TestVersionTags(TestCase):
 
+    def setUp(self) -> None:
+        cache.clear()
+
     @patch(MODULE_PATH + '.admin_status.__version__', TEST_VERSION)
     @patch(MODULE_PATH + '.admin_status.cache')
     def test_current_version_info_normal(self, mock_cache):
+        # given
         mock_cache.get_or_set.return_value = GITHUB_TAGS
-
+        # when
         result = _current_version_summary()
+        # then
         self.assertTrue(result['latest_major'])
         self.assertTrue(result['latest_minor'])
         self.assertTrue(result['latest_patch'])
@@ -152,23 +178,41 @@ class TestVersionTags(TestCase):
         self.assertEqual(result['latest_beta_version'], '2.4.6a1')
 
     @patch(MODULE_PATH + '.admin_status.__version__', TEST_VERSION)
-    @patch(MODULE_PATH + '.admin_status.cache')
+    @requests_mock.mock()
-    def test_current_version_info_failed(self, mock_cache):
+    def test_current_version_info_failed(self, requests_mocker):
-        mock_cache.get_or_set.side_effect = RequestException
+        # given
-        
+        url = (
-        expected = {}
+            'https://gitlab.com/api/v4/projects/allianceauth%2Fallianceauth'
+            '/repository/tags'
+        )
+        requests_mocker.get(url, status_code=500)
+        # when
         result = _current_version_summary()
-        self.assertEqual(result, expected)
+        # then
+        self.assertEqual(result, {})
 
     @requests_mock.mock()
     def test_fetch_tags_from_gitlab(self, requests_mocker):
+        # given
         url = (
             'https://gitlab.com/api/v4/projects/allianceauth%2Fallianceauth'
             '/repository/tags'
         )
         requests_mocker.get(url, json=GITHUB_TAGS)
-        result = _fetch_tags_from_gitlab()
+        # when
-        self.assertEqual(result, GITHUB_TAGS)
+        result = _current_version_summary()
+        # then
+        self.assertTrue(result)
+
+    @patch(MODULE_PATH + '.admin_status.__version__', TEST_VERSION)
+    @patch(MODULE_PATH + '.admin_status.cache')
+    def test_current_version_info_return_no_data(self, mock_cache):
+        # given
+        mock_cache.get_or_set.return_value = None
+        # when
+        result = _current_version_summary()
+        # then
+        self.assertEqual(result, {})
 
 
 class TestLatestsVersion(TestCase):

allianceauth/authentication/views.py

@@ -6,20 +6,21 @@ from django.contrib.auth import login, authenticate
 from django.contrib.auth.decorators import login_required
 from django.contrib.auth.models import User
 from django.core import signing
-from django.urls import reverse
+from django.http import JsonResponse
 from django.shortcuts import redirect, render
+from django.urls import reverse, reverse_lazy
 from django.utils.translation import gettext_lazy as _
 
 from allianceauth.eveonline.models import EveCharacter
 from esi.decorators import token_required
 from esi.models import Token
 
-from registration.backends.hmac.views import (
+from django_registration.backends.activation.views import (
     RegistrationView as BaseRegistrationView,
     ActivationView as BaseActivationView,
     REGISTRATION_SALT
 )
-from registration.signals import user_registered
+from django_registration.signals import user_registered
 
 from .models import CharacterOwnership
 from .forms import RegistrationForm
@@ -134,11 +135,14 @@ def sso_login(request, token):
 # Step 2
 class RegistrationView(BaseRegistrationView):
     form_class = RegistrationForm
-    success_url = 'authentication:dashboard'
+    template_name = "public/register.html"
+    email_body_template = "registration/activation_email.txt"
+    email_subject_template = "registration/activation_email_subject.txt"
+    success_url = reverse_lazy('registration_complete')
 
     def get_success_url(self, user):
         if not getattr(settings, 'REGISTRATION_VERIFY_EMAIL', True):
-            return 'authentication:dashboard', (), {}
+            return reverse_lazy('authentication:dashboard')
         return super().get_success_url(user)
 
     def dispatch(self, request, *args, **kwargs):
@@ -176,6 +180,9 @@ class RegistrationView(BaseRegistrationView):
 
 # Step 3
 class ActivationView(BaseActivationView):
+    template_name = "registration/activate.html"
+    success_url = reverse_lazy('registration_activation_complete')
+
     def validate_key(self, activation_key):
         try:
             dump = signing.loads(activation_key, salt=REGISTRATION_SALT,
@@ -207,5 +214,5 @@ def activation_complete(request):
 
 
 def registration_closed(request):
-    messages.error(request, _('Registraion of new accounts it not allowed at this time.'))
+    messages.error(request, _('Registration of new accounts is not allowed at this time.'))
     return redirect('authentication:login')

allianceauth/corputils/auth_hooks.py

@@ -6,11 +6,13 @@ from allianceauth.corputils import urls
 
 class CorpStats(MenuItemHook):
     def __init__(self):
-        MenuItemHook.__init__(self,
+        MenuItemHook.__init__(
+            self,
             _('Corporation Stats'),
             'fas fa-share-alt fa-fw',
             'corputils:view',
-                              navactive=['corputils:'])
+            navactive=['corputils:']
+        )
 
     def render(self, request):
         if request.user.has_perm('corputils.view_corp_corpstats') or request.user.has_perm(

allianceauth/corputils/migrations/0001_initial.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Generated by Django 1.10.1 on 2016-12-14 21:36
 from __future__ import unicode_literals
 

allianceauth/corputils/migrations/0002_migrate_permissions.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Generated by Django 1.10.1 on 2016-12-14 21:48
 from __future__ import unicode_literals
 

allianceauth/corputils/migrations/0003_granular_permissions.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Generated by Django 1.10.5 on 2017-03-22 23:35
 from __future__ import unicode_literals
 

allianceauth/corputils/migrations/0004_member_models.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Generated by Django 1.10.5 on 2017-03-26 20:13
 from __future__ import unicode_literals
 
@@ -13,8 +12,7 @@ def convert_json_to_members(apps, schema_editor):
     for cs in CorpStats.objects.all():
         members = json.loads(cs._members)
         CorpMember.objects.bulk_create(
-            [CorpMember(corpstats=cs, character_id=member_id, character_name=member_name) for member_id, member_name in
+            [CorpMember(corpstats=cs, character_id=member_id, character_name=member_name) for member_id, member_name in members.items()]
-             members.items()]
         )
 
 

allianceauth/corputils/migrations/0005_cleanup_permissions.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Generated by Django 1.11.2 on 2017-06-10 15:34
 from __future__ import unicode_literals
 

allianceauth/corputils/models.py

@@ -6,8 +6,7 @@ from bravado.exception import HTTPForbidden
 from django.db import models
 from esi.errors import TokenError
 from esi.models import Token
-from allianceauth.eveonline.models import EveCorporationInfo, EveCharacter,\
+from allianceauth.eveonline.models import EveCorporationInfo, EveCharacter, EveAllianceInfo
-     EveAllianceInfo
 from allianceauth.notifications import notify
 
 from allianceauth.corputils.managers import CorpStatsManager
@@ -49,8 +48,7 @@ class CorpStats(models.Model):
     def update(self):
         try:
             c = self.token.get_esi_client(spec_file=SWAGGER_SPEC_PATH)
-            assert c.Character.get_characters_character_id(character_id=self.token.character_id).result()[
+            assert c.Character.get_characters_character_id(character_id=self.token.character_id).result()['corporation_id'] == int(self.corp.corporation_id)
-                       'corporation_id'] == int(self.corp.corporation_id)
             member_ids = c.Corporation.get_corporations_corporation_id_members(
                 corporation_id=self.corp.corporation_id).result()
 
@@ -58,18 +56,15 @@ class CorpStats(models.Model):
             # the swagger spec doesn't have a maxItems count
             # manual testing says we can do over 350, but let's not risk it
             member_id_chunks = [member_ids[i:i + 255] for i in range(0, len(member_ids), 255)]
-            member_name_chunks = [c.Universe.post_universe_names(ids=id_chunk).result() for id_chunk in
+            member_name_chunks = [c.Universe.post_universe_names(ids=id_chunk).result() for id_chunk in member_id_chunks]
-                                  member_id_chunks]
             member_list = {}
             for name_chunk in member_name_chunks:
                 member_list.update({m['id']: m['name'] for m in name_chunk})
 
             # bulk create new member models
-            missing_members = [m_id for m_id in member_ids if
+            missing_members = [m_id for m_id in member_ids if not CorpMember.objects.filter(corpstats=self, character_id=m_id).exists()]
-                               not CorpMember.objects.filter(corpstats=self, character_id=m_id).exists()]
             CorpMember.objects.bulk_create(
-                [CorpMember(character_id=m_id, character_name=member_list[m_id], corpstats=self) for m_id in
+                [CorpMember(character_id=m_id, character_name=member_list[m_id], corpstats=self) for m_id in missing_members])
-                 missing_members])
 
             # purge old members
             self.members.exclude(character_id__in=member_ids).delete()
@@ -80,20 +75,21 @@ class CorpStats(models.Model):
         except TokenError as e:
             logger.warning("%s failed to update: %s" % (self, e))
             if self.token.user:
-                notify(self.token.user, "%s failed to update with your ESI token." % self,
+                notify(
+                    self.token.user, "%s failed to update with your ESI token." % self,
                     message="Your token has expired or is no longer valid. Please add a new one to create a new CorpStats.",
                     level="error")
             self.delete()
         except HTTPForbidden as e:
             logger.warning("%s failed to update: %s" % (self, e))
             if self.token.user:
-                notify(self.token.user, "%s failed to update with your ESI token." % self,
+                notify(self.token.user, "%s failed to update with your ESI token." % self, message="%s: %s" % (e.status_code, e.message), level="error")
-                       message="%s: %s" % (e.status_code, e.message), level="error")
             self.delete()
         except AssertionError:
             logger.warning("%s token character no longer in corp." % self)
             if self.token.user:
-                notify(self.token.user, "%s cannot update with your ESI token." % self,
+                notify(
+                    self.token.user, "%s cannot update with your ESI token." % self,
                     message="%s cannot update with your ESI token as you have left corp." % self, level="error")
             self.delete()
 
@@ -127,9 +123,7 @@ class CorpStats(models.Model):
 
     @property
     def mains(self):
-        return self.members.filter(pk__in=[m.pk for m in self.members.all() if
+        return self.members.filter(pk__in=[m.pk for m in self.members.all() if m.main_character and int(m.main_character.character_id) == int(m.character_id)])
-                                           m.main_character and int(m.main_character.character_id) == int(
-                                               m.character_id)])
 
     def visible_to(self, user):
         return CorpStats.objects.filter(pk=self.pk).visible_to(user).exists()

allianceauth/corputils/templates/corputils/corpstats.html

@@ -58,8 +58,7 @@
                                                 {% for id, main in mains.items %}
                                                     <tr>
                                                         <td class="text-center" style="vertical-align:middle">
-                                                            <div class="thumbnail"
+                                                            <div class="thumbnail" style="border: 0 none; box-shadow: none; background: transparent;">
-                                                                 style="border: 0 none; box-shadow: none; background: transparent;">
                                                                 <img src="{{ main.main.portrait_url_64 }}" class="img-circle">
                                                                 <div class="caption text-center">
                                                                     {{ main.main }}
@@ -88,8 +87,7 @@
                                                                         <td class="text-center" style="width:30%">{{ alt.corporation_name }}</td>
                                                                         <td class="text-center" style="width:30%">{{ alt.alliance_name }}</td>
                                                                         <td class="text-center" style="width:5%">
-                                                                            <a href="https://zkillboard.com/character/{{ alt.character_id }}/"
+                                                                            <a href="https://zkillboard.com/character/{{ alt.character_id }}/" class="label label-danger" target="_blank">
-                                                                               class="label label-danger" target="_blank">
                                                                                 {% trans "Killboard" %}
                                                                             </a>
                                                                         </td>
@@ -123,10 +121,9 @@
                                                     <tr>
                                                         <td><img src="{{ member.portrait_url }}" class="img-circle"></td>
                                                         <td class="text-center">{{ member }}</td>
-                                                        <td class="text-center"><a
+                                                        <td class="text-center">
-                                                                href="https://zkillboard.com/character/{{ member.character_id }}/"
+                                                            <a href="https://zkillboard.com/character/{{ member.character_id }}/" class="label label-danger" target="_blank">{% trans "Killboard" %}</a>
-                                                                class="label label-danger"
+                                                        </td>
-                                                                target="_blank">{% trans "Killboard" %}</a></td>
                                                         <td class="text-center">{{ member.character_ownership.user.profile.main_character.character_name }}</td>
                                                         <td class="text-center">{{ member.character_ownership.user.profile.main_character.corporation_name }}</td>
                                                         <td class="text-center">{{ member.character_ownership.user.profile.main_character.alliance_name }}</td>
@@ -136,10 +133,9 @@
                                                     <tr class="danger">
                                                         <td><img src="{{ member.portrait_url }}" class="img-circle"></td>
                                                         <td class="text-center">{{ member.character_name }}</td>
-                                                        <td class="text-center"><a
+                                                        <td class="text-center">
-                                                                href="https://zkillboard.com/character/{{ member.character_id }}/"
+                                                            <a href="https://zkillboard.com/character/{{ member.character_id }}/" class="label label-danger" target="_blank">{% trans "Killboard" %}</a>
-                                                                class="label label-danger"
+                                                        </td>
-                                                                target="_blank">{% trans "Killboard" %}</a></td>
                                                         <td class="text-center"></td>
                                                         <td class="text-center"></td>
                                                         <td class="text-center"></td>
@@ -167,9 +163,7 @@
                                                         <td><img src="{{ member.portrait_url }}" class="img-circle"></td>
                                                         <td class="text-center">{{ member.character_name }}</td>
                                                         <td class="text-center">
-                                                            <a href="https://zkillboard.com/character/{{ member.character_id }}/"
+                                                            <a href="https://zkillboard.com/character/{{ member.character_id }}/" class="label label-danger" target="_blank">
-                                                               class="label label-danger"
-                                                               target="_blank">
                                                                 {% trans "Killboard" %}
                                                             </a>
                                                         </td>

allianceauth/eveonline/admin.py

@@ -96,24 +96,62 @@ class EveAllianceForm(EveEntityForm):
 
 @admin.register(EveCorporationInfo)
 class EveCorporationInfoAdmin(admin.ModelAdmin):
+    search_fields = ['corporation_name']
+    list_display = ('corporation_name', 'alliance')
+    list_select_related = ('alliance',)
+    list_filter = (('alliance', admin.RelatedOnlyFieldListFilter),)
+    ordering = ('corporation_name',)
+
+    def has_change_permission(self, request, obj=None):
+        return False
+
     def get_form(self, request, obj=None, **kwargs):
         if not obj or not obj.pk:
             return EveCorporationForm
-        return super(EveCorporationInfoAdmin, self).get_form(request, obj=obj, **kwargs)
+        return super().get_form(request, obj=obj, **kwargs)
 
 
 @admin.register(EveAllianceInfo)
 class EveAllianceInfoAdmin(admin.ModelAdmin):
+    search_fields = ['alliance_name']
+    list_display = ('alliance_name',)
+    ordering = ('alliance_name',)
+
+    def has_change_permission(self, request, obj=None):
+        return False
+
     def get_form(self, request, obj=None, **kwargs):
         if not obj or not obj.pk:
             return EveAllianceForm
-        return super(EveAllianceInfoAdmin, self).get_form(request, obj=obj, **kwargs)
+        return super().get_form(request, obj=obj, **kwargs)
 
 
 @admin.register(EveCharacter)
 class EveCharacterAdmin(admin.ModelAdmin):
-    search_fields = ['character_name', 'corporation_name', 'alliance_name', 'character_ownership__user__username']
+    search_fields = [
-    list_display = ('character_name', 'corporation_name', 'alliance_name', 'user', 'main_character')
+        'character_name',
+        'corporation_name',
+        'alliance_name',
+        'character_ownership__user__username'
+    ]
+    list_display = (
+        'character_name', 'corporation_name', 'alliance_name', 'user', 'main_character'
+    )
+    list_select_related = (
+        'character_ownership', 'character_ownership__user__profile__main_character'
+    )
+    list_filter = (
+        'corporation_name',
+        'alliance_name',
+        (
+            'character_ownership__user__profile__main_character',
+            admin.RelatedOnlyFieldListFilter
+        ),
+    )
+    ordering = ('character_name', )
+
+    def has_change_permission(self, request, obj=None):
+        return False
 
     @staticmethod
     def user(obj):

allianceauth/eveonline/autogroups/admin.py

@@ -39,4 +39,3 @@ class AutogroupsConfigAdmin(admin.ModelAdmin):
 admin.site.register(AutogroupsConfig, AutogroupsConfigAdmin)
 admin.site.register(ManagedCorpGroup)
 admin.site.register(ManagedAllianceGroup)
-

allianceauth/eveonline/autogroups/migrations/0001_initial.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Generated by Django 1.11.6 on 2017-12-23 04:30
 from __future__ import unicode_literals
 

allianceauth/eveonline/autogroups/models.py

@@ -57,25 +57,21 @@ class AutogroupsConfig(models.Model):
 
     states = models.ManyToManyField(State, related_name='autogroups')
 
-    corp_groups = models.BooleanField(default=False,
+    corp_groups = models.BooleanField(default=False, help_text="Setting this to false will delete all the created groups.")
-                                      help_text="Setting this to false will delete all the created groups.")
     corp_group_prefix = models.CharField(max_length=50, default='Corp ', blank=True)
     corp_name_source = models.CharField(max_length=20, choices=NAME_OPTIONS, default=OPT_NAME)
 
-    alliance_groups = models.BooleanField(default=False,
+    alliance_groups = models.BooleanField(default=False, help_text="Setting this to false will delete all the created groups.")
-                                          help_text="Setting this to false will delete all the created groups.")
     alliance_group_prefix = models.CharField(max_length=50, default='Alliance ', blank=True)
     alliance_name_source = models.CharField(max_length=20, choices=NAME_OPTIONS, default=OPT_NAME)
 
     corp_managed_groups = models.ManyToManyField(
         Group, through='ManagedCorpGroup', related_name='corp_managed_config',
-        help_text='A list of corporation groups created and maintained by this AutogroupConfig. '
+        help_text='A list of corporation groups created and maintained by this AutogroupConfig. You should not edit this list unless you know what you\'re doing.')
-                  'You should not edit this list unless you know what you\'re doing.')
 
     alliance_managed_groups = models.ManyToManyField(
         Group, through='ManagedAllianceGroup', related_name='alliance_managed_config',
-        help_text='A list of alliance groups created and maintained by this AutogroupConfig. '
+        help_text='A list of alliance groups created and maintained by this AutogroupConfig. You should not edit this list unless you know what you\'re doing.')
-                  'You should not edit this list unless you know what you\'re doing.')
 
     replace_spaces = models.BooleanField(default=False)
     replace_spaces_with = models.CharField(

allianceauth/eveonline/managers.py

@@ -32,9 +32,11 @@ class EveCharacterManager(models.Manager):
     def update_character(self, character_id):
         return self.get(character_id=character_id).update_character()
 
-    def get_character_by_id(self, char_id):
+    def get_character_by_id(self, character_id: int):
-        if self.filter(character_id=char_id).exists():
+        """Return character by character ID or None if not found."""
-            return self.get(character_id=char_id)
+        try:
+            return self.get(character_id=character_id)
+        except self.model.DoesNotExist:
             return None
 
 

allianceauth/eveonline/migrations/0001_initial.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Generated by Django 1.10.1 on 2016-09-05 21:39
 from __future__ import unicode_literals
 

allianceauth/eveonline/migrations/0002_remove_eveapikeypair_error_count.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Generated by Django 1.10.1 on 2016-09-10 20:20
 from __future__ import unicode_literals
 

allianceauth/eveonline/migrations/0003_auto_20161026_0149.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Generated by Django 1.10.2 on 2016-10-26 01:49
 from __future__ import unicode_literals
 

allianceauth/eveonline/migrations/0004_eveapikeypair_sso_verified.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Generated by Django 1.10.2 on 2016-11-01 04:20
 from __future__ import unicode_literals
 

allianceauth/eveonline/migrations/0005_remove_eveallianceinfo_member_count.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Generated by Django 1.10.1 on 2016-12-16 23:22
 from __future__ import unicode_literals
 

allianceauth/eveonline/migrations/0006_allow_null_evecharacter_alliance.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Generated by Django 1.10.1 on 2017-01-02 19:23
 from __future__ import unicode_literals
 

allianceauth/eveonline/migrations/0007_unique_id_name.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Generated by Django 1.10.5 on 2017-01-18 13:20
 from __future__ import unicode_literals
 

allianceauth/eveonline/migrations/0008_remove_apikeys.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Generated by Django 1.10.5 on 2017-03-22 23:09
 from __future__ import unicode_literals
 

allianceauth/eveonline/migrations/0009_on_delete.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Generated by Django 1.11.5 on 2017-09-28 02:16
 from __future__ import unicode_literals
 

allianceauth/eveonline/migrations/0013_evecorporationinfo_ceo_id.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.1.1 on 2021-01-05 14:11
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('eveonline', '0012_index_additions'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='evecorporationinfo',
+            name='ceo_id',
+            field=models.PositiveIntegerField(blank=True, default=None, null=True),
+        ),
+    ]

allianceauth/eveonline/migrations/0014_auto_20210105_1413.py

@@ -0,0 +1,17 @@
+# Generated by Django 3.1.1 on 2021-01-05 14:13
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('eveonline', '0013_evecorporationinfo_ceo_id'),
+    ]
+
+    operations = [
+        migrations.AddIndex(
+            model_name='evecorporationinfo',
+            index=models.Index(fields=['ceo_id'], name='eveonline_e_ceo_id_eea7b8_idx'),
+        ),
+    ]

allianceauth/eveonline/models.py

@@ -82,6 +82,7 @@ class EveCorporationInfo(models.Model):
     corporation_name = models.CharField(max_length=254, unique=True)
     corporation_ticker = models.CharField(max_length=254)
     member_count = models.IntegerField()
+    ceo_id = models.PositiveIntegerField(blank=True, null=True, default=None)
     alliance = models.ForeignKey(
         EveAllianceInfo, blank=True, null=True, on_delete=models.SET_NULL
     )
@@ -89,10 +90,14 @@ class EveCorporationInfo(models.Model):
     objects = EveCorporationManager()
     provider = EveCorporationProviderManager()
 
+    class Meta:
+        indexes = [models.Index(fields=['ceo_id',]),]
+
     def update_corporation(self, corp: providers.Corporation = None):
         if corp is None:
             corp = self.provider.get_corporation(self.corporation_id)
         self.member_count = corp.members
+        self.ceo_id = corp.ceo_id
         try:
             self.alliance = EveAllianceInfo.objects.get(alliance_id=corp.alliance_id)
         except EveAllianceInfo.DoesNotExist:

allianceauth/eveonline/providers.py

@@ -7,6 +7,8 @@ from jsonschema.exceptions import RefResolutionError
 from django.conf import settings
 from esi.clients import esi_client_factory
 
+from allianceauth import __version__
+
 
 SWAGGER_SPEC_PATH = os.path.join(os.path.dirname(
     os.path.abspath(__file__)), 'swagger.json'
@@ -166,7 +168,7 @@ class EveSwaggerProvider(EveProvider):
         else:
             try:
                 self._client = esi_client_factory(
-                    token=token, spec_file=SWAGGER_SPEC_PATH
+                    token=token, spec_file=SWAGGER_SPEC_PATH, app_info_text=("allianceauth v" + __version__)
                 )
             except (HTTPError, RefResolutionError):
                 logger.exception(
@@ -182,7 +184,7 @@ class EveSwaggerProvider(EveProvider):
     def client(self):
         if self._client is None:
             self._client = esi_client_factory(
-                token=self._token, spec_file=SWAGGER_SPEC_PATH
+                token=self._token, spec_file=SWAGGER_SPEC_PATH, app_info_text=("allianceauth v" + __version__)
             )
         return self._client
 

allianceauth/eveonline/tasks.py

@@ -5,35 +5,96 @@ from .models import EveAllianceInfo
 from .models import EveCharacter
 from .models import EveCorporationInfo
 
+from . import providers
+
 logger = logging.getLogger(__name__)
 
 TASK_PRIORITY = 7
+CHUNK_SIZE = 500
+
+
+def chunks(lst, n):
+    """Yield successive n-sized chunks from lst."""
+    for i in range(0, len(lst), n):
+        yield lst[i:i + n]
+
 
 @shared_task
 def update_corp(corp_id):
+    """Update given corporation from ESI"""
     EveCorporationInfo.objects.update_corporation(corp_id)
 
 
 @shared_task
 def update_alliance(alliance_id):
+    """Update given alliance from ESI"""
     EveAllianceInfo.objects.update_alliance(alliance_id).populate_alliance()
 
 
 @shared_task
 def update_character(character_id):
+    """Update given character from ESI"""
     EveCharacter.objects.update_character(character_id)
 
 
 @shared_task
 def run_model_update():
+    """Update all alliances, corporations and characters from ESI"""
+
     # update existing corp models
     for corp in EveCorporationInfo.objects.all().values('corporation_id'):
-        update_corp.apply_async(args=[corp['corporation_id']], priority=TASK_PRIORITY)
+        update_corp.apply_async(
+            args=[corp['corporation_id']], priority=TASK_PRIORITY
+        )
 
     # update existing alliance models
     for alliance in EveAllianceInfo.objects.all().values('alliance_id'):
-        update_alliance.apply_async(args=[alliance['alliance_id']], priority=TASK_PRIORITY)
+        update_alliance.apply_async(
+            args=[alliance['alliance_id']], priority=TASK_PRIORITY
+        )
 
     # update existing character models
-    for character in EveCharacter.objects.all().values('character_id'):
+    character_ids = EveCharacter.objects.all().values_list('character_id', flat=True)
-        update_character.apply_async(args=[character['character_id']], priority=TASK_PRIORITY)
+    for character_ids_chunk in chunks(character_ids, CHUNK_SIZE):
+        affiliations_raw = providers.provider.client.Character\
+            .post_characters_affiliation(characters=character_ids_chunk).result()
+        character_names = providers.provider.client.Universe\
+            .post_universe_names(ids=character_ids_chunk).result()
+
+        affiliations = {
+            affiliation.get('character_id'): affiliation
+            for affiliation in affiliations_raw
+        }
+        # add character names to affiliations
+        for character in character_names:
+            character_id = character.get('id')
+            if character_id in affiliations:
+                affiliations[character_id]['name'] = character.get('name')
+
+        # fetch current characters
+        characters = EveCharacter.objects.filter(character_id__in=character_ids_chunk)\
+            .values('character_id', 'corporation_id', 'alliance_id', 'character_name')
+
+        for character in characters:
+            character_id = character.get('character_id')
+            if character_id in affiliations:
+                affiliation = affiliations[character_id]
+
+                corp_changed = (
+                    character.get('corporation_id') != affiliation.get('corporation_id')
+                )
+
+                alliance_id = character.get('alliance_id')
+                if not alliance_id:
+                    alliance_id = None
+                alliance_changed = alliance_id != affiliation.get('alliance_id')
+
+                name_changed = False
+                fetched_name = affiliation.get('name', False)
+                if fetched_name:
+                    name_changed = character.get('character_name') != fetched_name
+
+                if corp_changed or alliance_changed or name_changed:
+                    update_character.apply_async(
+                        args=[character.get('character_id')], priority=TASK_PRIORITY
+                    )

allianceauth/eveonline/tests/test_providers.py

@@ -592,3 +592,12 @@ class TestEveSwaggerProvider(TestCase):
         self.assertTrue(mock_esi_client_factory.called)
         self.assertIsNotNone(my_provider._client)
         self.assertEqual(my_client, 'my_client')
+
+    @patch(MODULE_PATH + '.__version__', '1.0.0')
+    def test_user_agent_header(self):
+        my_provider = EveSwaggerProvider()
+        my_client = my_provider.client
+        operation = my_client.Status.get_status()
+        self.assertEqual(
+            operation.future.request.headers['User-Agent'], 'allianceauth v1.0.0'
+        )

allianceauth/eveonline/tests/test_tasks.py

@@ -1,4 +1,4 @@
-from unittest.mock import patch
+from unittest.mock import patch, Mock
 
 from django.test import TestCase
 
@@ -44,15 +44,17 @@ class TestTasks(TestCase):
             mock_EveCharacter.objects.update_character.call_args[0][0], 42
         )
 
+
 @patch('allianceauth.eveonline.tasks.update_character')
 @patch('allianceauth.eveonline.tasks.update_alliance')
 @patch('allianceauth.eveonline.tasks.update_corp')
-    def test_run_model_update(
+@patch('allianceauth.eveonline.providers.provider')
-        self,
+@patch('allianceauth.eveonline.tasks.CHUNK_SIZE', 2)
-        mock_update_corp,
+class TestRunModelUpdate(TestCase):
-        mock_update_alliance,
+
-        mock_update_character,
+    @classmethod
-    ):
+    def setUpClass(cls):
+        super().setUpClass()
         EveCorporationInfo.objects.all().delete()
         EveAllianceInfo.objects.all().delete()
         EveCharacter.objects.all().delete()
@@ -60,39 +62,184 @@ class TestTasks(TestCase):
         EveCorporationInfo.objects.create(
             corporation_id=2345,
             corporation_name='corp.name',
-            corporation_ticker='corp.ticker',
+            corporation_ticker='c.c.t',
             member_count=10,
             alliance=None,
         )
         EveAllianceInfo.objects.create(
             alliance_id=3456,
             alliance_name='alliance.name',
-            alliance_ticker='alliance.ticker',
+            alliance_ticker='a.t',
-            executor_corp_id='78910',
+            executor_corp_id=5,
         )
         EveCharacter.objects.create(
-            character_id=1234,
+            character_id=1,
-            character_name='character.name',
+            character_name='character.name1',
             corporation_id=2345,
             corporation_name='character.corp.name',
             corporation_ticker='c.c.t',  # max 5 chars
+            alliance_id=None
+        )
+        EveCharacter.objects.create(
+            character_id=2,
+            character_name='character.name2',
+            corporation_id=9876,
+            corporation_name='character.corp.name',
+            corporation_ticker='c.c.t',  # max 5 chars
             alliance_id=3456,
             alliance_name='character.alliance.name',
         )
+        EveCharacter.objects.create(
+            character_id=3,
+            character_name='character.name3',
+            corporation_id=9876,
+            corporation_name='character.corp.name',
+            corporation_ticker='c.c.t',  # max 5 chars
+            alliance_id=3456,
+            alliance_name='character.alliance.name',
+        )
+        EveCharacter.objects.create(
+            character_id=4,
+            character_name='character.name4',
+            corporation_id=9876,
+            corporation_name='character.corp.name',
+            corporation_ticker='c.c.t',  # max 5 chars
+            alliance_id=3456,
+            alliance_name='character.alliance.name',
+        )
+        """
+        EveCharacter.objects.create(
+            character_id=5,
+            character_name='character.name5',
+            corporation_id=9876,
+            corporation_name='character.corp.name',
+            corporation_ticker='c.c.t',  # max 5 chars
+            alliance_id=3456,
+            alliance_name='character.alliance.name',
+        )
+        """
+
+    def setUp(self):
+        self.affiliations = [
+            {'character_id': 1, 'corporation_id': 5},
+            {'character_id': 2, 'corporation_id': 9876, 'alliance_id': 3456},
+            {'character_id': 3, 'corporation_id': 9876, 'alliance_id': 7456},
+            {'character_id': 4, 'corporation_id': 9876, 'alliance_id': 3456}
+        ]
+        self.names = [
+            {'id': 1, 'name': 'character.name1'},
+            {'id': 2, 'name': 'character.name2'},
+            {'id': 3, 'name': 'character.name3'},
+            {'id': 4, 'name': 'character.name4_new'}
+        ]
+
+    def test_normal_run(
+        self,
+        mock_provider,
+        mock_update_corp,
+        mock_update_alliance,
+        mock_update_character,
+    ):
+        def get_affiliations(characters: list):
+            response = [x for x in self.affiliations if x['character_id'] in characters]
+            mock_operator = Mock(**{'result.return_value': response})
+            return mock_operator
+
+        def get_names(ids: list):
+            response = [x for x in self.names if x['id'] in ids]
+            mock_operator = Mock(**{'result.return_value': response})
+            return mock_operator
+
+        mock_provider.client.Character.post_characters_affiliation.side_effect \
+            = get_affiliations
+
+        mock_provider.client.Universe.post_universe_names.side_effect = get_names
 
         run_model_update()
 
+        self.assertEqual(
+            mock_provider.client.Character.post_characters_affiliation.call_count, 2
+        )
+        self.assertEqual(
+            mock_provider.client.Universe.post_universe_names.call_count, 2
+        )
+
+        # character 1 has changed corp
+        # character 2 no change
+        # character 3 has changed alliance
+        # character 4 has changed name
         self.assertEqual(mock_update_corp.apply_async.call_count, 1)
         self.assertEqual(
             int(mock_update_corp.apply_async.call_args[1]['args'][0]), 2345
         )
-        
         self.assertEqual(mock_update_alliance.apply_async.call_count, 1)
         self.assertEqual(
             int(mock_update_alliance.apply_async.call_args[1]['args'][0]), 3456
         )
+        characters_updated = {
+            x[1]['args'][0] for x in mock_update_character.apply_async.call_args_list
+        }
+        excepted = {1, 3, 4}
+        self.assertSetEqual(characters_updated, excepted)
 
-        self.assertEqual(mock_update_character.apply_async.call_count, 1)
+    def test_ignore_character_not_in_affiliations(
-        self.assertEqual(
+        self,
-            int(mock_update_character.apply_async.call_args[1]['args'][0]), 1234
+        mock_provider,
-        )
+        mock_update_corp,
+        mock_update_alliance,
+        mock_update_character,
+    ):
+        def get_affiliations(characters: list):
+            response = [x for x in self.affiliations if x['character_id'] in characters]
+            mock_operator = Mock(**{'result.return_value': response})
+            return mock_operator
+
+        def get_names(ids: list):
+            response = [x for x in self.names if x['id'] in ids]
+            mock_operator = Mock(**{'result.return_value': response})
+            return mock_operator
+
+        del self.affiliations[0]
+
+        mock_provider.client.Character.post_characters_affiliation.side_effect \
+            = get_affiliations
+
+        mock_provider.client.Universe.post_universe_names.side_effect = get_names
+
+        run_model_update()
+        characters_updated = {
+            x[1]['args'][0] for x in mock_update_character.apply_async.call_args_list
+        }
+        excepted = {3, 4}
+        self.assertSetEqual(characters_updated, excepted)
+
+    def test_ignore_character_not_in_names(
+        self,
+        mock_provider,
+        mock_update_corp,
+        mock_update_alliance,
+        mock_update_character,
+    ):
+        def get_affiliations(characters: list):
+            response = [x for x in self.affiliations if x['character_id'] in characters]
+            mock_operator = Mock(**{'result.return_value': response})
+            return mock_operator
+
+        def get_names(ids: list):
+            response = [x for x in self.names if x['id'] in ids]
+            mock_operator = Mock(**{'result.return_value': response})
+            return mock_operator
+
+        del self.names[3]
+
+        mock_provider.client.Character.post_characters_affiliation.side_effect \
+            = get_affiliations
+
+        mock_provider.client.Universe.post_universe_names.side_effect = get_names
+
+        run_model_update()
+        characters_updated = {
+            x[1]['args'][0] for x in mock_update_character.apply_async.call_args_list
+        }
+        excepted = {1, 3}
+        self.assertSetEqual(characters_updated, excepted)

allianceauth/eveonline/views.py

@@ -1,2 +0,0 @@
-
-

allianceauth/fleetactivitytracking/forms.py

@@ -5,6 +5,4 @@ from django.utils.translation import ugettext_lazy as _
 
 class FatlinkForm(forms.Form):
     fleet = forms.CharField(label=_("Fleet Name"), max_length=50)
-    duration = forms.IntegerField(label=_("Duration of fat-link"), required=True, initial=30, min_value=1,
+    duration = forms.IntegerField(label=_("Duration of fat-link"), required=True, initial=30, min_value=1, max_value=2147483647, help_text=_('minutes'))
-                                  max_value=2147483647, help_text=_('minutes'))
-

allianceauth/fleetactivitytracking/migrations/0001_initial.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Generated by Django 1.10.1 on 2016-09-05 21:39
 from __future__ import unicode_literals
 

allianceauth/fleetactivitytracking/migrations/0002_auto_20160905_2220.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Generated by Django 1.10.1 on 2016-09-05 22:20
 from __future__ import unicode_literals
 

allianceauth/fleetactivitytracking/migrations/0003_auto_20160906_2354.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Generated by Django 1.10.1 on 2016-09-06 23:54
 from __future__ import unicode_literals
 

allianceauth/fleetactivitytracking/migrations/0004_make_strings_more_stringy.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Generated by Django 1.10.5 on 2017-03-22 23:35
 from __future__ import unicode_literals
 

allianceauth/fleetactivitytracking/templates/fleetactivitytracking/fatlinkformatter.html

@@ -1,6 +1,6 @@
 {% extends "allianceauth/base.html" %}
 {% load bootstrap %}
-{% load staticfiles %}
+{% load static %}
 {% load i18n %}
 
 {% block page_title %}{% trans "Create Fatlink" %}{% endblock page_title %}
@@ -30,4 +30,3 @@
     </div>
 
 {% endblock content %}
-

allianceauth/fleetactivitytracking/templates/fleetactivitytracking/fatlinkmodify.html

@@ -1,6 +1,6 @@
 {% extends "allianceauth/base.html" %}
 {% load bootstrap %}
-{% load staticfiles %}
+{% load static %}
 {% load i18n %}
 {% block page_title %}{% trans "Fatlink view" %}{% endblock page_title %}
 

allianceauth/fleetactivitytracking/templates/fleetactivitytracking/fatlinkpersonalmonthlystatisticsview.html

@@ -1,6 +1,6 @@
 {% extends "allianceauth/base.html" %}
 {% load bootstrap %}
-{% load staticfiles %}
+{% load static %}
 {% load i18n %}
 
 {% block page_title %}{% trans "Personal fatlink statistics" %}{% endblock page_title %}

allianceauth/fleetactivitytracking/templates/fleetactivitytracking/fatlinkpersonalstatisticsview.html

@@ -1,6 +1,6 @@
 {% extends "allianceauth/base.html" %}
 {% load bootstrap %}
-{% load staticfiles %}
+{% load static %}
 {% load i18n %}
 
 {% block page_title %}{% trans "Personal fatlink statistics" %}{% endblock page_title %}

allianceauth/fleetactivitytracking/templates/fleetactivitytracking/fatlinkstatisticscorpview.html

@@ -1,6 +1,6 @@
 {% extends "allianceauth/base.html" %}
 {% load bootstrap %}
-{% load staticfiles %}
+{% load static %}
 {% load i18n %}
 
 {% block page_title %}{% trans "Fatlink Corp Statistics" %}{% endblock page_title %}
@@ -44,4 +44,5 @@
 {% block extra_script %}
 $(document).ready(function () {
     $("[rel=tooltip]").tooltip();
+});
 {% endblock extra_script %}

allianceauth/fleetactivitytracking/templates/fleetactivitytracking/fatlinkstatisticsview.html

@@ -1,6 +1,6 @@
 {% extends "allianceauth/base.html" %}
 {% load bootstrap %}
-{% load staticfiles %}
+{% load static %}
 {% load i18n %}
 
 {% block page_title %}{% trans "Fatlink statistics" %}{% endblock page_title %}
@@ -46,4 +46,5 @@
 {% block extra_script %}
 $(document).ready(function () {
     $("[rel=tooltip]").tooltip();
+});
 {% endblock extra_script %}

allianceauth/fleetactivitytracking/templates/fleetactivitytracking/fatlinkview.html

@@ -1,6 +1,6 @@
 {% extends "allianceauth/base.html" %}
 {% load bootstrap %}
-{% load staticfiles %}
+{% load static %}
 {% load i18n %}
 
 {% block page_title %}{% trans "Fatlink view" %}{% endblock page_title %}

allianceauth/fleetactivitytracking/views.py

@@ -135,8 +135,7 @@ def fatlink_statistics_corp_view(request, corpid, year=None, month=None):
     stat_list.sort(key=lambda stat: stat.mainchar.character_name)
     stat_list.sort(key=lambda stat: (stat.n_fats, stat.avg_fat), reverse=True)
 
-    context = {'fatStats': stat_list, 'month': start_of_month.strftime("%B"), 'year': year,
+    context = {'fatStats': stat_list, 'month': start_of_month.strftime("%B"), 'year': year, 'previous_month': start_of_previous_month, 'corpid': corpid}
-           'previous_month': start_of_previous_month, 'corpid': corpid}
     if datetime.datetime.now() > start_of_next_month:
         context.update({'next_month': start_of_next_month})
 
@@ -163,16 +162,14 @@ def fatlink_statistics_view(request, year=datetime.date.today().year, month=date
 
     for fat in fats_in_span.exclude(character__corporation_id__in=fat_stats):
         if EveCorporationInfo.objects.filter(corporation_id=fat.character.corporation_id).exists():
-            fat_stats[fat.character.corporation_id] = CorpStat(fat.character.corporation_id, start_of_month,
+            fat_stats[fat.character.corporation_id] = CorpStat(fat.character.corporation_id, start_of_month, start_of_next_month)
-                                                               start_of_next_month)
 
     # collect and sort stats
     stat_list = [fat_stats[x] for x in fat_stats]
     stat_list.sort(key=lambda stat: stat.corp.corporation_name)
     stat_list.sort(key=lambda stat: (stat.n_fats, stat.avg_fat), reverse=True)
 
-    context = {'fatStats': stat_list, 'month': start_of_month.strftime("%B"), 'year': year,
+    context = {'fatStats': stat_list, 'month': start_of_month.strftime("%B"), 'year': year, 'previous_month': start_of_previous_month}
-           'previous_month': start_of_previous_month}
     if datetime.datetime.now() > start_of_next_month:
         context.update({'next_month': start_of_next_month})
 
@@ -199,8 +196,7 @@ def fatlink_personal_statistics_view(request, year=datetime.date.today().year):
     monthlystats = [(i + 1, datetime.date(year, i + 1, 1).strftime("%h"), monthlystats[i]) for i in range(12)]
 
     if datetime.datetime.now() > datetime.datetime(year + 1, 1, 1):
-        context = {'user': user, 'monthlystats': monthlystats, 'year': year, 'previous_year': year - 1,
+        context = {'user': user, 'monthlystats': monthlystats, 'year': year, 'previous_year': year - 1, 'next_year': year + 1}
-                   'next_year': year + 1}
     else:
         context = {'user': user, 'monthlystats': monthlystats, 'year': year, 'previous_year': year - 1}
 
@@ -229,9 +225,11 @@ def fatlink_monthly_personal_statistics_view(request, year, month, char_id=None)
     for fat in personal_fats:
         ship_statistics[fat.shiptype] = ship_statistics.setdefault(fat.shiptype, 0) + 1
         n_fats += 1
-    context = {'user': user, 'shipStats': sorted(ship_statistics.items()), 'month': start_of_month.strftime("%h"),
+    context = {
+        'user': user, 'shipStats': sorted(ship_statistics.items()), 'month': start_of_month.strftime("%h"),
         'year': year, 'n_fats': n_fats, 'char_id': char_id, 'previous_month': start_of_previous_month,
-               'next_month': start_of_next_month}
+        'next_month': start_of_next_month
+    }
 
     created_fats = Fatlink.objects.filter(creator=user).filter(fatdatetime__gte=start_of_month).filter(
         fatdatetime__lt=start_of_next_month)
@@ -257,8 +255,7 @@ def click_fatlink_view(request, token, fat_hash=None):
             location = c.Location.get_characters_character_id_location(character_id=token.character_id).result()
             ship = c.Location.get_characters_character_id_ship(character_id=token.character_id).result()
             location['solar_system_name'] = \
-                c.Universe.get_universe_systems_system_id(system_id=location['solar_system_id']).result()[
+                c.Universe.get_universe_systems_system_id(system_id=location['solar_system_id']).result()['name']
-                    'name']
             if location['station_id']:
                 location['station_name'] = \
                     c.Universe.get_universe_stations_station_id(station_id=location['station_id']).result()['name']

allianceauth/groupmanagement/admin.py

@@ -1,5 +1,5 @@
-from django.conf import settings
+from django.apps import apps
-
+from django.contrib.auth.models import Permission
 from django.contrib import admin
 from django.contrib.auth.models import Group as BaseGroup, User
 from django.db.models import Count
@@ -10,9 +10,8 @@ from django.dispatch import receiver
 
 from .models import AuthGroup
 from .models import GroupRequest
-from . import signals
 
-if 'allianceauth.eveonline.autogroups' in settings.INSTALLED_APPS:
+if 'eve_autogroups' in apps.app_configs:
     _has_auto_groups = True
 else:
     _has_auto_groups = False
@@ -41,9 +40,6 @@ class AuthGroupInlineAdmin(admin.StackedInline):
             kwargs["queryset"] = Group.objects.order_by(Lower('name'))
         return super().formfield_for_manytomany(db_field, request, **kwargs)
 
-    def has_add_permission(self, request):
-        return False
-
     def has_delete_permission(self, request, obj=None):
         return False
 
@@ -97,8 +93,9 @@ class HasLeaderFilter(admin.SimpleListFilter):
         else:
             return queryset
 
+
 class GroupAdmin(admin.ModelAdmin):
-    list_select_related = True
+    list_select_related = ('authgroup',)
     ordering = ('name',)
     list_display = (
         'name',
@@ -121,6 +118,9 @@ class GroupAdmin(admin.ModelAdmin):
 
     def get_queryset(self, request):
         qs = super().get_queryset(request)
+        if _has_auto_groups:
+            qs = qs.prefetch_related('managedalliancegroup_set', 'managedcorpgroup_set')
+        qs = qs.prefetch_related('authgroup__group_leaders').select_related('authgroup')
         qs = qs.annotate(
             member_count=Count('user', distinct=True),
         )
@@ -136,7 +136,7 @@ class GroupAdmin(admin.ModelAdmin):
     _member_count.admin_order_field = 'member_count'
 
     def has_leader(self, obj):
-        return obj.authgroup.group_leaders.exists()
+        return obj.authgroup.group_leaders.exists() or obj.authgroup.group_leader_groups.exists()
 
     has_leader.boolean = True
 
@@ -166,6 +166,18 @@ class GroupAdmin(admin.ModelAdmin):
     filter_horizontal = ('permissions',)
     inlines = (AuthGroupInlineAdmin,)
 
+    def formfield_for_manytomany(self, db_field, request, **kwargs):
+        if db_field.name == "permissions":
+            kwargs["queryset"] = Permission.objects.select_related("content_type").all()
+        return super().formfield_for_manytomany(db_field, request, **kwargs)
+
+    def save_formset(self, request, form, formset, change):
+        for inline_form in formset:
+            ag_instance = inline_form.save(commit=False)
+            ag_instance.group = form.instance
+            ag_instance.save()
+        formset.save()
+
 
 class Group(BaseGroup):
     class Meta:
@@ -173,13 +185,29 @@ class Group(BaseGroup):
         verbose_name = BaseGroup._meta.verbose_name
         verbose_name_plural = BaseGroup._meta.verbose_name_plural
 
+
 try:
     admin.site.unregister(BaseGroup)
 finally:
     admin.site.register(Group, GroupAdmin)
 
 
-admin.site.register(GroupRequest)
+@admin.register(GroupRequest)
+class GroupRequestAdmin(admin.ModelAdmin):
+    search_fields = ('user__username', )
+    list_display = ('id', 'group', 'user', '_leave_request', 'status')
+    list_filter = (
+        ('group', admin.RelatedOnlyFieldListFilter),
+        ('user', admin.RelatedOnlyFieldListFilter),
+        'leave_request',
+        'status'
+    )
+
+    def _leave_request(self, obj) -> True:
+        return obj.leave_request
+
+    _leave_request.short_description = 'is leave request'
+    _leave_request.boolean = True
 
 
 @receiver(pre_save, sender=Group)

allianceauth/groupmanagement/apps.py

@@ -5,3 +5,6 @@ class GroupManagementConfig(AppConfig):
     name = 'allianceauth.groupmanagement'
     label = 'groupmanagement'
     verbose_name = 'Group Management'
+
+    def ready(self):
+        from . import signals  # noqa: F401

allianceauth/groupmanagement/auth_hooks.py

@@ -0,0 +1,43 @@
+from django.utils.translation import ugettext_lazy as _
+
+from allianceauth.services.hooks import MenuItemHook, UrlHook
+from allianceauth import hooks
+
+from . import urls
+from .managers import GroupManager
+
+
+class GroupManagementMenuItem(MenuItemHook):
+    """ This class ensures only authorized users will see the menu entry """
+
+    def __init__(self):
+        # setup menu entry for sidebar
+        MenuItemHook.__init__(
+            self,
+            text=_("Group Management"),
+            classes="fas fa-users-cog fa-fw",
+            url_name="groupmanagement:management",
+            order=50,
+            navactive=[
+                "groupmanagement:management",  # group requests view
+                "groupmanagement:membership",  # group membership view
+                "groupmanagement:audit_log",  # group audit log view
+            ],
+        )
+
+    def render(self, request):
+        if GroupManager.can_manage_groups(request.user):
+            app_count = GroupManager.pending_requests_count_for_user(request.user)
+            self.count = app_count if app_count and app_count > 0 else None
+            return MenuItemHook.render(self, request)
+        return ""
+
+
+@hooks.register("menu_item_hook")
+def register_menu():
+    return GroupManagementMenuItem()
+
+
+@hooks.register("url_hook")
+def register_urls():
+    return UrlHook(urls, "group", r"^groups/")