Corpstats views for mains and unregistered

Remove json blob from corpstats model and replace with discrete member models

corputils/admin.py

@@ -1,5 +1,6 @@
 from __future__ import unicode_literals
-from corputils.models import CorpStats
+from corputils.models import CorpStats, CorpMember
 from django.contrib import admin
 
 admin.site.register(CorpStats)
+admin.site.register(CorpMember)

corputils/migrations/0004_member_models.py

@@ -0,0 +1,60 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.10.5 on 2017-03-26 20:13
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+import django.db.models.deletion
+import json
+
+
+def convert_json_to_members(apps, schema_editor):
+    CorpStats = apps.get_model('corputils', 'CorpStats')
+    CorpMember = apps.get_model('corputils', 'CorpMember')
+    for cs in CorpStats.objects.all():
+        members = json.loads(cs._members)
+        CorpMember.objects.bulk_create(
+            [CorpMember(corpstats=cs, character_id=member_id, character_name=member_name) for member_id, member_name in
+             members.items()]
+        )
+
+
+def convert_members_to_json(apps, schema_editor):
+    CorpStats = apps.get_model('corputils', 'CorpStats')
+    for cs in CorpStats.objects.all():
+        cs._members = json.dumps({m.character_id: m.character_name for m in cs.members.all()})
+        cs.save()
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ('corputils', '0003_make_strings_more_stringy'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='CorpMember',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('character_id', models.PositiveIntegerField()),
+                ('character_name', models.CharField(max_length=37)),
+            ],
+            options={
+                'ordering': ['character_name'],
+            },
+        ),
+        migrations.AddField(
+            model_name='corpmember',
+            name='corpstats',
+            field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='members',
+                                    to='corputils.CorpStats'),
+        ),
+        migrations.AlterUniqueTogether(
+            name='corpmember',
+            unique_together=set([('corpstats', 'character_id')]),
+        ),
+        migrations.RunPython(convert_json_to_members, convert_members_to_json),
+        migrations.RemoveField(
+            model_name='corpstats',
+            name='_members',
+        ),
+    ]

corputils/models.py

@@ -8,8 +8,6 @@ from notifications import notify
 from authentication.models import CharacterOwnership, UserProfile
 from bravado.exception import HTTPForbidden
 from corputils.managers import CorpStatsManager
-from operator import attrgetter
-import json
 import logging
 
 logger = logging.getLogger(__name__)
@@ -20,7 +18,6 @@ class CorpStats(models.Model):
     token = models.ForeignKey(Token, on_delete=models.CASCADE)
     corp = models.OneToOneField(EveCorporationInfo)
     last_update = models.DateTimeField(auto_now=True)
-    _members = models.TextField(default='{}')
 
     class Meta:
         permissions = (
@@ -52,15 +49,23 @@ class CorpStats(models.Model):
             # the swagger spec doesn't have a maxItems count
             # manual testing says we can do over 350, but let's not risk it
             member_id_chunks = [member_ids[i:i + 255] for i in range(0, len(member_ids), 255)]
-            c = self.token.get_esi_client(Character='v1') # ccplease bump versions of whole resources
+            c = self.token.get_esi_client(Character='v1')  # ccplease bump versions of whole resources
             member_name_chunks = [c.Character.get_characters_names(character_ids=id_chunk).result() for id_chunk in
                                   member_id_chunks]
             member_list = {}
             for name_chunk in member_name_chunks:
                 member_list.update({m['character_id']: m['character_name'] for m in name_chunk})
 
-            self.members = member_list
+            # bulk create new member models
-            self.save()
+            missing_members = [m_id for m_id in member_ids if
+                               not CorpMember.objects.filter(corpstats=self, character_id=m_id).exists()]
+            CorpMember.objects.bulk_create(
+                [CorpMember(character_id=m_id, character_name=member_list[m_id], corpstats=self) for m_id in
+                 missing_members])
+
+            # purge old members
+            self.members.exclude(character_id__in=member_ids).delete()
+
         except TokenError as e:
             logger.warning("%s failed to update: %s" % (self, e))
             if self.token.user:
@@ -82,87 +87,95 @@ class CorpStats(models.Model):
             self.delete()
 
     @property
-    def members(self):
-        return json.loads(self._members)
-
-    @members.setter
-    def members(self, dict):
-        self._members = json.dumps(dict)
-
-    @property
-    def member_ids(self):
-        return [id for id, name in self.members.items()]
-
-    @property
-    def member_names(self):
-        return [name for id, name in self.members.items()]
-
     def member_count(self):
-        return len(self.members)
+        return self.members.count()
 
-    @staticmethod
+    @property
-    def user_count(members):
+    def user_count(self):
-        mainchars = []
+        return len(set([m.main_character for m in self.members.all() if m.main_character]))
-        for member in members:
-            if hasattr(member.main, 'character_name'):
-                mainchars.append(member.main.character_name)
-        return len(set(mainchars))
 
-    def registered_characters(self):
+    @property
-        return len(CharacterOwnership.objects.filter(character__character_id__in=self.member_ids))
+    def registered_member_count(self):
+        return len(self.registered_members)
 
-    @python_2_unicode_compatible
+    @property
-    class MemberObject(object):
+    def registered_members(self):
-        def __init__(self, character_id, character_name):
+        return self.members.filter(pk__in=[m.pk for m in self.members.all() if m.registered])
-            self.character_id = character_id
-            self.character_name = character_name
-            try:
-                char = EveCharacter.objects.get(character_id=character_id)
-                self.main_user = char.character_ownership.user
-                self.main = self.main_user.profile.main_character
-                self.registered = True
-            except (EveCharacter.DoesNotExist, CharacterOwnership.DoesNotExist, UserProfile.DoesNotExist, AttributeError):
-                self.main = None
-                self.registered = False
-                self.main_user = ''
 
-        def __str__(self):
+    @property
-            return self.character_name
+    def unregistered_member_count(self):
+        return self.member_count - self.registered_member_count
 
-        def portrait_url(self, size=32):
+    @property
-            return "https://image.eveonline.com/Character/%s_%s.jpg" % (self.character_id, size)
+    def unregistered_members(self):
+        return self.members.filter(pk__in=[m.pk for m in self.members.all() if not m.registered])
 
-    def get_member_objects(self):
+    @property
-        member_list = [CorpStats.MemberObject(id, name) for id, name in self.members.items()]
+    def main_count(self):
-        outlist = sorted([m for m in member_list if m.main_user], key=attrgetter('main_user', 'character_name'))
+        return len(self.mains)
-        outlist = outlist + sorted([m for m in member_list if not m.main_user], key=attrgetter('character_name'))
+
-        return outlist
+    @property
+    def mains(self):
+        return self.members.filter(pk__in=[m.pk for m in self.members.all() if
+                                           m.main_character and int(m.main_character.character_id) == int(
+                                               m.character_id)])
 
     def can_update(self, user):
         return user.is_superuser or user == self.token.user
 
-    @python_2_unicode_compatible
+    def corp_logo(self, size=128):
-    class ViewModel(object):
+        return "https://image.eveonline.com/Corporation/%s_%s.png" % (self.corp.corporation_id, size)
-        def __init__(self, corpstats, user):
-            self.corp = corpstats.corp
-            self.members = corpstats.get_member_objects()
-            self.can_update = corpstats.can_update(user)
-            self.total_members = len(self.members)
-            self.total_users = corpstats.user_count(self.members)
-            self.registered_members = corpstats.registered_characters()
-            self.last_updated = corpstats.last_update
 
-        def __str__(self):
+    def alliance_logo(self, size=128):
-            return str(self.corp)
+        if self.corp.alliance:
+            return "https://image.eveonline.com/Alliance/%s_%s.png" % (self.corp.alliance.alliance_id, size)
+        else:
+            return "https://image.eveonline.com/Alliance/1_%s.png" % size
 
-        def corp_logo(self, size=128):
-            return "https://image.eveonline.com/Corporation/%s_%s.png" % (self.corp.corporation_id, size)
 
-        def alliance_logo(self, size=128):
+class CorpMember(models.Model):
-            if self.corp.alliance:
+    character_id = models.PositiveIntegerField()
-                return "https://image.eveonline.com/Alliance/%s_%s.png" % (self.corp.alliance.alliance_id, size)
+    character_name = models.CharField(max_length=37)
-            else:
+    corpstats = models.ForeignKey(CorpStats, on_delete=models.CASCADE, related_name='members')
-                return "https://image.eveonline.com/Alliance/1_%s.png" % size
 
-    def get_view_model(self, user):
+    class Meta:
-        return CorpStats.ViewModel(self, user)
+        # not making character_id unique in case a character moves between two corps while only one updates
+        unique_together = ('corpstats', 'character_id')
+        ordering = ['character_name']
+
+    def __str__(self):
+        return self.character_name
+
+    @property
+    def character(self):
+        try:
+            return EveCharacter.objects.get(character_id=self.character_id)
+        except EveCharacter.DoesNotExist:
+            return None
+
+    @property
+    def main_character(self):
+        try:
+            return self.character.character_ownership.user.profile.main_character
+        except (CharacterOwnership.DoesNotExist, UserProfile.DoesNotExist, AttributeError):
+            return None
+
+    @property
+    def alts(self):
+        if self.main_character:
+            return [co.character for co in self.main_character.character_ownership.user.character_ownerships.all()]
+        else:
+            return []
+
+    @property
+    def registered(self):
+        return CharacterOwnership.objects.filter(character__character_id=self.character_id).exists()
+
+    def portrait_url(self, size=32):
+        return "https://image.eveonline.com/Character/%s_%s.jpg" % (self.character_id, size)
+
+    def __getattr__(self, item):
+        if item.startswith('portrait_url_'):
+            size = item.strip('portrait_url_')
+            return self.portrait_url(size)
+        return super(CorpMember, self).__getattr__(item)

corputils/templates/corputils/corpstats.html

@@ -4,55 +4,108 @@
 {% load bootstrap_pagination %}
 {% load eveonline_extras %}
 {% block member_data %}
-            {% if corpstats %}
+    {% if corpstats %}
-                <div class="row">
+        <div class="row">
-                    <div class="col-lg-12 text-center">
+            <div class="col-lg-12 text-center">
-                        <table class="table">
+                <table class="table">
-                            <tr>
+                    <tr>
-                                <td class="text-center col-lg-6 {% if corpstats.corp.alliance %}{% else %}col-lg-offset-3{% endif %}"><img class="ra-avatar" src="{{ corpstats.corp_logo }}"></td>
+                        <td class="text-center col-lg-6
-                                {% if corpstats.corp.alliance %}
+                                {% if corpstats.corp.alliance %}{% else %}col-lg-offset-3{% endif %}"><img
-                                    <td class="text-center col-lg-6"><img class="ra-avatar" src="{{ corpstats.alliance_logo }}"></td>
+                                class="ra-avatar" src="{{ corpstats.corp_logo }}"></td>
-                                {% endif %}
+                        {% if corpstats.corp.alliance %}
-                            </tr>
+                            <td class="text-center col-lg-6"><img class="ra-avatar" src="{{ corpstats.alliance_logo }}">
-                            <tr>
+                            </td>
-                                <td class="text-center"><h4>{{ corpstats.corp.corporation_name }}</h4></td>
+                        {% endif %}
-                                {% if corpstats.corp.alliance %}
+                    </tr>
-                                    <td class="text-center"><h4>{{ corpstats.corp.alliance.alliance_name }}</h4></td>
+                    <tr>
-                                {% endif %}
+                        <td class="text-center"><h4>{{ corpstats.corp.corporation_name }}</h4></td>
-                            </tr>
+                        {% if corpstats.corp.alliance %}
-                        </table>
+                            <td class="text-center"><h4>{{ corpstats.corp.alliance.alliance_name }}</h4></td>
+                        {% endif %}
+                    </tr>
+                </table>
+            </div>
+        </div>
+        <div class="row">
+            <div class="col-lg-12">
+                <b>{% trans "Registration Index" %}</b>
+                <div class="progress">
+                    <div class="progress-bar" role="progressbar" aria-valuenow="{{ corpstats.registered_member_count }}"
+                         aria-valuemin="0" aria-valuemax="{{ corpstats.member_count }}"
+                         style="width: {% widthratio corpstats.registered_member_count corpstats.member_count 100 %}%;">
+                        {{ corpstats.registered_member_count }}/{{ corpstats.member_count }}
                     </div>
                 </div>
-                <div class="row">
+            </div>
-                    <div class="col-lg-12">
+        </div>
-                        <b>{% trans "Registration Index: " %}</b> {{ corpstats.total_users }} Main Character{{ corpstats.total_users|pluralize }}
+        <div class="row">
-                        <div class="progress">
+            <div class="col-lg-12">
-                            <div class="progress-bar" role="progressbar" aria-valuenow="{{ corpstats.registered_members }}" aria-valuemin="0" aria-valuemax="{{ corpstats.total_members }}" style="width: {% widthratio corpstats.registered_members corpstats.total_members 100 %}%;">
+                <div class="panel panel-default">
-                                {{ corpstats.registered_members }}/{{ corpstats.total_members }}
+                    <div class="panel-heading">
-                            </div>
+                        <ul class="nav nav-pills pull-left">
+                            <li class="active"><a href="#mains" data-toggle="pill">{% trans 'Mains' %} ({{ corpstats.main_count }})</a></li>
+                            <li><a href="#members" data-toggle="pill">{% trans 'Members' %} ({{ corpstats.member_count }})</a></li>
+                            <li><a href="#unregistered" data-toggle="pill">{% trans 'Unregistered' %} ({{ corpstats.unregistered_member_count }})</a></li>
+                        </ul>
+                        <div class="pull-right">
+                            {% trans "Last update:" %} {{ corpstats.last_update|naturaltime }}
+                                <a class="btn btn-success" type="button" href="{% url 'corputils:update' corpstats.corp.corporation_id %}" title="Update Now">
+                                    <span class="glyphicon glyphicon-refresh"></span>
+                                </a>
                         </div>
+                        <div class="clearfix"></div>
                     </div>
-                </div>
+                    <div class="panel-body">
-                <div class="row">
+                        <div class="tab-content">
-                    <div class="col-lg-12">
+                            <div class="tab-pane fade in active" id="mains">
-                        <div class="panel panel-default">
+                                <div class="text-center">{% bootstrap_paginate mains range=10 %}</div>
-                            <div class="panel-heading clearfix">
+                                <div class="table-responsive">
-                                <div class="panel-title pull-left">
+                                    <table class="table table-hover">
-                                    <h4>{% trans "Members" %}</h4>
+                                        <tr>
-                                </div>
+                                            <th class="text-center">{% trans "Main" %}</th>
-                                <div class="panel-title pull-right">
+                                            <th class="text-center">{% trans "Characters" %}</th>
-                                    {% trans "Last update:" %} {{ corpstats.last_updated|naturaltime }}
+                                        </tr>
-                                    {% if corpstats.can_update %}
+                                        {% for main in mains %}
-                                        <a class="btn btn-success" type="button" href="{% url 'corputils:update' corpstats.corp.corporation_id %}" title="Update Now">
+                                            <tr>
-                                            <span class="glyphicon glyphicon-refresh"></span>
+                                                <td class="text-center" style="vertical-align:middle">
-                                        </a>
+                                                    <div class="thumbnail"
-                                    {% endif %}
+                                                         style="border: 0 none; box-shadow: none; background: transparent;">
+                                                        <img src="{{ main.portrait_url_64 }}" class="img-circle">
+                                                        <div class="caption text-center">
+                                                            {{ main }}
+                                                        </div>
+                                                    </div>
+                                                </td>
+                                                <td>
+                                                    <table class="table table-hover">
+                                                        {% for alt in main.alts %}
+                                                            {% if forloop.first %}
+                                                                <tr>
+                                                                    <th class="text-center">{% trans "Character" %}</th>
+                                                                    <th class="text-center">{% trans "zKillboard" %}</th>
+                                                                    <th class="text-center">{% trans "Corporation" %}</th>
+                                                                    <th class="text-center">{% trans "Alliance" %}</th>
+                                                                </tr>
+                                                            {% endif %}
+                                                            <tr>
+                                                                <td class="text-center">{{ alt.character_name }}</td>
+                                                                <td class="text-center"><a
+                                                                        href="https://zkillboard.com/character/{{ alt.character_id }}/"
+                                                                        class="label label-danger"
+                                                                        target="_blank">{% trans "Killboard" %}</a></td>
+                                                                <td class="text-center">{{ alt.corporation_name }}</td>
+                                                                <td class="text-center">{{ alt.alliance_name }}</td>
+                                                            </tr>
+                                                        {% endfor %}
+                                                    </table>
+                                                </td>
+                                            </tr>
+                                        {% endfor %}
+                                    </table>
                                 </div>
                             </div>
-                            <div class="panel-body">
+                            <div class="tab-pane fade" id="members">
-                                <div class="text-center">
+                                <div class="text-center">{% bootstrap_paginate members range=10 %}</div>
-                                    {% bootstrap_paginate members range=10 %}
-                                </div>
                                 <div class="table-responsive">
                                     <table class="table table-hover">
                                         <tr>
@@ -67,10 +120,38 @@
                                             <tr {% if not member.registered %}class="danger"{% endif %}>
                                                 <td><img src="{{ member.portrait_url }}" class="img-circle"></td>
                                                 <td class="text-center">{{ member.character_name }}</td>
-                                                <td class="text-center"><a href="https://zkillboard.com/character/{{ member.character_id }}/" class="label label-danger" target="_blank">{% trans "Killboard" %}</a></td>
+                                                <td class="text-center"><a
-                                                <td class="text-center">{{ member.main.character_name }}</td>
+                                                        href="https://zkillboard.com/character/{{ member.character_id }}/"
-                                                <td class="text-center">{{ member.main.corporation_name }}</td>
+                                                        class="label label-danger"
-                                                <td class="text-center">{{ member.main.alliance_name }}</td>
+                                                        target="_blank">{% trans "Killboard" %}</a></td>
+                                                <td class="text-center">{{ member.main_character.character_name }}</td>
+                                                <td class="text-center">{{ member.main_character.corporation_name }}</td>
+                                                <td class="text-center">{{ member.main_character.alliance_name }}</td>
+                                            </tr>
+                                        {% endfor %}
+                                    </table>
+                                </div>
+                            </div>
+                            <div class="tab-pane fade" id="unregistered">
+                                <div class="text-center">{% bootstrap_paginate unregistered range=10 %}</div>
+                                <div class="table-responsive">
+                                    <table class="table table-hover">
+                                        <tr>
+                                            <th></th>
+                                            <th class="text-center">{% trans "Character" %}</th>
+                                            <th class="text-center">{% trans "zKillboard" %}</th>
+                                        </tr>
+                                        {% for member in unregistered %}
+                                            <tr class="danger">
+                                                <td><img src="{{ member.portrait_url }}" class="img-circle"></td>
+                                                <td class="text-center">{{ member.character_name }}</td>
+                                                <td class="text-center">
+                                                    <a href="https://zkillboard.com/character/{{ member.character_id }}/"
+                                                       class="label label-danger"
+                                                       target="_blank">
+                                                        {% trans "Killboard" %}
+                                                    </a>
+                                                </td>
                                             </tr>
                                         {% endfor %}
                                     </table>
@@ -79,5 +160,7 @@
                         </div>
                     </div>
                 </div>
-            {% endif %}
+            </div>
+        </div>
+    {% endif %}
 {% endblock %}

corputils/templates/corputils/search.html

@@ -27,9 +27,9 @@
                         <td class="text-center">{{ result.1.character_name }}</td>
                         <td class="text-center">{{ result.0.corp.corporation_name }}</td>
                         <td class="text-center"><a href="https://zkillboard.com/character/{{ result.1.character_id }}/" class="label label-danger" target="_blank">{% trans "Killboard" %}</a></td>
-                        <td class="text-center">{{ result.1.main.character_name }}</td>
+                        <td class="text-center">{{ result.1.main_character.character_name }}</td>
-                        <td class="text-center">{{ result.1.main.corporation_name }}</td>
+                        <td class="text-center">{{ result.1.main_character.corporation_name }}</td>
-                        <td class="text-center">{{ result.1.main.alliance_name }}</td>
+                        <td class="text-center">{{ result.1.main_character.alliance_name }}</td>
                     </tr>
                 {% endfor %}
             </table>

corputils/views.py

@@ -94,14 +94,20 @@ def corpstats_view(request, corp_id=None):
 
     # paginate
     members = []
+    mains = []
+    unregistered = []
     if corpstats:
         page = request.GET.get('page', 1)
-        members = get_page(corpstats.get_member_objects(), page)
+        members = get_page(corpstats.members.all(), page)
+        mains = get_page(corpstats.mains.all(), page)
+        unregistered = get_page(corpstats.unregistered_members.all(), page)
 
     if corpstats:
         context.update({
-            'corpstats': corpstats.get_view_model(request.user),
+            'corpstats': corpstats,
             'members': members,
+            'mains': mains,
+            'unregistered': unregistered,
         })
 
     return render(request, 'corputils/corpstats.html', context=context)
@@ -112,14 +118,10 @@ def corpstats_view(request, corp_id=None):
 def corpstats_update(request, corp_id):
     corp = get_object_or_404(EveCorporationInfo, corporation_id=corp_id)
     corpstats = get_object_or_404(CorpStats, corp=corp)
-    if corpstats.can_update(request.user):
+    try:
-        try:
+        corpstats.update()
-            corpstats.update()
+    except HTTPError as e:
-        except HTTPError as e:
+        messages.error(request, str(e))
-            messages.error(request, str(e))
-    else:
-        raise PermissionDenied(
-            'You do not have permission to update member data for the selected corporation statistics module.')
     if corpstats.pk:
         return redirect('corputils:view_corp', corp_id=corp.corporation_id)
     else:
@@ -132,13 +134,11 @@ def corpstats_search(request):
     results = []
     search_string = request.GET.get('search_string', None)
     if search_string:
-        has_similar = CorpStats.objects.filter(_members__icontains=search_string).visible_to(request.user)
+        has_similar = CorpStats.objects.filter(members__character_name__icontains=search_string).visible_to(request.user).distinct()
         for corpstats in has_similar:
-            similar = [(member_id, corpstats.members[member_id]) for member_id in corpstats.members if
+            similar = corpstats.members.filter(character_name__icontains=search_string)
-                       search_string.lower() in corpstats.members[member_id].lower()]
             for s in similar:
-                results.append(
+                results.append((corpstats, s))
-                    (corpstats, CorpStats.MemberObject(s[0], s[1])))
         page = request.GET.get('page', 1)
         results = sorted(results, key=lambda x: x[1].character_name)
         results_page = get_page(results, page)