Services Modules migration improvements (#676)

* Added a check to prevent dataloss from missing service modules

Migration will raise an exception and refuse to run if a model has data
for a field which is missing its target service.

* Add setting to allow services to be installed after service migration

Previously django would complain about migrations being out of order. By
setting `SERVICES_MIGRATED=True` the
`authentication.0013_service_modules` migration drops all of its
'optional' dependencies which allows the initial migrations of service
modules to run normally. If the setting is missing or set to False, the
migration will require all installed and required services migrations to
have run before the `authentication.0013_service_modules` migration.

* Move setting to somewhere it makes more sense

* Modify celerybeat registration to automatically register services

alliance_auth/settings.py.example

@@ -22,15 +22,8 @@ djcelery.setup_loader()
 # Celery configuration
 BROKER_URL = 'redis://localhost:6379/0'
 CELERYBEAT_SCHEDULER = "djcelery.schedulers.DatabaseScheduler"
-CELERYBEAT_SCHEDULE = {
-    """
-    Uncomment this if you are using the Teamspeak3 service
-    'run_ts3_group_update': {
-        'task': 'services.modules.teamspeak3.tasks.Teamspeak3Tasks.run_ts3_group_update',
-        'schedule': crontab(minute='*/30'),
-    },
-    """
-}
+CELERYBEAT_SCHEDULE = dict()
+
 
 # Build paths inside the project like this: os.path.join(BASE_DIR, ...)
 BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
@@ -157,6 +150,15 @@ DATABASES = {
     },
 }
 
+# If you have run the authentication.0013_service_modules migration
+# you will need to set this to True in order to install service modules
+# which were involved in that migration after it has been run.
+# If you are on a fresh install with no existing database you can safely
+# set this to True
+# If you have not run the authentication.0013_service_modules migration
+# leave this set to False.
+SERVICES_MIGRATED = False
+
 # Password validation
 # https://docs.djangoproject.com/en/1.10/ref/settings/#auth-password-validators
 
@@ -699,3 +701,9 @@ if ENABLE_AUTH_IPS4 or ENABLE_BLUE_IPS4:
 # Ensure corp/alliance IDs are expected types
 STR_CORP_IDS = [str(id) for id in CORP_IDS]
 STR_ALLIANCE_IDS = [str(id) for id in ALLIANCE_IDS]
+
+if 'services.modules.teamspeak3' in INSTALLED_APPS:
+    CELERYBEAT_SCHEDULE['run_ts3_group_update'] = {
+        'task': 'services.modules.teamspeak3.tasks.Teamspeak3Tasks.run_ts3_group_update',
+        'schedule': crontab(minute='*/30'),
+    }

authentication/migrations/0013_service_modules.py

@@ -22,6 +22,10 @@ def optional_dependencies():
 
     dependencies = []
 
+    # Skip adding module dependencies if the settings specifies that services have been migrated
+    if hasattr(settings, 'SERVICES_MIGRATED') and settings.SERVICES_MIGRATED:
+        return dependencies
+
     if 'services.modules.xenforo' in installed_apps:
         dependencies.append(('xenforo', '0001_initial'))
     if 'services.modules.discord' in installed_apps:
@@ -48,10 +52,54 @@ def optional_dependencies():
     return dependencies
 
 
+class DatalossException(Exception):
+    def __init__(self, field, app):
+        message = "This migration would cause a loss of data for the %s field, ensure the %s app is installed and" \
+                  " run the migration again" % (field, app)
+        super(Exception, self).__init__(message)
+
+
+def check_for_dataloss(authservicesinfo):
+    """
+    Check if any of the authservicesinfo contain a field which contains data
+    that would be lost because the target module for migration is not installed.
+    If a record is found with no matching target installed an exception is raised.
+    :param authservicesinfo: AuthServicesInfo records to check
+    """
+    installed_apps = settings.INSTALLED_APPS
+
+    for authinfo in authservicesinfo:
+        if authinfo.xenforo_username and 'services.modules.xenforo' not in installed_apps:
+            raise DatalossException('xenforo_username', 'services.modules.xenforo')
+        if authinfo.discord_uid and 'services.modules.discord' not in installed_apps:
+            raise DatalossException('discord_uid', 'services.modules.discord')
+        if authinfo.discourse_enabled and 'services.modules.discourse' not in installed_apps:
+            raise DatalossException('discourse_enabled', 'services.modules.discourse')
+        if authinfo.ipboard_username and 'services.modules.ipboard' not in installed_apps:
+            raise DatalossException('ipboard_username', 'services.modules.ipboard')
+        if authinfo.ips4_id and 'services.modules.ips4' not in installed_apps:
+            raise DatalossException('ips4_id', 'services.modules.ips4')
+        if authinfo.market_username and 'services.modules.market' not in installed_apps:
+            raise DatalossException('market_username', 'services.modules.market')
+        if authinfo.jabber_username and 'services.modules.openfire' not in installed_apps:
+            raise DatalossException('jabber_username', 'services.modules.openfire')
+        if authinfo.smf_username and 'services.modules.smf' not in installed_apps:
+            raise DatalossException('smf_username', 'services.modules.smf')
+        if authinfo.teamspeak3_uid and 'services.modules.teamspeak3' not in installed_apps:
+            raise DatalossException('teamspeak3_uid', 'services.modules.teamspeak3')
+        if authinfo.mumble_username and 'services.modules.mumble' not in installed_apps:
+            raise DatalossException('mumble_username', 'services.modules.mumble')
+        if authinfo.forum_username and 'services.modules.phpbb3' not in installed_apps:
+            raise DatalossException('forum_username', 'services.modules.phpbb3')
+
+
 def forward(apps, schema_editor):
     installed_apps = settings.INSTALLED_APPS
     AuthServicesInfo = apps.get_model("authentication", "AuthServicesInfo")
 
+    # Check if any records would result in dataloss
+    check_for_dataloss(AuthServicesInfo.objects.all())
+
     XenforoUser = apps.get_model('xenforo', 'XenforoUser') if 'services.modules.xenforo' in installed_apps else None
     DiscordUser = apps.get_model('discord', 'DiscordUser') if 'services.modules.discord' in installed_apps else None
     DiscourseUser = apps.get_model('discourse', 'DiscourseUser') if 'services.modules.discourse' in installed_apps else None