remakemigrations.py

#!/usr/bin/env python3

# Copyright (C) 2018    ASTRON (Netherlands Institute for Radio Astronomy)
# P.O. Box 2, 7990 AA Dwingeloo, The Netherlands
#
# This file is part of the LOFAR software suite.
# The LOFAR software suite is free software: you can redistribute it and/or
# modify it under the terms of the GNU General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# The LOFAR software suite is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.    See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.

# $Id:  $

# This script automates the procedure to replace the existing migrations on the source tree with initital migrations
# based on the current datamodel. Django offers a call 'makemigrations' through manage.py, which creates new migrations
# after the datamodel implementation has changed. These additional migrations apply those changes to an existing
# database reflecting the previous datamodel.
# This is a very nice feature for production, but there are a few downsides that this script tackles:
#
# 1. During development, where the datamodel constantly changes, whe typically don't want a ton of iterative migrations,
# but just have a clean start with a fresh initial database state without the whole provenance is perfectly fine. (We
# start up a fresh database anyway for every test or test deployment.) This can be achieved by removing all existing
# migrations prior to creating new ones.
# A difficulty with this approach is that we do have a manual migration to populate the database with fixtures.
# This migration needs to be restored or re-created after Django created fresh migrations for the database itself.
#
# 2. Since in settings.py we refer to the tmss app in the lofar environment, Django uses the build or installed version.
# A consequence is that the created migrations are placed in there and need to be copied to the source tree.
#
# This script requires a running postgres database instance to work against.
# To use specific database credentials, run e.g. ./remakemigrations.py -C b5f881c4-d41a-4f24-b9f5-23cd6a7f37d0


import os
from glob import glob
import subprocess as sp
import logging
import argparse
from shutil import copy
import lofar.sas.tmss

logger = logging.getLogger(__file__)

# set up paths
tmss_source_directory = os.path.dirname(__file__)
if tmss_source_directory == '':
    tmss_source_directory = '.'
tmss_env_directory = os.path.dirname(lofar.sas.tmss.__file__)
relative_migrations_directory = '/tmss/tmssapp/migrations/'

# template for manual changes and fixture (applied last):
template = """
#
# auto-generated by remakemigrations.py
#
# ! Please make sure to apply any changes to the template in that script !  
#
from django.db import migrations

from lofar.sas.tmss.tmss.tmssapp.populate import *

class Migration(migrations.Migration):

    dependencies = [
        ('tmssapp', '%s'),
    ]

    # Start SubTask id with 2 000 000 to avoid overlap with 'old' (test/production) OTDB
    operations = [ migrations.RunSQL('ALTER SEQUENCE tmssapp_SubTask_id_seq RESTART WITH 2000000;'),
                   migrations.RunPython(populate_choices),
                   migrations.RunPython(populate_settings),
                   migrations.RunPython(populate_misc),
                   migrations.RunPython(populate_resources),
                   migrations.RunPython(populate_cycles),
                   migrations.RunPython(populate_projects) ]
"""


def execute_and_log(cmd):

    logger.info('COMMAND: %s' % cmd)
    p = sp.Popen(cmd, shell=True, stdout=sp.PIPE, stderr=sp.PIPE)
    out, err = p.communicate()
    if out is not None:
        logger.info("STDOUT: %s" % out.decode('utf-8').strip())
    if err is not None:
        logger.info("STDERR: %s" % err.decode('utf-8').strip())


def delete_old_migrations():
    logger.info('Removing old migrations...')

    files = glob_migrations()
    for f in [path for path in files if ("initial" in path or "auto" in path or "populate" in path)]:
        logger.info('Deleting: %s' % f)
        os.remove(f)


def make_django_migrations(dbcredentials=None):

    logger.info('Making Django migrations...')
    if dbcredentials:
        os.environ['TMSS_DBCREDENTIALS'] = dbcredentials
    execute_and_log('/usr/bin/env python3 %s/manage.py makemigrations' % tmss_source_directory)


def make_populate_migration():

    logger.info('Making migration for populating database...')
    last_migration = determine_last_migration()
    migration = template % last_migration

    path = tmss_env_directory + relative_migrations_directory + '%s_populate.py' % str(int(last_migration.split('_')[0])+1).zfill(4)
    logger.info('Writing to: %s' % path)
    with open(path,'w') as f:
        f.write(migration)


def glob_migrations(directories=(tmss_source_directory, tmss_env_directory)):
    paths = []
    for directory in directories:
        paths += glob(directory + '/' + relative_migrations_directory + '0*_*')
    return paths


def copy_migrations_to_source():
    logger.info('Copying over migrations to source directory...')
    files = glob_migrations(directories=[tmss_env_directory])
    for file in files:
        logger.info('Copying %s to %s' % (file, tmss_source_directory + '/' + relative_migrations_directory))
        copy(file, tmss_source_directory + '/' + relative_migrations_directory)


def determine_last_migration():
    logger.info('Determining last migration...')
    files = glob_migrations()
    files = [os.path.basename(path) for path in files]
    f = max(files)
    last_migration = f.split('.py')[0]
    logger.info('Determined last migration: %s' % last_migration)
    return last_migration


def remake_migrations(dbcredentials=None):
    delete_old_migrations()
    make_django_migrations(dbcredentials)
    make_populate_migration()
    copy_migrations_to_source()


if __name__ == "__main__":

    logger.setLevel(logging.DEBUG)

    handler = logging.StreamHandler()
    handler.setLevel(logging.INFO)
    logger.addHandler(handler)

    parser = argparse.ArgumentParser()
    parser.add_argument("-C", action="store", dest="dbcredentials", help="use database specified in these dbcredentials")
    args = parser.parse_args()
    remake_migrations(args.dbcredentials)