-
Jorrit Schaap authoredJorrit Schaap authored
Code owners
Assign users and groups as approvers for specific file changes. Learn more.
remakemigrations.py 9.28 KiB
#!/usr/bin/env python3
# Copyright (C) 2018 ASTRON (Netherlands Institute for Radio Astronomy)
# P.O. Box 2, 7990 AA Dwingeloo, The Netherlands
#
# This file is part of the LOFAR software suite.
# The LOFAR software suite is free software: you can redistribute it and/or
# modify it under the terms of the GNU General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# The LOFAR software suite is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
# $Id: $
# This script automates the procedure to replace the existing migrations on the source tree with initital migrations
# based on the current datamodel. Django offers a call 'makemigrations' through manage.py, which creates new migrations
# after the datamodel implementation has changed. These additional migrations apply those changes to an existing
# database reflecting the previous datamodel.
# This is a very nice feature for production, but there are a few downsides that this script tackles:
#
# 1. During development, where the datamodel constantly changes, whe typically don't want a ton of iterative migrations,
# but just have a clean start with a fresh initial database state without the whole provenance is perfectly fine. (We
# start up a fresh database anyway for every test or test deployment.) This can be achieved by removing all existing
# migrations prior to creating new ones.
# A difficulty with this approach is that we do have a manual migration to populate the database with fixtures.
# This migration needs to be restored or re-created after Django created fresh migrations for the database itself.
#
# 2. Since in settings.py we refer to the tmss app in the lofar environment, Django uses the build or installed version.
# A consequence is that the created migrations are placed in there and need to be copied to the source tree.
#
# This script requires a running postgres database instance to work against.
# To use specific database credentials, run e.g. ./remakemigrations.py -C b5f881c4-d41a-4f24-b9f5-23cd6a7f37d0
import os
from glob import glob
import subprocess as sp
import logging
import argparse
from shutil import copy
import lofar.sas.tmss
logger = logging.getLogger(__file__)
# set up paths
tmss_source_directory = os.path.dirname(__file__)
if tmss_source_directory == '':
tmss_source_directory = '.'
tmss_env_directory = os.path.dirname(lofar.sas.tmss.__file__)
relative_migrations_directory = '/tmss/tmssapp/migrations/'
# template for manual changes and fixture (applied last):
template = """
#
# auto-generated by remakemigrations.py
#
# ! Please make sure to apply any changes to the template in that script !
#
from django.db import migrations
from lofar.sas.tmss.tmss.tmssapp.populate import *
class Migration(migrations.Migration):
dependencies = [
('tmssapp', '{migration_dependency}'),
]
operations = [ migrations.RunSQL('ALTER SEQUENCE tmssapp_SubTask_id_seq RESTART WITH 2000000;'), # Start SubTask id with 2 000 000 to avoid overlap with 'old' (test/production) OTDB
# add an SQL trigger in the database enforcing correct state transitions.
# it is crucial that illegal subtask state transitions are block at the "lowest level" (i.e.: in the database) so we can guarantee that the subtask state machine never breaks.
# see: https://support.astron.nl/confluence/display/TMSS/Subtask+State+Machine
# Explanation of SQl below: A trigger function is called upon each create/update of the subtask.
# If the state changes, then it is checked if the state transition from old to new is present in the SubtaskAllowedStateTransitions table.
# If not an Exception is raised, thus enforcing a rollback, thus enforcing the state machine to follow the design.
# It is thereby enforced upon the user/caller to handle these blocked illegal state transitions, and act more wisely.
migrations.RunSQL('''CREATE OR REPLACE FUNCTION tmssapp_check_subtask_state_transition()
RETURNS trigger AS
$BODY$
BEGIN
IF TG_OP = 'INSERT' THEN
IF NOT (SELECT EXISTS(SELECT id FROM tmssapp_subtaskallowedstatetransitions WHERE old_state_id IS NULL AND new_state_id=NEW.state_id)) THEN
RAISE EXCEPTION 'ILLEGAL SUBTASK STATE TRANSITION FROM % TO %', NULL, NEW.state_id;
END IF;
END IF;
IF TG_OP = 'UPDATE' THEN
IF OLD.state_id <> NEW.state_id AND NOT (SELECT EXISTS(SELECT id FROM tmssapp_subtaskallowedstatetransitions WHERE old_state_id=OLD.state_id AND new_state_id=NEW.state_id)) THEN
RAISE EXCEPTION 'ILLEGAL SUBTASK STATE TRANSITION FROM "%" TO "%"', OLD.state_id, NEW.state_id;
END IF;
END IF;
RETURN NEW;
END;
$BODY$
LANGUAGE plpgsql VOLATILE;
DROP TRIGGER IF EXISTS tmssapp_trigger_on_check_subtask_state_transition ON tmssapp_SubTask ;
CREATE TRIGGER tmssapp_trigger_on_check_subtask_state_transition
BEFORE INSERT OR UPDATE ON tmssapp_SubTask
FOR EACH ROW EXECUTE PROCEDURE tmssapp_check_subtask_state_transition();'''),
migrations.RunPython(populate_choices),
migrations.RunPython(populate_subtask_allowed_state_transitions),
migrations.RunPython(populate_settings),
migrations.RunPython(populate_misc),
migrations.RunPython(populate_resources),
migrations.RunPython(populate_cycles),
migrations.RunPython(populate_projects) ]
"""
def execute_and_log(cmd):
logger.info('COMMAND: %s' % cmd)
p = sp.Popen(cmd, shell=True, stdout=sp.PIPE, stderr=sp.PIPE)
out, err = p.communicate()
if out is not None:
logger.info("STDOUT: %s" % out.decode('utf-8').strip())
if err is not None:
logger.info("STDERR: %s" % err.decode('utf-8').strip())
def delete_old_migrations():
logger.info('Removing old migrations...')
files = glob_migrations()
for f in [path for path in files if ("initial" in path or "auto" in path or "populate" in path)]:
logger.info('Deleting: %s' % f)
os.remove(f)
def make_django_migrations(dbcredentials=None):
logger.info('Making Django migrations...')
if dbcredentials:
os.environ['TMSS_DBCREDENTIALS'] = dbcredentials
execute_and_log('/usr/bin/env python3 %s/manage.py makemigrations' % tmss_source_directory)
def make_populate_migration():
logger.info('Making migration for populating database...')
last_migration = determine_last_migration()
migration = template.format(migration_dependency=last_migration)
path = tmss_env_directory + relative_migrations_directory + '%s_populate.py' % str(int(last_migration.split('_')[0])+1).zfill(4)
logger.info('Writing to: %s' % path)
with open(path,'w') as f:
f.write(migration)
def glob_migrations(directories=(tmss_source_directory, tmss_env_directory)):
paths = []
for directory in directories:
paths += glob(directory + '/' + relative_migrations_directory + '0*_*')
return paths
def copy_migrations_to_source():
logger.info('Copying over migrations to source directory...')
files = glob_migrations(directories=[tmss_env_directory])
for file in files:
logger.info('Copying %s to %s' % (file, tmss_source_directory + '/' + relative_migrations_directory))
copy(file, tmss_source_directory + '/' + relative_migrations_directory)
def determine_last_migration():
logger.info('Determining last migration...')
files = glob_migrations()
files = [os.path.basename(path) for path in files]
f = max(files)
last_migration = f.split('.py')[0]
logger.info('Determined last migration: %s' % last_migration)
return last_migration
def remake_migrations(dbcredentials=None):
delete_old_migrations()
make_django_migrations(dbcredentials)
make_populate_migration()
copy_migrations_to_source()
if __name__ == "__main__":
logger.setLevel(logging.DEBUG)
handler = logging.StreamHandler()
handler.setLevel(logging.INFO)
logger.addHandler(handler)
parser = argparse.ArgumentParser()
parser.add_argument("-C", action="store", dest="dbcredentials", help="use database specified in these dbcredentials")
args = parser.parse_args()
remake_migrations(args.dbcredentials)