t_subtasks.py

#!/usr/bin/env python3

# Copyright (C) 2018    ASTRON (Netherlands Institute for Radio Astronomy)
# P.O. Box 2, 7990 AA Dwingeloo, The Netherlands
#
# This file is part of the LOFAR software suite.
# The LOFAR software suite is free software: you can redistribute it and/or
# modify it under the terms of the GNU General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# The LOFAR software suite is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.    See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.

# $Id:  $

import os
import unittest

import logging
logger = logging.getLogger(__name__)
logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s', level=logging.INFO)

# Do Mandatory setup step:
# use setup/teardown magic for tmss test database, ldap server and django server
# (ignore pycharm unused import statement, python unittests does use at RunTime the tmss_test_environment_unittest_setup module)
from lofar.sas.tmss.test.tmss_test_environment_unittest_setup import *


from lofar.sas.tmss.test.tmss_test_data_django_models import *

# import and setup rest test data creator
from lofar.sas.tmss.test.tmss_test_data_rest import TMSSRESTTestDataCreator

from lofar.sas.tmss.tmss.tmssapp import models

from lofar.sas.tmss.tmss.tmssapp.subtasks import *

# The following methods should be tested
# check_prerequities_for_subtask_creation
# create_subtasks_from_task_blueprint
# create_observation_control_subtask_from_task_blueprint
# create_qafile_subtask_from_task_blueprint
# create_qafile_subtask_from_observation_subtask
# create_qaplots_subtask_from_task_blueprint
# create_qaplots_subtask_from_qafile_subtask
# create_preprocessing_subtask_from_task_blueprint
#
# schedule_subtask
# check_prerequities_for_scheduling
# schedule_qafile_subtask
# schedule_qaplots_subtask
# schedule_observation_subtask
# schedule_pipeline_subtask
#
# create_and_schedule_subtasks_from_task_blueprint





class SubTasksCreationFromSubTask(unittest.TestCase):

    @staticmethod
    def create_subtask_object(subtask_type_value, subtask_state_value):
        """
        Helper function to create a subtask object for testing with given subtask value and subtask state value
        as string (no object)
        """
        template_type = models.SubtaskType.objects.get(value=subtask_type_value)
        subtask_template_obj = SubTasksCreationFromSubTask.create_subtask_template(template_type)
        subtask_state_obj = models.SubtaskState.objects.get(value=subtask_state_value)
        subtask_data = Subtask_test_data(subtask_template=subtask_template_obj, state=subtask_state_obj)
        return models.Subtask.objects.create(**subtask_data)

    @staticmethod
    def create_subtask_template(template_type: object):
        subtask_template_data = SubtaskTemplate_test_data()
        subtask_template_data['type'] = template_type
        return models.SubtaskTemplate.objects.create(**subtask_template_data)

    def test_create_qafile_subtask_from_observation_subtask_failed(self):
        """
        Test if creation of subtask qafile failed due to wrong state or wrong type of the predecessor subtask
        Correct state should be 'defined' and correct type should be 'observation' (for this test of course it is not)
        """
        subtasks = [self.create_subtask_object("pipeline", "defined"),
                    self.create_subtask_object("observation", "defining"),
                    self.create_subtask_object("observation", "defining") ]
        for subtask in subtasks:
            with self.assertRaises(ValueError):
                create_qafile_subtask_from_observation_subtask(subtask)

    def test_create_qafile_subtask_from_observation_subtask_succeed(self):
        """
        Test if creation of subtask qafile succeed
        Check if the created subtask has correct subtask state and value (TODO)
        """
        predecessor_subtask = self.create_subtask_object("observation", "defined")
        subtask = create_qafile_subtask_from_observation_subtask(predecessor_subtask)
        # subtask object is None because QA file conversion is  by default not enabled!!!!
        self.assertEqual(None, subtask)

    def test_create_qaplots_subtask_from_qafile_subtask_failed(self):
        """
        Test if creation of subtask qaplots failed due to wrong state or wrong type of the predecessor subtask
        Correct type should be 'qa_files' (for this test of course it is not)
        """
        subtasks = [self.create_subtask_object("pipeline", "defined"),
                    self.create_subtask_object("observation", "defining"),
                    self.create_subtask_object("observation", "defining") ]
        for subtask in subtasks:
            with self.assertRaises(ValueError):
                create_qaplots_subtask_from_qafile_subtask(subtask)

    def test_create_qaplots_subtask_from_qafile_subtask_succeed(self):
        """
        Test if creation of subtask qaplots succeed
        Check if the created subtask has correct subtask state and value (TODO)
        """
        predecessor_subtask = self.create_subtask_object("qa_files", "defined")
        subtask = create_qaplots_subtask_from_qafile_subtask(predecessor_subtask)
        # subtask object is None because QA plots is by default not enabled!!!!
        self.assertEqual(None, subtask)


class SubTasksCreationFromTaskBluePrint(unittest.TestCase):

    @staticmethod
    def create_task_blueprint_object(task_template_name="correlator schema", QA_enabled=False):
        """
        Helper function to create a task blueprint object for testing with given task template name value
        as string (no object)
        """
        task_blueprint_data = TaskBlueprint_test_data()
        task_blueprint_obj = models.TaskBlueprint.objects.create(**task_blueprint_data)
        task_blueprint_obj.specifications_template.name = task_template_name
        task_blueprint_obj.specifications_doc = {
                "QA": {
                    "plots": {
                        "enabled": QA_enabled,
                        "autocorrelation": True,
                        "crosscorrelation": True
                    },
                    "file_conversion": {
                        "enabled": QA_enabled,
                        "nr_of_subbands": -1,
                        "nr_of_timestamps": 256
                    }
                }
        }
        return task_blueprint_obj

    def test_create_sequence_of_subtask_from_task_blueprint(self):
        """
        Create multiple subtasks from a task blueprint, executed in correct order.
        No exception should occur, check name, type and state of the subtask
        """
        task_blueprint = self.create_task_blueprint_object()

        subtask = create_observation_control_subtask_from_task_blueprint(task_blueprint)
        self.assertEqual("defined", str(subtask.state))
        self.assertEqual("observationcontrol schema", str(subtask.specifications_template.name))
        self.assertEqual("observation", str(subtask.specifications_template.type))

        # Next call requires an observation subtask already created
        subtask = create_qafile_subtask_from_task_blueprint(task_blueprint)
        # subtask object is None because QA file conversion is by default not enabled!!!!
        self.assertEqual(None, subtask)

        # Next call will fail due to no qa_files object
        # ValueError: Cannot create qa_plots subtask for task_blueprint id=1 because it has no qafile subtask(s)
        with self.assertRaises(ValueError):
            subtask = create_qaplots_subtask_from_task_blueprint(task_blueprint)

        # subtask = create_preprocessing_subtask_from_task_blueprint(task_blueprint)

    def test_create_sequence_of_subtask_from_task_blueprint_with_QA_enabled(self):
        """
        Create multiple subtasks from a task blueprint, executed in correct order.
        QA plots and QA file conversion enabled
        No exception should occur, check name, type and state of the subtasks
        """
        # Enable QA plot and QA conversion
        task_blueprint = self.create_task_blueprint_object(QA_enabled=True)
        task_blueprint_preprocessing = self.create_task_blueprint_object("preprocessing schema")

        subtask = create_observation_control_subtask_from_task_blueprint(task_blueprint)
        self.assertEqual("defined", str(subtask.state))
        self.assertEqual("observationcontrol schema", str(subtask.specifications_template.name))
        self.assertEqual("observation", str(subtask.specifications_template.type))
        # Next call requires an observation subtask already created
        subtask = create_qafile_subtask_from_task_blueprint(task_blueprint)
        self.assertEqual("defined", str(subtask.state))
        self.assertEqual("QA file conversion", str(subtask.specifications_template.name))
        self.assertEqual("qa_files", str(subtask.specifications_template.type))
        # Next call requires an qaplots subtask already created
        subtask = create_qaplots_subtask_from_task_blueprint(task_blueprint)
        self.assertEqual("defined", str(subtask.state))
        self.assertEqual("QA plots", str(subtask.specifications_template.name))
        self.assertEqual("qa_plots", str(subtask.specifications_template.type))
        # TODO: check why next call failed?
        #subtask = create_preprocessing_subtask_from_task_blueprint(task_blueprint_preprocessing)
        #self.assertEqual("defined", str(subtask.state))


    def test_create_subtasks_from_task_blueprint_failure_on_schema(self):
        """
        Test creation failure due to unknown schema (no correlator or preprocessing schema)
        Check exception
            "SubtaskCreationException: Cannot create subtasks for task id=1 because no generator exists for its schema name=unknown schema"
        """
        task_blueprint = self.create_task_blueprint_object("unknown schema")
        with self.assertRaises(SubtaskCreationException):
            create_subtasks_from_task_blueprint(task_blueprint)

    def test_create_subtasks_from_task_blueprint_succeed(self):
        """
        """
        task_blueprint = self.create_task_blueprint_object(QA_enabled=True)
        subtasks = create_subtasks_from_task_blueprint(task_blueprint)
        self.assertEqual(3, len(subtasks))

# TODO Test the Schedule calls

if __name__ == "__main__":
    os.environ['TZ'] = 'UTC'
    unittest.main()