-
Jorrit Schaap authoredJorrit Schaap authored
Code owners
Assign users and groups as approvers for specific file changes. Learn more.
populate.py 11.51 KiB
"""
This module 'populate' defines methods to populate the database with predefined ('static') data,
according to the proposed Django way: https://docs.djangoproject.com/en/2.0/topics/migrations/#data-migrations
import this module in your empty migration step file, and add the following migration:
from ..populate import *
class Migration(migrations.Migration):
dependencies = [ <the dependency is automatically inserted here> ]
operations = [ migrations.RunPython(populate_choices) ]
"""
import logging
logger = logging.getLogger(__name__)
import json
import os
from datetime import datetime, timezone
from lofar.sas.tmss.tmss.tmssapp import models
from lofar.sas.tmss.tmss.tmssapp.models.specification import *
from lofar.sas.tmss.tmss.tmssapp.models.scheduling import *
from lofar.common import isTestEnvironment, isDevelopmentEnvironment
working_dir = os.path.dirname(os.path.abspath(__file__))
def populate_choices(apps, schema_editor):
'''
populate each 'choice' table in the database with the 'static' list of 'choice'.Choices for
each 'choice'type in Role, Datatype, Dataformat, CopyReason
:return: None
'''
for choice_class in [Role, Datatype, Dataformat, CopyReason,
SubtaskState, SubtaskType, StationType, Algorithm, ScheduleMethod, SchedulingRelationPlacement,
Flag, ProjectCategory, PeriodCategory, Quantity, TaskType]:
choice_class.objects.bulk_create([choice_class(value=x.value) for x in choice_class.Choices])
def populate_settings(apps, schema_editor):
Setting.objects.create(name=Flag.objects.get(value='allow_scheduling_observations'), value=True)
def populate_test_data():
"""
Create a Test Schedule Set to be able to refer to when Scheduling Unit Draft is created from a
scheduling unit json
:return:
"""
try:
# only add (with expensive setup time) example data when developing/testing and we're not unittesting
if isTestEnvironment() or isDevelopmentEnvironment():
from lofar.sas.tmss.tmss.exceptions import TMSSException
from lofar.sas.tmss.test.tmss_test_data_django_models import SchedulingSet_test_data, SchedulingUnitDraft_test_data
from lofar.sas.tmss.tmss.tmssapp.tasks import create_task_blueprints_and_subtasks_from_scheduling_unit_draft, create_task_blueprints_and_subtasks_and_schedule_subtasks_from_scheduling_unit_draft
from lofar.sas.tmss.tmss.tmssapp.subtasks import schedule_subtask
# create a Test Scheduling Set UC1 under project TMSS-Commissioning
tmss_project = models.Project.objects.get(name="TMSS-Commissioning")
for set_nr in range(2):
scheduling_set_data = SchedulingSet_test_data(name="Test Scheduling Set UC1 example %s" % (set_nr,), project=tmss_project)
scheduling_set = models.SchedulingSet.objects.create(**scheduling_set_data)
scheduling_set.tags = ["TEST", "UC1"]
scheduling_set.save()
logger.info('created test scheduling_set: %s', scheduling_set.name)
for unit_nr in range(5):
strategy_template = models.SchedulingUnitObservingStrategyTemplate.objects.get(name="UC1 CTC+pipelines")
# the 'template' in the strategy_template is a predefined json-data blob which validates against the given scheduling_unit_template
# a user might 'upload' a partial json-data blob, so add all the known defaults
scheduling_unit_spec = add_defaults_to_json_object_for_schema(strategy_template.template, strategy_template.scheduling_unit_template.schema)
# add the scheduling_unit_doc to a new SchedulingUnitDraft instance, and were ready to use it!
scheduling_unit_draft = models.SchedulingUnitDraft.objects.create(name="UC1 test scheduling unit %s.%s" % (set_nr+1, unit_nr+1),
scheduling_set=scheduling_set,
requirements_template=strategy_template.scheduling_unit_template,
requirements_doc=scheduling_unit_spec,
observation_strategy_template=strategy_template)
scheduling_unit_draft.tags = ["TEST", "UC1"]
scheduling_unit_draft.save()
logger.info('created test scheduling_unit_draft: %s', scheduling_unit_draft.name)
try:
if set_nr==0 and unit_nr==0:
scheduling_unit_blueprint = create_task_blueprints_and_subtasks_from_scheduling_unit_draft(scheduling_unit_draft)
scheduled_subtasks = models.Subtask.objects.filter(task_blueprint__scheduling_unit_blueprint=scheduling_unit_blueprint, task_blueprint__name='Calibrator Observation 1', specifications_template__type='observation').all()
for subtask in scheduled_subtasks:
schedule_subtask(subtask)
for state in [SubtaskState.Choices.QUEUEING, SubtaskState.Choices.QUEUED, SubtaskState.Choices.STARTING, SubtaskState.Choices.STARTED, SubtaskState.Choices.FINISHING, SubtaskState.Choices.FINISHED]:
subtask.state = SubtaskState.objects.get(value=state.value)
subtask.save()
else:
create_task_blueprints_and_subtasks_from_scheduling_unit_draft(scheduling_unit_draft)
except TMSSException as e:
logger.exception(e)
except ImportError:
pass
def populate_cycles(apps, schema_editor):
for nr in range(0, 18):
cycle = models.Cycle.objects.create(name="Cycle %02d" % nr,
description="Lofar Cycle %s" % nr,
start=datetime(2013+nr//2, 6 if nr%2==0 else 11, 1, 0, 0, 0, 0, tzinfo=timezone.utc),
stop=datetime(2013+(nr+1)//2, 6 if nr%2==1 else 11, 1, 0, 0, 0, 0, tzinfo=timezone.utc))
models.CycleQuota.objects.create(cycle=cycle,
resource_type=ResourceType.objects.get(name="observing_time"),
value=0.8*cycle.duration.total_seconds()) # rough guess. 80% of total time available for observing
models.CycleQuota.objects.create(cycle=cycle,
resource_type=ResourceType.objects.get(name="cep_processing_time"),
value=0.8*cycle.duration.total_seconds())
models.CycleQuota.objects.create(cycle=cycle,
resource_type=ResourceType.objects.get(name="lta_storage"),
value=0) # needs to be filled in by user (SOS)
models.CycleQuota.objects.create(cycle=cycle,
resource_type=ResourceType.objects.get(name="support_time"),
value=0) # needs to be filled in by user (SOS)
models.CycleQuota.objects.create(cycle=cycle,
resource_type=ResourceType.objects.get(name="observing_time_commissioning"),
value=0.05*cycle.duration.total_seconds()) # rough guess. 5% of total time available for observing
models.CycleQuota.objects.create(cycle=cycle,
resource_type=ResourceType.objects.get(name="observing_time_prio_a"),
value=0) # needs to be filled in by user (SOS)
models.CycleQuota.objects.create(cycle=cycle,
resource_type=ResourceType.objects.get(name="observing_time_prio_b"),
value=0) # needs to be filled in by user (SOS)
def populate_projects(apps, schema_editor):
tmss_project = models.Project.objects.create(name="TMSS-Commissioning",
description="Project for all TMSS tests and commissioning",
priority_rank=1.0,
can_trigger=False,
private_data=True,
expert=True,
filler=False)
tmss_project.cycles.set([models.Cycle.objects.get(name="Cycle 14")])
def populate_resources(apps, schema_editor):
ResourceType.objects.create(name="lta_storage", description="Amount of storage in the LTA (in bytes)", quantity=Quantity.objects.get(value=Quantity.Choices.BYTES.value))
ResourceType.objects.create(name="cep_storage", description="Amount of storage on the CEP processing cluster (in bytes)", quantity=Quantity.objects.get(value=Quantity.Choices.BYTES.value))
ResourceType.objects.create(name="cep_processing_time", description="Processing time on the CEP processing cluster (in seconds)", quantity=Quantity.objects.get(value=Quantity.Choices.TIME.value))
ResourceType.objects.create(name="observing_time", description="Observing time (in seconds)", quantity=Quantity.objects.get(value=Quantity.Choices.TIME.value))
ResourceType.objects.create(name="observing_time_prio_a", description="Observing time with priority A (in seconds)", quantity=Quantity.objects.get(value=Quantity.Choices.TIME.value))
ResourceType.objects.create(name="observing_time_prio_b", description="Observing time with priority B (in seconds)", quantity=Quantity.objects.get(value=Quantity.Choices.TIME.value))
ResourceType.objects.create(name="observing_time_commissioning", description="Observing time for Commissioning/DDT (in seconds)", quantity=Quantity.objects.get(value=Quantity.Choices.TIME.value))
ResourceType.objects.create(name="support_time", description="Support time by human (in seconds)", quantity=Quantity.objects.get(value=Quantity.Choices.TIME.value))
ResourceType.objects.create(name="number_of_triggers", description="Number of trigger events (as integer)", quantity=Quantity.objects.get(value=Quantity.Choices.NUMBER.value))
def populate_misc(apps, schema_editor):
cluster = Cluster.objects.create(name="CEP4", location="CIT", archive_site=False)
fs = Filesystem.objects.create(name="LustreFS", cluster=cluster, capacity=3.6e15)
def populate_connectors():
# the TaskConnectorType's define how the Task[Draft/Blueprint] *can* be connected.
# TODO Need overview which we do actually need
TaskConnectorType.objects.create(role=Role.objects.get(value=Role.Choices.INPUT.value),
datatype=Datatype.objects.get(value=Datatype.Choices.VISIBILITIES.value),
output_of=TaskTemplate.objects.get(name='calibrator observation'),
input_of=TaskTemplate.objects.get(name='preprocessing pipeline'))
TaskConnectorType.objects.create(role=Role.objects.get(value=Role.Choices.CORRELATOR.value),
datatype=Datatype.objects.get(value=Datatype.Choices.VISIBILITIES.value),
output_of=TaskTemplate.objects.get(name='calibrator observation'),
input_of=TaskTemplate.objects.get(name='preprocessing pipeline'))