-
Jorrit Schaap authoredJorrit Schaap authored
Code owners
Assign users and groups as approvers for specific file changes. Learn more.
populate.py 62.31 KiB
"""
This module 'populate' defines methods to populate the database with predefined ('static') data,
according to the proposed Django way: https://docs.djangoproject.com/en/2.0/topics/migrations/#data-migrations
import this module in your empty migration step file, and add the following migration:
from ..populate import *
class Migration(migrations.Migration):
dependencies = [ <the dependency is automatically inserted here> ]
operations = [ migrations.RunPython(populate_choices) ]
"""
import logging
logger = logging.getLogger(__name__)
import inspect
import re
from datetime import timezone, datetime, date
from lofar.sas.tmss.tmss.tmssapp import models
from lofar.sas.tmss.tmss.tmssapp import viewsets
from lofar.sas.tmss.tmss.tmssapp.models.specification import *
from lofar.sas.tmss.tmss.tmssapp.models.scheduling import *
from lofar.sas.tmss.tmss.tmssapp.models.permissions import *
from lofar.sas.tmss.tmss.tmssapp.conversions import timestamps_and_stations_to_sun_rise_and_set, get_all_stations
from lofar.common import isTestEnvironment, isDevelopmentEnvironment
from concurrent.futures import ThreadPoolExecutor
from django.contrib.auth.models import Group, Permission
from django.contrib.auth import get_user_model
User = get_user_model()
from django.contrib.contenttypes.models import ContentType
from django.db.utils import IntegrityError
working_dir = os.path.dirname(os.path.abspath(__file__))
permission_groups = {'system_setting': [viewsets.AntennaSetViewSet,
viewsets.ClusterViewSet,
viewsets.CommonSchemaTemplateViewSet,
viewsets.CopyReasonViewSet,
viewsets.CycleViewSet,
viewsets.CycleQuotaViewSet,
viewsets.DataformatViewSet,
viewsets.DataproductFeedbackTemplateViewSet,
viewsets.DataproductSpecificationsTemplateViewSet,
viewsets.DatatypeViewSet,
viewsets.DefaultDataproductSpecificationsTemplateViewSet,
viewsets.DefaultGeneratorTemplateViewSet,
viewsets.DefaultReservationTemplateViewSet,
viewsets.DefaultSchedulingConstraintsTemplateViewSet,
viewsets.DefaultSchedulingUnitTemplateViewSet,
viewsets.DefaultSubtaskTemplateViewSet,
viewsets.DefaultTaskRelationSelectionTemplateViewSet,
viewsets.DefaultTaskTemplateViewSet,
viewsets.FilesystemViewSet,
viewsets.GeneratorTemplateViewSet,
viewsets.HashAlgorithmViewSet,
viewsets.IOTypeViewSet,
viewsets.PeriodCategoryViewSet,
viewsets.PriorityQueueTypeViewSet,
viewsets.ProjectCategoryViewSet,
viewsets.ProjectPermissionViewSet,
viewsets.ProjectRoleViewSet,
viewsets.QuantityViewSet,
viewsets.ReservationTemplateViewSet,
viewsets.ResourceTypeViewSet,
viewsets.RoleViewSet,
viewsets.SAPViewSet, # assuming JD means 'system setting' by 'system table'
viewsets.SAPTemplateViewSet,
viewsets.SchedulingConstraintsTemplateViewSet,
viewsets.SchedulingRelationPlacementViewSet, # assuming JD means 'system setting' by 'system table'
viewsets.SchedulingUnitTemplateViewSet,
viewsets.StationTypeViewSet,
viewsets.SubtaskViewSet, # assuming JD means 'system setting' by 'system table'
viewsets.SubtaskAllowedStateTransitionsViewSet,
viewsets.SubtaskTemplateViewSet,
viewsets.SubtaskTypeViewSet,
viewsets.TaskConnectorTypeViewSet,
viewsets.TaskTemplateViewSet,
viewsets.TaskTypeViewSet,
viewsets.UserViewSet], # assuming JD means 'system setting' by 'system table'
'system_feedback': [viewsets.DataproductViewSet,
viewsets.DataproductArchiveInfoViewSet,
viewsets.DataproductHashViewSet,
viewsets.DataproductTransformViewSet,
viewsets.SIPidentifierViewSet,
viewsets.SubtaskInputViewSet,
viewsets.SubtaskOutputViewSet,
viewsets.SubtaskStateViewSet,
viewsets.SubtaskStateLogViewSet],
'cycle_administration': [viewsets.ProjectViewSet,
viewsets.ProjectQuotaViewSet,
viewsets.ProjectQuotaArchiveLocationViewSet,
viewsets.ReservationViewSet,
viewsets.ReservationStrategyTemplateViewSet,
viewsets.SchedulingUnitObservingStrategyTemplateViewSet],
'project_administration': [viewsets.SchedulingSetViewSet,
viewsets.SchedulingUnitDraftViewSet,
viewsets.SchedulingUnitDraftTriggerViewSet, # todo: I assume that responsive telescope users do not have access to this. so to what group should this belong to? System setting?
viewsets.TagsViewSet, # todo: I don't think this should be here. Tags are present on almost every model we have. They are also unrelated to particular projects.
viewsets.TaskDraftViewSet,
viewsets.TaskRelationBlueprintViewSet,
viewsets.TaskRelationDraftViewSet,
viewsets.TaskRelationSelectionTemplateViewSet,
viewsets.TaskSchedulingRelationDraftViewSet],
'blueprint_alteration': [viewsets.SchedulingUnitBlueprintViewSet,
viewsets.TaskSchedulingRelationBlueprintViewSet],
'blueprint_fixed': [viewsets.TaskBlueprintViewSet],
'operational': [viewsets.StationTimelineViewSet,
viewsets.SettingViewSet,
viewsets.SystemSettingFlagViewSet]
}
def populate_choices(apps, schema_editor):
'''
populate each 'choice' table in the database with the 'static' list of 'choice'.Choices for
each 'choice'type in Role, Datatype, Dataformat, CopyReason
:return: None
'''
choice_classes = [Role, IOType, Datatype, Dataformat, CopyReason,
SubtaskState, SubtaskType, StationType, HashAlgorithm, SchedulingRelationPlacement,
SystemSettingFlag, ProjectCategory, PeriodCategory, Quantity, TaskType, ProjectRole, PriorityQueueType]
# upload choices in parallel
with ThreadPoolExecutor() as executor:
executor.map(lambda choice_class: choice_class.objects.bulk_create([choice_class(value=x.value) for x in choice_class.Choices]),
choice_classes)
def populate_subtask_allowed_state_transitions(apps, schema_editor):
'''populate the SubtaskAllowedStateTransitions table with the allowed state transitions as defined by the design in https://support.astron.nl/confluence/display/TMSS/Subtask+State+Machine'''
DEFINING = SubtaskState.objects.get(value=SubtaskState.Choices.DEFINING.value)
DEFINED = SubtaskState.objects.get(value=SubtaskState.Choices.DEFINED.value)
SCHEDULING = SubtaskState.objects.get(value=SubtaskState.Choices.SCHEDULING.value)
SCHEDULED = SubtaskState.objects.get(value=SubtaskState.Choices.SCHEDULED.value)
UNSCHEDULING = SubtaskState.objects.get(value=SubtaskState.Choices.UNSCHEDULING.value)
QUEUEING = SubtaskState.objects.get(value=SubtaskState.Choices.QUEUEING.value)
QUEUED = SubtaskState.objects.get(value=SubtaskState.Choices.QUEUED.value)
STARTING = SubtaskState.objects.get(value=SubtaskState.Choices.STARTING.value)
STARTED = SubtaskState.objects.get(value=SubtaskState.Choices.STARTED.value)
FINISHING = SubtaskState.objects.get(value=SubtaskState.Choices.FINISHING.value)
FINISHED = SubtaskState.objects.get(value=SubtaskState.Choices.FINISHED.value)
CANCELLING = SubtaskState.objects.get(value=SubtaskState.Choices.CANCELLING.value)
CANCELLED = SubtaskState.objects.get(value=SubtaskState.Choices.CANCELLED.value)
ERROR = SubtaskState.objects.get(value=SubtaskState.Choices.ERROR.value)
UNSCHEDULABLE = SubtaskState.objects.get(value=SubtaskState.Choices.UNSCHEDULABLE.value)
OBSOLETE = SubtaskState.objects.get(value=SubtaskState.Choices.OBSOLETE.value)
SubtaskAllowedStateTransitions.objects.bulk_create([
SubtaskAllowedStateTransitions(old_state=None, new_state=DEFINING),
SubtaskAllowedStateTransitions(old_state=DEFINING, new_state=DEFINED),
SubtaskAllowedStateTransitions(old_state=DEFINED, new_state=SCHEDULING),
SubtaskAllowedStateTransitions(old_state=SCHEDULING, new_state=SCHEDULED),
SubtaskAllowedStateTransitions(old_state=SCHEDULING, new_state=UNSCHEDULABLE),
SubtaskAllowedStateTransitions(old_state=SCHEDULED, new_state=STARTING), # this is an odd one, as most (all?) subtasks are queued before execution...
SubtaskAllowedStateTransitions(old_state=SCHEDULED, new_state=QUEUEING),
SubtaskAllowedStateTransitions(old_state=SCHEDULED, new_state=UNSCHEDULING),
SubtaskAllowedStateTransitions(old_state=UNSCHEDULING, new_state=DEFINED),
SubtaskAllowedStateTransitions(old_state=UNSCHEDULING, new_state=CANCELLING), # directly after unscheduling we want to be able to go to cancelling and not trigger any schedulers on the defined state
SubtaskAllowedStateTransitions(old_state=QUEUEING, new_state=QUEUED),
SubtaskAllowedStateTransitions(old_state=QUEUED, new_state=STARTING),
SubtaskAllowedStateTransitions(old_state=STARTING, new_state=STARTED),
SubtaskAllowedStateTransitions(old_state=STARTED, new_state=FINISHING),
SubtaskAllowedStateTransitions(old_state=FINISHING, new_state=FINISHED),
SubtaskAllowedStateTransitions(old_state=CANCELLING, new_state=CANCELLED),
SubtaskAllowedStateTransitions(old_state=DEFINING, new_state=ERROR),
SubtaskAllowedStateTransitions(old_state=SCHEDULING, new_state=ERROR),
SubtaskAllowedStateTransitions(old_state=UNSCHEDULING, new_state=ERROR),
SubtaskAllowedStateTransitions(old_state=QUEUEING, new_state=ERROR),
SubtaskAllowedStateTransitions(old_state=STARTING, new_state=ERROR),
SubtaskAllowedStateTransitions(old_state=STARTED, new_state=ERROR),
SubtaskAllowedStateTransitions(old_state=FINISHING, new_state=ERROR),
SubtaskAllowedStateTransitions(old_state=CANCELLING, new_state=ERROR),
# allow transition from the "end"-states cancelled/error to obsolete to indicate user-intent
SubtaskAllowedStateTransitions(old_state=CANCELLED, new_state=OBSOLETE),
SubtaskAllowedStateTransitions(old_state=ERROR, new_state=OBSOLETE),
SubtaskAllowedStateTransitions(old_state=DEFINED, new_state=CANCELLING),
SubtaskAllowedStateTransitions(old_state=SCHEDULED, new_state=CANCELLING),
SubtaskAllowedStateTransitions(old_state=QUEUED, new_state=CANCELLING),
SubtaskAllowedStateTransitions(old_state=STARTED, new_state=CANCELLING),
SubtaskAllowedStateTransitions(old_state=FINISHING, new_state=CANCELLING) # when feedback is not complete after a (1 hour) timeout, then the subtask is cancelled.
])
def populate_settings(apps, schema_editor):
Setting.objects.create(name=SystemSettingFlag.objects.get(value='dynamic_scheduling_enabled'), value=isDevelopmentEnvironment())
def populate_test_data():
"""
Create a Test Schedule Set to be able to refer to when Scheduling Unit Draft is created from a
scheduling unit json
:return:
"""
try:
# only add (with expensive setup time) example data when developing/testing and we're not unittesting
if isTestEnvironment() or isDevelopmentEnvironment():
from lofar.sas.tmss.tmss.exceptions import TMSSException
from lofar.sas.tmss.test.tmss_test_data_django_models import SchedulingUnitDraft_test_data
from lofar.sas.tmss.tmss.tmssapp.tasks import create_task_blueprints_and_subtasks_from_scheduling_unit_draft, create_task_blueprints_and_subtasks_and_schedule_subtasks_from_scheduling_unit_draft
from lofar.sas.tmss.tmss.tmssapp.subtasks import schedule_subtask
from lofar.common.json_utils import get_default_json_object_for_schema
constraints_template = models.SchedulingConstraintsTemplate.objects.get(name="constraints")
constraints_spec = get_default_json_object_for_schema(constraints_template.schema)
uc1_strategy_template = models.SchedulingUnitObservingStrategyTemplate.objects.get(name="UC1 CTC+pipelines")
simple_obs_strategy_template = models.SchedulingUnitObservingStrategyTemplate.objects.get(name="Simple Observation")
short_obs_pl_ingest_strategy_template = models.SchedulingUnitObservingStrategyTemplate.objects.get(name="Short Test Observation - Pipeline - Ingest")
simple_beamforming_strategy_template = models.SchedulingUnitObservingStrategyTemplate.objects.get(name="Simple Beamforming Observation")
projects = models.Project.objects.order_by('-priority_rank').all()
for tmss_project in projects:
if 'Commissioning' not in tmss_project.tags:
continue
# for test purposes also create reservation objects from all reservation strategies
for strategy_template in ReservationStrategyTemplate.objects.all():
reservation_spec = add_defaults_to_json_object_for_schema(strategy_template.template,
strategy_template.reservation_template.schema)
reservation = Reservation.objects.create(name=strategy_template.name,
description=" %s created from reservation strategy" % strategy_template.description,
project=None,
specifications_template=strategy_template.reservation_template,
specifications_doc=reservation_spec,
start_time=datetime.now()+timedelta(days=1),
stop_time=None)
logger.info('created test reservation: %s', reservation.name)
for scheduling_set in tmss_project.scheduling_sets.all():
for unit_nr in range(2):
for strategy_template in [short_obs_pl_ingest_strategy_template, simple_obs_strategy_template, simple_beamforming_strategy_template, uc1_strategy_template]:
# the 'template' in the strategy_template is a predefined json-data blob which validates against the given scheduling_unit_template
# a user might 'upload' a partial json-data blob, so add all the known defaults
scheduling_unit_spec = add_defaults_to_json_object_for_schema(strategy_template.template, strategy_template.scheduling_unit_template.schema)
# limit target obs duration for demo data
if strategy_template == uc1_strategy_template:
scheduling_unit_spec['tasks']['Calibrator Observation 1']['specifications_doc']['duration'] = 2*60
scheduling_unit_spec['tasks']['Target Observation']['specifications_doc']['duration'] = 2*3600
scheduling_unit_spec['tasks']['Calibrator Observation 2']['specifications_doc']['duration'] = 2*60
elif strategy_template == simple_obs_strategy_template:
scheduling_unit_spec['tasks']['Observation']['specifications_doc']['duration'] = 5*60
# set some constraints, so the dynamic scheduler has something to chew on.
# DISABLED for now, because the 'daily' constraint solver is not ready yet.
# constraints_spec['daily']['require_day'] = unit_nr%2==0
# constraints_spec['daily']['require_night'] = unit_nr%2==1
# constraints_spec['daily']['avoid_twilight'] = unit_nr%4>1
# add the scheduling_unit_doc to a new SchedulingUnitDraft instance, and were ready to use it!
scheduling_unit_draft = models.SchedulingUnitDraft.objects.create(name="%s %s %0d" % ('UC1' if strategy_template==uc1_strategy_template else 'Obs', tmss_project.name, unit_nr+1),
scheduling_set=scheduling_set,
description="Test scheduling unit",
requirements_template=strategy_template.scheduling_unit_template,
requirements_doc=scheduling_unit_spec,
observation_strategy_template=strategy_template,
scheduling_constraints_doc=constraints_spec,
scheduling_constraints_template=constraints_template)
logger.info('created test scheduling_unit_draft: %s', scheduling_unit_draft.name)
try:
create_task_blueprints_and_subtasks_from_scheduling_unit_draft(scheduling_unit_draft)
except TMSSException as e:
logger.exception(e)
return
except ImportError:
pass
def populate_cycles(apps, schema_editor):
# Cycle 0 deviates from any patterns
cycle = models.Cycle.objects.create(name="Cycle 00",
description="Lofar Cycle 0",
start=datetime(2013, 2, 11, 0, 0, 0, 0, tzinfo=timezone.utc),
stop=datetime(2013, 11, 14, 0, 0, 0, 0, tzinfo=timezone.utc))
models.CycleQuota.objects.bulk_create([models.CycleQuota(cycle=cycle,
resource_type=ResourceType.objects.get(
name="LOFAR Observing Time"),
value=0.8 * cycle.duration.total_seconds()),
# rough guess. 80% of total time available for observing
models.CycleQuota(cycle=cycle,
resource_type=ResourceType.objects.get(
name="CEP Processing Time"),
value=0.8 * cycle.duration.total_seconds()),
models.CycleQuota(cycle=cycle,
resource_type=ResourceType.objects.get(name="LTA Storage"),
value=0), # needs to be filled in by user (SOS)
models.CycleQuota(cycle=cycle,
resource_type=ResourceType.objects.get(
name="LOFAR Support Time"),
value=0), # needs to be filled in by user (SOS)
models.CycleQuota(cycle=cycle,
resource_type=ResourceType.objects.get(
name="LOFAR Observing Time Commissioning"),
value=0.05 * cycle.duration.total_seconds()),
# rough guess. 5% of total time available for observing
models.CycleQuota(cycle=cycle,
resource_type=ResourceType.objects.get(
name="LOFAR Observing Time prio A"),
value=0), # needs to be filled in by user (SOS)
models.CycleQuota(cycle=cycle,
resource_type=ResourceType.objects.get(
name="LOFAR Observing Time prio B"),
value=0) # needs to be filled in by user (SOS)
])
# Cycles 1-10 follow the same pattern
for nr in range(1, 11):
cycle = models.Cycle.objects.create(name="Cycle %02d" % nr,
description="Lofar Cycle %s" % nr,
start=datetime(2013+nr//2, 5 if nr%2==0 else 11, 15, 0, 0, 0, 0, tzinfo=timezone.utc),
stop=datetime(2013+(nr+1)//2, 5 if nr%2==1 else 11, 14, 0, 0, 0, 0, tzinfo=timezone.utc))
models.CycleQuota.objects.bulk_create([models.CycleQuota(cycle=cycle,
resource_type=ResourceType.objects.get(name="LOFAR Observing Time"),
value=0.8*cycle.duration.total_seconds()), # rough guess. 80% of total time available for observing
models.CycleQuota(cycle=cycle,
resource_type=ResourceType.objects.get(name="CEP Processing Time"),
value=0.8*cycle.duration.total_seconds()),
models.CycleQuota(cycle=cycle,
resource_type=ResourceType.objects.get(name="LTA Storage"),
value=0), # needs to be filled in by user (SOS)
models.CycleQuota(cycle=cycle,
resource_type=ResourceType.objects.get(name="LOFAR Support Time"),
value=0), # needs to be filled in by user (SOS)
models.CycleQuota(cycle=cycle,
resource_type=ResourceType.objects.get(name="LOFAR Observing Time Commissioning"),
value=0.05*cycle.duration.total_seconds()), # rough guess. 5% of total time available for observing
models.CycleQuota(cycle=cycle,
resource_type=ResourceType.objects.get(name="LOFAR Observing Time prio A"),
value=0), # needs to be filled in by user (SOS)
models.CycleQuota(cycle=cycle,
resource_type=ResourceType.objects.get(name="LOFAR Observing Time prio B"),
value=0) # needs to be filled in by user (SOS)
])
# Cycle 11 deviates from any patterns
cycle = models.Cycle.objects.create(name="Cycle 11",
description="Lofar Cycle 11",
start=datetime(2018, 11, 15, 0, 0, 0, 0, tzinfo=timezone.utc),
stop=datetime(2019, 5, 31, 0, 0, 0, 0, tzinfo=timezone.utc))
models.CycleQuota.objects.bulk_create([models.CycleQuota(cycle=cycle,
resource_type=ResourceType.objects.get(
name="LOFAR Observing Time"),
value=0.8 * cycle.duration.total_seconds()),
# rough guess. 80% of total time available for observing
models.CycleQuota(cycle=cycle,
resource_type=ResourceType.objects.get(
name="CEP Processing Time"),
value=0.8 * cycle.duration.total_seconds()),
models.CycleQuota(cycle=cycle,
resource_type=ResourceType.objects.get(
name="LTA Storage"),
value=0), # needs to be filled in by user (SOS)
models.CycleQuota(cycle=cycle,
resource_type=ResourceType.objects.get(
name="LOFAR Support Time"),
value=0), # needs to be filled in by user (SOS)
models.CycleQuota(cycle=cycle,
resource_type=ResourceType.objects.get(
name="LOFAR Observing Time Commissioning"),
value=0.05 * cycle.duration.total_seconds()),
# rough guess. 5% of total time available for observing
models.CycleQuota(cycle=cycle,
resource_type=ResourceType.objects.get(
name="LOFAR Observing Time prio A"),
value=0), # needs to be filled in by user (SOS)
models.CycleQuota(cycle=cycle,
resource_type=ResourceType.objects.get(
name="LOFAR Observing Time prio B"),
value=0) # needs to be filled in by user (SOS)
])
# Cycles 12-19 follow the same pattern
for nr in range(12, 20):
cycle = models.Cycle.objects.create(name="Cycle %02d" % nr,
description="Lofar Cycle %s" % nr,
start=datetime(2013 + nr // 2, 6 if nr % 2 == 0 else 12, 1, 0, 0, 0, 0,
tzinfo=timezone.utc),
stop=datetime(2013 + (nr + 1) // 2, 5 if nr % 2 == 1 else 11,
30 if nr % 2 == 0 else 31, 0, 0,
0, 0, tzinfo=timezone.utc))
models.CycleQuota.objects.bulk_create([models.CycleQuota(cycle=cycle,
resource_type=ResourceType.objects.get(
name="LOFAR Observing Time"),
value=0.8 * cycle.duration.total_seconds()),
# rough guess. 80% of total time available for observing
models.CycleQuota(cycle=cycle,
resource_type=ResourceType.objects.get(
name="CEP Processing Time"),
value=0.8 * cycle.duration.total_seconds()),
models.CycleQuota(cycle=cycle,
resource_type=ResourceType.objects.get(
name="LTA Storage"),
value=0), # needs to be filled in by user (SOS)
models.CycleQuota(cycle=cycle,
resource_type=ResourceType.objects.get(
name="LOFAR Support Time"),
value=0), # needs to be filled in by user (SOS)
models.CycleQuota(cycle=cycle,
resource_type=ResourceType.objects.get(
name="LOFAR Observing Time Commissioning"),
value=0.05 * cycle.duration.total_seconds()),
# rough guess. 5% of total time available for observing
models.CycleQuota(cycle=cycle,
resource_type=ResourceType.objects.get(
name="LOFAR Observing Time prio A"),
value=0), # needs to be filled in by user (SOS)
models.CycleQuota(cycle=cycle,
resource_type=ResourceType.objects.get(
name="LOFAR Observing Time prio B"),
value=0) # needs to be filled in by user (SOS)
])
def populate_projects(apps, schema_editor):
for name, rank in (("high", 3), ("normal", 2), ("low", 1)):
tmss_project = models.Project.objects.create(name=name,
description="Project for all TMSS tests and commissioning (%s priority)" % (name,),
priority_rank=rank,
can_trigger=name=='high',
private_data=True,
expert=True,
filler=False)
tmss_project.tags = ["Commissioning"]
tmss_project.cycles.set([models.Cycle.objects.get(name="Cycle 16")])
tmss_project.save()
# for convenience, create a schedulingset for each project
models.SchedulingSet.objects.create(name="Test Scheduling Set", project=tmss_project)
project_quota = ProjectQuota.objects.create(project=tmss_project, value=1e12, resource_type=ResourceType.objects.get(name="LTA Storage"))
sara_fs = Filesystem.objects.get(name="Lofar Storage (SARA)")
models.ProjectQuotaArchiveLocation.objects.create(project_quota=project_quota, archive_location=sara_fs)
def populate_resources(apps, schema_editor):
bytes_q = Quantity.objects.get(value=Quantity.Choices.BYTES.value)
time_q = Quantity.objects.get(value=Quantity.Choices.TIME.value)
number_q = Quantity.objects.get(value=Quantity.Choices.NUMBER.value)
ResourceType.objects.bulk_create([ResourceType(name="LTA Storage", description="Amount of storage in the LTA (in bytes)", quantity=bytes_q),
ResourceType(name="CEP Storage", description="Amount of storage on the CEP processing cluster (in bytes)", quantity=bytes_q),
ResourceType(name="CEP Processing Time", description="Processing time on the CEP processing cluster (in seconds)", quantity=time_q),
ResourceType(name="LOFAR Observing Time", description="Observing time (in seconds)", quantity=time_q),
ResourceType(name="LOFAR Observing Time prio A", description="Observing time with priority A (in seconds)", quantity=time_q),
ResourceType(name="LOFAR Observing Time prio B", description="Observing time with priority B (in seconds)", quantity=time_q),
ResourceType(name="LOFAR Observing Time Commissioning", description="Observing time for Commissioning/DDT (in seconds)", quantity=time_q),
ResourceType(name="LOFAR Support Time", description="Support time by human (in seconds)", quantity=time_q),
ResourceType(name="Number of triggers", description="Number of trigger events (as integer)", quantity=number_q) ])
def populate_misc(apps, schema_editor):
cluster = Cluster.objects.create(name="CEP4", location="CIT", archive_site=False)
fs = Filesystem.objects.create(name="LustreFS", cluster=cluster, capacity=3.6e15)
sara_cluster = Cluster.objects.create(name="SARA", location="SARA", archive_site=True)
juelich_cluster = Cluster.objects.create(name="Jülich", location="Jülich", archive_site=True)
poznan_cluster = Cluster.objects.create(name="Poznan", location="Poznan", archive_site=True)
sara_fs = Filesystem.objects.create(name="Lofar Storage (SARA)", cluster=sara_cluster, capacity=3.6e15,
directory="srm://srm.grid.sara.nl:8443/pnfs/grid.sara.nl/data/lofar/ops/projects/")
sara_test_fs = Filesystem.objects.create(name="Lofar Test Storage (SARA)", cluster=sara_cluster, capacity=3.6e15,
directory="srm://srm.grid.sara.nl:8443/pnfs/grid.sara.nl/data/lofar/ops/test/projects/")
sara_user_fs = Filesystem.objects.create(name="Lofar User Disk Storage (SARA)", cluster=sara_cluster, capacity=3.6e15,
directory="srm://srm.grid.sara.nl/pnfs/grid.sara.nl/data/lofar/user/disk/projects/")
juelich_fs = Filesystem.objects.create(name="Lofar Storage (Jülich)", cluster=juelich_cluster, capacity=3.6e15,
directory="srm://lofar-srm.fz-juelich.de:8443/pnfs/fz-juelich.de/data/lofar/ops/projects/")
pozname_fs = Filesystem.objects.create(name="Lofar Storage (Poznan)", cluster=poznan_cluster, capacity=3.6e15,
directory="srm://lta-head.lofar.psnc.pl:8443/lofar/ops/projects/")
def populate_connectors():
# the TaskConnectorType's define how the Task[Draft/Blueprint] *can* be connected.
# NOTE: This is an explicit list of each possible link between tasks. This model suffices
# until the number of connectors throw too large. By then, we could consider introducing
# wild cards, like output_of=NULL meaning "any".
logger.info("Populating TaskConnectorType's")
# calibrator observation
TaskConnectorType.objects.create(role=Role.objects.get(value=Role.Choices.CORRELATOR.value),
datatype=Datatype.objects.get(value=Datatype.Choices.VISIBILITIES.value),
dataformat=Dataformat.objects.get(value=Dataformat.Choices.MEASUREMENTSET.value),
task_template=TaskTemplate.objects.get(name='calibrator observation'),
iotype=IOType.objects.get(value=IOType.Choices.OUTPUT.value))
# target observation
TaskConnectorType.objects.create(role=Role.objects.get(value=Role.Choices.CORRELATOR.value),
datatype=Datatype.objects.get(value=Datatype.Choices.VISIBILITIES.value),
dataformat=Dataformat.objects.get(value=Dataformat.Choices.MEASUREMENTSET.value),
task_template=TaskTemplate.objects.get(name='target observation'),
iotype=IOType.objects.get(value=IOType.Choices.OUTPUT.value))
# beamforming observation
TaskConnectorType.objects.create(role=Role.objects.get(value=Role.Choices.BEAMFORMER.value),
datatype=Datatype.objects.get(value=Datatype.Choices.TIME_SERIES.value),
dataformat=Dataformat.objects.get(value=Dataformat.Choices.BEAMFORMED.value),
task_template=TaskTemplate.objects.get(name='beamforming observation'),
iotype=IOType.objects.get(value=IOType.Choices.OUTPUT.value))
# pulsar pipeline
TaskConnectorType.objects.create(role=Role.objects.get(value=Role.Choices.BEAMFORMER.value),
datatype=Datatype.objects.get(value=Datatype.Choices.TIME_SERIES.value),
dataformat=Dataformat.objects.get(value=Dataformat.Choices.BEAMFORMED.value),
task_template=TaskTemplate.objects.get(name='pulsar pipeline'),
iotype=IOType.objects.get(value=IOType.Choices.INPUT.value))
TaskConnectorType.objects.create(role=Role.objects.get(value=Role.Choices.ANY.value),
datatype=Datatype.objects.get(value=Datatype.Choices.QUALITY.value),
dataformat=Dataformat.objects.get(value=Dataformat.Choices.PULP_SUMMARY.value),
task_template=TaskTemplate.objects.get(name='pulsar pipeline'),
iotype=IOType.objects.get(value=IOType.Choices.OUTPUT.value))
TaskConnectorType.objects.create(role=Role.objects.get(value=Role.Choices.ANY.value),
datatype=Datatype.objects.get(value=Datatype.Choices.PULSAR_PROFILE.value),
dataformat=Dataformat.objects.get(value=Dataformat.Choices.PULP_ANALYSIS.value),
task_template=TaskTemplate.objects.get(name='pulsar pipeline'),
iotype=IOType.objects.get(value=IOType.Choices.OUTPUT.value))
# preprocessing pipeline
for iotype_value in (IOType.Choices.INPUT.value, IOType.Choices.OUTPUT.value):
TaskConnectorType.objects.create(role=Role.objects.get(value=Role.Choices.ANY.value),
datatype=Datatype.objects.get(value=Datatype.Choices.VISIBILITIES.value),
dataformat=Dataformat.objects.get(value=Dataformat.Choices.MEASUREMENTSET.value),
task_template=TaskTemplate.objects.get(name='preprocessing pipeline'),
iotype=IOType.objects.get(value=iotype_value))
# Ingest and Cleanup can/should accept all kinds of data.
# So we could loop over all combinations of Datatype, Dataformat and Role and create an input connector for each.
# This would result however in "unrealistic"/non-existing types like: TIME_SERIES-MEASUREMENTSET, or VISIBILITIES-BEAMFORMED, etc, which do not make any sense.
# So, instead, lets loop over all exising output connectors, and accept those as input.
for task_template_name in ('ingest', 'cleanup'):
task_template = TaskTemplate.objects.get(name=task_template_name)
# loop over all existing output types
any_role = Role.objects.get(value=Role.Choices.ANY.value)
for output_connector_type in TaskConnectorType.objects.filter(iotype=IOType.objects.get(value=IOType.Choices.OUTPUT.value)).all():
# always create two input connectors for the specific output_connector_type.role and the any_role
for role in [output_connector_type.role, any_role]:
try:
connector = TaskConnectorType.objects.create(role=role,
datatype=output_connector_type.datatype,
dataformat=output_connector_type.dataformat,
task_template=task_template,
iotype=IOType.objects.get(value=IOType.Choices.INPUT.value))
except IntegrityError:
# we just tried to create a duplicate... It's ok to silently continue...
pass
def populate_permissions():
logger.info('Populating permissions')
populate_project_permissions()
populate_system_permissions()
populate_system_roles()
populate_system_test_users()
def populate_project_permissions():
# For each viewset and for each extra action create a project permission entry.
for name, obj in inspect.getmembers(viewsets):
if inspect.isclass(obj):
try:
permission_name = obj.serializer_class.Meta.model.__name__.lower()
logger.info('creating project permission %s' % permission_name)
try:
ProjectPermission.objects.create(name=permission_name)
except IntegrityError as e:
logger.debug('Skipping project permission creation for obj=%s: %s' % (obj, e))
extra_actions = obj.get_extra_actions()
if extra_actions:
for action in extra_actions:
action_permission_name = '%s-%s' % (permission_name, action.__name__)
logger.info('creating project permission %s' % action_permission_name)
try:
ProjectPermission.objects.create(name=action_permission_name)
except IntegrityError as e:
logger.debug('Skipping project permission creation for obj=%s: %s' % (obj, e))
except Exception as e:
logger.debug('Skipping project permission creation for obj=%s: %s' % (obj, e))
# Then Assign initial permission matrix
#
def _get_all_project_permissions_for_viewset(view):
name = view.serializer_class.Meta.model.__name__.lower()
# permission for the model itself
perms = [ProjectPermission.objects.get(name=name)]
# permissions for all extra actions
extra_actions = view.get_extra_actions()
if extra_actions:
for action in extra_actions:
action_permission_name = '%s-%s' % (name, action.__name__)
perms.append(ProjectPermission.objects.get(name=action_permission_name))
return perms
# system settings
for view in permission_groups['system_setting']:
perms = _get_all_project_permissions_for_viewset(view)
for perm in perms:
perm.GET.set([ProjectRole.objects.get(value=role) for role in ['pi', 'co_i', 'contact', 'shared_support', 'friend_of_project', 'friend_of_project_primary']])
# Note: we only grant GET permissions, since other methods are only permitted to users who are superuser anyway.
# system feedback
for view in permission_groups['system_feedback']:
perms = _get_all_project_permissions_for_viewset(view)
for perm in perms:
perm.GET.set([ProjectRole.objects.get(value=role) for role in ['pi', 'co_i', 'contact', 'shared_support', 'friend_of_project', 'friend_of_project_primary']])
# Note: we only grant GET permissions, since other methods are only permitted to users who are superuser anyway.
# cycle administration
for view in permission_groups['cycle_administration']:
perms = _get_all_project_permissions_for_viewset(view)
for perm in perms:
perm.GET.set([ProjectRole.objects.get(value=role) for role in ['pi', 'co_i', 'contact', 'shared_support', 'friend_of_project', 'friend_of_project_primary']])
# Note: we only grant GET permissions, since other methods are only permitted to users with particular system roles.
# project administration
for view in permission_groups['project_administration']:
perms = _get_all_project_permissions_for_viewset(view)
for perm in perms:
perm.GET.set([ProjectRole.objects.get(value=role) for role in ['pi', 'co_i', 'contact', 'shared_support', 'friend_of_project', 'friend_of_project_primary']])
perm.POST.set([ProjectRole.objects.get(value=role) for role in ['shared_support']])
perm.PUT.set([ProjectRole.objects.get(value=role) for role in ['shared_support']])
perm.PATCH.set([ProjectRole.objects.get(value=role) for role in ['shared_support']])
perm.DELETE.set([ProjectRole.objects.get(value=role) for role in ['shared_support']])
# blueprint alteration
for view in permission_groups['blueprint_alteration']:
perms = _get_all_project_permissions_for_viewset(view)
for perm in perms:
perm.GET.set([ProjectRole.objects.get(value=role) for role in ['pi', 'co_i', 'contact', 'shared_support', 'friend_of_project', 'friend_of_project_primary']])
perm.POST.set([ProjectRole.objects.get(value=role) for role in ['shared_support']])
perm.PUT.set([ProjectRole.objects.get(value=role) for role in ['shared_support']])
perm.PATCH.set([ProjectRole.objects.get(value=role) for role in ['shared_support']])
perm.DELETE.set([ProjectRole.objects.get(value=role) for role in ['shared_support']])
# blueprint fixed
for view in permission_groups['blueprint_fixed']:
perms = _get_all_project_permissions_for_viewset(view)
for perm in perms:
perm.GET.set([ProjectRole.objects.get(value=role) for role in ['pi', 'co_i', 'contact', 'shared_support', 'friend_of_project', 'friend_of_project_primary']])
# Note: we only grant GET permissions, since other methods are only permitted to users who are superuser anyway.
# operational
for view in permission_groups['operational']:
perms = _get_all_project_permissions_for_viewset(view)
for perm in perms:
perm.GET.set([ProjectRole.objects.get(value=role) for role in ['pi', 'co_i', 'contact', 'shared_support', 'friend_of_project', 'friend_of_project_primary']])
# Note: we only grant GET permissions, since other methods are only permitted to users who are superuser anyway.
def populate_system_permissions():
# For each viewset create custom permissions for extra actions.
for name, obj in inspect.getmembers(viewsets):
if inspect.isclass(obj):
try:
ct = ContentType.objects.get_for_model(obj.serializer_class.Meta.model)
extra_actions = obj.get_extra_actions()
if extra_actions:
for action in extra_actions:
codename = ("%s_%s" % (action.__name__, obj.serializer_class.Meta.model.__name__)).lower()
name = f'Can {action.__name__} {obj.serializer_class.Meta.model.__name__.lower()}'
Permission.objects.create(codename=codename, name=name, content_type=ct)
except:
pass
def populate_system_roles():
operator_group = Group.objects.create(name='operator')
support_group = Group.objects.create(name='support')
developer_group = Group.objects.create(name='developer')
admin_permission_group = Group.objects.create(name='admin')
maintenance_group = Group.objects.create(name='maintenance')
scientist_group = Group.objects.create(name='scientist')
assign_system_permissions()
def assign_system_permissions():
'''
Assign system permission to each system role, accordingly.
'''
# Get system roles
operator_group = Group.objects.get(name='operator')
support_group = Group.objects.get(name='support')
developer_group = Group.objects.get(name='developer')
admin_group = Group.objects.get(name='admin')
maintenance_group = Group.objects.get(name='maintenance')
scientist_group = Group.objects.get(name='scientist')
# system settings
for view in permission_groups['system_setting']:
name = view.serializer_class.Meta.model.__name__.lower()
perm = Permission.objects.get(codename='view_%s' % name)
operator_group.permissions.add(perm)
support_group.permissions.add(perm)
developer_group.permissions.add(perm)
admin_group.permissions.add(perm)
maintenance_group.permissions.add(perm)
scientist_group.permissions.add(perm)
extra_actions = view.get_extra_actions()
if extra_actions:
for action in extra_actions:
perm = Permission.objects.get(codename='%s_%s' % (action.__name__, name))
if 'get' in action.mapping:
operator_group.permissions.add(perm)
support_group.permissions.add(perm)
developer_group.permissions.add(perm)
admin_group.permissions.add(perm)
maintenance_group.permissions.add(perm)
scientist_group.permissions.add(perm)
# Note: we only grant view permissions, since alteration is only permitted to users who are superuser anyway.
# system feedback
for view in permission_groups['system_feedback']:
name = view.serializer_class.Meta.model.__name__.lower()
perm = Permission.objects.get(codename='view_%s' % name)
operator_group.permissions.add(perm)
support_group.permissions.add(perm)
developer_group.permissions.add(perm)
admin_group.permissions.add(perm)
maintenance_group.permissions.add(perm)
scientist_group.permissions.add(perm)
extra_actions = view.get_extra_actions()
if extra_actions:
for action in extra_actions:
perm = Permission.objects.get(codename='%s_%s' % (action.__name__, name))
if 'get' in action.mapping:
operator_group.permissions.add(perm)
support_group.permissions.add(perm)
developer_group.permissions.add(perm)
admin_group.permissions.add(perm)
maintenance_group.permissions.add(perm)
scientist_group.permissions.add(perm)
# Note: we only grant view permissions, since alteration is only permitted to users who are superuser anyway.
# cycle administration
for view in permission_groups['cycle_administration']:
name = view.serializer_class.Meta.model.__name__.lower()
perm = Permission.objects.get(codename='view_%s' % name)
operator_group.permissions.add(perm)
support_group.permissions.add(perm)
developer_group.permissions.add(perm)
admin_group.permissions.add(perm)
maintenance_group.permissions.add(perm)
scientist_group.permissions.add(perm)
extra_actions = view.get_extra_actions()
if extra_actions:
for action in extra_actions:
perm = Permission.objects.get(codename='%s_%s' % (action.__name__, name))
if 'get' in action.mapping:
operator_group.permissions.add(perm)
support_group.permissions.add(perm)
developer_group.permissions.add(perm)
admin_group.permissions.add(perm)
maintenance_group.permissions.add(perm)
scientist_group.permissions.add(perm)
perm = Permission.objects.get(codename='change_%s' % name)
operator_group.permissions.add(perm)
support_group.permissions.add(perm)
perm = Permission.objects.get(codename='add_%s' % name)
operator_group.permissions.add(perm)
support_group.permissions.add(perm)
perm = Permission.objects.get(codename='delete_%s' % name)
operator_group.permissions.add(perm)
support_group.permissions.add(perm) # todo: confirm
if extra_actions:
for action in extra_actions:
perm = Permission.objects.get(codename='%s_%s' % (action.__name__, name))
if 'post' in action.mapping:
operator_group.permissions.add(perm)
support_group.permissions.add(perm)
# project administration
for view in permission_groups['project_administration']:
name = view.serializer_class.Meta.model.__name__.lower()
perm = Permission.objects.get(codename='view_%s' % name)
operator_group.permissions.add(perm)
support_group.permissions.add(perm)
developer_group.permissions.add(perm)
admin_group.permissions.add(perm)
maintenance_group.permissions.add(perm)
scientist_group.permissions.add(perm)
extra_actions = view.get_extra_actions()
if extra_actions:
for action in extra_actions:
perm = Permission.objects.get(codename='%s_%s' % (action.__name__, name))
if 'get' in action.mapping:
operator_group.permissions.add(perm)
support_group.permissions.add(perm)
developer_group.permissions.add(perm)
admin_group.permissions.add(perm)
maintenance_group.permissions.add(perm)
scientist_group.permissions.add(perm)
perm = Permission.objects.get(codename='change_%s' % name)
operator_group.permissions.add(perm)
support_group.permissions.add(perm)
perm = Permission.objects.get(codename='add_%s' % name)
operator_group.permissions.add(perm)
support_group.permissions.add(perm)
perm = Permission.objects.get(codename='delete_%s' % name)
operator_group.permissions.add(perm)
support_group.permissions.add(perm)
if extra_actions:
for action in extra_actions:
perm = Permission.objects.get(codename='%s_%s' % (action.__name__, name))
if 'post' in action.mapping:
operator_group.permissions.add(perm)
support_group.permissions.add(perm)
# blueprint alteration
for view in permission_groups['blueprint_alteration']:
name = view.serializer_class.Meta.model.__name__.lower()
perm = Permission.objects.get(codename='view_%s' % name)
operator_group.permissions.add(perm)
support_group.permissions.add(perm)
developer_group.permissions.add(perm)
admin_group.permissions.add(perm)
maintenance_group.permissions.add(perm)
scientist_group.permissions.add(perm)
extra_actions = view.get_extra_actions()
if extra_actions:
for action in extra_actions:
perm = Permission.objects.get(codename='%s_%s' % (action.__name__, name))
if 'get' in action.mapping:
operator_group.permissions.add(perm)
support_group.permissions.add(perm)
developer_group.permissions.add(perm)
admin_group.permissions.add(perm)
maintenance_group.permissions.add(perm)
scientist_group.permissions.add(perm)
perm = Permission.objects.get(codename='change_%s' % name) # todo: allow 'add' to same groups?
operator_group.permissions.add(perm)
support_group.permissions.add(perm)
perm = Permission.objects.get(codename='add_%s' % name)
operator_group.permissions.add(perm)
support_group.permissions.add(perm)
perm = Permission.objects.get(codename='delete_%s' % name)
operator_group.permissions.add(perm)
support_group.permissions.add(perm)
if extra_actions:
for action in extra_actions:
perm = Permission.objects.get(codename='%s_%s' % (action.__name__, name))
if 'post' in action.mapping:
operator_group.permissions.add(perm)
support_group.permissions.add(perm)
# blueprint fixed
for view in permission_groups['blueprint_fixed']:
name = view.serializer_class.Meta.model.__name__.lower()
perm = Permission.objects.get(codename='view_%s' % name)
operator_group.permissions.add(perm)
support_group.permissions.add(perm)
developer_group.permissions.add(perm)
admin_group.permissions.add(perm)
maintenance_group.permissions.add(perm)
scientist_group.permissions.add(perm)
# Note: we only grant view permissions, since alteration is only permitted to users who are superuser anyway.
# operational
for view in permission_groups['operational']:
name = view.serializer_class.Meta.model.__name__.lower()
perm = Permission.objects.get(codename='view_%s' % name)
operator_group.permissions.add(perm)
support_group.permissions.add(perm)
developer_group.permissions.add(perm)
admin_group.permissions.add(perm)
maintenance_group.permissions.add(perm)
scientist_group.permissions.add(perm)
extra_actions = view.get_extra_actions()
if extra_actions:
for action in extra_actions:
perm = Permission.objects.get(codename='%s_%s' % (action.__name__, name))
if 'get' in action.mapping:
operator_group.permissions.add(perm)
support_group.permissions.add(perm)
developer_group.permissions.add(perm)
admin_group.permissions.add(perm)
maintenance_group.permissions.add(perm)
scientist_group.permissions.add(perm)
perm = Permission.objects.get(codename='change_%s' % name)
operator_group.permissions.add(perm)
perm = Permission.objects.get(codename='add_%s' % name)
operator_group.permissions.add(perm)
perm = Permission.objects.get(codename='delete_%s' % name)
operator_group.permissions.add(perm)
if extra_actions:
for action in extra_actions:
perm = Permission.objects.get(codename='%s_%s' % (action.__name__, name))
if 'post' in action.mapping:
operator_group.permissions.add(perm)
def populate_system_test_users():
# TODO: Set proper credentials (passwords at least) or remove before we hit production
operator_user, _ = User.objects.get_or_create(username='operator', password='operator')
operator_user.groups.add(Group.objects.get(name='operator'))
support_user, _ = User.objects.get_or_create(username='support', password='support')
support_user.groups.add(Group.objects.get(name='support'))
developer_user, _ = User.objects.get_or_create(username='developer', password='developer')
developer_user.groups.add(Group.objects.get(name='developer'))
admin_user, _ = User.objects.get_or_create(username='admin', password='admin')
admin_user.groups.add(Group.objects.get(name='admin'))
maintenance_user, _ = User.objects.get_or_create(username='maintenance', password='maintenance')
maintenance_user.groups.add(Group.objects.get(name='maintenance'))
scientist_user, _ = User.objects.get_or_create(username='scientist', password='scientist')
scientist_user.groups.add(Group.objects.get(name='scientist'))
def populate_sunrise_and_sunset_for_all_stations(nbr_days=3, start_date=date.today()):
"""
Populate station timeline data of all stations for given number of days the starting at given date
Note: If data is not in database yet, it will take about 6 seconds to calculate it for all (51) stations
"""
starttime_for_logging = datetime.utcnow()
logger.info("Populate sunrise and sunset for ALL known stations from %s up to %d days" % (start_date, nbr_days))
lst_timestamps = []
for i in range(0, nbr_days):
dt = datetime.combine(start_date, datetime.min.time()) + timedelta(days=i)
lst_timestamps.append(dt)
timestamps_and_stations_to_sun_rise_and_set(tuple(lst_timestamps), tuple(get_all_stations()), create_when_not_found=True)
logger.info("Populate sunrise and sunset done in %.1fs", (datetime.utcnow()-starttime_for_logging).total_seconds())