diff --git a/LTA/sip/lib/station_coordinates.conf b/LTA/sip/lib/station_coordinates.conf
index 07e488f9a72ccf960c6e6f30c9bc39823e3c7613..741cd1395f2a6a362e70335b7c97d0ac383eb746 100644
--- a/LTA/sip/lib/station_coordinates.conf
+++ b/LTA/sip/lib/station_coordinates.conf
@@ -154,3 +154,9 @@ coordinate_system='ITRF2005', x='3850973.9872', y='1439061.04111', z='4860478.99
coordinate_system='ITRF2005', x='3850980.8812', y='1438994.87911', z='4860498.993' , name='PL611_HBA'
coordinate_system='ITRF2005', x='3551478.64311', y='1334128.4928', z='5110179.160' , name='PL612_LBA'
coordinate_system='ITRF2005', x='3551481.8171', y='1334203.5728', z='5110157.410' , name='PL612_HBA'
+
+coordinate_system='ITRF2005', x='3801633.528060000', y='-529021.899396000', z='5076997.185' , name='IE613_LBA'
+coordinate_system='ITRF2005', x='3801691.943300000', y='-528983.966429000', z='5076957.924' , name='IE613_HBA'
+
+coordinate_system='ITRF2005', x='3183318.032280000', y='1276777.654760000', z='5359435.077' , name='LV614_LBA'
+coordinate_system='ITRF2005', x='3183249.285620000', y='1276801.742170000', z='5359469.949' , name='LV614_HBA'
\ No newline at end of file
diff --git a/SAS/TMSS/src/tmss/tmssapp/adapters/sip.py b/SAS/TMSS/src/tmss/tmssapp/adapters/sip.py
index 50e2a205555195be071027cdc0894164c2726335..49c160a383e56cb9b54f6c44ddc0e0e400a71ad9 100644
--- a/SAS/TMSS/src/tmss/tmssapp/adapters/sip.py
+++ b/SAS/TMSS/src/tmss/tmssapp/adapters/sip.py
@@ -31,6 +31,7 @@ mapping_filterset_type_TMSS_2_SIP = {
"HBA_210_250": constants.FILTERSELECTIONTYPE_210_250_MHZ
}
+
def get_number_of_dataproducts_of_type(subtask, dataproduct_datatype):
"""
Retrieve the number of dataproducts of given data type in subtask:
@@ -42,7 +43,7 @@ def get_number_of_dataproducts_of_type(subtask, dataproduct_datatype):
subtask_outputs = list(SubtaskOutput.objects.filter(subtask_id=subtask.id))
for subtask_output in subtask_outputs:
dataproducts = list(Dataproduct.objects.filter(producer_id=subtask_output.id,
- dataformat=dataproduct_datatype))
+ datatype=dataproduct_datatype))
nbr_dataproduct += len(dataproducts)
return nbr_dataproduct
@@ -248,8 +249,7 @@ def create_sip_representation_for_dataproduct(dataproduct: Dataproduct):
"unknown": constants.STORAGEWRITERTYPE_UNKNOWN}
try:
- # todo: why is this not with the data but with the connector? The type of data should not depend on what it is used for? I don't get it... - what if we have several connectors?
- dataproduct_type = type_map[dataproduct.producer.subtask.task_blueprint.specifications_template.output_connector_types.first().datatype.value] # todo: does not work on the main dataproduct?
+ dataproduct_type = type_map[dataproduct.datatype.value]
except Exception as err:
dataproduct_type = constants.DATAPRODUCTTYPE_UNKNOWN
logger.warning("Could not determine the type of dataproduct id %s (%s). Falling back to %s" % (dataproduct.id, err, dataproduct_type))
diff --git a/SAS/TMSS/src/tmss/tmssapp/migrations/0001_initial.py b/SAS/TMSS/src/tmss/tmssapp/migrations/0001_initial.py
index 607273e7c9f438c01d81c5a90d077e7e79b3bd95..5284783ea0c1845ae9a9feaf1d890c7d0797e1da 100644
--- a/SAS/TMSS/src/tmss/tmssapp/migrations/0001_initial.py
+++ b/SAS/TMSS/src/tmss/tmssapp/migrations/0001_initial.py
@@ -1136,6 +1136,11 @@ class Migration(migrations.Migration):
name='dataformat',
field=models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='tmssapp.Dataformat'),
),
+ migrations.AddField(
+ model_name='dataproduct',
+ name='datatype',
+ field=models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='tmssapp.Datatype'),
+ ),
migrations.AddField(
model_name='dataproduct',
name='feedback_template',
diff --git a/SAS/TMSS/src/tmss/tmssapp/models/scheduling.py b/SAS/TMSS/src/tmss/tmssapp/models/scheduling.py
index 4ac8634bab3ccf3a644f423d5fca7330ef387a2f..dbbc9ba3248624ef7b310e56075583387a86f5aa 100644
--- a/SAS/TMSS/src/tmss/tmssapp/models/scheduling.py
+++ b/SAS/TMSS/src/tmss/tmssapp/models/scheduling.py
@@ -269,6 +269,7 @@ class Dataproduct(BasicCommon):
filename = CharField(max_length=128, help_text='Name of the file (or top-level directory) of the dataproduct. Adheres to a naming convention, but is not meant for parsing.')
directory = CharField(max_length=1024, help_text='Directory where this dataproduct is (to be) stored.')
dataformat = ForeignKey('Dataformat', null=False, on_delete=PROTECT)
+ datatype = ForeignKey('Datatype', null=False, on_delete=PROTECT)
deleted_since = DateTimeField(null=True, help_text='When this dataproduct was removed from disk, or NULL if not deleted (NULLable).')
pinned_since = DateTimeField(null=True, help_text='When this dataproduct was pinned to disk, that is, forbidden to be removed, or NULL if not pinned (NULLable).')
specifications_doc = JSONField(help_text='Dataproduct properties (f.e. beam, subband), to distinguish them when produced by the same task, and to act as input for selections in the Task Input and Work Request Relation Blueprint objects.')
diff --git a/SAS/TMSS/src/tmss/tmssapp/subtasks.py b/SAS/TMSS/src/tmss/tmssapp/subtasks.py
index ddb337c7407fb40f5f3db99fee0da27c1a6856b4..6a5db732c5e58f98caf51656155e9762bcfca881 100644
--- a/SAS/TMSS/src/tmss/tmssapp/subtasks.py
+++ b/SAS/TMSS/src/tmss/tmssapp/subtasks.py
@@ -546,6 +546,7 @@ def schedule_qafile_subtask(qafile_subtask: Subtask):
qafile_subtask_dataproduct = Dataproduct.objects.create(filename="L%s_QA.h5" % (qa_input.producer.subtask_id, ),
directory="/data/qa/qa_files",
dataformat=Dataformat.objects.get(value=Dataformat.Choices.QA_HDF5.value),
+ datatype=Datatype.objects.get(value=Datatype.Choices.QUALITY.value), # todo: is this correct?
producer=qafile_subtask.outputs.first(),
specifications_doc="",
specifications_template=DataproductSpecificationsTemplate.objects.first(), # ?????
@@ -596,6 +597,7 @@ def schedule_qaplots_subtask(qaplots_subtask: Subtask):
obs_subtask = qafile_subtask.predecessors.first()
qaplots_subtask_dataproduct = Dataproduct.objects.create(directory="/data/qa/plots/L%s" % (obs_subtask.id, ),
dataformat=Dataformat.objects.get(value=Dataformat.Choices.QA_PLOTS.value),
+ datatype=Datatype.objects.get(value=Datatype.Choices.QUALITY.value), # todo: is this correct?
producer=qaplots_subtask.outputs.first(),
specifications_doc="",
specifications_template=DataproductSpecificationsTemplate.objects.first(), # ?????
@@ -715,6 +717,7 @@ def schedule_observation_subtask(observation_subtask: Subtask):
Dataproduct.objects.create(filename="L%d_SAP%03d_SB%03d_uv.MS" % (observation_subtask.id, sap_nr, sb_nr),
directory=directory,
dataformat=Dataformat.objects.get(value="MeasurementSet"),
+ datatype=Datatype.objects.get(value="visibilities"), # todo: is this correct?
producer=subtask_output,
specifications_doc={"sap": [sap_nr]}, # todo: set correct value. This will be provided by the RA somehow
specifications_template=dataproduct_specifications_template,
@@ -795,6 +798,7 @@ def schedule_pipeline_subtask(pipeline_subtask: Subtask):
output_dp = Dataproduct.objects.create(filename=filename,
directory=input_dp.directory.replace(str(pipeline_subtask_input.producer.subtask.pk), str(pipeline_subtask.pk)),
dataformat=Dataformat.objects.get(value="MeasurementSet"),
+ datatype=Datatype.objects.get(value="visibilities"), # todo: is this correct?
producer=pipeline_subtask_output,
specifications_doc={},
specifications_template=dataproduct_specifications_template,
diff --git a/SAS/TMSS/test/tmss_test_data_django_models.py b/SAS/TMSS/test/tmss_test_data_django_models.py
index dd093be160512794fd2c8a7025d4e8f6d0e2b5cf..3242c7e89f2f990fd1162c002c6c8488ccce787d 100644
--- a/SAS/TMSS/test/tmss_test_data_django_models.py
+++ b/SAS/TMSS/test/tmss_test_data_django_models.py
@@ -389,6 +389,7 @@ def Dataproduct_test_data(producer: models.SubtaskOutput=None,
filename: str="my_file.ext",
directory: str="/data/test-projects",
dataformat: models.Dataformat=None,
+ datatype: models.Datatype=None,
specifications_doc: object=None) -> dict:
if producer is None:
@@ -397,12 +398,16 @@ def Dataproduct_test_data(producer: models.SubtaskOutput=None,
if dataformat is None:
dataformat = models.Dataformat.objects.get(value="MeasurementSet")
+ if datatype is None:
+ datatype = models.Datatype.objects.get(value="visibilities")
+
if specifications_doc is None:
specifications_doc={}
return {"filename": filename,
"directory": directory,
"dataformat": dataformat,
+ "datatype": datatype,
"deleted_since": None,
"pinned_since": None,
"specifications_doc": specifications_doc,
diff --git a/SAS/TMSS/test/tmss_test_data_rest.py b/SAS/TMSS/test/tmss_test_data_rest.py
index d919fbbcc46cddd25b80ccc6e091b43802775c64..eaa5801124ac19072048d948a5b898bd9f0b36ca 100644
--- a/SAS/TMSS/test/tmss_test_data_rest.py
+++ b/SAS/TMSS/test/tmss_test_data_rest.py
@@ -480,7 +480,7 @@ class TMSSRESTTestDataCreator():
return {"subtask": subtask_url,
"tags": []}
- def Dataproduct(self, filename="my_filename", directory="/tmp/", specifications_template_url=None, subtask_output_url=None, dataproduct_feedback_template_url=None, dataformat="MeasurementSet"):
+ def Dataproduct(self, filename="my_filename", directory="/tmp/", specifications_template_url=None, subtask_output_url=None, dataproduct_feedback_template_url=None, dataformat="MeasurementSet", datatype="visibilities"):
if specifications_template_url is None:
specifications_template_url = self.post_data_and_get_url(self.SubtaskTemplate(), '/dataproduct_specifications_template/')
@@ -493,6 +493,7 @@ class TMSSRESTTestDataCreator():
return {"filename": filename,
"directory": directory,
"dataformat": "%s/dataformat/%s" % (self.django_api_url, dataformat),
+ "datatype": "%s/datatype/%s" % (self.django_api_url, datatype),
"deleted_since": None,
"pinned_since": None,
"specifications_doc": "{}",