diff --git a/.gitattributes b/.gitattributes
index b61af3cd41b448dfb07648b53eb4a0a025281449..cbb30e06f2f61e68e960bde55893c839c482cbd7 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -11,9 +11,9 @@ CAL/CalibrationCommon/lib/__init__.py -text
CAL/CalibrationCommon/lib/coordinates.py -text
CAL/CalibrationCommon/lib/datacontainers/__init__.py -text
CAL/CalibrationCommon/lib/datacontainers/holography_dataset.py -text
+CAL/CalibrationCommon/lib/datacontainers/holography_measurementset.py -text
CAL/CalibrationCommon/lib/datacontainers/holography_observation.py -text
CAL/CalibrationCommon/lib/datacontainers/holography_specification.py -text
-CAL/CalibrationCommon/lib/datacontainers/measurementset.py -text
CAL/CalibrationCommon/lib/mshologextract.py -text
CAL/CalibrationCommon/lib/utils.py -text
CAL/CalibrationCommon/test/CMakeLists.txt -text
diff --git a/CAL/CalibrationCommon/lib/datacontainers/__init__.py b/CAL/CalibrationCommon/lib/datacontainers/__init__.py
index dec6722fd9dc2c26c8d948c9aa9df9950bb40576..daf421e1e2655a92b721197ec4d6913f01d95888 100644
--- a/CAL/CalibrationCommon/lib/datacontainers/__init__.py
+++ b/CAL/CalibrationCommon/lib/datacontainers/__init__.py
@@ -1,8 +1,8 @@
-from .holography_dataset import HolographyDataset
from .holography_specification import HolographySpecification
from .holography_observation import HolographyObservation
-from .measurementset import MeasurementSet
+from .holography_measurementset import HolographyMeasurementSet
+from .holography_dataset import HolographyDataset
-__all__ = [HolographyDataset, HolographyObservation, HolographySpecification, MeasurementSet]
\ No newline at end of file
+__all__ = [HolographyDataset, HolographyObservation, HolographySpecification, HolographyMeasurementSet]
\ No newline at end of file
diff --git a/CAL/CalibrationCommon/lib/datacontainers/holography_dataset.py b/CAL/CalibrationCommon/lib/datacontainers/holography_dataset.py
index 621e0468850ffcfd6795e2d17e96d20645c359cc..2d869071728b081059d7b1c6b48b8ce89d9b1dd1 100644
--- a/CAL/CalibrationCommon/lib/datacontainers/holography_dataset.py
+++ b/CAL/CalibrationCommon/lib/datacontainers/holography_dataset.py
@@ -1,6 +1,8 @@
from .holography_specification import HolographySpecification
-from .holography_observation import HolographyObservation
+from lofar.calibration.common.datacontainers.holography_observation import HolographyObservation
+import logging
+logger = logging.getLogger(__name__)
class HolographyDataset():
def __init__(self):
@@ -50,16 +52,20 @@ class HolographyDataset():
mode = set()
source_name = set()
source_position = set()
-
+ target_stations = set()
for hbs, ho in list_of_hbs_ho_tuples:
- if station_name in hbs.target_station_names:
+ target_stations.update(hbs.target_station_names)
+
+ if station_name in hbs.target_station_names:
beam_specifications = hbs.get_beam_specifications_per_station_name(station_name)
for beam_specification in beam_specifications:
self.rcu_list.update(beam_specification.rcus_involved)
- mode.update(beam_specification.rcus_mode)
- source_name.update(ho.source_name)
- source_position.update(
+
+ mode.add(beam_specification.rcus_mode)
+
+ source_name.add(ho.source_name)
+ source_position.add(
(beam_specification.station_pointing['ra'],
beam_specification.station_pointing['dec'],
beam_specification.station_pointing['coordinate_system']
@@ -67,29 +73,52 @@ class HolographyDataset():
self.start_time = ho.start_mjd
self.end_time = ho.end_mjd
- self.frequencies = ho.frequency
- self.sas_ids.update(ho.sas_id)
+ self.frequencies.add(ho.frequency)
+ self.sas_ids.add(ho.sas_id)
self.target_station_name = station_name
self.reference_stations.update(hbs.reference_station_names)
+
+
+
else:
continue
+ # reads the target station position and the coordinate of its axes
+ # and does this only once since the coordinate will not change
+ first_holography_observation = list_of_hbs_ho_tuples[0][1]
+ first_ms = first_holography_observation.ms_for_a_given_beamlet_number.values()[0]
+ station_position, tile_offset, axes_coordinates = first_ms.\
+ get_station_position_tile_offsets_and_axes_coordinate_for_station_name(
+ station_name)
+
+ self.antenna_field_position = [list(station_position - antenna_offset)
+ for antenna_offset in tile_offset]
+ self.target_station_position = list(station_position)
+ self.rotation_matrix = axes_coordinates
+
+ if station_name not in target_stations:
+ logger.error('Station %s was not involved in the observation.'
+ ' The target stations for this observation are %s',
+ station_name, target_stations)
+ raise Exception('Station %s was not involved in the observation.'
+ % station_name,)
if len(mode) > 1:
raise ValueError('Multiple RCUs mode are not supported')
else:
- self.mode = self.mode.pop()
+ self.mode = mode.pop()
if len(source_position) > 1:
+ logger.error('Multiple source positions are not supported: %s', source_position)
raise ValueError('Multiple source positions are not supported')
else:
- self.source_position = source_position
+ self.source_position = source_position.pop()
if len(source_name) > 1:
raise ValueError('Multiple source name are not supported')
else:
- self.source_name = source_name
+ self.source_name = source_name.pop()
@staticmethod
def load_from_file(path):
diff --git a/CAL/CalibrationCommon/lib/datacontainers/measurementset.py b/CAL/CalibrationCommon/lib/datacontainers/holography_measurementset.py
similarity index 75%
rename from CAL/CalibrationCommon/lib/datacontainers/measurementset.py
rename to CAL/CalibrationCommon/lib/datacontainers/holography_measurementset.py
index a515c5d37e9df41754323c7fea5620a8ad7635ec..7365265c9ddd6d8e1396e5aa9c83868814b8a277 100644
--- a/CAL/CalibrationCommon/lib/datacontainers/measurementset.py
+++ b/CAL/CalibrationCommon/lib/datacontainers/holography_measurementset.py
@@ -1,31 +1,49 @@
from casacore.tables import table as MS_Table
-import astropy.time as astrotime
+
import os
import re
-class MeasurementSet(object):
+class HolographyMeasurementSet(object):
ms_name_pattern = r'L(?P<sas_id>\d{6})_SB(?P<sub_band_id>\d{3})_uv\.MS'
def __init__(self, ms_name, ms_path):
self.path = os.path.join(ms_path, ms_name)
- if not MeasurementSet.is_a_valid_ms_name(ms_name):
+ if not HolographyMeasurementSet.is_a_valid_ms_name(ms_name):
raise ValueError('The measurement set located in %s has not a valid name' % self.path,)
self.name = ms_name
self.sas_id, self.beamlet = \
- MeasurementSet.parse_sas_id_and_sub_band_from_ms_name(self.name)
+ HolographyMeasurementSet.parse_sas_id_and_sub_band_from_ms_name(self.name)
def get_frequency(self):
- spectral_window_table = self.get_spectral_window_table()
+ observation_table = self.get_spectral_window_table()
try:
- reference_frequency = spectral_window_table.getcol('REF_FREQUENCY')
+ reference_frequency = observation_table.getcol('REF_FREQUENCY')[0]
finally:
- spectral_window_table.close()
+ observation_table.close()
return reference_frequency
+ def get_subband(self):
+ """
+ Return the sub band associated to this measurement set
+ :return: sub band number
+ :rtype: int
+ """
+ observation_table = self.get_observation_table()
+ try:
+ clock = observation_table.getcol('LOFAR_CLOCK_FREQUENCY')[0]
+ central_frequency = observation_table.getcol('LOFAR_OBSERVATION_FREQUENCY_CENTER')[0]
+
+ bit_sampling = 1024
+ subband = int(round(central_frequency / clock * bit_sampling) % 512)
+
+ finally:
+ observation_table.close()
+ return subband
+
def get_data_table(self):
data_table = MS_Table(self.path)
return data_table
@@ -38,6 +56,11 @@ class MeasurementSet(object):
antenna_table = MS_Table(self.path + '/ANTENNA')
return antenna_table
+ def get_spectral_window_table(self):
+ antenna_table = MS_Table(self.path + '/SPECTRAL_WINDOW')
+ return antenna_table
+
+
def get_lofar_antenna_field_table(self):
antenna_field_table = MS_Table(self.path + '/LOFAR_ANTENNA_FIELD')
return antenna_field_table
@@ -72,19 +95,19 @@ class MeasurementSet(object):
return source_name
- def get_spectral_window_table(self):
- spectral_window_table = MS_Table(self.path + '/SPECTRAL_WINDOW')
- return spectral_window_table
-
def get_observation_table(self):
observation_table = MS_Table(self.path + '/OBSERVATION')
return observation_table
+
+
def get_start_end_observation(self):
observation_table = self.get_observation_table()
try:
- start_time, end_time = observation_table.getcol('TIME_RANGE')
+ time_range = observation_table.getcol('TIME_RANGE')[0]
+
+ start_time, end_time = time_range
finally:
observation_table.close()
@@ -125,18 +148,18 @@ class MeasurementSet(object):
@staticmethod
def parse_sas_id_and_sub_band_from_ms_name(ms_name):
- if not MeasurementSet.is_a_valid_ms_name(ms_name):
+ if not HolographyMeasurementSet.is_a_valid_ms_name(ms_name):
raise ValueError('The measurement set %s has not a valid name' % ms_name,)
- match = re.match(MeasurementSet.ms_name_pattern, ms_name)
+ match = re.match(HolographyMeasurementSet.ms_name_pattern, ms_name)
return (int(match.group('sas_id')), int(match.group('sub_band_id')))
@staticmethod
def is_a_valid_ms_name(ms_name):
- pattern = MeasurementSet.ms_name_pattern
+ pattern = HolographyMeasurementSet.ms_name_pattern
return re.match(pattern, ms_name.strip()) # is not None
@staticmethod
def filter_valid_ms_names(list_of_ms_names):
- return list(filter(MeasurementSet.is_a_valid_ms_name, list_of_ms_names))
+ return list(filter(HolographyMeasurementSet.is_a_valid_ms_name, list_of_ms_names))
diff --git a/CAL/CalibrationCommon/lib/datacontainers/holography_observation.py b/CAL/CalibrationCommon/lib/datacontainers/holography_observation.py
index 6f32bceb04e21431a403f5828e03445c0ca753e4..71aeef558c68f2da6af93b644b67a7532e278e1a 100644
--- a/CAL/CalibrationCommon/lib/datacontainers/holography_observation.py
+++ b/CAL/CalibrationCommon/lib/datacontainers/holography_observation.py
@@ -2,21 +2,25 @@ import re
import os
import astropy.time as astrotime
from datetime import datetime
-from .measurementset import MeasurementSet
+from .holography_measurementset import HolographyMeasurementSet
class HolographyObservation():
def __init__(self, path, sas_id, ms_for_a_given_beamlet_number, start_mjd_in_seconds,
- end_mjd_in_seconds, frequency, source_name):
+ end_mjd_in_seconds, sub_band, frequency, source_name):
"""
+
:param path:
:type path: str
:param sas_id:
:type sas_id: int
:param ms_for_a_given_beamlet_number:
+ :type ms_for_a_given_beamlet_number: dict[int, HolographyMeasurementSet]
:param start_mjd_in_seconds:
:type start_mjd_in_seconds: float
:param end_mjd_in_seconds:
:type end_mjd_in_seconds: float
+ :param sub_band:
+ :type sub_band: int
:param frequency:
:type frequency: float
:param source_name:
@@ -29,6 +33,7 @@ class HolographyObservation():
self.end_datetime = HolographyObservation.__mjd_to_datetime(end_mjd_in_seconds)
self.start_mjd = start_mjd_in_seconds
self.end_mjd = end_mjd_in_seconds
+ self.sub_band = sub_band
self.frequency = frequency
self.source_name = source_name
@@ -61,14 +66,33 @@ class HolographyObservation():
"""
Returns a set of unique source names given a list of measurement sets
:param ms_list: a list of measurement set where to extract the reference frequencies
- :type ms_list: list[MeasurementSet]
+ :type ms_list: list[HolographyMeasurementSet]
:return: a set of frequencies
:rtype: set[str]
"""
return {ms.get_source_name() for ms in ms_list}
+ @staticmethod
+ def extract_unique_subband_from_ms_list(ms_list):
+ """
+ Returns a set of unique rcu modes given a list of measurement sets
+ :param ms_list: a list of measurement set where to extract the reference frequencies
+ :type ms_list: list[HolographyMeasurementSet]
+ :return: a set of rcu modes
+ :rtype: set[int]
+ """
+ return {ms.get_subband() for ms in ms_list}
+
@staticmethod
def list_observations_in_path(path):
+ """
+ List all the observations in the given path and return a list of HolographyObservation
+
+ :param path: path to the directory where the holography observation is stored\
+ :type path: str
+ :return: a list of HolographyObservation
+ :rtype: list[HolographyObservation]
+ """
ms_dir_name_pattern = 'L(?P<sas_id>\d{6})'
ms_dirs_path_pattern = '^' + os.path.join(path, ms_dir_name_pattern, 'uv$')
observations_list = []
@@ -105,9 +129,20 @@ class HolographyObservation():
else:
source_name = unique_source_names.pop()
+ unique_subband = HolographyObservation.\
+ extract_unique_subband_from_ms_list(ms_indexed_per_beamlet_number.values())
+
+ if len(unique_subband) > 1:
+ raise ValueError(
+ 'Multiple subband per observation are not supported'
+ )
+ else:
+ sub_band = unique_subband.pop()
+
observations_list.append(
HolographyObservation(path, sas_id, ms_indexed_per_beamlet_number,
- start_mjd_in_seconds, end_mjd_in_seconds, frequency,
+ start_mjd_in_seconds, end_mjd_in_seconds, sub_band,
+ frequency,
source_name))
return observations_list
@@ -116,7 +151,7 @@ class HolographyObservation():
"""
Returns a set of reference frequencies given a list of measurement setss
:param ms_list: a list of measurement set where to extract the reference frequencies
- :type ms_list: list[MeasurementSet]
+ :type ms_list: list[HolographyMeasurementSet]
:return: returns a set of frequencies
:rtype: set[float]
"""
@@ -130,10 +165,11 @@ class HolographyObservation():
:param path: a path were the ms are stored
:return: a dict containing the map of the ms indexed by their beamlet number
ex. { 0 : ms_beam0 ....}
+ :rtype: dict[int, HolographyMeasurementSet]
"""
- filtered_list_of_ms_names = MeasurementSet.filter_valid_ms_names(list_of_ms_names)
- ms_list = [MeasurementSet(ms_name, path) for ms_name in filtered_list_of_ms_names]
+ filtered_list_of_ms_names = HolographyMeasurementSet.filter_valid_ms_names(list_of_ms_names)
+ ms_list = [HolographyMeasurementSet(ms_name, path) for ms_name in filtered_list_of_ms_names]
- beamlet_ms_map = {ms.beamlet:ms for ms in ms_list}
+ beamlet_ms_map = {ms.beamlet:ms for ms in ms_list} #
- return beamlet_ms_map
\ No newline at end of file
+ return beamlet_ms_map
diff --git a/CAL/CalibrationCommon/lib/datacontainers/holography_specification.py b/CAL/CalibrationCommon/lib/datacontainers/holography_specification.py
index f65c26f2eb9d35bd89376db965c5b644de8ebac5..5da59b4bf38cf2b56dfe171bc41244c7c6010dcc 100644
--- a/CAL/CalibrationCommon/lib/datacontainers/holography_specification.py
+++ b/CAL/CalibrationCommon/lib/datacontainers/holography_specification.py
@@ -2,7 +2,7 @@ import datetime
from collections import defaultdict
import re
import os
-
+from glob import glob
class HolographyStationBeamSpecification(object):
station_name = None
@@ -121,6 +121,26 @@ class HolographySpecification(object):
return beam_specification
+ @staticmethod
+ def list_bsf_files_in_path(path):
+ """
+ List all the Holography beam specification files in the given path
+ :param path: path to the beam specification files
+ :type path: str
+ :return: the list of Holography Specifications
+ :rtype: list[HolographySpecification]
+ """
+ bsf_files_name_pattern = 'Holog-*.txt'
+ bsf_files_path_pattern = os.path.join(path, bsf_files_name_pattern)
+ matched_path_list = glob(bsf_files_path_pattern)
+ matched_file_path_list = list(
+ filter(lambda path_i: os.path.isfile(path_i), matched_path_list))
+ matched_file_name_list = list(map(lambda path_i: os.path.basename(path_i),
+ matched_file_path_list))
+ holography_beam_specification = HolographySpecification.create_hs_list_from_name_list_and_path(
+ matched_file_name_list, path)
+ return holography_beam_specification
+
def _parse_lines(self, lines):
splitted_lines = HolographySpecification._split_lines(lines)