From 24b9805b5004bc301da349de6c85eb72c63833d2 Mon Sep 17 00:00:00 2001
From: Mattia Mancini <mancini@astron.nl>
Date: Tue, 14 May 2019 14:37:47 +0000
Subject: [PATCH] SSB-42: Implemented reviewer suggestions
---
CAL/CalibrationCommon/lib/coordinates.py | 36 +-
.../lib/datacontainers/holography_dataset.py | 8 +-
.../holography_measurementset.py | 260 +++++++-------
.../datacontainers/holography_observation.py | 284 ++++++++--------
.../holography_specification.py | 317 +++++++++---------
CAL/CalibrationCommon/lib/mshologextract.py | 7 +-
CAL/CalibrationCommon/lib/utils.py | 14 +-
.../test/t_holography_dataset_class.py | 10 +-
.../test/t_holography_dataset_class2.py | 9 +-
.../test/t_holography_ms_class.py | 6 +-
.../test/t_holography_observation.py | 4 +-
11 files changed, 481 insertions(+), 474 deletions(-)
diff --git a/CAL/CalibrationCommon/lib/coordinates.py b/CAL/CalibrationCommon/lib/coordinates.py
index 58fe75967a6..bd3c452a193 100644
--- a/CAL/CalibrationCommon/lib/coordinates.py
+++ b/CAL/CalibrationCommon/lib/coordinates.py
@@ -6,6 +6,7 @@ import astropy.units as u
# Always try to download the most recent IERS tables.
from astropy.utils.data import download_file
from astropy.utils import iers
+
iers.IERS.iers_table = iers.IERS_A.open(download_file(iers.IERS_A_URL, cache=True))
@@ -54,21 +55,18 @@ def lm_from_radec(ra, dec, ra0, dec0):
... ra0=Angle(2.0, u.rad), dec0=Angle(1.0, u.rad))
(-0.1234948364118721, -0.089406906258670149)
'''
- cos_dec = cos(dec.rad)
- sin_dec = sin(dec.rad)
+ cos_dec = cos(dec.rad)
+ sin_dec = sin(dec.rad)
cos_dec0 = cos(dec0.rad)
sin_dec0 = sin(dec0.rad)
- sin_dra = sin(float(ra.rad - ra0.rad))
- cos_dra = cos(float(ra.rad - ra0.rad))
+ sin_dra = sin(float(ra.rad - ra0.rad))
+ cos_dra = cos(float(ra.rad - ra0.rad))
- l_rad = cos_dec*sin_dra
- m_rad = sin_dec*cos_dec0 - cos_dec*sin_dec0*cos_dra
+ l_rad = cos_dec * sin_dra
+ m_rad = sin_dec * cos_dec0 - cos_dec * sin_dec0 * cos_dra
return (l_rad, m_rad)
-
-
-
def radec_from_lm(l_rad, m_rad, ra0, dec0):
r'''
Calculate right ascension and declination given direction cosines
@@ -111,17 +109,14 @@ def radec_from_lm(l_rad, m_rad, ra0, dec0):
(<Angle 1.8 rad>, <Angle 0.9 rad>)
'''
- n_rad = sqrt(1.0 - l_rad*l_rad - m_rad*m_rad)
+ n_rad = sqrt(1.0 - l_rad * l_rad - m_rad * m_rad)
cos_dec0 = cos(dec0.rad)
sin_dec0 = sin(dec0.rad)
- ra_rad = ra0.rad + arctan2(l_rad, cos_dec0*n_rad - m_rad*sin_dec0)
- dec_rad = arcsin(m_rad*cos_dec0 + sin_dec0*n_rad)
+ ra_rad = ra0.rad + arctan2(l_rad, cos_dec0 * n_rad - m_rad * sin_dec0)
+ dec_rad = arcsin(m_rad * cos_dec0 + sin_dec0 * n_rad)
return (Angle(ra_rad, u.rad), Angle(dec_rad, u.rad))
-
-
-
def icrs_from_itrs(unit_vector_itrs, obstime):
r'''
Convert a geocentric cartesian unit vector in the ITRS system into
@@ -161,7 +156,7 @@ def icrs_from_itrs(unit_vector_itrs, obstime):
(358.7928571, 89.91033405)>
'''
x, y, z = array(unit_vector_itrs).T
- c_itrs = SkyCoord(x, y, z, representation='cartesian',frame='itrs',
+ c_itrs = SkyCoord(x, y, z, representation='cartesian', frame='itrs',
obstime=obstime, equinox='J2000')
return c_itrs.icrs
@@ -200,15 +195,13 @@ def itrs_from_icrs(icrs_position_rad, obstime):
[ 1.70157684e-01, -4.68937638e-01, 8.66685557e-01]])
'''
- ra, dec = array(icrs_position_rad, dtype='float64').T*u.rad
- icrs = SkyCoord(ra, dec,frame='icrs',
+ ra, dec = array(icrs_position_rad, dtype='float64').T * u.rad
+ icrs = SkyCoord(ra, dec, frame='icrs',
obstime=obstime, equinox='J2000')
itrs = icrs.itrs
return array([itrs.x, itrs.y, itrs.z], dtype=float64).T
-
-
def pqr_from_icrs(icrs_rad, obstime, pqr_to_itrs_matrix):
r'''
Compute geocentric station-local PQR coordinates of a certain ICRS
@@ -261,8 +254,6 @@ def pqr_from_icrs(icrs_rad, obstime, pqr_to_itrs_matrix):
return dot(pqr_to_itrs_matrix.T, itrs_from_icrs(icrs_rad, obstime).T).T.squeeze()
-
-
def icrs_from_pqr(pqr, obstime, pqr_to_itrs_matrix):
r'''
Convert directions from the station-local PQR system into
@@ -337,4 +328,3 @@ def icrs_from_pqr(pqr, obstime, pqr_to_itrs_matrix):
level at which we see the difference.
'''
return icrs_from_itrs(dot(pqr_to_itrs_matrix, pqr.T).T, obstime=obstime)
-
diff --git a/CAL/CalibrationCommon/lib/datacontainers/holography_dataset.py b/CAL/CalibrationCommon/lib/datacontainers/holography_dataset.py
index 77c2b4aa0b5..9c4a511391b 100644
--- a/CAL/CalibrationCommon/lib/datacontainers/holography_dataset.py
+++ b/CAL/CalibrationCommon/lib/datacontainers/holography_dataset.py
@@ -7,6 +7,7 @@ from lofar.calibration.common.datacontainers.holography_observation import Holog
from .holography_dataset_definitions import *
from .holography_specification import HolographySpecification
+
logger = logging.getLogger(__file__)
@@ -258,13 +259,17 @@ class HolographyDataset():
"""
logger.info("Creating a holography data set for station \"%s\"...", station_name)
try:
+ logger.debug('collecting preliminary information')
self.__collect_preliminary_information(station_name, list_of_hbs_ms_tuples)
+ logger.debug('collected preliminary information')
+ logger.debug('reading data')
self.__read_data(station_name, list_of_hbs_ms_tuples)
+ logger.debug('read data')
self.central_beamlets = self.find_central_beamlets(self.source_position, self.ra_dec,
self.frequencies, self.beamlets)
logger.info("Creation of a holography data set for station \"%s\" done.", station_name)
except Exception as e:
- logger.exception("Errore creating dataset for station \"%s\": %s", station_name, e)
+ logger.exception("Error creating dataset for station \"%s\": %s", station_name, e)
raise e
def __read_data(self, station_name, list_of_hbs_ms_tuples):
@@ -321,7 +326,6 @@ class HolographyDataset():
target_stations = set()
reference_stations = set()
beamlets = set()
- central_beamlet = None
virtual_pointing = dict()
frequencies = set()
sas_ids = set()
diff --git a/CAL/CalibrationCommon/lib/datacontainers/holography_measurementset.py b/CAL/CalibrationCommon/lib/datacontainers/holography_measurementset.py
index 9ffc8c0f407..b390b8a56c1 100644
--- a/CAL/CalibrationCommon/lib/datacontainers/holography_measurementset.py
+++ b/CAL/CalibrationCommon/lib/datacontainers/holography_measurementset.py
@@ -1,6 +1,6 @@
import os
import re
-from enum import Enum
+from enum import IntEnum
import numpy
from astropy.time import Time
@@ -10,127 +10,185 @@ from lofar.calibration.common.coordinates import pqr_from_icrs
from .holography_dataset_definitions import *
-class CASA_POLARIZATION_INDEX(Enum):
+class CASA_POLARIZATION_INDEX(IntEnum):
XX = 0
XY = 1
YX = 2
YY = 3
X = 0
- Y = 0
+ Y = 1
+
+
+ms_name_pattern = r'L(?P<sas_id>\d{6})_SB(?P<sub_band_id>\d{3})_uv\.MS'
+
+
+def __mjd_to_astropy_time(mjd_time_seconds):
+ """
+ Convert the modified julian date in seconds in a datetime object
+ :param mjd_time_seconds: modified julian data in seconds
+ :return: the date time of the given julian date
+ :rtype: datetime
+ """
+ hour_in_seconds = 60 * 60
+ day_in_seconds = hour_in_seconds * 24
+
+ return Time(mjd_time_seconds / day_in_seconds, format='mjd', scale='utc')
+
+
+def _compute_lm_from_ra_dec_station_position_rotation_matrix_and_time(ra_dec_epoch,
+ rotation_matrix,
+ mjd_times):
+ if isinstance(ra_dec_epoch, numpy.ndarray):
+ ra, dec, epoch = ra_dec_epoch.tolist()
+
+ astropy_times = [__mjd_to_astropy_time(mjd_time)
+ for mjd_time in mjd_times]
+ n_samples = len(astropy_times)
+ return_value_dtype = [('l', numpy.float64),
+ ('m', numpy.float64)]
+
+ return_value = numpy.empty(n_samples, dtype=return_value_dtype)
+ l_m_arrays = pqr_from_icrs(numpy.array((ra, dec)), astropy_times, rotation_matrix)
+
+ return_value['l'][:] = l_m_arrays[:, 0]
+ return_value['m'][:] = l_m_arrays[:, 1]
+ else:
+ raise TypeError('Expected a structured numpy array for ra_dec obtained {}'.
+ format(ra_dec_epoch))
+
+ return return_value
+
+
+def parse_sas_id_and_sub_band_from_ms_name(ms_name):
+ if is_a_valid_ms_name(ms_name):
+ match = re.match(ms_name_pattern, ms_name)
+ else:
+ raise ValueError('The measurement set %s has not a valid name' % ms_name, )
+ return str(match.group('sas_id')), int(match.group('sub_band_id'))
+
+
+def is_a_valid_ms_name(ms_name):
+ pattern = ms_name_pattern
+ return re.match(pattern, ms_name.strip()) # is not None
+
+
+def filter_valid_ms_names(list_of_ms_names):
+ return list(filter(is_a_valid_ms_name, list_of_ms_names))
class HolographyMeasurementSet(object):
- ms_name_pattern = r'L(?P<sas_id>\d{6})_SB(?P<sub_band_id>\d{3})_uv\.MS'
def __init__(self, ms_name, ms_path):
self.path = os.path.join(ms_path, ms_name)
- if HolographyMeasurementSet.is_a_valid_ms_name(ms_name):
+ if is_a_valid_ms_name(ms_name):
self.name = ms_name
- self.sas_id, self.beamlet = \
- HolographyMeasurementSet.parse_sas_id_and_sub_band_from_ms_name(self.name)
+ self.sas_id, self.beamlet = parse_sas_id_and_sub_band_from_ms_name(self.name)
else:
raise ValueError('The measurement set located in %s has not a valid name' % self.path, )
- def get_frequency(self):
- observation_table = self.get_spectral_window_table()
- try:
- reference_frequency = observation_table.getcol('REF_FREQUENCY')[0]
- finally:
- observation_table.close()
-
- return reference_frequency
-
- def get_subband(self):
- """
- Return the sub band associated to this measurement set
- :return: sub band number
- :rtype: int
+ def get_data_table(self):
"""
- observation_table = self.get_observation_table()
- try:
- clock = observation_table.getcol('LOFAR_CLOCK_FREQUENCY')[0]
- central_frequency = observation_table.getcol('LOFAR_OBSERVATION_FREQUENCY_CENTER')[0]
- bit_sampling = 1024
- subband = int(round(central_frequency / clock * bit_sampling) % 512)
-
- finally:
- observation_table.close()
- return subband
-
- def get_data_table(self):
+ :return:
+ :rtype: MS_Table
+ """
data_table = MS_Table(self.path, ack=False, readonly=True)
return data_table
def get_pointing_table(self):
+ """
+
+ :return:
+ :rtype: MS_Table
+ """
pointing_table = MS_Table(self.path + '/POINTING', ack=False, readonly=True)
return pointing_table
def get_antenna_table(self):
+ """
+
+ :return:
+ :rtype: MS_Table
+ """
antenna_table = MS_Table(self.path + '/ANTENNA', ack=False, readonly=True)
return antenna_table
def get_spectral_window_table(self):
- antenna_table = MS_Table(self.path + '/SPECTRAL_WINDOW', ack=False, readonly=True)
- return antenna_table
+ """
+
+ :return:
+ :rtype: MS_Table
+ """
+ spectral_window_table = MS_Table(self.path + '/SPECTRAL_WINDOW', ack=False, readonly=True)
+ return spectral_window_table
def get_lofar_antenna_field_table(self):
+ """
+
+ :return:
+ :rtype: MS_Table
+ """
antenna_field_table = MS_Table(self.path + '/LOFAR_ANTENNA_FIELD', ack=False, readonly=True)
return antenna_field_table
+ def get_observation_table(self):
+ observation_table = MS_Table(self.path + '/OBSERVATION', ack=False)
+ return observation_table
+
+ def get_frequency(self):
+ with self.get_spectral_window_table() as observation_table:
+ reference_frequency = observation_table.getcol('REF_FREQUENCY')[0]
+ return reference_frequency
+
+ def get_subband(self):
+ """
+ Return the sub band associated to this measurement set
+ :return: sub band number
+ :rtype: int
+ """
+ with self.get_observation_table() as observation_table:
+ clock = observation_table.getcol('LOFAR_CLOCK_FREQUENCY')[0]
+ central_frequency = observation_table.getcol('LOFAR_OBSERVATION_FREQUENCY_CENTER')[0]
+
+ bit_sampling = 1024
+ subband = int(round(central_frequency / clock * bit_sampling) % 512)
+ return subband
+
def get_station_position_tile_offsets_and_axes_coordinate_for_station_name(self, station_name):
- antenna_table = self.get_antenna_table()
- antenna_field_table = self.get_lofar_antenna_field_table()
- try:
- station_name_index = antenna_table.index('NAME').rownr(station_name)
+ with self.get_antenna_table() as antenna_table:
+ with self.get_lofar_antenna_field_table() as antenna_field_table:
+ station_name_index = antenna_table.index('NAME').rownr(station_name)
- station_position = antenna_field_table.getcell('POSITION', station_name_index)
- tile_offsets = antenna_field_table.getcell('ELEMENT_OFFSET', station_name_index)
- tiles_not_used = antenna_field_table.getcell('ELEMENT_FLAG', station_name_index)
- index_tiles_used = numpy.where(
- (tiles_not_used[:, CASA_POLARIZATION_INDEX.CASA_X_INDEX] == False) &
- (tiles_not_used[:, CASA_POLARIZATION_INDEX.CASA_Y_INDEX] == False))[0]
- tile_offsets = tile_offsets[index_tiles_used, :]
+ station_position = antenna_field_table.getcell('POSITION', station_name_index)
+ tile_offsets = antenna_field_table.getcell('ELEMENT_OFFSET', station_name_index)
+ tiles_not_used = antenna_field_table.getcell('ELEMENT_FLAG', station_name_index)
+ index_tiles_used = numpy.where(
+ (tiles_not_used[:, CASA_POLARIZATION_INDEX.X] == False) &
+ (tiles_not_used[:, CASA_POLARIZATION_INDEX.Y] == False))[0]
+ tile_offsets = tile_offsets[index_tiles_used, :]
- axes_coordinate = antenna_field_table.getcell('COORDINATE_AXES', station_name_index)
+ axes_coordinate = antenna_field_table.getcell('COORDINATE_AXES', station_name_index)
- finally:
- antenna_table.close()
- antenna_field_table.close()
- return station_position, tile_offsets, axes_coordinate
+ return station_position, tile_offsets, axes_coordinate
def __extract_source_name_from_pointing(self):
- pointing_table = self.get_pointing_table()
- try:
+ with self.get_pointing_table() as pointing_table:
unique_names = {name for name in pointing_table.getcol('NAME')}
if len(unique_names) == 1:
source_name = unique_names.pop()
else:
raise ValueError('Expected only a source as a target for the observation')
- finally:
- pointing_table.close()
- return source_name
-
- def get_observation_table(self):
- observation_table = MS_Table(self.path + '/OBSERVATION', ack=False)
- return observation_table
+ return source_name
def get_start_end_observation(self):
- observation_table = self.get_observation_table()
-
- try:
+ with self.get_observation_table() as observation_table:
time_range = observation_table.getcol('TIME_RANGE')[0]
-
start_time, end_time = time_range
-
- finally:
- observation_table.close()
-
- return start_time, end_time
+ return start_time, end_time
def get_source_name(self):
return self.__extract_source_name_from_pointing()
@@ -196,16 +254,16 @@ class HolographyMeasurementSet(object):
beams_crosscorrelations_array[reference_station_index, :]['XX'] = \
crosscorrelations[
- reference_station_index, :, CASA_POLARIZATION_INDEX.CASA_XX_INDEX]
+ reference_station_index, :, CASA_POLARIZATION_INDEX.XX]
beams_crosscorrelations_array[reference_station_index, :]['XY'] = \
crosscorrelations[
- reference_station_index, :, CASA_POLARIZATION_INDEX.CASA_XY_INDEX]
+ reference_station_index, :, CASA_POLARIZATION_INDEX.XY]
beams_crosscorrelations_array[reference_station_index, :]['YX'] = \
crosscorrelations[
- reference_station_index, :, CASA_POLARIZATION_INDEX.CASA_YX_INDEX]
+ reference_station_index, :, CASA_POLARIZATION_INDEX.YX]
beams_crosscorrelations_array[reference_station_index, :]['YY'] = \
crosscorrelations[
- reference_station_index, :, CASA_POLARIZATION_INDEX.CASA_YY_INDEX]
+ reference_station_index, :, CASA_POLARIZATION_INDEX.YY]
beams_crosscorrelations_array[reference_station_index, :]['flag'] = \
flags[reference_station_index, :]
@@ -240,7 +298,7 @@ class HolographyMeasurementSet(object):
reference_stations)
timestamps = beam_crosscorrelations_array[0, :]['t']
- lm_for_target_station = HolographyMeasurementSet._compute_lm_from_ra_dec_station_position_rotation_matrix_and_time(
+ lm_for_target_station = _compute_lm_from_ra_dec_station_position_rotation_matrix_and_time(
pointing, rotation_matrix, timestamps)
for reference_stations_index in range(beam_crosscorrelations_array.shape[0]):
beam_crosscorrelations_array['l'][reference_stations_index, :] = \
@@ -250,62 +308,8 @@ class HolographyMeasurementSet(object):
return reference_station_names, beam_crosscorrelations_array
- @staticmethod
- def __mjd_to_astropy_time(mjd_time_seconds):
- """
- Convert the modified julian date in seconds in a datetime object
- :param mjd_time_seconds: modified julian data in seconds
- :return: the date time of the given julian date
- :rtype: datetime
- """
- hour_in_seconds = 60 * 60
- day_in_seconds = hour_in_seconds * 24
-
- return Time(mjd_time_seconds / day_in_seconds, format='mjd', scale='utc')
-
- @staticmethod
- def _compute_lm_from_ra_dec_station_position_rotation_matrix_and_time(ra_dec_epoch,
- rotation_matrix,
- mjd_times):
- if isinstance(ra_dec_epoch, numpy.ndarray):
- ra, dec, epoch = ra_dec_epoch.tolist()
-
- astropy_times = [HolographyMeasurementSet.__mjd_to_astropy_time(mjd_time)
- for mjd_time in mjd_times]
- n_samples = len(astropy_times)
- return_value_dtype = [('l', numpy.float64),
- ('m', numpy.float64)]
-
- return_value = numpy.empty(n_samples, dtype=return_value_dtype)
- l_m_arrays = pqr_from_icrs(numpy.array((ra, dec)), astropy_times, rotation_matrix)
-
- return_value['l'][:] = l_m_arrays[:, 0]
- return_value['m'][:] = l_m_arrays[:, 1]
- else:
- raise TypeError('Expected a structured numpy array for ra_dec obtained {}'.
- format(ra_dec_epoch))
-
- return return_value
-
def __repr__(self):
return 'MeasurementSet(%d) located in %s for sas_id %s and sub_band_id %d' % (id(self),
self.name,
self.sas_id,
self.beamlet)
-
- @staticmethod
- def parse_sas_id_and_sub_band_from_ms_name(ms_name):
- if HolographyMeasurementSet.is_a_valid_ms_name(ms_name):
- match = re.match(HolographyMeasurementSet.ms_name_pattern, ms_name)
- else:
- raise ValueError('The measurement set %s has not a valid name' % ms_name, )
- return str(match.group('sas_id')), int(match.group('sub_band_id'))
-
- @staticmethod
- def is_a_valid_ms_name(ms_name):
- pattern = HolographyMeasurementSet.ms_name_pattern
- return re.match(pattern, ms_name.strip()) # is not None
-
- @staticmethod
- def filter_valid_ms_names(list_of_ms_names):
- return list(filter(HolographyMeasurementSet.is_a_valid_ms_name, list_of_ms_names))
diff --git a/CAL/CalibrationCommon/lib/datacontainers/holography_observation.py b/CAL/CalibrationCommon/lib/datacontainers/holography_observation.py
index 81a2d333efc..dcf074025d7 100644
--- a/CAL/CalibrationCommon/lib/datacontainers/holography_observation.py
+++ b/CAL/CalibrationCommon/lib/datacontainers/holography_observation.py
@@ -1,12 +1,61 @@
-import re
+import logging
import os
-import astropy.time as astrotime
+import re
from datetime import datetime
-from .holography_measurementset import HolographyMeasurementSet
-import logging
+
+import astropy.time as astrotime
+
+from .holography_measurementset import HolographyMeasurementSet, filter_valid_ms_names
logger = logging.getLogger(__file__)
+
+def _mjd_to_datetime(mjd_time_seconds):
+ """
+ Convert the modified julian date in seconds in a datetime object
+ :param mjd_time_seconds: modified julian data in seconds
+ :return: the date time of the given julian date
+ :rtype: datetime
+ """
+ hour_in_seconds = 60 * 60
+ day_in_seconds = hour_in_seconds * 24
+ return astrotime.Time(mjd_time_seconds / day_in_seconds, format='mjd',
+ scale='utc').to_datetime()
+
+
+def _compute_time_range_from_ms_list(ms_list):
+ observation_start, observation_end = ms_list[0].get_start_end_observation()
+ for ms in ms_list:
+ ms_start_time, ms_end_time = ms.get_start_end_observation()
+ if observation_start > ms_start_time:
+ observation_start = ms_start_time
+ if observation_end < ms_end_time:
+ observation_end = ms_end_time
+ return observation_start, observation_end
+
+
+def extract_unique_source_names_from_ms_list(ms_list):
+ """
+ Returns a set of unique source names given a list of measurement sets
+ :param ms_list: a list of measurement set where to extract the reference _frequencies
+ :type ms_list: list[HolographyMeasurementSet]
+ :return: a set of _frequencies
+ :rtype: set[str]
+ """
+ return {ms.get_source_name() for ms in ms_list}
+
+
+def extract_unique_subband_from_ms_list(ms_list):
+ """
+ Returns a set of unique rcu modes given a list of measurement sets
+ :param ms_list: a list of measurement set where to extract the reference _frequencies
+ :type ms_list: list[HolographyMeasurementSet]
+ :return: a set of rcu modes
+ :rtype: set[int]
+ """
+ return {ms.get_subband() for ms in ms_list}
+
+
class HolographyObservation():
def __init__(self, path, sas_id, ms_for_a_given_beamlet_number, start_mjd_in_seconds,
end_mjd_in_seconds, sub_band, frequency, source_name):
@@ -32,149 +81,98 @@ class HolographyObservation():
self.path = path
self.sas_id = sas_id
self.ms_for_a_given_beamlet_number = ms_for_a_given_beamlet_number
- self.start_datetime = HolographyObservation.__mjd_to_datetime(start_mjd_in_seconds)
- self.end_datetime = HolographyObservation.__mjd_to_datetime(end_mjd_in_seconds)
+ self.start_datetime = _mjd_to_datetime(start_mjd_in_seconds)
+ self.end_datetime = _mjd_to_datetime(end_mjd_in_seconds)
self.start_mjd = start_mjd_in_seconds
self.end_mjd = end_mjd_in_seconds
self.sub_band = sub_band
self.frequency = frequency
self.source_name = source_name
- @staticmethod
- def __mjd_to_datetime(mjd_time_seconds):
- """
- Convert the modified julian date in seconds in a datetime object
- :param mjd_time_seconds: modified julian data in seconds
- :return: the date time of the given julian date
- :rtype: datetime
- """
- hour_in_seconds = 60 * 60
- day_in_seconds = hour_in_seconds * 24
- return astrotime.Time(mjd_time_seconds / day_in_seconds, format='mjd',
- scale='utc').to_datetime()
-
- @staticmethod
- def __compute_time_range_from_ms_list(ms_list):
- observation_start, observation_end = ms_list[0].get_start_end_observation()
- for ms in ms_list:
- ms_start_time, ms_end_time = ms.get_start_end_observation()
- if observation_start > ms_start_time:
- observation_start = ms_start_time
- if observation_end < ms_end_time:
- observation_end = ms_end_time
- return observation_start, observation_end
-
- @staticmethod
- def extract_unique_source_names_from_ms_list(ms_list):
- """
- Returns a set of unique source names given a list of measurement sets
- :param ms_list: a list of measurement set where to extract the reference _frequencies
- :type ms_list: list[HolographyMeasurementSet]
- :return: a set of _frequencies
- :rtype: set[str]
- """
- return {ms.get_source_name() for ms in ms_list}
-
- @staticmethod
- def extract_unique_subband_from_ms_list(ms_list):
- """
- Returns a set of unique rcu modes given a list of measurement sets
- :param ms_list: a list of measurement set where to extract the reference _frequencies
- :type ms_list: list[HolographyMeasurementSet]
- :return: a set of rcu modes
- :rtype: set[int]
- """
- return {ms.get_subband() for ms in ms_list}
-
- @staticmethod
- def list_observations_in_path(path):
- """
- List all the observations in the given path and return a list of HolographyObservation
-
- :param path: path to the directory where the holography observation is stored\
- :type path: str
- :return: a list of HolographyObservation
- :rtype: list[HolographyObservation]
- """
- logger.info("Loading holography observations from \"%s\"...", path)
- ms_dir_name_pattern = 'L(?P<sas_id>\d{6})'
- ms_dirs_path_pattern = '^' + os.path.join(path, ms_dir_name_pattern, 'uv$')
- observations_list = []
- for root, dirnames, filenames in os.walk(path):
- match = re.match(ms_dirs_path_pattern, root)
- if match:
- sas_id = match.group('sas_id')
-
- ms_indexed_per_beamlet_number = HolographyObservation. \
- create_ms_dict_from_ms_name_list_and_path(dirnames, root)
-
-
- start_mjd_in_seconds, end_mjd_in_seconds = HolographyObservation.\
- __compute_time_range_from_ms_list(
- list(ms_indexed_per_beamlet_number.values()))
-
- unique_frequencies = HolographyObservation. \
- extract_unique_reference_frequencies_from_ms_list(
- list(ms_indexed_per_beamlet_number.values()))
-
- if len(unique_frequencies) == 1:
- frequency = unique_frequencies.pop()
- else:
- raise ValueError(
- 'Multiple reference _frequencies per observation are not supported')
-
- unique_source_names = HolographyObservation.\
- extract_unique_source_names_from_ms_list(
- list(ms_indexed_per_beamlet_number.values()))
-
- if len(unique_source_names) == 1:
- source_name = unique_source_names.pop()
- else:
- raise ValueError(
- 'Multiple source target per observation are not supported')
-
- unique_subband = HolographyObservation.\
- extract_unique_subband_from_ms_list(
- list(ms_indexed_per_beamlet_number.values()))
-
- if len(unique_subband) == 1:
- sub_band = unique_subband.pop()
- else:
- raise ValueError(
- 'Multiple subband per observation are not supported')
-
- observations_list.append(
- HolographyObservation(path, sas_id, ms_indexed_per_beamlet_number,
- start_mjd_in_seconds, end_mjd_in_seconds, sub_band,
- frequency,
- source_name))
- logger.info("Holography observations were successfully loaded from \"%s\".", path)
- return observations_list
-
- @staticmethod
- def extract_unique_reference_frequencies_from_ms_list(ms_list):
- """
- Returns a set of reference _frequencies given a list of measurement setss
- :param ms_list: a list of measurement set where to extract the reference _frequencies
- :type ms_list: list[HolographyMeasurementSet]
- :return: returns a set of _frequencies
- :rtype: set[float]
- """
- return {ms.get_frequency() for ms in ms_list}
-
- @staticmethod
- def create_ms_dict_from_ms_name_list_and_path(list_of_ms_names, path):
- """
- Creates a dict measurement sets indexed by beamlet id
- :param list_of_ms_names: a list of the ms to process
- :param path: a path were the ms are stored
- :return: a dict containing the map of the ms indexed by their beamlet number
- ex. { 0 : ms_beam0 ....}
- :rtype: dict[int, HolographyMeasurementSet]
- """
- filtered_list_of_ms_names = HolographyMeasurementSet.filter_valid_ms_names(list_of_ms_names)
- ms_list = [HolographyMeasurementSet(ms_name, path) for ms_name in filtered_list_of_ms_names]
-
- beamlet_ms_map = {ms.beamlet:ms for ms in ms_list} #
- return beamlet_ms_map
+def list_observations_in_path(path):
+ """
+ List all the observations in the given path and return a list of HolographyObservation
+
+ :param path: path to the directory where the holography observation is stored\
+ :type path: str
+ :return: a list of HolographyObservation
+ :rtype: list[HolographyObservation]
+ """
+ logger.info("Loading holography observations from \"%s\"...", path)
+ ms_dir_name_pattern = 'L(?P<sas_id>\d{6})'
+ ms_dirs_path_pattern = '^' + os.path.join(path, ms_dir_name_pattern, 'uv$')
+ observations_list = []
+ for root, dirnames, filenames in os.walk(path):
+ match = re.match(ms_dirs_path_pattern, root)
+ if match:
+ sas_id = match.group('sas_id')
+
+ ms_indexed_per_beamlet_number = create_ms_dict_from_ms_name_list_and_path(dirnames,
+ root)
+
+ start_mjd_in_seconds, end_mjd_in_seconds = _compute_time_range_from_ms_list(
+ list(ms_indexed_per_beamlet_number.values()))
+
+ unique_frequencies = extract_unique_reference_frequencies_from_ms_list(
+ list(ms_indexed_per_beamlet_number.values()))
+
+ if len(unique_frequencies) == 1:
+ frequency = unique_frequencies.pop()
+ else:
+ raise ValueError(
+ 'Multiple reference _frequencies per observation are not supported')
+
+ unique_source_names = extract_unique_source_names_from_ms_list(
+ list(ms_indexed_per_beamlet_number.values()))
+
+ if len(unique_source_names) == 1:
+ source_name = unique_source_names.pop()
+ else:
+ raise ValueError(
+ 'Multiple source target per observation are not supported')
+
+ unique_subband = extract_unique_subband_from_ms_list(
+ list(ms_indexed_per_beamlet_number.values()))
+
+ if len(unique_subband) == 1:
+ sub_band = unique_subband.pop()
+ else:
+ raise ValueError(
+ 'Multiple subband per observation are not supported')
+
+ observations_list.append(
+ HolographyObservation(path, sas_id, ms_indexed_per_beamlet_number,
+ start_mjd_in_seconds, end_mjd_in_seconds, sub_band,
+ frequency,
+ source_name))
+ logger.info("Holography observations were successfully loaded from \"%s\".", path)
+ return observations_list
+
+
+def extract_unique_reference_frequencies_from_ms_list(ms_list):
+ """
+ Returns a set of reference _frequencies given a list of measurement setss
+ :param ms_list: a list of measurement set where to extract the reference _frequencies
+ :type ms_list: list[HolographyMeasurementSet]
+ :return: returns a set of _frequencies
+ :rtype: set[float]
+ """
+ return {ms.get_frequency() for ms in ms_list}
+
+
+def create_ms_dict_from_ms_name_list_and_path(list_of_ms_names, path):
+ """
+ Creates a dict measurement sets indexed by beamlet id
+ :param list_of_ms_names: a list of the ms to process
+ :param path: a path were the ms are stored
+ :return: a dict containing the map of the ms indexed by their beamlet number
+ ex. { 0 : ms_beam0 ....}
+ :rtype: dict[int, HolographyMeasurementSet]
+ """
+ filtered_list_of_ms_names = filter_valid_ms_names(list_of_ms_names)
+ ms_list = [HolographyMeasurementSet(ms_name, path) for ms_name in filtered_list_of_ms_names]
+
+ beamlet_ms_map = {ms.beamlet: ms for ms in ms_list} #
+
+ return beamlet_ms_map
diff --git a/CAL/CalibrationCommon/lib/datacontainers/holography_specification.py b/CAL/CalibrationCommon/lib/datacontainers/holography_specification.py
index b43f124e1cf..62d142740af 100644
--- a/CAL/CalibrationCommon/lib/datacontainers/holography_specification.py
+++ b/CAL/CalibrationCommon/lib/datacontainers/holography_specification.py
@@ -1,12 +1,13 @@
import datetime
+import logging
import os
import re
from collections import defaultdict
from glob import glob
-import logging
logger = logging.getLogger(__file__)
+
class HolographyStationBeamSpecification(object):
"""
Contains the data regarding the beam specification for a single station.
@@ -22,6 +23,158 @@ class HolographyStationBeamSpecification(object):
station_type = ()
+def _parse_line(split_line):
+ """
+ Parses a line in the holography specification file
+ :param split_line: dict containing the row indexed by the column name
+ :type split_line: dict[str, str]
+ """
+ beam_specification = HolographyStationBeamSpecification()
+
+ beam_specification.rcus_mode = int(split_line['rcus_mode'])
+ beam_specification.sub_band_ids = [int(sub_band) for sub_band in
+ split_line['sub_band'].split(',')]
+ beam_specification.mode_description = split_line['mode_description']
+ beam_specification.rcus_involved = _parse_integer_range_or_list(
+ split_line['rcus'])
+ beam_specification.beamlets = split_line['beamlets']
+
+ beam_specification.station_pointing = _parse_pointing(
+ split_line['station_pointing'])
+ beam_specification.virtual_pointing = _parse_pointing(
+ split_line['virtual_pointing'])
+ if len(beam_specification.sub_band_ids) == 1:
+ beam_specification.station_type = 'target'
+ else:
+ beam_specification.station_type = 'reference'
+ beam_specification.station_name = split_line['station_name']
+
+ return beam_specification
+
+
+def list_bsf_files_in_path(path):
+ """
+ List all the Holography beam specification files in the given path
+ :param path: path to the beam specification files
+ :type path: str
+ :return: the list of Holography Specifications
+ :rtype: list[HolographySpecification]
+ """
+ logger.info("Loading holography beam specifications from \"%s\"...", path)
+ bsf_files_name_pattern = 'Holog-*.txt'
+ bsf_files_path_pattern = os.path.join(path, bsf_files_name_pattern)
+ matched_path_list = glob(bsf_files_path_pattern)
+ matched_file_path_list = list(
+ filter(lambda path_i: os.path.isfile(path_i), matched_path_list))
+ matched_file_name_list = list(map(lambda path_i: os.path.basename(path_i),
+ matched_file_path_list))
+ holography_beam_specification = create_hs_list_from_name_list_and_path(
+ matched_file_name_list, path)
+ logger.info("Loading holography beam specifications were successfully loaded from \"%s\".",
+ path)
+ return holography_beam_specification
+
+
+def create_hs_list_from_name_list_and_path(name_list, path):
+ return [HolographySpecification(name, path) for name in name_list]
+
+
+def is_holography_specification_file_name(name):
+ return re.match(HolographySpecification.hs_name_pattern, name) is not None
+
+
+def extract_id_date_comment_from_name(name):
+ match = re.match(HolographySpecification.hs_name_pattern, name)
+ date = match.group('date')
+ hs_id = int(match.group('id'))
+ comment = match.group('comment')
+ date = datetime.datetime.strptime(date, '%Y%m%d')
+ return hs_id, date, comment
+
+
+def _split_header(line):
+ """
+ Split the header of holography specification file
+ :param line: line containing the header
+ :return: a dict containing the start_date,
+ the end_date, the rcu_mode, and the beam_switch_delay
+ :rtype: dict
+ """
+ date_regex = '\d{4}-\d{2}-\d{2}\s\d{2}:\d{2}:\d{2}'
+ format = r'(?P<start_date>{date_regex})\s*' \
+ '(?P<end_date>{date_regex})\s*' \
+ '(?P<rcu_mode>\d*)\s*' \
+ '(?P<beam_switch_delay>\d*.\d*)'.format(date_regex=date_regex)
+ match = re.match(format, line)
+ return match.groupdict()
+
+
+def _split_line(line):
+ """
+ Split the header of holography specification file
+ :param line: line containing the header
+ :return: a dict containing the start_date,
+ the end_date, the rcu_mode, and the beam_switch_delay
+ :rtype: dict
+ """
+ range_regex = '(\d*\:\d*)|(\d*)'
+ ra_dec_regex = '\d*\.\d*,-?\d*\.\d*,\w*'
+ regex = r'^(?P<station_name>\w*)\s*' \
+ r'(?P<mode_description>\w*)\s*' \
+ r'(?P<sub_band>[\d,]*)\s*' \
+ r'(?P<beamlets>{range_regex})\s*' \
+ r'(?P<rcus>{range_regex})\s*' \
+ r'(?P<rcus_mode>(\d*))\s*' \
+ r'(?P<virtual_pointing>{ra_dec_regex})\s*' \
+ r'(?P<station_pointing>{ra_dec_regex})'.format(range_regex=range_regex,
+ ra_dec_regex=ra_dec_regex)
+ match = re.match(regex, line)
+ if match is None:
+ raise ValueError('Cannot parse line {}'.format(line))
+ return match.groupdict()
+
+
+def _split_lines(lines):
+ return [_split_line(line)
+ for line in lines]
+
+
+def _parse_pointing(pointing_string):
+ ra, dec, coordinate_system = pointing_string.split(',')
+ ra = float(ra)
+ dec = float(dec)
+ return dict(ra=ra, dec=dec, coordinate_system=coordinate_system)
+
+
+def _parse_integer_range_or_list(string_to_be_parsed):
+ """
+ Parses a string containing a list of int or an inclusive range and returns a list of int
+
+ ex "1,2,3,4,5" -> [1, 2, 3, 4, 5]
+ "1:4" -> [1, 2, 3, 4]
+ :param string_to_be_parsed: the string representing a list of int or a range
+ :return: a list of int
+ :rtype: list(int)
+ """
+ if ':' in string_to_be_parsed:
+ try:
+ start, end = map(int, string_to_be_parsed.split(':'))
+ except ValueError as e:
+ raise ValueError('Cannot parse string %s expected [start]:[end] -> %s' %
+ (string_to_be_parsed, e))
+ return_value = [x for x in range(start, end)]
+ elif ',' in string_to_be_parsed:
+ try:
+ return_value = list(map(int, string_to_be_parsed.split(',')))
+ except ValueError as e:
+ raise ValueError('Cannot parse string %s expected [int],[int],... -> %s' %
+ (string_to_be_parsed, e))
+ else:
+ raise ValueError('Cannot parse string %s expected [int],[int],... or [start]:[end] %s' %
+ string_to_be_parsed)
+ return return_value
+
+
class HolographySpecification(object):
"""
The HolographySpecification represents the set of input used to specify an holography
@@ -40,8 +193,7 @@ class HolographySpecification(object):
"""
self.path = os.path.join(path, name)
self.name = name
- self.id, self.date, self.comment = HolographySpecification. \
- extract_id_date_comment_from_name(name)
+ self.id, self.date, self.comment = extract_id_date_comment_from_name(name)
self.reference_station_names = None
self.target_station_names = None
self.station_specification_map = defaultdict(list)
@@ -59,28 +211,6 @@ class HolographySpecification(object):
self.path,
)
- @staticmethod
- def list_bsf_files_in_path(path):
- """
- List all the Holography beam specification files in the given path
- :param path: path to the beam specification files
- :type path: str
- :return: the list of Holography Specifications
- :rtype: list[HolographySpecification]
- """
- logger.info("Loading holography beam specifications from \"%s\"...", path)
- bsf_files_name_pattern = 'Holog-*.txt'
- bsf_files_path_pattern = os.path.join(path, bsf_files_name_pattern)
- matched_path_list = glob(bsf_files_path_pattern)
- matched_file_path_list = list(
- filter(lambda path_i: os.path.isfile(path_i), matched_path_list))
- matched_file_name_list = list(map(lambda path_i: os.path.basename(path_i),
- matched_file_path_list))
- holography_beam_specification = HolographySpecification.create_hs_list_from_name_list_and_path(
- matched_file_name_list, path)
- logger.info("Loading holography beam specifications were successfully loaded from \"%s\".", path)
- return holography_beam_specification
-
def get_beam_specifications_per_station_name(self, station_name):
"""
Returns a list of beam specifications for the given station name
@@ -99,44 +229,10 @@ class HolographySpecification(object):
self._update_class_attributes()
logger.debug("Reading holography file \"%s\" done.", self.path)
- @staticmethod
- def create_hs_list_from_name_list_and_path(name_list, path):
- return [HolographySpecification(name, path) for name in name_list]
-
- @staticmethod
- def is_holography_specification_file_name(name):
- return re.match(HolographySpecification.hs_name_pattern, name) is not None
-
- @staticmethod
- def extract_id_date_comment_from_name(name):
- match = re.match(HolographySpecification.hs_name_pattern, name)
- date = match.group('date')
- hs_id = int(match.group('id'))
- comment = match.group('comment')
- date = datetime.datetime.strptime(date, '%Y%m%d')
- return hs_id, date, comment
-
def _read_lines(self):
with open(self.path, 'r') as fstream_in:
return fstream_in.readlines()
- @staticmethod
- def _split_header(line):
- """
- Split the header of holography specification file
- :param line: line containing the header
- :return: a dict containing the start_date,
- the end_date, the rcu_mode, and the beam_switch_delay
- :rtype: dict
- """
- date_regex = '\d{4}-\d{2}-\d{2}\s\d{2}:\d{2}:\d{2}'
- format = r'(?P<start_date>{date_regex})\s*' \
- '(?P<end_date>{date_regex})\s*' \
- '(?P<rcu_mode>\d*)\s*' \
- '(?P<beam_switch_delay>\d*.\d*)'.format(date_regex=date_regex)
- match = re.match(format, line)
- return match.groupdict()
-
def _parse_header(self, header):
"""
Parse the header
@@ -144,7 +240,7 @@ class HolographySpecification(object):
"""
date_time_format = '%Y-%m-%d %H:%M:%S'
- split_header = HolographySpecification._split_header(header)
+ split_header = _split_header(header)
self.start_datetime = datetime.datetime.strptime(split_header['start_date'],
date_time_format)
@@ -153,111 +249,16 @@ class HolographySpecification(object):
self.rcu_mode = split_header['rcu_mode']
self.beam_set_interval = split_header['beam_switch_delay']
- @staticmethod
- def _split_line(line):
- """
- Split the header of holography specification file
- :param line: line containing the header
- :return: a dict containing the start_date,
- the end_date, the rcu_mode, and the beam_switch_delay
- :rtype: dict
- """
- range_regex = '(\d*\:\d*)|(\d*)'
- ra_dec_regex = '\d*\.\d*,-?\d*\.\d*,\w*'
- regex = r'^(?P<station_name>\w*)\s*' \
- r'(?P<mode_description>\w*)\s*' \
- r'(?P<sub_band>[\d,]*)\s*' \
- r'(?P<beamlets>{range_regex})\s*' \
- r'(?P<rcus>{range_regex})\s*' \
- r'(?P<rcus_mode>(\d*))\s*' \
- r'(?P<virtual_pointing>{ra_dec_regex})\s*' \
- r'(?P<station_pointing>{ra_dec_regex})'.format(range_regex=range_regex,
- ra_dec_regex=ra_dec_regex)
- match = re.match(regex, line)
- if match is None:
- raise ValueError('Cannot parse line {}'.format(line))
- return match.groupdict()
-
- @staticmethod
- def _split_lines(lines):
- return [HolographySpecification._split_line(line)
- for line in lines]
-
- @staticmethod
- def _parse_pointing(pointing_string):
- ra, dec, coordinate_system = pointing_string.split(',')
- ra = float(ra)
- dec = float(dec)
- return dict(ra=ra, dec=dec, coordinate_system=coordinate_system)
-
- @staticmethod
- def _parse_integer_range_or_list(string_to_be_parsed):
- """
- Parses a string containing a list of int or an inclusive range and returns a list of int
-
- ex "1,2,3,4,5" -> [1, 2, 3, 4, 5]
- "1:4" -> [1, 2, 3, 4]
- :param string_to_be_parsed: the string representing a list of int or a range
- :return: a list of int
- :rtype: list(int)
- """
- if ':' in string_to_be_parsed:
- try:
- start, end = map(int, string_to_be_parsed.split(':'))
- except ValueError as e:
- raise ValueError('Cannot parse string %s expected [start]:[end] -> %s' %
- (string_to_be_parsed, e))
- return_value = [x for x in range(start, end)]
- elif ',' in string_to_be_parsed:
- try:
- return_value = list(map(int, string_to_be_parsed.split(',')))
- except ValueError as e:
- raise ValueError('Cannot parse string %s expected [int],[int],... -> %s' %
- (string_to_be_parsed, e))
- else:
- raise ValueError('Cannot parse string %s expected [int],[int],... or [start]:[end] %s' %
- string_to_be_parsed)
- return return_value
-
- @staticmethod
- def _parse_line(split_line):
- """
- Parses a line in the holography specification file
- :param split_line: dict containing the row indexed by the column name
- :type split_line: dict[str, str]
- """
- beam_specification = HolographyStationBeamSpecification()
-
- beam_specification.rcus_mode = int(split_line['rcus_mode'])
- beam_specification.sub_band_ids = [int(sub_band) for sub_band in
- split_line['sub_band'].split(',')]
- beam_specification.mode_description = split_line['mode_description']
- beam_specification.rcus_involved = HolographySpecification._parse_integer_range_or_list(
- split_line['rcus'])
- beam_specification.beamlets = split_line['beamlets']
-
- beam_specification.station_pointing = HolographySpecification._parse_pointing(
- split_line['station_pointing'])
- beam_specification.virtual_pointing = HolographySpecification._parse_pointing(
- split_line['virtual_pointing'])
- if len(beam_specification.sub_band_ids) == 1:
- beam_specification.station_type = 'target'
- else:
- beam_specification.station_type = 'reference'
- beam_specification.station_name = split_line['station_name']
-
- return beam_specification
-
def _parse_lines(self, lines):
- split_lines = HolographySpecification._split_lines(lines)
+ split_lines = _split_lines(lines)
for line in split_lines:
- parsed_line = HolographySpecification._parse_line(line)
+ parsed_line = _parse_line(line)
self.station_specification_map[parsed_line.station_name] += [parsed_line]
def _update_class_attributes(self):
- self.station_names = self.station_specification_map.keys()
+ self.station_names = list(self.station_specification_map.keys())
self.reference_station_names = [station_name for station_name in
self.station_specification_map
if len(self.station_specification_map[station_name]) == 1]
diff --git a/CAL/CalibrationCommon/lib/mshologextract.py b/CAL/CalibrationCommon/lib/mshologextract.py
index 5ba8ffea6eb..921cfa03603 100644
--- a/CAL/CalibrationCommon/lib/mshologextract.py
+++ b/CAL/CalibrationCommon/lib/mshologextract.py
@@ -9,6 +9,7 @@ from multiprocessing import Pool as ThreadPool
import os
from lofar.calibration.common.utils import *
+from lofar.calibration.common.datacontainers.holography_dataset import HolographyDataset
DEFAULT_SLEEP_TIME = 1
@@ -80,7 +81,7 @@ def read_and_store_single_target_station(target_station_name, matched_bsf_ms_pai
logger.info('Loading data for station %s ', target_station_name)
holography_dataset.load_from_beam_specification_and_ms(target_station_name,
matched_bsf_ms_pair)
- filename = '%s.hdf5' % (target_station_name)
+ filename = '%s.hdf5' % target_station_name
outpath = os.path.join(store_path, filename)
logger.info('Storing data for station %s in %s', target_station_name, outpath)
@@ -150,9 +151,11 @@ def read_holography_datasets_and_store(holography_observation_path,
raise NotImplementedError()
target_station_names = list_all_target_stations(holography_observation_path)
+ logger.debug('target station names %s', target_station_names)
matched_bsf_ms_pair =\
match_holography_beam_specification_file_with_observation(holography_observation_path)
-
+ logger.debug('matched beam specification files and measurement sets: %s',
+ matched_bsf_ms_pair)
try:
if not os.path.exists(store_path):
os.makedirs(store_path)
diff --git a/CAL/CalibrationCommon/lib/utils.py b/CAL/CalibrationCommon/lib/utils.py
index 807f1c706a6..587e3fbae67 100644
--- a/CAL/CalibrationCommon/lib/utils.py
+++ b/CAL/CalibrationCommon/lib/utils.py
@@ -1,5 +1,9 @@
-from lofar.calibration.common.datacontainers import *
+from lofar.calibration.common.datacontainers.holography_specification import list_bsf_files_in_path
+from lofar.calibration.common.datacontainers.holography_observation import list_observations_in_path
+import logging
+
+logger = logging.getLogger(__name__)
def is_observation_in_range(start, end, from_datetime, to_datetime):
"""
@@ -21,8 +25,10 @@ def is_observation_in_range(start, end, from_datetime, to_datetime):
def match_holography_beam_specification_file_with_observation(path):
- bsf_files = HolographySpecification.list_bsf_files_in_path(path)
- observation_list = HolographyObservation.list_observations_in_path(path)
+ bsf_files = list_bsf_files_in_path(path)
+ observation_list = list_observations_in_path(path)
+ logger.debug('specification files found %s', bsf_files)
+ logger.debug('observation files found %s', observation_list)
matched_observation_bsf_pair = []
for bsf_file in bsf_files:
bsf_file.read_file()
@@ -38,7 +44,7 @@ def match_holography_beam_specification_file_with_observation(path):
def list_all_target_stations(observation_path):
- bsf_files = HolographySpecification.list_bsf_files_in_path(observation_path)
+ bsf_files = list_bsf_files_in_path(observation_path)
target_station_name_set = set()
for bsf_file in bsf_files:
bsf_file.read_file()
diff --git a/CAL/CalibrationCommon/test/t_holography_dataset_class.py b/CAL/CalibrationCommon/test/t_holography_dataset_class.py
index c7840898ccd..2ce17f2aecd 100755
--- a/CAL/CalibrationCommon/test/t_holography_dataset_class.py
+++ b/CAL/CalibrationCommon/test/t_holography_dataset_class.py
@@ -4,9 +4,9 @@ import tempfile
import h5py
import numpy
import os
-from lofar.calibration.common.datacontainers import HolographyDataset, HolographySpecification
-from lofar.calibration.common.datacontainers import HolographyObservation
-
+from lofar.calibration.common.datacontainers import HolographyDataset
+from lofar.calibration.common.datacontainers.holography_observation import list_observations_in_path
+from lofar.calibration.common.datacontainers.holography_specification import list_bsf_files_in_path
logger = logging.getLogger('t_holography_dataset_class')
# READ doc/Holography_Data_Set.md! It contains the location from which the
@@ -20,8 +20,8 @@ class TestHolographyDatasetClass(unittest.TestCase):
Reads a Measurement Set from a file and converts it to an HDS.
'''
# Read the MS into memory.
- holist = HolographyObservation.list_observations_in_path(path_to_test_data)
- hbsflist = HolographySpecification.list_bsf_files_in_path(path_to_test_data)
+ holist = list_observations_in_path(path_to_test_data)
+ hbsflist = list_bsf_files_in_path(path_to_test_data)
for hbsf in hbsflist:
hbsf.read_file()
ho_per_ms = [(hbsf, ho) for hbsf, ho in zip(hbsflist, holist)]
diff --git a/CAL/CalibrationCommon/test/t_holography_dataset_class2.py b/CAL/CalibrationCommon/test/t_holography_dataset_class2.py
index 9dfd40ad671..064f1b36858 100755
--- a/CAL/CalibrationCommon/test/t_holography_dataset_class2.py
+++ b/CAL/CalibrationCommon/test/t_holography_dataset_class2.py
@@ -2,8 +2,9 @@ import logging
import unittest
import tempfile
import os
-from lofar.calibration.common.datacontainers import HolographyDataset, HolographySpecification
-from lofar.calibration.common.datacontainers import HolographyObservation
+from lofar.calibration.common.datacontainers import HolographyDataset
+from lofar.calibration.common.datacontainers.holography_specification import list_bsf_files_in_path
+from lofar.calibration.common.datacontainers.holography_observation import list_observations_in_path
logger = logging.getLogger('t_holography_dataset_class')
@@ -21,8 +22,8 @@ class TestHolographyDatasetClass(unittest.TestCase):
the file is compared with the HDS in memory.
'''
# Read the MS into memory.
- holist = HolographyObservation.list_observations_in_path(path_to_test_data)
- hbsflist = HolographySpecification.list_bsf_files_in_path(path_to_test_data)
+ holist = list_observations_in_path(path_to_test_data)
+ hbsflist = list_bsf_files_in_path(path_to_test_data)
for hbsf in hbsflist:
hbsf.read_file()
ho_per_ms = [(hbsf, ho) for hbsf, ho in zip(hbsflist, holist)]
diff --git a/CAL/CalibrationCommon/test/t_holography_ms_class.py b/CAL/CalibrationCommon/test/t_holography_ms_class.py
index 3765b1a40ed..36ffc1d4dc2 100755
--- a/CAL/CalibrationCommon/test/t_holography_ms_class.py
+++ b/CAL/CalibrationCommon/test/t_holography_ms_class.py
@@ -5,7 +5,8 @@ import numpy
from astropy.coordinates import SkyCoord
from astropy.time import Time
-from lofar.calibration.common.datacontainers import HolographyMeasurementSet
+from lofar.calibration.common.datacontainers.holography_measurementset import \
+ _compute_lm_from_ra_dec_station_position_rotation_matrix_and_time
logger = logging.getLogger('t_holography_ms_class')
@@ -27,8 +28,7 @@ class TestHolographyDatasetClass(unittest.TestCase):
[9.92823000e-01, -9.54190000e-02, 7.20990000e-02],
[3.30000000e-05, 6.03078000e-01, 7.97682000e-01]],
dtype=numpy.float64)
- result = HolographyMeasurementSet.\
- _compute_lm_from_ra_dec_station_position_rotation_matrix_and_time(
+ result = _compute_lm_from_ra_dec_station_position_rotation_matrix_and_time(
target_icrs_rad, core_pqr_itrs_mat, [sample_time, sample_time])
return_value_dtype = [('l', numpy.float64),
diff --git a/CAL/CalibrationCommon/test/t_holography_observation.py b/CAL/CalibrationCommon/test/t_holography_observation.py
index 1e764c67712..cb42013f387 100755
--- a/CAL/CalibrationCommon/test/t_holography_observation.py
+++ b/CAL/CalibrationCommon/test/t_holography_observation.py
@@ -1,6 +1,6 @@
import logging
import unittest
-from lofar.calibration.common.datacontainers import HolographyObservation
+from lofar.calibration.common.datacontainers.holography_observation import list_observations_in_path
logger = logging.getLogger('t_holography_dataset_class')
path_to_test_data = '/var/tmp/holography'
@@ -8,7 +8,7 @@ path_to_test_data = '/var/tmp/holography'
class TestHolographyDatasetClass(unittest.TestCase):
def test_holography_observation_path_listing(self):
- list = HolographyObservation.list_observations_in_path(path_to_test_data)
+ list = list_observations_in_path(path_to_test_data)
## Assert correct reading of the files
HolographyObservation_freq_one = list.pop()
--
GitLab