Commit 1f937738 authored by mancini's avatar mancini

OSB-54: Fix merge

parents 1c4af069 cefbd846
This diff is collapsed.
......@@ -19,3 +19,4 @@ SAS/OTB/jRSP/Makefile.am
SAS/OTB/jRSP/autoconf_share
SAS/OTB/jRSP/bootstrap
SAS/OTB/jRSP/configure.in
**/.idea
stages:
- prepare
- build
- unit_test
# - integration_test
#
# PREPARE STAGE
#
prepare_RAServices:
stage: prepare
script:
- docker build -t ci_raservices -f SubSystems/RAServices/Dockerfile .
#
# BUILD STAGE
#
build_RAServices:
stage: build
image: ci_raservices:latest
script:
- PACKAGE=RAServices
- echo "Building $PACKAGE..."
- mkdir install
- mkdir -p build/gnucxx11_opt
- cd build/gnucxx11_opt
- cmake -DBUILD_PACKAGES=$PACKAGE -DCASACORE_ROOT_DIR=/opt/casacore/ -DCASAREST_ROOT_DIR=/opt/casarest/ -DCMAKE_INSTALL_PREFIX=/opt/lofar -DUSE_LOG4CPLUS=false ../..
- make
- make DESTDIR=../../install install
- cd ../../install/opt/lofar
- tar --ignore-failed-read --exclude=include -czf ../../RAServices_$CI_COMMIT_SHORT_SHA.ztar *
dependencies:
- prepare_RAServices
artifacts:
expire_in: 6 hours
paths:
- build/gnucxx11_opt
- install/*.ztar
#
# UNIT TEST STAGE
#
unit_test_RAServices:
stage: unit_test
image: ci_raservices:latest
script:
- PACKAGE=RAServices
- echo "Testing $PACKAGE..."
- cd build/gnucxx11_opt
- SKIP_INTEGRATION_TESTS=true ctest
dependencies:
- build_RAServices
artifacts:
name: unit-test-report
when: always
paths:
- build/gnucxx11_opt/Testing/Temporary/LastTest.log
#
# INTEGRATION TEST STAGE
#
# integration_test_RAServices:
# stage: integration_test
# image: ci_raservices:latest
# services:
# - rabbitmq:latest
# variables:
# LOFAR_DEFAULT_BROKER: "rabbitmq"
# script:
# - PACKAGE=RAServices
# - echo "Testing $PACKAGE..."
# - cd build/gnucxx11_opt
# - SKIP_UNIT_TESTS=true ctest
# dependencies:
# - build_RAServices
# artifacts:
# name: integration-test-report
# when: always
# paths:
# - build/gnucxx11_opt/Testing/Temporary/LastTest.log
# Copyright (C) 2018 ASTRON (Netherlands Institute for Radio Astronomy)
# P.O. Box 2, 7990 AA Dwingeloo, The Netherlands
#
# This file is part of the LOFAR software suite.
# The LOFAR software suite is free software: you can redistribute it and/or
# modify it under the terms of the GNU General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# The LOFAR software suite is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
# $Id$
find_package(PythonInterp 3.0)
include(FindPythonModule)
find_python_module(numpy)
find_python_module(scipy)
find_python_module(h5py)
find_python_module(astropy)
find_python_module(casacore)
lofar_add_package(CalibrationCommon)
lofar_add_package(CalibrationProcessing)
# Copyright (C) 2018 ASTRON (Netherlands Institute for Radio Astronomy)
# P.O. Box 2, 7990 AA Dwingeloo, The Netherlands
#
# This file is part of the LOFAR software suite.
# The LOFAR software suite is free software: you can redistribute it and/or
# modify it under the terms of the GNU General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# The LOFAR software suite is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
# $Id$
lofar_package(CalibrationCommon 1.0 DEPENDS PyCommon)
include(PythonInstall)
add_subdirectory(lib)
add_subdirectory(bin)
add_subdirectory(test)
# CalibrationCommon {#CalibrationCommon}
## GENERAL
The CalibrationCommon package contains documentation and software that concerns all areas of LOFAR calibration.
### Description
- *What does it do?*
- *Why is it needed?*
### Author/Owner
### Overview
- *Add a diagram*
- *Add a link to the overview diagram*
- *Add a link in the overview diagram to link back to this documentation*.
- - -
## DEVELOPMENT
### Analyses
*Add non-technical information and functional considerations here, like user requirements and links to minutes of
meetings with stakeholders.*
### Design
*Add technical considerations and design choices here*
### Source Code
- *Add a link to svn (trunk).*
- *Add a link to (generated?) source code documentation.*
### Testing
- *How do you run unit tests?*
- *How do you run integration tests?*
- *Add a link to Jenkins jobs (if available)*
### Build & Deploy
- *Add a link to general instructions or describe specifics here.*
- *Add a link to Jenkins jobs (if available)*
- - -
## OPERATIONS
### Configuration
- *Where is the configuration file?*
- *What are the configuration options?*
### Log Files
- *Where are the log files?*
### Runtime
- *Where does it run? (which user machine)*
- *How do I run it? (user documentation? examples? commandline parameters?)*
- *Other considerations? (what happens elsewhere when I start or stop it?)*
### Interfaces (API)
- *Describe interfaces to other applications (REST API? http requests? Messagebus?)*
- *Other communication (user? import/export?)*
### Files/Databases
- *Which databases are used?*
- *Which files are used?*
### Dependencies
- *To/from other applications?*
- *Files?*
- *Network locations?*
- *Other?*
### Security
- *Special privileges needed?*
- *User login?*
- *Certificates needed?*
- *Other considerations?*
- - -
## ADDITIONAL INFORMATION
### User Documentation
*e.g. Please refer to URL X for the User Documentation*
### Operations Documentation
*e.g. Please refer to URL X for Operations Documentation*
# Definition of the LOFAR Holography Data Set {#HolographyDataset}
Authors: [Thomas Jürges](mailto:jurges@astron.nl), [Mattia Mancini](mailto:mancini@astron.nl)
HOLOGRAPHY_DATA_SET_VERSION
Date: 2018-09-10
Version: 1.0
References: n.a.
Abbreviations:
- *HDS*: Holography Data Set
- *LOFAR*: LOw Frequency ARray
- *MJD*: Modified Julian Date
- *MS*: Measurement Set
## The LOFAR Holography Data Set (HDS)
The LOFAR Holography Data Set (HDS) serves as data container for cross-correlation results that have been calculated from the Measurement Sets of holography observations. It also contains the complete specification that was provided when the holography observation was entered into the observation planning system.
### HDS from planning to data
Once all information is known, the attributes of the HDS will get filled in and the first instance of the HDS will be stored as a file on disk. After the observation is finished and the cross-correlations have been calculated the results are stored in MS. A conversion script takes the MS and converts it into the HDS.
## The HDS in real life
### Attributes of the HDS
Attributes are entities that are specified during the planning of a holography observation. They define what the *intention* at the time of the planning is. After a holography observation is scheduled its attributes will not change any more. Hence they are an immutable description of the planned observation. This information allows to repeat the same holography observation at a later time.
Below is the list of defined attributes in this version of HDS:
- **HDS_version**: The HDS is versioned and begins at version 1.0. The versions are incremental. *Unit: [n.a.]*
- **RCU_list**: An array that contains the list of RCUs that are planned to be used during the holography observation. *Unit: [n.a.]*
- **Mode**: The RCU mode that will be set during the holography observation. *Unit: [n.a.]*
- **SAS_ids**: An array that contains the SAS ids that were created for all observation runs of the holography observation. *Unit: [n.a.]*
- **Target_station_name**: Name of the LOFAR target station. *Unit: [n.a.]*
- **Target_station_position**: Position in cartesian coordinates. *Unit: [m, m, m]*
- **Central_beamlets**: An array that contains the central beamlet number for each frequency. *Unit: [n.a.]*
- **Source_name**: Name of the source observed during the holography observation. *Unit: [n.a.]*
- **Source_position**: Celestial coordinates (RA, DEC, EPOCH) of the source. *Unit: [rad, rad, n.a.]*
- **Observation_time**: Planned start and end time-stamps of the entire holography observation in MJD. *Unit: [s]*
- **Rotation_matrix**: Rotation matrix that allows to convert (RA, DEC) coordinates to (l, m) coordinates and vice versa. *Unit: [n.a.]*
- **Antenna_field_position**: Cartesian coordinates with reference to IRTF of the centre of each antenna field that will be used in the holography observation. *Unit: [m, m, m]*
Additionally if the HDS has been used to specify the holography observation these fields will also be present:
- **Specified reference station**: List of reference stations that were planned for the holography observation. *Unit: [n.a.]*
- **Specified frequency**: List of frequencies that were planned to be used for the holography observation. *Unit: [Hz]*
- **Specified RA DEC**: List of the planned target station's beamlet pointings for the holography observation. *Unit: [rad, rad]*
### Data tables in the HDS
Data tables serve as the work horses for the cross-correlations in an HDS. They contain a list of the actually used reference stations, a list of the actually used frequencies, the actually used target station beam positions and the calculated cross-calculation values.
Below is the list of defined data tables in this version of HDS:
- **Reference_station**: The reference station data table contains the list of all reference stations that were actually used in the holography observation. This list can contain less stations than the **Specified reference station** list. For a given reference station the index of it in this list serves as one of the three indices of **data**. *Unit: [n.a.]
- **frequency**: List of frequencies that were actually used for the holography observation. This list can contain less elements than the **Specified frequency** list. For a given frequency the index of it in this list serves as one of the three indices of **data**. *Unit [Hz]*
Additionally, a table containing the cross correlations and the right ascension and declination of each beam are stored in a group structure that will be described in the next sub section.
### Groups in the HDS
The right ascension, declination and the crosscorrelation are stored in the HDS in a hierarchical group structure. This allows to access a specific set of samples by the reference station name, the frequency of the samples and the beamlet number.
The notation ```%(station_name)``` is used to refer to a string containing a station name. The same notation is also used to refer to the string representation of the frequency in Hz (```%(frequency)```) and the beamlet number (```%(beamlet)```).
The structure of the groups is the following:
```
/CROSSCORRELATION
/%(station_name)
/%(frequency)
%(beamlet) This data table contains the cross-correlations *XX*, *XY*, *YX*, *YY*, l, m, the time stamp of the sample, and if or not the sample has been flagged for a given frequency, beamlet# and reference station. *Unit: [(n.a, n.a.), (n.a, n.a), (n.a, n.a), (n.a, n.a), (n.a), (n.a), (s), (True|False)]*\
/RA_DEC
/%(frequency)
%(beamlet) This data table contains the (right ascension, declination, epoch) of the target station beam pointings. This list makes it possible to translate between (frequency, beamlet) and (RA, DEC). *Unit: [rad, rad]*
```
## Cross-correlation data in HDS
Values in the cross-correlation tables are stored in a pseudo-structure. A simple representation of a single element in cross-correlation table can be given in Python language:
```
data_element = dict(
"XX" = c_XX,
"XY" = c_XY,
"YX" = c_YX,
"YY" = c_YY,
"t" = t_MJD,
"l" = l,
"m" = m,
"flag"= flag)
```
Here the indices "XX", "XY", "YX", "YY" allow accessing the cross-correlation values, "t" allows to access the MJD of the observation and "l", "m" allow to access the coordinates l and m and finally flag to access to the flag value.
As described before each cross-correlation table can be accessed by indices, meaning that the observation samples at a given frequency `%(frequency)`, for a reference station `%(station_name)` and a beamlet `%(beamlet)` can be directly accessed as `/%(station_name)/%(frequency)/%(beamlet)`. Example: `/RS409C/114843750.0/0`
## Layout of the HDS in HDF5 files
```
# Root
/
# Attributes
/ATTRS
/Antenna field position (1-d array, double[3]), cartesian coordinates of the centres of each antenna field
/HDS_version (double), holography file format version
/Mode (int), RCU mode used during all observations
/Observation_time (double[2]), Start and end time stamps of the entire holography observation
/RCU_list (vararray[int]), entries are RCU#s used in all observations
/Rotation_matrix (2-d array, double[3][3]), translation matrix for (RA, DEC) <-> (l, m) conversion
/SAS_ids (vararray[string]), contains all SAS ids of all observations
/Source_name (string), name of the observed source in the sky
/Source_position (compound(double, double, string), RA, DEC and Epoch of the source in the sky
/Target_station_name (string), name of the target station
/Target_station_position (double[3]), position of the centre of the station in cartesian coordinates
/Central_beamlets (1-d array, int), beamlet number of the central beamlet, index is the frequency
/Specified_reference_station (1-d array, string), entries are reference station names [optional]
/Specified_frequency (1-d array, double), entries are the frequencies of all observations that were specified [optional]
/Specified_RA_DEC (2-d array, compound[double, double, string[10]]), indices are frequency index, beamlet# [optional]
# Groups
/Reference_station (1-d array, string[8]), entries are reference station names
/Frequency (1-d array, double), entries are the frequencies of all observations
/RA_DEC
/[One entry for each of the frequencies] Index is the frequency in Hz as string.
/[One entry for each of the beamlets] Index is the beamlet number as string.
/(1-d array, compound[double, double, string[10]]), entries contain a struct of {RA, DEC, Epoch}
/CROSSCORRELATION
/[One entry for each of the reference stations] Index is the name of the reference station.
/[One entry for each of the frequencies] Index is the frequency in Hz as string.
/[One entry for each of the beamlets] Index is the beamlet number as string.
/data (1-d array, vararray[compound[compound[4][double, double], double[3], boolean]], entries contain a struct of {XX, XY, YX, YY, l, m, t, flag}.
```
## Example data for tests
Test data for two frequencies can be found on LOFAR's CEP3 in the directories
/data014/scratch/veen/holography/20180718/obs3/L661022 and
/data014/scratch/veen/holography/20180718/obs3/L661024.
The MS to HDS conversion utility will create an HDF5 file containing an HDS from a MS. This file can then be used as input for some of the tests of the conversion utility that verifies a correct conversion.
/**
\ingroup Packages
\defgroup CalibrationCommon CalibrationCommon
\ref CalibrationCommon
The CalibrationCommon package contains documentation and software that concerns all areas of LOFAR calibration.
*/
# Copyright (C) 2018 ASTRON (Netherlands Institute for Radio Astronomy)
# P.O. Box 2, 7990 AA Dwingeloo, The Netherlands
#
# This file is part of the LOFAR software suite.
# The LOFAR software suite is free software: you can redistribute it and/or
# modify it under the terms of the GNU General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# The LOFAR software suite is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
# $Id$
python_install(
__init__.py
datacontainers/holography_dataset_definitions.py
datacontainers/holography_dataset.py
datacontainers/holography_specification.py
datacontainers/holography_observation.py
datacontainers/holography_measurementset.py
datacontainers/calibration_table.py
datacontainers/__init__.py
coordinates.py
utils.py
DESTINATION lofar/calibration/common)
lofar_add_bin_script(mshologextract.py RENAME mshologextract)
\ No newline at end of file
This diff is collapsed.
__all__ = ['HolographyDataset', 'HolographyObservation', 'HolographySpecification', 'HolographyMeasurementSet']
from .holography_specification import HolographySpecification
from .holography_observation import HolographyObservation
from .holography_measurementset import HolographyMeasurementSet
from .holography_dataset import HolographyDataset
import logging
from glob import glob
from os import path
from re import fullmatch
from struct import iter_unpack, pack
from typing import BinaryIO
from typing import Dict, Tuple
from typing import List
from dataclasses import dataclass, asdict, field
from h5py import File
from numpy import empty as empty_ndarray, ndarray, fromiter as array_from_iter, float64, \
array_equal, arange, array
from copy import deepcopy
logger = logging.getLogger(__name__)
_MODE_TO_CLOCK = {1: 200, 3: 200, 5: 200, 6: 160, 7: 200}
_MODE_TO_NYQ_ZONE = {1: 1, 3: 1, 5: 2, 6: 1, 7: 3}
_ATTRIBUTE_NAME_TO_SERIALIZED_NAME = {
'observation_station': 'CalTableHeader.Observation.Station',
'observation_mode': 'CalTableHeader.Observation.Mode',
'observation_antennaset': 'CalTableHeader.Observation.AntennaSet',
'observation_band': 'CalTableHeader.Observation.Band',
'observation_source': 'CalTableHeader.Observation.Source',
'observation_date': 'CalTableHeader.Observation.Date',
'calibration_version': 'CalTableHeader.Calibration.Version',
'calibration_name': 'CalTableHeader.Calibration.Name',
'calibration_date': 'CalTableHeader.Calibration.Date',
'calibration_ppsdelay': 'CalTableHeader.Calibration.PPSDelay',
'comment': 'CalTableHeader.Comment'