Commit fac03fd8 authored by Jakob Maljaars's avatar Jakob Maljaars

Merge branch 'idg-cal-jakob-1' into 'master'

Idg cal jakob 1 (rebased)

See merge request !51
parents ea8e5c44 009945f9
Pipeline #8125 passed with stages
in 56 minutes and 31 seconds
......@@ -131,8 +131,8 @@ idg-integration-docker:
- cd build
# Run in verbose mode -VV as long as it's not properly working
- ctest -L integration
# Test the h5parm writer
- cd $HOME_DIR/scripts/test
# Test idg-cal scripts
- cd $HOME_DIR/idg-cal/unit_tests
- pytest .
rules:
- when: on_success
......@@ -185,8 +185,8 @@ idg-integration-das:
# Run integration tests
- "python3 -m pip install --user --only-binary=:all: python-casacore"
- ctest --verbose -L integration
# Test the h5parm writer
- cd $CURRENT_SOURCE/scripts/test
# Test idg-cal scripts
- cd $CURRENT_SOURCE/idg-cal/unit_tests
- pytest .
rules:
- when: on_success
......
......@@ -44,10 +44,10 @@ option (BUILD_WITH_PYTHON "Build Python bindings" OFF)
if(${CMAKE_VERSION} VERSION_GREATER "3.12.4")
find_package(Python REQUIRED)
set(PYTHON_INSTALL_DIR ${CMAKE_INSTALL_PREFIX}/lib/python${Python_VERSION_MAJOR}.${Python_VERSION_MINOR}/dist-packages/idg)
set(PYTHON_INSTALL_DIR ${CMAKE_INSTALL_PREFIX}/lib/python${Python_VERSION_MAJOR}.${Python_VERSION_MINOR}/site-packages/idg)
else()
find_package(PythonInterp REQUIRED)
set(PYTHON_INSTALL_DIR ${CMAKE_INSTALL_PREFIX}/lib/python${PYTHON_VERSION_MAJOR}.${PYTHON_VERSION_MINOR}/dist-packages/idg)
set(PYTHON_INSTALL_DIR ${CMAKE_INSTALL_PREFIX}/lib/python${PYTHON_VERSION_MAJOR}.${PYTHON_VERSION_MINOR}/site-packages/idg)
endif()
if (BUILD_TESTING)
......@@ -58,6 +58,7 @@ add_subdirectory("idg-util")
add_subdirectory("idg-lib")
add_subdirectory("idg-bin")
add_subdirectory("idg-api")
add_subdirectory("idg-cal")
# Write environment-module
configure_file (
......
# Copyright (C) 2020 ASTRON (Netherlands Institute for Radio Astronomy)
# SPDX-License-Identifier: GPL-3.0-or-later
# Install Python modules.
install(FILES
# __init__.py
h5parmwriter.py
idgcaldpstep.py
DESTINATION
${PYTHON_INSTALL_DIR})
\ No newline at end of file
......@@ -21,7 +21,9 @@ class H5ParmWriter:
radio astronomical data.
"""
def __init__(self, file_name, solution_set_name="sol000", attributes=None):
def __init__(
self, file_name, solution_set_name="sol000", attributes=None, overwrite=True
):
"""
Initialize H5Parmwriter. The file, specified by file_name, should not yet
exist. If it does, an error is thrown.
......@@ -36,7 +38,7 @@ class H5ParmWriter:
Dictionary containing attributes for the sol set
"""
if os.path.isfile(file_name):
if os.path.isfile(file_name) and not overwrite:
raise ValueError(
f"File {file_name} already exists, choose a different name!"
)
......@@ -161,11 +163,15 @@ class H5ParmWriter:
hdf_location = self.solution_set + "/" + name
self.h5file[self.solution_set].create_group(name)
self.h5file[hdf_location].attrs["TITLE"] = str.encode(type_name)
self.h5file[hdf_location].attrs["TITLE"] = np.asarray(
type_name, dtype=f"<S{len(type_name)}"
)
self.__write_version_stamp(self.solution_set + "/" + name)
# Convert axes labels to S-type
axes_labels = list(axes.keys())
axes_labels = ",".join(axes_labels).encode()
axes_labels = np.asarray(axes_labels, dtype=f"<S{len(axes_labels)}")
axes_shape = tuple(axes.values())
dtype = values.dtype if values is not None else dtype
......@@ -203,7 +209,9 @@ class H5ParmWriter:
self.h5file[hdf_location + "/weight"][:] = weights
if history is not None:
self.h5file[hdf_location + "/val"].attrs["HISTORY000"] = str.encode(history)
self.h5file[hdf_location + "/val"].attrs["HISTORY000"] = np.asarray(
history, dtype=f"<S{len(history)}"
)
def fill_solution_table(self, name, values, offset=None):
"""
......@@ -315,7 +323,7 @@ class H5ParmWriter:
if hdf_location + "/" + axis not in self.h5file:
# Create axis meta data set
self.h5file[hdf_location].create_dataset(
axis, meta_data.size, data_type_h5
axis, meta_data.shape, data_type_h5
)
self.h5file[hdf_location + "/" + axis][:] = meta_data.astype(
data_type_h5
......@@ -357,7 +365,9 @@ class H5ParmWriter:
hdf_location : str
Location where the version stamp will be added.
"""
self.h5file[hdf_location].attrs["h5parm_version"] = str.encode(H5PARM_VERSION)
self.h5file[hdf_location].attrs["h5parm_version"] = np.asarray(
H5PARM_VERSION, dtype=f"<S{len(H5PARM_VERSION)}"
)
def __write_attributes(self, hdf_location, attributes, overwrite):
"""
......@@ -382,7 +392,9 @@ class H5ParmWriter:
else:
if isinstance(value, str):
# Byte encoded string
self.h5file[hdf_location].attrs[key] = str.encode(value)
self.h5file[hdf_location].attrs[key] = np.asarray(
value, dtype=f"<S{len(value)}"
)
elif isinstance(value, np.ndarray) and value.dtype.type == np.str_:
# Convert to S string
data_type_h5 = re.sub(r"[a-zA-Z]", "S", value.dtype.str)
......
This diff is collapsed.
......@@ -622,6 +622,7 @@ int Plan::get_nr_timesteps(int baseline, int n) const {
}
int Plan::get_max_nr_timesteps_subgrid() const {
if (!metadata.size()) return 0;
auto max_nr_timesteps = metadata[0].nr_timesteps;
for (const Metadata& m : metadata) {
if (m.nr_timesteps > max_nr_timesteps) {
......
......@@ -194,6 +194,11 @@ void Proxy::calibrate_init(
// subgrid_size, frequencies, visibilities, uvw, baselines,
// grid, aterms, aterms_offsets, spheroidal);
float image_size = grid.get_x_dim() * cell_size;
auto grid_shared_ptr =
std::shared_ptr<Grid>(const_cast<Grid*>(&grid), [](Grid*) {});
set_grid(grid_shared_ptr, subgrid_size, image_size, w_step, shift.data());
int nr_w_layers;
if (w_step != 0.0) {
......@@ -304,6 +309,7 @@ void Proxy::calibrate_init(
kernel_size, subgrid_size, frequencies,
std::move(visibilities1), std::move(weights1),
std::move(uvw1), std::move(baselines1), grid, spheroidal);
set_grid(nullptr);
}
void Proxy::calibrate_update(
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment