Merge branch 'idg-cal-jakob-1' into 'master'

Idg cal jakob 1 (rebased)

See merge request !51

.gitlab-ci.yml

@@ -131,8 +131,8 @@ idg-integration-docker:
     - cd build
     # Run in verbose mode -VV as long as it's not properly working
     - ctest -L integration
-    # Test the h5parm writer
-    - cd $HOME_DIR/scripts/test
+    # Test idg-cal scripts
+    - cd $HOME_DIR/idg-cal/unit_tests
     - pytest .
   rules:
     - when: on_success
@@ -185,8 +185,8 @@ idg-integration-das:
     # Run integration tests
     - "python3 -m pip install --user --only-binary=:all: python-casacore"
     - ctest --verbose -L integration
-    # Test the h5parm writer
-    - cd $CURRENT_SOURCE/scripts/test
+    # Test idg-cal scripts
+    - cd $CURRENT_SOURCE/idg-cal/unit_tests
     - pytest .
   rules:
     - when: on_success

CMakeLists.txt

@@ -44,10 +44,10 @@ option (BUILD_WITH_PYTHON "Build Python bindings" OFF)
 
 if(${CMAKE_VERSION} VERSION_GREATER "3.12.4")
 find_package(Python REQUIRED)
-set(PYTHON_INSTALL_DIR ${CMAKE_INSTALL_PREFIX}/lib/python${Python_VERSION_MAJOR}.${Python_VERSION_MINOR}/dist-packages/idg)
+set(PYTHON_INSTALL_DIR ${CMAKE_INSTALL_PREFIX}/lib/python${Python_VERSION_MAJOR}.${Python_VERSION_MINOR}/site-packages/idg)
 else()
 find_package(PythonInterp REQUIRED)
-set(PYTHON_INSTALL_DIR ${CMAKE_INSTALL_PREFIX}/lib/python${PYTHON_VERSION_MAJOR}.${PYTHON_VERSION_MINOR}/dist-packages/idg)
+set(PYTHON_INSTALL_DIR ${CMAKE_INSTALL_PREFIX}/lib/python${PYTHON_VERSION_MAJOR}.${PYTHON_VERSION_MINOR}/site-packages/idg)
 endif()
 
 if (BUILD_TESTING)
@@ -58,6 +58,7 @@ add_subdirectory("idg-util")
 add_subdirectory("idg-lib")
 add_subdirectory("idg-bin")
 add_subdirectory("idg-api")
+add_subdirectory("idg-cal")
 
 # Write environment-module
 configure_file (

idg-cal/CMakeLists.txt

+# Copyright (C) 2020 ASTRON (Netherlands Institute for Radio Astronomy)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# Install Python modules.
+install(FILES
+#        __init__.py
+        h5parmwriter.py
+        idgcaldpstep.py
+
+        DESTINATION
+        ${PYTHON_INSTALL_DIR})
\ No newline at end of file

scripts/h5parmwriter.py → idg-cal/h5parmwriter.py

@@ -21,7 +21,9 @@ class H5ParmWriter:
     radio astronomical data.
     """
 
-    def __init__(self, file_name, solution_set_name="sol000", attributes=None):
+    def __init__(
+        self, file_name, solution_set_name="sol000", attributes=None, overwrite=True
+    ):
         """
         Initialize H5Parmwriter. The file, specified by file_name, should not yet
         exist. If it does, an error is thrown.
@@ -36,7 +38,7 @@ class H5ParmWriter:
             Dictionary containing attributes for the sol set
         """
 
-        if os.path.isfile(file_name):
+        if os.path.isfile(file_name) and not overwrite:
             raise ValueError(
                 f"File {file_name} already exists, choose a different name!"
             )
@@ -161,11 +163,15 @@ class H5ParmWriter:
 
         hdf_location = self.solution_set + "/" + name
         self.h5file[self.solution_set].create_group(name)
-        self.h5file[hdf_location].attrs["TITLE"] = str.encode(type_name)
+        self.h5file[hdf_location].attrs["TITLE"] = np.asarray(
+            type_name, dtype=f"<S{len(type_name)}"
+        )
         self.__write_version_stamp(self.solution_set + "/" + name)
 
+        # Convert axes labels to S-type
         axes_labels = list(axes.keys())
         axes_labels = ",".join(axes_labels).encode()
+        axes_labels = np.asarray(axes_labels, dtype=f"<S{len(axes_labels)}")
         axes_shape = tuple(axes.values())
 
         dtype = values.dtype if values is not None else dtype
@@ -203,7 +209,9 @@ class H5ParmWriter:
         self.h5file[hdf_location + "/weight"][:] = weights
 
         if history is not None:
-            self.h5file[hdf_location + "/val"].attrs["HISTORY000"] = str.encode(history)
+            self.h5file[hdf_location + "/val"].attrs["HISTORY000"] = np.asarray(
+                history, dtype=f"<S{len(history)}"
+            )
 
     def fill_solution_table(self, name, values, offset=None):
         """
@@ -315,7 +323,7 @@ class H5ParmWriter:
             if hdf_location + "/" + axis not in self.h5file:
                 # Create axis meta data set
                 self.h5file[hdf_location].create_dataset(
-                    axis, meta_data.size, data_type_h5
+                    axis, meta_data.shape, data_type_h5
                 )
                 self.h5file[hdf_location + "/" + axis][:] = meta_data.astype(
                     data_type_h5
@@ -357,7 +365,9 @@ class H5ParmWriter:
         hdf_location : str
             Location where the version stamp will be added.
         """
-        self.h5file[hdf_location].attrs["h5parm_version"] = str.encode(H5PARM_VERSION)
+        self.h5file[hdf_location].attrs["h5parm_version"] = np.asarray(
+            H5PARM_VERSION, dtype=f"<S{len(H5PARM_VERSION)}"
+        )
 
     def __write_attributes(self, hdf_location, attributes, overwrite):
         """
@@ -382,7 +392,9 @@ class H5ParmWriter:
             else:
                 if isinstance(value, str):
                     # Byte encoded string
-                    self.h5file[hdf_location].attrs[key] = str.encode(value)
+                    self.h5file[hdf_location].attrs[key] = np.asarray(
+                        value, dtype=f"<S{len(value)}"
+                    )
                 elif isinstance(value, np.ndarray) and value.dtype.type == np.str_:
                     # Convert to S string
                     data_type_h5 = re.sub(r"[a-zA-Z]", "S", value.dtype.str)

idg-lib/src/common/Plan.cpp

@@ -622,6 +622,7 @@ int Plan::get_nr_timesteps(int baseline, int n) const {
 }
 
 int Plan::get_max_nr_timesteps_subgrid() const {
+  if (!metadata.size()) return 0;
   auto max_nr_timesteps = metadata[0].nr_timesteps;
   for (const Metadata& m : metadata) {
     if (m.nr_timesteps > max_nr_timesteps) {

idg-lib/src/common/Proxy.cpp

@@ -194,6 +194,11 @@ void Proxy::calibrate_init(
   //    subgrid_size, frequencies, visibilities, uvw, baselines,
   //    grid, aterms, aterms_offsets, spheroidal);
 
+  float image_size = grid.get_x_dim() * cell_size;
+  auto grid_shared_ptr =
+      std::shared_ptr<Grid>(const_cast<Grid*>(&grid), [](Grid*) {});
+  set_grid(grid_shared_ptr, subgrid_size, image_size, w_step, shift.data());
+
   int nr_w_layers;
 
   if (w_step != 0.0) {
@@ -304,6 +309,7 @@ void Proxy::calibrate_init(
                     kernel_size, subgrid_size, frequencies,
                     std::move(visibilities1), std::move(weights1),
                     std::move(uvw1), std::move(baselines1), grid, spheroidal);
+  set_grid(nullptr);
 }
 
 void Proxy::calibrate_update(