Task SSB-42: Store the data in HDF5 groups

CAL/CalibrationCommon/lib/datacontainers/holography_dataset.py

@@ -333,25 +333,28 @@ class HolographyDataset():
             result.frequencies = list(f[HDS_FREQUENCY])
 
             result.ra_dec = dict()
+            for frequency in f["RA_DEC"].keys():
+                for beamlet in f["RA_DEC"][frequency].keys():
+                    if frequency not in result.data:
+                        result.ra_dec[frequency] = dict()
+                    result.ra_dec[frequency][beamlet] = f["RA_DEC"][frequency][beamlet]
+
+            beamlets = set()
             result.data = dict()
-            result.beamlets = [i for i in range(f[HDS_DATA].shape[2])]
-            for reference_station_index, reference_station_name in enumerate(result.reference_stations):
-                if reference_station_name not in result.data:
-                    result.data[reference_station_name] = dict()
-                for frequency_index, frequency in enumerate(result.frequencies):
-                    if str(frequency) not in result.ra_dec:
-                            result.ra_dec[str(frequency)] = dict()
-                    if str(frequency) not in result.data[reference_station_name]:
-                            result.data[reference_station_name][str(frequency)] = dict()
-                    for beamlet in result.beamlets:
-                        result.ra_dec[str(frequency)][beamlet] = \
-                        tuple(f[HDS_RA_DEC][frequency_index, beamlet].tolist())
-                        
-                        result.data[reference_station_name][str(frequency)][beamlet] = \
-                        f[HDS_DATA][reference_station_index, frequency_index, beamlet]
-            #result.ra_dec = list(f[HDS_RA_DEC])
-            #result.data = f[HDS_DATA]
-            
+            for reference_station in f["CROSSCORRELATION"].keys():
+                for frequency in f["CROSSCORRELATION"][reference_station].keys():
+                    for beamlet in f["CROSSCORRELATION"][reference_station][frequency].keys():
+                        beamlets.add(int(beamlet))
+                        if reference_station not in result.data:
+                            result.data[reference_station] = dict()
+                        if frequency not in result.data[reference_station]:
+                            result.data[reference_station][frequency] = dict()
+                        result.data[reference_station][frequency][beamlet] = f["CROSSCORRELATION"][reference_station][frequency][beamlet]
+
+            result.beamlets = list(beamlets)
+        except Exception as e:
+            logger.exception("Cannot read the Holography Data Set data from the HDF5 file \"%s\".  This is the exception that was thrown:  %s", path, e)
+            raise e
         finally:
             if f is not None:
                 f.close()
@@ -395,30 +398,34 @@ class HolographyDataset():
             f.attrs[HDS_ROTATION_MATRIX] = self.rotation_matrix
             f.attrs[HDS_ANTENNA_FIELD_POSITION] = self.antenna_field_position
 
-            # Data tables
-            f[HDS_REFERENCE_STATION] = numpy.array(self.reference_stations,
-                                                   dtype = h5py.special_dtype(vlen = str))
-
-            f[HDS_FREQUENCY] = numpy.array(self.frequencies, dtype = float)
-
-            f[HDS_RA_DEC] = numpy.empty([len(self.frequencies),
-                                        len(self.beamlets)],
-                                        dtype = coordinate_type)
-            ra_dec_dset = f[HDS_RA_DEC] 
-            dataset = f.create_dataset("data",
-                                       (len(self.reference_stations),
-                                        len(self.frequencies),
-                                        len(self.beamlets)),
-                                       dtype = h5py.special_dtype(vlen = data_sample_type))
-            
+            # Store the list of reference stations and frequencies.  We just
+            # want to keep 'em around for quick reference.
+            f[HDS_REFERENCE_STATION] = self.reference_stations
+            f[HDS_FREQUENCY] = self.frequencies
+
+            # We create groups for the reference stations and the frequencies.
+            # Then we store the data samples [XX, YY, XY, YX, t, l, m, flag]
+            # in an array.  The reference station name, the frequency and the
+            # beamlet number (index of the data sample array) allow random
+            # access of the data.
             
-            for reference_station_index, reference_station in enumerate(self.reference_stations):
-                for frequency_index, frequency in enumerate(self.frequencies):
-                    for beamlet in self.beamlets:
-                        ra_dec_dset [frequency_index, beamlet] = \
-                            self.ra_dec[frequency][beamlet]
-                        dataset[reference_station_index, frequency_index, beamlet] = \
-                            self.data[reference_station][frequency][beamlet]
+            f.create_group("RA_DEC")
+            for frequency in self.ra_dec.keys():
+                f["RA_DEC"].create_group(frequency)
+                for beamlet in self.ra_dec[frequency].keys():
+                    f["RA_DEC"][frequency][beamlet] = self.ra_dec[frequency][beamlet]
+
+            f.create_group("CROSSCORRELATION")
+            for reference_station in self.data.keys():
+                f["CROSSCORRELATION"].create_group(reference_station)
+                for frequency in self.data[reference_station].keys():
+                    f["CROSSCORRELATION"][reference_station].create_group(frequency)
+                    for beamlet in self.data[reference_station][frequency].keys():
+                        
+                        f["CROSSCORRELATION"][reference_station][frequency][beamlet] = self.data[reference_station][frequency][beamlet]
+        except Exception as e:
+            logger.exception("Cannot write the Holography Data Set data to the HDF5 file \"%s\".  This is the exception that was thrown:  %s", path, e)
+            raise e
         finally:
             if f is not None:
                 f.close()