Skip to content
Snippets Groups Projects
Commit 1f106361 authored by Corné Lukken's avatar Corné Lukken
Browse files

Resolve L2SS-2213 "Generate caltables"

parent fa9f9c96
No related branches found
No related tags found
1 merge request!1091Resolve L2SS-2213 "Generate caltables"
...@@ -150,6 +150,7 @@ Next change the version in the following places: ...@@ -150,6 +150,7 @@ Next change the version in the following places:
through [https://git.astron.nl/lofar2.0/tango/-/tags](Deploy Tags) through [https://git.astron.nl/lofar2.0/tango/-/tags](Deploy Tags)
# Release Notes # Release Notes
* 0.51.1 Generate caltables from LOFAR1 data or by generating dummies
* 0.51.0 Add Calibration service to gRPC server * 0.51.0 Add Calibration service to gRPC server
* 0.50.1 Generate and combine CDB files for all stations in Gitlab CI/CD * 0.50.1 Generate and combine CDB files for all stations in Gitlab CI/CD
* 0.50.0 Add pqr to etrs rotation matrix to metadata * 0.50.0 Add pqr to etrs rotation matrix to metadata
......
0.50.1 0.50.2
from datetime import datetime
import logging import logging
from glob import glob from glob import glob
from os import path from os import path
...@@ -62,6 +63,7 @@ class CalibrationTable: ...@@ -62,6 +63,7 @@ class CalibrationTable:
_HEADER_LINE_PATTERN = ( _HEADER_LINE_PATTERN = (
r"(^[A-z]*\.[A-z]*\.[A-z]*\s=\s.*$)|(^[A-z]*\.[A-z]*\s=\s.*$)" r"(^[A-z]*\.[A-z]*\.[A-z]*\s=\s.*$)|(^[A-z]*\.[A-z]*\s=\s.*$)"
) )
_LINK_LINE_PATTERN = r"link (CalTable-[0-9]{3}-(LBA|HBA)\_(INNER|OUTER|SPARSE_(ODD|EVEN))-[0-9]{2,3}\_[0-9]{2,3}\.dat)$"
_FREQUENCIES = 512 _FREQUENCIES = 512
_FLOATS_PER_FREQUENCY = 2 _FLOATS_PER_FREQUENCY = 2
_N_ANTENNAS_DUTCH = 96 _N_ANTENNAS_DUTCH = 96
...@@ -82,10 +84,48 @@ class CalibrationTable: ...@@ -82,10 +84,48 @@ class CalibrationTable:
observation_band: str = "" observation_band: str = ""
@staticmethod @staticmethod
def load_from_file(file_path): def generate_dummy(
observation_station: str, observation_antennaset: str, observation_band: str
):
logger.warning(
"generating dummy calibration for %s %s_%s due to missing LOFAR1 files",
observation_station,
observation_antennaset,
observation_band,
)
header: dict = {
"calibration_name": "Dummy",
"calibration_date": datetime.now().strftime("%Y%m%d"),
"calibration_version": "0",
"calibration_ppsdelay": "[0]",
"observation_mode": "0",
"observation_source": "None",
"observation_date": datetime.now().strftime("%Y%m%d%H%M"),
"observation_station": observation_station,
"observation_antennaset": observation_antennaset,
"observation_band": observation_band,
}
if "CS" in observation_station or "RS" in observation_station:
data = CalibrationTable._generate_data(CalibrationTable._N_ANTENNAS_DUTCH)
else:
data = CalibrationTable._generate_data(
CalibrationTable._N_ANTENNAS_INTERNATIONAL
)
return CalibrationTable(**header, data=data)
@staticmethod
def load_from_file(directory: str, filename: str):
file_path = f"{directory}/{filename}"
logger.info("loading file %s", file_path) logger.info("loading file %s", file_path)
with open(file_path, "rb") as file_stream: with open(file_path, "rb") as file_stream:
header = CalibrationTable._extract_header(file_stream) header = CalibrationTable._extract_header(file_stream)
if "link" in header:
logger.warning(
"linked file, using file %s as replacement for %s",
header["link"],
file_path,
)
return CalibrationTable.load_from_file(directory, header["link"])
data_raw = file_stream.read().rstrip(b"\n") data_raw = file_stream.read().rstrip(b"\n")
try: try:
data = CalibrationTable._parse_data(data_raw) data = CalibrationTable._parse_data(data_raw)
...@@ -180,7 +220,7 @@ class CalibrationTable: ...@@ -180,7 +220,7 @@ class CalibrationTable:
f_stream.write(data_packed) f_stream.write(data_packed)
@staticmethod @staticmethod
def _extract_header(fstream: BinaryIO): def _extract_header(fstream: BinaryIO) -> dict:
header = {} header = {}
for _ in range(CalibrationTable._MAX_HEADER_LINES): for _ in range(CalibrationTable._MAX_HEADER_LINES):
line = fstream.readline().decode("utf8").rstrip("\n") line = fstream.readline().decode("utf8").rstrip("\n")
...@@ -197,6 +237,10 @@ class CalibrationTable: ...@@ -197,6 +237,10 @@ class CalibrationTable:
) )
value = value.strip() value = value.strip()
header[key] = value header[key] = value
elif fullmatch(CalibrationTable._LINK_LINE_PATTERN, line):
key, value = line.split(" ")
header[key] = value
break
else: else:
logger.error('unrecognized line "%s"', line) logger.error('unrecognized line "%s"', line)
raise InvalidFileException('unrecognized line "%s"' % line) raise InvalidFileException('unrecognized line "%s"' % line)
...@@ -267,6 +311,16 @@ class CalibrationTable: ...@@ -267,6 +311,16 @@ class CalibrationTable:
return complex_data return complex_data
@staticmethod
def _generate_data(n_antennas: int):
complex_data = empty_ndarray(
[CalibrationTable._FREQUENCIES, n_antennas], dtype=complex
)
for elem in complex_data:
elem.real = 1.0
elem.imag = 0.0
return complex_data
def __post_init__(self): def __post_init__(self):
self._parse_header() self._parse_header()
......
...@@ -22,9 +22,9 @@ class LOFAR1CalTables(dict): ...@@ -22,9 +22,9 @@ class LOFAR1CalTables(dict):
} }
""" """
def __init__(self, station, tmpdir="."): def __init__(self, station: str, tmpdir: str = "."):
self.station = station self.station: str = station
self.tmpdir = tmpdir self.tmpdir: str = tmpdir
self.files = [ self.files = [
f"CalTable-{self.station_nr:03}-HBA-110_190.dat", f"CalTable-{self.station_nr:03}-HBA-110_190.dat",
...@@ -34,25 +34,45 @@ class LOFAR1CalTables(dict): ...@@ -34,25 +34,45 @@ class LOFAR1CalTables(dict):
f"CalTable-{self.station_nr:03}-LBA_OUTER-10_90.dat", f"CalTable-{self.station_nr:03}-LBA_OUTER-10_90.dat",
] ]
self.download_all() failed = self.download_all()
self.parse_all() self.parse_files(list(filter(lambda v: v not in failed, self.files)))
self.generate_failed(failed)
def extract_file_info(self, file: str) -> tuple[str, str]:
filename, ext = file.rsplit(".", 1)
_caltable, _stationnr, antennaset, band = filename.split("-")
return (
antennaset,
band,
)
@property @property
def station_nr(self): def station_nr(self):
return int(self.station[2:]) return int(self.station[2:])
def download_all(self): def download_all(self) -> list[str]:
failed: list[str] = []
for filename in self.files: for filename in self.files:
with open(f"{self.tmpdir}/{filename}", "wb") as f: with open(f"{self.tmpdir}/{filename}", "wb") as f:
url = f"https://git.astron.nl/ro/lofar1-caltables/-/raw/main/{self.station.upper()}/{filename}" url = f"https://git.astron.nl/ro/lofar1-caltables/-/raw/main/{self.station.upper()}/{filename}"
try:
data = urllib.request.urlopen(url).read() data = urllib.request.urlopen(url).read()
f.write(data) f.write(data)
except Exception:
failed.append(filename)
return failed
def parse_all(self): def parse_files(self, files: list[str]):
for f in self.files: for f in files:
filename, ext = f.rsplit(".", 1) antennaset, band = self.extract_file_info(f)
_caltable, _stationnr, antennaset, band = filename.split("-")
self[f"{antennaset}-{band}"] = CalibrationTable.load_from_file( self[f"{antennaset}-{band}"] = CalibrationTable.load_from_file(
f"{self.tmpdir}/{f}" self.tmpdir, f
)
def generate_failed(self, files: list[str]):
for f in files:
antennaset, band = self.extract_file_info(f)
self[f"{antennaset}-{band}"] = CalibrationTable.generate_dummy(
self.station, antennaset, band
) )
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment