diff --git a/docker-compose/device-beam.yml b/docker-compose/device-beam.yml index 97385f16492ec123044033713d0c7b835d2062fd..b7572dda6db2dbebcff62fc90bbb1e99b04b8000 100644 --- a/docker-compose/device-beam.yml +++ b/docker-compose/device-beam.yml @@ -4,6 +4,9 @@ # version: '2' +volumes: + iers-data: {} + services: device-beam: image: device-beam @@ -22,6 +25,7 @@ services: - "host.docker.internal:host-gateway" volumes: - ..:/opt/lofar/tango:rw + - iers-data:/opt/IERS environment: - TANGO_HOST=${TANGO_HOST} working_dir: /opt/lofar/tango diff --git a/docker-compose/lofar-device-base/Dockerfile b/docker-compose/lofar-device-base/Dockerfile index 34dde933732b08f6986149210fa9a497fb371737..f81445d208272885c579e3da18c3417ca5e7473d 100644 --- a/docker-compose/lofar-device-base/Dockerfile +++ b/docker-compose/lofar-device-base/Dockerfile @@ -6,5 +6,14 @@ RUN sudo apt-get update && sudo apt-get install -y git && sudo apt-get clean COPY lofar-requirements.txt /lofar-requirements.txt RUN sudo pip3 install -r /lofar-requirements.txt +# install and use ephimerides and geodetic ("measures") tables for casacore. +# we install a _stub_ since the tables need to be deployed explicitly from within the software. +RUN sudo mkdir -p /opt/IERS && sudo chmod a+rwx /opt/IERS +ARG IERS_DIRNAME=IERS-1970-01-01T00:00:00-stub +COPY WSRT_Measures_stub /opt/IERS/${IERS_DIRNAME} +RUN ln -sfT /opt/IERS/${IERS_DIRNAME} /opt/IERS/current + +COPY casarc /home/tango/.casarc + ENV TANGO_LOG_PATH=/var/log/tango RUN sudo mkdir -p /var/log/tango && sudo chmod a+rwx /var/log/tango diff --git a/docker-compose/lofar-device-base/WSRT_Measures_stub/geodetic/TAI_UTC/table.dat b/docker-compose/lofar-device-base/WSRT_Measures_stub/geodetic/TAI_UTC/table.dat new file mode 100644 index 0000000000000000000000000000000000000000..bc00ed3ee6d2eed34d9b02f755c07429a78a0980 Binary files /dev/null and b/docker-compose/lofar-device-base/WSRT_Measures_stub/geodetic/TAI_UTC/table.dat differ diff --git a/docker-compose/lofar-device-base/WSRT_Measures_stub/geodetic/TAI_UTC/table.f0 b/docker-compose/lofar-device-base/WSRT_Measures_stub/geodetic/TAI_UTC/table.f0 new file mode 100644 index 0000000000000000000000000000000000000000..60f2139e506a2e278c9600170c1013488aa42201 Binary files /dev/null and b/docker-compose/lofar-device-base/WSRT_Measures_stub/geodetic/TAI_UTC/table.f0 differ diff --git a/docker-compose/lofar-device-base/WSRT_Measures_stub/geodetic/TAI_UTC/table.info b/docker-compose/lofar-device-base/WSRT_Measures_stub/geodetic/TAI_UTC/table.info new file mode 100644 index 0000000000000000000000000000000000000000..62db44592e98cff62f07dbc6c4c9092dd1ab81d6 Binary files /dev/null and b/docker-compose/lofar-device-base/WSRT_Measures_stub/geodetic/TAI_UTC/table.info differ diff --git a/docker-compose/lofar-device-base/WSRT_Measures_stub/geodetic/TAI_UTC/table.lock b/docker-compose/lofar-device-base/WSRT_Measures_stub/geodetic/TAI_UTC/table.lock new file mode 100644 index 0000000000000000000000000000000000000000..87dc3b3fdb54598a55a3cedcb6ca1b35081d70d4 Binary files /dev/null and b/docker-compose/lofar-device-base/WSRT_Measures_stub/geodetic/TAI_UTC/table.lock differ diff --git a/docker-compose/lofar-device-base/casarc b/docker-compose/lofar-device-base/casarc new file mode 100644 index 0000000000000000000000000000000000000000..78e7a19ed26fd4aa5b2fd7aad6f0b3bd4179f6f8 --- /dev/null +++ b/docker-compose/lofar-device-base/casarc @@ -0,0 +1 @@ +measures.directory: /opt/IERS/current diff --git a/tangostationcontrol/requirements.txt b/tangostationcontrol/requirements.txt index 1e85d16c127bdf5e4351b63b7be94078a2569a41..f8bad33a66897ce01bfb5084430a9e7af8ce621b 100644 --- a/tangostationcontrol/requirements.txt +++ b/tangostationcontrol/requirements.txt @@ -11,3 +11,4 @@ h5py >= 3.1.0 # BSD psutil >= 5.8.0 # BSD docker >= 5.0.3 # Apache 2 python-logstash-async >= 2.3.0 # MIT +python-casacore >= 3.3.1 # GPL2 diff --git a/tangostationcontrol/tangostationcontrol/common/measures.py b/tangostationcontrol/tangostationcontrol/common/measures.py new file mode 100644 index 0000000000000000000000000000000000000000..58b3c90f456ef3060d1431a7809395e1e63be237 --- /dev/null +++ b/tangostationcontrol/tangostationcontrol/common/measures.py @@ -0,0 +1,143 @@ +# -*- coding: utf-8 -*- +# +# Distributed under the terms of the APACHE license. +# See LICENSE.txt for more info. + +""" Utility functions for managing the casacore 'measures' tables. + +The 'measures' tables contain the ephemerides and geodetic calibrations +as input for its time and space calculations. These tables are externally +available (see MEASURES_URL). + +Casacore is expected to be configured to look in /opt/IERS/current for +its 'measures' tables, through setting this in the ~/.casarc file as: + + measures.directory: /opt/IERS/current + +This can be verified by running the 'findmeastable' utility, which +is part of the 'casacore-tools' package. + +Periodically new measures need to be installed, especially if a leap +second is introduced. Measures are maintained in directories called +/opt/IERS/IERS-YYYY-MM-DDTHH:MM:SS, and /opt/IERS/current is a symlink +to the active set. + +Usage: + +The download_measures() function can be used to download new measures, +which can then be activated using use_measures_directory(). If +casacore.measures already accessed the measures, the python program +needs to be restarted in order to clear the cache. + +""" + +import pathlib +import urllib.request +import tarfile +import datetime +import os +import sys + +# Where to store the measures table sets +IERS_ROOTDIR = "/opt/IERS" + +# Where to download files to +DOWNLOAD_DIR = "/tmp" + +# Where new measures can be downloaded +MEASURES_URL = "ftp://ftp.astron.nl/outgoing/Measures/WSRT_Measures.ztar" + +def get_measures_directory(): + """ Return the directory of the current measures table in use. """ + + return str(pathlib.Path(IERS_ROOTDIR, "current").resolve()) + +def use_measures_directory(newdir): + """ Select a new set of measures tables to use. + + NOTE: Python must be restarted if the 'casacore.measures' module + already loaded the measures table before this switch. + + The 'restart_python()' function can be used for this purpose. + """ + + newdir = pathlib.Path(newdir) + + # newdir must be one of the available measures + if str(newdir) not in get_available_measures_directories(): + raise ValueError(f"Target is not an available measures directory: {newdir}") + + # be sure newdir must point to a directory containing measures + for subdir in ['ephemerides', 'geodetic']: + subdir = pathlib.Path(newdir, subdir) + + if not subdir.is_dir(): + raise ValueError(f"Subdirectory {subdir} does not exist") + + # switch to new directory + current_symlink = pathlib.Path(IERS_ROOTDIR, "current") + if current_symlink.exists(): + current_symlink.unlink() + current_symlink.symlink_to(newdir) + +def restart_python(): + """ Force a restart this python program. + + This function does not return. """ + + exe_path = pathlib.Path(sys.executable) + + # NOTE: Python 3.4+ closes all file descriptors > 2 automatically, see https://www.python.org/dev/peps/pep-0446/ + os.execv(exe_path, [exe_path.name] + sys.argv) + +def get_available_measures_directories() -> list: + """ Returns the set of installed measures tables. """ + return [str(d) for d in pathlib.Path(IERS_ROOTDIR).glob("IERS-*") if d.is_dir() and not d.is_symlink()] + +def download_measures() -> str: + """ Download new measures and return the directory in which they were installed. + """ + + # create target directory for new measures + now = datetime.datetime.now() + iers_dir_download = pathlib.Path(now.strftime(f"{IERS_ROOTDIR}/IERS-%FT%T")) + iers_dir_download.mkdir() + + try: + measures_filename = pathlib.Path(DOWNLOAD_DIR, "WSRT_Measures.ztar") + + # download measures + urllib.request.urlretrieve(MEASURES_URL, str(measures_filename)) + + # untar measures + tarball = tarfile.open(str(measures_filename)) + tarball.extractall(path=str(iers_dir_download)) + + # remove download + measures_filename.unlink() + except Exception as e: + # don't linger our new directory if we could not install measures in it + iers_dir_download.rmdir() + + raise + + # update the timestamp used in the directory name to reflect the time of the measures, + # not the time of download. + file_with_final_timestamp = pathlib.Path(iers_dir_download, "geodetic", "TAI_UTC") + mtime = datetime.datetime.fromtimestamp(file_with_final_timestamp.stat().st_mtime) + + iers_dir_final = pathlib.Path(mtime.strftime(f"{IERS_ROOTDIR}/IERS-%FT%T")) + + if iers_dir_final.exists(): + # these measures were already downloaded earlier, delete them and use ours, + # which allows the user to fix previously broken downloads. + try: + shutil.rmtree(iers_dir_final) + except Exception as e: + # move out of the way instead then + iers_dir_final.rename(iers_final.with_suffix("delete-me")) + + # update our name to reflect the correct timestamp + iers_dir_download.rename(iers_dir_final) + + return str(iers_dir_final) diff --git a/tangostationcontrol/tangostationcontrol/devices/beam.py b/tangostationcontrol/tangostationcontrol/devices/beam.py index fc407d485fa31ca3bc9f46470a2759552073fdf0..a87808b3946db0525d35a8999d064501cd3a3425 100644 --- a/tangostationcontrol/tangostationcontrol/devices/beam.py +++ b/tangostationcontrol/tangostationcontrol/devices/beam.py @@ -7,11 +7,18 @@ """ +# PyTango imports +from tango.server import attribute, command +from tango import AttrWriteType, DebugIt # Additional import from tangostationcontrol.common.entrypoint import entry from tangostationcontrol.devices.lofar_device import lofar_device -from tangostationcontrol.common.lofar_logging import device_logging_to_python +from tangostationcontrol.common.lofar_logging import device_logging_to_python, log_exceptions +from tangostationcontrol.common.measures import get_measures_directory, use_measures_directory, download_measures, restart_python, get_available_measures_directories + +import logging +logger = logging.getLogger() __all__ = ["Beam", "main"] @@ -26,7 +33,12 @@ class Beam(lofar_device): # Attributes # ---------- - pass + # Directory where the casacore measures that we use, reside. We configure ~/.casarc to + # use the symlink /opt/IERS/current, which we switch to the actual set of files to use. + measures_directory_R = attribute(dtype=str, access=AttrWriteType.READ, fget = lambda self: get_measures_directory()) + + # List of dowloaded measures (the latest 64, anyway) + measures_directories_available_R = attribute(dtype=(str,), max_dim_x=64, access=AttrWriteType.READ, fget = lambda self: sorted(get_available_measures_directories())[-64:]) # -------- # overloaded functions @@ -37,6 +49,36 @@ class Beam(lofar_device): # Commands # -------- + @command(dtype_out=str, doc_out="Name of newly installed measures directory") + @DebugIt() + @log_exceptions() + def download_measures(self): + """ Download new measures tables into /opt/IERS, but do not activate them. + + NOTE: This may take a while to complete. You are advised to increase + the timeout of the proxy using `my_device.set_timeout_millis(10000)`. """ + + return download_measures() + + @command(dtype_in=str, doc_in="Measures directory to activate") + @DebugIt() + @log_exceptions() + def use_measures(self, newdir): + """ Activate a downloaded set of measures tables. + + NOTE: This will turn off and restart this device!! """ + + # switch to requested measures + use_measures_directory(newdir) + logger.info(f"Switched measures table to {newdir}") + + # turn off our device, to prepare for a python restart + self.Off() + + # restart this program to force casacore to adopt + # the new tables + logger.warning("Restarting device to activate new measures tables") + restart_python() # ---------- # Run server diff --git a/tangostationcontrol/tangostationcontrol/test/common/fake_measures.ztar b/tangostationcontrol/tangostationcontrol/test/common/fake_measures.ztar new file mode 100644 index 0000000000000000000000000000000000000000..f7ca0f87529b5ba31e6620023e7e70c3f2bf5d99 Binary files /dev/null and b/tangostationcontrol/tangostationcontrol/test/common/fake_measures.ztar differ diff --git a/tangostationcontrol/tangostationcontrol/test/common/fake_measures_newer.ztar b/tangostationcontrol/tangostationcontrol/test/common/fake_measures_newer.ztar new file mode 100644 index 0000000000000000000000000000000000000000..dc392d0ed63fbba86dd5bf9003c6a9aad0277820 Binary files /dev/null and b/tangostationcontrol/tangostationcontrol/test/common/fake_measures_newer.ztar differ diff --git a/tangostationcontrol/tangostationcontrol/test/common/test_measures.py b/tangostationcontrol/tangostationcontrol/test/common/test_measures.py new file mode 100644 index 0000000000000000000000000000000000000000..bcb020a049a1d4e756623ba80c3e94fb526fb04e --- /dev/null +++ b/tangostationcontrol/tangostationcontrol/test/common/test_measures.py @@ -0,0 +1,71 @@ +# -*- coding: utf-8 -*- +# +# This file is part of the LOFAR 2.0 Station Software +# +# +# +# Distributed under the terms of the APACHE license. +# See LICENSE.txt for more info. + +import urllib.request +import os.path +from unittest import mock +import shutil +import tempfile + +from tangostationcontrol.common import measures + +from tangostationcontrol.test import base + +# where our WSRT_Measures.ztar surrogate is located +# two versions with different timestamps are provided +fake_measures = os.path.dirname(__file__) + "/fake_measures.ztar" +fake_measures_newer = os.path.dirname(__file__) + "/fake_measures_newer.ztar" + +class TestMeasures(base.TestCase): + @mock.patch.object(urllib.request, 'urlretrieve') + def test_download_and_use(self, m_urlretrieve): + """ Test downloading and using new measures tables. """ + + with tempfile.TemporaryDirectory() as tmpdirname, \ + mock.patch('tangostationcontrol.common.measures.IERS_ROOTDIR', tmpdirname) as rootdir, \ + mock.patch('tangostationcontrol.common.measures.DOWNLOAD_DIR', tmpdirname) as downloaddir: + + # emulate the download + m_urlretrieve.side_effect = lambda *args, **kw: shutil.copyfile(fake_measures, tmpdirname + "/WSRT_Measures.ztar") + + # 'download' and process our fake measures + newdir = measures.download_measures() + + # active them + measures.use_measures_directory(newdir) + + # check if they're activated + self.assertIn(newdir, measures.get_available_measures_directories()) + self.assertEqual(newdir, measures.get_measures_directory()) + + @mock.patch.object(urllib.request, 'urlretrieve') + def test_switch_tables(self, m_urlretrieve): + """ Test switching between available sets of measures tables. """ + + with tempfile.TemporaryDirectory() as tmpdirname, \ + mock.patch('tangostationcontrol.common.measures.IERS_ROOTDIR', tmpdirname) as rootdir, \ + mock.patch('tangostationcontrol.common.measures.DOWNLOAD_DIR', tmpdirname) as downloaddir: + + # 'download' two measures with different timestamps + m_urlretrieve.side_effect = lambda *args, **kw: shutil.copyfile(fake_measures, tmpdirname + "/WSRT_Measures.ztar") + newdir1 = measures.download_measures() + m_urlretrieve.side_effect = lambda *args, **kw: shutil.copyfile(fake_measures_newer, tmpdirname + "/WSRT_Measures.ztar") + newdir2 = measures.download_measures() + + # check if both are available + self.assertIn(newdir1, measures.get_available_measures_directories()) + self.assertIn(newdir2, measures.get_available_measures_directories()) + + # switch between the two + measures.use_measures_directory(newdir1) + self.assertEqual(newdir1, measures.get_measures_directory()) + measures.use_measures_directory(newdir2) + self.assertEqual(newdir2, measures.get_measures_directory()) + measures.use_measures_directory(newdir1) + self.assertEqual(newdir1, measures.get_measures_directory())