L2SS-480: merge with branch L2SS-497

docker-compose/device-beam.yml

@@ -4,6 +4,9 @@
 #
 version: '2'
 
+volumes:
+  iers-data: {}
+
 services:
   device-beam:
     image: device-beam
@@ -22,6 +25,7 @@ services:
       - "host.docker.internal:host-gateway"
     volumes:
       - ..:/opt/lofar/tango:rw
+      - iers-data:/opt/IERS
     environment:
       - TANGO_HOST=${TANGO_HOST}
     working_dir: /opt/lofar/tango

docker-compose/lofar-device-base/Dockerfile

@@ -6,5 +6,11 @@ RUN sudo apt-get update && sudo apt-get install -y git && sudo apt-get clean
 COPY lofar-requirements.txt /lofar-requirements.txt
 RUN sudo pip3 install -r /lofar-requirements.txt
 
+# install and use ephimerides and geodetic ("measures") tables for casacore
+RUN sudo mkdir -p /opt/IERS && sudo chmod a+rwx /opt/IERS
+RUN IERS_DIRNAME=IERS-`date +%FT%T` && mkdir -p /opt/IERS/${IERS_DIRNAME} && ln -sfT /opt/IERS/${IERS_DIRNAME} /opt/IERS/current
+RUN cd /opt/IERS/current && curl ftp://ftp.astron.nl/outgoing/Measures/WSRT_Measures.ztar | tar xz
+COPY casarc /home/tango/.casarc
+
 ENV TANGO_LOG_PATH=/var/log/tango
 RUN sudo mkdir -p /var/log/tango && sudo chmod a+rwx /var/log/tango

docker-compose/lofar-device-base/casarc

+measures.directory: /opt/IERS/current

tangostationcontrol/requirements.txt

@@ -11,4 +11,4 @@ h5py >= 3.5.0 # BSD
 psutil >= 5.8.0 # BSD
 docker >= 5.0.3 # Apache 2
 python-logstash-async >= 2.3.0 # MIT
-python-casacore
+python-casacore >= 3.3.1 # GPL2

tangostationcontrol/tangostationcontrol/common/measures.py

+# -*- coding: utf-8 -*-
+#
+# Distributed under the terms of the APACHE license.
+# See LICENSE.txt for more info.
+
+""" Utility functions for managing the casacore 'measures' tables.
+
+The 'measures' tables contain the ephemerides and geodetic calibrations
+as input for its time and space calculations. These tables are externally
+available (see MEASURES_URL).
+
+Casacore is expected to be configured to look in /opt/IERS/current for
+its 'measures' tables, through setting this in the ~/.casarc file as:
+
+    measures.directory: /opt/IERS/current
+
+This can be verified by running the 'findmeastable' utility, which
+is part of the 'casacore-tools' package.
+
+Periodically new measures need to be installed, especially if a leap
+second is introduced. Measures are maintained in directories called
+/opt/IERS/IERS-YYYY-MM-DDTHH:MM:SS, and /opt/IERS/current is a symlink
+to the active set.
+
+Usage:
+
+The download_measures() function can be used to download new measures,
+which can then be activated using use_measures_directory(). If
+casacore.measures already accessed the measures, the python program
+needs to be restarted in order to clear the cache.
+
+"""
+
+import pathlib
+import urllib.request
+import tarfile
+import datetime
+import os
+import sys
+
+# Where to store the measures table sets
+IERS_ROOTDIR = "/opt/IERS"
+
+# Where to download files to
+DOWNLOAD_DIR = "/tmp"
+
+# Where new measures can be downloaded
+MEASURES_URL = "ftp://ftp.astron.nl/outgoing/Measures/WSRT_Measures.ztar"
+
+def get_measures_directory():
+    """ Return the directory of the current measures table in use. """
+
+    return str(pathlib.Path(IERS_ROOTDIR, "current").resolve())
+
+def use_measures_directory(newdir):
+    """ Select a new set of measures tables to use.
+    
+        NOTE: Python must be restarted if the 'casacore.measures' module
+              already loaded the measures table before this switch.
+
+              The 'restart_python()' function can be used for this purpose.
+    """
+
+    newdir = pathlib.Path(newdir)
+
+    # newdir must be one of the available measures
+    if str(newdir) not in get_available_measures_directories():
+        raise ValueError(f"Target is not an available measures directory: {newdir}")
+
+    # be sure newdir must point to a directory containing measures
+    for subdir in ['ephemerides', 'geodetic']:
+        subdir = pathlib.Path(newdir, subdir)
+
+        if not subdir.is_dir():
+            raise ValueError(f"Subdirectory {subdir} does not exist")
+
+    # switch to new directory
+    current_symlink = pathlib.Path(IERS_ROOTDIR, "current")
+    if current_symlink.exists():
+        current_symlink.unlink()
+    current_symlink.symlink_to(newdir)
+
+def restart_python():
+    """ Force a restart this python program. 
+    
+        This function does not return. """
+
+    exe_path = pathlib.Path(sys.executable)
+    os.execv(exe_path, [exe_path.name] + sys.argv)
+
+    # NOTE: Python 3.4+ closes all file descriptors > 2 automatically, see https://www.python.org/dev/peps/pep-0446/
+
+def get_available_measures_directories() -> list:
+    """ Returns the set of installed measures tables. """
+    return [str(d) for d in pathlib.Path(IERS_ROOTDIR).glob("IERS-*") if d.is_dir() and not d.is_symlink()]
+
+def download_measures() -> str:
+    """ Download new measures and return the directory in which they were installed.
+    """
+
+    # create target directory for new measures
+    now = datetime.datetime.now()
+    iers_dir = pathlib.Path(now.strftime(f"{IERS_ROOTDIR}/IERS-%FT%T"))
+    iers_dir.mkdir()
+
+    try:
+        measures_filename = pathlib.Path(DOWNLOAD_DIR, "WSRT_Measures.ztar")
+
+        # download measures
+        urllib.request.urlretrieve(MEASURES_URL, str(measures_filename))
+
+        # untar measures
+        tarball = tarfile.open(str(measures_filename))
+        tarball.extractall(path=str(iers_dir))
+
+        # remove download
+        measures_filename.unlink()
+    except Exception as e:
+        # don't linger our new directory if we could not install measures in it
+        iers_dir.rmdir()
+
+        raise
+
+    return str(iers_dir)

tangostationcontrol/tangostationcontrol/devices/beam.py

@@ -8,10 +8,13 @@
 """
 
 # PyTango imports
-from tango import AttrWriteType, DevState
-from tango import DebugIt
-from tango.server import command
+from tango.server import attribute, command
+from tango import AttrWriteType, DebugIt
 import numpy
+import pathlib
+import urllib.request
+import tarfile
+import datetime
 # Additional import
 
 from tangostationcontrol.devices.device_decorators import *
@@ -19,7 +22,10 @@ from tangostationcontrol.common.entrypoint import entry
 from tangostationcontrol.clients.attribute_wrapper import attribute_wrapper
 from tangostationcontrol.devices.lofar_device import lofar_device
 from tangostationcontrol.common.lofar_logging import device_logging_to_python, log_exceptions
-from tangostationcontrol.beam.delays import delay_calculator
+from tangostationcontrol.common.measures import get_measures_directory, use_measures_directory, download_measures, restart_python, get_available_measures_directories
+
+import logging
+logger = logging.getLogger()
 
 __all__ = ["Beam", "main"]
 
@@ -34,6 +40,12 @@ class Beam(lofar_device):
     # Attributes
     # ----------
 
+    # Directory where the casacore measures that we use, reside. We configure ~/.casarc to
+    # use the symlink /opt/IERS/current, which we switch to the actual set of files to use.
+    measures_directory_R = attribute(dtype=str, access=AttrWriteType.READ, fget = lambda self: get_measures_directory())
+
+    # List of installed measures (the latest 64, anyway)
+    measures_directories_available_R = attribute(dtype=(str,), max_dim_x=64, access=AttrWriteType.READ, fget = lambda self: sorted(get_available_measures_directories())[-64:])
 
     # --------
     # overloaded functions
@@ -43,8 +55,37 @@ class Beam(lofar_device):
     # --------
     # Commands
     # --------
-    pass
 
+    @command(dtype_out=str, doc_out="Name of newly installed measures directory")
+    @DebugIt()
+    @log_exceptions()
+    def download_measures(self):
+        """ Download new measures tables into /opt/IERS.
+        
+            NOTE: This may take a while to complete. You are advised to increase
+                  the timeout of the proxy using `my_device.set_timeout_millis(10000)`. """
+
+        return download_measures()
+
+    @command(dtype_in=str, doc_in="Measures directory to activate")
+    @DebugIt()
+    @log_exceptions()
+    def use_measures(self, newdir):
+        """ Activate an installed set of measures tables.
+        
+            NOTE: This will turn off and restart this device!! """
+
+        # switch to requested measures
+        use_measures_directory(newdir)
+        logger.info(f"Switched measures table to {newdir}")
+
+        # turn off our device, to prepare for a python restart
+        self.Off()
+
+        # restart this program to force casacore to adopt
+        # the new tables
+        logger.warning("Restarting device to activate new measures tables")
+        restart_python()
 
 # ----------
 # Run server

tangostationcontrol/tangostationcontrol/test/common/test_measures.py

+# -*- coding: utf-8 -*-
+#
+# This file is part of the LOFAR 2.0 Station Software
+#
+#
+#
+# Distributed under the terms of the APACHE license.
+# See LICENSE.txt for more info.
+
+import urllib.request
+import os.path
+import datetime
+from unittest import mock
+import shutil
+import tempfile
+import time
+
+from tangostationcontrol.common import measures
+
+from tangostationcontrol.test import base
+
+# where our WSRT_Measures.ztar surrogate is located
+fake_measures = os.path.dirname(__file__) + "/fake_measures.ztar"
+
+class TestMeasures(base.TestCase):
+    @mock.patch.object(urllib.request, 'urlretrieve')
+    def test_download_and_use(self, m_urlretrieve):
+        """ Test downloading and using new measures tables. """
+
+        with tempfile.TemporaryDirectory() as tmpdirname, \
+             mock.patch('tangostationcontrol.common.measures.IERS_ROOTDIR', tmpdirname) as rootdir, \
+             mock.patch('tangostationcontrol.common.measures.DOWNLOAD_DIR', tmpdirname) as downloaddir:
+                
+            # emulate the download
+            m_urlretrieve.side_effect = lambda *args, **kw: shutil.copyfile(fake_measures, tmpdirname + "/WSRT_Measures.ztar")
+
+            # 'download' and process our fake measures
+            newdir = measures.download_measures()
+
+            # active them
+            measures.use_measures_directory(newdir)
+
+            # check if they're activated
+            self.assertIn(newdir, measures.get_available_measures_directories())
+            self.assertEqual(newdir, measures.get_measures_directory())
+
+    @mock.patch.object(urllib.request, 'urlretrieve')
+    def test_switch_tables(self, m_urlretrieve):
+        """ Test switching between available sets of measures tables. """
+
+        with tempfile.TemporaryDirectory() as tmpdirname, \
+             mock.patch('tangostationcontrol.common.measures.IERS_ROOTDIR', tmpdirname) as rootdir, \
+             mock.patch('tangostationcontrol.common.measures.DOWNLOAD_DIR', tmpdirname) as downloaddir:
+                
+            # emulate the download
+            m_urlretrieve.side_effect = lambda *args, **kw: shutil.copyfile(fake_measures, tmpdirname + "/WSRT_Measures.ztar")
+
+            # 'download' two measures with different timestamps
+            newdir1 = measures.download_measures()
+            time.sleep(1)
+            newdir2 = measures.download_measures()
+
+            # check if both are available
+            self.assertIn(newdir1, measures.get_available_measures_directories())
+            self.assertIn(newdir2, measures.get_available_measures_directories())
+
+            # switch between the two
+            measures.use_measures_directory(newdir1)
+            self.assertEqual(newdir1, measures.get_measures_directory())
+            measures.use_measures_directory(newdir2)
+            self.assertEqual(newdir2, measures.get_measures_directory())
+            measures.use_measures_directory(newdir1)
+            self.assertEqual(newdir1, measures.get_measures_directory())