Merge branch 'master' into 'L2SS-575-pointing-attributes'

# Conflicts:
#   tangostationcontrol/tangostationcontrol/devices/beam.py

docker-compose/device-beam.yml

@@ -4,6 +4,9 @@
 #
 version: '2'
 
+volumes:
+  iers-data: {}
+
 services:
   device-beam:
     image: device-beam
@@ -22,6 +25,7 @@ services:
       - "host.docker.internal:host-gateway"
     volumes:
       - ..:/opt/lofar/tango:rw
+      - iers-data:/opt/IERS
     environment:
       - TANGO_HOST=${TANGO_HOST}
     working_dir: /opt/lofar/tango

docker-compose/lofar-device-base/Dockerfile

@@ -6,5 +6,14 @@ RUN sudo apt-get update && sudo apt-get install -y git && sudo apt-get clean
 COPY lofar-requirements.txt /lofar-requirements.txt
 RUN sudo pip3 install -r /lofar-requirements.txt
 
+# install and use ephimerides and geodetic ("measures") tables for casacore.
+# we install a _stub_ since the tables need to be deployed explicitly from within the software.
+RUN sudo mkdir -p /opt/IERS && sudo chmod a+rwx /opt/IERS
+ARG IERS_DIRNAME=IERS-1970-01-01T00:00:00-stub
+COPY WSRT_Measures_stub /opt/IERS/${IERS_DIRNAME}
+RUN ln -sfT /opt/IERS/${IERS_DIRNAME} /opt/IERS/current
+
+COPY casarc /home/tango/.casarc
+
 ENV TANGO_LOG_PATH=/var/log/tango
 RUN sudo mkdir -p /var/log/tango && sudo chmod a+rwx /var/log/tango

docker-compose/lofar-device-base/casarc

+measures.directory: /opt/IERS/current

tangostationcontrol/requirements.txt

@@ -11,3 +11,4 @@ h5py >= 3.1.0 # BSD
 psutil >= 5.8.0 # BSD
 docker >= 5.0.3 # Apache 2
 python-logstash-async >= 2.3.0 # MIT
+python-casacore >= 3.3.1 # GPL2

tangostationcontrol/tangostationcontrol/common/measures.py

+# -*- coding: utf-8 -*-
+#
+# Distributed under the terms of the APACHE license.
+# See LICENSE.txt for more info.
+
+""" Utility functions for managing the casacore 'measures' tables.
+
+The 'measures' tables contain the ephemerides and geodetic calibrations
+as input for its time and space calculations. These tables are externally
+available (see MEASURES_URL).
+
+Casacore is expected to be configured to look in /opt/IERS/current for
+its 'measures' tables, through setting this in the ~/.casarc file as:
+
+    measures.directory: /opt/IERS/current
+
+This can be verified by running the 'findmeastable' utility, which
+is part of the 'casacore-tools' package.
+
+Periodically new measures need to be installed, especially if a leap
+second is introduced. Measures are maintained in directories called
+/opt/IERS/IERS-YYYY-MM-DDTHH:MM:SS, and /opt/IERS/current is a symlink
+to the active set.
+
+Usage:
+
+The download_measures() function can be used to download new measures,
+which can then be activated using use_measures_directory(). If
+casacore.measures already accessed the measures, the python program
+needs to be restarted in order to clear the cache.
+
+"""
+
+import pathlib
+import urllib.request
+import tarfile
+import datetime
+import os
+import sys
+
+# Where to store the measures table sets
+IERS_ROOTDIR = "/opt/IERS"
+
+# Where to download files to
+DOWNLOAD_DIR = "/tmp"
+
+# Where new measures can be downloaded
+MEASURES_URL = "ftp://ftp.astron.nl/outgoing/Measures/WSRT_Measures.ztar"
+
+def get_measures_directory():
+    """ Return the directory of the current measures table in use. """
+
+    return str(pathlib.Path(IERS_ROOTDIR, "current").resolve())
+
+def use_measures_directory(newdir):
+    """ Select a new set of measures tables to use.
+    
+        NOTE: Python must be restarted if the 'casacore.measures' module
+              already loaded the measures table before this switch.
+
+              The 'restart_python()' function can be used for this purpose.
+    """
+
+    newdir = pathlib.Path(newdir)
+
+    # newdir must be one of the available measures
+    if str(newdir) not in get_available_measures_directories():
+        raise ValueError(f"Target is not an available measures directory: {newdir}")
+
+    # be sure newdir must point to a directory containing measures
+    for subdir in ['ephemerides', 'geodetic']:
+        subdir = pathlib.Path(newdir, subdir)
+
+        if not subdir.is_dir():
+            raise ValueError(f"Subdirectory {subdir} does not exist")
+
+    # switch to new directory
+    current_symlink = pathlib.Path(IERS_ROOTDIR, "current")
+    if current_symlink.exists():
+        current_symlink.unlink()
+    current_symlink.symlink_to(newdir)
+
+def restart_python():
+    """ Force a restart this python program. 
+    
+        This function does not return. """
+
+    exe_path = pathlib.Path(sys.executable)
+
+    # NOTE: Python 3.4+ closes all file descriptors > 2 automatically, see https://www.python.org/dev/peps/pep-0446/
+    os.execv(exe_path, [exe_path.name] + sys.argv)
+
+def get_available_measures_directories() -> list:
+    """ Returns the set of installed measures tables. """
+    return [str(d) for d in pathlib.Path(IERS_ROOTDIR).glob("IERS-*") if d.is_dir() and not d.is_symlink()]
+
+def download_measures() -> str:
+    """ Download new measures and return the directory in which they were installed.
+    """
+
+    # create target directory for new measures
+    now = datetime.datetime.now()
+    iers_dir_download = pathlib.Path(now.strftime(f"{IERS_ROOTDIR}/IERS-%FT%T"))
+    iers_dir_download.mkdir()
+
+    try:
+        measures_filename = pathlib.Path(DOWNLOAD_DIR, "WSRT_Measures.ztar")
+
+        # download measures
+        urllib.request.urlretrieve(MEASURES_URL, str(measures_filename))
+
+        # untar measures
+        tarball = tarfile.open(str(measures_filename))
+        tarball.extractall(path=str(iers_dir_download))
+
+        # remove download
+        measures_filename.unlink()
+    except Exception as e:
+        # don't linger our new directory if we could not install measures in it
+        iers_dir_download.rmdir()
+
+        raise
+
+    # update the timestamp used in the directory name to reflect the time of the measures,
+    # not the time of download.
+    file_with_final_timestamp = pathlib.Path(iers_dir_download, "geodetic", "TAI_UTC")
+    mtime = datetime.datetime.fromtimestamp(file_with_final_timestamp.stat().st_mtime)
+
+    iers_dir_final = pathlib.Path(mtime.strftime(f"{IERS_ROOTDIR}/IERS-%FT%T"))
+
+    if iers_dir_final.exists():
+        # these measures were already downloaded earlier, delete them and use ours,
+        # which allows the user to fix previously broken downloads.
+        try:
+            shutil.rmtree(iers_dir_final)
+        except Exception as e:
+            # move out of the way instead then
+            iers_dir_final.rename(iers_final.with_suffix("delete-me"))
+
+    # update our name to reflect the correct timestamp
+    iers_dir_download.rename(iers_dir_final)
+
+    return str(iers_dir_final)

tangostationcontrol/tangostationcontrol/devices/beam.py

@@ -15,12 +15,14 @@ from tango import AttrWriteType
 from tango import DevState
 from tango import DebugIt
 
-# Additional import
-
 from tangostationcontrol.common.entrypoint import entry
 from tangostationcontrol.devices.device_decorators import only_in_states
 from tangostationcontrol.devices.lofar_device import lofar_device
-from tangostationcontrol.common.lofar_logging import device_logging_to_python
+from tangostationcontrol.common.lofar_logging import device_logging_to_python, log_exceptions
+from tangostationcontrol.common.measures import get_measures_directory, use_measures_directory, download_measures, restart_python, get_available_measures_directories
+
+import logging
+logger = logging.getLogger()
 
 __all__ = ["Beam", "main"]
 
@@ -55,6 +57,13 @@ class Beam(lofar_device):
         dtype=(numpy.double,), max_dim_x=96,
         fget=lambda self: self.hbat_pointing_epoch)
 
+    # Directory where the casacore measures that we use, reside. We configure ~/.casarc to
+    # use the symlink /opt/IERS/current, which we switch to the actual set of files to use.
+    measures_directory_R = attribute(dtype=str, access=AttrWriteType.READ, fget = lambda self: get_measures_directory())
+
+    # List of dowloaded measures (the latest 64, anyway)
+    measures_directories_available_R = attribute(dtype=(str,), max_dim_x=64, access=AttrWriteType.READ, fget = lambda self: sorted(get_available_measures_directories())[-64:])
+
     # --------
     # overloaded functions
     # --------
@@ -85,6 +94,36 @@ class Beam(lofar_device):
 
         return 0
 
+    @command(dtype_out=str, doc_out="Name of newly installed measures directory")
+    @DebugIt()
+    @log_exceptions()
+    def download_measures(self):
+        """ Download new measures tables into /opt/IERS, but do not activate them.
+        
+            NOTE: This may take a while to complete. You are advised to increase
+                  the timeout of the proxy using `my_device.set_timeout_millis(10000)`. """
+
+        return download_measures()
+
+    @command(dtype_in=str, doc_in="Measures directory to activate")
+    @DebugIt()
+    @log_exceptions()
+    def use_measures(self, newdir):
+        """ Activate a downloaded set of measures tables.
+        
+            NOTE: This will turn off and restart this device!! """
+
+        # switch to requested measures
+        use_measures_directory(newdir)
+        logger.info(f"Switched measures table to {newdir}")
+
+        # turn off our device, to prepare for a python restart
+        self.Off()
+
+        # restart this program to force casacore to adopt
+        # the new tables
+        logger.warning("Restarting device to activate new measures tables")
+        restart_python()
 
 # ----------
 # Run server

tangostationcontrol/tangostationcontrol/devices/sdp/xst.py

@@ -107,6 +107,9 @@ class XST(Statistics):
     FPGA_xst_subband_select_RW = attribute_wrapper(comms_id=OPCUAConnection, comms_annotation=["FPGA_xst_subband_select_RW"], datatype=numpy.uint32, dims=(8,16), access=AttrWriteType.READ_WRITE)
     FPGA_xst_subband_select_R = attribute_wrapper(comms_id=OPCUAConnection, comms_annotation=["FPGA_xst_subband_select_R"], datatype=numpy.uint32, dims=(8,16))
 
+     FPGA_xst_offload_nof_crosslets_RW = attribute_wrapper(comms_id=OPCUAConnection, comms_annotation=["FPGA_xst_offload_nof_crosslets_RW"], datatype=numpy.uint32, dims=(16,), access=AttrWriteType.READ_WRITE)
+     FPGA_xst_offload_nof_crosslets_R = attribute_wrapper(comms_id=OPCUAConnection, comms_annotation=["FPGA_xst_offload_nof_crosslets_R"], datatype=numpy.uint32, dims=(16,))
+
     # number of packets with valid payloads
     nof_valid_payloads_R    = attribute_wrapper(comms_id=StatisticsClient, comms_annotation={"type": "statistics", "parameter": "nof_valid_payloads"}, dims=(XSTCollector.MAX_FPGAS,), datatype=numpy.uint64)
     # number of packets with invalid payloads

tangostationcontrol/tangostationcontrol/test/common/test_measures.py

+# -*- coding: utf-8 -*-
+#
+# This file is part of the LOFAR 2.0 Station Software
+#
+#
+#
+# Distributed under the terms of the APACHE license.
+# See LICENSE.txt for more info.
+
+import urllib.request
+import os.path
+from unittest import mock
+import shutil
+import tempfile
+
+from tangostationcontrol.common import measures
+
+from tangostationcontrol.test import base
+
+# where our WSRT_Measures.ztar surrogate is located
+# two versions with different timestamps are provided
+fake_measures = os.path.dirname(__file__) + "/fake_measures.ztar"
+fake_measures_newer = os.path.dirname(__file__) + "/fake_measures_newer.ztar"
+
+class TestMeasures(base.TestCase):
+    @mock.patch.object(urllib.request, 'urlretrieve')
+    def test_download_and_use(self, m_urlretrieve):
+        """ Test downloading and using new measures tables. """
+
+        with tempfile.TemporaryDirectory() as tmpdirname, \
+             mock.patch('tangostationcontrol.common.measures.IERS_ROOTDIR', tmpdirname) as rootdir, \
+             mock.patch('tangostationcontrol.common.measures.DOWNLOAD_DIR', tmpdirname) as downloaddir:
+                
+            # emulate the download
+            m_urlretrieve.side_effect = lambda *args, **kw: shutil.copyfile(fake_measures, tmpdirname + "/WSRT_Measures.ztar")
+
+            # 'download' and process our fake measures
+            newdir = measures.download_measures()
+
+            # active them
+            measures.use_measures_directory(newdir)
+
+            # check if they're activated
+            self.assertIn(newdir, measures.get_available_measures_directories())
+            self.assertEqual(newdir, measures.get_measures_directory())
+
+    @mock.patch.object(urllib.request, 'urlretrieve')
+    def test_switch_tables(self, m_urlretrieve):
+        """ Test switching between available sets of measures tables. """
+
+        with tempfile.TemporaryDirectory() as tmpdirname, \
+             mock.patch('tangostationcontrol.common.measures.IERS_ROOTDIR', tmpdirname) as rootdir, \
+             mock.patch('tangostationcontrol.common.measures.DOWNLOAD_DIR', tmpdirname) as downloaddir:
+                
+            # 'download' two measures with different timestamps
+            m_urlretrieve.side_effect = lambda *args, **kw: shutil.copyfile(fake_measures, tmpdirname + "/WSRT_Measures.ztar")
+            newdir1 = measures.download_measures()
+            m_urlretrieve.side_effect = lambda *args, **kw: shutil.copyfile(fake_measures_newer, tmpdirname + "/WSRT_Measures.ztar")
+            newdir2 = measures.download_measures()
+
+            # check if both are available
+            self.assertIn(newdir1, measures.get_available_measures_directories())
+            self.assertIn(newdir2, measures.get_available_measures_directories())
+
+            # switch between the two
+            measures.use_measures_directory(newdir1)
+            self.assertEqual(newdir1, measures.get_measures_directory())
+            measures.use_measures_directory(newdir2)
+            self.assertEqual(newdir2, measures.get_measures_directory())
+            measures.use_measures_directory(newdir1)
+            self.assertEqual(newdir1, measures.get_measures_directory())