From f924edcea94c61d8223189c5c82fd64cce0f4fd8 Mon Sep 17 00:00:00 2001
From: stedif <stefano.difrischia@inaf.it>
Date: Thu, 23 Dec 2021 11:30:04 +0100
Subject: [PATCH] L2SS-480: merge with branch L2SS-497
---
docker-compose/device-beam.yml | 4 +
docker-compose/lofar-device-base/Dockerfile | 6 +
docker-compose/lofar-device-base/casarc | 1 +
tangostationcontrol/requirements.txt | 2 +-
.../tangostationcontrol/common/measures.py | 124 ++++++++++++++++++
.../tangostationcontrol/devices/beam.py | 51 ++++++-
.../test/common/test_measures.py | 73 +++++++++++
7 files changed, 255 insertions(+), 6 deletions(-)
create mode 100644 docker-compose/lofar-device-base/casarc
create mode 100644 tangostationcontrol/tangostationcontrol/common/measures.py
create mode 100644 tangostationcontrol/tangostationcontrol/test/common/test_measures.py
diff --git a/docker-compose/device-beam.yml b/docker-compose/device-beam.yml
index 97385f164..b7572dda6 100644
--- a/docker-compose/device-beam.yml
+++ b/docker-compose/device-beam.yml
@@ -4,6 +4,9 @@
#
version: '2'
+volumes:
+ iers-data: {}
+
services:
device-beam:
image: device-beam
@@ -22,6 +25,7 @@ services:
- "host.docker.internal:host-gateway"
volumes:
- ..:/opt/lofar/tango:rw
+ - iers-data:/opt/IERS
environment:
- TANGO_HOST=${TANGO_HOST}
working_dir: /opt/lofar/tango
diff --git a/docker-compose/lofar-device-base/Dockerfile b/docker-compose/lofar-device-base/Dockerfile
index 34dde9337..2cc3aeb19 100644
--- a/docker-compose/lofar-device-base/Dockerfile
+++ b/docker-compose/lofar-device-base/Dockerfile
@@ -6,5 +6,11 @@ RUN sudo apt-get update && sudo apt-get install -y git && sudo apt-get clean
COPY lofar-requirements.txt /lofar-requirements.txt
RUN sudo pip3 install -r /lofar-requirements.txt
+# install and use ephimerides and geodetic ("measures") tables for casacore
+RUN sudo mkdir -p /opt/IERS && sudo chmod a+rwx /opt/IERS
+RUN IERS_DIRNAME=IERS-`date +%FT%T` && mkdir -p /opt/IERS/${IERS_DIRNAME} && ln -sfT /opt/IERS/${IERS_DIRNAME} /opt/IERS/current
+RUN cd /opt/IERS/current && curl ftp://ftp.astron.nl/outgoing/Measures/WSRT_Measures.ztar | tar xz
+COPY casarc /home/tango/.casarc
+
ENV TANGO_LOG_PATH=/var/log/tango
RUN sudo mkdir -p /var/log/tango && sudo chmod a+rwx /var/log/tango
diff --git a/docker-compose/lofar-device-base/casarc b/docker-compose/lofar-device-base/casarc
new file mode 100644
index 000000000..78e7a19ed
--- /dev/null
+++ b/docker-compose/lofar-device-base/casarc
@@ -0,0 +1 @@
+measures.directory: /opt/IERS/current
diff --git a/tangostationcontrol/requirements.txt b/tangostationcontrol/requirements.txt
index 9f910e1da..c30f6b56c 100644
--- a/tangostationcontrol/requirements.txt
+++ b/tangostationcontrol/requirements.txt
@@ -11,4 +11,4 @@ h5py >= 3.5.0 # BSD
psutil >= 5.8.0 # BSD
docker >= 5.0.3 # Apache 2
python-logstash-async >= 2.3.0 # MIT
-python-casacore
+python-casacore >= 3.3.1 # GPL2
diff --git a/tangostationcontrol/tangostationcontrol/common/measures.py b/tangostationcontrol/tangostationcontrol/common/measures.py
new file mode 100644
index 000000000..eb2489d61
--- /dev/null
+++ b/tangostationcontrol/tangostationcontrol/common/measures.py
@@ -0,0 +1,124 @@
+# -*- coding: utf-8 -*-
+#
+# Distributed under the terms of the APACHE license.
+# See LICENSE.txt for more info.
+
+""" Utility functions for managing the casacore 'measures' tables.
+
+The 'measures' tables contain the ephemerides and geodetic calibrations
+as input for its time and space calculations. These tables are externally
+available (see MEASURES_URL).
+
+Casacore is expected to be configured to look in /opt/IERS/current for
+its 'measures' tables, through setting this in the ~/.casarc file as:
+
+ measures.directory: /opt/IERS/current
+
+This can be verified by running the 'findmeastable' utility, which
+is part of the 'casacore-tools' package.
+
+Periodically new measures need to be installed, especially if a leap
+second is introduced. Measures are maintained in directories called
+/opt/IERS/IERS-YYYY-MM-DDTHH:MM:SS, and /opt/IERS/current is a symlink
+to the active set.
+
+Usage:
+
+The download_measures() function can be used to download new measures,
+which can then be activated using use_measures_directory(). If
+casacore.measures already accessed the measures, the python program
+needs to be restarted in order to clear the cache.
+
+"""
+
+import pathlib
+import urllib.request
+import tarfile
+import datetime
+import os
+import sys
+
+# Where to store the measures table sets
+IERS_ROOTDIR = "/opt/IERS"
+
+# Where to download files to
+DOWNLOAD_DIR = "/tmp"
+
+# Where new measures can be downloaded
+MEASURES_URL = "ftp://ftp.astron.nl/outgoing/Measures/WSRT_Measures.ztar"
+
+def get_measures_directory():
+ """ Return the directory of the current measures table in use. """
+
+ return str(pathlib.Path(IERS_ROOTDIR, "current").resolve())
+
+def use_measures_directory(newdir):
+ """ Select a new set of measures tables to use.
+
+ NOTE: Python must be restarted if the 'casacore.measures' module
+ already loaded the measures table before this switch.
+
+ The 'restart_python()' function can be used for this purpose.
+ """
+
+ newdir = pathlib.Path(newdir)
+
+ # newdir must be one of the available measures
+ if str(newdir) not in get_available_measures_directories():
+ raise ValueError(f"Target is not an available measures directory: {newdir}")
+
+ # be sure newdir must point to a directory containing measures
+ for subdir in ['ephemerides', 'geodetic']:
+ subdir = pathlib.Path(newdir, subdir)
+
+ if not subdir.is_dir():
+ raise ValueError(f"Subdirectory {subdir} does not exist")
+
+ # switch to new directory
+ current_symlink = pathlib.Path(IERS_ROOTDIR, "current")
+ if current_symlink.exists():
+ current_symlink.unlink()
+ current_symlink.symlink_to(newdir)
+
+def restart_python():
+ """ Force a restart this python program.
+
+ This function does not return. """
+
+ exe_path = pathlib.Path(sys.executable)
+ os.execv(exe_path, [exe_path.name] + sys.argv)
+
+ # NOTE: Python 3.4+ closes all file descriptors > 2 automatically, see https://www.python.org/dev/peps/pep-0446/
+
+def get_available_measures_directories() -> list:
+ """ Returns the set of installed measures tables. """
+ return [str(d) for d in pathlib.Path(IERS_ROOTDIR).glob("IERS-*") if d.is_dir() and not d.is_symlink()]
+
+def download_measures() -> str:
+ """ Download new measures and return the directory in which they were installed.
+ """
+
+ # create target directory for new measures
+ now = datetime.datetime.now()
+ iers_dir = pathlib.Path(now.strftime(f"{IERS_ROOTDIR}/IERS-%FT%T"))
+ iers_dir.mkdir()
+
+ try:
+ measures_filename = pathlib.Path(DOWNLOAD_DIR, "WSRT_Measures.ztar")
+
+ # download measures
+ urllib.request.urlretrieve(MEASURES_URL, str(measures_filename))
+
+ # untar measures
+ tarball = tarfile.open(str(measures_filename))
+ tarball.extractall(path=str(iers_dir))
+
+ # remove download
+ measures_filename.unlink()
+ except Exception as e:
+ # don't linger our new directory if we could not install measures in it
+ iers_dir.rmdir()
+
+ raise
+
+ return str(iers_dir)
diff --git a/tangostationcontrol/tangostationcontrol/devices/beam.py b/tangostationcontrol/tangostationcontrol/devices/beam.py
index b2ccb5b54..203bfeac0 100644
--- a/tangostationcontrol/tangostationcontrol/devices/beam.py
+++ b/tangostationcontrol/tangostationcontrol/devices/beam.py
@@ -8,10 +8,13 @@
"""
# PyTango imports
-from tango import AttrWriteType, DevState
-from tango import DebugIt
-from tango.server import command
+from tango.server import attribute, command
+from tango import AttrWriteType, DebugIt
import numpy
+import pathlib
+import urllib.request
+import tarfile
+import datetime
# Additional import
from tangostationcontrol.devices.device_decorators import *
@@ -19,7 +22,10 @@ from tangostationcontrol.common.entrypoint import entry
from tangostationcontrol.clients.attribute_wrapper import attribute_wrapper
from tangostationcontrol.devices.lofar_device import lofar_device
from tangostationcontrol.common.lofar_logging import device_logging_to_python, log_exceptions
-from tangostationcontrol.beam.delays import delay_calculator
+from tangostationcontrol.common.measures import get_measures_directory, use_measures_directory, download_measures, restart_python, get_available_measures_directories
+
+import logging
+logger = logging.getLogger()
__all__ = ["Beam", "main"]
@@ -34,6 +40,12 @@ class Beam(lofar_device):
# Attributes
# ----------
+ # Directory where the casacore measures that we use, reside. We configure ~/.casarc to
+ # use the symlink /opt/IERS/current, which we switch to the actual set of files to use.
+ measures_directory_R = attribute(dtype=str, access=AttrWriteType.READ, fget = lambda self: get_measures_directory())
+
+ # List of installed measures (the latest 64, anyway)
+ measures_directories_available_R = attribute(dtype=(str,), max_dim_x=64, access=AttrWriteType.READ, fget = lambda self: sorted(get_available_measures_directories())[-64:])
# --------
# overloaded functions
@@ -43,8 +55,37 @@ class Beam(lofar_device):
# --------
# Commands
# --------
- pass
+ @command(dtype_out=str, doc_out="Name of newly installed measures directory")
+ @DebugIt()
+ @log_exceptions()
+ def download_measures(self):
+ """ Download new measures tables into /opt/IERS.
+
+ NOTE: This may take a while to complete. You are advised to increase
+ the timeout of the proxy using `my_device.set_timeout_millis(10000)`. """
+
+ return download_measures()
+
+ @command(dtype_in=str, doc_in="Measures directory to activate")
+ @DebugIt()
+ @log_exceptions()
+ def use_measures(self, newdir):
+ """ Activate an installed set of measures tables.
+
+ NOTE: This will turn off and restart this device!! """
+
+ # switch to requested measures
+ use_measures_directory(newdir)
+ logger.info(f"Switched measures table to {newdir}")
+
+ # turn off our device, to prepare for a python restart
+ self.Off()
+
+ # restart this program to force casacore to adopt
+ # the new tables
+ logger.warning("Restarting device to activate new measures tables")
+ restart_python()
# ----------
# Run server
diff --git a/tangostationcontrol/tangostationcontrol/test/common/test_measures.py b/tangostationcontrol/tangostationcontrol/test/common/test_measures.py
new file mode 100644
index 000000000..d70e01257
--- /dev/null
+++ b/tangostationcontrol/tangostationcontrol/test/common/test_measures.py
@@ -0,0 +1,73 @@
+# -*- coding: utf-8 -*-
+#
+# This file is part of the LOFAR 2.0 Station Software
+#
+#
+#
+# Distributed under the terms of the APACHE license.
+# See LICENSE.txt for more info.
+
+import urllib.request
+import os.path
+import datetime
+from unittest import mock
+import shutil
+import tempfile
+import time
+
+from tangostationcontrol.common import measures
+
+from tangostationcontrol.test import base
+
+# where our WSRT_Measures.ztar surrogate is located
+fake_measures = os.path.dirname(__file__) + "/fake_measures.ztar"
+
+class TestMeasures(base.TestCase):
+ @mock.patch.object(urllib.request, 'urlretrieve')
+ def test_download_and_use(self, m_urlretrieve):
+ """ Test downloading and using new measures tables. """
+
+ with tempfile.TemporaryDirectory() as tmpdirname, \
+ mock.patch('tangostationcontrol.common.measures.IERS_ROOTDIR', tmpdirname) as rootdir, \
+ mock.patch('tangostationcontrol.common.measures.DOWNLOAD_DIR', tmpdirname) as downloaddir:
+
+ # emulate the download
+ m_urlretrieve.side_effect = lambda *args, **kw: shutil.copyfile(fake_measures, tmpdirname + "/WSRT_Measures.ztar")
+
+ # 'download' and process our fake measures
+ newdir = measures.download_measures()
+
+ # active them
+ measures.use_measures_directory(newdir)
+
+ # check if they're activated
+ self.assertIn(newdir, measures.get_available_measures_directories())
+ self.assertEqual(newdir, measures.get_measures_directory())
+
+ @mock.patch.object(urllib.request, 'urlretrieve')
+ def test_switch_tables(self, m_urlretrieve):
+ """ Test switching between available sets of measures tables. """
+
+ with tempfile.TemporaryDirectory() as tmpdirname, \
+ mock.patch('tangostationcontrol.common.measures.IERS_ROOTDIR', tmpdirname) as rootdir, \
+ mock.patch('tangostationcontrol.common.measures.DOWNLOAD_DIR', tmpdirname) as downloaddir:
+
+ # emulate the download
+ m_urlretrieve.side_effect = lambda *args, **kw: shutil.copyfile(fake_measures, tmpdirname + "/WSRT_Measures.ztar")
+
+ # 'download' two measures with different timestamps
+ newdir1 = measures.download_measures()
+ time.sleep(1)
+ newdir2 = measures.download_measures()
+
+ # check if both are available
+ self.assertIn(newdir1, measures.get_available_measures_directories())
+ self.assertIn(newdir2, measures.get_available_measures_directories())
+
+ # switch between the two
+ measures.use_measures_directory(newdir1)
+ self.assertEqual(newdir1, measures.get_measures_directory())
+ measures.use_measures_directory(newdir2)
+ self.assertEqual(newdir2, measures.get_measures_directory())
+ measures.use_measures_directory(newdir1)
+ self.assertEqual(newdir1, measures.get_measures_directory())
--
GitLab