L2SS-1633: Sync IERS tables through minio.

docker-compose/lofar-device-base/Dockerfile

@@ -32,9 +32,7 @@ RUN if [ $TANGO_STATION_CONTROL ]; then \
 # install and use ephimerides and geodetic ("measures") tables for casacore.
 # we install a _stub_ since the tables need to be deployed explicitly from within the software.
 RUN sudo mkdir -p /opt/IERS && sudo chmod a+rwx /opt/IERS
-ARG IERS_DIRNAME=IERS-1970-01-01T00:00:00-stub
-COPY lofar-device-base/WSRT_Measures_stub /opt/IERS/${IERS_DIRNAME}
-RUN ln -sfT /opt/IERS/${IERS_DIRNAME} /opt/IERS/current
+COPY lofar-device-base/WSRT_Measures_stub /opt/IERS
 
 COPY lofar-device-base/casarc /home/tango/.casarc
 

docker-compose/lofar-device-base/casarc

-measures.directory: /opt/IERS/current
+measures.directory: /opt/IERS

docker-compose/object-storage.yml

@@ -20,8 +20,12 @@ services:
     entrypoint: ''
     command: >
       sh -c "mc alias set object-storage http://s3.service.consul:9000 $MINIO_ROOT_USER $MINIO_ROOT_PASSWORD
+             echo 'Initialising caltables'
              mc mb --with-versioning object-storage/caltables
              mc cp --recursive /opt/lofar/tango/docker-compose/object-storage/caltables/ object-storage/caltables/
+             echo 'Initialising IERS tables'
+             mc mb --with-versioning object-storage/iers
+             mc cp --recursive /opt/lofar/tango/docker-compose/object-storage/iers/ object-storage/iers/
              date +'%F %T'
              echo 'Initialisation completed'"
 

infra/jobs/station/device-server.levant.nomad

@@ -8,6 +8,56 @@ job "device-servers" {
       delay_function = "constant"
     }
 
+    group "sync-IERS" {
+        count = 1
+
+        network {
+          mode = "bridge"
+
+          port "metrics" {
+            to = 8081
+          }
+        }
+
+        service {
+            tags = ["scrape"]
+            name = "sync-IERS"
+            port = "sync-IERS"
+            meta {
+                metrics_address = "${NOMAD_ADDR_metrics}"
+                metrics_path = "/"
+            }
+        }
+
+        volume "IERS" {
+            type      = "host"
+            read_only = false
+            source    = "IERS"
+        }
+
+        task "sync-IERS" {
+            driver = "docker"
+
+            volume_mount {
+                volume      = "IERS"
+                destination = "/opt/IERS"
+                read_only   = false
+            }
+
+            config {
+                image       = "minio/mc:latest"
+                entrypoint  = ""
+                command     = "/bin/bash"
+                args        = ["-c", "mc alias set object-storage http://s3.service.consul:9000 [[.object_storage.user.name]] [[.object_storage.user.pass]] && mc mirror --preserve --watch object-storage/iers/ /opt/IERS/" ]
+            }
+
+            resources {
+                cpu    = 10
+                memory = 128
+            }
+        }
+    }
+
     [[ range $device := $.devices ]]
     [[ with $device ]]
     [[ $class := .class ]]
@@ -31,6 +81,12 @@ job "device-servers" {
             [[ end ]]
         }
 
+        volume "IERS" {
+          type      = "host"
+          read_only = true
+          source    = "IERS"
+        }
+
         service {
             tags = ["scrape"]
             name = "device-[[ $name ]]"
@@ -44,6 +100,12 @@ job "device-servers" {
         task "device-[[ $name ]]" {
             driver = "docker"
 
+            volume_mount {
+              volume      = "IERS"
+              destination = "/opt/IERS"
+              read_only   = true
+            }
+
             config {
                 image = "[[ $.registry.astron.url ]]/lofar-device-base:[[ $.image_tag ]]"
                 ports = [

infra/jobs/station/nomad-client.nomad

@@ -69,6 +69,7 @@ job "nomad-client" {
           - mkdir -p /localdata/volumes/monitoring-loki-data
           - mkdir -p /localdata/volumes/tango-database
           - mkdir -p /localdata/volumes/object-storage-data
+          - mkdir -p /localdata/volumes/IERS-data
           - chmod 0777 /localdata/volumes/*
           - [systemctl, enable, consul.service]
           - [systemctl, start, consul.service]
@@ -157,6 +158,10 @@ job "nomad-client" {
                 path = "/localdata/volumes/object-storage-data"
               }
 
+              host_volume "IERS" {
+                path = "/localdata/volumes/IERS-data"
+              }
+
               host_volume "jupyter-notebooks" {
                 path = "/localdata/volumes/jupyter-notebooks"
               }

infra/jobs/station/object-storage.levant.nomad

@@ -59,8 +59,11 @@ job "object-storage" {
       }
 
       env {
+        MINIO_REGION               = "[[.station]]"
         MINIO_ROOT_USER            = "[[.object_storage.user.name]]"
         MINIO_ROOT_PASSWORD        = "[[.object_storage.user.pass]]"
+        MINIO_PROMETHEUS_URL       = "http://prometheus.service.consul:9090"
+        MINIO_PROMETHEUS_JOBID     = "consul_services"
         MINIO_PROMETHEUS_AUTH_TYPE = "public"
         #MINIO_BROWSER_REDIRECT_URL = "http://[[.station]]c.control.lofar/minio"
       }

tangostationcontrol/tangostationcontrol/beam/delays.py

@@ -6,8 +6,37 @@ from functools import lru_cache
 from typing import TypedDict
 
 import casacore.measures
+import casacore.tables
 import numpy
 
+"""
+The 'measures' tables contain the ephemerides and geodetic calibrations
+as input for its time and space calculations. These tables are externally
+available at ftp://ftp.astron.nl/outgoing/Measures/WSRT_Measures.ztar.
+
+Casacore is expected to be configured to look in /opt/IERS for
+its 'measures' tables, through setting this in the ~/.casarc file as:
+
+    measures.directory: /opt/IERS
+
+This can be verified by running the 'findmeastable' utility, which
+is part of the 'casacore-tools' package.
+
+The measures
+"""
+
+# Where to store the measures table sets
+IERS_ROOTDIR = "/opt/IERS"
+
+
+def get_IERS_timestamp() -> datetime.datetime:
+    """Return the date of the currently installed IERS tables."""
+
+    with casacore.tables.table(f"{IERS_ROOTDIR}/geodetic/TAI_UTC") as t:
+        return datetime.datetime.strptime(t.VS_DATE, "%Y/%m/%d/%H:%M").replace(
+            tzinfo=datetime.timezone.utc
+        )
+
 
 class CasacoreQuantum(TypedDict):
     """A casacore::Quantum object as returned by casacore.measures."""

tangostationcontrol/tangostationcontrol/common/entrypoint.py

@@ -2,6 +2,8 @@
 # SPDX-License-Identifier: Apache-2.0
 
 import sys
+import os
+from pathlib import Path
 
 from tango.server import run
 from tangostationcontrol.common.lofar_logging import configure_logger
@@ -21,3 +23,14 @@ def entry(Device, **kwargs):
         return run(Device, args=args, **kwargs)
     else:
         return run((Device,), args=args, **kwargs)
+
+
+def restart_python():
+    """Force a restart this python program.
+
+    This function does not return."""
+
+    exe_path = Path(sys.executable)
+
+    # NOTE: Python 3.4+ closes all file descriptors > 2 automatically, see https://www.python.org/dev/peps/pep-0446/
+    os.execv(exe_path, [exe_path.name] + sys.argv)

tangostationcontrol/tangostationcontrol/common/measures.py

-# Copyright (C) 2022 ASTRON (Netherlands Institute for Radio Astronomy)
-# SPDX-License-Identifier: Apache-2.0
-
-""" Utility functions for managing the casacore 'measures' tables.
-
-The 'measures' tables contain the ephemerides and geodetic calibrations
-as input for its time and space calculations. These tables are externally
-available (see MEASURES_URL).
-
-Casacore is expected to be configured to look in /opt/IERS/current for
-its 'measures' tables, through setting this in the ~/.casarc file as:
-
-    measures.directory: /opt/IERS/current
-
-This can be verified by running the 'findmeastable' utility, which
-is part of the 'casacore-tools' package.
-
-Periodically new measures need to be installed, especially if a leap
-second is introduced. Measures are maintained in directories called
-/opt/IERS/IERS-YYYY-MM-DDTHH:MM:SS, and /opt/IERS/current is a symlink
-to the active set.
-
-Usage:
-
-The download_measures() function can be used to download new measures,
-which can then be activated using use_measures_directory(). If
-casacore.measures already accessed the measures, the python program
-needs to be restarted in order to clear the cache.
-
-"""
-
-import datetime
-import os
-import pathlib
-import shutil
-import sys
-import tarfile
-import urllib.request
-
-# Where to store the measures table sets
-IERS_ROOTDIR = "/opt/IERS"
-
-# Where to download files to
-DOWNLOAD_DIR = "/tmp"
-
-# Where new measures can be downloaded
-MEASURES_URL = "ftp://ftp.astron.nl/outgoing/Measures/WSRT_Measures.ztar"
-
-
-def get_measures_directory():
-    """Return the directory of the current measures table in use."""
-
-    return str(pathlib.Path(IERS_ROOTDIR, "current").resolve())
-
-
-def use_measures_directory(newdir):
-    """Select a new set of measures tables to use.
-
-    NOTE: Python must be restarted if the 'casacore.measures' module
-          already loaded the measures table before this switch.
-
-          The 'restart_python()' function can be used for this purpose.
-    """
-
-    newdir = pathlib.Path(newdir)
-
-    # newdir must be one of the available measures
-    if str(newdir) not in get_available_measures_directories():
-        raise ValueError(f"Target is not an available measures directory: {newdir}")
-
-    # be sure newdir must point to a directory containing measures
-    for subdir in ["ephemerides", "geodetic"]:
-        subdir = pathlib.Path(newdir, subdir)
-
-        if not subdir.is_dir():
-            raise ValueError(f"Subdirectory {subdir} does not exist")
-
-    # switch to new directory
-    current_symlink = pathlib.Path(IERS_ROOTDIR, "current")
-    if current_symlink.exists():
-        current_symlink.unlink()
-    current_symlink.symlink_to(newdir)
-
-
-def restart_python():
-    """Force a restart this python program.
-
-    This function does not return."""
-
-    exe_path = pathlib.Path(sys.executable)
-
-    # NOTE: Python 3.4+ closes all file descriptors > 2 automatically, see https://www.python.org/dev/peps/pep-0446/
-    os.execv(exe_path, [exe_path.name] + sys.argv)
-
-
-def get_available_measures_directories() -> list:
-    """Returns the set of installed measures tables."""
-    return [
-        str(d)
-        for d in pathlib.Path(IERS_ROOTDIR).glob("IERS-*")
-        if d.is_dir() and not d.is_symlink()
-    ]
-
-
-def download_measures() -> str:
-    """Download new measures and return the directory in which they were installed."""
-
-    # create target directory for new measures
-    now = datetime.datetime.now()
-    iers_dir_download = pathlib.Path(now.strftime(f"{IERS_ROOTDIR}/IERS-%FT%T"))
-    iers_dir_download.mkdir()
-
-    try:
-        measures_filename = pathlib.Path(DOWNLOAD_DIR, "WSRT_Measures.ztar")
-
-        # download measures
-        urllib.request.urlretrieve(MEASURES_URL, str(measures_filename))
-
-        # untar measures
-        tarball = tarfile.open(str(measures_filename))
-        tarball.extractall(path=str(iers_dir_download))
-
-        # remove download
-        measures_filename.unlink()
-    except Exception as e:
-        # don't linger our new directory if we could not install measures in it
-        iers_dir_download.rmdir()
-
-        raise
-
-    # update the timestamp used in the directory name to reflect the time of the measures,
-    # not the time of download.
-    file_with_final_timestamp = pathlib.Path(iers_dir_download, "geodetic", "TAI_UTC")
-    mtime = datetime.datetime.fromtimestamp(file_with_final_timestamp.stat().st_mtime)
-
-    iers_dir_final = pathlib.Path(mtime.strftime(f"{IERS_ROOTDIR}/IERS-%FT%T"))
-
-    if iers_dir_final.exists():
-        # these measures were already downloaded earlier, delete them and use ours,
-        # which allows the user to fix previously broken downloads.
-        try:
-            shutil.rmtree(iers_dir_final)
-        except Exception as e:
-            # move out of the way instead then
-            iers_dir_final.rename(iers_dir_final.with_suffix("delete-me"))
-
-    # update our name to reflect the correct timestamp
-    iers_dir_download.rename(iers_dir_final)
-
-    return str(iers_dir_final)

tangostationcontrol/tangostationcontrol/devices/base_device_classes/beam_device.py

@@ -26,7 +26,7 @@ from tango import (
 # PyTango imports
 from tango.server import attribute, command, device_property
 
-from tangostationcontrol.beam.delays import Delays, pointing_to_str
+from tangostationcontrol.beam.delays import Delays, pointing_to_str, get_IERS_timestamp
 from tangostationcontrol.beam.managers import AbstractBeamManager
 from tangostationcontrol.common.constants import MAX_POINTINGS, N_point_prop
 from tangostationcontrol.common.device_decorators import (
@@ -36,13 +36,6 @@ from tangostationcontrol.common.device_decorators import (
 
 # Additional import
 from tangostationcontrol.common.lofar_logging import log_exceptions, exception_to_str
-from tangostationcontrol.common.measures import (
-    download_measures,
-    get_available_measures_directories,
-    get_measures_directory,
-    restart_python,
-    use_measures_directory,
-)
 from tangostationcontrol.common.states import DEFAULT_COMMAND_STATES
 from tangostationcontrol.devices.base_device_classes.async_device import AsyncDevice
 
@@ -400,50 +393,14 @@ class BeamDevice(AsyncDevice):
     # --------
 
     # Directory where the casacore measures that we use, reside. We configure ~/.casarc to
-    # use the symlink /opt/IERS/current, which we switch to the actual set of files to use.
-    measures_directory_R = attribute(
-        dtype=str, access=AttrWriteType.READ, fget=lambda self: get_measures_directory()
-    )
-
-    # List of dowloaded measures (the latest 64, anyway)
-    measures_directories_available_R = attribute(
-        dtype=(str,),
-        max_dim_x=64,
+    # use the symlink /opt/IERS, which we switch to the actual set of files to use.
+    IERS_timestamp_R = attribute(
+        doc="Timestamp of the IERS tables used.",
+        dtype=numpy.float64,
         access=AttrWriteType.READ,
-        fget=lambda self: sorted(get_available_measures_directories())[-64:],
+        fget=lambda self: get_IERS_timestamp().timestamp(),
     )
 
-    @command(dtype_out=str, doc_out="Name of newly installed measures directory")
-    @DebugIt()
-    @log_exceptions()
-    def download_measures(self):
-        """Download new measures tables into /opt/IERS, but do not activate them.
-
-        NOTE: This may take a while to complete. You are advised to increase
-              the timeout of the proxy using `my_device.set_timeout_millis(10000)`."""
-
-        return download_measures()
-
-    @command(dtype_in=str, doc_in="Measures directory to activate")
-    @DebugIt()
-    @log_exceptions()
-    def use_measures(self, newdir):
-        """Activate a downloaded set of measures tables.
-
-        NOTE: This will turn off and restart this device!!"""
-
-        # switch to requested measures
-        use_measures_directory(newdir)
-        logger.info("Switched measures table to %s", newdir)
-
-        # turn off our device, to prepare for a python restart
-        self.Off()
-
-        # restart this program to force casacore to adopt
-        # the new tables
-        logger.warning("Restarting device to activate new measures tables")
-        restart_python()
-
 
 # ----------
 # Beam Tracker