Skip to content
Snippets Groups Projects
Commit 0256c47d authored by Stefano Di Frischia's avatar Stefano Di Frischia
Browse files

Merge branch 'master' into L2SS-984-fix-beamlet-subband-tests

parents 3b06aff6 fd862f35
No related branches found
No related tags found
1 merge request!440Resolve L2SS-984 "Fix beamlet subband tests"
Showing
with 196 additions and 116 deletions
......@@ -125,6 +125,7 @@ docker_build_image_all:
- bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh device-xst latest
- bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh device-temperature-manager latest
- bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh archiver-timescale latest
- bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh hdbpp latest
- bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh hdbppts-cm latest
- bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh hdbppts-es latest
......@@ -494,6 +495,17 @@ docker_build_image_archiver_timescale:
script:
# Do not remove 'bash' or statement will be ignored by primitive docker shell
- bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh archiver-timescale $tag
docker_build_image_hdbpp:
extends: .base_docker_images_except
only:
refs:
- merge_requests
changes:
- docker-compose/archiver-timescale.yml
- docker-compose/hdbpp/*
script:
# Do not remove 'bash' or statement will be ignored by primitive docker shell
- bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh hdbpp $tag
docker_build_image_hdbppts_cm:
extends: .base_docker_images_except
only:
......@@ -501,7 +513,7 @@ docker_build_image_hdbppts_cm:
- merge_requests
changes:
- docker-compose/archiver-timescale.yml
- docker-compose/tango-archiver-ts/*
- docker-compose/hdbppts-cm/*
script:
# Do not remove 'bash' or statement will be ignored by primitive docker shell
- bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh hdbppts-cm $tag
......@@ -512,7 +524,7 @@ docker_build_image_hdbppts_es:
- merge_requests
changes:
- docker-compose/archiver-timescale.yml
- docker-compose/tango-archiver-ts/*
- docker-compose/hdbppts-es/*
script:
# Do not remove 'bash' or statement will be ignored by primitive docker shell
- bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh hdbppts-es $tag
......
......@@ -15,6 +15,7 @@ TANGO_POGO_VERSION=9.6.34
TANGO_REST_VERSION=1.14.6
TANGO_STARTER_VERSION=2021-05-28
PG_TIMESCALEDB_VERSION=latest-pg12
PG_SUPERUSER_PASSWORD=password
PG_HDB_PASSWORD=hdbpp
MYSQL_ROOT_PASSWORD=secret
......
......@@ -39,6 +39,23 @@ registry and uploaded to our own using matching tags.
Services, same mechanism as devices.
### HDB++ image updates
The hdbpp Docker image is used as a base image for the hdbppts-cm (ConfigurationManager) and hdbppts-es (EventSubscriber)
images. If one is developing on a branch and any updates is made in hdbpp/Dockerfile, those won't be automatically picked up from hdbppts-cm and hdbppts-es, because the argument SOURCE_IMAGE in the docker-compose yml file always refers to the remote hdbpp image in the repository.
A temporary workaround for locally testing on these archiving containers is the following:
- Stop and remove any running hdbpp* container
- In the archiver-timescale.yml file, replace the hdbppts-cm and hdbppts-es SOURCE_IMAGE tag 'latest' with the branch name
(e.g. SOURCE_IMAGE: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/hdbpp:l2ss-new-branch)
- Rebuild all the hdbpp* container (make build hdbpp hdbppts-cm hdbppts-es), and then start them
- Test the new features
After the branch has been correctly developed, tested, the merge has been approved, and the new images have been built on the repository:
- Put back 'latest' tag on the archiver-timescale.yml file, replacing the branch name
- Merge on master
### Setup and maintenance
All behavioral logic to orchestrate change detection and image pushing can be
......
......@@ -8,6 +8,8 @@ services:
image: timescaledb
build:
context: timescaledb
args:
SOURCE_IMAGE: timescale/timescaledb:${PG_TIMESCALEDB_VERSION}
container_name: ${CONTAINER_NAME_PREFIX}archiver-timescale
networks:
- control
......@@ -31,10 +33,30 @@ services:
tag: "{{.Name}}"
restart: unless-stopped
hdbpp:
image: hdbpp
build:
context: hdbpp
args:
SOURCE_IMAGE: ${DOCKER_REGISTRY_HOST}/${DOCKER_REGISTRY_USER}-tango-cpp:${TANGO_CPP_VERSION}
container_name: ${CONTAINER_NAME_PREFIX}hdbpp
networks:
- control
depends_on:
- databaseds
- dsconfig
- archiver-timescale
extra_hosts:
- "host.docker.internal:host-gateway"
environment:
- TANGO_HOST=${TANGO_HOST}
hdbppts-cm:
image: hdbppts-cm
build:
context: tango-archiver-ts
context: hdbppts-cm
args:
SOURCE_IMAGE: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/hdbpp:latest
container_name: ${CONTAINER_NAME_PREFIX}hdbppts-cm
networks:
- control
......@@ -62,11 +84,14 @@ services:
hdbppts-es:
image: hdbppts-es
build:
context: tango-archiver-ts
context: hdbppts-es
args:
SOURCE_IMAGE: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/hdbpp:latest
container_name: ${CONTAINER_NAME_PREFIX}hdbppts-es
networks:
- control
depends_on:
- hdbppts-cm
- databaseds
- dsconfig
- archiver-timescale
......
#ARG DOCKER_REGISTRY_USER
#ARG DOCKER_REGISTRY_HOST
#FROM ${DOCKER_REGISTRY_HOST}/${DOCKER_REGISTRY_USER}/tango-cpp:latest
FROM artefact.skao.int/ska-tango-images-tango-cpp:9.3.9
ARG SOURCE_IMAGE
FROM ${SOURCE_IMAGE}
USER root
......@@ -33,7 +31,7 @@ RUN echo "deb http://deb.debian.org/debian buster-backports main contrib non-fre
# ----------- LIBHDB++ -----------------------------
RUN git clone -b master https://gitlab.com/tango-controls/hdbpp/libhdbpp.git
RUN git clone https://gitlab.com/tango-controls/hdbpp/libhdbpp.git
RUN cd libhdbpp \
&& mkdir build \
......@@ -48,7 +46,7 @@ RUN cd libhdbpp/build \
--showinstall=no \
--backup=no \
--type=debian \
--pkgsource="https://github.com/tango-controls-hdbpp/libhdbpp" \
--pkgsource="https://gitlab.com/tango-controls/hdbpp/libhdbpp" \
--pkglicense="LGPLv3" \
--deldesc=no \
--nodoc \
......@@ -95,70 +93,6 @@ RUN cd libhdbpp-timescale/build \
--requires="libpq5" \
make install
# ----------- HDB++ CONFIGURATION MANAGER ---------------
RUN git clone -b master https://github.com/tango-controls-hdbpp/hdbpp-cm.git
RUN cd hdbpp-cm \
&& mkdir -p build \
&& cd build \
&& cmake .. -DCMAKE_PREFIX_PATH=/usr/local/include/tango \
&& make -j4
RUN cd hdbpp-cm/build \
&& checkinstall \
--install=yes \
--fstrans=no \
--showinstall=no \
--backup=no \
--type=debian \
--pkgsource="https://github.com/tango-controls-hdbpp/hdbpp-cm" \
--pkglicense="GPLv3" \
--deldesc=no \
--nodoc \
--strip \
--stripso \
--maintainer="tango" \
--pkgarch=$(dpkg --print-architecture) \
--pkgversion="2.0.0" \
--pkgrelease="SNAPSHOT" \
--pkgname="hdbpp-cm" \
--requires="libzmq5,libomniorb4-2,libcos4-2,libomnithread4" \
make install
# ----------- HDB++ EVENT SUBSCRIBER ---------------
RUN git clone -b master https://github.com/tango-controls-hdbpp/hdbpp-es.git
RUN cd hdbpp-es \
&& mkdir -p build \
&& cd build \
&& cmake .. -DCMAKE_PREFIX_PATH=/usr/local/include/tango -DFETCH_LIBHDBPP=OFF -DLIBHDBPP_BACKEND=timescale -DPostgreSQL_TYPE_INCLUDE_DIR=/usr/local/include/postgresql \
&& make -j4
RUN cd hdbpp-es/build \
&& checkinstall \
--install=yes \
--fstrans=no \
--showinstall=no \
--backup=no \
--type=debian \
--pkgsource="https://github.com/tango-controls-hdbpp/hdbpp-es" \
--pkglicense="GPLv3" \
--deldesc=no \
--nodoc \
--strip \
--stripso \
--maintainer="tango" \
--pkgarch=$(dpkg --print-architecture) \
--pkgversion="2.0.0" \
--pkgrelease="SNAPSHOT" \
--pkgname="hdbpp-es" \
--requires="libzmq5,libomniorb4-2,libcos4-2,libomnithread4" \
make install
RUN apt-get update && \
apt-get install -y \
build-essential && \
......@@ -166,10 +100,5 @@ RUN apt-get update && \
RUN dpkg -i /libhdbpp/build/libhdbpp_2.0.0-SNAPSHOT_amd64.deb
RUN dpkg -i /libhdbpp-timescale/build/libhdbpp-timescale_2.0.0-SNAPSHOT_amd64.deb
RUN dpkg -i /hdbpp-cm/build/hdbpp-cm_2.0.0-SNAPSHOT_amd64.deb
RUN dpkg -i /hdbpp-es/build/hdbpp-es_2.0.0-SNAPSHOT_amd64.deb
RUN ldconfig
RUN mv /usr/local/bin/hdb++cm-srv /usr/local/bin/hdbppcm-srv
RUN mv /usr/local/bin/hdb++es-srv /usr/local/bin/hdbppes-srv
ARG SOURCE_IMAGE
FROM ${SOURCE_IMAGE}
# ----------- HDB++ CONFIGURATION MANAGER ---------------
RUN git clone https://gitlab.com/tango-controls/hdbpp/hdbpp-cm.git
RUN cd hdbpp-cm \
&& mkdir -p build \
&& cd build \
&& cmake .. -DCMAKE_PREFIX_PATH=/usr/local/include/tango \
&& make -j4
RUN cd hdbpp-cm/build \
&& checkinstall \
--install=yes \
--fstrans=no \
--showinstall=no \
--backup=no \
--type=debian \
--pkgsource="https://gitlab.com/tango-controls/hdbpp/hdbpp-cm" \
--pkglicense="GPLv3" \
--deldesc=no \
--nodoc \
--strip \
--stripso \
--maintainer="tango" \
--pkgarch=$(dpkg --print-architecture) \
--pkgversion="2.0.0" \
--pkgrelease="SNAPSHOT" \
--pkgname="hdbpp-cm" \
--requires="libzmq5,libomniorb4-2,libcos4-2,libomnithread4" \
make install
RUN apt-get update && \
apt-get install -y \
build-essential && \
apt-get clean
RUN dpkg -i /libhdbpp/build/libhdbpp_2.0.0-SNAPSHOT_amd64.deb
RUN dpkg -i /libhdbpp-timescale/build/libhdbpp-timescale_2.0.0-SNAPSHOT_amd64.deb
RUN dpkg -i /hdbpp-cm/build/hdbpp-cm_2.0.0-SNAPSHOT_amd64.deb
RUN ldconfig
RUN mv /usr/local/bin/hdb++cm-srv /usr/local/bin/hdbppcm-srv
ARG SOURCE_IMAGE
FROM ${SOURCE_IMAGE}
# ----------- HDB++ EVENT SUBSCRIBER ---------------
RUN git clone https://gitlab.com/tango-controls/hdbpp/hdbpp-es.git
RUN cd hdbpp-es \
&& mkdir -p build \
&& cd build \
&& cmake .. -DCMAKE_PREFIX_PATH=/usr/local/include/tango -DFETCH_LIBHDBPP=OFF -DLIBHDBPP_BACKEND=timescale -DPostgreSQL_TYPE_INCLUDE_DIR=/usr/local/include/postgresql \
&& make -j4
RUN cd hdbpp-es/build \
&& checkinstall \
--install=yes \
--fstrans=no \
--showinstall=no \
--backup=no \
--type=debian \
--pkgsource="https://gitlab.com/tango-controls/hdbpp/hdbpp-es" \
--pkglicense="GPLv3" \
--deldesc=no \
--nodoc \
--strip \
--stripso \
--maintainer="tango" \
--pkgarch=$(dpkg --print-architecture) \
--pkgversion="2.0.0" \
--pkgrelease="SNAPSHOT" \
--pkgname="hdbpp-es" \
--requires="libzmq5,libomniorb4-2,libcos4-2,libomnithread4" \
make install
RUN apt-get update && \
apt-get install -y \
build-essential && \
apt-get clean
RUN dpkg -i /libhdbpp/build/libhdbpp_2.0.0-SNAPSHOT_amd64.deb
RUN dpkg -i /libhdbpp-timescale/build/libhdbpp-timescale_2.0.0-SNAPSHOT_amd64.deb
RUN dpkg -i /hdbpp-es/build/hdbpp-es_2.0.0-SNAPSHOT_amd64.deb
RUN ldconfig
RUN mv /usr/local/bin/hdb++es-srv /usr/local/bin/hdbppes-srv
......@@ -16,6 +16,9 @@ RUN sudo apt-get install -y g++ gcc python3-dev
# Install git to install pip requirements from git
RUN sudo apt-get install -y git
# Install dependencies of our scripts (bin/start-ds.sh)
RUN sudo apt-get install -y rsync
COPY requirements.txt ./
RUN sudo pip3 install -r requirements.txt
......
FROM timescale/timescaledb:latest-pg12
ARG SOURCE_IMAGE
FROM ${SOURCE_IMAGE}
# Set admin role to perform DB creation
COPY resources/01_admin.sh docker-entrypoint-initdb.d/002_admin.sh
......
......@@ -82,8 +82,8 @@ LOCAL_IMAGES=(
"itango docker-compose_itango y"
"archiver-timescale timescaledb n" "hdbppts-cm hdbppts-cm n"
"hdbppts-es hdbppts-es n"
"archiver-timescale timescaledb y" "hdbpp hdbpp y" "hdbppts-cm hdbppts-cm y"
"hdbppts-es hdbppts-es y"
"grafana grafana n" "prometheus prometheus n"
"jupyter docker-compose_jupyter n"
......
File suppressed by a .gitattributes entry or the file's encoding is unsupported.
File suppressed by a .gitattributes entry or the file's encoding is unsupported.
File suppressed by a .gitattributes entry or the file's encoding is unsupported.
......@@ -68,7 +68,7 @@ class Delays:
These are the delays that have to be applied to the signal chain in order to line up the signal.
Positions closer to the source will result in a positive delay.
Returns delays[antenna]. """
return self.delays_bulk([direction], numpy.array(antenna_absolute_itrf) - self.reference_itrf).flatten()
......@@ -78,7 +78,7 @@ class Delays:
These are the delays that have to be applied to the signal chain in order to line up the signal.
Positions closer to the source will result in a positive delay.
Returns delays[antenna][direction]. """
# obtain the direction vector for each pointing
......
......@@ -21,11 +21,11 @@ import geohash
def _apply_fn_on_one_element_or_array(fn, array: numpy.array) -> numpy.array:
if array.ndim == 1:
# convert a single coordinate triple
return fn(array)
# convert a single coordinate triple
return fn(array)
else:
# convert each coordinate triple
return numpy.apply_along_axis(fn, 1, array)
# convert each coordinate triple
return numpy.apply_along_axis(fn, 1, array)
def ETRS_to_ITRF(ETRS_coordinates: numpy.array, ITRF_reference_frame: str = "ITRF2005", ITRF_reference_epoch: float = 2015.5) -> numpy.array:
""" Convert an array of coordinate triples from ETRS to ITRF, in the given reference frame and epoch. """
......
......@@ -8,11 +8,11 @@ class HBATAntennaOffsets(object):
"""
This class helps calculate the absolute offsets of the antennas within a tile,
based on their relative orientation to ETRS.
These offsets are Known in LOFAR1 as "iHBADeltas".
Within LOFAR, we use a relative "PQR" coordinate system for each station:
* The origin is the same as the ETRS reference position of the station,
* "Q" points to true North as seen from CS002LBA, the center of LOFAR,
* "P" points (roughly) east,
......@@ -27,7 +27,7 @@ class HBATAntennaOffsets(object):
` relative positions of each antenna element within a tile in ETRS space.
The relative ITRF positions are subsequently equal to the relative ETRS positions.
For reference, see:
https://git.astron.nl/RD/lofar-referentie-vlak/-/blob/master/data/dts/dts.ipynb
https://git.astron.nl/ro/lofar/-/blob/master/MAC/Deployment/data/Coordinates/calc_hba_deltas.py
......
......@@ -17,7 +17,7 @@ class AbstractCommClient(ABC):
def ping(self):
""" Check whether the connection is still alive.
Clients that override this method must raise an Exception if the
connection died. """
pass
......@@ -31,7 +31,7 @@ class AbstractCommClient(ABC):
as well as a reference to the attribute itself.
The read_function must return a single value, representing the current value of the attribute.
The write_function must take a single value, write it, and return None.
Examples:
......@@ -105,7 +105,7 @@ class CommClient(AbstractCommClient, Thread):
def ping(self):
""" Check whether the connection is still alive.
Clients that override this method must raise an Exception if the
connection died. """
pass
......
......@@ -131,7 +131,7 @@ class StatisticsClient(AsyncCommClient):
return self.collector.parameters[parameter].reshape(attribute.dim_x)
else:
return self.collector.parameters[parameter]
def _process_replicator_annotation():
parameters_dict = {"clients": numpy.array(self.tcp.clients(),dtype=str),
"nof_bytes_sent": numpy.uint64(self.tcp.nof_bytes_sent),
......@@ -141,7 +141,7 @@ class StatisticsClient(AsyncCommClient):
if parameter == k:
return v
raise ValueError(f"Unknown replicator parameter requested: {parameter}")
def write_function(value):
"""
Not used here
......@@ -149,4 +149,4 @@ class StatisticsClient(AsyncCommClient):
pass
return read_function, write_function
......@@ -55,7 +55,7 @@ def get_measures_directory():
def use_measures_directory(newdir):
""" Select a new set of measures tables to use.
NOTE: Python must be restarted if the 'casacore.measures' module
already loaded the measures table before this switch.
......@@ -83,7 +83,7 @@ def use_measures_directory(newdir):
def restart_python():
""" Force a restart this python program.
This function does not return. """
exe_path = pathlib.Path(sys.executable)
......
......@@ -61,10 +61,10 @@ class mapped_attribute(attribute):
return device.get_mapped_attribute(mapping_attribute)
self.fget = read_func_wrapper
super().__init__(dtype=dtype, max_dim_y=max_dim_y, max_dim_x=max_dim_x, access=access, fisallowed="is_attribute_access_allowed", **kwargs)
@device_logging_to_python()
class AntennaField(lofar_device):
......@@ -143,7 +143,7 @@ class AntennaField(lofar_device):
dtype='DevVarFloatArray',
mandatory=False
)
Antenna_Reference_ETRS = device_property(
doc="ETRS position (XYZ) of each Antenna",
dtype='DevVarFloatArray',
......@@ -188,7 +188,7 @@ class AntennaField(lofar_device):
)
# ----- RECV mapping
Power_to_RECV_mapping = device_property(
dtype=(numpy.int32,),
doc='The mapping of Antenna power lines to RECV mapping. Each RECV can handle 96 inputs. The Antenna number is the index and the value shows to which receiver device it is connected and on which input. The first integer is the input. The second integer is the RECV id. Example: [0, 3] = first receiver of property RECV_devices with input 3. -1 means that the Antenna is not connected. The property is stored in a one dimensional structure. It needs to be reshaped to a list of lists of two items.',
......@@ -280,16 +280,16 @@ class AntennaField(lofar_device):
def read_Antenna_Names_R(self):
return self.Antenna_Names
def read_Antenna_Use_R(self):
return self.Antenna_Use
def read_Antenna_Quality_R(self):
return self.Antenna_Quality
def read_Antenna_Use_str_R(self):
return [AntennaUse(x).name for x in self.Antenna_Use]
def read_Antenna_Quality_str_R(self):
return [AntennaQuality(x).name for x in self.Antenna_Quality]
......@@ -312,7 +312,7 @@ class AntennaField(lofar_device):
#
# Parsing a property here is quickest, so we chose that.
return len(self.Control_to_RECV_mapping) // 2
def read_Antenna_Field_Reference_ITRF_R(self):
# provide ITRF field coordinates if they were configured
if self.Antenna_Field_Reference_ITRF:
......@@ -321,7 +321,7 @@ class AntennaField(lofar_device):
# calculate them from ETRS coordinates if not, using the configured ITRF reference
ETRS_coordinates = numpy.array(self.Antenna_Field_Reference_ETRS).reshape(3)
return ETRS_to_ITRF(ETRS_coordinates, self.ITRF_Reference_Frame, self.ITRF_Reference_Epoch)
def read_Antenna_Field_Reference_GEO_R(self):
return ITRF_to_GEO(self.read_Antenna_Field_Reference_ITRF_R())
......@@ -339,7 +339,7 @@ class AntennaField(lofar_device):
and finally translated into global ETRS coordinates using the HBAT_PQR_to_ETRS_rotation_matrix.
The relative ITRF offsets are the same as relative ETRS offsets.
NB: In all of this, the absolute position of each tile is actually irrelevant, as all the
tiles lie on the same plane in ITRF. """
......@@ -357,7 +357,7 @@ class AntennaField(lofar_device):
for angle_deg in self.HBAT_PQR_rotation_angles_deg])
return all_offsets.reshape(-1, NUMBER_OF_ELEMENTS_PER_TILE * 3)
def read_Antenna_Reference_ITRF_R(self):
# provide ITRF coordinates if they were configured
if self.Antenna_Reference_ITRF:
......@@ -399,7 +399,7 @@ class AntennaField(lofar_device):
mapped_values = self.__mapper.map_read(mapped_point, recv_results)
return mapped_values
def set_mapped_attribute(self, mapped_point: str, value, cast_type: type):
"""Set the attribute to new value only for controlled points
......@@ -418,7 +418,7 @@ class AntennaField(lofar_device):
self.atomic_read_modify_write_attribute(
new_values, recv_proxy, mapped_point, cast_type=cast_type
)
# --------
# Overloaded functions
# --------
......@@ -428,7 +428,7 @@ class AntennaField(lofar_device):
super().configure_for_initialise()
self.__setup_all_receiver_proxies()
self.__setup_mapper()
@log_exceptions()
def _prepare_hardware(self):
usage_mask = self.read_attribute('Antenna_Usage_Mask_R')
......@@ -464,7 +464,7 @@ class AntennaField(lofar_device):
# convert them into delay steps
flatten_delay_steps = numpy.array(recv_proxy.calculate_HBAT_bf_delay_steps(recv_delays.flatten()), dtype=numpy.int64)
delay_steps = numpy.reshape(flatten_delay_steps,(-1, NUMBER_OF_ELEMENTS_PER_TILE * 2))
# write back into same positions we collected them from
result_values[recv_result_indices] = delay_steps
......@@ -533,9 +533,9 @@ class AntennaToRecvMapper(object):
if mapped_attribute in self._reshape_attributes_in:
recv_results = numpy.reshape(recv_results,
(self._number_of_receivers,) + self._reshape_attributes_in[mapped_attribute])
return self._mapped_r_values(recv_results, default_values)
def map_write(self, mapped_attribute: str, set_values: List[any]) -> List[any]:
"""Perform a mapped write for the attribute using the set_values
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment