Skip to content
Snippets Groups Projects
Commit ce2a6e95 authored by Corné Lukken's avatar Corné Lukken
Browse files

L2SS-1487: Install station control into lofar-device-base docker image

parent 31b9005d
No related branches found
No related tags found
1 merge request!716L2SS-1487: Install station control into lofar-device-base docker image
...@@ -20,6 +20,7 @@ include: ...@@ -20,6 +20,7 @@ include:
stages: stages:
- prepare - prepare
- packaging
- images - images
- building - building
- linting - linting
...@@ -27,7 +28,6 @@ stages: ...@@ -27,7 +28,6 @@ stages:
- static-analysis - static-analysis
- unit-tests - unit-tests
- integration-tests - integration-tests
- packaging
- publish - publish
- deploy - deploy
...@@ -37,16 +37,30 @@ trigger_prepare: ...@@ -37,16 +37,30 @@ trigger_prepare:
strategy: depend strategy: depend
include: .prepare.gitlab-ci.yml include: .prepare.gitlab-ci.yml
wheel_packaging:
stage: packaging
artifacts:
paths:
- tangostationcontrol/dist/*.whl
script:
- cd tangostationcontrol
- tox -e build
# See docker-compose/README.md for docker image behavior and explanation # See docker-compose/README.md for docker image behavior and explanation
.base_docker_images: .base_docker_images:
stage: images stage: images
image: docker:latest image: docker:latest
dependencies:
- wheel_packaging
needs:
- wheel_packaging
tags: tags:
- privileged - privileged
services: services:
- name: docker:dind - name: docker:dind
variables: variables:
DOCKER_TLS_CERTDIR: "/certs" DOCKER_TLS_CERTDIR: "/certs"
TANGO_SKIP_BUILD: "true"
before_script: before_script:
- | - |
if [[ "$CI_COMMIT_BRANCH" == "$CI_DEFAULT_BRANCH" && -z "$CI_COMMIT_TAG" ]]; then if [[ "$CI_COMMIT_BRANCH" == "$CI_DEFAULT_BRANCH" && -z "$CI_COMMIT_TAG" ]]; then
...@@ -58,7 +72,7 @@ trigger_prepare: ...@@ -58,7 +72,7 @@ trigger_prepare:
fi fi
- apk add --update make bash docker-compose - apk add --update make bash docker-compose
- docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
- touch /root/.Xauthority - touch ~/.Xauthority
# Hack BASH_SOURCE into sourced files, docker its sh shell won't set this # Hack BASH_SOURCE into sourced files, docker its sh shell won't set this
- export BASH_SOURCE=$(pwd)/setup.sh - export BASH_SOURCE=$(pwd)/setup.sh
# source the lofarrc file and mask its non zero exit code # source the lofarrc file and mask its non zero exit code
...@@ -228,6 +242,9 @@ integration_test_docker: ...@@ -228,6 +242,9 @@ integration_test_docker:
image: docker:23.0.5 # latest ships with docker compose v2.19, which has the following bug: https://github.com/docker/compose/issues/10668 image: docker:23.0.5 # latest ships with docker compose v2.19, which has the following bug: https://github.com/docker/compose/issues/10668
needs: needs:
- unit_test - unit_test
- wheel_packaging
dependencies:
- wheel_packaging
tags: tags:
- privileged - privileged
services: services:
...@@ -289,17 +306,6 @@ multi_project_integration_test: ...@@ -289,17 +306,6 @@ multi_project_integration_test:
project: lofar2.0/multi-project-integration project: lofar2.0/multi-project-integration
strategy: depend strategy: depend
wheel_packaging:
stage: packaging
rules:
- if: ($CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH) || $CI_COMMIT_TAG
artifacts:
paths:
- tangostationcontrol/dist/*.whl
script:
- cd tangostationcontrol
- tox -e build
publish_on_gitlab: publish_on_gitlab:
stage: publish stage: publish
environment: gitlab environment: gitlab
...@@ -319,6 +325,17 @@ publish_on_gitlab: ...@@ -319,6 +325,17 @@ publish_on_gitlab:
- echo "run twine for gitlab" - echo "run twine for gitlab"
- python3 -m twine upload --repository-url ${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/pypi tangostationcontrol/dist/* - python3 -m twine upload --repository-url ${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/pypi tangostationcontrol/dist/*
release_job:
stage: publish
image: registry.gitlab.com/gitlab-org/release-cli:latest
rules:
- if: '$CI_COMMIT_TAG && $CI_COMMIT_REF_PROTECTED == "true"'
script:
- echo "running release_job"
release:
tag_name: '$CI_COMMIT_TAG'
description: '$CI_COMMIT_TAG'
.base_deploy: .base_deploy:
stage: deploy stage: deploy
image: ubuntu:bionic image: ubuntu:bionic
......
...@@ -29,10 +29,8 @@ CWD=$(pwd) ...@@ -29,10 +29,8 @@ CWD=$(pwd)
cd /opt/lofar/tango || exit 1 cd /opt/lofar/tango || exit 1
# Check if configured for specific version # Check if configured for specific version
if [[ $TANGOSTATIONCONTROL ]]; then if [[ $TANGO_STATION_CONTROL ]]; then
# TODO (Corne): Download version from artifacts or pypi. echo "Assuming station control install performed as part of base image"
# Consider exit 2 an UnImplementedError
exit 2
else else
# Install the package, exit 1 if it fails # Install the package, exit 1 if it fails
# pip install ./ will _NOT_ install dependencies in requirements.txt! # pip install ./ will _NOT_ install dependencies in requirements.txt!
......
...@@ -221,9 +221,17 @@ endif ...@@ -221,9 +221,17 @@ endif
$(DOCKER_COMPOSE_ARGS) $(DOCKER_COMPOSE) -f tango.yml -f networks.yml up --no-recreate -d $(DOCKER_COMPOSE_ARGS) $(DOCKER_COMPOSE) -f tango.yml -f networks.yml up --no-recreate -d
context: ## Move the necessary files to create minimal docker context context: ## Move and build the necessary files to create minimal docker context
@mkdir -p tmp rm -rf ./tmp; \
@cp ../tangostationcontrol/requirements.txt tmp/ mkdir -p tmp; \
if [ -z "$${TANGO_SKIP_BUILD}" ]; then \
rm -rf ../tangostationcontrol/dist; \
cd ../tangostationcontrol; \
tox -e build; \
cd ../docker-compose; \
fi; \
cp ../tangostationcontrol/dist/*.whl tmp/; \
cp ../tangostationcontrol/requirements.txt tmp/; \
bootstrap: pull build # first start, initialise from scratch bootstrap: pull build # first start, initialise from scratch
$(MAKE) start dsconfig # boot up containers to load configurations $(MAKE) start dsconfig # boot up containers to load configurations
......
...@@ -7,3 +7,4 @@ RUN echo 'debconf debconf/frontend select Noninteractive' | sudo debconf-set-sel ...@@ -7,3 +7,4 @@ RUN echo 'debconf debconf/frontend select Noninteractive' | sudo debconf-set-sel
RUN sudo apt-get update && sudo apt-get install -y tzdata RUN sudo apt-get update && sudo apt-get install -y tzdata
RUN sudo apt-get update && sudo apt-get install -y tox python3-dev libboost-python-dev pkg-config git shellcheck graphviz casacore-data RUN sudo apt-get update && sudo apt-get install -y tox python3-dev libboost-python-dev pkg-config git shellcheck graphviz casacore-data
RUN sudo ln -s /var/lib/casacore/data/geodetic/TAI_UTC /usr/share/casacore/data/geodetic/TAI_UTC RUN sudo ln -s /var/lib/casacore/data/geodetic/TAI_UTC /usr/share/casacore/data/geodetic/TAI_UTC
RUN sudo pip install --upgrade tox
...@@ -23,6 +23,7 @@ services: ...@@ -23,6 +23,7 @@ services:
dockerfile: lofar-device-base/Dockerfile dockerfile: lofar-device-base/Dockerfile
args: args:
SOURCE_IMAGE: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-itango:${TANGO_ITANGO_VERSION} SOURCE_IMAGE: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-itango:${TANGO_ITANGO_VERSION}
TANGO_STATION_CONTROL: ${TANGO_STATION_CONTROL}
hostname: lofar-device-base hostname: lofar-device-base
container_name: lofar-device-base container_name: lofar-device-base
# These parameters are just visual queues, you have to define them again # These parameters are just visual queues, you have to define them again
......
ARG SOURCE_IMAGE ARG SOURCE_IMAGE
FROM ${SOURCE_IMAGE} FROM ${SOURCE_IMAGE}
ARG TANGO_STATION_CONTROL
ENV TANGO_STATION_CONTROL $TANGO_STATION_CONTROL
RUN --mount=type=cache,target=/var/cache/apt \ RUN --mount=type=cache,target=/var/cache/apt \
sudo apt-get update sudo apt-get update
RUN --mount=type=cache,target=/var/cache/apt \ RUN --mount=type=cache,target=/var/cache/apt \
...@@ -16,12 +19,15 @@ RUN --mount=type=cache,target=/var/cache/apt \ ...@@ -16,12 +19,15 @@ RUN --mount=type=cache,target=/var/cache/apt \
RUN --mount=type=cache,target=/var/cache/apt \ RUN --mount=type=cache,target=/var/cache/apt \
sudo apt-get install -y netcat sudo apt-get install -y netcat
# Manually install all requirements from the .txt as part of the base image
# This reduces runtime overhead as well as preventing issues around dependency
# installation for development builds (pip install ./ ignores requirements.txt)
COPY tmp/requirements.txt /tangostationcontrol-requirements.txt COPY tmp/requirements.txt /tangostationcontrol-requirements.txt
COPY tmp/*.whl /
RUN sudo pip3 install -r /tangostationcontrol-requirements.txt RUN echo "TANGO_STATION_CONTROL: ${TANGO_STATION_CONTROL}"
RUN if [ $TANGO_STATION_CONTROL ]; then \
sudo pip3 install /*.whl; \
else \
sudo pip3 install -r /tangostationcontrol-requirements.txt; \
fi
# install and use ephimerides and geodetic ("measures") tables for casacore. # install and use ephimerides and geodetic ("measures") tables for casacore.
# we install a _stub_ since the tables need to be deployed explicitly from within the software. # we install a _stub_ since the tables need to be deployed explicitly from within the software.
......
...@@ -76,14 +76,23 @@ if [ -z "$LOFAR20_DIR" ]; then ...@@ -76,14 +76,23 @@ if [ -z "$LOFAR20_DIR" ]; then
LOFAR20_DIR=$(readlink -f "${LOFAR20_DIR_RELATIVE}") LOFAR20_DIR=$(readlink -f "${LOFAR20_DIR_RELATIVE}")
fi fi
export TANGO_SKIP_BUILD=1
cd "$LOFAR20_DIR/docker-compose" || exit 1
# Start the database server first, `-z ${y+x}` is the inverse of `-n ${y}`
if [ -z "${no_build+x}" ]; then
rm -rf "${LOFAR20_DIR}/tangostationcontrol/dist"
cd "${LOFAR20_DIR}/tangostationcontrol" || exit 1
tox -e build
cd "$LOFAR20_DIR/docker-compose" || exit 1 cd "$LOFAR20_DIR/docker-compose" || exit 1
make build databaseds dsconfig
fi
# Start the database server first
[ -n "${no_build}" ] || make build databaseds dsconfig
make start databaseds dsconfig make start databaseds dsconfig
# Give dsconfig and databaseds time to start # Give dsconfig and databaseds time to start
sleep 1 # dsconfig container must be up and running... sleep 5 # dsconfig container must be up and running...
# shellcheck disable=SC2016 # shellcheck disable=SC2016
echo '/usr/local/bin/wait-for-it.sh ${TANGO_HOST} --strict --timeout=300 -- true' | make run dsconfig bash - echo '/usr/local/bin/wait-for-it.sh ${TANGO_HOST} --strict --timeout=300 -- true' | make run dsconfig bash -
......
...@@ -25,5 +25,9 @@ export NETWORK_MODE=tangonet ...@@ -25,5 +25,9 @@ export NETWORK_MODE=tangonet
# port. Example: export TANGO_HOST=station-xk25.astron.nl:10000 # port. Example: export TANGO_HOST=station-xk25.astron.nl:10000
export TANGO_HOST=$(hostname):10000 export TANGO_HOST=$(hostname):10000
# Configure to install station control in lofar-device-base image
# TODO(L2SS-520): Extend to support debug and expose as property in devices
export TANGO_STATION_CONTROL=1
# Allow read access for everybody to allow Docker forwarding of X11. # Allow read access for everybody to allow Docker forwarding of X11.
chmod a+r ~/.Xauthority chmod a+r ~/.Xauthority
...@@ -3,6 +3,38 @@ Developer information ...@@ -3,6 +3,38 @@ Developer information
This chapter describes key areas useful for developers. This chapter describes key areas useful for developers.
Environment variables
---------------------
Several environment variables fundamentally control the deployment and
development environment. These include:
- `TANGO_HOST`
- `HOSTNAME`
- `NETWORK_MODE`
- `TANGO_STATION_CONTROL`
- `TANGO_SKIP_BUILD`
Firstly, `TANGO_HOST` should point to the tango database server including its port.
An example would be `10.14.0.205:10000`. If `TANGO_HOST` is not set instead
`$HOSTNAME:10000` is used.
With `NETWORK_MODE` the name of the docker network can be controlled or `host` can
be used as value to point directly to the host. This feature has not been used
in a very long time.
Finally `TANGO_STATION_CONTROL` can be used to control if device containers should
build software from source (developer mode). Or if the software should be built
into the `lofar-device-base` docker image directly. If `TANGO_STATION_CONTROL` is set
the makefile will build a wheel package which will be installed into the docker image.
If instead a particular wheel package needs to be installed `TANGO_SKIP_BUILD` can be
set as well. Be sure the wheel package is placed in the `tangostationcontrol/dist/`
directory.
In the future the actual value of the `TANGO_STATION_CONTROL` variable might be used
to control various types of different behavior.
Docker compose Docker compose
------------------------- -------------------------
......
# Copyright (C) 2022 ASTRON (Netherlands Institute for Radio Astronomy) # Copyright (C) 2022 ASTRON (Netherlands Institute for Radio Astronomy)
# SPDX-License-Identifier: Apache-2.0 # SPDX-License-Identifier: Apache-2.0
import logging
import sys import sys
from tango import DeviceProxy from tango import DeviceProxy
...@@ -16,5 +17,6 @@ def main(*args, **kwargs): ...@@ -16,5 +17,6 @@ def main(*args, **kwargs):
DeviceProxy(args[0]).ping() DeviceProxy(args[0]).ping()
return 0 return 0
except BaseException: except BaseException as e:
logging.getLogger().exception(e)
return 1 return 1
...@@ -23,7 +23,10 @@ envdir = {toxworkdir}/testenv ...@@ -23,7 +23,10 @@ envdir = {toxworkdir}/testenv
deps = deps =
-r{toxinidir}/requirements.txt -r{toxinidir}/requirements.txt
-r{toxinidir}/test-requirements.txt -r{toxinidir}/test-requirements.txt
allowlist_externals =
{work_dir}/.tox/bin/python
commands_pre = commands_pre =
{work_dir}/.tox/bin/python -m tox --version
{envpython} --version {envpython} --version
commands = commands =
{envpython} -m pytest --version {envpython} -m pytest --version
...@@ -35,7 +38,9 @@ commands = ...@@ -35,7 +38,9 @@ commands =
envdir = {toxworkdir}/testenvpy310 envdir = {toxworkdir}/testenvpy310
[testenv:integration] [testenv:integration]
allowlist_externals = echo allowlist_externals =
{work_dir}/.tox/bin/python
echo
passenv = TANGO_HOST passenv = TANGO_HOST
setenv = setenv =
VIRTUAL_ENV={envdir} VIRTUAL_ENV={envdir}
...@@ -57,7 +62,6 @@ envdir = {toxworkdir}/coverage ...@@ -57,7 +62,6 @@ envdir = {toxworkdir}/coverage
setenv = setenv =
VIRTUAL_ENV={envdir} VIRTUAL_ENV={envdir}
commands = commands =
{envpython} --version
{envpython} -m pytest --version {envpython} -m pytest --version
{envpython} -m coverage --version {envpython} -m coverage --version
{envpython} -m coverage erase {envpython} -m coverage erase
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment