diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7afde25971ac6078ecb990aebec678af2d2360cf..91f2d43c802ff7b33d231d5af947a325ad21318f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -20,6 +20,7 @@ include: stages: - prepare + - packaging - images - building - linting @@ -27,7 +28,6 @@ stages: - static-analysis - unit-tests - integration-tests - - packaging - publish - deploy @@ -37,16 +37,30 @@ trigger_prepare: strategy: depend include: .prepare.gitlab-ci.yml +wheel_packaging: + stage: packaging + artifacts: + paths: + - tangostationcontrol/dist/*.whl + script: + - cd tangostationcontrol + - tox -e build + # See docker-compose/README.md for docker image behavior and explanation .base_docker_images: stage: images image: docker:latest + dependencies: + - wheel_packaging + needs: + - wheel_packaging tags: - privileged services: - name: docker:dind variables: DOCKER_TLS_CERTDIR: "/certs" + TANGO_SKIP_BUILD: "true" before_script: - | if [[ "$CI_COMMIT_BRANCH" == "$CI_DEFAULT_BRANCH" && -z "$CI_COMMIT_TAG" ]]; then @@ -58,7 +72,7 @@ trigger_prepare: fi - apk add --update make bash docker-compose - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY - - touch /root/.Xauthority + - touch ~/.Xauthority # Hack BASH_SOURCE into sourced files, docker its sh shell won't set this - export BASH_SOURCE=$(pwd)/setup.sh # source the lofarrc file and mask its non zero exit code @@ -228,6 +242,9 @@ integration_test_docker: image: docker:23.0.5 # latest ships with docker compose v2.19, which has the following bug: https://github.com/docker/compose/issues/10668 needs: - unit_test + - wheel_packaging + dependencies: + - wheel_packaging tags: - privileged services: @@ -289,17 +306,6 @@ multi_project_integration_test: project: lofar2.0/multi-project-integration strategy: depend -wheel_packaging: - stage: packaging - rules: - - if: ($CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH) || $CI_COMMIT_TAG - artifacts: - paths: - - tangostationcontrol/dist/*.whl - script: - - cd tangostationcontrol - - tox -e build - publish_on_gitlab: stage: publish environment: gitlab @@ -319,6 +325,17 @@ publish_on_gitlab: - echo "run twine for gitlab" - python3 -m twine upload --repository-url ${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/pypi tangostationcontrol/dist/* +release_job: + stage: publish + image: registry.gitlab.com/gitlab-org/release-cli:latest + rules: + - if: '$CI_COMMIT_TAG && $CI_COMMIT_REF_PROTECTED == "true"' + script: + - echo "running release_job" + release: + tag_name: '$CI_COMMIT_TAG' + description: '$CI_COMMIT_TAG' + .base_deploy: stage: deploy image: ubuntu:bionic diff --git a/bin/start-ds.sh b/bin/start-ds.sh index 723a8d9d76741b9e375deaab2996bbc1e00d1ca2..6732784f2ab448be70b17b73dab17e516c8e4ef2 100755 --- a/bin/start-ds.sh +++ b/bin/start-ds.sh @@ -29,10 +29,8 @@ CWD=$(pwd) cd /opt/lofar/tango || exit 1 # Check if configured for specific version -if [[ $TANGOSTATIONCONTROL ]]; then - # TODO (Corne): Download version from artifacts or pypi. - # Consider exit 2 an UnImplementedError - exit 2 +if [[ $TANGO_STATION_CONTROL ]]; then + echo "Assuming station control install performed as part of base image" else # Install the package, exit 1 if it fails # pip install ./ will _NOT_ install dependencies in requirements.txt! diff --git a/docker-compose/Makefile b/docker-compose/Makefile index c1f5a12c5f62b541a28a36d5a5bbe1838f3791e7..4929af30299e3eea95c6c03c93f05690e6394258 100644 --- a/docker-compose/Makefile +++ b/docker-compose/Makefile @@ -221,9 +221,17 @@ endif $(DOCKER_COMPOSE_ARGS) $(DOCKER_COMPOSE) -f tango.yml -f networks.yml up --no-recreate -d -context: ## Move the necessary files to create minimal docker context - @mkdir -p tmp - @cp ../tangostationcontrol/requirements.txt tmp/ +context: ## Move and build the necessary files to create minimal docker context + rm -rf ./tmp; \ + mkdir -p tmp; \ + if [ -z "$${TANGO_SKIP_BUILD}" ]; then \ + rm -rf ../tangostationcontrol/dist; \ + cd ../tangostationcontrol; \ + tox -e build; \ + cd ../docker-compose; \ + fi; \ + cp ../tangostationcontrol/dist/*.whl tmp/; \ + cp ../tangostationcontrol/requirements.txt tmp/; \ bootstrap: pull build # first start, initialise from scratch $(MAKE) start dsconfig # boot up containers to load configurations diff --git a/docker-compose/ci-runner/Dockerfile b/docker-compose/ci-runner/Dockerfile index acfb8a8c435ca20dbf7a86c394688523e2680942..d15d54b7a66545685253052e25da54211c918b17 100644 --- a/docker-compose/ci-runner/Dockerfile +++ b/docker-compose/ci-runner/Dockerfile @@ -7,3 +7,4 @@ RUN echo 'debconf debconf/frontend select Noninteractive' | sudo debconf-set-sel RUN sudo apt-get update && sudo apt-get install -y tzdata RUN sudo apt-get update && sudo apt-get install -y tox python3-dev libboost-python-dev pkg-config git shellcheck graphviz casacore-data RUN sudo ln -s /var/lib/casacore/data/geodetic/TAI_UTC /usr/share/casacore/data/geodetic/TAI_UTC +RUN sudo pip install --upgrade tox diff --git a/docker-compose/lofar-device-base.yml b/docker-compose/lofar-device-base.yml index 9bb7d5768ac630cae34d308453f21bbc2b893311..46ee4a2cfc7194ee5287e79382f0849eea3cb4a4 100644 --- a/docker-compose/lofar-device-base.yml +++ b/docker-compose/lofar-device-base.yml @@ -23,6 +23,7 @@ services: dockerfile: lofar-device-base/Dockerfile args: SOURCE_IMAGE: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-itango:${TANGO_ITANGO_VERSION} + TANGO_STATION_CONTROL: ${TANGO_STATION_CONTROL} hostname: lofar-device-base container_name: lofar-device-base # These parameters are just visual queues, you have to define them again diff --git a/docker-compose/lofar-device-base/Dockerfile b/docker-compose/lofar-device-base/Dockerfile index 82df78ca809cd5fd2061ea7163b9138e611eab52..77818acac2a9954d2fd6177d1948ce4e84c7619d 100644 --- a/docker-compose/lofar-device-base/Dockerfile +++ b/docker-compose/lofar-device-base/Dockerfile @@ -1,6 +1,9 @@ ARG SOURCE_IMAGE FROM ${SOURCE_IMAGE} +ARG TANGO_STATION_CONTROL +ENV TANGO_STATION_CONTROL $TANGO_STATION_CONTROL + RUN --mount=type=cache,target=/var/cache/apt \ sudo apt-get update RUN --mount=type=cache,target=/var/cache/apt \ @@ -16,12 +19,15 @@ RUN --mount=type=cache,target=/var/cache/apt \ RUN --mount=type=cache,target=/var/cache/apt \ sudo apt-get install -y netcat -# Manually install all requirements from the .txt as part of the base image -# This reduces runtime overhead as well as preventing issues around dependency -# installation for development builds (pip install ./ ignores requirements.txt) COPY tmp/requirements.txt /tangostationcontrol-requirements.txt +COPY tmp/*.whl / -RUN sudo pip3 install -r /tangostationcontrol-requirements.txt +RUN echo "TANGO_STATION_CONTROL: ${TANGO_STATION_CONTROL}" +RUN if [ $TANGO_STATION_CONTROL ]; then \ + sudo pip3 install /*.whl; \ + else \ + sudo pip3 install -r /tangostationcontrol-requirements.txt; \ + fi # install and use ephimerides and geodetic ("measures") tables for casacore. # we install a _stub_ since the tables need to be deployed explicitly from within the software. diff --git a/sbin/run_integration_test.sh b/sbin/run_integration_test.sh index 2a908c1636097cc23b65fccf09430f0bd1371de8..afed289d534538656b245244f157379b5157e151 100755 --- a/sbin/run_integration_test.sh +++ b/sbin/run_integration_test.sh @@ -76,14 +76,23 @@ if [ -z "$LOFAR20_DIR" ]; then LOFAR20_DIR=$(readlink -f "${LOFAR20_DIR_RELATIVE}") fi +export TANGO_SKIP_BUILD=1 + cd "$LOFAR20_DIR/docker-compose" || exit 1 -# Start the database server first -[ -n "${no_build}" ] || make build databaseds dsconfig +# Start the database server first, `-z ${y+x}` is the inverse of `-n ${y}` +if [ -z "${no_build+x}" ]; then + rm -rf "${LOFAR20_DIR}/tangostationcontrol/dist" + cd "${LOFAR20_DIR}/tangostationcontrol" || exit 1 + tox -e build + cd "$LOFAR20_DIR/docker-compose" || exit 1 + make build databaseds dsconfig +fi + make start databaseds dsconfig # Give dsconfig and databaseds time to start -sleep 1 # dsconfig container must be up and running... +sleep 5 # dsconfig container must be up and running... # shellcheck disable=SC2016 echo '/usr/local/bin/wait-for-it.sh ${TANGO_HOST} --strict --timeout=300 -- true' | make run dsconfig bash - diff --git a/setup.sh b/setup.sh index ecab83f49583b812de428516c5b23a03ef9be651..ee556bb84e2e5badd8767145db70dd3ef368064d 100755 --- a/setup.sh +++ b/setup.sh @@ -25,5 +25,9 @@ export NETWORK_MODE=tangonet # port. Example: export TANGO_HOST=station-xk25.astron.nl:10000 export TANGO_HOST=$(hostname):10000 +# Configure to install station control in lofar-device-base image +# TODO(L2SS-520): Extend to support debug and expose as property in devices +export TANGO_STATION_CONTROL=1 + # Allow read access for everybody to allow Docker forwarding of X11. chmod a+r ~/.Xauthority diff --git a/tangostationcontrol/docs/source/developer.rst b/tangostationcontrol/docs/source/developer.rst index 10758ac3d098eb5100e60b2e1a276c59ef479cc4..5739419b643daa67cecf9b9c095e3c130a774fc3 100644 --- a/tangostationcontrol/docs/source/developer.rst +++ b/tangostationcontrol/docs/source/developer.rst @@ -3,6 +3,38 @@ Developer information This chapter describes key areas useful for developers. +Environment variables +--------------------- + +Several environment variables fundamentally control the deployment and +development environment. These include: + +- `TANGO_HOST` +- `HOSTNAME` +- `NETWORK_MODE` +- `TANGO_STATION_CONTROL` +- `TANGO_SKIP_BUILD` + +Firstly, `TANGO_HOST` should point to the tango database server including its port. +An example would be `10.14.0.205:10000`. If `TANGO_HOST` is not set instead +`$HOSTNAME:10000` is used. + +With `NETWORK_MODE` the name of the docker network can be controlled or `host` can +be used as value to point directly to the host. This feature has not been used +in a very long time. + +Finally `TANGO_STATION_CONTROL` can be used to control if device containers should +build software from source (developer mode). Or if the software should be built +into the `lofar-device-base` docker image directly. If `TANGO_STATION_CONTROL` is set +the makefile will build a wheel package which will be installed into the docker image. + +If instead a particular wheel package needs to be installed `TANGO_SKIP_BUILD` can be +set as well. Be sure the wheel package is placed in the `tangostationcontrol/dist/` +directory. + +In the future the actual value of the `TANGO_STATION_CONTROL` variable might be used +to control various types of different behavior. + Docker compose ------------------------- diff --git a/tangostationcontrol/tangostationcontrol/common/health.py b/tangostationcontrol/tangostationcontrol/common/health.py index f37de1c982575579e3b6c0ca8b917ef8bebb7b3f..a080ee3d0d7bb1f7f67a692adf179fb674de1078 100644 --- a/tangostationcontrol/tangostationcontrol/common/health.py +++ b/tangostationcontrol/tangostationcontrol/common/health.py @@ -1,6 +1,7 @@ # Copyright (C) 2022 ASTRON (Netherlands Institute for Radio Astronomy) # SPDX-License-Identifier: Apache-2.0 +import logging import sys from tango import DeviceProxy @@ -16,5 +17,6 @@ def main(*args, **kwargs): DeviceProxy(args[0]).ping() return 0 - except BaseException: + except BaseException as e: + logging.getLogger().exception(e) return 1 diff --git a/tangostationcontrol/tox.ini b/tangostationcontrol/tox.ini index 788c2fb706ce9aa8057ba55793d14889f65cce14..9c848792b19e852856f68247c40455efd5e48d1e 100644 --- a/tangostationcontrol/tox.ini +++ b/tangostationcontrol/tox.ini @@ -23,7 +23,10 @@ envdir = {toxworkdir}/testenv deps = -r{toxinidir}/requirements.txt -r{toxinidir}/test-requirements.txt +allowlist_externals = + {work_dir}/.tox/bin/python commands_pre = + {work_dir}/.tox/bin/python -m tox --version {envpython} --version commands = {envpython} -m pytest --version @@ -35,7 +38,9 @@ commands = envdir = {toxworkdir}/testenvpy310 [testenv:integration] -allowlist_externals = echo +allowlist_externals = + {work_dir}/.tox/bin/python + echo passenv = TANGO_HOST setenv = VIRTUAL_ENV={envdir} @@ -57,7 +62,6 @@ envdir = {toxworkdir}/coverage setenv = VIRTUAL_ENV={envdir} commands = - {envpython} --version {envpython} -m pytest --version {envpython} -m coverage --version {envpython} -m coverage erase