diff --git a/.prepare.gitlab-ci.yml b/.prepare.gitlab-ci.yml
index e5b805977a55eb9cf26f6621293b3a284c55aa56..6e6f1685872ed810e5104bea6db2656d601ab08c 100644
--- a/.prepare.gitlab-ci.yml
+++ b/.prepare.gitlab-ci.yml
@@ -9,11 +9,12 @@ build_ci_runner_image:
script:
- docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
- |
+ source docker-compose/.env
if docker pull $CI_REGISTRY_IMAGE/ci-build-runner:$CI_COMMIT_REF_SLUG; then
- docker build --cache-from $CI_REGISTRY_IMAGE/ci-build-runner:$CI_COMMIT_REF_SLUG --tag $CI_REGISTRY_IMAGE/ci-build-runner:$CI_COMMIT_REF_SLUG docker-compose/ci-runner
+ docker build --cache-from $CI_REGISTRY_IMAGE/ci-build-runner:$CI_COMMIT_REF_SLUG --tag $CI_REGISTRY_IMAGE/ci-build-runner:$CI_COMMIT_REF_SLUG --build-arg SOURCE_IMAGE=${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-itango:${TANGO_ITANGO_VERSION} docker-compose/ci-runner
else
docker pull $CI_REGISTRY_IMAGE/ci-build-runner:latest || true
- docker build --cache-from $CI_REGISTRY_IMAGE/ci-build-runner:latest --tag $CI_REGISTRY_IMAGE/ci-build-runner:$CI_COMMIT_REF_SLUG docker-compose/ci-runner
+ docker build --cache-from $CI_REGISTRY_IMAGE/ci-build-runner:latest --tag $CI_REGISTRY_IMAGE/ci-build-runner:$CI_COMMIT_REF_SLUG --build-arg SOURCE_IMAGE=${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-itango:${TANGO_ITANGO_VERSION} docker-compose/ci-runner
fi
- docker push $CI_REGISTRY_IMAGE/ci-build-runner:$CI_COMMIT_REF_SLUG # push the image
- |
diff --git a/README.md b/README.md
index efcffd5641884276e3eb03f1800fceb286705acf..1d9513ced416c37b6f7f86289c6df21c376efe4a 100644
--- a/README.md
+++ b/README.md
@@ -6,7 +6,8 @@
# Tango Station Control
-Station Control software related to Tango devices.
+Station Control software related to Tango devices. \
+[Please consult the online documentation!](https://readthedocs.org/projects/lofar20-station-control/badge/?version=latest)
# Index
@@ -14,7 +15,7 @@ Station Control software related to Tango devices.
* [Prerequisites](#prerequisites)
* [Bootstrap](#bootstrap)
* [User documentation (ReadTheDocs (Sphinx / ReStructuredText))](tangostationcontrol/docs/README.md)
-* [Docker compose documentation](docker-compose/README.md)
+* [Docker compose & station services documentation](docker-compose/README.md)
* [Timescaledb](docker-compose/timescaledb/README.md)
* [Jupyter startup files](docker-compose/jupyterlab/ipython-profiles/stationcontrol-jupyter/startup/README.md)
* [Tango Prometheus exporter](https://git.astron.nl/lofar2.0/ska-tango-grafana-exporter)
diff --git a/docker-compose/README.md b/docker-compose/README.md
index c68719dc3e38e3b89f92a810664e59995aca288c..964f08cca268e049b6173efdc1ebb898f358efd3 100644
--- a/docker-compose/README.md
+++ b/docker-compose/README.md
@@ -1,24 +1,63 @@
-# Docker Compose
+# Docker Compose & Station Services
Documentation on how the LOFAR station control software utilizes docker-compose.
This documentation is intended for developers listing strategies and their
respective advantages and disadvantages. In addition, this documentation
contains developer expectations that they should uphold to.
-## Image tagging and change detection
+## Index
-Preventing unnecessary builds of docker images reduces build times and increases
-iteration speed. In order to achieve this the project requires properly tagged
-images and mechanisms for change detection.
+1. [Station services]()
+ 1. [Types of container services and specific strategies](#types-of-containers-and-specific-strategies)
+2. [HDB++ image updates](#hdb-image-updates)
+3. [Gitlab CI/CD](#gitlab-cicd)
+ 1. [Image tagging and change detection](#image-tagging-and-change-detection)
+ 1. [Setup and maintenance](#setup-and-maintenance)
+ 2. [Gitlab CI phases](#gitlab-ci-phases)
-For change detection the system relies on git. Git is used to determine the
-directories and files that have changes between the current and previous commit.
-All image related change detection mechanisms are based on this difference.
+## Station Services
-Using docker cache within the dind service is impractical see:
-https://gitlab.com/gitlab-org/gitlab-foss/-/issues/17861
+The station offers different services, this documentation provides
+names and an overview of their behavior as well as if these services
+are used in production.
-### Types of containers and specific strategies.
+- [Devices \[external\]](https://lofar20-station-control.readthedocs.io/en/latest/devices/overview.html)
+- Simulators
+ - [sdptr \[external\]](https://git.astron.nl/lofar2.0/sdptr)
+ - [sdptr-sim](sdptr-sim.yml)
+ - [pypcc \[external\]](https://git.astron.nl/lofar2.0/pypcc)
+ - [pypcc-sim-base](pypcc-sim-base/Dockerfile)
+ - [apsct-sim](apsct-sim.yml)
+ - [apspu-sim](apspu-sim.yml)
+ - [ccd-sim](ccd-sim.yml)
+ - [recv-sim](recv-sim.yml)
+ - [unb2-sim](unb2-sim.yml)
+- Base images
+ - [ci-runner](ci-runner/Dockerfile)
+ - [lofar-device-base](lofar-device-base.yml)
+- Services
+ - databases
+ - dsconfig
+ - [timescaledb](timescaledb/README.md)
+ - archiver-timescale
+ - hbdpp
+ - hbdpp-cm (ConfigurationManager)
+ - hbdpp-es (EventSubscriber)
+ - prometheus
+ - webservers / user interfaces
+ - jupyterlab
+ - [grafana](grafana/README.md)
+ - [http-json-schemas](http-json-schemas/README.md)
+ - logging / monitoring
+ - prometheus-node-exporter
+ - [tango-prometheus-exporter](tango-prometheus-exporter/README.md)
+ - logstash (grafana-logstash-output-loki)
+ - loki
+ - tango
+ - itango
+ - tango-rest
+
+### Types of containers and specific strategies
- Devices
- Simulators
@@ -41,22 +80,49 @@ Services, same mechanism as devices.
### HDB++ image updates
-The hdbpp Docker image is used as a base image for the hdbppts-cm (ConfigurationManager) and hdbppts-es (EventSubscriber)
-images. If one is developing on a branch and any updates is made in hdbpp/Dockerfile, those won't be automatically picked up from hdbppts-cm and hdbppts-es, because the argument SOURCE_IMAGE in the docker-compose yml file always refers to the remote hdbpp image in the repository.
+The hdbpp Docker image is used as a base image for the `hdbppts-cm`
+(ConfigurationManager) and `hdbppts-es` (EventSubscriber)
+images. If one is developing on a branch and any updates is made
+in hdbpp/Dockerfile, those won't be automatically picked up from `hdbppts-cm`
+and `hdbppts-es`, because the argument `SOURCE_IMAGE` in the docker-compose
+yml file always refers to the remote `hdbpp` image in the repository.
+
+A temporary workaround for locally testing on these archiving containers
+is the following:
+
+- Stop and remove any running `hdbpp*` container
+- In the archiver-timescale.yml file, replace the `hdbppts-cm` and `hdbppts-es`
+ `SOURCE_IMAGE` tag 'latest' with the branch name
+ (e.g. `SOURCE_IMAGE: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/hdbpp:l2ss-new-branch`)
+- Rebuild all the hdbpp* container (`make build hdbpp hdbppts-cm hdbppts-es`),
+ and then start them (`make start hdbpp hdbppts-cm hdbppts-es`)
+- Test the new features
-A temporary workaround for locally testing on these archiving containers is the following:
+After the branch has been correctly developed, tested, the merge has been
+approved, and the new images have been built on the repository:
+- Put back 'latest' tag on the `archiver-timescale.yml` file, replacing the branch name
+- Merge on master
-- Stop and remove any running hdbpp* container
-- In the archiver-timescale.yml file, replace the hdbppts-cm and hdbppts-es SOURCE_IMAGE tag 'latest' with the branch name
-(e.g. SOURCE_IMAGE: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/hdbpp:l2ss-new-branch)
-- Rebuild all the hdbpp* container (make build hdbpp hdbppts-cm hdbppts-es), and then start them
-- Test the new features
+## Gitlab CI/CD
-After the branch has been correctly developed, tested, the merge has been approved, and the new images have been built on the repository:
-- Put back 'latest' tag on the archiver-timescale.yml file, replacing the branch name
-- Merge on master
+1. [Image tagging and change detection](#image-tagging-and-change-detection)
+ 1. [Setup and maintenance](#setup-and-maintenance)
+2. [Gitlab CI phases](#gitlab-ci-phases)
+
+### Image tagging and change detection
+
+Preventing unnecessary builds of docker images reduces build times and increases
+iteration speed. In order to achieve this the project requires properly tagged
+images and mechanisms for change detection.
+
+For change detection the system relies on git. Git is used to determine the
+directories and files that have changes between the current and previous commit.
+All image related change detection mechanisms are based on this difference.
+
+Using docker cache within the dind service is impractical see:
+https://gitlab.com/gitlab-org/gitlab-foss/-/issues/17861
-### Setup and maintenance
+#### Setup and maintenance
All behavioral logic to orchestrate change detection and image pushing can be
found in the sbin/tag_and_push_docker_images.sh script as well as the
diff --git a/docker-compose/ci-runner/Dockerfile b/docker-compose/ci-runner/Dockerfile
index 5491f77de48032d03b13391955fb00a1354b29b1..acfb8a8c435ca20dbf7a86c394688523e2680942 100644
--- a/docker-compose/ci-runner/Dockerfile
+++ b/docker-compose/ci-runner/Dockerfile
@@ -1,4 +1,5 @@
-FROM git.astron.nl:5000/lofar2.0/tango/tango-itango:9.3.10
+ARG SOURCE_IMAGE
+FROM ${SOURCE_IMAGE}
RUN echo 'tzdata tzdata/Areas select Etc' | sudo debconf-set-selections
RUN echo 'tzdata tzdata/Zones/Etc select UTC' | sudo debconf-set-selections
diff --git a/docker-compose/device-configuration.yml b/docker-compose/device-configuration.yml
index 2d6059a6bdd7ce07e3407947ffe9e8c9efb30f47..5c59a5fa36667899d9eb7306ae51341daaa8fcb6 100644
--- a/docker-compose/device-configuration.yml
+++ b/docker-compose/device-configuration.yml
@@ -60,4 +60,4 @@ services:
stop_signal: SIGINT # request a graceful shutdown of Tango
stop_grace_period: 2s
depends_on:
- - schemas
+ - http-json-schemas
diff --git a/docker-compose/device-observation-control.yml b/docker-compose/device-observation-control.yml
index ee26dbc706665e9430c4d78a5885f5295ae26827..cce4c5e8159bedcd6fd90ce0948611c5edff9408 100644
--- a/docker-compose/device-observation-control.yml
+++ b/docker-compose/device-observation-control.yml
@@ -59,4 +59,4 @@ services:
stop_signal: SIGINT # request a graceful shutdown of Tango
stop_grace_period: 2s
depends_on:
- - schemas
+ - http-json-schemas
diff --git a/docker-compose/schemas.yml b/docker-compose/http-json-schemas.yml
similarity index 76%
rename from docker-compose/schemas.yml
rename to docker-compose/http-json-schemas.yml
index 94d01dc65e1905b0e0bedd42befc050bf1c2943c..c8661f3f96a8be7b50c5d55c20d567c7c10f1897 100644
--- a/docker-compose/schemas.yml
+++ b/docker-compose/http-json-schemas.yml
@@ -6,22 +6,22 @@
# created Observation devices.
#
# Defines:
-# - schemas: LOFAR2.0 station
+# - http-json-schemas: LOFAR2.0 station
#
version: '2.1'
services:
- schemas:
+ http-json-schemas:
build:
- context: schemas
- container_name: schemas
+ context: http-json-schemas
+ container_name: http-json-schemas
networks:
- control
# set the hostname, otherwise duplicate device registrations result every
# time the hostname changes as the container is restarted.
- hostname: schemas
+ hostname: http-json-schemas
environment:
- - NGINX_HOST=schemas
+ - NGINX_HOST=http-json-schemas
- NGINX_PORT=80
ports:
- 9999:80
diff --git a/docker-compose/schemas/Dockerfile b/docker-compose/http-json-schemas/Dockerfile
similarity index 100%
rename from docker-compose/schemas/Dockerfile
rename to docker-compose/http-json-schemas/Dockerfile
diff --git a/docker-compose/http-json-schemas/README.md b/docker-compose/http-json-schemas/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..ba91c82f03d07e024dc9dad77e9f5af2edb1c5a8
--- /dev/null
+++ b/docker-compose/http-json-schemas/README.md
@@ -0,0 +1,3 @@
+# HTTP JSON Schemas
+
+Service to serve JSON schema files over HTTP. Used by configuration manager device.
diff --git a/docker-compose/schemas/definitions/observation-settings.json b/docker-compose/http-json-schemas/definitions/observation-settings.json
similarity index 100%
rename from docker-compose/schemas/definitions/observation-settings.json
rename to docker-compose/http-json-schemas/definitions/observation-settings.json
diff --git a/docker-compose/schemas/definitions/pointing.json b/docker-compose/http-json-schemas/definitions/pointing.json
similarity index 100%
rename from docker-compose/schemas/definitions/pointing.json
rename to docker-compose/http-json-schemas/definitions/pointing.json
diff --git a/docker-compose/schemas/definitions/sap.json b/docker-compose/http-json-schemas/definitions/sap.json
similarity index 100%
rename from docker-compose/schemas/definitions/sap.json
rename to docker-compose/http-json-schemas/definitions/sap.json
diff --git a/docker-compose/schemas/definitions/station-configuration.json b/docker-compose/http-json-schemas/definitions/station-configuration.json
similarity index 97%
rename from docker-compose/schemas/definitions/station-configuration.json
rename to docker-compose/http-json-schemas/definitions/station-configuration.json
index 1f6886386642e4942e8309fa0db6eb6ed17a92e0..1dce35d0e1c00a8f91a83bb3b0a0a447e26a9faf 100644
--- a/docker-compose/schemas/definitions/station-configuration.json
+++ b/docker-compose/http-json-schemas/definitions/station-configuration.json
@@ -1,6 +1,6 @@
{
"$schema": "http://json-schema.org/draft-07/schema#",
- "$id": "http://schemas/station-configuration.json",
+ "$id": "http://http-json-schemas/station-configuration.json",
"title": "Tango device generator JSON canonical format",
"type": "object",
"additionalProperties": false,
diff --git a/docker-compose/integration-test.yml b/docker-compose/integration-test.yml
index 5ee8edb51f150ba162ba1cf5e0b4e8d7d384137a..1d771d6d4ccc5a3c558f6b088a6cc29795fc04b5 100644
--- a/docker-compose/integration-test.yml
+++ b/docker-compose/integration-test.yml
@@ -14,7 +14,7 @@ services:
context: .
dockerfile: ci-runner/Dockerfile
args:
- SOURCE_IMAGE: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/ci-runner:latest
+ SOURCE_IMAGE: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-itango:${TANGO_ITANGO_VERSION}
container_name: integration-test
networks:
- control
diff --git a/docker-compose/loki.yml b/docker-compose/loki.yml
index 924fad29e7c9d0da262695b3f4e2f152cfc7965b..fa114fcdecf72042f97100af47f90673fb2a57ec 100644
--- a/docker-compose/loki.yml
+++ b/docker-compose/loki.yml
@@ -23,30 +23,3 @@ services:
- "3100:3100"
command: -config.file=/etc/loki/local-config.yaml
restart: unless-stopped
-
- promtail:
- image: promtail
- build:
- context: promtail
- args:
- SOURCE_IMAGE: grafana/promtail:main
- container_name: promtail
- logging:
- driver: "json-file"
- options:
- max-size: "100m"
- max-file: "10"
- volumes:
- - /var/log:/var/log
- - lofar_log:/var/lofarlog
- # Setting up the Docker Driver Client for Grafana requires a mapping with the O.S. Docker Data Path
- # f.e. in Windows-WSL -> /mnt/wsl/docker-desktop-data/version-pack-data/community/docker:/var/lib/docker
- command: -config.file=/etc/promtail/config.yml
- networks:
- - control
- ports:
- - "9080:9080"
- restart: unless-stopped
-
-volumes:
- lofar_log:
diff --git a/docker-compose/prometheus-node-exporter.yml b/docker-compose/prometheus-node-exporter.yml
index 4089bbff76d58a4a8cdbb9ec925afc430aa88957..84fffd2949534aa51bb9adda6dfd5a7d813ad740 100644
--- a/docker-compose/prometheus-node-exporter.yml
+++ b/docker-compose/prometheus-node-exporter.yml
@@ -12,11 +12,6 @@ services:
prometheus-node-exporter:
image: prom/node-exporter
container_name: prometheus-node-exporter
- logging:
- driver: "json-file"
- options:
- max-size: "100m"
- max-file: "10"
network_mode: host # run on the host to be able to access host network statistics
logging:
driver: syslog
diff --git a/docker-compose/prometheus.yml b/docker-compose/prometheus.yml
index 23bc926af849460b750023031b2d9b0616655c48..f398a9d2a3ad8322a0e6de77073b8be1519b8081 100644
--- a/docker-compose/prometheus.yml
+++ b/docker-compose/prometheus.yml
@@ -18,11 +18,6 @@ services:
build:
context: prometheus
container_name: prometheus
- logging:
- driver: "json-file"
- options:
- max-size: "100m"
- max-file: "10"
networks:
- control
extra_hosts:
diff --git a/docker-compose/promtail/Dockerfile b/docker-compose/promtail/Dockerfile
deleted file mode 100644
index 6c18602c533b0b3b5c528156d9005a375e0fc736..0000000000000000000000000000000000000000
--- a/docker-compose/promtail/Dockerfile
+++ /dev/null
@@ -1,5 +0,0 @@
-ARG SOURCE_IMAGE
-FROM ${SOURCE_IMAGE}
-
-# Provide our promtail config
-COPY config.yml /etc/promtail/
diff --git a/docker-compose/promtail/README.md b/docker-compose/promtail/README.md
deleted file mode 100644
index 50ebf9ec8de9da11abe53f8c0ede54f80d6b3a20..0000000000000000000000000000000000000000
--- a/docker-compose/promtail/README.md
+++ /dev/null
@@ -1,24 +0,0 @@
-# Promtail
-
-Promtail is an agent which ships the contents of local logs to a private Grafana Loki instance.
-
-## Usage and configuration
-
-Promtail can only be configured to scrape logs from a file, pod, or journal.
-
-Since we do not save our TANGO device logs to file, it currently appears inconvenient using it for our lofar devices logs.
-
-It can rather be used for systemlogs, server logs saved to disk, and docker container logs (but, in this case, GrafanaLabs have developed a tailored Docker plugin that allows to send logs directly to Loki - [https://grafana.com/docs/loki/latest/clients/docker-driver/])
-
-Since our devices use a python log handler, this last one can be configured to send logs directly to Loki without passing through Promtail [https://medium.com/geekculture/pushing-logs-to-loki-without-using-promtail-fc31dfdde3c6]
-
-## Using Docker Driver Client
-
-To get the Docker Driver working, the steps are:
-
-* Install the Docker Driver Plugin
- `docker plugin install grafana/loki-docker-driver:latest --alias loki --grant-all-permissions`
- see [https://grafana.com/docs/loki/latest/clients/docker-driver/]
-* Change the default Logging Docker Driver, i.e. update the Docker daemon.json file
- see [https://grafana.com/docs/loki/latest/clients/docker-driver/configuration/)]
-* Restart the Docker daemon and rebuild all the containers
diff --git a/docker-compose/promtail/config.yml b/docker-compose/promtail/config.yml
deleted file mode 100644
index 089bd8699826f8debbd2f19e12103e9419a31475..0000000000000000000000000000000000000000
--- a/docker-compose/promtail/config.yml
+++ /dev/null
@@ -1,50 +0,0 @@
-# Configures the server for Promtail.
-server:
- http_listen_port: 9080
- grpc_listen_port: 0
-
-# Describes how to save read file offsets to disk
-positions:
- filename: /tmp/positions.yaml
-
-# Describes how Promtail connects to multiple instances
-# of Grafana Loki, sending logs to each.
-# WARNING: If one of the remote Loki servers fails to respond or responds
-# with any error which is retryable, this will impact sending logs to any
-# other configured remote Loki servers. Sending is done on a single thread!
-# It is generally recommended to run multiple Promtail clients in parallel
-# if you want to send to multiple remote Loki instances.
-clients:
- - url: http://loki:3100/loki/api/v1/push # Loki
-
-# Configures how Promtail can scrape logs from a series of targets
-# using a specified discovery method
-
-# Local machine logs
-scrape_configs:
-- job_name: system
- static_configs:
- - targets:
- - localhost
- labels:
- job: varlogs
- __path__: /var/log/*log
-
-# Docker logs
-- job_name: docker
- pipeline_stages:
- - docker: {}
- static_configs:
- - labels:
- job: docker
- __path__: /var/lib/docker/containers/*/*-json.log
-
-# Tango logs (disabled)
-- job_name: lofar
- static_configs:
- - targets:
- - localhost
- labels:
- job: lofar
- __path__: /var/lofarlog/*
-
diff --git a/docker-compose/rest.yml b/docker-compose/tango-rest.yml
similarity index 98%
rename from docker-compose/rest.yml
rename to docker-compose/tango-rest.yml
index b9f16755b3d17d94d412ac02d93a37482af88cb7..27f106dca2724b1890a42e8206d0e97f78579283 100644
--- a/docker-compose/rest.yml
+++ b/docker-compose/tango-rest.yml
@@ -14,7 +14,7 @@
version: '2.1'
services:
- rest:
+ tango-rest:
image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-rest:${TANGO_REST_VERSION}
container_name: tango-rest
networks:
diff --git a/docker-compose/timescaledb/README.md b/docker-compose/timescaledb/README.md
index 75f9446ef254658f8282bcf6255c87aa0f39a94b..1aee7e0eddb909b8902e1442b14cc81a7611c98c 100644
--- a/docker-compose/timescaledb/README.md
+++ b/docker-compose/timescaledb/README.md
@@ -1,18 +1,38 @@
# TimescaleDB Docker Image
-The Dockerfile in this directory allows to create a container with a PostrgreSQL-Timescale DBMS (https://www.timescale.com/), and then initialise it with the DB schema required by the Tango Archiving framework.
+The Dockerfile in this directory allows to create a container with a
+PostrgreSQL-Timescale DBMS (https://www.timescale.com/), and then initialise
+it with the DB schema required by the Tango Archiving framework.
-The main image is pulled from the official PostgreSQL repository in the Docker Hub (https://hub.docker.com/_/postgres). This image offers several features to customize and extend itself.
+The main image is pulled from the official PostgreSQL repository in the
+Docker Hub (https://hub.docker.com/_/postgres). This image offers several
+features to customize and extend itself.
## Initialization scripts
-If you would like to do additional initialization in an image derived from the Postgres official one, add one or more *.sql, *.sql.gz, or *.sh scripts under /docker-entrypoint-initdb.d (creating the directory if necessary). After the entrypoint calls initdb to create the default postgres user and database, it will run any *.sql files, run any executable *.sh scripts, and source any non-executable *.sh scripts found in that directory to do further initialization before starting the service.
+If you would like to do additional initialization in an image derived from
+the Postgres official one, add one or more *.sql, *.sql.gz, or *.sh scripts
+under /docker-entrypoint-initdb.d (creating the directory if necessary).
+After the entrypoint calls initdb to create the default postgres user and
+database, it will run any *.sql files, run any executable *.sh scripts, and
+source any non-executable *.sh scripts found in that directory to do further
+initialization before starting the service.
-The script files in the directory /docker-entrypoint-initdb.d are sequentially executed following their preempted number in the filename. Hence, the first ones (000_install_timescaledb.sh and 001_timescaledb_tune.sh) are provided by default.
-The next ones have been pulled from the official Tango repository in order to create the desired DB schema. These files are in the 'resources' directory and they have been pulled from Tango-Hdbpp_Timescale_Project (https://github.com/tango-controls-hdbpp/hdbpp-timescale-project/tree/master/resources/schema):
+The script files in the directory /docker-entrypoint-initdb.d are sequentially
+executed following their preempted number in the filename. Hence, the first
+ones (000_install_timescaledb.sh and 001_timescaledb_tune.sh) are provided by
+default.
+
+The next ones have been pulled from the official Tango repository in order to
+create the desired DB schema. These files are in the 'resources' directory
+and they have been pulled from Tango-Hdbpp_Timescale_Project
+(https://github.com/tango-controls-hdbpp/hdbpp-timescale-project/tree/master/resources/schema):
- admin.sql creates the admin user that will create the tables
-- hdb_schema.sql creates the standard Tango Archiving DB (This is the only MANDATORY script)
+- hdb_schema.sql creates the standard Tango Archiving DB (This is the only
+ MANDATORY script)
- hdb_roles.sql creates additional roles
-- hdb_ext_aggregates.sql creates the continuous aggregate views (https://docs.timescale.com/timescaledb/latest/how-to-guides/continuous-aggregates/)
+- hdb_ext_aggregates.sql creates the continuous aggregate views
+ (https://docs.timescale.com/timescaledb/latest/how-to-guides/continuous-aggregates/)
- cleanup.sql strips the SUPERUSER trait from hdb_admin
-Last updates on these scripts are dated to August 2021 (more info can be found at https://github.com/tango-controls-hdbpp/hdbpp-timescale-project/blob/master/doc/db-schema-config.md)
+Last updates on these scripts are dated to August 2021 (more info can be found
+ at https://github.com/tango-controls-hdbpp/hdbpp-timescale-project/blob/master/doc/db-schema-config.md)
diff --git a/sbin/run_integration_test.sh b/sbin/run_integration_test.sh
index 881a631a1984354511b589f847ef0b082b76dfde..c39992fe6c50d497d99973fc964e11294fdeb2e5 100755
--- a/sbin/run_integration_test.sh
+++ b/sbin/run_integration_test.sh
@@ -85,14 +85,14 @@ SIMULATORS=(sdptr-sim recv-sim unb2-sim apsct-sim apspu-sim ccd-sim)
# jupyter is physically large > 2.5gb and overlayfs is really slow.
make build "${DEVICES[@]}" "${SIMULATORS[@]}"
-make build logstash integration-test schemas
+make build logstash integration-test http-json-schemas
make build archiver-timescale hdbppts-cm hdbppts-es
# Start and stop sequence
-make stop schemas
+make stop http-json-schemas
make stop "${DEVICES[@]}" "${SIMULATORS[@]}" hdbppts-es hdbppts-cm archiver-timescale
make stop device-docker # this one does not test well in docker-in-docker
-make stop logstash schemas
+make stop logstash
# Run dummy integration test to install pytango in tox virtualenv without
# the memory pressure of the ELK stack.
@@ -102,7 +102,7 @@ make stop logstash schemas
# TODO(L2SS-992): Remove me and above documentation
integration_test dummy
-make start logstash schemas
+make start logstash http-json-schemas
# Update the dsconfig
# Do not remove `bash`, otherwise statement ignored by gitlab ci shell!
diff --git a/tangostationcontrol/docs/source/developer.rst b/tangostationcontrol/docs/source/developer.rst
index d9ecd655f20f4a42be1ed08bedb4928a09ac62ed..d3a725740cd81b61c7ac5eb2998934d148047c70 100644
--- a/tangostationcontrol/docs/source/developer.rst
+++ b/tangostationcontrol/docs/source/developer.rst
@@ -63,6 +63,11 @@ For more information, see:
Logging
-------------------------
+Overview of the data flow between docker services to facilitate logging
+
+.. image:: static/images/logging-data-flow.png
+ :target: _images/logging-data-flow.png
+
The Logstash pipeline collects the logs from the containers, as well as any external processes that send theirs. The following interfaces are available for this purpose:
+-------------+------------+-------------------------------------------------------------------------------------------------------------+
@@ -89,7 +94,7 @@ Log from Docker
Not all Docker containers run our Python programs, and can forward the logs themselves. For those, we use the ``syslog`` log driver in Docker. Extend the ``docker compose`` files with:
-.. literalinclude:: ../../../docker-compose/rest.yml
+.. literalinclude:: ../../../docker-compose/tango-rest.yml
:start-at: logging:
:end-before: restart:
diff --git a/tangostationcontrol/docs/source/static/drawio/logging-data-flow-diagram.drawio b/tangostationcontrol/docs/source/static/drawio/logging-data-flow-diagram.drawio
new file mode 100644
index 0000000000000000000000000000000000000000..6cef348269c23e2dda8f5c4563475765f119a750
--- /dev/null
+++ b/tangostationcontrol/docs/source/static/drawio/logging-data-flow-diagram.drawio
@@ -0,0 +1 @@
+<mxfile host="app.diagrams.net" modified="2023-03-10T09:30:32.391Z" agent="5.0 (X11)" etag="04sI2TFf57dfzGZP1dRO" version="20.8.23" type="device"><diagram name="Page-1" id="wlMN7mhJMfydnIccLdaM">5V1bj6M2FP41kdqHjADbQB7bme1FaquVZqvuPnqCJ6FLQgrOTtJfXxJwAsdMQhgbm05Wmg3G3D6+c3xudibofrX7OaOb5e9pxJKJ50S7CXqYeN4s9Iu/h4Z92UAQLhsWWRyVTe654TH+l1WNTtW6jSOWNzryNE14vGk2ztP1ms15o41mWfrS7PacJs2rbuiiuqJzbnic04RJ3f6KI74sW0NS6/0LixdLcWXXqfasqOhcNeRLGqUvtSb0YYLuszTl5bfV7p4lB+wELuVxP72y93RjGVvzLgfQbPXpj3z656ffFlOOHv75lfBkGpDyNN9osq2euLpbvhcQsHX0wwHJYmue0DyP5xP045KvkqLBLb5m6XYdscNVnGKL7WL++fD9ziPV5pfaroddfWMvNtY823+ubxyPuSNi83zYcUscV94rixbwbZ1BqZrydJvN2QUkMKrYRbMF45cgO727gvMsXbHihorjXs7sEORY1ogh2jKWUB5/a94vrUi6OJ3udIWPaVw8iedU8uSK8whxEuwSpyifszqqzgZwIg+cCCNwohIH6UQFE+i+1m1z6JBfuGEHXKfJ0uJLeUaxVcP03HRk8i2sxvpY3Z3UCskZOBOTpAvBO8R9SYfbyaCcdK9cRyvpsH+ddE1SvSxjzh439PjqX4rhs8nA6nQs42x3mUEyMaoDEHhzSDCiRhwUtDDHhW+mzpIajrfDJFCxCSYPwOQGLTChNpgcXTC5rnIV1nOMfWVAd4fSfe44BmYwznl9B2Yosh6k2JgHZreD8A/NajOkHgWnMVHEaTSQsWli3O+gqIcez3ygQlqGM9dv4Y2vbdCXMHpg3+I5y0cLlTb7KJCgWtFd0XDU186KLejTnr8VuOc4Se7TJM2Ox6KIsDDCRXvOs/Qrq+0JvSfk+4qsLCCdaNaCtTckLUMJ67/zdD0t0GHvCV9tXO4QXDIs7wibVo2y1/gYr7bFuJ1m1mnHzmhpY5RwBiym1MlgN0YpVw5/cbpepNNNdrjCkm3zKdtt0uKZs9Gip49i6kPib/ZRAiNOSgWE5U4KQU0CWe+kwGjqIE6KjcFJGDJpEf3ZoHpTtr2PejN6Gi1W+rSkbDsbBwlbR6iZBFJEOX2iOYvMG3d94dKXGnDs4xSxjVOeHG+K8nm6fo4XowVLH6Nkd+Go0e/2xWObRqt31kkfXB3y5vbHQXyviav5OJMnuxT5Pk/SQmKdbbQp/rpEOG3vA2p9FJZtOOOCDlxcbDzm5MnG25LzzfQY/cznS7ai5s2Tvqjpo5ZszUkg9a0qCoyUFZU8MOa5z3DzDbvgzVlXVmTCcxcGgF0KrQkEcU1rNCRbxWXIM2M5Hy9e2nQZ8pTrMu0lvHbHIa8qIesqc00USyALw0ahZ5t9hmRLo5a6WacRsyd/0xs9bboNWzhiQpTM552xPGKeOTZevPSxqsOIadhFImFHkPSVLWPZ/U7SRc5pvixav1tk9JmuaXFq0TpNt3yz5dMk/Rp/bxzQ3vXy+mgnRzAPUBlHqi/19AGl2rIwElA7BdCsiV1i2Rbh80PI8hAsKv4jMzJ7T0DrK5brYLb0Cy+JgpGaQ9ZhJubZc/tS3/eKG1e+LDFF1lMbp+rqAJZcNeYBwsEDOm5dPUApqg4DY4o8QKlg1HEu3pfUH1/uj8nF/no8TKLaTgPq6Zkc/rWpJ//4ORyRrnmtvfwoyrBCarSMu20ZVkhFdVpLtk9Oqa1ynFCe2hrnG9A3bmhO2toF9ykaZozwci73idGDmnWIg//PTO8KvT6mdyhiVDGt/3KBrZUWkkDiqolUliKYMpF8FyhP0tdEgjMTNM2SxWAKJIKLrMD7gjbgEEF10sH17ZvMaXLcvcLxPstkaJUL1NV1KEufTMkFLJwjQU+5INfWh1EkF5DneDYEzzsUYNgfeQDI+W254EEjPOLEoBqoaEGnQPG7QFib4eKrX8zjLYaLW1fpovaos+WC5NfqO3TmBm2v1Tl+1Op0sdDOVZ1OjIaDMJhPdAqR36rTMeAz0VQQQPymreNXro5Wne5bOKETZoE6l+voy6v5spN/yqWNFS196lbjDM6R28OCR5bbw4GqZQ59kObB0IpS5SfCauWmPXxV117rb8Te9rXFW/plpJpmy21WC1Zsg3SNt5Tjmy0pKQLp33dyNNGUkkIgQOSjy3IB+4sH1Lu8qOrJ1Ea8JDj9k7SVWA3riKr27xWgBOtcjFe6i8hQfdGi7WbPWTZNqPmJ6n3x0mcMyhVpIrrhjDK60Vtu9S1ZZqPcggGrdXJt28irz4cLZck1DxMwtFuX33GH1G+hHCuj2XxZ2EDZlMcrlh/X5h8tbtqEMPTsY1cQXEdp0MUTQjnatIyeNpvpfDVasPQxSnVhiQKQQusYJceaSkZZsLZpX7D0MUp2nGTdbsXaHL3NB33YdVjY4J0Wvwj70/LiFxyC5bT7JoSkoKam4pfQGUHxi8BCg1z0ClLezWYIBCqd8GbpOO7/yLJCKR4meSpf+UH4jJbnAUK4Ar3tIhNAEQguiwzsP8gk7FmHnOuAIgPKEW6L66uuFwvHIRdwrVKxdOWtchGI6V2nGl88iFxgK+VC24/hjX8GVtcf4TE8BQvIhTRzqnPNDcznahov4JQq70q+y/AUrGLz/IOVZffzr36iD/8B</diagram></mxfile>
\ No newline at end of file
diff --git a/tangostationcontrol/docs/source/static/images/logging-data-flow.png b/tangostationcontrol/docs/source/static/images/logging-data-flow.png
new file mode 100644
index 0000000000000000000000000000000000000000..8fdaff0fc7f8abd66ce8d26feb4cc9b383730c9b
Binary files /dev/null and b/tangostationcontrol/docs/source/static/images/logging-data-flow.png differ
diff --git a/tangostationcontrol/tangostationcontrol/configuration/configuration_base.py b/tangostationcontrol/tangostationcontrol/configuration/configuration_base.py
index d375aa812f35d34ed2c0f290e7a3f2d652599ca5..eb5fcc3928e3d7b8ea36f4aee1ce4f4e57b2a23f 100644
--- a/tangostationcontrol/tangostationcontrol/configuration/configuration_base.py
+++ b/tangostationcontrol/tangostationcontrol/configuration/configuration_base.py
@@ -53,7 +53,7 @@ jsonschema.validators.Draft7Validator.TYPE_CHECKER = (
class _ConfigurationBase(ABC):
- BASE_URL = "http://schemas/"
+ BASE_URL = "http://http-json-schemas/"
@staticmethod
def _class_to_url(cls_name):
diff --git a/tangostationcontrol/tangostationcontrol/test/configuration/_mock_requests.py b/tangostationcontrol/tangostationcontrol/test/configuration/_mock_requests.py
index df19c68f338c540791a29e888aa8c886af62a25d..01f6190a82a1ac22173d693d27f8684de17d5fd8 100644
--- a/tangostationcontrol/tangostationcontrol/test/configuration/_mock_requests.py
+++ b/tangostationcontrol/tangostationcontrol/test/configuration/_mock_requests.py
@@ -137,11 +137,11 @@ def mocked_requests_get(*args, **kwargs):
def json(self):
return json.loads(self.text)
- if args[0] == "http://schemas/pointing.json":
+ if args[0] == "http://http-json-schemas/pointing.json":
return MockResponse(POINTING_SCHEMA, 200)
- elif args[0] == "http://schemas/sap.json":
+ elif args[0] == "http://http-json-schemas/sap.json":
return MockResponse(SAP_SCHEMA, 200)
- elif args[0] == "http://schemas/observation-settings.json":
+ elif args[0] == "http://http-json-schemas/observation-settings.json":
return MockResponse(OBSERVATION_SETTINGS_SCHEMA, 200)
return MockResponse(None, 404)