diff --git a/docker-compose/lofar-device-base.yml b/docker-compose/lofar-device-base.yml index 8f31696433aaee502eea2af83f15c54119b22ec7..43bbcf635b594298e56081c021095fd07e8b6488 100644 --- a/docker-compose/lofar-device-base.yml +++ b/docker-compose/lofar-device-base.yml @@ -33,3 +33,8 @@ services: - control extra_hosts: - "host.docker.internal:host-gateway" + volumes: + - lofar_log:/var/log/tango + +volumes: + lofar_log: diff --git a/docker-compose/logstash/loki.conf b/docker-compose/logstash/loki.conf index 3a594d1532e5c44fc85301163d4009104d95869f..0423b33144dbe6d61fb22e583c9385ce99a7dd90 100644 --- a/docker-compose/logstash/loki.conf +++ b/docker-compose/logstash/loki.conf @@ -3,9 +3,15 @@ input { port => 5044 # TODO (L2SS-748) add SSL encryption } +} + +input { syslog { port => 1514 } +} + +input { tcp { port => 5959 codec => json diff --git a/docker-compose/loki.yml b/docker-compose/loki.yml index 6f634767bd04610a70e969cff2105c062f7ecb1e..317b2e3ff67fcc9316f5895509deebf22ecc3e66 100644 --- a/docker-compose/loki.yml +++ b/docker-compose/loki.yml @@ -23,7 +23,11 @@ services: restart: unless-stopped promtail: - image: grafana/promtail:2.6.0 + image: promtail + build: + context: promtail + args: + SOURCE_IMAGE: grafana/promtail:main container_name: ${CONTAINER_NAME_PREFIX}promtail logging: driver: "json-file" @@ -32,9 +36,15 @@ services: max-file: "10" volumes: - /var/log:/var/log + - lofar_log:/var/lofarlog + # Setting up the Docker Driver Client for Grafana requires a mapping with the O.S. Docker Data Path + # f.e. in Windows-WSL -> /mnt/wsl/docker-desktop-data/version-pack-data/community/docker:/var/lib/docker command: -config.file=/etc/promtail/config.yml networks: - control ports: - "9080:9080" restart: unless-stopped + +volumes: + lofar_log: diff --git a/docker-compose/promtail/Dockerfile b/docker-compose/promtail/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..6c18602c533b0b3b5c528156d9005a375e0fc736 --- /dev/null +++ b/docker-compose/promtail/Dockerfile @@ -0,0 +1,5 @@ +ARG SOURCE_IMAGE +FROM ${SOURCE_IMAGE} + +# Provide our promtail config +COPY config.yml /etc/promtail/ diff --git a/docker-compose/promtail/README.md b/docker-compose/promtail/README.md new file mode 100644 index 0000000000000000000000000000000000000000..50ebf9ec8de9da11abe53f8c0ede54f80d6b3a20 --- /dev/null +++ b/docker-compose/promtail/README.md @@ -0,0 +1,24 @@ +# Promtail + +Promtail is an agent which ships the contents of local logs to a private Grafana Loki instance. + +## Usage and configuration + +Promtail can only be configured to scrape logs from a file, pod, or journal. + +Since we do not save our TANGO device logs to file, it currently appears inconvenient using it for our lofar devices logs. + +It can rather be used for systemlogs, server logs saved to disk, and docker container logs (but, in this case, GrafanaLabs have developed a tailored Docker plugin that allows to send logs directly to Loki - [https://grafana.com/docs/loki/latest/clients/docker-driver/]) + +Since our devices use a python log handler, this last one can be configured to send logs directly to Loki without passing through Promtail [https://medium.com/geekculture/pushing-logs-to-loki-without-using-promtail-fc31dfdde3c6] + +## Using Docker Driver Client + +To get the Docker Driver working, the steps are: + +* Install the Docker Driver Plugin + `docker plugin install grafana/loki-docker-driver:latest --alias loki --grant-all-permissions` + see [https://grafana.com/docs/loki/latest/clients/docker-driver/] +* Change the default Logging Docker Driver, i.e. update the Docker daemon.json file + see [https://grafana.com/docs/loki/latest/clients/docker-driver/configuration/)] +* Restart the Docker daemon and rebuild all the containers diff --git a/docker-compose/promtail/config.yml b/docker-compose/promtail/config.yml new file mode 100644 index 0000000000000000000000000000000000000000..089bd8699826f8debbd2f19e12103e9419a31475 --- /dev/null +++ b/docker-compose/promtail/config.yml @@ -0,0 +1,50 @@ +# Configures the server for Promtail. +server: + http_listen_port: 9080 + grpc_listen_port: 0 + +# Describes how to save read file offsets to disk +positions: + filename: /tmp/positions.yaml + +# Describes how Promtail connects to multiple instances +# of Grafana Loki, sending logs to each. +# WARNING: If one of the remote Loki servers fails to respond or responds +# with any error which is retryable, this will impact sending logs to any +# other configured remote Loki servers. Sending is done on a single thread! +# It is generally recommended to run multiple Promtail clients in parallel +# if you want to send to multiple remote Loki instances. +clients: + - url: http://loki:3100/loki/api/v1/push # Loki + +# Configures how Promtail can scrape logs from a series of targets +# using a specified discovery method + +# Local machine logs +scrape_configs: +- job_name: system + static_configs: + - targets: + - localhost + labels: + job: varlogs + __path__: /var/log/*log + +# Docker logs +- job_name: docker + pipeline_stages: + - docker: {} + static_configs: + - labels: + job: docker + __path__: /var/lib/docker/containers/*/*-json.log + +# Tango logs (disabled) +- job_name: lofar + static_configs: + - targets: + - localhost + labels: + job: lofar + __path__: /var/lofarlog/* +