L2SS-1905: Auto install

docker/snmp-exporter/generator.yml

@@ -4,6 +4,9 @@ auths:
     version: 1
   public_v2:
     version: 2
+  lcu:
+    community: monitoring
+    version: 2
   pcon_acc:
     community: accread
     version: 1
@@ -109,6 +112,22 @@ modules:
         scale: 0.1
       outputVoltage:
         scale: 0.1
+#
+# Supermicro X12/X13
+#
+  aten:
+    walk:
+      - ipmi
+      - boardinfo
+      - hardwareinfo
+      - powerinfo
+    overrides:
+      powerStatus:
+        type: EnumAsStateSet
+      sensorReading:
+        type: OctetString
+      biosVesion:
+        type: OctetString
 
 #
 # White Rabbit

infra/env/common.yaml

@@ -37,7 +37,7 @@ object_storage:
     pass: minioadmin
 
 vector:
-  version: 0.39.X-distroless-static
+  version: 0.40.X-distroless-static
 
 pypcc:
   version: v1-0

infra/jobs/station/logging.levant.nomad

@@ -173,16 +173,12 @@ healthchecks.require_healthy = true
 [sources.host_metrics]
   type                 = "host_metrics"
   scrape_interval_secs = 10
-[sources.nomad_metrics]
-  type                 = "prometheus_scrape"
-  scrape_interval_secs = 10
-  endpoints            = [ "http://(( env "attr.unique.network.ip-address" )):4646/v1/metrics?format=prometheus" ]
 [sources.vector_metrics]
   type                 = "internal_metrics"
   scrape_interval_secs = 10
 [sinks.prometheus_remote_write]
   type                 = "prometheus_remote_write"
-  inputs               = [ "host_metrics", "nomad_metrics", "vector_metrics" ]
+  inputs               = [ "host_metrics", "vector_metrics" ]
   endpoint             = "http://prometheus.service.consul:9090/api/v1/write"
   healthcheck.enabled  = false
 EOF

infra/jobs/station/monitoring.levant.nomad

@@ -278,9 +278,9 @@ postgres.service.consul:5432:grafana:postgres:password
             consul_sd_configs:
               - server: 'consul.service.consul:8500'
                 services:
-          {{range services}}{{if in .Tags "scrape"}}{{ if .Name | regexMatch "(.+)-sidecar-proxy$" }}{{ else }}
+          {{range services}}{{ if and (in .Tags "scrape") (.Name | contains "sidecar" | not) }}
                   - '{{.Name}}'
-          {{end}}{{end}}{{end}}
+          {{ end }}{{end}}
             relabel_configs:
               - target_label: host
                 replacement: localhost
@@ -302,9 +302,15 @@ postgres.service.consul:5432:grafana:postgres:password
             consul_sd_configs:
               - server: 'consul.service.consul:8500'
                 services:
-          {{range services}}{{if in .Tags "snmp-scrape"}}{{ if .Name | regexMatch "(.+)-sidecar-proxy$" }}{{ else }}
+          {{range services}}{{ if and (in .Tags "snmp-scrape") (.Name | contains "sidecar" | not) }}
                   - '{{.Name}}'
-          {{end}}{{end}}{{end}}
+          {{ end }}{{end}}
+            metric_relabel_configs:
+              - source_labels: [prefix, __name__]
+                separator: ""
+                target_label: __name__
+              - action: labeldrop
+                regex: prefix
             relabel_configs:
               - target_label: host
                 replacement: localhost
@@ -316,6 +322,8 @@ postgres.service.consul:5432:grafana:postgres:password
                 target_label: __param_auth
               - target_label: __address__
                 replacement: "snmp-exporter.service.consul:9116"
+              - source_labels: [__meta_consul_service_metadata_prefix]
+                target_label: prefix
               - source_labels: [__meta_consul_service]
                 target_label: instance # avoid a dynamic ip:port or hostname
         EOH
@@ -391,6 +399,12 @@ postgres.service.consul:5432:grafana:postgres:password
           ring:
             kvstore:
               store: inmemory
+        limits_config:
+          ingestion_rate_mb: 1000
+          ingestion_burst_size_mb: 10000
+          max_global_streams_per_user: 0
+          per_stream_rate_limit: 100MB
+          per_stream_rate_limit_burst: 1000MB
 
 [[ if eq .station "dev" ]]
         distributor:

infra/station/README.md

+# LCU installation
+
+## Base system
+
+The base system is installed using [debian preseeding](https://wiki.debian.org/DebianInstaller/Preseed).
+The setup is rather simple. The following settings are applied during installation:
+* system user lcuadmin is created
+* openssh-server is installed
+* custom disk partitioning is used
+
+The full configuration can be found in the `preseed.cfg` in the debian directory.
+
+### Partitioning
+
+| dev            | size   | type | mount     |
+|----------------|--------|------|-----------|
+| /dev/nvme0n1p1 | 1 GiB  | EFI  | /boot/efi |
+| /dev/nvme0n1p2 | 1 GiB  | ext4 | /boot     |
+| /dev/nvme0n1p3 | 100%   | lvm  | vg0       |
+| /dev/vg0/root  | 50 GiB | ext4 | /         |
+| /dev/vg0/home  | 10 GiB | ext4 | /home     |
+| /dev/vg0/swap  | 8 GiB  | swap |           |
+
+## Installation
+
+The software of the LCU is installed using ansible.
+
+The following playbooks have to be applied:
+
+#### base.yml
+
+Copies ssh keys, sets a new user password for lcuadmin and disables password login for ssh.
+
+#### consul.yml
+
+Installs the consul server and applies configuration to connect to the central consul datacenter.
+
+Configures dnsmasq as the primary DNS server for the OS and to forward `.consul` queries to the consul DNS server
+
+Creates a role on the central Vault to allow the consul server to sign TLS certificates used for mTLS inter-service configuration
+
+#### nomad.yml
+Installs the nomad server as well as docker and the required CNI plugins
+
+Increases settings for inotify and the UDP max receive buffer size
+
+Alters the nomad server to allow memory over-subscription
+
+Deploys nomads own consul agent to join the consul datacenter
+
+#### ovs.yml
+Installs OVS on the host system
+
+Configures the internal network interface(s) to be part of the ovs bridge
+
+Installs and configures the [ovs CNI plugin](https://github.com/k8snetworkplumbingwg/ovs-cni)
+
+#### time.yml
+Installs and configures chrony as a NTP server
+
+Sets the timezone to UTC
+
+### Run a playbook
+Create a file `ansible.cfg` with the content
+```ini
+[defaults]
+host_key_checking = False
+inventory = hosts.yaml
+```
+
+Create a file `hosts.yaml` with the content
+```yaml
+station:
+  hosts:
+    <station>:
+      ansible_host: lcu.<station>.lofar
+      station_name: <station>
+vault:
+  hosts:
+    monitor:
+      ansible_host: monitor.control.lofar
+all:
+  vars:
+    consul_vault_token: "<token>"
+    nomad_encrypt: "<encrypt key>"
+    nomad_vault_token: "<token>"
+    ansible_user: lcuadmin
+    consul_encrypt: "<encrypt key>>"
+    volume_group: vg0
+    eth:
+      - enp1s0f0
+      - enp1s0f1
+    dns_servers:
+      - 10.149.64.20
+      - 10.149.64.21
+
+```
+
+Run
+```bash
+ansible-playbook -v <playbook>.yml
+```

infra/station/base.yml

+---
+- name: setup base system
+  hosts: station
+  remote_user: root
+  become: true
+  become_user: root
+  tasks:
+    - name: install packages
+      apt:
+        pkg:
+          - htop
+        update_cache: yes
+        state: present
+
+    - name: Set a hostname specifying strategy
+      hostname:
+        name: "{{ station_name }}c"
+
+    - name: Set up multiple authorized keys
+      ansible.posix.authorized_key:
+        user: lcuadmin
+        state: present
+        key: '{{ item }}'
+      with_file:
+        - public_keys/mol
+        - public_keys/feldt
+        - public_keys/lukken
+
+    - name: Create sudoers
+      copy:
+        content: 'lcuadmin ALL=(ALL)NOPASSWD: ALL'
+        dest: /etc/sudoers.d/lcuadmin
+#
+#    - name: Set lcuadmin password
+#      user:
+#        name: lcuadmin
+#        password: "{{ lcu_admin_password }}"
+#
+#    - name: disable sshd password login
+#      copy:
+#        content: 'PasswordAuthentication no'
+#        dest: /etc/ssh/sshd_config.d/no-passwd.conf
+#
+#    - name: Restart sshd
+#      service:
+#            name: ssh
+#            enabled: true
+#            state: restarted

infra/station/cni/nftables.conf.j2

@@ -9,6 +9,6 @@ table ip ovs_nat {
 
         chain postrouting {
                 type nat hook postrouting priority srcnat; policy accept;
-                oifname "{{ eth_ext }}" masquerade
+                oifname "ctrl0" masquerade
         }
 }

infra/station/cni/stationnet.conflist

@@ -5,11 +5,11 @@
         {
             "type": "ovs",
             "bridge": "br0",
-            "interface_type": "system",
             "configuration_path": "/opt/cni/net.d/ovs.d/ovs.conf",
             "isDefaultGateway": true,
             "ipMasq": true,
             "hairpinMode": true,
+            "vlan": 999,
             "mtu": 9000,
             "ipam": {
                 "type": "host-local",
@@ -21,6 +21,9 @@
                     { "dst": "0.0.0.0/0" }
                 ]
             }
+        },
+        {
+            "type": "firewall"
         }
     ]
 }

infra/station/consul/consul.hcl.j2

@@ -5,12 +5,12 @@ encrypt = "{{ consul_encrypt }}"
 data_dir = "/opt/consul"
 bind_addr = "0.0.0.0"
 
-client_addr = "10.99.250.250 {{ '{{' }} GetInterfaceIP \"{{ eth_ext }}\" {{ '}}' }} 127.0.0.1"
+client_addr = "10.99.250.250 {{ '{{' }} GetInterfaceIP \"ctrl0\" {{ '}}' }} 127.0.0.1"
 advertise_addr = "10.99.250.250"
 retry_join = ["127.0.0.1"]
 
-serf_wan = "{{ '{{' }} GetInterfaceIP \"{{ eth_ext }}\" {{ '}}' }}"
-#advertise_addr_wan = "{{ '{{' }} GetInterfaceIP \"{{ eth_ext }}\" {{ '}}' }}"
+serf_wan = "{{ '{{' }} GetInterfaceIP \"ctrl0\" {{ '}}' }}"
+#advertise_addr_wan = "{{ '{{' }} GetInterfaceIP \"ctrl0\" {{ '}}' }}"
 primary_gateways = [ "mesh-gateway.central.lofar.net:8443" ]
 
 bootstrap=true
@@ -50,7 +50,7 @@ telemetry {
 services {
   name = "white-rabbit-snmp"
   id = "white-rabbit-snmp"
-  address = "10.151.1.20"
+  address = "10.151.68.20"
   port = 161
   tags = ["snmp-scrape"]
   meta = {

infra/station/debian/grub/grub.cfg

+if [ x$feature_default_font_path = xy ] ; then
+   font=unicode
+else
+   font=$prefix/font.pf2
+fi
+
+if loadfont $font ; then
+  set gfxmode=800x600
+  set gfxpayload=keep
+  insmod efi_gop
+  insmod efi_uga
+  insmod video_bochs
+  insmod video_cirrus
+  insmod gfxterm
+  insmod png
+  terminal_output gfxterm
+fi
+
+if background_image /isolinux/splash.png; then
+  set color_normal=light-gray/black
+  set color_highlight=white/black
+elif background_image /splash.png; then
+  set color_normal=light-gray/black
+  set color_highlight=white/black
+else
+  set menu_color_normal=cyan/blue
+  set menu_color_highlight=white/blue
+fi
+
+insmod play
+play 960 440 1 0 4 440 1
+set theme=/boot/grub/theme/1
+menuentry --hotkey=a 'Automated install' {
+    set background_color=black
+    linux    /install.amd/vmlinuz auto=true url=file://cdrom/preseed.cfg priority=critical vga=788 --- quiet
+    initrd   /install.amd/initrd.gz
+}

infra/station/debian/isolinux/auto.cfg

+label autodark
+	menu label ^Automated LCU install
+	kernel /install.amd/vmlinuz
+	append auto=true url=file://cdrom/preseed.cfg priority=critical vga=788 initrd=/install.amd/initrd.gz theme=dark --- quiet

infra/station/debian/isolinux/menu.cfg

+menu hshift 4
+menu width 70
+
+menu title Debian GNU/Linux installer menu (BIOS mode)
+include stdmenu.cfg
+include auto.cfg
+include txt.cfg
+menu begin advanced
+    menu label ^Advanced options
+	menu title Advanced options
+	include stdmenu.cfg
+	label mainmenu
+		menu label ^Back..
+		menu exit
+	include adgtk.cfg
+	include adtxt.cfg
+	include adspkgtk.cfg
+	include adspk.cfg
+menu end
+menu begin dark
+    menu label Accessible ^dark contrast installer menu
+	menu title Accessible dark contrast option
+	include drkmenu.cfg
+	label mainmenu
+		menu label ^Back..
+		menu exit
+	include drkgtk.cfg
+	include drk.cfg
+	menu begin advanced
+	    menu label ^Advanced options
+		menu title Advanced options
+		include drkmenu.cfg
+		label mainmenu
+			menu label ^Back..
+			menu exit
+		include addrkgtk.cfg
+		include addrk.cfg
+	menu end
+	include x86drkme.cfg
+	label help
+		menu label ^Help
+		text help
+   Display help screens; type 'menu' at boot prompt to return to this menu
+		endtext
+		config prompt.cfg
+menu end
+include x86menu.cfg
+label help
+	menu label ^Help
+	text help
+   Display help screens; type 'menu' at boot prompt to return to this menu
+	endtext
+	config prompt.cfg
+include spkgtk.cfg
+include spk.cfg

infra/station/debian/preseed.cfg

+#_preseed_V1
+#### Contents of the preconfiguration file (for bookworm)
+### Localization
+# Preseeding only locale sets language, country and locale.
+#d-i debian-installer/locale string en_US
+
+# The values can also be preseeded individually for greater flexibility.
+d-i debian-installer/language string en
+d-i debian-installer/country string NL
+d-i debian-installer/locale string en_US.UTF-8
+# Optionally specify additional locales to be generated.
+#d-i localechooser/supported-locales multiselect en_US.UTF-8, nl_NL.UTF-8
+
+# Keyboard selection.
+d-i keyboard-configuration/xkb-keymap select us
+# d-i keyboard-configuration/toggle select No toggling
+
+### Network configuration
+# Disable network configuration entirely. This is useful for cdrom
+# installations on non-networked devices where the network questions,
+# warning and long timeouts are a nuisance.
+#d-i netcfg/enable boolean false
+
+# netcfg will choose an interface that has link if possible. This makes it
+# skip displaying a list if there is more than one interface.
+d-i netcfg/choose_interface select auto
+
+# To pick a particular interface instead:
+d-i netcfg/choose_interface select enp1s0f0
+
+# To set a different link detection timeout (default is 3 seconds).
+# Values are interpreted as seconds.
+d-i netcfg/link_wait_timeout string 10
+
+# If you have a slow dhcp server and the installer times out waiting for
+# it, this might be useful.
+d-i netcfg/dhcp_timeout string 60
+d-i netcfg/dhcpv6_timeout string 60
+
+# Automatic network configuration is the default.
+# If you prefer to configure the network manually, uncomment this line and
+# the static network configuration below.
+#d-i netcfg/disable_autoconfig boolean true
+
+# If you want the preconfiguration file to work on systems both with and
+# without a dhcp server, uncomment these lines and the static network
+# configuration below.
+#d-i netcfg/dhcp_failed note
+#d-i netcfg/dhcp_options select Configure network manually
+
+# Static network configuration.
+#
+# IPv4 example
+#d-i netcfg/get_ipaddress string 192.168.1.42
+#d-i netcfg/get_netmask string 255.255.255.0
+#d-i netcfg/get_gateway string 192.168.1.1
+#d-i netcfg/get_nameservers string 192.168.1.1
+#d-i netcfg/confirm_static boolean true
+#
+# IPv6 example
+#d-i netcfg/get_ipaddress string fc00::2
+#d-i netcfg/get_netmask string ffff:ffff:ffff:ffff::
+#d-i netcfg/get_gateway string fc00::1
+#d-i netcfg/get_nameservers string fc00::1
+#d-i netcfg/confirm_static boolean true
+
+# Any hostname and domain names assigned from dhcp take precedence over
+# values set here. However, setting the values still prevents the questions
+# from being shown, even if values come from dhcp.
+#d-i netcfg/get_hostname string unassigned-hostname
+#d-i netcfg/get_domain string unassigned-domain
+
+# If you want to force a hostname, regardless of what either the DHCP
+# server returns or what the reverse DNS entry for the IP is, uncomment
+# and adjust the following line.
+d-i netcfg/hostname string lcu-install
+d-i netcfg/domain string control.lofar
+
+# Disable that annoying WEP key dialog.
+d-i netcfg/wireless_wep string
+
+# If non-free firmware is needed for the network or other hardware, you can
+# configure the installer to always try to load it, without prompting. Or
+# change to false to disable asking.
+d-i hw-detect/load_firmware boolean true
+
+### Network console
+# Use the following settings if you wish to make use of the network-console
+# component for remote installation over SSH. This only makes sense if you
+# intend to perform the remainder of the installation manually.
+#d-i anna/choose_modules string network-console
+#d-i network-console/authorized_keys_url string http://10.0.0.1/openssh-key
+#d-i network-console/password password r00tme
+#d-i network-console/password-again password r00tme
+
+### Mirror settings
+# Mirror protocol:
+# If you select ftp, the mirror/country string does not need to be set.
+# Default value for the mirror protocol: http.
+d-i mirror/protocol string ftp
+#d-i mirror/country string manual
+d-i mirror/http/hostname string ftp.nl.debian.org
+d-i mirror/http/directory string /debian
+#d-i mirror/http/proxy string
+
+# Suite to install.
+#d-i mirror/suite string testing
+# Suite to use for loading installer components (optional).
+#d-i mirror/udeb/suite string testing
+
+### Account setup
+# Skip creation of a root account (normal user account will be able to
+# use sudo).
+d-i passwd/root-login boolean false
+# Alternatively, to skip creation of a normal user account.
+#d-i passwd/make-user boolean false
+
+d-i passwd/user-fullname string lcuadmin
+d-i passwd/username string lcuadmin
+d-i passwd/user-password password Start123
+d-i passwd/user-password-again password Start123
+# Root password, either in clear text
+#d-i passwd/root-password password Start123
+#d-i passwd/root-password-again password Start123
+# or encrypted using a crypt(3)  hash.
+#d-i passwd/root-password-crypted password [crypt(3) hash]
+
+# To create a normal user account.
+#d-i passwd/user-fullname string Debian User
+#d-i passwd/username string debian
+# Normal user's password, either in clear text
+#d-i passwd/user-password password insecure
+#d-i passwd/user-password-again password insecure
+# or encrypted using a crypt(3) hash.
+#d-i passwd/user-password-crypted password [crypt(3) hash]
+# Create the first user with the specified UID instead of the default.
+#d-i passwd/user-uid string 1010
+
+# The user account will be added to some standard initial groups. To
+# override that, use this.
+#d-i passwd/user-default-groups string audio cdrom video
+
+### Clock and time zone setup
+# Controls whether or not the hardware clock is set to UTC.
+d-i clock-setup/utc boolean true
+
+# You may set this to any valid setting for $TZ; see the contents of
+# /usr/share/zoneinfo/ for valid values.
+d-i time/zone string Europe/Amsterdam
+
+# Controls whether to use NTP to set the clock during the install
+d-i clock-setup/ntp boolean true
+# NTP server to use. The default is almost always fine here.
+d-i clock-setup/ntp-server string ntp1.control.lofar
+
+### Partitioning
+d-i partman-auto/disk string /dev/nvme0n1
+d-i partman-auto/method string lvm
+
+d-i partman-auto/purge_lvm_from_device boolean true
+d-i partman-md/device_remove_md boolean true
+d-i partman-lvm/device_remove_lvm boolean true
+
+d-i partman-auto/choose_recipe select boot-root
+d-i partman-auto-lvm/new_vg_name string vg0
+
+d-i partman-efi/non_efi_system boolean true
+d-i partman-partitioning/choose_label select gpt
+d-i partman-partitioning/default_label string gpt
+
+d-i partman/choose_partition select finish
+
+d-i partman-basicfilesystems/no_mount_point boolean true
+d-i partman-partitioning/confirm_write_new_label boolean true
+d-i partman-lvm/confirm boolean true
+d-i partman-lvm/confirm_nooverwrite boolean true
+
+d-i partman-auto/expert_recipe string                       \
+    boot-root ::                                            \
+            1024 1024 1024 fat32                            \
+                    $iflabel{ gpt }                         \
+                    $primary{ }                             \
+                    method{ efi }                           \
+                    format{ }                               \
+            .                                               \
+            1024 1024 1024 ext4                             \
+                    $primary{ } $bootable{ }                \
+                    method{ format } format{ }              \
+                    use_filesystem{ } filesystem{ ext4 }    \
+                    mountpoint{ /boot }                     \
+            .                                               \
+            500 10000 1000000000 free                       \
+                    $defaultignore{ }                       \
+                    $primary{ }                             \
+                    method{ lvm }                           \
+                    vg_name{ vg0 }                          \
+            .                                               \
+            8192 8192 8192 linux-swap                       \
+                    $lvmok{ }                               \
+                    in_vg{ vg0 }                            \
+                    lv_name{ swap }                         \
+                    method{ swap }                          \
+                    format{ }                               \
+            .                                               \
+            51200 51200 51200 ext4                          \
+                    $lvmok{ }                               \
+                    in_vg{ vg0 }                            \
+                    lv_name{ root }                         \
+                    method{ format }                        \
+                    format{ }                               \
+	                use_filesystem{ }                       \
+	                filesystem{ ext4 }                      \
+	                mountpoint{ / }                         \
+            .                                               \
+            10240 10240 10240 ext4                          \
+                    $lvmok{ }                               \
+                    in_vg{ vg0 }                            \
+                    lv_name{ home }                         \
+                    method{ format }                        \
+                    format{ }                               \
+	                use_filesystem{ }                       \
+	                filesystem{ ext4 }                      \
+	                mountpoint{ /home }                     \
+            .
+
+
+## Controlling how partitions are mounted
+# The default is to mount by UUID, but you can also choose "traditional" to
+# use traditional device names, or "label" to try filesystem labels before
+# falling back to UUIDs.
+#d-i partman/mount_style select uuid
+
+### Base system installation
+# Configure APT to not install recommended packages by default. Use of this
+# option can result in an incomplete system and should only be used by very
+# experienced users.
+#d-i base-installer/install-recommends boolean false
+
+# The kernel image (meta) package to be installed; "none" can be used if no
+# kernel is to be installed.
+#d-i base-installer/kernel/image string linux-image-686
+
+### Apt setup
+# Choose, if you want to scan additional installation media
+# (default: false).
+d-i apt-setup/cdrom/set-first boolean false
+# You can choose to install non-free firmware.
+d-i apt-setup/non-free-firmware boolean true
+# You can choose to install non-free and contrib software.
+d-i apt-setup/non-free boolean true
+d-i apt-setup/contrib boolean true
+# Uncomment the following line, if you don't want to have the sources.list
+# entry for a DVD/BD installation image active in the installed system
+# (entries for netinst or CD images will be disabled anyway, regardless of
+# this setting).
+#d-i apt-setup/disable-cdrom-entries boolean true
+# Uncomment this if you don't want to use a network mirror.
+d-i apt-setup/use_mirror boolean false
+# Select which update services to use; define the mirrors to be used.
+# Values shown below are the normal defaults.
+#d-i apt-setup/services-select multiselect security, updates
+#d-i apt-setup/security_host string security.debian.org
+
+# Additional repositories, local[0-9] available
+#d-i apt-setup/local0/repository string \
+#       http://local.server/debian stable main
+#d-i apt-setup/local0/comment string local server
+# Enable deb-src lines
+#d-i apt-setup/local0/source boolean true
+# URL to the public key of the local repository; you must provide a key or
+# apt will complain about the unauthenticated repository and so the
+# sources.list line will be left commented out.
+#d-i apt-setup/local0/key string http://local.server/key
+# or one can provide it in-line by base64 encoding the contents of the
+# key file (with `base64 -w0`) and specifying it thus:
+#d-i apt-setup/local0/key string base64://LS0tLS1CRUdJTiBQR1AgUFVCTElDIEtFWSBCTE9DSy0tLS0tCi4uLgo=
+# The content of the key file is checked to see if it appears to be ASCII-armoured.
+# If so it will be saved with an ".asc" extension, otherwise it gets a '.gpg' extension.
+# "keybox database" format is currently not supported. (see generators/60local in apt-setup's source)
+
+# By default the installer requires that repositories be authenticated
+# using a known gpg key. This setting can be used to disable that
+# authentication. Warning: Insecure, not recommended.
+#d-i debian-installer/allow_unauthenticated boolean true
+
+# Uncomment this to add multiarch configuration for i386
+#d-i apt-setup/multiarch string i386
+
+
+### Package selection
+#tasksel tasksel/first multiselect standard, openssh-server
+
+# Or choose to not get the tasksel dialog displayed at all (and don't install
+# any packages):
+d-i pkgsel/run_tasksel boolean false
+
+# Individual additional packages to install
+d-i pkgsel/include string openssh-server sudo
+# Whether to upgrade packages after debootstrap.
+# Allowed values: none, safe-upgrade, full-upgrade
+d-i pkgsel/upgrade select full-upgrade
+
+# You can choose, if your system will report back on what software you have
+# installed, and what software you use. The default is not to report back,
+# but sending reports helps the project determine what software is most
+# popular and should be included on the first CD/DVD.
+#popularity-contest popularity-contest/participate boolean false
+
+### Boot loader installation
+# Grub is the boot loader (for x86).
+
+# This is fairly safe to set, it makes grub install automatically to the UEFI
+# partition/boot record if no other operating system is detected on the machine.
+d-i grub-installer/only_debian boolean true
+
+# This one makes grub-installer install to the UEFI partition/boot record, if
+# it also finds some other OS, which is less safe as it might not be able to
+# boot that other OS.
+d-i grub-installer/with_other_os boolean false
+
+# Due notably to potential USB sticks, the location of the primary drive can
+# not be determined safely in general, so this needs to be specified:
+#d-i grub-installer/bootdev  string /dev/sda
+# To install to the primary device (assuming it is not a USB stick):
+d-i grub-installer/bootdev  string default
+
+# Alternatively, if you want to install to a location other than the UEFI
+# parition/boot record, uncomment and edit these lines:
+#d-i grub-installer/only_debian boolean false
+#d-i grub-installer/with_other_os boolean false
+#d-i grub-installer/bootdev  string (hd0,1)
+# To install grub to multiple disks:
+#d-i grub-installer/bootdev  string (hd0,1) (hd1,1) (hd2,1)
+
+# Optional password for grub, either in clear text
+#d-i grub-installer/password password r00tme
+#d-i grub-installer/password-again password r00tme
+# or encrypted using an MD5 hash, see grub-md5-crypt(8).
+#d-i grub-installer/password-crypted password [MD5 hash]
+
+# Use the following option to add additional boot parameters for the
+# installed system (if supported by the bootloader installer).
+# Note: options passed to the installer will be added automatically.
+#d-i debian-installer/add-kernel-opts string nousb
+
+### Finishing up the installation
+# During installations from serial console, the regular virtual consoles
+# (VT1-VT6) are normally disabled in /etc/inittab. Uncomment the next
+# line to prevent this.
+#d-i finish-install/keep-consoles boolean true
+
+# Avoid that last message about the install being complete.
+d-i finish-install/reboot_in_progress note
+
+# This will prevent the installer from ejecting the CD during the reboot,
+# which is useful in some situations.
+#d-i cdrom-detect/eject boolean false
+
+# This is how to make the installer shutdown when finished, but not
+# reboot into the installed system.
+#d-i debian-installer/exit/halt boolean true
+# This will power off the machine instead of just halting it.
+#d-i debian-installer/exit/poweroff boolean true
+
+### Preseeding other packages
+# Depending on what software you choose to install, or if things go wrong
+# during the installation process, it's possible that other questions may
+# be asked. You can preseed those too, of course. To get a list of every
+# possible question that could be asked during an install, do an
+# installation, and then run these commands:
+#   debconf-get-selections --installer > file
+#   debconf-get-selections >> file
+
+
+#### Advanced options
+### Running custom commands during the installation
+# d-i preseeding is inherently not secure. Nothing in the installer checks
+# for attempts at buffer overflows or other exploits of the values of a
+# preconfiguration file like this one. Only use preconfiguration files from
+# trusted locations! To drive that home, and because it's generally useful,
+# here's a way to run any shell command you'd like inside the installer,
+# automatically.
+
+# This first command is run as early as possible, just after
+# preseeding is read.
+#d-i preseed/early_command string anna-install some-udeb
+# This command is run immediately before the partitioner starts. It may be
+# useful to apply dynamic partitioner preseeding that depends on the state
+# of the disks (which may not be visible when preseed/early_command runs).
+#d-i partman/early_command \
+#       string debconf-set partman-auto/disk "$(list-devices disk | head -n1)"
+# This command is run just before the install finishes, but when there is
+# still a usable /target directory. You can chroot to /target and use it
+# directly, or use the apt-install and in-target commands to easily install
+# packages and run commands in the target system.
+#d-i preseed/late_command string apt-install zsh; in-target chsh -s /bin/zsh

infra/station/docker/daemon.json.j2

+{
+  "registry-mirrors": ["https://docker.lofar.net"]
+}

infra/station/ipmi-metrics.yml

+---
+- name: setup ipmi scraping
+  hosts: station
+  remote_user: root
+  become: true
+  become_user: root
+  tasks:
+    - name: install packages
+      apt:
+        pkg:
+          - prometheus-ipmi-exporter
+          - freeipmi-tools
+        update_cache: yes
+        state: present
+    - name: copy prometheus-ipmi config
+      template:
+        src: ipmi/ipmi.yml
+        dest: /etc/prometheus/ipmi.yml
+    - name: copy prometheus-ipmi config
+      template:
+        src: ipmi/default
+        dest: /etc/default/prometheus-ipmi-exporter
+    - name: copy sudoers
+      template:
+        src: ipmi/prometheus.sudoers
+        dest: /etc/sudoers.d/prometheus
+    - name: copy lcu-ipmi-service.hcl
+      template:
+        src: ipmi/lcu-ipmi-service.hcl
+        dest: /etc/consul.d/lcu-ipmi-service.hcl
+
+    - name: Start prometheus-ipmi-exporter.service
+      ansible.builtin.service:
+        name: prometheus-ipmi-exporter.service
+        state: restarted
+        enabled: yes
+    - name: Reload consul.service
+      ansible.builtin.service:
+        name: consul.service
+        state: reloaded
+
+

infra/station/ipmi/default

+ARGS="--config.file=/etc/prometheus/ipmi.yml"
+
+# prometheus-ipmi-exporter supports the following options:
+#
+#  --config.file=CONFIG.FILE
+#    Path to configuration file.
+#  --freeipmi.path="/usr/sbin"
+#    Path to FreeIPMI executables.
+#  --log.level=info
+#    Only log messages with the given severity or above. One of: [debug, info,
+#    warn, error]
+#  --log.format=logfmt
+#    Output format of log messages. One of: [logfmt, json]
+#  --web.listen-address=":9290"
+#    Address to listen on for web interface and telemetry.

infra/station/ipmi/ipmi.yml

+modules:
+  default:
+    # Available collectors are bmc, ipmi, chassis, dcmi, sel, and sm-lan-mode
+    collectors:
+      - bmc
+      - ipmi
+      - dcmi
+      - sel
+    # Got any sensors you don't care about? Add them here.
+    collector_cmd:
+      ipmi: ../bin/sudo
+      bmc: ../bin/sudo
+      dcmi: ../bin/sudo
+      sel: ../bin/sudo
+    custom_args:
+      ipmi:
+        - "ipmimonitoring"
+      bmc:
+        - "bmc-info"
+      dcmi:
+        - "ipmi-dcmi"
+      sel:
+        - "ipmi-sel"
+    exclude_sensor_ids:
+      - 2
+      - 29
+      - 32

infra/station/ipmi/lcu-ipmi-service.hcl

+services {
+  name = "lcu-ipmi"
+  id = "lcu-ipmi"
+  address = "10.99.250.250"
+  port = 9290
+  tags = ["scrape"]
+}

infra/station/ipmi/prometheus.sudoers

+prometheus ALL = NOPASSWD: /usr/sbin/ipmimonitoring,\
+                           /usr/sbin/ipmi-sensors,\
+                           /usr/sbin/ipmi-dcmi,\
+                           /usr/sbin/ipmi-raw,\
+                           /usr/sbin/bmc-info,\
+                           /usr/sbin/ipmi-chassis,\
+                           /usr/sbin/ipmi-sel