Merge branch 'vlans-per-station' into 'master'

Generate hosts.yaml, with variable VLANs per station & secret generation/lookups

See merge request !1107

.gitlab-ci.yml

@@ -140,7 +140,7 @@ render_CDB_environment_ilt:
       python -m pip install .
 
       # fetch IPs from netbox (netbox.astron.nl)
-      curl -X GET -k 'https://10.87.2.131/api/ipam/ip-addresses/?dns_name__isw=lcu-&limit=100' -H 'Accept: application/json' -H "Authorization: Token ${NETBOX_TOKEN}" > netbox-lcu-ips.json
+      curl -X GET -k 'https://10.87.2.131/api/ipam/ip-addresses/?dns_name__isw=lcu-&limit=0' -H 'Accept: application/json' -H "Authorization: Token ${NETBOX_TOKEN}" > netbox-lcu-ips.json
       curl -X GET -k  'https://10.87.2.131/api/ipam/prefixes/?description__ic=multicast%20range' -H 'accept: application/json' -H "Authorization: Token ${NETBOX_TOKEN}" > netbox-data-networks.json
 
       mkdir -p generated/CDB/environments
@@ -160,6 +160,80 @@ render_CDB_environment_ilt:
     paths:
       - generated/CDB/environments/
 
+# Generate the ansible hosts file for each station.
+render_ansible_hosts:
+  stage: render
+  needs: []
+  extends:
+    - .components
+    - .local_job
+  rules:
+    - !reference [.local_job, rules]
+    - if: $CI
+  when: manual
+  before_script:
+    - sudo apt install -y jq
+  script:
+    - |
+      python -m pip install .
+
+      # fetch IPs from netbox (netbox.astron.nl)
+      curl -X GET -k 'https://10.87.2.131/api/ipam/vlans/?limit=0' -H 'Accept: application/json' -H "Authorization: Token ${NETBOX_TOKEN}" > netbox-vlans.json
+
+      OUTPUT=hosts.yaml
+
+      (
+      echo "station:"
+      echo "  hosts:"
+      ) > ${OUTPUT}
+
+      for STATION in ${LOFAR2_STATIONS}; do
+        echo "Generating ansible host information for ${STATION}"
+
+        CONTROL_VLAN=`jq < netbox-vlans.json '.results[] | select(.name == "'${STATION}'-control") | .vid' -r`
+        DATA_VLAN=`jq < netbox-vlans.json '.results[] | select(.name == "'${STATION}'-data") | .vid' -r`
+
+        (
+        echo "    ${STATION}:"
+        echo "      ansible_host: ${STATION}-lcu.lofar.eu"
+        echo "      station_name: ${STATION}"
+        echo "      control_vlan: ${CONTROL_VLAN}"
+        echo "      data_vlan:    ${DATA_VLAN}"
+        ) >> $OUTPUT
+      done
+
+      DNS1_HOST=lcs020.control.lofar
+      DNS1_IP=`getent ahostsv4 ${DNS1_HOST} | awk '/STREAM/ { print $1; }'`
+      DNS2_HOST=lcs021.control.lofar
+      DNS2_IP=`getent ahostsv4 ${DNS2_HOST} | awk '/STREAM/ { print $1; }'`
+
+      (
+      echo "vault:"
+      echo "  hosts:"
+      echo "    monitor:"
+      echo "      ansible_host: monitor.control.lofar"
+      echo "all:"
+      echo "  vars:"
+      echo "    ansible_user: lcuadmin"
+      echo "    volume_group: vg0"
+      echo "    eth:"
+      echo "      - enp1s0f0"
+      echo "      - enp1s0f1"
+      echo "    dns_servers:"
+      echo "      - ${DNS1_IP}"
+      echo "      - ${DNS2_IP}"
+      ) >> $OUTPUT
+
+      echo "----------------"
+      cat $OUTPUT
+      echo "----------------"
+  artifacts:
+    expire_in: 4 weeks
+    expose_as: 'ansible hosts file'
+    name: ansible-hosts
+    paths:
+      - hosts.yaml
+
 # Collect full TangoDB configuration files, to provide a full
 # configuration of each station. This combines all the relevant
 # CDB files for a station into one, providing a full configuration.

infra/station/README.md

@@ -68,23 +68,22 @@ host_key_checking = False
 inventory = hosts.yaml
 ```
 
-Create a file `hosts.yaml` with the content
+Create a file `hosts.yaml` with this content. NB: This file can be generated by the `render_ansible` GitLab CI/CD job.
 ```yaml
 station:
   hosts:
     <station>:
       ansible_host: lcu.<station>.lofar
       station_name: <station>
+      control_vlan: <vlannr>
+      data_vlan: <vlannr>
 vault:
   hosts:
     monitor:
       ansible_host: monitor.control.lofar
 all:
   vars:
-    consul_vault_token: "<token>"
-    nomad_encrypt: "<encrypt key>"
     ansible_user: lcuadmin
-    consul_encrypt: "<encrypt key>>"
     volume_group: vg0
     eth:
       - enp1s0f0

infra/station/consul.yml

@@ -3,9 +3,11 @@
 - name: setup vault role
   hosts: station
   connection: ssh
+  become: true
+  become_user: root
   gather_facts: true
   tasks:
-    - name: Lookup ansible_hostname in getent database
+    - name: Setup station-specific role in vault
       command:
         argv:
           - vault
@@ -16,9 +18,29 @@
           - allow_subdomains=true
           - allowed_uri_sans=*.server.lofar-{{station_name}}.consul
           - generate_lease=true
-          - max_ttl=720h
+          - max_ttl=2208h
+      delegate_to: monitor
+      register: result
+
+- name: obtain secrets
+  hosts: station
+  connection: ssh
+  become: true
+  become_user: root
+  tasks:
+    - name: Generate vault consul token
+      command: bash -c 'vault token create -policy=connect-ca -address=http://vault.lofar.net -display-name=consul-{{station_name}} | awk "/^token / { print \$2; }"'
+      delegate_to: monitor
+      register: result
+    - set_fact:
+        consul_vault_token: "{{ result.stdout }}"
+    - name: Lookup consul encrypt key
+      command: awk '/^encrypt = / { print $3; }' /etc/consul.d/consul.hcl
       delegate_to: monitor
       register: result
+    - set_fact:
+        consul_encrypt: "{{ result.stdout }}"
+
 - name: setup consul
   hosts: station
   remote_user: root
@@ -31,6 +53,9 @@
       path: '/opt/consul'
       size: 20g
   tasks:
+    - name: debug
+      debug:
+        
     - name: install packages
       apt:
         pkg:
@@ -40,7 +65,6 @@
         update_cache: yes
         state: present
 
-
     - name: configure consul
       ansible.builtin.template:
         src: consul/consul.hcl.j2
@@ -85,7 +109,7 @@
         - agent.key
         - agent.crt
 
-    - name: create nomad keys templates directory
+    - name: create nomad variables templates directory
       file:
         path: /opt/consul/templates/nomad-variables
         state: directory
@@ -93,10 +117,10 @@
         owner: consul
         group: consul
 
-    - name: copy pki templates
+    - name: copy nomad variables templates
       template:
-        src: consul/nomad-variables/{{item}}
-        dest: /opt/consul/templates/nomad-variables/{{item}}
+        src: consul/nomad-variables/{{item}}.tpl
+        dest: /opt/consul/templates/nomad-variables/{{item}}.tpl
         owner: consul
         group: consul
       loop:

infra/station/consul/consul-agent.hcl.j2

@@ -4,7 +4,7 @@ node_name = "agent"
 server = false
 bind_addr = "0.0.0.0"
 retry_join = [ "10.99.250.250:8301" ]
-encrypt = "{{ consul_encrypt }}"
+encrypt = {{ consul_encrypt }}
 advertise_addr = "10.99.250.250"
 ports {
   dns = -1

infra/station/consul/consul-pki/agent.crt.tpl

-[[ with secret "pki_int/issue/consul-lofar-{{ station_name }}" "common_name=server.lofar-{{ station_name }}.consul" "ttl=720h" "alt_names=localhost,{{ ansible_facts['hostname'] }}.server.lofar-{{ station_name }}.consul" "ip_sans=127.0.0.1"]]
+[[ with secret "pki_int/issue/consul-lofar-{{ station_name }}" "common_name=server.lofar-{{ station_name }}.consul" "ttl=2208h" "alt_names=localhost,{{ ansible_facts['hostname'] }}.server.lofar-{{ station_name }}.consul" "ip_sans=127.0.0.1"]]
 [[ .Data.certificate ]]
 [[ end ]]

infra/station/consul/consul-pki/agent.key.tpl

-[[ with secret "pki_int/issue/consul-lofar-{{ station_name }}" "common_name=server.lofar-{{ station_name }}.consul" "ttl=720h" "alt_names=localhost,{{ ansible_facts['hostname'] }}.server.lofar-{{ station_name }}.consul" "ip_sans=127.0.0.1"]]
+[[ with secret "pki_int/issue/consul-lofar-{{ station_name }}" "common_name=server.lofar-{{ station_name }}.consul" "ttl=2208h" "alt_names=localhost,{{ ansible_facts['hostname'] }}.server.lofar-{{ station_name }}.consul" "ip_sans=127.0.0.1"]]
 [[ .Data.private_key ]]
 [[ end ]]

infra/station/consul/consul-pki/ca.crt.tpl

-[[ with secret "pki_int/issue/consul-lofar-{{ station_name }}" "common_name=server.lofar-{{ station_name }}.consul" "ttl=720h"]]
+[[ with secret "pki_int/issue/consul-lofar-{{ station_name }}" "common_name=server.lofar-{{ station_name }}.consul" "ttl=2208h"]]
 [[ .Data.issuing_ca ]]
 [[ end ]]

infra/station/consul/consul-template.hcl

@@ -3,7 +3,6 @@ vault {
   token = "{{ consul_vault_token }}"
   unwrap_token = false
   renew_token  = true
-  lease_renewal_threshold = 0.10
 }
 
 # This block defines the configuration for a template. Unlike other blocks,

infra/station/consul/consul.hcl.j2

 datacenter = "lofar-{{ station_name }}"
 primary_datacenter = "lofar-central"
 server = true
-encrypt = "{{ consul_encrypt }}"
+encrypt = {{ consul_encrypt }}
 data_dir = "/opt/consul"
 bind_addr = "0.0.0.0"
 
 client_addr = "10.99.250.250 {{ '{{' }} GetInterfaceIP \"ctrl0\" {{ '}}' }} 127.0.0.1"
 advertise_addr = "10.99.250.250"
 retry_join = ["127.0.0.1"]
+server_rejoin_age_max = "2208h"
 
 serf_wan = "{{ '{{' }} GetInterfaceIP \"ctrl0\" {{ '}}' }}"
 #advertise_addr_wan = "{{ '{{' }} GetInterfaceIP \"ctrl0\" {{ '}}' }}"
@@ -29,12 +30,21 @@ connect {
   enable_mesh_gateway_wan_federation = true
 }
 
+tls {
+  defaults {
     verify_incoming = true
     verify_outgoing = true
-verify_server_hostname = true
+
     ca_file = "/opt/consul/agent-certs/ca.crt"
     cert_file = "/opt/consul/agent-certs/agent.crt"
     key_file = "/opt/consul/agent-certs/agent.key"
+  }
+
+  internal_rpc {
+    verify_server_hostname = true
+  }
+}
+
 auto_encrypt {
   allow_tls = true
 }

infra/station/nomad.yml

 ---
+- name: obtain secrets
+  hosts: station
+  tasks:
+    - name: Generate nomad encrypt key
+      command: openssl rand -base64 32
+      register: result
+    - set_fact:
+        nomad_encrypt: "{{ result.stdout }}"
+
 - name: setup nomad
   hosts: station
   remote_user: root
@@ -142,18 +151,20 @@
         state: started
         enabled: yes
 
+- name: runtime configure nomad
+  hosts: station
+  tasks:
+     - name: enable memory oversubscription
+       command: bash -c 'nomad operator scheduler get-config -json | jq -e ".SchedulerConfig.MemoryOversubscriptionEnabled" >/dev/null || nomad operator scheduler set-config -memory-oversubscription=true'
+       register: result
+       changed_when: '"Scheduler configuration updated" in result.stdout'
+
 - name: deploy consul agent
   hosts: station
   connection: ssh
   gather_facts: true
   tasks:
-#    - name: Lookup ansible_hostname in getent database
-#      command:
-#        argv:
-#          - data=$(wget -qO- http://localhost:4646/v1/operator/scheduler/configuration | jq '.SchedulerConfig | .MemoryOversubscriptionEnabled=true')
-#          - wget --method=PUT --body-data="$data" -qO- http://localhost:4646/v1/operator/scheduler/configuration
-#      register: result
-    - name: Lookup ansible_hostname in getent database
+    - name: deploy nomad job
       command:
         argv:
           - nomad
@@ -161,4 +172,3 @@
           - -address=http://{{ansible_host}}:4646
           - consul-agent.nomad
       delegate_to: localhost
-      register: result

infra/station/ovs.yml

@@ -99,7 +99,7 @@
     - openvswitch.openvswitch.openvswitch_bridge:
         bridge: data0
         parent: br0
-        vlan: 1000
+        vlan: {{data_vlan}}
         set: "int data0 mtu_request=9000"
         state: present
 
@@ -113,7 +113,7 @@
     - openvswitch.openvswitch.openvswitch_bridge:
         bridge: ctrl0
         parent: br0
-        vlan: 102
+        vlan: {{control_vlan}}
         set: "int ctrl0 mtu_request=9000"
         state: present