From 1f0f17a33504f30d1ea9b2b74e8625e763e74d07 Mon Sep 17 00:00:00 2001
From: Jan David Mol <mol@astron.nl>
Date: Wed, 9 Apr 2014 13:45:38 +0000
Subject: [PATCH] Task #5681: Initial system validation framework for Cobalt
 nodes

---
 .gitattributes                                | 22 ++++++++
 .../tests/gpu/persistence-mode.test           |  6 +++
 .../validation/tests/hardware/sata-ahci.test  |  4 ++
 .../validation/tests/mpi/basic-mpi.test       |  5 ++
 .../tests/mpi/custom-openmpi-config.test      |  4 ++
 .../validation/tests/network/hosts.test       | 14 ++++++
 .../validation/tests/network/resolvconf.test  |  7 +++
 .../validation/tests/network/routes.cbt001    | 46 +++++++++++++++++
 .../validation/tests/network/routes.cbt002    | 42 ++++++++++++++++
 .../validation/tests/network/routes.cbt003    | 46 +++++++++++++++++
 .../validation/tests/network/routes.cbt004    | 50 +++++++++++++++++++
 .../validation/tests/network/routes.cbt005    | 50 +++++++++++++++++++
 .../validation/tests/network/routes.cbt006    | 50 +++++++++++++++++++
 .../validation/tests/network/routes.cbt007    | 50 +++++++++++++++++++
 .../validation/tests/network/routes.cbt008    | 46 +++++++++++++++++
 .../validation/tests/network/routes.cbt009    | 14 ++++++
 .../validation/tests/network/routes.test      |  5 ++
 .../validation/tests/os/gdb.test              |  5 ++
 .../validation/tests/os/limits.test           |  6 +++
 .../validation/tests/packages/atop.test       |  2 +
 .../validation/tests/packages/autogen.test    |  2 +
 .../validation/tests/packages/ipmitool.test   |  2 +
 .../tests/packages/linux-tools.test           |  2 +
 23 files changed, 480 insertions(+)
 create mode 100755 SubSystems/Online_Cobalt/validation/tests/gpu/persistence-mode.test
 create mode 100755 SubSystems/Online_Cobalt/validation/tests/hardware/sata-ahci.test
 create mode 100755 SubSystems/Online_Cobalt/validation/tests/mpi/basic-mpi.test
 create mode 100755 SubSystems/Online_Cobalt/validation/tests/mpi/custom-openmpi-config.test
 create mode 100755 SubSystems/Online_Cobalt/validation/tests/network/hosts.test
 create mode 100755 SubSystems/Online_Cobalt/validation/tests/network/resolvconf.test
 create mode 100644 SubSystems/Online_Cobalt/validation/tests/network/routes.cbt001
 create mode 100644 SubSystems/Online_Cobalt/validation/tests/network/routes.cbt002
 create mode 100644 SubSystems/Online_Cobalt/validation/tests/network/routes.cbt003
 create mode 100644 SubSystems/Online_Cobalt/validation/tests/network/routes.cbt004
 create mode 100644 SubSystems/Online_Cobalt/validation/tests/network/routes.cbt005
 create mode 100644 SubSystems/Online_Cobalt/validation/tests/network/routes.cbt006
 create mode 100644 SubSystems/Online_Cobalt/validation/tests/network/routes.cbt007
 create mode 100644 SubSystems/Online_Cobalt/validation/tests/network/routes.cbt008
 create mode 100644 SubSystems/Online_Cobalt/validation/tests/network/routes.cbt009
 create mode 100755 SubSystems/Online_Cobalt/validation/tests/network/routes.test
 create mode 100755 SubSystems/Online_Cobalt/validation/tests/os/gdb.test
 create mode 100755 SubSystems/Online_Cobalt/validation/tests/os/limits.test
 create mode 100755 SubSystems/Online_Cobalt/validation/tests/packages/atop.test
 create mode 100755 SubSystems/Online_Cobalt/validation/tests/packages/autogen.test
 create mode 100755 SubSystems/Online_Cobalt/validation/tests/packages/ipmitool.test
 create mode 100755 SubSystems/Online_Cobalt/validation/tests/packages/linux-tools.test

diff --git a/.gitattributes b/.gitattributes
index f3f9c91190d..087f32fed06 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -4522,6 +4522,28 @@ SubSystems/Online_Cobalt/test/tgenerateStationStreams.sh eol=lf
 SubSystems/Online_Cobalt/test/tstartBGL.in_parset -text
 SubSystems/Online_Cobalt/test/tstartBGL.run -text
 SubSystems/Online_Cobalt/test/tstartBGL.sh -text
+SubSystems/Online_Cobalt/validation/tests/gpu/persistence-mode.test -text
+SubSystems/Online_Cobalt/validation/tests/hardware/sata-ahci.test -text
+SubSystems/Online_Cobalt/validation/tests/mpi/basic-mpi.test -text
+SubSystems/Online_Cobalt/validation/tests/mpi/custom-openmpi-config.test -text
+SubSystems/Online_Cobalt/validation/tests/network/hosts.test -text
+SubSystems/Online_Cobalt/validation/tests/network/resolvconf.test -text
+SubSystems/Online_Cobalt/validation/tests/network/routes.cbt001 -text
+SubSystems/Online_Cobalt/validation/tests/network/routes.cbt002 -text
+SubSystems/Online_Cobalt/validation/tests/network/routes.cbt003 -text
+SubSystems/Online_Cobalt/validation/tests/network/routes.cbt004 -text
+SubSystems/Online_Cobalt/validation/tests/network/routes.cbt005 -text
+SubSystems/Online_Cobalt/validation/tests/network/routes.cbt006 -text
+SubSystems/Online_Cobalt/validation/tests/network/routes.cbt007 -text
+SubSystems/Online_Cobalt/validation/tests/network/routes.cbt008 -text
+SubSystems/Online_Cobalt/validation/tests/network/routes.cbt009 -text
+SubSystems/Online_Cobalt/validation/tests/network/routes.test -text
+SubSystems/Online_Cobalt/validation/tests/os/gdb.test -text
+SubSystems/Online_Cobalt/validation/tests/os/limits.test -text
+SubSystems/Online_Cobalt/validation/tests/packages/atop.test -text
+SubSystems/Online_Cobalt/validation/tests/packages/autogen.test -text
+SubSystems/Online_Cobalt/validation/tests/packages/ipmitool.test -text
+SubSystems/Online_Cobalt/validation/tests/packages/linux-tools.test -text
 /jenkins_make -text
 /lofar_config.h.cmake -text
 /lofstorman -text
diff --git a/SubSystems/Online_Cobalt/validation/tests/gpu/persistence-mode.test b/SubSystems/Online_Cobalt/validation/tests/gpu/persistence-mode.test
new file mode 100755
index 00000000000..37f4beb4a17
--- /dev/null
+++ b/SubSystems/Online_Cobalt/validation/tests/gpu/persistence-mode.test
@@ -0,0 +1,6 @@
+#!/bin/bash
+
+# All GPUs need to be in persistent mode
+nvidia-smi -q | grep 'Persistence Mode *: Disabled' && exit 1
+
+exit 0
diff --git a/SubSystems/Online_Cobalt/validation/tests/hardware/sata-ahci.test b/SubSystems/Online_Cobalt/validation/tests/hardware/sata-ahci.test
new file mode 100755
index 00000000000..75c03191145
--- /dev/null
+++ b/SubSystems/Online_Cobalt/validation/tests/hardware/sata-ahci.test
@@ -0,0 +1,4 @@
+#!/bin/bash
+
+# Check whether the SATA controller is in AHCI mode
+lspci -vvv | awk '/SATA/,/^$/ { print; }' | grep 'Kernel driver in use: ahci' || exit 1
diff --git a/SubSystems/Online_Cobalt/validation/tests/mpi/basic-mpi.test b/SubSystems/Online_Cobalt/validation/tests/mpi/basic-mpi.test
new file mode 100755
index 00000000000..88c85231f04
--- /dev/null
+++ b/SubSystems/Online_Cobalt/validation/tests/mpi/basic-mpi.test
@@ -0,0 +1,5 @@
+#!/bin/bash
+
+cd `dirname $0`
+/opt/openmpi/bin/mpicc mpi-hello-world.c -o mpi-hello-world
+/opt/openmpi/bin/mpirun --tag-output -H localhost,localhost ./mpi-hello-world | sort | fgrep 'Hello world' | wc -l | grep '^2$'
diff --git a/SubSystems/Online_Cobalt/validation/tests/mpi/custom-openmpi-config.test b/SubSystems/Online_Cobalt/validation/tests/mpi/custom-openmpi-config.test
new file mode 100755
index 00000000000..a5221c19511
--- /dev/null
+++ b/SubSystems/Online_Cobalt/validation/tests/mpi/custom-openmpi-config.test
@@ -0,0 +1,4 @@
+#!/bin/bash
+
+# Check whether we use our custom OpenMPI configuration file
+fgrep 'OpenMPI parameters for cbt00X' /opt/openmpi/etc/openmpi-mca-params.conf || exit 1
diff --git a/SubSystems/Online_Cobalt/validation/tests/network/hosts.test b/SubSystems/Online_Cobalt/validation/tests/network/hosts.test
new file mode 100755
index 00000000000..61a626d4762
--- /dev/null
+++ b/SubSystems/Online_Cobalt/validation/tests/network/hosts.test
@@ -0,0 +1,14 @@
+#!/bin/bash
+
+# Check for specific /etc/hosts entries
+
+fgrep ccu001 /etc/hosts || exit 1
+fgrep kis001 /etc/hosts || exit 1
+fgrep lhn001 /etc/hosts || exit 1
+
+fgrep lcs015 /etc/hosts || exit 1
+
+for i in `seq -w 1 102`
+do
+  fgrep "locus$i" /etc/hosts || exit 1
+done
diff --git a/SubSystems/Online_Cobalt/validation/tests/network/resolvconf.test b/SubSystems/Online_Cobalt/validation/tests/network/resolvconf.test
new file mode 100755
index 00000000000..bce80bf9940
--- /dev/null
+++ b/SubSystems/Online_Cobalt/validation/tests/network/resolvconf.test
@@ -0,0 +1,7 @@
+#!/bin/bash
+
+# Check for the presence of the various LOFAR domains
+for DOMAIN in cep2.lofar online.lofar control.lofar
+do
+  grep "^search.*\b$DOMAIN\b" /etc/resolv.conf || exit 1
+done
diff --git a/SubSystems/Online_Cobalt/validation/tests/network/routes.cbt001 b/SubSystems/Online_Cobalt/validation/tests/network/routes.cbt001
new file mode 100644
index 00000000000..590eee7676a
--- /dev/null
+++ b/SubSystems/Online_Cobalt/validation/tests/network/routes.cbt001
@@ -0,0 +1,46 @@
+default via 10.151.255.254 dev eth0  metric 100 
+10.135.252.0/24 via 10.175.255.201 dev eth2 
+10.135.253.0/24 via 10.175.255.202 dev eth3 
+10.135.254.0/24 via 10.175.255.203 dev eth4 
+10.135.255.0/24 via 10.175.255.204 dev eth5 
+10.144.0.0/13 dev eth0  proto kernel  scope link  src 10.149.128.1 
+10.168.0.0/13 dev eth2  proto kernel  scope link  src 10.168.96.1 
+10.168.0.0/13 dev eth5  proto kernel  scope link  src 10.168.96.4 
+10.168.0.0/13 dev eth4  proto kernel  scope link  src 10.168.96.3 
+10.168.0.0/13 dev eth3  proto kernel  scope link  src 10.168.96.2 
+10.175.5.1 dev eth2  scope link 
+10.175.5.2 dev eth2  scope link 
+10.175.5.3 dev eth2  scope link 
+10.175.5.4 dev eth2  scope link 
+10.175.5.7 dev eth2  scope link 
+10.175.5.8 dev eth2  scope link 
+10.175.5.9 dev eth2  scope link 
+10.175.5.10 dev eth2  scope link 
+10.175.39.1 dev eth3  scope link 
+10.175.39.2 dev eth3  scope link 
+10.175.39.3 dev eth3  scope link 
+10.175.39.4 dev eth3  scope link 
+10.175.39.7 dev eth3  scope link 
+10.175.39.8 dev eth3  scope link 
+10.175.39.9 dev eth3  scope link 
+10.175.39.10 dev eth3  scope link 
+10.175.129.1 dev eth4  scope link 
+10.175.129.2 dev eth4  scope link 
+10.175.129.3 dev eth4  scope link 
+10.175.129.4 dev eth4  scope link 
+10.175.129.7 dev eth4  scope link 
+10.175.129.8 dev eth4  scope link 
+10.175.129.9 dev eth4  scope link 
+10.175.129.10 dev eth4  scope link 
+10.175.152.1 dev eth5  scope link 
+10.175.152.2 dev eth5  scope link 
+10.175.152.3 dev eth5  scope link 
+10.175.152.4 dev eth5  scope link 
+10.175.183.1 dev eth5  scope link 
+10.175.183.2 dev eth5  scope link 
+10.175.183.3 dev eth5  scope link 
+10.175.183.4 dev eth5  scope link 
+10.175.255.201 dev eth2  scope link 
+10.175.255.202 dev eth3  scope link 
+10.175.255.203 dev eth4  scope link 
+10.175.255.204 dev eth5  scope link 
diff --git a/SubSystems/Online_Cobalt/validation/tests/network/routes.cbt002 b/SubSystems/Online_Cobalt/validation/tests/network/routes.cbt002
new file mode 100644
index 00000000000..2bf29dae41f
--- /dev/null
+++ b/SubSystems/Online_Cobalt/validation/tests/network/routes.cbt002
@@ -0,0 +1,42 @@
+default via 10.151.255.254 dev eth0  metric 100 
+10.135.252.0/24 via 10.175.255.201 dev eth2 
+10.135.253.0/24 via 10.175.255.202 dev eth3 
+10.135.254.0/24 via 10.175.255.203 dev eth4 
+10.135.255.0/24 via 10.175.255.204 dev eth5 
+10.144.0.0/13 dev eth0  proto kernel  scope link  src 10.149.128.2 
+10.168.0.0/13 dev eth5  proto kernel  scope link  src 10.168.97.4 
+10.168.0.0/13 dev eth3  proto kernel  scope link  src 10.168.97.2 
+10.168.0.0/13 dev eth2  proto kernel  scope link  src 10.168.97.1 
+10.168.0.0/13 dev eth4  proto kernel  scope link  src 10.168.97.3 
+10.175.35.1 dev eth2  scope link 
+10.175.35.2 dev eth2  scope link 
+10.175.35.3 dev eth2  scope link 
+10.175.35.4 dev eth2  scope link 
+10.175.35.7 dev eth2  scope link 
+10.175.35.8 dev eth2  scope link 
+10.175.35.9 dev eth2  scope link 
+10.175.35.10 dev eth2  scope link 
+10.175.68.1 dev eth3  scope link 
+10.175.68.2 dev eth3  scope link 
+10.175.68.3 dev eth3  scope link 
+10.175.68.4 dev eth3  scope link 
+10.175.68.7 dev eth3  scope link 
+10.175.68.8 dev eth3  scope link 
+10.175.68.9 dev eth3  scope link 
+10.175.68.10 dev eth3  scope link 
+10.175.170.1 dev eth5  scope link 
+10.175.170.2 dev eth5  scope link 
+10.175.170.3 dev eth5  scope link 
+10.175.170.4 dev eth5  scope link 
+10.175.185.1 dev eth4  scope link 
+10.175.185.2 dev eth4  scope link 
+10.175.185.3 dev eth4  scope link 
+10.175.185.4 dev eth4  scope link 
+10.175.186.1 dev eth5  scope link 
+10.175.186.2 dev eth5  scope link 
+10.175.186.3 dev eth5  scope link 
+10.175.186.4 dev eth5  scope link 
+10.175.255.201 dev eth2  scope link 
+10.175.255.202 dev eth3  scope link 
+10.175.255.203 dev eth4  scope link 
+10.175.255.204 dev eth5  scope link 
diff --git a/SubSystems/Online_Cobalt/validation/tests/network/routes.cbt003 b/SubSystems/Online_Cobalt/validation/tests/network/routes.cbt003
new file mode 100644
index 00000000000..f51ff743734
--- /dev/null
+++ b/SubSystems/Online_Cobalt/validation/tests/network/routes.cbt003
@@ -0,0 +1,46 @@
+default via 10.151.255.254 dev eth0  metric 100 
+10.135.252.0/24 via 10.175.255.201 dev eth2 
+10.135.253.0/24 via 10.175.255.202 dev eth3 
+10.135.254.0/24 via 10.175.255.203 dev eth4 
+10.135.255.0/24 via 10.175.255.204 dev eth5 
+10.144.0.0/13 dev eth0  proto kernel  scope link  src 10.149.128.3 
+10.168.0.0/13 dev eth5  proto kernel  scope link  src 10.168.98.4 
+10.168.0.0/13 dev eth4  proto kernel  scope link  src 10.168.98.3 
+10.168.0.0/13 dev eth3  proto kernel  scope link  src 10.168.98.2 
+10.168.0.0/13 dev eth2  proto kernel  scope link  src 10.168.98.1 
+10.175.2.1 dev eth2  scope link 
+10.175.2.2 dev eth2  scope link 
+10.175.2.3 dev eth2  scope link 
+10.175.2.4 dev eth2  scope link 
+10.175.2.7 dev eth2  scope link 
+10.175.2.8 dev eth2  scope link 
+10.175.2.9 dev eth2  scope link 
+10.175.2.10 dev eth2  scope link 
+10.175.20.1 dev eth3  scope link 
+10.175.20.2 dev eth3  scope link 
+10.175.20.3 dev eth3  scope link 
+10.175.20.4 dev eth3  scope link 
+10.175.20.7 dev eth3  scope link 
+10.175.20.8 dev eth3  scope link 
+10.175.20.9 dev eth3  scope link 
+10.175.20.10 dev eth3  scope link 
+10.175.55.1 dev eth4  scope link 
+10.175.55.2 dev eth4  scope link 
+10.175.55.3 dev eth4  scope link 
+10.175.55.4 dev eth4  scope link 
+10.175.55.7 dev eth4  scope link 
+10.175.55.8 dev eth4  scope link 
+10.175.55.9 dev eth4  scope link 
+10.175.55.10 dev eth4  scope link 
+10.175.145.1 dev eth3  scope link 
+10.175.145.2 dev eth3  scope link 
+10.175.145.3 dev eth3  scope link 
+10.175.145.4 dev eth3  scope link 
+10.175.145.7 dev eth5  scope link 
+10.175.145.8 dev eth5  scope link 
+10.175.145.9 dev eth5  scope link 
+10.175.145.10 dev eth5  scope link 
+10.175.255.201 dev eth2  scope link 
+10.175.255.202 dev eth3  scope link 
+10.175.255.203 dev eth4  scope link 
+10.175.255.204 dev eth5  scope link 
diff --git a/SubSystems/Online_Cobalt/validation/tests/network/routes.cbt004 b/SubSystems/Online_Cobalt/validation/tests/network/routes.cbt004
new file mode 100644
index 00000000000..1dd618024bb
--- /dev/null
+++ b/SubSystems/Online_Cobalt/validation/tests/network/routes.cbt004
@@ -0,0 +1,50 @@
+default via 10.151.255.254 dev eth0  metric 100 
+10.135.252.0/24 via 10.175.255.201 dev eth2 
+10.135.253.0/24 via 10.175.255.202 dev eth3 
+10.135.254.0/24 via 10.175.255.203 dev eth4 
+10.135.255.0/24 via 10.175.255.204 dev eth5 
+10.144.0.0/13 dev eth0  proto kernel  scope link  src 10.149.128.4 
+10.168.0.0/13 dev eth5  proto kernel  scope link  src 10.168.99.4 
+10.168.0.0/13 dev eth2  proto kernel  scope link  src 10.168.99.1 
+10.168.0.0/13 dev eth3  proto kernel  scope link  src 10.168.99.2 
+10.168.0.0/13 dev eth4  proto kernel  scope link  src 10.168.99.3 
+10.175.6.1 dev eth2  scope link 
+10.175.6.2 dev eth2  scope link 
+10.175.6.3 dev eth2  scope link 
+10.175.6.4 dev eth2  scope link 
+10.175.6.7 dev eth2  scope link 
+10.175.6.8 dev eth2  scope link 
+10.175.6.9 dev eth2  scope link 
+10.175.6.10 dev eth2  scope link 
+10.175.53.1 dev eth3  scope link 
+10.175.53.2 dev eth3  scope link 
+10.175.53.3 dev eth3  scope link 
+10.175.53.4 dev eth3  scope link 
+10.175.53.7 dev eth3  scope link 
+10.175.53.8 dev eth3  scope link 
+10.175.53.9 dev eth3  scope link 
+10.175.53.10 dev eth3  scope link 
+10.175.67.1 dev eth4  scope link 
+10.175.67.2 dev eth4  scope link 
+10.175.67.3 dev eth4  scope link 
+10.175.67.4 dev eth4  scope link 
+10.175.67.7 dev eth4  scope link 
+10.175.67.8 dev eth4  scope link 
+10.175.67.9 dev eth4  scope link 
+10.175.67.10 dev eth4  scope link 
+10.175.162.1 dev eth5  scope link 
+10.175.162.2 dev eth5  scope link 
+10.175.162.3 dev eth5  scope link 
+10.175.162.4 dev eth5  scope link 
+10.175.162.7 dev eth5  scope link 
+10.175.162.8 dev eth5  scope link 
+10.175.162.9 dev eth5  scope link 
+10.175.162.10 dev eth5  scope link 
+10.175.255.201 dev eth2  scope link 
+10.175.255.202 dev eth3  scope link 
+10.175.255.203 dev eth4  scope link 
+10.175.255.204 dev eth5  scope link 
+10.211.0.0/16 dev vlan2011  proto kernel  scope link  src 10.211.3.50 
+10.212.0.0/16 dev vlan2012  proto kernel  scope link  src 10.212.3.50 
+10.213.0.0/16 dev vlan2013  proto kernel  scope link  src 10.213.3.50 
+10.214.0.0/16 dev vlan2014  proto kernel  scope link  src 10.214.3.50 
diff --git a/SubSystems/Online_Cobalt/validation/tests/network/routes.cbt005 b/SubSystems/Online_Cobalt/validation/tests/network/routes.cbt005
new file mode 100644
index 00000000000..4f801991ad1
--- /dev/null
+++ b/SubSystems/Online_Cobalt/validation/tests/network/routes.cbt005
@@ -0,0 +1,50 @@
+default via 10.151.255.254 dev eth0  metric 100 
+10.135.252.0/24 via 10.175.255.201 dev eth2 
+10.135.253.0/24 via 10.175.255.202 dev eth3 
+10.135.254.0/24 via 10.175.255.203 dev eth4 
+10.135.255.0/24 via 10.175.255.204 dev eth5 
+10.144.0.0/13 dev eth0  proto kernel  scope link  src 10.149.128.5 
+10.168.0.0/13 dev eth5  proto kernel  scope link  src 10.168.100.4 
+10.168.0.0/13 dev eth2  proto kernel  scope link  src 10.168.100.1 
+10.168.0.0/13 dev eth3  proto kernel  scope link  src 10.168.100.2 
+10.168.0.0/13 dev eth4  proto kernel  scope link  src 10.168.100.3 
+10.175.3.1 dev eth2  scope link 
+10.175.3.2 dev eth2  scope link 
+10.175.3.3 dev eth2  scope link 
+10.175.3.4 dev eth2  scope link 
+10.175.3.7 dev eth2  scope link 
+10.175.3.8 dev eth2  scope link 
+10.175.3.9 dev eth2  scope link 
+10.175.3.10 dev eth2  scope link 
+10.175.22.1 dev eth3  scope link 
+10.175.22.2 dev eth3  scope link 
+10.175.22.3 dev eth3  scope link 
+10.175.22.4 dev eth3  scope link 
+10.175.22.7 dev eth3  scope link 
+10.175.22.8 dev eth3  scope link 
+10.175.22.9 dev eth3  scope link 
+10.175.22.10 dev eth3  scope link 
+10.175.154.1 dev eth4  scope link 
+10.175.154.2 dev eth4  scope link 
+10.175.154.3 dev eth4  scope link 
+10.175.154.4 dev eth4  scope link 
+10.175.161.1 dev eth4  scope link 
+10.175.161.2 dev eth4  scope link 
+10.175.161.3 dev eth4  scope link 
+10.175.161.4 dev eth4  scope link 
+10.175.161.7 dev eth4  scope link 
+10.175.161.8 dev eth4  scope link 
+10.175.161.9 dev eth4  scope link 
+10.175.161.10 dev eth4  scope link 
+10.175.166.1 dev eth2  scope link 
+10.175.166.2 dev eth2  scope link 
+10.175.166.3 dev eth2  scope link 
+10.175.166.4 dev eth2  scope link 
+10.175.255.201 dev eth2  scope link 
+10.175.255.202 dev eth3  scope link 
+10.175.255.203 dev eth4  scope link 
+10.175.255.204 dev eth5  scope link 
+10.211.0.0/16 dev vlan2011  proto kernel  scope link  src 10.211.8.50 
+10.212.0.0/16 dev vlan2012  proto kernel  scope link  src 10.212.8.50 
+10.213.0.0/16 dev vlan2013  proto kernel  scope link  src 10.213.8.50 
+10.214.0.0/16 dev vlan2014  proto kernel  scope link  src 10.214.8.50 
diff --git a/SubSystems/Online_Cobalt/validation/tests/network/routes.cbt006 b/SubSystems/Online_Cobalt/validation/tests/network/routes.cbt006
new file mode 100644
index 00000000000..c705c64e551
--- /dev/null
+++ b/SubSystems/Online_Cobalt/validation/tests/network/routes.cbt006
@@ -0,0 +1,50 @@
+default via 10.151.255.254 dev eth0  metric 100 
+10.135.252.0/24 via 10.175.255.201 dev eth2 
+10.135.253.0/24 via 10.175.255.202 dev eth3 
+10.135.254.0/24 via 10.175.255.203 dev eth4 
+10.135.255.0/24 via 10.175.255.204 dev eth5 
+10.144.0.0/13 dev eth0  proto kernel  scope link  src 10.149.128.6 
+10.168.0.0/13 dev eth5  proto kernel  scope link  src 10.168.101.4 
+10.168.0.0/13 dev eth2  proto kernel  scope link  src 10.168.101.1 
+10.168.0.0/13 dev eth3  proto kernel  scope link  src 10.168.101.2 
+10.168.0.0/13 dev eth4  proto kernel  scope link  src 10.168.101.3 
+10.175.7.1 dev eth2  scope link 
+10.175.7.2 dev eth2  scope link 
+10.175.7.3 dev eth2  scope link 
+10.175.7.4 dev eth2  scope link 
+10.175.7.7 dev eth2  scope link 
+10.175.7.8 dev eth2  scope link 
+10.175.7.9 dev eth2  scope link 
+10.175.7.10 dev eth2  scope link 
+10.175.66.1 dev eth3  scope link 
+10.175.66.2 dev eth3  scope link 
+10.175.66.3 dev eth3  scope link 
+10.175.66.4 dev eth3  scope link 
+10.175.66.7 dev eth3  scope link 
+10.175.66.8 dev eth3  scope link 
+10.175.66.9 dev eth3  scope link 
+10.175.66.10 dev eth3  scope link 
+10.175.165.1 dev eth5  scope link 
+10.175.165.2 dev eth5  scope link 
+10.175.165.3 dev eth5  scope link 
+10.175.165.4 dev eth5  scope link 
+10.175.193.1 dev eth4  scope link 
+10.175.193.2 dev eth4  scope link 
+10.175.193.3 dev eth4  scope link 
+10.175.193.4 dev eth4  scope link 
+10.175.193.7 dev eth4  scope link 
+10.175.193.8 dev eth4  scope link 
+10.175.193.9 dev eth4  scope link 
+10.175.193.10 dev eth4  scope link 
+10.175.195.1 dev eth5  scope link 
+10.175.195.2 dev eth5  scope link 
+10.175.195.3 dev eth5  scope link 
+10.175.195.4 dev eth5  scope link 
+10.175.255.201 dev eth2  scope link 
+10.175.255.202 dev eth3  scope link 
+10.175.255.203 dev eth4  scope link 
+10.175.255.204 dev eth5  scope link 
+10.211.0.0/16 dev vlan2011  proto kernel  scope link  src 10.211.7.50 
+10.212.0.0/16 dev vlan2012  proto kernel  scope link  src 10.212.7.50 
+10.213.0.0/16 dev vlan2013  proto kernel  scope link  src 10.213.7.50 
+10.214.0.0/16 dev vlan2014  proto kernel  scope link  src 10.214.7.50 
diff --git a/SubSystems/Online_Cobalt/validation/tests/network/routes.cbt007 b/SubSystems/Online_Cobalt/validation/tests/network/routes.cbt007
new file mode 100644
index 00000000000..2f50bced0b0
--- /dev/null
+++ b/SubSystems/Online_Cobalt/validation/tests/network/routes.cbt007
@@ -0,0 +1,50 @@
+default via 10.151.255.254 dev eth0  metric 100 
+10.135.252.0/24 via 10.175.255.201 dev eth2 
+10.135.253.0/24 via 10.175.255.202 dev eth3 
+10.135.254.0/24 via 10.175.255.203 dev eth4 
+10.135.255.0/24 via 10.175.255.204 dev eth5 
+10.144.0.0/13 dev eth0  proto kernel  scope link  src 10.149.128.7 
+10.168.0.0/13 dev eth2  proto kernel  scope link  src 10.168.102.1 
+10.168.0.0/13 dev eth4  proto kernel  scope link  src 10.168.102.3 
+10.168.0.0/13 dev eth3  proto kernel  scope link  src 10.168.102.2 
+10.168.0.0/13 dev eth5  proto kernel  scope link  src 10.168.102.4 
+10.175.1.1 dev eth2  scope link 
+10.175.1.2 dev eth2  scope link 
+10.175.1.3 dev eth2  scope link 
+10.175.1.4 dev eth2  scope link 
+10.175.1.7 dev eth2  scope link 
+10.175.1.8 dev eth2  scope link 
+10.175.1.9 dev eth2  scope link 
+10.175.1.10 dev eth2  scope link 
+10.175.51.1 dev eth3  scope link 
+10.175.51.2 dev eth3  scope link 
+10.175.51.3 dev eth3  scope link 
+10.175.51.4 dev eth3  scope link 
+10.175.51.7 dev eth3  scope link 
+10.175.51.8 dev eth3  scope link 
+10.175.51.9 dev eth3  scope link 
+10.175.51.10 dev eth3  scope link 
+10.175.149.1 dev eth5  scope link 
+10.175.149.2 dev eth5  scope link 
+10.175.149.3 dev eth5  scope link 
+10.175.149.4 dev eth5  scope link 
+10.175.177.1 dev eth4  scope link 
+10.175.177.2 dev eth4  scope link 
+10.175.177.3 dev eth4  scope link 
+10.175.177.4 dev eth4  scope link 
+10.175.177.7 dev eth4  scope link 
+10.175.177.8 dev eth4  scope link 
+10.175.177.9 dev eth4  scope link 
+10.175.177.10 dev eth4  scope link 
+10.175.182.1 dev eth4  scope link 
+10.175.182.2 dev eth4  scope link 
+10.175.182.3 dev eth4  scope link 
+10.175.182.4 dev eth4  scope link 
+10.175.201.1 dev eth5  scope link 
+10.175.201.2 dev eth5  scope link 
+10.175.201.3 dev eth5  scope link 
+10.175.201.4 dev eth5  scope link 
+10.175.255.201 dev eth2  scope link 
+10.175.255.202 dev eth3  scope link 
+10.175.255.203 dev eth4  scope link 
+10.175.255.204 dev eth5  scope link 
diff --git a/SubSystems/Online_Cobalt/validation/tests/network/routes.cbt008 b/SubSystems/Online_Cobalt/validation/tests/network/routes.cbt008
new file mode 100644
index 00000000000..ca2d0b9c291
--- /dev/null
+++ b/SubSystems/Online_Cobalt/validation/tests/network/routes.cbt008
@@ -0,0 +1,46 @@
+default via 10.151.255.254 dev eth0  metric 100 
+10.135.252.0/24 via 10.175.255.201 dev eth2 
+10.135.253.0/24 via 10.175.255.202 dev eth3 
+10.135.254.0/24 via 10.175.255.203 dev eth4 
+10.135.255.0/24 via 10.175.255.204 dev eth5 
+10.144.0.0/13 dev eth0  proto kernel  scope link  src 10.149.128.8 
+10.168.0.0/13 dev eth5  proto kernel  scope link  src 10.168.103.4 
+10.168.0.0/13 dev eth3  proto kernel  scope link  src 10.168.103.2 
+10.168.0.0/13 dev eth4  proto kernel  scope link  src 10.168.103.3 
+10.168.0.0/13 dev eth2  proto kernel  scope link  src 10.168.103.1 
+10.175.4.1 dev eth2  scope link 
+10.175.4.2 dev eth2  scope link 
+10.175.4.3 dev eth2  scope link 
+10.175.4.4 dev eth2  scope link 
+10.175.4.7 dev eth2  scope link 
+10.175.4.8 dev eth2  scope link 
+10.175.4.9 dev eth2  scope link 
+10.175.4.10 dev eth2  scope link 
+10.175.131.1 dev eth3  scope link 
+10.175.131.2 dev eth3  scope link 
+10.175.131.3 dev eth3  scope link 
+10.175.131.4 dev eth3  scope link 
+10.175.131.7 dev eth3  scope link 
+10.175.131.8 dev eth3  scope link 
+10.175.131.9 dev eth3  scope link 
+10.175.131.10 dev eth3  scope link 
+10.175.134.1 dev eth4  scope link 
+10.175.134.2 dev eth4  scope link 
+10.175.134.3 dev eth4  scope link 
+10.175.134.4 dev eth4  scope link 
+10.175.167.1 dev eth5  scope link 
+10.175.167.2 dev eth5  scope link 
+10.175.167.3 dev eth5  scope link 
+10.175.167.4 dev eth5  scope link 
+10.175.200.1 dev eth4  scope link 
+10.175.200.2 dev eth4  scope link 
+10.175.200.3 dev eth4  scope link 
+10.175.200.4 dev eth4  scope link 
+10.175.255.201 dev eth2  scope link 
+10.175.255.202 dev eth3  scope link 
+10.175.255.203 dev eth4  scope link 
+10.175.255.204 dev eth5  scope link 
+10.211.0.0/16 dev vlan2011  proto kernel  scope link  src 10.211.1.50 
+10.212.0.0/16 dev vlan2012  proto kernel  scope link  src 10.212.1.50 
+10.213.0.0/16 dev vlan2013  proto kernel  scope link  src 10.213.1.50 
+10.214.0.0/16 dev vlan2014  proto kernel  scope link  src 10.214.1.50 
diff --git a/SubSystems/Online_Cobalt/validation/tests/network/routes.cbt009 b/SubSystems/Online_Cobalt/validation/tests/network/routes.cbt009
new file mode 100644
index 00000000000..a7f0813b854
--- /dev/null
+++ b/SubSystems/Online_Cobalt/validation/tests/network/routes.cbt009
@@ -0,0 +1,14 @@
+default via 10.151.255.254 dev eth0  metric 100 
+10.135.252.0/24 via 10.175.255.201 dev eth2 
+10.135.253.0/24 via 10.175.255.202 dev eth3 
+10.135.254.0/24 via 10.175.255.203 dev eth4 
+10.135.255.0/24 via 10.175.255.204 dev eth5 
+10.144.0.0/13 dev eth0  proto kernel  scope link  src 10.149.128.9 
+10.168.0.0/13 dev eth5  proto kernel  scope link  src 10.168.104.4 
+10.168.0.0/13 dev eth4  proto kernel  scope link  src 10.168.104.3 
+10.168.0.0/13 dev eth3  proto kernel  scope link  src 10.168.104.2 
+10.168.0.0/13 dev eth2  proto kernel  scope link  src 10.168.104.1 
+10.175.255.201 dev eth2  scope link 
+10.175.255.202 dev eth3  scope link 
+10.175.255.203 dev eth4  scope link 
+10.175.255.204 dev eth5  scope link 
diff --git a/SubSystems/Online_Cobalt/validation/tests/network/routes.test b/SubSystems/Online_Cobalt/validation/tests/network/routes.test
new file mode 100755
index 00000000000..91ab62fe432
--- /dev/null
+++ b/SubSystems/Online_Cobalt/validation/tests/network/routes.test
@@ -0,0 +1,5 @@
+#!/bin/bash
+
+CWD=`dirname $0`
+
+diff <(sort $CWD/routes.`hostname`) <(ip route | sort) || exit 1
diff --git a/SubSystems/Online_Cobalt/validation/tests/os/gdb.test b/SubSystems/Online_Cobalt/validation/tests/os/gdb.test
new file mode 100755
index 00000000000..36a9308aa54
--- /dev/null
+++ b/SubSystems/Online_Cobalt/validation/tests/os/gdb.test
@@ -0,0 +1,5 @@
+#!/bin/bash
+
+# We need kernel.yama.ptrace_scope == 0 to be able to attach
+# gdb to our processes.
+[ `sysctl -n kernel.yama.ptrace_scope` == "0" ] || exit 1
diff --git a/SubSystems/Online_Cobalt/validation/tests/os/limits.test b/SubSystems/Online_Cobalt/validation/tests/os/limits.test
new file mode 100755
index 00000000000..ec9d18ec77a
--- /dev/null
+++ b/SubSystems/Online_Cobalt/validation/tests/os/limits.test
@@ -0,0 +1,6 @@
+#!/bin/bash
+
+# For now, require exact settings for all users
+grep '\*\s*-\s*memlock\s*unlimited' /etc/security/limits.conf || exit 1
+grep '\*\s*-\s*nice\s*19' /etc/security/limits.conf || exit 1
+grep '\*\s*-\s*rtprio\s*99' /etc/security/limits.conf || exit 1
diff --git a/SubSystems/Online_Cobalt/validation/tests/packages/atop.test b/SubSystems/Online_Cobalt/validation/tests/packages/atop.test
new file mode 100755
index 00000000000..60671bde3c2
--- /dev/null
+++ b/SubSystems/Online_Cobalt/validation/tests/packages/atop.test
@@ -0,0 +1,2 @@
+#!/bin/bash
+dpkg -l atop || exit 1
diff --git a/SubSystems/Online_Cobalt/validation/tests/packages/autogen.test b/SubSystems/Online_Cobalt/validation/tests/packages/autogen.test
new file mode 100755
index 00000000000..eac6bf503bd
--- /dev/null
+++ b/SubSystems/Online_Cobalt/validation/tests/packages/autogen.test
@@ -0,0 +1,2 @@
+#!/bin/bash
+dpkg -l autogen || exit 1
diff --git a/SubSystems/Online_Cobalt/validation/tests/packages/ipmitool.test b/SubSystems/Online_Cobalt/validation/tests/packages/ipmitool.test
new file mode 100755
index 00000000000..fd70461034f
--- /dev/null
+++ b/SubSystems/Online_Cobalt/validation/tests/packages/ipmitool.test
@@ -0,0 +1,2 @@
+#!/bin/bash
+dpkg -l ipmitool || exit 1
diff --git a/SubSystems/Online_Cobalt/validation/tests/packages/linux-tools.test b/SubSystems/Online_Cobalt/validation/tests/packages/linux-tools.test
new file mode 100755
index 00000000000..770b6c194db
--- /dev/null
+++ b/SubSystems/Online_Cobalt/validation/tests/packages/linux-tools.test
@@ -0,0 +1,2 @@
+#!/bin/bash
+dpkg -l linux-tools || exit 1
-- 
GitLab