COB-4: Harden CPU bindings across OpenMP

RTCP/Cobalt/CoInterface/src/cpu_utils.cc

@@ -26,6 +26,7 @@
 #include <sys/mman.h>
 #include <fstream>
 #include <algorithm>
+#include <pthread.h>
 
 #include <Common/LofarLogger.h>
 #include <CoInterface/Parset.h>
@@ -58,6 +59,17 @@ namespace LOFAR
        return cores;
     }
 
+    vector<unsigned> cpuBinding()
+    {
+      cpu_set_t mask;
+      pthread_t thread = pthread_self();
+
+      if (pthread_getaffinity_np(thread, sizeof(cpu_set_t), &mask) != 0)
+        THROW_SYSCALL("pthread_getaffinity_np");
+
+      return cpusetToVector(mask);
+    }
+
     void bindCPU(unsigned socket, const std::vector<unsigned> &skipCores)
     {
       LOG_DEBUG_STR("[NUMA] Binding to CPU " << socket);

RTCP/Cobalt/CoInterface/src/cpu_utils.h

@@ -42,6 +42,9 @@ namespace LOFAR
     // Bind the process to the specified socket
     void bindCPU(unsigned socket, const std::vector<unsigned> &skipCores = std::vector<unsigned>());
 
+    // Returns all cores we are bound to
+    std::vector<unsigned> cpuBinding();
+
     // Bind the memory to the specified socket. Moves already allocated memory.
     //
     // NOTE: Does NOT move pinned memory, f.e. created by pinAllMemory() or cuMemHostAlloc()

RTCP/Cobalt/CoInterface/test/t_cpu_utils.cc

@@ -25,12 +25,15 @@
 #include <cstring>
 #include <cstdio>
 #include <sched.h>
+#include <cstdlib>
+#include <omp.h>
 #include <string>
 #include <iostream>
 
 #include <Common/LofarLogger.h>
 #include <Common/SystemCallException.h>
 #include <Common/SystemUtil.h>
+#include <CoInterface/PrintVector.h>
 
 #include <UnitTest++.h>
 
@@ -126,6 +129,43 @@ TEST(cpu_avoid_cores) {
   CHECK(CPU_ISSET(second, &mask));
 }
 
+TEST(openmp_binding) {
+  bindCPU(0);
+
+  const vector<unsigned> allCores = cpuBinding();
+
+  CHECK(allCores.size() >= 2);
+
+  // Avoid binding on first core
+  const unsigned forbiddenCore = allCores[0];
+
+  bindCPU(0, vector<unsigned>{ forbiddenCore });
+
+  cout << "Selected forbidden core: " << forbiddenCore << endl;
+
+  #pragma omp parallel for num_threads(16)
+  for(int i = 0; i < 16; ++i)
+  {
+     const vector<unsigned> threadCores = cpuBinding();
+
+     #pragma omp critical (cout)
+     { cout << "Thread is bound to " << threadCores << endl; }
+
+     // forbidden core should NOT be in our list
+     CHECK(find(threadCores.begin(), threadCores.end(), forbiddenCore) == threadCores.end());
+
+     // other cores should be in our list
+     for(size_t i = 0; i < allCores.size(); i++) {
+       unsigned core = allCores[i];
+
+       if (core == forbiddenCore)
+         continue;
+
+       CHECK(find(threadCores.begin(), threadCores.end(), core) != threadCores.end());
+     }
+  }
+} 
+
 TEST(memory0) {
   test_memory(0);
 }

RTCP/Cobalt/GPUProc/src/scripts/runObservation.sh

@@ -481,6 +481,7 @@ else
   mpirun.sh -x LOFARROOT="$LOFARROOT" \
             -x QUEUE_PREFIX="$QUEUE_PREFIX" \
             -x LOFARENV="$LOFARENV" \
+            -x OMP_PROC_BIND=FALSE \
             -H "$HOSTS" \
             $MPIRUN_PARAMS \
             $CHECK_TOOL \