bug 225:

Do not allow time-critical threads to run on core 0, which handles all
ethernet and tree interrupts.

RTCP/IONProc/src/ION_Allocator.h

@@ -26,7 +26,7 @@
 
 #include <Interface/Allocator.h>
 
-#undef FLAT_MEMORY
+#define FLAT_MEMORY
 
 namespace LOFAR {
 namespace RTCP {

RTCP/IONProc/src/InputSection.cc

@@ -31,6 +31,7 @@
 #include <WH_DelayCompensation.h>
 #include <InputThread.h>
 #include <ION_Allocator.h>
+#include <Scheduling.h>
 //#include <TH_ZoidServer.h>
 #include <Interface/AlignedStdAllocator.h>
 #include <Interface/CN_Command.h>
@@ -64,6 +65,10 @@ template<typename SAMPLE_TYPE> InputSection<SAMPLE_TYPE>::InputSection(const std
   itsLogThread(0),
   itsDelayTimer("delay")
 {
+#if defined HAVE_BGP_ION
+  doNotRunOnCore0();
+#endif
+
   raisePriority();
 }
 

RTCP/IONProc/src/InputThread.cc

@@ -36,6 +36,7 @@
 #include <BeamletBuffer.h>
 #include <InputThread.h>
 #include <RSP.h>
+#include <Scheduling.h>
 
 #include <errno.h>
 #include <signal.h>
@@ -102,24 +103,6 @@ template <typename SAMPLE_TYPE> void InputThread<SAMPLE_TYPE>::sigHandler(int)
 }
 
 
-template <typename SAMPLE_TYPE> void InputThread<SAMPLE_TYPE>::setAffinity()
-{
-#if 1 && __linux__
-  cpu_set_t cpu_set;
-
-  CPU_ZERO(&cpu_set);
-
-  for (unsigned cpu = 1; cpu < 4; cpu ++)
-    CPU_SET(cpu, &cpu_set);
-
-  if (sched_setaffinity(0, sizeof cpu_set, &cpu_set) != 0) {
-    std::clog << "WARNING: sched_setaffinity failed" << std::endl;
-    perror("sched_setaffinity");
-  }
-#endif
-}
-
-
 template <typename SAMPLE_TYPE> void *InputThread<SAMPLE_TYPE>::mainLoopStub(void *inputThread)
 {
   try {
@@ -139,7 +122,9 @@ template <typename SAMPLE_TYPE> void *InputThread<SAMPLE_TYPE>::mainLoopStub(voi
 
 template <typename SAMPLE_TYPE> void InputThread<SAMPLE_TYPE>::mainLoop()
 {
-  setAffinity();
+#if defined HAVE_BGP_ION
+  doNotRunOnCore0();
+#endif
 
   const unsigned maxNrPackets = 128;
   TimeStamp	 actualstamp  = itsArgs.startTime - itsArgs.nrTimesPerPacket;

RTCP/IONProc/src/InputThread.h

@@ -64,7 +64,6 @@ template<typename SAMPLE_TYPE> class InputThread
 
   private:
     static void		  sigHandler(int);
-    static void		  setAffinity();
   
     volatile bool	  stop, stopped;
 

RTCP/IONProc/src/LogThread.cc

@@ -24,6 +24,7 @@
 #include <lofar_config.h>
 
 #include <LogThread.h>
+#include <Scheduling.h>
 #include <Interface/PrintVector.h>
 
 #include <algorithm>
@@ -71,6 +72,10 @@ void *LogThread::logThreadStub(void *arg)
 
 void LogThread::logThread()
 {
+#if defined HAVE_BGP_ION
+  runOnCore0();
+#endif
+
   std::clog << "LogThread running" << std::endl;
 
   // non-atomic updates from other threads cause race conditions, but who cares

RTCP/IONProc/src/Makefile.am

@@ -13,6 +13,7 @@ OutputThread.h			\
 OutputSection.h			\
 ReaderWriterSynchronization.h	\
 RSP.h				\
+Scheduling.h			\
 SlidingPointer.h		\
 WallClockTime.h			\
 WH_DelayCompensation.h		\
@@ -30,6 +31,7 @@ LogThread.cc			\
 OutputThread.cc			\
 OutputSection.cc		\
 ReaderWriterSynchronization.cc	\
+Scheduling.cc			\
 WH_DelayCompensation.cc		\
 FCNP_ServerStream.cc		\
 Package__Version.cc

RTCP/IONProc/src/OutputThread.cc

@@ -39,8 +39,19 @@ OutputThread::OutputThread(Stream *streamToStorage, unsigned nrBaselines, unsign
   for (unsigned i = 0; i < maxSendQueueSize; i ++)
     itsFreeQueue.append(new CorrelatedData(nrBaselines, nrChannels, hugeMemoryAllocator));
 
-  if (pthread_create(&thread, 0, mainLoopStub, this) != 0)
+  pthread_attr_t attr;
+
+  if (pthread_attr_init(&attr) != 0)
+    throw SystemCallException("pthread_attr_init output thread", errno, THROW_ARGS);
+
+  if (pthread_attr_setstacksize(&attr, 65536) != 0)
+    throw SystemCallException("pthread_attr_setstacksize output thread", errno, THROW_ARGS);
+
+  if (pthread_create(&thread, &attr, mainLoopStub, this) != 0)
     throw SystemCallException("pthread_create output thread", errno, THROW_ARGS);
+
+  if (pthread_attr_destroy(&attr) != 0)
+    throw SystemCallException("pthread_attr_destroy output thread", errno, THROW_ARGS);
 }
 
 

RTCP/IONProc/src/Scheduling.cc

+//#  Scheduling.cc:
+//#
+//#  Copyright (C) 2008
+//#  ASTRON (Netherlands Foundation for Research in Astronomy)
+//#  P.O.Box 2, 7990 AA Dwingeloo, The Netherlands, seg@astron.nl
+//#
+//#  This program is free software; you can redistribute it and/or modify
+//#  it under the terms of the GNU General Public License as published by
+//#  the Free Software Foundation; either version 2 of the License, or
+//#  (at your option) any later version.
+//#
+//#  This program is distributed in the hope that it will be useful,
+//#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+//#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+//#  GNU General Public License for more details.
+//#
+//#  You should have received a copy of the GNU General Public License
+//#  along with this program; if not, write to the Free Software
+//#  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+//#
+//#  $Id$
+
+//# Always #include <lofar_config.h> first!
+#include <lofar_config.h>
+
+#if defined HAVE_BGP_ION
+
+#include <IONProc/Scheduling.h>
+
+#include <iostream>
+#include <cstdio>
+
+#include <sched.h>
+
+
+namespace LOFAR {
+namespace RTCP {
+
+void doNotRunOnCore0()
+{
+  cpu_set_t cpu_set;
+
+  CPU_ZERO(&cpu_set);
+
+  for (unsigned cpu = 1; cpu < 4; cpu ++)
+    CPU_SET(cpu, &cpu_set);
+
+  if (sched_setaffinity(0, sizeof cpu_set, &cpu_set) != 0) {
+    std::clog << "WARNING: sched_setaffinity failed" << std::endl;
+    perror("sched_setaffinity");
+  }
+}
+
+
+void runOnCore0()
+{
+  cpu_set_t cpu_set;
+
+  CPU_ZERO(&cpu_set);
+  CPU_SET(0, &cpu_set);
+
+  if (sched_setaffinity(0, sizeof cpu_set, &cpu_set) != 0) {
+    std::clog << "WARNING: sched_setaffinity failed" << std::endl;
+    perror("sched_setaffinity");
+  }
+}
+
+} // namespace RTCP
+} // namespace LOFAR
+
+#endif

RTCP/IONProc/src/Scheduling.h

+//#  OutputThread.h
+//#
+//#  Copyright (C) 2006
+//#  ASTRON (Netherlands Foundation for Research in Astronomy)
+//#  P.O.Box 2, 7990 AA Dwingeloo, The Netherlands, seg@astron.nl
+//#
+//#  This program is free software; you can redistribute it and/or modify
+//#  it under the terms of the GNU General Public License as published by
+//#  the Free Software Foundation; either version 2 of the License, or
+//#  (at your option) any later version.
+//#
+//#  This program is distributed in the hope that it will be useful,
+//#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+//#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+//#  GNU General Public License for more details.
+//#
+//#  You should have received a copy of the GNU General Public License
+//#  along with this program; if not, write to the Free Software
+//#  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+//#
+//#  $Id$
+
+#ifndef LOFAR_IONPROC_SCHEDULING_H
+#define LOFAR_IONPROC_SCHEDULING_H
+
+//# Never #include <config.h> or #include <lofar_config.h> in a header file!
+
+namespace LOFAR {
+namespace RTCP {
+
+#if defined HAVE_BGP_ION
+// Core 0 handles all ethernet and tree interrupts.  Do not run time-critical
+// threads on this core.
+extern void doNotRunOnCore0();
+extern void runOnCore0();
+#endif
+
+} // namespace RTCP
+} // namespace LOFAR
+
+#endif