From 03d19eeae1bad723406c462aeaecac37435d54bc Mon Sep 17 00:00:00 2001
From: John Romein <romein@astron.nl>
Date: Thu, 18 Dec 2008 20:32:40 +0000
Subject: [PATCH] bug 225: Do not allow time-critical threads to run on core 0,
which handles all ethernet and tree interrupts.
---
RTCP/IONProc/src/ION_Allocator.h | 2 +-
RTCP/IONProc/src/InputSection.cc | 5 +++
RTCP/IONProc/src/InputThread.cc | 23 ++---------
RTCP/IONProc/src/InputThread.h | 1 -
RTCP/IONProc/src/LogThread.cc | 5 +++
RTCP/IONProc/src/Makefile.am | 2 +
RTCP/IONProc/src/OutputThread.cc | 13 +++++-
RTCP/IONProc/src/Scheduling.cc | 71 ++++++++++++++++++++++++++++++++
RTCP/IONProc/src/Scheduling.h | 41 ++++++++++++++++++
9 files changed, 141 insertions(+), 22 deletions(-)
create mode 100644 RTCP/IONProc/src/Scheduling.cc
create mode 100644 RTCP/IONProc/src/Scheduling.h
diff --git a/RTCP/IONProc/src/ION_Allocator.h b/RTCP/IONProc/src/ION_Allocator.h
index 6e83f880fb2..4bb1fa73b07 100644
--- a/RTCP/IONProc/src/ION_Allocator.h
+++ b/RTCP/IONProc/src/ION_Allocator.h
@@ -26,7 +26,7 @@
#include <Interface/Allocator.h>
-#undef FLAT_MEMORY
+#define FLAT_MEMORY
namespace LOFAR {
namespace RTCP {
diff --git a/RTCP/IONProc/src/InputSection.cc b/RTCP/IONProc/src/InputSection.cc
index 25386af060a..018ab716cc7 100644
--- a/RTCP/IONProc/src/InputSection.cc
+++ b/RTCP/IONProc/src/InputSection.cc
@@ -31,6 +31,7 @@
#include <WH_DelayCompensation.h>
#include <InputThread.h>
#include <ION_Allocator.h>
+#include <Scheduling.h>
//#include <TH_ZoidServer.h>
#include <Interface/AlignedStdAllocator.h>
#include <Interface/CN_Command.h>
@@ -64,6 +65,10 @@ template<typename SAMPLE_TYPE> InputSection<SAMPLE_TYPE>::InputSection(const std
itsLogThread(0),
itsDelayTimer("delay")
{
+#if defined HAVE_BGP_ION
+ doNotRunOnCore0();
+#endif
+
raisePriority();
}
diff --git a/RTCP/IONProc/src/InputThread.cc b/RTCP/IONProc/src/InputThread.cc
index 6f928b48cd5..9dfcb0c54cb 100644
--- a/RTCP/IONProc/src/InputThread.cc
+++ b/RTCP/IONProc/src/InputThread.cc
@@ -36,6 +36,7 @@
#include <BeamletBuffer.h>
#include <InputThread.h>
#include <RSP.h>
+#include <Scheduling.h>
#include <errno.h>
#include <signal.h>
@@ -102,24 +103,6 @@ template <typename SAMPLE_TYPE> void InputThread<SAMPLE_TYPE>::sigHandler(int)
}
-template <typename SAMPLE_TYPE> void InputThread<SAMPLE_TYPE>::setAffinity()
-{
-#if 1 && __linux__
- cpu_set_t cpu_set;
-
- CPU_ZERO(&cpu_set);
-
- for (unsigned cpu = 1; cpu < 4; cpu ++)
- CPU_SET(cpu, &cpu_set);
-
- if (sched_setaffinity(0, sizeof cpu_set, &cpu_set) != 0) {
- std::clog << "WARNING: sched_setaffinity failed" << std::endl;
- perror("sched_setaffinity");
- }
-#endif
-}
-
-
template <typename SAMPLE_TYPE> void *InputThread<SAMPLE_TYPE>::mainLoopStub(void *inputThread)
{
try {
@@ -139,7 +122,9 @@ template <typename SAMPLE_TYPE> void *InputThread<SAMPLE_TYPE>::mainLoopStub(voi
template <typename SAMPLE_TYPE> void InputThread<SAMPLE_TYPE>::mainLoop()
{
- setAffinity();
+#if defined HAVE_BGP_ION
+ doNotRunOnCore0();
+#endif
const unsigned maxNrPackets = 128;
TimeStamp actualstamp = itsArgs.startTime - itsArgs.nrTimesPerPacket;
diff --git a/RTCP/IONProc/src/InputThread.h b/RTCP/IONProc/src/InputThread.h
index a9956a640c0..1a98b4dae0e 100644
--- a/RTCP/IONProc/src/InputThread.h
+++ b/RTCP/IONProc/src/InputThread.h
@@ -64,7 +64,6 @@ template<typename SAMPLE_TYPE> class InputThread
private:
static void sigHandler(int);
- static void setAffinity();
volatile bool stop, stopped;
diff --git a/RTCP/IONProc/src/LogThread.cc b/RTCP/IONProc/src/LogThread.cc
index 3598e0bee04..16a14c58df4 100644
--- a/RTCP/IONProc/src/LogThread.cc
+++ b/RTCP/IONProc/src/LogThread.cc
@@ -24,6 +24,7 @@
#include <lofar_config.h>
#include <LogThread.h>
+#include <Scheduling.h>
#include <Interface/PrintVector.h>
#include <algorithm>
@@ -71,6 +72,10 @@ void *LogThread::logThreadStub(void *arg)
void LogThread::logThread()
{
+#if defined HAVE_BGP_ION
+ runOnCore0();
+#endif
+
std::clog << "LogThread running" << std::endl;
// non-atomic updates from other threads cause race conditions, but who cares
diff --git a/RTCP/IONProc/src/Makefile.am b/RTCP/IONProc/src/Makefile.am
index 8f822b5de84..d82c94fb4cd 100644
--- a/RTCP/IONProc/src/Makefile.am
+++ b/RTCP/IONProc/src/Makefile.am
@@ -13,6 +13,7 @@ OutputThread.h \
OutputSection.h \
ReaderWriterSynchronization.h \
RSP.h \
+Scheduling.h \
SlidingPointer.h \
WallClockTime.h \
WH_DelayCompensation.h \
@@ -30,6 +31,7 @@ LogThread.cc \
OutputThread.cc \
OutputSection.cc \
ReaderWriterSynchronization.cc \
+Scheduling.cc \
WH_DelayCompensation.cc \
FCNP_ServerStream.cc \
Package__Version.cc
diff --git a/RTCP/IONProc/src/OutputThread.cc b/RTCP/IONProc/src/OutputThread.cc
index 748b61b7b3b..bfabde7be93 100644
--- a/RTCP/IONProc/src/OutputThread.cc
+++ b/RTCP/IONProc/src/OutputThread.cc
@@ -39,8 +39,19 @@ OutputThread::OutputThread(Stream *streamToStorage, unsigned nrBaselines, unsign
for (unsigned i = 0; i < maxSendQueueSize; i ++)
itsFreeQueue.append(new CorrelatedData(nrBaselines, nrChannels, hugeMemoryAllocator));
- if (pthread_create(&thread, 0, mainLoopStub, this) != 0)
+ pthread_attr_t attr;
+
+ if (pthread_attr_init(&attr) != 0)
+ throw SystemCallException("pthread_attr_init output thread", errno, THROW_ARGS);
+
+ if (pthread_attr_setstacksize(&attr, 65536) != 0)
+ throw SystemCallException("pthread_attr_setstacksize output thread", errno, THROW_ARGS);
+
+ if (pthread_create(&thread, &attr, mainLoopStub, this) != 0)
throw SystemCallException("pthread_create output thread", errno, THROW_ARGS);
+
+ if (pthread_attr_destroy(&attr) != 0)
+ throw SystemCallException("pthread_attr_destroy output thread", errno, THROW_ARGS);
}
diff --git a/RTCP/IONProc/src/Scheduling.cc b/RTCP/IONProc/src/Scheduling.cc
new file mode 100644
index 00000000000..a65a1795176
--- /dev/null
+++ b/RTCP/IONProc/src/Scheduling.cc
@@ -0,0 +1,71 @@
+//# Scheduling.cc:
+//#
+//# Copyright (C) 2008
+//# ASTRON (Netherlands Foundation for Research in Astronomy)
+//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands, seg@astron.nl
+//#
+//# This program is free software; you can redistribute it and/or modify
+//# it under the terms of the GNU General Public License as published by
+//# the Free Software Foundation; either version 2 of the License, or
+//# (at your option) any later version.
+//#
+//# This program is distributed in the hope that it will be useful,
+//# but WITHOUT ANY WARRANTY; without even the implied warranty of
+//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+//# GNU General Public License for more details.
+//#
+//# You should have received a copy of the GNU General Public License
+//# along with this program; if not, write to the Free Software
+//# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+//#
+//# $Id$
+
+//# Always #include <lofar_config.h> first!
+#include <lofar_config.h>
+
+#if defined HAVE_BGP_ION
+
+#include <IONProc/Scheduling.h>
+
+#include <iostream>
+#include <cstdio>
+
+#include <sched.h>
+
+
+namespace LOFAR {
+namespace RTCP {
+
+void doNotRunOnCore0()
+{
+ cpu_set_t cpu_set;
+
+ CPU_ZERO(&cpu_set);
+
+ for (unsigned cpu = 1; cpu < 4; cpu ++)
+ CPU_SET(cpu, &cpu_set);
+
+ if (sched_setaffinity(0, sizeof cpu_set, &cpu_set) != 0) {
+ std::clog << "WARNING: sched_setaffinity failed" << std::endl;
+ perror("sched_setaffinity");
+ }
+}
+
+
+void runOnCore0()
+{
+ cpu_set_t cpu_set;
+
+ CPU_ZERO(&cpu_set);
+ CPU_SET(0, &cpu_set);
+
+ if (sched_setaffinity(0, sizeof cpu_set, &cpu_set) != 0) {
+ std::clog << "WARNING: sched_setaffinity failed" << std::endl;
+ perror("sched_setaffinity");
+ }
+}
+
+} // namespace RTCP
+} // namespace LOFAR
+
+#endif
diff --git a/RTCP/IONProc/src/Scheduling.h b/RTCP/IONProc/src/Scheduling.h
new file mode 100644
index 00000000000..2da71385ca4
--- /dev/null
+++ b/RTCP/IONProc/src/Scheduling.h
@@ -0,0 +1,41 @@
+//# OutputThread.h
+//#
+//# Copyright (C) 2006
+//# ASTRON (Netherlands Foundation for Research in Astronomy)
+//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands, seg@astron.nl
+//#
+//# This program is free software; you can redistribute it and/or modify
+//# it under the terms of the GNU General Public License as published by
+//# the Free Software Foundation; either version 2 of the License, or
+//# (at your option) any later version.
+//#
+//# This program is distributed in the hope that it will be useful,
+//# but WITHOUT ANY WARRANTY; without even the implied warranty of
+//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+//# GNU General Public License for more details.
+//#
+//# You should have received a copy of the GNU General Public License
+//# along with this program; if not, write to the Free Software
+//# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+//#
+//# $Id$
+
+#ifndef LOFAR_IONPROC_SCHEDULING_H
+#define LOFAR_IONPROC_SCHEDULING_H
+
+//# Never #include <config.h> or #include <lofar_config.h> in a header file!
+
+namespace LOFAR {
+namespace RTCP {
+
+#if defined HAVE_BGP_ION
+// Core 0 handles all ethernet and tree interrupts. Do not run time-critical
+// threads on this core.
+extern void doNotRunOnCore0();
+extern void runOnCore0();
+#endif
+
+} // namespace RTCP
+} // namespace LOFAR
+
+#endif
--
GitLab