diff --git a/.gitattributes b/.gitattributes
index c734edee1c82b652e16f131d452837f1b4d1116f..28743d21e359b3784325dfee0e1a60bb87e00f38 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -2572,6 +2572,7 @@ doc/papers/2011/europar/ION-processing.pdf -text
doc/papers/2011/europar/ION-processing.svg -text
doc/papers/2011/europar/LBA-field.jpg -text
doc/papers/2011/europar/Makefile -text
+doc/papers/2011/europar/coherent-dedispersion.jgr -text
doc/papers/2011/europar/colinear.fig -text
doc/papers/2011/europar/delay.fig -text
doc/papers/2011/europar/execution_times.jgr -text
@@ -2581,6 +2582,7 @@ doc/papers/2011/europar/lofar-stations.fig -text
doc/papers/2011/europar/lofar.bib -text
doc/papers/2011/europar/lofar.pdf -text
doc/papers/2011/europar/lofar.tex -text
+doc/papers/2011/europar/pencilbeams.svg -text
doc/papers/2011/europar/stations-beams.jgr -text
doc/papers/references.bib -text
/lofar_config.h.cmake -text
diff --git a/doc/papers/2011/europar/Makefile b/doc/papers/2011/europar/Makefile
index 0eb3f90410c670917278674e0b3f0ef7f6e1d90a..551095e5950bdb37440e617100f7c9959c34897c 100644
--- a/doc/papers/2011/europar/Makefile
+++ b/doc/papers/2011/europar/Makefile
@@ -4,7 +4,7 @@ BIB_SOURCES = lofar.bib
FIG_SOURCES = delay.fig lofar-stations.fig
-JGR_SOURCES = stations-beams.jgr execution_times.jgr
+JGR_SOURCES = stations-beams.jgr execution_times.jgr coherent-dedispersion.jgr
JPG_SOURCES =
diff --git a/doc/papers/2011/europar/coherent-dedispersion.jgr b/doc/papers/2011/europar/coherent-dedispersion.jgr
new file mode 100644
index 0000000000000000000000000000000000000000..715210ea1712d579d5a4e1fdb5b9e0b2219789e7
--- /dev/null
+++ b/doc/papers/2011/europar/coherent-dedispersion.jgr
@@ -0,0 +1,55 @@
+newgraph
+ xaxis
+ min 0 max 2
+ label : Rotational phase
+
+ no_auto_hash_labels
+ hash_label at 0 : 0
+ hash_label at 1 : 2p
+ hash_label at 2 : 4p
+
+ hash_labels
+ font Symbol
+
+ yaxis
+ min 0 max 1.5
+ nodraw
+
+ legend
+ x 0.5 y 1.2
+
+newline
+ label : Subband-level dedispersion
+ linetype dashed
+ linethickness 2.0
+ color 0 0 1
+ pts
+ shell : N=0;for i in \
+ 0.02011669 0.00507009 -0.00771127 0.02275786 -0.13795437 0.01486767\
+ 0.04283439 0.18166798 0.20039363 0.43340311 0.61501703 0.80407534\
+ 0.83333333 0.75695429 0.69960432 0.66769064 0.65099889 0.45159893\
+ 0.43001391 0.30665736 0.12644565 0.01286941 0.084529\
+ 0.02011669 0.00507009 -0.00771127 0.02275786 -0.13795437 0.01486767\
+ 0.04283439 0.18166798 0.20039363 0.43340311 0.61501703 0.80407534\
+ 0.83333333 0.75695429 0.69960432 0.66769064 0.65099889 0.45159893\
+ 0.43001391 0.30665736 0.12644565 0.01286941 0.084529 0.02011669;\
+ do echo $N / 23 | bc -l; echo "($i + 0.13795437)/(0.83333333 + 0.13795437)" | bc -l; N=$[$N+1]; done
+
+newline
+ label : Channel-level dedispersion
+ linetype solid
+ linethickness 2.0
+ color 1 0 0
+ pts
+ pts
+ shell : N=0;for i in \
+ 0.01427185 -0.05419144 0.0065101 0.06195318 -0.07190334 -0.00771236\
+ -0.05202656 -0.04786229 0.01148896 0.39690428 0.71437241 0.83333333\
+ 0.62925924 0.49068521 0.41371978 0.49633274 0.53869321 0.46915986\
+ 0.23657243 0.14340703 0.06047267 0.01298272 -0.06736764\
+ 0.01427185 -0.05419144 0.0065101 0.06195318 -0.07190334 -0.00771236\
+ -0.05202656 -0.04786229 0.01148896 0.39690428 0.71437241 0.83333333\
+ 0.62925924 0.49068521 0.41371978 0.49633274 0.53869321 0.46915986\
+ 0.23657243 0.14340703 0.06047267 0.01298272 -0.06736764 0.01427185;\
+ do echo $N / 23 | bc -l; echo "($i + 0.07190334)/(0.83333333 + 0.07190334)" | bc -l; N=$[$N+1]; done
+
diff --git a/doc/papers/2011/europar/execution_times.jgr b/doc/papers/2011/europar/execution_times.jgr
index 194d198dccc29a61c17409710cce4ba098a3fd9a..dad7110bb083ad8657dc2cf0b142b7b00fd0c77f 100644
--- a/doc/papers/2011/europar/execution_times.jgr
+++ b/doc/papers/2011/europar/execution_times.jgr
@@ -71,7 +71,7 @@ newgraph
marktype xbar marksize 0.8 fill 0.2 pattern solid
pts
shell : awk -v N=3 '/->/ {a+=$3;b+=$5;c+=$7;d+=$9;e+=$11;f+=$13;n++; if(n==N) print "1 " a " 2 " b " 3 " c " 4 " d " 5 " e " 6 " f;}' <execution_times.jgr
- label : Coherent dedispersion \
+ label : Channel dedispersion \
(cases D-F)
newcurve
diff --git a/doc/papers/2011/europar/lofar.pdf b/doc/papers/2011/europar/lofar.pdf
index 4b5eec457e58f929956187c0a4dbc8f50fab32f0..2e6d76a5006d7329d9f1a8994bbd791676a24f0e 100644
Binary files a/doc/papers/2011/europar/lofar.pdf and b/doc/papers/2011/europar/lofar.pdf differ
diff --git a/doc/papers/2011/europar/lofar.tex b/doc/papers/2011/europar/lofar.tex
index 032deacba1879859b989f8d4c2d5a796c560a03e..cba47ff136fa65c7d3082a3aebc0e8a3faf5ebb3 100644
--- a/doc/papers/2011/europar/lofar.tex
+++ b/doc/papers/2011/europar/lofar.tex
@@ -225,7 +225,21 @@ The beamformer transforms chunks representing station data into chunks represent
\subsection{Dedispersion}
-TODO
+\subsection{Channel-level dedispersion}
+
+Another major component in the pulsar-observation pipeline is real-time dedispersion. Since light of a high frequency travels faster through the interstellar medium than light of a lower frequency, the arrival time of a pulse differs for different wave lengths. To combine data from multiple frequency channels, the channels must be aligned (i.e., shifted in time), otherwise, the pulse will be smeared and becomes invisible. This process, called \emph{dedispersion}, is done by post-processing software that runs after the observation has finished. However, to observe at the lowest frequencies, or to observe fast-rotating millisecond pulsars, dedispersion must also be performed \emph{within\/} a channel, since our channels (typically 12~KHz) are too wide to ignore dispersion. Channel-level dedispersion can only be done before integration (Section ...), thus in the real-time pipeline.
+
+Dedispersion is performed in the frequency domain, effectively by doing a 4K~Fourier transform that splits a 12~KHz channel into 3~Hz subchannels. The phases of the observed samples are corrected by applying a Chirp function~\cite{...}, i.e., by multiplication with precomputed, subchannel-dependent, complex weights. These multiplications are programmed in assembly, to reduce the computational costs. A backward FFT is done to revert to 12~KHz channels.
+
+\begin{figure}[ht]
+\includegraphics[width=\textwidth]{coherent-dedispersion.pdf}
+\label{fig:dedispersion-result}
+\caption{Pulse profiles of pulsar J0034-0534, with dedispersion applied at channel and at subband level.}
+\end{figure}
+
+Figure~\ref{fig:dedispersion-result} shows the effectiveness of channel-level dedispersion, where we observed pulsar J0034-0534 with a pulse period of 1.88~ms with and without dedispersion. With dedispersion, the pulse is narrower and has the correct shape, while the noise floor is lower.
+
+Dedispersion is done before or after beam forming, depending on the number of stations and the number of beams, wherever the computational costs are minimal. Still, the computational costs are high (see ...). Yet, this pipeline component significantly contributes to the data quality of the LOFAR telescope. It also demonstrates the power of using a \emph{software\/} telescope; the pipeline component was implemented, verified, and optimized in only one month time.
\subsection{Second All-to-all Exchange}
@@ -272,8 +286,8 @@ In the Stokes I mode, we applied several integration factors (1, 2, 4, 8, and 12
\begin{table}
\begin{tabular}{l|l|r|r|r|r|r|r|l|l}
-Case & Mode & Coherent & Integration & Stations & Beams & Input & Output & Bound & Used for \\
- & & dedispersion & factor & & & rate & rate & & \\
+Case & Mode & Channel & Int. & Stations & Beams & Input & Output & Bound & Used for \\
+ & & dedisp. & factor & & & rate & rate & & \\
\hline
\hline
A & Stokes I & N & 12 & 4 & 543 & 12 Gb/s & 70 Gb/s & Memory & Surveys \\
@@ -287,7 +301,7 @@ F & Stokes I & Y & 1 & 64 & 42 & 198 Gb/s & 65 Gb/s & CPU & Known sources
\label{table:cases}
\end{table}
-We further analyse the workload of the compute cores by highlighting a set of interesting cases, summarised in Table \ref{table:cases}. We will focus on a memory-bound case (A), which also creates the highest number of beams, on CPU-bound cases interesting for performing surveys, with either 24 stations (B) or 64 stations (C) as input. Cases D and E focus on high-resolution observations of known sources, and are I/O bound configurations with 24 and 64 stations, respectively. Case F focusses on the observations of known sources as well, using Stokes I output, which allows more beams to be created. Coherent dedispersion requires the DM of the sources to be known, and is thus only enabled in the cases which observe known sources.
+We further analyse the workload of the compute cores by highlighting a set of interesting cases, summarised in Table \ref{table:cases}. We will focus on a memory-bound case (A), which also creates the highest number of beams, on CPU-bound cases interesting for performing surveys, with either 24 stations (B) or 64 stations (C) as input. Cases D and E focus on high-resolution observations of known sources, and are I/O bound configurations with 24 and 64 stations, respectively. Case F focusses on the observations of known sources as well, using Stokes I output, which allows more beams to be created. Channel dedispersion requires the DM of the sources to be known, and is thus only enabled in the cases which observe known sources.
The workload of the compute cores for each case is shown in Figure \ref{fig:execution-times}, which shows the average workload per core. For the CPU-bound cases B and C, the average load has to be lower than 100\% in order to prevent fluctuations from slowing down our real-time system. These fluctuations typically occur due to clashes within the BG/P 3D-torus network which is used for both all-to-all-exchanges, and cannot be avoided in all cases.
@@ -295,7 +309,7 @@ The cases which create many beams (A-C) spend most of the cycles performing beam
The costs for both the first and the second all-to-all exchange are mostly hidden due to overlaps with computation. The remaining cost for the second exchange is proportional to the output bandwidth required in each case.
-For the I/O-bound cases D-F, only a few tied-array beams are formed and transformed into Stokes I(QUV) parameters, which produces a lot of data but requires little CPU time. Enough CPU time is therefore avaialable to include coherent dedispersion, which scales with the number of beams and, as Figure \ref{fig:execution-times} shows, is an expensive operation.
+For the I/O-bound cases D-F, only a few tied-array beams are formed and transformed into Stokes I(QUV) parameters, which produces a lot of data but requires little CPU time. Enough CPU time is therefore avaialable to include channel dedispersion, which scales with the number of beams and, as Figure \ref{fig:execution-times} shows, is an expensive operation.
\comment{
- hit CPU, memory and I/O bounds
diff --git a/doc/papers/2011/europar/pencilbeams.svg b/doc/papers/2011/europar/pencilbeams.svg
new file mode 100644
index 0000000000000000000000000000000000000000..ec0335d50a31610d6b128bb819c95795650fdad1
--- /dev/null
+++ b/doc/papers/2011/europar/pencilbeams.svg
@@ -0,0 +1,177 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!-- Creator: fig2dev Version 3.2 Patchlevel 5a -->
+
+<!-- CreationDate: Thu Jan 27 09:28:38 2011 -->
+
+<!-- Magnification: 2.100 -->
+
+<svg
+ xmlns:dc="http://purl.org/dc/elements/1.1/"
+ xmlns:cc="http://creativecommons.org/ns#"
+ xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+ xmlns:svg="http://www.w3.org/2000/svg"
+ xmlns="http://www.w3.org/2000/svg"
+ xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
+ xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
+ width="12.6in"
+ height="13.1in"
+ viewBox="17009 5954 15113 15680"
+ id="svg2"
+ version="1.1"
+ inkscape:version="0.47pre4 r22446"
+ sodipodi:docname="pencilbeams.svg">
+ <metadata
+ id="metadata54">
+ <rdf:RDF>
+ <cc:Work
+ rdf:about="">
+ <dc:format>image/svg+xml</dc:format>
+ <dc:type
+ rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
+ <dc:title></dc:title>
+ </cc:Work>
+ </rdf:RDF>
+ </metadata>
+ <defs
+ id="defs52">
+ <inkscape:perspective
+ sodipodi:type="inkscape:persp3d"
+ inkscape:vp_x="0 : 589.5 : 1"
+ inkscape:vp_y="0 : 1000 : 0"
+ inkscape:vp_z="1134 : 589.5 : 1"
+ inkscape:persp3d-origin="567 : 393 : 1"
+ id="perspective56" />
+ </defs>
+ <sodipodi:namedview
+ pagecolor="#ffffff"
+ bordercolor="#666666"
+ borderopacity="1"
+ objecttolerance="10"
+ gridtolerance="10"
+ guidetolerance="10"
+ inkscape:pageopacity="0"
+ inkscape:pageshadow="2"
+ inkscape:window-width="1048"
+ inkscape:window-height="829"
+ id="namedview50"
+ showgrid="false"
+ inkscape:zoom="0.80067854"
+ inkscape:cx="972.28554"
+ inkscape:cy="889.24576"
+ inkscape:window-x="0"
+ inkscape:window-y="830"
+ inkscape:window-maximized="0"
+ inkscape:current-layer="svg2" />
+ <polygon
+ id="polygon8"
+ style="fill:#e3e3e3;stroke:#656565;stroke-width:95;stroke-linecap:butt;stroke-linejoin:miter;fill-opacity:1;stroke-opacity:1"
+ points="31477,12361 25778,6452 25778,6452 17385,21259 " />
+ <ellipse
+ sodipodi:ry="1725"
+ sodipodi:rx="4125"
+ sodipodi:cy="0"
+ sodipodi:cx="0"
+ cy="0"
+ cx="0"
+ d="M 4125,0 C 4125,952.69119 2278.1746,1725 0,1725 -2278.1746,1725 -4125,952.69119 -4125,0 c 0,-952.69119 1846.8254,-1725 4125,-1725 2278.1746,0 4125,772.30881 4125,1725 z"
+ id="ellipse10"
+ style="fill:#ffffe0;stroke:#656565;stroke-width:95;stroke-opacity:1"
+ ry="1725"
+ rx="4125"
+ transform="matrix(0.70711587,0.70709769,-0.70709769,0.70711587,28741,9310)" />
+ <polygon
+ id="polygon12"
+ style="fill:#ffe0e0;stroke:#ffff00;stroke-width:95;stroke-linecap:butt;stroke-linejoin:miter"
+ points="27590,9125 26920,8455 26828,7880 27403,7974 27403,7974 28073,8640 28165,9217 "
+ transform="matrix(0.90647511,0,0,0.90647511,2687.9989,870.71675)" />
+ <polygon
+ id="polygon14"
+ style="fill:#ffe0e0;stroke:#ff0000;stroke-width:95;stroke-linecap:butt;stroke-linejoin:miter"
+ points="27315,7397 26643,6729 26553,6154 27128,6246 27128,6246 27796,6914 27886,7491 "
+ transform="matrix(0.90647511,0,0,0.90647511,2687.9989,870.71675)" />
+ <polygon
+ id="polygon16"
+ style="fill:#ffe0e0;stroke:#ff0000;stroke-width:95;stroke-linecap:butt;stroke-linejoin:miter"
+ points="26828,7880 26162,7212 26066,6637 26643,6729 26643,6729 27315,7397 27403,7974 "
+ transform="matrix(0.90647511,0,0,0.90647511,2687.9989,870.71675)" />
+ <polygon
+ id="polygon18"
+ style="fill:#ffe0e0;stroke:#ff0000;stroke-width:95;stroke-linecap:butt;stroke-linejoin:miter"
+ points="26345,8365 25677,7699 25587,7122 26162,7212 26162,7212 26828,7880 26920,8455 "
+ transform="matrix(0.90647511,0,0,0.90647511,2687.9989,870.71675)" />
+ <polygon
+ id="polygon20"
+ style="fill:#ffe0e0;stroke:#ff0000;stroke-width:95;stroke-linecap:butt;stroke-linejoin:miter"
+ points="27107,9610 26439,8940 26345,8365 26920,8455 26920,8455 27590,9125 27680,9700 "
+ transform="matrix(0.90647511,0,0,0.90647511,2687.9989,870.71675)" />
+ <polygon
+ id="polygon22"
+ style="fill:#ffe0e0;stroke:#ff0000;stroke-width:95;stroke-linecap:butt;stroke-linejoin:miter"
+ points="28073,8640 27403,7974 27315,7397 27886,7491 27886,7491 28556,8157 28653,8732 "
+ transform="matrix(0.90647511,0,0,0.90647511,2687.9989,870.71675)" />
+ <polygon
+ id="polygon24"
+ style="fill:#ffe0e0;stroke:#ff0000;stroke-width:95;stroke-linecap:butt;stroke-linejoin:miter"
+ points="28556,8157 27886,7491 27796,6914 28371,7006 28371,7006 29041,7674 29131,8249 "
+ transform="matrix(0.90647511,0,0,0.90647511,2687.9989,870.71675)" />
+ <polygon
+ id="polygon26"
+ style="fill:#ffe0e0;stroke:#ff0000;stroke-width:95;stroke-linecap:butt;stroke-linejoin:miter"
+ points="29803,8919 29131,8249 29041,7674 29614,7766 29614,7766 30282,8436 30376,9009 "
+ transform="matrix(0.90647511,0,0,0.90647511,2687.9989,870.71675)" />
+ <polygon
+ id="polygon28"
+ style="fill:#ffe0e0;stroke:#ff0000;stroke-width:95;stroke-linecap:butt;stroke-linejoin:miter"
+ points="29318,9400 28653,8732 28556,8157 29131,8249 29131,8249 29803,8919 29891,9492 "
+ transform="matrix(0.90647511,0,0,0.90647511,2687.9989,870.71675)" />
+ <polygon
+ id="polygon30"
+ style="fill:#ffe0e0;stroke:#ff0000;stroke-width:95;stroke-linecap:butt;stroke-linejoin:miter"
+ points="28833,9883 28165,9217 28073,8640 28653,8732 28653,8732 29318,9400 29408,9980 "
+ transform="matrix(0.90647511,0,0,0.90647511,2687.9989,870.71675)" />
+ <polygon
+ id="polygon32"
+ style="fill:#ffe0e0;stroke:#ff0000;stroke-width:95;stroke-linecap:butt;stroke-linejoin:miter"
+ points="28348,10368 27680,9700 27590,9125 28165,9217 28165,9217 28833,9883 28923,10460 "
+ transform="matrix(0.90647511,0,0,0.90647511,2687.9989,870.71675)" />
+ <polygon
+ id="polygon34"
+ style="fill:#ffe0e0;stroke:#ff0000;stroke-width:95;stroke-linecap:butt;stroke-linejoin:miter"
+ points="27867,10851 27197,10183 27107,9610 27680,9700 27680,9700 28348,10368 28445,10945 "
+ transform="matrix(0.90647511,0,0,0.90647511,2687.9989,870.71675)" />
+ <polygon
+ id="polygon36"
+ style="fill:#ffe0e0;stroke:#ff0000;stroke-width:95;stroke-linecap:butt;stroke-linejoin:miter"
+ points="29112,11609 28445,10945 28348,10368 28923,10460 28923,10460 29595,11130 29686,11706 "
+ transform="matrix(0.90647511,0,0,0.90647511,2687.9989,870.71675)" />
+ <polygon
+ id="polygon38"
+ style="fill:#ffe0e0;stroke:#ff0000;stroke-width:95;stroke-linecap:butt;stroke-linejoin:miter"
+ points="30076,10645 29408,9980 29318,9400 29891,9492 29891,9492 30559,10164 30654,10735 "
+ transform="matrix(0.90647511,0,0,0.90647511,2687.9989,870.71675)" />
+ <polygon
+ id="polygon40"
+ style="fill:#ffe0e0;stroke:#ff0000;stroke-width:95;stroke-linecap:butt;stroke-linejoin:miter"
+ points="30559,10164 29891,9492 29803,8919 30376,9009 30376,9009 31044,9679 31137,10253 "
+ transform="matrix(0.90647511,0,0,0.90647511,2687.9989,870.71675)" />
+ <polygon
+ id="polygon42"
+ style="fill:#ffe0e0;stroke:#ff0000;stroke-width:95;stroke-linecap:butt;stroke-linejoin:miter"
+ points="31321,11405 30654,10735 30559,10164 31137,10253 31137,10253 31806,10922 31895,11500 "
+ transform="matrix(0.90647511,0,0,0.90647511,2687.9989,870.71675)" />
+ <polygon
+ id="polygon44"
+ style="fill:#ffe0e0;stroke:#ff0000;stroke-width:95;stroke-linecap:butt;stroke-linejoin:miter"
+ points="30836,11890 30171,11220 30076,10645 30654,10735 30654,10735 31321,11405 31414,11981 "
+ transform="matrix(0.90647511,0,0,0.90647511,2687.9989,870.71675)" />
+ <polygon
+ id="polygon46"
+ style="fill:#ffe0e0;stroke:#ff0000;stroke-width:95;stroke-linecap:butt;stroke-linejoin:miter"
+ points="29595,11130 28923,10460 28833,9883 29408,9980 29408,9980 30076,10645 30171,11220 "
+ transform="matrix(0.90647511,0,0,0.90647511,2687.9989,870.71675)" />
+ <polygon
+ id="polygon48"
+ style="fill:#ffe0e0;stroke:#ff0000;stroke-width:95;stroke-linecap:butt;stroke-linejoin:miter"
+ points="30353,12373 29686,11706 29595,11130 30171,11220 30171,11220 30836,11890 30931,12466 "
+ transform="matrix(0.90647511,0,0,0.90647511,2687.9989,870.71675)" />
+</svg>