diff --git a/.gitattributes b/.gitattributes
index 415afbbf2e0e1e09f242b8a1d6f0ca446971e2a4..eb7e5409fb7a83c5efb457e379dfbdd4e0771bd6 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -2591,6 +2591,7 @@ doc/papers/2011/europar/Europe_countries_map.png -text
doc/papers/2011/europar/ION-processing.pdf -text
doc/papers/2011/europar/ION-processing.svg -text
doc/papers/2011/europar/LBA-field.jpg -text
+doc/papers/2011/europar/LBAfield-delay.svg -text
doc/papers/2011/europar/Makefile -text
doc/papers/2011/europar/coherent-dedispersion.jgr -text
doc/papers/2011/europar/colinear.fig -text
diff --git a/doc/papers/2011/europar/LBA-field.jpg b/doc/papers/2011/europar/LBA-field.jpg
index 2a5c4d8b7be382c85fcc03a9343e501bd3ad61f3..cb6d8a0348cf8122ac61994ecf72e48615058de9 100644
Binary files a/doc/papers/2011/europar/LBA-field.jpg and b/doc/papers/2011/europar/LBA-field.jpg differ
diff --git a/doc/papers/2011/europar/LBAfield-delay.svg b/doc/papers/2011/europar/LBAfield-delay.svg
new file mode 100644
index 0000000000000000000000000000000000000000..9e91c73231bc1486d8b84478b892fb88e85931b0
--- /dev/null
+++ b/doc/papers/2011/europar/LBAfield-delay.svg
@@ -0,0 +1,224 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!-- Created with Inkscape (http://www.inkscape.org/) -->
+
+<svg
+ xmlns:dc="http://purl.org/dc/elements/1.1/"
+ xmlns:cc="http://creativecommons.org/ns#"
+ xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+ xmlns:svg="http://www.w3.org/2000/svg"
+ xmlns="http://www.w3.org/2000/svg"
+ xmlns:xlink="http://www.w3.org/1999/xlink"
+ xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
+ xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
+ sodipodi:docname="LBAfield-delay.svg"
+ inkscape:version="0.47pre4 r22446"
+ version="1.1"
+ id="svg2"
+ height="682.79382"
+ width="665.84027">
+ <sodipodi:namedview
+ id="base"
+ pagecolor="#ffffff"
+ bordercolor="#666666"
+ borderopacity="1.0"
+ inkscape:pageopacity="0.0"
+ inkscape:pageshadow="2"
+ inkscape:zoom="0.35"
+ inkscape:cx="-205.53156"
+ inkscape:cy="588.64453"
+ inkscape:document-units="px"
+ inkscape:current-layer="layer1"
+ showgrid="false"
+ inkscape:window-width="1048"
+ inkscape:window-height="828"
+ inkscape:window-x="0"
+ inkscape:window-y="0"
+ inkscape:window-maximized="0" />
+ <defs
+ id="defs4">
+ <marker
+ inkscape:stockid="Arrow1Mend"
+ orient="auto"
+ refY="0"
+ refX="0"
+ id="Arrow1Mend"
+ style="overflow:visible">
+ <path
+ id="path4960"
+ d="M 0,0 5,-5 -12.5,0 5,5 0,0 z"
+ style="fill-rule:evenodd;stroke:#000000;stroke-width:1pt;marker-start:none"
+ transform="matrix(-0.4,0,0,-0.4,-4,0)" />
+ </marker>
+ <marker
+ inkscape:stockid="Arrow2Mstart"
+ orient="auto"
+ refY="0"
+ refX="0"
+ id="Arrow2Mstart"
+ style="overflow:visible">
+ <path
+ id="path4975"
+ style="font-size:12px;fill-rule:evenodd;stroke-width:0.625;stroke-linejoin:round"
+ d="M 8.7185878,4.0337352 -2.2072895,0.01601326 8.7185884,-4.0017078 c -1.7454984,2.3720609 -1.7354408,5.6174519 -6e-7,8.035443 z"
+ transform="scale(0.6,0.6)" />
+ </marker>
+ <marker
+ inkscape:stockid="Arrow1Lend"
+ orient="auto"
+ refY="0"
+ refX="0"
+ id="Arrow1Lend"
+ style="overflow:visible">
+ <path
+ id="path4954"
+ d="M 0,0 5,-5 -12.5,0 5,5 0,0 z"
+ style="fill-rule:evenodd;stroke:#000000;stroke-width:1pt;marker-start:none"
+ transform="matrix(-0.8,0,0,-0.8,-10,0)" />
+ </marker>
+ <marker
+ inkscape:stockid="Arrow1Lstart"
+ orient="auto"
+ refY="0"
+ refX="0"
+ id="Arrow1Lstart"
+ style="overflow:visible">
+ <path
+ id="path4951"
+ d="M 0,0 5,-5 -12.5,0 5,5 0,0 z"
+ style="fill-rule:evenodd;stroke:#000000;stroke-width:1pt;marker-start:none"
+ transform="matrix(0.8,0,0,0.8,10,0)" />
+ </marker>
+ <inkscape:perspective
+ id="perspective10"
+ inkscape:persp3d-origin="372.04724 : 350.78739 : 1"
+ inkscape:vp_z="744.09448 : 526.18109 : 1"
+ inkscape:vp_y="0 : 1000 : 0"
+ inkscape:vp_x="0 : 526.18109 : 1"
+ sodipodi:type="inkscape:persp3d" />
+ <inkscape:perspective
+ sodipodi:type="inkscape:persp3d"
+ inkscape:vp_x="0 : 0.5 : 1"
+ inkscape:vp_y="0 : 1000 : 0"
+ inkscape:vp_z="1 : 0.5 : 1"
+ inkscape:persp3d-origin="0.5 : 0.33333333 : 1"
+ id="perspective2884" />
+ <inkscape:perspective
+ sodipodi:type="inkscape:persp3d"
+ inkscape:vp_x="0 : 0.5 : 1"
+ inkscape:vp_y="0 : 1000 : 0"
+ inkscape:vp_z="1 : 0.5 : 1"
+ inkscape:persp3d-origin="0.5 : 0.33333333 : 1"
+ id="perspective2997" />
+ <inkscape:perspective
+ sodipodi:type="inkscape:persp3d"
+ inkscape:vp_x="0 : 0.5 : 1"
+ inkscape:vp_y="0 : 1000 : 0"
+ inkscape:vp_z="1 : 0.5 : 1"
+ inkscape:persp3d-origin="0.5 : 0.33333333 : 1"
+ id="perspective3019" />
+ <inkscape:perspective
+ sodipodi:type="inkscape:persp3d"
+ inkscape:vp_x="0 : 0.5 : 1"
+ inkscape:vp_y="0 : 1000 : 0"
+ inkscape:vp_z="1 : 0.5 : 1"
+ inkscape:persp3d-origin="0.5 : 0.33333333 : 1"
+ id="perspective3041" />
+ <inkscape:perspective
+ id="perspective4886"
+ inkscape:persp3d-origin="0.5 : 0.33333333 : 1"
+ inkscape:vp_z="1 : 0.5 : 1"
+ inkscape:vp_y="0 : 1000 : 0"
+ inkscape:vp_x="0 : 0.5 : 1"
+ sodipodi:type="inkscape:persp3d" />
+ <inkscape:perspective
+ id="perspective4908"
+ inkscape:persp3d-origin="0.5 : 0.33333333 : 1"
+ inkscape:vp_z="1 : 0.5 : 1"
+ inkscape:vp_y="0 : 1000 : 0"
+ inkscape:vp_x="0 : 0.5 : 1"
+ sodipodi:type="inkscape:persp3d" />
+ <inkscape:perspective
+ id="perspective4932"
+ inkscape:persp3d-origin="0.5 : 0.33333333 : 1"
+ inkscape:vp_z="1 : 0.5 : 1"
+ inkscape:vp_y="0 : 1000 : 0"
+ inkscape:vp_x="0 : 0.5 : 1"
+ sodipodi:type="inkscape:persp3d" />
+ <inkscape:perspective
+ id="perspective6350"
+ inkscape:persp3d-origin="0.5 : 0.33333333 : 1"
+ inkscape:vp_z="1 : 0.5 : 1"
+ inkscape:vp_y="0 : 1000 : 0"
+ inkscape:vp_x="0 : 0.5 : 1"
+ sodipodi:type="inkscape:persp3d" />
+ <inkscape:perspective
+ id="perspective6372"
+ inkscape:persp3d-origin="0.5 : 0.33333333 : 1"
+ inkscape:vp_z="1 : 0.5 : 1"
+ inkscape:vp_y="0 : 1000 : 0"
+ inkscape:vp_x="0 : 0.5 : 1"
+ sodipodi:type="inkscape:persp3d" />
+ </defs>
+ <metadata
+ id="metadata7">
+ <rdf:RDF>
+ <cc:Work
+ rdf:about="">
+ <dc:format>image/svg+xml</dc:format>
+ <dc:type
+ rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
+ <dc:title></dc:title>
+ </cc:Work>
+ </rdf:RDF>
+ </metadata>
+ <g
+ id="layer1"
+ inkscape:groupmode="layer"
+ inkscape:label="Layer 1"
+ transform="translate(-1.4285644,-369.42549)">
+ <image
+ xlink:href="file:///home/mol/svn/lofar-papers/2011/europar/LBA-field.jpg"
+ width="600"
+ height="454"
+ id="image2886"
+ x="1.4285644"
+ y="598.2193" />
+ <path
+ id="path4070"
+ title="sin(x)"
+ d="m 109.44983,649.45164 c -1.80697,-2.03049 -3.60765,-4.05812 -4.95171,-5.87584 -1.34407,-1.81773 -2.22139,-3.4209 -2.35702,-4.68315 -0.13563,-1.26224 0.47417,-2.18184 1.82186,-2.76225 1.3477,-0.58041 3.42918,-0.82352 5.95741,-0.86129 2.52823,-0.0378 5.49282,0.12507 8.439,0.27943 2.94619,0.15436 5.86135,0.29446 8.30015,0.21559 2.43879,-0.0789 4.39128,-0.38128 5.59494,-1.0279 1.20365,-0.64662 1.65512,-1.639 1.37694,-2.96677 -0.27819,-1.32777 -1.28151,-2.98886 -2.71112,-4.8459 -1.42961,-1.85705 -3.27489,-3.90517 -5.07708,-5.93347 -1.80219,-2.02831 -3.54871,-4.03103 -4.79953,-5.80588 -1.25082,-1.77487 -1.99629,-3.31743 -1.98653,-4.51285 0.01,-1.19541 0.77772,-2.04231 2.26634,-2.55795 1.48861,-0.51563 3.69299,-0.70225 6.30277,-0.70252 2.60979,-2.8e-4 5.61414,0.18082 8.55076,0.33079 2.93661,0.14996 5.79293,0.26301 8.13476,0.13957 2.34183,-0.12344 4.15977,-0.4877 5.21683,-1.2017 1.05706,-0.71401 1.35066,-1.77895 0.93336,-3.17067 -0.41732,-1.39173 -1.54032,-3.10783 -3.04742,-5.00049 -1.50709,-1.89267 -3.38726,-3.95682 -5.17511,-5.97853 -1.78786,-2.02172 -3.47088,-3.99525 -4.62112,-5.72388 -1.15024,-1.72863 -1.75858,-3.20816 -1.60117,-4.33571 0.1574,-1.12754 1.08274,-1.9021 2.70857,-2.35466 1.62583,-0.45256 3.94655,-0.58571 6.62966,-0.55227 2.68313,0.0334 5.71745,0.22832 8.63497,0.3695 2.91752,0.14119 5.70578,0.22295 7.94352,0.0517 2.23772,-0.17129 3.91609,-0.5997 4.82459,-1.382 0.90851,-0.78228 1.04537,-1.91927 0.4929,-3.37312 -0.55246,-1.45386 -1.78832,-3.22184 -3.36454,-5.14627 -1.57621,-1.92443 -3.48141,-4.00009 -5.24542,-6.01085 -1.76402,-2.01076 -3.37448,-3.95094 -4.41722,-5.63015 -1.04273,-1.67921 -1.50919,-3.09352 -1.20247,-4.15244 0.30672,-1.05891 1.388,-1.76178 3.14679,-2.15322 1.7588,-0.39145 4.18878,-0.47437 6.93674,-0.41113 2.74795,0.0632 5.80232,0.26734 8.69129,0.3954 2.88897,0.12806 5.60025,0.17444 7.72717,-0.0478 2.12692,-0.22223 3.66123,-0.71686 4.41983,-1.56806 0.75859,-0.8512 0.74046,-2.05944 0.0574,-3.57334 -0.6831,-1.5139 -2.02454,-3.33042 -3.66124,-5.28265 -1.6367,-1.95223 -3.55694,-4.03481 -5.28773,-6.03029 -1.73078,-1.99548 -3.2599,-3.89827 -4.18864,-5.52508 -0.92873,-1.62681 -1.24913,-2.97399 -0.79203,-3.96378 0.45709,-0.98979 1.69226,-1.62192 3.57923,-1.95445 1.88697,-0.33253 4.41874,-0.36866 7.22276,-0.27964 2.80401,0.089 5.86843,0.29772 8.71952,0.40837 2.85108,0.11064 5.47675,0.11767 7.48659,-0.15837 2.00984,-0.27605 3.39623,-0.83867 4.00415,-1.75914 0.6079,-0.92047 0.43716,-2.19885 -0.37154,-3.77047 -0.80869,-1.57164 -2.24802,-3.43315 -3.9363,-5.4091 -1.68829,-1.97595 -3.61356,-4.06083 -5.30184,-6.03678 -1.68829,-1.97595 -3.12762,-3.83745 -3.93631,-5.40909 -0.80871,-1.57163 -0.97946,-2.85002 -0.37154,-3.77048 0.6079,-0.92048 1.99431,-1.48309 4.00415,-1.75914 2.00984,-0.27604 4.63551,-0.26901 7.48659,-0.15836 2.85109,0.11064 5.9155,0.31935 8.71952,0.40836 2.80402,0.089 5.33579,0.0529 7.22276,-0.27964 1.88696,-0.33253 3.12214,-0.96466 3.57924,-1.95445 0.45709,-0.9898 0.13669,-2.33697 -0.79205,-3.96378 -0.92873,-1.62681 -2.45786,-3.5296 -4.18864,-5.52508 -1.73079,-1.99548 -3.65103,-4.07805 -5.28772,-6.03029 -1.6367,-1.95223 -2.97814,-3.76875 -3.66124,-5.28266 -0.68311,-1.51389 -0.70124,-2.72213 0.0574,-3.57333 0.75859,-0.8512 2.2929,-1.34583 4.41983,-1.56806 2.12692,-0.22222 4.83819,-0.17584 7.72716,-0.0478 2.88897,0.12807 5.94335,0.33215 8.6913,0.3954 2.74795,0.0632 5.17794,-0.0197 6.93673,-0.41113 1.75879,-0.39144 2.84008,-1.09431 3.1468,-2.15322 0.30672,-1.05892 -0.15973,-2.47323 -1.20248,-4.15244 -1.04273,-1.67921 -2.65319,-3.61939 -4.41721,-5.63015 -1.76402,-2.01075 -3.66921,-4.08641 -5.24543,-6.01085 -1.57621,-1.92443 -2.81207,-3.69241 -3.36454,-5.14626 -0.55247,-1.45386 -0.4156,-2.59085 0.49291,-3.37313 0.90851,-0.7823 2.58686,-1.21071 4.82459,-1.38201 2.23774,-0.17128 5.026,-0.0895 7.94352,0.0517 2.91752,0.14118 5.95184,0.33605 8.63496,0.3695 2.68312,0.0334 5.00383,-0.0997 6.62966,-0.55228 1.62583,-0.45256 2.55117,-1.22711 2.70858,-2.35466 0.15741,-1.12755 -0.45092,-2.60708 -1.60117,-4.33571 -1.15025,-1.72862 -2.83327,-3.70216 -4.62112,-5.72387 -1.78785,-2.02172 -3.66802,-4.08587 -5.17512,-5.97853 -1.50709,-1.89266 -2.63009,-3.60877 -3.0474,-5.0005 -0.41732,-1.39172 -0.12372,-2.45666 0.93335,-3.17066 1.05707,-0.71401 2.875,-1.07827 5.21683,-1.2017 2.34183,-0.12344 5.19814,-0.0104 8.13476,0.13957 2.93662,0.14996 5.94098,0.33106 8.55075,0.33078 2.60979,-2.6e-4 4.81417,-0.18688 6.30278,-0.70252 1.48862,-0.51564 2.25658,-1.36253 2.26634,-2.55794 0.01,-1.19542 -0.73571,-2.73799 -1.98653,-4.51285 -1.25083,-1.77486 -2.99734,-3.77758 -4.79953,-5.80588 -1.80219,-2.02831 -3.64747,-4.07642 -5.07708,-5.93347 -1.42961,-1.85704 -2.43293,-3.51814 -2.71112,-4.84591 -0.27818,-1.32777 0.17329,-2.32015 1.37694,-2.96677 1.20366,-0.64662 3.15615,-0.94903 5.59494,-1.0279 2.43879,-0.0789 5.35397,0.0612 8.30015,0.2156 2.94618,0.15435 5.91076,0.31718 8.439,0.27942 2.52822,-0.0378 4.6097,-0.28087 5.9574,-0.86128 1.3477,-0.58042 1.9575,-1.50002 1.82186,-2.76226 -0.13563,-1.26225 -1.01294,-2.86542 -2.35701,-4.68315 -1.34407,-1.81771 -3.14474,-3.84533 -4.95172,-5.87583"
+ style="color:#000000;fill:none;stroke:#ff6565;stroke-width:6;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:6.4000001;stroke-opacity:1;stroke-dasharray:none;stroke-dashoffset:0;marker:none;visibility:visible;display:inline;overflow:visible;enable-background:accumulate" />
+ <path
+ style="fill:none;stroke:#000000;stroke-width:6;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:6.4000001;stroke-opacity:1;stroke-dasharray:6,12;stroke-dashoffset:0"
+ d="M 492.02031,704.8009 113.36477,475.94232"
+ id="path4922"
+ sodipodi:nodetypes="cc" />
+ <path
+ style="fill:none;stroke:#000000;stroke-width:6;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:6.4000001;stroke-opacity:1;stroke-dasharray:6,12;stroke-dashoffset:0"
+ d="M 115,652.36218 31.135253,601.80454"
+ id="path4922-3"
+ sodipodi:nodetypes="cc" />
+ <path
+ style="fill:none;stroke:#000000;stroke-width:6;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:6.4000001;stroke-opacity:1;stroke-dasharray:none;marker-start:url(#Arrow2Mstart);marker-mid:none;marker-end:url(#Arrow1Mend)"
+ d="M 110.9235,480.05345 34.459029,597.93905"
+ id="path4946"
+ sodipodi:nodetypes="cc" />
+ <text
+ xml:space="preserve"
+ style="font-size:64px;font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;text-align:start;line-height:125%;writing-mode:lr-tb;text-anchor:start;fill:#000000;fill-opacity:1;stroke:none;font-family:Verdana;-inkscape-font-specification:Verdana"
+ x="-426.14566"
+ y="359.95926"
+ id="text6338"
+ transform="matrix(0.58809037,-0.80879522,0.80879522,0.58809037,0,0)"
+ sodipodi:linespacing="125%"><tspan
+ sodipodi:role="line"
+ id="tspan6340"
+ x="-426.14566"
+ y="359.95926">Δt</tspan></text>
+ <path
+ id="path4070-8"
+ title="sin(x)"
+ d="m 489.09172,701.70127 c -1.80697,-2.03049 -3.60765,-4.05812 -4.95171,-5.87584 -1.34407,-1.81773 -2.22139,-3.4209 -2.35702,-4.68315 -0.13563,-1.26224 0.47417,-2.18184 1.82186,-2.76225 1.3477,-0.58041 3.42918,-0.82352 5.95741,-0.86129 2.52823,-0.0378 5.49282,0.12507 8.439,0.27943 2.94619,0.15436 5.86135,0.29446 8.30015,0.21559 2.43879,-0.0789 4.39128,-0.38128 5.59494,-1.0279 1.20365,-0.64662 1.65512,-1.639 1.37694,-2.96677 -0.27819,-1.32777 -1.28151,-2.98886 -2.71112,-4.8459 -1.42961,-1.85705 -3.27489,-3.90517 -5.07708,-5.93347 -1.80219,-2.02831 -3.54871,-4.03103 -4.79953,-5.80588 -1.25082,-1.77487 -1.99629,-3.31743 -1.98653,-4.51285 0.01,-1.19541 0.77772,-2.04231 2.26634,-2.55795 1.48861,-0.51563 3.69299,-0.70225 6.30277,-0.70252 2.60979,-2.8e-4 5.61414,0.18082 8.55076,0.33079 2.93661,0.14996 5.79293,0.26301 8.13476,0.13957 2.34183,-0.12344 4.15977,-0.4877 5.21683,-1.2017 1.05706,-0.71401 1.35066,-1.77895 0.93336,-3.17067 -0.41732,-1.39173 -1.54032,-3.10783 -3.04742,-5.00049 -1.50709,-1.89267 -3.38726,-3.95682 -5.17511,-5.97853 -1.78786,-2.02172 -3.47088,-3.99525 -4.62112,-5.72388 -1.15024,-1.72863 -1.75858,-3.20816 -1.60117,-4.33571 0.1574,-1.12754 1.08274,-1.9021 2.70857,-2.35466 1.62583,-0.45256 3.94655,-0.58571 6.62966,-0.55227 2.68313,0.0334 5.71745,0.22832 8.63497,0.3695 2.91752,0.14119 5.70578,0.22295 7.94352,0.0517 2.23772,-0.17129 3.91609,-0.5997 4.82459,-1.382 0.90851,-0.78228 1.04537,-1.91927 0.4929,-3.37312 -0.55246,-1.45386 -1.78832,-3.22184 -3.36454,-5.14627 -1.57621,-1.92443 -3.48141,-4.00009 -5.24542,-6.01085 -1.76402,-2.01076 -3.37448,-3.95094 -4.41722,-5.63015 -1.04273,-1.67921 -1.50919,-3.09352 -1.20247,-4.15244 0.30672,-1.05891 1.388,-1.76178 3.14679,-2.15322 1.7588,-0.39145 4.18878,-0.47437 6.93674,-0.41113 2.74795,0.0632 5.80232,0.26734 8.69129,0.3954 2.88897,0.12806 5.60025,0.17444 7.72717,-0.0478 2.12692,-0.22223 3.66123,-0.71686 4.41983,-1.56806 0.75859,-0.8512 0.74046,-2.05944 0.0574,-3.57334 -0.6831,-1.5139 -2.02454,-3.33042 -3.66124,-5.28265 -1.6367,-1.95223 -3.55694,-4.03481 -5.28773,-6.03029 -1.73078,-1.99548 -3.2599,-3.89827 -4.18864,-5.52508 -0.92873,-1.62681 -1.24913,-2.97399 -0.79203,-3.96378 0.45709,-0.98979 1.69226,-1.62192 3.57923,-1.95445 1.88697,-0.33253 4.41874,-0.36866 7.22276,-0.27964 2.80401,0.089 5.86843,0.29772 8.71952,0.40837 2.85108,0.11064 5.47675,0.11767 7.48659,-0.15837 2.00984,-0.27605 3.39623,-0.83867 4.00414,-1.75914 0.6079,-0.92047 0.43716,-2.19885 -0.37154,-3.77047 -0.80868,-1.57164 -2.24801,-3.43315 -3.93629,-5.4091 -1.68829,-1.97595 -3.61356,-4.06083 -5.30184,-6.03678 -1.68829,-1.97595 -3.12762,-3.83745 -3.93631,-5.40909 -0.80871,-1.57163 -0.97946,-2.85002 -0.37154,-3.77048 0.6079,-0.92048 1.99431,-1.48309 4.00415,-1.75914 2.00984,-0.27604 4.63551,-0.26901 7.48659,-0.15836 2.85108,0.11064 5.91549,0.31935 8.71951,0.40836 2.80402,0.089 5.33579,0.0529 7.22276,-0.27964 1.88696,-0.33253 3.12214,-0.96466 3.57924,-1.95445 0.45709,-0.9898 0.13669,-2.33697 -0.79205,-3.96378 -0.92873,-1.62681 -2.45786,-3.5296 -4.18864,-5.52508 -1.73079,-1.99548 -3.65103,-4.07805 -5.28772,-6.03029 -1.6367,-1.95223 -2.97814,-3.76875 -3.66124,-5.28266 -0.68311,-1.51389 -0.70124,-2.72213 0.0574,-3.57333 0.75859,-0.8512 2.2929,-1.34583 4.41983,-1.56806 2.12692,-0.22222 4.83819,-0.17584 7.72716,-0.0478 2.88897,0.12807 5.94335,0.33215 8.6913,0.3954 2.74795,0.0632 5.17794,-0.0197 6.93673,-0.41113 1.75879,-0.39144 2.84008,-1.09431 3.1468,-2.15322 0.30672,-1.05892 -0.15973,-2.47323 -1.20248,-4.15244 -1.04273,-1.67921 -2.65319,-3.61939 -4.41721,-5.63015 -1.76402,-2.01075 -3.66921,-4.08641 -5.24543,-6.01085 -1.57621,-1.92443 -2.81207,-3.69241 -3.36454,-5.14626 -0.55247,-1.45386 -0.4156,-2.59085 0.49291,-3.37313 0.90851,-0.7823 2.58686,-1.21071 4.82459,-1.38201 2.23774,-0.17128 5.026,-0.0895 7.94352,0.0517 2.91752,0.14118 5.95184,0.33605 8.63496,0.3695 2.68312,0.0334 5.00383,-0.0997 6.62966,-0.55228 1.62583,-0.45256 2.55117,-1.22711 2.70858,-2.35466 0.15741,-1.12755 -0.45092,-2.60708 -1.60117,-4.33571 -1.15025,-1.72862 -2.83327,-3.70216 -4.62112,-5.72387 -1.78785,-2.02172 -3.66802,-4.08587 -5.17512,-5.97853 -1.50709,-1.89266 -2.63009,-3.60877 -3.0474,-5.0005 -0.41732,-1.39172 -0.12372,-2.45666 0.93335,-3.17066 1.05707,-0.71401 2.875,-1.07827 5.21683,-1.2017 2.34183,-0.12344 5.19814,-0.0104 8.13476,0.13957 2.93662,0.14996 5.94098,0.33106 8.55075,0.33078 2.60979,-2.6e-4 4.81417,-0.18688 6.30278,-0.70252 1.48862,-0.51564 2.25658,-1.36253 2.26634,-2.55794 0.01,-1.19542 -0.73571,-2.73799 -1.98653,-4.51285 -1.25083,-1.77486 -2.99734,-3.77758 -4.79953,-5.80588 -1.80219,-2.02831 -3.64747,-4.07642 -5.07708,-5.93347 -1.42961,-1.85704 -2.43293,-3.51814 -2.71112,-4.84591 -0.27818,-1.32777 0.17329,-2.32015 1.37694,-2.96677 1.20366,-0.64662 3.15615,-0.94903 5.59494,-1.0279 2.43879,-0.0789 5.35397,0.0612 8.30015,0.2156 2.94618,0.15435 5.91076,0.31718 8.439,0.27942 2.52822,-0.0378 4.6097,-0.28087 5.9574,-0.86128 1.3477,-0.58042 1.9575,-1.50002 1.82186,-2.76226 -0.13563,-1.26225 -1.01294,-2.86542 -2.35701,-4.68315 -1.34407,-1.81771 -3.14474,-3.84533 -4.95172,-5.87583"
+ style="color:#000000;fill:none;stroke:#ff6565;stroke-width:6;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:6.4000001;stroke-opacity:1;stroke-dasharray:none;stroke-dashoffset:0;marker:none;visibility:visible;display:inline;overflow:visible;enable-background:accumulate" />
+ </g>
+</svg>
diff --git a/doc/papers/2011/europar/Makefile b/doc/papers/2011/europar/Makefile
index 66589dca4b5b68e536c593fd4e84d14d9f0e1c55..a2bc0cd2ee374ba62d575a1fe1f55e0768c8df65 100644
--- a/doc/papers/2011/europar/Makefile
+++ b/doc/papers/2011/europar/Makefile
@@ -2,7 +2,7 @@ TEX_SOURCES = lofar.tex
BIB_SOURCES = lofar.bib
-FIG_SOURCES = delay.fig lofar-stations.fig processing.fig
+FIG_SOURCES = lofar-stations.fig processing.fig
JGR_SOURCES = stations-beams.jgr execution_times.jgr coherent-dedispersion.jgr dispersed-signal.jgr
@@ -14,7 +14,7 @@ PNG_SOURCES =
STY_SOURCES =
-SVG_SOURCES = pencilbeams.svg
+SVG_SOURCES = pencilbeams.svg LBAfield-delay.svg
AUX_FILES = $(TEX_SOURCES:%.tex=%.aux)
GEN_FIGURES = $(FIG_SOURCES:%.fig=%.pdf) $(JGR_SOURCES:%.jgr=%.pdf) $(SVG_SOURCES:%.svg=%.pdf)
diff --git a/doc/papers/2011/europar/execution_times.jgr b/doc/papers/2011/europar/execution_times.jgr
index bb1dd5592a7bfd090f6729c2de83a29b740fa563..e49eb0568cb9401585c556a53a9e525218dce75f 100644
--- a/doc/papers/2011/europar/execution_times.jgr
+++ b/doc/papers/2011/europar/execution_times.jgr
@@ -49,35 +49,35 @@ newgraph
newcurve
(* output core pure I/O *)
- marktype xbar marksize 0.8 fill 0 pattern solid
+ marktype xbar marksize 0.8 fill 0 pattern solid cfill 1 0 0
pts
shell : awk -v N=7 '/->/ {a+=$3;b+=$5;c+=$7;d+=$9;e+=$11;f+=$13;n++; if(n==N) print "1 " a " 2 " b " 3 " c " 4 " d " 5 " e " 6 " f;}' <execution_times.jgr
label : Output to I/O node
newcurve
(* output reorder *)
- marktype xbar marksize 0.8 fill 0.8 pattern solid
+ marktype xbar marksize 0.8 fill 0.8 pattern solid cfill 1 1 0
pts
shell : awk -v N=6 '/->/ {a+=$3;b+=$5;c+=$7;d+=$9;e+=$11;f+=$13;n++; if(n==N) print "1 " a " 2 " b " 3 " c " 4 " d " 5 " e " 6 " f;}' <execution_times.jgr
label : Output reordering
newcurve
(* 2nd xpose *)
- marktype xbar marksize 0.8 fill 1 pattern solid
+ marktype xbar marksize 0.8 fill 1 pattern solid cfill 0 0.7 0
pts
shell : awk -v N=5 '/->/ {a+=$3;b+=$5;c+=$7;d+=$9;e+=$11;f+=$13;n++; if(n==N) print "1 " a " 2 " b " 3 " c " 4 " d " 5 " e " 6 " f;}' <execution_times.jgr
label : 2nd all-to-all exchange
newcurve
(* stokes calculation *)
- marktype xbar marksize 0.8 pattern stripe 45
+ marktype xbar marksize 0.8 pattern stripe 45 cfill 0 1 1
pts
shell : awk -v N=4 '/->/ {a+=$3;b+=$5;c+=$7;d+=$9;e+=$11;f+=$13;n++; if(n==N) print "1 " a " 2 " b " 3 " c " 4 " d " 5 " e " 6 " f;}' <execution_times.jgr
label : Stokes calculations
newcurve
(* coh dd *)
- marktype xbar marksize 0.8 fill 0.2 pattern solid
+ marktype xbar marksize 0.8 fill 0.2 pattern solid cfill 1 0 1
pts
shell : awk -v N=3 '/->/ {a+=$3;b+=$5;c+=$7;d+=$9;e+=$11;f+=$13;n++; if(n==N) print "1 " a " 2 " b " 3 " c " 4 " d " 5 " e " 6 " f;}' <execution_times.jgr
label : Channel dedispersion \
@@ -85,14 +85,14 @@ newgraph
newcurve
(* beam forming *)
- marktype xbar marksize 0.8 fill 0.5 pattern solid
+ marktype xbar marksize 0.8 fill 0.5 pattern solid cfill 0 0 1
pts
shell : awk -v N=2 '/->/ {a+=$3;b+=$5;c+=$7;d+=$9;e+=$11;f+=$13;n++; if(n==N) print "1 " a " 2 " b " 3 " c " 4 " d " 5 " e " 6 " f;}' <execution_times.jgr
label : Beam forming
newcurve
(* input core I/O + prebf dsp *)
- marktype xbar marksize 0.8 fill 0.5 pattern stripe -45
+ marktype xbar marksize 0.8 fill 0.5 pattern stripe -45 cfill 0 0.7 0
pts
shell : awk -v N=1 '/->/ {a+=$3;b+=$5;c+=$7;d+=$9;e+=$11;f+=$13;n++; if(n==N) print "1 " a " 2 " b " 3 " c " 4 " d " 5 " e " 6 " f;}' <execution_times.jgr
label : 1st all-to-all exchange & \
diff --git a/doc/papers/2011/europar/lofar.pdf b/doc/papers/2011/europar/lofar.pdf
index 004c6f15d4b2b0a307920a943ae0fe6229e5a0a6..b2379bf1ad852c6456b5afc52ac65c1e3a53106d 100644
Binary files a/doc/papers/2011/europar/lofar.pdf and b/doc/papers/2011/europar/lofar.pdf differ
diff --git a/doc/papers/2011/europar/lofar.tex b/doc/papers/2011/europar/lofar.tex
index c690ba9c664d9fd1e8e85f6f707133bcbd8ac618..19d21824434a21c87bc5f2a15c25664d3c49519e 100644
--- a/doc/papers/2011/europar/lofar.tex
+++ b/doc/papers/2011/europar/lofar.tex
@@ -49,44 +49,28 @@ software correlator benefits:
}
-The LOFAR (LOw Frequency ARray) telescope is the first of a new generation of radio telescopes. Instead of using a set of large, expensive dishes, LOFAR uses many thousands of simple antennas. Every antenna observes the full sky, and the telescope can be aimed through signal processing techniques. LOFAR's novel design allows the telescope to perform wide-angle observations as well as to observe in multiple directions simultaneously, neither of which are possible when using traditional dishes. In several ways, LOFAR will be the largest telescope in the world, and will enable ground-breaking research in several areas of astronomy and particle physics~\cite{Bruyn:02}.
+The LOFAR (LOw Frequency ARray) telescope is the first of a new generation of radio telescopes. Instead of using a set of large, expensive dishes, LOFAR uses many thousands of simple antennas. Every antenna observes the full sky, and the telescope is aimed through signal-processing techniques. LOFAR's novel design allows the telescope to perform wide-angle observations as well as to observe in multiple directions simultaneously, neither of which are possible when using traditional dishes. In several ways, LOFAR will be the largest telescope in the world, and will enable ground-breaking research in several areas of astronomy and particle physics~\cite{Bruyn:02}.
-Another novelty is the elaborate use of software to process the telescope data in real time. Previous generations of telescopes depended on custom-made hardware to combine data, because of the high data rates and processing requirements. The availability of sufficiently powerful supercomputers however, allow the use of software to combine telescope data, creating a more flexible and reconfigurable instrument.
+Another novelty is the elaborate use of software to process the telescope data in real time. Previous generations of telescopes depended on custom-made hardware to combine data, because of the high data rates and processing requirements. The availability of sufficiently powerful supercomputers however, allow the use of software to combine telescope data, creating a more flexible and reconfigurable instrument. Because LOFAR is driven by new science, flexibility in the design is essential in order to explore the possibilities and limits of our telescope.
-For processing LOFAR data, we use an IBM BlueGene/P (BG/P) supercomputer. The LOFAR antennas are grouped into stations, and each station sends its data (up to 198 Gb/s for all stations) to the BG/P super computer. Inside the BG/P, the data are split and recombined using both real-time signal processing routines as well as two all-to-all exchanges. The output data streams are sufficiently reduced in size in order to be able to stream them out of the BG/P and store them on disks in our storage cluster.
-
-The stations can be configured to observe in several directions in parallel, but have to divide their output bandwidth among them. In this paper, we present the \emph{beam former}, an extension to the LOFAR software which allows the telescope to be aimed in tens of directions simultaneously at LOFAR's full observational bandwidth, and in hundreds of directions at reduced bandwidth. Both feats cannot be matched by any other telescope. The data streams corresponding to each observational direction, called \emph{tied-array beams}, are generated through (weighted) summations of the station inputs, which are demultiplexed using an all-to-all exchange, and routed to the storage cluster.
-
-The primary scientific use case driving the work presented in this paper is pulsar research. A pulsar is a rapidly rotating, highly magnetised neutron star, which emits electromagnetic radiation from its poles. Similar to the behaviour of a lighthouse, the radiation is visible to us only if one of the poles points towards Earth, and subsequently appears to us as a very regular series of pulses, with a period as low as 1.4~ms~\cite{Hessels:06}. Pulsars are weak radio sources, and their individual pulses often do not rise above the background noise that fills our universe. LOFAR is one of the few telescopes which operates in the frequency range (10 -- 240 MHz) in which pulsars are typically at their brightest. Our beam former also makes LOFAR the only telescope that can observe in hundreds of directions simultaneously with high sensitivity. These aspects make LOFAR an ideal instrument to discover unknown pulsars by doing a sensitive sky survey in a short amount of time, as well as an ideal instrument to study known pulsars in more detail. Apart from pulsar research, our beam former can be used to focus on planets, exoplanets, the sun, and other radio objects, with unprecedented sensitivity. Furthermore, our pipeline allows fast broad-sky surveys to discover not only new pulsars but also other radio sources.
-
-In this paper, we will show how a software solution and the use of a massively parallel machine allows us to achieve this feat. We provide an in-depth study on all performance aspects, real-time behaviour, and scaling characteristics.
-
-\section{Related Work}
-
-Traditional radio dishes are unsuitable for beam forming due to their narrow field-of-view. A radio dish can be extended to focus on multiple sources by deploying multiple receivers in its focal point (a \emph{focal plane array})~\cite{Staveley-Smith:96}, a solution which does not scale. The Murchison Widefield Array (MWA) uses a design similar to LOFAR (omnidirectional antennas), and plans to build a beam former, but is still under construction~\cite{Lonsdale:09}. The LOFAR beam former is thus the only beam former capable of producing hundreds of tied-array beams.
-
-This paper builds upon the design of the imaging pipeline~\cite{Romein:10a}, further showing the flexibility and the power of a software telescope. Although many parameters in our imaging pipeline are platform-specific, a comparison across several hardware platforms~\cite{Nieuwpoort:09} showed that the issues faced are platform agnostic.
-
-\section{LOFAR}
-\label{Sec:LOFAR}
+For processing LOFAR data, we use an IBM BlueGene/P (BG/P) supercomputer. The LOFAR antennas are grouped into stations, and each station sends its data (up to 198 Gb/s for all stations) to the BG/P. Inside the BG/P, the data are split and combined using both real-time signal-processing routines as well as two all-to-all exchanges. The output data streams are sufficiently reduced in size in order to be able to stream them out of the BG/P and store them on disks in our storage cluster.
\begin{figure}[ht]
-\comment{
-\subfigure[A field with low-band antennas (dipoles).]{
+\subfigure[The left antenna receives the wave later.]{
\makebox[35mm][c]{
- \includegraphics[width=0.27\textwidth]{LBA-field.jpg}
- \label{fig:lbafield}
+ \includegraphics[width=0.27\textwidth]{LBAfield-delay.pdf}
+ \label{fig:delay}
}
}
\hfill
-}
\subfigure[Locations of the stations.]{
- \makebox[37mm][c]{
+ \makebox[35mm][c]{
\includegraphics[width=0.35\textwidth]{lofar-stations.pdf}
\label{fig:map}
}
}
\hfill
+\comment{
\subfigure[The left antenna receives the wave later.]{
\makebox[28mm][c]{
\includegraphics[width=0.20\textwidth]{delay.pdf}
@@ -94,8 +78,9 @@ This paper builds upon the design of the imaging pipeline~\cite{Romein:10a}, fur
}
}
\hfill
+}
\subfigure[Tied-array beams (hexagons) formed within two station beams (ellipse).]{
- \makebox[50mm][c]{
+ \makebox[40mm][c]{
\includegraphics[width=0.3\textwidth]{pencilbeams.pdf}
\label{fig:pencilbeams}
}
@@ -103,23 +88,22 @@ This paper builds upon the design of the imaging pipeline~\cite{Romein:10a}, fur
\caption{LOFAR antennas}
\end{figure}
-The LOFAR telescope consists of many thousands of simple dipole antennas (see Figure \ref{fig:lbafield}), grouped in \emph{stations}. The stations are strategically placed, with 20 stations acting as its centre (the \emph{core}) and 24 stations at increasing distances from that core (see Figure \ref{fig:map}). A core station can act as two individual stations in some observational modes. Every station collects and combines the signals from its antennas, and sends the resulting data stream to our IBM BlueGene/P (BG/P) supercomputer at our central processing facility. The BG/P combines the data streams from one or more stations and reduces the resulting stream in size sufficiently to be able to store it on disks in our storage cluster. Both the stations and the BG/P perform hard-real-time signal processing. Once the data has been stored on disk, off-line processing takes over. The off-line processing transforms and further reduces the data produced by the BG/P into data products such as images or time series, and are made available to the astronomer(s) that requested the observation.
+In this paper, we will present the LOFAR \emph{beam former}: a collection of software pipelines that allow the LOFAR telescope to be aimed at hundreds of directions simultaneously. This feat is made possible by LOFAR's unique design and the resources offered by the BG/P. Traditional radio telescopes are aimed by focussing their dishes on a source, resulting in a sharp but narrow field-of-view per dish. The LOFAR antennas are omnidirectional, and do not have to be moved or rotated. A station is focussed on a source by taking advantage of the fact that the speed of electromagnetic waves is finite, causing an electromagnetic wave to arrive at different antennas at different times (see Figure \ref{fig:delay}). A process called \emph{delay compensation} counters the differences in arrival times by delaying the signals from different antennas such that they are synchronised with respect to the observed source. The synchronised signals from all antennas are accumulated (by the station \emph{beam former}) and sent to the BG/P. The resulting \emph{station beam} has a wide field-of-view around the source. It contains samples for both the X and the Y polarisations as 16-bit complex integers, resulting in 3.1~Gb/s of data per station.
-The antennas are omnidirectional and have no moving parts. Instead, all telescope functions are performed electronically through signal processing done at the stations and on the BG/P. The telescope can be aimed because the speed of light is finite: the light emitted by a source will arrive at different antennas at different times (see Figure \ref{fig:delay}). By adding appropriate delays to the signals from individual antennas before accumulating them, the signals from the source will be amplified with respect to signals from other directions. Once the samples from all antennas are combined, the data are transmitted to the BG/P, which uses the same technique to combine the data from the individual stations. The latter will be explained further in Section \ref{Sec:Beamforming}.
+The BG/P, which receives the signals from all stations, again performs delay compensation and beam forming, this time in software. The BG/P beam former can focus on sources anywhere in the fields of view of the station beams, creating \emph{tied-array beams} (beams). An example is shown in Figure \ref{fig:pencilbeams}, in which station beams (represented by an ellipse) contains several tied-array beams (represented by hexagons). The actual width of the station beams, as well as the width of the tied-array beams, depends on the number as well as the locations of the stations used. Hundreds of tied-array beams are typically needed to fully cover the field of view of a station beam. Different tied-array beams are created by adding the signals from the individual stations using different delays, which depend on the relative positions of the stations and the relative direction of the tied-array beam with respect to the station beam. The BG/P applies delay compensation in two steps. First, coarse-grain compensation is performed by shifting the samples from different stations with respect to each other. Then, for each tied-array beam, the remaining sub-sample delays are compensated for by shifting the phases of the signals. The phase of each sample is changed through a complex multiplication with a precomputed weight. Tied-array beams are thus a linear combination of the (shifted) signals from the stations.
-A LOFAR station is able to produce 248 frequency subbands of 195~kHz out of the sensitivity range of 80~MHz to 250~MHz. Each sample consists of two complex 16-bit integers, representing the amplitude and phase of the X and Y polarisations of the antennas. The resulting data stream from a station is a 3.1~Gb/s UDP stream.
+Our beam former supports several pipelines: \emph{XY polarisations}, \emph{Stokes IQUV}, and \emph{Stokes I}. The XY polarisations pipeline outputs the raw tied-array beams, which consist of two 3.1~Gb/s streams of 32-bit complex floating points numbers (floats), one stream for each polarisation. The Stokes IQUV pipeline applies a domain transformation to each sample of the raw tied-array beams, which is useful for polarisation-related studies. The four Stokes parameters, calculated through $I = X\overline{X} + Y\overline{Y}$, $Q = X\overline{X} - Y\overline{Y}$, $U = 2\mathrm{Re}(X\overline{Y})$, $V = 2\mathrm{Im}(X\overline{Y})$, with each parameter being a 32-bit float, resulting in four 1.5~Gb/s streams. The Stokes I pipeline calculates only the first Stokes parameter, which represents the signal strength in both polarisations. The Stokes I pipeline supports temporal integration to trade time resolution for a reduced bandwidth per beam, allowing more beams to be created.
+
+Finally, our software can produce the Stokes parameters (I or IQUV) of an \emph{incoherent} beam, which is an accumulation of the uncompensated station signals. The incoherent beam is less sensitive than a tied-array beam, but it maintains the wide field-of-view of the stations. The incoherent beam is used to detect the presence of sources, but does not reveal their location within the station beams.
+
+The primary scientific use case driving the work presented in this paper is pulsar research. A pulsar is a rapidly rotating, highly magnetised neutron star, which emits electromagnetic radiation from its poles. Similar to the behaviour of a lighthouse, the radiation is visible to us only if one of the poles points towards Earth, and subsequently appears to us as a very regular series of pulses, with a period as low as 1.4~ms~\cite{Hessels:06}. Pulsars are weak radio sources, and their individual pulses often do not rise above the background noise that fills our universe. Our beam former can focus on several pulsars at LOFAR's full observational bandwidth, producing either XY polarisation or Stokes IQUV data. Alternatively, the beam former is capable of efficiently performing sky surveys to discover new pulsars (or other radio sources) by covering the sky with hundreds of tied-array beams.
+
+% The delays are applied in two phases. First, the streams are aligned by shifting them a whole number of samples with respect to each other, which resolves delay differences up to the granularity of a single sample. Then, the remaining sub-sample delays are compensated for by shifting the phase of the signal. In order to obtain different tied-array beams, only the sub-sample delays have to be adjusted. A phase shift is performed by applying a complex multiplication. To form a beam, the beam former gathers the streams of samples from the stations, multiplies them with precomputed weights representing the required phase shift, and adds the streams together. The same weights are applied to both the X and the Y polarisations. The resulting data stream is called the \emph{XY polarisations}, and consists of 32-bit complex floating point numbers (complex floats).
-\comment{
- Hardware:
- - stations and antennas
- - network
-
- Processing in general:
- - station processing
- - how aiming works
- - online processing
- - offline processing
-}
+
+%The stations can be configured to observe in several directions in parallel, but have to divide their output bandwidth among them. In this paper, we present the \emph{beam former}, which allows the telescope to be aimed in tens of directions simultaneously at LOFAR's full observational bandwidth, and in hundreds of directions at reduced bandwidth. Both feats cannot be matched by any other telescope. The data streams corresponding to each observational direction, called \emph{tied-array beams}, are generated through (weighted) summations of the station inputs, which are demultiplexed using an all-to-all exchange, and routed to the storage cluster.
+
+In this paper, we will show how a software solution and the use of a massively parallel machine allows us to achieve these feats. We provide an in-depth study on all performance aspects, real-time behaviour, and scaling characteristics.
\section{IBM BlueGene/P}
@@ -134,53 +118,27 @@ The BG/P contains several networks. A fast \emph{3-dimensional torus\/} connects
\subsection{External I/O}
\label{Sec:Networks}
-We customised the I/O node software stack~\cite{Yoshii:10} and run a multi-threaded program on each I/O~node which is responsible for the handling of both the input and the output. Even though the I/O nodes each have a 10~Gb/s Ethernet interface, they do not have enough computation power to handle 10~Gb/s of data. The overhead of handling IRQs, IP, and UDP/TCP put a high load on the 850~MHz cores of the I/O nodes, limiting performance. An I/O node can output at most 3.1~Gb/s, unless it has to handle station input (3.1~Gb/s), in which case it can output at most 1.1~Gb/s. We implemented a low-overhead communication protocol called FCNP~\cite{Romein:09a} to efficiently transport data to and from the compute nodes. Each I/O node forwards its data to the compute nodes, which perform all of the necessary processing. Once the compute nodes have finished processing the data, the results are sent back to the I/O nodes. The I/O nodes forward these results to our storage cluster, which can sustain a throughput up to 80~Gb/s. The I/O node drops output data if it cannot be sent, in order to keep the system running at real time.
-
-\comment{
- BG/P explanation:
- - basic architecture and size of our installation
- - IO nodes and compute nodes
- - internal networks
- - external networks and storage nodes
-}
-
-\section{Beam Forming}
-\label{Sec:Beamforming}
-
-A station focusses at a source by applying different delays to the signals from its antennas, and subsequently adding the delayed signals. Delaying the signals is known as \emph{delay compensation}, and subsequently adding them as \emph{beam forming}. The station beam former is implemented in hardware, in which the signal is delayed by switching it over wires of different lengths. The signals from the different antennas are subsequently added using an FPGA. The resulting \emph{station beam}, even though it is focussed on a certain source, still has a wide field of view.
-
-The BG/P, which receives the signals from all stations, again performs delay compensation and beam forming, this time in software. The BG/P beam former can focus on sources anywhere in the fields of view of the station beams, creating \emph{tied-array beams} (beams). An example is shown in Figure \ref{fig:pencilbeams}, in which a station beam (represented by an ellipse) contains several tied-array beams (represented by hexagons). The actual width of the station beam, as well as the width of the tied-array beams, depends on the number as well as the locations of the stations used. Hundreds of tied-array beams are typically needed to fully cover the field of view of a station beam.
-
-Different tied-array beams are created by adding the signals from the individual stations using different delays. The delays that have to be applied to obtain a tied-array beam depends on the relative positions of the stations and the relative direction of the tied-array beam with respect to the station beam. The delays are applied in two phases. First, the streams are aligned by shifting them a whole number of samples with respect to each other, which resolves delay differences up to the granularity of a single sample. Then, the remaining sub-sample delays are compensated for by shifting the phase of the signal. In order to obtain different tied-array beams, only the sub-sample delays have to be adjusted. A phase shift is performed by applying a complex multiplication. To form a beam, the beam former gathers the streams of samples from the stations, multiplies them with precomputed weights representing the required phase shift, and adds the streams together. The same weights are applied to both the X and the Y polarisations. The resulting data stream is called the \emph{XY polarisations}, and consists of 32-bit complex floating point numbers (complex floats).
-
-The XY polarisations can optionally be converted into \emph{Stokes IQUV} parameters, which represent the polarisation aspects in an alternative way. The Stokes parameters are defined as $I = X\overline{X} + Y\overline{Y}$, $Q = X\overline{X} - Y\overline{Y}$, $U = 2\mathrm{Re}(X\overline{Y})$, $V = 2\mathrm{Im}(X\overline{Y})$, with each parameter being a 32-bit float.
+We customised the I/O node software stack~\cite{Yoshii:10} and run a multi-threaded program on each I/O~node which is responsible for the handling of both the input and the output. Even though the I/O nodes each have a 10~Gb/s Ethernet interface, they do not have enough computation power to handle 10~Gb/s of data. The overhead of handling IRQs, IP, and UDP/TCP put a high load on the 850~MHz cores of the I/O nodes, limiting performance. An I/O node can output at most 3.1~Gb/s, unless it has to handle station input (3.1~Gb/s), in which case it can output at most 1.1~Gb/s. We implemented a low-overhead communication protocol called FCNP~\cite{Romein:09a} to efficiently transport data to and from the compute nodes, which perform the required signal processing. The I/O nodes forward the results to our storage cluster, which can sustain a throughput up to 80~Gb/s.
-Both the XY polarisations and the Stokes IQUV parameters require up to 6.2~Gb/s per beam, which severely limits the number of beams that can be produced, and which represents a time resolution which is not always necessary. For example, in sky surveys, it is desirable to create many beams. The data rate per beam thus has to be lowered. For many-beam observations, we convert the XY polarisations into just the \emph{Stokes I} parameter, which represents the amplitude of the signal in the X and Y polarisations combined. The resulting data rate is 1.5~Gb/s per beam, but we also allow temporal integration to further reduce the data rate with an integer factor, allowing even more beams to be created. For each beam, each polarisation or Stokes parameter is stored in a separate file. If too many I/O nodes are limited to 1.1~Gb/s of output, full polarisation or Stokes parameter streams are too wide to transport. In such cases, we split the streams into \emph{slices} of 83 or 124 subbands per substream instead of 248.
-
-Finally, our software can produce the Stokes parameters (I or IQUV) of an \emph{incoherent} beam, which is an accumulation of the uncompensated station signals. The incoherent beam is less sensitive than a tied-array beam, but it maintains the wide field-of-view of the stations. The incoherent beam is used to detect the presence of sources, but does not reveal their location within the station beams.
-
-% TODO: incoherent stokes
-\section{Pulsar Pipeline}
-
-%To observe known pulsars, our beam former is put in the high-resolution mode, in which Complex Voltages or Stokes IQUV parameters are recorded at full bandwidth in order to closely study the shapes of the individual pulses.
+\section{Beam Former Pipelines}
In this section, we will describe in detail how the full signal-processing pipeline operates, in and around the beam former. The use of a software pipeline allows us to reconfigure the components and design of our standard imaging pipeline, described in \cite{Romein:10a}. Both pipelines can be run simultaneously. Figure \ref{fig:processing} gives an overview of our system.
\begin{figure}[ht]
\center
\includegraphics[width=0.8\textwidth]{processing.pdf}
-\caption{Data flow diagram describing three pipelines.}
+\caption{Data flow diagram describing the on-line pipelines of LOFAR. The imaging and UHEP modes are outside the scope of this work.}
\label{fig:processing}
\end{figure}
\subsection{Input from Stations}
-The first step in the pipeline is receiving and collecting from the stations on the I/O nodes. Each I/O node receives the data of (at most) one station, and stores the received data in a circular buffer. If necessary, the read pointer of the circular buffer is shifted a number of samples to reflect the coarse-grain delay compensation that will be necessary to align the streams from different stations.
+The BG/P receives data from up to 64 stations. The stations are strategically placed, with 20 stations acting as its centre (the \emph{core}) and 24 stations at increasing distances from the core (see Figure \ref{fig:map}). A core station can act as two individual stations in some observational modes. A station is able to produce 248 frequency subbands of 195~kHz out of the sensitivity range of 10~MHz to 250~MHz. Each sample consists of two complex 16-bit integers, representing the amplitude and phase of the X and Y polarisations of the antennas. The resulting data stream from a station is a 3.1~Gb/s UDP stream, which is sent to an I/O node in our BG/P.
-The station data are split into chunks of one subband and 0.25 seconds. The chunk size is chosen such that the compute cores have enough memory to perform all of the necessary processing. Due to the BG/P design, an I/O node sends chunks to its own compute cores only. The compute cores exchange the chunks they obtain from their I/O node using an all-to-all exchange.
+At the I/O nodes, the station data are split into chunks of one subband and 0.25 seconds. The chunk size is chosen such that the compute cores have enough memory to perform all of the necessary processing. Due to the BG/P design, an I/O node sends chunks to its own compute cores only. The compute cores exchange the chunks they obtain from their I/O node using an all-to-all exchange.
\subsection{First All-to-all Exchange}
-The first all-to-all exchange in the pipeline allows the compute cores to distribute the chunks from a single station, and to collect all the chunks of the same subband from all of the stations. The exchange is performed over the fast 3D-torus network, but with up to 198~Gb/s of station data to be exchanged (64 stations producing 3.1~Gb/s), special care still has to be taken to avoid network bottlenecks. It is impossible to optimise for short network paths due to the physical distances between the different psets across a BG/P rack. Instead, we optimised the data exchange by creating as many paths as possible between compute cores that have to exchange data. Within each pset, we employ a virtual remapping such that the number of possible routes between communicating cores in different psets is maximised.
+The first all-to-all exchange in the pipeline allows the compute cores to distribute the chunks from a single station, and to collect all the chunks of the same subband from all of the stations. The exchange is performed over the fast 3D-torus network, but with up to 198~Gb/s of station data to be exchanged, special care still has to be taken to avoid network bottlenecks. It is impossible to optimise for short network paths due to the physical distances between the different psets across a BG/P rack. Instead, we optimised the data exchange by creating as many paths as possible between compute cores that have to exchange data. Within each pset, we employ a virtual remapping such that the number of possible routes between communicating cores in different psets is maximised.
The communications in the all-to-all exchange are asynchronous, which allows a compute core to start processing a subband from a station as soon as it arrives, up to the point that data from all stations are required. Communication and computation are thus overlapped as much as possible.
@@ -200,7 +158,7 @@ Up to this point, processing chunks from different stations can be done independ
\subsection{Beam Forming}
-The beam former creates the beams as described in Section \ref{Sec:Beamforming}. First, the different weights required for the different beams are computed, based on the station positions and the beam directions. Note that the data in the chunks are already delay compensated with respect to the source at which the stations are pointed. Any delay compensation performed by the beam former is therefore to compensate the delay differences between the desired beams and the station's source. The reason for this two-stage approach is flexibility. By already compensating for the station's source in the previous step, the resulting data can not only be fed to the beam former, but also to other pipelines, such as the imaging pipeline. Because we have a software pipeline, we can implement and connect different processing pipelines with only a small increase in complexity.
+First, the different weights required for the different tied-array beams are computed, based on the station positions and the beam directions. Note that the data in the chunks are already delay compensated with respect to the source at which the stations are pointed. Any delay compensation performed by the beam former is therefore to compensate the delay differences between the desired beams and the station's source. The reason for this two-stage approach is flexibility. By already compensating for the station's source in the previous step, the resulting data can not only be fed to the beam former pipelines, but also to other pipelines, such as the imaging pipeline. Because we have a software pipeline, we can implement and connect different processing pipelines with only a small increase in complexity.
The delays are applied to the station data through complex multiplications and additions, programmed in assembly. In order to take full advantage of the L1 cache and the available registers, data is processed in sets of 6 stations, producing 3 beams, or a subset thereof to cover the remaining stations and beams. While the exact ideal set size in which the data is to be processed is platform specific, we have shown in previous work that similar tradeoffs exist for similar problems across different architectures~\cite{Nieuwpoort:09}.
@@ -238,7 +196,9 @@ Figure~\ref{fig:dedispersion-result} shows the observed effectiveness of channel
In the second all-to-all exchange, the chunks made by the beam former are again exchanged over the 3D-torus network. Due to memory constrains on the compute cores, the cores that performed the beam forming cannot be the same cores that receive the beam data after the exchange. We assign a set of cores (\emph{output cores}) to receive the chunks. The output cores are chosen before an observation, and are distinct from the \emph{input cores} which perform the earlier computations in the pipeline.
-An output core gathers the chunks that contain different subbands but belong to the same slice (see Section \ref{Sec:Beamforming}). Then, it rearranges the dimensions of the data into their final ordering, which is necessary, because the data order that will be written to disk is not the same order that can be produced by our computations without taking heavy L1 cache penalties. We hide this reordering cost at the output cores by overlapping computation (the reordering of a chunk) with communication (the arrival of other chunks). Once all of the chunks are received and reordered, they are sent back to the I/O node.
+An output core gathers the chunks that contain different subbands but belong to the same output stream. An output stream consists of all 248 subbands belonging to the same polarisation or Stokes parameter. If the full 248 subbands cannot be exported by the I/O node due to data rate limitations, the polarisation or Stokes parameter is split into multiple
+
+Then, it rearranges the dimensions of the data into their final ordering, which is necessary, because the data order that will be written to disk is not the same order that can be produced by our computations without taking heavy L1 cache penalties. We hide this reordering cost at the output cores by overlapping computation (the reordering of a chunk) with communication (the arrival of other chunks). Once all of the chunks are received and reordered, they are sent back to the I/O node.
For the distribution of the workload over the available output cores, three factors have to be considered. First, all of the data belonging to the same beam has to be processed by output cores in the same pset, to ensure that one I/O node can concatenate all of the 0.25 second chunks that belong to the beam. Second, the maximum output rate per I/O node has to be respected. Finally, the presence of the first all-to-all exchange, which uses the same network at up to 198~Gb/s. The second exchange uses up to 80~Gb/s. Even though each link sustains 3.4~Gb/s, it has to process the traffic from four cores, as well as traffic routed through it between other nodes. The network links in the BG/P become overloaded unless enough output cores are used to spread the load.
@@ -338,6 +298,12 @@ For the I/O-bound cases D-F, only a few tied-array beams are formed and transfor
- network bw use in 2nd transpose
}
+\section{Related Work}
+
+The LOFAR beam former is the only beam former capable of producing hundreds of tied-array beams. Traditional radio dishes are unsuitable for beam forming due to their narrow field-of-view. A radio dish can be extended to focus on multiple sources by adding additional receivers to its focal point (a \emph{focal plane array})~\cite{Staveley-Smith:96}, but such a solution does not scale. The Murchison Widefield Array (MWA) uses a design similar to LOFAR, and plans to build a beam former, but is still under construction~\cite{Lonsdale:09}.
+
+%This paper builds upon the design of the imaging pipeline~\cite{Romein:10a}, further showing the flexibility and the power of a software telescope. Although many parameters in our imaging pipeline are platform-specific, a comparison across several hardware platforms~\cite{Nieuwpoort:09} showed that the issues faced are platform agnostic.
+
\section{Discussion}
\comment{
diff --git a/doc/papers/2011/europar/pencilbeams.svg b/doc/papers/2011/europar/pencilbeams.svg
index 0d5fc9a49f200fc29d1a87241571495a8f405655..f1b163fae40e44a11bce70f969a078793340ba78 100644
--- a/doc/papers/2011/europar/pencilbeams.svg
+++ b/doc/papers/2011/europar/pencilbeams.svg
@@ -18,7 +18,7 @@
viewBox="17009 5954 17260.779 14290.346"
id="svg2"
version="1.1"
- inkscape:version="0.48.0 r9654"
+ inkscape:version="0.47pre4 r22446"
sodipodi:docname="pencilbeams.svg">
<metadata
id="metadata54">
@@ -41,6 +41,27 @@
inkscape:vp_z="1134 : 589.5 : 1"
inkscape:persp3d-origin="567 : 393 : 1"
id="perspective56" />
+ <inkscape:perspective
+ id="perspective6468"
+ inkscape:persp3d-origin="0.5 : 0.33333333 : 1"
+ inkscape:vp_z="1 : 0.5 : 1"
+ inkscape:vp_y="0 : 1000 : 0"
+ inkscape:vp_x="0 : 0.5 : 1"
+ sodipodi:type="inkscape:persp3d" />
+ <inkscape:perspective
+ id="perspective7002"
+ inkscape:persp3d-origin="0.5 : 0.33333333 : 1"
+ inkscape:vp_z="1 : 0.5 : 1"
+ inkscape:vp_y="0 : 1000 : 0"
+ inkscape:vp_x="0 : 0.5 : 1"
+ sodipodi:type="inkscape:persp3d" />
+ <inkscape:perspective
+ id="perspective7031"
+ inkscape:persp3d-origin="0.5 : 0.33333333 : 1"
+ inkscape:vp_z="1 : 0.5 : 1"
+ inkscape:vp_y="0 : 1000 : 0"
+ inkscape:vp_x="0 : 0.5 : 1"
+ sodipodi:type="inkscape:persp3d" />
</defs>
<sodipodi:namedview
pagecolor="#ffffff"
@@ -51,15 +72,15 @@
guidetolerance="10"
inkscape:pageopacity="0"
inkscape:pageshadow="2"
- inkscape:window-width="1022"
- inkscape:window-height="829"
+ inkscape:window-width="1048"
+ inkscape:window-height="828"
id="namedview50"
showgrid="false"
- inkscape:zoom="0.35355339"
- inkscape:cx="746.49403"
- inkscape:cy="466.33141"
+ inkscape:zoom="1.4142136"
+ inkscape:cx="197.77917"
+ inkscape:cy="126.92015"
inkscape:window-x="0"
- inkscape:window-y="397"
+ inkscape:window-y="0"
inkscape:window-maximized="0"
inkscape:current-layer="svg2"
fit-margin-top="0"
@@ -190,24 +211,37 @@
style="fill:#ffe0e0;stroke:#ff0000;stroke-width:95;stroke-linecap:butt;stroke-linejoin:miter"
points="29595,11130 30171,11220 30171,11220 30836,11890 30931,12466 30353,12373 29686,11706 "
transform="matrix(0.88407596,0.20026688,-0.20026688,0.88407596,8037.0631,-4399.335)" />
- <path
- inkscape:connector-curvature="0"
- id="path3102"
- d="m 17406.182,17943.438 a 1137.2016,1137.2016 0 0 0 856.574,1485.415"
- style="color:#000000;fill:none;stroke:#000000;stroke-width:95;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none;stroke-dashoffset:0;marker:none;visibility:visible;display:inline;overflow:visible;enable-background:accumulate" />
- <polyline
- id="polyline3116"
- style="color:#000000;fill:none;stroke:#000000;stroke-width:64.65872192;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none;stroke-dashoffset:0;marker:none;visibility:visible;display:inline;overflow:visible;enable-background:accumulate"
- points="1800,6300 2100,5400 2400,6300 "
- transform="matrix(1.4692527,0,0,1.4692527,14411.845,10946.857)" />
- <path
- inkscape:connector-curvature="0"
- id="path3102-0"
- d="m 20305.836,17943.438 a 1137.2016,1137.2016 0 0 0 856.574,1485.415"
- style="color:#000000;fill:none;stroke:#000000;stroke-width:95;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none;stroke-dashoffset:0;marker:none;visibility:visible;display:inline;overflow:visible;enable-background:accumulate" />
- <polyline
- id="polyline3116-2"
- style="color:#000000;fill:none;stroke:#000000;stroke-width:64.65872192;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none;stroke-dashoffset:0;marker:none;visibility:visible;display:inline;overflow:visible;enable-background:accumulate"
- points="1800,6300 2100,5400 2400,6300 "
- transform="matrix(1.4692527,0,0,1.4692527,17311.499,10946.857)" />
+ <g
+ id="g7018">
+ <polyline
+ transform="matrix(1.4692527,0,0,1.4692527,14411.845,10946.857)"
+ points="1800,6300 2100,5400 2400,6300 "
+ style="color:#000000;fill:none;stroke:#000000;stroke-width:36.28282602;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none;stroke-dashoffset:0;marker:none;visibility:visible;display:inline;overflow:visible;enable-background:accumulate"
+ id="polyline3116" />
+ <path
+ id="path6482"
+ d="m 17499.034,18923.426 c 0,1305.223 0,1305.223 0,1305.223"
+ style="fill:none;stroke:#000000;stroke-width:92.04850006;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none" />
+ <path
+ id="path6482-4"
+ d="m 17673.919,18840.987 c -349.77,0 -349.77,0 -349.77,0"
+ style="fill:none;stroke:#000000;stroke-width:126.35780334;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none" />
+ </g>
+ <g
+ transform="translate(2932.5366,0)"
+ id="g7018-4">
+ <polyline
+ transform="matrix(1.4692527,0,0,1.4692527,14411.845,10946.857)"
+ points="1800,6300 2100,5400 2400,6300 "
+ style="color:#000000;fill:none;stroke:#000000;stroke-width:36.28282547;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none;stroke-dashoffset:0;marker:none;visibility:visible;display:inline;overflow:visible;enable-background:accumulate"
+ id="polyline3116-0" />
+ <path
+ id="path6482-8"
+ d="m 17499.034,18923.426 c 0,1305.223 0,1305.223 0,1305.223"
+ style="fill:none;stroke:#000000;stroke-width:92.04850006;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none" />
+ <path
+ id="path6482-4-5"
+ d="m 17673.919,18840.987 c -349.77,0 -349.77,0 -349.77,0"
+ style="fill:none;stroke:#000000;stroke-width:126.35780334;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none" />
+ </g>
</svg>