Skip to content
Snippets Groups Projects
Commit c710749f authored by Adriaan Renting's avatar Adriaan Renting
Browse files

Task #9267: Merged fixes for issues 9267 and 9379 with the trunk.

parents 3a61bde7 54d4ea86
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
SAS/XML_generator/src/xmlgen.py
+ 23
− 17
View file @ c710749f
...@@ -2,7 +2,7 @@ ...@@ -2,7 +2,7 @@
# XML generator prototype # XML generator prototype
VERSION = "2.15.0" VERSION = "2.16.1"
import sys, getopt, time import sys, getopt, time
from xml.sax.saxutils import escape as XMLescape from xml.sax.saxutils import escape as XMLescape
...@@ -259,7 +259,7 @@ def processingCluster(cluster, number_of_tasks=244): ...@@ -259,7 +259,7 @@ def processingCluster(cluster, number_of_tasks=244):
CEP4 = r""" <processingCluster> CEP4 = r""" <processingCluster>
<name>CEP4</name> <name>CEP4</name>
<partition>/data</partition> <partition>/data/projects/</partition>
<numberOfTasks>%i</numberOfTasks> <numberOfTasks>%i</numberOfTasks>
<minRAMPerTask unit="byte">1000000000</minRAMPerTask> <minRAMPerTask unit="byte">1000000000</minRAMPerTask>
<minScratchPerTask unit="byte">100000000</minScratchPerTask> <minScratchPerTask unit="byte">100000000</minScratchPerTask>
...@@ -281,7 +281,7 @@ def dataProductCluster(cluster): ...@@ -281,7 +281,7 @@ def dataProductCluster(cluster):
</storageCluster>""" </storageCluster>"""
CEP4 = r"""<storageCluster> CEP4 = r"""<storageCluster>
<name>CEP4</name> <name>CEP4</name>
<partition>/data</partition> <partition>/data/projects/</partition>
</storageCluster>""" </storageCluster>"""
if cluster == "CEP4": if cluster == "CEP4":
...@@ -1060,16 +1060,25 @@ def readImagingBBS(value): ...@@ -1060,16 +1060,25 @@ def readImagingBBS(value):
imagingBBS[2] = toBool(imagingBBS[2]) imagingBBS[2] = toBool(imagingBBS[2])
return imagingBBS return imagingBBS
def checkDemixMultiples(avg_freq_step, avg_time_step, demix_freq_step, demix_time_step, name):
try:
if avg_freq_step and demix_freq_step:
if int(demix_freq_step) % int(avg_freq_step) <> 0:
raise GenException("demixFreqStep (%s) should be integer multiple of averagingFreqStep (%s) for %s" % (demix_freq_step, avg_freq_step, name))
if avg_time_step and demix_time_step:
if int(demix_time_step) % int(avg_time_step) <> 0:
raise GenException("demixTimeStep (%s) should be integer multiple of averagingTimeStep (%s) for %s" % (demix_time_step, avg_time_step, name))
except:
raise GenException("I can't read the Demix values for %s" % name)
def readGlobalDemix(value): def readGlobalDemix(value):
globalDemix = ['','','','','','',''] globalDemix = ['','','','','','','']
if value: if value:
valList = value.split(';') valList = value.split(';')
for i in range(0,len(valList)): for i in range(0,len(valList)):
globalDemix[i] = valList[i] globalDemix[i] = valList[i]
if (globalDemix[0] != '') and (globalDemix[2] != ''): checkDemixMultiples(globalDemix[0], globalDemix[1], globalDemix[2], globalDemix[3], "globalDemix")
if int(globalDemix[2]) % int(globalDemix[0]) <> 0: #TODO try ? globalDemix[6] = toBool(globalDemix[6]) # convert ignoreTarget to bool
raise GenException("demixFreqStep (" + globalDemix[2] + ") should be integer multiple of averagingFreqStep (" + globalDemix[0] + ") for globalDemix")
globalDemix[6] = toBool(globalDemix[6]) # convert ignoreTarget to bool
return globalDemix return globalDemix
def readGlobalPulsar(value): def readGlobalPulsar(value):
...@@ -1199,7 +1208,7 @@ def readCalibratorBeam(startLine, lines, globalSubbands, globalTABrings, globalB ...@@ -1199,7 +1208,7 @@ def readCalibratorBeam(startLine, lines, globalSubbands, globalTABrings, globalB
calibratorDemix = [] calibratorDemix = []
for pipeline in pipelines: for pipeline in pipelines:
if pipeline.startswith("BBS"): if pipeline.startswith("BBS"):
calibratorBBS.append(BBSDefault) calibratorBBS.append(BBSDefault[:]) # [:] is needed to make a deep copy
calBBS = readExtraParms("BBS", [pipeline]) calBBS = readExtraParms("BBS", [pipeline])
if len(calBBS) > 0: if len(calBBS) > 0:
for i in range(0,len(calBBS)): for i in range(0,len(calBBS)):
...@@ -1211,14 +1220,12 @@ def readCalibratorBeam(startLine, lines, globalSubbands, globalTABrings, globalB ...@@ -1211,14 +1220,12 @@ def readCalibratorBeam(startLine, lines, globalSubbands, globalTABrings, globalB
calibratorBBS[-1][i] = globalBBS[i] calibratorBBS[-1][i] = globalBBS[i]
if pipeline.startswith("Demix"): if pipeline.startswith("Demix"):
calibratorDemix.append(DemixDefault) calibratorDemix.append(DemixDefault[:]) # [:] is needed to make a deep copy
calDemix = readExtraParms("Demix", [pipeline]) calDemix = readExtraParms("Demix", [pipeline])
if len(calDemix) > 0: if len(calDemix) > 0:
for i in range(0,len(calDemix)): for i in range(0,len(calDemix)):
calibratorDemix[-1][i] = calDemix[i] calibratorDemix[-1][i] = calDemix[i]
if (calibratorDemix[-1][0] != '') and (calibratorDemix[-1][2] != ''): checkDemixMultiples(calibratorDemix[-1][0], calibratorDemix[-1][1], calibratorDemix[-1][2], calibratorDemix[-1][3], "calibratorDemix")
if int(calibratorDemix[-1][2]) % int(calibratorDemix[-1][0]) <> 0:
raise GenException("demixFreqStep (" + calibratorDemix[-1][2] + ") should be integer multiple of averagingFreqStep (" + calibratorDemix[-1][0] + ") for calibrator beam pipeline")
calibratorDemix[-1][6] = toBool(calibratorDemix[-1][6]) calibratorDemix[-1][6] = toBool(calibratorDemix[-1][6])
elif globalDemix != []: elif globalDemix != []:
printInfo('Using global demix settings for Calibrator beam pipeline') printInfo('Using global demix settings for Calibrator beam pipeline')
...@@ -1325,26 +1332,25 @@ def readTargetBeams(startLine, lines, globalSubbands, globalBBS, globalDemix, gl ...@@ -1325,26 +1332,25 @@ def readTargetBeams(startLine, lines, globalSubbands, globalBBS, globalDemix, gl
if targetBeams[nr_beams][7]: # pipeline created? if targetBeams[nr_beams][7]: # pipeline created?
for pipeline in pipelines: for pipeline in pipelines:
if pipeline.startswith("BBS"): if pipeline.startswith("BBS"):
targetBBS[nr_beams].append(BBSDefault) targetBBS[nr_beams].append(BBSDefault[:]) # [:] is needed to make a deep copy
tarBBS = readExtraParms("BBS", [pipeline]) tarBBS = readExtraParms("BBS", [pipeline])
for i in range(0, len(tarBBS)): for i in range(0, len(tarBBS)):
targetBBS[nr_beams][-1][i] = tarBBS[i] targetBBS[nr_beams][-1][i] = tarBBS[i]
targetBBS[nr_beams][-1][3] = toBool(targetBBS[nr_beams][-1][3]) targetBBS[nr_beams][-1][3] = toBool(targetBBS[nr_beams][-1][3])
if pipeline.startswith("Demix"): if pipeline.startswith("Demix"):
targetDemix[nr_beams].append(DemixDefault) targetDemix[nr_beams].append(DemixDefault[:]) # [:] is needed to make a deep copy
tarDemix = readExtraParms("Demix", [pipeline]) tarDemix = readExtraParms("Demix", [pipeline])
if len(tarDemix) >= 4: if len(tarDemix) >= 4:
for i in range(0,len(tarDemix)): for i in range(0,len(tarDemix)):
targetDemix[nr_beams][-1][i] = tarDemix[i] targetDemix[nr_beams][-1][i] = tarDemix[i]
if int(targetDemix[nr_beams][-1][2]) % int(targetDemix[nr_beams][-1][0]) <> 0: checkDemixMultiples(targetDemix[nr_beams][-1][0], targetDemix[nr_beams][-1][1], targetDemix[nr_beams][-1][2], targetDemix[nr_beams][-1][3], "targetDemix[%i]" % nr_beams)
raise GenException("demixFreqStep (" + targetDemix[nr_beams][-1][2] + ") should be integer multiple of averagingFreqStep (" + targetDemix[nr_beams][-1][0] + "), target beam pipeline:" + str(nr_beams))
targetDemix[nr_beams][-1][6] = toBool(targetDemix[nr_beams][-1][6]) # convert ignoreTarget to bool targetDemix[nr_beams][-1][6] = toBool(targetDemix[nr_beams][-1][6]) # convert ignoreTarget to bool
elif len(tarDemix) > 0: elif len(tarDemix) > 0:
raise GenException("Demixing parameters should at least have the first four averaging/demixing steps (block %s, targetBeam %s)" % (blockNr, nr_beams)) raise GenException("Demixing parameters should at least have the first four averaging/demixing steps (block %s, targetBeam %s)" % (blockNr, nr_beams))
if pipeline.startswith("Pulsar"): if pipeline.startswith("Pulsar"):
targetPulsar[nr_beams].append(PulsarDefault) targetPulsar[nr_beams].append(PulsarDefault[:]) # [:] is needed to make a deep copy
tarPulsar = readExtraParms("Pulsar", [pipeline]) tarPulsar = readExtraParms("Pulsar", [pipeline])
if len(tarPulsar) > 0: if len(tarPulsar) > 0:
for i in range(0,len(tarPulsar)): for i in range(0,len(tarPulsar)):
......
SAS/XML_generator/test/test_regression.in_data/xml/MSSS_20151207_testmultipipe.xml
+ 19
− 19
View file @ c710749f
<?xml version="1.0" encoding="UTF-8"?> <?xml version="1.0" encoding="UTF-8"?>
<lofar:project xmlns:lofar="http://www.astron.nl/MoM2-Lofar" xmlns:mom2="http://www.astron.nl/MoM2" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.astron.nl/MoM2-Lofar http://lofar.astron.nl:8080/mom3/schemas/LofarMoM2.xsd http://www.astron.nl/MoM2 http://lofar.astron.nl:8080/mom3/schemas/MoM2.xsd "> <lofar:project xmlns:lofar="http://www.astron.nl/MoM2-Lofar" xmlns:mom2="http://www.astron.nl/MoM2" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.astron.nl/MoM2-Lofar http://lofar.astron.nl:8080/mom3/schemas/LofarMoM2.xsd http://www.astron.nl/MoM2 http://lofar.astron.nl:8080/mom3/schemas/MoM2.xsd ">
<version>2.15.0</version> <version>2.16.1</version>
<template version="2.15.0" author="Alwin de Jong,Adriaan Renting" changedBy="Adriaan Renting"> <template version="2.16.1" author="Alwin de Jong,Adriaan Renting" changedBy="Adriaan Renting">
<description>XML Template generator version 2.15.0</description> <description>XML Template generator version 2.16.1</description>
</template> </template>
<name>LOFAROBS</name> <name>LOFAROBS</name>
<children> <children>
...@@ -413,8 +413,8 @@ ...@@ -413,8 +413,8 @@
<flaggingStrategy>LBAdefault</flaggingStrategy> <flaggingStrategy>LBAdefault</flaggingStrategy>
<duration>5000</duration> <duration>5000</duration>
<demixingParameters> <demixingParameters>
<averagingFreqStep>64</averagingFreqStep> <averagingFreqStep>8</averagingFreqStep>
<averagingTimeStep>5</averagingTimeStep> <averagingTimeStep>2</averagingTimeStep>
<demixFreqStep>64</demixFreqStep> <demixFreqStep>64</demixFreqStep>
<demixTimeStep>10</demixTimeStep> <demixTimeStep>10</demixTimeStep>
<demixAlways>[CygA,CasA]</demixAlways> <demixAlways>[CygA,CasA]</demixAlways>
...@@ -489,8 +489,8 @@ ...@@ -489,8 +489,8 @@
<flaggingStrategy>LBAdefault</flaggingStrategy> <flaggingStrategy>LBAdefault</flaggingStrategy>
<duration>5000</duration> <duration>5000</duration>
<demixingParameters> <demixingParameters>
<averagingFreqStep>64</averagingFreqStep> <averagingFreqStep>8</averagingFreqStep>
<averagingTimeStep>5</averagingTimeStep> <averagingTimeStep>2</averagingTimeStep>
<demixFreqStep>64</demixFreqStep> <demixFreqStep>64</demixFreqStep>
<demixTimeStep>10</demixTimeStep> <demixTimeStep>10</demixTimeStep>
<demixAlways>[CygA,CasA]</demixAlways> <demixAlways>[CygA,CasA]</demixAlways>
...@@ -565,8 +565,8 @@ ...@@ -565,8 +565,8 @@
<flaggingStrategy>LBAdefault</flaggingStrategy> <flaggingStrategy>LBAdefault</flaggingStrategy>
<duration>5000</duration> <duration>5000</duration>
<demixingParameters> <demixingParameters>
<averagingFreqStep>64</averagingFreqStep> <averagingFreqStep>8</averagingFreqStep>
<averagingTimeStep>5</averagingTimeStep> <averagingTimeStep>2</averagingTimeStep>
<demixFreqStep>64</demixFreqStep> <demixFreqStep>64</demixFreqStep>
<demixTimeStep>10</demixTimeStep> <demixTimeStep>10</demixTimeStep>
<demixAlways>[CygA,CasA]</demixAlways> <demixAlways>[CygA,CasA]</demixAlways>
...@@ -641,8 +641,8 @@ ...@@ -641,8 +641,8 @@
<flaggingStrategy>LBAdefault</flaggingStrategy> <flaggingStrategy>LBAdefault</flaggingStrategy>
<duration>5000</duration> <duration>5000</duration>
<demixingParameters> <demixingParameters>
<averagingFreqStep>64</averagingFreqStep> <averagingFreqStep>8</averagingFreqStep>
<averagingTimeStep>5</averagingTimeStep> <averagingTimeStep>2</averagingTimeStep>
<demixFreqStep>64</demixFreqStep> <demixFreqStep>64</demixFreqStep>
<demixTimeStep>10</demixTimeStep> <demixTimeStep>10</demixTimeStep>
<demixAlways>[CygA,CasA]</demixAlways> <demixAlways>[CygA,CasA]</demixAlways>
...@@ -717,8 +717,8 @@ ...@@ -717,8 +717,8 @@
<flaggingStrategy>LBAdefault</flaggingStrategy> <flaggingStrategy>LBAdefault</flaggingStrategy>
<duration>5000</duration> <duration>5000</duration>
<demixingParameters> <demixingParameters>
<averagingFreqStep>64</averagingFreqStep> <averagingFreqStep>8</averagingFreqStep>
<averagingTimeStep>5</averagingTimeStep> <averagingTimeStep>2</averagingTimeStep>
<demixFreqStep>64</demixFreqStep> <demixFreqStep>64</demixFreqStep>
<demixTimeStep>10</demixTimeStep> <demixTimeStep>10</demixTimeStep>
<demixAlways>[CygA,CasA]</demixAlways> <demixAlways>[CygA,CasA]</demixAlways>
...@@ -793,8 +793,8 @@ ...@@ -793,8 +793,8 @@
<flaggingStrategy>LBAdefault</flaggingStrategy> <flaggingStrategy>LBAdefault</flaggingStrategy>
<duration>5000</duration> <duration>5000</duration>
<demixingParameters> <demixingParameters>
<averagingFreqStep>64</averagingFreqStep> <averagingFreqStep>8</averagingFreqStep>
<averagingTimeStep>5</averagingTimeStep> <averagingTimeStep>2</averagingTimeStep>
<demixFreqStep>64</demixFreqStep> <demixFreqStep>64</demixFreqStep>
<demixTimeStep>10</demixTimeStep> <demixTimeStep>10</demixTimeStep>
<demixAlways>[CygA,CasA]</demixAlways> <demixAlways>[CygA,CasA]</demixAlways>
...@@ -869,8 +869,8 @@ ...@@ -869,8 +869,8 @@
<flaggingStrategy>LBAdefault</flaggingStrategy> <flaggingStrategy>LBAdefault</flaggingStrategy>
<duration>5000</duration> <duration>5000</duration>
<demixingParameters> <demixingParameters>
<averagingFreqStep>64</averagingFreqStep> <averagingFreqStep>8</averagingFreqStep>
<averagingTimeStep>5</averagingTimeStep> <averagingTimeStep>2</averagingTimeStep>
<demixFreqStep>64</demixFreqStep> <demixFreqStep>64</demixFreqStep>
<demixTimeStep>10</demixTimeStep> <demixTimeStep>10</demixTimeStep>
<demixAlways>[CygA,CasA]</demixAlways> <demixAlways>[CygA,CasA]</demixAlways>
...@@ -945,8 +945,8 @@ ...@@ -945,8 +945,8 @@
<flaggingStrategy>LBAdefault</flaggingStrategy> <flaggingStrategy>LBAdefault</flaggingStrategy>
<duration>5000</duration> <duration>5000</duration>
<demixingParameters> <demixingParameters>
<averagingFreqStep>64</averagingFreqStep> <averagingFreqStep>8</averagingFreqStep>
<averagingTimeStep>5</averagingTimeStep> <averagingTimeStep>2</averagingTimeStep>
<demixFreqStep>64</demixFreqStep> <demixFreqStep>64</demixFreqStep>
<demixTimeStep>10</demixTimeStep> <demixTimeStep>10</demixTimeStep>
<demixAlways>[CygA,CasA]</demixAlways> <demixAlways>[CygA,CasA]</demixAlways>
......
SAS/XML_generator/test/test_regression.in_data/xml/test_input_cep4.xml
+ 11
− 11
View file @ c710749f
<?xml version="1.0" encoding="UTF-8"?> <?xml version="1.0" encoding="UTF-8"?>
<lofar:project xmlns:lofar="http://www.astron.nl/MoM2-Lofar" xmlns:mom2="http://www.astron.nl/MoM2" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.astron.nl/MoM2-Lofar http://lofar.astron.nl:8080/mom3/schemas/LofarMoM2.xsd http://www.astron.nl/MoM2 http://lofar.astron.nl:8080/mom3/schemas/MoM2.xsd "> <lofar:project xmlns:lofar="http://www.astron.nl/MoM2-Lofar" xmlns:mom2="http://www.astron.nl/MoM2" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.astron.nl/MoM2-Lofar http://lofar.astron.nl:8080/mom3/schemas/LofarMoM2.xsd http://www.astron.nl/MoM2 http://lofar.astron.nl:8080/mom3/schemas/MoM2.xsd ">
<version>2.15.0</version> <version>2.16.1</version>
<template version="2.15.0" author="Alwin de Jong,Adriaan Renting" changedBy="Adriaan Renting"> <template version="2.16.1" author="Alwin de Jong,Adriaan Renting" changedBy="Adriaan Renting">
<description>XML Template generator version 2.15.0</description> <description>XML Template generator version 2.16.1</description>
</template> </template>
<name>test_lofar</name> <name>test_lofar</name>
<children> <children>
...@@ -103,7 +103,7 @@ ...@@ -103,7 +103,7 @@
<status>no_data</status> <status>no_data</status>
<storageCluster> <storageCluster>
<name>CEP4</name> <name>CEP4</name>
<partition>/data</partition> <partition>/data/projects/</partition>
</storageCluster> </storageCluster>
</lofar:uvDataProduct> </lofar:uvDataProduct>
</item> </item>
...@@ -114,7 +114,7 @@ ...@@ -114,7 +114,7 @@
<status>no_data</status> <status>no_data</status>
<storageCluster> <storageCluster>
<name>CEP4</name> <name>CEP4</name>
<partition>/data</partition> <partition>/data/projects/</partition>
</storageCluster> </storageCluster>
</lofar:bfDataProduct> </lofar:bfDataProduct>
</item> </item>
...@@ -132,7 +132,7 @@ ...@@ -132,7 +132,7 @@
<description>Crab test/1.0/PP (Pulsar Pipeline)</description> <description>Crab test/1.0/PP (Pulsar Pipeline)</description>
<processingCluster> <processingCluster>
<name>CEP4</name> <name>CEP4</name>
<partition>/data</partition> <partition>/data/projects/</partition>
<numberOfTasks>244</numberOfTasks> <numberOfTasks>244</numberOfTasks>
<minRAMPerTask unit="byte">1000000000</minRAMPerTask> <minRAMPerTask unit="byte">1000000000</minRAMPerTask>
<minScratchPerTask unit="byte">100000000</minScratchPerTask> <minScratchPerTask unit="byte">100000000</minScratchPerTask>
...@@ -180,7 +180,7 @@ ...@@ -180,7 +180,7 @@
<status>no_data</status> <status>no_data</status>
<storageCluster> <storageCluster>
<name>CEP4</name> <name>CEP4</name>
<partition>/data</partition> <partition>/data/projects/</partition>
</storageCluster> </storageCluster>
</lofar:pulsarDataProduct> </lofar:pulsarDataProduct>
</item> </item>
...@@ -278,7 +278,7 @@ ...@@ -278,7 +278,7 @@
<status>no_data</status> <status>no_data</status>
<storageCluster> <storageCluster>
<name>CEP4</name> <name>CEP4</name>
<partition>/data</partition> <partition>/data/projects/</partition>
</storageCluster> </storageCluster>
</lofar:uvDataProduct> </lofar:uvDataProduct>
</item> </item>
...@@ -289,7 +289,7 @@ ...@@ -289,7 +289,7 @@
<status>no_data</status> <status>no_data</status>
<storageCluster> <storageCluster>
<name>CEP4</name> <name>CEP4</name>
<partition>/data</partition> <partition>/data/projects/</partition>
</storageCluster> </storageCluster>
</lofar:bfDataProduct> </lofar:bfDataProduct>
</item> </item>
...@@ -307,7 +307,7 @@ ...@@ -307,7 +307,7 @@
<description>Crab test/2.0/PP (Pulsar Pipeline)</description> <description>Crab test/2.0/PP (Pulsar Pipeline)</description>
<processingCluster> <processingCluster>
<name>CEP4</name> <name>CEP4</name>
<partition>/data</partition> <partition>/data/projects/</partition>
<numberOfTasks>244</numberOfTasks> <numberOfTasks>244</numberOfTasks>
<minRAMPerTask unit="byte">1000000000</minRAMPerTask> <minRAMPerTask unit="byte">1000000000</minRAMPerTask>
<minScratchPerTask unit="byte">100000000</minScratchPerTask> <minScratchPerTask unit="byte">100000000</minScratchPerTask>
...@@ -355,7 +355,7 @@ ...@@ -355,7 +355,7 @@
<status>no_data</status> <status>no_data</status>
<storageCluster> <storageCluster>
<name>CEP4</name> <name>CEP4</name>
<partition>/data</partition> <partition>/data/projects/</partition>
</storageCluster> </storageCluster>
</lofar:pulsarDataProduct> </lofar:pulsarDataProduct>
</item> </item>
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment