diff --git a/SAS/XML_generator/src/xmlgen.py b/SAS/XML_generator/src/xmlgen.py
index 04884462d4fee930f8b89c73b78ee0d7a1d793c5..c696085d6e2784a18fc57a198892a5974b32b769 100755
--- a/SAS/XML_generator/src/xmlgen.py
+++ b/SAS/XML_generator/src/xmlgen.py
@@ -2,7 +2,7 @@
# XML generator prototype
-VERSION = "2.15.0"
+VERSION = "2.16.1"
import sys, getopt, time
from xml.sax.saxutils import escape as XMLescape
@@ -259,7 +259,7 @@ def processingCluster(cluster, number_of_tasks=244):
CEP4 = r""" <processingCluster>
<name>CEP4</name>
- <partition>/data</partition>
+ <partition>/data/projects/</partition>
<numberOfTasks>%i</numberOfTasks>
<minRAMPerTask unit="byte">1000000000</minRAMPerTask>
<minScratchPerTask unit="byte">100000000</minScratchPerTask>
@@ -281,7 +281,7 @@ def dataProductCluster(cluster):
</storageCluster>"""
CEP4 = r"""<storageCluster>
<name>CEP4</name>
- <partition>/data</partition>
+ <partition>/data/projects/</partition>
</storageCluster>"""
if cluster == "CEP4":
@@ -1060,16 +1060,25 @@ def readImagingBBS(value):
imagingBBS[2] = toBool(imagingBBS[2])
return imagingBBS
+def checkDemixMultiples(avg_freq_step, avg_time_step, demix_freq_step, demix_time_step, name):
+ try:
+ if avg_freq_step and demix_freq_step:
+ if int(demix_freq_step) % int(avg_freq_step) <> 0:
+ raise GenException("demixFreqStep (%s) should be integer multiple of averagingFreqStep (%s) for %s" % (demix_freq_step, avg_freq_step, name))
+ if avg_time_step and demix_time_step:
+ if int(demix_time_step) % int(avg_time_step) <> 0:
+ raise GenException("demixTimeStep (%s) should be integer multiple of averagingTimeStep (%s) for %s" % (demix_time_step, avg_time_step, name))
+ except:
+ raise GenException("I can't read the Demix values for %s" % name)
+
def readGlobalDemix(value):
globalDemix = ['','','','','','','']
if value:
valList = value.split(';')
for i in range(0,len(valList)):
globalDemix[i] = valList[i]
- if (globalDemix[0] != '') and (globalDemix[2] != ''):
- if int(globalDemix[2]) % int(globalDemix[0]) <> 0: #TODO try ?
- raise GenException("demixFreqStep (" + globalDemix[2] + ") should be integer multiple of averagingFreqStep (" + globalDemix[0] + ") for globalDemix")
- globalDemix[6] = toBool(globalDemix[6]) # convert ignoreTarget to bool
+ checkDemixMultiples(globalDemix[0], globalDemix[1], globalDemix[2], globalDemix[3], "globalDemix")
+ globalDemix[6] = toBool(globalDemix[6]) # convert ignoreTarget to bool
return globalDemix
def readGlobalPulsar(value):
@@ -1199,7 +1208,7 @@ def readCalibratorBeam(startLine, lines, globalSubbands, globalTABrings, globalB
calibratorDemix = []
for pipeline in pipelines:
if pipeline.startswith("BBS"):
- calibratorBBS.append(BBSDefault)
+ calibratorBBS.append(BBSDefault[:]) # [:] is needed to make a deep copy
calBBS = readExtraParms("BBS", [pipeline])
if len(calBBS) > 0:
for i in range(0,len(calBBS)):
@@ -1211,14 +1220,12 @@ def readCalibratorBeam(startLine, lines, globalSubbands, globalTABrings, globalB
calibratorBBS[-1][i] = globalBBS[i]
if pipeline.startswith("Demix"):
- calibratorDemix.append(DemixDefault)
+ calibratorDemix.append(DemixDefault[:]) # [:] is needed to make a deep copy
calDemix = readExtraParms("Demix", [pipeline])
if len(calDemix) > 0:
for i in range(0,len(calDemix)):
calibratorDemix[-1][i] = calDemix[i]
- if (calibratorDemix[-1][0] != '') and (calibratorDemix[-1][2] != ''):
- if int(calibratorDemix[-1][2]) % int(calibratorDemix[-1][0]) <> 0:
- raise GenException("demixFreqStep (" + calibratorDemix[-1][2] + ") should be integer multiple of averagingFreqStep (" + calibratorDemix[-1][0] + ") for calibrator beam pipeline")
+ checkDemixMultiples(calibratorDemix[-1][0], calibratorDemix[-1][1], calibratorDemix[-1][2], calibratorDemix[-1][3], "calibratorDemix")
calibratorDemix[-1][6] = toBool(calibratorDemix[-1][6])
elif globalDemix != []:
printInfo('Using global demix settings for Calibrator beam pipeline')
@@ -1325,26 +1332,25 @@ def readTargetBeams(startLine, lines, globalSubbands, globalBBS, globalDemix, gl
if targetBeams[nr_beams][7]: # pipeline created?
for pipeline in pipelines:
if pipeline.startswith("BBS"):
- targetBBS[nr_beams].append(BBSDefault)
+ targetBBS[nr_beams].append(BBSDefault[:]) # [:] is needed to make a deep copy
tarBBS = readExtraParms("BBS", [pipeline])
for i in range(0, len(tarBBS)):
targetBBS[nr_beams][-1][i] = tarBBS[i]
targetBBS[nr_beams][-1][3] = toBool(targetBBS[nr_beams][-1][3])
if pipeline.startswith("Demix"):
- targetDemix[nr_beams].append(DemixDefault)
+ targetDemix[nr_beams].append(DemixDefault[:]) # [:] is needed to make a deep copy
tarDemix = readExtraParms("Demix", [pipeline])
if len(tarDemix) >= 4:
for i in range(0,len(tarDemix)):
targetDemix[nr_beams][-1][i] = tarDemix[i]
- if int(targetDemix[nr_beams][-1][2]) % int(targetDemix[nr_beams][-1][0]) <> 0:
- raise GenException("demixFreqStep (" + targetDemix[nr_beams][-1][2] + ") should be integer multiple of averagingFreqStep (" + targetDemix[nr_beams][-1][0] + "), target beam pipeline:" + str(nr_beams))
+ checkDemixMultiples(targetDemix[nr_beams][-1][0], targetDemix[nr_beams][-1][1], targetDemix[nr_beams][-1][2], targetDemix[nr_beams][-1][3], "targetDemix[%i]" % nr_beams)
targetDemix[nr_beams][-1][6] = toBool(targetDemix[nr_beams][-1][6]) # convert ignoreTarget to bool
elif len(tarDemix) > 0:
raise GenException("Demixing parameters should at least have the first four averaging/demixing steps (block %s, targetBeam %s)" % (blockNr, nr_beams))
if pipeline.startswith("Pulsar"):
- targetPulsar[nr_beams].append(PulsarDefault)
+ targetPulsar[nr_beams].append(PulsarDefault[:]) # [:] is needed to make a deep copy
tarPulsar = readExtraParms("Pulsar", [pipeline])
if len(tarPulsar) > 0:
for i in range(0,len(tarPulsar)):
diff --git a/SAS/XML_generator/test/test_regression.in_data/xml/MSSS_20151207_testmultipipe.xml b/SAS/XML_generator/test/test_regression.in_data/xml/MSSS_20151207_testmultipipe.xml
index 7cadb0bf5f1686b48413067329225961acd4b904..915330590faa4a5e71061802c255ea8a6857d3c9 100644
--- a/SAS/XML_generator/test/test_regression.in_data/xml/MSSS_20151207_testmultipipe.xml
+++ b/SAS/XML_generator/test/test_regression.in_data/xml/MSSS_20151207_testmultipipe.xml
@@ -1,8 +1,8 @@
<?xml version="1.0" encoding="UTF-8"?>
<lofar:project xmlns:lofar="http://www.astron.nl/MoM2-Lofar" xmlns:mom2="http://www.astron.nl/MoM2" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.astron.nl/MoM2-Lofar http://lofar.astron.nl:8080/mom3/schemas/LofarMoM2.xsd http://www.astron.nl/MoM2 http://lofar.astron.nl:8080/mom3/schemas/MoM2.xsd ">
- <version>2.15.0</version>
- <template version="2.15.0" author="Alwin de Jong,Adriaan Renting" changedBy="Adriaan Renting">
- <description>XML Template generator version 2.15.0</description>
+ <version>2.16.1</version>
+ <template version="2.16.1" author="Alwin de Jong,Adriaan Renting" changedBy="Adriaan Renting">
+ <description>XML Template generator version 2.16.1</description>
</template>
<name>LOFAROBS</name>
<children>
@@ -413,8 +413,8 @@
<flaggingStrategy>LBAdefault</flaggingStrategy>
<duration>5000</duration>
<demixingParameters>
- <averagingFreqStep>64</averagingFreqStep>
- <averagingTimeStep>5</averagingTimeStep>
+ <averagingFreqStep>8</averagingFreqStep>
+ <averagingTimeStep>2</averagingTimeStep>
<demixFreqStep>64</demixFreqStep>
<demixTimeStep>10</demixTimeStep>
<demixAlways>[CygA,CasA]</demixAlways>
@@ -489,8 +489,8 @@
<flaggingStrategy>LBAdefault</flaggingStrategy>
<duration>5000</duration>
<demixingParameters>
- <averagingFreqStep>64</averagingFreqStep>
- <averagingTimeStep>5</averagingTimeStep>
+ <averagingFreqStep>8</averagingFreqStep>
+ <averagingTimeStep>2</averagingTimeStep>
<demixFreqStep>64</demixFreqStep>
<demixTimeStep>10</demixTimeStep>
<demixAlways>[CygA,CasA]</demixAlways>
@@ -565,8 +565,8 @@
<flaggingStrategy>LBAdefault</flaggingStrategy>
<duration>5000</duration>
<demixingParameters>
- <averagingFreqStep>64</averagingFreqStep>
- <averagingTimeStep>5</averagingTimeStep>
+ <averagingFreqStep>8</averagingFreqStep>
+ <averagingTimeStep>2</averagingTimeStep>
<demixFreqStep>64</demixFreqStep>
<demixTimeStep>10</demixTimeStep>
<demixAlways>[CygA,CasA]</demixAlways>
@@ -641,8 +641,8 @@
<flaggingStrategy>LBAdefault</flaggingStrategy>
<duration>5000</duration>
<demixingParameters>
- <averagingFreqStep>64</averagingFreqStep>
- <averagingTimeStep>5</averagingTimeStep>
+ <averagingFreqStep>8</averagingFreqStep>
+ <averagingTimeStep>2</averagingTimeStep>
<demixFreqStep>64</demixFreqStep>
<demixTimeStep>10</demixTimeStep>
<demixAlways>[CygA,CasA]</demixAlways>
@@ -717,8 +717,8 @@
<flaggingStrategy>LBAdefault</flaggingStrategy>
<duration>5000</duration>
<demixingParameters>
- <averagingFreqStep>64</averagingFreqStep>
- <averagingTimeStep>5</averagingTimeStep>
+ <averagingFreqStep>8</averagingFreqStep>
+ <averagingTimeStep>2</averagingTimeStep>
<demixFreqStep>64</demixFreqStep>
<demixTimeStep>10</demixTimeStep>
<demixAlways>[CygA,CasA]</demixAlways>
@@ -793,8 +793,8 @@
<flaggingStrategy>LBAdefault</flaggingStrategy>
<duration>5000</duration>
<demixingParameters>
- <averagingFreqStep>64</averagingFreqStep>
- <averagingTimeStep>5</averagingTimeStep>
+ <averagingFreqStep>8</averagingFreqStep>
+ <averagingTimeStep>2</averagingTimeStep>
<demixFreqStep>64</demixFreqStep>
<demixTimeStep>10</demixTimeStep>
<demixAlways>[CygA,CasA]</demixAlways>
@@ -869,8 +869,8 @@
<flaggingStrategy>LBAdefault</flaggingStrategy>
<duration>5000</duration>
<demixingParameters>
- <averagingFreqStep>64</averagingFreqStep>
- <averagingTimeStep>5</averagingTimeStep>
+ <averagingFreqStep>8</averagingFreqStep>
+ <averagingTimeStep>2</averagingTimeStep>
<demixFreqStep>64</demixFreqStep>
<demixTimeStep>10</demixTimeStep>
<demixAlways>[CygA,CasA]</demixAlways>
@@ -945,8 +945,8 @@
<flaggingStrategy>LBAdefault</flaggingStrategy>
<duration>5000</duration>
<demixingParameters>
- <averagingFreqStep>64</averagingFreqStep>
- <averagingTimeStep>5</averagingTimeStep>
+ <averagingFreqStep>8</averagingFreqStep>
+ <averagingTimeStep>2</averagingTimeStep>
<demixFreqStep>64</demixFreqStep>
<demixTimeStep>10</demixTimeStep>
<demixAlways>[CygA,CasA]</demixAlways>
diff --git a/SAS/XML_generator/test/test_regression.in_data/xml/test_input_cep4.xml b/SAS/XML_generator/test/test_regression.in_data/xml/test_input_cep4.xml
index a3a0f4589de04f3e540a0911be7deb4b0f8e5006..7d936dc182ede92a12d822df3cbbb9befdc65db1 100644
--- a/SAS/XML_generator/test/test_regression.in_data/xml/test_input_cep4.xml
+++ b/SAS/XML_generator/test/test_regression.in_data/xml/test_input_cep4.xml
@@ -1,8 +1,8 @@
<?xml version="1.0" encoding="UTF-8"?>
<lofar:project xmlns:lofar="http://www.astron.nl/MoM2-Lofar" xmlns:mom2="http://www.astron.nl/MoM2" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.astron.nl/MoM2-Lofar http://lofar.astron.nl:8080/mom3/schemas/LofarMoM2.xsd http://www.astron.nl/MoM2 http://lofar.astron.nl:8080/mom3/schemas/MoM2.xsd ">
- <version>2.15.0</version>
- <template version="2.15.0" author="Alwin de Jong,Adriaan Renting" changedBy="Adriaan Renting">
- <description>XML Template generator version 2.15.0</description>
+ <version>2.16.1</version>
+ <template version="2.16.1" author="Alwin de Jong,Adriaan Renting" changedBy="Adriaan Renting">
+ <description>XML Template generator version 2.16.1</description>
</template>
<name>test_lofar</name>
<children>
@@ -103,7 +103,7 @@
<status>no_data</status>
<storageCluster>
<name>CEP4</name>
- <partition>/data</partition>
+ <partition>/data/projects/</partition>
</storageCluster>
</lofar:uvDataProduct>
</item>
@@ -114,7 +114,7 @@
<status>no_data</status>
<storageCluster>
<name>CEP4</name>
- <partition>/data</partition>
+ <partition>/data/projects/</partition>
</storageCluster>
</lofar:bfDataProduct>
</item>
@@ -132,7 +132,7 @@
<description>Crab test/1.0/PP (Pulsar Pipeline)</description>
<processingCluster>
<name>CEP4</name>
- <partition>/data</partition>
+ <partition>/data/projects/</partition>
<numberOfTasks>244</numberOfTasks>
<minRAMPerTask unit="byte">1000000000</minRAMPerTask>
<minScratchPerTask unit="byte">100000000</minScratchPerTask>
@@ -180,7 +180,7 @@
<status>no_data</status>
<storageCluster>
<name>CEP4</name>
- <partition>/data</partition>
+ <partition>/data/projects/</partition>
</storageCluster>
</lofar:pulsarDataProduct>
</item>
@@ -278,7 +278,7 @@
<status>no_data</status>
<storageCluster>
<name>CEP4</name>
- <partition>/data</partition>
+ <partition>/data/projects/</partition>
</storageCluster>
</lofar:uvDataProduct>
</item>
@@ -289,7 +289,7 @@
<status>no_data</status>
<storageCluster>
<name>CEP4</name>
- <partition>/data</partition>
+ <partition>/data/projects/</partition>
</storageCluster>
</lofar:bfDataProduct>
</item>
@@ -307,7 +307,7 @@
<description>Crab test/2.0/PP (Pulsar Pipeline)</description>
<processingCluster>
<name>CEP4</name>
- <partition>/data</partition>
+ <partition>/data/projects/</partition>
<numberOfTasks>244</numberOfTasks>
<minRAMPerTask unit="byte">1000000000</minRAMPerTask>
<minScratchPerTask unit="byte">100000000</minScratchPerTask>
@@ -355,7 +355,7 @@
<status>no_data</status>
<storageCluster>
<name>CEP4</name>
- <partition>/data</partition>
+ <partition>/data/projects/</partition>
</storageCluster>
</lofar:pulsarDataProduct>
</item>