Skip to content
Snippets Groups Projects
Commit c710749f authored by Adriaan Renting's avatar Adriaan Renting
Browse files

Task #9267: Merged fixes for issues 9267 and 9379 with the trunk.

parents 3a61bde7 54d4ea86
Branches
Tags
No related merge requests found
......@@ -2,7 +2,7 @@
# XML generator prototype
VERSION = "2.15.0"
VERSION = "2.16.1"
import sys, getopt, time
from xml.sax.saxutils import escape as XMLescape
......@@ -259,7 +259,7 @@ def processingCluster(cluster, number_of_tasks=244):
CEP4 = r""" <processingCluster>
<name>CEP4</name>
<partition>/data</partition>
<partition>/data/projects/</partition>
<numberOfTasks>%i</numberOfTasks>
<minRAMPerTask unit="byte">1000000000</minRAMPerTask>
<minScratchPerTask unit="byte">100000000</minScratchPerTask>
......@@ -281,7 +281,7 @@ def dataProductCluster(cluster):
</storageCluster>"""
CEP4 = r"""<storageCluster>
<name>CEP4</name>
<partition>/data</partition>
<partition>/data/projects/</partition>
</storageCluster>"""
if cluster == "CEP4":
......@@ -1060,15 +1060,24 @@ def readImagingBBS(value):
imagingBBS[2] = toBool(imagingBBS[2])
return imagingBBS
def checkDemixMultiples(avg_freq_step, avg_time_step, demix_freq_step, demix_time_step, name):
try:
if avg_freq_step and demix_freq_step:
if int(demix_freq_step) % int(avg_freq_step) <> 0:
raise GenException("demixFreqStep (%s) should be integer multiple of averagingFreqStep (%s) for %s" % (demix_freq_step, avg_freq_step, name))
if avg_time_step and demix_time_step:
if int(demix_time_step) % int(avg_time_step) <> 0:
raise GenException("demixTimeStep (%s) should be integer multiple of averagingTimeStep (%s) for %s" % (demix_time_step, avg_time_step, name))
except:
raise GenException("I can't read the Demix values for %s" % name)
def readGlobalDemix(value):
globalDemix = ['','','','','','','']
if value:
valList = value.split(';')
for i in range(0,len(valList)):
globalDemix[i] = valList[i]
if (globalDemix[0] != '') and (globalDemix[2] != ''):
if int(globalDemix[2]) % int(globalDemix[0]) <> 0: #TODO try ?
raise GenException("demixFreqStep (" + globalDemix[2] + ") should be integer multiple of averagingFreqStep (" + globalDemix[0] + ") for globalDemix")
checkDemixMultiples(globalDemix[0], globalDemix[1], globalDemix[2], globalDemix[3], "globalDemix")
globalDemix[6] = toBool(globalDemix[6]) # convert ignoreTarget to bool
return globalDemix
......@@ -1199,7 +1208,7 @@ def readCalibratorBeam(startLine, lines, globalSubbands, globalTABrings, globalB
calibratorDemix = []
for pipeline in pipelines:
if pipeline.startswith("BBS"):
calibratorBBS.append(BBSDefault)
calibratorBBS.append(BBSDefault[:]) # [:] is needed to make a deep copy
calBBS = readExtraParms("BBS", [pipeline])
if len(calBBS) > 0:
for i in range(0,len(calBBS)):
......@@ -1211,14 +1220,12 @@ def readCalibratorBeam(startLine, lines, globalSubbands, globalTABrings, globalB
calibratorBBS[-1][i] = globalBBS[i]
if pipeline.startswith("Demix"):
calibratorDemix.append(DemixDefault)
calibratorDemix.append(DemixDefault[:]) # [:] is needed to make a deep copy
calDemix = readExtraParms("Demix", [pipeline])
if len(calDemix) > 0:
for i in range(0,len(calDemix)):
calibratorDemix[-1][i] = calDemix[i]
if (calibratorDemix[-1][0] != '') and (calibratorDemix[-1][2] != ''):
if int(calibratorDemix[-1][2]) % int(calibratorDemix[-1][0]) <> 0:
raise GenException("demixFreqStep (" + calibratorDemix[-1][2] + ") should be integer multiple of averagingFreqStep (" + calibratorDemix[-1][0] + ") for calibrator beam pipeline")
checkDemixMultiples(calibratorDemix[-1][0], calibratorDemix[-1][1], calibratorDemix[-1][2], calibratorDemix[-1][3], "calibratorDemix")
calibratorDemix[-1][6] = toBool(calibratorDemix[-1][6])
elif globalDemix != []:
printInfo('Using global demix settings for Calibrator beam pipeline')
......@@ -1325,26 +1332,25 @@ def readTargetBeams(startLine, lines, globalSubbands, globalBBS, globalDemix, gl
if targetBeams[nr_beams][7]: # pipeline created?
for pipeline in pipelines:
if pipeline.startswith("BBS"):
targetBBS[nr_beams].append(BBSDefault)
targetBBS[nr_beams].append(BBSDefault[:]) # [:] is needed to make a deep copy
tarBBS = readExtraParms("BBS", [pipeline])
for i in range(0, len(tarBBS)):
targetBBS[nr_beams][-1][i] = tarBBS[i]
targetBBS[nr_beams][-1][3] = toBool(targetBBS[nr_beams][-1][3])
if pipeline.startswith("Demix"):
targetDemix[nr_beams].append(DemixDefault)
targetDemix[nr_beams].append(DemixDefault[:]) # [:] is needed to make a deep copy
tarDemix = readExtraParms("Demix", [pipeline])
if len(tarDemix) >= 4:
for i in range(0,len(tarDemix)):
targetDemix[nr_beams][-1][i] = tarDemix[i]
if int(targetDemix[nr_beams][-1][2]) % int(targetDemix[nr_beams][-1][0]) <> 0:
raise GenException("demixFreqStep (" + targetDemix[nr_beams][-1][2] + ") should be integer multiple of averagingFreqStep (" + targetDemix[nr_beams][-1][0] + "), target beam pipeline:" + str(nr_beams))
checkDemixMultiples(targetDemix[nr_beams][-1][0], targetDemix[nr_beams][-1][1], targetDemix[nr_beams][-1][2], targetDemix[nr_beams][-1][3], "targetDemix[%i]" % nr_beams)
targetDemix[nr_beams][-1][6] = toBool(targetDemix[nr_beams][-1][6]) # convert ignoreTarget to bool
elif len(tarDemix) > 0:
raise GenException("Demixing parameters should at least have the first four averaging/demixing steps (block %s, targetBeam %s)" % (blockNr, nr_beams))
if pipeline.startswith("Pulsar"):
targetPulsar[nr_beams].append(PulsarDefault)
targetPulsar[nr_beams].append(PulsarDefault[:]) # [:] is needed to make a deep copy
tarPulsar = readExtraParms("Pulsar", [pipeline])
if len(tarPulsar) > 0:
for i in range(0,len(tarPulsar)):
......
<?xml version="1.0" encoding="UTF-8"?>
<lofar:project xmlns:lofar="http://www.astron.nl/MoM2-Lofar" xmlns:mom2="http://www.astron.nl/MoM2" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.astron.nl/MoM2-Lofar http://lofar.astron.nl:8080/mom3/schemas/LofarMoM2.xsd http://www.astron.nl/MoM2 http://lofar.astron.nl:8080/mom3/schemas/MoM2.xsd ">
<version>2.15.0</version>
<template version="2.15.0" author="Alwin de Jong,Adriaan Renting" changedBy="Adriaan Renting">
<description>XML Template generator version 2.15.0</description>
<version>2.16.1</version>
<template version="2.16.1" author="Alwin de Jong,Adriaan Renting" changedBy="Adriaan Renting">
<description>XML Template generator version 2.16.1</description>
</template>
<name>LOFAROBS</name>
<children>
......@@ -413,8 +413,8 @@
<flaggingStrategy>LBAdefault</flaggingStrategy>
<duration>5000</duration>
<demixingParameters>
<averagingFreqStep>64</averagingFreqStep>
<averagingTimeStep>5</averagingTimeStep>
<averagingFreqStep>8</averagingFreqStep>
<averagingTimeStep>2</averagingTimeStep>
<demixFreqStep>64</demixFreqStep>
<demixTimeStep>10</demixTimeStep>
<demixAlways>[CygA,CasA]</demixAlways>
......@@ -489,8 +489,8 @@
<flaggingStrategy>LBAdefault</flaggingStrategy>
<duration>5000</duration>
<demixingParameters>
<averagingFreqStep>64</averagingFreqStep>
<averagingTimeStep>5</averagingTimeStep>
<averagingFreqStep>8</averagingFreqStep>
<averagingTimeStep>2</averagingTimeStep>
<demixFreqStep>64</demixFreqStep>
<demixTimeStep>10</demixTimeStep>
<demixAlways>[CygA,CasA]</demixAlways>
......@@ -565,8 +565,8 @@
<flaggingStrategy>LBAdefault</flaggingStrategy>
<duration>5000</duration>
<demixingParameters>
<averagingFreqStep>64</averagingFreqStep>
<averagingTimeStep>5</averagingTimeStep>
<averagingFreqStep>8</averagingFreqStep>
<averagingTimeStep>2</averagingTimeStep>
<demixFreqStep>64</demixFreqStep>
<demixTimeStep>10</demixTimeStep>
<demixAlways>[CygA,CasA]</demixAlways>
......@@ -641,8 +641,8 @@
<flaggingStrategy>LBAdefault</flaggingStrategy>
<duration>5000</duration>
<demixingParameters>
<averagingFreqStep>64</averagingFreqStep>
<averagingTimeStep>5</averagingTimeStep>
<averagingFreqStep>8</averagingFreqStep>
<averagingTimeStep>2</averagingTimeStep>
<demixFreqStep>64</demixFreqStep>
<demixTimeStep>10</demixTimeStep>
<demixAlways>[CygA,CasA]</demixAlways>
......@@ -717,8 +717,8 @@
<flaggingStrategy>LBAdefault</flaggingStrategy>
<duration>5000</duration>
<demixingParameters>
<averagingFreqStep>64</averagingFreqStep>
<averagingTimeStep>5</averagingTimeStep>
<averagingFreqStep>8</averagingFreqStep>
<averagingTimeStep>2</averagingTimeStep>
<demixFreqStep>64</demixFreqStep>
<demixTimeStep>10</demixTimeStep>
<demixAlways>[CygA,CasA]</demixAlways>
......@@ -793,8 +793,8 @@
<flaggingStrategy>LBAdefault</flaggingStrategy>
<duration>5000</duration>
<demixingParameters>
<averagingFreqStep>64</averagingFreqStep>
<averagingTimeStep>5</averagingTimeStep>
<averagingFreqStep>8</averagingFreqStep>
<averagingTimeStep>2</averagingTimeStep>
<demixFreqStep>64</demixFreqStep>
<demixTimeStep>10</demixTimeStep>
<demixAlways>[CygA,CasA]</demixAlways>
......@@ -869,8 +869,8 @@
<flaggingStrategy>LBAdefault</flaggingStrategy>
<duration>5000</duration>
<demixingParameters>
<averagingFreqStep>64</averagingFreqStep>
<averagingTimeStep>5</averagingTimeStep>
<averagingFreqStep>8</averagingFreqStep>
<averagingTimeStep>2</averagingTimeStep>
<demixFreqStep>64</demixFreqStep>
<demixTimeStep>10</demixTimeStep>
<demixAlways>[CygA,CasA]</demixAlways>
......@@ -945,8 +945,8 @@
<flaggingStrategy>LBAdefault</flaggingStrategy>
<duration>5000</duration>
<demixingParameters>
<averagingFreqStep>64</averagingFreqStep>
<averagingTimeStep>5</averagingTimeStep>
<averagingFreqStep>8</averagingFreqStep>
<averagingTimeStep>2</averagingTimeStep>
<demixFreqStep>64</demixFreqStep>
<demixTimeStep>10</demixTimeStep>
<demixAlways>[CygA,CasA]</demixAlways>
......
<?xml version="1.0" encoding="UTF-8"?>
<lofar:project xmlns:lofar="http://www.astron.nl/MoM2-Lofar" xmlns:mom2="http://www.astron.nl/MoM2" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.astron.nl/MoM2-Lofar http://lofar.astron.nl:8080/mom3/schemas/LofarMoM2.xsd http://www.astron.nl/MoM2 http://lofar.astron.nl:8080/mom3/schemas/MoM2.xsd ">
<version>2.15.0</version>
<template version="2.15.0" author="Alwin de Jong,Adriaan Renting" changedBy="Adriaan Renting">
<description>XML Template generator version 2.15.0</description>
<version>2.16.1</version>
<template version="2.16.1" author="Alwin de Jong,Adriaan Renting" changedBy="Adriaan Renting">
<description>XML Template generator version 2.16.1</description>
</template>
<name>test_lofar</name>
<children>
......@@ -103,7 +103,7 @@
<status>no_data</status>
<storageCluster>
<name>CEP4</name>
<partition>/data</partition>
<partition>/data/projects/</partition>
</storageCluster>
</lofar:uvDataProduct>
</item>
......@@ -114,7 +114,7 @@
<status>no_data</status>
<storageCluster>
<name>CEP4</name>
<partition>/data</partition>
<partition>/data/projects/</partition>
</storageCluster>
</lofar:bfDataProduct>
</item>
......@@ -132,7 +132,7 @@
<description>Crab test/1.0/PP (Pulsar Pipeline)</description>
<processingCluster>
<name>CEP4</name>
<partition>/data</partition>
<partition>/data/projects/</partition>
<numberOfTasks>244</numberOfTasks>
<minRAMPerTask unit="byte">1000000000</minRAMPerTask>
<minScratchPerTask unit="byte">100000000</minScratchPerTask>
......@@ -180,7 +180,7 @@
<status>no_data</status>
<storageCluster>
<name>CEP4</name>
<partition>/data</partition>
<partition>/data/projects/</partition>
</storageCluster>
</lofar:pulsarDataProduct>
</item>
......@@ -278,7 +278,7 @@
<status>no_data</status>
<storageCluster>
<name>CEP4</name>
<partition>/data</partition>
<partition>/data/projects/</partition>
</storageCluster>
</lofar:uvDataProduct>
</item>
......@@ -289,7 +289,7 @@
<status>no_data</status>
<storageCluster>
<name>CEP4</name>
<partition>/data</partition>
<partition>/data/projects/</partition>
</storageCluster>
</lofar:bfDataProduct>
</item>
......@@ -307,7 +307,7 @@
<description>Crab test/2.0/PP (Pulsar Pipeline)</description>
<processingCluster>
<name>CEP4</name>
<partition>/data</partition>
<partition>/data/projects/</partition>
<numberOfTasks>244</numberOfTasks>
<minRAMPerTask unit="byte">1000000000</minRAMPerTask>
<minScratchPerTask unit="byte">100000000</minScratchPerTask>
......@@ -355,7 +355,7 @@
<status>no_data</status>
<storageCluster>
<name>CEP4</name>
<partition>/data</partition>
<partition>/data/projects/</partition>
</storageCluster>
</lofar:pulsarDataProduct>
</item>
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment