From 5eda83fcf8c0bfaf6fc8aead7e74a438562dac5c Mon Sep 17 00:00:00 2001
From: alex <alex@tls-tautenburg.de>
Date: Mon, 18 Oct 2021 12:13:18 +0000
Subject: [PATCH] Bugfix ddecal + minor updates
---
Docker/Dockerfile | 3 ++
docs/source/running.rst | 4 +-
lofar-cwl/steps/ddecal.cwl | 61 ++++++++++++++++++++-----
lofar-cwl/steps/ms_concat.cwl | 1 +
steps/check_unflagged_fraction.cwl | 18 ++++++--
steps/findRefAnt_join.cwl | 2 +-
steps/summary.cwl | 25 ++++++++++
workflows/prefactor_target.cwl | 8 ++++
workflows/prefactor_target/finalize.cwl | 22 +++++++++
workflows/prefactor_target/gsmcal.cwl | 13 ++++++
10 files changed, 139 insertions(+), 18 deletions(-)
diff --git a/Docker/Dockerfile b/Docker/Dockerfile
index 311e49dd..61221e79 100644
--- a/Docker/Dockerfile
+++ b/Docker/Dockerfile
@@ -10,6 +10,9 @@ RUN \
apt-get update && \
apt-get install -y vim wget casacore-tools && \
rm -rf /var/cache/apt
+RUN \
+ pip3 install --upgrade pip && \
+ pip3 install h5py
RUN \
pip3 install --upgrade pip && \
git clone https://github.com/lofar-astron/prefactor.git ${PREFACTOR_NAME} && \
diff --git a/docs/source/running.rst b/docs/source/running.rst
index 1e746da6..c66cec5c 100644
--- a/docs/source/running.rst
+++ b/docs/source/running.rst
@@ -41,9 +41,10 @@ Pipeline options for ``cwltool``
The following ``<cwl_options>`` are recommended to use for running **prefactor** with ``cwltool``:
- * **---outdir**: specifies the location of the pipeline outputs
+ * **---outdir**: specifies the location of the final pipeline output directory (results)
* **---tmpdir-prefix**: specifies the location of the intermediate data products
* **---parallel**: jobs will run in parallel
+ * **---singularity**: use Singularity instead of Docker
* **---no-container**: don't use Docker container (only for manual installation)
* **---preserve-entire-environment**: use system environment variables (only for manual installation)
* **---debug**: more verbose output (use only for debugging the pipeline)
@@ -74,6 +75,7 @@ The following ``<cwl_options>`` are recommended to use for running **prefactor**
* **---stats**: creates runtime statistics
* **---maxLocalJobs**: amount of local jobs to be run at the same time ("max_per_node")
* **---retryCount**: amount of retries for each failed pipeline job
+ * **---singularity**: use Singularity instead of Docker
* **---preserve-entire-environment**: use system environment variables (only for manual installation)
* **---no-container**: don't use Docker container (only for manual installation)
diff --git a/lofar-cwl/steps/ddecal.cwl b/lofar-cwl/steps/ddecal.cwl
index e89280ec..09011bba 100644
--- a/lofar-cwl/steps/ddecal.cwl
+++ b/lofar-cwl/steps/ddecal.cwl
@@ -18,14 +18,6 @@ hints:
arguments:
- steps=[ddecal,count]
- - ddecal.uvlambdamin=300
- - ddecal.maxiter=50
- - ddecal.nchan=1
- - ddecal.solint=1
- - ddecal.tolerance=1.e-3
- - ddecal.usemodelcolumn=True
- - ddecal.flagdivergedonly=True
- - ddecal.propagateconvergedonly=True
inputs:
- id: msin
@@ -71,6 +63,16 @@ inputs:
default: true
inputBinding:
prefix: ddecal.propagatesolutions=True
+ - id: propagate_converged_only
+ type: boolean?
+ default: true
+ inputBinding:
+ prefix: ddecal.propagateconvergedonly=True
+ - id: usemodelcolumn
+ type: boolean?
+ default: true
+ inputBinding:
+ prefix: ddecal.usemodelcolumn=True
- id: flagunconverged
type: boolean?
default: false
@@ -78,9 +80,9 @@ inputs:
Flag unconverged solutions (i.e., those from solves that did not converge
within maxiter iterations).
inputBinding:
- prefix: ddecal.flagdivergedonly=True
+ prefix: ddecal.flagunconverged=True
- id: flagdivergedonly
- default: false
+ default: true
type: boolean?
doc: |
Flag only the unconverged solutions for which divergence was detected.
@@ -119,14 +121,49 @@ inputs:
inputBinding:
prefix: ddecal.mode=
separate: false
-
+ - id: nchan
+ type: int?
+ default: 1
+ inputBinding:
+ position: 0
+ prefix: ddecal.nchan=
+ separate: false
+ - id: maxiter
+ type: int?
+ default: 50
+ inputBinding:
+ position: 0
+ prefix: ddecal.maxiter=
+ separate: false
+ - id: tolerance
+ type: float?
+ default: 1e-3
+ inputBinding:
+ position: 0
+ prefix: ddecal.tolerance=
+ separate: false
+ - id: uvlambdamin
+ type: int?
+ default: 300
+ inputBinding:
+ position: 0
+ prefix: ddecal.uvlambdamin=
+ separate: false
+ - id: solint
+ type: int?
+ default: 1
+ inputBinding:
+ position: 0
+ prefix: ddecal.solint=
+ separate: false
+
+#--------------------
- id: save2json
default: true
type: boolean?
inputBinding:
position: 0
prefix: count.savetojson=True
-
- id: jsonfilename
type: string?
default: 'out.json'
diff --git a/lofar-cwl/steps/ms_concat.cwl b/lofar-cwl/steps/ms_concat.cwl
index 87c36a77..5741e4ad 100644
--- a/lofar-cwl/steps/ms_concat.cwl
+++ b/lofar-cwl/steps/ms_concat.cwl
@@ -71,3 +71,4 @@ requirements:
inplaceUpdate: true
- class: InlineJavascriptRequirement
stdout: ms_concat.log
+stderr: ms_concat_err.log
\ No newline at end of file
diff --git a/steps/check_unflagged_fraction.cwl b/steps/check_unflagged_fraction.cwl
index 474352e5..59c6e95d 100644
--- a/steps/check_unflagged_fraction.cwl
+++ b/steps/check_unflagged_fraction.cwl
@@ -37,15 +37,19 @@ requirements:
output = check_unflagged_fraction(ms, min_fraction=min_fraction, print_fraction=True)
- filename = os.path.basename(output['flagged'])
+ filename = os.path.basename(output['filename'].rstrip('/'))
unflagged_fraction = output['unflagged_fraction']
-
- if filename != 'None':
- shutil.move(filename, filename.lstrip('out_'))
+ flagged = output['flagged']
cwl_output = {}
cwl_output['unflagged_fraction'] = unflagged_fraction
+ if flagged != 'None':
+ shutil.move(filename, filename.lstrip('out_'))
+ cwl_output['filename'] = 'None'
+ else:
+ cwl_output['filename'] = filename
+
with open('./out.json', 'w') as fp:
json.dump(cwl_output, fp)
@@ -60,6 +64,12 @@ outputs:
loadContents: true
glob: 'out.json'
outputEval: $(JSON.parse(self[0].contents).unflagged_fraction)
+ - id: filenames
+ type: string
+ outputBinding:
+ loadContents: true
+ glob: 'out.json'
+ outputEval: $(JSON.parse(self[0].contents).filename)
- id: logfile
type: File[]
outputBinding:
diff --git a/steps/findRefAnt_join.cwl b/steps/findRefAnt_join.cwl
index 20c5fa14..e5a3e114 100644
--- a/steps/findRefAnt_join.cwl
+++ b/steps/findRefAnt_join.cwl
@@ -92,7 +92,7 @@ requirements:
min_flagged_fraction = min(flagged_fraction_list)
refant = flagged_fraction_data[min_flagged_fraction][0]
- print('Selected station ' + str(refant) + ' as reference antenna. Fraction of flagged data is ' + '{:>3}'.format('{:.1f}'.format(min_flagged_fraction) + '%'))
+ print('Selected station ' + str(refant) + ' as reference antenna. Fraction of flagged data is ' + '{:>3}'.format('{:.1f}'.format(100 * min_flagged_fraction) + '%'))
flagged_fraction_antenna['state'] = inputs['state']
diff --git a/steps/summary.cwl b/steps/summary.cwl
index bc3075cc..8a9d9ef1 100644
--- a/steps/summary.cwl
+++ b/steps/summary.cwl
@@ -88,12 +88,37 @@ inputs:
separate: true
itemSeparator: ','
valueFrom: '$(self)'
+ - id: removed_bands
+ default: false
+ type:
+ - string?
+ - boolean?
+ inputBinding:
+ position: 0
+ prefix: '--removed_bands'
+ separate: true
+ itemSeparator: ','
+ valueFrom: '$(self)'
+ - id: min_unflagged_fraction
+ default: 0.5
+ type: float?
+ inputBinding:
+ position: 0
+ prefix: '--min_unflagged'
+ doc: minimum fraction of unflagged data per band to continue
- id: demix
default: false
type: boolean?
inputBinding:
position: 0
prefix: '--demix True'
+ - id: refant
+ default: ''
+ type: string?
+ inputBinding:
+ position: 0
+ prefix: '--refant'
+ doc: Reference antenna used
- id: output_fname
type:
- boolean?
diff --git a/workflows/prefactor_target.cwl b/workflows/prefactor_target.cwl
index b73396f4..19dd4510 100644
--- a/workflows/prefactor_target.cwl
+++ b/workflows/prefactor_target.cwl
@@ -413,7 +413,9 @@ steps:
- id: Ateam_flags_join_out
- id: inspection
- id: logfiles
+ - id: removed_bands
- id: total_bandwidth
+ - id: out_refant
run: ./prefactor_target/gsmcal.cwl
label: gsmcal
'sbg:x': 900
@@ -461,6 +463,12 @@ steps:
source: demix_sources
- id: ncores
source: ncores
+ - id: removed_bands
+ source: gsmcal/removed_bands
+ - id: min_unflagged_fraction
+ source: min_unflagged_fraction
+ - id: refant
+ source: gsmcal/out_refant
out:
- id: msout
- id: solutions
diff --git a/workflows/prefactor_target/finalize.cwl b/workflows/prefactor_target/finalize.cwl
index aba41969..f34e0ab0 100644
--- a/workflows/prefactor_target/finalize.cwl
+++ b/workflows/prefactor_target/finalize.cwl
@@ -5,6 +5,21 @@ label: finalize
$namespaces:
sbg: 'https://www.sevenbridges.com/'
inputs:
+ - id: refant
+ type: string?
+ default: 'CS001HBA0'
+ 'sbg:x': -1000
+ 'sbg:y': -1000
+ - id: min_unflagged_fraction
+ type: float?
+ default: 0.5
+ 'sbg:x': -1000
+ 'sbg:y': -900
+ - id: removed_bands
+ type: 'string[]?'
+ default: []
+ 'sbg:x': -1000
+ 'sbg:y': -800
- id: demix_sources
type: 'string[]?'
default: []
@@ -276,6 +291,13 @@ steps:
- id: demix_sources
source: demix_sources
valueFrom: "$(self.join(','))"
+ - id: removed_bands
+ source: removed_bands
+ valueFrom: "$(self.join(','))"
+ - id: min_unflagged_fraction
+ source: min_unflagged_fraction
+ - id: refant
+ source: refant
out:
- id: summary_file
- id: logfile
diff --git a/workflows/prefactor_target/gsmcal.cwl b/workflows/prefactor_target/gsmcal.cwl
index e6b89b06..83cac3a8 100644
--- a/workflows/prefactor_target/gsmcal.cwl
+++ b/workflows/prefactor_target/gsmcal.cwl
@@ -120,6 +120,12 @@ outputs:
linkMerge: merge_flattened
'sbg:x': 1900
'sbg:y': 800
+ - id: out_refant
+ outputSource:
+ - findRefAnt_join/refant
+ type: string
+ 'sbg:x': 1900
+ 'sbg:y': 900
- id: logfiles
outputSource:
- concat_logfiles_identify/output
@@ -137,6 +143,12 @@ outputs:
linkMerge: merge_flattened
'sbg:x': 1900
'sbg:y': 1000
+ - id: removed_bands
+ outputSource:
+ - check_unflagged_fraction/filenames
+ type: 'string[]'
+ 'sbg:x': 1900
+ 'sbg:y': 1100
- id: total_bandwidth
outputSource:
- sort_times_into_freqGroups/total_bandwidth
@@ -434,6 +446,7 @@ steps:
out:
- id: msout
- id: unflagged_fraction
+ - id: filenames
- id: logfile
run: ../../steps/check_unflagged_fraction.cwl
label: check_unflagged_fraction
--
GitLab