diff --git a/Docker/Dockerfile b/Docker/Dockerfile
index 311e49dd68e9effdbc6f917401dd38efa1d519c3..61221e79b04e6d112d64ee1253498a5b5d4ecb35 100644
--- a/Docker/Dockerfile
+++ b/Docker/Dockerfile
@@ -10,6 +10,9 @@ RUN \
apt-get update && \
apt-get install -y vim wget casacore-tools && \
rm -rf /var/cache/apt
+RUN \
+ pip3 install --upgrade pip && \
+ pip3 install h5py
RUN \
pip3 install --upgrade pip && \
git clone https://github.com/lofar-astron/prefactor.git ${PREFACTOR_NAME} && \
diff --git a/docs/source/running.rst b/docs/source/running.rst
index 1e746da656864d02930f3f012d96ca212da007a4..c66cec5cbe6ddb3e3b6c42704c1906eeeb9d8230 100644
--- a/docs/source/running.rst
+++ b/docs/source/running.rst
@@ -41,9 +41,10 @@ Pipeline options for ``cwltool``
The following ``<cwl_options>`` are recommended to use for running **prefactor** with ``cwltool``:
- * **---outdir**: specifies the location of the pipeline outputs
+ * **---outdir**: specifies the location of the final pipeline output directory (results)
* **---tmpdir-prefix**: specifies the location of the intermediate data products
* **---parallel**: jobs will run in parallel
+ * **---singularity**: use Singularity instead of Docker
* **---no-container**: don't use Docker container (only for manual installation)
* **---preserve-entire-environment**: use system environment variables (only for manual installation)
* **---debug**: more verbose output (use only for debugging the pipeline)
@@ -74,6 +75,7 @@ The following ``<cwl_options>`` are recommended to use for running **prefactor**
* **---stats**: creates runtime statistics
* **---maxLocalJobs**: amount of local jobs to be run at the same time ("max_per_node")
* **---retryCount**: amount of retries for each failed pipeline job
+ * **---singularity**: use Singularity instead of Docker
* **---preserve-entire-environment**: use system environment variables (only for manual installation)
* **---no-container**: don't use Docker container (only for manual installation)
diff --git a/lofar-cwl/steps/ddecal.cwl b/lofar-cwl/steps/ddecal.cwl
index e89280ec948302705401a758cf3fe62dbebc816d..09011bba50e3d7c6233f117e2b69818172ded767 100644
--- a/lofar-cwl/steps/ddecal.cwl
+++ b/lofar-cwl/steps/ddecal.cwl
@@ -18,14 +18,6 @@ hints:
arguments:
- steps=[ddecal,count]
- - ddecal.uvlambdamin=300
- - ddecal.maxiter=50
- - ddecal.nchan=1
- - ddecal.solint=1
- - ddecal.tolerance=1.e-3
- - ddecal.usemodelcolumn=True
- - ddecal.flagdivergedonly=True
- - ddecal.propagateconvergedonly=True
inputs:
- id: msin
@@ -71,6 +63,16 @@ inputs:
default: true
inputBinding:
prefix: ddecal.propagatesolutions=True
+ - id: propagate_converged_only
+ type: boolean?
+ default: true
+ inputBinding:
+ prefix: ddecal.propagateconvergedonly=True
+ - id: usemodelcolumn
+ type: boolean?
+ default: true
+ inputBinding:
+ prefix: ddecal.usemodelcolumn=True
- id: flagunconverged
type: boolean?
default: false
@@ -78,9 +80,9 @@ inputs:
Flag unconverged solutions (i.e., those from solves that did not converge
within maxiter iterations).
inputBinding:
- prefix: ddecal.flagdivergedonly=True
+ prefix: ddecal.flagunconverged=True
- id: flagdivergedonly
- default: false
+ default: true
type: boolean?
doc: |
Flag only the unconverged solutions for which divergence was detected.
@@ -119,14 +121,49 @@ inputs:
inputBinding:
prefix: ddecal.mode=
separate: false
-
+ - id: nchan
+ type: int?
+ default: 1
+ inputBinding:
+ position: 0
+ prefix: ddecal.nchan=
+ separate: false
+ - id: maxiter
+ type: int?
+ default: 50
+ inputBinding:
+ position: 0
+ prefix: ddecal.maxiter=
+ separate: false
+ - id: tolerance
+ type: float?
+ default: 1e-3
+ inputBinding:
+ position: 0
+ prefix: ddecal.tolerance=
+ separate: false
+ - id: uvlambdamin
+ type: int?
+ default: 300
+ inputBinding:
+ position: 0
+ prefix: ddecal.uvlambdamin=
+ separate: false
+ - id: solint
+ type: int?
+ default: 1
+ inputBinding:
+ position: 0
+ prefix: ddecal.solint=
+ separate: false
+
+#--------------------
- id: save2json
default: true
type: boolean?
inputBinding:
position: 0
prefix: count.savetojson=True
-
- id: jsonfilename
type: string?
default: 'out.json'
diff --git a/lofar-cwl/steps/ms_concat.cwl b/lofar-cwl/steps/ms_concat.cwl
index 87c36a77627c1b8891fb8e0d3bbb0a5dd0e0a874..5741e4addd9700cc1e62509a2a76fae589195260 100644
--- a/lofar-cwl/steps/ms_concat.cwl
+++ b/lofar-cwl/steps/ms_concat.cwl
@@ -71,3 +71,4 @@ requirements:
inplaceUpdate: true
- class: InlineJavascriptRequirement
stdout: ms_concat.log
+stderr: ms_concat_err.log
\ No newline at end of file
diff --git a/steps/check_unflagged_fraction.cwl b/steps/check_unflagged_fraction.cwl
index 474352e5227a9719c93c07d0d9d14cfa6725fa1d..59c6e95d52fef4c10258ed966a952629afef248a 100644
--- a/steps/check_unflagged_fraction.cwl
+++ b/steps/check_unflagged_fraction.cwl
@@ -37,15 +37,19 @@ requirements:
output = check_unflagged_fraction(ms, min_fraction=min_fraction, print_fraction=True)
- filename = os.path.basename(output['flagged'])
+ filename = os.path.basename(output['filename'].rstrip('/'))
unflagged_fraction = output['unflagged_fraction']
-
- if filename != 'None':
- shutil.move(filename, filename.lstrip('out_'))
+ flagged = output['flagged']
cwl_output = {}
cwl_output['unflagged_fraction'] = unflagged_fraction
+ if flagged != 'None':
+ shutil.move(filename, filename.lstrip('out_'))
+ cwl_output['filename'] = 'None'
+ else:
+ cwl_output['filename'] = filename
+
with open('./out.json', 'w') as fp:
json.dump(cwl_output, fp)
@@ -60,6 +64,12 @@ outputs:
loadContents: true
glob: 'out.json'
outputEval: $(JSON.parse(self[0].contents).unflagged_fraction)
+ - id: filenames
+ type: string
+ outputBinding:
+ loadContents: true
+ glob: 'out.json'
+ outputEval: $(JSON.parse(self[0].contents).filename)
- id: logfile
type: File[]
outputBinding:
diff --git a/steps/findRefAnt_join.cwl b/steps/findRefAnt_join.cwl
index 20c5fa1495c4e95822ef1140f51d588a3ac6ce64..e5a3e1148257a2b0f21ba4ac2d8708d3ccc80436 100644
--- a/steps/findRefAnt_join.cwl
+++ b/steps/findRefAnt_join.cwl
@@ -92,7 +92,7 @@ requirements:
min_flagged_fraction = min(flagged_fraction_list)
refant = flagged_fraction_data[min_flagged_fraction][0]
- print('Selected station ' + str(refant) + ' as reference antenna. Fraction of flagged data is ' + '{:>3}'.format('{:.1f}'.format(min_flagged_fraction) + '%'))
+ print('Selected station ' + str(refant) + ' as reference antenna. Fraction of flagged data is ' + '{:>3}'.format('{:.1f}'.format(100 * min_flagged_fraction) + '%'))
flagged_fraction_antenna['state'] = inputs['state']
diff --git a/steps/summary.cwl b/steps/summary.cwl
index bc3075ccb197e4514d1ea7b4664bc46f58ba107b..8a9d9ef1a449d9b83665df92eab12d0ebfc543a2 100644
--- a/steps/summary.cwl
+++ b/steps/summary.cwl
@@ -88,12 +88,37 @@ inputs:
separate: true
itemSeparator: ','
valueFrom: '$(self)'
+ - id: removed_bands
+ default: false
+ type:
+ - string?
+ - boolean?
+ inputBinding:
+ position: 0
+ prefix: '--removed_bands'
+ separate: true
+ itemSeparator: ','
+ valueFrom: '$(self)'
+ - id: min_unflagged_fraction
+ default: 0.5
+ type: float?
+ inputBinding:
+ position: 0
+ prefix: '--min_unflagged'
+ doc: minimum fraction of unflagged data per band to continue
- id: demix
default: false
type: boolean?
inputBinding:
position: 0
prefix: '--demix True'
+ - id: refant
+ default: ''
+ type: string?
+ inputBinding:
+ position: 0
+ prefix: '--refant'
+ doc: Reference antenna used
- id: output_fname
type:
- boolean?
diff --git a/workflows/prefactor_target.cwl b/workflows/prefactor_target.cwl
index b73396f46e96f8ee2824b9cc27c7c479b958709b..19dd45106239f203b3e8970a3d2ec53b2d1f5abd 100644
--- a/workflows/prefactor_target.cwl
+++ b/workflows/prefactor_target.cwl
@@ -413,7 +413,9 @@ steps:
- id: Ateam_flags_join_out
- id: inspection
- id: logfiles
+ - id: removed_bands
- id: total_bandwidth
+ - id: out_refant
run: ./prefactor_target/gsmcal.cwl
label: gsmcal
'sbg:x': 900
@@ -461,6 +463,12 @@ steps:
source: demix_sources
- id: ncores
source: ncores
+ - id: removed_bands
+ source: gsmcal/removed_bands
+ - id: min_unflagged_fraction
+ source: min_unflagged_fraction
+ - id: refant
+ source: gsmcal/out_refant
out:
- id: msout
- id: solutions
diff --git a/workflows/prefactor_target/finalize.cwl b/workflows/prefactor_target/finalize.cwl
index aba4196902cdd12ad5e3bd4a917af00ce8c0b2ed..f34e0ab06a50f3a526c3d473fd813c0eb2166d04 100644
--- a/workflows/prefactor_target/finalize.cwl
+++ b/workflows/prefactor_target/finalize.cwl
@@ -5,6 +5,21 @@ label: finalize
$namespaces:
sbg: 'https://www.sevenbridges.com/'
inputs:
+ - id: refant
+ type: string?
+ default: 'CS001HBA0'
+ 'sbg:x': -1000
+ 'sbg:y': -1000
+ - id: min_unflagged_fraction
+ type: float?
+ default: 0.5
+ 'sbg:x': -1000
+ 'sbg:y': -900
+ - id: removed_bands
+ type: 'string[]?'
+ default: []
+ 'sbg:x': -1000
+ 'sbg:y': -800
- id: demix_sources
type: 'string[]?'
default: []
@@ -276,6 +291,13 @@ steps:
- id: demix_sources
source: demix_sources
valueFrom: "$(self.join(','))"
+ - id: removed_bands
+ source: removed_bands
+ valueFrom: "$(self.join(','))"
+ - id: min_unflagged_fraction
+ source: min_unflagged_fraction
+ - id: refant
+ source: refant
out:
- id: summary_file
- id: logfile
diff --git a/workflows/prefactor_target/gsmcal.cwl b/workflows/prefactor_target/gsmcal.cwl
index e6b89b0660cb4cc5d0c3d200981a7f3eeb54170a..83cac3a8d3258c936c8f2ab6ca5d8c6036e59421 100644
--- a/workflows/prefactor_target/gsmcal.cwl
+++ b/workflows/prefactor_target/gsmcal.cwl
@@ -120,6 +120,12 @@ outputs:
linkMerge: merge_flattened
'sbg:x': 1900
'sbg:y': 800
+ - id: out_refant
+ outputSource:
+ - findRefAnt_join/refant
+ type: string
+ 'sbg:x': 1900
+ 'sbg:y': 900
- id: logfiles
outputSource:
- concat_logfiles_identify/output
@@ -137,6 +143,12 @@ outputs:
linkMerge: merge_flattened
'sbg:x': 1900
'sbg:y': 1000
+ - id: removed_bands
+ outputSource:
+ - check_unflagged_fraction/filenames
+ type: 'string[]'
+ 'sbg:x': 1900
+ 'sbg:y': 1100
- id: total_bandwidth
outputSource:
- sort_times_into_freqGroups/total_bandwidth
@@ -434,6 +446,7 @@ steps:
out:
- id: msout
- id: unflagged_fraction
+ - id: filenames
- id: logfile
run: ../../steps/check_unflagged_fraction.cwl
label: check_unflagged_fraction