merge HBA_calibrator to master

Former-commit-id: fd9dcce3 [formerly ef21e782]
Former-commit-id: b3fdae7a
Former-commit-id: 8ef341c6

Docker/Dockerfile

 FROM lofareosc/lofar-pipeline:latest
 USER root
+
+RUN apt update && apt install -y vim wget casacore-tools && rm -rf /var/cache/apt 
+RUN pip3 install --upgrade pip && pip3 install --upgrade numpy scipy
 COPY prefactor/scripts/* /opt/lofar/scripts/
 COPY prefactor/skymodels/* /data/skymodels/
 COPY prefactor/rfistrategies/* /data/rfistrategies/
-COPY prefactor/solutions/ /data/solutions/*
-RUN apt update && apt install -y vim 
-RUN pip3 install --upgrade pip && pip3 install --upgrade numpy scipy
-RUN ln -sf /opt/lofar/scripts/* /usr/local/bin/
+COPY prefactor/solutions/* /data/solutions/
 
+RUN ln -sf /opt/lofar/scripts/* /usr/local/bin/
 RUN chmod +rx /usr/local/bin/*
 ENV PYTHONPATH="/opt/lofar/scripts:${PYTHONPATH}"
 ENTRYPOINT ["/home/lofaruser/.entrypoint"]

Docker/build_docker.sh

@@ -17,11 +17,9 @@ Prefactor_TAG=production
 
 # FETCHES Prefactor
 git_clone_or_pull https://github.com/lofar-astron/prefactor.git ${Prefactor_TAG} prefactor
-
 SCRIPT_PATH=$(realpath ${BASH_SOURCE[0]})
 
 DOCKER_PATH=$(dirname ${SCRIPT_PATH})
 
 docker build ${DOCKER_PATH} -t lofareosc/prefactor
-
 docker build ${DOCKER_PATH} -f ${DOCKER_PATH}/Dockerfile_ci -t lofareosc/prefactor-ci

lofar-cwl/steps/AOFlagger.cwl

@@ -13,7 +13,7 @@ requirements:
 
 hints:
   DockerRequirement:
-    dockerPull: lofareosc/lofar-pipeline-ci:latest
+    dockerPull: lofareosc/prefactor:HBAcalibrator
 
 baseCommand:
   - aoflagger

lofar-cwl/steps/Ateamclipper.cwl

+class: CommandLineTool
+cwlVersion: v1.0
+$namespaces:
+  sbg: 'https://www.sevenbridges.com/'
+id: check_ateam_separation
+baseCommand:
+  - python3
+  - /usr/local/bin/Ateamclipper.py
+inputs:
+  - id: msin
+    type:
+      - Directory
+      - type: array
+        items: Directory
+    inputBinding:
+      position: 0
+    doc: Input measurement set
+outputs:
+  - id: msout
+    doc: Output MS
+    type: Directory
+    outputBinding:
+      glob: $(inputs.msin.basename)
+  - id: logfile
+    type: 'File[]'
+    outputBinding:
+      glob: Ateamclipper.log
+  - id: output
+    type: File
+    outputBinding:
+      glob: Ateamclipper.txt
+label: Ateamclipper
+hints:
+  - class: InitialWorkDirRequirement
+    listing:
+      - entry: $(inputs.msin)
+        writable: true
+  - class: DockerRequirement
+    dockerPull: lofareosc/prefactor:HBAcalibrator
+  - class: InlineJavascriptRequirement
+stdout: Ateamclipper.log

lofar-cwl/steps/DP3.ApplyCalStepGenerator.cwl

@@ -14,7 +14,7 @@ inputs:
     default: applycal
 
   - id: parmdb
-    type: string
+    type: File
     doc: Path of parmdb in which the parameters are stored. This can also be an H5Parm file, in that case the filename has to end in '.h5'
   - id: solset
     type: string?
@@ -36,6 +36,10 @@ inputs:
        - scalaramplitude
        - rotationmeasure
        - fulljones
+       - polalign
+       - bandpass
+       - RMextract
+       - phaseOrig
     doc: |
       Type of correction to perform, can be one of 'gain', 'tec', 'clock',
       '(common)rotationangle' / 'rotation', '(common)scalarphase',
@@ -88,8 +92,6 @@ steps:
     in:
       - id: step_type
         default: 'applycal'
-      - id: output_files
-        default: ["parmdb"]
       - id: step_id
         source: step_name
       - id: steps

lofar-cwl/steps/DP3.AveragerStepGenerator.cwl

@@ -41,7 +41,7 @@ inputs:
       zero, it overrides <step>.timestep
     default: 0
   - id: freqresolution
-    type: float
+    type: string
     doc: >-
       Target frequency resolution, in Hz (or append "MHz" or "kHz" to specify it
       in those units). If this is given, and bigger than zero, it overrides

lofar-cwl/steps/DP3.Execute.cwl

@@ -41,6 +41,18 @@ outputs:
     type: Directory
     'sbg:x': 224.60113525390625
     'sbg:y': 130
+  - id: logfile
+    outputSource:
+      - dppp/logfile
+    type: 'File[]'
+    'sbg:x': 185.1875
+    'sbg:y': 334.5
+  - id: parset
+    outputSource:
+      - generic_step/parset
+    type: File
+    'sbg:x': 11.1875
+    'sbg:y': -299.5
 steps:
   - id: generic_step
     in:

lofar-cwl/steps/DP3.FilterStepGenerator.cwl

+#!/usr/bin/env cwl-runner
+
+class: Workflow
+cwlVersion: v1.0
+$namespaces:
+  sbg: 'https://www.sevenbridges.com/'
+id: dp3-filter-step-generator
+
+inputs:
+  - id: steps
+    type: Any[]?
+    default: []
+  - id: step_name
+    type: string
+    doc: unique name for the step
+    default: filter
+  - id: baseline
+    type: string
+    doc: baseline to keeps
+  - id: remove
+    type: boolean
+    doc: Remove or flag
+outputs:
+- id: augmented_steps
+  outputSource:
+    - DP3_GenericStep/augmented_steps
+  type: Any[]
+
+steps:
+  - id: DP3_GenericStep
+    in:
+    - id: steps
+      source: steps
+    - id: parameters
+      valueFrom: $(inputs)
+    - id: step_id
+      source: step_name
+    - id: step_type
+      default: 'filter'
+    - id: baseline
+      source: baseline
+    - id: remove
+      source: remove
+    out:
+      - augmented_steps
+    run: ../steps/DP3.GenericStep.cwl
+requirements:
+  - class: StepInputExpressionRequirement
+  - class: InlineJavascriptRequirement
+  - class: MultipleInputFeatureRequirement
\ No newline at end of file

lofar-cwl/steps/DP3.GenericStep.cwl

@@ -39,7 +39,7 @@ expression: |-
    delete inputs.parameters.steps;
    delete inputs.parameters.parameters;
    delete inputs.parameters.steps;
-
+   delete inputs.parameters.output_files;
 
    var new_step = {'step_type': step_type,
                    'step_id': step_id,

lofar-cwl/steps/DP3.PreflaggerStepGenerator.cwl

@@ -133,39 +133,31 @@ inputs:
       31*nchan/32..nchan-1] to flag the first and last 2 or 8 channels
       (depending on 64 or 256 channels in the observation).
   - id: amplmin
-    default: -1e+30
-    type: float
+    type: float?
     doc: >-
       Correlation data with amplitude < amplmin will match. It can be given per
       correlation. For example, amplmin=[100,,,100] matches data points with XX
       or YY amplitude < 100. The non-specified amplitudes get the default value.
   - id: amplmax
-    default: 1e+30
-    type: float
+    type: float?
     doc: Correlation data with amplitude > amplmax will match.
   - id: phasemin
-    default: -1e+30
-    type: float
+    type: float?
     doc: Correlation data with phase < phasemin (in radians) will match.
   - id: phasemax
-    default: 1e+30
-    type: float
+    type: float?
     doc: Correlation data with phase > phasemax (in radians) will match.
   - id: realmin
-    default: -1e+30
-    type: float
+    type: float?
     doc: Correlation data with real complex part < realmin will match.
   - id: realmax
-    default: 1e+30
-    type: float
+    type: float?
     doc: Correlation data with real complex part > realmax will match.
   - id: imagmin
-    default: -1e+30
-    type: float
+    type: float?
     doc: Correlation data with imaginary complex part < imagmin will match.
   - id: imagmax
-    default: 1e+30
-    type: float
+    type: float?
     doc: Correlation data with imaginary complex part > imagmax will match.
 outputs:
 - id: augmented_steps

lofar-cwl/steps/DPPP.cwl

@@ -17,7 +17,7 @@ inputs:
       prefix: msin=
       separate: false
     doc: Input Measurement Set
-  - default: out.MS
+  - default: "."
     id: msout_name
     type: string
     inputBinding:
@@ -37,23 +37,61 @@ inputs:
   - id: output_directory_names
     type: Any
     doc: Expected output file names
-  - id: autoweight
+  - default: false
+    id: autoweight
     type: boolean
-    default: true
     inputBinding:
-      prefix: -msin.autoweight=True
-  - id: baseline
+      position: 0
+      prefix: 'msin.autoweight=True'
+  - default: ''
+    id: baseline
     type: string
-    default: ""
     inputBinding:
-      prefix: -msin.baseline=
+      position: 0
+      prefix: 'msin.baseline='
+      separate: false
+  - default: DATA
+    id: output_column
+    type: string?
+    inputBinding:
+      position: 0
+      prefix: 'msout.datacolumn='
+      separate: false
+  - default: DATA
+    id: input_column
+    type: string?
+    inputBinding:
+      position: 0
+      prefix: 'msin.datacolumn='
       separate: false
+  - id: writefullresflag
+    type: boolean
+    default: false
+    inputBinding:
+       prefix: msout.writefullresflag=True
+  - id: overwrite
+    type: boolean
+    default: false
+    inputBinding:
+       prefix: msout.overwrite=True
+  - id: storagemanager
+    type: string
+    default: ""
+    inputBinding:
+       prefix: msout.storagemanager=
+       separate: false
+  - id: databitrate
+    type: int
+    default: 0
+    inputBinding:
+       prefix: msout.storagemanager.databitrate=
+       separate: false
 outputs:
   - id: msout
     doc: Output Measurement Set
     type: Directory
     outputBinding:
-      glob: '$(inputs.msout_name=="."?inputs.msin:inputs.msout_name)'
+      glob: '$(inputs.msout_name=="." ? inputs.msin.basename : inputs.msout_name)'
   - id: secondary_output_files
     doc: Secondary output files
     type: Any
@@ -89,12 +127,20 @@ outputs:
           return output
         }
   - id: logfile
-    type: File?
+    type: File[]
     outputBinding:
-      glob: DPPP.log
+      glob: 'DPPP*.log'
 hints:
   - class: DockerRequirement
-    dockerPull: lofareosc/lofar-pipeline
+    dockerPull: lofareosc/prefactor:HBAcalibrator
 stdout: DPPP.log
+stderr: DPPP_err.log
 requirements:
   - class: InlineJavascriptRequirement
+  - class: InitialWorkDirRequirement
+    listing:
+      - entry: $(inputs.msin)
+        writable: true
+      - entry: $(inputs.secondary_files)
+      - entry: $(inputs.secondary_directories)
+      

lofar-cwl/steps/H5ParmCollector.cwl

@@ -12,7 +12,9 @@ baseCommand:
   - H5parm_collector.py
 inputs:
   - id: h5parmFiles
-    type: 'File[]'
+    type:
+      - 'File[]'
+      - File
     format: lofar:#H5Parm
     inputBinding:
       position: 0
@@ -24,10 +26,26 @@ inputs:
       position: 0
       prefix: '--insolset'
     doc: Input solset name
+  - default: sol000
+    id: outsolset
+    type: string?
+    inputBinding:
+      position: 0
+      prefix: '--outsolset'
+    doc: Output solset name
+  - id: insoltab
+    type: string?
+    inputBinding:
+      position: 0
+      prefix: '--insoltab'
+    doc: Output solset name
   - default: output.h5
     id: outh5parmname
     type: string
     doc: Output h5parm name
+    inputBinding:
+      position: 0
+      prefix: '--outh5parm'
   - id: squeeze
     type: boolean
     default: false
@@ -49,6 +67,7 @@ inputs:
       position: 0
       prefix: '-c'
     doc: overwrite output
+    
 outputs:
   - id: outh5parm
     doc: Output h5parm
@@ -57,14 +76,14 @@ outputs:
     outputBinding:
       glob: $(inputs.outh5parmname)
   - id: log
-    type: File
+    type: File[]
     outputBinding:
-      glob: $(inputs.outh5parmname)-parm_collector_output.log
+      glob: '$(inputs.outh5parmname)-parm_collector_output*.log'
 label: H5parm_collector
 hints:
   - class: DockerRequirement
-    dockerPull: 'lofareosc/lofar-pipeline-ci:latest'
+    dockerPull: 'lofareosc/prefactor:HBAcalibrator'
 stdout: $(inputs.outh5parmname)-parm_collector_output.log
+stderr: $(inputs.outh5parmname)-parm_collector_output_err.log
 requirements:
-  - class: InlineJavascriptRequirement
-
+  - class: InlineJavascriptRequirement
\ No newline at end of file

lofar-cwl/steps/LoSoTo.Abs.cwl

@@ -25,12 +25,13 @@ requirements:
 baseCommand: "losoto"
 
 arguments:
+  - '--verbose'
   - $(inputs.input_h5parm.basename)
   - parset.config
 
 hints:
   DockerRequirement:
-    dockerPull: lofareosc/lofar-pipeline-ci:latest
+    dockerPull: lofareosc/prefactor:HBAcalibrator
 
 inputs:
   - id: input_h5parm

lofar-cwl/steps/LoSoTo.Clip.cwl

@@ -26,12 +26,13 @@ requirements:
 baseCommand: "losoto"
 
 arguments:
+  - '--verbose'
   - $(inputs.input_h5parm.basename)
   - parset.config
 
 hints:
   DockerRequirement:
-    dockerPull: lofareosc/lofar-pipeline-ci:latest
+    dockerPull: lofareosc/prefactor:HBAcalibrator
 
 inputs:
   - id: input_h5parm

lofar-cwl/steps/LoSoTo.ClockTec.cwl

@@ -26,12 +26,13 @@ requirements:
 baseCommand: "losoto"
 
 arguments:
+  - '--verbose'
   - $(inputs.input_h5parm.basename)
   - parset.config
 
 hints:
   DockerRequirement:
-    dockerPull: lofareosc/lofar-pipeline-ci:latest
+    dockerPull: lofareosc/prefactor:HBAcalibrator
 
 inputs:
   - id: input_h5parm
@@ -52,14 +53,14 @@ inputs:
   - id: tec3rdsoltabOut
     doc: output soltab name for tec3rd offset
     type: string?
-  - id: flagBadChannels
+  - id: FlagBadChannels
     type: boolean?
     doc: Detect and remove bad channel before fitting, by default True.
   - id: flagCut
     type: float?
   - id: chi2cut
     type: float?
-  - id: combinePol
+  - id: CombinePol
     type: boolean?
     doc: |
       Find a combined polarization solution, by default False.
@@ -67,11 +68,11 @@ inputs:
     type: boolean?
     doc: |
       Detect and remove phase wraps, by default True.
-  - id: fit3rdorder
+  - id: Fit3rdOrder
     type: boolean?
     doc: |
       Fit a 3rd order ionospheric ocmponent (usefult <40 MHz). By default False.
-  - id: circular
+  - id: Circular
     type: boolean?
     doc: |
       Assume circular polarization with FR not removed. By default False.
@@ -91,5 +92,16 @@ outputs:
     format: lofar:#H5Parm
     outputBinding:
       glob: $(inputs.input_h5parm.basename)
+  - id: parset
+    type: File
+    outputBinding:
+      glob: parset.config
+  - id: log
+    type: File[]
+    outputBinding:
+      glob: '$(inputs.input_h5parm.basename)-losoto*.log'
+
+stdout: $(inputs.input_h5parm.basename)-losoto.log
+stderr: $(inputs.input_h5parm.basename)-losoto_err.log
 $schema:
   - https://git.astron.nl/eosc/ontologies/raw/master/schema/lofar.owl

lofar-cwl/steps/LoSoTo.Duplicate.cwl

@@ -25,12 +25,13 @@ requirements:
 baseCommand: "losoto"
 
 arguments:
+  - '--verbose'
   - $(inputs.input_h5parm.basename)
   - parset.config
 
 hints:
   DockerRequirement:
-    dockerPull: lofareosc/lofar-pipeline-ci:latest
+    dockerPull: lofareosc/prefactor:HBAcalibrator
 
 inputs:
   - id: input_h5parm
@@ -48,8 +49,13 @@ outputs:
     format: lofar:#H5Parm
     outputBinding:
       glob: $(inputs.input_h5parm.basename)
+  - id: log
+    type: File[]
+    outputBinding:
+      glob: '$(inputs.input_h5parm.basename)-losoto*.log'
 
-
+stdout: $(inputs.input_h5parm.basename)-losoto.log
+stderr: $(inputs.input_h5parm.basename)-losoto_err.log
 $namespaces:
   lofar: https://git.astron.nl/eosc/ontologies/raw/master/schema/lofar.owl
 $schema:

lofar-cwl/steps/LoSoTo.Faraday.cwl

@@ -27,12 +27,13 @@ requirements:
 baseCommand: "losoto"
 
 arguments:
+  - '--verbose'
   - $(inputs.input_h5parm.basename)
   - parset.config
 
 hints:
   DockerRequirement:
-    dockerPull: lofareosc/lofar-pipeline-ci:latest
+    dockerPull: lofareosc/prefactor:HBAcalibrator
 
 inputs:
   - id: input_h5parm
@@ -41,6 +42,9 @@ inputs:
   - id: soltab
     type: string
     doc: "Solution table"
+  - id: soltabout
+    type: string?
+    doc: output table name (same solset)
   - id: refAnt
     type: string?
     doc:  Reference antenna, by default the first.
@@ -57,5 +61,13 @@ outputs:
     format: lofar:#H5Parm
     outputBinding:
       glob: $(inputs.input_h5parm.basename)
+  - id: log
+    type: File[]
+    outputBinding:
+      glob: '$(inputs.input_h5parm.basename)-losoto*.log'
+
+stdout: $(inputs.input_h5parm.basename)-losoto.log
+stderr: $(inputs.input_h5parm.basename)-losoto_err.log
+
 $schema:
   - https://git.astron.nl/eosc/ontologies/raw/master/schema/lofar.owl

lofar-cwl/steps/LoSoTo.Flag.cwl

@@ -27,12 +27,13 @@ requirements:
 baseCommand: "losoto"
 
 arguments:
+  - '--verbose'
   - $(inputs.input_h5parm.basename)
   - parset.config
 
 hints:
   DockerRequirement:
-    dockerPull: lofareosc/lofar-pipeline-ci:latest
+    dockerPull: lofareosc/prefactor:HBAcalibrator
 
 inputs:
   - id: input_h5parm
@@ -101,5 +102,12 @@ outputs:
     format: lofar:#H5Parm
     outputBinding:
       glob: $(inputs.input_h5parm.basename)
+  - id: log
+    type: File[]
+    outputBinding:
+      glob: '$(inputs.input_h5parm.basename)-losoto*.log'
+
+stdout: $(inputs.input_h5parm.basename)-losoto.log
+stderr: $(inputs.input_h5parm.basename)-losoto_err.log
 $schema:
   - https://git.astron.nl/eosc/ontologies/raw/master/schema/lofar.owl

lofar-cwl/steps/LoSoTo.FlagStation.cwl

@@ -26,12 +26,13 @@ requirements:
 baseCommand: "losoto"
 
 arguments:
+  - '--verbose'
   - $(inputs.input_h5parm.basename)
   - parset.config
 
 hints:
   DockerRequirement:
-    dockerPull: lofareosc/lofar-pipeline-ci:latest
+    dockerPull: lofareosc/prefactor:HBAcalibrator
 
 inputs:
   - id: input_h5parm
@@ -76,7 +77,7 @@ inputs:
     type: string?
     doc: Specifies the telescope if mode = 'bandpass'.
   - id: skipInternational
-    type: string?
+    type: boolean?
     doc: |
       If True, skip flagging of international LOFAR stations (only used if
        telescope = 'lofar')
@@ -100,5 +101,12 @@ outputs:
     format: lofar:#H5Parm
     outputBinding:
       glob: $(inputs.input_h5parm.basename)
+  - id: log
+    type: File[]
+    outputBinding:
+      glob: '$(inputs.input_h5parm.basename)-losoto*.log'
+
+stdout: $(inputs.input_h5parm.basename)-losoto.log
+stderr: $(inputs.input_h5parm.basename)-losoto_err.log
 $schema:
   - https://git.astron.nl/eosc/ontologies/raw/master/schema/lofar.owl

lofar-cwl/steps/LoSoTo.Flagextend.cwl

@@ -28,12 +28,13 @@ requirements:
 baseCommand: "losoto"
 
 arguments:
+  - '--verbose'
   - $(inputs.input_h5parm.basename)
   - parset.config
 
 hints:
   DockerRequirement:
-    dockerPull: lofareosc/lofar-pipeline-ci:latest
+    dockerPull: lofareosc/prefactor:HBAcalibrator
 
 inputs:
   - id: input_h5parm
@@ -67,5 +68,12 @@ outputs:
     format: lofar:#H5Parm
     outputBinding:
       glob: $(inputs.input_h5parm.basename)
+  - id: log
+    type: File[]
+    outputBinding:
+      glob: '$(inputs.input_h5parm.basename)-losoto*.log'
+
+stdout: $(inputs.input_h5parm.basename)-losoto.log
+stderr: $(inputs.input_h5parm.basename)-losoto_err.log
 $schema:
   - https://git.astron.nl/eosc/ontologies/raw/master/schema/lofar.owl