diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 20374b5dce75e13086547aa2154e882c435a9fca..c6be0c29784542c0548ddb274c98bc2a0de64517 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -44,139 +44,18 @@ validate_scripts:
script:
- for file in `find . -name 'steps/*.cwl'`; do cwltool --validate $file; done
-average_step_generator:
- stage: test_steps
- allow_failure: true
- script:
- - cwl-runner steps/DP3.AveragerStepGenerator.cwl test_jobs/averager_step_generator.json
- - ! cwl-runner steps/DP3.AveragerStepGenerator.cwl test_jobs/averager_step_generator_error.json
-
-aoflagger_step_generator:
- stage: test_steps
- allow_failure: true
- script:
- - cwl-runner steps/DP3.AOFlaggerStepGenerator.cwl test_jobs/aoflagger_step_generator.json
-
-preflagger_step_generator:
- stage: test_steps
- allow_failure: true
- script:
- - cwl-runner steps/DP3.PreflaggerStepGenerator.cwl test_jobs/preflagger_step_generator.json
-
-losoto_abs:
- stage: test_steps
- allow_failure: true
- script:
- - cwl-runner --no-container steps/LoSoTo.Abs.cwl test_jobs/losoto_abs.json
-
-losoto_duplicate:
- stage: test_steps
- allow_failure: true
- script:
- - cwl-runner --no-container steps/LoSoTo.Duplicate.cwl test_jobs/losoto_duplicate.json
-
-losoto_plot:
- stage: test_steps
- allow_failure: true
- script:
- - cwl-runner --no-container steps/LoSoTo.Plot.cwl test_jobs/losoto_plot.json
-
-losoto_norm:
- stage: test_steps
- allow_failure: true
- script:
- - cwl-runner --no-container steps/LoSoTo.Norm.cwl test_jobs/losoto_norm.json
-
-losoto_replicateonaxis:
- stage: test_steps
- allow_failure: true
- script:
- - cwl-runner --no-container steps/LoSoTo.Replicateonaxis.cwl test_jobs/losoto_replicateonaxis.json
-
-losoto_reset:
- stage: test_steps
- allow_failure: true
- script:
- - cwl-runner --no-container steps/LoSoTo.Reset.cwl test_jobs/losoto_reset.json
-
-losoto_residual:
- stage: test_steps
- allow_failure: true
- script:
- - cwl-runner --no-container steps/LoSoTo.Residual.cwl test_jobs/losoto_residual.json
-
-parset_selector:
- stage: test_steps
- allow_failure: true
- script:
- - cwl-runner steps/DP3.Selector.cwl test_jobs/selector.json
-
-file_selector:
- stage: test_steps
- allow_failure: true
- script:
- - cwl-runner steps/FileSelector.cwl test_jobs/file_selector.json
-
-NDPPP:
- stage: test_steps
- allow_failure: true
- script:
- - cwltool --no-container --preserve-environment LD_LIBRARY_PATH --preserve-environment PATH steps/DPPP.cwl test_jobs/NDPPP.json
-
-DP3.Execute:
- stage: test_steps
- allow_failure: true
- script:
- - cwltool --no-container --preserve-environment LD_LIBRARY_PATH --preserve-environment PATH steps/DP3.Execute.cwl test_jobs/dp3_execute.json
-
-h5parm_collector:
- stage: test_steps
- allow_failure: true
- script:
- - cwltool --no-container --preserve-environment LD_LIBRARY_PATH --preserve-environment PATH --preserve-environment PYTHONPATH steps/H5ParmCollector.cwl test_jobs/h5parm_collector.json
-
-aoflagger:
- stage: test_steps
- allow_failure: true
- script:
- - cwltool --no-container --preserve-environment LD_LIBRARY_PATH --preserve-environment PATH steps/AOFlagger.cwl test_jobs/aoflagger.json
-
find_skymodel_cal:
stage: test_steps
allow_failure: true
script:
- cwltool --no-container --preserve-environment LD_LIBRARY_PATH --preserve-environment PATH --preserve-environment PYTHONPATH steps/find_skymodel_cal.cwl test_jobs/find_skymodel_cal.json
-interpolate_visibilities:
- stage: test_steps
- allow_failure: true
- script:
- - cwltool --no-container --preserve-environment LD_LIBRARY_PATH --preserve-environment PATH --preserve-environment PYTHONPATH steps/interpolate_visibilities.cwl test_jobs/interpolate_visibilities.json
-
-h5parm_collector:
- stage: test_steps
- allow_failure: true
- script:
- - cwltool --no-container --preserve-environment LD_LIBRARY_PATH --preserve-environment PATH --preserve-environment PYTHONPATH steps/H5ParmCollector.cwl test_jobs/h5parm_collector.json
-
blsmooth:
stage: test_steps
allow_failure: true
script:
- cwltool --no-container --preserve-environment LD_LIBRARY_PATH --preserve-environment PATH --preserve-environment PYTHONPATH steps/blsmooth.cwl test_jobs/blsmooth.json
-ddecal:
- stage: test_steps
- allow_failure: true
- script:
- - cwltool --no-container --preserve-environment LD_LIBRARY_PATH --preserve-environment PATH --preserve-environment PYTHONPATH steps/ddecal.cwl test_jobs/ddecal.json
-
-predict:
- stage: test_steps
- allow_failure: true
- script:
- - cwltool --no-container --preserve-environment LD_LIBRARY_PATH --preserve-environment PATH --preserve-environment PYTHONPATH steps/predict.cwl test_jobs/predict.json
-
prefactor_calibrator:
stage: integration_test
script:
diff --git a/Docker/Dockerfile b/Docker/Dockerfile
index dcbb851bd6429ba05f5ae0e35af786af0ae8b60b..7b8b78a8b7f7c7fff4f642458082b1559993bbfb 100644
--- a/Docker/Dockerfile
+++ b/Docker/Dockerfile
@@ -1,140 +1,4 @@
-FROM kernsuite/base:5 as builder
-SHELL ["/bin/bash", "-c"]
-
-RUN export DEBIAN_FRONTEND=noninteractive && \
- apt-get update && \
- apt-get install -y \
- casacore-data \
- casacore-dev \
- libboost-python-dev \
- libcfitsio-dev \
- python-dev \
- python3-numpy \
- cmake \
- build-essential \
- libhdf5-serial-dev \
- libarmadillo-dev \
- libboost-filesystem-dev \
- libboost-system-dev \
- libboost-date-time-dev \
- libboost-signals-dev \
- libboost-program-options-dev \
- libboost-test-dev \
- libxml2-dev \
- libpng-dev \
- pkg-config \
- aoflagger-dev \
- libgtkmm-3.0-dev \
- git \
- wget \
- libfftw3-dev \
- libgsl-dev
-
-RUN mkdir -p /src/
-WORKDIR /src/
-
-
-# Build the IDG version from source
-
-
-ADD ./idg /src/idg/src
-RUN mkdir idg/build && cd idg/build && cmake -DCMAKE_INSTALL_PREFIX=/usr/local/ ../src && make install -j2
-
-ADD ./DP3 /src/dp3/src
-RUN cd /src/dp3/ && \
- mkdir build && \
- cd build && \
- cmake ../src -DCMAKE_INSTALL_PREFIX=/usr/local/ -DPORTABLE=True &&\
- make -j2 && \
- make install -j2
-
-ADD ./STMAN /src/STMAN/src
-RUN cd /src/STMAN/ && \
- mkdir build && \
- cd build && \
- cmake ../src -DCMAKE_INSTALL_PREFIX=/usr/local/ -DPORTABLE=True && \
- make -j2 && \
- make install -j2
-
-ADD ./Dysco /src/Dysco/src
-RUN cd /src/Dysco/ && \
- mkdir build && \
- cd build && \
- cmake ../src -DCMAKE_INSTALL_PREFIX=/usr/local/ -DPORTABLE=True && \
- make -j2 && \
- make install -j2
-
-#----------------------------------------------
-FROM kernsuite/base:5 as runner
-
-COPY --from=builder /usr/local/ /usr/local
-
-SHELL ["/bin/bash", "-c"]
-
-RUN export DEBIAN_FRONTEND=noninteractive && \
- apt-get update && \
- apt-get install -y \
- aoflagger \
- casacore-data \
- libarmadillo8 \
- libboost-date-time1.65.1 \
- libboost-filesystem1.65.1 \
- libboost-python1.65.1 \
- libboost-program-options1.65.1 \
- libboost-system1.65.1 \
- libboost-signals1.65.1 \
- libboost-test1.65.1 \
- libboost-python1.65-dev \
- libstationresponse2 \
- libcasa-* \
- libcfitsio5 \
- libgtkmm-3.0 \
- libfftw3-3 \
- libhdf5-cpp-100 \
- libpng16-16 \
- libxml2 \
- python3.7 \
- python3-casacore \
- python3-numpy \
- python3-scipy && \
- rm -rf /var/lib/apt/lists/*
-
-
-RUN mkdir /home/lofaruser/ && \
- groupadd -r lofaruser && \
- useradd --no-log-init -r -g lofaruser lofaruser && \
- chown lofaruser:lofaruser /home/lofaruser
-
-ARG TOIL_VERSION=3.20.0
-LABEL toil.version=${TOIL_VERSION}
-
-ARG LSMTool_TAG=master
-LABEL LSMTool.version.tag=${LSMTool_TAG}
-
-ARG LoSoTo_TAG=master
-LABEL LoSoTo.version.tag=${LoSoTo_TAG}
-
-ARG STMAN_TAG=master
-LABEL STMAN.version.tag=${STMAN_TAG}
-
-ARG Dysco_TAG=master
-LABEL Dysco.version.tag=${Dysco_TAG}
-
-
-RUN apt update && apt install -y nodejs python3-pip git wsclean
-RUN python3 -m pip install cwltool cwl-runner -e "git://github.com/darafferty/LSMTool.git@${LSMTool_TAG}#egg=LSMTool" \
- "toil[cwl]"==${TOIL_VERSION} \
- matplotlib
-
-RUN python3 -m pip install -e "git+https://github.com/revoltek/losoto.git@${LoSoTo_TAG}#egg=LoSoTo"
-
-ADD losoto.patch /home/lofaruser/losoto.patch
-RUN patch /src/losoto/losoto/operations/__init__.py /home/lofaruser/losoto.patch
-ADD .entrypoint /home/lofaruser/.entrypoint
-RUN chown lofaruser:lofaruser /home/lofaruser/.entrypoint && \
- chmod +rx /home/lofaruser/.entrypoint
-
-WORKDIR /home/lofaruser
+FROM lofareosc/lofar-pipeline:latest
COPY prefactor/scripts/* /opt/lofar/scripts/
COPY prefactor/skymodels/* /data/skymodels/
@@ -148,8 +12,3 @@ ENV PYTHONPATH="/opt/lofar/scripts:${PYTHONPATH}"
ENTRYPOINT ["/home/lofaruser/.entrypoint"]
USER lofaruser
-ARG DP3_TAG=master
-ARG IDG_TAG=master
-
-LABEL IDG.version.tag=${IDG_TAG}
-LABEL DP3.version.tag=${DP3_TAG}
diff --git a/Docker/build_docker.sh b/Docker/build_docker.sh
index 36e430f2b9b07baf1a79d14bb06143e010255bfc..82c8963f9a5d34980555822cbd168ebcc3ea757a 100755
--- a/Docker/build_docker.sh
+++ b/Docker/build_docker.sh
@@ -13,28 +13,8 @@ git_clone_or_pull () {
fi
}
-# SOFTWARE VERSIONS
-DP3_TAG=v4.1
-IDG_TAG=0.6
-LSMTool_TAG=v1.4.2
-LoSoTo_TAG=f0398b
-TOIL_VERSION=3.20.0
-STMAN_TAG=master
-Dysco_TAG=v1.2
Prefactor_TAG=master
-# FETCHES THE IDG
-git_clone_or_pull https://gitlab.com/astron-idg/idg/ ${IDG_TAG} idg
-
-# FETCHES DP3
-git_clone_or_pull https://github.com/lofar-astron/DP3 ${DP3_TAG} DP3
-
-# FETCHES STMAN
-git_clone_or_pull https://github.com/lofar-astron/LofarStMan ${STMAN_TAG} STMAN
-
-# FETCHES Dysco
-git_clone_or_pull https://github.com/aroffringa/dysco.git ${Dysco_TAG} Dysco
-
# FETCHES Prefactor
git_clone_or_pull https://github.com/lofar-astron/prefactor.git ${Prefactor_TAG} prefactor
@@ -42,13 +22,6 @@ SCRIPT_PATH=$(realpath ${BASH_SOURCE[0]})
DOCKER_PATH=$(dirname ${SCRIPT_PATH})
-docker build ${DOCKER_PATH} --build-arg=IDG_TAG=${IDG_TAG}\
- --build-arg=DP3_TAG=${DP3_TAG}\
- --build-arg=LSMTool_TAG=${LSMTool_TAG}\
- --build-arg=LoSoTo_TAG=${LoSoTo_TAG}\
- --build-arg=STMAN_TAG=${STMAN_TAG}\
- --build-arg=Dysco_TAG=${Dysco_TAG}\
- --build-arg=TOIL_VERSION=${TOIL_VERSION}\
- -t lofareosc/prefactor
+docker build ${DOCKER_PATH} -t lofareosc/prefactor
docker build ${DOCKER_PATH} -f ${DOCKER_PATH}/Dockerfile_ci -t lofareosc/prefactor-ci
diff --git a/steps/AOFlagger.cwl b/steps/AOFlagger.cwl
deleted file mode 100644
index e3421e14bc87cb1c9615f4e21f6a981a2849ebc8..0000000000000000000000000000000000000000
--- a/steps/AOFlagger.cwl
+++ /dev/null
@@ -1,121 +0,0 @@
-#!/usr/bin/env cwl-runner
-
-class: CommandLineTool
-cwlVersion: v1.0
-id: aoflagger
-
-requirements:
- - class: InlineJavascriptRequirement
- - class: InitialWorkDirRequirement
- listing:
- - entry: $(inputs.msin)
- writable: true
-
-hints:
- DockerRequirement:
- dockerPull: lofareosc/prefactor-ci:master
-
-baseCommand:
- - aoflagger
-inputs:
- - id: msin
- type: Directory[]
- doc: Input measurement set
- inputBinding:
- position: 0
- - id: verbose
- type: boolean?
-
- doc: will produce verbose output
- inputBinding:
- valueFrom: -v
- - id: n_threads
- type: int?
- doc: |
- overrides the number of threads specified in the strategy
- (default: one thread for each CPU core)
- inputBinding:
- prefix: -j
-
- - id: strategy
- type: File?
- doc: |
- specifies a customized strategy
- inputBinding:
- prefix: -strategy
-
-#### READ MODES
- - id: direct_read
- type: boolean?
- doc: |
- Will perform the slowest IO but will always work.
- inputBinding:
- prefix: -direct-read
- - id: indirect_read
- type: boolean?
- doc: |
- Will reorder the measurement set before starting, which is normally
- faster but requires free disk space to reorder the data to.
- inputBinding:
- prefix: -indirect-read
- - id: memory_read
- type: boolean?
- doc: |
- Will read the entire measurement set in memory. This is the fastest,
- but requires much memory.
- inputBinding:
- prefix: -memory-read
- - id: auto_read
- type: boolean?
- doc: |
- Will select either memory or direct mode based on available memory (default).
- inputBinding:
- prefix: -auto-read
-##############
- - id: skip_flagged
- type: boolean?
- doc: |
- Will skip an ms if it has already been processed by AOFlagger according
- to its HISTORY table.
- inputBinding:
- prefix: -skip-flagged
- - id: uvw
- type: File?
- doc: Reads uvw values (some exotic strategies require these)
- inputBinding:
- prefix: -uvw
- - id: column
- type: string?
- doc: Specify column to flag
- inputBinding:
- prefix: -column
- - id: bands
- type: int[]?
- doc: Comma separated list of (zero-indexed) band ids to process
- inputBinding:
- prefix: -bands
- itemSeparator: ','
- - id: fields
- type: int[]?
- doc: Comma separated list of (zero-indexed) field ids to process
- inputBinding:
- prefix: -fields
- itemSeparator: ','
- - id: combine-spws
- type: boolean?
- doc: Join all SPWs together in frequency direction before flagging
- inputBinding:
- prefix: -combine-spws
- - id: bandpass
- type: File?
- doc: Set bandpass correction file for any 'Apply passband' action
- inputBinding:
- prefix: bandpass
-
-outputs:
- - id: output_ms
- doc: Output image
- type: Directory[]
- outputBinding:
- outputEval: $(inputs.msin)
-label: AOFlagger
diff --git a/steps/DP3.AOFlaggerStepGenerator.cwl b/steps/DP3.AOFlaggerStepGenerator.cwl
deleted file mode 100755
index 2a70f602771d7e5e669ba46ee395550807bb379d..0000000000000000000000000000000000000000
--- a/steps/DP3.AOFlaggerStepGenerator.cwl
+++ /dev/null
@@ -1,132 +0,0 @@
-#!/usr/bin/env cwl-runner
-
-class: Workflow
-cwlVersion: v1.0
-id: aoflagger_step_generator
-
-inputs:
- - id: steps
- type: Any[]?
- default: []
- - id: step_name
- type: string
- doc: unique name for the step
- default: aoflag
- - id: count.save
- type: boolean
- default: false
- doc: >-
- If true, the flag percentages per frequency are saved to a table with
- extension .flagfreq and percentages per station to a table with extension
- .flagstat. The basename of the table is the MS name (without extension)
- followed by the stepname and extension.
- - id: count.path
- type: string
- default: ""
- doc: >-
- The directory where to create the flag percentages table. If empty, the
- path of the input MS is used.
- - id: strategy
- type: File?
- doc: >-
- The name of the strategy file to use. If no name is given, the default
- strategy is used which is fine for HBA. For LBA data the strategy
- LBAdefault should be used.
- - id: memoryperc
- type: int
- default: 0
- doc: >-
- If >0, percentage of the machines memory to use. If memorymax nor
- memoryperc is given, all memory will be used (minus 2 GB (at most 50%) for
- other purposes). Accepts only integer values (LOFAR v2.16). Limiting the
- available memory too much affects flagging accuracy; in general try to use
- at least 10 GB of memory.
- - id: memorymax
- type: double
- default: 0
- doc: >-
- Maximum amount of memory (in GB) to use. â‡0 means no maximum. As stated
- above, this affects flagging accuracy.
- - id: timewindow
- type: int
- default: 0
- doc: >-
- Number of time slots to be flagged jointly. The larger the time window,
- the better the flagging performs. 0 means that it will be deduced from the
- memory to use. Note that the time window can be extended with an overlap
- on the left and right side to minimize possible boundary effects.
- - id: overlapperc
- type: double
- default: 0
- doc: >-
- If >0, percentage of time window to be added to the left and right side
- for overlap purposes (to minimize boundary effects). If overlapmax is not
- given, it defaults to 1%.
- - id: overlapmax
- type: int
- default: 0
- doc: Maximum overlap value (0 is no maximum).
- - id: autocorr
- type: boolean
- default: true
- doc: Flag autocorrelations?
- - id: pulsar
- type: boolean
- default: false
- doc: Use flagging strategy optimized for pulsar observations?
- - id: pedantic
- type: boolean
- default: false
- doc: Be more pedantic when flagging?
- - id: keepstatistics
- type: boolean
- default: true
- doc: Write the quality statistics?
-outputs:
- - id: augmented_steps
- outputSource:
- - DP3_GenericStep/augmented_steps
- type: Any[]
-
-steps:
- - id: DP3_GenericStep
- in:
- - id: step_type
- default: 'aoflagger'
- - id: step_id
- source: step_name
- - id: steps
- source: steps
- - id: parameters
- valueFrom: $(inputs)
- - id: count.save
- source: count.save
- - id: count.path
- source: count.save
- - id: strategy
- source: strategy
- - id: memoryperc
- source: memoryperc
- - id: memorymax
- source: memorymax
- - id: timewindow
- source: timewindow
- - id: overlapperc
- source: overlapperc
- - id: overlapmax
- source: overlapmax
- - id: autocorr
- source: autocorr
- - id: pulsar
- source: pulsar
- - id: pedantic
- source: pedantic
- - id: keepstatistics
- source: keepstatistics
- out:
- - augmented_steps
- run: ../steps/DP3.GenericStep.cwl
-requirements:
- - class: StepInputExpressionRequirement
- - class: InlineJavascriptRequirement
- - class: MultipleInputFeatureRequirement
diff --git a/steps/DP3.AveragerStepGenerator.cwl b/steps/DP3.AveragerStepGenerator.cwl
deleted file mode 100755
index d5424919111a6c38f65053bfec5f2e299a98c076..0000000000000000000000000000000000000000
--- a/steps/DP3.AveragerStepGenerator.cwl
+++ /dev/null
@@ -1,85 +0,0 @@
-#!/usr/bin/env cwl-runner
-
-class: Workflow
-cwlVersion: v1.0
-id: average_step_generator
-
-inputs:
- - id: steps
- type: Any[]?
- default: []
- - id: step_name
- type: string
- doc: unique name for the step
- default: average
- - id: timestep
- type: int
- doc: >-
- Number of time slots to average. It is truncated if exceeding the actual
- number of times.
- default: 1
- - id: freqstep
- type: int
- doc: >-
- Number of channels to average. It is truncated if exceeding the actual
- number of channels.
- default: 1
- - id: minpoints
- type: int
- doc: >-
- If number of averaged unflagged input points < minpoints, the averaged
- point is flagged.
- default: 0
- - id: minperc
- type: float
- doc: 'Like minpoints, but expressed as a percentage of timestep*freqstep.'
- default: 0
- - id: timeresolution
- type: float
- doc: >-
- Target time resolution, in seconds. If this is given, and bigger than
- zero, it overrides <step>.timestep
- default: 0
- - id: freqresolution
- type: float
- doc: >-
- Target frequency resolution, in Hz (or append "MHz" or "kHz" to specify it
- in those units). If this is given, and bigger than zero, it overrides
- <step>.freqstep
- default: 0
-outputs:
- - id: augmented_steps
- outputSource:
- - DP3_GenericStep/augmented_steps
- type: Any[]
-
-steps:
- - id: DP3_GenericStep
- in:
- - id: step_type
- default: 'averager'
- - id: step_id
- source: step_name
- - id: steps
- source: steps
- - id: parameters
- valueFrom: $(inputs)
- - id: timestep
- source: timestep
- - id: freqstep
- source: freqstep
- - id: minpoints
- source: minpoints
- - id: minperc
- source: minperc
- - id: timeresolution
- source: timeresolution
- - id: freqresolution
- source: freqresolution
- out:
- - augmented_steps
- run: ../steps/DP3.GenericStep.cwl
-requirements:
- - class: StepInputExpressionRequirement
- - class: InlineJavascriptRequirement
- - class: MultipleInputFeatureRequirement
diff --git a/steps/DP3.DemixerStepGenerator.cwl b/steps/DP3.DemixerStepGenerator.cwl
deleted file mode 100755
index c88bdd1721d9a544f872ef89f814c361eb56dee5..0000000000000000000000000000000000000000
--- a/steps/DP3.DemixerStepGenerator.cwl
+++ /dev/null
@@ -1,190 +0,0 @@
-#!/usr/bin/env cwl-runner
-
-class: Workflow
-cwlVersion: v1.0
-$namespaces:
- sbg: 'https://www.sevenbridges.com/'
-id: demix_step_generator
-
-inputs:
- - id: steps
- type: Any[]?
- default: []
- - id: step_name
- type: string
- doc: unique name for the step
- default: demix
- - id: baseline
- default: ""
- type: string
- doc: Baselines to demix. See Description of baseline selection parameters.
- - id: blrange
- default: []
- type: double[]
- doc: Baselines to demix. See Description of baseline selection parameters.
- - id: corrtype
- default: cross
- type: string
- doc: >-
- Baselines to demix. Correlation type to match? Must be auto, cross, or an
- empty string.
- - id: timestep
- default: 1
- type: int
- doc: >-
- Number of time slots to average when subtracting. It is truncated if
- exceeding the actual number of times. Note that the data itself will also
- be averaged by this amount.
- - id: freqstep
- default: 1
- type: int
- doc: >-
- Number of channels to average when subtracting. It is truncated if
- exceeding the actual number of channels. Note that the data itself will
- also be averaged by this amount.
- - id: demixtimestep
- default: 0
- type: int
- doc: >-
- Number of time slots to average when demixing. It is truncated if
- exceeding the actual number of times. It defaults to the averaging used
- for the subtract.
- - id: demixfreqstep
- default: 0
- type: int
- doc: >-
- Number of channels to average when demixing. It is truncated if exceeding
- the actual number of channels. It defaults to the averaging used for the
- subtract.
- - id: ntimechunk
- default: null
- type: int
- doc: >-
- Number of demix time slots (after averaging) that are processed jointly in
- as much a parallel way as possible. If subtract uses different time
- averaging, it has to fit integrally.
- - id: skymodel
- default: sky
- type: [File, Directory, string]
- doc: The name of the SourceDB to use (i.e., the output of makesourcedb).
- - id: instrumentmodel
- default: instrument
- type: string
- doc: >-
- The name of the ParmDB to use. The ParmDB does not need to exist. If it
- does not exist it will be created.
- - id: subtractsources
- default: null
- type: string[]?
- doc: >-
- Names of the sources to subtract. If none are given, demixing comes down
- to averaging. The sources must exist as patches in the SourceDB.
- - id: modelsources
- default: []
- type: string[]
- doc: >-
- Names of sources with models to take into account when solving. the
- sources must exist as patches in the SourceDB. Note that the target should
- NOT be part of this parameter. If a model of the target has to be used, it
- has to be given in parameter targetsource.
- - id: targetsource
- default: ''
- type: string
- doc: >-
- It can be used to specify the name of the source model of the target. If
- given, the target source model (its patch in the SourceDB) is taken into
- account when solving; in this case parameter othersources cannot be given.
- It cannot be given if ignoretarget=true. If not given, the target is
- projected away or ignored (depending on parameter ignoretarget).
- - id: ignoretarget
- default: false
- type: boolean
- doc: false = project the target source away; true = ignore the target
- - id: othersources
- default: []
- type: string[]
- doc: >-
- Names of sources of which the direction is taken into account when
- demixing by projecting the directions away. The direction needs to be
- specified if the source is unknown (which is usually the case). It can be
- done using parameters <step>.<sourcename>.phasecenter.
- - id: propagatesolutions
- default: true
- type: boolean
- doc: >-
- If set to true, solutions of a time slot are used as initial values for
- the next time slot. If set to false, the diagonal elements of the Jones
- matrix are initialized to one and the off-diagonal elements to zero.
- - id: defaultgain
- default: 1
- type: double
- doc: >-
- The default and initial gain for the directional gains that are computed
- internally.
- - id: maxiter
- default: 50
- type: int
- doc: Maximum number of iterations used in the LM solve
-outputs:
-- id: augmented_steps
- outputSource:
- - DP3_GenericStep/augmented_steps
- type: Any[]
-
-steps:
- - id: DP3_GenericStep
- in:
- - id: steps
- source: steps
- - id: output_files
- default: ["instrumentmodel"]
-
- - id: parameters
- valueFrom: $(inputs)
- - id: step_id
- source: step_name
- - id: step_type
- default: 'demixer'
- - id: baseline
- source: baseline
- - id: blrange
- source: blrange
- - id: corrtype
- source: corrtype
- - id: timestep
- source: timestep
- - id: freqstep
- source: freqstep
- - id: demixtimestep
- source: demixtimestep
- - id: demixfreqstep
- source: demixfreqstep
- - id: ntimechunk
- source: ntimechunk
- - id: skymodel
- source: skymodel
- - id: instrumentmodel
- source: instrumentmodel
- - id: subtractsources
- source: subtractsources
- - id: modelsources
- source: modelsources
- - id: targetsource
- source: targetsource
- - id: ignoretarget
- source: ignoretarget
- - id: othersources
- source: othersources
- - id: propagatesolutions
- source: propagatesolutions
- - id: defaultgain
- source: defaultgain
- - id: maxiter
- source: maxiter
- out:
- - augmented_steps
- run: ../steps/DP3.GenericStep.cwl
-requirements:
- - class: StepInputExpressionRequirement
- - class: InlineJavascriptRequirement
- - class: MultipleInputFeatureRequirement
diff --git a/steps/DP3.EmptyParsetGenerator.cwl b/steps/DP3.EmptyParsetGenerator.cwl
deleted file mode 100755
index f3e2c175ced26ad327e9d56e473bbaf0039996a5..0000000000000000000000000000000000000000
--- a/steps/DP3.EmptyParsetGenerator.cwl
+++ /dev/null
@@ -1,25 +0,0 @@
-#!/usr/bin/env cwl-runner
-
-cwlVersion: v1.0
-class: CommandLineTool
-id: empty_parset_generator
-baseCommand: [echo, 'steps=[]']
-
-requirements:
- InlineJavascriptRequirement: {}
-
-inputs: []
-
-stdout: output_parset
-outputs:
- - id: output_parset
- doc: Empty parset file
- streamable: True
- type: File
- outputBinding:
- glob: output_parset
- - id: output_secondary_files
- doc: files needed to execute the step
- type: File[]
- outputBinding:
- outputEval: $([])
diff --git a/steps/DP3.Execute.cwl b/steps/DP3.Execute.cwl
deleted file mode 100644
index bf15ea45d9ac61ab6d23521605ea0955324e7e45..0000000000000000000000000000000000000000
--- a/steps/DP3.Execute.cwl
+++ /dev/null
@@ -1,79 +0,0 @@
-class: Workflow
-cwlVersion: v1.0
-id: dp3_execute
-label: DP3.Execute
-$namespaces:
- sbg: 'https://www.sevenbridges.com/'
-inputs:
- - id: msout_name
- type: string
- 'sbg:x': -425.39886474609375
- 'sbg:y': 127
- - id: msin
- type: Directory
- 'sbg:x': -254.39886474609375
- 'sbg:y': 242
- - id: steps
- type: 'Any[]?'
- 'sbg:x': -518.3988647460938
- 'sbg:y': -88
-outputs:
- - id: secondary_output_files
- outputSource:
- - dppp/secondary_output_files
- type: Any
- 'sbg:x': 213.60113525390625
- 'sbg:y': -184
- - id: secondary_output_directories
- outputSource:
- - dppp/secondary_output_directories
- type: Any
- 'sbg:x': 248.60113525390625
- 'sbg:y': -47
- - id: msout
- outputSource:
- - dppp/msout
- type: Directory
- 'sbg:x': 224.60113525390625
- 'sbg:y': 130
-steps:
- - id: generic_step
- in:
- - id: steps
- source:
- - steps
- out:
- - id: parset
- - id: input_files
- - id: input_directories
- - id: output_file_names
- - id: output_directory_names
- run: ./DP3.ParsetGenerator.cwl
- 'sbg:x': -294
- 'sbg:y': -84
- - id: dppp
- in:
- - id: parset
- source: generic_step/parset
- - id: msin
- source: msin
- - id: msout_name
- source: msout_name
- - id: secondary_files
- source:
- - generic_step/input_files
- - id: secondary_directories
- source:
- - generic_step/input_directories
- - id: output_file_names
- source: generic_step/output_file_names
- - id: output_directory_names
- source: generic_step/output_directory_names
- out:
- - id: msout
- - id: secondary_output_files
- - id: secondary_output_directories
- run: ./DPPP.cwl
- 'sbg:x': 26
- 'sbg:y': -46
-requirements: []
diff --git a/steps/DP3.GenericStep.cwl b/steps/DP3.GenericStep.cwl
deleted file mode 100755
index 38829da42ed0deaac51bcf92c7c105e9139d4108..0000000000000000000000000000000000000000
--- a/steps/DP3.GenericStep.cwl
+++ /dev/null
@@ -1,63 +0,0 @@
-#!/usr/bin/env cwl-runner
-
-cwlVersion: v1.0
-class: ExpressionTool
-id: generic_step
-
-requirements:
- - class: InlineJavascriptRequirement
-
-inputs:
- - id: steps
- type: Any[]
-
- - id: parameters
- type: Any?
-
-outputs:
- - id: augmented_steps
- type: Any[]
-expression: |-
- ${
- var step_type = inputs.parameters.step_type;
- var step_id = inputs.parameters.step_id;
-
- var output_file_type = inputs.parameters.output_files;
- var output_files = {}
-
- for(var index in output_file_type){
- var file_type = output_file_type[index]
- if(file_type != undefined && file_type != null){
- output_files[file_type] = inputs.parameters[file_type]
- }
- }
-
- var input_secondary_files = inputs.parameters.input_secondary_files;
-
- delete inputs.parameters.step_type;
- delete inputs.parameters.step_id;
- delete inputs.parameters.steps;
- delete inputs.parameters.parameters;
- delete inputs.parameters.steps;
-
-
- var new_step = {'step_type': step_type,
- 'step_id': step_id,
- 'parameters': inputs.parameters,
- 'output_files': output_files};
-
-
- var in_parameters = inputs.steps;
-
- // Checks if the step_id is duplicate if so raised an exception
- var step_ids = []
- for(var step in in_parameters){
- step = in_parameters[step]
- step_ids.push(step.step_id)
- }
- if(step_ids.indexOf(step_id) >= 0) throw 'step_id has to be unique'
-
- var out_parameters = in_parameters == null ? [new_step] : in_parameters.concat(new_step)
-
- return {'augmented_steps': out_parameters};
- }
diff --git a/steps/DP3.ParsetGenerator.cwl b/steps/DP3.ParsetGenerator.cwl
deleted file mode 100644
index f5837e9a3528135180143bee7fb406ef736b1efb..0000000000000000000000000000000000000000
--- a/steps/DP3.ParsetGenerator.cwl
+++ /dev/null
@@ -1,148 +0,0 @@
-#!/usr/bin/env cwl-runner
-
-cwlVersion: v1.0
-class: CommandLineTool
-id: generic_step
-
-requirements:
- - class: InlineJavascriptRequirement
-
-baseCommand:
- - python3
-
-arguments:
- - prefix: -c
- valueFrom: |
- import sys
-
- true = True
- false = False
- null = None
- steps_list = $(inputs.steps)
- steps_id_list = list(map(lambda item: item['step_id'], steps_list))
-
- for step in steps_list:
- try:
- step_id = step.get('step_id')
- step_type = step.get('step_type')
- parameters = step.get('parameters')
-
-
- line = '.'.join([step_id, 'type'])
- line += '=%s' % step_type
- print(line)
-
- for parameter_name, parameter in parameters.items():
- if isinstance(parameter, dict):
- if parameter['class'] in ['File', 'Directory']:
- parameter = parameter['basename']
- else:
- raise NotImplemented()
- if parameter is None:
- continue
- line = '.'.join([step_id, parameter_name])
- line = '%s=%s' % (line, parameter)
-
- print(line)
- except KeyError as e:
- print('invalid input for step ', step, ':', e)
- sys.exit(1)
- last_step_line = 'steps=[%s]' % ', '.join(steps_id_list)
- print(last_step_line)
-inputs:
- - id: steps
- type: Any[]?
-
-outputs:
- - id: parset
- doc: Parset output file
- type: File
- outputBinding:
- glob: parset
- - id: input_files
- doc: additional input files
- type: File[]
- outputBinding:
- outputEval: |-
- ${
- var outfiles = []
- for(var step in inputs.steps){
- step = inputs.steps[step]
- for(var parameter in step['parameters']){
- parameter = step['parameters'][parameter]
-
- if (parameter != null && parameter.class =='File'){
- outfiles.push(parameter)
- }
- }
- }
- return outfiles
- }
- - id: input_directories
- doc: additional input directory
- type: Directory[]
- outputBinding:
- outputEval: |
- ${
- var outdirs = []
- for(var step in inputs.steps){
- step = inputs.steps[step]
- for(var parameter in step['parameters']){
- parameter = step['parameters'][parameter]
- if (parameter != null && parameter.class=='Directory'){
- outdirs.push(parameter)
- }
- }
- }
- return outdirs
- }
-
- - id: output_file_names
- doc: expected output files
- type: Any
- outputBinding:
- outputEval: |
- ${
- var out_names = {}
- for(var step_idx in inputs.steps){
- var step = inputs.steps[step_idx]
- var step_id = step['step_id']
- var step_dirs = {}
- if(!step.hasOwnProperty("output_files")) continue;
-
- var step_files = {}
-
- for(var index in step['output_files']){
- var file_id = index
- var file_name = step['output_files'][file_id]
- step_files[file_id] = file_name
- }
-
- out_names[step_id] = step_files
- }
- return out_names
- }
- - id: output_directory_names
- doc: expected output directories
- type: Any
- outputBinding:
- outputEval: |-
- ${
- var out_names = {}
- for(var step_idx in inputs.steps){
- var step = inputs.steps[step_idx]
- var step_id = step['step_id']
- var step_dirs = {}
- if(!step.hasOwnProperty("output_directories")) continue;
-
- for(var index in step['output_directories']){
- var file_id = index
- var file_name = step['output_directories'][file_id]
- step_dirs[file_id] = file_name
- }
- out_names[step_id] = step_dirs
- }
- return out_names
- }
-stdout:
- parset
diff --git a/steps/DP3.PreflaggerStepGenerator.cwl b/steps/DP3.PreflaggerStepGenerator.cwl
deleted file mode 100755
index 03f6c11d6f16db4d72a38666e4a294a2f568fb28..0000000000000000000000000000000000000000
--- a/steps/DP3.PreflaggerStepGenerator.cwl
+++ /dev/null
@@ -1,246 +0,0 @@
-#!/usr/bin/env cwl-runner
-
-class: Workflow
-cwlVersion: v1.0
-id: preflag_step_generator
-
-inputs:
- - id: steps
- type: Any[]?
- default: []
- - id: step_name
- type: string
- doc: unique name for the step
- default: preflag
- - id: count.save
- default: false
- type: boolean
- doc: >-
- If true, the flag percentages per frequency are saved to a table with
- extension .flagfreq and percentages per station to a table with extension
- .flagstat. The basename of the table is the MS name (without extension)
- followed by the stepname and extension.
- - id: count.path
- type: string?
- doc: >-
- The directory where to create the flag percentages table. If empty, the
- path of the input MS is used.
- - id: mode
- default: set
- type: string
- doc: >-
- Case-insensitive string telling what to do with the flags of the data
- matching (or not matching) the selection criteria given in the other
- parameters.
- - id: expr
- default: ''
- type: string?
- doc: >-
- Expression of preflagger keyword sets (see above). Operators AND, OR, and
- NOT are possible (or their equivalents &&,&, ||, |, and !). Parentheses
- can be used to change precedence order. For example:: c1 and (c2 or c3)
- - default:
- id: timeofday
- type: string?
- doc: >-
- Ranges of UTC time-of-day given as st..end or val+-delta. Each value must
- be given as 12:34:56.789, 12h34m56.789, or as a value followed by a unit
- like h, min, or s.
- - id: abstime
- default: []
- type: string[]
- doc: >-
- Ranges of absolute UTC date/time given as st..end or val+-delta. Each
- value (except delta) must be given as a date/time in casacore MVTime
- format, for instance 12-Mar-2010/11:31:00.000. A delta value must be given
- as a time (for instance 1:30:0 or 20s).
- - id: reltime
- default: []
- type: string[]
- doc: >-
- Ranges of times (using .. or +-) since the start of the observation. A
- time can be given like 1:30:0 or 20s.
- - id: timeslot
- default: []
- type: string[]
- doc: >-
- Time slot sequence numbers. First time slot is 0. st..end means end
- inclusive.
- - id: lst
- default: []
- type: string[]
- doc: >-
- Ranges of Local Apparent Sidereal Times like 1:30:0 +- 20min. The LST of a
- time slot is calculated for the array position, thus not per antenna.
- - id: azimuth
- default: []
- type: float[]
- doc: >-
- Ranges of azimuth angles given as st..end or val+-delta. Each value has to
- be given as a casacore direction like 12:34:56.789 or 12h34m56.789,
- 12.34.56.789 or 12d34m56.789, or a value followed by a unit like rad or
- deg.
- - id: elevation
- type: string?
- doc: >-
- Ranges of elevation angles (similar to azimuth). For example::
- 0deg..10deg
- - id: baseline
- default: []
- type: string[]
- doc: See Description of baseline selection parameters.
- - id: corrtype
- default: ''
- type: string
- doc: Correlation type to match? Must be auto, cross, or an empty string.
- - id: blmin
- default: -1
- type: float
- doc: If blmin > 0, baselines with length < blmin meter will match.
- - id: blmax
- default: -1
- type: float
- doc: If blmax > 0, baselines with length > blmax meter will match.
- - id: uvmmin
- default: -1
- type: float
- doc: >-
- If uvmmin > 0, baselines with UV-distance < uvmmin meter will match. Note
- that the UV-distance is the projected baseline length.
- - id: uvmmax
- default: -1
- type: float
- doc: If uvmmax > 0, baselines with UV-distance > uvmmax meter will match.
- - id: freqrange
- default: ''
- type: string
- doc: >-
- Channels in the given frequency ranges will match. Each value in the
- vector is a range which can be given as start..end or start+-delta. A
- value can be followed by a unit like KHz. If only one value in a range has
- a unit, the unit is also applied to the other value. If a range has no
- unit, it defaults to MHz. For example: freqrange=[1.2 .. 1.4 MHz,
- 1.8MHz+-50KHz] flags channels between 1.2MHz and 1.4MHz and between
- 1.75MHz and 1.85MHz. The example shows that blanks can be used at will.
- - id: chan
- default: ''
- type: string
- doc: >-
- The given channels will match (start counting at 0). Channels exceeding
- the number of channels are ignored. Similar to msin, it is possible to
- specify the channels as an expression of nchan. Furthermore, .. can be
- used to specify ranges. For example: chan=[0..nchan/32-1,
- 31*nchan/32..nchan-1] to flag the first and last 2 or 8 channels
- (depending on 64 or 256 channels in the observation).
- - id: amplmin
- default: -1e+30
- type: float
- doc: >-
- Correlation data with amplitude < amplmin will match. It can be given per
- correlation. For example, amplmin=[100,,,100] matches data points with XX
- or YY amplitude < 100. The non-specified amplitudes get the default value.
- - id: amplmax
- default: 1e+30
- type: float
- doc: Correlation data with amplitude > amplmax will match.
- - id: phasemin
- default: -1e+30
- type: float
- doc: Correlation data with phase < phasemin (in radians) will match.
- - id: phasemax
- default: 1e+30
- type: float
- doc: Correlation data with phase > phasemax (in radians) will match.
- - id: realmin
- default: -1e+30
- type: float
- doc: Correlation data with real complex part < realmin will match.
- - id: realmax
- default: 1e+30
- type: float
- doc: Correlation data with real complex part > realmax will match.
- - id: imagmin
- default: -1e+30
- type: float
- doc: Correlation data with imaginary complex part < imagmin will match.
- - id: imagmax
- default: 1e+30
- type: float
- doc: Correlation data with imaginary complex part > imagmax will match.
-outputs:
-- id: augmented_steps
- outputSource:
- - DP3_GenericStep/augmented_steps
- type: Any[]
-
-steps:
- - id: DP3_GenericStep
- in:
- - id: step_type
- default: 'preflagger'
- - id: step_id
- source: step_name
- - id: steps
- source: steps
- - id: parameters
- valueFrom: $(inputs)
- - id: count.save
- source: count.save
- - id: count.path
- source: count.path
- - id: mode
- source: mode
- - id: expr
- source: expr
- - id: abstime
- source: abstime
- - id: reltime
- source: reltime
- - id: timeslot
- source: timeslot
- - id: lst
- source: lst
- - id: azimuth
- source: azimuth
- - id: elevation
- source: elevation
- - id: baseline
- source: baseline
- - id: corrtype
- source: corrtype
- - id: blmin
- source: blmin
- - id: blmax
- source: blmax
- - id: uvmmin
- source: uvmmin
- - id: uvmmax
- source: uvmmax
- - id: freqrange
- source: freqrange
- - id: chan
- source: chan
- - id: amplmin
- source: amplmin
- - id: amplmax
- source: amplmax
- - id: phasemin
- source: phasemin
- - id: phasemax
- source: phasemax
- - id: realmin
- source: realmin
- - id: realmax
- source: realmax
- - id: imagmin
- source: imagmin
- - id: imagmax
- source: imagmax
- out:
- - augmented_steps
- run: ../steps/DP3.GenericStep.cwl
-
-requirements:
- - class: StepInputExpressionRequirement
- - class: InlineJavascriptRequirement
- - class: MultipleInputFeatureRequirement
diff --git a/steps/DP3.Selector.cwl b/steps/DP3.Selector.cwl
deleted file mode 100755
index b50754a85ab040136df19917a0e01a5464e8c7c9..0000000000000000000000000000000000000000
--- a/steps/DP3.Selector.cwl
+++ /dev/null
@@ -1,24 +0,0 @@
-class: ExpressionTool
-cwlVersion: v1.0
-id: selector
-inputs:
- - id: select_a
- type: boolean
- doc: 'if true selects a_steps if false selects b_steps'
- - id: a_steps
- type: Any[]
- default: []
- - id: b_steps
- type: Any[]
- default: []
-outputs:
- - id: selected_steps
- type: Any
-expression: |
- $(
- {'selected_steps': inputs.select_a ? inputs.a_steps : inputs.b_steps}
- )
-label: Selector
-
-requirements:
- - class: InlineJavascriptRequirement
diff --git a/steps/DPPP.cwl b/steps/DPPP.cwl
deleted file mode 100755
index 70d824f02518f594ecf5d9b8975041b1febee5bc..0000000000000000000000000000000000000000
--- a/steps/DPPP.cwl
+++ /dev/null
@@ -1,89 +0,0 @@
-class: CommandLineTool
-cwlVersion: v1.0
-$namespaces:
- sbg: 'https://www.sevenbridges.com/'
-id: dppp
-baseCommand:
- - DPPP
-inputs:
- - id: parset
- type: File?
- inputBinding:
- position: -1
- - id: msin
- type: Directory?
- inputBinding:
- position: 0
- prefix: msin=
- separate: false
- doc: Input Measurement Set
- - default: out.MS
- id: msout_name
- type: string
- inputBinding:
- position: 0
- prefix: msout=
- separate: false
- doc: Output Measurement Set
- - id: secondary_files
- type: 'File[]?'
- doc: Secondary files needed for the step
- - id: secondary_directories
- type: 'Directory[]?'
- doc: Secondary directories needed for the step
- - id: output_file_names
- type: Any
- doc: Expected output file names
- - id: output_directory_names
- type: Any
- doc: Expected output file names
-outputs:
- - id: msout
- doc: Output Measurement Set
- type: Directory
- outputBinding:
- glob: '$(inputs.msout_name=="."?inputs.msin:inputs.msout_name)'
- - id: secondary_output_files
- doc: Secondary output files
- type: Any
- outputBinding:
- outputEval: |-
- ${
- var output={}
- for(var step_name in inputs.output_file_names){
- var file_per_idx = inputs.output_file_names[step_name];
- for(var file_idx in file_per_idx){
- var file_name = file_per_idx[file_idx];
-
- output[file_idx] = {'class':'File', 'path': file_name};
- }
- }
- return output
- }
- - id: secondary_output_directories
- doc: Secondary output directories
- type: Any
- outputBinding:
- outputEval: |-
- ${
- var output={}
- for(var step_name in inputs.output_directory_names){
- var file_per_idx = inputs.output_directory_names[step_name];
- for(var file_idx in file_per_idx){
- var file_name = file_per_idx[file_idx];
-
- output[file_idx] = {'class':'Directory', 'path': file_name};
- }
- }
- return output
- }
- - id: logfile
- type: File?
- outputBinding:
- glob: DPPP.log
-hints:
- - class: DockerRequirement
- dockerPull: lofareosc/prefactor
-stdout: DPPP.log
-requirements:
- - class: InlineJavascriptRequirement
diff --git a/steps/DirSelector.cwl b/steps/DirSelector.cwl
deleted file mode 100644
index d5bd04038a703c3ecfbc5f07aa917892b87bd85d..0000000000000000000000000000000000000000
--- a/steps/DirSelector.cwl
+++ /dev/null
@@ -1,39 +0,0 @@
-class: ExpressionTool
-cwlVersion: v1.0
-id: dirselector
-inputs:
- - id: inputs
- type: Any
- doc: input files
- - id: step_name
- type: string
- doc: >-
- id of the step that generates the output file or directory
- (e.g. 'demixstepgenerator')
- - id: key_name
- type: string
- doc: >-
- key of the parset that contains the name of the outputfile
- (e.g. 'instrumentmodel')
-outputs:
- - id: output
- type: [Directory, 'Directory[]']
-
-expression: |
- ${
- var step_name = inputs.step_name
- var key_name = inputs.key_name
-
- if(inputs.inputs.hasOwnProperty(step_name)){
- var inputs_per_step = inputs.inputs[step_name]
- if(inputs_per_step.hasOwnProperty(key_name)){
- return {'output': inputs_per_step[key_name]}
- }
- }
- return {'output': []}
-
- }
-label: DirSelector
-
-requirements:
- - class: InlineJavascriptRequirement
diff --git a/steps/FileSelector.cwl b/steps/FileSelector.cwl
deleted file mode 100644
index f935adca0125043812f663d913ef039819f6a52e..0000000000000000000000000000000000000000
--- a/steps/FileSelector.cwl
+++ /dev/null
@@ -1,31 +0,0 @@
-class: ExpressionTool
-cwlVersion: v1.0
-id: fileselector
-inputs:
- - id: inputs
- type: Any
- doc: input files
- - id: step_name
- type: string
- doc: >-
- id of the step that generates the output file or directory
- (e.g. 'demixstepgenerator')
- - id: key_name
- type: string
- doc: >-
- key of the parset that contains the name of the outputfile
- (e.g. 'instrumentmodel')
-outputs:
- - id: output
- type: File
-
-expression: |
- ${
- var step_name = inputs.step_name
- var key_name = inputs.key_name
- return {'output': inputs.inputs[step_name][key_name]}
- }
-label: FileSelector
-
-requirements:
- - class: InlineJavascriptRequirement
diff --git a/steps/H5ParmCollector.cwl b/steps/H5ParmCollector.cwl
deleted file mode 100755
index e2e33e20acab94f1e092866d37a2086185e93530..0000000000000000000000000000000000000000
--- a/steps/H5ParmCollector.cwl
+++ /dev/null
@@ -1,70 +0,0 @@
-class: CommandLineTool
-cwlVersion: v1.0
-$namespaces:
- sbg: 'https://www.sevenbridges.com/'
- lofar: https://git.astron.nl/eosc/ontologies/raw/master/schema/lofar.owl
-$schema:
- - https://git.astron.nl/eosc/ontologies/raw/master/schema/lofar.owl
-
-id: h5parm_collector
-
-baseCommand:
- - H5parm_collector.py
-inputs:
- - id: h5parmFiles
- type: 'File[]'
- format: lofar:#H5Parm
- inputBinding:
- position: 0
- doc: List of h5parm files
- - default: sol000
- id: insolset
- type: string?
- inputBinding:
- position: 0
- prefix: '--insolset'
- doc: Input solset name
- - default: output.h5
- id: outh5parmname
- type: string
- doc: Output h5parm name
- - id: squeeze
- type: boolean
- default: false
- inputBinding:
- position: 0
- prefix: '-q'
- doc: removes all axes with the length of 1
- - default: true
- id: verbose
- type: boolean
- inputBinding:
- position: 0
- prefix: '-v'
- doc: verbose output
- - default: true
- id: clobber
- type: boolean
- inputBinding:
- position: 0
- prefix: '-c'
- doc: overwrite output
-outputs:
- - id: outh5parm
- doc: Output h5parm
- type: File
- format: lofar:#H5Parm
- outputBinding:
- glob: $(inputs.outh5parmname)
- - id: log
- type: File
- outputBinding:
- glob: $(inputs.outh5parmname)-parm_collector_output.log
-label: H5parm_collector
-hints:
- - class: DockerRequirement
- dockerPull: 'lofareosc/prefactor-ci:master'
-stdout: $(inputs.outh5parmname)-parm_collector_output.log
-requirements:
- - class: InlineJavascriptRequirement
-
diff --git a/steps/LoSoTo.Abs.cwl b/steps/LoSoTo.Abs.cwl
deleted file mode 100755
index 43f5c28c5a5fd18157707d9ba273bd85c6620532..0000000000000000000000000000000000000000
--- a/steps/LoSoTo.Abs.cwl
+++ /dev/null
@@ -1,52 +0,0 @@
-#!/usr/bin/env cwl-runner
-
-class: CommandLineTool
-cwlVersion: v1.0
-id: losoto_abs
-
-$namespaces:
- lofar: https://git.astron.nl/eosc/ontologies/raw/master/schema/lofar.owl
-doc: |
- Take absolute value. Needed before smooth if amplitudes are negative!
- WEIGHT: no need to be weight compliant
-
-requirements:
- InitialWorkDirRequirement:
- listing:
- - entryname: 'parset.config'
- entry: |
- [abs]
- soltab = $(inputs.soltab)
- operation = ABS
- - entryname: $(inputs.input_h5parm.basename)
- entry: $(inputs.input_h5parm)
- writable: true
-
-baseCommand: "losoto"
-
-arguments:
- - $(inputs.input_h5parm.basename)
- - parset.config
-
-hints:
- DockerRequirement:
- dockerPull: lofareosc/prefactor-ci:master
-
-inputs:
- - id: input_h5parm
- type: File
- format: lofar:#H5Parm
- - id: soltab
- type: string
- doc: "Solution table"
-
-outputs:
- - id: output_h5parm
- type: File
- format: lofar:#H5Parm
- outputBinding:
- glob: $(inputs.input_h5parm.basename)
-
-
-$schema:
- - https://git.astron.nl/eosc/ontologies/raw/master/schema/lofar.owl
diff --git a/steps/LoSoTo.Duplicate.cwl b/steps/LoSoTo.Duplicate.cwl
deleted file mode 100644
index 9111602fa6ebde8aaca0b425afeef3c3d1e39d75..0000000000000000000000000000000000000000
--- a/steps/LoSoTo.Duplicate.cwl
+++ /dev/null
@@ -1,56 +0,0 @@
-#!/usr/bin/env cwl-runner
-
-class: CommandLineTool
-cwlVersion: v1.0
-id: losoto_duplicate
-
-doc: |
- Duplicate a table
-
-requirements:
- InlineJavascriptRequirement: {}
- InitialWorkDirRequirement:
- listing:
- - entryname: 'parset.config'
- entry: |
- [duplicate]
- soltab = $(inputs.soltab)
- operation = DUPLICATE
- $(inputs.soltabOut !== null? 'soltabOut=' + inputs.soltabOut: '')
-
- - entryname: $(inputs.input_h5parm.basename)
- entry: $(inputs.input_h5parm)
- writable: true
-
-baseCommand: "losoto"
-
-arguments:
- - $(inputs.input_h5parm.basename)
- - parset.config
-
-hints:
- DockerRequirement:
- dockerPull: lofareosc/prefactor-ci:master
-
-inputs:
- - id: input_h5parm
- type: File
- format: lofar:#H5Parm
- - id: soltab
- type: string
- doc: "Solution table"
- - id: soltabOut
- type: string?
- doc: Output table name. By default choose next available from table type.
-outputs:
- - id: output_h5parm
- type: File
- format: lofar:#H5Parm
- outputBinding:
- glob: $(inputs.input_h5parm.basename)
-
-
-$namespaces:
- lofar: https://git.astron.nl/eosc/ontologies/raw/master/schema/lofar.owl
-$schema:
- - https://git.astron.nl/eosc/ontologies/raw/master/schema/lofar.owl
diff --git a/steps/LoSoTo.Norm.cwl b/steps/LoSoTo.Norm.cwl
deleted file mode 100644
index f60ee4c469c938e01c52b5d226e275bba4fe2a1f..0000000000000000000000000000000000000000
--- a/steps/LoSoTo.Norm.cwl
+++ /dev/null
@@ -1,61 +0,0 @@
-#!/usr/bin/env cwl-runner
-
-class: CommandLineTool
-cwlVersion: v1.0
-id: losoto_abs
-
-doc: |
- Normalize the solutions to a given value WEIGHT: Weights compliant
-
-requirements:
- InlineJavascriptRequirement: {}
- InitialWorkDirRequirement:
- listing:
- - entryname: 'parset.config'
- entry: |
- [norm]
- soltab = $(inputs.soltab)
- operation = NORM
- axesToNorm = $(inputs.axesToNorm)
- $(inputs.normVal !== null? 'normVal=' + inputs.normVal: '')
-
- - entryname: $(inputs.input_h5parm.basename)
- entry: $(inputs.input_h5parm)
- writable: true
-
-baseCommand: "losoto"
-
-arguments:
- - $(inputs.input_h5parm.basename)
- - parset.config
-
-hints:
- DockerRequirement:
- dockerPull: lofareosc/prefactor-ci:master
-
-inputs:
- - id: input_h5parm
- type: File
- format: lofar:#H5Parm
- - id: soltab
- type: string
- doc: "Solution table"
- - id: axesToNorm
- type: string[]
- doc: Axes along which compute the normalization
- - id: normVal
- type: float?
- doc: Number to normalize to vals = vals * (normVal/valsMean), by default 1.
-
-outputs:
- - id: output_h5parm
- type: File
- format: lofar:#H5Parm
- outputBinding:
- glob: $(inputs.input_h5parm.basename)
-
-
-$namespaces:
- lofar: https://git.astron.nl/eosc/ontologies/raw/master/schema/lofar.owl
-$schema:
- - https://git.astron.nl/eosc/ontologies/raw/master/schema/lofar.owl
diff --git a/steps/LoSoTo.Plot.cwl b/steps/LoSoTo.Plot.cwl
deleted file mode 100644
index e5d5cfb054bbda082cf00a311d9df0904ea22f1f..0000000000000000000000000000000000000000
--- a/steps/LoSoTo.Plot.cwl
+++ /dev/null
@@ -1,148 +0,0 @@
-#!/usr/bin/env cwl-runner
-
-class: CommandLineTool
-cwlVersion: v1.0
-id: losoto_plot
-
-doc: |
- This operation for LoSoTo implements basic plotting WEIGHT:
- flag-only compliant, no need for weight
-
-requirements:
- InlineJavascriptRequirement:
- expressionLib:
- - |
- function get_config() {
- var par = ['soltab = ' + inputs.soltab]
- if (inputs.ncpu !== null) par.push('ncpu='+inputs.ncpu);
- par.push('[plot]')
- par.push('operation = PLOT')
- for(var field_name in inputs){
- if(field_name === 'input_h5parm' ||
- field_name === 'soltab' ||
- field_name === 'ncpu') continue;
- if(inputs[field_name] === null ||
- inputs[field_name] === 'null') continue;
- par.push(field_name+'='+inputs[field_name])
- }
- return par
- }
- InitialWorkDirRequirement:
- listing:
- - entryname: 'parset.config'
- entry: $(get_config().join('\n'))
- - entryname: $(inputs.input_h5parm.basename)
- entry: $(inputs.input_h5parm)
- writable: true
-
-
-baseCommand: "losoto"
-
-arguments:
- - $(inputs.input_h5parm.basename)
- - parset.config
-
-hints:
- DockerRequirement:
- dockerPull: lofareosc/prefactor-ci:master
-
-inputs:
- - id: input_h5parm
- type: File
- format: lofar:#H5Parm
- - id: soltab
- type: string[]
- doc: "Tabs to plot"
-
- - id: axesInPlot
- type: string[]?
- default: []
- doc: |
- 1- or 2-element array which says the coordinates to plot (2 for 3D plots).
- - id: axisInTable
- type: string?
- doc: |
- the axis to plot on a page - e.g. ant to get all antenna’s on one file.
- - id: axisInCol
- type: string?
- doc: |
- The axis to plot in different colours - e.g. pol to get correlations with
- different colors.
- - id: axisDiff
- type: string?
- doc: |
- This must be a len=2 axis and the plot will have the differential value
- - e.g. ‘pol’ to plot XX-YY.
- - id: NColFig
- type: int?
- doc: |
- Number of columns in a multi-table image. By default is automatically
- chosen.
- - id: figSize
- type: int[]
- default: [0,0]
- doc: |
- Size of the image [x,y], if one of the values is 0, then it is
- automatically chosen. By default automatic set.
- - id: markerSize
- type: int?
- default: 2
- doc: |
- Size of the markers in the 2D plot. By default 2.
- - id: minmax
- type: float[]?
- doc: |
- Min max value for the independent variable (0 means automatic).
- - id: log
- type: string?
- doc: |
- Use Log=’XYZ’ to set which axes to put in Log.
- - id: plotFlag
- type: boolean?
- default: false
- doc: Whether to plot also flags as red points in 2D plots.
- - id: doUnwrap
- type: boolean?
- default: false
- doc: Unwrap phases.
- - id: refAnt
- type: string?
- default: ''
- doc: |
- Reference antenna for phases. By default None.
- - id: soltabsToAdd
- type: string?
- doc: |
- Tables to “add†(e.g. ‘sol000/tec000’), it works only for tec and clock
- to be added to phases.
- - id: makeAntPlot
- default: false
- type: boolean?
- doc: |
- Make a plot containing antenna coordinates in x,y and in color the value
- to plot, axesInPlot must be [ant].
- - id: makeMovie
- default: false
- type: boolean?
- doc: |
- Make a movie summing up all the produced plots.
- - id: prefix
- type: string?
- default: 'losoto.plot.'
- doc: |
- Prefix to add before the self-generated filename.
- - id: ncpu
- type: int?
- doc: Number of cpus, by default all available.
-
-outputs:
- - id: output_plots
- type: File[]
- outputBinding:
- glob: "$(inputs.prefix)*.png"
-
-
-$namespaces:
- lofar: https://git.astron.nl/eosc/ontologies/raw/master/schema/lofar.owl
-$schema:
- - https://git.astron.nl/eosc/ontologies/raw/master/schema/lofar.owl
diff --git a/steps/LoSoTo.Replicateonaxis.cwl b/steps/LoSoTo.Replicateonaxis.cwl
deleted file mode 100644
index c676467d19611e57252c17bdfd83b99d8a5fc821..0000000000000000000000000000000000000000
--- a/steps/LoSoTo.Replicateonaxis.cwl
+++ /dev/null
@@ -1,77 +0,0 @@
-#!/usr/bin/env cwl-runner
-
-class: CommandLineTool
-cwlVersion: v1.0
-id: losoto_replicate_on_axis
-
-doc: |
- Replace the values along a certain axis taking them from one specific axis
- cell
-
-requirements:
- InlineJavascriptRequirement: {}
- InitialWorkDirRequirement:
- listing:
- - entryname: "replicate_axis.py"
- entry: |
- import sys
- from losoto.h5parm import openSoltab
- from losoto.operations import replicateonaxis
-
- soltab_address = "$(inputs.soltab)"
- soltab = openSoltab(sys.argv[1], address=soltab_address)
-
- axisReplicate = "$(inputs.axisReplicate)"
- fromCell = "$(inputs.fromCell !== null ? inputs.fromCell : 0)"
- updateWeights = "$(inputs.updateWeights? inputs.updateWeights : 'True')"
-
- replicateonaxis.run(soltab, axisReplicate=axisReplicate,
- fromCell=fromCell,
- updateWeights=updateWeights)
-
- - entryname: $(inputs.input_h5parm.basename)
- entry: $(inputs.input_h5parm)
- writable: true
-
-baseCommand: "python3"
-
-arguments:
- - replicate_axis.py
- - $(inputs.input_h5parm.basename)
-
-
-hints:
- DockerRequirement:
- dockerPull: lofareosc/prefactor-ci:master
-
-inputs:
- - id: input_h5parm
- type: File
- format: lofar:#H5Parm
- - id: soltab
- type: string
- doc: "Solution table"
- - id: axisReplicate
- type: string
- doc: Axis along which replicate the values.
- - id: fromCell
- type: string?
- doc: A cell value in axisReplicate from which to copy the data values.
- - id: updateWeights
- type: boolean?
- doc: |
- If False then weights are untouched, if True they are replicated
- like data. Default: True.
-
-outputs:
- - id: output_h5parm
- type: File
- format: lofar:#H5Parm
- outputBinding:
- glob: $(inputs.input_h5parm.basename)
-
-
-$namespaces:
- lofar: https://git.astron.nl/eosc/ontologies/raw/master/schema/lofar.owl
-$schema:
- - https://git.astron.nl/eosc/ontologies/raw/master/schema/lofar.owl
diff --git a/steps/LoSoTo.Reset.cwl b/steps/LoSoTo.Reset.cwl
deleted file mode 100644
index 99594d12046ee2d48622dda0e08de72e25c8efcd..0000000000000000000000000000000000000000
--- a/steps/LoSoTo.Reset.cwl
+++ /dev/null
@@ -1,57 +0,0 @@
-#!/usr/bin/env cwl-runner
-
-class: CommandLineTool
-cwlVersion: v1.0
-id: losoto_reset
-
-doc: Subtract/divide two tables or a clock/tec/tec3rd/rm from a phase.
-
-requirements:
- InlineJavascriptRequirement: {}
- InitialWorkDirRequirement:
- listing:
- - entryname: 'parset.config'
- entry: |
- [reset]
- soltab = $(inputs.soltab)
- operation = RESET
- $(inputs.dataVal !== null? 'dataVal=' + inputs.dataVal: '')
-
- - entryname: $(inputs.input_h5parm.basename)
- entry: $(inputs.input_h5parm)
- writable: true
-
-baseCommand: "losoto"
-
-arguments:
- - $(inputs.input_h5parm.basename)
- - parset.config
-
-hints:
- DockerRequirement:
- dockerPull: lofareosc/prefactor-ci:master
-
-inputs:
- - id: input_h5parm
- type: File
- format: lofar:#H5Parm
- - id: soltab
- type: string
- doc: "Solution table"
- - id: dataVal
- type: float?
- doc: |
- If given set values to this number, otherwise uses 1 for amplitude and 0
- for all other soltab types.
-outputs:
- - id: output_h5parm
- type: File
- format: lofar:#H5Parm
- outputBinding:
- glob: $(inputs.input_h5parm.basename)
-
-
-$namespaces:
- lofar: https://git.astron.nl/eosc/ontologies/raw/master/schema/lofar.owl
-$schema:
- - https://git.astron.nl/eosc/ontologies/raw/master/schema/lofar.owl
diff --git a/steps/LoSoTo.Residual.cwl b/steps/LoSoTo.Residual.cwl
deleted file mode 100644
index 4bc566cd7d878f8532d69cd74aad7a6d4a72bf8b..0000000000000000000000000000000000000000
--- a/steps/LoSoTo.Residual.cwl
+++ /dev/null
@@ -1,59 +0,0 @@
-#!/usr/bin/env cwl-runner
-
-class: CommandLineTool
-cwlVersion: v1.0
-id: losoto_residual
-
-doc: Subtract/divide two tables or a clock/tec/tec3rd/rm from a phase.
-
-requirements:
- InitialWorkDirRequirement:
- listing:
- - entryname: 'parset.config'
- entry: |
- [residuals]
- soltab = $(inputs.soltab)
- operation=RESIDUALS
- soltabsToSub=$(inputs.soltabsToSub)
- ratio=$(inputs.ratio)
-
- - entryname: $(inputs.input_h5parm.basename)
- entry: $(inputs.input_h5parm)
- writable: true
-
-baseCommand: "losoto"
-
-arguments:
- - $(inputs.input_h5parm.basename)
- - parset.config
-
-hints:
- DockerRequirement:
- dockerPull: lofareosc/prefactor-ci:master
-
-inputs:
- - id: input_h5parm
- type: File
- format: lofar:#H5Parm
- - id: soltab
- type: string
- doc: "Solution table"
- - id: soltabsToSub
- type: string[]
- doc: List of soltabs to subtract
- - id: ratio
- type: boolean?
- default: false
- doc: Return the ratio instead of subtracting.
-
-outputs:
- - id: output_h5parm
- type: File
- format: lofar#H5Parm
- outputBinding:
- glob: $(inputs.input_h5parm.basename)
-
-$namespaces:
- lofar: https://git.astron.nl/eosc/ontologies/raw/master/schema/lofar.owl
-$schema:
- - https://git.astron.nl/eosc/ontologies/raw/master/schema/lofar.owl
diff --git a/steps/RadioDatatypes.yaml b/steps/RadioDatatypes.yaml
deleted file mode 100644
index 0a1679541e2bf66a9bc447669dfd27a6d94c2ce1..0000000000000000000000000000000000000000
--- a/steps/RadioDatatypes.yaml
+++ /dev/null
@@ -1,8 +0,0 @@
-type: array
-name: MeasurementSet
-label: The measurement set data location
-items:
- - name: main_directory
- type: Directory
- inputBinding:
- position: 0
diff --git a/steps/ddecal.cwl b/steps/ddecal.cwl
deleted file mode 100644
index c534d5362f563941d9503b980ae0dd6a4b766acf..0000000000000000000000000000000000000000
--- a/steps/ddecal.cwl
+++ /dev/null
@@ -1,111 +0,0 @@
-#!/usr/bin/env cwl-runner
-
-class: CommandLineTool
-cwlVersion: v1.0
-id: ddecal
-baseCommand: [DPPP]
-
-requirements:
- InlineJavascriptRequirement: {}
-
-hints:
- DockerRequirement:
- dockerPull: lofareosc/prefactor-ci:master
-
-arguments:
- - steps=[ddecal]
- - ddecal.mode=rotation+diagonal
- - ddecal.uvlambdamin=300
- - ddecal.maxiter=50
- - ddecal.nchan=1
- - ddecal.solint=1
- - ddecal.propagateconvergedonly=True
- - ddecal.flagdivergedonly=True
- - ddecal.tolerance=1.e-3
- - ddecal.usemodelcolumn=True
-
-
-inputs:
- - id: msin
- type: Directory?
- doc: Input Measurement Set
- inputBinding:
- prefix: msin=
- separate: false
-
- - id: msin_datacolumn
- type: string
- default: DATA
- doc: Input data Column
- inputBinding:
- prefix: msin.datacolumn=
- separate: false
-
- - id: msin_modelcolum
- type: string
- default: MODEL_DATA
- doc: Model data Column
- inputBinding:
- prefix: msin.modelcolumn=
- separate: false
-
- - id: output_name_h5parm
- type: string
- default: instrument.h5
- inputBinding:
- prefix: ddecal.h5parm=
- separate: false
- - id: msout_name
- type: string
- doc: Output Measurement Set
- default: out.MS
- inputBinding:
- prefix: msout=
- separate: false
-
-#--------------------
- - id: propagate_solutions
- type: boolean
- default: true
- inputBinding:
- prefix: propagatesolutions=True
- - id: flagunconverged
- type: boolean
- default: false
- doc: |
- Flag unconverged solutions (i.e., those from solves that did not converge
- within maxiter iterations).
- inputBinding:
- prefix: flagdivergedonly=True
- - id: flagdivergedonly
- default: false
- type: boolean
- doc: |
- Flag only the unconverged solutions for which divergence was detected.
- At the moment, this option is effective only for rotation+diagonal
- solves, where divergence is detected when the amplitudes of any station
- are found to be more than a factor of 5 from the mean amplitude over all
- stations.
- If divergence for any one station is detected, all stations are flagged
- for that solution interval. Only effective when flagunconverged=true
- and mode=rotation+diagonal.
- inputBinding:
- prefix: flagdivergedonly=True
-outputs:
- - id: msout
- doc: Output Measurement Set
- type: Directory
- outputBinding:
- glob: $(inputs.msout_name=="."?inputs.msin:inputs.msout_name)
-
- - id: h5parm
- doc: Filename of output H5Parm (to be read by e.g. losoto)
- type: File
- format: lofar:#H5Parm
- outputBinding:
- glob: $(inputs.output_name_h5parm)
-
-$namespaces:
- lofar: https://git.astron.nl/eosc/ontologies/raw/master/schema/lofar.owl
-$schema:
- - https://git.astron.nl/eosc/ontologies/raw/master/schema/lofar.owl
diff --git a/steps/interpolate_visibilities.cwl b/steps/interpolate_visibilities.cwl
deleted file mode 100644
index 90772b4f072cdecd9f03fd5f1d6258b66b995d11..0000000000000000000000000000000000000000
--- a/steps/interpolate_visibilities.cwl
+++ /dev/null
@@ -1,52 +0,0 @@
-class: CommandLineTool
-cwlVersion: v1.0
-$namespaces:
- sbg: 'https://www.sevenbridges.com/'
-id: interpolate_visibilities
-
-baseCommand:
- - DPPP
-arguments:
- - msout.datacolumn=INTERP_DATA
- - msout=.
- - msin.datacolumn=DATA
- - steps=[interpolate]
-inputs:
- - id: msin
- type: Directory
- inputBinding:
- prefix: 'msin='
- separate: false
- - id: storage_manager
- type: string
- default: Dysco
- inputBinding:
- separate: false
- prefix: 'msout.storagemanager='
- - id: storage_manager_databitrate
- type: int
- default: 0
- inputBinding:
- prefix: 'msout.storagemanager.databitrate='
- separate: false
- - id: window_size
- type: int
- inputBinding:
- prefix: 'interpolate.windowsize='
- separate: false
-
-outputs:
- - id: msout
- type: Directory
- outputBinding:
- glob: $(inputs.msin.basename)
-label: interpolate_visibilities
-hints:
- - class: DockerRequirement
- dockerPull: lofareosc/prefactor-ci:master
-
-requirements:
- - class: InitialWorkDirRequirement
- listing:
- - entry: $(inputs.msin)
- writable: true
diff --git a/steps/makesourcedb.cwl b/steps/makesourcedb.cwl
deleted file mode 100755
index eb5c2e7c7b8f09c0a2c97ea4512bb241cff17fb5..0000000000000000000000000000000000000000
--- a/steps/makesourcedb.cwl
+++ /dev/null
@@ -1,56 +0,0 @@
-class: CommandLineTool
-cwlVersion: v1.0
-$namespaces:
- sbg: 'https://www.sevenbridges.com/'
-id: make_sourcedb
-baseCommand:
- - makesourcedb
-inputs:
- - id: sky_model
- type:
- - File
- - string
- inputBinding:
- position: 0
- prefix: in=
- separate: false
- - default: Ateam.sourcedb
- id: output_file_name
- type: string?
- inputBinding:
- position: 1
- prefix: out=
- separate: false
- valueFrom: Ateam.sourcedb
- - default: blob
- id: outtype
- type: string?
- inputBinding:
- position: 2
- prefix: outtype=
- separate: false
- - default: <
- id: format
- type: string?
- inputBinding:
- position: 3
- prefix: format=
- separate: false
-outputs:
- - id: sourcedb
- type:
- - Directory
- - File
- outputBinding:
- glob: $(inputs.output_file_name)
- - id: log
- type: File?
- outputBinding:
- glob: make_sourcedb_ateam.log
-label: make_sourcedb_ateam
-hints:
- - class: DockerRequirement
- dockerPull: 'lofareosc/prefactor:latest'
-stdout: make_sourcedb_ateam.log
-requirements:
- - class: InlineJavascriptRequirement
diff --git a/steps/ms_concat.cwl b/steps/ms_concat.cwl
deleted file mode 100644
index d9038e8df9506c24c6093b7ed9c4b27fb2a88196..0000000000000000000000000000000000000000
--- a/steps/ms_concat.cwl
+++ /dev/null
@@ -1,59 +0,0 @@
-class: CommandLineTool
-cwlVersion: v1.0
-$namespaces:
- sbg: 'https://www.sevenbridges.com/'
-id: ms_concat
-
-
-baseCommand:
- - concat_MS.py
-inputs:
- - id: min_length
- type: int?
- default: 50
- inputBinding:
- position: 0
- prefix: '--min_length'
- doc: Minimum amount of subbands to concatenate in frequency.
- - id: overhead
- type: float?
- inputBinding:
- position: 0
- prefix: '--overhead'
- doc: |
- Only use this fraction of the available memory for deriving the amount
- of data to be concatenated.
- - id: msin
- type: 'Directory[]'
- inputBinding:
- position: 1
- shellQuote: false
- - default: out.MS
- id: msout
- type: string
- inputBinding:
- position: 2
-outputs:
- - id: concat_meta_ms
- type: Directory[]
- outputBinding:
- glob: >
- $(inputs.msout)_[0-9*]
- - id: concat_additional_ms
- type: Directory[]
- outputBinding:
- glob: $(inputs.msout)_*_CONCAT
- - id: ms_outs
- type: Directory[]
- outputBinding:
- outputEval: $(inputs.msin)
-label: ms_concat
-requirements:
- - class: ShellCommandRequirement
- - class: DockerRequirement
- dockerPull: lofareosc/prefactor-ci:master
- - class: InlineJavascriptRequirement
- - class: InitialWorkDirRequirement
- listing:
- - entry: $(inputs.msin)
- writable: true
diff --git a/steps/msinfo.cwl b/steps/msinfo.cwl
deleted file mode 100644
index 0c7e2f5b779360baef059c0fe61f753a428d8707..0000000000000000000000000000000000000000
--- a/steps/msinfo.cwl
+++ /dev/null
@@ -1,40 +0,0 @@
-#!/usr/bin/env cwl-runner
-
-class: CommandLineTool
-cwlVersion: v1.1
-id: msinfo
-
-requirements:
- - class: InlineJavascriptRequirement
- - class: InitialWorkDirRequirement
- listing:
- - entryname: msinfo.py
- entry: |
- import sys
- print(sys.argv)
- - class: SchemaDefRequirement
- types:
- - $import: RadioDatatypes.yaml
-hints:
- DockerRequirement:
- dockerPull: lofareosc/prefactor-ci:master
-
-baseCommand:
- - python3
- - msinfo.py
-
-inputs:
- data:
- type: RadioDatatypes.yaml#MeasurementSet
- inputBinding:
- position: 0
-
-stdout: std.output
-
-outputs:
- info:
- type: File
- outputBinding:
- glob: std.output
-label: msinfo
-
diff --git a/steps/predict.cwl b/steps/predict.cwl
deleted file mode 100644
index eeb14aef083ee2e7ec43056e68fd018c47023e50..0000000000000000000000000000000000000000
--- a/steps/predict.cwl
+++ /dev/null
@@ -1,71 +0,0 @@
-class: CommandLineTool
-cwlVersion: v1.0
-$namespaces:
- sbg: 'https://www.sevenbridges.com/'
-id: predict
-baseCommand:
- - DPPP
-inputs:
- - id: msin
- type: Directory
- inputBinding:
- position: 0
- prefix: msin=
- separate: false
- doc: Input Measurement Set
- - default: DATA
- id: msin_datacolumn
- type: string
- inputBinding:
- position: 0
- prefix: msin.datacolumn=
- separate: false
- doc: Input data Column
- - default: MODEL_DATA
- id: msout_datacolumn
- type: string
- inputBinding:
- position: 0
- prefix: msout.datacolumn=
- separate: false
- - id: sources_db
- type: File
- inputBinding:
- position: 0
- prefix: predict.sourcedb=
- separate: false
- - default: null
- id: sources
- type: 'string[]'
- inputBinding:
- position: 0
- prefix: predict.sources=
- separate: false
- itemSeparator: ','
- valueFrom: '[$(self)]'
- - default: false
- id: usebeammodel
- type: boolean
- inputBinding:
- position: 0
- prefix: predict.usebeammodel=True
-outputs:
- - id: msout
- doc: Output Measurement Set
- type: Directory
- outputBinding:
- glob: $(inputs.msin.basename)
-arguments:
- - 'steps=[predict]'
- - predict.beammode=array_factor
- - predict.usechannelfreq=False
- - msout=.
-requirements:
- - class: InitialWorkDirRequirement
- listing:
- - entry: $(inputs.msin)
- writable: true
- - class: InlineJavascriptRequirement
-hints:
- - class: DockerRequirement
- dockerPull: lofareosc/prefactor-ci:master
diff --git a/steps/wsclean.cwl b/steps/wsclean.cwl
deleted file mode 100644
index 2bb63f2066cf8237103211ced00a92b2c5c33170..0000000000000000000000000000000000000000
--- a/steps/wsclean.cwl
+++ /dev/null
@@ -1,42 +0,0 @@
-class: CommandLineTool
-cwlVersion: v1.0
-id: wsclean
-baseCommand:
- - wsclean
-inputs:
- - id: image_size
- type: int[]
- default: [1000, 1000]
- inputBinding:
- position: 1
- prefix: -size
- - id: image_scale
- type: float
- default: 0.01
- inputBinding:
- position: 1
- prefix: -scale
- valueFrom: $(self)deg
- - id: image_name
- default: image
- type: string
- inputBinding:
- position: 1
- prefix: -name
- - id: msin
- type: Directory
- inputBinding:
- position: 2
-outputs:
- dirty_image:
- type: File
- outputBinding:
- glob: $(inputs.image_name)-dirty.fits
- image:
- type: File
- outputBinding:
- glob: $(inputs.image_name)-image.fits
-label: WSClean
-requirements:
- - class: DockerRequirement
- dockerPull: lofareosc/prefactor-ci:master
diff --git a/test_jobs/NDPPP.json b/test_jobs/NDPPP.json
deleted file mode 100644
index d010d681ec66a6e25ae050193c8a40ac871d40d6..0000000000000000000000000000000000000000
--- a/test_jobs/NDPPP.json
+++ /dev/null
@@ -1,21 +0,0 @@
-{
- "parset": {
- "class": "File",
- "path": "ndppp_step.parset"
- },
- "msin": {
- "class": "Directory",
- "path": "/data/L570745_SB000_uv_first10.MS"
- },
- "output_file_names": {
- "gaincal": {
- "name": "test.h5"
- },
- },
- "output_directory_names": {
- "count": {
- "count_flagfreq": "L570745_SB000_uv_first10_count.flagfreq",
- "count_flagstat": "L570745_SB000_uv_first10_count.flagstat"
- }
- }
-}
diff --git a/test_jobs/aoflagger.json b/test_jobs/aoflagger.json
deleted file mode 100644
index 138c5022e2e071f4f84a79fdadbdd063b51bfe3f..0000000000000000000000000000000000000000
--- a/test_jobs/aoflagger.json
+++ /dev/null
@@ -1,12 +0,0 @@
-{
- "msin": [
- {
- "class": "Directory",
- "path": "/data/L570745_SB000_uv_first10.MS"
- },
- {
- "class": "Directory",
- "path": "/data/L570745_SB001_uv_first10.MS"
- }
- ]
-}
diff --git a/test_jobs/aoflagger_step_generator.json b/test_jobs/aoflagger_step_generator.json
deleted file mode 100644
index 4c6b5085d4492d2a008d0c0cdd0bbd4139c97132..0000000000000000000000000000000000000000
--- a/test_jobs/aoflagger_step_generator.json
+++ /dev/null
@@ -1,8 +0,0 @@
-{
- "input_parset": [{
- "step_type": "name",
- "step_id": "id",
- "parameters": []
-}],
- "strategy": {"class":"File", "path":"/dev/null"}
-}
diff --git a/test_jobs/averager_step_generator.json b/test_jobs/averager_step_generator.json
deleted file mode 100644
index 4bfe8ddc068c92271e8bb29fad6219214d24a199..0000000000000000000000000000000000000000
--- a/test_jobs/averager_step_generator.json
+++ /dev/null
@@ -1,7 +0,0 @@
-{
- "input_parset": [{
- "step_type": "name",
- "step_id": "id",
- "parameters": []
-}]
-}
diff --git a/test_jobs/averager_step_generator_error.json b/test_jobs/averager_step_generator_error.json
deleted file mode 100644
index c0a124a2a47027e9261e360d5557c31a5de16b4e..0000000000000000000000000000000000000000
--- a/test_jobs/averager_step_generator_error.json
+++ /dev/null
@@ -1,9 +0,0 @@
-{
- "input_parset": [{
- "step_type": "name",
- "step_id": "id",
- "parameters": []
-}],
- "step_id": "id",
- "strategy": {"class":"File", "path":"/dev/null"}
-}
diff --git a/test_jobs/ddecal.json b/test_jobs/ddecal.json
deleted file mode 100644
index 13a89c1f35db2da2d1be6966d5f213619360e514..0000000000000000000000000000000000000000
--- a/test_jobs/ddecal.json
+++ /dev/null
@@ -1,8 +0,0 @@
-{
- "msin": {
- "class": "Directory",
- "path": "/data/L570745_SB001_uv_MODEL.MS"
- },
- "msin_datacolumn": "DATA",
- "usemodelcolumn": true
-}
diff --git a/test_jobs/demo_prefactor.json b/test_jobs/demo_prefactor.json
deleted file mode 100644
index ef2e0db960d13f382cdd2272969ce975e4c7d77f..0000000000000000000000000000000000000000
--- a/test_jobs/demo_prefactor.json
+++ /dev/null
@@ -1,17 +0,0 @@
-{
- "msin": [
- {
- "class": "Directory",
- "path": "/data/L570745_SB003_uv_first10.MS"
- },
- {
- "class": "Directory",
- "path": "/data/L570745_SB005_uv_first10.MS"
- },
- {
- "class": "Directory",
- "path": "/data/L570745_SB008_uv_first10.MS"
- }
- ]
-}
-
diff --git a/test_jobs/dp3_execute.json b/test_jobs/dp3_execute.json
deleted file mode 100644
index f21dff5b5c3fc6f6c7bd58edc1d6aee45ce5446b..0000000000000000000000000000000000000000
--- a/test_jobs/dp3_execute.json
+++ /dev/null
@@ -1,19 +0,0 @@
-{
- "msin": {"class": "Directory", "path":"/data/L570745_SB003_uv_first10.MS"},
- "steps": [
- {
- "step_type": "averager",
- "step_id": "average",
- "parameters": {
- "timestep": 1,
- "freqstep": 1,
- "minpoints": 0,
- "minperc": 0,
- "timeresolution": 0,
- "freqresolution": 0
- },
- "output_files": {}
- }
- ],
- "msout_name": "TEST"
-}
diff --git a/test_jobs/file_selector.json b/test_jobs/file_selector.json
deleted file mode 100644
index 77e4a30217226c2250882fd1ca55810695577c13..0000000000000000000000000000000000000000
--- a/test_jobs/file_selector.json
+++ /dev/null
@@ -1,12 +0,0 @@
-{
- "inputs": {
- "average": {
- "parmdb": {
- "class": "File",
- "location": "ndppp_step.parset"
- }
- }
- },
- "step_name": "average",
- "key_name": "parmdb"
-}
diff --git a/test_jobs/h5parm_collector.json b/test_jobs/h5parm_collector.json
deleted file mode 100644
index 42c60bb8d15f9f8e49673c916c7d19cec07ae35a..0000000000000000000000000000000000000000
--- a/test_jobs/h5parm_collector.json
+++ /dev/null
@@ -1,9 +0,0 @@
-{
- "h5parmFiles": [
- {
- "class": "File",
- "path": "/data/example.h5",
- "format": "lofar:#H5Parm"
- }
- ]
-}
diff --git a/test_jobs/interpolate_visibilities.json b/test_jobs/interpolate_visibilities.json
deleted file mode 100644
index a9ce3dea491423d5a858e8a500f78b3df24a63d3..0000000000000000000000000000000000000000
--- a/test_jobs/interpolate_visibilities.json
+++ /dev/null
@@ -1,4 +0,0 @@
-{
- "msin": { "class":"Directory", "path":"/data/L570745_SB000_uv_first10.MS"},
- "window_size": 5
-}
diff --git a/test_jobs/losoto_abs.json b/test_jobs/losoto_abs.json
deleted file mode 100644
index a283362c08eefe03c690a9a3ad3791dbf4ed74bc..0000000000000000000000000000000000000000
--- a/test_jobs/losoto_abs.json
+++ /dev/null
@@ -1,8 +0,0 @@
-{
- "input_h5parm": {
- "class": "File",
- "path": "/data/example.h5",
- "format": "lofar:#H5Parm"
- },
- "soltab": "sol000/amplitude000"
-}
diff --git a/test_jobs/losoto_duplicate.json b/test_jobs/losoto_duplicate.json
deleted file mode 100644
index a283362c08eefe03c690a9a3ad3791dbf4ed74bc..0000000000000000000000000000000000000000
--- a/test_jobs/losoto_duplicate.json
+++ /dev/null
@@ -1,8 +0,0 @@
-{
- "input_h5parm": {
- "class": "File",
- "path": "/data/example.h5",
- "format": "lofar:#H5Parm"
- },
- "soltab": "sol000/amplitude000"
-}
diff --git a/test_jobs/losoto_norm.json b/test_jobs/losoto_norm.json
deleted file mode 100644
index d6d689949f55ce2c769956e474508c78c7c28f32..0000000000000000000000000000000000000000
--- a/test_jobs/losoto_norm.json
+++ /dev/null
@@ -1,9 +0,0 @@
-{
- "input_h5parm": {
- "class": "File",
- "path": "/data/example.h5",
- "format": "lofar:#H5Parm"
- },
- "soltab": "sol000/amplitude000",
- "axesToNorm": ["time"]
-}
diff --git a/test_jobs/losoto_plot.json b/test_jobs/losoto_plot.json
deleted file mode 100644
index 1316e959d7b7af2e331e320f412a16357d69643d..0000000000000000000000000000000000000000
--- a/test_jobs/losoto_plot.json
+++ /dev/null
@@ -1,6 +0,0 @@
-{
- "input_h5parm": {"class": "File", "path": "/data/example.h5", "format": "lofar:#H5Parm"},
- "soltab": ["sol000/phase000"],
- "axesInPlot": ["time"],
- "ncpu": 1
-}
diff --git a/test_jobs/losoto_replicateonaxis.json b/test_jobs/losoto_replicateonaxis.json
deleted file mode 100644
index 0abfc35d9280bdc84605590f00479a9b5d36d7c6..0000000000000000000000000000000000000000
--- a/test_jobs/losoto_replicateonaxis.json
+++ /dev/null
@@ -1,10 +0,0 @@
-{
- "input_h5parm": {
- "class": "File",
- "path": "/data/example.h5",
- "format": "lofar:#H5Parm"
- },
- "soltab": "sol000/amplitude000",
- "axisReplicate": "time",
- "updateWeight": true
-}
diff --git a/test_jobs/losoto_reset.json b/test_jobs/losoto_reset.json
deleted file mode 100644
index 2d773afb6baa7bb4d83c8fe0d85becf472b54f69..0000000000000000000000000000000000000000
--- a/test_jobs/losoto_reset.json
+++ /dev/null
@@ -1,8 +0,0 @@
-{
- "input_h5parm": {
- "class": "File",
- "path": "/data/example.h5",
- "format": "lofar:#H5Parm"
- },
- "soltab": "sol000/amplitude000"
-}
diff --git a/test_jobs/losoto_residual.json b/test_jobs/losoto_residual.json
deleted file mode 100644
index 4a56f1c6e77dcb29aac66414ba1cb19253597bd6..0000000000000000000000000000000000000000
--- a/test_jobs/losoto_residual.json
+++ /dev/null
@@ -1,9 +0,0 @@
-{
- "input_h5parm": {
- "class": "File",
- "path": "/data/example.h5",
- "format": "lofar:#H5Parm"
- },
- "soltab": "sol000/amplitude000",
- "soltabsToSub": ["amplitude000"]
-}
diff --git a/test_jobs/ms_concat.json b/test_jobs/ms_concat.json
deleted file mode 100644
index 935a917a5ee9ed30ebbb7782305899cecf88bd70..0000000000000000000000000000000000000000
--- a/test_jobs/ms_concat.json
+++ /dev/null
@@ -1,14 +0,0 @@
-{
- "min_length": 1,
- "msin": [
- {
- "class": "Directory",
- "path": "../test_data/L570745_SB000_uv_first10.MS"
- },
- {
- "class": "Directory",
- "path": "../test_data/L570745_SB001_uv_first10.MS"
- }
- ]
-}
-
diff --git a/test_jobs/parset_generator.json b/test_jobs/parset_generator.json
deleted file mode 100644
index 5b3c748396c5cc9928b04ce0d57ae41842137835..0000000000000000000000000000000000000000
--- a/test_jobs/parset_generator.json
+++ /dev/null
@@ -1,7 +0,0 @@
-{
- "parameters_input": [{
- "step_type": "name",
- "step_id": "id",
- "parameters": {"timestep": 1}
-}]
-}
diff --git a/test_jobs/parset_generator_file_input.json b/test_jobs/parset_generator_file_input.json
deleted file mode 100644
index c29a5a08f0230adfe440117df3bac1a4ca0023fd..0000000000000000000000000000000000000000
--- a/test_jobs/parset_generator_file_input.json
+++ /dev/null
@@ -1,22 +0,0 @@
-{
- "parameters_input": [{
- "step_type": "averager",
- "step_id": "average",
- "output_files": {
- "parmdb": "fily"
- },
- "output_directories": {
- "parmdb": "diry"
- },
- "parameters": {
- "timestep": 1,
- "freqstep": 1,
- "strategy": {
- "class": "File",
- "location": "file:///etc/hostname",
- "size": 0,
- "nameext": ""
- }
- }
-}]
-}
diff --git a/test_jobs/predict.json b/test_jobs/predict.json
deleted file mode 100644
index 2a493f8755d1350243a7d0f04783dbddc2dda7ea..0000000000000000000000000000000000000000
--- a/test_jobs/predict.json
+++ /dev/null
@@ -1,10 +0,0 @@
-{
- "msin": {
- "class": "Directory",
- "path": "/data/L570745_SB001_uv_first10.MS"
- },
- "sources_db": {
- "class": "File",
- "path": "/data/A-Team_lowres.sourcedb"
- }
-}
diff --git a/test_jobs/preflagger_step_generator.json b/test_jobs/preflagger_step_generator.json
deleted file mode 100644
index 4c6b5085d4492d2a008d0c0cdd0bbd4139c97132..0000000000000000000000000000000000000000
--- a/test_jobs/preflagger_step_generator.json
+++ /dev/null
@@ -1,8 +0,0 @@
-{
- "input_parset": [{
- "step_type": "name",
- "step_id": "id",
- "parameters": []
-}],
- "strategy": {"class":"File", "path":"/dev/null"}
-}
diff --git a/test_jobs/selector.json b/test_jobs/selector.json
deleted file mode 100644
index 3025390d76a1ae4ba3d6aab8482129ece27319cb..0000000000000000000000000000000000000000
--- a/test_jobs/selector.json
+++ /dev/null
@@ -1,4 +0,0 @@
-{
- "select_a": true,
- "a_arguments": [{"name": "a"}]
-}