diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 0b99ee9f10ad5cf948202d1b84ad90ab6ff162a3..9d942088effa0416e62dc05eb56e7ab076ff2177 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -70,7 +70,7 @@ run_hba_calibrator:
allow_failure: true
needs: ["download_data"]
script:
- - cwltool --no-container --preserve-environment LD_LIBRARY_PATH --preserve-environment PATH --preserve-environment PYTHONPATH workflows/HBA_calibrator.cwl test_jobs/HBA_calibrator.json
+ - cwltool --no-container --preserve-environment LD_LIBRARY_PATH --preserve-environment PATH --preserve-environment PYTHONPATH --outdir results workflows/HBA_calibrator.cwl test_jobs/HBA_calibrator.json
- test_jobs/check_workflow_results.py results /builds/eosc/prefactor3-cwl/data/results_calibrator
run_hba_target:
@@ -78,7 +78,7 @@ run_hba_target:
allow_failure: true
needs: ["download_data"]
script:
- - cwltool --no-container --preserve-environment LD_LIBRARY_PATH --preserve-environment PATH --preserve-environment PYTHONPATH workflows/HBA_target.cwl test_jobs/HBA_target.json
+ - cwltool --no-container --preserve-environment LD_LIBRARY_PATH --preserve-environment PATH --preserve-environment PYTHONPATH --outdir results workflows/HBA_target.cwl test_jobs/HBA_target.json
- test_jobs/check_workflow_results.py results /builds/eosc/prefactor3-cwl/data/results_target
build-doc:
diff --git a/README.md b/README.md
index cd054be2705cd06c20a8ba8551fad0c1def5226c..28f8984f278395d3b0b77ab3c60692c296b086ff 100644
--- a/README.md
+++ b/README.md
@@ -1,7 +1,92 @@
-# Prefactor3-CWL
+# prefactor
+## The LOFAR pre-facet calibration pipeline
-This is a CWL implementation of the Prefactor 3 pipeline, which lives at https://github.com/lofar-astron/prefactor.
+**prefactor** is a pipeline to correct for various instrumental and ionospheric effects in both **LOFAR HBA** and **LOFAR LBA** observations.
+It will prepare your data so that you will be able to use any direction-dependent calibration software, like [Rapthor](https://github.com/darafferty/rapthor), [factor](https://github.com/lofar-astron/factor), or [killMS](https://github.com/saopicc/killMS/).
+It includes:
+* removal of clock offsets between core and remote stations (using clock-TEC separation)
+* correction of the polarization alignment between XX and YY
+* robust time-independent bandpass correction
+* ionospheric RM corrections with [RMextract](https://github.com/lofar-astron/RMextract/)
+* removal of the element beam
+* advanced flagging and removal of bad data
+* mitigation of broad-band RFI and bad stations
+* direction-independent phase correction of the target, using a global sky model from [TGSS ADR](https://http://tgssadr.strw.leidenuniv.nl/) or the new Global Sky Model [GSM](http://172.104.228.177/)
+* detailled diagnostics
+
+The full documentation can be found at the [prefactor webpage](https://www.astron.nl/citt/prefactor/).
+
+**WARNING**: The current skymodel used for 3C295 is not using the Scaife&Heald flux density scale.
+
+### Software requirements
+* [DP3](https://github.com/lofar-astron/DP3) (v5.1 or later)
+* [LoSoTo](https://github.com/revoltek/losoto) (v2.2.1 or later)
+* [LSMTool](https://github.com/darafferty/LSMTool) (v1.4.3 or later)
+* [EveryBeam](https://git.astron.nl/RD/EveryBeam) (v0.2.0 or later)
+* [RMextract](https://github.com/maaijke/RMextract) (v0.4.4 or later)
+* [AOFlagger](https://gitlab.com/aroffringa/aoflagger) (v3.1 or later)
+* [WSClean](https://gitlab.com/aroffringa/wsclean) (v3.0 or later)
+* [IDG](https://gitlab.com/astron-idg/idg) (v0.8 or later)
+* [LofarStMan](https://github.com/lofar-astron/LofarStMan)
+* [Dysco](https://github.com/aroffringa/dysco.git) (v1.2 or later)
+* casacore
+* Python3 (including matplotlib, scipy, and astropy)
+* [cwltool](https://github.com/common-workflow-language/cwltool) or [toil-cwl-runner](https://github.com/DataBiosphere/toil)
+
+### Installation
+Detailled installation instructions for prefactor are available at the [prefactor documentation](https://www.astron.nl/citt/prefactor/installation.html).
+
+It is recommended to use [Docker](https://docs.docker.com/engine/install/ubuntu/) or [Singularity](https://sylabs.io/guides/3.0/user-guide/installation.html) for the use of software containers.
+
+To download and install the repository run the following commands:
+
+ git clone https://git.astron.nl/eosc/prefactor3-cwl.git prefactor --depth 1
+ cd prefactor
+ pip3 install --upgrade $PWD
+
+### Running prefactor
+Instructions how to [setup](https://www.astron.nl/citt/prefactor/parset.html) and [run](https://www.astron.nl/citt/prefactor/running.html) prefactor are also available at the prefactor documentation.
+
+### Directory structure
+prefactor contains the following sub-directories:
+* **Docker**: contains the Dockerfile for creating the Docker image which contains all necessary software packages
+* **docs**: contains the documentation for prefactor
+* **rfistrategies**: strategies for statistical RFI mitigation using [AOFlagger](https://sourceforge.net/p/aoflagger/wiki/Home/)
+* **scripts**: scripts that the pipeline calls to process data, generate plots, etc.
+* **skymodels**: skymodels that are used by the pipeline (e.g. for demixing or calibrating the calibrator)
+* **solutions**: template solutions for the use of non-supported calibrator sources
+* **steps**: steps of the pipeline described in the [Common Workflow Language (CWL)](https://www.commonwl.org/)
+* **test_jobs**: descriptions for running continouos integration (CI)
+* **workflows**: pipeline workflow desriptions in the [Common Workflow Language (CWL)](https://www.commonwl.org/)
+
+**prefactor** and its scripts were developed by:
+* Alexander Drabent
+* David Rafferty
+* Mattia Mancini
+* Marcel Loose
+* Andreas Horneffer
+* Francesco de Gasperin
+* Marco Iacobelli
+* Emanuela Orru
+* Björn Adebahr
+* Martin Hardcastle
+* George Heald
+* Soumyajit Mandal
+* Carole Roskowinski
+* Jose Sabater Montes
+* Timothy Shimwell
+* Sarrvesh Sridhar
+* Reinout van Weeren
+* Wendy Williams
+
+### Acknowledgements
<img src="https://www.egi.eu/wp-content/uploads/2020/01/eu-logo.jpeg" alt="EU Flag" width="80">
<img src="https://www.egi.eu/wp-content/uploads/2020/01/eosc-hub-v-web.png" alt="EOSC-hub logo" height="60">
-This work is co-funded by the EOSC-hub project (Horizon 2020) under Grant number 777536.
+This work is co-funded by the EOSC-hub project (Horizon 2020) under Grant number 777536.
+
+This work was supported by the BMBF Verbundforschung under the grant 05A20STA.
+
+
+The Prefactor procedure is described in this paper:
+* de Gasperin, F.; Dijkema, T. J.; Drabent, A.; Mevius, M.; Rafferty, van Weeren, R., et al. 2019, [A&A, 662, A5](http://adsabs.harvard.edu/abs/2018arXiv181107954D)
diff --git a/docs/source/calibrator.rst b/docs/source/calibrator.rst
index 5491e2e6f830a180940a86bca6bb3c0074aeaa45..34fd5720c99fb4bb313bbc1ff8b5c079b283ecd6 100644
--- a/docs/source/calibrator.rst
+++ b/docs/source/calibrator.rst
@@ -5,7 +5,7 @@ Calibrator pipeline
.. note::
- If you are running the deprecated genericpipeline version of the pipeline (**prefactor** 3.2 or older), please check the :doc:`old instrunctions page<calibrator_old>`.
+ If you are running the deprecated genericpipeline version of the pipeline (**prefactor** 3.2 or older), please check the :doc:`old instructions page<calibrator_old>`.
This pipeline processes the calibrator data in order to derive direction-independent corrections.
It will take into account the correct order of distortions to be calibrated for.
@@ -48,7 +48,7 @@ The workflow ``prep`` consists of:
Calibration of the polarization alignment (``PA``)
--------------------------------------------------
The phase solutions derived from the preparation step are now collected and loaded into **LoSoTo**.
-**LoSoTo** will derive the polarizion alignment and provide diagnostic plots under ``results/inspection/``:
+**LoSoTo** will derive the polarizion alignment and provide diagnostic plots under ``inspection/``:
- ``polalign_ph_pol??``: matrix plot of the phase solutions for the XX and YY polarization
.. image:: polalign_ph_polXX.png
@@ -127,24 +127,24 @@ The workflow ``ion`` consists of:
.. note::
All solutions are written in the h5parm file format via the steps ``H5parm_collector`` and called during all the workflows.
- The solutions are stored in the final calibrator solution set ``results/cal_values/cal_solutions.h5``.
+ The solutions are stored in the final calibrator solution set ``cal_solutions.h5``.
Further diagnostics
-------------------
-The ``results`` directory will contain all relevant outputs of the current **prefactor** run, once the pipeline has finished:
- - logfiles in ``results/logs``
- - summary file (JSON format) in ``results/summary``
- - calibration solutions in ``results/cal_values/cal_solutions.h5``
- - inspection plots in ``results/inspection``
+The results directory will contain all relevant outputs of the current **prefactor** run, once the pipeline has finished:
+ - logfiles in ``logs``
+ - summary file (JSON format) in ``??_prefactor_calibrator_summary.json``
+ - calibration solutions in ``cal_solutions.h5``
+ - inspection plots in ``inspection``
``Ateam_separation.png`` shows the distance and the elevation of A-Team sources with respect to the analyzed observation.
You can also check the calibration solutions for more details::
- $ losoto -i results/cal_values/cal_solutions.h5
+ $ losoto -i cal_solutions.h5
- Summary of results/cal_values/cal_solutions.h5
+ Summary of cal_solutions.h5
Solution set 'calibrator':
==========================
@@ -181,7 +181,7 @@ You can also check the calibration solutions for more details::
For an overall summary it is advised to check the summary logfile::
- $ cat results/logs/3c286_summary.log
+ $ cat logs/3c286_summary.log
*********************************************
*** prefactor calibrator pipeline summary ***
diff --git a/docs/source/calibrator_old.rst b/docs/source/calibrator_old.rst
index 305554f9aa0f3c7d5c5ab78b14e97091c9d4bd4c..5a51ba088287ef8e84f3cbae22aa3dcdf8affe97 100644
--- a/docs/source/calibrator_old.rst
+++ b/docs/source/calibrator_old.rst
@@ -5,7 +5,7 @@ Calibrator pipeline
.. note::
- These instructions are outdated and only valid for **prefactor** 3.2 or older. Please check the :doc:`recent instrunctions page<calibrator>`.
+ These instructions are outdated and only valid for **prefactor** 3.2 or older. Please check the :doc:`recent instructions page<calibrator>`.
This pipeline processes the calibrator data in order to derive direction-independent corrections.
diff --git a/docs/source/parset.rst b/docs/source/parset.rst
index 1b77b6911fd7f5b3789fbf632f85b28da6bb3dbb..cf6a442ede78f7aed06256fa22df501ae57d503c 100644
--- a/docs/source/parset.rst
+++ b/docs/source/parset.rst
@@ -4,7 +4,7 @@ Configuring prefactor
=====================
.. note::
- If you are running the deprecated genericpipeline version of the pipeline (**prefactor** 3.2 or older), please check the :doc:`old instrunctions page<parset_old>`.
+ If you are running the deprecated genericpipeline version of the pipeline (**prefactor** 3.2 or older), please check the :doc:`old instructions page<parset_old>`.
Preparing the configuration file
--------------------------------
@@ -114,7 +114,7 @@ If you run the target pipeline you also need to provide the calibrator solution
{"class": "Directory", "path": "?????/L228163_SB008_uv.dppp.MS"},
{"class": "Directory", "path": "?????/L228163_SB009_uv.dppp.MS"}
],
- "cal_solutions": {"class": "File", "path": "results/cal_values/cal_solutions.h5"}
+ "cal_solutions": {"class": "File", "path": "cal_solutions.h5"}
}
A detailed description of the input parameters can be found in the :doc:`calibrator` and :doc:`target` section.
diff --git a/docs/source/parset_old.rst b/docs/source/parset_old.rst
index 242c6f0335d27439174be8b37f74428c94ca2a75..7aa9f5757bdfffdb08a7f80ab1c24ed4c6d3160f 100644
--- a/docs/source/parset_old.rst
+++ b/docs/source/parset_old.rst
@@ -4,7 +4,7 @@ Configuring prefactor
=====================
.. note::
- These instructions are outdated and only valid for **prefactor** 3.2 or older. Please check the :doc:`recent instrunctions page<parset>`.
+ These instructions are outdated and only valid for **prefactor** 3.2 or older. Please check the :doc:`recent instructions page<parset>`.
Preparing the configuration file
--------------------------------
diff --git a/docs/source/pipelineoverview.rst b/docs/source/pipelineoverview.rst
index e2947afe1f9de2a13283d545c46caab86ebb8c98..d38a4a7e6a0bca2f4812edf54c5b924eeef85bf9 100644
--- a/docs/source/pipelineoverview.rst
+++ b/docs/source/pipelineoverview.rst
@@ -4,7 +4,7 @@ Pipeline overview
=================
.. note::
- If you are running the deprecated genericpipeline version of the pipeline (**prefactor** 3.2 or older), please check the :doc:`old instrunctions page<pipelineoverview_old>`.
+ If you are running the deprecated genericpipeline version of the pipeline (**prefactor** 3.2 or older), please check the :doc:`old instructions page<pipelineoverview_old>`.
**Prefactor** is organized in three major parts to process **LOFAR** data:
diff --git a/docs/source/pipelineoverview_old.rst b/docs/source/pipelineoverview_old.rst
index 23fcdd858aa0075e2ee672b5dda4281fc9607adb..074e276b32599de30aaa1ceb40802e5f761386fc 100644
--- a/docs/source/pipelineoverview_old.rst
+++ b/docs/source/pipelineoverview_old.rst
@@ -5,7 +5,7 @@ Pipeline overview
.. note::
- These instructions are outdated and only valid for **prefactor** 3.2 or older. Please check the :doc:`recent instrunctions page<pipelineoverview>`.
+ These instructions are outdated and only valid for **prefactor** 3.2 or older. Please check the :doc:`recent instructions page<pipelineoverview>`.
**Prefactor** is organized in three major parts to process **LOFAR** data:
diff --git a/docs/source/running.rst b/docs/source/running.rst
index c66cec5cbe6ddb3e3b6c42704c1906eeeb9d8230..c31de4af3675a1afb3dec3d0721fc77e168f7645 100644
--- a/docs/source/running.rst
+++ b/docs/source/running.rst
@@ -5,7 +5,7 @@ Starting a pipeline
.. note::
- If you are running the deprecated genericpipeline version of the pipeline (**prefactor** 3.2 or older), please check the :doc:`old instrunctions page<running_old>`.
+ If you are running the deprecated genericpipeline version of the pipeline (**prefactor** 3.2 or older), please check the :doc:`old instructions page<running_old>`.
Once you have the data and the input JSON file ready, you can run the pipeline, e.g., with ``cwltool`` or ``toil`` for the HBA calibrator pipeline::
diff --git a/docs/source/running_old.rst b/docs/source/running_old.rst
index b3d6f007fc37e978993b6613fe1b4b8a8327aa24..ed08be45ec4b32755711f1f7b6b8bd097260d75f 100644
--- a/docs/source/running_old.rst
+++ b/docs/source/running_old.rst
@@ -4,7 +4,7 @@ Starting a pipeline
.. note::
- These instructions are outdated and only valid for **prefactor** 3.2 or older. Please check the :doc:`recent instrunctions page<running>`.
+ These instructions are outdated and only valid for **prefactor** 3.2 or older. Please check the :doc:`recent instructions page<running>`.
Once you have the data and the parsets ready, you can run the pipeline using the
diff --git a/docs/source/target.rst b/docs/source/target.rst
index 21d49bf90de405eb701cef9b538314c8e655003e..45f7b5674c45abbaf7ab7867b4b734a2fe68b031 100644
--- a/docs/source/target.rst
+++ b/docs/source/target.rst
@@ -5,7 +5,7 @@ Target pipeline
.. note::
- If you are running the deprecated genericpipeline version of the pipeline (**prefactor** 3.2 or older), please check the :doc:`old instrunctions page<target_old>`.
+ If you are running the deprecated genericpipeline version of the pipeline (**prefactor** 3.2 or older), please check the :doc:`old instructions page<target_old>`.
This pipeline processes the target data in order to apply the direction-independent corrections from the calibrator pipeline. A first initial direction-independent self-calibration of the target field is performed, using a global sky model based on the `TGSS ADR`_ or the new `Global Sky Model`_ (GSM), and applied to the data.
@@ -89,18 +89,18 @@ The last step also incorporates full `Dysco`_ compression to save disk space. Th
.. note::
All solutions are written in the h5parm file format via the steps ``H5parm_collector`` and called during all the workflows.
-
- The solutions are stored in the final calibrator solution set ``results/cal_values/cal_solutions.h5``.
+
+ The solutions are stored in the final calibrator solution set ``cal_solutions.h5``.
Further diagnostics
-------------------
-The ``results`` directory will contain all relevant outputs of the current **prefactor** run, once the pipeline has finished:
- - fully calibrated datasets, concatenated with ``num_SBs_per_group`` subbands per MS file and averaged, if desired (see averaging options below). The DATA column of each MS contains the calibrated data (with the direction-independent solutions applied).
- - logfiles in ``results/logs``
- - summary file (JSON format) in ``results/summary``
- - calibration solutions in ``results/cal_values/cal_solutions.h5``
- - inspection plots in ``results/inspection``
+The output directory will contain all relevant outputs of the current **prefactor** run, once the pipeline has finished:
+ - fully calibrated datasets in ``results``, concatenated with ``num_SBs_per_group`` subbands per MS file and averaged, if desired (see averaging options below). The DATA column of each MS contains the calibrated data (with the direction-independent solutions applied).
+ - logfiles in ``logs``
+ - summary file (JSON format) in ``??_prefactor_target_summary.json``
+ - calibration solutions in ``cal_solutions.h5``
+ - inspection plots in ``inspection``
The following diagnostic help to assess the quality of the data reduction:
- ``Ateam_separation.png``: shows the distance and the elevation of A-Team sources with respect to the analyzed observation
@@ -120,9 +120,10 @@ The following diagnostic help to assess the quality of the data reduction:
You can also check the calibration solutions for more details::
- $ losoto -i results/cal_values/cal_solutions.h5
+ $ losoto -i cal_solutions.h5
+
+ Summary of cal_solutions.h5
- Summary of results/cal_values/cal_solutions.h5
Solution set 'calibrator':
==========================
@@ -188,8 +189,8 @@ You can also check the calibration solutions for more details::
For an overall summary it is advised to check the summary logfile::
- $ cat results/logs/???_summary.log
-
+ $ cat logs/???_summary.log
+
*****************************************
*** prefactor target pipeline summary ***
*****************************************
diff --git a/docs/source/target_old.rst b/docs/source/target_old.rst
index 68d48329bde612fe57627bb06a6f2c2a27b1620e..9145403e05ce60421fcdb3c452e51517bdb0c020 100644
--- a/docs/source/target_old.rst
+++ b/docs/source/target_old.rst
@@ -5,7 +5,7 @@ Target pipeline
.. note::
- These instructions are outdated and only valid for **prefactor** 3.2 or older. Please check the :doc:`recent instrunctions page<target>`.
+ These instructions are outdated and only valid for **prefactor** 3.2 or older. Please check the :doc:`recent instructions page<target>`.
This pipeline processes the target data in order to apply the direction-independent corrections from the calibrator pipeline (line 26). A first initial direction-independent self-calibration of the target field is performed, using a global sky model based on the `TGSS ADR`_ or the new Global Sky Model (GSM), and applied to the data.
diff --git a/steps/collectlog.cwl b/steps/collectfiles.cwl
similarity index 89%
rename from steps/collectlog.cwl
rename to steps/collectfiles.cwl
index 40c78c211ce99ba9cd6fd3eca20635216ae46386..c929ccef1d9cceab075c900acb54f7e89ef36734 100644
--- a/steps/collectlog.cwl
+++ b/steps/collectfiles.cwl
@@ -1,13 +1,13 @@
class: CommandLineTool
cwlVersion: v1.2
-id: collectlog
+id: collectfiles
baseCommand:
- bash
- - collect_logs.sh
+ - collect_files.sh
inputs:
- id: start_directory
type: Directory?
- - id: log_files
+ - id: files
type:
- File
- type: array
@@ -19,16 +19,16 @@ inputs:
- id: sub_directory_name
type: string
outputs:
- - id: log_dir
+ - id: dir
type: Directory
outputBinding:
glob: |
$(inputs.start_directory === null ? inputs.sub_directory_name: inputs.start_directory.basename)
-label: CollectLog
+label: CollectFiles
requirements:
- class: InitialWorkDirRequirement
listing:
- - entryname: collect_logs.sh
+ - entryname: collect_files.sh
entry: |
#!/bin/bash
set -e
diff --git a/test_jobs/HBA_target.json b/test_jobs/HBA_target.json
index 30777f0ce2576e9822a7fa7f79730840e08209ea..24fdcf26bc485a56b22a99e54bb1e4a62ed61ec4 100644
--- a/test_jobs/HBA_target.json
+++ b/test_jobs/HBA_target.json
@@ -85,7 +85,7 @@
"ncores": 12,
"cal_solutions": {
"class": "File",
- "path": "/builds/eosc/prefactor3-cwl/data/results_calibrator/cal_values/cal_solutions.h5"
+ "path": "/builds/eosc/prefactor3-cwl/data/results_calibrator/cal_solutions.h5"
},
"A-Team_skymodel": {
"class": "File",
diff --git a/test_jobs/check_workflow_results.py b/test_jobs/check_workflow_results.py
index e9dab56d1e93edb5e841bf75145576af6311d456..844997ff5d2c81495572c1ac83dbb6a3864ea033 100755
--- a/test_jobs/check_workflow_results.py
+++ b/test_jobs/check_workflow_results.py
@@ -8,6 +8,7 @@ from filecmp import dircmp
import h5py
import sys
import numpy as np
+import os
def check_all_files_present(dcmp, leftname='left', rightname='right'):
@@ -62,8 +63,8 @@ def main(check, control):
if not all_present:
error = True
- check_path = check + '/cal_values/cal_solutions.h5'
- control_path = control + '/cal_values/cal_solutions.h5'
+ check_path = os.path.join(check, 'cal_solutions.h5')
+ control_path = os.path.join(control, 'cal_solutions.h5')
check_h5 = h5py.File(check_path, 'r')
control_h5 = h5py.File(control_path, 'r')
diff --git a/workflows/HBA_calibrator.cwl b/workflows/HBA_calibrator.cwl
index 044e63712ee12a0d674c45f87c14a4a8657d1768..258a99bd1ffb0879c7aa67e4b5313505263595de 100644
--- a/workflows/HBA_calibrator.cwl
+++ b/workflows/HBA_calibrator.cwl
@@ -121,10 +121,22 @@ inputs:
type: string?
default: 195.3125kHz
outputs:
- - id: results
+ - id: log_files
outputSource:
- - save_results/log_dir
+ - save_logfiles/dir
type: Directory
+ - id: inspection_plots
+ outputSource:
+ - save_inspection/dir
+ type: Directory
+ - id: summary
+ outputSource:
+ - prefactor/summary_file
+ type: File
+ - id: solutions
+ outputSource:
+ - prefactor/solutions
+ type: File
steps:
- id: prefactor
in:
@@ -219,65 +231,28 @@ steps:
label: prefactor_calibrator
- id: save_logfiles
in:
- - id: log_files
+ - id: files
linkMerge: merge_flattened
source:
- prefactor/logfiles
- id: sub_directory_name
default: logs
out:
- - id: log_dir
- run: ./../steps/collectlog.cwl
+ - id: dir
+ run: ./../steps/collectfiles.cwl
label: save_logfiles
- id: save_inspection
in:
- - id: log_files
+ - id: files
linkMerge: merge_flattened
source:
- prefactor/inspection
- id: sub_directory_name
default: inspection
out:
- - id: log_dir
- run: ./../steps/collectlog.cwl
+ - id: dir
+ run: ./../steps/collectfiles.cwl
label: save_inspection
- - id: save_solutions
- in:
- - id: log_files
- source:
- - prefactor/solutions
- - id: sub_directory_name
- default: cal_values
- out:
- - id: log_dir
- run: ./../steps/collectlog.cwl
- label: save_solutions
- - id: save_summary
- in:
- - id: log_files
- source:
- - prefactor/summary_file
- - id: sub_directory_name
- default: summary
- out:
- - id: log_dir
- run: ./../steps/collectlog.cwl
- label: save_summary
- - id: save_results
- in:
- - id: log_files
- linkMerge: merge_flattened
- source:
- - save_solutions/log_dir
- - save_inspection/log_dir
- - save_logfiles/log_dir
- - save_summary/log_dir
- - id: sub_directory_name
- default: results
- out:
- - id: log_dir
- run: ./../steps/collectlog.cwl
- label: save_results
requirements:
- class: SubworkflowFeatureRequirement
- - class: MultipleInputFeatureRequirement
\ No newline at end of file
+ - class: MultipleInputFeatureRequirement
diff --git a/workflows/HBA_target.cwl b/workflows/HBA_target.cwl
index c8045687c50408ef3c1ec0128e58d025d755c2ae..834175e9c2bdd8e3eb4d1a67fe116103426ac2d6 100644
--- a/workflows/HBA_target.cwl
+++ b/workflows/HBA_target.cwl
@@ -153,10 +153,26 @@ inputs:
type: int?
default: 16
outputs:
- - id: results
+ - id: calibrated_data
outputSource:
- - save_results/log_dir
+ - save_results/dir
type: Directory
+ - id: log_files
+ outputSource:
+ - save_logfiles/dir
+ type: Directory
+ - id: inspection_plots
+ outputSource:
+ - save_inspection/dir
+ type: Directory
+ - id: summary
+ outputSource:
+ - prefactor/summary_file
+ type: File
+ - id: solutions
+ outputSource:
+ - prefactor/solutions
+ type: File
steps:
- id: prefactor
in:
@@ -269,66 +285,40 @@ steps:
label: prefactor_target
- id: save_logfiles
in:
- - id: log_files
+ - id: files
linkMerge: merge_flattened
source:
- prefactor/logfiles
- id: sub_directory_name
default: logs
out:
- - id: log_dir
- run: ./../steps/collectlog.cwl
+ - id: dir
+ run: ./../steps/collectfiles.cwl
label: save_logfiles
- id: save_inspection
in:
- - id: log_files
+ - id: files
linkMerge: merge_flattened
source:
- prefactor/inspection
- id: sub_directory_name
default: inspection
out:
- - id: log_dir
- run: ./../steps/collectlog.cwl
+ - id: dir
+ run: ./../steps/collectfiles.cwl
label: save_inspection
- - id: save_solutions
- in:
- - id: log_files
- source:
- - prefactor/solutions
- - id: sub_directory_name
- default: cal_values
- out:
- - id: log_dir
- run: ./../steps/collectlog.cwl
- label: save_solutions
- - id: save_summary
- in:
- - id: log_files
- source:
- - prefactor/summary_file
- - id: sub_directory_name
- default: summary
- out:
- - id: log_dir
- run: ./../steps/collectlog.cwl
- label: save_summary
- id: save_results
in:
- - id: log_files
+ - id: files
linkMerge: merge_flattened
source:
- - save_solutions/log_dir
- - save_inspection/log_dir
- - save_logfiles/log_dir
- - save_summary/log_dir
- prefactor/msout
- id: sub_directory_name
default: results
out:
- - id: log_dir
- run: ./../steps/collectlog.cwl
+ - id: dir
+ run: ./../steps/collectfiles.cwl
label: save_results
requirements:
- class: SubworkflowFeatureRequirement
- - class: MultipleInputFeatureRequirement
\ No newline at end of file
+ - class: MultipleInputFeatureRequirement