diff --git a/rfistrategies/lofar-default.lua b/rfistrategies/lofar-default.lua index 810f306b075447274ded8c663c5dc6a9adff09c6..9cd5d6ab4dd447e572115a3bba476912a0636bfb 100644 --- a/rfistrategies/lofar-default.lua +++ b/rfistrategies/lofar-default.lua @@ -27,11 +27,11 @@ function execute(input) input:clear_mask() -- For collecting statistics. Note that this is done after clear_mask(), - -- so that the statistics ignore any flags in the input data. + -- so that the statistics ignore any flags in the input data. local copy_of_input = input:copy() - + for ipol,polarization in ipairs(inpPolarizations) do - + local data = input:convert_to_polarization(polarization) data = data:convert_to_complex(representation) @@ -42,7 +42,7 @@ function execute(input) local sumthr_level = threshold_factor * base_threshold aoflagger.sumthreshold(data, sumthr_level, sumthr_level*transient_threshold_factor, true, true) - + -- Do timestep & channel flagging local chdata = data:copy() aoflagger.threshold_timestep_rms(data, 3.5) diff --git a/rfistrategies/lofar-lba-wideband.lua b/rfistrategies/lofar-lba-wideband.lua index d514af8760a6c956c12acd825d769c4db239525a..ffd8782cb5d8f0e39b73911b7339afd7347230fb 100644 --- a/rfistrategies/lofar-lba-wideband.lua +++ b/rfistrategies/lofar-lba-wideband.lua @@ -28,7 +28,7 @@ function execute(input) -- flags on input will be flagged on output. If set to false, existing flags are ignored. local exclude_original_flags = true local transient_threshold_factor = 1.0 -- decreasing this value makes detection of transient RFI more aggressive - + -- -- End of generic settings -- @@ -39,13 +39,13 @@ function execute(input) input:clear_mask() end -- For collecting statistics. Note that this is done after clear_mask(), - -- so that the statistics ignore any flags in the input data. + -- so that the statistics ignore any flags in the input data. local copy_of_input = input:copy() aoflagger.normalize_bandpass(input) - + for ipol,polarization in ipairs(flag_polarizations) do - + local pol_data = input:convert_to_polarization(polarization) local original_data @@ -140,4 +140,3 @@ function execute(input) end input:flag_nans() end - diff --git a/scripts/Ateamclipper.py b/scripts/Ateamclipper.py index c7ba705c22edb4bbcba09cf378837553aed4f452..3ca573304f2275d3b5b9e12369fed000b60d5b86 100755 --- a/scripts/Ateamclipper.py +++ b/scripts/Ateamclipper.py @@ -48,7 +48,7 @@ for pol in range(0,numpy.size(data[0,0,:])): flag[idx,chan,0] = True flag[idx,chan,1] = True flag[idx,chan,2] = True - flag[idx,chan,3] = True + flag[idx,chan,3] = True print('') for chan in range(0,numpy.size(data[0,:,0])): diff --git a/scripts/check_Ateam_separation.py b/scripts/check_Ateam_separation.py index 6e149f69bd36d4ade3f77ef2c4a5a0ec28438c50..6eed4aa79e198d9abad9c7b04310d9e0e7881886 100755 --- a/scripts/check_Ateam_separation.py +++ b/scripts/check_Ateam_separation.py @@ -28,7 +28,7 @@ targets = [ {'name' : 'CasA', 'ra' : 6.123487680622104, 'dec' : 1.0265153995604 ######################################################################## def input2strlist_nomapfile(invar): - """ + """ from bin/download_IONEX.py give the list of MSs from the list provided as a string """ @@ -48,30 +48,30 @@ def input2strlist_nomapfile(invar): def main(ms_input, min_separation = 30, outputimage = None): """ - Print seperation of the phase reference center of an input MS - + Print seperation of the phase reference center of an input MS + Parameters ---------- ms_input : str String from the list (map) of the calibrator MSs - + Returns ------- 0 --just for printing - """ + """ msname = input2strlist_nomapfile(ms_input)[0] - + # Create a measures object me = pm.measures() # Open the measurement set and the antenna and pointing table - ms = pt.table(msname) + ms = pt.table(msname) # Get the position of the first antenna and set it as reference frame - ant_table = pt.table(msname + '::ANTENNA') + ant_table = pt.table(msname + '::ANTENNA') ant_no = 0 pos = ant_table.getcol('POSITION') x = qa.quantity( pos[ant_no,0], 'm' ) @@ -107,7 +107,7 @@ def main(ms_input, min_separation = 30, outputimage = None): print('The minimal accepted distance to an A-Team source is: ' + str(min_separation) + ' deg.') json_output = [] for target in targets: - + t = qa.quantity(time[0], 's') t1 = me.epoch('utc', t) me.doframe(t1) @@ -118,7 +118,7 @@ def main(ms_input, min_separation = 30, outputimage = None): direction = me.direction('j2000', ra_qa, dec_qa) else : direction = me.direction(target['name']) - + separations.append(me.separation(pointing, direction)) # Loop through all time stamps and calculate the elevation of the pointing @@ -130,7 +130,7 @@ def main(ms_input, min_separation = 30, outputimage = None): a = me.measure(direction, 'azel') elevation = a['m1'] el.append(elevation['value']/pylab.pi*180) - + el = numpy.array(el) pylab.plot(time1, el) if target['name'] != 'Pointing': @@ -163,16 +163,16 @@ def main(ms_input, min_separation = 30, outputimage = None): json.dump(json_output, fp) return 0 - + ######################################################################## if __name__ == '__main__': import argparse parser = argparse.ArgumentParser(description='Print seperation of the phase reference center of an input MS to an A-team source') - + parser.add_argument('MSfile', type=str, nargs='+', help='One (or more MSs).') parser.add_argument('--min_separation', type=int, default=30, help='minimal accepted distance to an A-team source on the sky in degrees (will raise a WARNING). Default: 30') parser.add_argument('--outputimage', type=str, default=None, help='location of the elevation plot of the A-Team sources.') - + args = parser.parse_args() - + main(args.MSfile, args.min_separation, args.outputimage) \ No newline at end of file diff --git a/scripts/generate_input.sh b/scripts/generate_input.sh index 9294e8d944fb8fdefe30b1342ecddb6c286157f0..6f3a1ca7aef71efbab7bc94f4f1f18c511a751a4 100755 --- a/scripts/generate_input.sh +++ b/scripts/generate_input.sh @@ -71,7 +71,7 @@ cat >&3 <<-EOF solset: class: "File" path: "${SOLSET}" - h5merger: + h5merger: class: "Directory" path: "${H5DIR}" selfcal: diff --git a/scripts/make_summary.py b/scripts/make_summary.py index 2ca7ca3462857b33b978e19534e1cacc3708246c..a4e79c280cf5d91d76b30c1e8bfb1e038d779b95 100755 --- a/scripts/make_summary.py +++ b/scripts/make_summary.py @@ -218,7 +218,7 @@ def main(flagFiles=None, pipeline='LINC', run_type='calibrator', filtered_antenn return 0 if __name__=='__main__': - + parser = argparse.ArgumentParser(description='Creates summary of a given LINC run.') parser.add_argument('flagFiles', nargs='+', help='List of flag files in JSON format') parser.add_argument('--pipeline', type=str, default='LINC', help='Name of the pipeline.') @@ -244,7 +244,7 @@ if __name__=='__main__': output_fname=args.output_fname, structure_file=args.structure_file, Ateam_separation_file=args.Ateam_separation_file, solutions=args.solutions, clip_sources=args.clip_sources, demix_sources=args.demix_sources, - demix=args.demix, removed_bands=args.removed_bands, + demix=args.demix, removed_bands=args.removed_bands, min_unflagged=args.min_unflagged, refant=args.refant) sys.exit(0) \ No newline at end of file diff --git a/scripts/skynet.py b/scripts/skynet.py index f6d102ff5e5792171f1ce1a2899fe2a46b4c8c39..5440c549774deb80040718bf16fe7e134db084ac 100755 --- a/scripts/skynet.py +++ b/scripts/skynet.py @@ -17,7 +17,7 @@ def write_skymodel (ra, dec, model, outname = None): with open(outname, 'w') as skymodel: skymodel.write ('# (Name, Type, Ra, Dec, I, MajorAxis, MinorAxis, Orientation) = format\n') for i in range(len(model)): - # the angles RA and DEC should be sexigesimal coordinates, which for + # the angles RA and DEC should be sexigesimal coordinates, which for # RA is in hours, minutes, seconds (format "XXhYYmZZs") and for # DEC is in degrees, minutes, seconds (format "XXdYYmZZs"). # These should be formatted as strings. If, instead, the angles are diff --git a/scripts/sort_times_into_freqGroups.py b/scripts/sort_times_into_freqGroups.py index 4c401ff53cda51f6eb542bf22c9249aa4d971b51..0a64862d0607c8197ed23dbbac201e9752229b8a 100755 --- a/scripts/sort_times_into_freqGroups.py +++ b/scripts/sort_times_into_freqGroups.py @@ -9,7 +9,7 @@ import logging ######################################################################## def input2strlist_nomapfile(invar): - """ + """ from bin/download_IONEX.py give the list of MSs from the list provided as a string """ @@ -67,8 +67,8 @@ def main(MSfile, numSB=10, NDPPPfill=True, stepname=None, mergeLastGroup=False, ---------- MSfile : list or str List of MS filenames, or string with list, or path to a mapfile - numSB : int, optional - How many files should go into one frequency group. Values <= 0 mean put + numSB : int, optional + How many files should go into one frequency group. Values <= 0 mean put all files of the same time-step into one group. default = -1 NDPPPfill : bool, optional @@ -76,7 +76,7 @@ def main(MSfile, numSB=10, NDPPPfill=True, stepname=None, mergeLastGroup=False, fill the data with flagged dummy data. default = True target_path : str, optional - Change the path of the "groups" files to this. (I.e. write output files + Change the path of the "groups" files to this. (I.e. write output files into this directory with the subsequent NDPPP call.) default = keep path of input files stepname : str, optional @@ -85,15 +85,15 @@ def main(MSfile, numSB=10, NDPPPfill=True, stepname=None, mergeLastGroup=False, mergeLastGroup = True, truncateLastSBs = True: not allowed mergeLastGroup = True, truncateLastSBs = False: - put the files from the last group that doesn't have SBperGroup subbands - into the second last group (which will then have more than SBperGroup entries). + put the files from the last group that doesn't have SBperGroup subbands + into the second last group (which will then have more than SBperGroup entries). mergeLastGroup = False, truncateLastSBs = True: ignore last files, that don't make for a full group (not all files are used). mergeLastGroup = False, truncateLastSBs = False: keep inclomplete last group, or - with NDPPPfill=True - fill - last group with dummies. + last group with dummies. firstSB : int, optional - If set, then reference the grouping of files to this station-subband. As if a file + If set, then reference the grouping of files to this station-subband. As if a file with this station-subband would be included in the input files. (For HBA-low, i.e. 0 -> 100MHz, 55 -> 110.74MHz, 512 -> 200MHz) @@ -140,7 +140,7 @@ def main(MSfile, numSB=10, NDPPPfill=True, stepname=None, mergeLastGroup=False, for ms in time_groups[time]['files']: # Get the frequency info sw = pt.table(ms+'::SPECTRAL_WINDOW', ack=False) - freq = sw.col('REF_FREQUENCY')[0] + freq = sw.col('REF_FREQUENCY')[0] if first: file_bandwidth = sw.col('TOTAL_BANDWIDTH')[0] nchans = sw.col('CHAN_WIDTH')[0].shape[0] @@ -185,7 +185,7 @@ def main(MSfile, numSB=10, NDPPPfill=True, stepname=None, mergeLastGroup=False, # HBA-high minfreq = (float(firstSB)/512.*100e6)+200e6-freq_width/2. else: - raise ValueError('sort_times_into_freqGroups: Frequency of lowest input data is higher than 300MHz!') + raise ValueError('sort_times_into_freqGroups: Frequency of lowest input data is higher than 300MHz!') if np.min(freqliste) < minfreq: raise ValueError('sort_times_into_freqGroups: Frequency of lowest input data is lower than reference frequency!') else: @@ -212,7 +212,7 @@ def main(MSfile, numSB=10, NDPPPfill=True, stepname=None, mergeLastGroup=False, ngroups = len(freqborders)-1 if ngroups == 0: raise ValueError('sort_times_into_freqGroups: Not enough input subbands to create at least one full (frequency-)group!') - + logging.info("Will create " + str(ngroups) + " group(s) with " + str(numSB) + "file(s) each.") groupnames = [] @@ -255,7 +255,7 @@ def main(MSfile, numSB=10, NDPPPfill=True, stepname=None, mergeLastGroup=False, results = {'filenames': filenames, 'groupnames': groupnames, 'total_bandwidth': total_bandwidth} return(results) - + ######################################################################## if __name__ == '__main__': import argparse diff --git a/steps/Ateamclipper.cwl b/steps/Ateamclipper.cwl index 82be20d998cd4794643f3448e193965ffb6c9e05..a9bdcaf538ebe23509460b511f4fbe217cca04e3 100755 --- a/steps/Ateamclipper.cwl +++ b/steps/Ateamclipper.cwl @@ -1,7 +1,10 @@ class: CommandLineTool cwlVersion: v1.2 id: Ateamclipper -label: Ateamclipper +label: A-team clipper +doc: | + Clipping of the A-team sources + using LINC`s Ateamclipper script. baseCommand: - Ateamclipper.py @@ -14,25 +17,33 @@ inputs: items: Directory inputBinding: position: 0 - doc: Input measurement set + doc: Input measurementSet. + - id: number_cores type: int? default: 12 + doc: | + Number of cores to use per job for tasks with + high I/O or memory. outputs: - id: msout - doc: Output MS + doc: Output MeasurementSet. type: Directory outputBinding: glob: $(inputs.msin.basename) + - id: logfile type: File[] outputBinding: glob: Ateamclipper.log + doc: The files containing the stdout from the step. + - id: output type: File outputBinding: glob: Ateamclipper.txt + doc: A text file containing flagging fraction statistics. hints: - class: InitialWorkDirRequirement diff --git a/steps/aoflagger.cwl b/steps/aoflagger.cwl index d37daee39f51fe210a85a07d0b8f68f3754c6a8f..0db9669fdbb5c3cd6e602bff4cfaf6e188241284 100644 --- a/steps/aoflagger.cwl +++ b/steps/aoflagger.cwl @@ -1,7 +1,9 @@ class: CommandLineTool cwlVersion: v1.2 id: aoflagging -label: aoflagging +label: AOflagger +doc: | + Runs the AO-flagging module of DP3. baseCommand: DP3 @@ -13,7 +15,8 @@ inputs: prefix: msin= separate: false shellQuote: false - doc: Input Measurement Set + doc: Data to be processed in MeasurementSet format. + - id: msin_datacolumn type: string? default: DATA @@ -22,7 +25,8 @@ inputs: prefix: msin.datacolumn= separate: false shellQuote: true - doc: Input data Column + doc: The data column of the MeasurementSet to be processed. + - id: memoryperc type: int? default: 15 @@ -32,12 +36,14 @@ inputs: separate: false shellQuote: false doc: Indicates the percentage of pc memory to use + - id: keepstatistics type: boolean? default: true inputBinding: prefix: aoflagger.keepstatistics=True doc: Indicates whether statistics should be written to file. + - id: strategy type: - File? @@ -48,7 +54,8 @@ inputs: prefix: aoflagger.strategy= separate: false shellQuote: false - doc: specifies a customized strategy + doc: The name of the strategy file to use. + - id: max_dp3_threads type: int? default: 5 @@ -57,6 +64,7 @@ inputs: prefix: numthreads= separate: false shellQuote: false + doc: The number of threads per DP3 process. arguments: - steps=[aoflagger] @@ -65,14 +73,18 @@ arguments: outputs: - id: msout - doc: Output Measurement Set + doc: Output MeasurementSet. type: Directory outputBinding: glob: $(inputs.msin.basename) + - id: logfile type: File[] outputBinding: glob: aoflag*.log + doc: | + The files containing the stdout + and stderr from the step. requirements: - class: InitialWorkDirRequirement diff --git a/steps/check_ateam_separation.cwl b/steps/check_ateam_separation.cwl index 8469158cc74ef23747b23dcd15f4eceef9ebb489..cea174ea24318419fb79caf7bee291c55c548435 100644 --- a/steps/check_ateam_separation.cwl +++ b/steps/check_ateam_separation.cwl @@ -1,8 +1,15 @@ class: CommandLineTool cwlVersion: v1.2 id: check_ateam_separation +label: check Ateam separation +doc: | + Checks whether the source is sufficiently + far away from A-team sources. Produces an + image of the plotted distance. + baseCommand: - check_Ateam_separation.py + inputs: - id: ms type: @@ -11,36 +18,49 @@ inputs: items: Directory inputBinding: position: 0 - doc: Input measurement set - - default: Ateam_separation.png - id: output_image_name + doc: An array of input data in MeasurementSet format. + + - id: output_image_name + default: Ateam_separation.png type: string? inputBinding: position: 2 prefix: --outputimage + doc: The filename of the output image. + - id: min_separation type: int? inputBinding: position: 1 prefix: --min_separation + doc: | + The minimal accepted distance to an + A-team source on the sky in degrees. + outputs: - id: output_image - doc: Output image + doc: The output image containing the plotted distances. type: File outputBinding: glob: $(inputs.output_image_name) + - id: output_json - doc: Output JSON + doc: Distances to A-team sources in JSON format. type: File outputBinding: glob: '*.json' + - id: logfile type: File outputBinding: glob: Ateam_separation.log -label: check_Ateam_separation + doc: | + The files containing the stdout + and stderr from the step. + hints: - class: DockerRequirement dockerPull: astronrd/linc - class: InlineJavascriptRequirement + stdout: Ateam_separation.log diff --git a/steps/check_station_mismatch.cwl b/steps/check_station_mismatch.cwl index 40803c6b6e5f381b4844b65486fe39b8838195c2..a9fd1000a64a010e35d80c08b10c742a5b70e975 100644 --- a/steps/check_station_mismatch.cwl +++ b/steps/check_station_mismatch.cwl @@ -1,29 +1,55 @@ class: CommandLineTool cwlVersion: v1.2 id: check_station_mismatch -label: check_station_mismatch +label: Check station mismatch +doc: | + Compares the lists of stations contained in MeasurementSets + against the list of station in the solution file and ensures + both are consistent. -baseCommand: +baseCommand: - python3 - compare_station_list.py inputs: - id: msin type: Directory[] - doc: Calibrator measurement sets. + doc: Input MeasurementSets. inputBinding: position: 0 + - id: solset type: File - doc: The solution set from the prefactor pipeline. + doc: The solution set from the LINC pipeline. + - id: solset_name type: string? doc: Name of the solution set. default: vlbi + - id: filter_baselines type: string? default: "*&" - doc: Filter constrains for ndppp_prep_target. + doc: Filter constrains for the dp3_prep_target step. + +outputs: + - id: filter_out + type: string + outputBinding: + loadContents: true + glob: out.json + outputEval: $(JSON.parse(self[0].contents).filter_out) + doc: | + A JSON formatted filter command containing + the station names to filter. + + - id: logfile + type: File[] + outputBinding: + glob: compareStationMismatch*.log + doc: | + The files containing the stdout + and stderr from the step. requirements: - class: InlineJavascriptRequirement @@ -47,9 +73,9 @@ requirements: filter = inputs['filter_baselines'] print(mss) - output = compareStationList(mss, - h5parmdb = h5parmdb, - solset_name = solset_name, + output = compareStationList(mss, + h5parmdb = h5parmdb, + solset_name = solset_name, filter = filter) filter_out = output['filter'] @@ -58,19 +84,6 @@ requirements: with open('./out.json', 'w') as fp: json.dump(cwl_output, fp) -outputs: - - id: filter_out - type: string - outputBinding: - loadContents: true - glob: out.json - outputEval: $(JSON.parse(self[0].contents).filter_out) - - - id: logfile - type: File[] - outputBinding: - glob: compareStationMismatch*.log - hints: DockerRequirement: dockerPull: vlbi-cwl:latest diff --git a/steps/collectfiles.cwl b/steps/collectfiles.cwl index 6387499bf103d853f3be8b33ce4df3e454aebdba..af1a09f8de53ce3b17ce08226a965c50f9c17f9a 100644 --- a/steps/collectfiles.cwl +++ b/steps/collectfiles.cwl @@ -1,7 +1,10 @@ class: CommandLineTool cwlVersion: v1.2 id: collectfiles -label: CollectFiles +label: Collect files +doc: | + This step stores an array of files + or directories in an a directory. baseCommand: - bash @@ -10,6 +13,10 @@ baseCommand: inputs: - id: start_directory type: Directory? + doc: | + A string of the directory that + should contain the output directory. + - id: files type: - File @@ -19,10 +26,17 @@ inputs: - Directory inputBinding: position: 0 + doc: | + The files or directories that should be placed + in the output directory. + - id: sub_directory_name type: string + doc: | + A string that determines + the name of the output directory. -outputs: +outputs: - id: dir type: Directory outputBinding: @@ -30,9 +44,11 @@ outputs: $(inputs.start_directory === null ? inputs.sub_directory_name: inputs.start_directory.basename) requirements: + - class: InlineJavascriptRequirement - class: InitialWorkDirRequirement listing: - entryname: collect_files.sh + writable: false entry: | #!/bin/bash set -e @@ -47,5 +63,3 @@ requirements: echo $OUTPUT_PATH mkdir -p $OUTPUT_PATH cp -rL $* $OUTPUT_PATH - writable: false - - class: InlineJavascriptRequirement diff --git a/steps/concatenate_files.cwl b/steps/concatenate_files.cwl index 15972abc53861e3c5bc8b2f22e0bf9f01319bd05..6a277ae253339489148ccda117739159f7a01edb 100644 --- a/steps/concatenate_files.cwl +++ b/steps/concatenate_files.cwl @@ -1,10 +1,14 @@ class: CommandLineTool cwlVersion: v1.2 id: concatfiles -label: concatfiles +label: Concatenate files +doc: | + Takes an array of text files and concatenates them. + The output file can be given a file name and, optionally, + a suffix. baseCommand: - - bash + - bash - bulk_rename.sh requirements: @@ -26,14 +30,20 @@ inputs: type: File[] inputBinding: position: 0 + doc: The list of files to be concatenated. + - id: file_prefix type: string + doc: The output file name. + - id: file_suffix type: string? default: log + doc: The output file extension. outputs: - id: output type: File outputBinding: glob: "$(inputs.file_prefix).$(inputs.file_suffix)" + doc: The concatenated file. diff --git a/steps/copyST_gains.cwl b/steps/copyST_gains.cwl deleted file mode 100644 index 6f4a77d7fe3b8af1c747674bf528e40fa0c8c027..0000000000000000000000000000000000000000 --- a/steps/copyST_gains.cwl +++ /dev/null @@ -1,70 +0,0 @@ -cwlVersion: v1.2 -class: CommandLineTool -id: copyST_gains -label: copyST_gains - -baseCommand: - - python3 - - run_h5merger.py - -inputs: - - id: msin - type: Directory - doc: Input measurement set. - - id: selfcal_solution - type: File - doc: Self-calibration solution h5 file. - - id: h5merger - type: Directory - doc: Path to a copy of the LOFAR_helper scripts. - -outputs: - - id: h5parm - type: File - doc: | - Selfcal_solution with linear polarization and - addition of ST001 solutions for core stations. - outputBinding: - glob: '*_toapply.h5' - - id: logfile - type: File[] - outputBinding: - glob: copyST_gains*.log - -requirements: - - class: InitialWorkDirRequirement - listing: - - entry: $(inputs.selfcal_solution) - writable: true - - entryname: run_h5merger.py - entry: | - import subprocess - import json - - inputs = json.loads(r"""$(inputs)""") - msin = inputs['msin']['path'] - h5parm = inputs['selfcal_solution'] - # ensure that the path points at the actual script - h5_merger = inputs['h5merger']['path'] + '/h5_merger.py' - - h5parm_in = h5parm['basename'] - h5parm_out = h5parm['basename'].replace('.h5','_toapply.h5') - - ## run on the h5parm using a measurement set for the core station info - h5_modify = subprocess.run(['python3', f'{h5_merger}', - '--ms', f'{msin}', - '--h5_tables', f'{h5parm_in}', - '--h5_out', f'{h5parm_out}', - '--circ2lin', '--add_cs', - '--h5_time', f'{h5parm_in}'], - capture_output=True, text=True) - - print("run_h5merger stdout:", h5_modify.stdout) - print("run_h5merger stderr:", h5_modify.stderr) - -hints: - - class: DockerRequirement - dockerPull: vlbi-cwl - -stdout: copyST_gains.log -stderr: copyST_gains_err.log diff --git a/steps/delay_cal_model.cwl b/steps/delay_cal_model.cwl index 1fc3f99cbc2510717d2ee0f244663b7d79615bc8..95f9b5c2496c63b9cb8abf26ba1b0c0950791584 100644 --- a/steps/delay_cal_model.cwl +++ b/steps/delay_cal_model.cwl @@ -1,19 +1,22 @@ class: CommandLineTool cwlVersion: v1.2 id: delay_cal_model -label: delay_cal_model +label: Delay cal model +doc: | + Creates a skymodel for use in the self-calibration. baseCommand: skynet.py inputs: - id: msin type: Directory - doc: Input measurement set. + doc: Input data in MeasurementSet format. inputBinding: position: 0 + - id: delay_calibrator type: File - doc: Coordinates of best delay calibrator. + doc: Coordinates of a suitable delay calibrator. inputBinding: position: 1 prefix: --delay-cal-file @@ -24,14 +27,21 @@ outputs: type: File outputBinding: glob: $(inputs.msin.basename)/skymodel + doc: The skymodel of the delay calibrator. + - id: msout type: Directory outputBinding: glob: $(inputs.msin.basename) + doc: The input data. + - id: logfile type: File[] outputBinding: glob: delay_cal_model*.log + doc: | + The files containing the stdout + and stderr from the step. hints: - class: DockerRequirement diff --git a/steps/delay_solve.cwl b/steps/delay_solve.cwl index a30ccb7456f88bec5e0b6a3dedfa4f84c87012bf..97d3aaabf61c76b29f9f90c6ea0fd5c1ea56a114 100644 --- a/steps/delay_solve.cwl +++ b/steps/delay_solve.cwl @@ -1,7 +1,10 @@ cwlVersion: v1.2 class: CommandLineTool id: delay_solve -label: delay_solve +label: Delay solve +doc: | + Performs direction independent calibration + of the international antenna array. baseCommand: - python3 @@ -10,37 +13,55 @@ baseCommand: inputs: - id: msin type: Directory - doc: Delay calibrator measurement set. + doc: | + Input data phase-shifted to the + delay calibrator in MeasurementSet format. + - id: skymodel type: File - doc: The skymodel to be used in the first cycle in the self-calibration. + doc: | + The skymodel to be used in the first + cycle in the self-calibration. inputBinding: prefix: --skymodel= separate: false shellQuote: false + - id: configfile type: File - doc: Configuration options for self-calibration. + doc: A plain-text file containing configuration options for self-calibration. + - id: selfcal type: Directory doc: External self-calibration script. + - id: h5merger type: Directory - doc: External LOFAR helper scripts for merging h5 files. + doc: External LOFAR helper scripts for merging HDF5 files. outputs: - id: h5parm type: File outputBinding: glob: merged_addCS_selfcalcyle009_linear*.h5 + doc: | + The calibration solution files generated + by lofar_facet_selfcal in HDF5 format. + - id: images type: File[] outputBinding: glob: image*.png + doc: | + Delay calibrator images generated by lofar_facet_selfcal. + - id: logfile type: File[] outputBinding: glob: delay_solve*.log + doc: | + The files containing the stdout + and stderr from the step. requirements: - class: ShellCommandRequirement @@ -57,7 +78,7 @@ requirements: import json inputs = json.loads(r"""$(inputs)""") - + msin = inputs['msin']['basename'] configfile = inputs['configfile']['path'] skymodel = inputs['skymodel']['path'] diff --git a/steps/dp3_applycal.cwl b/steps/dp3_applycal.cwl index fa396a4ebac512e6bcf43b8a5d5b56b2d460d5c6..23b3ba6a7c202c81b6a00867bb19069ec201a4ec 100644 --- a/steps/dp3_applycal.cwl +++ b/steps/dp3_applycal.cwl @@ -1,7 +1,10 @@ cwlVersion: v1.2 class: CommandLineTool id: dp3_applycal -label: dp3_applycal +label: DP3 applycal +doc: | + Applies the solutions generated by the + DDFacet pipeline to the input data. baseCommand: DP3 @@ -15,18 +18,22 @@ arguments: inputs: - id: msin type: Directory - doc: Input measurement set. + doc: Input data in MeasurementSet format. inputBinding: position: 0 prefix: msin= separate: false + - id: ddf-solset type: File? - doc: The solution tables generated by the DDF pipeline in an HDF5 format. + doc: | + The solution tables generated by + the DDF pipeline in an HDF5 format. inputBinding: position: 0 prefix: applyddf.parmdb= separate: false + - id: max_dp3_threads type: int? default: 5 @@ -34,6 +41,7 @@ inputs: inputBinding: prefix: numthreads= separate: false + - id: msin_datacolumn type: string? default: DATA @@ -41,34 +49,52 @@ inputs: prefix: msin.datacolumn= separate: false shellQuote: false + doc: The name of the input data column. + - id: msout_datacolumn type: string? default: DATA inputBinding: prefix: msout.datacolumn= separate: false + doc: The column to write the output data to. + - id: storagemanager type: string? default: dysco inputBinding: prefix: msout.storagemanager= separate: false + doc: | + String that specifies what storage manager + to use. By default uses dysco compression. + - id: fullresflags type: boolean? default: False inputBinding: prefix: msout.writefullresflag= separate: false + doc: | + Specifies whether to write + the full resolution flags. outputs: - id: msout type: Directory outputBinding: glob: $(msin.basename) + doc: | + The output data with corrected + data in MeasurementSet format. + - id: logfile type: File[] outputBinding: glob: dp3_applycal*.log + doc: | + The files containing the stdout + and stderr from the step. hints: - class: DockerRequirement diff --git a/steps/dp3_concat.cwl b/steps/dp3_concat.cwl index ded3d4b95b32d0c20702acfdc63809fe204aed93..16e882548b61872c168d3e7f0fe77eaa50693b14 100644 --- a/steps/dp3_concat.cwl +++ b/steps/dp3_concat.cwl @@ -1,6 +1,10 @@ class: CommandLineTool cwlVersion: v1.2 id: dp3concat +label: DP3 concatenate +doc: | + Reduces the dataset by concatenating + subbands into frequency groups. baseCommand: - DP3 @@ -8,9 +12,10 @@ baseCommand: inputs: - id: msin type: Directory[] - doc: Input measurement sets + doc: Input data in MeasurementSet format. + - id: msin_filenames - doc: Input measurement set string (including dummy.ms) + doc: An array of MeasurementSets to be concatenated. type: string[] inputBinding: position: 0 @@ -18,13 +23,16 @@ inputs: separate: false itemSeparator: ',' valueFrom: "[$(self.join(','))]" + - id: msout_name type: string + doc: The name of the output data in MeasurementSet format. inputBinding: position: 0 prefix: msout= separate: false shellQuote: false + - id: storagemanager type: string? default: 'dysco' @@ -33,6 +41,10 @@ inputs: separate: false shellQuote: false position: 0 + doc: | + A string that specifies what storage manager + to use. By default uses `dysco` compression. + - id: datacolumn_in type: string? default: 'DATA' @@ -41,6 +53,10 @@ inputs: separate: false shellQuote: false position: 0 + doc: | + The name of the data column from + which the input data is read. + - id: datacolumn_out type: string? default: 'DATA' @@ -49,6 +65,10 @@ inputs: separate: false shellQuote: false position: 0 + doc: | + The name of the data column into + which the output data is written. + - id: max_dp3_threads type: int? default: 5 @@ -57,23 +77,34 @@ inputs: separate: false shellQuote: false position: 0 + doc: The number of CPU threads to use. outputs: - id: msout - doc: Output Measurement Set + doc: | + The output data with corrected + data in MeasurementSet format. type: Directory outputBinding: glob: $(inputs.msout_name) + - id: flagged_statistics type: string outputBinding: loadContents: true glob: out.json outputEval: $(JSON.parse(self[0].contents).flagged_fraction_dict) + doc: | + A JSON formatted file containing flagging + statistics of the data after concatenation. + - id: logfile type: File[] outputBinding: glob: dp3_concat*.log + doc: | + The files containing the stdout + and stderr from the step. arguments: - steps=[count] diff --git a/steps/dp3_make_parset.cwl b/steps/dp3_make_parset.cwl index e60099a6cdd65cd2dbafa1493300555d30c3d93a..ff5e5ea8e1b4f046b0f0bf93deff6347eb4c3966 100755 --- a/steps/dp3_make_parset.cwl +++ b/steps/dp3_make_parset.cwl @@ -1,7 +1,8 @@ class: CommandLineTool cwlVersion: v1.2 id: make_parset -label: make_parset +label: Make parset +doc: Creates a DP3 parameter set file. baseCommand: - cp @@ -20,23 +21,30 @@ inputs: - id: flag_baselines type: string? default: "[]" - doc: flag baseline pattern, eg "[ CS013HBA*&&* ]". + doc: The flag baseline pattern, eg "[ CS013HBA*&&* ]". + - id: station_mismatch type: string? default: "*&" - doc: Filter of mismatches between solset and MSs. + doc: Filter of mismatches between solset and the input data. + - id: solset type: File - doc: The solution set from the prefactor pipeline. + doc: | + The solution tables generated by the LINC target pipeline + in an HDF5 format. + - id: phasesol type: string? default: TGSSphase + doc: The type of correction to perform. outputs: - id: parset type: File outputBinding: glob: dp3.parset + doc: A file containing input parameters for a call to DP3. requirements: - class: InlineJavascriptRequirement diff --git a/steps/dp3_phaseup.cwl b/steps/dp3_phaseup.cwl index 8927f46ed17944e1ee6b9a28103f8ebd2ab87719..14acdeeeecab7f85a4659812a0231ecc7ea74aab 100644 --- a/steps/dp3_phaseup.cwl +++ b/steps/dp3_phaseup.cwl @@ -1,7 +1,11 @@ class: CommandLineTool cwlVersion: v1.2 id: dp3_phaseup -label: dp3_phaseup +label: DP3 phaseup +doc: | + Phaseshifts data to the delay calibrator, + averages the data, combines the core + stations into one superstation. baseCommand: DP3 @@ -16,12 +20,13 @@ arguments: inputs: - id: msin type: Directory - doc: Input measurement set. + doc: Input data in MeasurementSet format. inputBinding: position: 0 prefix: msin= separate: false shellQuote: false + - id: msout_name type: string? default: "dp3-phaseup-" @@ -31,40 +36,56 @@ inputs: separate: false valueFrom: | $(self + "_" + inputs.msin.basename) + doc: | + Filename prefix for the output MeasurementSet + to be prepended to the name of the msin input. + - id: storagemanager type: string? - doc: default: 'dysco' inputBinding: position: 1 prefix: msout.storagemanager= separate: false shellQuote: false + doc: | + String that specifies what storage manager + to use. By default uses dysco compression. + - id: datacolumn_in type: string? default: 'DATA' - doc: Data column input measurement set. + doc: | + Name of the data column from which + input data is read. inputBinding: position: 1 prefix: msin.datacolumn= separate: false shellQuote: false + - id: datacolumn_out type: string? default: 'DATA' - doc: Data column output measurement set. + doc: | + Name of the data column into which + output data is written. inputBinding: position: 1 prefix: msout.datacolumn= separate: false shellQuote: false + - id: phase_center type: string - doc: 'source RA and DEC.' + doc: | + Source coordinates (right ascension, declination) + to shift the phase centre to. inputBinding: position: 1 separate: false prefix: shift.phasecenter= + - id: freqresolution type: string? default: '48.82kHz' @@ -72,6 +93,10 @@ inputs: position: 1 separate: false prefix: average1.freqresolution= + doc: | + Target frequency resolution for + the first averaging. + - id: timeresolution type: float? default: 4.0 @@ -79,13 +104,20 @@ inputs: position: 1 separate: false prefix: average1.timeresolution= + doc: | + Target time resolution in seconds + for the first averaging. + - id: beam_direction type: string - doc: 'source RA and DEC.' + doc: | + Source coordinates (right ascension, declination) + to apply beam corrections. inputBinding: position: 1 separate: false prefix: applybeam.direction= + - id: max_dp3_threads type: int? default: 5 @@ -93,6 +125,9 @@ inputs: position: 1 separate: false prefix: numthreads= + doc: | + The number of CPU threads to use. + - id: beam_mode type: string? default: full @@ -101,6 +136,7 @@ inputs: position: 1 separate: false prefix: applybeam.beammode= + - id: frequency_resolution type: string? default: 390.56kHz @@ -108,6 +144,10 @@ inputs: position: 1 separate: false prefix: average2.freqresolution= + doc: | + Target frequency resolution for + the second averaging. + - id: time_resolution type: string? default: '32.0' @@ -115,20 +155,29 @@ inputs: position: 1 separate: false prefix: average2.timeresolution= + doc: | + Target time resolution in seconds + for the second averaging. outputs: - id: msout type: Directory outputBinding: glob: $(inputs.msout_name + "_" + inputs.msin.basename) + doc: | + The phase-shifted output data in MeasurementSet format. + - id: logfile type: File outputBinding: glob: dp3_phaseup.log + doc: The file containing the stdout from the step. + - id: errorfile type: File outputBinding: glob: dp3_phaseup_err.log + doc: The file containing the stderr from the step. hints: DockerRequirement: diff --git a/steps/dp3_prep_target.cwl b/steps/dp3_prep_target.cwl index 338b2021cef0a71a41d846eeea2d4013616ae35e..25159a46fc00d64119bb3dc006eed34526a22ede 100644 --- a/steps/dp3_prep_target.cwl +++ b/steps/dp3_prep_target.cwl @@ -1,7 +1,11 @@ class: CommandLineTool cwlVersion: v1.2 id: dp3_prep_target -label: dp3_prep_target +label: DP3 prep target +doc: | + Flags requested antennas, flags erroneously + low amplitudes, applies LINC calibrator and + target solutions. baseCommand: DP3 @@ -10,14 +14,16 @@ inputs: type: File inputBinding: position: -1 - doc: DP3 parset file. + doc: DP3 parameter set file. + - id: msin type: Directory inputBinding: position: 0 prefix: msin= separate: false - doc: Input measurement set. + doc: Input data in MeasurementSet format. + - id: msout_name type: string? default: "." @@ -25,33 +31,55 @@ inputs: position: 0 prefix: msout= separate: false + doc: | + The name of the output MeasurementSet. + Defaults to overwriting the input MeasurementSet. + - id: solset type: File - doc: Input solutions file. + doc: | + The solution tables generated by the LINC target pipeline + in an HDF5 format. + - id: collect_flag_statistics_before default: true type: boolean? inputBinding: position: 0 prefix: count1.savetojson=True + doc: | + Boolean to determine whether to write flagging + information of the data to a file in JSON format. + - id: flag_statistics_filename_before type: string? default: 'out1.json' inputBinding: prefix: count1.jsonfilename= separate: false + doc: | + The filename for the flagging + information of the initial data. + - id: collect_flag_statistics_after default: true type: boolean? inputBinding: position: 0 prefix: count2.savetojson=True + doc: | + Boolean to determine whether to write flagging + information of the output data to a file in JSON format. + - id: flag_statistics_filename_after type: string? default: 'out2.json' inputBinding: prefix: count2.jsonfilename= separate: false + doc: | + The filename for the flagging + information of the output data. arguments: - applyPA.parmdb=$(inputs.solset.path) @@ -70,23 +98,33 @@ outputs: type: File[] outputBinding: glob: dp3_prep_target*.log + doc: | + The files containing the stdout + and stderr from the step. + - id: msout - doc: Output Measurement Set. + doc: Output data in MeasurementSet format. type: Directory outputBinding: glob: '$(inputs.msout_name=="." ? inputs.msin.basename : inputs.msout_name)' + - id: flag_statistics_before type: string outputBinding: loadContents: true glob: $(inputs.flag_statistics_filename_before) outputEval: $(JSON.parse(self[0].contents).flagged_fraction_dict) + doc: | + The flagging information of the initial data as a parsed JSON string. + - id: flag_statistics_after type: string outputBinding: loadContents: true glob: $(inputs.flag_statistics_filename_after) outputEval: $(JSON.parse(self[0].contents).flagged_fraction_dict) + doc: | + The flagging information of the output data as a parsed JSON string. hints: DockerRequirement: diff --git a/steps/filter_ms_group.cwl b/steps/filter_ms_group.cwl index 9b044d0b0f324a448b8fa85d23d7598877a0cb40..8c64bc7701265d791386f869e78398ae36948d68 100644 --- a/steps/filter_ms_group.cwl +++ b/steps/filter_ms_group.cwl @@ -1,7 +1,10 @@ class: CommandLineTool cwlVersion: v1.2 id: filter_ms_group -label: filter_ms_group +label: Filter MeasurementSet group +doc: | + Collects MeasurementSets for concatenation, + excluding dummy data. baseCommand: - python3 @@ -13,14 +16,22 @@ arguments: inputs: - id: group_id type: string + doc: | + A string that determines which + MeasurementSets should be combined. + - id: groups_specification type: File inputBinding: position: 1 + doc: | + A file containing directories of MeasurementSets. + - id: measurement_sets type: Directory[] inputBinding: position: 2 + doc: The total number of input MeasurementSets. outputs: - id: selected_ms @@ -29,6 +40,8 @@ outputs: loadContents: true glob: 'out.json' outputEval: $(JSON.parse(self[0].contents).selected_ms) + doc: | + The names of the selected MeasurementSets. requirements: - class: InlineJavascriptRequirement diff --git a/steps/findRefAnt_join.cwl b/steps/findRefAnt_join.cwl index 324cf8bd19590d99d34f46fe955215195371b044..e5ecf40b1891aa5fec52ecc035e9b3d5c43fd876 100644 --- a/steps/findRefAnt_join.cwl +++ b/steps/findRefAnt_join.cwl @@ -1,7 +1,11 @@ class: CommandLineTool cwlVersion: v1.2 id: findRefAnt_join -label: findRefAnt_join +label: Find Reference Antenna join +doc: | + Selects station data from a dictionary, determines + and outputs the fraction of flagged data and the + station name with the least amount of flagged data. baseCommand: - python3 @@ -11,15 +15,17 @@ inputs: - id: flagged_fraction_dict type: string[]? default: [] - doc: list of flagged antennas per MS + doc: A list of flagged antennas per MeasurementSet. + - id: filter_station type: string? default: '*&' - doc: Filter these baselines for the comparison + doc: A regular expression pattern for station names to select. + - id: state type: string? default: 'NONE' - doc: Provide state information for collecting antenna statistics + doc: State information for the collection of antenna statistics. outputs: - id: refant @@ -28,14 +34,25 @@ outputs: loadContents: true glob: out.json outputEval: $(JSON.parse(self[0].contents).refant) + doc: | + The reference antenna, containing + the least amount of flagged data. + - id: flagged_fraction_antenna type: File outputBinding: glob: flagged_fraction_antenna.json + doc: | + The fraction of flagged data per + antenna in a CSV format. + - id: logfile type: File outputBinding: glob: findRefAnt.log + doc: | + The files containing the stdout + and stderr from the step. requirements: - class: InlineJavascriptRequirement diff --git a/steps/predict.cwl b/steps/predict.cwl index 0803ccf51a1ef11a189a210ac7c2809dcf99c146..66fb847b11b9672e6c65afbd4a2a602a044eeb48 100644 --- a/steps/predict.cwl +++ b/steps/predict.cwl @@ -1,7 +1,12 @@ class: CommandLineTool cwlVersion: v1.2 id: predict_ateam -label: predict_ateam +label: DP3 predict A-team +doc: | + Simulates data for the A-team sources + based off a skymodel. Writes the simulated + data to the MSOUT datacolumn of the given + MeasurementSet. baseCommand: - DP3 @@ -14,7 +19,8 @@ inputs: prefix: msin= separate: false shellQuote: false - doc: Input Measurement Set + doc: Input data in MeasurementSet format. + - id: msin_datacolumn type: string? default: DATA @@ -23,7 +29,10 @@ inputs: prefix: msin.datacolumn= separate: false shellQuote: false - doc: Input data Column + doc: | + Data column of the MeasurementSet + from which input data is read. + - id: msout_datacolumn type: string? default: MODEL_DATA @@ -32,6 +41,10 @@ inputs: prefix: msout.datacolumn= separate: false shellQuote: false + doc: | + Data column of the MeasurementSet + into which output data is written. + - id: skymodel type: - File? @@ -42,6 +55,10 @@ inputs: prefix: predict.sourcedb= separate: false shellQuote: false + doc: | + A file containing a suitable skymodel or + the path to such a file. + - id: sources type: string[]? default: @@ -56,6 +73,10 @@ inputs: itemSeparator: ',' valueFrom: '"$(self)"' shellQuote: false + doc: | + Labels of the skymodel patches to + use to simulate visibilities. + - id: usebeammodel type: boolean? default: true @@ -63,6 +84,9 @@ inputs: position: 0 prefix: predict.usebeammodel=True shellQuote: false + doc: | + Determines whether to use the beam model. + - id: storagemanager type: string? default: "dysco" @@ -70,6 +94,10 @@ inputs: prefix: msout.storagemanager= separate: false shellQuote: false + doc: | + String that specifies what storage manager + to use. By default uses `dysco` compression. + - id: databitrate type: int? default: 0 @@ -77,6 +105,11 @@ inputs: prefix: msout.storagemanager.databitrate= separate: false shellQuote: false + doc: | + Number of bits per float used for columns + containing visibilities. Default compresses + weights only. + - id: max_dp3_threads type: int? default: 5 @@ -84,6 +117,7 @@ inputs: prefix: numthreads= separate: false shellQuote: false + doc: The number of threads per DP3 process. requirements: - class: InitialWorkDirRequirement @@ -104,14 +138,18 @@ arguments: outputs: - id: msout - doc: Output Measurement Set + doc: Output data in MeasurementSet format. type: Directory outputBinding: glob: $(inputs.msin.basename) + - id: logfile type: File[] outputBinding: glob: predict_ateam*.log + doc: | + The files containing the stdout + and stderr from the step. hints: - class: DockerRequirement diff --git a/steps/prep_delay.cwl b/steps/prep_delay.cwl index 9c18409818b9a690c3baf3eed77e7fc40205e497..d46bf841270485f66345bc179ae9e77cd0424622 100644 --- a/steps/prep_delay.cwl +++ b/steps/prep_delay.cwl @@ -1,7 +1,9 @@ class: CommandLineTool cwlVersion: v1.2 id: prep_delay -label: prep_delay +label: Prepare delay +doc: | + Converts the delay calibrator information into strings. baseCommand: - python3 @@ -10,10 +12,17 @@ baseCommand: inputs: - id: delay_calibrator type: File - doc: file containing target info. + doc: | + The file containing the properties and + coordinates of the delay calibrator. + - id: extract_single type: boolean? default: false + doc: | + A boolean that, if set to true, ensures that + the step will only extract the source_id and + coordinates of the first entry of the catalogue. requirements: - class: InlineJavascriptRequirement @@ -50,6 +59,7 @@ outputs: loadContents: true glob: out.json outputEval: $(JSON.parse(self[0].contents).name) + - id: coordinates type: string doc: Catalogue source coordinates. @@ -57,10 +67,14 @@ outputs: loadContents: true glob: out.json outputEval: $(JSON.parse(self[0].contents).coords) + - id: logfile type: File[] outputBinding: glob: prep_delay*.log + doc: | + The files containing the stdout + and stderr outputs from the step. hints: DockerRequirement: diff --git a/steps/sort_concatmap.cwl b/steps/sort_concatmap.cwl index cc9e2a7fb7e17fea01ce4da45f15f81a27ffe77d..a104f6127eab5d64b77f7fa74dfc9431ca403e2f 100755 --- a/steps/sort_concatmap.cwl +++ b/steps/sort_concatmap.cwl @@ -1,7 +1,10 @@ class: CommandLineTool cwlVersion: v1.2 id: sort_concatmap -label: Sort Concatmap +label: Sort concatenate map +doc: | + Sorts the subbands into a given number + of regularly spaced frequency groups. baseCommand: - python3 @@ -13,31 +16,46 @@ inputs: - Directory[] inputBinding: position: 0 - doc: Input measurement sets + doc: Input MeasurementSets to be sorted. + - id: numbands type: int? default: 10 - doc: The number of files that have to be grouped together. + doc: The number of elements in each group. + - id: DP3fill type: boolean? default: True - doc: Add dummy file names for missing frequencies, so that DP3 can fill the data with flagged dummy data. + doc: | + Add dummy file names for missing frequencies, + so that DP3 can fill the data with flagged dummy data. + - id: stepname type: string? default: '.dp3-concat' - doc: Add this stepname into the file names of the output files. + doc: | + A string to be appended to the file names of the output files. + - id: mergeLastGroup type: boolean? default: False - doc: Add dummy file names for missing frequencies, so that DP3 can fill the data with flagged dummy data. + doc: | + Add dummy file names for missing frequencies, + so that DP3 can fill the data with flagged dummy data. + - id: truncateLastSBs type: boolean? default: False - doc: Add dummy file names for missing frequencies, so that DP3 can fill the data with flagged dummy data. + doc: | + Add dummy file names for missing frequencies, + so that DP3 can fill the data with flagged dummy data. + - id: firstSB type: int? default: null - doc: If set, reference the grouping of files to this station subband. + doc: | + If set, reference the grouping of + files to this station subband. requirements: - class: InlineJavascriptRequirement @@ -83,16 +101,23 @@ outputs: type: File outputBinding: glob: filenames.json + doc: | + A list of filenames that have to + be concatenated, in JSON format. + - id: groupnames type: string[] outputBinding: loadContents: true glob: out.json outputEval: $(JSON.parse(self[0].contents).groupnames) + doc: A string of names of the frequency groups. + - id: logfile type: File outputBinding: glob: sort_concatmap.log + doc: The file containing the stdout output from the step. hints: - class: DockerRequirement diff --git a/steps/summary.cwl b/steps/summary.cwl index 81eebd7e99199383013bc0565acc36231b5eca3a..c39b530d63b0ad84a34565e6ccc0ad9250465d8a 100644 --- a/steps/summary.cwl +++ b/steps/summary.cwl @@ -1,7 +1,10 @@ class: CommandLineTool cwlVersion: v1.2 id: summary -label: summary +label: VLBI summary +doc: | + This step creates a JSON formatted file + containing pipeline run summary statistics. baseCommand: - make_summary.py @@ -13,62 +16,70 @@ inputs: - File inputBinding: position: 1 - doc: List of files with flag information (JSON) + doc: List of files with flag information in JSON format. + - id: pipeline type: string? default: 'VLBI' inputBinding: position: 0 prefix: --pipeline - doc: Name of the pipeline + doc: Name of the pipeline. + - id: run_type default: 'calibrator' type: string? inputBinding: position: 0 prefix: --run_type - doc: Type of the pipeline + doc: The type of the pipeline. + - id: filter default: '*&' type: string? inputBinding: position: 0 prefix: --filtered_antennas - doc: Filter these antenna string from the processing. + doc: A pattern of antenna names to filter from the processing. + - id: bad_antennas default: '*&' type: string? inputBinding: position: 0 prefix: --bad_antennas - doc: Antenna string to be processed + doc: A pattern of names of antennas with bad data. + - id: structure_file default: false - type: + type: - boolean? - File? inputBinding: position: 0 prefix: --structure_file + - id: Ateam_separation_file default: false - type: + type: - boolean? - File? inputBinding: position: 0 prefix: --Ateam_separation_file + - id: solutions default: false - type: + type: - boolean? - File? inputBinding: position: 0 prefix: --solutions + - id: clip_sources default: false - type: + type: - string? - boolean? inputBinding: @@ -77,6 +88,7 @@ inputs: separate: true itemSeparator: ',' valueFrom: '$(self)' + - id: demix_sources default: false type: @@ -88,6 +100,7 @@ inputs: separate: true itemSeparator: ',' valueFrom: '$(self)' + - id: removed_bands default: false type: @@ -99,13 +112,15 @@ inputs: separate: true itemSeparator: ',' valueFrom: '$(self)' + - id: min_unflagged_fraction default: 0.5 type: float? inputBinding: position: 0 prefix: --min_unflagged - doc: minimum fraction of unflagged data per band to continue + doc: The minimum required fraction of unflagged data per band. + - id: demix default: "False" type: string? @@ -120,27 +135,32 @@ inputs: inputBinding: position: 0 prefix: --refant - doc: Reference antenna used + doc: The reference antenna used. + - id: output_fname - type: + type: - boolean? - string? default: false inputBinding: position: 0 prefix: --output_fname - doc: Name of the output file + doc: The name of the output file. outputs: - id: summary_file - doc: Summary File in JSON format + doc: Summary File in JSON format. type: File outputBinding: glob: '*.json' + - id: logfile type: File[] outputBinding: glob: summary*.log + doc: | + The files containing the stdout + and stderr from the step. hints: - class: DockerRequirement diff --git a/workflows/concatenate-flag.cwl b/workflows/concatenate-flag.cwl index 733f33c3ecab4c657744df00664d454e8eaa8016..fb6e172b00d68dde5566cbdf419d6bd254aa0acc 100644 --- a/workflows/concatenate-flag.cwl +++ b/workflows/concatenate-flag.cwl @@ -1,26 +1,46 @@ class: Workflow cwlVersion: v1.2 id: sort-concat-flag -label: sort-concat-flag +label: VLBI concatenation and flagging +doc: | + Reduces the number of MeasurementSets by concatenating + of subbands into groups by frequency, and flags bad data + in the resulting MeasurementSets. Optionally applies the + solutions from the DDF pipeline, if these are given. inputs: - id: msin type: Directory[] + doc: | + Input data in MeasurementSets. A-team data + has been removed in the setup workflow. + - id: ddf_solset type: File? - doc: The solution tables generated by the DDF pipeline in an HDF5 format. + doc: | + The solution tables generated by the + DDF pipeline in an HDF5 format. + - id: numbands type: int? default: 10 - doc: The number of files that have to be grouped together. + doc: | + The number of files that have to be + grouped together in frequency. + - id: firstSB type: int? default: null - doc: If set, reference the grouping of files to this station subband. + doc: | + If set, reference the grouping of files + to this station subband. + - id: max_dp3_threads type: int? default: 5 - doc: The maximum number of threads that DP3 should use per process. + doc: | + The maximum number of threads that DP3 + should use per process. steps: - id: sort_concatenate @@ -114,12 +134,23 @@ outputs: - id: logdir outputSource: save_logfiles/dir type: Directory + doc: | + The directory containing all the stdin + and stderr files from the workflow. + - id: msout outputSource: concatenate-flag/msout type: Directory[] + doc: | + An array of MeasurementSets + containing the concatenated data. + - id: concat_flags type: File outputSource: concat_flags_join/flagged_fraction_antenna + doc: | + A JSON formatted file containing flagging statistics + of the MeasurementSet data after concatenation. requirements: - class: SubworkflowFeatureRequirement diff --git a/workflows/delay-calibration.cwl b/workflows/delay-calibration.cwl index 4d2a822997cf5444cf28bc8aa26ee6cf736ff6bd..8890409b93eebfed1c80268d71e635160db36697 100644 --- a/workflows/delay-calibration.cwl +++ b/workflows/delay-calibration.cwl @@ -1,7 +1,17 @@ class: Workflow cwlVersion: v1.2 id: delay-calibration -label: delay-calibration +label: VLBI delay calibration +doc: | + The delay calibration pipeline does the following: + * applies LINC solutions to target data and flags + A-team sources, + * concatenates the data in groups of 10, performs + flagging on the international stations, (optionally) + applies DDF solutions to the data, + * creates a MeasurementSet with data phase-shifted + to a given delay calibrator, calibrated for direction- + independent effects. requirements: - class: SubworkflowFeatureRequirement @@ -10,40 +20,65 @@ requirements: inputs: - id: msin type: Directory[] + doc: The raw data in a MeasurementSet version 2.0 format. + - id: solset type: File - doc: The solution set from the prefactor pipeline. + doc: | + The solution tables generated by the LINC target pipeline + in an HDF5 format. + - id: delay_calibrator type: File doc: A delay calibrator catalogue in CSV format. + - id: ddf_solset type: File? - doc: The solution tables generated by the DDF pipeline in an HDF5 format. + doc: | + The solution tables generated by the DDF pipeline + in an HDF5 format. + - id: filter_baselines type: string? default: "*&" + doc: The default filter constraints for the dp3_prep_target step. + - id: flag_baselines type: string? default: "[]" + doc: The pattern used by DP3 to flag baselines, e.g. [ CS013HBA*&&* ]. + - id: phasesol type: string? default: TGSSphase + doc: The name of the target solution table to use from the solset input. + - id: configfile type: File doc: Settings for the delay calibration in delay_solve. + - id: selfcal type: Directory doc: Path of external calibration scripts. + - id: h5merger type: Directory - doc: External LOFAR helper scripts for mergin h5 files. + doc: External LOFAR helper scripts for merging h5 files. + - id: reference_stationSB type: int? default: 104 + doc: | + Subbands are concatenated in the concatenate-flag + workflow relative to this station subband. + - id: number_cores type: int? default: 12 - doc: Number of cores to use per job for tasks with high I/O or memory. + doc: | + Number of cores to use per job for tasks with + high I/O or memory. + - id: max_dp3_threads type: int? default: 5 @@ -140,23 +175,37 @@ outputs: - id: msout outputSource: phaseup/msout type: Directory + doc: | + The fully concatenated data in MeasurementSet + format, phase-shifted to the delay calibrator. - id: msouts outputSource: sort-concatenate-flag/msout type: Directory[] + doc: | + The concatenated data in MeasurementSet format after + A-team clipping and optional DDF solutions applied. - id: logs outputSource: store_logs/dir type: Directory + doc: | + The logfiles generated by the pipeline steps, + sorted per subworkflow. - id: pictures outputSource: phaseup/pictures type: File[] + doc: Inspection plots generated by lofar_facet_selfcal. - id: solutions outputSource: phaseup/solutions type: File + doc: | + The calibrated data solutions generated by lofar_facet_selfcal + in HDF5 format. - id: summary_file outputSource: phaseup/summary_file type: File + doc: Pipeline summary statistics in JSON format. diff --git a/workflows/phaseup-concat.cwl b/workflows/phaseup-concat.cwl index eb45b9a8f5f258079ea74f986547d0244e0ba3ba..2b8f55c3049dc45d44335cc5670d1b092212b099 100644 --- a/workflows/phaseup-concat.cwl +++ b/workflows/phaseup-concat.cwl @@ -1,67 +1,101 @@ class: Workflow cwlVersion: v1.2 id: phaseup-concat -label: phaseup-concat +label: VLBI phaseup and concatenation +doc: | + Phase shifts data to the in-field calibrator + and performs the direction-independent calibration. inputs: - id: msin type: Directory[] - doc: Input measurement sets. + doc: Input data in MeasurementSet format. + - id: delay_calibrator type: File doc: Catalogue file with information on in-field calibrator. + - id: numbands type: int? default: -1 doc: The number of files that have to be grouped together. + - id: firstSB type: int? default: null doc: If set, reference the grouping of files to this station subband. + - id: do_flagging type: boolean? default: false + doc: Boolean to determine whether to perform flagging of the data. + - id: configfile type: File doc: Settings for the delay calibration in delay_solve. + - id: selfcal type: Directory doc: Path of external calibration scripts. + - id: h5merger type: Directory doc: External LOFAR helper scripts for mergin h5 files. - id: flags type: File[] + doc: Flagging information in JSON format. + - id: pipeline type: string? default: 'VLBI' + doc: Name of the pipeline. + - id: run_type type: string? default: '' + doc: Type of the pipeline. + - id: filter_baselines type: string? default: '[CR]S*&' + doc: Name pattern of antennas to be filtered from the processing. + - id: bad_antennas type: string? default: '[CR]S*&' + doc: Antenna string to be processed. + - id: compare_stations_filter type: string? default: '[CR]S*&' + - id: check_Ateam_separation.json type: File + doc: | + A list of angular distances between the + delay calibrator and the A-team sources. + - id: clip_sources type: string[]? default: [] + doc: List of sources that were clipped. + - id: removed_bands type: string[]? default: [] + doc: The list of bands that were removed from the data. + - id: min_unflagged_fraction type: float? default: 0.5 + doc: The minimum fraction of unflagged data per band to continue. + - id: refant type: string? default: 'CS001HBA0' + doc: The reference antenna used. + - id: max_dp3_threads type: int? default: 5 @@ -80,6 +114,7 @@ steps: - id: logfile run: ../steps/prep_delay.cwl label: prep_delay + - id: dp3_phaseup in: - id: msin @@ -99,6 +134,7 @@ steps: run: ../steps/dp3_phaseup.cwl scatter: msin label: dp3_phaseup + - id: sort_concatenate in: - id: msin @@ -113,6 +149,7 @@ steps: - id: logfile run: ../steps/sort_concatmap.cwl label: sort_concatmap + - id: phaseup_concatenate in: - id: msin @@ -132,6 +169,7 @@ steps: run: ./subworkflows/concatenation.cwl scatter: group_id label: phaseup_concatenate + - id: phaseup_flags_join in: - id: flagged_fraction_dict @@ -146,6 +184,7 @@ steps: - id: logfile run: ../steps/findRefAnt_join.cwl label: prep_target_flags_join + - id: concat_logfiles_phaseup label: concat_logfiles_phaseup in: @@ -158,6 +197,7 @@ steps: out: - id: output run: ../steps/concatenate_files.cwl + - id: concat_logfiles_concatenate label: concat_logfiles_concatenate in: @@ -170,6 +210,7 @@ steps: out: - id: output run: ../steps/concatenate_files.cwl + - id: delay_cal_model label: delay_cal_model in: @@ -264,18 +305,37 @@ outputs: - id: msout type: Directory outputSource: delay_cal_model/msout + doc: | + The data in MeasurementSet format after + phase-shifting to the delay calibrator. + - id: solutions type: File outputSource: delay_solve/h5parm + doc: | + The calibrated solutions for the + delay calibrator in HDF5 format. + - id: logdir outputSource: save_logfiles/dir type: Directory + doc: | + The directory containing all the stdin + and stderr files from the workflow. + - id: pictures type: File[] outputSource: delay_solve/images + doc: | + The inspection plots generated + by delay_solve. + - id: summary_file type: File outputSource: summary/summary_file + doc: | + Pipeline summary statistics + in JSON format. requirements: - class: SubworkflowFeatureRequirement diff --git a/workflows/setup.cwl b/workflows/setup.cwl index 88971f66a265a14325d02f4ccb55ebdc6a104016..36872b8330ebf9c5214bc6c58cd647d86c066791 100644 --- a/workflows/setup.cwl +++ b/workflows/setup.cwl @@ -1,33 +1,66 @@ class: Workflow cwlVersion: v1.2 -id: vlbi-setup -label: vlbi-setup +id: setup +label: VLBI setup +doc: | + Applies the LINC calibrator and target solutions to the + input data and clips A-team contributions based on + predicted skymodel data. inputs: - id: msin type: Directory[] + doc: | + The raw input data in a MeasurementSet + version 2.0 format. + - id: solset type: File - doc: The solution set from the prefactor pipeline. + doc: | + The solution tables generated by the LINC + target pipeline in an HDF5 format. + - id: filter_baselines type: string? default: "*&" + doc: | + The default filter constraints + for the dp3_prep_target step. + - id: flag_baselines type: string? default: "[]" + doc: | + The pattern used by DP3 to flag + baselines, e.g. [ CS013HBA*&&* ]. + - id: phasesol type: string? default: TGSSphase + doc: + The name of the target solution table + to use from the solset input. + - id: min_separation type: int? default: 30 + doc: | + The minimal accepted angular distance + to an A-team source on the sky in degrees. + - id: number_cores type: int? default: 12 + doc: | + The minimum number of cores that should be + available for steps that require high I/O. + - id: max_dp3_threads type: int? default: 5 - doc: The maximum number of threads DP3 should use per process. + doc: | + The maximum number of threads DP3 + should use per process. requirements: - class: SubworkflowFeatureRequirement @@ -48,8 +81,9 @@ steps: - id: logfile run: ../steps/check_station_mismatch.cwl label: check_station_mismatch + - id: check_ateam_separation - in: + in: - id: ms source: - msin @@ -61,6 +95,7 @@ steps: - id: logfile run: ../steps/check_ateam_separation.cwl label: check_Ateam_separation + - id: dp3_make_parset in: - id: flag_baselines @@ -74,13 +109,14 @@ steps: out: - id: parset run: ../steps/dp3_make_parset.cwl + - id: clip_A-team in: - id: parset source: dp3_make_parset/parset - id: msin linkMerge: merge_flattened - source: + source: - msin - id: solset source: solset @@ -96,6 +132,7 @@ steps: run: ./subworkflows/clip_A-team.cwl scatter: msin label: clip_A-team + - id: concat_logfiles_clip_A-team label: concat_logfiles_clip_A-team in: @@ -108,6 +145,7 @@ steps: out: - id: output run: ../steps/concatenate_files.cwl + - id: initial_flags_join in: - id: flagged_fraction_dict @@ -122,6 +160,7 @@ steps: - id: logfile run: ../steps/findRefAnt_join.cwl label: initial_flags_join + - id: prep_target_flags_join in: - id: flagged_fraction_dict @@ -136,6 +175,7 @@ steps: - id: logfile run: ../steps/findRefAnt_join.cwl label: prep_target_flags_join + - id: save_logfiles in: - id: files @@ -155,18 +195,42 @@ outputs: - id: logdir outputSource: save_logfiles/dir type: Directory + doc: | + The directory containing all the stdin and + stderr files from the workflow. + - id: parset outputSource: dp3_make_parset/parset type: File + doc: | + The parameter set used by DP3. + Used for debugging purposes. + - id: msout outputSource: clip_A-team/msout type: Directory[] + doc: | + The input data in MeasurementSet format after + the LINC solutions have been applied and A-team + sources have been removed. + - id: initial_flags outputSource: initial_flags_join/flagged_fraction_antenna type: File + doc: | + A JSON formatted file containing flagging + statistics of all the initial data. + - id: prep_target_flags outputSource: prep_target_flags_join/flagged_fraction_antenna type: File + doc: | + A JSON formatted file containing flagging + statistics of all the data after application of LINC solutions. + - id: check_Ateam_separation_file outputSource: check_ateam_separation/output_json type: File + doc: | + JSON formatted file containing the A-team sources + within min_separation degrees of the target. diff --git a/workflows/subworkflows/clip_A-team.cwl b/workflows/subworkflows/clip_A-team.cwl index 5358e0d31328fa5f1b3901538e80a2dbba0d0ff5..174b4ec5fe8ca6fcc6b44fd468ca1d957f556b8c 100644 --- a/workflows/subworkflows/clip_A-team.cwl +++ b/workflows/subworkflows/clip_A-team.cwl @@ -1,7 +1,12 @@ class: Workflow cwlVersion: v1.2 id: clip_A-team -label: clip_A-team +label: Clip A-team +doc: | + This workflow performs the actual application of + LINC solutions to the data. It then simulates + A-team visibilities based on the given catalogue, + and clips the resulting A-team sources from the data. requirements: - class: InlineJavascriptRequirement @@ -9,17 +14,22 @@ requirements: inputs: - id: parset - doc: DP3 parset. type: File + doc: Parameter set file used by DP3. + - id: msin - doc: input measurement set. type: Directory + doc: input data in MeasurementSet format. + - id: solset - doc: H5 solutions file. type: File + doc: The solution tables generated by the LINC target pipeline in an HDF5 format. + - id: number_cores type: int? default: 12 + doc: The minimum number of cores that should be available for steps that require high I/O. + - id: max_dp3_threads type: int? default: 5 @@ -121,15 +131,31 @@ outputs: outputSource: - concat_logfiles_clip_A-team/output type: File + doc: | + The file containing the `stdin` and `stderr` + from the `clip_A-team` workflow. + - id: msout outputSource: - Ateamcliptar/msout type: Directory + doc: | + The input data in MeasurementSet format after + the LINC solutions have been applied and A-team + sources have been removed. + - id: flag_statistics_before outputSource: - dp3_prep_target/flag_statistics_before type: string + doc: | + A JSON formatted file containing flagging + statistics of the initial data. + - id: flag_statistics_after outputSource: - dp3_prep_target/flag_statistics_after type: string + doc: | + A JSON formatted file containing flagging statistics + of the data after application of LINC solutions diff --git a/workflows/subworkflows/concatenation.cwl b/workflows/subworkflows/concatenation.cwl index 5998f271d4f42c00075a688a6384214c51da1213..263f81b435a2d53801f27455858e52fcfd47a84c 100644 --- a/workflows/subworkflows/concatenation.cwl +++ b/workflows/subworkflows/concatenation.cwl @@ -1,21 +1,42 @@ class: Workflow cwlVersion: v1.2 id: concatenation -label: concatenation +label: Concatenation +doc: | + Concatenation selects MeasurementSets from the + input data based on predetermined frequency + grouping, and concatenates the resulting groups. + If specified, the workflow applies DDF solutions to + the resulting MeasurementSet. inputs: - id: msin type: Directory[] + doc: | + Input data in MeasurementSets. A-team + data has been removed in `setup` workflow. + - id: ddf_solset type: File? - doc: The solution tables generated by the DDF pipeline in an HDF5 format. + doc: | + The solution tables generated by the + DDF pipeline in an HDF5 format. + - id: group_id type: string + doc: The name of the final concatenated MeasurementSet. + - id: groups_specification type: File + doc: | + A list of filenames that have to + be concatenated, in JSON format. + - id: do_flagging type: boolean? default: true + doc: Boolean to determine whether to perform flagging of the data. + - id: max_dp3_threads type: int? default: 5 @@ -67,7 +88,7 @@ steps: - id: dp3_applycal in: - id: msin - source: + source: - AOflagging/msout pickValue: first_non_null - id: ddf_solset @@ -113,17 +134,30 @@ outputs: - dp3_concat/msout pickValue: first_non_null type: Directory + doc: The data in a concatenated MeasurementSet. + - id: concatenate_logfile outputSource: dp3_concatenate_logfiles/output type: File + doc: | + The file containing the stdout and + stderr from the dp3_concatenate step. + - id: aoflag_logfile outputSource: - concat_logfiles_aoflagging/output pickValue: all_non_null type: File + doc: | + The file containing the stdout and + stderr from the AOflagging step. + - id: concat_flag_statistics type: string outputSource: dp3_concat/flagged_statistics + doc: | + A JSON formatted file containing flagging + statistics of the data after concatenation. requirements: - class: InlineJavascriptRequirement diff --git a/workflows/subworkflows/predictAteam.cwl b/workflows/subworkflows/predictAteam.cwl deleted file mode 100644 index c19bcc1a48d5f7354aafe8a235451d34925b1965..0000000000000000000000000000000000000000 --- a/workflows/subworkflows/predictAteam.cwl +++ /dev/null @@ -1,86 +0,0 @@ -class: Workflow -cwlVersion: v1.2 -id: predictAteam -label: predictAteam - -inputs: - - id: msin - type: Directory - doc: Input Measurement Set - - id: solset - type: File - doc: LINC target solutions file. - -steps: - - id: prep_target - in: - - id: parset - source: dp3_make_parset/parset - - id: msin - linkMerge: merge_flattened - source: - - msin - - id: solset - source: solset - out: - - id: logfiles - - id: msout - run: ./subworkflows/prep_target.cwl - scatter: msin - label: prep_target - - - id: predict - in: - - id: msin - source: msin - out: - - id: msout - - id: logfile - run: ../../steps/predict.cwl - label: predict - - id: Ateamcliptar - in: - - id: msin - source: predict/msout - out: - - id: msout - - id: logfile - - id: output - run: ../../steps/Ateamclipper.cwl - label: Ateamcliptar - - id: concat_logfiles_predict - in: - - id: file_list - linkMerge: merge_flattened - source: predict/logfile - - id: file_prefix - default: predict - out: - - id: output - run: ../../steps/concatenate_files.cwl - label: concat_logfiles_predict - - id: concat_logfiles_cliptar - in: - - id: file_list - linkMerge: merge_flattened - source: Ateamcliptar/logfile - - id: file_prefix - default: Ateamcliptar - out: - - id: output - run: ../../steps/concatenate_files.cwl - label: concat_logfiles_cliptar - -outputs: - - id: predict_logfile - outputSource: concat_logfiles_predict/output - type: File - - id: cliptar_logfile - outputSource: concat_logfiles_cliptar/output - type: File - - id: msout - outputSource: Ateamcliptar/msout - type: Directory - -requirements: - - class: MultipleInputFeatureRequirement diff --git a/workflows/subworkflows/prep_target.cwl b/workflows/subworkflows/prep_target.cwl deleted file mode 100644 index 243891d0480a7aae4b34cab96721402aad492d38..0000000000000000000000000000000000000000 --- a/workflows/subworkflows/prep_target.cwl +++ /dev/null @@ -1,68 +0,0 @@ -class: Workflow -cwlVersion: v1.2 -id: prep_target -label: prep_target - -requirements: - - class: InlineJavascriptRequirement - - class: SubworkflowFeatureRequirement - - class: StepInputExpressionRequirement - - class: MultipleInputFeatureRequirement - -inputs: - - id: parset - type: File - - id: msin - type: Directory - - id: solset - type: File - -steps: - - id: dp3_prep_target - label: dp3_prep_target - in: - - id: parset - source: parset - - id: msin - source: msin - - id: msout_name - source: msin - valueFrom: $("out_"+self.basename) - - id: solset - source: solset - out: - - id: logfile - - id: msout - - id: flag_statistics_before - - id: flag_statistics_after - run: ../../steps/dp3_prep_target.cwl - - id: concat_logfiles_prep_targ - label: concat_logfiles_prep_target - in: - - id: file_list - linkMerge: merge_flattened - source: - - dp3_prep_target/logfile - - id: file_prefix - default: dp3_prep_targ - out: - - id: output - run: ../../steps/concatenate_files.cwl - -outputs: - - id: logfiles - outputSource: - - concat_logfiles_prep_targ/output - type: File - - id: msout - outputSource: - - dp3_prep_target/msout - type: Directory - - id: flag_statistics_before - outputSource: - - dp3_prep_target/flag_statistics_before - type: string - - id: flag_statistics_after - outputSource: - - dp3_prep_target/flag_statistics_after - type: string