diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 531c86abb638ca6db6a09318cd07c118f71059eb..985db901d7e8cc55a97a1227b5832668e21ff7bb 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -8,7 +8,6 @@ stages: - download_data - validate_cwl_scripts - test_steps - - integration_test before_script: - mkdir workdir @@ -52,10 +51,4 @@ check_ateam_separation: allow_failure: true needs: ["download_data"] script: - - cwltool --no-container steps/check_ateam_separation.cwl test_jobs/check_ateam_separation.json - -prefactor_calibrator: - stage: integration_test - needs: ["download_data"] - script: - - cwltool --no-container --preserve-environment PYTHONPATH --preserve-environment PREFACTOR_DATA_ROOT workflows/HBA_calibrator.cwl test_jobs/HBA_calibrator.json + - cwltool --no-container --preserve-environment LD_LIBRARY_PATH --preserve-environment PATH --preserve-environment PYTHONPATH steps/check_ateam_separation.cwl test_jobs/check_ateam_separation.json \ No newline at end of file diff --git a/Docker/build_docker.sh b/Docker/build_docker.sh index 550bc861cdf1d10f58cc0bde20a1cfc722569048..23a6f12e00727094e750c856b4bb37b41beb4d34 100755 --- a/Docker/build_docker.sh +++ b/Docker/build_docker.sh @@ -3,8 +3,8 @@ set -e DOCKER_TAG=latest -docker build ${PWD} -f Dockerfile-base -t lofareosc/lofar-pipeline:${DOCKER_TAG} -docker push lofareosc/lofar-pipeline:${DOCKER_TAG} +#docker build ${PWD} -f Dockerfile-base -t lofareosc/lofar-pipeline:${DOCKER_TAG} +#docker push lofareosc/lofar-pipeline:${DOCKER_TAG} -docker build ${PWD} -t lofareosc/prefactor3-cwl:${DOCKER_TAG} +docker build --no-cache ${PWD} -t lofareosc/prefactor3-cwl:${DOCKER_TAG} docker push lofareosc/prefactor3-cwl:${DOCKER_TAG} diff --git a/steps/check_ateam_separation.cwl b/steps/check_ateam_separation.cwl index de6df72d7ead2c7750ce51157d5ae442fe10de59..911dfb45a99f03a1e73664ffd96711a37aa7b757 100644 --- a/steps/check_ateam_separation.cwl +++ b/steps/check_ateam_separation.cwl @@ -29,6 +29,11 @@ outputs: type: File? outputBinding: glob: $(inputs.output_image_name) + - id: output_json + doc: Output JSON + type: File + outputBinding: + glob: '*.json' - id: logfile type: File? outputBinding: diff --git a/steps/findRefAnt_join.cwl b/steps/findRefAnt_join.cwl index bf9bd2857977ed5255a9f3d37bdc52769f3854cc..e8ffabc885bc687a81010e3584d3fe8a74b1d6da 100644 --- a/steps/findRefAnt_join.cwl +++ b/steps/findRefAnt_join.cwl @@ -15,6 +15,10 @@ inputs: type: string? default: '*&' doc: Filter these baselines for the comparison + - id: state + type: string? + default: 'NONE' + doc: Provide state information for collecting antenna statistics label: findRefAnt_join @@ -25,6 +29,10 @@ outputs: loadContents: true glob: 'out.json' outputEval: $(JSON.parse(self[0].contents).refant) + - id: flagged_fraction_antenna + type: File? + outputBinding: + glob: 'flagged_fraction_antenna.json' - id: logfile type: File? outputBinding: @@ -69,10 +77,13 @@ requirements: flagged_fraction_list = [] sorted_stations = sorted(flagged_fraction_data.keys()) + + flagged_fraction_antenna = {} for antenna in sorted_stations: flagged_fraction = sum(flagged_fraction_data[antenna]) / len(flagged_fraction_data[antenna]) flagged_fraction_list.append(flagged_fraction) + flagged_fraction_antenna[antenna] = flagged_fraction try: flagged_fraction_data[flagged_fraction].append(antenna) except KeyError: @@ -82,10 +93,15 @@ requirements: refant = flagged_fraction_data[min_flagged_fraction][0] print('Selected station ' + str(refant) + ' as reference antenna. Fraction of flagged data is ' + '{:>3}'.format('{:.1f}'.format(min_flagged_fraction) + '%')) + flagged_fraction_antenna['state'] = '$(inputs.state)' + cwl_output = {'refant': str(refant)} with open('./out.json', 'w') as fp: json.dump(cwl_output, fp) + + with open('./flagged_fraction_antenna.json', 'w') as fp: + json.dump(flagged_fraction_antenna, fp) stdout: findRefAnt.log stderr: findRefAnt_err.log diff --git a/steps/summary.cwl b/steps/summary.cwl new file mode 100644 index 0000000000000000000000000000000000000000..809299e1c578b10bb3363d6bb8b9710bfe43cabd --- /dev/null +++ b/steps/summary.cwl @@ -0,0 +1,123 @@ +class: CommandLineTool +cwlVersion: v1.0 +$namespaces: + lofar: 'https://git.astron.nl/eosc/ontologies/raw/master/schema/lofar.owl' + sbg: 'https://www.sevenbridges.com/' +id: summary +baseCommand: + - make_summary_CWL.py +inputs: + - id: flagFiles + type: + - 'File[]' + - File + inputBinding: + position: 1 + doc: List of files with flag information (JSON) + - id: pipeline + type: string? + default: 'prefactor' + inputBinding: + position: 0 + prefix: '--pipeline' + doc: Name of the pipeline + - id: run_type + default: 'calibrator' + type: string? + inputBinding: + position: 0 + prefix: '--run_type' + doc: Type of the pipeline + - id: filter + default: '*&' + type: string? + inputBinding: + position: 0 + prefix: '--filtered_antennas' + doc: Filter these antenna string from the processing. + - id: bad_antennas + default: '*&' + type: string? + inputBinding: + position: 0 + prefix: '--bad_antennas' + doc: Antenna string to be processed + - id: structure_file + default: false + type: + - boolean? + - File? + inputBinding: + position: 0 + prefix: '--structure_file' + - id: Ateam_separation_file + default: false + type: + - boolean? + - File? + inputBinding: + position: 0 + prefix: '--Ateam_separation_file' + - id: solutions + default: false + type: + - boolean? + - File? + inputBinding: + position: 0 + prefix: '--solutions' + - id: clip_sources + default: false + type: + - string? + - boolean? + inputBinding: + position: 0 + prefix: '--clip_sources' + separate: true + itemSeparator: ',' + valueFrom: '$(self)' + - id: demix_sources + default: false + type: + - string? + - boolean? + inputBinding: + position: 0 + prefix: '--demix_sources' + separate: true + itemSeparator: ',' + valueFrom: '$(self)' + - id: demix + default: false + type: boolean? + inputBinding: + position: 0 + prefix: '--demix True' + - id: output_fname + type: + - boolean? + - string? + default: false + inputBinding: + position: 0 + prefix: '--output_fname' + doc: Name of the output file +outputs: + - id: summary_file + doc: Summary File in JSON format + type: File + outputBinding: + glob: '*.json' + - id: logfile + type: File[] + outputBinding: + glob: 'summary*.log' +label: summary +hints: + - class: DockerRequirement + dockerPull: lofareosc/prefactor3-cwl +stdout: summary.log +stderr: summary_err.log +$schema: + - 'https://git.astron.nl/eosc/ontologies/raw/master/schema/lofar.owl' diff --git a/steps/uvplot.cwl b/steps/uvplot.cwl new file mode 100644 index 0000000000000000000000000000000000000000..34a2a2aaa7821f20f115f6ef7672b380d8bcb2c1 --- /dev/null +++ b/steps/uvplot.cwl @@ -0,0 +1,82 @@ +class: CommandLineTool +cwlVersion: v1.0 +$namespaces: + lofar: 'https://git.astron.nl/eosc/ontologies/raw/master/schema/lofar.owl' + sbg: 'https://www.sevenbridges.com/' +id: uvplot +baseCommand: + - plot_uvcov.py +inputs: + - id: MSfiles + type: + - 'Directory[]' + - Directory + inputBinding: + position: 2 + shellQuote: false + doc: List of MS for UV-plotting + - id: output_name + type: string? + default: 'uv-coverage.png' + inputBinding: + position: 1 + prefix: '--output' + shellQuote: false + doc: Name of the output image + - id: title + default: 'uv-coverage' + type: string? + inputBinding: + position: 1 + prefix: '--title' + shellQuote: false + doc: Title of the plots + - id: limits + default: ',,,' + type: string? + inputBinding: + position: 1 + prefix: '--limits' + shellQuote: false + - id: timeslots + default: '0,10,0' + type: string? + inputBinding: + position: 1 + prefix: '--timeslots' + shellQuote: false + - id: antennas + default: '-1' + type: string? + inputBinding: + position: 1 + prefix: '--antennas' + shellQuote: false + - id: wideband + default: true + type: boolean? + inputBinding: + position: 0 + prefix: '--wideband True' + shellQuote: false +outputs: + - id: output_image + doc: Plot of the uv-coverage + type: File[] + outputBinding: + glob: "*$(inputs.output_name.split('.').pop())" + - id: logfile + type: File[] + outputBinding: + glob: 'uvplot*.log' +label: uvplot +hints: + - class: DockerRequirement + dockerPull: lofareosc/prefactor3-cwl +requirements: + - class: ShellCommandRequirement + - class: InlineJavascriptRequirement +stdout: uvplot.log +stderr: uvplot_err.log +$schema: + - 'https://git.astron.nl/eosc/ontologies/raw/master/schema/lofar.owl' diff --git a/subworkflow/apply_calibrate_bp.cwl b/subworkflow/apply_calibrate_bp.cwl index d859a3b67962d4d3265ee2ef424f1132eb9dca55..de5863ce7ef4f7b3c374d25960533cd07bde3a1e 100644 --- a/subworkflow/apply_calibrate_bp.cwl +++ b/subworkflow/apply_calibrate_bp.cwl @@ -32,6 +32,12 @@ outputs: type: File 'sbg:x': 1000 'sbg:y': -300 + - id: final_flags_out + outputSource: + - final_flags/flagged_fraction_dict + type: string? + 'sbg:x': 1000 + 'sbg:y': -200 - id: BLsmooth.log outputSource: - BLsmooth/logfile @@ -79,12 +85,12 @@ steps: source: input_h5parm - id: msout_datacolumn default: CORRECTED_DATA + - id: correction + default: polalign - id: storagemanager default: Dysco - id: databitrate default: 0 - - id: correction - default: polalign out: - id: msout - id: logfile @@ -102,12 +108,12 @@ steps: source: input_h5parm - id: msout_datacolumn default: CORRECTED_DATA + - id: correction + default: bandpass - id: storagemanager default: Dysco - id: databitrate default: 0 - - id: correction - default: bandpass - id: updateweights default: 'True' out: @@ -125,12 +131,12 @@ steps: default: CORRECTED_DATA - id: msout_datacolumn default: CORRECTED_DATA + - id: type + default: applybeam - id: storagemanager default: Dysco - id: databitrate default: 0 - - id: type - default: applybeam - id: updateweights default: 'true' - id: usechannelfreq @@ -156,12 +162,12 @@ steps: source: input_h5parm - id: msout_datacolumn default: CORRECTED_DATA + - id: correction + default: faraday - id: storagemanager default: Dysco - id: databitrate default: 0 - - id: correction - default: faraday out: - id: msout - id: logfile @@ -199,6 +205,7 @@ steps: - id: mode default: scalarphase out: + - id: msout - id: h5parm - id: logfile run: ../lofar-cwl/steps/ddecal.cwl @@ -269,4 +276,15 @@ steps: label: concat_logfiles_applyPA 'sbg:x': -200 'sbg:y': 200 + - id: final_flags + in: + - id: msin + source: applyFR/msout + out: + - id: flagged_fraction_dict + - id: logfile + run: ./../steps/findRefAnt.cwl + label: final_flags + 'sbg:x': 700 + 'sbg:y': -200 requirements: [] diff --git a/subworkflow/ndppp_prep_targ.cwl b/subworkflow/ndppp_prep_targ.cwl index 45d923d9430a8745aa6aace52a5759d98ee54cf8..f40d3ad325d74958e85d2202722f28203ce1034e 100644 --- a/subworkflow/ndppp_prep_targ.cwl +++ b/subworkflow/ndppp_prep_targ.cwl @@ -101,7 +101,22 @@ inputs: type: boolean 'sbg:x': 2234.90185546875 'sbg:y': -393.8929748535156 + - id: clip_sources + type: 'string[]?' + default: + - VirA_4_patch + - CygAGG + - CasA_4_patch + - TauAGG + 'sbg:x': 4000 + 'sbg:y': -700 outputs: + - id: prep_flags_out + outputSource: + - prep_flags/flagged_fraction_dict + type: string? + 'sbg:x': 5500 + 'sbg:y': 200 - id: prep_logfile outputSource: - concat_logfiles_dppp/output @@ -122,7 +137,7 @@ outputs: 'sbg:y': -100 - id: msout outputSource: - - predict/msout + - Ateamclipper/msout type: Directory 'sbg:x': 5502.318359375 'sbg:y': -279.49835205078125 @@ -437,11 +452,7 @@ steps: - id: sources_db source: skymodel - id: sources - default: - - VirA_4_patch - - CygAGG - - CasA_4_patch - - TauAGG + source: clip_sources - id: usebeammodel default: true - id: onebeamperpatch @@ -456,6 +467,17 @@ steps: run: ../lofar-cwl/steps/filter_predict.cwl 'sbg:x': 4500 'sbg:y': -344.5472106933594 + - id: prep_flags + in: + - id: msin + source: dp3_execute/msout + out: + - id: flagged_fraction_dict + - id: logfile + run: ./../steps/findRefAnt.cwl + label: prep_flags + 'sbg:x': 4500 + 'sbg:y': -600 - id: Ateamclipper in: - id: msin @@ -470,6 +492,6 @@ steps: 'sbg:x': 4700 'sbg:y': -344.5472106933594 requirements: - - class: SubworkflowFeatureRequirement - class: InlineJavascriptRequirement - class: StepInputExpressionRequirement + - class: SubworkflowFeatureRequirement diff --git a/workflows/HBA_calibrator.cwl b/workflows/HBA_calibrator.cwl index 16c0f9b1c9f75bceba252895e7a16ad49c45a327..906d905d83090627f8842ff00eb1e59bca522e26 100644 --- a/workflows/HBA_calibrator.cwl +++ b/workflows/HBA_calibrator.cwl @@ -296,6 +296,7 @@ steps: - id: logfiles - id: solutions - id: inspection + - id: summary_file run: ./prefactor_calibrator.cwl label: prefactor_calibrator 'sbg:x': 0 @@ -313,7 +314,7 @@ steps: run: ./../steps/collectlog.cwl label: save_logfiles 'sbg:x': 1000 - 'sbg:y': -200 + 'sbg:y': -300 - id: save_inspection in: - id: log_files @@ -327,7 +328,7 @@ steps: run: ./../steps/collectlog.cwl label: save_inspection 'sbg:x': 1000 - 'sbg:y': 0 + 'sbg:y': -100 - id: save_solutions in: - id: log_files @@ -340,7 +341,20 @@ steps: run: ./../steps/collectlog.cwl label: save_solutions 'sbg:x': 1000 - 'sbg:y': 200 + 'sbg:y': 100 + - id: save_summary + in: + - id: log_files + source: + - prefactor/summary_file + - id: sub_directory_name + default: summary + out: + - id: log_dir + run: ./../steps/collectlog.cwl + label: save_summary + 'sbg:x': 1000 + 'sbg:y': 300 - id: save_results in: - id: log_files @@ -349,6 +363,7 @@ steps: - save_solutions/log_dir - save_inspection/log_dir - save_logfiles/log_dir + - save_summary/log_dir - id: sub_directory_name default: results out: diff --git a/workflows/HBA_target.cwl b/workflows/HBA_target.cwl index 70b1aa5b7906334a79286c10cf20c112825df121..d5d1399ca570eb7c0c06cd30501c525b3811f533 100644 --- a/workflows/HBA_target.cwl +++ b/workflows/HBA_target.cwl @@ -238,11 +238,20 @@ inputs: default: null 'sbg:x': -1000 'sbg:y': 2900 + - id: clip_sources + type: 'string[]?' + default: + - VirA_4_patch + - CygAGG + - CasA_4_patch + - TauAGG + 'sbg:x': -1000 + 'sbg:y': 3000 - id: ncores type: int? default: 1 'sbg:x': -1000 - 'sbg:y': 3000 + 'sbg:y': 3100 outputs: - id: results outputSource: @@ -348,6 +357,8 @@ steps: source: proxy_user - id: proxy_pass source: proxy_pass + - id: clip_sources + source: clip_sources - id: ncores source: ncores out: @@ -355,6 +366,7 @@ steps: - id: msout - id: solutions - id: inspection + - id: summary_file run: ./prefactor_target.cwl label: prefactor_target 'sbg:x': 0 @@ -372,7 +384,7 @@ steps: run: ./../steps/collectlog.cwl label: save_logfiles 'sbg:x': 1000 - 'sbg:y': -200 + 'sbg:y': -300 - id: save_inspection in: - id: log_files @@ -386,7 +398,7 @@ steps: run: ./../steps/collectlog.cwl label: save_inspection 'sbg:x': 1000 - 'sbg:y': 0 + 'sbg:y': -100 - id: save_solutions in: - id: log_files @@ -399,7 +411,20 @@ steps: run: ./../steps/collectlog.cwl label: save_solutions 'sbg:x': 1000 - 'sbg:y': 200 + 'sbg:y': 100 + - id: save_summary + in: + - id: log_files + source: + - prefactor/summary_file + - id: sub_directory_name + default: summary + out: + - id: log_dir + run: ./../steps/collectlog.cwl + label: save_summary + 'sbg:x': 1000 + 'sbg:y': 300 - id: save_results in: - id: log_files @@ -408,6 +433,7 @@ steps: - save_solutions/log_dir - save_inspection/log_dir - save_logfiles/log_dir + - save_summary/log_dir - prefactor/msout - id: sub_directory_name default: results diff --git a/workflows/prefactor_calibrator.cwl b/workflows/prefactor_calibrator.cwl index 0f41382529729c8a7fad595ccdc92ca486096f55..ded68950f95d3ac652f50f751a520f534def8f49 100644 --- a/workflows/prefactor_calibrator.cwl +++ b/workflows/prefactor_calibrator.cwl @@ -230,7 +230,40 @@ outputs: linkMerge: merge_flattened 'sbg:x': 2000 'sbg:y': 600 + - id: summary_file + outputSource: + - ion/summary_file + type: File + 'sbg:x': 2000 + 'sbg:y': 0 steps: + - id: initial_flags + in: + - id: msin + source: msin + out: + - id: flagged_fraction_dict + run: ./../steps/findRefAnt.cwl + label: initial_flags + scatter: + - msin + 'sbg:x': -500 + 'sbg:y': -500 + - id: initial_flags_join + in: + - id: flagged_fraction_dict + source: + - initial_flags/flagged_fraction_dict + - id: filter_station + default: '' + - id: state + default: 'initial' + out: + - id: flagged_fraction_antenna + run: ./../steps/findRefAnt_join.cwl + label: initial_flags_join + 'sbg:x': -300 + 'sbg:y': -500 - id: prep in: - id: msin @@ -287,6 +320,7 @@ steps: - id: logfiles - id: outh5parm_logfile - id: check_Ateam_separation.png + - id: check_Ateam_separation.json - id: msout - id: parset - id: calibrator_name @@ -449,6 +483,7 @@ steps: - id: logfiles - id: outh5parm - id: outh5parm_logfile + - id: final_flags_join_out run: ./prefactor_calibrator/bp.cwl label: BP 'sbg:x': 1300 @@ -475,7 +510,21 @@ steps: source: clock_smooth - id: calibrator_name source: prep/calibrator_name + - id: flags + linkMerge: merge_flattened + source: + - initial_flags_join/flagged_fraction_antenna + - bp/final_flags_join_out + - id: demix + source: demix + - id: demix_sources + source: demix_sources + - id: filter_baselines + source: filter_baselines + - id: check_Ateam_separation.json + source: prep/check_Ateam_separation.json out: + - id: summary_file - id: outsolutions - id: inspection - id: logfiles diff --git a/workflows/prefactor_calibrator/bp.cwl b/workflows/prefactor_calibrator/bp.cwl index 285946f1cc63647bc1153b8a35749ec608c1d1ad..1a43352a5941e0fc907c24d63ddba2c72dff2421 100644 --- a/workflows/prefactor_calibrator/bp.cwl +++ b/workflows/prefactor_calibrator/bp.cwl @@ -87,6 +87,12 @@ outputs: type: File 'sbg:x': 1000 'sbg:y': -200 + - id: final_flags_join_out + outputSource: + - final_flags_join/flagged_fraction_antenna + type: File? + 'sbg:x': 1000 + 'sbg:y': -100 - id: inspection outputSource: - losoto_plot_A1/output_plots @@ -423,12 +429,28 @@ steps: - id: outh5parm - id: apply_pa.log - id: applybeam.log + - id: final_flags_out run: ../../subworkflow/apply_calibrate_bp.cwl label: apply_calibrate_bp scatter: - msin 'sbg:x': 500 'sbg:y': 0 + - id: final_flags_join + in: + - id: flagged_fraction_dict + source: + - apply_calibrate_bp/final_flags_out + - id: filter_station + default: '' + - id: state + default: 'final' + out: + - id: flagged_fraction_antenna + run: ./../../steps/findRefAnt_join.cwl + label: final_flags_join + 'sbg:x': 750 + 'sbg:y': 150 - id: concat_logfiles_blsmooth in: - id: file_list diff --git a/workflows/prefactor_calibrator/ion.cwl b/workflows/prefactor_calibrator/ion.cwl index 94f73066cf133126a2ec61f50786d632cec47769..d38478bcd69d93cfdd9bb891f4993a6d8dd19131 100644 --- a/workflows/prefactor_calibrator/ion.cwl +++ b/workflows/prefactor_calibrator/ion.cwl @@ -5,6 +5,27 @@ label: ion $namespaces: sbg: 'https://www.sevenbridges.com/' inputs: + - id: filter_baselines + type: string? + default: '*&' + 'sbg:x': -1000 + 'sbg:y': -600 + - id: demix_sources + type: 'string[]?' + default: + - CasA + - CygA + 'sbg:x': -1000 + 'sbg:y': -500 + - id: demix + type: boolean? + default: false + 'sbg:x': -1000 + 'sbg:y': -400 + - id: flags + type: File[] + 'sbg:x': -1000 + 'sbg:y': -300 - id: calibrator_name type: string 'sbg:x': -1000 @@ -41,7 +62,17 @@ inputs: type: File 'sbg:x': -1000 'sbg:y': 600 + - id: check_Ateam_separation.json + type: File + 'sbg:x': -1000 + 'sbg:y': 700 outputs: + - id: summary_file + outputSource: + - summary/summary_file + type: File + 'sbg:x': 1000 + 'sbg:y': -100 - id: inspection outputSource: - losoto_plot_P3/output_plots @@ -63,6 +94,7 @@ outputs: - id: logfiles outputSource: - concat_logfiles_ion/output + - concat_logfiles_summary/output type: 'File[]' linkMerge: merge_flattened 'sbg:x': 1000 @@ -255,6 +287,35 @@ steps: label: losoto_plot_Pr 'sbg:x': 0 'sbg:y': 300 + - id: summary + in: + - id: flagFiles + source: flags + linkMerge: merge_flattened + - id: pipeline + default: 'prefactor' + - id: run_type + default: 'calibrator' + - id: filter + source: filter_baselines + - id: bad_antennas + default: '' + - id: Ateam_separation_file + source: check_Ateam_separation.json + - id: solutions + source: h5parm_pointingname/outh5parm + - id: demix + source: demix + - id: demix_sources + source: demix_sources + valueFrom: "$(self.join(','))" + out: + - id: summary_file + - id: logfile + run: ../../steps/summary.cwl + label: summary + 'sbg:x': 500 + 'sbg:y': 0 - id: concat_logfiles_ion in: - id: file_list @@ -278,6 +339,21 @@ steps: label: concat_logfiles_ion 'sbg:x': 500 'sbg:y': 500 + - id: concat_logfiles_summary + in: + - id: file_list + linkMerge: merge_flattened + source: + - summary/logfile + - id: file_prefix + source: calibrator_name + valueFrom: $(self+'_summary') + out: + - id: output + run: ../../steps/concatenate_files.cwl + label: concat_logfiles_summary + 'sbg:x': 700 + 'sbg:y': 500 - id: write_solutions in: - id: h5parmFile @@ -315,5 +391,7 @@ steps: 'sbg:x': 0 'sbg:y': 700 requirements: + - class: InlineJavascriptRequirement + - class: StepInputExpressionRequirement - class: SubworkflowFeatureRequirement - class: MultipleInputFeatureRequirement diff --git a/workflows/prefactor_calibrator/prep.cwl b/workflows/prefactor_calibrator/prep.cwl index 4d40b75b588383b38e56da2ced250b3884503794..d711b7ff19169f979e6910d1d1ff5ae953e9653c 100644 --- a/workflows/prefactor_calibrator/prep.cwl +++ b/workflows/prefactor_calibrator/prep.cwl @@ -146,6 +146,12 @@ outputs: type: 'File[]' 'sbg:x': 1554.5106201171875 'sbg:y': 777 + - id: check_Ateam_separation.json + outputSource: + - check_ateam_separation/output_json + type: File + 'sbg:x': 1554.5106201171875 + 'sbg:y': 500 - id: logfiles outputSource: - concat_logfiles_calib/output @@ -310,6 +316,7 @@ steps: source: min_separation out: - id: output_imag + - id: output_json - id: logfile run: ../../steps/check_ateam_separation.cwl label: check_Ateam_separation diff --git a/workflows/prefactor_target.cwl b/workflows/prefactor_target.cwl index cb825e2ba918ac9299b516a72b1a6c6cf0a95c1f..d52c978104bdf52c4efbc326a59a64b2d472293c 100644 --- a/workflows/prefactor_target.cwl +++ b/workflows/prefactor_target.cwl @@ -238,11 +238,20 @@ inputs: default: null 'sbg:x': -1000 'sbg:y': 2900 + - id: clip_sources + type: 'string[]?' + default: + - VirA_4_patch + - CygAGG + - CasA_4_patch + - TauAGG + 'sbg:x': -1000 + 'sbg:y': 3000 - id: ncores type: int? default: 1 'sbg:x': -1000 - 'sbg:y': 3000 + 'sbg:y': 3100 outputs: - id: inspection outputSource: @@ -275,7 +284,40 @@ outputs: linkMerge: merge_flattened 'sbg:x': 2000 'sbg:y': 600 + - id: summary_file + outputSource: + - finalize/summary_file + type: File + 'sbg:x': 2000 + 'sbg:y': 300 steps: + - id: initial_flags + in: + - id: msin + source: msin + out: + - id: flagged_fraction_dict + run: ./../steps/findRefAnt.cwl + label: initial_flags + scatter: + - msin + 'sbg:x': -500 + 'sbg:y': -500 + - id: initial_flags_join + in: + - id: flagged_fraction_dict + source: + - initial_flags/flagged_fraction_dict + - id: filter_station + default: '' + - id: state + default: 'initial' + out: + - id: flagged_fraction_antenna + run: ./../steps/findRefAnt_join.cwl + label: initial_flags_join + 'sbg:x': -300 + 'sbg:y': -500 - id: prep in: - id: msin @@ -292,8 +334,6 @@ steps: source: filter_baselines - id: raw_data source: raw_data - - id: propagatesolutions - source: propagatesolutions - id: demix_sources source: - demix_sources @@ -345,10 +385,15 @@ steps: source: proxy_user - id: proxy_pass source: proxy_pass + - id: clip_sources + source: clip_sources out: + - id: compare_stations_filter - id: outh5parm - id: inspection - id: msout + - id: check_Ateam_separation.json + - id: prep_flags_join_out - id: logfiles run: ./prefactor_target/prep.cwl label: prep @@ -381,6 +426,21 @@ steps: label: findRefAnt_join 'sbg:x': 600 'sbg:y': 0 + - id: Ateam_flags_join + in: + - id: flagged_fraction_dict + source: + - findRefAnt/flagged_fraction_dict + - id: filter_station + default: '' + - id: state + default: 'Ateam' + out: + - id: flagged_fraction_antenna + run: ./../steps/findRefAnt_join.cwl + label: Ateam_flags_join + 'sbg:x': 600 + 'sbg:y': 300 - id: concat_logfiles_RefAnt in: - id: file_list @@ -425,18 +485,18 @@ steps: source: min_unflagged_fraction - id: refant source: findRefAnt_join/refant - - id: overhead - source: overhead - id: min_length source: min_length + - id: overhead + source: overhead out: - id: msout - id: outh5parm - id: bad_antennas - - id: total_bandwidth - id: outh5parm_logfile - id: inspection - id: logfiles + - id: total_bandwidth run: ./prefactor_target/gsmcal.cwl label: gsmcal 'sbg:x': 900 @@ -464,6 +524,24 @@ steps: source: skymodel_source - id: total_bandwidth source: gsmcal/total_bandwidth + - id: check_Ateam_separation.json + source: prep/check_Ateam_separation.json + - id: filter_baselines + source: filter_baselines + - id: compare_stations_filter + source: prep/compare_stations_filter + - id: flags + linkMerge: merge_flattened + source: + - initial_flags_join/flagged_fraction_antenna + - prep/prep_flags_join_out + - Ateam_flags_join/flagged_fraction_antenna + - id: clip_sources + source: clip_sources + - id: demix + source: demix + - id: demix_sources + source: demix_sources - id: ncores source: ncores out: @@ -471,6 +549,7 @@ steps: - id: solutions - id: logfiles - id: inspection + - id: summary_file run: ./prefactor_target/finalize.cwl label: finalize 'sbg:x': 1200 diff --git a/workflows/prefactor_target/finalize.cwl b/workflows/prefactor_target/finalize.cwl index d03724b7f17afbefa7bcda11378a8b27db165270..08bd48c65cd80e80e9bcae32151f0b7d4f7be97a 100644 --- a/workflows/prefactor_target/finalize.cwl +++ b/workflows/prefactor_target/finalize.cwl @@ -5,6 +5,18 @@ label: finalize $namespaces: sbg: 'https://www.sevenbridges.com/' inputs: + - id: demix_sources + type: 'string[]' + 'sbg:x': -1000 + 'sbg:y': -700 + - id: demix + type: boolean + 'sbg:x': -1000 + 'sbg:y': -600 + - id: clip_sources + type: 'string[]' + 'sbg:x': -1000 + 'sbg:y': -500 - id: msin type: 'Directory[]' 'sbg:x': -1000 @@ -45,11 +57,27 @@ inputs: type: int 'sbg:x': -1000 'sbg:y': 500 + - id: check_Ateam_separation.json + type: File + 'sbg:x': -1000 + 'sbg:y': 600 + - id: filter_baselines + type: string + 'sbg:x': -1000 + 'sbg:y': 700 + - id: compare_stations_filter + type: string + 'sbg:x': -1000 + 'sbg:y': 800 + - id: flags + type: File[] + 'sbg:x': -1000 + 'sbg:y': 900 - id: ncores type: int? default: 1 'sbg:x': -1000 - 'sbg:y': 600 + 'sbg:y': 1000 outputs: - id: msout outputSource: @@ -69,6 +97,8 @@ outputs: - concat_logfiles_solutions/output - concat_logfiles_structure/output - concat_logfiles_wsclean/output + - concat_logfiles_summary/output + - concat_logfiles_uvplot/output type: 'File[]' linkMerge: merge_flattened 'sbg:x': 1300 @@ -77,10 +107,17 @@ outputs: outputSource: - structure_function/structure_plot - wsclean/image + - uvplot/output_image type: 'File[]?' linkMerge: merge_flattened 'sbg:x': 1300 'sbg:y': 500 + - id: summary_file + outputSource: + - summary/summary_file + type: File + 'sbg:x': 1300 + 'sbg:y': 1000 steps: - id: add_missing_stations in: @@ -148,6 +185,33 @@ steps: scatterMethod: dotproduct 'sbg:x': 400 'sbg:y': 0 + - id: final_flags + in: + - id: msin + source: apply_gsmcal/msout + out: + - id: flagged_fraction_dict + run: ./../../steps/findRefAnt.cwl + label: final_flags + scatter: + - msin + 'sbg:x': 400 + 'sbg:y': -400 + - id: final_flags_join + in: + - id: flagged_fraction_dict + source: + - final_flags/flagged_fraction_dict + - id: filter_station + default: '' + - id: state + default: 'final' + out: + - id: flagged_fraction_antenna + run: ./../../steps/findRefAnt_join.cwl + label: final_flags_join + 'sbg:x': 600 + 'sbg:y': -400 - id: average in: - id: msin @@ -180,6 +244,64 @@ steps: scatterMethod: dotproduct 'sbg:x': 600 'sbg:y': -200 + - id: summary + in: + - id: flagFiles + source: + - flags + - final_flags_join/flagged_fraction_antenna + linkMerge: merge_flattened + - id: pipeline + default: 'prefactor' + - id: run_type + default: 'target' + - id: filter + source: filter_baselines + - id: bad_antennas + source: + - bad_antennas + - compare_stations_filter + valueFrom: $(self.join('')) + - id: structure_file + source: structure_function/structure_txt + - id: Ateam_separation_file + source: check_Ateam_separation.json + - id: solutions + source: h5parm_pointingname/outh5parm + - id: clip_sources + source: clip_sources + valueFrom: "$(self.join(','))" + - id: demix + source: demix + - id: demix_sources + source: demix_sources + valueFrom: "$(self.join(','))" + out: + - id: summary_file + - id: logfile + run: ../../steps/summary.cwl + label: summary + 'sbg:x': 800 + 'sbg:y': 0 + - id: uvplot + in: + - id: MSfiles + source: apply_gsmcal/msout + - id: output_name + source: get_targetname/targetname + valueFrom: $(self)_uv-coverage.png + - id: title + source: get_targetname/targetname + valueFrom: '"uv coverage of the target pointing: $(self)"' + - id: wideband + default: true + out: + - id: output_image + - id: logfile + run: ../../steps/uvplot.cwl + label: uvplot + 'sbg:x': 800 + 'sbg:y': 300 - id: wsclean in: - id: msin @@ -359,6 +481,35 @@ steps: label: concat_logfiles_wsclean 'sbg:x': 1100 'sbg:y': 500 + - id: concat_logfiles_summary + in: + - id: file_list + linkMerge: merge_flattened + source: + - summary/logfile + - id: file_prefix + source: get_targetname/targetname + valueFrom: $(self+'_summary') + out: + - id: output + run: ../../steps/concatenate_files.cwl + label: concat_logfiles_summary + 'sbg:x': 1050 + 'sbg:y': 0 + - id: concat_logfiles_uvplot + in: + - id: file_list + linkMerge: merge_flattened + source: + - uvplot/logfile + - id: file_prefix + default: uvplot + out: + - id: output + run: ../../steps/concatenate_files.cwl + label: concat_logfiles_uvplot + 'sbg:x': 1050 + 'sbg:y': 300 - id: get_targetname in: - id: msin diff --git a/workflows/prefactor_target/prep.cwl b/workflows/prefactor_target/prep.cwl index 73a796aa0df5f9d5f3891378f40e5eccbec8ae45..4f46e388f17d7e7b6cfbcb620f5a1777a5fb3a0d 100644 --- a/workflows/prefactor_target/prep.cwl +++ b/workflows/prefactor_target/prep.cwl @@ -38,9 +38,13 @@ inputs: default: false 'sbg:x': -1000 'sbg:y': -1100 - - id: propagatesolutions - type: boolean? - default: true + - id: clip_sources + type: 'string[]?' + default: + - VirA_4_patch + - CygAGG + - CasA_4_patch + - TauAGG 'sbg:x': -1000 'sbg:y': -1000 - id: demix_sources @@ -180,6 +184,12 @@ inputs: 'sbg:x': -1000 'sbg:y': 1700 outputs: + - id: compare_stations_filter + outputSource: + - compare_station_list/filter_out + type: string + 'sbg:x': 1000 + 'sbg:y': -800 - id: outh5parm outputSource: - createRMh5parm/h5parmout @@ -201,6 +211,12 @@ outputs: type: 'Directory[]' 'sbg:x': 1000 'sbg:y': 0 + - id: check_Ateam_separation.json + outputSource: + - check_ateam_separation/output_json + type: File + 'sbg:x': 1000 + 'sbg:y': 500 - id: logfiles outputSource: - make_sourcedb_ateam/log @@ -214,7 +230,28 @@ outputs: linkMerge: merge_flattened 'sbg:x': 1000 'sbg:y': 800 + - id: prep_flags_join_out + outputSource: + - prep_flags_join/flagged_fraction_antenna + type: File? + 'sbg:x': 1000 + 'sbg:y': 1100 steps: + - id: prep_flags_join + in: + - id: flagged_fraction_dict + source: + - dppp_prep_target/prep_flags_out + - id: filter_station + default: '' + - id: state + default: 'prep' + out: + - id: flagged_fraction_antenna + run: ./../../steps/findRefAnt_join.cwl + label: prep_flags_join + 'sbg:x': 400 + 'sbg:y': -200 - id: check_ateam_separation in: - id: ms @@ -224,6 +261,7 @@ steps: source: min_separation out: - id: output_imag + - id: output_json - id: logfile run: ../../steps/check_ateam_separation.cwl label: check_Ateam_separation @@ -381,12 +419,15 @@ steps: source: compare_station_list/filter_out - id: updateweights source: updateweights + - id: clip_sources + source: clip_sources out: - id: msout - id: prep_logfile - id: predict_logfile - id: clipper_logfile - id: clipper_output + - id: prep_flags_out - id: parset run: ../../subworkflow/ndppp_prep_targ.cwl label: ndppp_prep_target