Attempt to fix issue with too many directories

Former-commit-id: 221905e6 [formerly 0667dfd4]
Former-commit-id: 058ccdd8
Former-commit-id: 75573ba5

lofar-cwl/steps/check_ateam_separation.cwl

 class: CommandLineTool
 cwlVersion: v1.0
-$namespaces:
-  sbg: 'https://www.sevenbridges.com/'
 id: check_ateam_separation
 baseCommand:
   - python3

steps/copy_in_one_directory.cwl

+id: copy_in_one_directory
+label: copy_in_one_directory
+class: ExpressionTool
+
+cwlVersion: v1.0
+inputs: 
+    - id: input
+      type: Directory[]
+outputs: 
+    - id: output
+      type: Directory
+      glob: .
+
+requirements:
+    - class: InitialWorkDirRequirement
+      listing:
+        - $(inputs.input)
+      
+
+expression: |
+  ${
+    
+  }
+
+
+requirements:
+  - class: InlineJavascriptRequirement

workflows/HBA_calibrator.cwl

@@ -344,9 +344,6 @@ steps:
     label: save_results
     'sbg:x': 1200
     'sbg:y': 0
-hints:
-    cwltool:LoadListingRequirement:
-        loadListing: shallow_listing
 requirements:
   - class: SubworkflowFeatureRequirement
   - class: MultipleInputFeatureRequirement

workflows/dowload_surl_and_create_input_file-json.cwl

@@ -2,26 +2,19 @@ class: Workflow
 cwlVersion: v1.0
 id: dowload_surl_and_create_input_file_json
 label: dowload_surl_and_create_input_file.json
-$namespaces:
-  sbg: 'https://www.sevenbridges.com/'
+
 inputs:
   - id: surl_list
     type: File
-    'sbg:x': -329.7890625
-    'sbg:y': -35
 outputs:
   - id: results
     outputSource:
       - _h_b_a_calibrator/results
     type: Directory
-    'sbg:x': 739.1641235351562
-    'sbg:y': -95.5
   - id: logs
     outputSource:
       - _h_b_a_calibrator/logs
     type: Directory
-    'sbg:x': 725.1641235351562
-    'sbg:y': 153.5
 steps:
   - id: read_surl_list
     in:
@@ -31,8 +24,6 @@ steps:
       - id: surls
     run: ../steps/read_surl_list.cwl
     label: ReadSurlList
-    'sbg:x': -158
-    'sbg:y': -40
   - id: surl_copy
     in:
       - id: surl
@@ -43,8 +34,6 @@ steps:
     label: surl_copy
     scatter:
       - surl
-    'sbg:x': 145
-    'sbg:y': -19
   - id: untar
     in:
       - id: tar_file
@@ -55,8 +44,6 @@ steps:
     label: untar
     scatter:
       - tar_file
-    'sbg:x': 350
-    'sbg:y': 8
   - id: stage
     in:
       - id: input
@@ -67,8 +54,6 @@ steps:
     label: stage
     scatter:
       - input
-    'sbg:x': -8
-    'sbg:y': -29
   - id: _h_b_a_calibrator
     in:
       - id: msin
@@ -79,8 +64,6 @@ steps:
       - id: results
     run: ./HBA_calibrator.cwl
     label: HBA_calibrator
-    'sbg:x': 563
-    'sbg:y': 17
 requirements:
   - class: SubworkflowFeatureRequirement
   - class: ScatterFeatureRequirement

workflows/prefactor_calibrator/prep.cwl

@@ -7,147 +7,146 @@ $namespaces:
 inputs:
   - id: msin
     type: 'Directory[]'
-    'sbg:x': -1000
-    'sbg:y': -900
+    'sbg:x': -515.1839599609375
+    'sbg:y': 330.674560546875
   - id: filter_baselines
     type: string
-    'sbg:x': -1000
-    'sbg:y': -800
+    'sbg:x': 254.234375
+    'sbg:y': 684
   - id: raw_data
     type: boolean
-    'sbg:x': -1000
-    'sbg:y': -700
+    'sbg:x': 0
+    'sbg:y': 563
   - id: propagatesolutions
     type: boolean
-    'sbg:x': -1000
-    'sbg:y': -600
+    'sbg:x': 0
+    'sbg:y': 670
   - id: flagunconverged
     type: boolean
-    'sbg:x': -1000
-    'sbg:y': -500
+    'sbg:x': 559.319091796875
+    'sbg:y': 837.5
   - id: demix
     type: boolean
-    'sbg:x': -1000
-    'sbg:y': -400
+    'sbg:x': 254.234375
+    'sbg:y': 1326
   - id: max_dppp_threads
     type: int?
     default: 10
-    'sbg:x': -1000
-    'sbg:y': -300
+    'sbg:x': 254.234375
+    'sbg:y': 214
   - id: memoryperc
     type: int
-    'sbg:x': -1000
-    'sbg:y': -200
+    'sbg:x': 254.234375
+    'sbg:y': 107
   - id: flag_baselines
     type: 'string[]'
-    'sbg:x': -1000
-    'sbg:y': -100
+    'sbg:x': 254.234375
+    'sbg:y': 442
   - id: avg_timeresolution
     type: float
-    'sbg:x': -1000
-    'sbg:y': 0
+    'sbg:x': 254.234375
+    'sbg:y': 1554
   - id: avg_freqresolution
     type: string
-    'sbg:x': -1000
-    'sbg:y': 100
+    'sbg:x': 254.234375
+    'sbg:y': 1661
   - id: process_baselines_cal
     type: string
-    'sbg:x': -1000
-    'sbg:y': 200
+    'sbg:x': 0
+    'sbg:y': 777
   - id: demix_timestep
     type: int
-    'sbg:x': -1000
-    'sbg:y': 300
+    'sbg:x': 254.234375
+    'sbg:y': 898
   - id: demix_freqstep
     type: int
-    'sbg:x': -1000
-    'sbg:y': 400
+    'sbg:x': 254.234375
+    'sbg:y': 1219
   - id: demix_target
     type: string
-    'sbg:x': -1000
-    'sbg:y': 500
+    'sbg:x': 254.234375
+    'sbg:y': 1005
   - id: demix_sources
     type: 'string[]'
-    'sbg:x': -1000
-    'sbg:y': 600
+    'sbg:x': 254.234375
+    'sbg:y': 1112
   - id: min_length
     type: int?
-    'sbg:x': -1000
-    'sbg:y': 700
+    'sbg:x': 360
+    'sbg:y': 1919.42626953125
   - id: overhead
     type: float?
-    'sbg:x': -1000
-    'sbg:y': 800
+    'sbg:x': 180
+    'sbg:y': 1903.963134765625
   - id: min_separation
     type: int?
-    'sbg:x': -1000
-    'sbg:y': 900
+    'sbg:x': 0
+    'sbg:y': 884
   - id: do_smooth
     type: boolean?
     default: false
-    'sbg:x': -1000
-    'sbg:y': 1000
+    'sbg:x': 559.319091796875
+    'sbg:y': 944.5
   - id: max_separation_arcmin
     type: float?
-    'sbg:x': -1000
-    'sbg:y': 1100
+    'sbg:x': 0
+    'sbg:y': 991
   - id: calibrator_path_skymodel
     type: Directory?
-    'sbg:x': -1000
-    'sbg:y': 1200
+    'sbg:x': 0
+    'sbg:y': 1098
   - id: A-Team_skymodel
     type: File?
-    'sbg:x': -1000
-    'sbg:y': 1300
+    'sbg:x': 0
+    'sbg:y': 1205
   - id: elevation
     type: string
     default: 0deg..20deg
-    'sbg:x': -1000
-    'sbg:y': 1400
+    'sbg:x': 254.234375
+    'sbg:y': 791
   - id: amplmin
     type: float
     default: 1.e-30
-    'sbg:x': -1000
-    'sbg:y': 1500
+    'sbg:x': 254.234375
+    'sbg:y': 1768
 outputs:
   - id: outh5parm
     outputSource:
       - h5parm_collector/outh5parm
     type: File
-    'sbg:x': 1000
-    'sbg:y': -500
+    'sbg:x': 1554.5106201171875
+    'sbg:y': 884
   - id: check_Ateam_separation.png
     outputSource:
       - check_ateam_separation/output_imag
     type: File
-    'sbg:x': 1000
-    'sbg:y': -300
+    'sbg:x': 559.319091796875
+    'sbg:y': 1051.5
   - id: msout
     outputSource:
       - predict_calibrate/msout
     type: 'Directory[]'
-    'sbg:x': 1000
-    'sbg:y': 0
+    'sbg:x': 1329.937255859375
+    'sbg:y': 663
   - id: parset
     outputSource:
       - ndppp_prep_cal/parset
     type: 'File[]'
-    'sbg:x': 1000
-    'sbg:y': 200
+    'sbg:x': 1027.490966796875
+    'sbg:y': 884
   - id: calibrator_name
     outputSource:
       - find_skymodel_cal/model_name
     type: string
-    'sbg:x': 1000
-    'sbg:y': 400
+    'sbg:x': 559.319091796875
+    'sbg:y': 1158.5
   - id: outh5parm_logfile
     outputSource:
       - h5parm_collector/log
-    type: File[]
-    'sbg:x': 1000
-    'sbg:y': 650
+    type: 'File[]'
+    'sbg:x': 1554.5106201171875
+    'sbg:y': 777
   - id: logfiles
-    linkMerge: merge_flattened
     outputSource:
       - concat_logfiles_calib/output
       - concat_logfiles_predict/output
@@ -158,18 +157,23 @@ outputs:
       - make_sourcedb_ateam/log
       - check_ateam_separation/logfile
     type: 'File[]'
-    'sbg:x': 1000
-    'sbg:y': 800
+    linkMerge: merge_flattened
+    'sbg:x': 1554.5106201171875
+    'sbg:y': 991
 steps:
-  - id: ndppp_prep_cal
+  - id: copy
     in:
-      - id: msin
-        linkMerge: merge_flattened
+      - id: input
         source:
           - msin
-      - id: msin_baseline
-        default: '*'
-        source: filter_baselines
+    out:
+      - id: output
+    run: ../../steps/copy_in_one_directory.cwl
+    label: copy_in_one_directory
+    'sbg:x': -267.1643981933594
+    'sbg:y': 480.32098388671875
+  - id: ndppp_prep_cal
+    in:
       - id: baselines_to_flag
         default: []
         source:
@@ -187,6 +191,13 @@ steps:
       - id: demix
         default: false
         source: demix
+      - id: msin
+        linkMerge: merge_flattened
+        source:
+          - msin
+      - id: msin_baseline
+        default: '*'
+        source: filter_baselines
       - id: skymodel
         source: make_sourcedb_ateam/sourcedb
       - id: timeresolution
@@ -220,8 +231,8 @@ steps:
     label: ndppp_prep_cal
     scatter:
       - msin
-    'sbg:x': 0
-    'sbg:y': 0
+    'sbg:x': 559.319091796875
+    'sbg:y': 497.5
   - id: predict_calibrate
     in:
       - id: msin
@@ -246,13 +257,13 @@ steps:
     label: predict_calibrate
     scatter:
       - msin
-    'sbg:x': 500
-    'sbg:y': 0
+    'sbg:x': 1027.490966796875
+    'sbg:y': 749
   - id: check_ateam_separation
     in:
       - id: ms
         source:
-          - msin
+          - copy/output
       - id: min_separation
         source: min_separation
     out:
@@ -260,8 +271,8 @@ steps:
       - id: logfile
     run: ../../lofar-cwl/steps/check_ateam_separation.cwl
     label: check_Ateam_separation
-    'sbg:x': 0
-    'sbg:y': -300
+    'sbg:x': 254.234375
+    'sbg:y': 1440
   - id: make_sourcedb_ateam
     in:
       - id: sky_model
@@ -273,13 +284,13 @@ steps:
       - id: log
     run: ../../lofar-cwl/steps/makesourcedb.cwl
     label: make_sourcedb_ateam
-    'sbg:x': -500
-    'sbg:y': 300
+    'sbg:x': 254.234375
+    'sbg:y': 328
   - id: find_skymodel_cal
     in:
       - id: msin
         source:
-          - msin
+          - copy/output
       - id: skymodels
         source: calibrator_path_skymodel
       - id: max_separation_arcmin
@@ -290,8 +301,8 @@ steps:
       - id: logfile
     run: ../../steps/find_skymodel_cal.cwl
     label: find_skymodel_cal
-    'sbg:x': -500
-    'sbg:y': 500
+    'sbg:x': 254.234375
+    'sbg:y': 563
   - id: make_sourcedb
     in:
       - id: sky_model
@@ -303,21 +314,22 @@ steps:
       - id: log
     run: ../../lofar-cwl/steps/makesourcedb.cwl
     label: make_sourcedb
-    'sbg:x': 0
-    'sbg:y': 500
+    'sbg:x': 559.319091796875
+    'sbg:y': 723.5
   - id: concat_logfiles_prep_cal
     in:
       - id: file_list
         linkMerge: merge_flattened
-        source: ndppp_prep_cal/logfile
+        source:
+          - ndppp_prep_cal/logfile
       - id: file_prefix
         default: ndppp_prep_cal
     out:
       - id: output
     run: ../../steps/concatenate_files.cwl
     label: concat_logfiles_prep_cal
-    'sbg:x': 250
-    'sbg:y': 300
+    'sbg:x': 1027.490966796875
+    'sbg:y': 991
   - id: concat_logfiles_blsmooth
     in:
       - id: file_list
@@ -329,8 +341,8 @@ steps:
       - id: output
     run: ../../steps/concatenate_files.cwl
     label: concat_logfiles_blsmooth
-    'sbg:x': 750
-    'sbg:y': 300
+    'sbg:x': 1329.937255859375
+    'sbg:y': 1105
   - id: concat_logfiles_predict
     in:
       - id: file_list
@@ -343,8 +355,8 @@ steps:
       - id: output
     run: ../../steps/concatenate_files.cwl
     label: concat_logfiles_predict
-    'sbg:x': 750
-    'sbg:y': 500
+    'sbg:x': 1329.937255859375
+    'sbg:y': 891
   - id: concat_logfiles_calib
     in:
       - id: file_list
@@ -356,8 +368,8 @@ steps:
       - id: output
     run: ../../steps/concatenate_files.cwl
     label: concat_logfiles_calib
-    'sbg:x': 750
-    'sbg:y': 800
+    'sbg:x': 1329.937255859375
+    'sbg:y': 998
   - id: h5parm_collector
     in:
       - id: h5parmFiles
@@ -374,8 +386,10 @@ steps:
       - id: log
     run: ../../lofar-cwl/steps/H5ParmCollector.cwl
     label: H5parm_collector
-    'sbg:x': 750
-    'sbg:y': -300
+    'sbg:x': 1329.937255859375
+    'sbg:y': 777
 requirements:
   - class: SubworkflowFeatureRequirement
   - class: ScatterFeatureRequirement
+  - class: InlineJavascriptRequirement
+  - class: StepInputExpressionRequirement