From 9fec4cf021a75cf16cc024f4ce2fd515c5698d96 Mon Sep 17 00:00:00 2001
From: alex <alex@tls-tautenburg.de>
Date: Tue, 20 Jul 2021 14:28:03 +0000
Subject: [PATCH] Resolve RAP-165
Former-commit-id: c266d813169ad8b015f14a5c05a935a86b61a576 [formerly 4240981a6c1ed139ce9ba462ef29348b2556f156]
Former-commit-id: 95fc5dfeed3007ef5004eda40612d22492662603
Former-commit-id: e8db6557fe542a669c875f774f96de46bf1346e0
---
.gitlab-ci.yml | 59 +++++++++++--------
lofar-cwl/steps/Ateamclipper.cwl | 3 +-
steps/add_missing_stations.cwl | 3 +-
steps/aoflag.cwl | 1 -
steps/blsmooth.cwl | 3 +-
steps/check_ateam_separation.cwl | 3 +-
steps/createRMh5parm.cwl | 3 +-
steps/directory_listing.cwl | 4 +-
steps/h5parm_pointingname.cwl | 3 +-
steps/plot_Ateamclipper.cwl | 3 +-
steps/structure_function.cwl | 3 +-
steps/transfer_solutions.cwl | 3 +-
test_jobs/HBA_calibrator.json | 9 +++
test_jobs/blsmooth.json | 2 +-
test_jobs/check_ateam_separation.json | 6 ++
test_jobs/find_skymodel_cal.json | 4 +-
...integration_test_prefactor_calibrator.json | 14 -----
test_jobs/pol_align.json | 2 +-
18 files changed, 63 insertions(+), 65 deletions(-)
create mode 100644 test_jobs/HBA_calibrator.json
create mode 100644 test_jobs/check_ateam_separation.json
delete mode 100644 test_jobs/integration_test_prefactor_calibrator.json
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 5a7f55a4..956d19c6 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,62 +1,69 @@
-image: lofareosc/prefactor-ci:latest
+image: lofareosc/prefactor3-cwl:latest
variables:
TEST_DATASET_NAME: "L570745_uv_first10.MS.tar.xz"
PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
stages:
- - build_docker
- - push_docker
+ - download_data
+ - test_prefactor
- validate_cwl_scripts
- test_steps
- integration_test
before_script:
- - mkdir workdir
- - mkdir logs
+ - mkdir workdir
+ - mkdir logs
after_script:
- echo "All done"
-build_docker:
- image: docker:latest
- stage: build_docker
- only:
- - tags
+download_data:
+ stage: download_data
script:
- - apk add git subversion git-lfs bash
- - cd Docker
- - bash build_docker.sh
+ - mkdir data && cd data
+ - wget -q https://git.astron.nl/eosc/prefactor3-cwl/-/raw/master/test_data/$TEST_DATASET_NAME -O $TEST_DATASET_NAME && tar -xf $TEST_DATASET_NAME && rm -f $TEST_DATASET_NAME
+ - wget -q https://git.astron.nl/eosc/prefactor3-cwl/-/raw/master/test_data/example.h5 -O example.h5
+ - wget -q https://git.astron.nl/eosc/prefactor3-cwl/-/raw/master/test_data/A-Team_lowres.sourcedb -O A-Team_lowres.sourcedb
+ artifacts:
+ paths:
+ - data
-push_docker_tags:
- image: docker:latest
- allow_failure: true
- stage: push_docker
- only:
- - tags
+test_prefactor:
+ stage: test_prefactor
+ needs: ["download_data"]
script:
- - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
- - docker tag lofareosc/prefactor-ci:latest $CI_REGISTRY/prefactor-ci:$CI_COMMIT_REF_NAME
- - docker push $CI_REGISTRY/prefactor-ci:$CI_COMMIT_REF_NAME
+ - cd data
+ - ls -l
validate_scripts:
stage: validate_cwl_scripts
script:
- for file in `find . -name 'steps/*.cwl'`; do cwltool --validate $file; done
+blsmooth:
+ stage: test_steps
+ allow_failure: true
+ needs: ["download_data"]
+ script:
+ - cwltool --no-container --preserve-environment LD_LIBRARY_PATH --preserve-environment PATH --preserve-environment PYTHONPATH steps/blsmooth.cwl test_jobs/blsmooth.json
+
find_skymodel_cal:
stage: test_steps
allow_failure: true
+ needs: ["download_data"]
script:
- cwltool --no-container --preserve-environment LD_LIBRARY_PATH --preserve-environment PATH --preserve-environment PYTHONPATH steps/find_skymodel_cal.cwl test_jobs/find_skymodel_cal.json
-blsmooth:
+check_ateam_separation:
stage: test_steps
allow_failure: true
+ needs: ["download_data"]
script:
- - cwltool --no-container --preserve-environment LD_LIBRARY_PATH --preserve-environment PATH --preserve-environment PYTHONPATH steps/blsmooth.cwl test_jobs/blsmooth.json
+ - cwltool --no-container --preserve-environment LD_LIBRARY_PATH --preserve-environment PATH --preserve-environment PYTHONPATH steps/check_ateam_separation.cwl test_jobs/check_ateam_separation.json
-pol_align:
+prefactor_calibrator:
stage: integration_test
+ needs: ["download_data"]
script:
- - cwltool --no-container subworkflow/pol_align.cwl test_jobs/pol_align.json
+ - cwltool --no-container workflows/HBA_calibrator.cwl test_jobs/HBA_calibrator.json
diff --git a/lofar-cwl/steps/Ateamclipper.cwl b/lofar-cwl/steps/Ateamclipper.cwl
index e74db690..0badc835 100755
--- a/lofar-cwl/steps/Ateamclipper.cwl
+++ b/lofar-cwl/steps/Ateamclipper.cwl
@@ -4,8 +4,7 @@ $namespaces:
sbg: 'https://www.sevenbridges.com/'
id: check_ateam_separation
baseCommand:
- - python3
- - /usr/local/bin/Ateamclipper.py
+ - Ateamclipper.py
inputs:
- id: msin
type:
diff --git a/steps/add_missing_stations.cwl b/steps/add_missing_stations.cwl
index 18627726..5370829b 100644
--- a/steps/add_missing_stations.cwl
+++ b/steps/add_missing_stations.cwl
@@ -5,8 +5,7 @@ $namespaces:
sbg: 'https://www.sevenbridges.com/'
id: add_missing_stations
baseCommand:
- - python3
- - /usr/local/bin/add_missing_stations.py
+ - add_missing_stations.py
inputs:
- default: output.h5
id: h5parm
diff --git a/steps/aoflag.cwl b/steps/aoflag.cwl
index d93d2bd6..4f6c03a5 100644
--- a/steps/aoflag.cwl
+++ b/steps/aoflag.cwl
@@ -140,7 +140,6 @@ requirements:
mkdir -pv workdir
cp -r \$directories workdir/. && cd workdir
concat=out.MS_[0-9*]
- echo \$1 \$2 \$3 \$4 \$concat
aoflagger \$1 \$2 \$3 \$4 \$concat
stdout: aoflag.log
diff --git a/steps/blsmooth.cwl b/steps/blsmooth.cwl
index 0bda55d6..6f4efd6b 100644
--- a/steps/blsmooth.cwl
+++ b/steps/blsmooth.cwl
@@ -3,8 +3,7 @@ cwlVersion: v1.0
id: blsmooth
label: BLsmooth
baseCommand:
- - python3
- - /usr/local/bin/BLsmooth.py
+ - BLsmooth.py
inputs:
- id: msin
type: Directory
diff --git a/steps/check_ateam_separation.cwl b/steps/check_ateam_separation.cwl
index 699cbdab..a6da2abc 100644
--- a/steps/check_ateam_separation.cwl
+++ b/steps/check_ateam_separation.cwl
@@ -2,8 +2,7 @@ class: CommandLineTool
cwlVersion: v1.0
id: check_ateam_separation
baseCommand:
- - python3
- - /usr/local/bin/check_Ateam_separation.py
+ - check_Ateam_separation.py
inputs:
- id: ms
type:
diff --git a/steps/createRMh5parm.cwl b/steps/createRMh5parm.cwl
index d73f6281..06c98593 100644
--- a/steps/createRMh5parm.cwl
+++ b/steps/createRMh5parm.cwl
@@ -3,8 +3,7 @@ cwlVersion: v1.0
id: createRMh5parm
label: createRMh5parm
baseCommand:
- - python3
- - /usr/local/bin/createRMh5parm.py
+ - createRMh5parm.py
inputs:
- id: msin
type:
diff --git a/steps/directory_listing.cwl b/steps/directory_listing.cwl
index b4267867..73301837 100644
--- a/steps/directory_listing.cwl
+++ b/steps/directory_listing.cwl
@@ -47,7 +47,7 @@ arguments:
valueFrom: d
hints:
- class: DockerRequirement
- dockerPull: 'lofareosc/prefactor-ci:master'
+ dockerPull: lofareosc/prefactor3-cwl
+stdout: out_file
requirements:
- class: InlineJavascriptRequirement
-stdout: out_file
diff --git a/steps/h5parm_pointingname.cwl b/steps/h5parm_pointingname.cwl
index 81e2fedd..e1a99a38 100644
--- a/steps/h5parm_pointingname.cwl
+++ b/steps/h5parm_pointingname.cwl
@@ -5,8 +5,7 @@ $namespaces:
sbg: 'https://www.sevenbridges.com/'
id: h5parm_pointingname
baseCommand:
- - python3
- - /usr/local/bin/h5parm_pointingname.py
+ - h5parm_pointingname.py
inputs:
- format: 'lofar:#H5Parm'
id: h5parmFile
diff --git a/steps/plot_Ateamclipper.cwl b/steps/plot_Ateamclipper.cwl
index 9b019713..c9e19821 100644
--- a/steps/plot_Ateamclipper.cwl
+++ b/steps/plot_Ateamclipper.cwl
@@ -4,8 +4,7 @@ $namespaces:
sbg: 'https://www.sevenbridges.com/'
id: plot_Ateamclipper
baseCommand:
- - python3
- - /usr/local/bin/plot_Ateamclipper.py
+ - plot_Ateamclipper.py
inputs:
- id: clipper_output
type: File
diff --git a/steps/structure_function.cwl b/steps/structure_function.cwl
index 3ae8620f..bc138367 100644
--- a/steps/structure_function.cwl
+++ b/steps/structure_function.cwl
@@ -5,8 +5,7 @@ $namespaces:
sbg: 'https://www.sevenbridges.com/'
id: structure_function
baseCommand:
- - python3
- - /usr/local/bin/getStructure_from_phases.py
+ - getStructure_from_phases.py
inputs:
- format: 'lofar:#H5Parm'
id: h5parmFile
diff --git a/steps/transfer_solutions.cwl b/steps/transfer_solutions.cwl
index 7b5298d0..d58d3145 100644
--- a/steps/transfer_solutions.cwl
+++ b/steps/transfer_solutions.cwl
@@ -4,8 +4,7 @@ $namespaces:
sbg: 'https://www.sevenbridges.com/'
id: transfer_solutions
baseCommand:
- - python3
- - /usr/local/bin/transfer_solutions.py
+ - transfer_solutions.py
inputs:
- default: output.h5
id: h5parm
diff --git a/test_jobs/HBA_calibrator.json b/test_jobs/HBA_calibrator.json
new file mode 100644
index 00000000..9e44c2dc
--- /dev/null
+++ b/test_jobs/HBA_calibrator.json
@@ -0,0 +1,9 @@
+{
+ "msin" : [
+ {"class": "Directory", "path": "../data/L570745_SB000_uv_first10.MS"},
+ {"class": "Directory", "path": "../data/L570745_SB001_uv_first10.MS"},
+ {"class": "Directory", "path": "../data/L570745_SB002_uv_first10.MS"}
+ ],
+ "raw_data": false,
+ "demix": false,
+}
diff --git a/test_jobs/blsmooth.json b/test_jobs/blsmooth.json
index 5b300249..a7fbbce8 100644
--- a/test_jobs/blsmooth.json
+++ b/test_jobs/blsmooth.json
@@ -1,7 +1,7 @@
{
"msin": {
"class": "Directory",
- "path": "/data/L570745_SB003_uv_first10.MS"
+ "path": "../data/L570745_SB003_uv_first10.MS"
},
"do_smooth": true
}
diff --git a/test_jobs/check_ateam_separation.json b/test_jobs/check_ateam_separation.json
new file mode 100644
index 00000000..37f4990c
--- /dev/null
+++ b/test_jobs/check_ateam_separation.json
@@ -0,0 +1,6 @@
+{
+ "ms": {
+ "class": "Directory",
+ "path": "../data/L570745_SB001_uv_first10.MS"
+ }
+}
diff --git a/test_jobs/find_skymodel_cal.json b/test_jobs/find_skymodel_cal.json
index 1a006724..b22f5079 100644
--- a/test_jobs/find_skymodel_cal.json
+++ b/test_jobs/find_skymodel_cal.json
@@ -1,10 +1,10 @@
{
"msin": {
"class": "Directory",
- "path": "/data/L570745_SB000_uv_first10.MS"
+ "path": "../data/L570745_SB000_uv_first10.MS"
},
"skymodels": {
"class": "Directory",
- "path": "/data/skymodels"
+ "path": "/usr/local/share/prefactor/skymodels"
}
}
diff --git a/test_jobs/integration_test_prefactor_calibrator.json b/test_jobs/integration_test_prefactor_calibrator.json
deleted file mode 100644
index 672f9798..00000000
--- a/test_jobs/integration_test_prefactor_calibrator.json
+++ /dev/null
@@ -1,14 +0,0 @@
-{
- "msin" : [
- {"class": "Directory", "path": "/data/L570745_SB000_uv_first10.MS"},
- {"class": "Directory", "path": "/data/L570745_SB001_uv_first10.MS"},
- {"class": "Directory", "path": "/data/L570745_SB002_uv_first10.MS"}
- ],
- "raw_data": false,
- "demix": false,
- "skymodels": {
- "class": "Directory",
- "path": "/data/skymodels"
- },
- "A-Team_sky_model": "/data/skymodels/Ateam_LBA_CC.skymodel"
-}
diff --git a/test_jobs/pol_align.json b/test_jobs/pol_align.json
index 0cdbed89..aad9492c 100644
--- a/test_jobs/pol_align.json
+++ b/test_jobs/pol_align.json
@@ -2,6 +2,6 @@
"input_h5parm": {
"class": "File",
"format": "lofar:#H5Parm",
- "path": "/data/example.h5"
+ "path": "../data/example.h5"
}
}
--
GitLab