Skip to content
Snippets Groups Projects
Commit 1cb42061 authored by David Rafferty's avatar David Rafferty
Browse files

Add initial version of calibrator workflow ci test 

Former-commit-id: 96282492
parent 9b7f9cd4
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
.gitlab-ci.yml
+ 18
− 2
View file @ 1cb42061
......@@ -2,12 +2,17 @@ image: lofareosc/prefactor3-cwl:latest
variables:
TEST_DATASET_NAME: "L570745_uv_first10.MS.tar.xz"
TEST_HBA_DATASET_NAME: "test_data_30subbands_12timeslots.tar.gz"
CALIBRATOR_HBA_RESULTS_NAME: "results_calibrator.tar.gz"
TARGET_HBA_RESULTS_NAME: "results_target.tar.gz"
PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
stages:
- download_data
- validate_cwl_scripts
- test_steps
- test_calibrator
- test_target
before_script:
- mkdir workdir
......@@ -20,7 +25,10 @@ download_data:
stage: download_data
script:
- mkdir data && cd data
- wget -q https://git.astron.nl/eosc/prefactor3-cwl/-/raw/master/test_data/$TEST_DATASET_NAME -O $TEST_DATASET_NAME && tar -xf $TEST_DATASET_NAME && rm -f $TEST_DATASET_NAME
- wget -q https://git.astron.nl/eosc/prefactor3-cwl/-/raw/master/test_data/$TEST_DATASET_NAME -O $TEST_DATASET_NAME && tar -xf $TEST_DATASET_NAME && rm -f $TEST_DATA_NAME
- wget -q www.astron.nl/citt/ci_data/prefactor/$TEST_HBA_DATASET_NAME -O $TEST_HBA_DATASET_NAME && tar -xfz $TEST_HBA_DATASET_NAME && rm -f $TEST_HBA_DATASET_NAME
- wget -q www.astron.nl/citt/ci_data/prefactor/$CALIBRATOR_HBA_RESULTS_NAME -O $CALIBRATOR_HBA_RESULTS_NAME && tar -xfz $CALIBRATOR_HBA_RESULTS_NAME && rm -f $CALIBRATOR_HBA_RESULTS_NAME
- wget -q www.astron.nl/citt/ci_data/prefactor/$TARGET_HBA_RESULTS_NAME -O $TARGET_HBA_RESULTS_NAME && tar -xfz $TARGET_HBA_RESULTS_NAME && rm -f $TARGET_HBA_RESULTS_NAME
- wget -q https://git.astron.nl/eosc/prefactor3-cwl/-/raw/master/test_data/example.h5 -O example.h5
- wget -q https://git.astron.nl/eosc/prefactor3-cwl/-/raw/master/test_data/A-Team_lowres.sourcedb -O A-Team_lowres.sourcedb
artifacts:
......@@ -51,4 +59,12 @@ check_ateam_separation:
allow_failure: true
needs: ["download_data"]
script:
- cwltool --no-container --preserve-environment LD_LIBRARY_PATH --preserve-environment PATH --preserve-environment PYTHONPATH steps/check_ateam_separation.cwl test_jobs/check_ateam_separation.json
\ No newline at end of file
- cwltool --no-container --preserve-environment LD_LIBRARY_PATH --preserve-environment PATH --preserve-environment PYTHONPATH steps/check_ateam_separation.cwl test_jobs/check_ateam_separation.json
run_hba_calibrator:
stage: test_calibrator
allow_failure: true
needs: ["download_data"]
script:
- cwltool --no-container --preserve-environment LD_LIBRARY_PATH --preserve-environment PATH --preserve-environment PYTHONPATH workflows/HBA_calibrator.cwl test_jobs/HBA_calibrator.json
- test_jobs/check_workflow_results.py results ../data/results_calibrator
test_jobs/HBA_calibrator.json
+ 130
− 7
View file @ 1cb42061
{
"msin" : [
{"class": "Directory", "path": "../data/L570745_SB000_uv_first10.MS"},
{"class": "Directory", "path": "../data/L570745_SB001_uv_first10.MS"},
{"class": "Directory", "path": "../data/L570745_SB002_uv_first10.MS"}
"msin": [
{
"class": "Directory",
"path": "../data/L667520_SB000_uv_12timeslots.MS"
},
{
"class": "Directory",
"path": "../data/L667520_SB001_uv_12timeslots.MS"
},
{
"class": "Directory",
"path": "../data/L667520_SB002_uv_12timeslots.MS"
},
{
"class": "Directory",
"path": "../data/L667520_SB003_uv_12timeslots.MS"
},
{
"class": "Directory",
"path": "../data/L667520_SB004_uv_12timeslots.MS"
},
{
"class": "Directory",
"path": "../data/L667520_SB005_uv_12timeslots.MS"
},
{
"class": "Directory",
"path": "../data/L667520_SB006_uv_12timeslots.MS"
},
{
"class": "Directory",
"path": "../data/L667520_SB007_uv_12timeslots.MS"
},
{
"class": "Directory",
"path": "../data/L667520_SB008_uv_12timeslots.MS"
},
{
"class": "Directory",
"path": "../data/L667520_SB009_uv_12timeslots.MS"
},
{
"class": "Directory",
"path": "../data/L667520_SB010_uv_12timeslots.MS"
},
{
"class": "Directory",
"path": "../data/L667520_SB011_uv_12timeslots.MS"
},
{
"class": "Directory",
"path": "../data/L667520_SB012_uv_12timeslots.MS"
},
{
"class": "Directory",
"path": "../data/L667520_SB013_uv_12timeslots.MS"
},
{
"class": "Directory",
"path": "../data/L667520_SB014_uv_12timeslots.MS"
},
{
"class": "Directory",
"path": "../data/L667520_SB015_uv_12timeslots.MS"
},
{
"class": "Directory",
"path": "../data/L667520_SB016_uv_12timeslots.MS"
},
{
"class": "Directory",
"path": "../data/L667520_SB017_uv_12timeslots.MS"
},
{
"class": "Directory",
"path": "../data/L667520_SB018_uv_12timeslots.MS"
},
{
"class": "Directory",
"path": "../data/L667520_SB019_uv_12timeslots.MS"
},
{
"class": "Directory",
"path": "../data/L667520_SB020_uv_12timeslots.MS"
},
{
"class": "Directory",
"path": "../data/L667520_SB021_uv_12timeslots.MS"
},
{
"class": "Directory",
"path": "../data/L667520_SB022_uv_12timeslots.MS"
},
{
"class": "Directory",
"path": "../data/L667520_SB023_uv_12timeslots.MS"
},
{
"class": "Directory",
"path": "../data/L667520_SB024_uv_12timeslots.MS"
},
{
"class": "Directory",
"path": "../data/L667520_SB025_uv_12timeslots.MS"
},
{
"class": "Directory",
"path": "../data/L667520_SB026_uv_12timeslots.MS"
},
{
"class": "Directory",
"path": "../data/L667520_SB027_uv_12timeslots.MS"
},
{
"class": "Directory",
"path": "../data/L667520_SB028_uv_12timeslots.MS"
},
{
"class": "Directory",
"path": "../data/L667520_SB029_uv_12timeslots.MS"
},
],
"raw_data": false,
"demix": false,
}
"A-Team_skymodel": {
"class": "File",
"path": "/usr/local/share/prefactor/skymodels/Ateam_LBA_CC.skymodel"
},
"calibrator_path_skymodel": {
"class": "Directory",
"path": "/usr/local/share/prefactor/skymodels"
}
}
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment