From 8cee244b2443c8d47bf701e6750baf406020fc11 Mon Sep 17 00:00:00 2001
From: mancini <mancini@astron.nl>
Date: Tue, 14 Jan 2020 11:00:24 +0100
Subject: [PATCH] Add LOFAR Beam
---
.gitignore | 1 +
.gitlab-ci.yml | 5 +++
Docker/Dockerfile | 16 +++++++-
Docker/build_docker.sh | 4 ++
steps/LoSoTo.Lofarbeam.cwl | 68 +++++++++++++++++++++++++++++++++
steps/utils.js | 10 ++++-
test_jobs/losoto_lofarbeam.json | 8 ++++
7 files changed, 109 insertions(+), 3 deletions(-)
create mode 100644 steps/LoSoTo.Lofarbeam.cwl
create mode 100644 test_jobs/losoto_lofarbeam.json
diff --git a/.gitignore b/.gitignore
index 8fe27bc4..0db6769a 100644
--- a/.gitignore
+++ b/.gitignore
@@ -4,6 +4,7 @@ Docker/DP3
Docker/Dysco
Docker/STMAN
Docker/idg
+Docker/LOFARBeam
*.sif
# Byte-compiled / optimized / DLL files
__pycache__/
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index ba19298f..2a2eaa59 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -148,6 +148,11 @@ losoto_reweight:
script:
- cwl-runner --no-container steps/LoSoTo.Reweight.cwl test_jobs/losoto_reweight.json
+losoto_lofarbeam:
+ stage: test_steps
+ allow_failure: true
+ script:
+ - cwl-runner --no-container --preserve-environment PYTHONPATH steps/LoSoTo.Lofarbeam.cwl test_jobs/losoto_lofarbeam.json
losoto_residual:
stage: test_steps
diff --git a/Docker/Dockerfile b/Docker/Dockerfile
index ca101ba3..7eb1e967 100644
--- a/Docker/Dockerfile
+++ b/Docker/Dockerfile
@@ -17,6 +17,7 @@ RUN export DEBIAN_FRONTEND=noninteractive && \
libboost-filesystem-dev \
libboost-system-dev \
libboost-date-time-dev \
+ libboost-numpy-dev \
libboost-signals-dev \
libboost-program-options-dev \
libboost-test-dev \
@@ -64,6 +65,14 @@ RUN cd /src/Dysco/ && \
make -j2 && \
make install -j2
+ADD ./LOFARBeam /src/LOFARBeam/src
+RUN cd /src/LOFARBeam/ && \
+ mkdir build && \
+ cd build && \
+ cmake ../src -DCMAKE_INSTALL_PREFIX=/usr/local/ -DPYTHON_EXECUTABLE=/usr/bin/python3 && \
+ make -j2 && \
+ make install -j2
+
#----------------------------------------------
FROM kernsuite/base:5 as runner
@@ -79,6 +88,7 @@ RUN export DEBIAN_FRONTEND=noninteractive && \
libarmadillo8 \
libboost-date-time1.65.1 \
libboost-filesystem1.65.1 \
+ libboost-numpy1.65.1 \
libboost-python1.65.1 \
libboost-program-options1.65.1 \
libboost-system1.65.1 \
@@ -120,8 +130,10 @@ LABEL STMAN.version.tag=${STMAN_TAG}
ARG Dysco_TAG=master
LABEL Dysco.version.tag=${Dysco_TAG}
+ARG LofarBeam_TAG=master
+LABEL LOFARBeam.version.tag=${LofarBeam_TAG}
-RUN apt update && apt install -y nodejs python3-pip git wsclean
+RUN apt-get --allow-releaseinfo-change update -y && apt install -y nodejs python3-pip git wsclean
RUN python3 -m pip install cwltool cwl-runner -e "git://github.com/darafferty/LSMTool.git@${LSMTool_TAG}#egg=LSMTool" \
"toil[cwl]"==${TOIL_VERSION} \
matplotlib
@@ -140,7 +152,7 @@ WORKDIR /home/lofaruser
RUN chmod +rx /usr/local/bin/*
ENTRYPOINT ["/home/lofaruser/.entrypoint"]
USER lofaruser
-
+ENV PYTHONPATH=/usr/local/lib/python3.6/site-packages/
ARG DP3_TAG=master
ARG IDG_TAG=master
diff --git a/Docker/build_docker.sh b/Docker/build_docker.sh
index b908b90f..83c6dd8d 100755
--- a/Docker/build_docker.sh
+++ b/Docker/build_docker.sh
@@ -22,6 +22,7 @@ TOIL_VERSION=3.20.0
STMAN_TAG=master
Dysco_TAG=v1.2
Prefactor_TAG=master
+LofarBeam_TAG=master
# FETCHES THE IDG
git_clone_or_pull https://gitlab.com/astron-idg/idg/ ${IDG_TAG} idg
@@ -35,6 +36,8 @@ git_clone_or_pull https://github.com/lofar-astron/LofarStMan ${STMAN_TAG} STMAN
# FETCHES Dysco
git_clone_or_pull https://github.com/aroffringa/dysco.git ${Dysco_TAG} Dysco
+# FETCHES LOFARBeam
+git_clone_or_pull https://github.com/lofar-astron/LOFARBeam ${LofarBeam_TAG} LOFARBeam
SCRIPT_PATH=$(realpath ${BASH_SOURCE[0]})
@@ -47,6 +50,7 @@ docker build ${DOCKER_PATH} --build-arg=IDG_TAG=${IDG_TAG}\
--build-arg=STMAN_TAG=${STMAN_TAG}\
--build-arg=Dysco_TAG=${Dysco_TAG}\
--build-arg=TOIL_VERSION=${TOIL_VERSION}\
+ --build-arg=LofarBeam_TAG=${LofarBeam_TAG}\
-t lofareosc/lofar-pipeline
docker build ${DOCKER_PATH} -f ${DOCKER_PATH}/Dockerfile_ci -t lofareosc/lofar-pipeline-ci
diff --git a/steps/LoSoTo.Lofarbeam.cwl b/steps/LoSoTo.Lofarbeam.cwl
new file mode 100644
index 00000000..ce421868
--- /dev/null
+++ b/steps/LoSoTo.Lofarbeam.cwl
@@ -0,0 +1,68 @@
+#!/usr/bin/env cwl-runner
+
+class: CommandLineTool
+cwlVersion: v1.0
+id: losoto_lofarbeam
+
+$namespaces:
+ lofar: https://git.astron.nl/eosc/ontologies/raw/master/schema/lofar.owl
+doc: |
+ LOFAR beam prediction
+
+requirements:
+ InlineJavascriptRequirement:
+ expressionLib:
+ - { $include: utils.js}
+ InitialWorkDirRequirement:
+ listing:
+ - entryname: 'parset.config'
+ entry: $(get_losoto_config('LOFARBEAM').join('\n'))
+
+ - entryname: $(inputs.input_h5parm.basename)
+ entry: $(inputs.input_h5parm)
+ writable: true
+
+baseCommand: "losoto"
+
+arguments:
+ - $(inputs.input_h5parm.basename)
+ - parset.config
+
+hints:
+ DockerRequirement:
+ dockerPull: lofareosc/lofar-pipeline-ci:latest
+
+inputs:
+ - id: input_h5parm
+ type: File
+ format: lofar:#H5Parm
+ - id: soltab
+ type: string
+ doc: "Solution table"
+ - id: ms
+ type: Directory
+ doc: "Measurement set"
+ - id: inverse
+ type: boolean?
+ default: false
+ - id: useElementResponse
+ type: boolean?
+ default: true
+ - id: useArrayFactor
+ type: boolean?
+ default: true
+ - id: useChanFreq
+ type: boolean?
+ default: true
+outputs:
+ - id: output_h5parm
+ type: File
+ format: lofar:#H5Parm
+ outputBinding:
+ glob: $(inputs.input_h5parm.basename)
+ - id: config
+ type: File
+ outputBinding:
+ glob: "*.config"
+$schema:
+ - https://git.astron.nl/eosc/ontologies/raw/master/schema/lofar.owl
diff --git a/steps/utils.js b/steps/utils.js
index 3b3804b5..2077e7ba 100644
--- a/steps/utils.js
+++ b/steps/utils.js
@@ -8,9 +8,17 @@ function get_losoto_config(step_name) {
if(field_name === 'input_h5parm' ||
field_name === 'soltab' ||
field_name === 'ncpu') continue;
+
if(inputs[field_name] === null ||
inputs[field_name] === 'null') continue;
- par.push(field_name+'='+inputs[field_name])
+
+ if(inputs[field_name]["class"] !== undefined &&
+ (inputs[field_name]["class"] ==="Directory" ||
+ inputs[field_name]["class"] ==="File")){
+ par.push(field_name+'='+inputs[field_name].path)
+ } else {
+ par.push(field_name+'='+inputs[field_name])
+ }
}
return par
}
diff --git a/test_jobs/losoto_lofarbeam.json b/test_jobs/losoto_lofarbeam.json
new file mode 100644
index 00000000..e6c2ed20
--- /dev/null
+++ b/test_jobs/losoto_lofarbeam.json
@@ -0,0 +1,8 @@
+{
+ "ms": {
+ "class": "Directory",
+ "path": "/data/L570745_SB001_uv_MODEL.MS"
+ },
+ "input_h5parm": {"class": "File", "path": "/data/example.h5", "format": "lofar:#H5Parm"},
+ "soltab": "sol000/phase000"
+}
--
GitLab