Merge branch 'LoSoTo-LOFARBeam' into 'master'

Add LOFAR Beam

See merge request eosc/lofar-cwl!12

Former-commit-id: 6620eb81

.gitignore

@@ -4,6 +4,7 @@ Docker/DP3
 Docker/Dysco
 Docker/STMAN
 Docker/idg
+Docker/LOFARBeam
 *.sif
 # Byte-compiled / optimized / DLL files
 __pycache__/

.gitlab-ci.yml

@@ -167,6 +167,11 @@ losoto_reweight:
   script:
    - cwl-runner --no-container steps/LoSoTo.Reweight.cwl test_jobs/losoto_reweight.json
 
+losoto_lofarbeam:
+  stage: test_steps
+  allow_failure: true
+  script:
+   - cwl-runner --no-container --preserve-environment PYTHONPATH steps/LoSoTo.Lofarbeam.cwl test_jobs/losoto_lofarbeam.json
 
 losoto_residual:
   stage: test_steps

Docker/Dockerfile

@@ -17,6 +17,7 @@ RUN export DEBIAN_FRONTEND=noninteractive && \
         libboost-filesystem-dev \
         libboost-system-dev \
         libboost-date-time-dev \
+        libboost-numpy-dev \
         libboost-signals-dev \ 
         libboost-program-options-dev \
         libboost-test-dev \
@@ -64,6 +65,14 @@ RUN cd /src/Dysco/ && \
     make -j2 && \
     make install -j2
 
+ADD ./LOFARBeam /src/LOFARBeam/src
+RUN cd /src/LOFARBeam/ && \
+    mkdir build && \
+    cd build && \
+    cmake ../src -DCMAKE_INSTALL_PREFIX=/usr/local/ -DPYTHON_EXECUTABLE=/usr/bin/python3 && \
+    make -j2 && \
+    make install -j2
+
 #----------------------------------------------
 FROM kernsuite/base:5 as runner
 
@@ -79,6 +88,7 @@ RUN export DEBIAN_FRONTEND=noninteractive && \
         libarmadillo8 \
         libboost-date-time1.65.1 \
         libboost-filesystem1.65.1 \
+        libboost-numpy1.65.1 \
         libboost-python1.65.1 \
         libboost-program-options1.65.1 \
         libboost-system1.65.1 \
@@ -120,8 +130,10 @@ LABEL STMAN.version.tag=${STMAN_TAG}
 ARG Dysco_TAG=master
 LABEL Dysco.version.tag=${Dysco_TAG}
 
+ARG LofarBeam_TAG=master
+LABEL LOFARBeam.version.tag=${LofarBeam_TAG}
     
-RUN apt update && apt install -y nodejs python3-pip git wsclean
+RUN apt-get --allow-releaseinfo-change update -y  && apt install -y nodejs python3-pip git wsclean
 RUN python3 -m pip install cwltool cwl-runner -e "git://github.com/darafferty/LSMTool.git@${LSMTool_TAG}#egg=LSMTool" \
                                    "toil[cwl]"==${TOIL_VERSION} \
                                    matplotlib
@@ -140,7 +152,7 @@ WORKDIR /home/lofaruser
 RUN chmod +rx /usr/local/bin/*
 ENTRYPOINT ["/home/lofaruser/.entrypoint"]
 USER lofaruser
-
+ENV PYTHONPATH=/usr/local/lib/python3.6/site-packages/
 ARG DP3_TAG=master
 ARG IDG_TAG=master
 

Docker/build_docker.sh

@@ -22,6 +22,7 @@ TOIL_VERSION=3.20.0
 STMAN_TAG=master
 Dysco_TAG=v1.2
 Prefactor_TAG=master
+LofarBeam_TAG=master
 
 # FETCHES THE IDG
 git_clone_or_pull https://gitlab.com/astron-idg/idg/ ${IDG_TAG} idg
@@ -35,6 +36,8 @@ git_clone_or_pull https://github.com/lofar-astron/LofarStMan ${STMAN_TAG} STMAN
 # FETCHES Dysco
 git_clone_or_pull https://github.com/aroffringa/dysco.git ${Dysco_TAG} Dysco
 
+# FETCHES LOFARBeam
+git_clone_or_pull https://github.com/lofar-astron/LOFARBeam ${LofarBeam_TAG} LOFARBeam
 
 SCRIPT_PATH=$(realpath ${BASH_SOURCE[0]})
 
@@ -47,6 +50,7 @@ docker build ${DOCKER_PATH} --build-arg=IDG_TAG=${IDG_TAG}\
                             --build-arg=STMAN_TAG=${STMAN_TAG}\
                             --build-arg=Dysco_TAG=${Dysco_TAG}\
                             --build-arg=TOIL_VERSION=${TOIL_VERSION}\
+			    --build-arg=LofarBeam_TAG=${LofarBeam_TAG}\
                             -t lofareosc/lofar-pipeline
 
 docker build ${DOCKER_PATH} -f ${DOCKER_PATH}/Dockerfile_ci -t lofareosc/lofar-pipeline-ci

steps/LoSoTo.Lofarbeam.cwl

+#!/usr/bin/env cwl-runner
+
+class: CommandLineTool
+cwlVersion: v1.0
+id: losoto_lofarbeam
+
+$namespaces:
+  lofar: https://git.astron.nl/eosc/ontologies/raw/master/schema/lofar.owl
+doc: |
+  LOFAR beam prediction
+
+requirements:
+  InlineJavascriptRequirement:
+    expressionLib:
+      - { $include: utils.js}
+  InitialWorkDirRequirement:
+    listing:
+      - entryname: 'parset.config'
+        entry: $(get_losoto_config('LOFARBEAM').join('\n'))
+
+      - entryname: $(inputs.input_h5parm.basename)
+        entry: $(inputs.input_h5parm)
+        writable: true
+
+baseCommand: "losoto"
+
+arguments:
+  - $(inputs.input_h5parm.basename)
+  - parset.config
+
+hints:
+  DockerRequirement:
+    dockerPull: lofareosc/lofar-pipeline-ci:latest
+
+inputs:
+  - id: input_h5parm
+    type: File
+    format: lofar:#H5Parm
+  - id: soltab
+    type: string
+    doc: "Solution table"
+  - id: ms
+    type: Directory
+    doc: "Measurement set"
+  - id: inverse
+    type: boolean?
+    default: false
+  - id: useElementResponse
+    type: boolean?
+    default: true
+  - id: useArrayFactor
+    type: boolean?
+    default: true
+  - id: useChanFreq
+    type: boolean?
+    default: true
+outputs:
+  - id: output_h5parm
+    type: File
+    format: lofar:#H5Parm
+    outputBinding:
+      glob: $(inputs.input_h5parm.basename)
+  - id: config
+    type: File
+    outputBinding:
+      glob: "*.config"
+$schema:
+  - https://git.astron.nl/eosc/ontologies/raw/master/schema/lofar.owl

steps/utils.js

@@ -8,9 +8,17 @@ function get_losoto_config(step_name) {
         if(field_name === 'input_h5parm' ||
            field_name === 'soltab' ||
            field_name === 'ncpu') continue;
+
         if(inputs[field_name] === null ||
            inputs[field_name] === 'null') continue;
-        par.push(field_name+'='+inputs[field_name])
+        
+        if(inputs[field_name]["class"] !== undefined &&
+           (inputs[field_name]["class"] ==="Directory" ||
+            inputs[field_name]["class"] ==="File")){
+            par.push(field_name+'='+inputs[field_name].path)
+        } else {
+            par.push(field_name+'='+inputs[field_name])
+        }
     }
     return par
 }

test_jobs/losoto_lofarbeam.json

+{
+  "ms": {
+   "class": "Directory",
+   "path": "/data/L570745_SB001_uv_MODEL.MS"
+  },
+  "input_h5parm": {"class": "File", "path": "/data/example.h5", "format": "lofar:#H5Parm"},
+  "soltab":  "sol000/phase000"
+}