Commit 6620eb81 authored by mancini's avatar mancini

Merge branch 'LoSoTo-LOFARBeam' into 'master'

Add LOFAR Beam

See merge request lofar-cwl!12
parents 878593e1 8cee244b
Pipeline #513 canceled with stages
......@@ -4,6 +4,7 @@ Docker/DP3
Docker/Dysco
Docker/STMAN
Docker/idg
Docker/LOFARBeam
*.sif
# Byte-compiled / optimized / DLL files
__pycache__/
......
......@@ -167,6 +167,11 @@ losoto_reweight:
script:
- cwl-runner --no-container steps/LoSoTo.Reweight.cwl test_jobs/losoto_reweight.json
losoto_lofarbeam:
stage: test_steps
allow_failure: true
script:
- cwl-runner --no-container --preserve-environment PYTHONPATH steps/LoSoTo.Lofarbeam.cwl test_jobs/losoto_lofarbeam.json
losoto_residual:
stage: test_steps
......
......@@ -17,6 +17,7 @@ RUN export DEBIAN_FRONTEND=noninteractive && \
libboost-filesystem-dev \
libboost-system-dev \
libboost-date-time-dev \
libboost-numpy-dev \
libboost-signals-dev \
libboost-program-options-dev \
libboost-test-dev \
......@@ -64,6 +65,14 @@ RUN cd /src/Dysco/ && \
make -j2 && \
make install -j2
ADD ./LOFARBeam /src/LOFARBeam/src
RUN cd /src/LOFARBeam/ && \
mkdir build && \
cd build && \
cmake ../src -DCMAKE_INSTALL_PREFIX=/usr/local/ -DPYTHON_EXECUTABLE=/usr/bin/python3 && \
make -j2 && \
make install -j2
#----------------------------------------------
FROM kernsuite/base:5 as runner
......@@ -79,6 +88,7 @@ RUN export DEBIAN_FRONTEND=noninteractive && \
libarmadillo8 \
libboost-date-time1.65.1 \
libboost-filesystem1.65.1 \
libboost-numpy1.65.1 \
libboost-python1.65.1 \
libboost-program-options1.65.1 \
libboost-system1.65.1 \
......@@ -120,8 +130,10 @@ LABEL STMAN.version.tag=${STMAN_TAG}
ARG Dysco_TAG=master
LABEL Dysco.version.tag=${Dysco_TAG}
ARG LofarBeam_TAG=master
LABEL LOFARBeam.version.tag=${LofarBeam_TAG}
RUN apt update && apt install -y nodejs python3-pip git wsclean
RUN apt-get --allow-releaseinfo-change update -y && apt install -y nodejs python3-pip git wsclean
RUN python3 -m pip install cwltool cwl-runner -e "git://github.com/darafferty/LSMTool.git@${LSMTool_TAG}#egg=LSMTool" \
"toil[cwl]"==${TOIL_VERSION} \
matplotlib
......@@ -140,7 +152,7 @@ WORKDIR /home/lofaruser
RUN chmod +rx /usr/local/bin/*
ENTRYPOINT ["/home/lofaruser/.entrypoint"]
USER lofaruser
ENV PYTHONPATH=/usr/local/lib/python3.6/site-packages/
ARG DP3_TAG=master
ARG IDG_TAG=master
......
......@@ -22,6 +22,7 @@ TOIL_VERSION=3.20.0
STMAN_TAG=master
Dysco_TAG=v1.2
Prefactor_TAG=master
LofarBeam_TAG=master
# FETCHES THE IDG
git_clone_or_pull https://gitlab.com/astron-idg/idg/ ${IDG_TAG} idg
......@@ -35,6 +36,8 @@ git_clone_or_pull https://github.com/lofar-astron/LofarStMan ${STMAN_TAG} STMAN
# FETCHES Dysco
git_clone_or_pull https://github.com/aroffringa/dysco.git ${Dysco_TAG} Dysco
# FETCHES LOFARBeam
git_clone_or_pull https://github.com/lofar-astron/LOFARBeam ${LofarBeam_TAG} LOFARBeam
SCRIPT_PATH=$(realpath ${BASH_SOURCE[0]})
......@@ -47,6 +50,7 @@ docker build ${DOCKER_PATH} --build-arg=IDG_TAG=${IDG_TAG}\
--build-arg=STMAN_TAG=${STMAN_TAG}\
--build-arg=Dysco_TAG=${Dysco_TAG}\
--build-arg=TOIL_VERSION=${TOIL_VERSION}\
--build-arg=LofarBeam_TAG=${LofarBeam_TAG}\
-t lofareosc/lofar-pipeline
docker build ${DOCKER_PATH} -f ${DOCKER_PATH}/Dockerfile_ci -t lofareosc/lofar-pipeline-ci
#!/usr/bin/env cwl-runner
class: CommandLineTool
cwlVersion: v1.0
id: losoto_lofarbeam
$namespaces:
lofar: https://git.astron.nl/eosc/ontologies/raw/master/schema/lofar.owl
doc: |
LOFAR beam prediction
requirements:
InlineJavascriptRequirement:
expressionLib:
- { $include: utils.js}
InitialWorkDirRequirement:
listing:
- entryname: 'parset.config'
entry: $(get_losoto_config('LOFARBEAM').join('\n'))
- entryname: $(inputs.input_h5parm.basename)
entry: $(inputs.input_h5parm)
writable: true
baseCommand: "losoto"
arguments:
- $(inputs.input_h5parm.basename)
- parset.config
hints:
DockerRequirement:
dockerPull: lofareosc/lofar-pipeline-ci:latest
inputs:
- id: input_h5parm
type: File
format: lofar:#H5Parm
- id: soltab
type: string
doc: "Solution table"
- id: ms
type: Directory
doc: "Measurement set"
- id: inverse
type: boolean?
default: false
- id: useElementResponse
type: boolean?
default: true
- id: useArrayFactor
type: boolean?
default: true
- id: useChanFreq
type: boolean?
default: true
outputs:
- id: output_h5parm
type: File
format: lofar:#H5Parm
outputBinding:
glob: $(inputs.input_h5parm.basename)
- id: config
type: File
outputBinding:
glob: "*.config"
$schema:
- https://git.astron.nl/eosc/ontologies/raw/master/schema/lofar.owl
......@@ -8,9 +8,17 @@ function get_losoto_config(step_name) {
if(field_name === 'input_h5parm' ||
field_name === 'soltab' ||
field_name === 'ncpu') continue;
if(inputs[field_name] === null ||
inputs[field_name] === 'null') continue;
par.push(field_name+'='+inputs[field_name])
if(inputs[field_name]["class"] !== undefined &&
(inputs[field_name]["class"] ==="Directory" ||
inputs[field_name]["class"] ==="File")){
par.push(field_name+'='+inputs[field_name].path)
} else {
par.push(field_name+'='+inputs[field_name])
}
}
return par
}
{
"ms": {
"class": "Directory",
"path": "/data/L570745_SB001_uv_MODEL.MS"
},
"input_h5parm": {"class": "File", "path": "/data/example.h5", "format": "lofar:#H5Parm"},
"soltab": "sol000/phase000"
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment