Commit 24168d81 authored by mancini's avatar mancini

First commit migrating from prefactor3

parent be047a63
test_data/*.MS
Docker/scripts
*.sif
# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]
*$py.class
# C extensions
*.so
# Distribution / packaging
.Python
build/
develop-eggs/
dist/
downloads/
eggs/
.eggs/
lib/
lib64/
parts/
sdist/
var/
wheels/
pip-wheel-metadata/
share/python-wheels/
*.egg-info/
.installed.cfg
*.egg
MANIFEST
# PyInstaller
# Usually these files are written by a python script from a template
# before PyInstaller builds the exe, so as to inject date/other infos into it.
*.manifest
*.spec
# Installer logs
pip-log.txt
pip-delete-this-directory.txt
# Unit test / coverage reports
htmlcov/
.tox/
.nox/
.coverage
.coverage.*
.cache
nosetests.xml
coverage.xml
*.cover
*.py,cover
.hypothesis/
.pytest_cache/
# Translations
*.mo
*.pot
# Django stuff:
*.log
local_settings.py
db.sqlite3
db.sqlite3-journal
# Flask stuff:
instance/
.webassets-cache
# Scrapy stuff:
.scrapy
# Sphinx documentation
docs/_build/
# PyBuilder
target/
# Jupyter Notebook
.ipynb_checkpoints
# IPython
profile_default/
ipython_config.py
# pyenv
.python-version
# pipenv
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
# However, in case of collaboration, if having platform-specific dependencies or dependencies
# having no cross-platform support, pipenv may install dependencies that don't work, or not
# install all needed dependencies.
#Pipfile.lock
# celery beat schedule file
celerybeat-schedule
# SageMath parsed files
*.sage.py
# Environments
.env
.venv
env/
venv/
ENV/
env.bak/
venv.bak/
# Spyder project settings
.spyderproject
.spyproject
# Rope project settings
.ropeproject
# mkdocs documentation
/site
# mypy
.mypy_cache/
.dmypy.json
dmypy.json
# Pyre type checker
.pyre/
# Not idea
.idea
image: lofareosc/lofar-pipeline-ci:latest
variables:
TEST_DATASET_NAME: "L570745_uv_first10.MS.tar.xz"
PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
stages:
- build_docker
- push_docker
- validate_cwl_scripts
- test_steps
- integration_test
before_script:
- mkdir workdir
- mkdir logs
after_script:
- echo "All done"
build_docker:
image: docker:latest
stage: build_docker
only:
- tags
script:
- apk add git subversion git-lfs bash
- cd Docker
- bash build_docker.sh
push_docker_tags:
image: docker:latest
allow_failure: true
stage: push_docker
only:
- tags
script:
- docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
- docker tag lofareosc/prefactor-ci:latest $CI_REGISTRY/prefactor-ci:$CI_COMMIT_REF_NAME
- docker push $CI_REGISTRY/prefactor-ci:$CI_COMMIT_REF_NAME
validate_scripts:
stage: validate_cwl_scripts
script:
- for file in `find . -name 'steps/*.cwl'`; do cwltool --validate $file; done
average_step_generator:
stage: test_steps
allow_failure: true
script:
- cwl-runner steps/DP3.AveragerStepGenerator.cwl test_jobs/averager_step_generator.json
- ! cwl-runner steps/DP3.AveragerStepGenerator.cwl test_jobs/averager_step_generator_error.json
aoflagger_step_generator:
stage: test_steps
allow_failure: true
script:
- cwl-runner steps/DP3.AOFlaggerStepGenerator.cwl test_jobs/aoflagger_step_generator.json
preflagger_step_generator:
stage: test_steps
allow_failure: true
script:
- cwl-runner steps/DP3.PreflaggerStepGenerator.cwl test_jobs/preflagger_step_generator.json
losoto_abs:
stage: test_steps
allow_failure: true
script:
- cwl-runner --no-container steps/LoSoTo.Abs.cwl test_jobs/losoto_abs.json
losoto_duplicate:
stage: test_steps
allow_failure: true
script:
- cwl-runner --no-container steps/LoSoTo.Duplicate.cwl test_jobs/losoto_duplicate.json
losoto_plot:
stage: test_steps
allow_failure: true
script:
- cwl-runner --no-container steps/LoSoTo.Plot.cwl test_jobs/losoto_plot.json
losoto_norm:
stage: test_steps
allow_failure: true
script:
- cwl-runner --no-container steps/LoSoTo.Norm.cwl test_jobs/losoto_norm.json
losoto_replicateonaxis:
stage: test_steps
allow_failure: true
script:
- cwl-runner --no-container steps/LoSoTo.Replicateonaxis.cwl test_jobs/losoto_replicateonaxis.json
losoto_reset:
stage: test_steps
allow_failure: true
script:
- cwl-runner --no-container steps/LoSoTo.Reset.cwl test_jobs/losoto_reset.json
losoto_residual:
stage: test_steps
allow_failure: true
script:
- cwl-runner --no-container steps/LoSoTo.Residual.cwl test_jobs/losoto_residual.json
parset_selector:
stage: test_steps
allow_failure: true
script:
- cwl-runner steps/DP3.Selector.cwl test_jobs/selector.json
file_selector:
stage: test_steps
allow_failure: true
script:
- cwl-runner steps/FileSelector.cwl test_jobs/file_selector.json
NDPPP:
stage: test_steps
allow_failure: true
script:
- cwltool --no-container --preserve-environment LD_LIBRARY_PATH --preserve-environment PATH steps/DPPP.cwl test_jobs/NDPPP.json
DP3.Execute:
stage: test_steps
allow_failure: true
script:
- cwltool --no-container --preserve-environment LD_LIBRARY_PATH --preserve-environment PATH steps/DP3.Execute.cwl test_jobs/dp3_execute.json
h5parm_collector:
stage: test_steps
allow_failure: true
script:
- cwltool --no-container --preserve-environment LD_LIBRARY_PATH --preserve-environment PATH --preserve-environment PYTHONPATH steps/H5ParmCollector.cwl test_jobs/h5parm_collector.json
aoflagger:
stage: test_steps
allow_failure: true
script:
- cwltool --no-container --preserve-environment LD_LIBRARY_PATH --preserve-environment PATH steps/AOFlagger.cwl test_jobs/aoflagger.json
interpolate_visibilities:
stage: test_steps
allow_failure: true
script:
- cwltool --no-container --preserve-environment LD_LIBRARY_PATH --preserve-environment PATH --preserve-environment PYTHONPATH steps/interpolate_visibilities.cwl test_jobs/interpolate_visibilities.json
h5parm_collector:
stage: test_steps
allow_failure: true
script:
- cwltool --no-container --preserve-environment LD_LIBRARY_PATH --preserve-environment PATH --preserve-environment PYTHONPATH steps/H5ParmCollector.cwl test_jobs/h5parm_collector.json
blsmooth:
stage: test_steps
allow_failure: true
script:
- cwltool --no-container --preserve-environment LD_LIBRARY_PATH --preserve-environment PATH --preserve-environment PYTHONPATH steps/blsmooth.cwl test_jobs/blsmooth.json
ddecal:
stage: test_steps
allow_failure: true
script:
- cwltool --no-container --preserve-environment LD_LIBRARY_PATH --preserve-environment PATH --preserve-environment PYTHONPATH steps/ddecal.cwl test_jobs/ddecal.json
predict:
stage: test_steps
allow_failure: true
script:
- cwltool --no-container --preserve-environment LD_LIBRARY_PATH --preserve-environment PATH --preserve-environment PYTHONPATH steps/predict.cwl test_jobs/predict.json
#!/bin/bash
set -e
exec "$@"
FROM kernsuite/base:5 as builder
SHELL ["/bin/bash", "-c"]
RUN export DEBIAN_FRONTEND=noninteractive && \
apt-get update && \
apt-get install -y \
casacore-data \
casacore-dev \
libboost-python-dev \
libcfitsio-dev \
python-dev \
python3-numpy \
cmake \
build-essential \
libhdf5-serial-dev \
libarmadillo-dev \
libboost-filesystem-dev \
libboost-system-dev \
libboost-date-time-dev \
libboost-signals-dev \
libboost-program-options-dev \
libboost-test-dev \
libxml2-dev \
libpng-dev \
pkg-config \
aoflagger-dev \
libgtkmm-3.0-dev \
git \
wget \
libfftw3-dev \
libgsl-dev
RUN mkdir -p /src/
WORKDIR /src/
# Build the IDG version from source
ADD ./idg /src/idg/src
RUN mkdir idg/build && cd idg/build && cmake -DCMAKE_INSTALL_PREFIX=/usr/local/ ../src && make install -j2
ADD ./DP3 /src/dp3/src
RUN cd /src/dp3/ && \
mkdir build && \
cd build && \
cmake ../src -DCMAKE_INSTALL_PREFIX=/usr/local/ -DPORTABLE=True &&\
make -j2 && \
make install -j2
ADD ./STMAN /src/STMAN/src
RUN cd /src/STMAN/ && \
mkdir build && \
cd build && \
cmake ../src -DCMAKE_INSTALL_PREFIX=/usr/local/ -DPORTABLE=True && \
make -j2 && \
make install -j2
ADD ./Dysco /src/Dysco/src
RUN cd /src/Dysco/ && \
mkdir build && \
cd build && \
cmake ../src -DCMAKE_INSTALL_PREFIX=/usr/local/ -DPORTABLE=True && \
make -j2 && \
make install -j2
#----------------------------------------------
FROM kernsuite/base:5 as runner
COPY --from=builder /usr/local/ /usr/local
SHELL ["/bin/bash", "-c"]
RUN export DEBIAN_FRONTEND=noninteractive && \
apt-get update && \
apt-get install -y \
aoflagger \
casacore-data \
libarmadillo8 \
libboost-date-time1.65.1 \
libboost-filesystem1.65.1 \
libboost-python1.65.1 \
libboost-program-options1.65.1 \
libboost-system1.65.1 \
libboost-signals1.65.1 \
libboost-test1.65.1 \
libboost-python1.65-dev \
libstationresponse2 \
libcasa-* \
libcfitsio5 \
libgtkmm-3.0 \
libfftw3-3 \
libhdf5-cpp-100 \
libpng16-16 \
libxml2 \
python3.7 \
python3-casacore \
python3-numpy \
python3-scipy && \
rm -rf /var/lib/apt/lists/*
RUN mkdir /home/lofaruser/ && \
groupadd -r lofaruser && \
useradd --no-log-init -r -g lofaruser lofaruser && \
chown lofaruser:lofaruser /home/lofaruser
ARG TOIL_VERSION=3.20.0
LABEL toil.version=${TOIL_VERSION}
ARG LSMTool_TAG=master
LABEL LSMTool.version.tag=${LSMTool_TAG}
ARG LoSoTo_TAG=master
LABEL LoSoTo.version.tag=${LoSoTo_TAG}
ARG STMAN_TAG=master
LABEL STMAN.version.tag=${STMAN_TAG}
ARG Dysco_TAG=master
LABEL Dysco.version.tag=${Dysco_TAG}
RUN apt update && apt install -y nodejs python3-pip git wsclean
RUN python3 -m pip install cwltool cwl-runner -e "git://github.com/darafferty/LSMTool.git@${LSMTool_TAG}#egg=LSMTool" \
"toil[cwl]"==${TOIL_VERSION} \
matplotlib
RUN python3 -m pip install -e "git+https://github.com/revoltek/losoto.git@${LoSoTo_TAG}#egg=LoSoTo"
ADD losoto.patch /home/lofaruser/losoto.patch
RUN patch /src/losoto/losoto/operations/__init__.py /home/lofaruser/losoto.patch
ADD .entrypoint /home/lofaruser/.entrypoint
RUN chown lofaruser:lofaruser /home/lofaruser/.entrypoint && \
chmod +rx /home/lofaruser/.entrypoint
WORKDIR /home/lofaruser
COPY prefactor/skymodels/* /data/skymodels/
COPY prefactor/rfistrategies/* /data/rfistrategies/
COPY prefactor/solutions/ /data/solutions/*
RUN chmod +rx /usr/local/bin/*
ENTRYPOINT ["/home/lofaruser/.entrypoint"]
USER lofaruser
ARG DP3_TAG=master
ARG IDG_TAG=master
LABEL IDG.version.tag=${IDG_TAG}
LABEL DP3.version.tag=${DP3_TAG}
FROM lofareosc/lofar-pipeline:latest
SHELL ["/bin/bash", "-c"]
ADD L570745_uv_first10.MS.tar.xz /data
ADD skymodels.tar.gz /data/skymodels
ADD test_data/A-Team_lowres.sourcedb /data/A-Team_lowres.sourcedb
ADD test_data/example.h5 /data/example.h5
ADD test_data/L570745_SB001_uv_MODEL.MS /data/L570745_SB001_uv_MODEL.MS
#! /bin/bash
set -e
git_clone_or_pull () {
REPO=$1
BRANCH=$2
DIR=$3
if [ -d $DIR ]
then
git -C "${DIR}" pull
else
git clone --depth 1 -b "${BRANCH}" "${REPO}" "${DIR}"
fi
}
# SOFTWARE VERSIONS
DP3_TAG=v4.1
IDG_TAG=0.6
LSMTool_TAG=v1.4.2
LoSoTo_TAG=f0398b
TOIL_VERSION=3.20.0
STMAN_TAG=master
Dysco_TAG=v1.2
Prefactor_TAG=master
# FETCHES THE IDG
git_clone_or_pull https://gitlab.com/astron-idg/idg/ ${IDG_TAG} idg
# FETCHES DP3
git_clone_or_pull https://github.com/lofar-astron/DP3 ${DP3_TAG} DP3
# FETCHES STMAN
git_clone_or_pull https://github.com/lofar-astron/LofarStMan ${STMAN_TAG} STMAN
# FETCHES Dysco
git_clone_or_pull https://github.com/aroffringa/dysco.git ${Dysco_TAG} Dysco
SCRIPT_PATH=$(realpath ${BASH_SOURCE[0]})
DOCKER_PATH=$(dirname ${SCRIPT_PATH})
docker build ${DOCKER_PATH} --build-arg=IDG_TAG=${IDG_TAG}\
--build-arg=DP3_TAG=${DP3_TAG}\
--build-arg=LSMTool_TAG=${LSMTool_TAG}\
--build-arg=LoSoTo_TAG=${LoSoTo_TAG}\
--build-arg=STMAN_TAG=${STMAN_TAG}\
--build-arg=Dysco_TAG=${Dysco_TAG}\
--build-arg=TOIL_VERSION=${TOIL_VERSION}\
-t lofareosc/lofar-pipeline
docker build ${DOCKER_PATH} -f ${DOCKER_PATH}/Dockerfile_ci -t lofareosc/lofar-pipeline-ci
--- /src/losoto/losoto/operations/__init__.py 2019-11-07 12:43:46.220577945 +0000
+++ /src/losoto/losoto/operations/__init__.py 2019-11-07 12:43:34.401765786 +0000
@@ -2,8 +2,7 @@
__all__ = [ os.path.basename(f)[:-3] for f in glob.glob(os.path.dirname(__file__)+"/*.py") if f[0] != '_']
-for x in __all__:
- __import__(x, locals(), globals())
+from . import *
class Timer(object):
"""