...
 
Commits (7)
test_data/*.MS
Docker/scripts
Docker/DP3
Docker/Dysco
Docker/STMAN
Docker/idg
*.sif
# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]
*$py.class
# C extensions
*.so
# Distribution / packaging
.Python
build/
develop-eggs/
dist/
downloads/
eggs/
.eggs/
lib/
lib64/
parts/
sdist/
var/
wheels/
pip-wheel-metadata/
share/python-wheels/
*.egg-info/
.installed.cfg
*.egg
MANIFEST
# PyInstaller
# Usually these files are written by a python script from a template
# before PyInstaller builds the exe, so as to inject date/other infos into it.
*.manifest
*.spec
# Installer logs
pip-log.txt
pip-delete-this-directory.txt
# Unit test / coverage reports
htmlcov/
.tox/
.nox/
.coverage
.coverage.*
.cache
nosetests.xml
coverage.xml
*.cover
*.py,cover
.hypothesis/
.pytest_cache/
# Translations
*.mo
*.pot
# Django stuff:
*.log
local_settings.py
db.sqlite3
db.sqlite3-journal
# Flask stuff:
instance/
.webassets-cache
# Scrapy stuff:
.scrapy
# Sphinx documentation
docs/_build/
# PyBuilder
target/
# Jupyter Notebook
.ipynb_checkpoints
# IPython
profile_default/
ipython_config.py
# pyenv
.python-version
# pipenv
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
# However, in case of collaboration, if having platform-specific dependencies or dependencies
# having no cross-platform support, pipenv may install dependencies that don't work, or not
# install all needed dependencies.
#Pipfile.lock
# celery beat schedule file
celerybeat-schedule
# SageMath parsed files
*.sage.py
# Environments
.env
.venv
env/
venv/
ENV/
env.bak/
venv.bak/
# Spyder project settings
.spyderproject
.spyproject
# Rope project settings
.ropeproject
# mkdocs documentation
/site
# mypy
.mypy_cache/
.dmypy.json
dmypy.json
# Pyre type checker
.pyre/
# Not idea
.idea
image: lofareosc/lofar-pipeline-ci:latest
variables:
TEST_DATASET_NAME: "L570745_uv_first10.MS.tar.xz"
PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
stages:
- build_docker
- push_docker
- validate_cwl_scripts
- test_steps
- integration_test
before_script:
- mkdir workdir
- mkdir logs
after_script:
- echo "All done"
build_docker:
image: docker:latest
stage: build_docker
only:
- tags
script:
- apk add git subversion git-lfs bash
- cd Docker
- bash build_docker.sh
push_docker_tags:
image: docker:latest
allow_failure: true
stage: push_docker
only:
- tags
script:
- docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
- docker tag lofareosc/prefactor-ci:latest $CI_REGISTRY/prefactor-ci:$CI_COMMIT_REF_NAME
- docker push $CI_REGISTRY/prefactor-ci:$CI_COMMIT_REF_NAME
validate_scripts:
stage: validate_cwl_scripts
script:
- for file in `find . -name 'steps/*.cwl'`; do cwltool --validate $file; done
average_step_generator:
stage: test_steps
allow_failure: true
script:
- cwl-runner steps/DP3.AveragerStepGenerator.cwl test_jobs/averager_step_generator.json
- ! cwl-runner steps/DP3.AveragerStepGenerator.cwl test_jobs/averager_step_generator_error.json
aoflagger_step_generator:
stage: test_steps
allow_failure: true
script:
- cwl-runner steps/DP3.AOFlaggerStepGenerator.cwl test_jobs/aoflagger_step_generator.json
preflagger_step_generator:
stage: test_steps
allow_failure: true
script:
- cwl-runner steps/DP3.PreflaggerStepGenerator.cwl test_jobs/preflagger_step_generator.json
losoto_abs:
stage: test_steps
allow_failure: true
script:
- cwl-runner --no-container steps/LoSoTo.Abs.cwl test_jobs/losoto_abs.json
losoto_duplicate:
stage: test_steps
allow_failure: true
script:
- cwl-runner --no-container steps/LoSoTo.Duplicate.cwl test_jobs/losoto_duplicate.json
losoto_plot:
stage: test_steps
allow_failure: true
script:
- cwl-runner --no-container steps/LoSoTo.Plot.cwl test_jobs/losoto_plot.json
losoto_norm:
stage: test_steps
allow_failure: true
script:
- cwl-runner --no-container steps/LoSoTo.Norm.cwl test_jobs/losoto_norm.json
losoto_replicateonaxis:
stage: test_steps
allow_failure: true
script:
- cwl-runner --no-container steps/LoSoTo.Replicateonaxis.cwl test_jobs/losoto_replicateonaxis.json
losoto_reset:
stage: test_steps
allow_failure: true
script:
- cwl-runner --no-container steps/LoSoTo.Reset.cwl test_jobs/losoto_reset.json
losoto_residual:
stage: test_steps
allow_failure: true
script:
- cwl-runner --no-container steps/LoSoTo.Residual.cwl test_jobs/losoto_residual.json
parset_selector:
stage: test_steps
allow_failure: true
script:
- cwl-runner steps/DP3.Selector.cwl test_jobs/selector.json
file_selector:
stage: test_steps
allow_failure: true
script:
- cwl-runner steps/FileSelector.cwl test_jobs/file_selector.json
NDPPP:
stage: test_steps
allow_failure: true
script:
- cwltool --no-container --preserve-environment LD_LIBRARY_PATH --preserve-environment PATH steps/DPPP.cwl test_jobs/NDPPP.json
DP3.Execute:
stage: test_steps
allow_failure: true
script:
- cwltool --no-container --preserve-environment LD_LIBRARY_PATH --preserve-environment PATH steps/DP3.Execute.cwl test_jobs/dp3_execute.json
h5parm_collector:
stage: test_steps
allow_failure: true
script:
- cwltool --no-container --preserve-environment LD_LIBRARY_PATH --preserve-environment PATH --preserve-environment PYTHONPATH steps/H5ParmCollector.cwl test_jobs/h5parm_collector.json
aoflagger:
stage: test_steps
allow_failure: true
script:
- cwltool --no-container --preserve-environment LD_LIBRARY_PATH --preserve-environment PATH steps/AOFlagger.cwl test_jobs/aoflagger.json
interpolate_visibilities:
stage: test_steps
allow_failure: true
script:
- cwltool --no-container --preserve-environment LD_LIBRARY_PATH --preserve-environment PATH --preserve-environment PYTHONPATH steps/interpolate_visibilities.cwl test_jobs/interpolate_visibilities.json
h5parm_collector:
stage: test_steps
allow_failure: true
script:
- cwltool --no-container --preserve-environment LD_LIBRARY_PATH --preserve-environment PATH --preserve-environment PYTHONPATH steps/H5ParmCollector.cwl test_jobs/h5parm_collector.json
ddecal:
stage: test_steps
allow_failure: true
script:
- cwltool --no-container --preserve-environment LD_LIBRARY_PATH --preserve-environment PATH --preserve-environment PYTHONPATH steps/ddecal.cwl test_jobs/ddecal.json
gaincal:
stage: test_steps
allow_failure: true
script:
- cwltool --no-container --preserve-environment LD_LIBRARY_PATH --preserve-environment PATH --preserve-environment PYTHONPATH steps/gaincal.cwl test_jobs/gaincal.json
predict:
stage: test_steps
allow_failure: true
script:
- cwltool --no-container --preserve-environment LD_LIBRARY_PATH --preserve-environment PATH --preserve-environment PYTHONPATH steps/predict.cwl test_jobs/predict.json
#!/bin/bash
set -e
exec "$@"
FROM kernsuite/base:5 as builder
SHELL ["/bin/bash", "-c"]
RUN export DEBIAN_FRONTEND=noninteractive && \
apt-get update && \
apt-get install -y \
casacore-data \
casacore-dev \
libboost-python-dev \
libcfitsio-dev \
python-dev \
python3-numpy \
cmake \
build-essential \
libhdf5-serial-dev \
libarmadillo-dev \
libboost-filesystem-dev \
libboost-system-dev \
libboost-date-time-dev \
libboost-signals-dev \
libboost-program-options-dev \
libboost-test-dev \
libxml2-dev \
libpng-dev \
pkg-config \
aoflagger-dev \
libgtkmm-3.0-dev \
git \
wget \
libfftw3-dev \
libgsl-dev
RUN mkdir -p /src/
WORKDIR /src/
# Build the IDG version from source
ADD ./idg /src/idg/src
RUN mkdir idg/build && cd idg/build && cmake -DCMAKE_INSTALL_PREFIX=/usr/local/ ../src && make install -j2
ADD ./DP3 /src/dp3/src
RUN cd /src/dp3/ && \
mkdir build && \
cd build && \
cmake ../src -DCMAKE_INSTALL_PREFIX=/usr/local/ -DPORTABLE=True &&\
make -j2 && \
make install -j2
ADD ./STMAN /src/STMAN/src
RUN cd /src/STMAN/ && \
mkdir build && \
cd build && \
cmake ../src -DCMAKE_INSTALL_PREFIX=/usr/local/ -DPORTABLE=True && \
make -j2 && \
make install -j2
ADD ./Dysco /src/Dysco/src
RUN cd /src/Dysco/ && \
mkdir build && \
cd build && \
cmake ../src -DCMAKE_INSTALL_PREFIX=/usr/local/ -DPORTABLE=True && \
make -j2 && \
make install -j2
#----------------------------------------------
FROM kernsuite/base:5 as runner
COPY --from=builder /usr/local/ /usr/local
SHELL ["/bin/bash", "-c"]
RUN export DEBIAN_FRONTEND=noninteractive && \
apt-get update && \
apt-get install -y \
aoflagger \
casacore-data \
libarmadillo8 \
libboost-date-time1.65.1 \
libboost-filesystem1.65.1 \
libboost-python1.65.1 \
libboost-program-options1.65.1 \
libboost-system1.65.1 \
libboost-signals1.65.1 \
libboost-test1.65.1 \
libboost-python1.65-dev \
libstationresponse2 \
libcasa-* \
libcfitsio5 \
libgtkmm-3.0 \
libfftw3-3 \
libhdf5-cpp-100 \
libpng16-16 \
libxml2 \
python3.7 \
python3-casacore \
python3-numpy \
python3-scipy && \
rm -rf /var/lib/apt/lists/*
RUN mkdir /home/lofaruser/ && \
groupadd -r lofaruser && \
useradd --no-log-init -r -g lofaruser lofaruser && \
chown lofaruser:lofaruser /home/lofaruser
ARG TOIL_VERSION=3.20.0
LABEL toil.version=${TOIL_VERSION}
ARG LSMTool_TAG=master
LABEL LSMTool.version.tag=${LSMTool_TAG}
ARG LoSoTo_TAG=master
LABEL LoSoTo.version.tag=${LoSoTo_TAG}
ARG STMAN_TAG=master
LABEL STMAN.version.tag=${STMAN_TAG}
ARG Dysco_TAG=master
LABEL Dysco.version.tag=${Dysco_TAG}
RUN apt update && apt install -y nodejs python3-pip git wsclean
RUN python3 -m pip install cwltool cwl-runner -e "git://github.com/darafferty/LSMTool.git@${LSMTool_TAG}#egg=LSMTool" \
"toil[cwl]"==${TOIL_VERSION} \
matplotlib
RUN python3 -m pip install -e "git+https://github.com/revoltek/losoto.git@${LoSoTo_TAG}#egg=LoSoTo"
ADD losoto.patch /home/lofaruser/losoto.patch
RUN patch /src/losoto/losoto/operations/__init__.py /home/lofaruser/losoto.patch
ADD .entrypoint /home/lofaruser/.entrypoint
RUN chown lofaruser:lofaruser /home/lofaruser/.entrypoint && \
chmod +rx /home/lofaruser/.entrypoint
WORKDIR /home/lofaruser
RUN chmod +rx /usr/local/bin/*
ENTRYPOINT ["/home/lofaruser/.entrypoint"]
USER lofaruser
ARG DP3_TAG=master
ARG IDG_TAG=master
LABEL IDG.version.tag=${IDG_TAG}
LABEL DP3.version.tag=${DP3_TAG}
FROM lofareosc/lofar-pipeline:latest
SHELL ["/bin/bash", "-c"]
ADD L570745_uv_first10.MS.tar.xz /data
ADD skymodels.tar.gz /data/skymodels
ADD test_data/A-Team_lowres.sourcedb /data/A-Team_lowres.sourcedb
ADD test_data/example.h5 /data/example.h5
ADD test_data/L570745_SB001_uv_MODEL.MS /data/L570745_SB001_uv_MODEL.MS
#! /bin/bash
set -e
git_clone_or_pull () {
REPO=$1
BRANCH=$2
DIR=$3
if [ -d $DIR ]
then
git -C "${DIR}" pull
else
git clone --depth 1 -b "${BRANCH}" "${REPO}" "${DIR}"
fi
}
# SOFTWARE VERSIONS
DP3_TAG=v4.1
IDG_TAG=0.6
LSMTool_TAG=v1.4.2
LoSoTo_TAG=f0398b
TOIL_VERSION=3.20.0
STMAN_TAG=master
Dysco_TAG=v1.2
Prefactor_TAG=master
# FETCHES THE IDG
git_clone_or_pull https://gitlab.com/astron-idg/idg/ ${IDG_TAG} idg
# FETCHES DP3
git_clone_or_pull https://github.com/lofar-astron/DP3 ${DP3_TAG} DP3
# FETCHES STMAN
git_clone_or_pull https://github.com/lofar-astron/LofarStMan ${STMAN_TAG} STMAN
# FETCHES Dysco
git_clone_or_pull https://github.com/aroffringa/dysco.git ${Dysco_TAG} Dysco
SCRIPT_PATH=$(realpath ${BASH_SOURCE[0]})
DOCKER_PATH=$(dirname ${SCRIPT_PATH})
docker build ${DOCKER_PATH} --build-arg=IDG_TAG=${IDG_TAG}\
--build-arg=DP3_TAG=${DP3_TAG}\
--build-arg=LSMTool_TAG=${LSMTool_TAG}\
--build-arg=LoSoTo_TAG=${LoSoTo_TAG}\
--build-arg=STMAN_TAG=${STMAN_TAG}\
--build-arg=Dysco_TAG=${Dysco_TAG}\
--build-arg=TOIL_VERSION=${TOIL_VERSION}\
-t lofareosc/lofar-pipeline
docker build ${DOCKER_PATH} -f ${DOCKER_PATH}/Dockerfile_ci -t lofareosc/lofar-pipeline-ci
--- /src/losoto/losoto/operations/__init__.py 2019-11-07 12:43:46.220577945 +0000
+++ /src/losoto/losoto/operations/__init__.py 2019-11-07 12:43:34.401765786 +0000
@@ -2,8 +2,7 @@
__all__ = [ os.path.basename(f)[:-3] for f in glob.glob(os.path.dirname(__file__)+"/*.py") if f[0] != '_']
-for x in __all__:
- __import__(x, locals(), globals())
+from . import *
class Timer(object):
"""
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.