Commit 3694958d authored by Jakob Maljaars's avatar Jakob Maljaars

Merge branch 'add-integration-tests' into 'master'

Add integration tests

See merge request !5
parents 0af882c5 aaecc745
Pipeline #4388 passed with stages
in 32 minutes and 18 seconds
default:
image: ubuntu:20.04
before_script:
- apt-get update && apt-get install -y g++ cmake libfftw3-dev liblapacke-dev python3 python3-pip
stages:
- prepare
- build
- integration_and_deploy
# Avoid getting stuck on requested user input, i.e. avoid
# https://git.astron.nl/RD/idg/-/jobs/37678#L767
variables:
DEBIAN_FRONTEND: "noninteractive"
build-base:
stage: prepare
script:
- docker build --tag idg_base:${CI_COMMIT_SHORT_SHA} -f ./docker/ubuntu_20_04_base .
build-integration:
stage: prepare
script:
- docker build --tag idg_integration:${CI_COMMIT_SHORT_SHA} -f ./docker/ubuntu_20_04_integration .
clang-format:
script:
stage: build
dependencies:
- build-base
image: idg_base:${CI_COMMIT_SHORT_SHA}
before_script:
- pip3 install clang-format==9.0.0
- export PATH=/home/gitlab-runner/.local/bin:$PATH
script:
- ./scripts/run-clang-format.sh
idg-python:
stage: build
dependencies:
- build-base
image: idg_base:${CI_COMMIT_SHORT_SHA}
script:
- mkdir build
- cd build
- cmake -DBUILD_WITH_PYTHON=ON ..
- make install -j4
idg-release:
stage: build
dependencies:
- build-base
image: idg_base:${CI_COMMIT_SHORT_SHA}
script:
- mkdir build
- cd build
......@@ -22,6 +46,10 @@ idg-release:
- make install -j4
idg-debug:
stage: build
dependencies:
- build-base
image: idg_base:${CI_COMMIT_SHORT_SHA}
script:
- mkdir build
- cd build
......@@ -29,7 +57,12 @@ idg-debug:
- make install -j4
idg-test:
stage: build
dependencies:
- build-base
image: idg_base:${CI_COMMIT_SHORT_SHA}
script:
# TODO: export probably not needed
- export PATH="${PATH}:/home/gitlab-runner/.local/bin"
- pip3 install gcovr
- mkdir build
......@@ -37,10 +70,46 @@ idg-test:
- cmake .. -DCMAKE_CXX_FLAGS="-coverage" -DCMAKE_EXE_LINKER_FLAGS="-coverage" -DBUILD_WITH_TESTS=On
- make install -j4
- export LD_LIBRARY_PATH=$(pwd)/lib:$LD_LIBRARY_PATH
- ctest -j8 --verbose
- ctest -j8 --verbose -LE integration
# Capture coverage
- gcovr -r .. -e '.*/tests/.*' -e '.*/CompilerIdCXX/.*' -e '.*/external/.*'
- gcovr -r .. -e '.*/tests/.*' -e '.*/CompilerIdCXX/.*' -e '.*/external/.*' --xml > coverage.xml
artifacts:
reports:
cobertura: build/coverage.xml
idg-integration:
stage: integration_and_deploy
image: idg_integration:${CI_COMMIT_SHORT_SHA}
dependencies:
- build-integration
before_script:
- export HOME_DIR=$PWD
# Install IDG
- mkdir /opt/idg && mkdir build
- cd build
- cmake -DCMAKE_BUILD_TYPE=Release -DWRITE_OUT_SCALAR_BEAM=ON -DBUILD_WITH_PYTHON=ON -DCMAKE_INSTALL_PREFIX=/opt/idg ..
# - cmake -DCMAKE_INSTALL_PREFIX=/opt/idg ..
- make install -j4
- export LD_LIBRARY_PATH="/opt/idg/lib:$LD_LIBRARY_PATH"
# Compile against development branch of wsclean
- mkdir /wsclean && cd /wsclean && git clone https://gitlab.com/aroffringa/wsclean.git src
- cd src/ && git checkout development && cd ..
- mkdir build && cd build
- cmake -DCMAKE_PREFIX_PATH=/opt/idg -DCMAKE_INSTALL_PREFIX=/usr ../src
- make install -j4
- cd $HOME_DIR && rm -rf /wsclean
# Install dp3
- mkdir /dp3 && cd /dp3 && git clone https://github.com/lofar-astron/DP3.git src
- cd src/ && git checkout development && cd ..
- mkdir build && cd build
- cmake -DCMAKE_INSTALL_PREFIX=/usr ../src
- make install -j4
- cd $HOME_DIR && rm -rf /dp3
script:
# Return to idg build directory to run tests
- cd build
# Run in verbose mode -VV as long as it's not properly working
- ctest -VV -L integration
cmake_minimum_required(VERSION 2.8)
project(idg)
option (WRITE_OUT_SCALAR_BEAM "Write scalar beam to scalar_beam.npy" OFF)
# Set rpath to install directory
SET(CMAKE_INSTALL_RPATH ${CMAKE_INSTALL_PREFIX}/lib)
......@@ -48,3 +50,6 @@ install(
DESTINATION
${CMAKE_INSTALL_PREFIX}/share/
)
# Add integration tests
add_subdirectory(test/integration)
FROM ubuntu:20.04
RUN export DEBIAN_FRONTEND=noninteractive && apt-get update && \
apt-get install -y \
g++ cmake libfftw3-dev liblapacke-dev python3 python3-pip
\ No newline at end of file
FROM ubuntu:20.04
RUN export DEBIAN_FRONTEND=noninteractive && apt-get update && \
apt-get install -y \
g++ cmake libfftw3-dev liblapack-dev libblas-dev python3 python3-pip \
libboost-python-dev libcfitsio-dev python3-numpy wget \
build-essential libhdf5-dev libarmadillo-dev libgsl-dev \
libboost-filesystem-dev libboost-system-dev libboost-date-time-dev \
libboost-program-options-dev libboost-test-dev \
libxml2-dev libpng-dev pkg-config \
libgtkmm-3.0-dev git libfftw3-dev \
gfortran flex bison wcslib-dev \
libboost-numpy-dev liblua5.3-dev \
casacore-dev casacore-tools pybind11-dev \
# Build aoflagger3
&& mkdir /aoflagger && cd /aoflagger \
&& git clone https://gitlab.com/aroffringa/aoflagger.git src \
&& mkdir build && cd build \
&& cmake -DCMAKE_INSTALL_PREFIX=/usr ../src && make install -j4 \
&& cd / && rm -rf aoflagger \
# Build everybeam, checked-out at master (date: 2020/09/11)
&& mkdir /everybeam && cd /everybeam && git clone https://git.astron.nl/RD/EveryBeam.git src \
&& cd src/ && git checkout 5f3669a06f1b1a96155a929c6355816867333ff1 && cd .. \
&& mkdir build && cd build \
&& cmake -DCMAKE_INSTALL_PREFIX=/usr ../src -DPYTHON_EXECUTABLE=/usr/bin/python3 \
&& make install -j4 \
&& cd / && rm -rf everybeam \
# Build DP3, checked out at development (data: 2020/09/11)
# Install WSRT Measures (extra casacore data, for integration tests)
# Note: The file on the ftp site is updated daily. When warnings regarding leap
# seconds appear, ignore them or regenerate the docker image.
&& wget -O /WSRT_Measures.ztar ftp://ftp.astron.nl/outgoing/Measures/WSRT_Measures.ztar \
&& cd /var/lib/casacore/data \
&& tar xfz /WSRT_Measures.ztar \
&& rm /WSRT_Measures.ztar \
# Install some python packages
&& pip3 install astropy scipy
......@@ -847,7 +847,7 @@ namespace api {
}
}
#ifndef NDEBUG
#if !defined(NDEBUG) || defined(WRITE_OUT_SCALAR_BEAM)
{
const long unsigned leshape [] = {m_size, m_size};
npy::SaveArrayAsNumpy("scalar_beam.npy", false, 2, leshape, *m_scalar_beam);
......
......@@ -14,6 +14,7 @@
#cmakedefine BUILD_WITH_PYTHON
#cmakedefine PERFORMANCE_REPORT
#cmakedefine COMPILE_VERBOSE
#cmakedefine WRITE_OUT_SCALAR_BEAM
#cmakedefine CLANG_CXX_COMPILER
#cmakedefine GNU_CXX_COMPILER
......
#!/bin/bash
#
# Author: Jakob Maljaars
# Email: jakob.maljaars_@_stcorp.nl
# Script for downloading a mock LOFAR Measurement set
set -e
SCRIPT_PATH="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
cd $SCRIPT_PATH
# Move up to parent folder containing the source
cd ..
mkdir -p test/integration/tmp/data
cd test/integration/tmp/data
LOFAR_MOCK_ARCHIVE=LOFAR_ARCHIVE.tar.bz2
LOFAR_MOCK_MS=LOFAR_MOCK.ms
if [ ! -f "$LOFAR_MOCK_ARCHIVE" ]; then
wget -q https://www.astron.nl/citt/EveryBeam/L258627-150-timesteps.tar.bz2 -O $LOFAR_MOCK_ARCHIVE
fi
if [ -d $LOFAR_MOCK_MS ]
then
echo "Directory already exists"
else
mkdir $LOFAR_MOCK_MS
fi
tar -xf $LOFAR_MOCK_ARCHIVE -C $LOFAR_MOCK_MS --strip-components=1
rm $LOFAR_MOCK_ARCHIVE
\ No newline at end of file
# List of integration tests
set(INTEGRATION_TESTS
tSinglePointSource
tGridDegrid
)
# Add integration tests to "integration", can be invoked with
# "ctest -L integration"
foreach(TEST ${INTEGRATION_TESTS})
add_test(${TEST} ${CMAKE_CURRENT_SOURCE_DIR}/${TEST}.sh)
set_tests_properties(${TEST} PROPERTIES LABELS "integration")
endforeach()
\ No newline at end of file
This diff is collapsed.
msin =
msout=
msout.datacolumn = DATA
steps = [predict, applybeam]
predict.type=predict
predict.sourcedb=model.sourcedb
applybeam.type=applybeam
applybeam.invert=true
msin =
msout=
msout.datacolumn = DATA
steps = [predict]
predict.type=predict
predict.sourcedb=model.sourcedb
This diff is collapsed.
#!/usr/bin/env python3
import os
from astropy.io import fits
from subprocess import call, check_call
# Extract some environment variables
source_dir = os.path.dirname(os.path.realpath(__file__))
datadir = os.environ["DATADIR"]
msname = os.environ["MSNAME"]
# Write template fits image
ms = os.path.join(datadir, msname)
check_call(
[
"wsclean",
"-size",
"1000",
"1000",
"-scale",
"1asec",
"-interval",
"0",
"1",
"-no-reorder",
ms,
]
)
template = os.path.join(datadir, "template.fits")
check_call(["mv", "wsclean-image.fits", template])
with fits.open(template) as img:
N = img[0].data.shape[-1]
img[0].data[:] = 0.0
for stokes1 in ["I", "Q", "U", "V"]:
for stokes2 in ["I", "Q", "U", "V"]:
if stokes1 != stokes2:
img.writeto(
f"pointsource-{stokes1}-{stokes2}-model.fits", overwrite=True,
)
img[0].data[0, 0, int(N / 2), int(N / 2)] = 1.0
for stokes in ["I", "Q", "U", "V"]:
img.writeto(f"pointsource-{stokes}-{stokes}-model.fits", overwrite=True)
# Convert the non-zero (center) pixel to ra-dec coordinates and write cat file
check_call(["casapy2bbs.py", "pointsource-I-I-model.fits", "pointsource.cat"])
stokes = ["I", "Q", "U", "V"]
stokes1 = "I"
stokes1_idx = 5 + stokes.index(stokes1)
# This loop takes pointsource.cat and:
# - copies/writes its contents into pointsource-[stokes2].cat files (only line 8 is modified)
# - writes the sourcedb directories based on the .cat files
# TODO: once the integration test is up-and-running, this loop probably can
# be cleaned up a bit.
for stokes2 in stokes:
stokes2_idx = 5 + stokes.index(stokes2)
file_in = "pointsource.cat"
file_out = f"pointsource-{stokes2}.cat"
with open(file_in) as f_in, open(file_out, "w") as f_out:
for cnt, line in enumerate(f_in):
line = line[:-1]
s = line.split(",")
if cnt > 6 and len(s) == 9:
s[stokes1_idx], s[stokes2_idx] = "0.0", s[stokes1_idx]
line = ",".join(s)
print(line)
f_out.write(line + "\n")
sourcedb = f"pointsource-{stokes2}.sourcedb"
check_call(["rm", "-rf", sourcedb])
check_call(
[
"makesourcedb",
f"in={file_out}",
"format=Name, Type, Patch, Ra, Dec, I, Q, U, V",
f"out={sourcedb}",
]
)
exit(0)
#!/usr/bin/env python3
import os
from subprocess import check_call, call
import casacore.tables
import numpy as np
import sys
# Get stokes component from argument list
stokes = sys.argv[1]
# Get some environment variables
common_dir = os.environ["COMMON"]
datadir = os.environ["DATADIR"]
msname = os.environ["MSNAME"]
sourcedb = f"pointsource-{stokes}.sourcedb"
ms = os.path.join(datadir, msname)
T = casacore.tables.taql(
"SELECT TIME, cdatetime(TIME-.1) AS TIMESTR FROM $ms GROUPBY TIME"
)
interval_start = 0
interval_end = 100
if interval_end >= len(T):
print(
f"Interval end set to {interval_end}, but test measurment set only contains {len(T)} timesteps"
)
exit(1)
starttime = T[interval_start]["TIME"] - 0.1
endtime = T[interval_end]["TIME"] - 0.1
starttimestr = T[interval_start]["TIMESTR"]
endtimestr = T[interval_end]["TIMESTR"]
check_call(
[
"DPPP",
os.path.join(common_dir, "dppp-predict.parset"),
"msin=" + ms,
"msin.starttime=" + starttimestr,
"msin.endtime=" + endtimestr,
"predict.sourcedb=" + sourcedb,
]
)
check_call(
[
"wsclean",
"-name",
f"pointsource-{stokes}",
"-predict",
"-interval",
"0",
"100",
"-pol",
"IQUV",
"-use-idg",
"-idg-mode",
"cpu",
"-no-reorder",
ms,
]
)
t = casacore.tables.taql("SELECT * FROM $ms WHERE TIME>$starttime AND TIME<$endtime")
data = t.getcol("DATA") # generated by DPPP
model_data = t.getcol("MODEL_DATA") # generated by wsclean
print(f"Norm of DATA column: {np.linalg.norm(data)}")
print(f"Norm of MODEL_DATA column: {np.linalg.norm(model_data)}")
err = np.amax(abs(model_data - data))
if err < 1e-4:
exit(0)
else:
print(f"max error {err} (inf norm) exceeds specified threshold of 1e-4")
exit(1)
#!/bin/bash
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
# Set some environment variables
export DATADIR=${DIR}/tmp/data
export WORKDIR=${DIR}/tmp/workdir
export MSNAME="LOFAR_MOCK.ms"
export PATH="$PATH:${DIR}/common"
export COMMON=${DIR}/common
# Download measurement set into test/tmp/data directory (if needed)
cd $DIR
if [ -d $DATADIR/LOFAR_MOCK.ms ]
then
echo "LOFAR_MOCK.ms already exists"
else
./../../scripts/download_lofar_ms.sh
fi
mkdir -p $WORKDIR
cd $WORKDIR
${DIR}/gridding/test_gridding.py -v
#!/bin/bash
#
# Integration test for single point source, centered at the image center
#
# Make sure that:
# - casacore/lib
# - aoflagger/lib
# - (boost/lib)
# - idg/lib
# are on your $LD_LIBRARY_PATH!
# Get full path to this script
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
# Set some environment variables
export DATADIR=${DIR}/tmp/data
export WORKDIR=${DIR}/tmp/workdir
export MSNAME="LOFAR_MOCK.ms"
export PATH="$PATH:${DIR}/common"
export COMMON=${DIR}/common
function testrc {
if [ $1 -eq 0 ]; then
echo -e "\033[32mPassed\e[0m"
else
# Remove tmp directory and exit
rm -rf ${DIR}/tmp
echo -e "\033[31mFailed\e[0m"
exit 1
fi
}
function printtestname {
echo ============================ >> log.txt
echo -n $1 | tee -a log.txt
echo -n "................"
echo -e "\n============================" >> log.txt
}
function printpreparetestname {
echo ============================ >> log.txt
echo $1 >> log.txt
echo -e "============================" >> log.txt
}
# Download measurement set into test/tmp/data directory (if needed)
cd $DIR
# ./../../scripts/download_lofar_ms.sh
if [ -d $DATADIR/LOFAR_MOCK.ms ]
then
echo "LOFAR_MOCK.ms already exists"
else
./../../scripts/download_lofar_ms.sh
fi
mkdir -p $WORKDIR
cd $WORKDIR
# Prepare the testset
printpreparetestname prepare-testset-1
${DIR}/singlepointsource/preparetestset.py
printtestname "test degridding pointsource in center, stokes I"
${DIR}/singlepointsource/test_pointsource.py I
testrc $?
printtestname "test degridding pointsource in center, stokes Q"
${DIR}/singlepointsource/test_pointsource.py Q
testrc $?
printtestname "test degridding pointsource in center, stokes U"
${DIR}/singlepointsource/test_pointsource.py U
testrc $?
printtestname "test degridding pointsource in center, stokes V"
${DIR}/singlepointsource/test_pointsource.py V
testrc $?
# Remove tmp directory
# rm -rf ${DIR}/tmp
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment