Skip to content
Snippets Groups Projects
Commit dd3cc731 authored by Wouter Klijn's avatar Wouter Klijn
Browse files

Task #3670: Add backup of the current working jenkins build configs (the config.xml)

parent 6bf61284
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
Showing
with 697 additions and 0 deletions
.gitattributes
+ 4
− 0
View file @ dd3cc731
...@@ -1321,6 +1321,10 @@ CEP/Pipeline/test/recipes/nodes/imager_create_dbs_test.py eol=lf ...@@ -1321,6 +1321,10 @@ CEP/Pipeline/test/recipes/nodes/imager_create_dbs_test.py eol=lf
CEP/Pipeline/test/recipes/nodes/imager_prepare_test.py eol=lf CEP/Pipeline/test/recipes/nodes/imager_prepare_test.py eol=lf
CEP/Pipeline/test/regression_tests/calibrator_pipeline.py -text CEP/Pipeline/test/regression_tests/calibrator_pipeline.py -text
CEP/Pipeline/test/regression_tests/imaging_pipeline.py -text CEP/Pipeline/test/regression_tests/imaging_pipeline.py -text
CEP/Pipeline/test/regression_tests/jenkins_config/calibrator_regression.config.xml -text
CEP/Pipeline/test/regression_tests/jenkins_config/imaging_regression.config.xml -text
CEP/Pipeline/test/regression_tests/jenkins_config/pipeline_task_regression.config.xml -text
CEP/Pipeline/test/regression_tests/jenkins_config/target_regression.config.xml -text
CEP/Pipeline/test/regression_tests/target_pipeline.py -text CEP/Pipeline/test/regression_tests/target_pipeline.py -text
CEP/Pipeline/test/support/__init__.py eol=lf CEP/Pipeline/test/support/__init__.py eol=lf
CEP/Pipeline/test/support/pipelinexml_standalone.py -text CEP/Pipeline/test/support/pipelinexml_standalone.py -text
......
CEP/Pipeline/test/regression_tests/jenkins_config/calibrator_regression.config.xml 0 → 100644
+ 176
− 0
View file @ dd3cc731
<?xml version="1.0" encoding="UTF-8"?>
<project>
<actions/>
<description>Regression/delta test for the LOFAR calibration pipeline.
Triggers after a build on changes in the trunk at 2:00 AM</description>
<logRotator>
<daysToKeep>-1</daysToKeep>
<numToKeep>15</numToKeep>
<artifactDaysToKeep>-1</artifactDaysToKeep>
<artifactNumToKeep>-1</artifactNumToKeep>
</logRotator>
<keepDependencies>false</keepDependencies>
<properties>
<hudson.plugins.redmine.RedmineProjectProperty>
<redmineWebsite>https://support.astron.nl/lofar_issuetracker/</redmineWebsite>
<projectName>lofarsys</projectName>
<redmineVersion>true</redmineVersion>
</hudson.plugins.redmine.RedmineProjectProperty>
</properties>
<scm class="hudson.scm.SubversionSCM">
<locations>
<hudson.scm.SubversionSCM_-ModuleLocation>
<remote>https://svn.astron.nl/LOFAR/trunk/CEP</remote>
<local>LOFAR/CEP</local>
</hudson.scm.SubversionSCM_-ModuleLocation>
<hudson.scm.SubversionSCM_-ModuleLocation>
<remote>https://svn.astron.nl/LOFAR/trunk/CMake</remote>
<local>LOFAR/CMake</local>
</hudson.scm.SubversionSCM_-ModuleLocation>
</locations>
<browser class="hudson.plugins.viewVC.ViewVCRepositoryBrowser">
<url>https://svn.astron.nl/</url>
<location>LOFAR</location>
</browser>
<excludedRegions/>
<includedRegions/>
<excludedUsers/>
<excludedRevprop/>
<excludedCommitMessages/>
<workspaceUpdater class="hudson.scm.subversion.UpdateUpdater"/>
</scm>
<assignedNode>LCE072</assignedNode>
<canRoam>false</canRoam>
<disabled>false</disabled>
<blockBuildWhenDownstreamBuilding>false</blockBuildWhenDownstreamBuilding>
<blockBuildWhenUpstreamBuilding>false</blockBuildWhenUpstreamBuilding>
<jdk>(Default)</jdk>
<triggers class="vector">
<hudson.triggers.SCMTrigger>
<spec>0 2 * * *</spec>
</hudson.triggers.SCMTrigger>
</triggers>
<concurrentBuild>true</concurrentBuild>
<customWorkspace>/home/lofarbuild/jenkins_builds/msss_calibrator</customWorkspace>
<builders>
<hudson.plugins.cmake.CmakeBuilder>
<sourceDir>LOFAR</sourceDir>
<buildDir>gnu_debug</buildDir>
<installDir>install</installDir>
<buildType>Debug</buildType>
<otherBuildType/>
<generator>Unix Makefiles</generator>
<makeCommand>make -j8</makeCommand>
<installCommand>make install -j8</installCommand>
<preloadScript/>
<cmakeArgs>-Wdev -DBUILD_ASKAPsoft=OFF -DUSE_OPENMP=ON -DBUILD_PACKAGES=Offline </cmakeArgs>
<projectCmakePath/>
<cleanBuild>false</cleanBuild>
<cleanInstallDir>false</cleanInstallDir>
<builderImpl/>
</hudson.plugins.cmake.CmakeBuilder>
<hudson.tasks.Shell>
<command># This command will perform the actual regression test steps
# 1. A unit test
ctest -R pipeline
# 2. set environment
. /opt/cep/login/bashrc
use LofIm
export PYTHONPATH=$WORKSPACE/install/lib/python2.6/dist-packages:$PYTHONPATH
# 3. run the pipeline
#remove old state file
rm -f $WORKSPACE/install/var/run/pipeline/jobs/Observation64405/statefile
# remove old data products
ssh lce068 "rm -rf /data/scratch/lofarbuild/Observation64405/*"
ssh lce069 "rm -rf /data/scratch/lofarbuild/Observation64405/*"
mkdir -p $WORKSPACE/install/var/run/pipeline
#copy the config file to a 'writable' location
cp $WORKSPACE/install/share/pipeline/pipeline.cfg $WORKSPACE/install/var/run/pipeline/pipeline_copy.cfg
# replace clusterdesc file in the cfg file: allows running in lce072
sed -i 's/cep2.clusterdesc/cep1_test.clusterdesc/g' $WORKSPACE/install/var/run/pipeline/pipeline_copy.cfg
cd $WORKSPACE/install/bin
# Copy the parsetfile from a local to a global location
mkdir -p $WORKSPACE/parset_files
cp /data/lofar/testdata/CEP/Pipeline/calibrator_pipeline/Observation64405 $WORKSPACE/parset_files/Observation64405
# Run the calibrator with adapted cfg file
python msss_calibrator_pipeline.py $WORKSPACE/parset_files/Observation64405 -c $WORKSPACE/install/var/run/pipeline/pipeline_copy.cfg -d
# 4. Test correct functioning
# a. copy target image data to the scratch directory
rm -rf /data/scratch/lofarbuild/pipeline_regression_test/calibrator
mkdir -p /data/scratch/lofarbuild/pipeline_regression_test/calibrator
# copy target data
scp -r /data/lofar/testdata/CEP/Pipeline/calibrator_pipeline/datasets/*.INST /data/scratch/lofarbuild/pipeline_regression_test/calibrator
# copy the run results
scp -r lce068:/data/scratch/lofarbuild/calibrator_test/L64405_SAP000_SB000_inst.INST /data/scratch/lofarbuild/pipeline_regression_test/calibrator/L64405_SAP000_SB000_inst.INST.result
scp -r lce069:/data/scratch/lofarbuild/calibrator_test/L64405_SAP000_SB001_inst.INST /data/scratch/lofarbuild/pipeline_regression_test/calibrator/L64405_SAP000_SB001_inst.INST.result
# c. Do actual comparison
#cd /data/scratch/lofarbuild/pipeline
# first sb
python $WORKSPACE/LOFAR/CEP/Pipeline/test/regression_tests/calibrator_pipeline.py /data/scratch/lofarbuild/pipeline_regression_test/calibrator/L64405_SAP000_SB000_inst.INST /data/scratch/lofarbuild/pipeline_regression_test/calibrator/L64405_SAP000_SB000_inst.INST.result 0.0001
#second subband
python $WORKSPACE/LOFAR/CEP/Pipeline/test/regression_tests/calibrator_pipeline.py /data/scratch/lofarbuild/pipeline_regression_test/calibrator/L64405_SAP000_SB001_inst.INST /data/scratch/lofarbuild/pipeline_regression_test/calibrator/L64405_SAP000_SB001_inst.INST.result 0.0001
# 5 remove the pipeline products
# On failure of the test this delete step is not performed and the data can be inspected
# ***************** DANGER: HERE BE DRAGONS!!*************
#ssh lce069 "rm -rf /data/scratch/lofarbuild/out/*"
#rm /home/lofarbuild/jenkins_builds/install/var/run/pipeline/jobs/out/statefile
# ***************** DANGER: HERE BE DRAGONS!!*************
</command>
</hudson.tasks.Shell>
</builders>
<publishers>
<hudson.plugins.warnings.WarningsPublisher>
<healthy/>
<unHealthy/>
<thresholdLimit>low</thresholdLimit>
<pluginName>[WARNINGS] </pluginName>
<defaultEncoding/>
<canRunOnFailed>true</canRunOnFailed>
<useDeltaValues>false</useDeltaValues>
<thresholds>
<unstableTotalAll/>
<unstableTotalHigh/>
<unstableTotalNormal/>
<unstableTotalLow/>
<unstableNewAll/>
<unstableNewHigh/>
<unstableNewNormal/>
<unstableNewLow/>
<failedTotalAll/>
<failedTotalHigh/>
<failedTotalNormal/>
<failedTotalLow/>
<failedNewAll/>
<failedNewHigh/>
<failedNewNormal/>
<failedNewLow/>
</thresholds>
<shouldDetectModules>false</shouldDetectModules>
<dontComputeNew>false</dontComputeNew>
<parserConfigurations/>
<consoleLogParsers>
<string>GNU compiler 4 (ld)</string>
<string>GNU compiler 4 (gcc)</string>
</consoleLogParsers>
</hudson.plugins.warnings.WarningsPublisher>
</publishers>
<buildWrappers>
<hudson.plugins.timestamper.TimestamperBuildWrapper/>
</buildWrappers>
</project>
\ No newline at end of file
CEP/Pipeline/test/regression_tests/jenkins_config/imaging_regression.config.xml 0 → 100644
+ 162
− 0
View file @ dd3cc731
<?xml version="1.0" encoding="UTF-8"?>
<project>
<actions/>
<description>Regression/delta test for the LOFAR imaging pipeline.
Triggers a build and test at midnight on changes in the trunk.</description>
<logRotator>
<daysToKeep>-1</daysToKeep>
<numToKeep>15</numToKeep>
<artifactDaysToKeep>-1</artifactDaysToKeep>
<artifactNumToKeep>-1</artifactNumToKeep>
</logRotator>
<keepDependencies>false</keepDependencies>
<properties>
<hudson.plugins.redmine.RedmineProjectProperty>
<redmineWebsite>https://support.astron.nl/lofar_issuetracker/</redmineWebsite>
<projectName>lofarsys</projectName>
<redmineVersion>true</redmineVersion>
</hudson.plugins.redmine.RedmineProjectProperty>
</properties>
<scm class="hudson.scm.SubversionSCM">
<locations>
<hudson.scm.SubversionSCM_-ModuleLocation>
<remote>https://svn.astron.nl/LOFAR/trunk/CEP</remote>
<local>LOFAR/CEP</local>
</hudson.scm.SubversionSCM_-ModuleLocation>
<hudson.scm.SubversionSCM_-ModuleLocation>
<remote>https://svn.astron.nl/LOFAR/trunk/CMake</remote>
<local>LOFAR/CMake</local>
</hudson.scm.SubversionSCM_-ModuleLocation>
</locations>
<browser class="hudson.plugins.viewVC.ViewVCRepositoryBrowser">
<url>https://svn.astron.nl/</url>
<location>LOFAR</location>
</browser>
<excludedRegions/>
<includedRegions/>
<excludedUsers/>
<excludedRevprop/>
<excludedCommitMessages/>
<workspaceUpdater class="hudson.scm.subversion.UpdateUpdater"/>
</scm>
<assignedNode>LCE072</assignedNode>
<canRoam>false</canRoam>
<disabled>false</disabled>
<blockBuildWhenDownstreamBuilding>false</blockBuildWhenDownstreamBuilding>
<blockBuildWhenUpstreamBuilding>false</blockBuildWhenUpstreamBuilding>
<jdk>(Default)</jdk>
<triggers class="vector">
<hudson.triggers.SCMTrigger>
<spec>0 5 * * *</spec>
</hudson.triggers.SCMTrigger>
</triggers>
<concurrentBuild>true</concurrentBuild>
<customWorkspace>/home/lofarbuild/jenkins_builds</customWorkspace>
<builders>
<hudson.tasks.Shell>
<command># Perform svn steps needed before building
if [ -f $WORKSPACE/LOFAR/CMakeLists.txt ]
then echo "Found a previous checkout: updating"; svn up --depth files LOFAR
else echo "not found a previous checkout: checkingout"; svn checkout --depth files https://svn.astron.nl/LOFAR/trunk LOFAR
fi</command>
</hudson.tasks.Shell>
<hudson.plugins.cmake.CmakeBuilder>
<sourceDir>LOFAR</sourceDir>
<buildDir>gnu_debug</buildDir>
<installDir>install</installDir>
<buildType>Debug</buildType>
<otherBuildType/>
<generator>Unix Makefiles</generator>
<makeCommand>make -j8</makeCommand>
<installCommand>make install</installCommand>
<preloadScript/>
<cmakeArgs>-Wdev -DBUILD_ASKAPsoft=OFF -DUSE_OPENMP=ON -DBUILD_PACKAGES=Offline </cmakeArgs>
<projectCmakePath/>
<cleanBuild>false</cleanBuild>
<cleanInstallDir>false</cleanInstallDir>
<builderImpl/>
</hudson.plugins.cmake.CmakeBuilder>
<hudson.tasks.Shell>
<command># This command will perform the actual regression test steps
# 1. A unit test
ctest -R pipeline
# 2. set environment
. /opt/cep/login/bashrc
use LofIm
use Pythonlibs
export PYTHONPATH=$WORKSPACE/install/lib/python2.6/dist-packages:$PYTHONPATH
# 3. run the pipeline
#remove old state file
rm -f $WORKSPACE/install/var/run/pipeline/jobs/Observation64405/statefile
ssh lce069 "rm -rf /data/scratch/lofarbuild/out/*"
mkdir -p $WORKSPACE/install/var/run/pipeline
cd $WORKSPACE/install/bin
python msss_imager_pipeline.py /data/lofar/testdata/CEP/Pipeline/imager_pipeline/basic_test/out.parset -c $WORKSPACE/install/share/pipeline/pipeline.cfg -d
# 4. Test correct functioning
# a. copy target image data to the scratch directory
scp -r /data/lofar/testdata/CEP/Pipeline/imager_pipeline/basic_test/image.restored /data/scratch/lofarbuild/pipeline/image.restored.target
scp -r lce069:/data/scratch/lofarbuild/out/awimage_cycle_0/image.restored /data/scratch/lofarbuild/pipeline/image.restored
# b. copy target sourcelist to the scratch directory
scp -r /data/lofar/testdata/CEP/Pipeline/imager_pipeline/basic_test/bdsm_catalog /data/scratch/lofarbuild/pipeline/bdsm_catalog.target
scp -r lce069:/data/scratch/lofarbuild/out/awimage_cycle_0/bdsm_catalog /data/scratch/lofarbuild/pipeline/bdsm_catalog
# c. Do actual comparison
cd /data/scratch/lofarbuild/pipeline
python $WORKSPACE/LOFAR/CEP/Pipeline/test/regression_tests/imaging_pipeline.py bdsm_catalog bdsm_catalog.target image.restored image.restored.target 0.0002
# 5 remove the pipeline products
# On failure of the test this delete step is not performed and the data can be inspected
# ***************** DANGER: HERE BE DRAGONS!!*************
ssh lce069 "rm -rf /data/scratch/lofarbuild/out/*"
rm /home/lofarbuild/jenkins_builds/install/var/run/pipeline/jobs/out/statefile
# ***************** DANGER: HERE BE DRAGONS!!*************
</command>
</hudson.tasks.Shell>
</builders>
<publishers>
<hudson.plugins.warnings.WarningsPublisher>
<healthy/>
<unHealthy/>
<thresholdLimit>low</thresholdLimit>
<pluginName>[WARNINGS] </pluginName>
<defaultEncoding/>
<canRunOnFailed>true</canRunOnFailed>
<useDeltaValues>false</useDeltaValues>
<thresholds>
<unstableTotalAll/>
<unstableTotalHigh/>
<unstableTotalNormal/>
<unstableTotalLow/>
<unstableNewAll/>
<unstableNewHigh/>
<unstableNewNormal/>
<unstableNewLow/>
<failedTotalAll/>
<failedTotalHigh/>
<failedTotalNormal/>
<failedTotalLow/>
<failedNewAll/>
<failedNewHigh/>
<failedNewNormal/>
<failedNewLow/>
</thresholds>
<shouldDetectModules>false</shouldDetectModules>
<dontComputeNew>false</dontComputeNew>
<parserConfigurations/>
<consoleLogParsers>
<string>GNU compiler 4 (ld)</string>
<string>GNU compiler 4 (gcc)</string>
</consoleLogParsers>
</hudson.plugins.warnings.WarningsPublisher>
</publishers>
<buildWrappers>
<hudson.plugins.timestamper.TimestamperBuildWrapper/>
</buildWrappers>
</project>
\ No newline at end of file
CEP/Pipeline/test/regression_tests/jenkins_config/pipeline_task_regression.config.xml 0 → 100644
+ 179
− 0
View file @ dd3cc731
<?xml version="1.0" encoding="UTF-8"?>
<project>
<actions/>
<description>This is a jenkins build to test the imaging pipeline regressions test on task branches.
It performs a complete build from scratch. Builds the task branche and performs a simple imaging pipeline.
Produced data products are compared default data.
This build will fail if the branche produces output different to the baseline.
</description>
<logRotator>
<daysToKeep>-1</daysToKeep>
<numToKeep>15</numToKeep>
<artifactDaysToKeep>-1</artifactDaysToKeep>
<artifactNumToKeep>-1</artifactNumToKeep>
</logRotator>
<keepDependencies>false</keepDependencies>
<properties>
<hudson.plugins.redmine.RedmineProjectProperty>
<redmineWebsite>https://support.astron.nl/lofar_issuetracker/</redmineWebsite>
<projectName>lofarsys</projectName>
<redmineVersion>true</redmineVersion>
</hudson.plugins.redmine.RedmineProjectProperty>
<hudson.model.ParametersDefinitionProperty>
<parameterDefinitions>
<hudson.scm.listtagsparameter.ListSubversionTagsParameterDefinition>
<name>BRANCHE</name>
<description>Select a Subversion entry</description>
<tagsDir>https://svn.astron.nl/LOFAR/branches</tagsDir>
<tagsFilter/>
<reverseByDate>true</reverseByDate>
<reverseByName>false</reverseByName>
<defaultValue/>
<maxTags/>
<uuid>41b88e15-dbbe-4a3b-96b0-4a4b9949e906</uuid>
</hudson.scm.listtagsparameter.ListSubversionTagsParameterDefinition>
</parameterDefinitions>
</hudson.model.ParametersDefinitionProperty>
</properties>
<scm class="hudson.scm.SubversionSCM">
<locations>
<hudson.scm.SubversionSCM_-ModuleLocation>
<remote>https://svn.astron.nl/LOFAR/branches/$BRANCHE/CEP</remote>
<local>LOFAR/CEP</local>
</hudson.scm.SubversionSCM_-ModuleLocation>
<hudson.scm.SubversionSCM_-ModuleLocation>
<remote>https://svn.astron.nl/LOFAR/branches/$BRANCHE/CMake</remote>
<local>LOFAR/CMake</local>
</hudson.scm.SubversionSCM_-ModuleLocation>
</locations>
<browser class="hudson.plugins.viewVC.ViewVCRepositoryBrowser">
<url>https://svn.astron.nl/</url>
<location>LOFAR</location>
</browser>
<excludedRegions/>
<includedRegions/>
<excludedUsers/>
<excludedRevprop/>
<excludedCommitMessages/>
<workspaceUpdater class="hudson.scm.subversion.UpdateUpdater"/>
</scm>
<assignedNode>LCE072</assignedNode>
<canRoam>false</canRoam>
<disabled>false</disabled>
<blockBuildWhenDownstreamBuilding>false</blockBuildWhenDownstreamBuilding>
<blockBuildWhenUpstreamBuilding>false</blockBuildWhenUpstreamBuilding>
<jdk>(Default)</jdk>
<triggers class="vector"/>
<concurrentBuild>true</concurrentBuild>
<customWorkspace>/home/lofarbuild/jenkins_builds/imager_branche_test</customWorkspace>
<builders>
<hudson.tasks.Shell>
<command># Perform svn steps needed before building
echo "This is a test"
if [ -f $WORKSPACE/LOFAR/CMakeLists.txt ]
then echo "Found a previous checkout: updating"; svn up --depth files LOFAR
else echo "not found a previous checkout: checkingout"; svn checkout --depth files https://svn.astron.nl/LOFAR/branches/$BRANCHE LOFAR
fi
#svn checkout --depth files https://svn.astron.nl/LOFAR/branches/$BRANCHE LOFAR
</command>
</hudson.tasks.Shell>
<hudson.plugins.cmake.CmakeBuilder>
<sourceDir>LOFAR</sourceDir>
<buildDir>gnu_debug</buildDir>
<installDir>install</installDir>
<buildType>Debug</buildType>
<otherBuildType/>
<generator>Unix Makefiles</generator>
<makeCommand>make -j8</makeCommand>
<installCommand>make install</installCommand>
<preloadScript/>
<cmakeArgs>-Wdev -DBUILD_ASKAPsoft=OFF -DUSE_OPENMP=ON -DBUILD_PACKAGES=Offline </cmakeArgs>
<projectCmakePath/>
<cleanBuild>false</cleanBuild>
<cleanInstallDir>false</cleanInstallDir>
<builderImpl/>
</hudson.plugins.cmake.CmakeBuilder>
<hudson.tasks.Shell>
<command># This command will perform the actual regression test steps
# 1. A unit test
ctest -R pipeline
# ***************** DANGER: HERE BE DRAGONS!!*************
# assure existance of a single file in dir
ssh lce069 "touch /data/scratch/lofarbuild/$BRANCHE/file.txt"
ssh lce069 "rm -rf /data/scratch/lofarbuild/$BRANCHE/*"
rm -f $WORKSPACE/install/var/run/pipeline/jobs/$BRANCHE/statefile
# ***************** DANGER: HERE BE DRAGONS!!*************
# 2. set environment
. /opt/cep/login/bashrc
use LofIm
use Pythonlibs
export PYTHONPATH=$WORKSPACE/install/lib/python2.6/dist-packages:$PYTHONPATH
# 3. run the pipeline
mkdir -p $WORKSPACE/install/var/run/pipeline
cd $WORKSPACE/install/bin
python msss_imager_pipeline.py /data/lofar/testdata/CEP/Pipeline/imager_pipeline/basic_test/out.parset -c $WORKSPACE/install/share/pipeline/pipeline.cfg --job $BRANCHE -d
# 4. Test correct functioning
# a. copy target image data to the scratch directory
scp -r /data/lofar/testdata/CEP/Pipeline/imager_pipeline/basic_test/image.restored /data/scratch/lofarbuild/pipeline/image.restored.target
scp -r lce069:/data/scratch/lofarbuild/out/awimage_cycle_0/image.restored /data/scratch/lofarbuild/pipeline/image.restored
# b. copy target sourcelist to the scratch directory
scp -r /data/lofar/testdata/CEP/Pipeline/imager_pipeline/basic_test/bdsm_catalog /data/scratch/lofarbuild/pipeline/bdsm_catalog.target
scp -r lce069:/data/scratch/lofarbuild/$BRANCHE/awimage_cycle_0/bdsm_catalog /data/scratch/lofarbuild/pipeline/bdsm_catalog
# c. Do actual comparison
cd /data/scratch/lofarbuild/pipeline
python $WORKSPACE/LOFAR/CEP/Pipeline/test/regression_tests/imaging_pipeline.py bdsm_catalog bdsm_catalog.target image.restored image.restored.target 0.0002
</command>
</hudson.tasks.Shell>
</builders>
<publishers>
<hudson.plugins.warnings.WarningsPublisher>
<healthy/>
<unHealthy/>
<thresholdLimit>low</thresholdLimit>
<pluginName>[WARNINGS] </pluginName>
<defaultEncoding/>
<canRunOnFailed>true</canRunOnFailed>
<useDeltaValues>false</useDeltaValues>
<thresholds>
<unstableTotalAll/>
<unstableTotalHigh/>
<unstableTotalNormal/>
<unstableTotalLow/>
<unstableNewAll/>
<unstableNewHigh/>
<unstableNewNormal/>
<unstableNewLow/>
<failedTotalAll/>
<failedTotalHigh/>
<failedTotalNormal/>
<failedTotalLow/>
<failedNewAll/>
<failedNewHigh/>
<failedNewNormal/>
<failedNewLow/>
</thresholds>
<shouldDetectModules>false</shouldDetectModules>
<dontComputeNew>false</dontComputeNew>
<parserConfigurations/>
<consoleLogParsers>
<string>GNU compiler 4 (ld)</string>
<string>GNU compiler 4 (gcc)</string>
</consoleLogParsers>
</hudson.plugins.warnings.WarningsPublisher>
</publishers>
<buildWrappers>
<hudson.plugins.timestamper.TimestamperBuildWrapper/>
</buildWrappers>
</project>
\ No newline at end of file
CEP/Pipeline/test/regression_tests/jenkins_config/target_regression.config.xml 0 → 100644
+ 176
− 0
View file @ dd3cc731
<?xml version="1.0" encoding="UTF-8"?>
<project>
<actions/>
<description>Regression/delta test for the LOFAR target pipeline.
Triggers a build and test after midnight on changes in the trunk.</description>
<logRotator>
<daysToKeep>-1</daysToKeep>
<numToKeep>15</numToKeep>
<artifactDaysToKeep>-1</artifactDaysToKeep>
<artifactNumToKeep>-1</artifactNumToKeep>
</logRotator>
<keepDependencies>false</keepDependencies>
<properties>
<hudson.plugins.redmine.RedmineProjectProperty>
<redmineWebsite>https://support.astron.nl/lofar_issuetracker/</redmineWebsite>
<projectName>lofarsys</projectName>
<redmineVersion>true</redmineVersion>
</hudson.plugins.redmine.RedmineProjectProperty>
</properties>
<scm class="hudson.scm.SubversionSCM">
<locations>
<hudson.scm.SubversionSCM_-ModuleLocation>
<remote>https://svn.astron.nl/LOFAR/trunk/CEP</remote>
<local>LOFAR/CEP</local>
</hudson.scm.SubversionSCM_-ModuleLocation>
<hudson.scm.SubversionSCM_-ModuleLocation>
<remote>https://svn.astron.nl/LOFAR/trunk/CMake</remote>
<local>LOFAR/CMake</local>
</hudson.scm.SubversionSCM_-ModuleLocation>
</locations>
<browser class="hudson.plugins.viewVC.ViewVCRepositoryBrowser">
<url>https://svn.astron.nl/</url>
<location>LOFAR</location>
</browser>
<excludedRegions/>
<includedRegions/>
<excludedUsers/>
<excludedRevprop/>
<excludedCommitMessages/>
<workspaceUpdater class="hudson.scm.subversion.UpdateUpdater"/>
</scm>
<assignedNode>LCE072</assignedNode>
<canRoam>false</canRoam>
<disabled>false</disabled>
<blockBuildWhenDownstreamBuilding>false</blockBuildWhenDownstreamBuilding>
<blockBuildWhenUpstreamBuilding>false</blockBuildWhenUpstreamBuilding>
<jdk>(Default)</jdk>
<triggers class="vector">
<hudson.triggers.SCMTrigger>
<spec>0 4 * * *</spec>
</hudson.triggers.SCMTrigger>
</triggers>
<concurrentBuild>true</concurrentBuild>
<customWorkspace>/home/lofarbuild/jenkins_builds/target_regression</customWorkspace>
<builders>
<hudson.plugins.cmake.CmakeBuilder>
<sourceDir>LOFAR</sourceDir>
<buildDir>gnu_debug</buildDir>
<installDir>install</installDir>
<buildType>Debug</buildType>
<otherBuildType/>
<generator>Unix Makefiles</generator>
<makeCommand>make -j4</makeCommand>
<installCommand>make install</installCommand>
<preloadScript/>
<cmakeArgs>-Wdev -DBUILD_ASKAPsoft=OFF -DUSE_OPENMP=ON -DBUILD_PACKAGES=Offline </cmakeArgs>
<projectCmakePath/>
<cleanBuild>false</cleanBuild>
<cleanInstallDir>false</cleanInstallDir>
<builderImpl/>
</hudson.plugins.cmake.CmakeBuilder>
<hudson.tasks.Shell>
<command># This command will perform the actual regression test steps
# 1. A unit test
ctest -R pipeline
# 2. set environment
. /opt/cep/login/bashrc
use LofIm
export PYTHONPATH=$WORKSPACE/install/lib/python2.6/dist-packages:$PYTHONPATH
ssh lce068 "rm -rf /data/scratch/lofarbuild/Observation64406/*"
ssh lce069 "rm -rf /data/scratch/lofarbuild/Observation64406/*"
ssh lce068 "rm -rf /data/scratch/lofarbuild/target_integration/*"
ssh lce069 "rm -rf /data/scratch/lofarbuild/target_integration/*"
# copy the test data to the correct place on the node
ssh lce068 "mkdir -p /data/scratch/lofarbuild/target_integration"
scp -rp /data/lofar/testdata/CEP/Pipeline/target_pipeline/datasets/L64372_SAP000_SB000_uv.MS lce068:/data/scratch/lofarbuild/target_integration
scp -rp /data/lofar/testdata/CEP/Pipeline/target_pipeline/datasets/L64405_SAP000_SB000_inst.INST lce068:/data/scratch/lofarbuild/target_integration
ssh lce069 "mkdir -p /data/scratch/lofarbuild/target_integration"
scp -rp /data/lofar/testdata/CEP/Pipeline/target_pipeline/datasets/L64372_SAP000_SB001_uv.MS lce069:/data/scratch/lofarbuild/target_integration
scp -rp /data/lofar/testdata/CEP/Pipeline/target_pipeline/datasets/L64405_SAP000_SB001_inst.INST lce069:/data/scratch/lofarbuild/target_integration
# 3. run the pipeline
#remove old state file
rm -f $WORKSPACE/install/var/run/pipeline/jobs/Observation64406/statefile
mkdir -p $WORKSPACE/install/var/run/pipeline
#copy the config file to a 'writable' location
cp $WORKSPACE/install/share/pipeline/pipeline.cfg $WORKSPACE/install/var/run/pipeline/pipeline_copy.cfg
#copy the parset to a writable location
cp /data/lofar/testdata/CEP/Pipeline/target_pipeline/Observation64406 $WORKSPACE/install/var/run/pipeline/Observation64406
# replace clusterdesc file in the cfg file: allows running in lce072
sed -i 's/cep2.clusterdesc/cep1_test.clusterdesc/g' $WORKSPACE/install/var/run/pipeline/pipeline_copy.cfg
cd $WORKSPACE/install/bin
python msss_target_pipeline.py $WORKSPACE/install/var/run/pipeline/Observation64406 -c $WORKSPACE/install/var/run/pipeline/pipeline_copy.cfg -d
## 4. Test correct functioning
# a. copy target calibrated data to the scratch directory
mkdir -p /data/scratch/lofarbuild/target_regression_test
scp -r /data/lofar/testdata/CEP/Pipeline/target_pipeline/datasets/L64406_SB000_uv.dppp.MS_target /data/scratch/lofarbuild/target_regression_test/L64406_SB000_uv.dppp.MS_target
scp -r /data/lofar/testdata/CEP/Pipeline/target_pipeline/datasets/L64406_SB001_uv.dppp.MS_target /data/scratch/lofarbuild/target_regression_test/L64406_SB001_uv.dppp.MS_target
# b. copy target sourcelist to the scratch directory
scp -r lce068:/data/scratch/lofarbuild/target_integration/L64406_SB000_uv.dppp.MS /data/scratch/lofarbuild/target_regression_test/L64406_SB000_uv.dppp.MS
scp -r lce069:/data/scratch/lofarbuild/target_integration/L64406_SB001_uv.dppp.MS /data/scratch/lofarbuild/target_regression_test/L64406_SB001_uv.dppp.MS
# c. Do actual comparison
python $WORKSPACE/LOFAR/CEP/Pipeline/test/regression_tests/target_pipeline.py /data/scratch/lofarbuild/target_regression_test/L64406_SB000_uv.dppp.MS_target /data/scratch/lofarbuild/target_regression_test/L64406_SB000_uv.dppp.MS
python $WORKSPACE/LOFAR/CEP/Pipeline/test/regression_tests/target_pipeline.py /data/scratch/lofarbuild/target_regression_test/L64406_SB001_uv.dppp.MS_target /data/scratch/lofarbuild/target_regression_test/L64406_SB001_uv.dppp.MS
</command>
</hudson.tasks.Shell>
</builders>
<publishers>
<hudson.plugins.warnings.WarningsPublisher>
<healthy/>
<unHealthy/>
<thresholdLimit>low</thresholdLimit>
<pluginName>[WARNINGS] </pluginName>
<defaultEncoding/>
<canRunOnFailed>true</canRunOnFailed>
<useDeltaValues>false</useDeltaValues>
<thresholds>
<unstableTotalAll/>
<unstableTotalHigh/>
<unstableTotalNormal/>
<unstableTotalLow/>
<unstableNewAll/>
<unstableNewHigh/>
<unstableNewNormal/>
<unstableNewLow/>
<failedTotalAll/>
<failedTotalHigh/>
<failedTotalNormal/>
<failedTotalLow/>
<failedNewAll/>
<failedNewHigh/>
<failedNewNormal/>
<failedNewLow/>
</thresholds>
<shouldDetectModules>false</shouldDetectModules>
<dontComputeNew>false</dontComputeNew>
<parserConfigurations/>
<consoleLogParsers>
<string>GNU compiler 4 (ld)</string>
<string>GNU compiler 4 (gcc)</string>
</consoleLogParsers>
</hudson.plugins.warnings.WarningsPublisher>
</publishers>
<buildWrappers>
<hudson.plugins.timestamper.TimestamperBuildWrapper/>
</buildWrappers>
</project>
\ No newline at end of file
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment