diff --git a/CMakeLists.txt b/CMakeLists.txt index c17a0929f22c78e0d072dd46e5b64b712a2d0cd3..83eb93053e88064040f7e4146831397923308b53 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -942,7 +942,7 @@ if(BUILD_TESTING) configure_file(${CMAKE_SOURCE_DIR}/scripts/test/testconfig.py.in testconfig.py) - configure_file(${CMAKE_SOURCE_DIR}/scripts/test/utils.py.in utils.py) + configure_file(${CMAKE_SOURCE_DIR}/scripts/test/utils.py utils.py COPYONLY) # The 'source' symbolic link simplifies running the tests manually inside # ${CMAKE_CURRENT_BINARY_DIR}: It allows using 'source/tApplyBeam.sh' instead diff --git a/ddecal/test/integration/tBdaDdeCal.py b/ddecal/test/integration/tBdaDdeCal.py index b44c1a90e1afbf94e2a146037f2d2692c5a7902a..b49698ddd2a38759e7ee88ef36fd7ad34c297d74 100755 --- a/ddecal/test/integration/tBdaDdeCal.py +++ b/ddecal/test/integration/tBdaDdeCal.py @@ -2,9 +2,6 @@ # SPDX-License-Identifier: GPL-3.0-or-later import pytest -import os -import shutil -import uuid import numpy as np from subprocess import check_call, check_output, run, CalledProcessError @@ -14,7 +11,7 @@ import sys sys.path.append(".") import testconfig as tcf -from utils import assert_taql, untar_ms +from utils import assert_taql, run_in_tmp_path, untar """ Script can be invoked in two ways: @@ -25,26 +22,13 @@ Script can be invoked in two ways: MSIN = "tNDPPP-bda.MS" MSIN_REGULAR = "tNDPPP-generic.MS" -CWD = os.getcwd() CORRUPTIONS = 3, 4, 7 # Corruption gain factors per antenna @pytest.fixture(autouse=True) -def source_env(): - os.chdir(CWD) - tmpdir = str(uuid.uuid4()) - os.mkdir(tmpdir) - os.chdir(tmpdir) - - untar_ms(f"{tcf.RESOURCEDIR}/{MSIN}.tgz") - untar_ms(f"{tcf.RESOURCEDIR}/{MSIN_REGULAR}.tgz") - - # Tests are executed here - yield - - # Post-test: clean up - os.chdir(CWD) - shutil.rmtree(tmpdir) +def source_env(run_in_tmp_path): + untar(f"{tcf.RESOURCEDIR}/{MSIN}.tgz") + untar(f"{tcf.RESOURCEDIR}/{MSIN_REGULAR}.tgz") @pytest.fixture() diff --git a/ddecal/test/integration/tDDECal.py b/ddecal/test/integration/tDDECal.py index f1d9e876f712e68f02dc6d591be8cc729f5bdf23..00526d8e8a1f30eacfc918e45d5f342a884e7f06 100755 --- a/ddecal/test/integration/tDDECal.py +++ b/ddecal/test/integration/tDDECal.py @@ -2,9 +2,6 @@ # SPDX-License-Identifier: GPL-3.0-or-later import pytest -import os -import shutil -import uuid from subprocess import check_call, check_output, CalledProcessError, STDOUT import numpy as np @@ -15,7 +12,7 @@ sys.path.append(".") import testconfig as tcf from testconfig import TAQLEXE -from utils import assert_taql, untar_ms, get_taql_result +from utils import assert_taql, run_in_tmp_path, untar, get_taql_result """ Script can be invoked in two ways: @@ -27,31 +24,18 @@ Script can be invoked in two ways: IDG_RESOURCES = "idg-fits-sources.tbz2" MSINTGZ = "tDDECal.in_MS.tgz" MSIN = "tDDECal.MS" -CWD = os.getcwd() @pytest.fixture(autouse=True) -def source_env(): - os.chdir(CWD) - tmpdir = str(uuid.uuid4()) - os.mkdir(tmpdir) - os.chdir(tmpdir) - - untar_ms(f"{tcf.RESOURCEDIR}/{MSINTGZ}") +def source_env(run_in_tmp_path): + untar(f"{tcf.RESOURCEDIR}/{MSINTGZ}") check_call([tcf.MAKESOURCEDBEXE, f"in={MSIN}/sky.txt", f"out={MSIN}/sky"]) - # Tests are executed here - yield - - # Post-test: clean up - os.chdir(CWD) - shutil.rmtree(tmpdir) - @pytest.fixture() def idgpredict_env(): """Extract data for testing DDECal with IDGPredict""" - untar_ms(f"{tcf.DDECAL_RESOURCEDIR}/{IDG_RESOURCES}") + untar(f"{tcf.DDECAL_RESOURCEDIR}/{IDG_RESOURCES}") @pytest.fixture() diff --git a/ddecal/test/integration/tIDGPredict.py b/ddecal/test/integration/tIDGPredict.py index 563b7f0ff522ea56f282557f50b31f2dcc0b2c91..5d73fd792194dbe172c0b58fe4da3f831172bbd3 100755 --- a/ddecal/test/integration/tIDGPredict.py +++ b/ddecal/test/integration/tIDGPredict.py @@ -2,9 +2,6 @@ # SPDX-License-Identifier: GPL-3.0-or-later import pytest -import os -import shutil -import uuid from subprocess import check_call, check_output """ Append current directory to system path in order to import testconfig """ @@ -13,7 +10,7 @@ import sys sys.path.append(".") import testconfig as tcf -from utils import assert_taql, untar_ms +from utils import assert_taql, run_in_tmp_path, untar """ Script can be invoked in two ways: @@ -25,25 +22,12 @@ Script can be invoked in two ways: REF_SOLUTIONS = "idg-fits-sources.tbz2" MSINTGZ = "tDDECal.in_MS.tgz" MSIN = "tDDECal.MS" -CWD = os.getcwd() @pytest.fixture(autouse=True) -def source_env(): - os.chdir(CWD) - tmpdir = str(uuid.uuid4()) - os.mkdir(tmpdir) - os.chdir(tmpdir) - - untar_ms(f"{tcf.RESOURCEDIR}/{MSINTGZ}") - untar_ms(f"{tcf.DDECAL_RESOURCEDIR}/{REF_SOLUTIONS}") - - # Tests are executed here - yield - - # Post-test: clean up - os.chdir(CWD) - shutil.rmtree(tmpdir) +def source_env(run_in_tmp_path): + untar(f"{tcf.RESOURCEDIR}/{MSINTGZ}") + untar(f"{tcf.DDECAL_RESOURCEDIR}/{REF_SOLUTIONS}") def compare_results(source_name): diff --git a/parmdb/test/integration/tShowSourceDb.py b/parmdb/test/integration/tShowSourceDb.py index f1d856338229386346df3f6a1619dfb75fd823ea..83bc684a89b7fff1ef334ff5c0ee55df90f3100f 100644 --- a/parmdb/test/integration/tShowSourceDb.py +++ b/parmdb/test/integration/tShowSourceDb.py @@ -1,16 +1,12 @@ import pytest -import shutil -import os import sys -import uuid -from subprocess import check_call -from subprocess import check_output +from subprocess import check_call, check_output # Append current directory to system path in order to import testconfig sys.path.append(".") import testconfig as tcf -from utils import untar_ms +from utils import run_in_tmp_path, untar """ Tests for the showsourcedb tool @@ -22,24 +18,11 @@ Script can be invoked in two ways: """ MSIN = "tDemix.in_MS" -CWD = os.getcwd() @pytest.fixture(autouse=True) -def source_env(): - os.chdir(CWD) - tmpdir = str(uuid.uuid4()) - os.mkdir(tmpdir) - os.chdir(tmpdir) - - untar_ms(f"{tcf.RESOURCEDIR}/{MSIN}.tgz") - - # Tests are executed here - yield - - # Post-test: clean up - os.chdir(CWD) - shutil.rmtree(tmpdir) +def source_env(run_in_tmp_path): + untar(f"{tcf.RESOURCEDIR}/{MSIN}.tgz") def test_skymodel_sourcedb_roundtrip(): diff --git a/pythondp3/test/integration/tMakeMainSteps.py b/pythondp3/test/integration/tMakeMainSteps.py index 6aaf098caded62dfab4124b6f146f61730bf461e..b422f25373bd42763c38fe017e002858ee5eadde 100644 --- a/pythondp3/test/integration/tMakeMainSteps.py +++ b/pythondp3/test/integration/tMakeMainSteps.py @@ -2,7 +2,7 @@ # SPDX-License-Identifier: GPL-3.0-or-later """ -These tests are checking the python bindings of the DP3::makeMainSteps function. +These tests are checking the python bindings of the DP3::makeMainSteps function. Script can be invoked in two ways: - as standalone from the build/pythondp3/test/integration directory, @@ -21,7 +21,7 @@ import testconfig as tcf sys.path.insert(0, tcf.PYTHONDIR) -from utils import assert_taql, untar_ms +from utils import assert_taql, run_in_tmp_path, untar try: "The import may fail while running pytest --collect-only" @@ -30,34 +30,20 @@ except ImportError: pass MSIN = "tNDPPP-generic.MS" -CWD = os.getcwd() -@pytest.fixture(autouse=True) -def source_env(tmp_path): - os.chdir(tmp_path) - - untar_ms(f"{tcf.RESOURCEDIR}/{MSIN}.tgz") - - # Tests are executed here - yield - - # Post-test: clean up - os.chdir(CWD) - - -def test_make_main_steps(tmp_path): +def test_make_main_steps(run_in_tmp_path): """ This test creates an input, averaging and output step by calling the make_main_steps() factory function. """ - msout = str(tmp_path / "averaged_tmp.MS") + untar(f"{tcf.RESOURCEDIR}/{MSIN}.tgz") parset = dp3.parameterset.ParameterSet() parset.add("msin", MSIN) - parset.add("msout", msout) + parset.add("msout", "averaged_tmp.MS") parset.add("msout.overwrite", "true") parset.add("steps", "[average]") parset.add("average.timestep", "2") diff --git a/scripts/test/utils.py b/scripts/test/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..8a40af2626086450ed386917ccf1e726e0476f53 --- /dev/null +++ b/scripts/test/utils.py @@ -0,0 +1,42 @@ +# Copyright (C) 2021 ASTRON (Netherlands Institute for Radio Astronomy) +# SPDX-License-Identifier: GPL-3.0-or-later + +import os +import pytest +import shutil +import uuid +from subprocess import check_call, check_output +from testconfig import TAQLEXE + + +def assert_taql(command, expected_rows=0): + result = ( + check_output([TAQLEXE, "-noph", "-nopa", command]).decode().strip() + ) + assert result == f"select result of {expected_rows} rows" + + +def untar(source): + if not os.path.isfile(source): + raise IOError( + f"Not able to find {source} containing test input files." + ) + check_call(["tar", "xf", source]) + + +def get_taql_result(command): + """Get the output of a taql command""" + result = ( + check_output([TAQLEXE, "-noph", "-nopr", command]).decode().strip() + ) + return result + + +@pytest.fixture() +def run_in_tmp_path(tmp_path): + """ + Creates a temporary directory, runs the test in it, and removes the + directory. + """ + # 'tmp_path' is a base fixture from Pytest that already does everything else, including cleaning up. + os.chdir(tmp_path) diff --git a/scripts/test/utils.py.in b/scripts/test/utils.py.in deleted file mode 100644 index 8d35649c7c52bd2b05a6bd30279c5a5ea3c46a26..0000000000000000000000000000000000000000 --- a/scripts/test/utils.py.in +++ /dev/null @@ -1,22 +0,0 @@ -# Copyright (C) 2021 ASTRON (Netherlands Institute for Radio Astronomy) -# SPDX-License-Identifier: GPL-3.0-or-later - -import os -from subprocess import check_call, check_output -from testconfig import TAQLEXE - - -def assert_taql(command, expected_rows=0): - result = check_output([TAQLEXE, "-noph", "-nopa", command]).decode().strip() - assert result == f"select result of {expected_rows} rows" - - -def untar_ms(source): - if not os.path.isfile(source): - raise IOError(f"Not able to find {source} containing the reference solutions.") - check_call(["tar", "xf", source]) - -def get_taql_result(command): - """Get the output of a taql command""" - result = check_output([TAQLEXE, "-noph", "-nopr", command]).decode().strip() - return result diff --git a/steps/test/integration/tApplyBeam.py b/steps/test/integration/tApplyBeam.py index 278f339da1f4e59ccd49a809dd3b68514303f7ee..e47e5ba369261718a81afec9da3cdbdc40971861 100644 --- a/steps/test/integration/tApplyBeam.py +++ b/steps/test/integration/tApplyBeam.py @@ -2,10 +2,7 @@ # SPDX-License-Identifier: GPL-3.0-or-later import pytest -import os -import shutil -import uuid -from subprocess import check_call, check_output +from subprocess import check_call # Append current directory to system path in order to import testconfig import sys @@ -13,7 +10,7 @@ import sys sys.path.append(".") import testconfig as tcf -from utils import assert_taql, untar_ms, get_taql_result +from utils import assert_taql, run_in_tmp_path, untar """ Tests for applying the beam model. @@ -27,7 +24,6 @@ Script can be invoked in two ways: MSIN = "tNDPPP-generic.MS" DISH_MSIN = "tDish.MS" MSAPPLYBEAM = "tApplyBeam.tab" -CWD = os.getcwd() """ The tDish.MS is a reduced version of a MEERKAT dataset, generated with the following command: @@ -45,22 +41,10 @@ filter.blrange="[0,100,0,100]" @pytest.fixture(autouse=True) -def source_env(): - os.chdir(CWD) - tmpdir = str(uuid.uuid4()) - os.mkdir(tmpdir) - os.chdir(tmpdir) - - untar_ms(f"{tcf.RESOURCEDIR}/{MSIN}.tgz") - untar_ms(f"{tcf.RESOURCEDIR}/{DISH_MSIN}.tgz") - untar_ms(f"{tcf.SRCDIR}/{MSAPPLYBEAM}.tgz") - - # Tests are executed here - yield - - # Post-test: clean up - os.chdir(CWD) - shutil.rmtree(tmpdir) +def source_env(run_in_tmp_path): + untar(f"{tcf.RESOURCEDIR}/{MSIN}.tgz") + untar(f"{tcf.RESOURCEDIR}/{DISH_MSIN}.tgz") + untar(f"{tcf.SRCDIR}/{MSAPPLYBEAM}.tgz") @pytest.mark.parametrize("usechannelfreq", [False, True]) diff --git a/steps/test/integration/tApplyCal2.py b/steps/test/integration/tApplyCal2.py index 8f03c50aebfb7653c7badbaa3049016ddec55e85..40cda2dd010c7fab6544510345a34f4aa051ed79 100755 --- a/steps/test/integration/tApplyCal2.py +++ b/steps/test/integration/tApplyCal2.py @@ -2,9 +2,6 @@ # SPDX-License-Identifier: GPL-3.0-or-later import pytest -import os -import shutil -import uuid from subprocess import check_call # Append current directory to system path in order to import testconfig @@ -13,7 +10,7 @@ import sys sys.path.append(".") import testconfig as tcf -from utils import assert_taql, untar_ms +from utils import assert_taql, run_in_tmp_path, untar """ Tests for applying the calibration. @@ -28,25 +25,12 @@ Script can be invoked in two ways: MSIN = "tNDPPP-generic.MS" PARMDB_TGZ = "tApplyCal2.parmdb.tgz" PARMDB = "tApplyCal.parmdb" -CWD = os.getcwd() @pytest.fixture(autouse=True) -def source_env(): - os.chdir(CWD) - tmpdir = str(uuid.uuid4()) - os.mkdir(tmpdir) - os.chdir(tmpdir) - - untar_ms(f"{tcf.RESOURCEDIR}/{MSIN}.tgz") - untar_ms(f"{tcf.SRCDIR}/{PARMDB_TGZ}") - - # Tests are executed here - yield - - # Post-test: clean up - os.chdir(CWD) - shutil.rmtree(tmpdir) +def source_env(run_in_tmp_path): + untar(f"{tcf.RESOURCEDIR}/{MSIN}.tgz") + untar(f"{tcf.SRCDIR}/{PARMDB_TGZ}") @pytest.mark.parametrize("updateweights", [False, True]) diff --git a/steps/test/integration/tBdaExpander.py b/steps/test/integration/tBdaExpander.py index b06ca594d5e02a38020caf70660a5823da60dfac..21ffc7963d0c6512c906a77676162cfb214e6451 100644 --- a/steps/test/integration/tBdaExpander.py +++ b/steps/test/integration/tBdaExpander.py @@ -2,17 +2,14 @@ # SPDX-License-Identifier: GPL-3.0-or-later import pytest -import shutil -import os import sys -import uuid from subprocess import check_call # Append current directory to system path in order to import testconfig sys.path.append(".") import testconfig as tcf -from utils import assert_taql, untar_ms +from utils import assert_taql, run_in_tmp_path, untar """ Script can be invoked in two ways: @@ -27,25 +24,12 @@ Script can be invoked in two ways: MSIN_REGULAR = "tNDPPP-generic.MS" MSIN_BDA = "tNDPPP-bda.MS" -CWD = os.getcwd() @pytest.fixture(autouse=True) -def source_env(): - os.chdir(CWD) - tmpdir = str(uuid.uuid4()) - os.mkdir(tmpdir) - os.chdir(tmpdir) - - untar_ms(f"{tcf.RESOURCEDIR}/{MSIN_REGULAR}.tgz") - untar_ms(f"{tcf.RESOURCEDIR}/{MSIN_BDA}.tgz") - - # Tests are executed here - yield - - # Post-test: clean up - os.chdir(CWD) - shutil.rmtree(tmpdir) +def source_env(run_in_tmp_path): + untar(f"{tcf.RESOURCEDIR}/{MSIN_REGULAR}.tgz") + untar(f"{tcf.RESOURCEDIR}/{MSIN_BDA}.tgz") @pytest.fixture() diff --git a/steps/test/integration/tBdaPredict.py b/steps/test/integration/tBdaPredict.py index ecf43613661f3a5b04cb1f5b518c0ce700a50be1..404994555fb5a48388f9c97893598c94b1a8e679 100755 --- a/steps/test/integration/tBdaPredict.py +++ b/steps/test/integration/tBdaPredict.py @@ -2,17 +2,14 @@ # SPDX-License-Identifier: GPL-3.0-or-later import pytest -import shutil -import os import sys -import uuid from subprocess import check_call # Append current directory to system path in order to import testconfig sys.path.append(".") import testconfig as tcf -from utils import assert_taql, untar_ms +from utils import assert_taql, run_in_tmp_path, untar """ Script can be invoked in two ways: @@ -24,25 +21,12 @@ Script can be invoked in two ways: MSIN_REGULAR = "tNDPPP-generic.MS" MSIN_BDA = "tNDPPP-bda.MS" -CWD = os.getcwd() @pytest.fixture(autouse=True) -def source_env(): - os.chdir(CWD) - tmpdir = str(uuid.uuid4()) - os.mkdir(tmpdir) - os.chdir(tmpdir) - - untar_ms(f"{tcf.RESOURCEDIR}/{MSIN_REGULAR}.tgz") - untar_ms(f"{tcf.RESOURCEDIR}/{MSIN_BDA}.tgz") - - # Tests are executed here - yield - - # Post-test: clean up - os.chdir(CWD) - shutil.rmtree(tmpdir) +def source_env(run_in_tmp_path): + untar(f"{tcf.RESOURCEDIR}/{MSIN_REGULAR}.tgz") + untar(f"{tcf.RESOURCEDIR}/{MSIN_BDA}.tgz") @pytest.fixture() diff --git a/steps/test/integration/tClipper.py b/steps/test/integration/tClipper.py index 9cc346dfe398936c5fe96acfe8c36c025fa8a793..e20dc60ca30eee91d85dfc4469fd84a428cfe2d8 100644 --- a/steps/test/integration/tClipper.py +++ b/steps/test/integration/tClipper.py @@ -2,10 +2,7 @@ # SPDX-License-Identifier: GPL-3.0-or-later import pytest -import shutil -import os -import uuid -from subprocess import check_call, check_output +from subprocess import check_call # Append current directory to system path in order to import testconfig import sys @@ -13,7 +10,7 @@ import sys sys.path.append(".") import testconfig as tcf -from utils import assert_taql, untar_ms, get_taql_result +from utils import assert_taql, get_taql_result, run_in_tmp_path, untar """ Tests for clipper (coarsely predict and clip bright sources). @@ -26,25 +23,12 @@ Script can be invoked in two ways: MSIN = "tNDPPP-generic.MS" MSOUT = "clipper.MS" -CWD = os.getcwd() TEST_AMPLMAX = 1.5 @pytest.fixture(autouse=True) -def source_env(): - os.chdir(CWD) - tmpdir = str(uuid.uuid4()) - os.mkdir(tmpdir) - os.chdir(tmpdir) - - untar_ms(f"{tcf.RESOURCEDIR}/{MSIN}.tgz") - - # Tests are executed here - yield - - # Post-test: clean up - os.chdir(CWD) - shutil.rmtree(tmpdir) +def source_env(run_in_tmp_path): + untar(f"{tcf.RESOURCEDIR}/{MSIN}.tgz") @pytest.fixture() diff --git a/steps/test/integration/tColumnReader.py b/steps/test/integration/tColumnReader.py index 69e2c047f73f5b0cd04c4fea5805da405ff762ae..d07d81f71dcb2a983a467e5bb2d7871d151cf635 100644 --- a/steps/test/integration/tColumnReader.py +++ b/steps/test/integration/tColumnReader.py @@ -12,7 +12,7 @@ import sys sys.path.append(".") import testconfig as tcf -from utils import untar_ms +from utils import run_in_tmp_path, untar """ Script can be invoked in two ways: @@ -25,17 +25,8 @@ MSIN = "tNDPPP-generic.MS" @pytest.fixture(autouse=True) -def source_env(tmpdir_factory): - tmpdir = str(tmpdir_factory.mktemp("data")) - os.chdir(tmpdir) - - untar_ms(f"{tcf.RESOURCEDIR}/{MSIN}.tgz") - - # Tests are executed here - yield - - # Post-test: clean up - shutil.rmtree(tmpdir) +def source_env(run_in_tmp_path): + untar(f"{tcf.RESOURCEDIR}/{MSIN}.tgz") def test_filter_and_columnreader(): diff --git a/steps/test/integration/tDemix.py b/steps/test/integration/tDemix.py index d08d07a17ede2cd7647bfc9ac2999a501c7bcb93..84348462ff5827dc7a645dc36bb0de249dc622e5 100755 --- a/steps/test/integration/tDemix.py +++ b/steps/test/integration/tDemix.py @@ -2,17 +2,14 @@ # SPDX-License-Identifier: GPL-3.0-or-later import pytest -import shutil -import os import sys -import uuid from subprocess import check_call # Append current directory to system path in order to import testconfig sys.path.append(".") import testconfig as tcf -from utils import assert_taql, untar_ms +from utils import assert_taql, run_in_tmp_path, untar """ Tests for applying the beam model. @@ -24,7 +21,6 @@ Script can be invoked in two ways: """ MSIN = "tDemix.in_MS" -CWD = os.getcwd() common_args = [ "msin=tDemix_tmp/tDemix.MS", @@ -47,13 +43,8 @@ skymodel_arg = "demix.skymodel='tDemix_tmp/{}'" @pytest.fixture(autouse=True) -def source_env(): - os.chdir(CWD) - tmpdir = str(uuid.uuid4()) - os.mkdir(tmpdir) - os.chdir(tmpdir) - - untar_ms(f"{tcf.RESOURCEDIR}/{MSIN}.tgz") +def source_env(run_in_tmp_path): + untar(f"{tcf.RESOURCEDIR}/{MSIN}.tgz") check_call( [ tcf.MAKESOURCEDBEXE, @@ -62,13 +53,6 @@ def source_env(): ] ) - # Tests are executed here - yield - - # Post-test: clean up - os.chdir(CWD) - shutil.rmtree(tmpdir) - @pytest.mark.parametrize("skymodel", ["sky.txt", "sourcedb"]) def test_without_target(skymodel): diff --git a/steps/test/integration/tGainCal.py b/steps/test/integration/tGainCal.py index 46cf2628264c064f8c9a5be21d7598e2503873e7..5f8c7fbf8be0a8ff294c9cc8361b4cdc60305e81 100755 --- a/steps/test/integration/tGainCal.py +++ b/steps/test/integration/tGainCal.py @@ -1,8 +1,6 @@ import pytest -import shutil import os import sys -import uuid from subprocess import check_call, check_output import numpy as np @@ -10,7 +8,7 @@ import numpy as np sys.path.append(".") import testconfig as tcf -from utils import assert_taql, untar_ms +from utils import assert_taql, run_in_tmp_path, untar """ Tests for gaincal (direction independent calibration). @@ -23,25 +21,12 @@ Script can be invoked in two ways: MSGAINCAL = "tGainCal.tab" MSIN = "tNDPPP-generic.MS" -CWD = os.getcwd() @pytest.fixture(autouse=True) -def source_env(): - os.chdir(CWD) - tmpdir = str(uuid.uuid4()) - os.mkdir(tmpdir) - os.chdir(tmpdir) - - untar_ms(f"{tcf.RESOURCEDIR}/{MSIN}.tgz") - untar_ms(f"{tcf.SRCDIR}/{MSGAINCAL}.tgz") - - # Tests are executed here - yield - - # Post-test: clean up - os.chdir(CWD) - shutil.rmtree(tmpdir) +def source_env(run_in_tmp_path): + untar(f"{tcf.RESOURCEDIR}/{MSIN}.tgz") + untar(f"{tcf.SRCDIR}/{MSGAINCAL}.tgz") @pytest.fixture() diff --git a/steps/test/integration/tGainCalH5Parm.py b/steps/test/integration/tGainCalH5Parm.py index 68a873a57f647cc6c377169f11df1e74fe2997b7..986029fb61297ac9814147bbfcf2196b56fe21be 100755 --- a/steps/test/integration/tGainCalH5Parm.py +++ b/steps/test/integration/tGainCalH5Parm.py @@ -2,9 +2,6 @@ # SPDX-License-Identifier: GPL-3.0-or-later import pytest -import os -import shutil -import uuid from subprocess import check_call # Append current directory to system path in order to import testconfig @@ -13,7 +10,7 @@ import sys sys.path.append(".") import testconfig as tcf -from utils import assert_taql, untar_ms +from utils import assert_taql, run_in_tmp_path, untar """ Tests for applying the calibration. @@ -27,25 +24,12 @@ Script can be invoked in two ways: MSIN = "tNDPPP-generic.MS" REF_DATA = "tGainCal.tab" -CWD = os.getcwd() @pytest.fixture(autouse=True) -def source_env(): - os.chdir(CWD) - tmpdir = str(uuid.uuid4()) - os.mkdir(tmpdir) - os.chdir(tmpdir) - - untar_ms(f"{tcf.RESOURCEDIR}/{MSIN}.tgz") - untar_ms(f"{tcf.SRCDIR}/{REF_DATA}.tgz") - - # Tests are executed here - yield - - # Post-test: clean up - os.chdir(CWD) - shutil.rmtree(tmpdir) +def source_env(run_in_tmp_path): + untar(f"{tcf.RESOURCEDIR}/{MSIN}.tgz") + untar(f"{tcf.SRCDIR}/{REF_DATA}.tgz") @pytest.fixture() diff --git a/steps/test/integration/tIDGImager.py b/steps/test/integration/tIDGImager.py index 3eabfe5e09ef28ad7266e0bd2d0c77433460895f..a97f3065541360857134632298a3beda39865e10 100755 --- a/steps/test/integration/tIDGImager.py +++ b/steps/test/integration/tIDGImager.py @@ -1,15 +1,13 @@ import pytest -import shutil import os import sys -import uuid from subprocess import check_call # Append current directory to system path in order to import testconfig sys.path.append(".") import testconfig as tcf -from utils import get_taql_result, untar_ms +from utils import get_taql_result, run_in_tmp_path, untar """ Tests for testing IDGImager @@ -24,19 +22,10 @@ MSIN = "tNDPPP-generic.MS" @pytest.fixture(autouse=True) -def source_env(tmpdir_factory): - tmpdir = str(tmpdir_factory.mktemp("data")) - os.chdir(tmpdir) - - untar_ms(f"{tcf.RESOURCEDIR}/{MSIN}.tgz") +def source_env(run_in_tmp_path): + untar(f"{tcf.RESOURCEDIR}/{MSIN}.tgz") get_taql_result(f"UPDATE {MSIN} SET DATA=1") - # Tests are executed here - yield - - # Post-test: clean up - shutil.rmtree(tmpdir) - def test_idg_image_generation_cpu(): """ diff --git a/steps/test/integration/tMsIn.py b/steps/test/integration/tMsIn.py index ebf3071dc9bd6f633cee0419108151ded4042c7b..d1436a434dc073019970774fdb1142a9c4d07a69 100644 --- a/steps/test/integration/tMsIn.py +++ b/steps/test/integration/tMsIn.py @@ -2,10 +2,7 @@ # SPDX-License-Identifier: GPL-3.0-or-later import pytest -import os -import re import shutil -import uuid from subprocess import check_call # Append current directory to system path in order to import testconfig @@ -14,7 +11,7 @@ import sys sys.path.append(".") import testconfig as tcf -from utils import untar_ms, assert_taql +from utils import assert_taql, run_in_tmp_path, untar """ Script can be invoked in two ways: @@ -24,24 +21,11 @@ Script can be invoked in two ways: """ MSIN = "tNDPPP-generic.MS" -CWD = os.getcwd() @pytest.fixture(autouse=True) -def source_env(): - os.chdir(CWD) - tmpdir = str(uuid.uuid4()) - os.mkdir(tmpdir) - os.chdir(tmpdir) - - untar_ms(f"{tcf.RESOURCEDIR}/{MSIN}.tgz") - - # Tests are executed here - yield - - # Post-test: clean up - os.chdir(CWD) - shutil.rmtree(tmpdir) +def source_env(run_in_tmp_path): + untar(f"{tcf.RESOURCEDIR}/{MSIN}.tgz") def test_update_flags(): diff --git a/steps/test/integration/tMsOut.py b/steps/test/integration/tMsOut.py index c3c306276fd9f04dc215b7f9bc0ae9ee2e14f989..afd6afd5b456eb0c7100a41001e9ea6c3977b83a 100644 --- a/steps/test/integration/tMsOut.py +++ b/steps/test/integration/tMsOut.py @@ -4,8 +4,6 @@ import pytest import os import re -import shutil -import uuid from subprocess import check_call # Append current directory to system path in order to import testconfig @@ -14,27 +12,14 @@ import sys sys.path.append(".") import testconfig as tcf -from utils import untar_ms, get_taql_result, check_output +from utils import check_output, get_taql_result, run_in_tmp_path, untar MSIN = "tNDPPP-generic.MS" -CWD = os.getcwd() @pytest.fixture(autouse=True) -def source_env(): - os.chdir(CWD) - tmpdir = str(uuid.uuid4()) - os.mkdir(tmpdir) - os.chdir(tmpdir) - - untar_ms(f"{tcf.RESOURCEDIR}/{MSIN}.tgz") - - # Tests are executed here - yield - - # Post-test: clean up - os.chdir(CWD) - shutil.rmtree(tmpdir) +def source_env(run_in_tmp_path): + untar(f"{tcf.RESOURCEDIR}/{MSIN}.tgz") def test_chunking(): diff --git a/steps/test/integration/tMultiApplyCal.py b/steps/test/integration/tMultiApplyCal.py index 5789a93c4ff4fd54b508129b80c122ee44ce36f5..f2bb64d87d20796b61da6b73e4afc4e1fad19742 100644 --- a/steps/test/integration/tMultiApplyCal.py +++ b/steps/test/integration/tMultiApplyCal.py @@ -2,9 +2,6 @@ # SPDX-License-Identifier: GPL-3.0-or-later import pytest -import os -import shutil -import uuid from subprocess import check_call # Append current directory to system path in order to import testconfig @@ -13,7 +10,7 @@ import sys sys.path.append(".") import testconfig as tcf -from utils import assert_taql, untar_ms +from utils import assert_taql, run_in_tmp_path, untar """ Script can be invoked in two ways: @@ -27,25 +24,12 @@ PARMDB_TGZ = ( "tApplyCal2.parmdb.tgz" # Note: This archive contains tApplyCal.parmdb. ) PARMDB = "tApplyCal.parmdb" -CWD = os.getcwd() @pytest.fixture(autouse=True) -def source_env(): - os.chdir(CWD) - tmpdir = str(uuid.uuid4()) - os.mkdir(tmpdir) - os.chdir(tmpdir) - - untar_ms(f"{tcf.RESOURCEDIR}/{MSIN}.tgz") - untar_ms(f"{tcf.SRCDIR}/{PARMDB_TGZ}") - - # Tests are executed here - yield - - # Post-test: clean up - os.chdir(CWD) - shutil.rmtree(tmpdir) +def source_env(run_in_tmp_path): + untar(f"{tcf.RESOURCEDIR}/{MSIN}.tgz") + untar(f"{tcf.SRCDIR}/{PARMDB_TGZ}") @pytest.mark.parametrize( diff --git a/steps/test/integration/tNullStokes.py b/steps/test/integration/tNullStokes.py index 86e250d847bca32cefa967394d39b76ed213a4ff..68fd5205004accaef0f81dc9a0e4f2e93c749d05 100755 --- a/steps/test/integration/tNullStokes.py +++ b/steps/test/integration/tNullStokes.py @@ -1,15 +1,12 @@ import pytest -import shutil -import os import sys -import uuid from subprocess import check_call # Append current directory to system path in order to import testconfig sys.path.append(".") import testconfig as tcf -from utils import assert_taql, untar_ms +from utils import assert_taql, run_in_tmp_path, untar """ Tests for nullstokes (zeroing out the Stokes parameters Q and/or U). @@ -22,24 +19,11 @@ Script can be invoked in two ways: MSIN = "tNDPPP-generic.MS" MSOUT = "test_data.MS" -CWD = os.getcwd() @pytest.fixture(autouse=True) -def source_env(): - os.chdir(CWD) - tmpdir = str(uuid.uuid4()) - os.mkdir(tmpdir) - os.chdir(tmpdir) - - untar_ms(f"{tcf.RESOURCEDIR}/{MSIN}.tgz") - - # Tests are executed here - yield - - # Post-test: clean up - os.chdir(CWD) - shutil.rmtree(tmpdir) +def source_env(run_in_tmp_path): + untar(f"{tcf.RESOURCEDIR}/{MSIN}.tgz") @pytest.fixture() diff --git a/steps/test/integration/tPhaseshiftPredict.py b/steps/test/integration/tPhaseshiftPredict.py index 3fad33403ba6b632828913ae9817fe2abd4ceebb..6b52c882796a35fa8fdbf63c16665beaee2e23c2 100755 --- a/steps/test/integration/tPhaseshiftPredict.py +++ b/steps/test/integration/tPhaseshiftPredict.py @@ -1,15 +1,12 @@ import pytest -import shutil -import os import sys -import uuid from subprocess import check_call # Append current directory to system path in order to import testconfig sys.path.append(".") import testconfig as tcf -from utils import get_taql_result, untar_ms +from utils import get_taql_result, run_in_tmp_path, untar """ Tests for predicting gaussian sources @@ -24,17 +21,8 @@ MSIN = "tNDPPP-generic.MS" @pytest.fixture(autouse=True) -def source_env(tmpdir_factory): - tmpdir = str(tmpdir_factory.mktemp("data")) - os.chdir(tmpdir) - - untar_ms(f"{tcf.RESOURCEDIR}/{MSIN}.tgz") - - # Tests are executed here - yield - - # Post-test: clean up - shutil.rmtree(tmpdir) +def source_env(run_in_tmp_path): + untar(f"{tcf.RESOURCEDIR}/{MSIN}.tgz") testdata = [ diff --git a/steps/test/integration/tPredict.py b/steps/test/integration/tPredict.py index 252e5e3773ddde8ddfab78f771f8b556e1c35ffa..747e407845817e3f11dc183db89a8f30307bbee5 100644 --- a/steps/test/integration/tPredict.py +++ b/steps/test/integration/tPredict.py @@ -2,10 +2,7 @@ # SPDX-License-Identifier: GPL-3.0-or-later import pytest -import shutil -import os -import uuid -from subprocess import check_call, check_output +from subprocess import check_call import numpy as np @@ -15,7 +12,7 @@ import sys sys.path.append(".") import testconfig as tcf -from utils import assert_taql, untar_ms +from utils import assert_taql, run_in_tmp_path, untar """ Replacement for tPredict.sh using pytest. @@ -28,25 +25,12 @@ Script can be invoked in two ways: MSIN = "tNDPPP-generic.MS" MSPREDICT = "tPredict.tab" -CWD = os.getcwd() @pytest.fixture(autouse=True) -def source_env(): - os.chdir(CWD) - tmpdir = str(uuid.uuid4()) - os.mkdir(tmpdir) - os.chdir(tmpdir) - - untar_ms(f"{tcf.RESOURCEDIR}/{MSIN}.tgz") - untar_ms(f"{tcf.SRCDIR}/{MSPREDICT}.tgz") - - # Tests are executed here - yield - - # Post-test: clean up - os.chdir(CWD) - shutil.rmtree(tmpdir) +def source_env(run_in_tmp_path): + untar(f"{tcf.RESOURCEDIR}/{MSIN}.tgz") + untar(f"{tcf.SRCDIR}/{MSPREDICT}.tgz") @pytest.fixture @@ -161,7 +145,6 @@ def test_without_and_with_time_smearing(use_time_smearing): f"center, POINT, 01:37:41.299, +{33 + dec_offset}.09.35.132, 10, , , , ,\n" ) - shutil.rmtree(sourcedb, ignore_errors=True) check_call( [tcf.MAKESOURCEDBEXE, "in=timesmearing.skymodel", f"out={sourcedb}"] ) diff --git a/steps/test/integration/tReadOnly.py b/steps/test/integration/tReadOnly.py index ef5672a68bae1da9fdd06831b557d203810dd30d..7852ac94e68b7ad74cb4db82a44442550242ba5d 100644 --- a/steps/test/integration/tReadOnly.py +++ b/steps/test/integration/tReadOnly.py @@ -2,9 +2,6 @@ # SPDX-License-Identifier: GPL-3.0-or-later import pytest -import os -import shutil -import uuid from subprocess import check_call # Append current directory to system path in order to import testconfig @@ -13,7 +10,7 @@ import sys sys.path.append(".") import testconfig as tcf -from utils import untar_ms +from utils import run_in_tmp_path, untar # FIXME: consider merging into tGainCal? @@ -26,25 +23,12 @@ Script can be invoked in two ways: MSIN = "tNDPPP-generic.MS" MSGAINCAL = "tGainCal.tab" -CWD = os.getcwd() @pytest.fixture(autouse=True) def source_env(): - os.chdir(CWD) - tmpdir = str(uuid.uuid4()) - os.mkdir(tmpdir) - os.chdir(tmpdir) - - untar_ms(f"{tcf.RESOURCEDIR}/{MSIN}.tgz") - untar_ms(f"{tcf.SRCDIR}/{MSGAINCAL}.tgz") - - # Tests are executed here - yield - - # Post-test: clean up - os.chdir(CWD) - shutil.rmtree(tmpdir) + untar(f"{tcf.RESOURCEDIR}/{MSIN}.tgz") + untar(f"{tcf.SRCDIR}/{MSGAINCAL}.tgz") def test_read_only(): diff --git a/steps/test/integration/tSagecalPredict.py b/steps/test/integration/tSagecalPredict.py index f3522c248412047d60b87eeccce54f00c4404834..9d6824caf904d4ed616e638197fd4d0dc909e71a 100644 --- a/steps/test/integration/tSagecalPredict.py +++ b/steps/test/integration/tSagecalPredict.py @@ -2,10 +2,7 @@ # SPDX-License-Identifier: GPL-3.0-or-later import pytest -import shutil -import os -import uuid -from subprocess import check_call, check_output +from subprocess import check_call # Append current directory to system path in order to import testconfig import sys @@ -13,7 +10,7 @@ import sys sys.path.append(".") import testconfig as tcf -from utils import assert_taql, untar_ms +from utils import assert_taql, run_in_tmp_path, untar """ Similar to tPredict.py, testing SagecalPredict using pytest. @@ -23,25 +20,12 @@ Similar to tPredict.py, testing SagecalPredict using pytest. MSIN = "tNDPPP-generic.MS" # But we have a different reference MS for comparison MSPREDICT = "tSagecalPredict.tab" -CWD = os.getcwd() @pytest.fixture(autouse=True) -def source_env(): - os.chdir(CWD) - tmpdir = str(uuid.uuid4()) - os.mkdir(tmpdir) - os.chdir(tmpdir) - - untar_ms(f"{tcf.RESOURCEDIR}/{MSIN}.tgz") - untar_ms(f"{tcf.SRCDIR}/{MSPREDICT}.tgz") - - # Tests are executed here - yield - - # Post-test: clean up - os.chdir(CWD) - shutil.rmtree(tmpdir) +def source_env(run_in_tmp_path): + untar(f"{tcf.RESOURCEDIR}/{MSIN}.tgz") + untar(f"{tcf.SRCDIR}/{MSPREDICT}.tgz") def test_with_beam_replace(): diff --git a/steps/test/integration/tSplit.py b/steps/test/integration/tSplit.py index 6aae31ddcab04725d5dec4f58423104f0070e28b..62b6083be0846c4cbf3c3193eb7ed8ef9379390d 100644 --- a/steps/test/integration/tSplit.py +++ b/steps/test/integration/tSplit.py @@ -2,9 +2,6 @@ # SPDX-License-Identifier: GPL-3.0-or-later import pytest -import os -import shutil -import uuid from subprocess import check_call # Append current directory to system path in order to import testconfig @@ -13,7 +10,7 @@ import sys sys.path.append(".") import testconfig as tcf -from utils import assert_taql, untar_ms +from utils import assert_taql, run_in_tmp_path, untar """ Script can be invoked in two ways: @@ -24,25 +21,12 @@ Script can be invoked in two ways: MSIN = "tNDPPP-generic.MS" MSAPPLYBEAM = "tApplyBeam.tab" -CWD = os.getcwd() @pytest.fixture(autouse=True) -def source_env(): - os.chdir(CWD) - tmpdir = str(uuid.uuid4()) - os.mkdir(tmpdir) - os.chdir(tmpdir) - - untar_ms(f"{tcf.RESOURCEDIR}/{MSIN}.tgz") - untar_ms(f"{tcf.SRCDIR}/{MSAPPLYBEAM}.tgz") - - # Tests are executed here - yield - - # Post-test: clean up - os.chdir(CWD) - shutil.rmtree(tmpdir) +def source_env(run_in_tmp_path): + untar(f"{tcf.RESOURCEDIR}/{MSIN}.tgz") + untar(f"{tcf.SRCDIR}/{MSAPPLYBEAM}.tgz") def test_split(): diff --git a/steps/test/integration/tUVWFlagger.py b/steps/test/integration/tUVWFlagger.py index ec8e4de1830ccccec967ba0e078a2466d1f40275..3594de63710fe97f08ce24b4d9629b3d0383aec2 100644 --- a/steps/test/integration/tUVWFlagger.py +++ b/steps/test/integration/tUVWFlagger.py @@ -1,12 +1,9 @@ # Copyright (C) 2022 ASTRON (Netherlands Institute for Radio Astronomy) # SPDX-License-Identifier: GPL-3.0-or-later -import os import pytest import re -import shutil -import uuid -from subprocess import check_call +from subprocess import check_call, check_output # Append current directory to system path in order to import testconfig import sys @@ -14,7 +11,7 @@ import sys sys.path.append(".") import testconfig as config -import utils +from utils import assert_taql, run_in_tmp_path, untar """ Script can be invoked in two ways: @@ -24,24 +21,11 @@ Script can be invoked in two ways: """ MSIN = "tNDPPP-generic.MS" -CWD = os.getcwd() @pytest.fixture(autouse=True) -def source_env(): - os.chdir(CWD) - tmpdir = str(uuid.uuid4()) - os.mkdir(tmpdir) - os.chdir(tmpdir) - - utils.untar_ms(f"{config.RESOURCEDIR}/{MSIN}.tgz") - - # Tests are executed here - yield - - # Post-test: clean up - os.chdir(CWD) - shutil.rmtree(tmpdir) +def source_env(run_in_tmp_path): + untar(f"{config.RESOURCEDIR}/{MSIN}.tgz") def test_update_flags_inplace(): @@ -52,11 +36,11 @@ def test_update_flags_inplace(): count_flags_set = f"select from {MSIN} where all(FLAG=True)" - utils.check_call([config.TAQLEXE, "update", MSIN, "set", "FLAG=False"]) - utils.assert_taql(count_flags_set) + check_call([config.TAQLEXE, "update", MSIN, "set", "FLAG=False"]) + assert_taql(count_flags_set) # The first run all visibilites are flagged. - result = utils.check_output( + result = check_output( [ config.DP3EXE, f"msin={MSIN}", @@ -66,14 +50,14 @@ def test_update_flags_inplace(): "uvwflag.ulambdamin=100000000000000", ] ) - utils.assert_taql(count_flags_set, 168) + assert_taql(count_flags_set, 168) assert re.search( b"\nTotal flagged: 100.000% \\(1344 out of 1344 visibilities\\)\n\n\n", result, ) # The second run nothing is flagged, since everything is already flagged - result = utils.check_output( + result = check_output( [ config.DP3EXE, f"msin={MSIN}", @@ -83,7 +67,7 @@ def test_update_flags_inplace(): "uvwflag.ulambdamin=100000000000000", ] ) - utils.assert_taql(count_flags_set, 168) + assert_taql(count_flags_set, 168) assert re.search( b"\nTotal flagged: 0.000% \\(0 out of 1344 visibilities\\)\n\n\n", result, @@ -94,10 +78,10 @@ def test_update_flags_new_table(): """Assert that updating the flags in a different output column works.""" count_flags_set = f"select from {MSIN} where all(FLAG=True)" - utils.check_call([config.TAQLEXE, "update", MSIN, "set", "FLAG=False"]) - utils.assert_taql(count_flags_set, 0) + check_call([config.TAQLEXE, "update", MSIN, "set", "FLAG=False"]) + assert_taql(count_flags_set, 0) - result = utils.check_output( + result = check_output( [ config.DP3EXE, f"msin={MSIN}", @@ -110,19 +94,19 @@ def test_update_flags_new_table(): ) # Assert that FLAG column is not changed - utils.assert_taql(count_flags_set, 0) + assert_taql(count_flags_set, 0) assert re.search( b"\nTotal flagged: 100.000% \\(1344 out of 1344 visibilities\\)\n\n\n", result, ) count_flags_set = f"select from {MSIN} where all(MODIFIED_FLAGS=True)" - utils.assert_taql(count_flags_set, 168) + assert_taql(count_flags_set, 168) # The second run compares the flagged changes based on the FLAG column # in the input. So it still will again flag 100% of the original data. - result = utils.check_output( + result = check_output( [ config.DP3EXE, f"msin={MSIN}", @@ -138,4 +122,4 @@ def test_update_flags_new_table(): result, ) count_flags_set = f"select from {MSIN} where all(MODIFIED_FLAGS=True)" - utils.assert_taql(count_flags_set, 168) + assert_taql(count_flags_set, 168)