diff --git a/.gitlab-ci.common.yml b/.gitlab-ci.common.yml index 9f75a18637981c7ad0cc1ea632618d2728eade6d..2af60374f89c74ebe23d2dffe1d9039a1058e9e7 100644 --- a/.gitlab-ci.common.yml +++ b/.gitlab-ci.common.yml @@ -24,10 +24,7 @@ stages: - build - linting - test - - publish - - pages -include: .gitlab-ci.wheels.yml # The 'IMAGE' variables allow reusing docker images between different pipelines. # See https://confluence.skatelescope.org/display/SE/Caching+Docker+images+using+GitLab+CI+registry @@ -102,7 +99,7 @@ prepare-base-2404: - cd build - cmake -G Ninja -DBUILD_TESTING=On .. - ninja install - - DP3 + - EDGEDP3 # Build (and run) DP3 on Ubuntu 20, ensuring backward compatiblity. # This build does not include IDG, since IDG no longer supports Ubuntu 20. @@ -315,22 +312,3 @@ integration-test-2204: - build/pytest_*.xml reports: junit: build/pytest_*.xml - -deploy-package-2204: - stage: publish - needs: ["versioning","build-package-2204"] - image: $BASE_IMAGE_2204 - script: - - pip3 install aptly-api-client - - chmod -R 700 external/schaap-packaging - - cd dp3_package - - export FILES=$(ls -1 $PWD/*.deb) - - echo UPLOADING files $FILES - # The following example command must be executed first or else the update will fail because there is no repository - # create_repo.py -a amd64 -c testing -d bionic --gpg-key ${GPG_KEY} --gpg-passphrase ${GPG_PASS} schaap - - ../external/schaap-packaging/update_repo.py --cleanup -d bionic --gpg-key ${GPG_KEY} --gpg-passphrase ${GPG_PASS} schaap ${FILES} - rules: - # Only run on master because GPG_KEY and GPG_PASS are protected and therefore only available on protected branches. - - if: '$CI_COMMIT_BRANCH != "master"' - when: never - - if: '$UPLOAD_PACKAGE' diff --git a/CMakeLists.txt b/CMakeLists.txt index 61cff2dbc97ed17b460a4036f71a12772f81d31e..1708d65eaf7cd3bc0fa5887643531a61db1fd642 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -13,27 +13,27 @@ endif() # Set version number and project name. # Please keep CPack/CMakeLists.txt and setup.py in sync with this. -set(DP3_VERSION 6.1.0) -if(DP3_VERSION MATCHES "^([0-9]+)\\.([0-9]+)\\.([0-9]+)") - set(DP3_VERSION_MAJOR "${CMAKE_MATCH_1}") - set(DP3_VERSION_MINOR "${CMAKE_MATCH_2}") - set(DP3_VERSION_PATCH "${CMAKE_MATCH_3}") +set(EDGEDP3_VERSION 6.1.0) +if(EDGEDP3_VERSION MATCHES "^([0-9]+)\\.([0-9]+)\\.([0-9]+)") + set(EDGEDP3_VERSION_MAJOR "${CMAKE_MATCH_1}") + set(EDGEDP3_VERSION_MINOR "${CMAKE_MATCH_2}") + set(EDGEDP3_VERSION_PATCH "${CMAKE_MATCH_3}") else() - message(FATAL_ERROR "Failed to parse DP3_VERSION='${DP3_VERSION}'") + message(FATAL_ERROR "Failed to parse EDGEDP3_VERSION='${EDGEDP3_VERSION}'") endif() # Get the latest abbreviated commit hash of the working branch execute_process( COMMAND git describe WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR} - OUTPUT_VARIABLE DP3_GIT_HASH + OUTPUT_VARIABLE EDGEDP3_GIT_HASH OUTPUT_STRIP_TRAILING_WHITESPACE) option(BUILD_WITH_CUDA "Build with CUDA support" FALSE) if(BUILD_WITH_CUDA) project( - DP3 - VERSION ${DP3_VERSION} + EDGEDP3 + VERSION ${EDGEDP3_VERSION} LANGUAGES CUDA C CXX) set(CUDA_PROPAGATE_HOST_FLAGS FALSE) set(CMAKE_CUDA_ARCHITECTURES @@ -53,7 +53,7 @@ if(BUILD_WITH_CUDA) FetchContent_MakeAvailable(cudawrappers) else() - project(DP3 VERSION ${DP3_VERSION}) + project(EDGEDP3 VERSION ${EDGEDP3_VERSION}) endif() include(CheckCXXCompilerFlag) @@ -232,14 +232,14 @@ if(${EVERYBEAM_FOUND}) VERSION_GREATER_EQUAL "0.7.0") message( FATAL_ERROR - "DP3 needs EveryBeam version >= 0.5.8 and < 0.7.0 - but found version ${EveryBeam_VERSION}" + "EDGEDP3 needs EveryBeam version >= 0.5.8 and < 0.7.0 - but found version ${EveryBeam_VERSION}" ) endif() include_directories(${EVERYBEAM_INCLUDE_DIR}) else(${EVERYBEAM_FOUND}) message( FATAL_ERROR - "DP3 requires EveryBeam, but EveryBeam was not found. " + "EDGEDP3 requires EveryBeam, but EveryBeam was not found. " "Please install https://git.astron.nl/RD/EveryBeam and make sure that " "EveryBeam is added to the CMAKE_PREFIX_PATH") endif(${EVERYBEAM_FOUND}) @@ -250,7 +250,7 @@ if(IDGAPI_FOUND) if((IDGAPI_VERSION VERSION_LESS "0.8") OR (NOT DEFINED IDGAPI_VERSION)) message( FATAL_ERROR - "IDGAPI was found, but DP3 requires IDGAPI to have version >= 0.8. " + "IDGAPI was found, but EDGEDP3 requires IDGAPI to have version >= 0.8. " "Please compile IDG repository at a version >= 0.8") endif() endif() @@ -412,7 +412,7 @@ if(BUILD_WITH_CUDA) # 1) if built as an object the kernel simbol is not exported propery by # the cuda ecosystem # 2) if built as a shared library the dependencies of DDECal are missing - # when DP3 is built in debug mode + # when EDGEDP3 is built in debug mode add_library( CudaSolvers STATIC ddecal/gain_solvers/IterativeDiagonalSolverCuda.cc ddecal/gain_solvers/kernels/IterativeDiagonal.cu) @@ -440,9 +440,9 @@ if(BUILD_WITH_CUDA) install(TARGETS CudaSolvers) endif() -# 'Base' files and steps are only included in DP3. +# 'Base' files and steps are only included in EDGEDP3. add_library( - DP3_OBJ OBJECT + EDGEDP3_OBJ OBJECT antennaflagger/Flagger.cc base/AartfaacSubtableWriter.cc base/Apply.cc @@ -514,7 +514,7 @@ add_library( steps/ApplyBeam.cc steps/NullStokes.cc steps/SagecalPredict.cc) -target_link_libraries(DP3_OBJ xsimd xtensor xtensor-blas) +target_link_libraries(EDGEDP3_OBJ xsimd xtensor xtensor-blas) add_library( ParmDB OBJECT @@ -544,7 +544,7 @@ add_library( parmdb/SourceInfo.cc) # dp3_testdyndp3 requires using position independent code. -set_property(TARGET Blob Common DDECal DP3_OBJ ParmDB +set_property(TARGET Blob Common DDECal EDGEDP3_OBJ ParmDB PROPERTY POSITION_INDEPENDENT_CODE ON) set(LIBDIRAC_PREFIX @@ -574,7 +574,7 @@ else() ) endif() -set(DP3_LIBRARIES +set(EDGEDP3_LIBRARIES ${AOFLAGGER_LIB} ${ARMADILLO_LIBRARIES} ${CASACORE_LIBRARIES} @@ -589,20 +589,20 @@ set(DP3_LIBRARIES ${Boost_LIBRARIES}) if(BUILD_WITH_CUDA) - list(APPEND DP3_LIBRARIES CudaSolvers) + list(APPEND EDGEDP3_LIBRARIES CudaSolvers) endif() # If libdirac is found, use it if(LIBDIRAC_FOUND) # add link flags - set(DP3_LIBRARIES ${DP3_LIBRARIES} ${LIBDIRAC_LINK_LIBRARIES}) + set(EDGEDP3_LIBRARIES ${EDGEDP3_LIBRARIES} ${LIBDIRAC_LINK_LIBRARIES}) if(HAVE_CUDA) # if we use libdirac with CUDA support, we enable a different preprocessor def # so as not to conflict with CPU only version of libdirac add_definitions(-DHAVE_LIBDIRAC_CUDA) add_definitions(-DHAVE_CUDA) - set(DP3_LIBRARIES - ${DP3_LIBRARIES} + set(EDGEDP3_LIBRARIES + ${EDGEDP3_LIBRARIES} ${CUDA_LIBRARIES} ${CUDA_CUBLAS_LIBRARIES} ${CUDA_CUFFT_LIBRARIES} @@ -619,7 +619,7 @@ endif() option(CHECK_BLAS "Perform validity checks on the BLAS library" On) if(CHECK_BLAS) include("${CMAKE_SOURCE_DIR}/external/aocommon/CMake/CheckBLAS.cmake") - check_blas(LIBRARIES ${DP3_LIBRARIES}) + check_blas(LIBRARIES ${EDGEDP3_LIBRARIES}) endif() add_library(Aocommon INTERFACE) @@ -629,16 +629,16 @@ add_library(Casacore INTERFACE) target_include_directories(Casacore INTERFACE ${CASACORE_INCLUDE_DIR}) target_link_libraries(Casacore INTERFACE ${CASACORE_LIBRARIES}) -add_library(LIBDP3 SHARED) +add_library(LIBEDGEDP3 SHARED) target_link_libraries( - LIBDP3 - PUBLIC Blob Common DDECal DP3_OBJ ParmDB - # The public headers of LIBDP3 include casacore and aocommon headers + LIBEDGEDP3 + PUBLIC Blob Common DDECal EDGEDP3_OBJ ParmDB + # The public headers of LIBEDGEDP3 include casacore and aocommon headers # These targets should therefore be linked as INTERFACE INTERFACE Casacore Aocommon - PRIVATE ${DP3_LIBRARIES}) -set_target_properties(LIBDP3 PROPERTIES LIBRARY_OUTPUT_NAME "DP3") -install(TARGETS LIBDP3) + PRIVATE ${EDGEDP3_LIBRARIES}) +set_target_properties(LIBEDGEDP3 PROPERTIES LIBRARY_OUTPUT_NAME "EDGEDP3") +install(TARGETS LIBEDGEDP3) # CMake 3.23 has FILE_SETs which are a cleaner solution. But this version # is too new to require, install(DIRECTORY include DESTINATION include) @@ -650,21 +650,22 @@ endif() set(SOURCEDB_LIBRARIES Blob Common ParmDB ${CASACORE_LIBRARIES} ${Boost_LIBRARIES}) -add_executable(DP3 base/Main.cc) -target_link_libraries(DP3 LIBDP3 ${HDF5_LIBRARIES} ${Boost_LIBRARIES}) +add_executable(EDGEDP3 base/Main.cc) +target_link_libraries(EDGEDP3 LIBEDGEDP3 ${HDF5_LIBRARIES} ${Boost_LIBRARIES}) -add_executable(makesourcedb parmdb/makesourcedb.cc) -target_link_libraries(makesourcedb ${SOURCEDB_LIBRARIES}) +add_executable(edgemakesourcedb parmdb/makesourcedb.cc) +target_link_libraries(edgemakesourcedb ${SOURCEDB_LIBRARIES}) -add_executable(showsourcedb parmdb/showsourcedb.cc) -target_link_libraries(showsourcedb ${SOURCEDB_LIBRARIES}) +add_executable(edgeshowsourcedb parmdb/showsourcedb.cc) +target_link_libraries(edgeshowsourcedb ${SOURCEDB_LIBRARIES}) -add_executable(msoverview base/msoverview.cc base/MS.cc) -target_link_libraries(msoverview ${CASACORE_LIBRARIES} ${Boost_LIBRARIES}) +add_executable(edgemsoverview base/msoverview.cc base/MS.cc) +target_link_libraries(edgemsoverview ${CASACORE_LIBRARIES} ${Boost_LIBRARIES}) -install(TARGETS DP3 makesourcedb showsourcedb msoverview DESTINATION bin) +install(TARGETS EDGEDP3 edgemakesourcedb edgeshowsourcedb edgemsoverview + DESTINATION bin) -# Install a script that warns users that DP3 is the new name of the executable +# Install a script that warns users that EDGEDP3 is the new name of the executable install( PROGRAMS scripts/DPPP-deprecation.sh DESTINATION bin @@ -827,7 +828,8 @@ if(BUILD_TESTING) if(BUILD_WITH_CUDA) set_target_properties(unittests PROPERTIES CUDA_RESOLVE_DEVICE_SYMBOLS ON) endif() - target_link_libraries(unittests LIBDP3 ${DP3_LIBRARIES} xtensor xtensor-blas) + target_link_libraries(unittests LIBEDGEDP3 ${EDGEDP3_LIBRARIES} xtensor + xtensor-blas) # Automatically (re)build the unit tests on every ctest run. add_test(buildunittests ${CMAKE_COMMAND} --build ${CMAKE_BINARY_DIR} --target diff --git a/CPack/CMakeLists.txt b/CPack/CMakeLists.txt index d492630854f7ff3457235b81356de29ab0c81f55..13e2ff1b457fc8d2be6004adbd243fe158e42aa7 100644 --- a/CPack/CMakeLists.txt +++ b/CPack/CMakeLists.txt @@ -25,9 +25,9 @@ include(${CMAKE_CURRENT_SOURCE_DIR}/DetermineTargetCPU.cmake) # Set package name: encode target CPU, if defined, in package name. if(DEFINED IDENTIFIED_TARGET_CPU) - set(CPACK_PACKAGE_NAME "dp3-${IDENTIFIED_TARGET_CPU}") + set(CPACK_PACKAGE_NAME "edgedp3-${IDENTIFIED_TARGET_CPU}") else() - set(CPACK_PACKAGE_NAME "dp3") + set(CPACK_PACKAGE_NAME "edgedp3") endif() set(CPACK_PACKAGE_VENDOR "ASTRON") @@ -54,14 +54,14 @@ set(CPACK_DEBIAN_PACKAGE_SHLIBDEPS ON) # Determine list of conflicting package names. Packages not built for current # target CPU always conflict. if(DEFINED IDENTIFIED_TARGET_CPU) - set(_conflicts dp3) + set(_conflicts edgedp3) else() set(_conflicts) endif() foreach(_cpu ${KNOWN_TARGET_CPUS}) if(NOT "${_cpu}" STREQUAL "${IDENTIFIED_TARGET_CPU}") if("${_conflicts}" STREQUAL "") - set(_conflicts "dp3-${_cpu}") + set(_conflicts "edgedp3-${_cpu}") else() set(_conflicts "${_conflicts}, dp3-${_cpu}") endif() diff --git a/README.md b/README.md index 5dc6e0e614c6c0e838f56d443302dad2736fcee6..7f7a2d46a2175afb0f4bb3b89d8e9d7ad9d0a86c 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,11 @@ -# DP3 -LOFAR preprocessing software, including averaging, flagging, various kinds of calibration and more. +# Edge DP3 -The DP3 documentation can be found at: https://dp3.readthedocs.org - -This repository is a continuation of the one at svn.astron.nl/LOFAR. In particular, it has branched off at LOFAR Release 3.2 (Sept 2018). The version of DP3 that is in the ASTRON repository is no longer maintained. +Experimental version of DP3 meant for testing new functionalities and optimization on edge computing devices. +Please refer to the stable version of DP3 for additional details and documentation on what DP3 can do: +https://git.astron.nl/RD/DP3.git ## Installation + Some non-standard dependencies of this project are: armadillo, boost, boost-python, casacore, hdf5, aoflagger, and EveryBeam. See the Dockerfiles [`docker/ubuntu_22_04_base`](docker/ubuntu_22_04_base) and/or [`docker/ubuntu_24_04_base`](docker/ubuntu_24_04_base) as examples. Typical installation commands: @@ -18,8 +18,10 @@ make install ``` ## Contributing +Feel free to contribute by submitting a pull request or by opening issues. + ### Want to Help? -Issues can be filed either at [gitlab](https://git.astron.nl/RD/DP3) or [github](https://github.com/lofar-astron/DP3). +Issues can be filed at [gitlab](https://git.astron.nl/mancini/EdgeDP3). -Want to contribute some code, or improve documentation? You can start by cloning our code from the [DP3 development repository](https://git.astron.nl/RD/DP3). \ No newline at end of file +Want to contribute some code, or improve documentation? You can start by cloning our code from the [DP3 development repository](https://git.astron.nl/mancini/EdgeDP3). \ No newline at end of file diff --git a/docs/CMakeLists.txt b/docs/CMakeLists.txt index aee72c5b286e268317352f1a0a5b55032879858a..a17aa89c93c76d4a199dd5ec7b3ad2d093572014 100644 --- a/docs/CMakeLists.txt +++ b/docs/CMakeLists.txt @@ -4,22 +4,23 @@ cmake_minimum_required(VERSION 3.9) set(CMAKE_MODULE_PATH ${CMAKE_CURRENT_LIST_DIR}/../CMake) -project(DP3_DOC NONE) +project(EDGEDP3_DOC NONE) find_package(Doxygen) -if(NOT DP3_VERSION) +if(NOT EDGEDP3_VERSION) # Building docs standalone, get version from top level CMakelists file(READ "../CMakeLists.txt" TOPLEVEL_CMAKELISTS) - set(DP3_VERSION_REGEX - ".*set\\(DP3_VERSION ([0-9]+)\\.([0-9]+)\\.([0-9]+).*\\)") - if("${TOPLEVEL_CMAKELISTS}" MATCHES ${DP3_VERSION_REGEX}) - set(DP3_VERSION_MAJOR "${CMAKE_MATCH_1}") - set(DP3_VERSION_MINOR "${CMAKE_MATCH_2}") - set(DP3_VERSION_PATCH "${CMAKE_MATCH_3}") + set(EDGEDP3_VERSION_REGEX + ".*set\\(EDGEDP3_VERSION ([0-9]+)\\.([0-9]+)\\.([0-9]+).*\\)") + if("${TOPLEVEL_CMAKELISTS}" MATCHES ${EDGEDP3_VERSION_REGEX}) + set(EDGEDP3_VERSION_MAJOR "${CMAKE_MATCH_1}") + set(EDGEDP3_VERSION_MINOR "${CMAKE_MATCH_2}") + set(EDGEDP3_VERSION_PATCH "${CMAKE_MATCH_3}") else() message( - FATAL_ERROR "Failed to parse DP3_VERSION from top level CMakeLists.txt") + FATAL_ERROR + "Failed to parse EDGEDP3_VERSION from top level CMakeLists.txt") endif() endif() diff --git a/pythondp3/CMakeLists.txt b/pythondp3/CMakeLists.txt index 5a19a3d129b59acd8ce944d1fbdc811efadf0ba8..986eabae98125244f19e9a9209987ab96a22b25d 100644 --- a/pythondp3/CMakeLists.txt +++ b/pythondp3/CMakeLists.txt @@ -21,7 +21,7 @@ pybind11_add_module(parameterset MODULE parameterset.cc) pybind11_add_module(fitters MODULE pyfitters.cc) foreach(PYLIB pydp3 parameterset fitters) - target_link_libraries(${PYLIB} PRIVATE LIBDP3) + target_link_libraries(${PYLIB} PRIVATE LIBEDGEDP3) # Output the library/module in the pseudo install directory. set_target_properties(${PYLIB} PROPERTIES LIBRARY_OUTPUT_DIRECTORY @@ -30,7 +30,8 @@ endforeach() target_link_libraries(fitters PRIVATE xtensor) install(TARGETS parameterset pydp3 fitters - DESTINATION "${PYTHON_INSTALL_DIR}/dp3") -install(FILES __init__.py DESTINATION "${PYTHON_INSTALL_DIR}/dp3") -install(FILES steps/__init__.py DESTINATION "${PYTHON_INSTALL_DIR}/dp3/steps") -install(FILES steps/queue.py DESTINATION "${PYTHON_INSTALL_DIR}/dp3/steps") + DESTINATION "${PYTHON_INSTALL_DIR}/edgedp3") +install(FILES __init__.py DESTINATION "${PYTHON_INSTALL_DIR}/edgedp3") +install(FILES steps/__init__.py + DESTINATION "${PYTHON_INSTALL_DIR}/edgedp3/steps") +install(FILES steps/queue.py DESTINATION "${PYTHON_INSTALL_DIR}/edgedp3/steps") diff --git a/scripts/test/testconfig.py.in b/scripts/test/testconfig.py.in index 9f00d94174430f44c591a3b4a932df40911cc900..d8faab5c3ca15288396d4506ad0a202854900b29 100644 --- a/scripts/test/testconfig.py.in +++ b/scripts/test/testconfig.py.in @@ -5,9 +5,9 @@ SRCDIR = "@CMAKE_CURRENT_SOURCE_DIR@" BINDIR = "@CMAKE_BINARY_DIR@" RESOURCEDIR = "@CMAKE_SOURCE_DIR@/resources" DDECAL_RESOURCEDIR = "@CMAKE_SOURCE_DIR@/ddecal/test/integration/resources/" -DP3EXE = "@CMAKE_BINARY_DIR@/DP3" -MAKESOURCEDBEXE = "@CMAKE_BINARY_DIR@/makesourcedb" -SHOWSOURCEDBEXE = "@CMAKE_BINARY_DIR@/showsourcedb" +DP3EXE = "@CMAKE_BINARY_DIR@/EDGEDP3" +MAKESOURCEDBEXE = "@CMAKE_BINARY_DIR@/edgemakesourcedb" +SHOWSOURCEDBEXE = "@CMAKE_BINARY_DIR@/edgeshowsourcedb" TAQLEXE = "@TAQL_EXECUTABLE@" PYTHONDIR = "@CMAKE_BINARY_DIR@/pythondp3/" PYTHONMOCKDIR = "@CMAKE_SOURCE_DIR@/pythondp3/test/unit/mock" \ No newline at end of file diff --git a/setup.py b/setup.py index 784de04f81625f2447f5c3f8e44cd1b1ffaaa3b2..28279c82adb0a14764706b1a4f6df0d73706ebfa 100755 --- a/setup.py +++ b/setup.py @@ -110,16 +110,16 @@ class CMakeBuild(build_ext): # The information here can also be placed in setup.cfg - better separation of # logic and declaration, and simpler if you include description/version in a file. setup( - name="DP3", + name="EdgeDP3", version="6.1.0", author="Astron", - author_email="dijkema@astron.nl", - description="DP3", - long_description="LOFAR preprocessing software, including averaging, flagging, various kinds of calibration and more.", - ext_modules=[CMakeExtension("dp3")], + author_email="mancini@astron.nl", + description="EdgeDP3", + long_description="Experimental LOFAR preprocessing software, including averaging, flagging, various kinds of calibration and more.", + ext_modules=[CMakeExtension("edgedp3")], cmdclass={"build_ext": CMakeBuild}, zip_safe=False, - python_requires=">=3.6", + python_requires=">=3.8", data_files=[ ( "../../DP3.libs/aoflagger/strategies", @@ -133,7 +133,7 @@ setup( ["build/__DP3_from_pip__"], ), ], - url="https://dp3.readthedocs.io/", + url="https://git.astron.nl/mancini/edgedp3.git", classifiers=[ "Development Status :: 5 - Production/Stable", "License :: OSI Approved :: GNU General Public License v3 (GPLv3)",