From 1f9205c635197199a56c1fda933e6ebf616ed546 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Josu=C3=A9=20M=2E=20Sehnem?= Date: Thu, 25 Apr 2024 16:57:31 -0300 Subject: [PATCH] Remove submodules, streamline build process (#51) Co-authored-by: tcmetzger <39711796+tcmetzger@users.noreply.github.com> --- .github/workflows/conda.yml | 3 +- .gitmodules | 6 -- CMakeLists.txt | 78 +++++++++++++------ README.md | 46 ++++------- build_rte_rrtmgp.sh | 9 --- conda.recipe/meta.yaml | 14 ++-- pyproject.toml | 2 +- pyrte_rrtmgp/rrtmgp_data.py | 61 +++++++++++++++ rrtmgp-data | 1 - rte-rrtmgp | 1 - tests/test_python_frontend/test_gas_optics.py | 3 +- tests/test_python_frontend/test_lw_solver.py | 3 +- tests/test_python_frontend/test_sw_solver.py | 5 +- 13 files changed, 147 insertions(+), 85 deletions(-) delete mode 100644 .gitmodules delete mode 100755 build_rte_rrtmgp.sh create mode 100644 pyrte_rrtmgp/rrtmgp_data.py delete mode 160000 rrtmgp-data delete mode 160000 rte-rrtmgp diff --git a/.github/workflows/conda.yml b/.github/workflows/conda.yml index 53bd7d5..dd019d8 100644 --- a/.github/workflows/conda.yml +++ b/.github/workflows/conda.yml @@ -18,7 +18,7 @@ jobs: strategy: fail-fast: false matrix: - platform: [ubuntu-latest] + platform: [ubuntu-latest, macos-latest] python-version: ["3.11"] runs-on: ${{ matrix.platform }} @@ -39,6 +39,7 @@ jobs: - name: Get conda uses: conda-incubator/setup-miniconda@v3.0.0 with: + miniconda-version: "latest" python-version: ${{ matrix.python-version }} channels: conda-forge diff --git a/.gitmodules b/.gitmodules deleted file mode 100644 index 53cf4ea..0000000 --- a/.gitmodules +++ /dev/null @@ -1,6 +0,0 @@ -[submodule "rte-rrtmgp"] - path = rte-rrtmgp - url = https://github.com/earth-system-radiation/rte-rrtmgp.git -[submodule "rrtmgp-data"] - path = rrtmgp-data - url = https://github.com/earth-system-radiation/rrtmgp-data.git diff --git a/CMakeLists.txt b/CMakeLists.txt index 4debb39..1834d3f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -9,43 +9,65 @@ if (UNIX AND NOT APPLE) set(LINUX TRUE) endif() -# Custom command for the compile-rte-rrtmgp target -add_custom_command( - TARGET compile-rte-rrtmgp - COMMAND cd ${CMAKE_SOURCE_DIR} && - ./build_rte_rrtmgp.sh && - mv rte-rrtmgp/build/*.a ${CMAKE_CURRENT_BINARY_DIR} && - mv rte-rrtmgp/build/*.o ${CMAKE_CURRENT_BINARY_DIR} && - mv rte-rrtmgp/build/*.mod ${CMAKE_CURRENT_BINARY_DIR} -) +# Check if FC environment variable is set +if(DEFINED ENV{FC}) + set(CMAKE_Fortran_COMPILER $ENV{FC}) + message(STATUS "Using Fortran compiler from FC environment variable: $ENV{FC}") +else() + # Define a list of preferred Fortran compilers + set(PREFERRED_FC_COMPILERS gfortran ifort gfortran-10 gfortran-11 f77) -# Compile C bindings -set(HEADERS - ${CMAKE_SOURCE_DIR}/rte-rrtmgp/rte-kernels/api/rte_kernels.h - ${CMAKE_SOURCE_DIR}/rte-rrtmgp/rrtmgp-kernels/api/rrtmgp_kernels.h + foreach(compiler IN LISTS PREFERRED_FC_COMPILERS) + find_program(FOUND_COMPILER NAMES ${compiler}) + if(FOUND_COMPILER) + set(CMAKE_Fortran_COMPILER ${FOUND_COMPILER}) + message(STATUS "Using Fortran compiler: ${FOUND_COMPILER}") + break() + endif() + endforeach() +endif() + +if(NOT CMAKE_Fortran_COMPILER) + message(FATAL_ERROR "No suitable Fortran compiler found") +endif() + +include(ExternalProject REQUIRED) + +if(CMAKE_HOST_SYSTEM_NAME STREQUAL "Windows") + set(BUILD_COMMAND_STRING "set FC=%CMAKE_Fortran_COMPILER% && cd build && nmake /A") +else() + set(BUILD_COMMAND_STRING "FC=${CMAKE_Fortran_COMPILER} make -C build -j ${N}") +endif() + +ExternalProject_Add( + rte-rrtmgp + GIT_REPOSITORY https://github.com/earth-system-radiation/rte-rrtmgp.git + GIT_TAG origin/develop + GIT_SHALLOW TRUE + SOURCE_DIR ${CMAKE_CURRENT_BINARY_DIR}/rte-rrtmgp + CONFIGURE_COMMAND "" + BUILD_IN_SOURCE TRUE + BUILD_COMMAND eval ${BUILD_COMMAND_STRING} + INSTALL_COMMAND "" ) -set(SOURCES ${CMAKE_SOURCE_DIR}/pybind_interface.cpp) -set(TARGET_NAME pyrte_rrtmgp) +# Compile C bindings find_package(pybind11 REQUIRED) -pybind11_add_module(${TARGET_NAME} ${SOURCES} ${HEADERS}) +set(TARGET_NAME pyrte_rrtmgp) +set(SOURCES ${CMAKE_SOURCE_DIR}/pybind_interface.cpp) -target_compile_definitions(${TARGET_NAME} PRIVATE - VERSION_INFO=${VERSION_INFO} - DBL_EPSILON=5.8e-2 - DCMAKE_LIBRARY_OUTPUT_DIRECTORY=pyrte_rrtmgp -) +pybind11_add_module(${TARGET_NAME} ${SOURCES}) -add_dependencies(${TARGET_NAME} compile-rte-rrtmgp) +add_dependencies(${TARGET_NAME} rte-rrtmgp) target_include_directories(${TARGET_NAME} PUBLIC - ${CMAKE_SOURCE_DIR}/rte-rrtmgp/rte-kernels/api/ - ${CMAKE_SOURCE_DIR}/rte-rrtmgp/rrtmgp-kernels/api/ + ${CMAKE_CURRENT_BINARY_DIR}/rte-rrtmgp/rte-kernels/api/ + ${CMAKE_CURRENT_BINARY_DIR}/rte-rrtmgp/rrtmgp-kernels/api/ ) target_link_directories(${TARGET_NAME} PUBLIC - ${CMAKE_CURRENT_BINARY_DIR} + ${CMAKE_CURRENT_BINARY_DIR}/rte-rrtmgp/build ) target_link_libraries(${TARGET_NAME} PUBLIC @@ -53,6 +75,12 @@ target_link_libraries(${TARGET_NAME} PUBLIC rte ) +target_compile_definitions(${TARGET_NAME} PRIVATE + VERSION_INFO=${VERSION_INFO} + DBL_EPSILON=5.8e-2 + DCMAKE_LIBRARY_OUTPUT_DIRECTORY=pyrte_rrtmgp +) + if (${LINUX}) target_link_libraries(${TARGET_NAME} PUBLIC gfortran) endif() diff --git a/README.md b/README.md index 29b0f2b..8bf1f76 100644 --- a/README.md +++ b/README.md @@ -106,12 +106,14 @@ Currently, the following functions are available in the `pyrte_rrtmgp` package: ### Prerequisites -pyRTE-RRTMGP is currently only tested on x86_64 architecture with Linux (and, to some extent, macOS on Intel processors). +pyRTE-RRTMGP is built on install time. Building the package requires a compatible Fortran compiler, a C++ compiler and CMake to be installed on your system. The package is compatible with POSIX systems and is tested on Linux and macOS using the GNU Fortran compiler (gfortran) and the GNU C++ compiler (g++). The package should also work with the Intel Fortran compiler (ifort) but was not tested with it. If you use ``conda``, the system packages are installed automatically. If you use ``pip``, you need to install those packages yourself (see below). -The package source code is hosted [on GitHub](https://github.com/earth-system-radiation/pyRTE-RRTMGP) and uses git submodules to include the [RTE+RRTMGP Fortran software](https://earth-system-radiation.github.io/rte-rrtmgp/). The easiest way to install pyRTE-RRTMGP is to use `git`. You can install git from [here](https://git-scm.com/downloads). +The package source code is hosted [on GitHub](https://github.com/earth-system-radiation/pyRTE-RRTMGP). The easiest way to install pyRTE-RRTMGP is to use `git`. You can install git from [here](https://git-scm.com/downloads). ### Installation with conda (recommended) +Using conda is the recommended method because conda will take care of the system dependencies for you. + 1. **Clone the repository**: ```bash @@ -130,13 +132,7 @@ The package source code is hosted [on GitHub](https://github.com/earth-system-ra cd pyRTE-RRTMGP ``` -2. **Update the submodules**: - - ```bash - git submodule update --init --recursive - ``` - -3. **Make sure you have conda installed**. If not, you can install it from [here](https://docs.conda.io/en/latest/miniconda.html). +2. **Make sure you have conda installed**. If not, you can install it from [here](https://docs.conda.io/en/latest/miniconda.html). To make sure your conda setup is working, run the command below: ```bash @@ -145,19 +141,19 @@ The package source code is hosted [on GitHub](https://github.com/earth-system-ra If this runs without errors, you are good to go. -4. **Install the conda build requirements** (if you haven't already): +3. **Install the conda build requirements** (if you haven't already): ```bash conda install conda-build conda-verify ``` -5. **Build the conda package locally**: +4. **Build the conda package locally**: ```bash conda build conda.recipe ``` -6. **Install the package** in your current conda environment: +5. **Install the package** in your current conda environment: ```bash conda install -c ${CONDA_PREFIX}/conda-bld/ pyrte_rrtmgp @@ -167,31 +163,23 @@ The package source code is hosted [on GitHub](https://github.com/earth-system-ra ### Installation with pip -You also have the option to build and install the package with pip. This might work on additional, untested architectures (such as macOS on M1). However, this is not recommended as it requires you to have a working Fortran compiler and other prerequisites installed on your system. +You also have the option to build and install the package with pip. This should work with macOS and Linux systems but requires you to install the system dependencies manually. -1. **Clone the repository** (``git clone git@github.com:earth-system-radiation/pyRTE-RRTMGP.git``) and update the submodules (``git submodule update --init --recursive``) as described in the conda installation instructions above. +#### Mac OS -2. **Install the necessary dependencies** for your operating system. +1. **Install the requirements** On MacOS systems you can use `brew` to install the dependencies as following `brew install git gcc cmake`, but you can also install de requirements using other package managers, such as conda. - With Ubuntu, for example, you would use: +2. **Install the package** Then you can install the package directly from the git repository `pip install git+https://github.com/earth-system-radiation/pyRTE-RRTMGP` - ``` bash - sudo apt install -y \ - libnetcdff-dev \ - gfortran-10 \ - python3-dev \ - cmake - ``` +#### Debian/Ubuntu - On other systems, you might be able to install the necessary dependencies with a package manager like `brew`. +1. **Install the requirements** On Debian/Ubuntu systems you can use `apt` to install the dependencies as following `sudo apt install build-essential gfortran cmake git`, but you can also install de requirements using other package managers, such as conda. -3. **Compile the Fortran code and build and install the Python package** in your current environment with: +2. **Install the package** Then you can install the package directly from the git repository `pip install git+https://github.com/earth-system-radiation/pyRTE-RRTMGP` - ``` bash - pip install . - ``` +Other linux distributions should also support the installation of the package, you just need to install the dependencies using the package manager of your distribution. - For development purposes, you can install the package in editable mode: ``pip install -e .``. +For development purposes, you can install the package in editable mode: ``pip install -e .``. Once built, the module will be located in a folder called `pyrte_rrtmgp` diff --git a/build_rte_rrtmgp.sh b/build_rte_rrtmgp.sh deleted file mode 100755 index 05a0972..0000000 --- a/build_rte_rrtmgp.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/sh - -if [ -n "$CONDA_PREFIX" ] && [ "$OS" = *"Windows"* ]; then - export FC=gfortran -elif [ -z "$CONDA_PREFIX" ]; then - export FC=gfortran -fi - -make -C rte-rrtmgp/build -j 2 diff --git a/conda.recipe/meta.yaml b/conda.recipe/meta.yaml index 9d077cd..ed61349 100644 --- a/conda.recipe/meta.yaml +++ b/conda.recipe/meta.yaml @@ -14,19 +14,15 @@ build: requirements: build: - - gcc_linux-64 # [linux] - - gxx_linux-64 # [linux] + - {{ compiler('c') }} + - {{ compiler('cxx') }} - gfortran_linux-64 # [linux] - - clang_osx-64 # [osx] - - clangxx_osx-64 # [osx] - gfortran_osx-64 # [osx] - - binutils # [not win] - m2w64-gcc-fortran # [win] - - m2w64-gcc # [win] - - m2w64-binutils # [win] - libuv==1.44.2 - cmake==3.26.4 - - make + - git==2.44.0 + - make==4.3 host: - python @@ -40,6 +36,7 @@ requirements: - numpy >=1.21.0 - xarray >=2023.5.0 - netcdf4 >=1.5.7 + - requests>=2.4.0 test: imports: @@ -49,6 +46,7 @@ test: - pytest>=7.4 - xarray >=2023.5.0 - netcdf4 >=1.5.7 + - requests>=2.4.0 source_files: - tests commands: diff --git a/pyproject.toml b/pyproject.toml index 26f6959..5e5fa93 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,7 +24,7 @@ build-backend = "scikit_build_core.build" [project.optional-dependencies] -test = ["pytest", "numpy>=1.21.0", "xarray>=2023.5.0", "netcdf4>=1.5.7"] +test = ["pytest", "numpy>=1.21.0", "xarray>=2023.5.0", "netcdf4>=1.5.7", "requests>=2.4.0"] [tool.scikit-build] diff --git a/pyrte_rrtmgp/rrtmgp_data.py b/pyrte_rrtmgp/rrtmgp_data.py new file mode 100644 index 0000000..69089aa --- /dev/null +++ b/pyrte_rrtmgp/rrtmgp_data.py @@ -0,0 +1,61 @@ +import hashlib +import os +import platform +import tarfile + +import requests + +# URL of the file to download +TAG = "v1.8" +DATA_URL = f"https://github.com/earth-system-radiation/rrtmgp-data/archive/refs/tags/{TAG}.tar.gz" + + +def get_cache_dir(): + # Determine the system cache folder + if platform.system() == "Windows": + cache_path = os.getenv("LOCALAPPDATA") + elif platform.system() == "Darwin": + cache_path = os.path.expanduser("~/Library/Caches") + else: + cache_path = os.getenv("XDG_CACHE_HOME", os.path.expanduser("~/.cache")) + cache_path = os.path.join(cache_path, "pyrte_rrtmgp") + + # Create the directory if it doesn't exist + if not os.path.exists(cache_path): + os.makedirs(cache_path) + + return cache_path + + +def download_rrtmgp_data(): + # Directory where the data will be stored + cache_dir = get_cache_dir() + + # Path to the downloaded file + file_path = os.path.join(cache_dir, f"{TAG}.tar.gz") + + # Path to the file containing the checksum of the downloaded file + checksum_file_path = os.path.join(cache_dir, f"{TAG}.tar.gz.sha256") + + # Download the file if it doesn't exist or if the checksum doesn't match + if not os.path.exists(file_path) or ( + os.path.exists(checksum_file_path) + and open(checksum_file_path).read() + != hashlib.sha256(open(file_path, "rb").read()).hexdigest() + ): + response = requests.get(DATA_URL, stream=True) + response.raise_for_status() + + with open(file_path, "wb") as f: + for chunk in response.iter_content(chunk_size=8192): + f.write(chunk) + + # Save the checksum of the downloaded file + with open(checksum_file_path, "w") as f: + f.write(hashlib.sha256(open(file_path, "rb").read()).hexdigest()) + + # Uncompress the file + with tarfile.open(file_path) as tar: + tar.extractall(path=cache_dir) + + return os.path.join(cache_dir, f"rrtmgp-data-{TAG[1:]}") diff --git a/rrtmgp-data b/rrtmgp-data deleted file mode 160000 index df02975..0000000 --- a/rrtmgp-data +++ /dev/null @@ -1 +0,0 @@ -Subproject commit df02975ab93165b34a59f0d04b4ae6148fe5127c diff --git a/rte-rrtmgp b/rte-rrtmgp deleted file mode 160000 index 8a955cb..0000000 --- a/rte-rrtmgp +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 8a955cbf8d8cd2d08d81019d6ccd7d55bc50f5ec diff --git a/tests/test_python_frontend/test_gas_optics.py b/tests/test_python_frontend/test_gas_optics.py index 35e5707..85cd6c5 100644 --- a/tests/test_python_frontend/test_gas_optics.py +++ b/tests/test_python_frontend/test_gas_optics.py @@ -4,6 +4,7 @@ import pytest import xarray as xr from pyrte_rrtmgp import rrtmgp_gas_optics +from pyrte_rrtmgp.rrtmgp_data import download_rrtmgp_data from pyrte_rrtmgp.kernels.rrtmgp import ( compute_planck_source, compute_tau_absorption, @@ -15,7 +16,7 @@ ERROR_TOLERANCE = 1e-4 -rte_rrtmgp_dir = os.environ.get("RRTMGP_DATA", "rrtmgp-data") +rte_rrtmgp_dir = download_rrtmgp_data() clear_sky_example_files = f"{rte_rrtmgp_dir}/examples/rfmip-clear-sky/inputs" rfmip = xr.load_dataset( diff --git a/tests/test_python_frontend/test_lw_solver.py b/tests/test_python_frontend/test_lw_solver.py index 3cf3aa7..878bb96 100644 --- a/tests/test_python_frontend/test_lw_solver.py +++ b/tests/test_python_frontend/test_lw_solver.py @@ -3,11 +3,12 @@ import numpy as np import xarray as xr from pyrte_rrtmgp import rrtmgp_gas_optics +from pyrte_rrtmgp.rrtmgp_data import download_rrtmgp_data from pyrte_rrtmgp.kernels.rte import lw_solver_noscat ERROR_TOLERANCE = 1e-4 -rte_rrtmgp_dir = os.environ.get("RRTMGP_DATA", "rrtmgp-data") +rte_rrtmgp_dir = download_rrtmgp_data() clear_sky_example_files = f"{rte_rrtmgp_dir}/examples/rfmip-clear-sky/inputs" rfmip = xr.load_dataset( diff --git a/tests/test_python_frontend/test_sw_solver.py b/tests/test_python_frontend/test_sw_solver.py index d10984a..940d887 100644 --- a/tests/test_python_frontend/test_sw_solver.py +++ b/tests/test_python_frontend/test_sw_solver.py @@ -4,12 +4,13 @@ import pytest import xarray as xr from pyrte_rrtmgp import rrtmgp_gas_optics +from pyrte_rrtmgp.rrtmgp_data import download_rrtmgp_data from pyrte_rrtmgp.kernels.rte import sw_solver_2stream from pyrte_rrtmgp.utils import compute_mu0, compute_toa_flux, get_usecols ERROR_TOLERANCE = 1e-4 -rte_rrtmgp_dir = os.environ.get("RRTMGP_DATA", "rrtmgp-data") +rte_rrtmgp_dir = download_rrtmgp_data() clear_sky_example_files = f"{rte_rrtmgp_dir}/examples/rfmip-clear-sky/inputs" rfmip = xr.load_dataset( @@ -29,7 +30,7 @@ ref_flux_down = rsd.isel(expt=0)["rsd"].values -def test_lw_solver_noscat(): +def test_sw_solver_noscat(): gas_optics = kdist.gas_optics.load_atmosferic_conditions(rfmip) surface_albedo = rfmip["surface_albedo"].data