Remove submodules, streamline build process (#51)

Co-authored-by: tcmetzger <[email protected]>

.github/workflows/conda.yml

@@ -18,7 +18,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        platform: [ubuntu-latest]
+        platform: [ubuntu-latest, macos-latest]
         python-version: ["3.11"]
 
     runs-on: ${{ matrix.platform }}
@@ -39,6 +39,7 @@ jobs:
       - name: Get conda
         uses: conda-incubator/[email protected]
         with:
+          miniconda-version: "latest"
           python-version: ${{ matrix.python-version }}
           channels: conda-forge
 

CMakeLists.txt

@@ -9,50 +9,78 @@ if (UNIX AND NOT APPLE)
     set(LINUX TRUE)
 endif()
 
-# Custom command for the compile-rte-rrtmgp target
-add_custom_command(
-    TARGET compile-rte-rrtmgp
-    COMMAND cd ${CMAKE_SOURCE_DIR} && 
-            ./build_rte_rrtmgp.sh &&
-            mv rte-rrtmgp/build/*.a ${CMAKE_CURRENT_BINARY_DIR} && 
-            mv rte-rrtmgp/build/*.o ${CMAKE_CURRENT_BINARY_DIR} && 
-            mv rte-rrtmgp/build/*.mod ${CMAKE_CURRENT_BINARY_DIR}
-)
+# Check if FC environment variable is set
+if(DEFINED ENV{FC})
+    set(CMAKE_Fortran_COMPILER $ENV{FC})
+    message(STATUS "Using Fortran compiler from FC environment variable: $ENV{FC}")
+else()
+    # Define a list of preferred Fortran compilers
+    set(PREFERRED_FC_COMPILERS gfortran ifort gfortran-10 gfortran-11 f77)
 
-# Compile C bindings
-set(HEADERS 
-    ${CMAKE_SOURCE_DIR}/rte-rrtmgp/rte-kernels/api/rte_kernels.h
-    ${CMAKE_SOURCE_DIR}/rte-rrtmgp/rrtmgp-kernels/api/rrtmgp_kernels.h
+    foreach(compiler IN LISTS PREFERRED_FC_COMPILERS)
+        find_program(FOUND_COMPILER NAMES ${compiler})
+        if(FOUND_COMPILER)
+            set(CMAKE_Fortran_COMPILER ${FOUND_COMPILER})
+            message(STATUS "Using Fortran compiler: ${FOUND_COMPILER}")
+            break()
+        endif()
+    endforeach()
+endif()
+
+if(NOT CMAKE_Fortran_COMPILER)
+    message(FATAL_ERROR "No suitable Fortran compiler found")
+endif()
+
+include(ExternalProject REQUIRED)
+
+if(CMAKE_HOST_SYSTEM_NAME STREQUAL "Windows")
+    set(BUILD_COMMAND_STRING "set FC=%CMAKE_Fortran_COMPILER% && cd build && nmake /A")
+else()
+    set(BUILD_COMMAND_STRING "FC=${CMAKE_Fortran_COMPILER} make -C build -j ${N}")
+endif()
+
+ExternalProject_Add(
+    rte-rrtmgp
+    GIT_REPOSITORY https://github.com/earth-system-radiation/rte-rrtmgp.git
+    GIT_TAG origin/develop
+    GIT_SHALLOW TRUE
+    SOURCE_DIR ${CMAKE_CURRENT_BINARY_DIR}/rte-rrtmgp
+    CONFIGURE_COMMAND ""
+    BUILD_IN_SOURCE TRUE
+    BUILD_COMMAND eval ${BUILD_COMMAND_STRING}
+    INSTALL_COMMAND ""
 )
-set(SOURCES ${CMAKE_SOURCE_DIR}/pybind_interface.cpp)
-set(TARGET_NAME pyrte_rrtmgp)
 
+# Compile C bindings
 find_package(pybind11 REQUIRED)
 
-pybind11_add_module(${TARGET_NAME} ${SOURCES} ${HEADERS})
+set(TARGET_NAME pyrte_rrtmgp)
+set(SOURCES ${CMAKE_SOURCE_DIR}/pybind_interface.cpp)
 
-target_compile_definitions(${TARGET_NAME} PRIVATE 
-    VERSION_INFO=${VERSION_INFO}
-    DBL_EPSILON=5.8e-2
-    DCMAKE_LIBRARY_OUTPUT_DIRECTORY=pyrte_rrtmgp
-)
+pybind11_add_module(${TARGET_NAME} ${SOURCES})
 
-add_dependencies(${TARGET_NAME} compile-rte-rrtmgp)
+add_dependencies(${TARGET_NAME} rte-rrtmgp)
 
 target_include_directories(${TARGET_NAME} PUBLIC 
-    ${CMAKE_SOURCE_DIR}/rte-rrtmgp/rte-kernels/api/
-    ${CMAKE_SOURCE_DIR}/rte-rrtmgp/rrtmgp-kernels/api/
+    ${CMAKE_CURRENT_BINARY_DIR}/rte-rrtmgp/rte-kernels/api/
+    ${CMAKE_CURRENT_BINARY_DIR}/rte-rrtmgp/rrtmgp-kernels/api/
 )
 
 target_link_directories(${TARGET_NAME} PUBLIC 
-    ${CMAKE_CURRENT_BINARY_DIR}
+    ${CMAKE_CURRENT_BINARY_DIR}/rte-rrtmgp/build
 )
 
 target_link_libraries(${TARGET_NAME} PUBLIC 
     rrtmgp
     rte
 )
 
+target_compile_definitions(${TARGET_NAME} PRIVATE 
+    VERSION_INFO=${VERSION_INFO}
+    DBL_EPSILON=5.8e-2
+    DCMAKE_LIBRARY_OUTPUT_DIRECTORY=pyrte_rrtmgp
+)
+
 if (${LINUX})
     target_link_libraries(${TARGET_NAME} PUBLIC gfortran)
 endif()

README.md

@@ -106,12 +106,14 @@ Currently, the following functions are available in the `pyrte_rrtmgp` package:
 
 ### Prerequisites
 
-pyRTE-RRTMGP is currently only tested on x86_64 architecture with Linux (and, to some extent, macOS on Intel processors).
+pyRTE-RRTMGP is built on install time. Building the package requires a compatible Fortran compiler, a C++ compiler and CMake to be installed on your system. The package is compatible with POSIX systems and is tested on Linux and macOS using the GNU Fortran compiler (gfortran) and the GNU C++ compiler (g++). The package should also work with the Intel Fortran compiler (ifort) but was not tested with it. If you use ``conda``, the system packages are installed automatically. If you use ``pip``, you need to install those packages yourself (see below).
 
-The package source code is hosted [on GitHub](https://github.com/earth-system-radiation/pyRTE-RRTMGP) and uses git submodules to include the [RTE+RRTMGP Fortran software](https://earth-system-radiation.github.io/rte-rrtmgp/). The easiest way to install pyRTE-RRTMGP is to use `git`. You can install git from [here](https://git-scm.com/downloads).
+The package source code is hosted [on GitHub](https://github.com/earth-system-radiation/pyRTE-RRTMGP). The easiest way to install pyRTE-RRTMGP is to use `git`. You can install git from [here](https://git-scm.com/downloads).
 
 ### Installation with conda (recommended)
 
+Using conda is the recommended method because conda will take care of the system dependencies for you.
+
 1. **Clone the repository**:
 
     ```bash
@@ -130,13 +132,7 @@ The package source code is hosted [on GitHub](https://github.com/earth-system-ra
     cd pyRTE-RRTMGP
     ```
 
-2. **Update the submodules**:
-
-    ```bash
-    git submodule update --init --recursive
-    ```
-
-3. **Make sure you have conda installed**. If not, you can install it from [here](https://docs.conda.io/en/latest/miniconda.html).
+2. **Make sure you have conda installed**. If not, you can install it from [here](https://docs.conda.io/en/latest/miniconda.html).
     To make sure your conda setup is working, run the command below:
 
     ```bash
@@ -145,19 +141,19 @@ The package source code is hosted [on GitHub](https://github.com/earth-system-ra
 
     If this runs without errors, you are good to go.
 
-4. **Install the conda build requirements** (if you haven't already):
+3. **Install the conda build requirements** (if you haven't already):
 
     ```bash
     conda install conda-build conda-verify
     ```
 
-5. **Build the conda package locally**:
+4. **Build the conda package locally**:
 
     ```bash
     conda build conda.recipe
     ```
 
-6. **Install the package** in your current conda environment:
+5. **Install the package** in your current conda environment:
 
     ```bash
     conda install -c ${CONDA_PREFIX}/conda-bld/ pyrte_rrtmgp
@@ -167,31 +163,23 @@ The package source code is hosted [on GitHub](https://github.com/earth-system-ra
 
 ### Installation with pip
 
-You also have the option to build and install the package with pip. This might work on additional, untested architectures (such as macOS on M1). However, this is not recommended as it requires you to have a working Fortran compiler and other prerequisites installed on your system.
+You also have the option to build and install the package with pip. This should work with macOS and Linux systems but requires you to install the system dependencies manually.
 
-1. **Clone the repository** (``git clone [email protected]:earth-system-radiation/pyRTE-RRTMGP.git``) and update the submodules (``git submodule update --init --recursive``) as described in the conda installation instructions above.
+#### Mac OS
 
-2. **Install the necessary dependencies** for your operating system.
+1. **Install the requirements** On MacOS systems you can use `brew` to install the dependencies as following `brew install git gcc cmake`, but you can also install de requirements using other package managers, such as conda.
 
-    With Ubuntu, for example, you would use:
+2. **Install the package** Then you can install the package directly from the git repository `pip install git+https://github.com/earth-system-radiation/pyRTE-RRTMGP`
 
-    ``` bash
-    sudo apt install -y \
-        libnetcdff-dev \
-        gfortran-10 \
-        python3-dev \
-        cmake
-    ```
+#### Debian/Ubuntu
 
-    On other systems, you might be able to install the necessary dependencies with a package manager like `brew`.
+1. **Install the requirements** On Debian/Ubuntu systems you can use `apt` to install the dependencies as following `sudo apt install build-essential gfortran cmake git`, but you can also install de requirements using other package managers, such as conda.
 
-3. **Compile the Fortran code and build and install the Python package** in your current environment with:
+2. **Install the package** Then you can install the package directly from the git repository `pip install git+https://github.com/earth-system-radiation/pyRTE-RRTMGP`
 
-    ``` bash
-    pip install .
-    ```
+Other linux distributions should also support the installation of the package, you just need to install the dependencies using the package manager of your distribution.
 
-    For development purposes, you can install the package in editable mode: ``pip install -e .``.
+For development purposes, you can install the package in editable mode: ``pip install -e .``.
 
 Once built, the module will be located in a folder called `pyrte_rrtmgp`
 

conda.recipe/meta.yaml

@@ -14,19 +14,15 @@ build:
 
 requirements:
   build:
-    - gcc_linux-64        # [linux]
-    - gxx_linux-64        # [linux]
+    - {{ compiler('c') }}
+    - {{ compiler('cxx') }}
     - gfortran_linux-64   # [linux]
-    - clang_osx-64        # [osx]
-    - clangxx_osx-64      # [osx]
     - gfortran_osx-64     # [osx]
-    - binutils            # [not win]
     - m2w64-gcc-fortran   # [win]
-    - m2w64-gcc           # [win]
-    - m2w64-binutils      # [win]
     - libuv==1.44.2
     - cmake==3.26.4
-    - make
+    - git==2.44.0
+    - make==4.3
 
   host:
     - python
@@ -40,6 +36,7 @@ requirements:
     - numpy >=1.21.0
     - xarray >=2023.5.0
     - netcdf4 >=1.5.7
+    - requests>=2.4.0
 
 test:
   imports:
@@ -49,6 +46,7 @@ test:
     - pytest>=7.4
     - xarray >=2023.5.0
     - netcdf4 >=1.5.7
+    - requests>=2.4.0
   source_files:
     - tests
   commands:

pyproject.toml

@@ -24,7 +24,7 @@ build-backend = "scikit_build_core.build"
 
 
 [project.optional-dependencies]
-test = ["pytest", "numpy>=1.21.0", "xarray>=2023.5.0", "netcdf4>=1.5.7"]
+test = ["pytest", "numpy>=1.21.0", "xarray>=2023.5.0", "netcdf4>=1.5.7", "requests>=2.4.0"]
 
 
 [tool.scikit-build]

pyrte_rrtmgp/rrtmgp_data.py

@@ -0,0 +1,61 @@
+import hashlib
+import os
+import platform
+import tarfile
+
+import requests
+
+# URL of the file to download
+TAG = "v1.8"
+DATA_URL = f"https://github.com/earth-system-radiation/rrtmgp-data/archive/refs/tags/{TAG}.tar.gz"
+
+
+def get_cache_dir():
+    # Determine the system cache folder
+    if platform.system() == "Windows":
+        cache_path = os.getenv("LOCALAPPDATA")
+    elif platform.system() == "Darwin":
+        cache_path = os.path.expanduser("~/Library/Caches")
+    else:
+        cache_path = os.getenv("XDG_CACHE_HOME", os.path.expanduser("~/.cache"))
+    cache_path = os.path.join(cache_path, "pyrte_rrtmgp")
+
+    # Create the directory if it doesn't exist
+    if not os.path.exists(cache_path):
+        os.makedirs(cache_path)
+
+    return cache_path
+
+
+def download_rrtmgp_data():
+    # Directory where the data will be stored
+    cache_dir = get_cache_dir()
+
+    # Path to the downloaded file
+    file_path = os.path.join(cache_dir, f"{TAG}.tar.gz")
+
+    # Path to the file containing the checksum of the downloaded file
+    checksum_file_path = os.path.join(cache_dir, f"{TAG}.tar.gz.sha256")
+
+    # Download the file if it doesn't exist or if the checksum doesn't match
+    if not os.path.exists(file_path) or (
+        os.path.exists(checksum_file_path)
+        and open(checksum_file_path).read()
+        != hashlib.sha256(open(file_path, "rb").read()).hexdigest()
+    ):
+        response = requests.get(DATA_URL, stream=True)
+        response.raise_for_status()
+
+        with open(file_path, "wb") as f:
+            for chunk in response.iter_content(chunk_size=8192):
+                f.write(chunk)
+
+        # Save the checksum of the downloaded file
+        with open(checksum_file_path, "w") as f:
+            f.write(hashlib.sha256(open(file_path, "rb").read()).hexdigest())
+
+    # Uncompress the file
+    with tarfile.open(file_path) as tar:
+        tar.extractall(path=cache_dir)
+
+    return os.path.join(cache_dir, f"rrtmgp-data-{TAG[1:]}")

tests/test_python_frontend/test_gas_optics.py

@@ -4,6 +4,7 @@
 import pytest
 import xarray as xr
 from pyrte_rrtmgp import rrtmgp_gas_optics
+from pyrte_rrtmgp.rrtmgp_data import download_rrtmgp_data
 from pyrte_rrtmgp.kernels.rrtmgp import (
     compute_planck_source,
     compute_tau_absorption,
@@ -15,7 +16,7 @@
 
 ERROR_TOLERANCE = 1e-4
 
-rte_rrtmgp_dir = os.environ.get("RRTMGP_DATA", "rrtmgp-data")
+rte_rrtmgp_dir = download_rrtmgp_data()
 clear_sky_example_files = f"{rte_rrtmgp_dir}/examples/rfmip-clear-sky/inputs"
 
 rfmip = xr.load_dataset(

tests/test_python_frontend/test_lw_solver.py

@@ -3,11 +3,12 @@
 import numpy as np
 import xarray as xr
 from pyrte_rrtmgp import rrtmgp_gas_optics
+from pyrte_rrtmgp.rrtmgp_data import download_rrtmgp_data
 from pyrte_rrtmgp.kernels.rte import lw_solver_noscat
 
 ERROR_TOLERANCE = 1e-4
 
-rte_rrtmgp_dir = os.environ.get("RRTMGP_DATA", "rrtmgp-data")
+rte_rrtmgp_dir = download_rrtmgp_data()
 clear_sky_example_files = f"{rte_rrtmgp_dir}/examples/rfmip-clear-sky/inputs"
 
 rfmip = xr.load_dataset(