Changed files to create docs env and build docs remotely on juwels (#251

)

* Changed files to create docs env and build docs remotely on juwels

* add docs extra to pyproject.toml

* Added updated information to README

* Grammar

* Trailing whitespaces (*melting emoji*)

---------

Co-authored-by: Anna Lappe <[email protected]>
Co-authored-by: Jarl Sondre Sæther <[email protected]>

.gitignore

@@ -42,6 +42,7 @@ mllogs/
 .logs/
 lightning_logs/
 mlruns/
+ray_checkpoints/
 
 # Byte-compiled / optimized / DLL files
 __pycache__/

.readthedocs.yaml

@@ -40,5 +40,5 @@ sphinx:
 python:
    install:
   #  - wheel
-   - requirements: docs/pre-requirements.txt
+   # - requirements: docs/pre-requirements.txt
    - requirements: docs/requirements.txt

docs/README.md

@@ -1,16 +1,11 @@
 # Read The Docs documentation page
 
-The python dependencies are organized in two requirements files, which
-must be installed in the following order:
-
-1. `pre-requirements.txt` contains torch and tensorflow.
-1. `requirements.txt` contains the packages which depend on torch and tensorflow,
-which should be installed *after* torch and tensorflow.
+The docs can be built either locally on your system or remotely on JSC.
 
 ## Build docs locally
 
-To build the docs locally and visualize them in your browser, without relying on external
-services (e.g., Read The Docs cloud), use the following commands
+To build the docs locally and visualize them in your browser without relying on external
+services (e.g., Read The Docs cloud), use the following commands:
 
 ```bash
 # Clone the repo, if not done yet
@@ -19,22 +14,24 @@ cd itwinai-docs
 
 # The first time, you may need to install some Linux packages (assuming Ubuntu system here)
 sudo apt update && sudo apt install libmysqlclient-dev
+sudo apt install pandoc
 sudo apt install python3-sphinx
 
 # Create a python virtual environment and install itwinai and its dependencies
 python3 -m venv .venv-docs
 source .venv-docs/bin/activate
-pip install -r docs/pre-requirements.txt
-pip install -r docs/requirements.txt
-pip install sphinx-rtd-theme
+
+# Choose the appropriate command for your OS here
+# pip install ".[torch,docs,macos]" 
+pip install ".[torch,docs,linux]"
 
 # Move to the docs folder and build them using Sphinx
 cd docs
 make clean
 make html
 
 # Serve a local HTTP server to navigate the newly created docs pages.
-# You can see the docs visiting http://localhost:8000 in your browser.
+# You can see the docs by visiting http://localhost:8000 in your browser.
 python -m http.server --directory  _build/html/
 ```
 
@@ -45,16 +42,15 @@ explained above. However, the environment setup must be slightly adapted to use
 some modules provided on the HPC system.
 
 To manage the docs, you can simply use the Makefile target
-belows.
+below.
 
 From the repository's root, create the docs virtual environment:
 
 ```bash
 make docs-env-jsc
 ```
 
-Once the environment is ready, build the docs
-and serve them on localhost:
+Once the environment is ready, build the docs and serve them on localhost:
 
 ```bash
 make docs-jsc

docs/requirements.txt

@@ -1,13 +1 @@
-sphinx-rtd-theme==2.0.0
-nbsphinx==0.9.4
-myst-parser==2.0.0
-
-git+https://github.com/thomas-bouvier/horovod.git@compile-cpp17
-deepspeed
-IPython
-# local path to itwinai module, assuming that pip install -r docs/requirements.txt is run form the repository root
-# If needed, you can add optional dependencies, like: ".[dev]"
-
-prov4ml[linux]@git+https://github.com/matbun/ProvML@main
-
-.[torch]
+.[torch,docs,linux]

env-files/docs/build-docs-jsc.sh

@@ -3,9 +3,7 @@
 # Build the documentation locally and serve it on localhost on JSC systems
 
 ml --force purge
-ml Stages/2024 GCC OpenMPI CUDA/12 cuDNN MPI-settings/CUDA
-ml Python CMake HDF5 PnetCDF libaio mpi4py
-ml Stages/2023 Pandoc/2.19.2
+ml Stages/2023  GCCcore/.11.3.0 Python/3.10.4 Pandoc/2.19.2
 
 source .venv-docs/bin/activate
 cd docs

env-files/docs/create-docs-env-jsc.sh

@@ -3,14 +3,13 @@
 # Create .venv-docs virtualenv to build the documentation locally on JSC systems
 
 ml --force purge
-ml Stages/2024 GCC OpenMPI CUDA/12 cuDNN MPI-settings/CUDA
-ml Python CMake HDF5 PnetCDF libaio mpi4py
+ml Stages/2023  GCCcore/.11.3.0 Python/3.10.4 Pandoc/2.19.2
 
 cmake --version
 gcc --version
 
 rm -rf .venv-docs
 python -m venv .venv-docs
 source .venv-docs/bin/activate
-pip install -r docs/pre-requirements.txt
+
 pip install -r docs/requirements.txt

pyproject.toml

@@ -43,18 +43,13 @@ dependencies = [
 # dynamic = ["version", "description"]
 
 [project.optional-dependencies]
-
-# Torch should be already installed, but it is specified here to
-# prevent itwinai dependencies from cheing its version.
-torch = ["lightning==2.*", "torchmetrics"]
-# torch-cpu = [
-#     "torch==2.1.*",
-#     "torchvision",
-#     "torchaudio",
-#     "lightning",
-#     "torchmetrics",
-#     "deepspeed",
-# ]
+torch = [
+  "torch==2.4.*",
+  "lightning==2.*",
+  "torchmetrics>=1.6.0",
+  "torchvision>=0.19.1",
+  "torchaudio>=2.4.1",
+]
 dev = [
     "pytest>=7.4.2",
     "pytest-mock>=3.11.1",
@@ -64,6 +59,19 @@ dev = [
     "isort",
     "tensorflow==2.16.*",  # needed by tests on tensorboard
 ]
+docs = [ 
+  "sphinx-rtd-theme==2.0.0",
+  "nbsphinx==0.9.4", 
+  "myst-parser==2.0.0",
+  "IPython",
+  "tensorflow==2.16.*",
+]
+macos = [
+    "prov4ml[apple]@git+https://github.com/matbun/ProvML"
+]
+linux = [
+    "prov4ml[linux]@git+https://github.com/matbun/ProvML"
+]
 
 [project.urls]
 Homepage = "https://www.intertwin.eu/"