From 662030eb34c3d98339c5aee3e873861d235eced1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 18 Nov 2024 12:55:04 +0000 Subject: [PATCH 01/10] Bump deptry from 0.21.0 to 0.21.1 in the python group Bumps the python group with 1 update: [deptry](https://github.com/fpgmaas/deptry). Updates `deptry` from 0.21.0 to 0.21.1 - [Release notes](https://github.com/fpgmaas/deptry/releases) - [Changelog](https://github.com/fpgmaas/deptry/blob/main/CHANGELOG.md) - [Commits](https://github.com/fpgmaas/deptry/compare/0.21.0...0.21.1) --- updated-dependencies: - dependency-name: deptry dependency-type: direct:production update-type: version-update:semver-patch dependency-group: python ... Signed-off-by: dependabot[bot] --- CI/requirements_ci.in | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CI/requirements_ci.in b/CI/requirements_ci.in index b95fdf60e..d4c6ec4ab 100644 --- a/CI/requirements_ci.in +++ b/CI/requirements_ci.in @@ -1,5 +1,5 @@ bump-my-version==0.28.1 -deptry==0.21.0 +deptry==0.21.1 flit==3.10.1 pip==24.3.1 pylint==3.3.1 diff --git a/pyproject.toml b/pyproject.toml index f4029a8db..91f4f18da 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -65,7 +65,7 @@ dev = [ "codespell ==2.3.0", "coverage[toml] >=7.5.0", "coveralls >=4.0.1", # coveralls is not yet compatible with Python 3.13 - "deptry ==0.21.0", + "deptry ==0.21.1", "flake8 >=7.1.1", "flake8-rst-docstrings >=0.3.0", "h5netcdf>=1.3.0", From dbef92087c5be6d1988f8d106b51097696689d4b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89ric=20Dupuis?= Date: Tue, 19 Nov 2024 10:22:55 -0500 Subject: [PATCH 02/10] only local changes for base_kws_vars --- CHANGELOG.rst | 1 + xclim/sdba/adjustment.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 15dd45bfe..c2b931d9e 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -14,6 +14,7 @@ Bug fixes ^^^^^^^^^ * Fixed pickling issue with ``xclim.sdba.Grouper`` and other classes for usage with `dask>=2024.11`. (:issue:`1992`, :pull:`1993`). * Fixed an issue with ``nimbus`` that was causing URL path components to be improperly joined. (:pull:`1997`). +* `base_kws_vars` in `MBCn` is now copied inside the `adjust` function so that in-place changes do not change the dict globally. Internal changes ^^^^^^^^^^^^^^^^ diff --git a/xclim/sdba/adjustment.py b/xclim/sdba/adjustment.py index 5c8506071..be1981d53 100644 --- a/xclim/sdba/adjustment.py +++ b/xclim/sdba/adjustment.py @@ -5,6 +5,7 @@ """ from __future__ import annotations +import copy from importlib.util import find_spec from inspect import signature from typing import Any @@ -1818,7 +1819,7 @@ def _adjust( period_dim=None, ): # set default values for non-specified parameters - base_kws_vars = base_kws_vars or {} + base_kws_vars = copy.deepcopy(base_kws_vars) or {} pts_dim = self.pts_dims[0] for v in sim[pts_dim].values: base_kws_vars.setdefault(v, {}) From 13965d913d794c8786f7cf4ca89211ec0aa2b932 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89ric=20Dupuis?= Date: Tue, 19 Nov 2024 10:26:02 -0500 Subject: [PATCH 03/10] add pull number --- CHANGELOG.rst | 2 +- xclim/sdba/adjustment.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index c2b931d9e..aed32045a 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -14,7 +14,7 @@ Bug fixes ^^^^^^^^^ * Fixed pickling issue with ``xclim.sdba.Grouper`` and other classes for usage with `dask>=2024.11`. (:issue:`1992`, :pull:`1993`). * Fixed an issue with ``nimbus`` that was causing URL path components to be improperly joined. (:pull:`1997`). -* `base_kws_vars` in `MBCn` is now copied inside the `adjust` function so that in-place changes do not change the dict globally. +* `base_kws_vars` in `MBCn` is now copied inside the `adjust` function so that in-place changes do not change the dict globally. (:pull:`1999`). Internal changes ^^^^^^^^^^^^^^^^ diff --git a/xclim/sdba/adjustment.py b/xclim/sdba/adjustment.py index be1981d53..5a1c5db01 100644 --- a/xclim/sdba/adjustment.py +++ b/xclim/sdba/adjustment.py @@ -5,7 +5,7 @@ """ from __future__ import annotations -import copy +from copy import deepcopy from importlib.util import find_spec from inspect import signature from typing import Any @@ -1819,7 +1819,7 @@ def _adjust( period_dim=None, ): # set default values for non-specified parameters - base_kws_vars = copy.deepcopy(base_kws_vars) or {} + base_kws_vars = deepcopy(base_kws_vars) or {} pts_dim = self.pts_dims[0] for v in sim[pts_dim].values: base_kws_vars.setdefault(v, {}) From d31f51ec73e76b1999a47eeb651dbe29e98721ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89ric=20Dupuis?= Date: Tue, 19 Nov 2024 10:27:26 -0500 Subject: [PATCH 04/10] check None first --- xclim/sdba/adjustment.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xclim/sdba/adjustment.py b/xclim/sdba/adjustment.py index 5a1c5db01..da5ab54fe 100644 --- a/xclim/sdba/adjustment.py +++ b/xclim/sdba/adjustment.py @@ -1819,7 +1819,7 @@ def _adjust( period_dim=None, ): # set default values for non-specified parameters - base_kws_vars = deepcopy(base_kws_vars) or {} + base_kws_vars = {} if base_kws_vars is None else deepcopy(base_kws_vars) pts_dim = self.pts_dims[0] for v in sim[pts_dim].values: base_kws_vars.setdefault(v, {}) From 6121ec70d2df28a88ba851b1942f645619783ca3 Mon Sep 17 00:00:00 2001 From: Ouranos Helper Bot Date: Tue, 19 Nov 2024 16:30:40 +0000 Subject: [PATCH 05/10] =?UTF-8?q?Bump=20version:=200.53.3-dev.2=20?= =?UTF-8?q?=E2=86=92=200.53.3-dev.3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pyproject.toml | 2 +- xclim/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 91f4f18da..dfd27d3d3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -134,7 +134,7 @@ target-version = [ ] [tool.bumpversion] -current_version = "0.53.3-dev.2" +current_version = "0.53.3-dev.3" commit = true commit_args = "--no-verify" tag = false diff --git a/xclim/__init__.py b/xclim/__init__.py index 9010e312d..6f5891522 100644 --- a/xclim/__init__.py +++ b/xclim/__init__.py @@ -13,7 +13,7 @@ __author__ = """Travis Logan""" __email__ = "logan.travis@ouranos.ca" -__version__ = "0.53.3-dev.2" +__version__ = "0.53.3-dev.3" with _resources.as_file(_resources.files("xclim.data")) as _module_data: From 89f169556f21599fa37cb7e70a991df4c1f50704 Mon Sep 17 00:00:00 2001 From: Trevor James Smith <10819524+Zeitsperre@users.noreply.github.com> Date: Wed, 20 Nov 2024 14:02:37 -0500 Subject: [PATCH 06/10] overload the fetch argument of nimbus to add a User-Agent --- xclim/testing/utils.py | 37 ++++++++++++++++++++++++++++++++++--- 1 file changed, 34 insertions(+), 3 deletions(-) diff --git a/xclim/testing/utils.py b/xclim/testing/utils.py index 25a626bd7..182a39adf 100644 --- a/xclim/testing/utils.py +++ b/xclim/testing/utils.py @@ -13,13 +13,14 @@ import sys import time import warnings -from collections.abc import Sequence +from collections.abc import Callable, Sequence from datetime import datetime as dt +from functools import wraps from importlib import import_module from io import StringIO from pathlib import Path from shutil import copytree -from typing import TextIO +from typing import IO, TextIO from urllib.error import HTTPError, URLError from urllib.parse import urljoin, urlparse from urllib.request import urlretrieve @@ -519,7 +520,8 @@ def nimbus( # noqa: PR01 remote = audit_url( urljoin(urljoin(repo, branch if branch.endswith("/") else f"{branch}/"), "data") ) - return pooch.create( + + _nimbus = pooch.create( path=cache_dir, base_url=remote, version=default_testdata_version, @@ -528,6 +530,35 @@ def nimbus( # noqa: PR01 registry=load_registry(branch=branch, repo=repo), ) + # Add a custom fetch method to the Pooch instance + # Needed to address: https://github.com/readthedocs/readthedocs.org/issues/11763 + # Fix inspired by @bjlittle (https://github.com/bjlittle/geovista/pull/1202) + _nimbus.fetch_diversion = _nimbus.fetch + + # Overload the fetch method to add user-agent headers + @wraps(_nimbus.fetch_diversion) + def _fetch(*args: str, **kwargs: bool | Callable) -> str: # numpydoc ignore=GL08 + + def _downloader( + url: str, + output_file: str | IO, + poocher: pooch.Pooch, + check_only: bool | None = False, + ) -> None: + """Download the file from the URL and save it to the save_path.""" + headers = {"User-Agent": f"xclim ({__xclim_version__})"} + downloader = pooch.HTTPDownloader(headers=headers) + return downloader(url, output_file, poocher, check_only=check_only) + + # default to our http/s downloader with user-agent headers + kwargs.setdefault("downloader", _downloader) + return _nimbus.fetch_diversion(*args, **kwargs) + + # Replace the fetch method with the custom fetch method + _nimbus.fetch = _fetch + + return _nimbus + # idea copied from raven that it borrowed from xclim that borrowed it from xarray that was borrowed from Seaborn def open_dataset( From fc14674640853dc176ae1ed8ff262954717d57ef Mon Sep 17 00:00:00 2001 From: Trevor James Smith <10819524+Zeitsperre@users.noreply.github.com> Date: Wed, 20 Nov 2024 14:36:20 -0500 Subject: [PATCH 07/10] better support of external testdata repositories --- xclim/testing/utils.py | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/xclim/testing/utils.py b/xclim/testing/utils.py index 182a39adf..7de871a76 100644 --- a/xclim/testing/utils.py +++ b/xclim/testing/utils.py @@ -436,6 +436,8 @@ def load_registry( dict Dictionary of filenames and hashes. """ + if not repo.endswith("/"): + repo = f"{repo}/" remote_registry = audit_url( urljoin( urljoin(repo, branch if branch.endswith("/") else f"{branch}/"), @@ -443,7 +445,7 @@ def load_registry( ) ) - if branch != default_testdata_version: + if repo == default_testdata_repo_url and branch != default_testdata_version: custom_registry_folder = Path( str(ilr.files("xclim").joinpath(f"testing/{branch}")) ) @@ -452,10 +454,20 @@ def load_registry( urlretrieve(remote_registry, registry_file) # noqa: S310 elif repo != default_testdata_repo_url: - registry_file = Path(str(ilr.files("xclim").joinpath("testing/registry.txt"))) + external_repo_name = urlparse(repo).path.split("/")[-2] + external_branch_name = branch.split("/")[-1] + registry_file = Path( + str( + ilr.files("xclim").joinpath( + f"testing/registry.{external_repo_name}.{external_branch_name}.txt" + ) + ) + ) urlretrieve(remote_registry, registry_file) # noqa: S310 - registry_file = Path(str(ilr.files("xclim").joinpath("testing/registry.txt"))) + else: + registry_file = Path(str(ilr.files("xclim").joinpath("testing/registry.txt"))) + if not registry_file.exists(): raise FileNotFoundError(f"Registry file not found: {registry_file}") @@ -517,6 +529,8 @@ def nimbus( # noqa: PR01 "The `pooch` package is required to fetch the xclim testing data. " "You can install it with `pip install pooch` or `pip install xclim[dev]`." ) + if not repo.endswith("/"): + repo = f"{repo}/" remote = audit_url( urljoin(urljoin(repo, branch if branch.endswith("/") else f"{branch}/"), "data") ) From 412219fc71157f0abfce88a473b0da165633d911 Mon Sep 17 00:00:00 2001 From: Trevor James Smith <10819524+Zeitsperre@users.noreply.github.com> Date: Wed, 20 Nov 2024 15:10:03 -0500 Subject: [PATCH 08/10] cleaner --- xclim/testing/utils.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/xclim/testing/utils.py b/xclim/testing/utils.py index 7de871a76..1969b6519 100644 --- a/xclim/testing/utils.py +++ b/xclim/testing/utils.py @@ -445,15 +445,7 @@ def load_registry( ) ) - if repo == default_testdata_repo_url and branch != default_testdata_version: - custom_registry_folder = Path( - str(ilr.files("xclim").joinpath(f"testing/{branch}")) - ) - custom_registry_folder.mkdir(parents=True, exist_ok=True) - registry_file = custom_registry_folder.joinpath("registry.txt") - urlretrieve(remote_registry, registry_file) # noqa: S310 - - elif repo != default_testdata_repo_url: + if repo != default_testdata_repo_url: external_repo_name = urlparse(repo).path.split("/")[-2] external_branch_name = branch.split("/")[-1] registry_file = Path( @@ -465,6 +457,14 @@ def load_registry( ) urlretrieve(remote_registry, registry_file) # noqa: S310 + elif branch != default_testdata_version: + custom_registry_folder = Path( + str(ilr.files("xclim").joinpath(f"testing/{branch}")) + ) + custom_registry_folder.mkdir(parents=True, exist_ok=True) + registry_file = custom_registry_folder.joinpath("registry.txt") + urlretrieve(remote_registry, registry_file) # noqa: S310 + else: registry_file = Path(str(ilr.files("xclim").joinpath("testing/registry.txt"))) From 2f57a894c6c316f118e4e9eb455d82af1f7f6cdd Mon Sep 17 00:00:00 2001 From: Trevor James Smith <10819524+Zeitsperre@users.noreply.github.com> Date: Wed, 20 Nov 2024 16:33:14 -0500 Subject: [PATCH 09/10] update CHANGELOG.rst --- CHANGELOG.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index aed32045a..223085d08 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -15,10 +15,12 @@ Bug fixes * Fixed pickling issue with ``xclim.sdba.Grouper`` and other classes for usage with `dask>=2024.11`. (:issue:`1992`, :pull:`1993`). * Fixed an issue with ``nimbus`` that was causing URL path components to be improperly joined. (:pull:`1997`). * `base_kws_vars` in `MBCn` is now copied inside the `adjust` function so that in-place changes do not change the dict globally. (:pull:`1999`). +* Fixed a bug in the logic of ``xclim.testing.utils.load_registry`` that impacted the ability to load a `registry.txt` from a non-default repository. (:pull:`2001`). Internal changes ^^^^^^^^^^^^^^^^ * Changed french translations with word "pluvieux" to "avec précipitations". (:issue:`1960`, :pull:`1994`). +* In order to address 403 (forbidden) request errors when retrieving data from GitHub via ReadTheDocs, the ``nimbus`` class has been modified to use an overloaded `fetch` method that appends a User-Agent header to the request. (:pull:`2001`). v0.53.2 (2024-10-31) -------------------- From aaf115c81509809054b9d2642efbbfc567890296 Mon Sep 17 00:00:00 2001 From: Ouranos Helper Bot Date: Wed, 20 Nov 2024 21:51:12 +0000 Subject: [PATCH 10/10] =?UTF-8?q?Bump=20version:=200.53.3-dev.3=20?= =?UTF-8?q?=E2=86=92=200.53.3-dev.4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pyproject.toml | 2 +- xclim/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index dfd27d3d3..b34efcc0f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -134,7 +134,7 @@ target-version = [ ] [tool.bumpversion] -current_version = "0.53.3-dev.3" +current_version = "0.53.3-dev.4" commit = true commit_args = "--no-verify" tag = false diff --git a/xclim/__init__.py b/xclim/__init__.py index 6f5891522..c4a9153d9 100644 --- a/xclim/__init__.py +++ b/xclim/__init__.py @@ -13,7 +13,7 @@ __author__ = """Travis Logan""" __email__ = "logan.travis@ouranos.ca" -__version__ = "0.53.3-dev.3" +__version__ = "0.53.3-dev.4" with _resources.as_file(_resources.files("xclim.data")) as _module_data: