From 4545ef406c4a9b443056536ed394a5d913273aee Mon Sep 17 00:00:00 2001 From: "Philipp A." Date: Tue, 10 Oct 2023 12:25:40 +0200 Subject: [PATCH 01/43] WIP fix warnings --- anndata/_core/merge.py | 10 ++++--- anndata/_io/h5ad.py | 2 ++ anndata/_io/read.py | 12 ++++----- anndata/_io/utils.py | 4 +-- .../multi_files/_anncollection.py | 2 +- anndata/tests/test_awkward.py | 4 +-- anndata/tests/test_dask_view_mem.py | 9 +++++-- anndata/tests/test_hdf5_backing.py | 11 +++++--- anndata/tests/test_readwrite.py | 4 +-- pyproject.toml | 27 ++++++++++++++++--- 10 files changed, 59 insertions(+), 26 deletions(-) diff --git a/anndata/_core/merge.py b/anndata/_core/merge.py index a99eda6ac..f18c05c6b 100644 --- a/anndata/_core/merge.py +++ b/anndata/_core/merge.py @@ -206,7 +206,9 @@ def as_cp_sparse(x) -> CupySparseMatrix: return cpsparse.csr_matrix(x) -def unify_dtypes(dfs: Iterable[pd.DataFrame]) -> list[pd.DataFrame]: +def unify_dtypes( + dfs: Iterable[pd.DataFrame], *, prune: bool = True +) -> list[pd.DataFrame]: """ Attempts to unify datatypes from multiple dataframes. @@ -237,7 +239,7 @@ def unify_dtypes(dfs: Iterable[pd.DataFrame]) -> list[pd.DataFrame]: if col in df: df[col] = df[col].astype(dtype) - return dfs + return [df for df in dfs if len(df)] if prune else dfs def try_unifying_dtype( @@ -1239,7 +1241,9 @@ def concat( [pd.Series(dim_indices(a, axis=axis)) for a in adatas], ignore_index=True ) if index_unique is not None: - concat_indices = concat_indices.str.cat(label_col.map(str), sep=index_unique) + concat_indices = concat_indices.str.cat( + label_col.map(str, na_action=None), sep=index_unique + ) concat_indices = pd.Index(concat_indices) alt_indices = merge_indices( diff --git a/anndata/_io/h5ad.py b/anndata/_io/h5ad.py index bfddc6504..337f13f92 100644 --- a/anndata/_io/h5ad.py +++ b/anndata/_io/h5ad.py @@ -29,6 +29,7 @@ ) from ..experimental import read_dispatched from .specs import read_elem, write_elem +from .specs.registry import IOSpec, write_spec from .utils import ( H5PY_V3, _read_legacy_raw, @@ -110,6 +111,7 @@ def write_h5ad( @report_write_key_on_error +@write_spec(IOSpec("array", "0.2.0")) def write_sparse_as_dense(f, key, value, dataset_kwargs=MappingProxyType({})): real_key = None # Flag for if temporary key was used if key in f: diff --git a/anndata/_io/read.py b/anndata/_io/read.py index 68f7fbd27..a50c4b2ef 100644 --- a/anndata/_io/read.py +++ b/anndata/_io/read.py @@ -274,16 +274,16 @@ def read_loom( uns = {} if cleanup: uns_obs = {} - for key in list(obs.keys()): - if len(set(obs[key])) == 1: - uns_obs[f"{key}"] = obs[key][0] + for key in obs.columns: + if len(obs[key].unique()) == 1: + uns_obs[key] = obs[key].iloc[0] del obs[key] if uns_obs: uns["loom-obs"] = uns_obs uns_var = {} - for key in list(var.keys()): - if len(set(var[key])) == 1: - uns_var[f"{key}"] = var[key][0] + for key in var.columns: + if len(var[key].unique()) == 1: + uns_var[key] = var[key].iloc[0] del var[key] if uns_var: uns["loom-var"] = uns_var diff --git a/anndata/_io/utils.py b/anndata/_io/utils.py index e6bccde01..42f82b494 100644 --- a/anndata/_io/utils.py +++ b/anndata/_io/utils.py @@ -182,7 +182,7 @@ def re_raise_error(e, elem, key, op=Literal["read", "writ"]): def report_read_key_on_error(func): """\ - A decorator for zarr element reading which makes keys involved in errors get reported. + A decorator for hdf5/zarr element reading which makes keys involved in errors get reported. Example ------- @@ -213,7 +213,7 @@ def func_wrapper(*args, **kwargs): def report_write_key_on_error(func): """\ - A decorator for zarr element reading which makes keys involved in errors get reported. + A decorator for hdf6/zarr element writing which makes keys involved in errors get reported. Example ------- diff --git a/anndata/experimental/multi_files/_anncollection.py b/anndata/experimental/multi_files/_anncollection.py index 5a6012709..fa60d3537 100644 --- a/anndata/experimental/multi_files/_anncollection.py +++ b/anndata/experimental/multi_files/_anncollection.py @@ -208,7 +208,7 @@ def __getitem__(self, key, use_convert=True): else: if vidx is not None: idx = np.ix_(*idx) if not isinstance(idx[1], slice) else idx - arrs.append(arr[idx]) + arrs.append(arr.iloc[idx] if isinstance(arr, pd.Series) else arr[idx]) if len(arrs) > 1: _arr = _merge(arrs) diff --git a/anndata/tests/test_awkward.py b/anndata/tests/test_awkward.py index 993fb91de..85ad080d7 100644 --- a/anndata/tests/test_awkward.py +++ b/anndata/tests/test_awkward.py @@ -196,8 +196,8 @@ def reversed(self): ] ), # categorical array - ak.to_categorical(ak.Array([["a", "b", "c"], ["a", "b"]])), - ak.to_categorical(ak.Array([[1, 1, 2], [3, 3]])), + ak.str.to_categorical(ak.Array([["a", "b", "c"], ["a", "b"]])), + ak.str.to_categorical(ak.Array([[1, 1, 2], [3, 3]])), # tyical record type with AIRR data consisting of different dtypes ak.Array( [ diff --git a/anndata/tests/test_dask_view_mem.py b/anndata/tests/test_dask_view_mem.py index bb758a223..d597d40aa 100644 --- a/anndata/tests/test_dask_view_mem.py +++ b/anndata/tests/test_dask_view_mem.py @@ -1,9 +1,14 @@ from __future__ import annotations +from typing import TYPE_CHECKING + import pytest import anndata as ad +if TYPE_CHECKING: + import pandas as pd + pytest.importorskip("pytest_memray") # ------------------------------------------------------------------------------ @@ -155,5 +160,5 @@ def test_modify_view_mapping_obs_var_memory(attr_name, give_chunks): ) subset = adata[:N, :N] assert subset.is_view - m = getattr(subset, attr_name)["m"] - m[0] = 100 + m: pd.Series = getattr(subset, attr_name)["m"] + m.iloc[0] = 100 diff --git a/anndata/tests/test_hdf5_backing.py b/anndata/tests/test_hdf5_backing.py index 61c0c905c..fb4bed483 100644 --- a/anndata/tests/test_hdf5_backing.py +++ b/anndata/tests/test_hdf5_backing.py @@ -303,10 +303,13 @@ def test_backed_modification_sparse(adata, backing_h5ad, sparse_format): assert adata.filename == backing_h5ad assert adata.isbacked - adata.X[0, [0, 2]] = 10 - adata.X[1, [0, 2]] = [11, 12] - with pytest.raises(ValueError): - adata.X[2, 1] = 13 + with pytest.warns( + PendingDeprecationWarning, match=r"__setitem__ will likely be removed" + ): + adata.X[0, [0, 2]] = 10 + adata.X[1, [0, 2]] = [11, 12] + with pytest.raises(ValueError): + adata.X[2, 1] = 13 assert adata.isbacked diff --git a/anndata/tests/test_readwrite.py b/anndata/tests/test_readwrite.py index 98de43a61..d18164f69 100644 --- a/anndata/tests/test_readwrite.py +++ b/anndata/tests/test_readwrite.py @@ -430,7 +430,7 @@ def test_readloom_deprecations(tmp_path): depr_result = ad.read_loom(loom_pth, obsm_names=obsm_mapping) actual_result = ad.read_loom(loom_pth, obsm_mapping=obsm_mapping) assert_equal(actual_result, depr_result) - with pytest.raises(ValueError, match="ambiguous"): + with pytest.raises(ValueError, match="ambiguous"), pytest.warns(FutureWarning): ad.read_loom(loom_pth, obsm_mapping=obsm_mapping, obsm_names=obsm_mapping) # varm_names -> varm_mapping @@ -439,7 +439,7 @@ def test_readloom_deprecations(tmp_path): depr_result = ad.read_loom(loom_pth, varm_names=varm_mapping) actual_result = ad.read_loom(loom_pth, varm_mapping=varm_mapping) assert_equal(actual_result, depr_result) - with pytest.raises(ValueError, match="ambiguous"): + with pytest.raises(ValueError, match="ambiguous"), pytest.warns(FutureWarning): ad.read_loom(loom_pth, varm_mapping=varm_mapping, varm_names=varm_mapping) # positional -> keyword diff --git a/pyproject.toml b/pyproject.toml index 7bfbe496a..cd001b353 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -88,9 +88,10 @@ test = [ "joblib", "boltons", "scanpy", - "httpx", # For data downloading + "httpx", # For data downloading "dask[array,distributed]", "awkward>=2.3", + "pyarrow", # For non-deprecated awkward array functionality "pytest_memray", ] gpu = ["cupy"] @@ -104,13 +105,31 @@ version-file = "anndata/_version.py" [tool.coverage.run] source = ["anndata"] -omit = ["setup.py", "versioneer.py", "anndata/_version.py", "**/test_*.py"] +omit = ["anndata/_version.py", "**/test_*.py"] [tool.pytest.ini_options] -addopts = "--doctest-modules" +addopts = [ + "--strict-markers", + "--doctest-modules", + "--ignore=anndata/core.py", # deprecated + "--ignore=anndata/readwrite.py", # deprecated +] python_files = "test_*.py" testpaths = ["anndata", "docs/concatenation.rst"] -filterwarnings = ['ignore:X\.dtype being converted to np.float32:FutureWarning'] +filterwarnings = [ + 'error', + # We still test deprecated functions + 'ignore:AnnData\.concatenate method is deprecated:FutureWarning', + 'ignore:`anndata\.read` is deprecated:FutureWarning', + # We test our experimental features too + 'ignore::anndata._warnings.ExperimentalFeatureWarning', + # TODO: catch these in tests instead, makes sure they aren’t triggered accidentally + 'ignore::anndata._warnings.ImplicitModificationWarning', + 'ignore:.*names are not unique. To make them unique:UserWarning', + 'ignore::scipy.sparse._base.SparseEfficiencyWarning', + # Third party issues + 'ignore::numba.core.errors.NumbaDeprecationWarning', # we don’t import numba +] # For some reason this effects how logging is shown when tests are run xfail_strict = true markers = ["gpu: mark test to run on GPU"] From 82e5971f0ecdf77ef560f84ea59d85ec7f14e375 Mon Sep 17 00:00:00 2001 From: "Philipp A." Date: Tue, 10 Oct 2023 12:41:43 +0200 Subject: [PATCH 02/43] All done --- anndata/_core/anndata.py | 10 ++++------ anndata/experimental/multi_files/_anncollection.py | 2 +- anndata/tests/test_concatenate.py | 3 ++- pyproject.toml | 11 ++++++----- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/anndata/_core/anndata.py b/anndata/_core/anndata.py index 944fc66a4..0ce979c13 100644 --- a/anndata/_core/anndata.py +++ b/anndata/_core/anndata.py @@ -875,23 +875,21 @@ def _prep_dim_index(self, value, attr: str) -> pd.Index: value = pd.Index(value) if not isinstance(value.name, (str, type(None))): value.name = None - # fmt: off if ( - not isinstance(value, pd.RangeIndex) + len(value) > 0 + and not isinstance(value, pd.RangeIndex) and infer_dtype(value) not in ("string", "bytes") ): sample = list(value[: min(len(value), 5)]) - warnings.warn(dedent( + msg = dedent( f""" AnnData expects .{attr}.index to contain strings, but got values like: {sample} Inferred to be: {infer_dtype(value)} """ - ), # noqa - stacklevel=2, ) - # fmt: on + warnings.warn(msg, stacklevel=2) return value def _set_dim_index(self, value: pd.Index, attr: str): diff --git a/anndata/experimental/multi_files/_anncollection.py b/anndata/experimental/multi_files/_anncollection.py index fa60d3537..ac26283b1 100644 --- a/anndata/experimental/multi_files/_anncollection.py +++ b/anndata/experimental/multi_files/_anncollection.py @@ -721,7 +721,7 @@ def __init__( ) if index_unique is not None: concat_indices = concat_indices.str.cat( - label_col.map(str), sep=index_unique + label_col.map(str, na_action=None), sep=index_unique ) self.obs_names = pd.Index(concat_indices) diff --git a/anndata/tests/test_concatenate.py b/anndata/tests/test_concatenate.py index 318e78b3f..42e2574c7 100644 --- a/anndata/tests/test_concatenate.py +++ b/anndata/tests/test_concatenate.py @@ -814,7 +814,8 @@ def gen_dim_array(m): # Check values of included elements full_inds = np.arange(w_pairwise.shape[axis]) - groups = getattr(w_pairwise, dim).groupby("orig").indices + obs_var: pd.DataFrame = getattr(w_pairwise, dim) + groups = obs_var.groupby("orig", observed=True).indices for k, inds in groups.items(): orig_arr = getattr(adatas[k], dim_attr)["arr"] full_arr = getattr(w_pairwise, dim_attr)["arr"] diff --git a/pyproject.toml b/pyproject.toml index cd001b353..7722d0d90 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -119,16 +119,17 @@ testpaths = ["anndata", "docs/concatenation.rst"] filterwarnings = [ 'error', # We still test deprecated functions - 'ignore:AnnData\.concatenate method is deprecated:FutureWarning', + 'ignore:The AnnData\.concatenate method is deprecated:FutureWarning', 'ignore:`anndata\.read` is deprecated:FutureWarning', # We test our experimental features too 'ignore::anndata._warnings.ExperimentalFeatureWarning', - # TODO: catch these in tests instead, makes sure they aren’t triggered accidentally - 'ignore::anndata._warnings.ImplicitModificationWarning', - 'ignore:.*names are not unique. To make them unique:UserWarning', - 'ignore::scipy.sparse._base.SparseEfficiencyWarning', # Third party issues 'ignore::numba.core.errors.NumbaDeprecationWarning', # we don’t import numba + # TODO: Catch the following ones in tests or fix them + 'default::anndata._warnings.ImplicitModificationWarning', + 'default:.*names are not unique. To make them unique:UserWarning', + 'default::scipy.sparse._base.SparseEfficiencyWarning', + 'default::dask.array.core.PerformanceWarning', ] # For some reason this effects how logging is shown when tests are run xfail_strict = true From 944c21a6ccc372ac38a5525a2d0a121983ef813b Mon Sep 17 00:00:00 2001 From: "Philipp A." Date: Tue, 10 Oct 2023 12:47:54 +0200 Subject: [PATCH 03/43] better na_action value --- anndata/_core/merge.py | 2 +- anndata/experimental/multi_files/_anncollection.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/anndata/_core/merge.py b/anndata/_core/merge.py index f18c05c6b..0480562a2 100644 --- a/anndata/_core/merge.py +++ b/anndata/_core/merge.py @@ -1242,7 +1242,7 @@ def concat( ) if index_unique is not None: concat_indices = concat_indices.str.cat( - label_col.map(str, na_action=None), sep=index_unique + label_col.map(str, na_action="ignore"), sep=index_unique ) concat_indices = pd.Index(concat_indices) diff --git a/anndata/experimental/multi_files/_anncollection.py b/anndata/experimental/multi_files/_anncollection.py index ac26283b1..0da6d6b4a 100644 --- a/anndata/experimental/multi_files/_anncollection.py +++ b/anndata/experimental/multi_files/_anncollection.py @@ -721,7 +721,7 @@ def __init__( ) if index_unique is not None: concat_indices = concat_indices.str.cat( - label_col.map(str, na_action=None), sep=index_unique + label_col.map(str, na_action="ignore"), sep=index_unique ) self.obs_names = pd.Index(concat_indices) From 19f39e6ec2d77f46966deb53db7369c9cc527983 Mon Sep 17 00:00:00 2001 From: "Philipp A." Date: Tue, 10 Oct 2023 12:56:19 +0200 Subject: [PATCH 04/43] typo --- anndata/_io/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/anndata/_io/utils.py b/anndata/_io/utils.py index 42f82b494..2f63a6e01 100644 --- a/anndata/_io/utils.py +++ b/anndata/_io/utils.py @@ -213,7 +213,7 @@ def func_wrapper(*args, **kwargs): def report_write_key_on_error(func): """\ - A decorator for hdf6/zarr element writing which makes keys involved in errors get reported. + A decorator for hdf5/zarr element writing which makes keys involved in errors get reported. Example ------- From 9321d387d2855909705036c0febcfa04be4fa4c9 Mon Sep 17 00:00:00 2001 From: "Philipp A." Date: Tue, 10 Oct 2023 16:26:47 +0200 Subject: [PATCH 05/43] WIP --- .azure-pipelines.yml | 16 +++++++++++-- anndata/_core/aligned_mapping.py | 15 ++++-------- anndata/_core/anndata.py | 30 +++++++++++++++--------- anndata/_core/merge.py | 11 +++------ anndata/_io/h5ad.py | 23 ++++++++++-------- anndata/_io/utils.py | 22 +++++++++--------- anndata/_io/write.py | 6 ++++- anndata/tests/helpers.py | 32 ++++++++++++++++---------- anndata/tests/test_concatenate.py | 3 +++ anndata/tests/test_concatenate_disk.py | 6 ++++- anndata/tests/test_deprecations.py | 2 +- anndata/tests/test_hdf5_backing.py | 4 ++-- anndata/tests/test_raw.py | 4 ++-- anndata/tests/test_readwrite.py | 17 ++++++++------ anndata/utils.py | 10 ++++++++ docs/benchmark-read-write.ipynb | 2 +- docs/concatenation.rst | 2 +- pyproject.toml | 15 ------------ 18 files changed, 125 insertions(+), 95 deletions(-) diff --git a/.azure-pipelines.yml b/.azure-pipelines.yml index 7e875a4b8..88d2e9cdc 100644 --- a/.azure-pipelines.yml +++ b/.azure-pipelines.yml @@ -55,12 +55,24 @@ jobs: - script: | pytest displayName: "PyTest" - condition: eq(variables['RUN_COVERAGE'], 'no') + condition: eq(variables['RUN_COVERAGE'], 'no') && eq(variables['PRERELEASE_DEPENDENCIES'], 'no') - script: | pytest --cov --cov-report=xml --cov-context=test displayName: "PyTest (coverage)" - condition: eq(variables['RUN_COVERAGE'], 'yes') + condition: eq(variables['RUN_COVERAGE'], 'yes') && eq(variables['PRERELEASE_DEPENDENCIES'], 'no') + + # TODO: fix all the exceptions here + - script: > + pytest + -W error + -W 'default::anndata._warnings.ImplicitModificationWarning' + -W 'default:Observation names are not unique. To make them unique:UserWarning' + -W 'default:Variable names are not unique. To make them unique:UserWarning' + -W 'default::scipy.sparse._base.SparseEfficiencyWarning' + -W 'default::dask.array.core.PerformanceWarning' + displayName: "PyTest (treat warnings as errors)" + condition: eq(variables['RUN_COVERAGE'], 'no') && eq(variables['PRERELEASE_DEPENDENCIES'], 'yes') - task: PublishCodeCoverageResults@1 inputs: diff --git a/anndata/_core/aligned_mapping.py b/anndata/_core/aligned_mapping.py index a69730e26..2383e517d 100644 --- a/anndata/_core/aligned_mapping.py +++ b/anndata/_core/aligned_mapping.py @@ -19,7 +19,7 @@ from anndata._warnings import ExperimentalFeatureWarning, ImplicitModificationWarning from anndata.compat import AwkArray -from ..utils import deprecated, dim_len, ensure_df_homogeneous +from ..utils import deprecated, dim_len, ensure_df_homogeneous, warn_once from .access import ElementRef from .index import _subset from .views import as_view, view_update @@ -61,19 +61,12 @@ def _ipython_key_completions_(self) -> list[str]: def _validate_value(self, val: V, key: str) -> V: """Raises an error if value is invalid""" if isinstance(val, AwkArray): - warnings.warn( + warn_once( "Support for Awkward Arrays is currently experimental. " "Behavior may change in the future. Please report any issues you may encounter!", ExperimentalFeatureWarning, # stacklevel=3, ) - # Prevent from showing up every time an awkward array is used - # You'd think `once` works, but it doesn't at the repl and in notebooks - warnings.filterwarnings( - "ignore", - category=ExperimentalFeatureWarning, - message="Support for Awkward Arrays is currently experimental.*", - ) for i, axis in enumerate(self.axes): if self.parent.shape[axis] != dim_len(val, i): right_shape = tuple(self.parent.shape[a] for a in self.axes) @@ -122,7 +115,9 @@ def copy(self): for k, v in self.items(): if isinstance(v, AwkArray): # Shallow copy since awkward array buffers are immutable - d[k] = copy(v) + with warnings.catch_warnings(): + warnings.simplefilter("ignore", ExperimentalFeatureWarning) + d[k] = copy(v) else: d[k] = v.copy() return d diff --git a/anndata/_core/anndata.py b/anndata/_core/anndata.py index 0ce979c13..bc3050d46 100644 --- a/anndata/_core/anndata.py +++ b/anndata/_core/anndata.py @@ -819,6 +819,8 @@ def raw(self) -> Raw: @raw.setter def raw(self, value: AnnData): + from .._warnings import ExperimentalFeatureWarning + if value is None: del self.raw elif not isinstance(value, AnnData): @@ -826,7 +828,9 @@ def raw(self, value: AnnData): else: if self.is_view: self._init_as_actual(self.copy()) - self._raw = Raw(self, X=value.X, var=value.var, varm=value.varm) + with warnings.catch_warnings(): + warnings.simplefilter("ignore", ExperimentalFeatureWarning) + self._raw = Raw(self, X=value.X, var=value.var, varm=value.varm) @raw.deleter def raw(self): @@ -1571,17 +1575,21 @@ def to_memory(self, copy=False) -> AnnData: def copy(self, filename: PathLike | None = None) -> AnnData: """Full copy, optionally on disk.""" + from .._warnings import ExperimentalFeatureWarning + if not self.isbacked: - if self.is_view and self._has_X(): - # TODO: How do I unambiguously check if this is a copy? - # Subsetting this way means we don’t have to have a view type - # defined for the matrix, which is needed for some of the - # current distributed backend. Specifically Dask. - return self._mutated_copy( - X=_subset(self._adata_ref.X, (self._oidx, self._vidx)).copy() - ) - else: - return self._mutated_copy() + with warnings.catch_warnings(): + warnings.simplefilter("ignore", ExperimentalFeatureWarning) + if self.is_view and self._has_X(): + # TODO: How do I unambiguously check if this is a copy? + # Subsetting this way means we don’t have to have a view type + # defined for the matrix, which is needed for some of the + # current distributed backend. Specifically Dask. + return self._mutated_copy( + X=_subset(self._adata_ref.X, (self._oidx, self._vidx)).copy() + ) + else: + return self._mutated_copy() else: from .._io import read_h5ad, write_h5ad diff --git a/anndata/_core/merge.py b/anndata/_core/merge.py index 0480562a2..44045be7b 100644 --- a/anndata/_core/merge.py +++ b/anndata/_core/merge.py @@ -17,7 +17,7 @@ from itertools import repeat from operator import and_, or_, sub from typing import Any, Literal, TypeVar -from warnings import filterwarnings, warn +from warnings import warn import numpy as np import pandas as pd @@ -28,7 +28,7 @@ from anndata._warnings import ExperimentalFeatureWarning from ..compat import AwkArray, CupyArray, CupyCSRMatrix, CupySparseMatrix, DaskArray -from ..utils import asarray, dim_len +from ..utils import asarray, dim_len, warn_once from .anndata import AnnData from .index import _subset, make_slice @@ -873,17 +873,12 @@ def gen_outer_reindexers(els, shapes, new_index: pd.Index, *, axis=0): raise NotImplementedError( "Cannot concatenate an AwkwardArray with other array types." ) - warn( + warn_once( "Outer joins on awkward.Arrays will have different return values in the future." "For details, and to offer input, please see:\n\n\t" "https://github.com/scverse/anndata/issues/898", ExperimentalFeatureWarning, ) - filterwarnings( - "ignore", - category=ExperimentalFeatureWarning, - message=r"Outer joins on awkward.Arrays will have different return values.*", - ) # all_keys = union_keys(el.fields for el in els if not_missing(el)) reindexers = [] for el in els: diff --git a/anndata/_io/h5ad.py b/anndata/_io/h5ad.py index 337f13f92..bf224f37a 100644 --- a/anndata/_io/h5ad.py +++ b/anndata/_io/h5ad.py @@ -1,6 +1,7 @@ from __future__ import annotations import re +import warnings from functools import partial from pathlib import Path from types import MappingProxyType @@ -17,7 +18,7 @@ import pandas as pd from scipy import sparse -from anndata._warnings import OldFormatWarning +from anndata._warnings import ExperimentalFeatureWarning, OldFormatWarning from .._core.anndata import AnnData from .._core.file_backing import AnnDataFileManager, filename @@ -231,15 +232,17 @@ def read_h5ad( def callback(func, elem_name: str, elem, iospec): if iospec.encoding_type == "anndata" or elem_name.endswith("/"): - return AnnData( - **{ - # This is covering up backwards compat in the anndata initializer - # In most cases we should be able to call `func(elen[k])` instead - k: read_dispatched(elem[k], callback) - for k in elem.keys() - if not k.startswith("raw.") - } - ) + with warnings.catch_warnings(): + warnings.simplefilter("ignore", ExperimentalFeatureWarning) + return AnnData( + **{ + # This is covering up backwards compat in the anndata initializer + # In most cases we should be able to call `func(elen[k])` instead + k: read_dispatched(elem[k], callback) + for k in elem.keys() + if not k.startswith("raw.") + } + ) elif elem_name.startswith("/raw."): return None elif elem_name == "/X" and "X" in as_sparse: diff --git a/anndata/_io/utils.py b/anndata/_io/utils.py index 2f63a6e01..964f94811 100644 --- a/anndata/_io/utils.py +++ b/anndata/_io/utils.py @@ -165,19 +165,17 @@ def _get_parent(elem): return parent -def re_raise_error(e, elem, key, op=Literal["read", "writ"]): +def add_key_note(e: BaseException, elem, key, op=Literal["read", "writ"]) -> None: if any( f"Error raised while {op}ing key" in note for note in getattr(e, "__notes__", []) ): - raise - else: - parent = _get_parent(elem) - add_note( - e, - f"Error raised while {op}ing key {key!r} of {type(elem)} to " f"{parent}", - ) - raise e + return + parent = _get_parent(elem) + add_note( + e, + f"Error raised while {op}ing key {key!r} of {type(elem)} to " f"{parent}", + ) def report_read_key_on_error(func): @@ -206,7 +204,8 @@ def func_wrapper(*args, **kwargs): try: return func(*args, **kwargs) except Exception as e: - re_raise_error(e, elem, elem.name, "read") + add_key_note(e, elem, elem.name, "read") + raise return func_wrapper @@ -239,7 +238,8 @@ def func_wrapper(*args, **kwargs): try: return func(*args, **kwargs) except Exception as e: - re_raise_error(e, elem, key, "writ") + add_key_note(e, elem, key, "writ") + raise return func_wrapper diff --git a/anndata/_io/write.py b/anndata/_io/write.py index 5a4d2ca62..75240b779 100644 --- a/anndata/_io/write.py +++ b/anndata/_io/write.py @@ -103,7 +103,11 @@ def write_loom(filename: PathLike, adata: AnnData, write_obsm_varm: bool = False for key in adata.layers.keys(): layers[key] = adata.layers[key].T - from loompy import create + from numba.core.errors import NumbaDeprecationWarning + + with warnings.catch_warnings(): + warnings.simplefilter("ignore", NumbaDeprecationWarning) + from loompy import create if filename.exists(): filename.unlink() diff --git a/anndata/tests/helpers.py b/anndata/tests/helpers.py index 641bdc791..ae8e61fa2 100644 --- a/anndata/tests/helpers.py +++ b/anndata/tests/helpers.py @@ -4,7 +4,7 @@ import re import warnings from collections.abc import Collection, Mapping -from contextlib import contextmanager +from contextlib import contextmanager, nullcontext from functools import partial, singledispatch, wraps from string import ascii_letters @@ -15,7 +15,7 @@ from pandas.api.types import is_numeric_dtype from scipy import sparse -from anndata import AnnData, Raw +from anndata import AnnData, ExperimentalFeatureWarning, Raw from anndata._core.aligned_mapping import AlignedMapping from anndata._core.sparse_dataset import BaseCompressedSparseDataset from anndata._core.views import ArrayView @@ -256,17 +256,25 @@ def gen_adata( awkward_ragged=gen_awkward((12, None, None)), # U_recarray=gen_vstr_recarray(N, 5, "U4") ) - adata = AnnData( - X=X, - obs=obs, - var=var, - obsm=obsm, - varm=varm, - layers=layers, - obsp=obsp, - varp=varp, - uns=uns, + + ctx = ( + pytest.warns(ExperimentalFeatureWarning) + if AwkArray in {*obsm_types, *varm_types} + else nullcontext() ) + + with ctx: + adata = AnnData( + X=X, + obs=obs, + var=var, + obsm=obsm, + varm=varm, + layers=layers, + obsp=obsp, + varp=varp, + uns=uns, + ) return adata diff --git a/anndata/tests/test_concatenate.py b/anndata/tests/test_concatenate.py index 42e2574c7..efd0f7509 100644 --- a/anndata/tests/test_concatenate.py +++ b/anndata/tests/test_concatenate.py @@ -145,6 +145,9 @@ def test_concat_interface_errors(): concat([]) +@pytest.mark.filterwarnings( + r"ignore:The AnnData\.concatenate method is deprecated:FutureWarning" +) @pytest.mark.parametrize( ["concat_func", "backwards_compat"], [ diff --git a/anndata/tests/test_concatenate_disk.py b/anndata/tests/test_concatenate_disk.py index f9eab9540..659fb98cf 100644 --- a/anndata/tests/test_concatenate_disk.py +++ b/anndata/tests/test_concatenate_disk.py @@ -109,7 +109,7 @@ def test_anndatas_without_reindex( M = 50 sparse_fmt = "csr" adatas = [] - for _ in range(5): + for i in range(5): if axis == 0: M = np.random.randint(1, 100) else: @@ -122,6 +122,10 @@ def test_anndatas_without_reindex( sparse_fmt=sparse_fmt, **GEN_ADATA_OOC_CONCAT_ARGS, ) + if axis == 0: + a.obs_names = f"{i}-" + a.obs_names + else: + a.var_names = f"{i}-" + a.var_names adatas.append(a) assert_eq_concat_on_disk( diff --git a/anndata/tests/test_deprecations.py b/anndata/tests/test_deprecations.py index 01f202aec..cfa813c34 100644 --- a/anndata/tests/test_deprecations.py +++ b/anndata/tests/test_deprecations.py @@ -123,7 +123,7 @@ def test_deprecated_read(tmp_path): memory.write_h5ad(tmp_path / "file.h5ad") with pytest.warns(FutureWarning, match="`anndata.read` is deprecated"): - from_disk = ad.read(tmp_path / "file.h5ad") + from_disk = ad.read_h5ad(tmp_path / "file.h5ad") assert_equal(memory, from_disk) diff --git a/anndata/tests/test_hdf5_backing.py b/anndata/tests/test_hdf5_backing.py index fb4bed483..5c00182a4 100644 --- a/anndata/tests/test_hdf5_backing.py +++ b/anndata/tests/test_hdf5_backing.py @@ -89,11 +89,11 @@ def test_read_write_X(tmp_path, mtx_format, backed_mode, as_dense): orig = ad.AnnData(mtx_format(asarray(sparse.random(10, 10, format="csr")))) orig.write(orig_pth) - backed = ad.read(orig_pth, backed=backed_mode) + backed = ad.read_h5ad(orig_pth, backed=backed_mode) backed.write(backed_pth, as_dense=as_dense) backed.file.close() - from_backed = ad.read(backed_pth) + from_backed = ad.read_h5ad(backed_pth) assert np.all(asarray(orig.X) == asarray(from_backed.X)) diff --git a/anndata/tests/test_raw.py b/anndata/tests/test_raw.py index 7e4689d60..b51376b9a 100644 --- a/anndata/tests/test_raw.py +++ b/anndata/tests/test_raw.py @@ -81,7 +81,7 @@ def test_raw_of_view(adata_raw: ad.AnnData): def test_raw_rw(adata_raw: ad.AnnData, backing_h5ad): adata_raw.write(backing_h5ad) - adata_read = ad.read(backing_h5ad) + adata_read = ad.read_h5ad(backing_h5ad) assert_equal(adata_read, adata_raw, exact=True) @@ -96,7 +96,7 @@ def test_raw_view_rw(adata_raw: ad.AnnData, backing_h5ad): assert_equal(adata_raw_view, adata_raw) with pytest.warns(ImplicitModificationWarning, match="initializing view as actual"): adata_raw_view.write(backing_h5ad) - adata_read = ad.read(backing_h5ad) + adata_read = ad.read_h5ad(backing_h5ad) assert_equal(adata_read, adata_raw_view, exact=True) diff --git a/anndata/tests/test_readwrite.py b/anndata/tests/test_readwrite.py index d18164f69..dbbee8be3 100644 --- a/anndata/tests/test_readwrite.py +++ b/anndata/tests/test_readwrite.py @@ -88,7 +88,7 @@ def rw(backing_h5ad): M, N = 100, 101 orig = gen_adata((M, N)) orig.write(backing_h5ad) - curr = ad.read(backing_h5ad) + curr = ad.read_h5ad(backing_h5ad) return curr, orig @@ -139,7 +139,7 @@ def test_readwrite_kitchensink(tmp_path, storage, typ, backing_h5ad, dataset_kwa if storage == "h5ad": adata_src.write(backing_h5ad, **dataset_kwargs) - adata_mid = ad.read(backing_h5ad) + adata_mid = ad.read_h5ad(backing_h5ad) adata_mid.write(tmp_path / "mid.h5ad", **dataset_kwargs) adata = ad.read_h5ad(tmp_path / "mid.h5ad") else: @@ -179,7 +179,7 @@ def test_readwrite_maintain_X_dtype(typ, backing_h5ad): adata_src = ad.AnnData(X) adata_src.write(backing_h5ad) - adata = ad.read(backing_h5ad) + adata = ad.read_h5ad(backing_h5ad) assert adata.X.dtype == adata_src.X.dtype @@ -212,7 +212,7 @@ def test_readwrite_h5ad_one_dimension(typ, backing_h5ad): adata_src = ad.AnnData(X, obs=obs_dict, var=var_dict, uns=uns_dict) adata_one = adata_src[:, 0].copy() adata_one.write(backing_h5ad) - adata = ad.read(backing_h5ad) + adata = ad.read_h5ad(backing_h5ad) assert adata.shape == (3, 1) assert_equal(adata, adata_one) @@ -224,7 +224,7 @@ def test_readwrite_backed(typ, backing_h5ad): adata_src.filename = backing_h5ad # change to backed mode adata_src.write() - adata = ad.read(backing_h5ad) + adata = ad.read_h5ad(backing_h5ad) assert isinstance(adata.obs["oanno1"].dtype, pd.CategoricalDtype) assert not isinstance(adata.obs["oanno2"].dtype, pd.CategoricalDtype) assert adata.obs.index.tolist() == ["name1", "name2", "name3"] @@ -728,10 +728,13 @@ def test_scanpy_krumsiek11(tmp_path, diskfmt): filepth = tmp_path / f"test.{diskfmt}" import scanpy as sc - orig = sc.datasets.krumsiek11() + # TODO: this should be fixed in scanpy instead + with pytest.warns(UserWarning, match=r"Observation names are not unique"): + orig = sc.datasets.krumsiek11() del orig.uns["highlights"] # Can’t write int keys getattr(orig, f"write_{diskfmt}")(filepth) - read = getattr(ad, f"read_{diskfmt}")(filepth) + with pytest.warns(UserWarning, match=r"Observation names are not unique"): + read = getattr(ad, f"read_{diskfmt}")(filepth) assert_equal(orig, read, exact=True) diff --git a/anndata/utils.py b/anndata/utils.py index b5fc5c16c..3eeb08b56 100644 --- a/anndata/utils.py +++ b/anndata/utils.py @@ -1,5 +1,7 @@ from __future__ import annotations +import os +import re import warnings from functools import singledispatch, wraps from typing import TYPE_CHECKING, Any @@ -311,6 +313,14 @@ def convert_dictionary_to_structured_array(source: Mapping[str, Sequence[Any]]): return arr +def warn_once(msg: str, category: type[Warning], stacklevel: int = 1): + warnings.warn(msg, category, stacklevel=stacklevel) + if "PYTEST_CURRENT_TEST" not in os.environ: + # Prevent from showing up every time an awkward array is used + # You'd think `'once'` works, but it doesn't at the repl and in notebooks + warnings.filterwarnings("ignore", category=category, message=re.escape(msg)) + + def deprecated(new_name: str): """\ This is a decorator which can be used to mark functions diff --git a/docs/benchmark-read-write.ipynb b/docs/benchmark-read-write.ipynb index 44356459d..3420ebc9a 100644 --- a/docs/benchmark-read-write.ipynb +++ b/docs/benchmark-read-write.ipynb @@ -103,7 +103,7 @@ ], "source": [ "%%time\n", - "adata = ad.read('test.h5ad')" + "adata = ad.read_h5ad('test.h5ad')" ] }, { diff --git a/docs/concatenation.rst b/docs/concatenation.rst index 17674188d..e8b07272b 100644 --- a/docs/concatenation.rst +++ b/docs/concatenation.rst @@ -33,7 +33,7 @@ Let's start off with an example: If we split this object up by clusters of observations, then stack those subsets we'll obtain the same values – just ordered differently. - >>> groups = pbmc.obs.groupby("louvain").indices + >>> groups = pbmc.obs.groupby("louvain", observed=True).indices >>> pbmc_concat = ad.concat([pbmc[inds] for inds in groups.values()], merge="same") >>> assert np.array_equal(pbmc.X, pbmc_concat[pbmc.obs_names].X) >>> pbmc_concat diff --git a/pyproject.toml b/pyproject.toml index 7722d0d90..bef135269 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -116,21 +116,6 @@ addopts = [ ] python_files = "test_*.py" testpaths = ["anndata", "docs/concatenation.rst"] -filterwarnings = [ - 'error', - # We still test deprecated functions - 'ignore:The AnnData\.concatenate method is deprecated:FutureWarning', - 'ignore:`anndata\.read` is deprecated:FutureWarning', - # We test our experimental features too - 'ignore::anndata._warnings.ExperimentalFeatureWarning', - # Third party issues - 'ignore::numba.core.errors.NumbaDeprecationWarning', # we don’t import numba - # TODO: Catch the following ones in tests or fix them - 'default::anndata._warnings.ImplicitModificationWarning', - 'default:.*names are not unique. To make them unique:UserWarning', - 'default::scipy.sparse._base.SparseEfficiencyWarning', - 'default::dask.array.core.PerformanceWarning', -] # For some reason this effects how logging is shown when tests are run xfail_strict = true markers = ["gpu: mark test to run on GPU"] From 0607419118bc9207f4c27d437fb74b02ae1fffac Mon Sep 17 00:00:00 2001 From: "Philipp A." Date: Tue, 10 Oct 2023 17:25:56 +0200 Subject: [PATCH 06/43] only doctest left --- anndata/_core/anndata.py | 45 +++++++++------ anndata/_core/merge.py | 88 ++++++++++++++---------------- anndata/_io/h5ad.py | 4 +- anndata/_io/specs/methods.py | 8 ++- anndata/_io/zarr.py | 29 +++++----- anndata/tests/test_awkward.py | 44 ++++++++++++--- anndata/tests/test_concatenate.py | 71 +++++++++++++++++------- anndata/tests/test_deprecations.py | 38 ++++++------- 8 files changed, 198 insertions(+), 129 deletions(-) diff --git a/anndata/_core/anndata.py b/anndata/_core/anndata.py index bc3050d46..67f2c710e 100644 --- a/anndata/_core/anndata.py +++ b/anndata/_core/anndata.py @@ -30,6 +30,7 @@ from anndata._warnings import ImplicitModificationWarning from .. import utils +from .._warnings import ExperimentalFeatureWarning from ..compat import ( CupyArray, CupySparseMatrix, @@ -899,7 +900,9 @@ def _prep_dim_index(self, value, attr: str) -> pd.Index: def _set_dim_index(self, value: pd.Index, attr: str): # Assumes _prep_dim_index has been run if self.is_view: - self._init_as_actual(self.copy()) + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", category=ExperimentalFeatureWarning) + self._init_as_actual(self.copy()) getattr(self, attr).index = value for v in getattr(self, f"{attr}m").values(): if isinstance(v, pd.DataFrame): @@ -1305,7 +1308,10 @@ def _inplace_subset_var(self, index: Index1D): Same as `adata = adata[:, index]`, but inplace. """ adata_subset = self[:, index].copy() - self._init_as_actual(adata_subset) + + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", category=ExperimentalFeatureWarning) + self._init_as_actual(adata_subset) def _inplace_subset_obs(self, index: Index1D): """\ @@ -1314,7 +1320,10 @@ def _inplace_subset_obs(self, index: Index1D): Same as `adata = adata[index, :]`, but inplace. """ adata_subset = self[index].copy() - self._init_as_actual(adata_subset) + + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", category=ExperimentalFeatureWarning) + self._init_as_actual(adata_subset) # TODO: Update, possibly remove def __setitem__( @@ -1352,18 +1361,20 @@ def transpose(self) -> AnnData: "which is currently not implemented. Call `.copy()` before transposing." ) - return AnnData( - X=_safe_transpose(X) if X is not None else None, - layers={k: _safe_transpose(v) for k, v in self.layers.items()}, - obs=self.var, - var=self.obs, - uns=self._uns, - obsm=self._varm, - varm=self._obsm, - obsp=self._varp, - varp=self._obsp, - filename=self.filename, - ) + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", category=ExperimentalFeatureWarning) + return AnnData( + X=_safe_transpose(X) if X is not None else None, + layers={k: _safe_transpose(v) for k, v in self.layers.items()}, + obs=self.var, + var=self.obs, + uns=self._uns, + obsm=self._varm, + varm=self._obsm, + obsp=self._varp, + varp=self._obsp, + filename=self.filename, + ) T = property(transpose) @@ -1571,7 +1582,9 @@ def to_memory(self, copy=False) -> AnnData: if self.isbacked: self.file.close() - return AnnData(**new) + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", category=ExperimentalFeatureWarning) + return AnnData(**new) def copy(self, filename: PathLike | None = None) -> AnnData: """Full copy, optionally on disk.""" diff --git a/anndata/_core/merge.py b/anndata/_core/merge.py index 44045be7b..dcf5f3e47 100644 --- a/anndata/_core/merge.py +++ b/anndata/_core/merge.py @@ -4,6 +4,7 @@ from __future__ import annotations import typing +import warnings from collections import OrderedDict from collections.abc import ( Callable, @@ -13,7 +14,7 @@ MutableSet, Sequence, ) -from functools import reduce, singledispatch +from functools import partial, reduce, singledispatch from itertools import repeat from operator import and_, or_, sub from typing import Any, Literal, TypeVar @@ -814,7 +815,7 @@ def concat_arrays(arrays, reindexers, axis=0, index=None, fill_value=None): ) -def inner_concat_aligned_mapping(mappings, reindexers=None, index=None, axis=0): +def inner_concat_aligned_mapping(mappings, *, reindexers=None, index=None, axis=0): result = {} for k in intersect_keys(mappings): @@ -874,7 +875,7 @@ def gen_outer_reindexers(els, shapes, new_index: pd.Index, *, axis=0): "Cannot concatenate an AwkwardArray with other array types." ) warn_once( - "Outer joins on awkward.Arrays will have different return values in the future." + "Outer joins on awkward.Arrays will have different return values in the future. " "For details, and to offer input, please see:\n\n\t" "https://github.com/scverse/anndata/issues/898", ExperimentalFeatureWarning, @@ -902,7 +903,7 @@ def gen_outer_reindexers(els, shapes, new_index: pd.Index, *, axis=0): def outer_concat_aligned_mapping( - mappings, reindexers=None, index=None, fill_value=None, axis=0 + mappings, *, reindexers=None, index=None, axis=0, fill_value=None ): result = {} ns = [m.parent.shape[axis] for m in mappings] @@ -1267,37 +1268,30 @@ def concat( X = concat_Xs(adatas, reindexers, axis=axis, fill_value=fill_value) if join == "inner": - layers = inner_concat_aligned_mapping( - [a.layers for a in adatas], axis=axis, reindexers=reindexers - ) - concat_mapping = inner_concat_aligned_mapping( - [getattr(a, f"{dim}m") for a in adatas], index=concat_indices - ) - if pairwise: - concat_pairwise = concat_pairwise_mapping( - mappings=[getattr(a, f"{dim}p") for a in adatas], - shapes=[a.shape[axis] for a in adatas], - join_keys=intersect_keys, - ) - else: - concat_pairwise = {} + concat_aligned_mapping = inner_concat_aligned_mapping + join_keys = intersect_keys elif join == "outer": - layers = outer_concat_aligned_mapping( - [a.layers for a in adatas], reindexers, axis=axis, fill_value=fill_value + concat_aligned_mapping = partial( + outer_concat_aligned_mapping, fill_value=fill_value ) - concat_mapping = outer_concat_aligned_mapping( - [getattr(a, f"{dim}m") for a in adatas], - index=concat_indices, - fill_value=fill_value, + join_keys = union_keys + else: + assert False + + layers = concat_aligned_mapping( + [a.layers for a in adatas], axis=axis, reindexers=reindexers + ) + concat_mapping = concat_aligned_mapping( + [getattr(a, f"{dim}m") for a in adatas], index=concat_indices + ) + if pairwise: + concat_pairwise = concat_pairwise_mapping( + mappings=[getattr(a, f"{dim}p") for a in adatas], + shapes=[a.shape[axis] for a in adatas], + join_keys=join_keys, ) - if pairwise: - concat_pairwise = concat_pairwise_mapping( - mappings=[getattr(a, f"{dim}p") for a in adatas], - shapes=[a.shape[axis] for a in adatas], - join_keys=union_keys, - ) - else: - concat_pairwise = {} + else: + concat_pairwise = {} # TODO: Reindex lazily, so we don't have to make those copies until we're sure we need the element alt_mapping = merge( @@ -1340,17 +1334,19 @@ def concat( "not concatenating `.raw` attributes.", UserWarning, ) - return AnnData( - **{ - "X": X, - "layers": layers, - dim: concat_annot, - alt_dim: alt_annot, - f"{dim}m": concat_mapping, - f"{alt_dim}m": alt_mapping, - f"{dim}p": concat_pairwise, - f"{alt_dim}p": alt_pairwise, - "uns": uns, - "raw": raw, - } - ) + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", category=ExperimentalFeatureWarning) + return AnnData( + **{ + "X": X, + "layers": layers, + dim: concat_annot, + alt_dim: alt_annot, + f"{dim}m": concat_mapping, + f"{alt_dim}m": alt_mapping, + f"{dim}p": concat_pairwise, + f"{alt_dim}p": alt_pairwise, + "uns": uns, + "raw": raw, + } + ) diff --git a/anndata/_io/h5ad.py b/anndata/_io/h5ad.py index bf224f37a..5fac0406d 100644 --- a/anndata/_io/h5ad.py +++ b/anndata/_io/h5ad.py @@ -153,7 +153,9 @@ def read_h5ad_backed(filename: str | Path, mode: Literal["r", "r+"]) -> AnnData: d["raw"] = _read_raw(f, attrs={"var", "varm"}) - adata = AnnData(**d) + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", category=ExperimentalFeatureWarning) + adata = AnnData(**d) # Backwards compat to <0.7 if isinstance(f["obs"], h5py.Dataset): diff --git a/anndata/_io/specs/methods.py b/anndata/_io/specs/methods.py index 00cd66ea7..a934656f4 100644 --- a/anndata/_io/specs/methods.py +++ b/anndata/_io/specs/methods.py @@ -1,5 +1,6 @@ from __future__ import annotations +import warnings from collections.abc import Mapping from functools import partial from itertools import product @@ -19,7 +20,7 @@ from anndata._core.merge import intersect_keys from anndata._core.sparse_dataset import CSCDataset, CSRDataset, sparse_dataset from anndata._io.utils import H5PY_V3, check_key -from anndata._warnings import OldFormatWarning +from anndata._warnings import ExperimentalFeatureWarning, OldFormatWarning from anndata.compat import ( AwkArray, CupyArray, @@ -281,7 +282,10 @@ def read_anndata(elem, _reader): ]: if k in elem: d[k] = _reader.read_elem(elem[k]) - return AnnData(**d) + + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", category=ExperimentalFeatureWarning) + return AnnData(**d) @_REGISTRY.register_write(H5Group, Raw, IOSpec("raw", "0.1.0")) diff --git a/anndata/_io/zarr.py b/anndata/_io/zarr.py index 022ee8a1d..20ec679be 100644 --- a/anndata/_io/zarr.py +++ b/anndata/_io/zarr.py @@ -1,5 +1,6 @@ from __future__ import annotations +import warnings from pathlib import Path from typing import TYPE_CHECKING, TypeVar from warnings import warn @@ -9,19 +10,13 @@ import zarr from scipy import sparse -from anndata._warnings import OldFormatWarning +from anndata._warnings import ExperimentalFeatureWarning, OldFormatWarning from .._core.anndata import AnnData -from ..compat import ( - _clean_uns, - _from_fixed_length_strings, -) +from ..compat import _clean_uns, _from_fixed_length_strings from ..experimental import read_dispatched, write_dispatched from .specs import read_elem -from .utils import ( - _read_legacy_raw, - report_read_key_on_error, -) +from .utils import _read_legacy_raw, report_read_key_on_error if TYPE_CHECKING: from collections.abc import MutableMapping @@ -74,13 +69,15 @@ def read_zarr(store: str | Path | MutableMapping | zarr.Group) -> AnnData: # Read with handling for backwards compat def callback(func, elem_name: str, elem, iospec): if iospec.encoding_type == "anndata" or elem_name.endswith("/"): - return AnnData( - **{ - k: read_dispatched(v, callback) - for k, v in elem.items() - if not k.startswith("raw.") - } - ) + with warnings.catch_warnings(): + warnings.simplefilter("ignore", ExperimentalFeatureWarning) + return AnnData( + **{ + k: read_dispatched(v, callback) + for k, v in elem.items() + if not k.startswith("raw.") + } + ) elif elem_name.startswith("/raw."): return None elif elem_name in {"/obs", "/var"}: diff --git a/anndata/tests/test_awkward.py b/anndata/tests/test_awkward.py index 85ad080d7..a7b7753e6 100644 --- a/anndata/tests/test_awkward.py +++ b/anndata/tests/test_awkward.py @@ -1,13 +1,20 @@ """Tests related to awkward arrays""" from __future__ import annotations +from contextlib import nullcontext + import numpy as np import numpy.testing as npt import pandas as pd import pytest import anndata -from anndata import AnnData, ImplicitModificationWarning, read_h5ad +from anndata import ( + AnnData, + ExperimentalFeatureWarning, + ImplicitModificationWarning, + read_h5ad, +) from anndata.compat import awkward as ak from anndata.tests.helpers import assert_equal, gen_adata, gen_awkward from anndata.utils import dim_len @@ -95,8 +102,11 @@ def test_set_awkward(field, value, valid): """ adata = gen_adata((10, 20), varm_types=(), obsm_types=(), layers_types=()) + ctx = pytest.warns(ExperimentalFeatureWarning) if field != "uns" else nullcontext() + def _assign(): - getattr(adata, field)["test"] = value + with ctx: + getattr(adata, field)["test"] = value if not valid: with pytest.raises(ValueError): @@ -109,7 +119,9 @@ def _assign(): def test_copy(key): """Check that modifying a copy does not modify the original""" adata = gen_adata((3, 3), varm_types=(), obsm_types=(), layers_types=()) - getattr(adata, key)["awk"] = ak.Array([{"a": [1], "b": [2], "c": [3]}] * 3) + ctx = pytest.warns(ExperimentalFeatureWarning) if key != "uns" else nullcontext() + with ctx: + getattr(adata, key)["awk"] = ak.Array([{"a": [1], "b": [2], "c": [3]}] * 3) adata_copy = adata.copy() getattr(adata_copy, key)["awk"]["c"] = np.full((3, 1), 4) getattr(adata_copy, key)["awk"]["d"] = np.full((3, 1), 5) @@ -128,7 +140,8 @@ def test_copy(key): def test_view(key): """Check that modifying a view does not modify the original""" adata = gen_adata((3, 3), varm_types=(), obsm_types=(), layers_types=()) - getattr(adata, key)["awk"] = ak.Array([{"a": [1], "b": [2], "c": [3]}] * 3) + with pytest.warns(ExperimentalFeatureWarning): + getattr(adata, key)["awk"] = ak.Array([{"a": [1], "b": [2], "c": [3]}] * 3) adata_view = adata[:2, :2] with pytest.warns(ImplicitModificationWarning, match="initializing view as actual"): @@ -159,9 +172,10 @@ def reversed(self): ak.behavior[BEHAVIOUR_ID] = ReversibleArray adata = gen_adata((3, 3), varm_types=(), obsm_types=(), layers_types=()) - adata.obsm["awk_string"] = ak.with_parameter( - ak.Array(["AAA", "BBB", "CCC"]), "__list__", BEHAVIOUR_ID - ) + with pytest.warns(ExperimentalFeatureWarning): + adata.obsm["awk_string"] = ak.with_parameter( + ak.Array(["AAA", "BBB", "CCC"]), "__list__", BEHAVIOUR_ID + ) adata_view = adata[:2] with pytest.raises(NotImplementedError): @@ -370,7 +384,13 @@ def test_concat_mixed_types(key, arrays, expected, join): tmp_adata.obs_names if key == "obsm" else tmp_adata.var_names, inplace=True, ) - getattr(tmp_adata, key)["test"] = a + ctx = ( + pytest.warns(ExperimentalFeatureWarning) + if isinstance(a, ak.Array) + else nullcontext() + ) + with ctx: + getattr(tmp_adata, key)["test"] = a to_concat.append(tmp_adata) @@ -379,6 +399,12 @@ def test_concat_mixed_types(key, arrays, expected, join): anndata.concat(to_concat, axis=axis, join=join) else: print(to_concat) - result_adata = anndata.concat(to_concat, axis=axis, join=join) + ctx = ( + pytest.warns(ExperimentalFeatureWarning) + if join == "outer" + else nullcontext() + ) + with ctx: + result_adata = anndata.concat(to_concat, axis=axis, join=join) result = getattr(result_adata, key).get("test", None) assert_equal(expected, result, exact=True) diff --git a/anndata/tests/test_concatenate.py b/anndata/tests/test_concatenate.py index efd0f7509..d29ba4cb8 100644 --- a/anndata/tests/test_concatenate.py +++ b/anndata/tests/test_concatenate.py @@ -2,6 +2,7 @@ import warnings from collections.abc import Hashable +from contextlib import nullcontext from copy import deepcopy from functools import partial, singledispatch from itertools import chain, permutations, product @@ -14,7 +15,7 @@ from numpy import ma from scipy import sparse -from anndata import AnnData, Raw, concat +from anndata import AnnData, ExperimentalFeatureWarning, Raw, concat from anndata._core import merge from anndata._core.index import _subset from anndata.compat import AwkArray, DaskArray @@ -30,6 +31,10 @@ ) from anndata.utils import asarray +mark_legacy_concatenate = pytest.mark.filterwarnings( + r"ignore:The AnnData\.concatenate method is deprecated:FutureWarning" +) + @singledispatch def filled_like(a, fill_value=None): @@ -145,9 +150,7 @@ def test_concat_interface_errors(): concat([]) -@pytest.mark.filterwarnings( - r"ignore:The AnnData\.concatenate method is deprecated:FutureWarning" -) +@mark_legacy_concatenate @pytest.mark.parametrize( ["concat_func", "backwards_compat"], [ @@ -176,6 +179,7 @@ def test_concatenate_roundtrip(join_type, array_type, concat_func, backwards_com assert_equal(result[orig.obs_names].copy(), orig) +@mark_legacy_concatenate def test_concatenate_dense(): # dense data X1 = np.array([[1, 2, 3], [4, 5, 6]]) @@ -251,6 +255,7 @@ def test_concatenate_dense(): assert np.allclose(var_ma.compressed(), var_ma_ref.compressed()) +@mark_legacy_concatenate def test_concatenate_layers(array_type, join_type): adatas = [] for _ in range(5): @@ -310,6 +315,7 @@ def gen_index(n): ] +@mark_legacy_concatenate def test_concatenate_obsm_inner(obsm_adatas): adata = obsm_adatas[0].concatenate(obsm_adatas[1:], join="inner") @@ -339,6 +345,7 @@ def test_concatenate_obsm_inner(obsm_adatas): pd.testing.assert_frame_equal(true_df, cur_df) +@mark_legacy_concatenate def test_concatenate_obsm_outer(obsm_adatas, fill_val): outer = obsm_adatas[0].concatenate( obsm_adatas[1:], join="outer", fill_value=fill_val @@ -409,6 +416,7 @@ def test_concat_annot_join(obsm_adatas, join_type): ) +@mark_legacy_concatenate def test_concatenate_layers_misaligned(array_type, join_type): adatas = [] for _ in range(5): @@ -422,6 +430,7 @@ def test_concatenate_layers_misaligned(array_type, join_type): assert_equal(merged.X, merged.layers["a"]) +@mark_legacy_concatenate def test_concatenate_layers_outer(array_type, fill_val): # Testing that issue #368 is fixed a = AnnData( @@ -437,6 +446,7 @@ def test_concatenate_layers_outer(array_type, fill_val): ) +@mark_legacy_concatenate def test_concatenate_fill_value(fill_val): def get_obs_els(adata): return { @@ -482,6 +492,7 @@ def get_obs_els(adata): ptr += orig.n_obs +@mark_legacy_concatenate def test_concatenate_dense_duplicates(): X1 = np.array([[1, 2, 3], [4, 5, 6]]) X2 = np.array([[1, 2, 3], [4, 5, 6]]) @@ -533,6 +544,7 @@ def test_concatenate_dense_duplicates(): ] +@mark_legacy_concatenate def test_concatenate_sparse(): # sparse data from scipy.sparse import csr_matrix @@ -578,6 +590,7 @@ def test_concatenate_sparse(): ] +@mark_legacy_concatenate def test_concatenate_mixed(): X1 = sparse.csr_matrix(np.array([[1, 2, 0], [4, 0, 6], [0, 0, 9]])) X2 = sparse.csr_matrix(np.array([[0, 2, 3], [4, 0, 0], [7, 0, 9]])) @@ -613,6 +626,7 @@ def test_concatenate_mixed(): assert isinstance(adata_all.layers["counts"], sparse.csr_matrix) +@mark_legacy_concatenate def test_concatenate_with_raw(): # dense data X1 = np.array([[1, 2, 3], [4, 5, 6]]) @@ -702,8 +716,9 @@ def test_concatenate_awkward(join_type): ] ) - adata_a = AnnData(np.zeros((2, 0), dtype=float), obsm={"awk": a}) - adata_b = AnnData(np.zeros((3, 0), dtype=float), obsm={"awk": b}) + with pytest.warns(ExperimentalFeatureWarning): + adata_a = AnnData(np.zeros((2, 0), dtype=float), obsm={"awk": a}) + adata_b = AnnData(np.zeros((3, 0), dtype=float), obsm={"awk": b}) if join_type == "inner": expected = ak.Array( @@ -746,7 +761,13 @@ def test_concatenate_awkward(join_type): ] ) - result = concat([adata_a, adata_b], join=join_type).obsm["awk"] + ctx = ( + pytest.warns(ExperimentalFeatureWarning) + if join_type == "outer" + else nullcontext() + ) + with ctx: + result = concat([adata_a, adata_b], join=join_type).obsm["awk"] assert_equal(expected, result) @@ -766,11 +787,12 @@ def test_awkward_does_not_mix(join_type, other): [[{"a": 1, "b": "foo"}], [{"a": 2, "b": "bar"}, {"a": 3, "b": "baz"}]] ) - adata_a = AnnData( - np.zeros((2, 3), dtype=float), - obs=pd.DataFrame(index=list("ab")), - obsm={"val": awk}, - ) + with pytest.warns(ExperimentalFeatureWarning): + adata_a = AnnData( + np.zeros((2, 3), dtype=float), + obs=pd.DataFrame(index=list("ab")), + obsm={"val": awk}, + ) adata_b = AnnData( np.zeros((3, 3), dtype=float), obs=pd.DataFrame(index=list("cde")), @@ -1318,14 +1340,24 @@ def test_concat_size_0_dim(axis, join_type, merge_strategy, shape): dim = ("obs", "var")[axis] expected_size = expected_shape(a, b, axis=axis, join=join_type) - result = concat( - {"a": a, "b": b}, - axis=axis, - join=join_type, - merge=merge_strategy, - pairwise=True, - index_unique="-", + + ctx = ( + pytest.warns( + ExperimentalFeatureWarning, + match=r"Outer joins on awkward.Arrays will have different return values in the future.", + ) + if join_type == "outer" + else nullcontext() ) + with ctx: + result = concat( + {"a": a, "b": b}, + axis=axis, + join=join_type, + merge=merge_strategy, + pairwise=True, + index_unique="-", + ) assert result.shape == expected_size if join_type == "outer": @@ -1374,6 +1406,7 @@ def test_concat_outer_aligned_mapping(elem): check_filled_like(result, elem_name=f"obsm/{elem}") +@mark_legacy_concatenate def test_concatenate_size_0_dim(): # https://github.com/scverse/anndata/issues/526 diff --git a/anndata/tests/test_deprecations.py b/anndata/tests/test_deprecations.py index cfa813c34..31e69cf25 100644 --- a/anndata/tests/test_deprecations.py +++ b/anndata/tests/test_deprecations.py @@ -38,26 +38,24 @@ def test_get_obsvar_array_warn(adata): adata._get_var_array("s1") -# TODO: Why doesn’t this mark work? -# @pytest.mark.filterwarnings("ignore::DeprecationWarning") +@pytest.mark.filterwarnings("ignore::DeprecationWarning") def test_get_obsvar_array(adata): - with pytest.warns(DeprecationWarning): # Just to hide warnings - assert np.allclose(adata._get_obs_array("a"), adata.obs_vector("a")) - assert np.allclose( - adata._get_obs_array("a", layer="x2"), - adata.obs_vector("a", layer="x2"), - ) - assert np.allclose( - adata._get_obs_array("a", use_raw=True), adata.raw.obs_vector("a") - ) - assert np.allclose(adata._get_var_array("s1"), adata.var_vector("s1")) - assert np.allclose( - adata._get_var_array("s1", layer="x2"), - adata.var_vector("s1", layer="x2"), - ) - assert np.allclose( - adata._get_var_array("s1", use_raw=True), adata.raw.var_vector("s1") - ) + assert np.allclose(adata._get_obs_array("a"), adata.obs_vector("a")) + assert np.allclose( + adata._get_obs_array("a", layer="x2"), + adata.obs_vector("a", layer="x2"), + ) + assert np.allclose( + adata._get_obs_array("a", use_raw=True), adata.raw.obs_vector("a") + ) + assert np.allclose(adata._get_var_array("s1"), adata.var_vector("s1")) + assert np.allclose( + adata._get_var_array("s1", layer="x2"), + adata.var_vector("s1", layer="x2"), + ) + assert np.allclose( + adata._get_var_array("s1", use_raw=True), adata.raw.var_vector("s1") + ) def test_obsvar_vector_Xlayer(adata): @@ -123,7 +121,7 @@ def test_deprecated_read(tmp_path): memory.write_h5ad(tmp_path / "file.h5ad") with pytest.warns(FutureWarning, match="`anndata.read` is deprecated"): - from_disk = ad.read_h5ad(tmp_path / "file.h5ad") + from_disk = ad.read(tmp_path / "file.h5ad") assert_equal(memory, from_disk) From 525b8fa57a80018f4125308a674e192f40463bfd Mon Sep 17 00:00:00 2001 From: "Philipp A." Date: Tue, 10 Oct 2023 17:39:51 +0200 Subject: [PATCH 07/43] done --- .azure-pipelines.yml | 6 +++--- anndata/_core/anndata.py | 10 +++++++++- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/.azure-pipelines.yml b/.azure-pipelines.yml index 88d2e9cdc..29cd6160e 100644 --- a/.azure-pipelines.yml +++ b/.azure-pipelines.yml @@ -55,12 +55,12 @@ jobs: - script: | pytest displayName: "PyTest" - condition: eq(variables['RUN_COVERAGE'], 'no') && eq(variables['PRERELEASE_DEPENDENCIES'], 'no') + condition: and(eq(variables['RUN_COVERAGE'], 'no'), eq(variables['PRERELEASE_DEPENDENCIES'], 'no')) - script: | pytest --cov --cov-report=xml --cov-context=test displayName: "PyTest (coverage)" - condition: eq(variables['RUN_COVERAGE'], 'yes') && eq(variables['PRERELEASE_DEPENDENCIES'], 'no') + condition: and(eq(variables['RUN_COVERAGE'], 'yes'), eq(variables['PRERELEASE_DEPENDENCIES'], 'no')) # TODO: fix all the exceptions here - script: > @@ -72,7 +72,7 @@ jobs: -W 'default::scipy.sparse._base.SparseEfficiencyWarning' -W 'default::dask.array.core.PerformanceWarning' displayName: "PyTest (treat warnings as errors)" - condition: eq(variables['RUN_COVERAGE'], 'no') && eq(variables['PRERELEASE_DEPENDENCIES'], 'yes') + condition: and(eq(variables['RUN_COVERAGE'], 'no'), eq(variables['PRERELEASE_DEPENDENCIES'], 'yes')) - task: PublishCodeCoverageResults@1 inputs: diff --git a/anndata/_core/anndata.py b/anndata/_core/anndata.py index 67f2c710e..6671d45f3 100644 --- a/anndata/_core/anndata.py +++ b/anndata/_core/anndata.py @@ -1687,7 +1687,7 @@ def concatenate( Examples -------- - Joining on intersection of variables. + Joining on intersection of variables. First, prepare example data: >>> adata1 = AnnData( ... np.array([[1, 2, 3], [4, 5, 6]]), @@ -1704,6 +1704,14 @@ def concatenate( ... dict(obs_names=['s1', 's2'], anno2=['d3', 'd4']), ... dict(var_names=['d', 'c', 'b'], annoA=[0, 2, 3], annoB=[0, 1, 2]), ... ) + + They can now be concatenated. Since this method is deprecated, it will raise a warning: + + >>> warnings.filterwarnings( + ... 'ignore', + ... r'The AnnData\\.concatenate method is deprecated', + ... FutureWarning, + ... ) >>> adata = adata1.concatenate(adata2, adata3) >>> adata AnnData object with n_obs × n_vars = 6 × 2 From 0adaa2c0f1bb6aae41aa9d202cc916e21020982d Mon Sep 17 00:00:00 2001 From: "Philipp A." Date: Tue, 10 Oct 2023 17:46:00 +0200 Subject: [PATCH 08/43] maybe this works --- .azure-pipelines.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.azure-pipelines.yml b/.azure-pipelines.yml index 29cd6160e..9a149a11c 100644 --- a/.azure-pipelines.yml +++ b/.azure-pipelines.yml @@ -63,10 +63,12 @@ jobs: condition: and(eq(variables['RUN_COVERAGE'], 'yes'), eq(variables['PRERELEASE_DEPENDENCIES'], 'no')) # TODO: fix all the exceptions here + # replacing “:initializing view as actual:UserWarning“ with “::anndata._warnings.ImplicitModificationWarning” + # breaks because we’re not using editable installs or a src/ layout - script: > pytest -W error - -W 'default::anndata._warnings.ImplicitModificationWarning' + -W 'default:initializing view as actual:UserWarning' -W 'default:Observation names are not unique. To make them unique:UserWarning' -W 'default:Variable names are not unique. To make them unique:UserWarning' -W 'default::scipy.sparse._base.SparseEfficiencyWarning' From c7a121b6f330a16a7116ac1c7f03fd5e15894f76 Mon Sep 17 00:00:00 2001 From: "Philipp A." Date: Tue, 10 Oct 2023 17:57:50 +0200 Subject: [PATCH 09/43] extract pruning from unify_dtypes --- anndata/_core/merge.py | 26 +++++++++++++++++--------- anndata/experimental/merge.py | 7 +++---- 2 files changed, 20 insertions(+), 13 deletions(-) diff --git a/anndata/_core/merge.py b/anndata/_core/merge.py index dcf5f3e47..9ba257138 100644 --- a/anndata/_core/merge.py +++ b/anndata/_core/merge.py @@ -207,9 +207,7 @@ def as_cp_sparse(x) -> CupySparseMatrix: return cpsparse.csr_matrix(x) -def unify_dtypes( - dfs: Iterable[pd.DataFrame], *, prune: bool = True -) -> list[pd.DataFrame]: +def _unify_dtypes(dfs: Iterable[pd.DataFrame]) -> list[pd.DataFrame]: """ Attempts to unify datatypes from multiple dataframes. @@ -240,7 +238,18 @@ def unify_dtypes( if col in df: df[col] = df[col].astype(dtype) - return [df for df in dfs if len(df)] if prune else dfs + return dfs + + +def concat_with_unified_dtypes( + dfs: Iterable[pd.DataFrame], + *, + join: Literal["inner", "outer"] = "outer", + axis: Literal[0, 1] = 0, +) -> pd.DataFrame: + dfs = _unify_dtypes(dfs) + nonempty_dfs = [df for df in dfs if len(df)] + return pd.concat(nonempty_dfs, ignore_index=True, join=join, axis=axis) def try_unifying_dtype( @@ -747,9 +756,8 @@ def concat_arrays(arrays, reindexers, axis=0, index=None, fill_value=None): "Cannot concatenate a dataframe with other array types." ) # TODO: behaviour here should be chosen through a merge strategy - df = pd.concat( - unify_dtypes([f(x) for f, x in zip(reindexers, arrays)]), - ignore_index=True, + df = concat_with_unified_dtypes( + [f(x) for f, x in zip(reindexers, arrays)], axis=axis, ) df.index = index @@ -1251,8 +1259,8 @@ def concat( # Annotation for concatenation axis check_combinable_cols([getattr(a, dim).columns for a in adatas], join=join) - concat_annot = pd.concat( - unify_dtypes([getattr(a, dim) for a in adatas]), + concat_annot = concat_with_unified_dtypes( + [getattr(a, dim) for a in adatas], join=join, ignore_index=True, ) diff --git a/anndata/experimental/merge.py b/anndata/experimental/merge.py index 69b97377a..4eba37572 100644 --- a/anndata/experimental/merge.py +++ b/anndata/experimental/merge.py @@ -22,13 +22,13 @@ StrategiesLiteral, _resolve_dim, concat_arrays, + concat_with_unified_dtypes, gen_inner_reindexers, gen_reindexer, intersect_keys, merge_dataframes, merge_indices, resolve_merge_strategy, - unify_dtypes, ) from .._core.sparse_dataset import BaseCompressedSparseDataset, sparse_dataset from .._io.specs import read_elem, write_elem @@ -383,10 +383,9 @@ def _write_alt_annot(groups, output_group, alt_dim, alt_indices, merge): def _write_dim_annot(groups, output_group, dim, concat_indices, label, label_col, join): - concat_annot = pd.concat( - unify_dtypes([read_elem(g[dim]) for g in groups]), + concat_annot = concat_with_unified_dtypes( + [read_elem(g[dim]) for g in groups], join=join, - ignore_index=True, ) concat_annot.index = concat_indices if label is not None: From c06845c0bdc77625c615205bdac78f91ea128efd Mon Sep 17 00:00:00 2001 From: "Philipp A." Date: Tue, 10 Oct 2023 17:59:44 +0200 Subject: [PATCH 10/43] =?UTF-8?q?circumvent=20other=20ImplicitModification?= =?UTF-8?q?Warning=E2=80=9D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .azure-pipelines.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.azure-pipelines.yml b/.azure-pipelines.yml index 9a149a11c..1595bb021 100644 --- a/.azure-pipelines.yml +++ b/.azure-pipelines.yml @@ -63,12 +63,14 @@ jobs: condition: and(eq(variables['RUN_COVERAGE'], 'yes'), eq(variables['PRERELEASE_DEPENDENCIES'], 'no')) # TODO: fix all the exceptions here - # replacing “:initializing view as actual:UserWarning“ with “::anndata._warnings.ImplicitModificationWarning” - # breaks because we’re not using editable installs or a src/ layout + # replacing “:initializing view as actual” and “:Transforming to str index” + # with “::anndata._warnings.ImplicitModificationWarning” breaks, + # because we’re neither using editable installs nor a src/ layout - script: > pytest -W error -W 'default:initializing view as actual:UserWarning' + -W 'default:Transforming to str index:UserWarning' -W 'default:Observation names are not unique. To make them unique:UserWarning' -W 'default:Variable names are not unique. To make them unique:UserWarning' -W 'default::scipy.sparse._base.SparseEfficiencyWarning' From 706afb8b5ba77819298f599e3422dc356d1d06ed Mon Sep 17 00:00:00 2001 From: "Philipp A." Date: Tue, 10 Oct 2023 18:17:36 +0200 Subject: [PATCH 11/43] whoops --- anndata/_core/merge.py | 1 - 1 file changed, 1 deletion(-) diff --git a/anndata/_core/merge.py b/anndata/_core/merge.py index 9ba257138..4788cd637 100644 --- a/anndata/_core/merge.py +++ b/anndata/_core/merge.py @@ -1262,7 +1262,6 @@ def concat( concat_annot = concat_with_unified_dtypes( [getattr(a, dim) for a in adatas], join=join, - ignore_index=True, ) concat_annot.index = concat_indices if label is not None: From fcac1906ba80d63e9cb76f51d37ffdaaec3b619e Mon Sep 17 00:00:00 2001 From: "Philipp A." Date: Tue, 10 Oct 2023 18:28:09 +0200 Subject: [PATCH 12/43] =?UTF-8?q?can=E2=80=99t=20use=20regexes?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .azure-pipelines.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.azure-pipelines.yml b/.azure-pipelines.yml index 1595bb021..4e51a9817 100644 --- a/.azure-pipelines.yml +++ b/.azure-pipelines.yml @@ -63,13 +63,13 @@ jobs: condition: and(eq(variables['RUN_COVERAGE'], 'yes'), eq(variables['PRERELEASE_DEPENDENCIES'], 'no')) # TODO: fix all the exceptions here - # replacing “:initializing view as actual” and “:Transforming to str index” + # replacing “:Trying to modify attribute” and “:Transforming to str index” # with “::anndata._warnings.ImplicitModificationWarning” breaks, # because we’re neither using editable installs nor a src/ layout - script: > pytest -W error - -W 'default:initializing view as actual:UserWarning' + -W 'default:Trying to modify attribute:UserWarning' -W 'default:Transforming to str index:UserWarning' -W 'default:Observation names are not unique. To make them unique:UserWarning' -W 'default:Variable names are not unique. To make them unique:UserWarning' From a55bb60683ffddddcc9d36b64f19b24513687195 Mon Sep 17 00:00:00 2001 From: Philipp A Date: Tue, 10 Oct 2023 19:11:23 +0200 Subject: [PATCH 13/43] Update pyproject.toml Co-authored-by: Isaac Virshup --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index bef135269..e108410f2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -91,7 +91,7 @@ test = [ "httpx", # For data downloading "dask[array,distributed]", "awkward>=2.3", - "pyarrow", # For non-deprecated awkward array functionality + "pyarrow", "pytest_memray", ] gpu = ["cupy"] From 825ab5fd4381be944cf0fbe7f8ce3d53dcf3b97c Mon Sep 17 00:00:00 2001 From: "Philipp A." Date: Thu, 12 Oct 2023 09:57:50 +0200 Subject: [PATCH 14/43] undo concat_with_unified_dtypes --- anndata/_core/merge.py | 23 +++++++---------------- anndata/experimental/merge.py | 7 ++++--- 2 files changed, 11 insertions(+), 19 deletions(-) diff --git a/anndata/_core/merge.py b/anndata/_core/merge.py index 08e780996..70c32248c 100644 --- a/anndata/_core/merge.py +++ b/anndata/_core/merge.py @@ -214,7 +214,7 @@ def as_cp_sparse(x) -> CupySparseMatrix: return cpsparse.csr_matrix(x) -def _unify_dtypes(dfs: Iterable[pd.DataFrame]) -> list[pd.DataFrame]: +def unify_dtypes(dfs: Iterable[pd.DataFrame]) -> list[pd.DataFrame]: """ Attempts to unify datatypes from multiple dataframes. @@ -248,17 +248,6 @@ def _unify_dtypes(dfs: Iterable[pd.DataFrame]) -> list[pd.DataFrame]: return dfs -def concat_with_unified_dtypes( - dfs: Iterable[pd.DataFrame], - *, - join: Literal["inner", "outer"] = "outer", - axis: Literal[0, 1] = 0, -) -> pd.DataFrame: - dfs = _unify_dtypes(dfs) - nonempty_dfs = [df for df in dfs if len(df)] - return pd.concat(nonempty_dfs, ignore_index=True, join=join, axis=axis) - - def try_unifying_dtype( col: Sequence[np.dtype | ExtensionDtype], ) -> pd.core.dtypes.base.ExtensionDtype | None: @@ -763,9 +752,10 @@ def concat_arrays(arrays, reindexers, axis=0, index=None, fill_value=None): "Cannot concatenate a dataframe with other array types." ) # TODO: behaviour here should be chosen through a merge strategy - df = concat_with_unified_dtypes( - [f(x) for f, x in zip(reindexers, arrays)], + df = pd.concat( + unify_dtypes(f(x) for f, x in zip(reindexers, arrays)), axis=axis, + ignore_index=True, ) df.index = index return df @@ -1266,9 +1256,10 @@ def concat( # Annotation for concatenation axis check_combinable_cols([getattr(a, dim).columns for a in adatas], join=join) - concat_annot = concat_with_unified_dtypes( - [getattr(a, dim) for a in adatas], + concat_annot = pd.concat( + unify_dtypes(getattr(a, dim) for a in adatas), join=join, + ignore_index=True, ) concat_annot.index = concat_indices if label is not None: diff --git a/anndata/experimental/merge.py b/anndata/experimental/merge.py index 4dd239cb5..95b0b215f 100644 --- a/anndata/experimental/merge.py +++ b/anndata/experimental/merge.py @@ -22,13 +22,13 @@ StrategiesLiteral, _resolve_dim, concat_arrays, - concat_with_unified_dtypes, gen_inner_reindexers, gen_reindexer, intersect_keys, merge_dataframes, merge_indices, resolve_merge_strategy, + unify_dtypes, ) from .._core.sparse_dataset import BaseCompressedSparseDataset, sparse_dataset from .._io.specs import read_elem, write_elem @@ -383,9 +383,10 @@ def _write_alt_annot(groups, output_group, alt_dim, alt_indices, merge): def _write_dim_annot(groups, output_group, dim, concat_indices, label, label_col, join): - concat_annot = concat_with_unified_dtypes( - [read_elem(g[dim]) for g in groups], + concat_annot = pd.concat( + unify_dtypes(read_elem(g[dim]) for g in groups), join=join, + ignore_index=True, ) concat_annot.index = concat_indices if label is not None: From 78c6f41ae2aa0b48afb4f87ceb94af4d3230705d Mon Sep 17 00:00:00 2001 From: "Philipp A." Date: Thu, 12 Oct 2023 10:27:52 +0200 Subject: [PATCH 15/43] make into list --- anndata/_core/merge.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/anndata/_core/merge.py b/anndata/_core/merge.py index 70c32248c..92fa34d18 100644 --- a/anndata/_core/merge.py +++ b/anndata/_core/merge.py @@ -36,7 +36,7 @@ DaskArray, _map_cat_to_str, ) -from ..utils import asarray, dim_len +from ..utils import asarray, dim_len, warn_once from .anndata import AnnData from .index import _subset, make_slice @@ -220,6 +220,7 @@ def unify_dtypes(dfs: Iterable[pd.DataFrame]) -> list[pd.DataFrame]: For catching cases where pandas would convert to object dtype. """ + dfs = list(dfs) # Get shared categorical columns df_dtypes = [dict(df.dtypes) for df in dfs] columns = reduce(lambda x, y: x.union(y), [df.columns for df in dfs]) From 2fd024c9d3a3182e81df61d84c528737de5d427f Mon Sep 17 00:00:00 2001 From: Philipp A Date: Thu, 12 Oct 2023 11:32:24 +0200 Subject: [PATCH 16/43] just ignore Co-authored-by: Isaac Virshup --- anndata/tests/helpers.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/anndata/tests/helpers.py b/anndata/tests/helpers.py index ae8e61fa2..b2741cde7 100644 --- a/anndata/tests/helpers.py +++ b/anndata/tests/helpers.py @@ -256,14 +256,12 @@ def gen_adata( awkward_ragged=gen_awkward((12, None, None)), # U_recarray=gen_vstr_recarray(N, 5, "U4") ) - - ctx = ( - pytest.warns(ExperimentalFeatureWarning) - if AwkArray in {*obsm_types, *varm_types} - else nullcontext() - ) - - with ctx: + with warnings.catch_warnings(): + warnings.filterwarnings( + "ignore", + "Support for Awkward Arrays is currently experimental", + ExperimentalFeatureWarning, + ) adata = AnnData( X=X, obs=obs, From c29d4d52a3fed318d20714056afbc4e8dc33ecdf Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 12 Oct 2023 09:32:43 +0000 Subject: [PATCH 17/43] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- anndata/tests/helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/anndata/tests/helpers.py b/anndata/tests/helpers.py index b2741cde7..342d365e7 100644 --- a/anndata/tests/helpers.py +++ b/anndata/tests/helpers.py @@ -4,7 +4,7 @@ import re import warnings from collections.abc import Collection, Mapping -from contextlib import contextmanager, nullcontext +from contextlib import contextmanager from functools import partial, singledispatch, wraps from string import ascii_letters From edde4bd3a29d042c26e4a5c96b9e5a4e09dcb887 Mon Sep 17 00:00:00 2001 From: "Philipp A." Date: Thu, 12 Oct 2023 11:33:10 +0200 Subject: [PATCH 18/43] Catch empty df concat warnings --- anndata/_io/utils.py | 4 ++-- anndata/compat/__init__.py | 4 ++-- anndata/tests/test_awkward.py | 10 +++++++++- anndata/tests/test_concatenate.py | 5 ++++- 4 files changed, 17 insertions(+), 6 deletions(-) diff --git a/anndata/_io/utils.py b/anndata/_io/utils.py index 964f94811..cd90be473 100644 --- a/anndata/_io/utils.py +++ b/anndata/_io/utils.py @@ -5,7 +5,7 @@ from warnings import warn import h5py -from packaging import version +from packaging.version import Version from anndata.compat import H5Group, ZarrGroup, add_note @@ -13,7 +13,7 @@ # For allowing h5py v3 # https://github.com/scverse/anndata/issues/442 -H5PY_V3 = version.parse(h5py.__version__).major >= 3 +H5PY_V3 = Version(h5py.__version__).major >= 3 # ------------------------------------------------------------------------------- # Type conversion diff --git a/anndata/compat/__init__.py b/anndata/compat/__init__.py index 0b0542a0d..a0a77977f 100644 --- a/anndata/compat/__init__.py +++ b/anndata/compat/__init__.py @@ -14,7 +14,7 @@ import h5py import numpy as np import pandas as pd -from packaging.version import parse as _parse_version +from packaging.version import Version from scipy.sparse import issparse, spmatrix from .exceptiongroups import add_note # noqa: F401 @@ -395,7 +395,7 @@ def _safe_transpose(x): def _map_cat_to_str(cat: pd.Categorical) -> pd.Categorical: - if _parse_version(pd.__version__) >= _parse_version("2.0"): + if Version(pd.__version__) >= Version("2.0"): # Argument added in pandas 2.0 return cat.map(str, na_action="ignore") else: diff --git a/anndata/tests/test_awkward.py b/anndata/tests/test_awkward.py index a7b7753e6..e43d6da7b 100644 --- a/anndata/tests/test_awkward.py +++ b/anndata/tests/test_awkward.py @@ -395,7 +395,15 @@ def test_concat_mixed_types(key, arrays, expected, join): to_concat.append(tmp_adata) if isinstance(expected, type) and issubclass(expected, Exception): - with pytest.raises(expected): + ctx = ( + pytest.warns( + FutureWarning, + match=r"The behavior of DataFrame concatenation with empty or all-NA entries is deprecated", + ) + if any(df.empty for df in arrays if isinstance(df, pd.DataFrame)) + else nullcontext() + ) + with pytest.raises(expected), ctx: anndata.concat(to_concat, axis=axis, join=join) else: print(to_concat) diff --git a/anndata/tests/test_concatenate.py b/anndata/tests/test_concatenate.py index d29ba4cb8..8d0cc970e 100644 --- a/anndata/tests/test_concatenate.py +++ b/anndata/tests/test_concatenate.py @@ -1349,7 +1349,10 @@ def test_concat_size_0_dim(axis, join_type, merge_strategy, shape): if join_type == "outer" else nullcontext() ) - with ctx: + with ctx, pytest.warns( + FutureWarning, + match=r"The behavior of DataFrame concatenation with empty or all-NA entries is deprecated", + ): result = concat( {"a": a, "b": b}, axis=axis, From 63f9dbb9a8373cb3914c7a1de5c7c9a7b56c293b Mon Sep 17 00:00:00 2001 From: Philipp A Date: Thu, 12 Oct 2023 11:33:57 +0200 Subject: [PATCH 19/43] =?UTF-8?q?don=E2=80=99t=20hide=20loompy=20warning?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Isaac Virshup --- anndata/_io/write.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/anndata/_io/write.py b/anndata/_io/write.py index 75240b779..5a4d2ca62 100644 --- a/anndata/_io/write.py +++ b/anndata/_io/write.py @@ -103,11 +103,7 @@ def write_loom(filename: PathLike, adata: AnnData, write_obsm_varm: bool = False for key in adata.layers.keys(): layers[key] = adata.layers[key].T - from numba.core.errors import NumbaDeprecationWarning - - with warnings.catch_warnings(): - warnings.simplefilter("ignore", NumbaDeprecationWarning) - from loompy import create + from loompy import create if filename.exists(): filename.unlink() From f3c74b6d586d69cad96f84a17e35614c0b0af44b Mon Sep 17 00:00:00 2001 From: "Philipp A." Date: Thu, 12 Oct 2023 11:41:35 +0200 Subject: [PATCH 20/43] Ignore loompy errors just in tests --- anndata/tests/test_readwrite.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/anndata/tests/test_readwrite.py b/anndata/tests/test_readwrite.py index dbbee8be3..0c9317526 100644 --- a/anndata/tests/test_readwrite.py +++ b/anndata/tests/test_readwrite.py @@ -13,6 +13,7 @@ import pandas as pd import pytest import zarr +from numba.core.errors import NumbaDeprecationWarning from scipy.sparse import csc_matrix, csr_matrix import anndata as ad @@ -388,7 +389,9 @@ def test_readwrite_loom(typ, obsm_mapping, varm_mapping, tmp_path): adata_src.obsm["X_a"] = np.zeros((adata_src.n_obs, 2)) adata_src.varm["X_b"] = np.zeros((adata_src.n_vars, 3)) - adata_src.write_loom(tmp_path / "test.loom", write_obsm_varm=True) + with warnings.catch_warnings(): + warnings.simplefilter("ignore", NumbaDeprecationWarning) + adata_src.write_loom(tmp_path / "test.loom", write_obsm_varm=True) adata = ad.read_loom( tmp_path / "test.loom", From 8eb9981dffebefb92b91be728a1d103bd8d21e76 Mon Sep 17 00:00:00 2001 From: "Philipp A." Date: Thu, 12 Oct 2023 11:46:45 +0200 Subject: [PATCH 21/43] add comment for assert false --- anndata/_core/merge.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/anndata/_core/merge.py b/anndata/_core/merge.py index 92fa34d18..7757acc0b 100644 --- a/anndata/_core/merge.py +++ b/anndata/_core/merge.py @@ -1282,7 +1282,7 @@ def concat( ) join_keys = union_keys else: - assert False + assert False, f"{join=} should have been validated above by pd.concat" layers = concat_aligned_mapping( [a.layers for a in adatas], axis=axis, reindexers=reindexers From 63aacb1840c1545884597b6ae5966bca8c5bebb7 Mon Sep 17 00:00:00 2001 From: "Philipp A." Date: Thu, 12 Oct 2023 12:04:29 +0200 Subject: [PATCH 22/43] fix test_concat_size_0_dim --- anndata/tests/test_concatenate.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/anndata/tests/test_concatenate.py b/anndata/tests/test_concatenate.py index 8d0cc970e..5ccba91ff 100644 --- a/anndata/tests/test_concatenate.py +++ b/anndata/tests/test_concatenate.py @@ -6,7 +6,7 @@ from copy import deepcopy from functools import partial, singledispatch from itertools import chain, permutations, product -from typing import Any, Callable +from typing import Any, Callable, Literal import numpy as np import pandas as pd @@ -98,7 +98,7 @@ def fill_val(request): @pytest.fixture(params=[0, 1]) -def axis(request): +def axis(request) -> Literal[0, 1]: return request.param @@ -1341,7 +1341,7 @@ def test_concat_size_0_dim(axis, join_type, merge_strategy, shape): expected_size = expected_shape(a, b, axis=axis, join=join_type) - ctx = ( + ctx_awk = ( pytest.warns( ExperimentalFeatureWarning, match=r"Outer joins on awkward.Arrays will have different return values in the future.", @@ -1349,10 +1349,15 @@ def test_concat_size_0_dim(axis, join_type, merge_strategy, shape): if join_type == "outer" else nullcontext() ) - with ctx, pytest.warns( - FutureWarning, - match=r"The behavior of DataFrame concatenation with empty or all-NA entries is deprecated", - ): + ctx_concat_empty = ( + pytest.warns( + FutureWarning, + match=r"The behavior of DataFrame concatenation with empty or all-NA entries is deprecated", + ) + if join_type == "inner" and shape[axis] == 0 + else nullcontext() + ) + with ctx_awk, ctx_concat_empty: result = concat( {"a": a, "b": b}, axis=axis, From 8035fd92643e25184772cc1ec0837848dc1b1f18 Mon Sep 17 00:00:00 2001 From: "Philipp A." Date: Thu, 12 Oct 2023 12:13:05 +0200 Subject: [PATCH 23/43] add release notes --- docs/release-notes/0.10.3.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/release-notes/0.10.3.md b/docs/release-notes/0.10.3.md index 7a0b9c163..9f1db07f5 100644 --- a/docs/release-notes/0.10.3.md +++ b/docs/release-notes/0.10.3.md @@ -5,6 +5,12 @@ ```{rubric} Documentation ``` +* Stop showing “Support for Awkward Arrays is currently experimental” warnings when + reading, concatenating, slicing, or transposing AnnData objects {pr}`1182` {user}`flying-sheep` ```{rubric} Performance ``` + +```{rubric} Other updates +``` +* Fail canary CI job when tests raise unexpected warnings. {pr}`1182` {user}`flying-sheep` From 391931ee4bb06edee64e666056aac6e425e7db12 Mon Sep 17 00:00:00 2001 From: "Philipp A." Date: Thu, 12 Oct 2023 12:16:41 +0200 Subject: [PATCH 24/43] Remaining warnings --- .azure-pipelines.yml | 3 ++- anndata/tests/test_layers.py | 6 +++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/.azure-pipelines.yml b/.azure-pipelines.yml index 4e51a9817..7edf6e430 100644 --- a/.azure-pipelines.yml +++ b/.azure-pipelines.yml @@ -63,12 +63,13 @@ jobs: condition: and(eq(variables['RUN_COVERAGE'], 'yes'), eq(variables['PRERELEASE_DEPENDENCIES'], 'no')) # TODO: fix all the exceptions here - # replacing “:Trying to modify attribute” and “:Transforming to str index” + # replacing “:Setting element”, “:Trying to modify attribute” and “:Transforming to str index” # with “::anndata._warnings.ImplicitModificationWarning” breaks, # because we’re neither using editable installs nor a src/ layout - script: > pytest -W error + -W 'default:Setting element:UserWarning' -W 'default:Trying to modify attribute:UserWarning' -W 'default:Transforming to str index:UserWarning' -W 'default:Observation names are not unique. To make them unique:UserWarning' diff --git a/anndata/tests/test_layers.py b/anndata/tests/test_layers.py index 4b6a7f287..34b088976 100644 --- a/anndata/tests/test_layers.py +++ b/anndata/tests/test_layers.py @@ -6,6 +6,7 @@ import numpy as np import pandas as pd import pytest +from numba.core.errors import NumbaDeprecationWarning from anndata import AnnData, read_h5ad, read_loom from anndata.tests.helpers import gen_typed_df_t2_size @@ -78,7 +79,10 @@ def test_readwrite(backing_h5ad): def test_readwrite_loom(tmp_path): loom_path = tmp_path / "test.loom" adata = AnnData(X=X, layers=dict(L=L.copy())) - adata.write_loom(loom_path) + + with warnings.catch_warnings(): + warnings.simplefilter("ignore", NumbaDeprecationWarning) + adata.write_loom(loom_path) adata_read = read_loom(loom_path, X_name="") assert adata.layers.keys() == adata_read.layers.keys() From 361aa054bb4ac68450244b8bb2f7749bf24a4fe7 Mon Sep 17 00:00:00 2001 From: "Philipp A." Date: Thu, 12 Oct 2023 18:21:40 +0200 Subject: [PATCH 25/43] Whew --- anndata/_core/aligned_mapping.py | 4 +--- anndata/_core/anndata.py | 30 +++++++++--------------- anndata/_io/h5ad.py | 20 ++++++++-------- anndata/_io/zarr.py | 19 +++++++--------- anndata/tests/test_awkward.py | 2 +- anndata/tests/test_base.py | 9 ++++---- anndata/tests/test_concatenate.py | 6 +++-- anndata/tests/test_dask.py | 32 ++++++++++++++------------ anndata/tests/test_hdf5_backing.py | 4 ++++ anndata/tests/test_helpers.py | 23 +++++++++++++------ anndata/tests/test_inplace_subset.py | 22 ++++++++++-------- anndata/tests/test_io_conversion.py | 29 ++++++++++++++++++------ anndata/tests/test_io_elementwise.py | 8 ++++--- anndata/tests/test_readwrite.py | 34 +++++++++++++++++++--------- anndata/tests/test_transpose.py | 8 +++++-- anndata/tests/test_views.py | 4 ++++ anndata/tests/test_x.py | 9 +++++--- 17 files changed, 155 insertions(+), 108 deletions(-) diff --git a/anndata/_core/aligned_mapping.py b/anndata/_core/aligned_mapping.py index 5000596d5..b3db938d3 100644 --- a/anndata/_core/aligned_mapping.py +++ b/anndata/_core/aligned_mapping.py @@ -115,9 +115,7 @@ def copy(self): for k, v in self.items(): if isinstance(v, AwkArray): # Shallow copy since awkward array buffers are immutable - with warnings.catch_warnings(): - warnings.simplefilter("ignore", ExperimentalFeatureWarning) - d[k] = copy(v) + d[k] = copy(v) else: d[k] = v.copy() return d diff --git a/anndata/_core/anndata.py b/anndata/_core/anndata.py index 6671d45f3..75006e3e2 100644 --- a/anndata/_core/anndata.py +++ b/anndata/_core/anndata.py @@ -820,8 +820,6 @@ def raw(self) -> Raw: @raw.setter def raw(self, value: AnnData): - from .._warnings import ExperimentalFeatureWarning - if value is None: del self.raw elif not isinstance(value, AnnData): @@ -829,9 +827,7 @@ def raw(self, value: AnnData): else: if self.is_view: self._init_as_actual(self.copy()) - with warnings.catch_warnings(): - warnings.simplefilter("ignore", ExperimentalFeatureWarning) - self._raw = Raw(self, X=value.X, var=value.var, varm=value.varm) + self._raw = Raw(self, X=value.X, var=value.var, varm=value.varm) @raw.deleter def raw(self): @@ -1588,21 +1584,17 @@ def to_memory(self, copy=False) -> AnnData: def copy(self, filename: PathLike | None = None) -> AnnData: """Full copy, optionally on disk.""" - from .._warnings import ExperimentalFeatureWarning - if not self.isbacked: - with warnings.catch_warnings(): - warnings.simplefilter("ignore", ExperimentalFeatureWarning) - if self.is_view and self._has_X(): - # TODO: How do I unambiguously check if this is a copy? - # Subsetting this way means we don’t have to have a view type - # defined for the matrix, which is needed for some of the - # current distributed backend. Specifically Dask. - return self._mutated_copy( - X=_subset(self._adata_ref.X, (self._oidx, self._vidx)).copy() - ) - else: - return self._mutated_copy() + if self.is_view and self._has_X(): + # TODO: How do I unambiguously check if this is a copy? + # Subsetting this way means we don’t have to have a view type + # defined for the matrix, which is needed for some of the + # current distributed backend. Specifically Dask. + return self._mutated_copy( + X=_subset(self._adata_ref.X, (self._oidx, self._vidx)).copy() + ) + else: + return self._mutated_copy() else: from .._io import read_h5ad, write_h5ad diff --git a/anndata/_io/h5ad.py b/anndata/_io/h5ad.py index 5fac0406d..17c5e9989 100644 --- a/anndata/_io/h5ad.py +++ b/anndata/_io/h5ad.py @@ -234,17 +234,15 @@ def read_h5ad( def callback(func, elem_name: str, elem, iospec): if iospec.encoding_type == "anndata" or elem_name.endswith("/"): - with warnings.catch_warnings(): - warnings.simplefilter("ignore", ExperimentalFeatureWarning) - return AnnData( - **{ - # This is covering up backwards compat in the anndata initializer - # In most cases we should be able to call `func(elen[k])` instead - k: read_dispatched(elem[k], callback) - for k in elem.keys() - if not k.startswith("raw.") - } - ) + return AnnData( + **{ + # This is covering up backwards compat in the anndata initializer + # In most cases we should be able to call `func(elen[k])` instead + k: read_dispatched(elem[k], callback) + for k in elem.keys() + if not k.startswith("raw.") + } + ) elif elem_name.startswith("/raw."): return None elif elem_name == "/X" and "X" in as_sparse: diff --git a/anndata/_io/zarr.py b/anndata/_io/zarr.py index 20ec679be..00f9766f0 100644 --- a/anndata/_io/zarr.py +++ b/anndata/_io/zarr.py @@ -1,6 +1,5 @@ from __future__ import annotations -import warnings from pathlib import Path from typing import TYPE_CHECKING, TypeVar from warnings import warn @@ -10,7 +9,7 @@ import zarr from scipy import sparse -from anndata._warnings import ExperimentalFeatureWarning, OldFormatWarning +from anndata._warnings import OldFormatWarning from .._core.anndata import AnnData from ..compat import _clean_uns, _from_fixed_length_strings @@ -69,15 +68,13 @@ def read_zarr(store: str | Path | MutableMapping | zarr.Group) -> AnnData: # Read with handling for backwards compat def callback(func, elem_name: str, elem, iospec): if iospec.encoding_type == "anndata" or elem_name.endswith("/"): - with warnings.catch_warnings(): - warnings.simplefilter("ignore", ExperimentalFeatureWarning) - return AnnData( - **{ - k: read_dispatched(v, callback) - for k, v in elem.items() - if not k.startswith("raw.") - } - ) + return AnnData( + **{ + k: read_dispatched(v, callback) + for k, v in elem.items() + if not k.startswith("raw.") + } + ) elif elem_name.startswith("/raw."): return None elif elem_name in {"/obs", "/var"}: diff --git a/anndata/tests/test_awkward.py b/anndata/tests/test_awkward.py index e43d6da7b..6f4c9548e 100644 --- a/anndata/tests/test_awkward.py +++ b/anndata/tests/test_awkward.py @@ -122,7 +122,7 @@ def test_copy(key): ctx = pytest.warns(ExperimentalFeatureWarning) if key != "uns" else nullcontext() with ctx: getattr(adata, key)["awk"] = ak.Array([{"a": [1], "b": [2], "c": [3]}] * 3) - adata_copy = adata.copy() + adata_copy = adata.copy() getattr(adata_copy, key)["awk"]["c"] = np.full((3, 1), 4) getattr(adata_copy, key)["awk"]["d"] = np.full((3, 1), 5) diff --git a/anndata/tests/test_base.py b/anndata/tests/test_base.py index 14127271f..b0959f3a6 100644 --- a/anndata/tests/test_base.py +++ b/anndata/tests/test_base.py @@ -11,7 +11,7 @@ from scipy import sparse as sp from scipy.sparse import csr_matrix, issparse -from anndata import AnnData +from anndata import AnnData, ExperimentalFeatureWarning from anndata.tests.helpers import assert_equal, gen_adata # some test objects that we use below @@ -224,9 +224,10 @@ def test_setting_dim_index(dim): mapping_attr = f"{dim}m" orig = gen_adata((5, 5)) - orig.raw = orig - curr = orig.copy() - view = orig[:, :] + with pytest.warns(ExperimentalFeatureWarning): + orig.raw = orig + curr = orig.copy() + view = orig[:, :] new_idx = pd.Index(list("abcde"), name="letters") setattr(curr, index_attr, new_idx) diff --git a/anndata/tests/test_concatenate.py b/anndata/tests/test_concatenate.py index 5ccba91ff..2f164fec8 100644 --- a/anndata/tests/test_concatenate.py +++ b/anndata/tests/test_concatenate.py @@ -1382,7 +1382,8 @@ def test_concat_size_0_dim(axis, join_type, merge_strategy, shape): check_filled_like(elem[altaxis_idx], elem_name=f"layers/{k}") if shape[axis] > 0: - b_result = result[axis_idx].copy() + with pytest.warns(ExperimentalFeatureWarning): + b_result = result[axis_idx].copy() mapping_elem = f"{dim}m" setattr(b_result, f"{dim}_names", getattr(b, f"{dim}_names")) for k, result_elem in getattr(b_result, mapping_elem).items(): @@ -1430,7 +1431,8 @@ def test_concat_null_X(): adatas_orig = {k: gen_adata((20, 10)) for k in list("abc")} adatas_no_X = {} for k, v in adatas_orig.items(): - v = v.copy() + with pytest.warns(ExperimentalFeatureWarning): + v = v.copy() del v.X adatas_no_X[k] = v diff --git a/anndata/tests/test_dask.py b/anndata/tests/test_dask.py index 7bd353f24..962609e63 100644 --- a/anndata/tests/test_dask.py +++ b/anndata/tests/test_dask.py @@ -3,6 +3,8 @@ """ from __future__ import annotations +import warnings + import pandas as pd import pytest @@ -107,21 +109,21 @@ def test_dask_distributed_write(adata, tmp_path, diskfmt): pth = tmp_path / f"test_write.{diskfmt}" g = as_group(pth, mode="w") - with dd.LocalCluster(n_workers=1, threads_per_worker=1, processes=False) as cluster: - with dd.Client(cluster): - M, N = adata.X.shape - adata.obsm["a"] = da.random.random((M, 10)) - adata.obsm["b"] = da.random.random((M, 10)) - adata.varm["a"] = da.random.random((N, 10)) - orig = adata - if diskfmt == "h5ad": - with pytest.raises( - ValueError, match="Cannot write dask arrays to hdf5" - ): - write_elem(g, "", orig) - return - write_elem(g, "", orig) - curr = read_elem(g) + with dd.LocalCluster( + n_workers=1, threads_per_worker=1, processes=False + ) as cluster, dd.Client(cluster), warnings.catch_warnings(): + warnings.simplefilter("ignore", ad.ExperimentalFeatureWarning) + M, N = adata.X.shape + adata.obsm["a"] = da.random.random((M, 10)) + adata.obsm["b"] = da.random.random((M, 10)) + adata.varm["a"] = da.random.random((N, 10)) + orig = adata + if diskfmt == "h5ad": + with pytest.raises(ValueError, match="Cannot write dask arrays to hdf5"): + write_elem(g, "", orig) + return + write_elem(g, "", orig) + curr = read_elem(g) with pytest.raises(Exception): assert_equal(curr.obsm["a"], curr.obsm["b"]) diff --git a/anndata/tests/test_hdf5_backing.py b/anndata/tests/test_hdf5_backing.py index 94f2af4b0..a987201da 100644 --- a/anndata/tests/test_hdf5_backing.py +++ b/anndata/tests/test_hdf5_backing.py @@ -177,6 +177,10 @@ def test_backed_raw(tmp_path): assert_equal(final_adata, mem_adata) +# This is thrown on too many lines to filter it with pytest.warns +@pytest.mark.filterwarnings( + "ignore:Support for Awkward Arrays is currently experimental:anndata._warnings.ExperimentalFeatureWarning" +) @pytest.mark.parametrize( "array_type", [ diff --git a/anndata/tests/test_helpers.py b/anndata/tests/test_helpers.py index 52f1d6cb1..9d4a17c74 100644 --- a/anndata/tests/test_helpers.py +++ b/anndata/tests/test_helpers.py @@ -106,8 +106,9 @@ def test_assert_equal(): assert_equal(np.array(list(ascii_letters)), np.array(list(ascii_letters))[::-1]) adata = gen_adata((10, 10)) - adata.raw = adata.copy() - assert_equal(adata, adata.copy(), exact=True) + with pytest.warns(ad.ExperimentalFeatureWarning): + adata.raw = adata.copy() + assert_equal(adata, adata.copy(), exact=True) # TODO: I’m not sure this is good behaviour, I’ve disabled in for now. # assert_equal( # adata, @@ -117,14 +118,16 @@ def test_assert_equal(): # ].copy(), # exact=False, # ) - adata2 = adata.copy() + with pytest.warns(ad.ExperimentalFeatureWarning): + adata2 = adata.copy() to_modify = list(adata2.layers.keys())[0] del adata2.layers[to_modify] with pytest.raises(AssertionError) as missing_layer_error: assert_equal(adata, adata2) assert "layers" in str(missing_layer_error.value) # `to_modify` will be in pytest info - adata2 = adata.copy() + with pytest.warns(ad.ExperimentalFeatureWarning): + adata2 = adata.copy() adata2.layers[to_modify][0, 0] = adata2.layers[to_modify][0, 0] + 1 with pytest.raises(AssertionError) as changed_layer_error: assert_equal(adata, adata2) @@ -150,6 +153,10 @@ def test_assert_equal(): assert_equal(ordered_cat, unordered_cat, exact=True) +# This is thrown on too many lines to filter it with pytest.warns +@pytest.mark.filterwarnings( + "ignore:Support for Awkward Arrays is currently experimental:anndata._warnings.ExperimentalFeatureWarning" +) def test_assert_equal_raw(): base = gen_adata((10, 10)) orig = base.copy() @@ -173,8 +180,9 @@ def test_assert_equal_raw_presence(): # This was causing some testing issues during # https://github.com/scverse/anndata/pull/542 a = gen_adata((10, 20)) - b = a.copy() - a.raw = a.copy() + with pytest.warns(ad.ExperimentalFeatureWarning): + b = a.copy() + a.raw = a.copy() assert b.raw is None with pytest.raises(AssertionError): @@ -187,7 +195,8 @@ def test_assert_equal_raw_presence(): # Should they not be if an exact comparison is made? def test_assert_equal_aligned_mapping(): adata1 = gen_adata((10, 10)) - adata2 = adata1.copy() + with pytest.warns(ad.ExperimentalFeatureWarning): + adata2 = adata1.copy() for attr in ["obsm", "varm", "layers", "obsp", "varp"]: assert_equal(getattr(adata1, attr), getattr(adata2, attr)) diff --git a/anndata/tests/test_inplace_subset.py b/anndata/tests/test_inplace_subset.py index 110d2574a..1e352b0d0 100644 --- a/anndata/tests/test_inplace_subset.py +++ b/anndata/tests/test_inplace_subset.py @@ -4,6 +4,7 @@ import pytest from scipy import sparse +from anndata import ExperimentalFeatureWarning from anndata.tests.helpers import ( as_dense_dask_array, assert_equal, @@ -32,9 +33,10 @@ def test_inplace_subset_var(matrix_type, subset_func): orig = gen_adata((30, 30), X_type=matrix_type) subset_idx = subset_func(orig.var_names) - modified = orig.copy() - from_view = orig[:, subset_idx].copy() - modified._inplace_subset_var(subset_idx) + with pytest.warns(ExperimentalFeatureWarning): + modified = orig.copy() + from_view = orig[:, subset_idx].copy() + modified._inplace_subset_var(subset_idx) assert_equal(asarray(from_view.X), asarray(modified.X), exact=True) assert_equal(from_view.obs, modified.obs, exact=True) @@ -52,9 +54,10 @@ def test_inplace_subset_obs(matrix_type, subset_func): orig = gen_adata((30, 30), X_type=matrix_type) subset_idx = subset_func(orig.obs_names) - modified = orig.copy() - from_view = orig[subset_idx, :].copy() - modified._inplace_subset_obs(subset_idx) + with pytest.warns(ExperimentalFeatureWarning): + modified = orig.copy() + from_view = orig[subset_idx, :].copy() + modified._inplace_subset_obs(subset_idx) assert_equal(asarray(from_view.X), asarray(modified.X), exact=True) assert_equal(from_view.obs, modified.obs, exact=True) @@ -75,8 +78,9 @@ def test_inplace_subset_no_X(subset_func, dim): subset_idx = subset_func(getattr(orig, f"{dim}_names")) - modified = orig.copy() - from_view = subset_dim(orig, **{dim: subset_idx}).copy() - getattr(modified, f"_inplace_subset_{dim}")(subset_idx) + with pytest.warns(ExperimentalFeatureWarning): + modified = orig.copy() + from_view = subset_dim(orig, **{dim: subset_idx}).copy() + getattr(modified, f"_inplace_subset_{dim}")(subset_idx) assert_equal(modified, from_view, exact=True) diff --git a/anndata/tests/test_io_conversion.py b/anndata/tests/test_io_conversion.py index 29a5d27e9..d0dded4e6 100644 --- a/anndata/tests/test_io_conversion.py +++ b/anndata/tests/test_io_conversion.py @@ -3,6 +3,8 @@ """ from __future__ import annotations +from contextlib import nullcontext + import h5py import numpy as np import pytest @@ -38,7 +40,8 @@ def test_sparse_to_dense_disk(tmp_path, mtx_format, to_convert): dense_from_mem_pth = tmp_path / "dense_mem.h5ad" dense_from_disk_pth = tmp_path / "dense_disk.h5ad" mem = gen_adata((50, 50), mtx_format) - mem.raw = mem + with pytest.warns(ad.ExperimentalFeatureWarning): + mem.raw = mem mem.write_h5ad(mem_pth) disk = ad.read_h5ad(mem_pth, backed="r") @@ -54,8 +57,14 @@ def test_sparse_to_dense_disk(tmp_path, mtx_format, to_convert): assert isinstance(f[k], h5py.Dataset) for backed in [None, "r"]: - from_mem = ad.read_h5ad(dense_from_mem_pth, backed=backed) - from_disk = ad.read_h5ad(dense_from_disk_pth, backed=backed) + ctx = ( + pytest.warns(ad.ExperimentalFeatureWarning) + if backed is None + else nullcontext() + ) + with ctx: + from_mem = ad.read_h5ad(dense_from_mem_pth, backed=backed) + from_disk = ad.read_h5ad(dense_from_disk_pth, backed=backed) assert_equal(mem, from_mem) assert_equal(mem, from_disk) assert_equal(disk, from_mem) @@ -65,11 +74,13 @@ def test_sparse_to_dense_disk(tmp_path, mtx_format, to_convert): def test_sparse_to_dense_inplace(tmp_path, spmtx_format): pth = tmp_path / "adata.h5ad" orig = gen_adata((50, 50), spmtx_format) - orig.raw = orig + with pytest.warns(ad.ExperimentalFeatureWarning): + orig.raw = orig orig.write(pth) backed = ad.read_h5ad(pth, backed="r+") backed.write(as_dense=("X", "raw/X")) - new = ad.read_h5ad(pth) + with pytest.warns(ad.ExperimentalFeatureWarning): + new = ad.read_h5ad(pth) assert_equal(orig, new) assert_equal(backed, new) @@ -95,12 +106,16 @@ def test_sparse_to_dense_errors(tmp_path): def test_dense_to_sparse_memory(tmp_path, spmtx_format, to_convert): dense_path = tmp_path / "dense.h5ad" orig = gen_adata((50, 50), np.array) - orig.raw = orig + with pytest.warns(ad.ExperimentalFeatureWarning): + orig.raw = orig orig.write_h5ad(dense_path) assert not isinstance(orig.X, sparse.spmatrix) assert not isinstance(orig.raw.X, sparse.spmatrix) - curr = ad.read_h5ad(dense_path, as_sparse=to_convert, as_sparse_fmt=spmtx_format) + with pytest.warns(ad.ExperimentalFeatureWarning): + curr = ad.read_h5ad( + dense_path, as_sparse=to_convert, as_sparse_fmt=spmtx_format + ) if "X" in to_convert: assert isinstance(curr.X, spmtx_format) diff --git a/anndata/tests/test_io_elementwise.py b/anndata/tests/test_io_elementwise.py index 34a42e7ff..3b33b86f5 100644 --- a/anndata/tests/test_io_elementwise.py +++ b/anndata/tests/test_io_elementwise.py @@ -150,7 +150,8 @@ def test_dask_write_sparse(store, sparse_format): def test_io_spec_raw(store): adata = gen_adata((3, 2)) - adata.raw = adata + with pytest.warns(ad.ExperimentalFeatureWarning): + adata.raw = adata write_elem(store, "adata", adata) @@ -297,5 +298,6 @@ def test_read_zarr_from_group(tmp_path, consolidated): else: read_func = zarr.open - with read_func(pth) as z: - assert_equal(ad.read_zarr(z["table/table"]), adata) + with read_func(pth) as z, pytest.warns(ad.ExperimentalFeatureWarning): + expected = ad.read_zarr(z["table/table"]) + assert_equal(adata, expected) diff --git a/anndata/tests/test_readwrite.py b/anndata/tests/test_readwrite.py index 0c9317526..cd072bfe3 100644 --- a/anndata/tests/test_readwrite.py +++ b/anndata/tests/test_readwrite.py @@ -89,7 +89,8 @@ def rw(backing_h5ad): M, N = 100, 101 orig = gen_adata((M, N)) orig.write(backing_h5ad) - curr = ad.read_h5ad(backing_h5ad) + with pytest.warns(ad.ExperimentalFeatureWarning): + curr = ad.read_h5ad(backing_h5ad) return curr, orig @@ -240,12 +241,14 @@ def test_readwrite_equivalent_h5ad_zarr(tmp_path, typ): M, N = 100, 101 adata = gen_adata((M, N), X_type=typ) - adata.raw = adata + with pytest.warns(ad.ExperimentalFeatureWarning): + adata.raw = adata adata.write_h5ad(h5ad_pth) adata.write_zarr(zarr_pth) - from_h5ad = ad.read_h5ad(h5ad_pth) - from_zarr = ad.read_zarr(zarr_pth) + with pytest.warns(ad.ExperimentalFeatureWarning): + from_h5ad = ad.read_h5ad(h5ad_pth) + from_zarr = ad.read_zarr(zarr_pth) assert_equal(from_h5ad, from_zarr, exact=True) @@ -325,7 +328,9 @@ def check_compressed(key, value): msg = "\n\t".join(not_compressed) raise AssertionError(f"These elements were not compressed correctly:\n\t{msg}") - assert_equal(adata, ad.read_h5ad(pth)) + with pytest.warns(ad.ExperimentalFeatureWarning): + expected = ad.read_h5ad(pth) + assert_equal(adata, expected) def test_zarr_compression(tmp_path): @@ -350,7 +355,9 @@ def check_compressed(key, value): msg = "\n\t".join(not_compressed) raise AssertionError(f"These elements were not compressed correctly:\n\t{msg}") - assert_equal(adata, ad.read_zarr(pth)) + with pytest.warns(ad.ExperimentalFeatureWarning): + expected = ad.read_zarr(pth) + assert_equal(adata, expected) def test_changed_obs_var_names(tmp_path, diskfmt): @@ -359,12 +366,14 @@ def test_changed_obs_var_names(tmp_path, diskfmt): orig = gen_adata((10, 10)) orig.obs_names.name = "obs" orig.var_names.name = "var" - modified = orig.copy() + with pytest.warns(ad.ExperimentalFeatureWarning): + modified = orig.copy() modified.obs_names.name = "cells" modified.var_names.name = "genes" getattr(orig, f"write_{diskfmt}")(filepth) - read = getattr(ad, f"read_{diskfmt}")(filepth) + with pytest.warns(ad.ExperimentalFeatureWarning): + read = getattr(ad, f"read_{diskfmt}")(filepth) assert_equal(orig, read, exact=True) assert orig.var.index.name == "var" @@ -690,7 +699,8 @@ def test_zarr_chunk_X(tmp_path): z = zarr.open(str(zarr_pth)) # As of v2.3.2 zarr won’t take a Path assert z["X"].chunks == (10, 10) - from_zarr = ad.read_zarr(zarr_pth) + with pytest.warns(ad.ExperimentalFeatureWarning): + from_zarr = ad.read_zarr(zarr_pth) assert_equal(from_zarr, adata) @@ -782,11 +792,13 @@ def test_adata_in_uns(tmp_path, diskfmt): "b": gen_adata((12, 8)), } another_one = gen_adata((2, 5)) - another_one.raw = gen_adata((2, 7)) + with pytest.warns(ad.ExperimentalFeatureWarning): + another_one.raw = gen_adata((2, 7)) orig.uns["adatas"]["b"].uns["another_one"] = another_one write(orig, pth) - curr = read(pth) + with pytest.warns(ad.ExperimentalFeatureWarning): + curr = read(pth) assert_equal(orig, curr) diff --git a/anndata/tests/test_transpose.py b/anndata/tests/test_transpose.py index 720733496..498dc8065 100644 --- a/anndata/tests/test_transpose.py +++ b/anndata/tests/test_transpose.py @@ -1,5 +1,7 @@ from __future__ import annotations +import warnings + import numpy as np import pytest from scipy import sparse @@ -23,8 +25,10 @@ def test_transpose_orig(): def _add_raw(adata, *, var_subset=slice(None)): - new = adata[:, var_subset].copy() - new.raw = adata + with warnings.catch_warnings(): + warnings.simplefilter("ignore", ad.ExperimentalFeatureWarning) + new = adata[:, var_subset].copy() + new.raw = adata return new diff --git a/anndata/tests/test_views.py b/anndata/tests/test_views.py index fd0ff8902..6784ba60a 100644 --- a/anndata/tests/test_views.py +++ b/anndata/tests/test_views.py @@ -502,6 +502,10 @@ def test_layers_view(): assert view_hash != joblib.hash(view_adata) +# This is thrown on too many lines to filter it with pytest.warns +@pytest.mark.filterwarnings( + "ignore:Support for Awkward Arrays is currently experimental:anndata._warnings.ExperimentalFeatureWarning" +) # TODO: This can be flaky. Make that stop def test_view_of_view(matrix_type, subset_func, subset_func2): adata = gen_adata((30, 15), X_type=matrix_type) diff --git a/anndata/tests/test_x.py b/anndata/tests/test_x.py index 5f381c8c1..d300df05f 100644 --- a/anndata/tests/test_x.py +++ b/anndata/tests/test_x.py @@ -7,7 +7,7 @@ from scipy import sparse import anndata as ad -from anndata import AnnData +from anndata import AnnData, ExperimentalFeatureWarning from anndata.tests.helpers import assert_equal, gen_adata from anndata.utils import asarray @@ -52,7 +52,8 @@ def test_del_set_equiv_X(): """Tests that `del adata.X` is equivalent to `adata.X = None`""" # test setter and deleter orig = gen_adata((10, 10)) - copy = orig.copy() + with pytest.warns(ad.ExperimentalFeatureWarning): + copy = orig.copy() del orig.X copy.X = None @@ -133,6 +134,8 @@ def test_io_missing_X(tmp_path, diskfmt): del adata.X write(adata, file_pth) - from_disk = read(file_pth) + + with pytest.warns(ExperimentalFeatureWarning): + from_disk = read(file_pth) assert_equal(from_disk, adata) From 8ae08d38c19804f91c97f5a0c8ee77091cc1b5c5 Mon Sep 17 00:00:00 2001 From: "Philipp A." Date: Fri, 13 Oct 2023 09:51:24 +0200 Subject: [PATCH 26/43] Filter deprecations from doctests --- anndata/_core/aligned_mapping.py | 2 +- anndata/_core/anndata.py | 26 +++++++------------- anndata/utils.py | 41 +++++++++++++++++++++++--------- conftest.py | 12 +++++++++- 4 files changed, 50 insertions(+), 31 deletions(-) diff --git a/anndata/_core/aligned_mapping.py b/anndata/_core/aligned_mapping.py index b3db938d3..4fdcb0b29 100644 --- a/anndata/_core/aligned_mapping.py +++ b/anndata/_core/aligned_mapping.py @@ -124,7 +124,7 @@ def _view(self, parent: AnnData, subset_idx: I): """Returns a subset copy-on-write view of the object.""" return self._view_class(self, parent, subset_idx) - @deprecated("dict(obj)") + @deprecated("dict(obj)", FutureWarning) def as_dict(self) -> dict: return dict(self) diff --git a/anndata/_core/anndata.py b/anndata/_core/anndata.py index 75006e3e2..fe5e0ce23 100644 --- a/anndata/_core/anndata.py +++ b/anndata/_core/anndata.py @@ -40,7 +40,7 @@ _move_adj_mtx, ) from ..logging import anndata_logger as logger -from ..utils import convert_to_dict, dim_len, ensure_df_homogeneous +from ..utils import convert_to_dict, deprecated, dim_len, ensure_df_homogeneous from .access import ElementRef from .aligned_mapping import ( AxisArrays, @@ -1608,6 +1608,12 @@ def copy(self, filename: PathLike | None = None) -> AnnData: write_h5ad(filename, self) return read_h5ad(filename, backed=mode) + @deprecated( + "anndata.concat", + FutureWarning, + "See the tutorial for concat at: " + "https://anndata.readthedocs.io/en/latest/concatenation.html", + ) def concatenate( self, *adatas: AnnData, @@ -1679,7 +1685,7 @@ def concatenate( Examples -------- - Joining on intersection of variables. First, prepare example data: + Joining on intersection of variables. >>> adata1 = AnnData( ... np.array([[1, 2, 3], [4, 5, 6]]), @@ -1696,14 +1702,6 @@ def concatenate( ... dict(obs_names=['s1', 's2'], anno2=['d3', 'd4']), ... dict(var_names=['d', 'c', 'b'], annoA=[0, 2, 3], annoB=[0, 1, 2]), ... ) - - They can now be concatenated. Since this method is deprecated, it will raise a warning: - - >>> warnings.filterwarnings( - ... 'ignore', - ... r'The AnnData\\.concatenate method is deprecated', - ... FutureWarning, - ... ) >>> adata = adata1.concatenate(adata2, adata3) >>> adata AnnData object with n_obs × n_vars = 6 × 2 @@ -1839,14 +1837,6 @@ def concatenate( """ from .merge import concat, merge_dataframes, merge_outer, merge_same - warnings.warn( - "The AnnData.concatenate method is deprecated in favour of the " - "anndata.concat function. Please use anndata.concat instead.\n\n" - "See the tutorial for concat at: " - "https://anndata.readthedocs.io/en/latest/concatenation.html", - FutureWarning, - ) - if self.isbacked: raise ValueError("Currently, concatenate only works in memory mode.") diff --git a/anndata/utils.py b/anndata/utils.py index 3eeb08b56..c28680028 100644 --- a/anndata/utils.py +++ b/anndata/utils.py @@ -21,6 +21,24 @@ logger = get_logger(__name__) +def import_name(name: str) -> Any: + from importlib import import_module + + parts = name.split(".") + obj = import_module(parts[0]) + for i, name in enumerate(parts[1:]): + try: + obj = import_module(f"{obj.__name__}.{name}") + except ModuleNotFoundError: + break + for name in parts[i + 1 :]: + try: + obj = getattr(obj, name) + except AttributeError: + raise RuntimeError(f"{parts[:i]}, {parts[i+1:]}, {obj} {name}") + return obj + + @singledispatch def asarray(x): """Convert x to a numpy array""" @@ -321,7 +339,9 @@ def warn_once(msg: str, category: type[Warning], stacklevel: int = 1): warnings.filterwarnings("ignore", category=category, message=re.escape(msg)) -def deprecated(new_name: str): +def deprecated( + new_name: str, category: type[Warning] = DeprecationWarning, add_msg: str = "" +): """\ This is a decorator which can be used to mark functions as deprecated. It will result in a warning being emitted @@ -329,20 +349,19 @@ def deprecated(new_name: str): """ def decorator(func): + msg = ( + f"Use {new_name} instead of {func.__name__}, " + f"{func.__name__} will be removed in the future." + ) + if add_msg: + msg += f" {add_msg}" + @wraps(func) def new_func(*args, **kwargs): - # turn off filter - warnings.simplefilter("always", DeprecationWarning) - warnings.warn( - f"Use {new_name} instead of {func.__name__}, " - f"{func.__name__} will be removed in the future.", - category=DeprecationWarning, - stacklevel=2, - ) - warnings.simplefilter("default", DeprecationWarning) # reset filter + warnings.warn(msg, category=category, stacklevel=2) return func(*args, **kwargs) - setattr(new_func, "__deprecated", True) + setattr(new_func, "__deprecated", (category, msg)) return new_func return decorator diff --git a/conftest.py b/conftest.py index 1825ef24c..643d184a6 100644 --- a/conftest.py +++ b/conftest.py @@ -4,11 +4,14 @@ # TODO: Fix that, e.g. with the `pytest -p anndata.testing._pytest` pattern. from __future__ import annotations +import re +import warnings from typing import TYPE_CHECKING import pytest from anndata.compat import chdir +from anndata.utils import import_name if TYPE_CHECKING: from pathlib import Path @@ -17,9 +20,16 @@ @pytest.fixture -def doctest_env(cache: pytest.Cache, tmp_path: Path) -> None: +def doctest_env( + request: pytest.FixtureRequest, cache: pytest.Cache, tmp_path: Path +) -> None: from scanpy import settings + func = import_name(request.node.name) + if warning_detail := getattr(func, "__deprecated", None): + cat, msg = warning_detail # type: tuple[type[Warning], str] + warnings.filterwarnings("ignore", category=cat, message=re.escape(msg)) + old_dd, settings.datasetdir = settings.datasetdir, cache.mkdir("scanpy-data") with chdir(tmp_path): yield From 18acba46ee7bb9e38ca4d6e91d2d7ddf1dde92e4 Mon Sep 17 00:00:00 2001 From: "Philipp A." Date: Fri, 13 Oct 2023 10:59:48 +0200 Subject: [PATCH 27/43] fix docs --- docs/release-notes/0.6.0.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/release-notes/0.6.0.md b/docs/release-notes/0.6.0.md index b2cc1b506..ab4316f64 100644 --- a/docs/release-notes/0.6.0.md +++ b/docs/release-notes/0.6.0.md @@ -26,7 +26,7 @@ ### 0.6.0 {small}`1 May, 2018` - compatibility with Seurat converter -- tremendous speedup for {func}`~anndata.AnnData.concatenate` +- tremendous speedup for {meth}`~anndata.AnnData.concatenate` - bug fix for deep copy of unstructured annotation after slicing - bug fix for reading HDF5 stored single-category annotations - `'outer join'` concatenation: adds zeros for concatenation of sparse data and nans for dense data From 7a672e0c79c3ea4dd612642fc209143bcd33c699 Mon Sep 17 00:00:00 2001 From: "Philipp A." Date: Fri, 13 Oct 2023 11:07:09 +0200 Subject: [PATCH 28/43] Fix regex --- anndata/tests/test_concatenate.py | 2 +- anndata/utils.py | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/anndata/tests/test_concatenate.py b/anndata/tests/test_concatenate.py index 2f164fec8..504ad00e4 100644 --- a/anndata/tests/test_concatenate.py +++ b/anndata/tests/test_concatenate.py @@ -1115,7 +1115,7 @@ def test_concatenate_uns(unss, merge_strategy, result, value_gen): print(merge_strategy, "\n", unss, "\n", result) result, *unss = permute_nested_values([result] + unss, value_gen) adatas = [uns_ad(uns) for uns in unss] - with pytest.warns(FutureWarning, match=r"concatenate method is deprecated"): + with pytest.warns(FutureWarning, match=r"concatenate is deprecated"): merged = AnnData.concatenate(*adatas, uns_merge=merge_strategy).uns assert_equal(merged, result, elem_name="uns") diff --git a/anndata/utils.py b/anndata/utils.py index c28680028..03d1379e9 100644 --- a/anndata/utils.py +++ b/anndata/utils.py @@ -349,9 +349,10 @@ def deprecated( """ def decorator(func): + name = func.__qualname__ msg = ( - f"Use {new_name} instead of {func.__name__}, " - f"{func.__name__} will be removed in the future." + f"Use {new_name} instead of {name}, " + f"{name} is deprecated and will be removed in the future." ) if add_msg: msg += f" {add_msg}" From 4e973653033bed226cce5732f850aa3de8d2e20e Mon Sep 17 00:00:00 2001 From: "Philipp A." Date: Fri, 13 Oct 2023 12:30:36 +0200 Subject: [PATCH 29/43] make hiding configurable --- anndata/_core/anndata.py | 1 + anndata/utils.py | 14 +++++++++----- conftest.py | 2 +- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/anndata/_core/anndata.py b/anndata/_core/anndata.py index fe5e0ce23..dc4188517 100644 --- a/anndata/_core/anndata.py +++ b/anndata/_core/anndata.py @@ -1613,6 +1613,7 @@ def copy(self, filename: PathLike | None = None) -> AnnData: FutureWarning, "See the tutorial for concat at: " "https://anndata.readthedocs.io/en/latest/concatenation.html", + hide=False, ) def concatenate( self, diff --git a/anndata/utils.py b/anndata/utils.py index 03d1379e9..d0c3bf4b9 100644 --- a/anndata/utils.py +++ b/anndata/utils.py @@ -340,7 +340,10 @@ def warn_once(msg: str, category: type[Warning], stacklevel: int = 1): def deprecated( - new_name: str, category: type[Warning] = DeprecationWarning, add_msg: str = "" + new_name: str, + category: type[Warning] = DeprecationWarning, + add_msg: str = "", + hide: bool = True, ): """\ This is a decorator which can be used to mark functions @@ -362,7 +365,7 @@ def new_func(*args, **kwargs): warnings.warn(msg, category=category, stacklevel=2) return func(*args, **kwargs) - setattr(new_func, "__deprecated", (category, msg)) + setattr(new_func, "__deprecated", (category, msg, hide)) return new_func return decorator @@ -375,13 +378,14 @@ class DeprecationMixinMeta(type): """ def __dir__(cls): - def is_deprecated(attr): + def is_hidden(attr) -> bool: if isinstance(attr, property): attr = attr.fget - return getattr(attr, "__deprecated", False) + _, _, hide = getattr(attr, "__deprecated", (None, None, False)) + return hide return [ item for item in type.__dir__(cls) - if not is_deprecated(getattr(cls, item, None)) + if not is_hidden(getattr(cls, item, None)) ] diff --git a/conftest.py b/conftest.py index 643d184a6..a878034e1 100644 --- a/conftest.py +++ b/conftest.py @@ -27,7 +27,7 @@ def doctest_env( func = import_name(request.node.name) if warning_detail := getattr(func, "__deprecated", None): - cat, msg = warning_detail # type: tuple[type[Warning], str] + cat, msg, _ = warning_detail # type: tuple[type[Warning], str, bool] warnings.filterwarnings("ignore", category=cat, message=re.escape(msg)) old_dd, settings.datasetdir = settings.datasetdir, cache.mkdir("scanpy-data") From 1e46cc0c0fb4447ca0ea84d8a3002373d475ea65 Mon Sep 17 00:00:00 2001 From: "Philipp A." Date: Fri, 13 Oct 2023 12:42:41 +0200 Subject: [PATCH 30/43] Fix doctest collection --- conftest.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/conftest.py b/conftest.py index a878034e1..588a82054 100644 --- a/conftest.py +++ b/conftest.py @@ -25,10 +25,13 @@ def doctest_env( ) -> None: from scanpy import settings - func = import_name(request.node.name) - if warning_detail := getattr(func, "__deprecated", None): - cat, msg, _ = warning_detail # type: tuple[type[Warning], str, bool] - warnings.filterwarnings("ignore", category=cat, message=re.escape(msg)) + # request.node.parent is either a DoctestModule or a DoctestTextFile. + # Only DoctestModule has a .obj attribute (the imported module). + if request.node.parent.obj: + func = import_name(request.node.name) + if warning_detail := getattr(func, "__deprecated", None): + cat, msg, _ = warning_detail # type: tuple[type[Warning], str, bool] + warnings.filterwarnings("ignore", category=cat, message=re.escape(msg)) old_dd, settings.datasetdir = settings.datasetdir, cache.mkdir("scanpy-data") with chdir(tmp_path): From da81e661c3d652b323ad772bd11a3681e552ebbe Mon Sep 17 00:00:00 2001 From: "Philipp A." Date: Fri, 13 Oct 2023 12:58:58 +0200 Subject: [PATCH 31/43] Fix remaining catch regex --- anndata/tests/test_concatenate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/anndata/tests/test_concatenate.py b/anndata/tests/test_concatenate.py index 504ad00e4..ce63473ba 100644 --- a/anndata/tests/test_concatenate.py +++ b/anndata/tests/test_concatenate.py @@ -32,7 +32,7 @@ from anndata.utils import asarray mark_legacy_concatenate = pytest.mark.filterwarnings( - r"ignore:The AnnData\.concatenate method is deprecated:FutureWarning" + r"ignore:AnnData\.concatenate is deprecated:FutureWarning" ) From 0585a97a8597aa11ef8adfbea39c36ae858a8b0f Mon Sep 17 00:00:00 2001 From: "Philipp A." Date: Fri, 13 Oct 2023 13:06:30 +0200 Subject: [PATCH 32/43] oops --- anndata/tests/test_concatenate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/anndata/tests/test_concatenate.py b/anndata/tests/test_concatenate.py index ce63473ba..5e683f9e4 100644 --- a/anndata/tests/test_concatenate.py +++ b/anndata/tests/test_concatenate.py @@ -32,7 +32,7 @@ from anndata.utils import asarray mark_legacy_concatenate = pytest.mark.filterwarnings( - r"ignore:AnnData\.concatenate is deprecated:FutureWarning" + r"ignore:.*AnnData\.concatenate is deprecated:FutureWarning" ) From eec3d5bd014589788f1315c1c2a6860aa695c2a7 Mon Sep 17 00:00:00 2001 From: Phil Schaf Date: Thu, 19 Oct 2023 12:14:43 +0200 Subject: [PATCH 33/43] Ignore globally --- .azure-pipelines.yml | 19 ++++++++++-- anndata/_core/anndata.py | 43 ++++++++++----------------- anndata/_core/merge.py | 31 +++++++++----------- anndata/_io/h5ad.py | 7 ++--- anndata/_io/specs/methods.py | 7 ++--- anndata/tests/helpers.py | 30 ++++++++----------- anndata/tests/test_awkward.py | 39 ++++++------------------ anndata/tests/test_base.py | 9 +++--- anndata/tests/test_concatenate.py | 44 ++++++++-------------------- anndata/tests/test_dask.py | 1 - anndata/tests/test_hdf5_backing.py | 4 --- anndata/tests/test_helpers.py | 23 +++++---------- anndata/tests/test_inplace_subset.py | 22 ++++++-------- anndata/tests/test_io_conversion.py | 29 +++++------------- anndata/tests/test_io_elementwise.py | 5 ++-- anndata/tests/test_io_warnings.py | 1 - anndata/tests/test_readwrite.py | 32 +++++++------------- anndata/tests/test_transpose.py | 8 ++--- anndata/tests/test_views.py | 4 --- anndata/tests/test_x.py | 8 ++--- anndata/utils.py | 8 ++--- pyproject.toml | 3 ++ 22 files changed, 135 insertions(+), 242 deletions(-) diff --git a/.azure-pipelines.yml b/.azure-pipelines.yml index 7edf6e430..b2cdca39e 100644 --- a/.azure-pipelines.yml +++ b/.azure-pipelines.yml @@ -63,12 +63,25 @@ jobs: condition: and(eq(variables['RUN_COVERAGE'], 'yes'), eq(variables['PRERELEASE_DEPENDENCIES'], 'no')) # TODO: fix all the exceptions here - # replacing “:Setting element”, “:Trying to modify attribute” and “:Transforming to str index” - # with “::anndata._warnings.ImplicitModificationWarning” breaks, - # because we’re neither using editable installs nor a src/ layout + # TODO: centralize using a custom pytes CLI arg, because: + # 1. This allows to easily locally run with the same settings as this job + # 2. Otherwise we can’t filter anndata warning classes directly, + # because we’re neither using editable installs nor a src/ layout. + # This is better because it’s a more specific filter and we don’t need one line per message. + # Setting the filters in code instead of per `-W` fixes that. + # This affects: + # - `anndata._warnings.ExperimentalFeatureWarning` + # - “Support for Awkward Arrays” + # - “Outer joins on awkward.Arrays” + # - `anndata._warnings.ImplicitModificationWarning` + # - “Trying to modify attribute” + # - “Transforming to str index” + # 3. The 'ignore' lines need to be kept in sync with `filterwarnings` config in pyproject.toml - script: > pytest -W error + -W 'ignore:Support for Awkward Arrays is currently experimental' + -W 'ignore:Outer joins on awkward.Arrays' -W 'default:Setting element:UserWarning' -W 'default:Trying to modify attribute:UserWarning' -W 'default:Transforming to str index:UserWarning' diff --git a/anndata/_core/anndata.py b/anndata/_core/anndata.py index dc4188517..3fc67dd92 100644 --- a/anndata/_core/anndata.py +++ b/anndata/_core/anndata.py @@ -30,7 +30,6 @@ from anndata._warnings import ImplicitModificationWarning from .. import utils -from .._warnings import ExperimentalFeatureWarning from ..compat import ( CupyArray, CupySparseMatrix, @@ -896,9 +895,7 @@ def _prep_dim_index(self, value, attr: str) -> pd.Index: def _set_dim_index(self, value: pd.Index, attr: str): # Assumes _prep_dim_index has been run if self.is_view: - with warnings.catch_warnings(): - warnings.filterwarnings("ignore", category=ExperimentalFeatureWarning) - self._init_as_actual(self.copy()) + self._init_as_actual(self.copy()) getattr(self, attr).index = value for v in getattr(self, f"{attr}m").values(): if isinstance(v, pd.DataFrame): @@ -1305,9 +1302,7 @@ def _inplace_subset_var(self, index: Index1D): """ adata_subset = self[:, index].copy() - with warnings.catch_warnings(): - warnings.filterwarnings("ignore", category=ExperimentalFeatureWarning) - self._init_as_actual(adata_subset) + self._init_as_actual(adata_subset) def _inplace_subset_obs(self, index: Index1D): """\ @@ -1317,9 +1312,7 @@ def _inplace_subset_obs(self, index: Index1D): """ adata_subset = self[index].copy() - with warnings.catch_warnings(): - warnings.filterwarnings("ignore", category=ExperimentalFeatureWarning) - self._init_as_actual(adata_subset) + self._init_as_actual(adata_subset) # TODO: Update, possibly remove def __setitem__( @@ -1357,20 +1350,18 @@ def transpose(self) -> AnnData: "which is currently not implemented. Call `.copy()` before transposing." ) - with warnings.catch_warnings(): - warnings.filterwarnings("ignore", category=ExperimentalFeatureWarning) - return AnnData( - X=_safe_transpose(X) if X is not None else None, - layers={k: _safe_transpose(v) for k, v in self.layers.items()}, - obs=self.var, - var=self.obs, - uns=self._uns, - obsm=self._varm, - varm=self._obsm, - obsp=self._varp, - varp=self._obsp, - filename=self.filename, - ) + return AnnData( + X=_safe_transpose(X) if X is not None else None, + layers={k: _safe_transpose(v) for k, v in self.layers.items()}, + obs=self.var, + var=self.obs, + uns=self._uns, + obsm=self._varm, + varm=self._obsm, + obsp=self._varp, + varp=self._obsp, + filename=self.filename, + ) T = property(transpose) @@ -1578,9 +1569,7 @@ def to_memory(self, copy=False) -> AnnData: if self.isbacked: self.file.close() - with warnings.catch_warnings(): - warnings.filterwarnings("ignore", category=ExperimentalFeatureWarning) - return AnnData(**new) + return AnnData(**new) def copy(self, filename: PathLike | None = None) -> AnnData: """Full copy, optionally on disk.""" diff --git a/anndata/_core/merge.py b/anndata/_core/merge.py index 7757acc0b..66a8c3459 100644 --- a/anndata/_core/merge.py +++ b/anndata/_core/merge.py @@ -4,7 +4,6 @@ from __future__ import annotations import typing -import warnings from collections import OrderedDict from collections.abc import ( Callable, @@ -1340,19 +1339,17 @@ def concat( "not concatenating `.raw` attributes.", UserWarning, ) - with warnings.catch_warnings(): - warnings.filterwarnings("ignore", category=ExperimentalFeatureWarning) - return AnnData( - **{ - "X": X, - "layers": layers, - dim: concat_annot, - alt_dim: alt_annot, - f"{dim}m": concat_mapping, - f"{alt_dim}m": alt_mapping, - f"{dim}p": concat_pairwise, - f"{alt_dim}p": alt_pairwise, - "uns": uns, - "raw": raw, - } - ) + return AnnData( + **{ + "X": X, + "layers": layers, + dim: concat_annot, + alt_dim: alt_annot, + f"{dim}m": concat_mapping, + f"{alt_dim}m": alt_mapping, + f"{dim}p": concat_pairwise, + f"{alt_dim}p": alt_pairwise, + "uns": uns, + "raw": raw, + } + ) diff --git a/anndata/_io/h5ad.py b/anndata/_io/h5ad.py index 17c5e9989..337f13f92 100644 --- a/anndata/_io/h5ad.py +++ b/anndata/_io/h5ad.py @@ -1,7 +1,6 @@ from __future__ import annotations import re -import warnings from functools import partial from pathlib import Path from types import MappingProxyType @@ -18,7 +17,7 @@ import pandas as pd from scipy import sparse -from anndata._warnings import ExperimentalFeatureWarning, OldFormatWarning +from anndata._warnings import OldFormatWarning from .._core.anndata import AnnData from .._core.file_backing import AnnDataFileManager, filename @@ -153,9 +152,7 @@ def read_h5ad_backed(filename: str | Path, mode: Literal["r", "r+"]) -> AnnData: d["raw"] = _read_raw(f, attrs={"var", "varm"}) - with warnings.catch_warnings(): - warnings.filterwarnings("ignore", category=ExperimentalFeatureWarning) - adata = AnnData(**d) + adata = AnnData(**d) # Backwards compat to <0.7 if isinstance(f["obs"], h5py.Dataset): diff --git a/anndata/_io/specs/methods.py b/anndata/_io/specs/methods.py index a934656f4..adbb1b4bc 100644 --- a/anndata/_io/specs/methods.py +++ b/anndata/_io/specs/methods.py @@ -1,6 +1,5 @@ from __future__ import annotations -import warnings from collections.abc import Mapping from functools import partial from itertools import product @@ -20,7 +19,7 @@ from anndata._core.merge import intersect_keys from anndata._core.sparse_dataset import CSCDataset, CSRDataset, sparse_dataset from anndata._io.utils import H5PY_V3, check_key -from anndata._warnings import ExperimentalFeatureWarning, OldFormatWarning +from anndata._warnings import OldFormatWarning from anndata.compat import ( AwkArray, CupyArray, @@ -283,9 +282,7 @@ def read_anndata(elem, _reader): if k in elem: d[k] = _reader.read_elem(elem[k]) - with warnings.catch_warnings(): - warnings.filterwarnings("ignore", category=ExperimentalFeatureWarning) - return AnnData(**d) + return AnnData(**d) @_REGISTRY.register_write(H5Group, Raw, IOSpec("raw", "0.1.0")) diff --git a/anndata/tests/helpers.py b/anndata/tests/helpers.py index 342d365e7..641bdc791 100644 --- a/anndata/tests/helpers.py +++ b/anndata/tests/helpers.py @@ -15,7 +15,7 @@ from pandas.api.types import is_numeric_dtype from scipy import sparse -from anndata import AnnData, ExperimentalFeatureWarning, Raw +from anndata import AnnData, Raw from anndata._core.aligned_mapping import AlignedMapping from anndata._core.sparse_dataset import BaseCompressedSparseDataset from anndata._core.views import ArrayView @@ -256,23 +256,17 @@ def gen_adata( awkward_ragged=gen_awkward((12, None, None)), # U_recarray=gen_vstr_recarray(N, 5, "U4") ) - with warnings.catch_warnings(): - warnings.filterwarnings( - "ignore", - "Support for Awkward Arrays is currently experimental", - ExperimentalFeatureWarning, - ) - adata = AnnData( - X=X, - obs=obs, - var=var, - obsm=obsm, - varm=varm, - layers=layers, - obsp=obsp, - varp=varp, - uns=uns, - ) + adata = AnnData( + X=X, + obs=obs, + var=var, + obsm=obsm, + varm=varm, + layers=layers, + obsp=obsp, + varp=varp, + uns=uns, + ) return adata diff --git a/anndata/tests/test_awkward.py b/anndata/tests/test_awkward.py index 6f4c9548e..9e780c8a8 100644 --- a/anndata/tests/test_awkward.py +++ b/anndata/tests/test_awkward.py @@ -11,7 +11,6 @@ import anndata from anndata import ( AnnData, - ExperimentalFeatureWarning, ImplicitModificationWarning, read_h5ad, ) @@ -102,11 +101,8 @@ def test_set_awkward(field, value, valid): """ adata = gen_adata((10, 20), varm_types=(), obsm_types=(), layers_types=()) - ctx = pytest.warns(ExperimentalFeatureWarning) if field != "uns" else nullcontext() - def _assign(): - with ctx: - getattr(adata, field)["test"] = value + getattr(adata, field)["test"] = value if not valid: with pytest.raises(ValueError): @@ -119,10 +115,8 @@ def _assign(): def test_copy(key): """Check that modifying a copy does not modify the original""" adata = gen_adata((3, 3), varm_types=(), obsm_types=(), layers_types=()) - ctx = pytest.warns(ExperimentalFeatureWarning) if key != "uns" else nullcontext() - with ctx: - getattr(adata, key)["awk"] = ak.Array([{"a": [1], "b": [2], "c": [3]}] * 3) - adata_copy = adata.copy() + getattr(adata, key)["awk"] = ak.Array([{"a": [1], "b": [2], "c": [3]}] * 3) + adata_copy = adata.copy() getattr(adata_copy, key)["awk"]["c"] = np.full((3, 1), 4) getattr(adata_copy, key)["awk"]["d"] = np.full((3, 1), 5) @@ -140,8 +134,7 @@ def test_copy(key): def test_view(key): """Check that modifying a view does not modify the original""" adata = gen_adata((3, 3), varm_types=(), obsm_types=(), layers_types=()) - with pytest.warns(ExperimentalFeatureWarning): - getattr(adata, key)["awk"] = ak.Array([{"a": [1], "b": [2], "c": [3]}] * 3) + getattr(adata, key)["awk"] = ak.Array([{"a": [1], "b": [2], "c": [3]}] * 3) adata_view = adata[:2, :2] with pytest.warns(ImplicitModificationWarning, match="initializing view as actual"): @@ -172,10 +165,9 @@ def reversed(self): ak.behavior[BEHAVIOUR_ID] = ReversibleArray adata = gen_adata((3, 3), varm_types=(), obsm_types=(), layers_types=()) - with pytest.warns(ExperimentalFeatureWarning): - adata.obsm["awk_string"] = ak.with_parameter( - ak.Array(["AAA", "BBB", "CCC"]), "__list__", BEHAVIOUR_ID - ) + adata.obsm["awk_string"] = ak.with_parameter( + ak.Array(["AAA", "BBB", "CCC"]), "__list__", BEHAVIOUR_ID + ) adata_view = adata[:2] with pytest.raises(NotImplementedError): @@ -384,13 +376,7 @@ def test_concat_mixed_types(key, arrays, expected, join): tmp_adata.obs_names if key == "obsm" else tmp_adata.var_names, inplace=True, ) - ctx = ( - pytest.warns(ExperimentalFeatureWarning) - if isinstance(a, ak.Array) - else nullcontext() - ) - with ctx: - getattr(tmp_adata, key)["test"] = a + getattr(tmp_adata, key)["test"] = a to_concat.append(tmp_adata) @@ -406,13 +392,6 @@ def test_concat_mixed_types(key, arrays, expected, join): with pytest.raises(expected), ctx: anndata.concat(to_concat, axis=axis, join=join) else: - print(to_concat) - ctx = ( - pytest.warns(ExperimentalFeatureWarning) - if join == "outer" - else nullcontext() - ) - with ctx: - result_adata = anndata.concat(to_concat, axis=axis, join=join) + result_adata = anndata.concat(to_concat, axis=axis, join=join) result = getattr(result_adata, key).get("test", None) assert_equal(expected, result, exact=True) diff --git a/anndata/tests/test_base.py b/anndata/tests/test_base.py index b0959f3a6..14127271f 100644 --- a/anndata/tests/test_base.py +++ b/anndata/tests/test_base.py @@ -11,7 +11,7 @@ from scipy import sparse as sp from scipy.sparse import csr_matrix, issparse -from anndata import AnnData, ExperimentalFeatureWarning +from anndata import AnnData from anndata.tests.helpers import assert_equal, gen_adata # some test objects that we use below @@ -224,10 +224,9 @@ def test_setting_dim_index(dim): mapping_attr = f"{dim}m" orig = gen_adata((5, 5)) - with pytest.warns(ExperimentalFeatureWarning): - orig.raw = orig - curr = orig.copy() - view = orig[:, :] + orig.raw = orig + curr = orig.copy() + view = orig[:, :] new_idx = pd.Index(list("abcde"), name="letters") setattr(curr, index_attr, new_idx) diff --git a/anndata/tests/test_concatenate.py b/anndata/tests/test_concatenate.py index 5e683f9e4..17c11ef70 100644 --- a/anndata/tests/test_concatenate.py +++ b/anndata/tests/test_concatenate.py @@ -15,7 +15,7 @@ from numpy import ma from scipy import sparse -from anndata import AnnData, ExperimentalFeatureWarning, Raw, concat +from anndata import AnnData, Raw, concat from anndata._core import merge from anndata._core.index import _subset from anndata.compat import AwkArray, DaskArray @@ -716,9 +716,8 @@ def test_concatenate_awkward(join_type): ] ) - with pytest.warns(ExperimentalFeatureWarning): - adata_a = AnnData(np.zeros((2, 0), dtype=float), obsm={"awk": a}) - adata_b = AnnData(np.zeros((3, 0), dtype=float), obsm={"awk": b}) + adata_a = AnnData(np.zeros((2, 0), dtype=float), obsm={"awk": a}) + adata_b = AnnData(np.zeros((3, 0), dtype=float), obsm={"awk": b}) if join_type == "inner": expected = ak.Array( @@ -761,13 +760,7 @@ def test_concatenate_awkward(join_type): ] ) - ctx = ( - pytest.warns(ExperimentalFeatureWarning) - if join_type == "outer" - else nullcontext() - ) - with ctx: - result = concat([adata_a, adata_b], join=join_type).obsm["awk"] + result = concat([adata_a, adata_b], join=join_type).obsm["awk"] assert_equal(expected, result) @@ -787,12 +780,11 @@ def test_awkward_does_not_mix(join_type, other): [[{"a": 1, "b": "foo"}], [{"a": 2, "b": "bar"}, {"a": 3, "b": "baz"}]] ) - with pytest.warns(ExperimentalFeatureWarning): - adata_a = AnnData( - np.zeros((2, 3), dtype=float), - obs=pd.DataFrame(index=list("ab")), - obsm={"val": awk}, - ) + adata_a = AnnData( + np.zeros((2, 3), dtype=float), + obs=pd.DataFrame(index=list("ab")), + obsm={"val": awk}, + ) adata_b = AnnData( np.zeros((3, 3), dtype=float), obs=pd.DataFrame(index=list("cde")), @@ -1341,23 +1333,15 @@ def test_concat_size_0_dim(axis, join_type, merge_strategy, shape): expected_size = expected_shape(a, b, axis=axis, join=join_type) - ctx_awk = ( - pytest.warns( - ExperimentalFeatureWarning, - match=r"Outer joins on awkward.Arrays will have different return values in the future.", - ) - if join_type == "outer" - else nullcontext() - ) ctx_concat_empty = ( pytest.warns( FutureWarning, match=r"The behavior of DataFrame concatenation with empty or all-NA entries is deprecated", ) - if join_type == "inner" and shape[axis] == 0 + if shape[axis] == 0 else nullcontext() ) - with ctx_awk, ctx_concat_empty: + with ctx_concat_empty: result = concat( {"a": a, "b": b}, axis=axis, @@ -1382,8 +1366,7 @@ def test_concat_size_0_dim(axis, join_type, merge_strategy, shape): check_filled_like(elem[altaxis_idx], elem_name=f"layers/{k}") if shape[axis] > 0: - with pytest.warns(ExperimentalFeatureWarning): - b_result = result[axis_idx].copy() + b_result = result[axis_idx].copy() mapping_elem = f"{dim}m" setattr(b_result, f"{dim}_names", getattr(b, f"{dim}_names")) for k, result_elem in getattr(b_result, mapping_elem).items(): @@ -1431,8 +1414,7 @@ def test_concat_null_X(): adatas_orig = {k: gen_adata((20, 10)) for k in list("abc")} adatas_no_X = {} for k, v in adatas_orig.items(): - with pytest.warns(ExperimentalFeatureWarning): - v = v.copy() + v = v.copy() del v.X adatas_no_X[k] = v diff --git a/anndata/tests/test_dask.py b/anndata/tests/test_dask.py index 962609e63..a5ac2eb5a 100644 --- a/anndata/tests/test_dask.py +++ b/anndata/tests/test_dask.py @@ -112,7 +112,6 @@ def test_dask_distributed_write(adata, tmp_path, diskfmt): with dd.LocalCluster( n_workers=1, threads_per_worker=1, processes=False ) as cluster, dd.Client(cluster), warnings.catch_warnings(): - warnings.simplefilter("ignore", ad.ExperimentalFeatureWarning) M, N = adata.X.shape adata.obsm["a"] = da.random.random((M, 10)) adata.obsm["b"] = da.random.random((M, 10)) diff --git a/anndata/tests/test_hdf5_backing.py b/anndata/tests/test_hdf5_backing.py index a987201da..94f2af4b0 100644 --- a/anndata/tests/test_hdf5_backing.py +++ b/anndata/tests/test_hdf5_backing.py @@ -177,10 +177,6 @@ def test_backed_raw(tmp_path): assert_equal(final_adata, mem_adata) -# This is thrown on too many lines to filter it with pytest.warns -@pytest.mark.filterwarnings( - "ignore:Support for Awkward Arrays is currently experimental:anndata._warnings.ExperimentalFeatureWarning" -) @pytest.mark.parametrize( "array_type", [ diff --git a/anndata/tests/test_helpers.py b/anndata/tests/test_helpers.py index 9d4a17c74..52f1d6cb1 100644 --- a/anndata/tests/test_helpers.py +++ b/anndata/tests/test_helpers.py @@ -106,9 +106,8 @@ def test_assert_equal(): assert_equal(np.array(list(ascii_letters)), np.array(list(ascii_letters))[::-1]) adata = gen_adata((10, 10)) - with pytest.warns(ad.ExperimentalFeatureWarning): - adata.raw = adata.copy() - assert_equal(adata, adata.copy(), exact=True) + adata.raw = adata.copy() + assert_equal(adata, adata.copy(), exact=True) # TODO: I’m not sure this is good behaviour, I’ve disabled in for now. # assert_equal( # adata, @@ -118,16 +117,14 @@ def test_assert_equal(): # ].copy(), # exact=False, # ) - with pytest.warns(ad.ExperimentalFeatureWarning): - adata2 = adata.copy() + adata2 = adata.copy() to_modify = list(adata2.layers.keys())[0] del adata2.layers[to_modify] with pytest.raises(AssertionError) as missing_layer_error: assert_equal(adata, adata2) assert "layers" in str(missing_layer_error.value) # `to_modify` will be in pytest info - with pytest.warns(ad.ExperimentalFeatureWarning): - adata2 = adata.copy() + adata2 = adata.copy() adata2.layers[to_modify][0, 0] = adata2.layers[to_modify][0, 0] + 1 with pytest.raises(AssertionError) as changed_layer_error: assert_equal(adata, adata2) @@ -153,10 +150,6 @@ def test_assert_equal(): assert_equal(ordered_cat, unordered_cat, exact=True) -# This is thrown on too many lines to filter it with pytest.warns -@pytest.mark.filterwarnings( - "ignore:Support for Awkward Arrays is currently experimental:anndata._warnings.ExperimentalFeatureWarning" -) def test_assert_equal_raw(): base = gen_adata((10, 10)) orig = base.copy() @@ -180,9 +173,8 @@ def test_assert_equal_raw_presence(): # This was causing some testing issues during # https://github.com/scverse/anndata/pull/542 a = gen_adata((10, 20)) - with pytest.warns(ad.ExperimentalFeatureWarning): - b = a.copy() - a.raw = a.copy() + b = a.copy() + a.raw = a.copy() assert b.raw is None with pytest.raises(AssertionError): @@ -195,8 +187,7 @@ def test_assert_equal_raw_presence(): # Should they not be if an exact comparison is made? def test_assert_equal_aligned_mapping(): adata1 = gen_adata((10, 10)) - with pytest.warns(ad.ExperimentalFeatureWarning): - adata2 = adata1.copy() + adata2 = adata1.copy() for attr in ["obsm", "varm", "layers", "obsp", "varp"]: assert_equal(getattr(adata1, attr), getattr(adata2, attr)) diff --git a/anndata/tests/test_inplace_subset.py b/anndata/tests/test_inplace_subset.py index 1e352b0d0..110d2574a 100644 --- a/anndata/tests/test_inplace_subset.py +++ b/anndata/tests/test_inplace_subset.py @@ -4,7 +4,6 @@ import pytest from scipy import sparse -from anndata import ExperimentalFeatureWarning from anndata.tests.helpers import ( as_dense_dask_array, assert_equal, @@ -33,10 +32,9 @@ def test_inplace_subset_var(matrix_type, subset_func): orig = gen_adata((30, 30), X_type=matrix_type) subset_idx = subset_func(orig.var_names) - with pytest.warns(ExperimentalFeatureWarning): - modified = orig.copy() - from_view = orig[:, subset_idx].copy() - modified._inplace_subset_var(subset_idx) + modified = orig.copy() + from_view = orig[:, subset_idx].copy() + modified._inplace_subset_var(subset_idx) assert_equal(asarray(from_view.X), asarray(modified.X), exact=True) assert_equal(from_view.obs, modified.obs, exact=True) @@ -54,10 +52,9 @@ def test_inplace_subset_obs(matrix_type, subset_func): orig = gen_adata((30, 30), X_type=matrix_type) subset_idx = subset_func(orig.obs_names) - with pytest.warns(ExperimentalFeatureWarning): - modified = orig.copy() - from_view = orig[subset_idx, :].copy() - modified._inplace_subset_obs(subset_idx) + modified = orig.copy() + from_view = orig[subset_idx, :].copy() + modified._inplace_subset_obs(subset_idx) assert_equal(asarray(from_view.X), asarray(modified.X), exact=True) assert_equal(from_view.obs, modified.obs, exact=True) @@ -78,9 +75,8 @@ def test_inplace_subset_no_X(subset_func, dim): subset_idx = subset_func(getattr(orig, f"{dim}_names")) - with pytest.warns(ExperimentalFeatureWarning): - modified = orig.copy() - from_view = subset_dim(orig, **{dim: subset_idx}).copy() - getattr(modified, f"_inplace_subset_{dim}")(subset_idx) + modified = orig.copy() + from_view = subset_dim(orig, **{dim: subset_idx}).copy() + getattr(modified, f"_inplace_subset_{dim}")(subset_idx) assert_equal(modified, from_view, exact=True) diff --git a/anndata/tests/test_io_conversion.py b/anndata/tests/test_io_conversion.py index d0dded4e6..29a5d27e9 100644 --- a/anndata/tests/test_io_conversion.py +++ b/anndata/tests/test_io_conversion.py @@ -3,8 +3,6 @@ """ from __future__ import annotations -from contextlib import nullcontext - import h5py import numpy as np import pytest @@ -40,8 +38,7 @@ def test_sparse_to_dense_disk(tmp_path, mtx_format, to_convert): dense_from_mem_pth = tmp_path / "dense_mem.h5ad" dense_from_disk_pth = tmp_path / "dense_disk.h5ad" mem = gen_adata((50, 50), mtx_format) - with pytest.warns(ad.ExperimentalFeatureWarning): - mem.raw = mem + mem.raw = mem mem.write_h5ad(mem_pth) disk = ad.read_h5ad(mem_pth, backed="r") @@ -57,14 +54,8 @@ def test_sparse_to_dense_disk(tmp_path, mtx_format, to_convert): assert isinstance(f[k], h5py.Dataset) for backed in [None, "r"]: - ctx = ( - pytest.warns(ad.ExperimentalFeatureWarning) - if backed is None - else nullcontext() - ) - with ctx: - from_mem = ad.read_h5ad(dense_from_mem_pth, backed=backed) - from_disk = ad.read_h5ad(dense_from_disk_pth, backed=backed) + from_mem = ad.read_h5ad(dense_from_mem_pth, backed=backed) + from_disk = ad.read_h5ad(dense_from_disk_pth, backed=backed) assert_equal(mem, from_mem) assert_equal(mem, from_disk) assert_equal(disk, from_mem) @@ -74,13 +65,11 @@ def test_sparse_to_dense_disk(tmp_path, mtx_format, to_convert): def test_sparse_to_dense_inplace(tmp_path, spmtx_format): pth = tmp_path / "adata.h5ad" orig = gen_adata((50, 50), spmtx_format) - with pytest.warns(ad.ExperimentalFeatureWarning): - orig.raw = orig + orig.raw = orig orig.write(pth) backed = ad.read_h5ad(pth, backed="r+") backed.write(as_dense=("X", "raw/X")) - with pytest.warns(ad.ExperimentalFeatureWarning): - new = ad.read_h5ad(pth) + new = ad.read_h5ad(pth) assert_equal(orig, new) assert_equal(backed, new) @@ -106,16 +95,12 @@ def test_sparse_to_dense_errors(tmp_path): def test_dense_to_sparse_memory(tmp_path, spmtx_format, to_convert): dense_path = tmp_path / "dense.h5ad" orig = gen_adata((50, 50), np.array) - with pytest.warns(ad.ExperimentalFeatureWarning): - orig.raw = orig + orig.raw = orig orig.write_h5ad(dense_path) assert not isinstance(orig.X, sparse.spmatrix) assert not isinstance(orig.raw.X, sparse.spmatrix) - with pytest.warns(ad.ExperimentalFeatureWarning): - curr = ad.read_h5ad( - dense_path, as_sparse=to_convert, as_sparse_fmt=spmtx_format - ) + curr = ad.read_h5ad(dense_path, as_sparse=to_convert, as_sparse_fmt=spmtx_format) if "X" in to_convert: assert isinstance(curr.X, spmtx_format) diff --git a/anndata/tests/test_io_elementwise.py b/anndata/tests/test_io_elementwise.py index 3b33b86f5..08853b6c4 100644 --- a/anndata/tests/test_io_elementwise.py +++ b/anndata/tests/test_io_elementwise.py @@ -150,8 +150,7 @@ def test_dask_write_sparse(store, sparse_format): def test_io_spec_raw(store): adata = gen_adata((3, 2)) - with pytest.warns(ad.ExperimentalFeatureWarning): - adata.raw = adata + adata.raw = adata write_elem(store, "adata", adata) @@ -298,6 +297,6 @@ def test_read_zarr_from_group(tmp_path, consolidated): else: read_func = zarr.open - with read_func(pth) as z, pytest.warns(ad.ExperimentalFeatureWarning): + with read_func(pth) as z: expected = ad.read_zarr(z["table/table"]) assert_equal(adata, expected) diff --git a/anndata/tests/test_io_warnings.py b/anndata/tests/test_io_warnings.py index 1dc341ffd..ac704c249 100644 --- a/anndata/tests/test_io_warnings.py +++ b/anndata/tests/test_io_warnings.py @@ -26,7 +26,6 @@ def test_old_format_warning_not_thrown(tmp_path): with warnings.catch_warnings(record=True) as record: warnings.simplefilter("always", ad.OldFormatWarning) - warnings.simplefilter("ignore", ad.ExperimentalFeatureWarning) ad.read_h5ad(pth) diff --git a/anndata/tests/test_readwrite.py b/anndata/tests/test_readwrite.py index cd072bfe3..22b1aaffc 100644 --- a/anndata/tests/test_readwrite.py +++ b/anndata/tests/test_readwrite.py @@ -89,8 +89,7 @@ def rw(backing_h5ad): M, N = 100, 101 orig = gen_adata((M, N)) orig.write(backing_h5ad) - with pytest.warns(ad.ExperimentalFeatureWarning): - curr = ad.read_h5ad(backing_h5ad) + curr = ad.read_h5ad(backing_h5ad) return curr, orig @@ -241,14 +240,12 @@ def test_readwrite_equivalent_h5ad_zarr(tmp_path, typ): M, N = 100, 101 adata = gen_adata((M, N), X_type=typ) - with pytest.warns(ad.ExperimentalFeatureWarning): - adata.raw = adata + adata.raw = adata adata.write_h5ad(h5ad_pth) adata.write_zarr(zarr_pth) - with pytest.warns(ad.ExperimentalFeatureWarning): - from_h5ad = ad.read_h5ad(h5ad_pth) - from_zarr = ad.read_zarr(zarr_pth) + from_h5ad = ad.read_h5ad(h5ad_pth) + from_zarr = ad.read_zarr(zarr_pth) assert_equal(from_h5ad, from_zarr, exact=True) @@ -328,8 +325,7 @@ def check_compressed(key, value): msg = "\n\t".join(not_compressed) raise AssertionError(f"These elements were not compressed correctly:\n\t{msg}") - with pytest.warns(ad.ExperimentalFeatureWarning): - expected = ad.read_h5ad(pth) + expected = ad.read_h5ad(pth) assert_equal(adata, expected) @@ -355,8 +351,7 @@ def check_compressed(key, value): msg = "\n\t".join(not_compressed) raise AssertionError(f"These elements were not compressed correctly:\n\t{msg}") - with pytest.warns(ad.ExperimentalFeatureWarning): - expected = ad.read_zarr(pth) + expected = ad.read_zarr(pth) assert_equal(adata, expected) @@ -366,14 +361,12 @@ def test_changed_obs_var_names(tmp_path, diskfmt): orig = gen_adata((10, 10)) orig.obs_names.name = "obs" orig.var_names.name = "var" - with pytest.warns(ad.ExperimentalFeatureWarning): - modified = orig.copy() + modified = orig.copy() modified.obs_names.name = "cells" modified.var_names.name = "genes" getattr(orig, f"write_{diskfmt}")(filepth) - with pytest.warns(ad.ExperimentalFeatureWarning): - read = getattr(ad, f"read_{diskfmt}")(filepth) + read = getattr(ad, f"read_{diskfmt}")(filepth) assert_equal(orig, read, exact=True) assert orig.var.index.name == "var" @@ -699,8 +692,7 @@ def test_zarr_chunk_X(tmp_path): z = zarr.open(str(zarr_pth)) # As of v2.3.2 zarr won’t take a Path assert z["X"].chunks == (10, 10) - with pytest.warns(ad.ExperimentalFeatureWarning): - from_zarr = ad.read_zarr(zarr_pth) + from_zarr = ad.read_zarr(zarr_pth) assert_equal(from_zarr, adata) @@ -792,13 +784,11 @@ def test_adata_in_uns(tmp_path, diskfmt): "b": gen_adata((12, 8)), } another_one = gen_adata((2, 5)) - with pytest.warns(ad.ExperimentalFeatureWarning): - another_one.raw = gen_adata((2, 7)) + another_one.raw = gen_adata((2, 7)) orig.uns["adatas"]["b"].uns["another_one"] = another_one write(orig, pth) - with pytest.warns(ad.ExperimentalFeatureWarning): - curr = read(pth) + curr = read(pth) assert_equal(orig, curr) diff --git a/anndata/tests/test_transpose.py b/anndata/tests/test_transpose.py index 498dc8065..720733496 100644 --- a/anndata/tests/test_transpose.py +++ b/anndata/tests/test_transpose.py @@ -1,7 +1,5 @@ from __future__ import annotations -import warnings - import numpy as np import pytest from scipy import sparse @@ -25,10 +23,8 @@ def test_transpose_orig(): def _add_raw(adata, *, var_subset=slice(None)): - with warnings.catch_warnings(): - warnings.simplefilter("ignore", ad.ExperimentalFeatureWarning) - new = adata[:, var_subset].copy() - new.raw = adata + new = adata[:, var_subset].copy() + new.raw = adata return new diff --git a/anndata/tests/test_views.py b/anndata/tests/test_views.py index 6784ba60a..fd0ff8902 100644 --- a/anndata/tests/test_views.py +++ b/anndata/tests/test_views.py @@ -502,10 +502,6 @@ def test_layers_view(): assert view_hash != joblib.hash(view_adata) -# This is thrown on too many lines to filter it with pytest.warns -@pytest.mark.filterwarnings( - "ignore:Support for Awkward Arrays is currently experimental:anndata._warnings.ExperimentalFeatureWarning" -) # TODO: This can be flaky. Make that stop def test_view_of_view(matrix_type, subset_func, subset_func2): adata = gen_adata((30, 15), X_type=matrix_type) diff --git a/anndata/tests/test_x.py b/anndata/tests/test_x.py index d300df05f..5115f9ee6 100644 --- a/anndata/tests/test_x.py +++ b/anndata/tests/test_x.py @@ -7,7 +7,7 @@ from scipy import sparse import anndata as ad -from anndata import AnnData, ExperimentalFeatureWarning +from anndata import AnnData from anndata.tests.helpers import assert_equal, gen_adata from anndata.utils import asarray @@ -52,8 +52,7 @@ def test_del_set_equiv_X(): """Tests that `del adata.X` is equivalent to `adata.X = None`""" # test setter and deleter orig = gen_adata((10, 10)) - with pytest.warns(ad.ExperimentalFeatureWarning): - copy = orig.copy() + copy = orig.copy() del orig.X copy.X = None @@ -135,7 +134,6 @@ def test_io_missing_X(tmp_path, diskfmt): write(adata, file_pth) - with pytest.warns(ExperimentalFeatureWarning): - from_disk = read(file_pth) + from_disk = read(file_pth) assert_equal(from_disk, adata) diff --git a/anndata/utils.py b/anndata/utils.py index d0c3bf4b9..9c700e28b 100644 --- a/anndata/utils.py +++ b/anndata/utils.py @@ -1,6 +1,5 @@ from __future__ import annotations -import os import re import warnings from functools import singledispatch, wraps @@ -333,10 +332,9 @@ def convert_dictionary_to_structured_array(source: Mapping[str, Sequence[Any]]): def warn_once(msg: str, category: type[Warning], stacklevel: int = 1): warnings.warn(msg, category, stacklevel=stacklevel) - if "PYTEST_CURRENT_TEST" not in os.environ: - # Prevent from showing up every time an awkward array is used - # You'd think `'once'` works, but it doesn't at the repl and in notebooks - warnings.filterwarnings("ignore", category=category, message=re.escape(msg)) + # Prevent from showing up every time an awkward array is used + # You'd think `'once'` works, but it doesn't at the repl and in notebooks + warnings.filterwarnings("ignore", category=category, message=re.escape(msg)) def deprecated( diff --git a/pyproject.toml b/pyproject.toml index e108410f2..8849cf6de 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -114,6 +114,9 @@ addopts = [ "--ignore=anndata/core.py", # deprecated "--ignore=anndata/readwrite.py", # deprecated ] +filterwarnings = [ + "ignore::anndata._warnings.ExperimentalFeatureWarning", +] python_files = "test_*.py" testpaths = ["anndata", "docs/concatenation.rst"] # For some reason this effects how logging is shown when tests are run From 89d0b458f639f65becf2a641075c8711ba3c35a6 Mon Sep 17 00:00:00 2001 From: Philipp A Date: Thu, 19 Oct 2023 13:12:10 +0200 Subject: [PATCH 34/43] Discard changes to anndata/tests/test_x.py --- anndata/tests/test_x.py | 1 - 1 file changed, 1 deletion(-) diff --git a/anndata/tests/test_x.py b/anndata/tests/test_x.py index 5115f9ee6..5f381c8c1 100644 --- a/anndata/tests/test_x.py +++ b/anndata/tests/test_x.py @@ -133,7 +133,6 @@ def test_io_missing_X(tmp_path, diskfmt): del adata.X write(adata, file_pth) - from_disk = read(file_pth) assert_equal(from_disk, adata) From a2feac2498cb0db63f9a9829835bacb258af5ab4 Mon Sep 17 00:00:00 2001 From: Phil Schaf Date: Thu, 19 Oct 2023 15:39:33 +0200 Subject: [PATCH 35/43] ugh --- pyproject.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 8849cf6de..6bc93a010 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -115,7 +115,8 @@ addopts = [ "--ignore=anndata/readwrite.py", # deprecated ] filterwarnings = [ - "ignore::anndata._warnings.ExperimentalFeatureWarning", + 'ignore:Support for Awkward Arrays is currently experimental', + 'ignore:Outer joins on awkward\.Arrays', ] python_files = "test_*.py" testpaths = ["anndata", "docs/concatenation.rst"] From f9adde77305fc0935cf51001288d47d13c16c631 Mon Sep 17 00:00:00 2001 From: Phil Schaf Date: Thu, 19 Oct 2023 16:40:37 +0200 Subject: [PATCH 36/43] make comment into issue --- .azure-pipelines.yml | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/.azure-pipelines.yml b/.azure-pipelines.yml index b2cdca39e..d6f0c0688 100644 --- a/.azure-pipelines.yml +++ b/.azure-pipelines.yml @@ -63,20 +63,7 @@ jobs: condition: and(eq(variables['RUN_COVERAGE'], 'yes'), eq(variables['PRERELEASE_DEPENDENCIES'], 'no')) # TODO: fix all the exceptions here - # TODO: centralize using a custom pytes CLI arg, because: - # 1. This allows to easily locally run with the same settings as this job - # 2. Otherwise we can’t filter anndata warning classes directly, - # because we’re neither using editable installs nor a src/ layout. - # This is better because it’s a more specific filter and we don’t need one line per message. - # Setting the filters in code instead of per `-W` fixes that. - # This affects: - # - `anndata._warnings.ExperimentalFeatureWarning` - # - “Support for Awkward Arrays” - # - “Outer joins on awkward.Arrays” - # - `anndata._warnings.ImplicitModificationWarning` - # - “Trying to modify attribute” - # - “Transforming to str index” - # 3. The 'ignore' lines need to be kept in sync with `filterwarnings` config in pyproject.toml + # TODO: Centralize, see https://github.com/scverse/anndata/issues/1204 - script: > pytest -W error From 842410360377c4a2ea44b29253bda29eb7c985d4 Mon Sep 17 00:00:00 2001 From: Philipp A Date: Thu, 19 Oct 2023 16:43:17 +0200 Subject: [PATCH 37/43] Apply suggestions from code review Co-authored-by: Isaac Virshup --- anndata/_io/specs/methods.py | 1 - anndata/tests/test_dask.py | 2 +- pyproject.toml | 2 +- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/anndata/_io/specs/methods.py b/anndata/_io/specs/methods.py index adbb1b4bc..00cd66ea7 100644 --- a/anndata/_io/specs/methods.py +++ b/anndata/_io/specs/methods.py @@ -281,7 +281,6 @@ def read_anndata(elem, _reader): ]: if k in elem: d[k] = _reader.read_elem(elem[k]) - return AnnData(**d) diff --git a/anndata/tests/test_dask.py b/anndata/tests/test_dask.py index a5ac2eb5a..7cad3a437 100644 --- a/anndata/tests/test_dask.py +++ b/anndata/tests/test_dask.py @@ -111,7 +111,7 @@ def test_dask_distributed_write(adata, tmp_path, diskfmt): with dd.LocalCluster( n_workers=1, threads_per_worker=1, processes=False - ) as cluster, dd.Client(cluster), warnings.catch_warnings(): + ) as cluster, dd.Client(cluster): M, N = adata.X.shape adata.obsm["a"] = da.random.random((M, 10)) adata.obsm["b"] = da.random.random((M, 10)) diff --git a/pyproject.toml b/pyproject.toml index 6bc93a010..9cf5426db 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -88,7 +88,7 @@ test = [ "joblib", "boltons", "scanpy", - "httpx", # For data downloading + "httpx", # For data downloading "dask[array,distributed]", "awkward>=2.3", "pyarrow", From 6dd8d6b13dd751af3d0aa348a6d0fa0df0166f82 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 19 Oct 2023 14:44:14 +0000 Subject: [PATCH 38/43] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- anndata/tests/test_dask.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/anndata/tests/test_dask.py b/anndata/tests/test_dask.py index 7cad3a437..56cb0f8c8 100644 --- a/anndata/tests/test_dask.py +++ b/anndata/tests/test_dask.py @@ -3,8 +3,6 @@ """ from __future__ import annotations -import warnings - import pandas as pd import pytest From 7bebcb39dd3014bae73db8636e9767c569e9589b Mon Sep 17 00:00:00 2001 From: Philipp A Date: Thu, 19 Oct 2023 16:44:40 +0200 Subject: [PATCH 39/43] Update anndata/tests/helpers.py Co-authored-by: Isaac Virshup --- anndata/tests/helpers.py | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/anndata/tests/helpers.py b/anndata/tests/helpers.py index 641bdc791..a886d7e68 100644 --- a/anndata/tests/helpers.py +++ b/anndata/tests/helpers.py @@ -256,17 +256,19 @@ def gen_adata( awkward_ragged=gen_awkward((12, None, None)), # U_recarray=gen_vstr_recarray(N, 5, "U4") ) - adata = AnnData( - X=X, - obs=obs, - var=var, - obsm=obsm, - varm=varm, - layers=layers, - obsp=obsp, - varp=varp, - uns=uns, - ) + with warnings.catch_warnings(): + warnings.simplefilter("ignore", ExperimentalFeatureWarning) + adata = AnnData( + X=X, + obs=obs, + var=var, + obsm=obsm, + varm=varm, + layers=layers, + obsp=obsp, + varp=varp, + uns=uns, + ) return adata From 27051194eb22dde9e88754b92980e1cf60cf4f7c Mon Sep 17 00:00:00 2001 From: Philipp A Date: Thu, 19 Oct 2023 16:47:19 +0200 Subject: [PATCH 40/43] Update pyproject.toml --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 9cf5426db..8c2508787 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -88,7 +88,7 @@ test = [ "joblib", "boltons", "scanpy", - "httpx", # For data downloading + "httpx", # For data downloading "dask[array,distributed]", "awkward>=2.3", "pyarrow", From 90cdb2ff0beec16018162129d3f4af9deb968bf3 Mon Sep 17 00:00:00 2001 From: Phil Schaf Date: Thu, 19 Oct 2023 16:51:00 +0200 Subject: [PATCH 41/43] done --- anndata/tests/helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/anndata/tests/helpers.py b/anndata/tests/helpers.py index a886d7e68..316fc991e 100644 --- a/anndata/tests/helpers.py +++ b/anndata/tests/helpers.py @@ -15,7 +15,7 @@ from pandas.api.types import is_numeric_dtype from scipy import sparse -from anndata import AnnData, Raw +from anndata import AnnData, ExperimentalFeatureWarning, Raw from anndata._core.aligned_mapping import AlignedMapping from anndata._core.sparse_dataset import BaseCompressedSparseDataset from anndata._core.views import ArrayView From 4a0792d83e6e7e167c8991fcbfad1fcf359a54e3 Mon Sep 17 00:00:00 2001 From: "Philipp A." Date: Tue, 24 Oct 2023 17:37:46 +0200 Subject: [PATCH 42/43] remove len condition --- anndata/_core/anndata.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/anndata/_core/anndata.py b/anndata/_core/anndata.py index 3fc67dd92..f357dcd59 100644 --- a/anndata/_core/anndata.py +++ b/anndata/_core/anndata.py @@ -875,10 +875,9 @@ def _prep_dim_index(self, value, attr: str) -> pd.Index: value = pd.Index(value) if not isinstance(value.name, (str, type(None))): value.name = None - if ( - len(value) > 0 - and not isinstance(value, pd.RangeIndex) - and infer_dtype(value) not in ("string", "bytes") + if not ( + isinstance(value, pd.RangeIndex) + or infer_dtype(value) in ("string", "bytes") ): sample = list(value[: min(len(value), 5)]) msg = dedent( From 37f5a03cf9a51c400a96254191aabcb151fca754 Mon Sep 17 00:00:00 2001 From: "Philipp A." Date: Tue, 24 Oct 2023 18:35:16 +0200 Subject: [PATCH 43/43] it was necessary after all --- anndata/_core/anndata.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/anndata/_core/anndata.py b/anndata/_core/anndata.py index f357dcd59..3fc67dd92 100644 --- a/anndata/_core/anndata.py +++ b/anndata/_core/anndata.py @@ -875,9 +875,10 @@ def _prep_dim_index(self, value, attr: str) -> pd.Index: value = pd.Index(value) if not isinstance(value.name, (str, type(None))): value.name = None - if not ( - isinstance(value, pd.RangeIndex) - or infer_dtype(value) in ("string", "bytes") + if ( + len(value) > 0 + and not isinstance(value, pd.RangeIndex) + and infer_dtype(value) not in ("string", "bytes") ): sample = list(value[: min(len(value), 5)]) msg = dedent(