From 96f092e714ca9777265c314850fb86191f2118ee Mon Sep 17 00:00:00 2001 From: Hagen Wierstorf Date: Wed, 4 Dec 2024 17:39:02 +0100 Subject: [PATCH 01/12] TST: enable tests for documentation files --- docs/conftest.py | 98 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 98 insertions(+) create mode 100644 docs/conftest.py diff --git a/docs/conftest.py b/docs/conftest.py new file mode 100644 index 00000000..aead547a --- /dev/null +++ b/docs/conftest.py @@ -0,0 +1,98 @@ +from doctest import ELLIPSIS +from doctest import NORMALIZE_WHITESPACE +import os + +import pytest +import sybil +from sybil.parsers.rest import DocTestParser +from sybil.parsers.rest import PythonCodeBlockParser +from sybil.parsers.rest import SkipParser + +import audb +from audb.core.conftest import cache # noqa: F401 +from audb.core.conftest import imports +from audb.core.conftest import public_repository # noqa: F401 + + +@pytest.fixture(scope="module") +def run_in_tmpdir(tmpdir_factory): + """Move to a persistent tmpdir for execution of a whole file.""" + tmpdir = tmpdir_factory.mktemp("tmp") + current_dir = os.getcwd() + os.chdir(tmpdir) + + yield + + os.chdir(current_dir) + + +@pytest.fixture(scope="module") +def default_configuration(): + """Set config values to default values from global config file.""" + # Read global config file + global_config = audb.core.config.load_configuration_file( + audb.core.config.global_config_file + ) + # Store current user settings + current_cache_root = audb.config.CACHE_ROOT + current_shared_cache_root = audb.config.SHARED_CACHE_ROOT + current_repositories = audb.config.REPOSITORIES + # Enforce default values + audb.config.CACHE_ROOT = global_config["cache_root"] + audb.config.SHARED_CACHE_ROOT = global_config["shared_cache_root"] + audb.config.REPOSITORIES = [ + audb.Repository(r["name"], r["host"], r["backend"]) + for r in global_config["repositories"] + ] + + yield audb + + # Restore user settings + audb.config.CACHE_ROOT = current_cache_root + audb.config.SHARED_CACHE_ROOT = current_shared_cache_root + audb.config.REPOSITORIES = current_repositories + + +# Collect doctests +# +# We use several `sybil.Sybil` instances +# to pass different fixtures for different files +# +parsers = [ + DocTestParser(optionflags=ELLIPSIS + NORMALIZE_WHITESPACE), + PythonCodeBlockParser(), + SkipParser(), +] +pytest_collect_file = sybil.sybil.SybilCollection( + ( + sybil.Sybil( + parsers=parsers, + filenames=[ + "authentication.rst", + "overview.rst", + "quickstart.rst", + "dependencies.rst", + "load.rst", + "audb.info.rst", + ], + fixtures=[ + "cache", + "run_in_tmpdir", + "public_repository", + ], + setup=imports, + ), + sybil.Sybil( + parsers=parsers, + filenames=["publish.rst"], + fixtures=["cache", "run_in_tmpdir"], + setup=imports, + ), + sybil.Sybil( + parsers=parsers, + filenames=["configuration.rst", "caching.rst"], + fixtures=["default_configuration"], + setup=imports, + ), + ) +).pytest() From 9d726ed79e0212296c2e2513aae9d26f8a8936db Mon Sep 17 00:00:00 2001 From: Hagen Wierstorf Date: Wed, 4 Dec 2024 17:49:26 +0100 Subject: [PATCH 02/12] Fix sorting of file listing --- docs/publish.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/publish.rst b/docs/publish.rst index eb599db4..f4698597 100644 --- a/docs/publish.rst +++ b/docs/publish.rst @@ -118,7 +118,7 @@ We can compare this with the files stored in the repository. indent = " " * 2 * (level) print(f"{indent}{os.path.basename(root)}/") subindent = " " * 2 * (level + 1) - for f in files: + for f in sorted(files): print(f"{subindent}{f}") >>> list_files(repository.host) @@ -133,8 +133,8 @@ data/ age.parquet media/ 1.0.0/ - e26ef45d-bdc1-6153-bdc4-852d83806e4a.zip 436c65ec-1e42-f9de-2708-ecafe07e827e.zip + e26ef45d-bdc1-6153-bdc4-852d83806e4a.zip fda7e4d6-f2b2-4cff-cab5-906ef5d57607.zip As you can see all media files are stored @@ -261,8 +261,8 @@ data/ 1.1.0/ ef4d1e81-6488-95cf-a165-604d1e47d575.zip 1.0.0/ - e26ef45d-bdc1-6153-bdc4-852d83806e4a.zip 436c65ec-1e42-f9de-2708-ecafe07e827e.zip + e26ef45d-bdc1-6153-bdc4-852d83806e4a.zip fda7e4d6-f2b2-4cff-cab5-906ef5d57607.zip And check which databases are available. From d0d857ca48ea64bf00c3607720aa4638bd13e0df Mon Sep 17 00:00:00 2001 From: Hagen Wierstorf Date: Wed, 4 Dec 2024 18:08:31 +0100 Subject: [PATCH 03/12] Use context manager for setting default values --- docs/conftest.py | 56 ++++++++++++++++++++++++++++-------------------- 1 file changed, 33 insertions(+), 23 deletions(-) diff --git a/docs/conftest.py b/docs/conftest.py index aead547a..6f0b64d5 100644 --- a/docs/conftest.py +++ b/docs/conftest.py @@ -14,6 +14,36 @@ from audb.core.conftest import public_repository # noqa: F401 +class DefaultConfiguration: + """Context manager to provide default configuration values.""" + + def __init__(self): + self.config = audb.core.config.load_configuration_file( + audb.core.config.global_config_file + ) + self._saved_state = {} + + def __enter__(self): + """Save current state.""" + self._saved_state = { + "cache_root": audb.config.CACHE_ROOT, + "shared_cache_root": audb.config.SHARED_CACHE_ROOT, + "repositories": audb.config.REPOSITORIES, + } + # Set default values + audb.config.CACHE_ROOT = self.config["cache_root"] + audb.config.SHARED_CACHE_ROOT = self.config["shared_cache_root"] + audb.config.REPOSITORIES = [ + audb.Repository(repo["name"], repo["host"], repo["backend"]) + for repo in self.config["repositories"] + ] + + def __exit__(self, *args): + """Restore previous state.""" + for key, value in self._saved_state.items(): + setattr(audb.config, key, value) + + @pytest.fixture(scope="module") def run_in_tmpdir(tmpdir_factory): """Move to a persistent tmpdir for execution of a whole file.""" @@ -28,29 +58,9 @@ def run_in_tmpdir(tmpdir_factory): @pytest.fixture(scope="module") def default_configuration(): - """Set config values to default values from global config file.""" - # Read global config file - global_config = audb.core.config.load_configuration_file( - audb.core.config.global_config_file - ) - # Store current user settings - current_cache_root = audb.config.CACHE_ROOT - current_shared_cache_root = audb.config.SHARED_CACHE_ROOT - current_repositories = audb.config.REPOSITORIES - # Enforce default values - audb.config.CACHE_ROOT = global_config["cache_root"] - audb.config.SHARED_CACHE_ROOT = global_config["shared_cache_root"] - audb.config.REPOSITORIES = [ - audb.Repository(r["name"], r["host"], r["backend"]) - for r in global_config["repositories"] - ] - - yield audb - - # Restore user settings - audb.config.CACHE_ROOT = current_cache_root - audb.config.SHARED_CACHE_ROOT = current_shared_cache_root - audb.config.REPOSITORIES = current_repositories + """Set config values to default values.""" + with DefaultConfiguration(): + yield # Collect doctests From 1573ff961bbcb2a24b8dff88145565bc4df8c878 Mon Sep 17 00:00:00 2001 From: Hagen Wierstorf Date: Wed, 4 Dec 2024 18:21:13 +0100 Subject: [PATCH 04/12] Fix sorting of os.walk --- docs/publish.rst | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/docs/publish.rst b/docs/publish.rst index f4698597..6509c416 100644 --- a/docs/publish.rst +++ b/docs/publish.rst @@ -113,7 +113,7 @@ We can compare this with the files stored in the repository. import os def list_files(path): - for root, dirs, files in os.walk(path): + for root, _, files in sorted(os.walk(path)): level = root.replace(path, "").count(os.sep) indent = " " * 2 * (level) print(f"{indent}{os.path.basename(root)}/") @@ -128,14 +128,14 @@ data/ 1.0.0/ db.parquet db.yaml - meta/ - 1.0.0/ - age.parquet media/ 1.0.0/ 436c65ec-1e42-f9de-2708-ecafe07e827e.zip e26ef45d-bdc1-6153-bdc4-852d83806e4a.zip fda7e4d6-f2b2-4cff-cab5-906ef5d57607.zip + meta/ + 1.0.0/ + age.parquet As you can see all media files are stored inside the ``media/`` folder, @@ -246,24 +246,24 @@ We can again inspect the repository. data/ data-local/ age-test/ - 1.1.0/ + 1.0.0/ db.parquet db.yaml - 1.0.0/ + 1.1.0/ db.parquet db.yaml - meta/ - 1.1.0/ - age.parquet - 1.0.0/ - age.parquet media/ - 1.1.0/ - ef4d1e81-6488-95cf-a165-604d1e47d575.zip 1.0.0/ 436c65ec-1e42-f9de-2708-ecafe07e827e.zip e26ef45d-bdc1-6153-bdc4-852d83806e4a.zip fda7e4d6-f2b2-4cff-cab5-906ef5d57607.zip + 1.1.0/ + ef4d1e81-6488-95cf-a165-604d1e47d575.zip + meta/ + 1.0.0/ + age.parquet + 1.1.0/ + age.parquet And check which databases are available. From e8a36500605988136ef9a1b56132b8c59074b387 Mon Sep 17 00:00:00 2001 From: Hagen Wierstorf Date: Wed, 4 Dec 2024 18:22:16 +0100 Subject: [PATCH 05/12] Fix order in conftest.py --- docs/conftest.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/conftest.py b/docs/conftest.py index 6f0b64d5..ecdeec2d 100644 --- a/docs/conftest.py +++ b/docs/conftest.py @@ -44,6 +44,13 @@ def __exit__(self, *args): setattr(audb.config, key, value) +@pytest.fixture(scope="module") +def default_configuration(): + """Set config values to default values.""" + with DefaultConfiguration(): + yield + + @pytest.fixture(scope="module") def run_in_tmpdir(tmpdir_factory): """Move to a persistent tmpdir for execution of a whole file.""" @@ -56,13 +63,6 @@ def run_in_tmpdir(tmpdir_factory): os.chdir(current_dir) -@pytest.fixture(scope="module") -def default_configuration(): - """Set config values to default values.""" - with DefaultConfiguration(): - yield - - # Collect doctests # # We use several `sybil.Sybil` instances From 06f8cf2350edfc34f906bb08e0dbe3127457a6cd Mon Sep 17 00:00:00 2001 From: Hagen Wierstorf Date: Wed, 4 Dec 2024 18:29:15 +0100 Subject: [PATCH 06/12] Skip test for pandas == 2.1.4 --- docs/dependencies.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/dependencies.rst b/docs/dependencies.rst index 66236ba7..1f360e18 100644 --- a/docs/dependencies.rst +++ b/docs/dependencies.rst @@ -48,6 +48,11 @@ If your database contains only WAV or FLAC files, we store the duration in seconds of every file in the database dependency table. +.. + >>> import pandas as pd + +.. skip: start if(pd.version == "2.1.4", reason="formats output differently") + >>> deps = audb.dependencies("emodb", version="1.4.1") >>> df = deps() >>> df.duration[:10] @@ -63,6 +68,8 @@ wav/03a02Wb.wav 2.123625 wav/03a02Wc.wav 1.498063 Name: duration, dtype: double[pyarrow] +.. skip: end + For those databases you can get their overall duration with: From 18d41b9bd463749db6813b8459203e914f0db89f Mon Sep 17 00:00:00 2001 From: Hagen Wierstorf Date: Wed, 4 Dec 2024 18:38:54 +0100 Subject: [PATCH 07/12] Fix typo --- docs/dependencies.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/dependencies.rst b/docs/dependencies.rst index 1f360e18..4ea83c6f 100644 --- a/docs/dependencies.rst +++ b/docs/dependencies.rst @@ -51,7 +51,7 @@ in the database dependency table. .. >>> import pandas as pd -.. skip: start if(pd.version == "2.1.4", reason="formats output differently") +.. skip: start if(pd.__version__ == "2.1.4", reason="formats output differently") >>> deps = audb.dependencies("emodb", version="1.4.1") >>> df = deps() From 72ad110db18fb2f99a5669af0483e3f7250d4868 Mon Sep 17 00:00:00 2001 From: Hagen Wierstorf Date: Wed, 4 Dec 2024 18:50:22 +0100 Subject: [PATCH 08/12] fix cache test under MacOS, skip under Windows --- docs/caching.rst | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/docs/caching.rst b/docs/caching.rst index e0a0a2f5..41454adc 100644 --- a/docs/caching.rst +++ b/docs/caching.rst @@ -1,5 +1,12 @@ .. _caching: +.. As the test outputs are mainly paths, +.. we skip the tests under Windows +.. + >>> import platform + +.. skip: start if(platform.system() == "Windows") + Caching ======= @@ -44,7 +51,7 @@ accessible to you only. By default it points to >>> audb.default_cache_root(shared=False) -'/home/.../audb' +'.../audb' When you request a database with :meth:`audb.load`, :mod:`audb` first looks for it in the shared cache folder @@ -88,5 +95,6 @@ Note, 2. overwrites 3. and 4., and so on. +.. skip: end .. _adjust the rights: https://superuser.com/a/264406 From 92121297955fdeb18f605b173fba136eb60f0934 Mon Sep 17 00:00:00 2001 From: Hagen Wierstorf Date: Wed, 4 Dec 2024 18:59:44 +0100 Subject: [PATCH 09/12] Fix typo --- docs/caching.rst | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/caching.rst b/docs/caching.rst index 41454adc..b65901ee 100644 --- a/docs/caching.rst +++ b/docs/caching.rst @@ -65,8 +65,6 @@ There are four ways to change the default locations: 1. By setting the argument ``cache_root`` during a function call, e.g. -.. skip: next - >>> db = audb.load("emodb", ..., cache_root="/cache/root/audb") 2. System-wide by setting the following system variables From 5d4c21ffdb72a7f56ccb218e9be7cdd5bee8ddf1 Mon Sep 17 00:00:00 2001 From: Hagen Wierstorf Date: Thu, 5 Dec 2024 08:49:30 +0100 Subject: [PATCH 10/12] Fix skipping of tests for caching --- docs/caching.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/caching.rst b/docs/caching.rst index b65901ee..8cc7a7cb 100644 --- a/docs/caching.rst +++ b/docs/caching.rst @@ -65,6 +65,9 @@ There are four ways to change the default locations: 1. By setting the argument ``cache_root`` during a function call, e.g. +.. skip: end +.. skip: next + >>> db = audb.load("emodb", ..., cache_root="/cache/root/audb") 2. System-wide by setting the following system variables @@ -76,6 +79,8 @@ There are four ways to change the default locations: 3. Program-wide by overwriting the default values in :class:`audb.config` +.. skip: start if(platform.system() == "Windows") + >>> audb.config.SHARED_CACHE_ROOT = "/new/shared/cache/audb" >>> audb.default_cache_root(shared=True) '/new/shared/cache/audb' From 9811840f5f0f731a8586f5610ef9ee67a774648c Mon Sep 17 00:00:00 2001 From: Hagen Wierstorf Date: Thu, 5 Dec 2024 09:04:04 +0100 Subject: [PATCH 11/12] Fix path in quickstart under Windows --- docs/quickstart.rst | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/docs/quickstart.rst b/docs/quickstart.rst index 4de7e14f..7527f049 100644 --- a/docs/quickstart.rst +++ b/docs/quickstart.rst @@ -32,11 +32,9 @@ Let's load version 1.4.1 of the emodb_ database. verbose=False, ) # Add flavor path, to mimic `full_path=True` + flavor_path = audb.flavor_path("emodb", "1.4.1").replace("\\", "/") for table in list(db.tables): - db[table]._df.index = _audformat.utils.expand_file_path( - db[table]._df.index, - f'...{audb.flavor_path("emodb", "1.4.1")}', - ) + db[table]._df.index = f"...{flavor_path}/" + db[table]._df.index .. skip: next From de590a3e364d7ad7c99bbd6cdb45d3d3e2da099a Mon Sep 17 00:00:00 2001 From: Hagen Wierstorf Date: Thu, 5 Dec 2024 09:11:48 +0100 Subject: [PATCH 12/12] Remove unneeded import --- docs/quickstart.rst | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/quickstart.rst b/docs/quickstart.rst index 7527f049..89c9c39d 100644 --- a/docs/quickstart.rst +++ b/docs/quickstart.rst @@ -22,8 +22,6 @@ Let's load version 1.4.1 of the emodb_ database. .. Load with only_metadata=True in the background .. invisible-code-block: python - import audformat as _audformat - db = audb.load( "emodb", version="1.4.1",