From 7155cf717845258d0abc2e93da5fff91078c3ac3 Mon Sep 17 00:00:00 2001 From: Hagen Wierstorf Date: Fri, 15 Nov 2024 09:17:23 +0100 Subject: [PATCH] Fix doctests --- audbackend/core/backend/minio.py | 4 +- audbackend/core/conftest.py | 117 +++++++++++++---------- audbackend/core/errors.py | 1 + audbackend/core/interface/maven.py | 7 ++ audbackend/core/interface/unversioned.py | 47 +++++++++ audbackend/core/interface/versioned.py | 92 +++++++++++------- 6 files changed, 179 insertions(+), 89 deletions(-) diff --git a/audbackend/core/backend/minio.py b/audbackend/core/backend/minio.py index 0b5ec80..ed18096 100644 --- a/audbackend/core/backend/minio.py +++ b/audbackend/core/backend/minio.py @@ -40,11 +40,11 @@ class Minio(Base): >>> auth = ("Q3AM3UQ867SPQQA43P2F", "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG") >>> repository = "my-data" + audeer.uid() >>> Minio.create(host, repository, authentication=auth) - >>> file = audeer.touch("file.txt") + >>> file = audeer.touch("src.txt") >>> backend = Minio(host, repository, authentication=auth) >>> try: ... with backend: - ... backend.put_file("src.txt", "/sub/file.txt") + ... backend.put_file(file, "/sub/file.txt") ... backend.ls() ... finally: ... Minio.delete(host, repository, authentication=auth) diff --git a/audbackend/core/conftest.py b/audbackend/core/conftest.py index 6e16463..0ee0b90 100644 --- a/audbackend/core/conftest.py +++ b/audbackend/core/conftest.py @@ -1,5 +1,6 @@ import datetime import doctest +import os import pytest import sybil @@ -14,74 +15,88 @@ pytest_collect_file = sybil.Sybil( parsers=[DocTestParser(optionflags=doctest.ELLIPSIS)], patterns=["*.py"], - fixtures=["prepare_docstring_tests", "filesystem_backend"], + fixtures=[ + "filesystem_backend", + "prepare_docstring_tests", + "clear", + ], ).pytest() class FileSystem(audbackend.backend.FileSystem): - def __init__( - self, - host: str, - repository: str, - ): + def __init__(self, host, repository): super().__init__(host, repository) self.opened = True - def _date( - self, - path: str, - ) -> str: + def _date(self, path): date = datetime.datetime(1991, 2, 20) date = audbackend.core.utils.date_format(date) return date - def _owner( - self, - path: str, - ) -> str: + def _owner(self, path): return "doctest" +@pytest.fixture(scope="session", autouse=True) +def clear(): + """Clear local files and filesystem backend. + + When using a tmpdir with the scope ``"function"`` or ``"class"``, + a new tmpdir is used each line + in the docstring tests. + When using the next greater scope ``"module"``, + the same tmpdir is used in the whole file, + which means there is no scope + that provides a new tmpdir + for each function/method + within a file. + To simulate this behavior, + we use the scope ``"module"``, + and provide this ``clear()`` function + to reset after a finished docstring. + + """ + + def clear_all(): + # Clear backend + audeer.rmdir("host", "repo") + audeer.mkdir("host", "repo") + # Clear local files + files = audeer.list_file_names(".", basenames=True) + files = [file for file in files if not file == "src.txt"] + for file in files: + os.remove(file) + + yield clear_all + + @pytest.fixture(scope="function") def filesystem_backend(): + """Filesystem backend with patched date and owner methods. + + The backend is also opened already. + + """ yield FileSystem("host", "repo") -@pytest.fixture(scope="function", autouse=True) -def prepare_docstring_tests(tmpdir, monkeypatch): +@pytest.fixture(scope="module", autouse=True) +def prepare_docstring_tests(tmpdir_factory): r"""Code to be run before each doctest.""" - # Change to tmp dir - monkeypatch.chdir(tmpdir) - - # Provide example file `src.txt` - audeer.touch("src.txt") - - # Prepare backend - audeer.mkdir("host", "repo") - - yield - - -# @pytest.fixture(scope="function", autouse=True) -# def prepare_docstring_tests(doctest_namespace): -# with tempfile.TemporaryDirectory() as tmp: -# # Change to tmp dir -# current_dir = os.getcwd() -# os.chdir(tmp) -# # Prepare backend -# audeer.mkdir("host") -# audbackend.backend.FileSystem.create("host", "repo") -# # Provide example file `src.txt` -# audeer.touch("src.txt") -# # Provide DoctestFileSystem as FileSystem, -# # and audbackend -# # in docstring examples -# doctest_namespace["DoctestFileSystem"] = DoctestFileSystem -# doctest_namespace["audbackend"] = audbackend -# -# yield -# -# # Remove backend -# audbackend.backend.FileSystem.delete("host", "repo") -# # Change back to current dir -# os.chdir(current_dir) + tmp_dir = tmpdir_factory.mktemp("tmp") + + try: + # Change to tmp dir + current_dir = os.getcwd() + os.chdir(tmp_dir) + + # Provide example file `src.txt` + audeer.touch("src.txt") + + # Prepare backend + audeer.mkdir("host", "repo") + + yield + + finally: + os.chdir(current_dir) diff --git a/audbackend/core/errors.py b/audbackend/core/errors.py index fed5a5d..c49c9aa 100644 --- a/audbackend/core/errors.py +++ b/audbackend/core/errors.py @@ -8,6 +8,7 @@ class BackendError(Exception): >>> import audeer >>> import audbackend + >>> _ = audeer.mkdir("host", "repo") Examples: >>> backend = audbackend.backend.FileSystem("host", "repo") diff --git a/audbackend/core/interface/maven.py b/audbackend/core/interface/maven.py index a7fca22..4e82aaa 100644 --- a/audbackend/core/interface/maven.py +++ b/audbackend/core/interface/maven.py @@ -65,6 +65,7 @@ class Maven(Versioned): .. >>> import audbackend + >>> import audeer Examples: >>> file = "src.txt" @@ -79,6 +80,9 @@ class Maven(Versioned): >>> interface.get_file("/file.txt", "dst.txt", "2.0.0") '...dst.txt' + .. + >>> clear() + """ # noqa: E501 def __init__( @@ -162,6 +166,9 @@ def ls( >>> interface.ls("/sub/") [('/sub/archive.zip', '1.0.0')] + .. + >>> clear() + """ # noqa: E501 if path.endswith("/"): # find files under sub-path paths = self.backend.ls( diff --git a/audbackend/core/interface/unversioned.py b/audbackend/core/interface/unversioned.py index cfcdfa6..0da09bd 100644 --- a/audbackend/core/interface/unversioned.py +++ b/audbackend/core/interface/unversioned.py @@ -17,6 +17,17 @@ class Unversioned(Base): .. >>> import audbackend + >>> import audeer + + .. >>> def clear(): + .. ... # Clear backend + .. ... audeer.rmdir("host", "repo") + .. ... audeer.mkdir("host", "repo") + .. ... # Clear local files + .. ... files = audeer.list_file_names(".", basenames=True) + .. ... files = [file for file in files if not file == "src.txt"] + .. ... for file in files: + .. ... os.remove(file) Examples: >>> file = "src.txt" @@ -30,6 +41,9 @@ class Unversioned(Base): >>> interface.get_file("/file.txt", "dst.txt") '...dst.txt' + .. + >>> clear() + """ def checksum( @@ -64,6 +78,9 @@ def checksum( >>> interface.checksum("/file.txt") 'd41d8cd98f00b204e9800998ecf8427e' + .. + >>> clear() + """ return self.backend.checksum(path) @@ -118,6 +135,9 @@ def copy_file( >>> interface.exists("/copy.txt") True + .. + >>> clear() + """ self.backend.copy_file( src_path, @@ -158,6 +178,9 @@ def date( >>> interface.date("/file.txt") '1991-02-20' + .. + >>> clear() + """ return self.backend.date(path) @@ -200,6 +223,9 @@ def exists( >>> interface.exists("/file.txt") True + .. + >>> clear() + """ return self.backend.exists( path, @@ -268,6 +294,9 @@ def get_archive( >>> interface.get_archive("/sub/archive.zip", ".") ['src.txt'] + .. + >>> clear() + """ return self.backend.get_archive( src_path, @@ -339,6 +368,9 @@ def get_file( >>> os.path.exists("dst.txt") True + .. + >>> clear() + """ return self.backend.get_file( src_path, @@ -411,6 +443,9 @@ def ls( >>> interface.ls("/sub/") ['/sub/archive.zip'] + .. + >>> clear() + """ # noqa: E501 return self.backend.ls( path, @@ -475,6 +510,9 @@ def move_file( >>> interface.exists("/file.txt") False + .. + >>> clear() + """ self.backend.move_file( src_path, @@ -516,6 +554,7 @@ def owner( >>> interface.owner("/file.txt") 'doctest' + >>> clear() """ return self.backend.owner(path) @@ -587,6 +626,8 @@ def put_archive( >>> interface.exists("/sub/archive.tar.gz") True + >>> clear() + """ self.backend.put_archive( src_root, @@ -647,6 +688,9 @@ def put_file( >>> interface.exists("/file.txt") True + .. + >>> clear() + """ self.backend.put_file( src_path, @@ -683,5 +727,8 @@ def remove_file( >>> interface.exists("/file.txt") False + .. + >>> clear() + """ self.backend.remove_file(path) diff --git a/audbackend/core/interface/versioned.py b/audbackend/core/interface/versioned.py index edec2d2..73a0185 100644 --- a/audbackend/core/interface/versioned.py +++ b/audbackend/core/interface/versioned.py @@ -37,6 +37,8 @@ class Versioned(Base): >>> interface.get_file("/file.txt", "dst.txt", "2.0.0") '...dst.txt' + .. + >>> clear() """ @@ -71,9 +73,7 @@ def checksum( RuntimeError: if backend was not opened .. - >>> backend = audbackend.backend.FileSystem("host", "repo") - >>> backend.open() - >>> interface = Versioned(backend) + >>> interface = Versioned(filesystem_backend) Examples: >>> file = "src.txt" @@ -84,6 +84,9 @@ def checksum( >>> interface.checksum("/file.txt", "1.0.0") 'd41d8cd98f00b204e9800998ecf8427e' + .. + >>> clear() + """ path_with_version = self._path_with_version(path, version) return self.backend.checksum(path_with_version) @@ -136,9 +139,7 @@ def copy_file( RuntimeError: if backend was not opened .. - >>> backend = audbackend.backend.FileSystem("host", "repo") - >>> backend.open() - >>> interface = Versioned(backend) + >>> interface = Versioned(filesystem_backend) Examples: >>> file = "src.txt" @@ -149,6 +150,9 @@ def copy_file( >>> interface.exists("/copy.txt", "1.0.0") True + .. + >>> clear() + """ if version is None: versions = self.versions(src_path) @@ -201,6 +205,9 @@ def date( >>> interface.date("/file.txt", "1.0.0") '1991-02-20' + .. + >>> clear() + """ path_with_version = self._path_with_version(path, version) return self.backend.date(path_with_version) @@ -236,9 +243,7 @@ def exists( RuntimeError: if backend was not opened .. - >>> backend = audbackend.backend.FileSystem("host", "repo") - >>> backend.open() - >>> interface = Versioned(backend) + >>> interface = Versioned(filesystem_backend) Examples: >>> file = "src.txt" @@ -248,6 +253,9 @@ def exists( >>> interface.exists("/file.txt", "1.0.0") True + .. + >>> clear() + """ path_with_version = self._path_with_version(path, version) return self.backend.exists( @@ -312,9 +320,7 @@ def get_archive( RuntimeError: if backend was not opened .. - >>> backend = audbackend.backend.FileSystem("host", "repo") - >>> backend.open() - >>> interface = Versioned(backend) + >>> interface = Versioned(filesystem_backend) Examples: >>> file = "src.txt" @@ -323,6 +329,9 @@ def get_archive( >>> interface.get_archive("/sub/archive.zip", ".", "1.0.0") ['src.txt'] + .. + >>> clear() + """ src_path_with_version = self._path_with_version(src_path, version) return self.backend.get_archive( @@ -388,9 +397,7 @@ def get_file( RuntimeError: if backend was not opened .. - >>> backend = audbackend.backend.FileSystem("host", "repo") - >>> backend.open() - >>> interface = Versioned(backend) + >>> interface = Versioned(filesystem_backend) Examples: >>> file = "src.txt" @@ -401,6 +408,9 @@ def get_file( >>> os.path.exists("dst.txt") True + .. + >>> clear() + """ src_path_with_version = self._path_with_version(src_path, version) return self.backend.get_file( @@ -431,9 +441,7 @@ def latest_version( RuntimeError: if backend was not opened .. - >>> backend = audbackend.backend.FileSystem("host", "repo") - >>> backend.open() - >>> interface = Versioned(backend) + >>> interface = Versioned(filesystem_backend) Examples: >>> file = "src.txt" @@ -442,6 +450,9 @@ def latest_version( >>> interface.latest_version("/file.txt") '2.0.0' + .. + >>> clear() + """ vs = self.versions(path) return vs[-1] @@ -496,9 +507,7 @@ def ls( RuntimeError: if backend was not opened .. - >>> backend = audbackend.backend.FileSystem("host", "repo") - >>> backend.open() - >>> interface = Versioned(backend) + >>> interface = Versioned(filesystem_backend) Examples: >>> file = "src.txt" @@ -518,6 +527,9 @@ def ls( >>> interface.ls("/sub/") [('/sub/archive.zip', '1.0.0')] + .. + >>> clear() + """ # noqa: E501 if path.endswith("/"): # find files under sub-path paths = self.backend.ls( @@ -638,9 +650,7 @@ def move_file( RuntimeError: if backend was not opened .. - >>> backend = audbackend.backend.FileSystem("host", "repo") - >>> backend.open() - >>> interface = Versioned(backend) + >>> interface = Versioned(filesystem_backend) Examples: >>> file = "src.txt" @@ -653,6 +663,9 @@ def move_file( >>> interface.exists("/file.txt", "1.0.0") False + .. + >>> clear() + """ if version is None: versions = self.versions(src_path) @@ -706,6 +719,9 @@ def owner( >>> interface.owner("/file.txt", "1.0.0") 'doctest' + .. + >>> clear() + """ path_with_version = self._path_with_version(path, version) return self.backend.owner(path_with_version) @@ -772,9 +788,7 @@ def put_archive( RuntimeError: if backend was not opened .. - >>> backend = audbackend.backend.FileSystem("host", "repo") - >>> backend.open() - >>> interface = Versioned(backend) + >>> interface = Versioned(filesystem_backend) Examples: >>> file = "src.txt" @@ -784,6 +798,9 @@ def put_archive( >>> interface.exists("/sub/archive.tar.gz", "1.0.0") True + .. + >>> clear() + """ dst_path_with_version = self._path_with_version(dst_path, version) self.backend.put_archive( @@ -842,9 +859,7 @@ def put_file( RuntimeError: if backend was not opened .. - >>> backend = audbackend.backend.FileSystem("host", "repo") - >>> backend.open() - >>> interface = Versioned(backend) + >>> interface = Versioned(filesystem_backend) Examples: >>> file = "src.txt" @@ -854,6 +869,9 @@ def put_file( >>> interface.exists("/file.txt", "3.0.0") True + .. + >>> clear() + """ dst_path_with_version = self._path_with_version(dst_path, version) return self.backend.put_file( @@ -885,9 +903,7 @@ def remove_file( RuntimeError: if backend was not opened .. - >>> backend = audbackend.backend.FileSystem("host", "repo") - >>> backend.open() - >>> interface = Versioned(backend) + >>> interface = Versioned(filesystem_backend) Examples: >>> file = "src.txt" @@ -898,6 +914,9 @@ def remove_file( >>> interface.exists("/file.txt", "1.0.0") False + .. + >>> clear() + """ path_with_version = self._path_with_version(path, version) self.backend.remove_file(path_with_version) @@ -929,9 +948,7 @@ def versions( RuntimeError: if backend was not opened .. - >>> backend = audbackend.backend.FileSystem("host", "repo") - >>> backend.open() - >>> interface = Versioned(backend) + >>> interface = Versioned(filesystem_backend) Examples: >>> file = "src.txt" @@ -940,6 +957,9 @@ def versions( >>> interface.versions("/file.txt") ['1.0.0', '2.0.0'] + .. + >>> clear() + """ utils.check_path(path)