From be5eccd9c5141a013f1e66c3b90d38736b6810cd Mon Sep 17 00:00:00 2001 From: Alex Wolf Date: Thu, 14 Mar 2024 21:01:46 +0100 Subject: [PATCH] =?UTF-8?q?=F0=9F=9A=9A=20Move=20transform=20settings=20in?= =?UTF-8?q?to=20settings=20(#1498)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/annotate.ipynb | 4 +-- docs/faq/key.ipynb | 4 +-- docs/faq/setup.ipynb | 4 +-- docs/faq/track-run-inputs.ipynb | 4 +-- docs/introduction.ipynb | 4 +-- docs/meta.ipynb | 4 +-- docs/track.ipynb | 4 +-- docs/transfer.ipynb | 4 +-- docs/tutorial.ipynb | 4 +-- docs/tutorial2.ipynb | 4 +-- lamindb/__init__.py | 2 +- lamindb/core/_run_context.py | 10 ++++---- lamindb/core/_settings.py | 7 +++++ lamindb/core/_sync_git.py | 40 ++++++++++++++++++++--------- lamindb/core/_transform_settings.py | 6 ++--- sub/lamin-cli | 2 +- sub/lamindb-setup | 2 +- tests/notebooks/no-title.ipynb | 4 +-- tests/test_run_context.py | 2 +- 19 files changed, 68 insertions(+), 47 deletions(-) diff --git a/docs/annotate.ipynb b/docs/annotate.ipynb index ecba0441d..21c8a413d 100644 --- a/docs/annotate.ipynb +++ b/docs/annotate.ipynb @@ -60,8 +60,8 @@ "source": [ "bt.settings.auto_save_parents = False # ignores ontological hierarchy\n", "ln.settings.verbosity = \"info\"\n", - "ln.transform.stem_uid = \"sU0y1kF3igep\"\n", - "ln.transform.version = \"0\"" + "ln.settings.transform.stem_uid = \"sU0y1kF3igep\"\n", + "ln.settings.transform.version = \"0\"" ] }, { diff --git a/docs/faq/key.ipynb b/docs/faq/key.ipynb index ff9ade878..2e7bc9b18 100644 --- a/docs/faq/key.ipynb +++ b/docs/faq/key.ipynb @@ -125,8 +125,8 @@ "metadata": {}, "outputs": [], "source": [ - "ln.transform.stem_uid = \"WIwaNDvlEkwS\"\n", - "ln.transform.version = \"1\"\n", + "ln.settings.transform.stem_uid = \"WIwaNDvlEkwS\"\n", + "ln.settings.transform.version = \"1\"\n", "ln.track()" ] }, diff --git a/docs/faq/setup.ipynb b/docs/faq/setup.ipynb index 65f6b0fdf..c8e96a43a 100644 --- a/docs/faq/setup.ipynb +++ b/docs/faq/setup.ipynb @@ -84,8 +84,8 @@ "metadata": {}, "outputs": [], "source": [ - "ln.transform.stem_uid = \"2lhqA4uTKSFP\"\n", - "ln.transform.version = \"0\"\n", + "ln.settings.transform.stem_uid = \"2lhqA4uTKSFP\"\n", + "ln.settings.transform.version = \"0\"\n", "ln.track()" ] }, diff --git a/docs/faq/track-run-inputs.ipynb b/docs/faq/track-run-inputs.ipynb index 20ed968cc..a182415cb 100644 --- a/docs/faq/track-run-inputs.ipynb +++ b/docs/faq/track-run-inputs.ipynb @@ -78,8 +78,8 @@ "metadata": {}, "outputs": [], "source": [ - "ln.transform.stem_uid = \"Rx2s9aPTMQLY\"\n", - "ln.transform.version = \"1\"\n", + "ln.settings.transform.stem_uid = \"Rx2s9aPTMQLY\"\n", + "ln.settings.transform.version = \"1\"\n", "ln.track()" ] }, diff --git a/docs/introduction.ipynb b/docs/introduction.ipynb index 18e052e2c..3b9465ee8 100644 --- a/docs/introduction.ipynb +++ b/docs/introduction.ipynb @@ -71,8 +71,8 @@ "ln.setup.init(schema=\"bionty\", storage=\"./lamin-intro\")\n", "\n", "# tag your code with auto-generated identifiers for a script or notebook\n", - "ln.transform.stem_uid = \"FPnfDtJz8qbE\"\n", - "ln.transform.version = \"1\"\n", + "ln.settings.transform.stem_uid = \"FPnfDtJz8qbE\"\n", + "ln.settings.transform.version = \"1\"\n", "\n", "# track the execution of a transform with a global run context\n", "ln.track()" diff --git a/docs/meta.ipynb b/docs/meta.ipynb index aecd347af..9348df71e 100644 --- a/docs/meta.ipynb +++ b/docs/meta.ipynb @@ -67,8 +67,8 @@ "metadata": {}, "outputs": [], "source": [ - "ln.transform.stem_uid = \"vldHzF3aTAiW\"\n", - "ln.transform.version = \"1\"\n", + "ln.settings.transform.stem_uid = \"vldHzF3aTAiW\"\n", + "ln.settings.transform.version = \"1\"\n", "ln.track()" ] }, diff --git a/docs/track.ipynb b/docs/track.ipynb index 05d0d7557..38e2bf052 100644 --- a/docs/track.ipynb +++ b/docs/track.ipynb @@ -67,8 +67,8 @@ "metadata": {}, "outputs": [], "source": [ - "ln.transform.stem_uid = \"9priar0hoE5u\"\n", - "ln.transform.version = \"0\"\n", + "ln.settings.transform.stem_uid = \"9priar0hoE5u\"\n", + "ln.settings.transform.version = \"0\"\n", "ln.track()" ] }, diff --git a/docs/transfer.ipynb b/docs/transfer.ipynb index 1b962e8ea..eadc571c2 100644 --- a/docs/transfer.ipynb +++ b/docs/transfer.ipynb @@ -48,8 +48,8 @@ "metadata": {}, "outputs": [], "source": [ - "ln.transform.stem_uid = \"ITeOtm7bhtdq\"\n", - "ln.transform.version = \"1\"\n", + "ln.settings.transform.stem_uid = \"ITeOtm7bhtdq\"\n", + "ln.settings.transform.version = \"1\"\n", "ln.track()" ] }, diff --git a/docs/tutorial.ipynb b/docs/tutorial.ipynb index c0a507ecf..6a92a2288 100644 --- a/docs/tutorial.ipynb +++ b/docs/tutorial.ipynb @@ -118,8 +118,8 @@ "metadata": {}, "outputs": [], "source": [ - "ln.transform.stem_uid = \"NJvdsWWbJlZS\"\n", - "ln.transform.version = \"0\"\n", + "ln.settings.transform.stem_uid = \"NJvdsWWbJlZS\"\n", + "ln.settings.transform.version = \"0\"\n", "ln.track()" ] }, diff --git a/docs/tutorial2.ipynb b/docs/tutorial2.ipynb index 7cf38b615..c6bdb1bb2 100644 --- a/docs/tutorial2.ipynb +++ b/docs/tutorial2.ipynb @@ -703,8 +703,8 @@ "metadata": {}, "outputs": [], "source": [ - "ln.transform.stem_uid = \"dMtrt8YMSdl6\"\n", - "ln.transform.version = \"1\"\n", + "ln.settings.transform.stem_uid = \"dMtrt8YMSdl6\"\n", + "ln.settings.transform.version = \"1\"\n", "\n", "ln.track()" ] diff --git a/lamindb/__init__.py b/lamindb/__init__.py index 06d0fdc0a..a2e0b0ff1 100644 --- a/lamindb/__init__.py +++ b/lamindb/__init__.py @@ -107,7 +107,7 @@ def __getattr__(name): from ._view import view from .core._run_context import run_context from .core._settings import settings - from .core._transform_settings import transform_settings as transform + from .core._transform_settings import transform # backward compat # schema modules if not _os.environ.get("LAMINDB_MULTI_INSTANCE") == "true": diff --git a/lamindb/core/_run_context.py b/lamindb/core/_run_context.py index e7b15475e..4e9cbf24c 100644 --- a/lamindb/core/_run_context.py +++ b/lamindb/core/_run_context.py @@ -16,7 +16,7 @@ from lnschema_core.types import TransformType from lnschema_core.users import current_user_id -from lamindb.core._transform_settings import transform_settings +from lamindb.core._transform_settings import transform as transform_settings from ._settings import settings from ._sync_git import get_transform_reference_from_git_repo @@ -77,7 +77,7 @@ def get_stem_uid_and_version_from_file(file_path: str) -> Tuple[str, str]: if stem_uid is None or version is None: raise SystemExit( - f"ln.transform.stem_uid and ln.transform.version aren't set in {file_path}\n" + f"ln.settings.transform.stem_uid and ln.settings.transform.version aren't set in {file_path}\n" "Call ln.track() and copy/paste the output into the notebook" ) return stem_uid, version @@ -124,7 +124,7 @@ def update_stem_uid_or_version( updated = new_version != version if updated: new_metadata = ( - f'ln.transform.stem_uid = "{new_stem_uid}"\nln.transform.version =' + f'ln.settings.transform.stem_uid = "{new_stem_uid}"\nln.settings.transform.version =' f' "{new_version}"\n' ) raise SystemExit( @@ -167,8 +167,8 @@ def get_notebook_name_colab() -> str: MESSAGE = """To track this {transform_type}, set the following two global variables: -ln.transform.stem_uid = "{stem_uid}" -ln.transform.version = "{version}" +ln.settings.transform.stem_uid = "{stem_uid}" +ln.settings.transform.version = "{version}" """ diff --git a/lamindb/core/_settings.py b/lamindb/core/_settings.py index b5797b738..4815bf83e 100644 --- a/lamindb/core/_settings.py +++ b/lamindb/core/_settings.py @@ -8,6 +8,8 @@ from lamindb_setup.core._settings_instance import sanitize_git_repo_url from upath import UPath +from ._transform_settings import TransformSettings, transform + VERBOSITY_TO_INT = { "error": 0, # 40 "warning": 1, # 30 @@ -91,6 +93,11 @@ def _storage_settings(self) -> ln_setup.dev.StorageSettings: storage_settings = ln_setup.dev.StorageSettings(root=self._using_storage) return storage_settings + @property + def transform(self) -> TransformSettings: + """Transform settings.""" + return transform + @property def sync_git_repo(self) -> Optional[str]: """Sync transforms with scripts in git repository. diff --git a/lamindb/core/_sync_git.py b/lamindb/core/_sync_git.py index f3baf4890..007da2122 100644 --- a/lamindb/core/_sync_git.py +++ b/lamindb/core/_sync_git.py @@ -9,6 +9,10 @@ from ._settings import sanitize_git_repo_url, settings +class BlobHashNotFound(SystemExit): + pass + + def get_git_repo_from_remote() -> Path: repo_url = settings.sync_git_repo repo_dir = setup_settings.storage.cache_dir / repo_url.split("/")[-1] @@ -27,14 +31,25 @@ def get_git_repo_from_remote() -> Path: return repo_dir -def check_remote_git_url_matches_setting(): +def check_local_git_repo() -> bool: result = subprocess.run( "git config --get remote.origin.url", shell=True, capture_output=True, ) - remote_url = sanitize_git_repo_url(result.stdout.decode().strip()) - assert remote_url == settings.sync_git_repo + result_str = result.stdout.decode().strip() + if result_str == "": + # running-not-in-a-git-repo + return False + else: + remote_url = sanitize_git_repo_url(result_str) + if remote_url == settings.sync_git_repo: + # running-in-correct-git-repo + return True + else: + # running-in-wrong-git-repo + logger.warning("running in wrong git repo") + return False def get_git_commit_hash( @@ -91,18 +106,19 @@ def get_filepath_within_git_repo( def get_transform_reference_from_git_repo(path: Path) -> str: blob_hash = hash_code(path).hexdigest() - commit_hash = get_git_commit_hash(blob_hash) - if commit_hash is not None: - logger.warning("found script in local repository") - check_remote_git_url_matches_setting() + commit_hash = None + if check_local_git_repo(): repo_dir = None else: repo_dir = get_git_repo_from_remote() - commit_hash = get_git_commit_hash(blob_hash, repo_dir=repo_dir) - if commit_hash is None: - raise RuntimeError( - f"Did not find blob hash {blob_hash} of {path} in git repo {repo_dir}" - ) + commit_hash = get_git_commit_hash(blob_hash, repo_dir=repo_dir) + if commit_hash is None: + if repo_dir is None: + repo_dir = Path.cwd() + raise BlobHashNotFound( + f"❌ Did not find blob hash {blob_hash} in git repo ({settings.sync_git_repo}) {repo_dir}\n" + f"Did you commit the script? -> {path}" + ) gitpath = get_filepath_within_git_repo(commit_hash, blob_hash, repo_dir) reference = f"{settings.sync_git_repo}/blob/{commit_hash}/{gitpath}" return reference diff --git a/lamindb/core/_transform_settings.py b/lamindb/core/_transform_settings.py index 3048336cd..8ae64703c 100644 --- a/lamindb/core/_transform_settings.py +++ b/lamindb/core/_transform_settings.py @@ -1,11 +1,9 @@ from typing import Optional -# user-facing name -class transform: +class TransformSettings: stem_uid: Optional[None] = None version: Optional[None] = None -# internal name -transform_settings = transform +transform = TransformSettings() diff --git a/sub/lamin-cli b/sub/lamin-cli index 0083923e1..47f89a9e2 160000 --- a/sub/lamin-cli +++ b/sub/lamin-cli @@ -1 +1 @@ -Subproject commit 0083923e1395528fde6f96b2db9e976791764e51 +Subproject commit 47f89a9e2d7e7814cae75951c2761c329ec95935 diff --git a/sub/lamindb-setup b/sub/lamindb-setup index d2863984b..0dad909c9 160000 --- a/sub/lamindb-setup +++ b/sub/lamindb-setup @@ -1 +1 @@ -Subproject commit d2863984b3e5365bbbcea8daba2fd97ce3c7e862 +Subproject commit 0dad909c91a75108909f23424b57829a01cf2be0 diff --git a/tests/notebooks/no-title.ipynb b/tests/notebooks/no-title.ipynb index 17bcde4e6..60c090a79 100644 --- a/tests/notebooks/no-title.ipynb +++ b/tests/notebooks/no-title.ipynb @@ -27,8 +27,8 @@ "metadata": {}, "outputs": [], "source": [ - "ln.transform.stem_uid = \"Irn3xQyQ40GU\"\n", - "ln.transform.version = \"1\"\n", + "ln.settings.transform.stem_uid = \"Irn3xQyQ40GU\"\n", + "ln.settings.transform.version = \"1\"\n", "\n", "with pytest.raises(NoTitleError):\n", " ln.track()" diff --git a/tests/test_run_context.py b/tests/test_run_context.py index 41c1f2bf7..e302164e7 100644 --- a/tests/test_run_context.py +++ b/tests/test_run_context.py @@ -103,6 +103,6 @@ def test_sync_git_repo(): transform = ln.Transform.filter(name="initialized.py").one() assert ( transform.reference - == "https://github.com/laminlabs/lamin-cli/blob/39fb29b1b3ccc891a025b5a631d6294413b6ee45/tests/scripts/initialized.py" + == "https://github.com/laminlabs/lamin-cli/blob/4244dadaf8e189549d614c54d49c78e35b00da85/tests/scripts/initialized.py" ) assert transform.reference_type == "url"