From 3e75bcdd1b8f6a559a1f4efb53b94a92e20af315 Mon Sep 17 00:00:00 2001 From: anders-albert Date: Wed, 1 Jan 2025 13:11:09 +0100 Subject: [PATCH 01/33] test: extend testing --- .../locations/data_model_based.LocationFilter.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/data/complete_org/modules/my_example_module/locations/data_model_based.LocationFilter.yaml b/tests/data/complete_org/modules/my_example_module/locations/data_model_based.LocationFilter.yaml index 83575a487..49774274a 100644 --- a/tests/data/complete_org/modules/my_example_module/locations/data_model_based.LocationFilter.yaml +++ b/tests/data/complete_org/modules/my_example_module/locations/data_model_based.LocationFilter.yaml @@ -5,3 +5,6 @@ dataModels: - space: DataModelSpace externalId: DataModelLocation version: v1 +assetCentric: + dataSetExternalIds: + - "" From 3518dcd632047ed1368d10a09db4bac8aa34737d Mon Sep 17 00:00:00 2001 From: anders-albert Date: Wed, 1 Jan 2025 13:22:16 +0100 Subject: [PATCH 02/33] fix: implemented emtpy --- cognite_toolkit/_cdf_tk/client/api/lookup.py | 33 +++++++++++++------ .../_resource_loaders/location_loaders.py | 8 +++-- 2 files changed, 29 insertions(+), 12 deletions(-) diff --git a/cognite_toolkit/_cdf_tk/client/api/lookup.py b/cognite_toolkit/_cdf_tk/client/api/lookup.py index 90cdf40f2..db928752e 100644 --- a/cognite_toolkit/_cdf_tk/client/api/lookup.py +++ b/cognite_toolkit/_cdf_tk/client/api/lookup.py @@ -36,14 +36,22 @@ def resource_name(self) -> str: return type(self).__name__.removesuffix("LookUpAPI") @overload - def id(self, external_id: str, is_dry_run: bool = False) -> int: ... + def id(self, external_id: str, is_dry_run: bool = False, allow_empty: bool = False) -> int: ... @overload - def id(self, external_id: SequenceNotStr[str], is_dry_run: bool = False) -> list[int]: ... + def id( + self, external_id: SequenceNotStr[str], is_dry_run: bool = False, allow_empty: bool = False + ) -> list[int]: ... - def id(self, external_id: str | SequenceNotStr[str], is_dry_run: bool = False) -> int | list[int]: + def id( + self, external_id: str | SequenceNotStr[str], is_dry_run: bool = False, allow_empty: bool = False + ) -> int | list[int]: ids = [external_id] if isinstance(external_id, str) else external_id missing = [id for id in ids if id not in self._cache] + if allow_empty and "" in missing: + # Note we do not want to put empty string in the cache. It is a special case that + # as of 01/02/2025 only applies to LocationFilters + missing.remove("") if missing: try: lookup = self._id(missing) @@ -63,14 +71,19 @@ def id(self, external_id: str | SequenceNotStr[str], is_dry_run: bool = False) - raise ResourceRetrievalError( f"Failed to retrieve {self.resource_name} with external_id {missing}." "Have you created it?" ) - if is_dry_run: - return ( - self._cache.get(external_id, self.dry_run_id) - if isinstance(external_id, str) - else [self._cache.get(id, self.dry_run_id) for id in ids] - ) + return ( + self._get_id_from_cache(external_id, is_dry_run, allow_empty) + if isinstance(external_id, str) + else [self._get_id_from_cache(id, is_dry_run, allow_empty) for id in ids] + ) - return self._cache[external_id] if isinstance(external_id, str) else [self._cache[id] for id in ids] + def _get_id_from_cache(self, external_id: str, is_dry_run: bool = False, allow_empty: bool = False) -> int: + if allow_empty and external_id == "": + return 0 + elif is_dry_run: + return self._cache.get(external_id, self.dry_run_id) + else: + return self._cache[external_id] @overload def external_id(self, id: int) -> str: ... diff --git a/cognite_toolkit/_cdf_tk/loaders/_resource_loaders/location_loaders.py b/cognite_toolkit/_cdf_tk/loaders/_resource_loaders/location_loaders.py index d1970f894..c155abf87 100644 --- a/cognite_toolkit/_cdf_tk/loaders/_resource_loaders/location_loaders.py +++ b/cognite_toolkit/_cdf_tk/loaders/_resource_loaders/location_loaders.py @@ -97,12 +97,16 @@ def load_resource(self, resource: dict[str, Any], is_dry_run: bool = False) -> L return LocationFilterWrite._load(resource) asset_centric = resource["assetCentric"] if data_set_external_ids := asset_centric.pop("dataSetExternalIds", None): - asset_centric["dataSetIds"] = self.client.lookup.data_sets.id(data_set_external_ids, is_dry_run) + asset_centric["dataSetIds"] = self.client.lookup.data_sets.id( + data_set_external_ids, is_dry_run, allow_empty=True + ) for subfilter_name in self.subfilter_names: subfilter = asset_centric.get(subfilter_name, {}) if data_set_external_ids := subfilter.pop("dataSetExternalIds", []): asset_centric[subfilter_name]["dataSetIds"] = self.client.lookup.data_sets.id( - data_set_external_ids, is_dry_run + data_set_external_ids, + is_dry_run, + allow_empty=True, ) return LocationFilterWrite._load(resource) From 3bc4d0cc17079d6627278182174acaa02d80bcfa Mon Sep 17 00:00:00 2001 From: anders-albert Date: Wed, 1 Jan 2025 13:30:41 +0100 Subject: [PATCH 03/33] tests: updated mocking --- tests/test_unit/approval_client/client.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/tests/test_unit/approval_client/client.py b/tests/test_unit/approval_client/client.py index 05a7c0e1a..54c65676c 100644 --- a/tests/test_unit/approval_client/client.py +++ b/tests/test_unit/approval_client/client.py @@ -86,7 +86,9 @@ def __init__(self): self._reverse_cache: dict[int, str] = {} @staticmethod - def _create_id(string: str) -> int: + def _create_id(string: str, allow_empty: bool) -> int: + if allow_empty and string == "": + return 0 # This simulates CDF setting the internal ID. # By using hashing, we will always get the same ID for the same string. # Thus, the ID will be consistent between runs and can be used in snapshots. @@ -95,15 +97,17 @@ def _create_id(string: str) -> int: hash_int = int(hex_dig[:16], 16) return hash_int - def id(self, external_id: str | SequenceNotStr[str], is_dry_run: bool = False) -> int | list[int]: + def id( + self, external_id: str | SequenceNotStr[str], is_dry_run: bool = False, allow_empty: bool = False + ) -> int | list[int]: if isinstance(external_id, str): - id_ = self._create_id(external_id) + id_ = self._create_id(external_id, allow_empty) if id_ not in self._reverse_cache: self._reverse_cache[id_] = external_id return id_ output: list[int] = [] for ext_id in external_id: - id_ = self._create_id(ext_id) + id_ = self._create_id(ext_id, allow_empty) if id_ not in self._reverse_cache: self._reverse_cache[id_] = ext_id output.append(id_) From 84e9999100d772ea6d027d0bc75fd5ad42c3e44a Mon Sep 17 00:00:00 2001 From: anders-albert Date: Wed, 1 Jan 2025 13:33:28 +0100 Subject: [PATCH 04/33] fix: quote version key in location filter --- cognite_toolkit/_cdf_tk/commands/build.py | 7 ++++++- .../loaders/_resource_loaders/location_loaders.py | 11 ++++++++++- .../locations/data_model_based.LocationFilter.yaml | 2 +- 3 files changed, 17 insertions(+), 3 deletions(-) diff --git a/cognite_toolkit/_cdf_tk/commands/build.py b/cognite_toolkit/_cdf_tk/commands/build.py index c40c049d9..41b3179f7 100644 --- a/cognite_toolkit/_cdf_tk/commands/build.py +++ b/cognite_toolkit/_cdf_tk/commands/build.py @@ -55,6 +55,7 @@ DataModelLoader, ExtractionPipelineConfigLoader, FileLoader, + LocationFilterLoader, NodeLoader, RawDatabaseLoader, RawTableLoader, @@ -455,7 +456,11 @@ def _replace_variables( source_files.append(BuildSourceFile(source, content, None)) continue - if resource_name in {TransformationLoader.folder_name, DataModelLoader.folder_name}: + if resource_name in { + TransformationLoader.folder_name, + DataModelLoader.folder_name, + LocationFilterLoader.folder_name, + }: # Ensure that all keys that are version gets read as strings. # This is required by DataModels, Views, and Transformations that reference DataModels and Views. content = quote_int_value_by_key_in_yaml(content, key="version") diff --git a/cognite_toolkit/_cdf_tk/loaders/_resource_loaders/location_loaders.py b/cognite_toolkit/_cdf_tk/loaders/_resource_loaders/location_loaders.py index c155abf87..29cd61809 100644 --- a/cognite_toolkit/_cdf_tk/loaders/_resource_loaders/location_loaders.py +++ b/cognite_toolkit/_cdf_tk/loaders/_resource_loaders/location_loaders.py @@ -2,6 +2,7 @@ from collections.abc import Hashable, Iterable, Sequence from functools import lru_cache +from pathlib import Path from typing import Any, final from cognite.client.data_classes.capabilities import Capability, LocationFiltersAcl @@ -16,7 +17,7 @@ LocationFilterWriteList, ) from cognite_toolkit._cdf_tk.loaders._base_loaders import ResourceLoader -from cognite_toolkit._cdf_tk.utils import in_dict +from cognite_toolkit._cdf_tk.utils import in_dict, quote_int_value_by_key_in_yaml, safe_read from cognite_toolkit._cdf_tk.utils.diff_list import diff_list_hashable, diff_list_identifiable, dm_identifier from .classic_loaders import AssetLoader, SequenceLoader @@ -90,6 +91,14 @@ def get_id(cls, item: LocationFilter | LocationFilterWrite | dict) -> str: def dump_id(cls, id: str) -> dict[str, Any]: return {"externalId": id} + def safe_read(self, filepath: Path | str) -> str: + # The version is a string, but the user often writes it as an int. + # YAML will then parse it as an int, for example, `3_0_2` will be parsed as `302`. + # This is technically a user mistake, as you should quote the version in the YAML file. + # However, we do not want to put this burden on the user (knowing the intricate workings of YAML), + # so we fix it here. + return quote_int_value_by_key_in_yaml(safe_read(filepath), key="version") + def load_resource(self, resource: dict[str, Any], is_dry_run: bool = False) -> LocationFilterWrite: if parent_external_id := resource.pop("parentExternalId", None): resource["parentId"] = self.client.lookup.location_filters.id(parent_external_id, is_dry_run) diff --git a/tests/data/complete_org/modules/my_example_module/locations/data_model_based.LocationFilter.yaml b/tests/data/complete_org/modules/my_example_module/locations/data_model_based.LocationFilter.yaml index 49774274a..780760557 100644 --- a/tests/data/complete_org/modules/my_example_module/locations/data_model_based.LocationFilter.yaml +++ b/tests/data/complete_org/modules/my_example_module/locations/data_model_based.LocationFilter.yaml @@ -4,7 +4,7 @@ description: LocationFilter for DataModel dataModels: - space: DataModelSpace externalId: DataModelLocation - version: v1 + version: 1_0_0 assetCentric: dataSetExternalIds: - "" From ca9c216d4536a3062601793535f1e6e591607949 Mon Sep 17 00:00:00 2001 From: anders-albert Date: Wed, 1 Jan 2025 13:38:38 +0100 Subject: [PATCH 05/33] feat: added data modeling type --- .../_cdf_tk/client/data_classes/location_filters.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cognite_toolkit/_cdf_tk/client/data_classes/location_filters.py b/cognite_toolkit/_cdf_tk/client/data_classes/location_filters.py index f24f4f1b4..9042b74e5 100644 --- a/cognite_toolkit/_cdf_tk/client/data_classes/location_filters.py +++ b/cognite_toolkit/_cdf_tk/client/data_classes/location_filters.py @@ -242,6 +242,7 @@ def __init__( scene: LocationFilterScene | None = None, asset_centric: AssetCentricFilter | None = None, views: list[LocationFilterView] | None = None, + data_modeling_type: Literal["HYBRID", "DATA_MODELING_ONLY"] | None = None, ) -> None: super().__init__( external_id, name, parent_id, description, data_models, instance_spaces, scene, asset_centric, views @@ -249,6 +250,8 @@ def __init__( self.id = id self.created_time = created_time self.updated_time = updated_time + # Inferred on the server side based on the filter. + self.data_modeling_type = data_modeling_type @classmethod def _load(cls, resource: dict[str, Any], cognite_client: CogniteClient | None = None) -> Self: @@ -267,6 +270,7 @@ def _load(cls, resource: dict[str, Any], cognite_client: CogniteClient | None = views=[LocationFilterView._load(view) for view in resource["views"]] if "views" in resource else None, created_time=resource["createdTime"], updated_time=resource["lastUpdatedTime"], + data_modeling_type=resource.get("dataModelingType"), ) From 59038a6bbb9815f078c6fa248ca1780974d2bac8 Mon Sep 17 00:00:00 2001 From: anders-albert Date: Wed, 1 Jan 2025 13:40:06 +0100 Subject: [PATCH 06/33] tests: updated test --- tests/test_integration/test_toolkit_client/test_locations.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_integration/test_toolkit_client/test_locations.py b/tests/test_integration/test_toolkit_client/test_locations.py index 05c5551ef..df1242669 100644 --- a/tests/test_integration/test_toolkit_client/test_locations.py +++ b/tests/test_integration/test_toolkit_client/test_locations.py @@ -64,6 +64,7 @@ def test_create_retrieve_delete(self, toolkit_client: ToolkitClient, my_data_mod created = toolkit_client.location_filters.create(location_filter) assert isinstance(created, LocationFilter) assert created.as_write().dump() == location_filter.dump() + assert created.data_modeling_type is not None retrieved = toolkit_client.location_filters.retrieve(created.id) assert isinstance(retrieved, LocationFilter) From d5349d2c68de5ac2223f9225bb664e3486879158 Mon Sep 17 00:00:00 2001 From: anders-albert Date: Wed, 1 Jan 2025 13:43:09 +0100 Subject: [PATCH 07/33] tests: regen --- .../test_cli/test_build_deploy_snapshots/complete_org.yaml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/test_unit/test_cli/test_build_deploy_snapshots/complete_org.yaml b/tests/test_unit/test_cli/test_build_deploy_snapshots/complete_org.yaml index 342709d87..a467561a6 100644 --- a/tests/test_unit/test_cli/test_build_deploy_snapshots/complete_org.yaml +++ b/tests/test_unit/test_cli/test_build_deploy_snapshots/complete_org.yaml @@ -257,10 +257,13 @@ Location: externalId: waterTreatmentPlant1_Windows_3_11_8 name: Water treatment plant LocationFilter: -- dataModels: +- assetCentric: + dataSetIds: + - 0 + dataModels: - externalId: DataModelLocation space: DataModelSpace - version: v1 + version: '1_0_0' description: LocationFilter for DataModel externalId: DataModelLocationFilter name: LocationFilter for DataModel From 651497b72268ab1eace75af0dea693931a6de440 Mon Sep 17 00:00:00 2001 From: anders-albert Date: Wed, 1 Jan 2025 13:44:46 +0100 Subject: [PATCH 08/33] feat: support data modeling type --- .../client/data_classes/location_filters.py | 17 ++++++++++++++--- .../test_toolkit_client/test_locations.py | 5 ++++- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/cognite_toolkit/_cdf_tk/client/data_classes/location_filters.py b/cognite_toolkit/_cdf_tk/client/data_classes/location_filters.py index 9042b74e5..2be0b82cb 100644 --- a/cognite_toolkit/_cdf_tk/client/data_classes/location_filters.py +++ b/cognite_toolkit/_cdf_tk/client/data_classes/location_filters.py @@ -148,6 +148,7 @@ def __init__( scene: LocationFilterScene | None = None, asset_centric: AssetCentricFilter | None = None, views: list[LocationFilterView] | None = None, + data_modeling_type: Literal["HYBRID", "DATA_MODELING_ONLY"] | None = None, ) -> None: self.external_id = external_id self.name = name @@ -158,6 +159,7 @@ def __init__( self.scene = scene self.asset_centric = asset_centric self.views = views + self.data_modeling_type = data_modeling_type def as_write(self) -> LocationFilterWrite: return LocationFilterWrite( @@ -170,6 +172,7 @@ def as_write(self) -> LocationFilterWrite: scene=self.scene, asset_centric=self.asset_centric, views=self.views, + data_modeling_type=self.data_modeling_type, ) def dump(self, camel_case: bool = True) -> dict[str, Any]: @@ -208,6 +211,7 @@ def _load(cls, resource: dict[str, Any], cognite_client: CogniteClient | None = scene=scene, asset_centric=asset_centric, views=views, + data_modeling_type=resource.get("dataModelingType"), ) @@ -245,13 +249,20 @@ def __init__( data_modeling_type: Literal["HYBRID", "DATA_MODELING_ONLY"] | None = None, ) -> None: super().__init__( - external_id, name, parent_id, description, data_models, instance_spaces, scene, asset_centric, views + external_id, + name, + parent_id, + description, + data_models, + instance_spaces, + scene, + asset_centric, + views, + data_modeling_type, ) self.id = id self.created_time = created_time self.updated_time = updated_time - # Inferred on the server side based on the filter. - self.data_modeling_type = data_modeling_type @classmethod def _load(cls, resource: dict[str, Any], cognite_client: CogniteClient | None = None) -> Self: diff --git a/tests/test_integration/test_toolkit_client/test_locations.py b/tests/test_integration/test_toolkit_client/test_locations.py index df1242669..c1d03ca3c 100644 --- a/tests/test_integration/test_toolkit_client/test_locations.py +++ b/tests/test_integration/test_toolkit_client/test_locations.py @@ -57,7 +57,10 @@ def my_data_model(toolkit_client: ToolkitClient, toolkit_space: Space) -> DataMo class TestLocationFilterAPI: def test_create_retrieve_delete(self, toolkit_client: ToolkitClient, my_data_model: DataModel) -> None: location_filter = LocationFilterWrite( - name="loc", external_id=SESSION_EXTERNAL_ID, data_models=[my_data_model.as_id()] + name="loc", + external_id=SESSION_EXTERNAL_ID, + data_models=[my_data_model.as_id()], + data_modeling_type="DATA_MODELING_ONLY", ) created: LocationFilter | None = None try: From faa738a0adf054deaf948d191568d0b7766e363b Mon Sep 17 00:00:00 2001 From: anders-albert Date: Wed, 1 Jan 2025 13:46:32 +0100 Subject: [PATCH 09/33] tests: update mock lookup --- tests/test_unit/approval_client/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_unit/approval_client/client.py b/tests/test_unit/approval_client/client.py index 54c65676c..7ac8e25a9 100644 --- a/tests/test_unit/approval_client/client.py +++ b/tests/test_unit/approval_client/client.py @@ -86,7 +86,7 @@ def __init__(self): self._reverse_cache: dict[int, str] = {} @staticmethod - def _create_id(string: str, allow_empty: bool) -> int: + def _create_id(string: str, allow_empty: bool = False) -> int: if allow_empty and string == "": return 0 # This simulates CDF setting the internal ID. From e651803cb0c1dc8dd30780901ed852f5e729cb7b Mon Sep 17 00:00:00 2001 From: anders-albert Date: Wed, 1 Jan 2025 13:48:41 +0100 Subject: [PATCH 10/33] build: changelog --- CHANGELOG.cdf-tk.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.cdf-tk.md b/CHANGELOG.cdf-tk.md index a266b0451..9cbd64cf4 100644 --- a/CHANGELOG.cdf-tk.md +++ b/CHANGELOG.cdf-tk.md @@ -23,6 +23,7 @@ Changes are grouped as follows: with the flag `--offline`. - [alpha feature] New subcommand `cdf modules pull` which pulls the configuration from the CDF project to the local modules directory. +- Support for property `dataModelingType` in `LocationFilter` resources. ### Fixed @@ -31,6 +32,8 @@ Changes are grouped as follows: - Cognite Toolkit has improved resources that have server set default values that can lead to redeploy even when unchanged. This includes `Sequences`, `Containers`, `DataSets`, `Views`, `Nodes`, `Edges`, `ExtractionPipelines`, `CogniteFiles`, `HostedExtractorJobs`, `Relationships`, `RobotMaps`, and `WorkflowVersions`. +- `LocationFilters` now parses the `version` key of `View` and `DataModel` correctly as a string. +- `LocationFilters` now converts an empty string of `dataSetExternalId` to `0` instead of ignoring it. ## [0.3.23] - 2024-12-13 From 23dfbdeddd822b71d65b446ed56d31c50ea42070 Mon Sep 17 00:00:00 2001 From: anders-albert Date: Wed, 1 Jan 2025 14:03:49 +0100 Subject: [PATCH 11/33] fix: introduced bugs --- cognite_toolkit/_cdf_tk/client/api/lookup.py | 14 +++++++++++++- .../loaders/_resource_loaders/location_loaders.py | 3 +++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/cognite_toolkit/_cdf_tk/client/api/lookup.py b/cognite_toolkit/_cdf_tk/client/api/lookup.py index db928752e..e1cd06abe 100644 --- a/cognite_toolkit/_cdf_tk/client/api/lookup.py +++ b/cognite_toolkit/_cdf_tk/client/api/lookup.py @@ -97,6 +97,8 @@ def external_id( ) -> str | list[str]: ids = [id] if isinstance(id, int) else id missing = [id_ for id_ in ids if id_ not in self._reverse_cache] + if 0 in missing: + missing.remove(0) if missing: try: lookup = self._external_id(missing) @@ -116,7 +118,17 @@ def external_id( raise ResourceRetrievalError( f"Failed to retrieve {self.resource_name} with id {missing}." "Have you created it?" ) - return self._reverse_cache[id] if isinstance(id, int) else [self._reverse_cache[id] for id in ids] + return ( + self._get_external_id_from_cache(id) + if isinstance(id, int) + else [self._get_external_id_from_cache(id) for id in ids] + ) + + def _get_external_id_from_cache(self, id: int) -> str: + if id == 0: + # Reverse of looking up an empty string. + return "" + return self._reverse_cache[id] @abstractmethod def _id(self, external_id: SequenceNotStr[str]) -> dict[str, int]: diff --git a/cognite_toolkit/_cdf_tk/loaders/_resource_loaders/location_loaders.py b/cognite_toolkit/_cdf_tk/loaders/_resource_loaders/location_loaders.py index 29cd61809..21e0ae2b3 100644 --- a/cognite_toolkit/_cdf_tk/loaders/_resource_loaders/location_loaders.py +++ b/cognite_toolkit/_cdf_tk/loaders/_resource_loaders/location_loaders.py @@ -135,6 +135,9 @@ def dump_resource(self, resource: LocationFilter, local: dict[str, Any]) -> dict asset_centric[subfilter_name]["dataSetExternalIds"] = self.client.lookup.data_sets.external_id( data_set_ids ) + if "dataModelingType" in dumped and "dataModelingType" not in dumped: + # Default set on server side + dumped.pop("dataModelingType") return dumped def diff_list( From 84d0958f9faa8afb37b5d2517e6b6bc85d5698af Mon Sep 17 00:00:00 2001 From: anders-albert Date: Wed, 1 Jan 2025 14:05:25 +0100 Subject: [PATCH 12/33] fix; bug bug --- .../_cdf_tk/loaders/_resource_loaders/location_loaders.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cognite_toolkit/_cdf_tk/loaders/_resource_loaders/location_loaders.py b/cognite_toolkit/_cdf_tk/loaders/_resource_loaders/location_loaders.py index 21e0ae2b3..8965e74b7 100644 --- a/cognite_toolkit/_cdf_tk/loaders/_resource_loaders/location_loaders.py +++ b/cognite_toolkit/_cdf_tk/loaders/_resource_loaders/location_loaders.py @@ -135,7 +135,7 @@ def dump_resource(self, resource: LocationFilter, local: dict[str, Any]) -> dict asset_centric[subfilter_name]["dataSetExternalIds"] = self.client.lookup.data_sets.external_id( data_set_ids ) - if "dataModelingType" in dumped and "dataModelingType" not in dumped: + if "dataModelingType" in dumped and "dataModelingType" not in local: # Default set on server side dumped.pop("dataModelingType") return dumped From 9cbd3be56b82b134d82d5a65844280bd332fb490 Mon Sep 17 00:00:00 2001 From: anders-albert Date: Wed, 1 Jan 2025 14:08:08 +0100 Subject: [PATCH 13/33] fix; bug bug bug --- .../_cdf_tk/loaders/_resource_loaders/location_loaders.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cognite_toolkit/_cdf_tk/loaders/_resource_loaders/location_loaders.py b/cognite_toolkit/_cdf_tk/loaders/_resource_loaders/location_loaders.py index 8965e74b7..dd69cf6b2 100644 --- a/cognite_toolkit/_cdf_tk/loaders/_resource_loaders/location_loaders.py +++ b/cognite_toolkit/_cdf_tk/loaders/_resource_loaders/location_loaders.py @@ -124,6 +124,9 @@ def dump_resource(self, resource: LocationFilter, local: dict[str, Any]) -> dict dumped = resource.as_write().dump() if parent_id := dumped.pop("parentId", None): dumped["parentExternalId"] = self.client.lookup.location_filters.external_id(parent_id) + if "dataModelingType" in dumped and "dataModelingType" not in local: + # Default set on server side + dumped.pop("dataModelingType") if "assetCentric" not in dumped: return dumped asset_centric = dumped["assetCentric"] @@ -135,9 +138,6 @@ def dump_resource(self, resource: LocationFilter, local: dict[str, Any]) -> dict asset_centric[subfilter_name]["dataSetExternalIds"] = self.client.lookup.data_sets.external_id( data_set_ids ) - if "dataModelingType" in dumped and "dataModelingType" not in local: - # Default set on server side - dumped.pop("dataModelingType") return dumped def diff_list( From 0d02410b7a1fa08b352da4ad8e65f63222a0563a Mon Sep 17 00:00:00 2001 From: anders-albert Date: Wed, 1 Jan 2025 14:12:27 +0100 Subject: [PATCH 14/33] tests: standardize --- .../locations/data_model_based.LocationFilter.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/data/complete_org/modules/my_example_module/locations/data_model_based.LocationFilter.yaml b/tests/data/complete_org/modules/my_example_module/locations/data_model_based.LocationFilter.yaml index 780760557..acf1e1231 100644 --- a/tests/data/complete_org/modules/my_example_module/locations/data_model_based.LocationFilter.yaml +++ b/tests/data/complete_org/modules/my_example_module/locations/data_model_based.LocationFilter.yaml @@ -4,7 +4,7 @@ description: LocationFilter for DataModel dataModels: - space: DataModelSpace externalId: DataModelLocation - version: 1_0_0 + version: '1_0_0' assetCentric: dataSetExternalIds: - - "" + - '' From 5530e993ef2568eabad69fa509086a07bf4821dd Mon Sep 17 00:00:00 2001 From: anders-albert Date: Wed, 1 Jan 2025 14:12:41 +0100 Subject: [PATCH 15/33] tests: standardize 2 --- .../locations/data_model_based.LocationFilter.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/data/complete_org/modules/my_example_module/locations/data_model_based.LocationFilter.yaml b/tests/data/complete_org/modules/my_example_module/locations/data_model_based.LocationFilter.yaml index acf1e1231..2a355fef8 100644 --- a/tests/data/complete_org/modules/my_example_module/locations/data_model_based.LocationFilter.yaml +++ b/tests/data/complete_org/modules/my_example_module/locations/data_model_based.LocationFilter.yaml @@ -7,4 +7,4 @@ dataModels: version: '1_0_0' assetCentric: dataSetExternalIds: - - '' + - '' From ce5129abfbdf47381f9c5e781e43758c1507a07f Mon Sep 17 00:00:00 2001 From: anders-albert Date: Wed, 18 Dec 2024 11:50:18 +0100 Subject: [PATCH 16/33] tests: set cpu and memory outside limits --- .../modules/my_example_module/functions/first.function.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/data/complete_org/modules/my_example_module/functions/first.function.yaml b/tests/data/complete_org/modules/my_example_module/functions/first.function.yaml index 8fb4a6acd..96940d6b7 100644 --- a/tests/data/complete_org/modules/my_example_module/functions/first.function.yaml +++ b/tests/data/complete_org/modules/my_example_module/functions/first.function.yaml @@ -20,4 +20,7 @@ functionPath: './src/handler.py' # Data set id for the zip file with the code that is uploaded. dataSetExternalId: 'ds_timeseries_{{example_variable}}' - indexUrl: https://pypi.org/simple \ No newline at end of file + indexUrl: https://pypi.org/simple + # The below are above the function limit + cpu: 10.0 + memory: 10.0 From 4413d27ffaec68b9cef0ba6b3f67814d875ee072 Mon Sep 17 00:00:00 2001 From: anders-albert Date: Wed, 18 Dec 2024 13:35:53 +0100 Subject: [PATCH 17/33] tests: try to recreate issue --- .../modules/my_example_module/functions/first.function.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/data/complete_org/modules/my_example_module/functions/first.function.yaml b/tests/data/complete_org/modules/my_example_module/functions/first.function.yaml index 96940d6b7..b9cb143e2 100644 --- a/tests/data/complete_org/modules/my_example_module/functions/first.function.yaml +++ b/tests/data/complete_org/modules/my_example_module/functions/first.function.yaml @@ -22,5 +22,5 @@ dataSetExternalId: 'ds_timeseries_{{example_variable}}' indexUrl: https://pypi.org/simple # The below are above the function limit - cpu: 10.0 - memory: 10.0 + cpu: 0.6 + memory: 0.6 From 9e241a499170b5189a1ea774c602728255942fc3 Mon Sep 17 00:00:00 2001 From: anders-albert Date: Mon, 30 Dec 2024 22:45:36 +0100 Subject: [PATCH 18/33] docs: documented case --- .../modules/my_example_module/functions/first.function.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/data/complete_org/modules/my_example_module/functions/first.function.yaml b/tests/data/complete_org/modules/my_example_module/functions/first.function.yaml index b9cb143e2..ce3e52f40 100644 --- a/tests/data/complete_org/modules/my_example_module/functions/first.function.yaml +++ b/tests/data/complete_org/modules/my_example_module/functions/first.function.yaml @@ -21,6 +21,6 @@ # Data set id for the zip file with the code that is uploaded. dataSetExternalId: 'ds_timeseries_{{example_variable}}' indexUrl: https://pypi.org/simple - # The below are above the function limit + # The cpu and memory limit are automatically increased to the default on an Azure or AWS cloud. cpu: 0.6 memory: 0.6 From c97e2d366839efd2a894c68b14b19f859a179ff4 Mon Sep 17 00:00:00 2001 From: anders-albert Date: Mon, 30 Dec 2024 22:58:13 +0100 Subject: [PATCH 19/33] fix: redeployment issue --- CHANGELOG.cdf-tk.md | 2 ++ .../_cdf_tk/loaders/_resource_loaders/function_loaders.py | 7 ++++++- .../test_cli/test_build_deploy_snapshots/complete_org.yaml | 4 +++- 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.cdf-tk.md b/CHANGELOG.cdf-tk.md index 9cbd64cf4..6752b615d 100644 --- a/CHANGELOG.cdf-tk.md +++ b/CHANGELOG.cdf-tk.md @@ -32,6 +32,8 @@ Changes are grouped as follows: - Cognite Toolkit has improved resources that have server set default values that can lead to redeploy even when unchanged. This includes `Sequences`, `Containers`, `DataSets`, `Views`, `Nodes`, `Edges`, `ExtractionPipelines`, `CogniteFiles`, `HostedExtractorJobs`, `Relationships`, `RobotMaps`, and `WorkflowVersions`. +- On CDF deployed on Azure and AWS clouds, setting the `CPU` and `memory` of a CogniteFiles to lower than + the required value no longer triggers a redeploy. - `LocationFilters` now parses the `version` key of `View` and `DataModel` correctly as a string. - `LocationFilters` now converts an empty string of `dataSetExternalId` to `0` instead of ignoring it. diff --git a/cognite_toolkit/_cdf_tk/loaders/_resource_loaders/function_loaders.py b/cognite_toolkit/_cdf_tk/loaders/_resource_loaders/function_loaders.py index df53294ad..7bc2724c9 100644 --- a/cognite_toolkit/_cdf_tk/loaders/_resource_loaders/function_loaders.py +++ b/cognite_toolkit/_cdf_tk/loaders/_resource_loaders/function_loaders.py @@ -153,6 +153,10 @@ def dump_resource(self, resource: Function, local: dict[str, Any]) -> dict[str, if key not in local: # Server set default values dumped.pop(key, None) + elif isinstance(local.get(key), float) and local[key] < dumped[key]: + # On Azure and AWS, the server sets the CPU and Memory to the default values, if the user + # pass in a lower value. This should not trigger a redeploy. + dumped[key] = local[key] for key in ["indexUrl", "extraIndexUrls"]: # Only in write (request) format of the function if key in local: @@ -224,7 +228,8 @@ def create(self, items: FunctionWriteList) -> FunctionList: else: raise RuntimeError("Could not retrieve file from files API") item.file_id = file_id - created.append(self.client.functions.create(item)) + created_item = self.client.functions.create(item) + created.append(created_item) return created def retrieve(self, ids: SequenceNotStr[str]) -> FunctionList: diff --git a/tests/test_unit/test_cli/test_build_deploy_snapshots/complete_org.yaml b/tests/test_unit/test_cli/test_build_deploy_snapshots/complete_org.yaml index a467561a6..f12b18e91 100644 --- a/tests/test_unit/test_cli/test_build_deploy_snapshots/complete_org.yaml +++ b/tests/test_unit/test_cli/test_build_deploy_snapshots/complete_org.yaml @@ -162,7 +162,8 @@ Frame: - externalId: rootCoordinateFrame name: Root coordinate frame Function: -- description: Returns the input data, secrets, and function info. +- cpu: 0.6 + description: Returns the input data, secrets, and function info. envVars: CDF_ENV: dev ENV_TYPE: dev @@ -170,6 +171,7 @@ Function: fileId: -1 functionPath: ./src/handler.py indexUrl: https://pypi.org/simple + memory: 0.6 metadata: cdf-toolkit-function-hash: a63720560f48030aaf994e92b02b1de1932514aa79b3115ced9d65fba8440aff cdf-toolkit-secret-hash: d218376a8813e2bb96b70b9f3df07157b0b58f18f3667c64e500032095f4f2cb0f1679ed7034189009131ce2e77cc363d840b478d94d8525cad7d9830a78e129 From 710e03b9f421b735adb11e10a080809c26018dfb Mon Sep 17 00:00:00 2001 From: anders-albert Date: Mon, 30 Dec 2024 23:07:52 +0100 Subject: [PATCH 20/33] feat: added warning on mistake --- .../_resource_loaders/function_loaders.py | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/cognite_toolkit/_cdf_tk/loaders/_resource_loaders/function_loaders.py b/cognite_toolkit/_cdf_tk/loaders/_resource_loaders/function_loaders.py index 7bc2724c9..b46d9d564 100644 --- a/cognite_toolkit/_cdf_tk/loaders/_resource_loaders/function_loaders.py +++ b/cognite_toolkit/_cdf_tk/loaders/_resource_loaders/function_loaders.py @@ -229,9 +229,35 @@ def create(self, items: FunctionWriteList) -> FunctionList: raise RuntimeError("Could not retrieve file from files API") item.file_id = file_id created_item = self.client.functions.create(item) + self._warn_if_cpu_or_memory_changed(created_item, item) created.append(created_item) return created + @staticmethod + def _warn_if_cpu_or_memory_changed(created_item: Function, item: FunctionWrite) -> None: + is_cpu_increased = ( + isinstance(item.cpu, float) and isinstance(created_item.cpu, float) and item.cpu < created_item.cpu + ) + is_mem_increased = ( + isinstance(item.memory, float) + and isinstance(created_item.memory, float) + and item.memory < created_item.memory + ) + if is_cpu_increased and is_mem_increased: + prefix = "CPU and Memory" + suffix = f"CPU {item.cpu} -> {created_item.cpu}, Memory {item.memory} -> {created_item.memory}" + elif is_cpu_increased: + prefix = "CPU" + suffix = f"{item.cpu} -> {created_item.cpu}" + elif is_mem_increased: + prefix = "Memory" + suffix = f"{item.memory} -> {created_item.memory}" + else: + return + # The server sets the CPU and Memory to the default values, if the user pass in a lower value. + # This happens on Azure and AWS. Warning the user about this. + LowSeverityWarning(f"{prefix} is not configurable. Function {item.external_id!r} set {suffix}").print_warning() + def retrieve(self, ids: SequenceNotStr[str]) -> FunctionList: if not self._is_activated("retrieve"): return FunctionList([]) From 729d45d718752a2d8641d57abbc1b41e78772404 Mon Sep 17 00:00:00 2001 From: anders-albert Date: Mon, 30 Dec 2024 23:10:27 +0100 Subject: [PATCH 21/33] style: warning message --- .../_cdf_tk/loaders/_resource_loaders/function_loaders.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cognite_toolkit/_cdf_tk/loaders/_resource_loaders/function_loaders.py b/cognite_toolkit/_cdf_tk/loaders/_resource_loaders/function_loaders.py index b46d9d564..4aec61fad 100644 --- a/cognite_toolkit/_cdf_tk/loaders/_resource_loaders/function_loaders.py +++ b/cognite_toolkit/_cdf_tk/loaders/_resource_loaders/function_loaders.py @@ -256,7 +256,9 @@ def _warn_if_cpu_or_memory_changed(created_item: Function, item: FunctionWrite) return # The server sets the CPU and Memory to the default values, if the user pass in a lower value. # This happens on Azure and AWS. Warning the user about this. - LowSeverityWarning(f"{prefix} is not configurable. Function {item.external_id!r} set {suffix}").print_warning() + LowSeverityWarning( + f"Function {prefix} is not configurable. Function {item.external_id!r} set {suffix}" + ).print_warning() def retrieve(self, ids: SequenceNotStr[str]) -> FunctionList: if not self._is_activated("retrieve"): From 6e5d94dd65029429a0d4fa25b80560b9046d80e0 Mon Sep 17 00:00:00 2001 From: anders-albert Date: Wed, 1 Jan 2025 12:50:40 +0100 Subject: [PATCH 22/33] feat; introduce ToolkitClientConfig --- cognite_toolkit/_cdf_tk/client/__init__.py | 4 +-- .../_cdf_tk/client/_toolkit_client.py | 34 ++++++++++++++++++- .../_resource_loaders/function_loaders.py | 6 +++- cognite_toolkit/_cdf_tk/utils/auth.py | 7 ++-- tests/test_integration/conftest.py | 27 ++++++++------- 5 files changed, 58 insertions(+), 20 deletions(-) diff --git a/cognite_toolkit/_cdf_tk/client/__init__.py b/cognite_toolkit/_cdf_tk/client/__init__.py index cd4e09694..3b7f5c569 100644 --- a/cognite_toolkit/_cdf_tk/client/__init__.py +++ b/cognite_toolkit/_cdf_tk/client/__init__.py @@ -1,3 +1,3 @@ -from ._toolkit_client import ToolkitClient +from ._toolkit_client import ToolkitClient, ToolkitClientConfig -__all__ = ["ToolkitClient"] +__all__ = ["ToolkitClient", "ToolkitClientConfig"] diff --git a/cognite_toolkit/_cdf_tk/client/_toolkit_client.py b/cognite_toolkit/_cdf_tk/client/_toolkit_client.py index e35f9da3d..2671a5b51 100644 --- a/cognite_toolkit/_cdf_tk/client/_toolkit_client.py +++ b/cognite_toolkit/_cdf_tk/client/_toolkit_client.py @@ -1,5 +1,7 @@ from __future__ import annotations +from typing import cast + from cognite.client import ClientConfig, CogniteClient from .api.dml import DMLAPI @@ -9,11 +11,41 @@ from .api.verify import VerifyAPI +class ToolkitClientConfig(ClientConfig): + @property + def cluster_provider(self) -> str: + cdf_cluster = self.cdf_cluster + if cdf_cluster is None: + return "unknown" + elif cdf_cluster.startswith("az-") or cdf_cluster in {"azure_dev", "bluefield", "westeurope-1"}: + return "azure" + elif cdf_cluster.startswith("aws-") or cdf_cluster in {"orangefield"}: + return "aws" + elif cdf_cluster.startswith("gc-") or cdf_cluster in { + "greenfield", + "asia-northeast1-1", + "cognitedata-development", + "cognitedata-production", + }: + return "gcp" + else: + return "unknown" + + class ToolkitClient(CogniteClient): - def __init__(self, config: ClientConfig | None = None) -> None: + def __init__(self, config: ToolkitClientConfig | None = None) -> None: super().__init__(config=config) self.location_filters = LocationFiltersAPI(self._config, self._API_VERSION, self) self.robotics = RoboticsAPI(self._config, self._API_VERSION, self) self.dml = DMLAPI(self._config, self._API_VERSION, self) self.verify = VerifyAPI(self._config, self._API_VERSION, self) self.lookup = LookUpGroup(self._config, self._API_VERSION, self) + + @property + def config(self) -> ToolkitClientConfig: + """Returns a config object containing the configuration for the current client. + + Returns: + ClientConfig: The configuration object. + """ + return cast(ToolkitClientConfig, self._config) diff --git a/cognite_toolkit/_cdf_tk/loaders/_resource_loaders/function_loaders.py b/cognite_toolkit/_cdf_tk/loaders/_resource_loaders/function_loaders.py index 4aec61fad..6989a6e09 100644 --- a/cognite_toolkit/_cdf_tk/loaders/_resource_loaders/function_loaders.py +++ b/cognite_toolkit/_cdf_tk/loaders/_resource_loaders/function_loaders.py @@ -153,7 +153,11 @@ def dump_resource(self, resource: Function, local: dict[str, Any]) -> dict[str, if key not in local: # Server set default values dumped.pop(key, None) - elif isinstance(local.get(key), float) and local[key] < dumped[key]: + elif ( + self.client.config.cluster_provider in ("azure", "aws") + and isinstance(local.get(key), float) + and local[key] < dumped[key] + ): # On Azure and AWS, the server sets the CPU and Memory to the default values, if the user # pass in a lower value. This should not trigger a redeploy. dumped[key] = local[key] diff --git a/cognite_toolkit/_cdf_tk/utils/auth.py b/cognite_toolkit/_cdf_tk/utils/auth.py index e109db76a..f53be8ab2 100644 --- a/cognite_toolkit/_cdf_tk/utils/auth.py +++ b/cognite_toolkit/_cdf_tk/utils/auth.py @@ -23,7 +23,6 @@ import questionary import typer -from cognite.client import ClientConfig from cognite.client.config import global_config from cognite.client.credentials import ( CredentialProvider, @@ -38,7 +37,7 @@ from rich import print from rich.prompt import Prompt -from cognite_toolkit._cdf_tk.client import ToolkitClient +from cognite_toolkit._cdf_tk.client import ToolkitClient, ToolkitClientConfig from cognite_toolkit._cdf_tk.constants import ( _RUNNING_IN_BROWSER, TOOLKIT_CLIENT_ENTRA_ID, @@ -659,7 +658,7 @@ def initialize_from_auth_variables(self, auth: AuthVariables, clear_cache: bool raise AuthenticationError(f"Login flow {auth.login_flow} is not supported.") self._token_url = auth.token_url self._toolkit_client = ToolkitClient( - ClientConfig( + ToolkitClientConfig( client_name=self._client_name, base_url=self._cdf_url, project=self._project, @@ -790,7 +789,7 @@ def create_client(self, credentials: ClientCredentials) -> ToolkitClient: ) return ToolkitClient( - config=ClientConfig( + config=ToolkitClientConfig( client_name=self._client_name, project=self._project, base_url=self._cdf_url, diff --git a/tests/test_integration/conftest.py b/tests/test_integration/conftest.py index 3dc441fb7..9c7d011e2 100644 --- a/tests/test_integration/conftest.py +++ b/tests/test_integration/conftest.py @@ -3,12 +3,12 @@ from pathlib import Path import pytest -from cognite.client import ClientConfig, CogniteClient, global_config +from cognite.client import CogniteClient, global_config from cognite.client.credentials import OAuthClientCredentials from cognite.client.data_classes.data_modeling import Space, SpaceApply from dotenv import load_dotenv -from cognite_toolkit._cdf_tk.client import ToolkitClient +from cognite_toolkit._cdf_tk.client import ToolkitClient, ToolkitClientConfig from cognite_toolkit._cdf_tk.commands import CollectCommand from cognite_toolkit._cdf_tk.utils import CDFToolConfig from tests.constants import REPO_ROOT @@ -18,7 +18,7 @@ @pytest.fixture(scope="session") -def cognite_client() -> CogniteClient: +def toolkit_client_config() -> ToolkitClientConfig: load_dotenv(REPO_ROOT / ".env", override=True) # Ensure that we do not collect data during tests cmd = CollectCommand() @@ -33,19 +33,22 @@ def cognite_client() -> CogniteClient: audience=f"https://{cdf_cluster}.cognitedata.com", ) global_config.disable_pypi_version_check = True - return CogniteClient( - ClientConfig( - client_name="cdf-toolkit-integration-tests", - base_url=f"https://{cdf_cluster}.cognitedata.com", - project=os.environ["CDF_PROJECT"], - credentials=credentials, - ) + return ToolkitClientConfig( + client_name="cdf-toolkit-integration-tests", + base_url=f"https://{cdf_cluster}.cognitedata.com", + project=os.environ["CDF_PROJECT"], + credentials=credentials, ) @pytest.fixture(scope="session") -def toolkit_client(cognite_client: CogniteClient) -> ToolkitClient: - return ToolkitClient(cognite_client._config) +def cognite_client(toolkit_client_config: ToolkitClientConfig) -> CogniteClient: + return CogniteClient(toolkit_client_config) + + +@pytest.fixture(scope="session") +def toolkit_client(toolkit_client_config: ToolkitClientConfig) -> ToolkitClient: + return ToolkitClient(toolkit_client_config) @pytest.fixture(scope="session") From d04e293cb263e8e0f1f81e5070e771e6534f34bf Mon Sep 17 00:00:00 2001 From: anders-albert Date: Wed, 1 Jan 2025 12:51:43 +0100 Subject: [PATCH 23/33] style: better return value --- cognite_toolkit/_cdf_tk/client/_toolkit_client.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cognite_toolkit/_cdf_tk/client/_toolkit_client.py b/cognite_toolkit/_cdf_tk/client/_toolkit_client.py index 2671a5b51..b219e1799 100644 --- a/cognite_toolkit/_cdf_tk/client/_toolkit_client.py +++ b/cognite_toolkit/_cdf_tk/client/_toolkit_client.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import cast +from typing import Literal, cast from cognite.client import ClientConfig, CogniteClient @@ -13,11 +13,11 @@ class ToolkitClientConfig(ClientConfig): @property - def cluster_provider(self) -> str: + def cluster_provider(self) -> Literal["azure", "aws", "gcp", "unknown"]: cdf_cluster = self.cdf_cluster if cdf_cluster is None: return "unknown" - elif cdf_cluster.startswith("az-") or cdf_cluster in {"azure_dev", "bluefield", "westeurope-1"}: + elif cdf_cluster.startswith("az-") or cdf_cluster in {"azure-dev", "bluefield", "westeurope-1"}: return "azure" elif cdf_cluster.startswith("aws-") or cdf_cluster in {"orangefield"}: return "aws" From 3041834c1d656847b63004e647a07adabedefe71 Mon Sep 17 00:00:00 2001 From: anders-albert Date: Wed, 1 Jan 2025 12:53:16 +0100 Subject: [PATCH 24/33] refactor: renaming --- cognite_toolkit/_cdf_tk/client/_toolkit_client.py | 2 +- .../_cdf_tk/loaders/_resource_loaders/function_loaders.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cognite_toolkit/_cdf_tk/client/_toolkit_client.py b/cognite_toolkit/_cdf_tk/client/_toolkit_client.py index b219e1799..4a1361058 100644 --- a/cognite_toolkit/_cdf_tk/client/_toolkit_client.py +++ b/cognite_toolkit/_cdf_tk/client/_toolkit_client.py @@ -13,7 +13,7 @@ class ToolkitClientConfig(ClientConfig): @property - def cluster_provider(self) -> Literal["azure", "aws", "gcp", "unknown"]: + def cloud_provider(self) -> Literal["azure", "aws", "gcp", "unknown"]: cdf_cluster = self.cdf_cluster if cdf_cluster is None: return "unknown" diff --git a/cognite_toolkit/_cdf_tk/loaders/_resource_loaders/function_loaders.py b/cognite_toolkit/_cdf_tk/loaders/_resource_loaders/function_loaders.py index 6989a6e09..bff0d0691 100644 --- a/cognite_toolkit/_cdf_tk/loaders/_resource_loaders/function_loaders.py +++ b/cognite_toolkit/_cdf_tk/loaders/_resource_loaders/function_loaders.py @@ -154,7 +154,7 @@ def dump_resource(self, resource: Function, local: dict[str, Any]) -> dict[str, # Server set default values dumped.pop(key, None) elif ( - self.client.config.cluster_provider in ("azure", "aws") + self.client.config.cloud_provider in ("azure", "aws") and isinstance(local.get(key), float) and local[key] < dumped[key] ): From c0c250a752d19af53670f3792fcbd08898685cc9 Mon Sep 17 00:00:00 2001 From: Anders Albert <60234212+doctrino@users.noreply.github.com> Date: Wed, 1 Jan 2025 12:55:06 +0100 Subject: [PATCH 25/33] Update cognite_toolkit/_cdf_tk/client/_toolkit_client.py --- cognite_toolkit/_cdf_tk/client/_toolkit_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cognite_toolkit/_cdf_tk/client/_toolkit_client.py b/cognite_toolkit/_cdf_tk/client/_toolkit_client.py index 4a1361058..08599773a 100644 --- a/cognite_toolkit/_cdf_tk/client/_toolkit_client.py +++ b/cognite_toolkit/_cdf_tk/client/_toolkit_client.py @@ -46,6 +46,6 @@ def config(self) -> ToolkitClientConfig: """Returns a config object containing the configuration for the current client. Returns: - ClientConfig: The configuration object. + ToolkitClientConfig: The configuration object. """ return cast(ToolkitClientConfig, self._config) From f38ff212ca1d1f55becc2e2fca78ce511d4ed854 Mon Sep 17 00:00:00 2001 From: anders-albert Date: Wed, 1 Jan 2025 12:56:49 +0100 Subject: [PATCH 26/33] docs; better docs --- .../_cdf_tk/loaders/_resource_loaders/function_loaders.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/cognite_toolkit/_cdf_tk/loaders/_resource_loaders/function_loaders.py b/cognite_toolkit/_cdf_tk/loaders/_resource_loaders/function_loaders.py index bff0d0691..bc936df86 100644 --- a/cognite_toolkit/_cdf_tk/loaders/_resource_loaders/function_loaders.py +++ b/cognite_toolkit/_cdf_tk/loaders/_resource_loaders/function_loaders.py @@ -158,8 +158,9 @@ def dump_resource(self, resource: Function, local: dict[str, Any]) -> dict[str, and isinstance(local.get(key), float) and local[key] < dumped[key] ): - # On Azure and AWS, the server sets the CPU and Memory to the default values, if the user - # pass in a lower value. This should not trigger a redeploy. + # On Azure and AWS, the server sets the CPU and Memory to the default values if the user + # pass in lower values. We set this to match the local to avoid triggering a redeploy. + # Note the user will get a warning about this when the function is created. dumped[key] = local[key] for key in ["indexUrl", "extraIndexUrls"]: # Only in write (request) format of the function From 00d9ee8d98ff5eea182ce6eeff94d4a0bbbca915 Mon Sep 17 00:00:00 2001 From: anders-albert Date: Wed, 1 Jan 2025 14:25:18 +0100 Subject: [PATCH 27/33] refactor: deal with special behavior --- .../_resource_loaders/function_loaders.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/cognite_toolkit/_cdf_tk/loaders/_resource_loaders/function_loaders.py b/cognite_toolkit/_cdf_tk/loaders/_resource_loaders/function_loaders.py index bc936df86..3d7e547a0 100644 --- a/cognite_toolkit/_cdf_tk/loaders/_resource_loaders/function_loaders.py +++ b/cognite_toolkit/_cdf_tk/loaders/_resource_loaders/function_loaders.py @@ -153,15 +153,19 @@ def dump_resource(self, resource: Function, local: dict[str, Any]) -> dict[str, if key not in local: # Server set default values dumped.pop(key, None) - elif ( - self.client.config.cloud_provider in ("azure", "aws") - and isinstance(local.get(key), float) - and local[key] < dumped[key] - ): + elif isinstance(local.get(key), float) and local[key] < dumped[key]: # On Azure and AWS, the server sets the CPU and Memory to the default values if the user # pass in lower values. We set this to match the local to avoid triggering a redeploy. # Note the user will get a warning about this when the function is created. - dumped[key] = local[key] + if self.client.config.cloud_provider in ("azure", "aws"): + dumped[key] = local[key] + elif self.client.config.cloud_provider == "gcp" and key == "cpu" and local[key] < 1.0: + # GCP does not allow CPU to be set to below 1.0 + dumped[key] = local[key] + elif self.client.config.cloud_provider == "gcp" and key == "memory" and local[key] < 1.5: + # GCP does not allow Memory to be set to below 1.5 + dumped[key] = local[key] + for key in ["indexUrl", "extraIndexUrls"]: # Only in write (request) format of the function if key in local: From c91e4df8e2a3118a03eea754886903883b6e7dce Mon Sep 17 00:00:00 2001 From: anders-albert Date: Wed, 1 Jan 2025 23:17:52 +0100 Subject: [PATCH 28/33] tests: added failing test --- cognite_toolkit/_cdf_tk/builders/_base.py | 8 ++++++-- cognite_toolkit/_cdf_tk/loaders/_base_loaders.py | 16 ++++++++++++++-- .../test_cdf_tk/test_builders/test_builder.py | 15 ++++++++++++++- 3 files changed, 34 insertions(+), 5 deletions(-) diff --git a/cognite_toolkit/_cdf_tk/builders/_base.py b/cognite_toolkit/_cdf_tk/builders/_base.py index 50ced4a7c..333e96799 100644 --- a/cognite_toolkit/_cdf_tk/builders/_base.py +++ b/cognite_toolkit/_cdf_tk/builders/_base.py @@ -103,7 +103,9 @@ def _get_loader(self, source_path: Path) -> tuple[None, ToolkitWarning] | tuple[ def get_loader( - source_path: Path, resource_folder: str + source_path: Path, + resource_folder: str, + force_pattern: bool = False, ) -> tuple[None, ToolkitWarning] | tuple[type[ResourceLoader], None]: folder_loaders = LOADER_BY_FOLDER_NAME.get(resource_folder, []) if not folder_loaders: @@ -112,7 +114,9 @@ def get_loader( details=f"Available resources are: {', '.join(LOADER_BY_FOLDER_NAME.keys())}", ) - loaders = [loader for loader in folder_loaders if loader.is_supported_file(source_path)] + loaders = [ + loader for loader in folder_loaders if loader.is_supported_file(source_path, force_pattern=force_pattern) + ] if len(loaders) == 0: suggestion: str | None = None if "." in source_path.stem: diff --git a/cognite_toolkit/_cdf_tk/loaders/_base_loaders.py b/cognite_toolkit/_cdf_tk/loaders/_base_loaders.py index 18b51af8e..e756c7b39 100644 --- a/cognite_toolkit/_cdf_tk/loaders/_base_loaders.py +++ b/cognite_toolkit/_cdf_tk/loaders/_base_loaders.py @@ -109,12 +109,24 @@ def any_supported_files(cls, directory: Path) -> bool: return any(cls.is_supported_file(file) for file in directory.glob("**/*")) @classmethod - def is_supported_file(cls, file: Path) -> bool: + def is_supported_file(cls, file: Path, force_pattern: bool = False) -> bool: + """Check if hte file is supported by this loader. + + Args: + file: The filepath to check. + force_pattern: If True, the filename pattern is used to determine if the file is supported. If False, the + file extension is used to determine if the file is supported (given that the + RequireKind flag is enabled). + + Returns: + bool: True if the file is supported, False otherwise. + + """ if cls.filetypes and file.suffix[1:] not in cls.filetypes: return False if cls.exclude_filetypes and file.suffix[1:] in cls.exclude_filetypes: return False - if Flags.REQUIRE_KIND.is_enabled() and not issubclass(cls, DataLoader): + if force_pattern is False and Flags.REQUIRE_KIND.is_enabled() and not issubclass(cls, DataLoader): return file.stem.casefold().endswith(cls.kind.casefold()) else: if cls.filename_pattern: diff --git a/tests/test_unit/test_cdf_tk/test_builders/test_builder.py b/tests/test_unit/test_cdf_tk/test_builders/test_builder.py index d00f70be4..56e2f5a1f 100644 --- a/tests/test_unit/test_cdf_tk/test_builders/test_builder.py +++ b/tests/test_unit/test_cdf_tk/test_builders/test_builder.py @@ -1,12 +1,14 @@ from __future__ import annotations from pathlib import Path +from unittest.mock import MagicMock import pytest -from cognite_toolkit._cdf_tk.builders import TransformationBuilder +from cognite_toolkit._cdf_tk.builders import TransformationBuilder, get_loader from cognite_toolkit._cdf_tk.exceptions import AmbiguousResourceFileError from cognite_toolkit._cdf_tk.feature_flags import Flags +from cognite_toolkit._cdf_tk.loaders import RawDatabaseLoader if not Flags.REQUIRE_KIND.is_enabled(): @@ -18,3 +20,14 @@ def test_get_loader_raises_ambiguous_error(): source_path=Path("my_module") / "transformations" / "notification.yaml", ) assert "Ambiguous resource file" in str(e.value) + + +def test_get_loader() -> None: + filepath = MagicMock(spec=Path) + filepath.name = "filelocation.yaml" + filepath.stem = "filelocation" + filepath.suffix = ".yaml" + filepath.read_text.return_value = "dbName: my_database\n" + _, loader = get_loader(filepath, "raw", force_pattern=True) + + assert loader is RawDatabaseLoader From 539af06cc2aea5b5b3c4591d8c929ef9afc3fe8c Mon Sep 17 00:00:00 2001 From: anders-albert Date: Wed, 1 Jan 2025 23:21:09 +0100 Subject: [PATCH 29/33] refactor; extending test --- cognite_toolkit/_cdf_tk/commands/_changes.py | 2 +- .../test_cdf_tk/test_builders/test_builder.py | 19 ++++++++++++++----- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/cognite_toolkit/_cdf_tk/commands/_changes.py b/cognite_toolkit/_cdf_tk/commands/_changes.py index 4969fc6e5..29bab27f7 100644 --- a/cognite_toolkit/_cdf_tk/commands/_changes.py +++ b/cognite_toolkit/_cdf_tk/commands/_changes.py @@ -64,7 +64,7 @@ def do(self) -> set[Path]: for source_file in source_files: if source_file.suffix not in {".yaml", ".yml"}: continue - loader, warning = get_loader(source_file, resource_folder) + loader, warning = get_loader(source_file, resource_folder, force_pattern=True) if loader is None: print(f"Could not find loader for {source_file}") continue diff --git a/tests/test_unit/test_cdf_tk/test_builders/test_builder.py b/tests/test_unit/test_cdf_tk/test_builders/test_builder.py index 56e2f5a1f..d9c9fad50 100644 --- a/tests/test_unit/test_cdf_tk/test_builders/test_builder.py +++ b/tests/test_unit/test_cdf_tk/test_builders/test_builder.py @@ -8,7 +8,7 @@ from cognite_toolkit._cdf_tk.builders import TransformationBuilder, get_loader from cognite_toolkit._cdf_tk.exceptions import AmbiguousResourceFileError from cognite_toolkit._cdf_tk.feature_flags import Flags -from cognite_toolkit._cdf_tk.loaders import RawDatabaseLoader +from cognite_toolkit._cdf_tk.loaders import RawDatabaseLoader, RawTableLoader, ResourceLoader if not Flags.REQUIRE_KIND.is_enabled(): @@ -22,12 +22,21 @@ def test_get_loader_raises_ambiguous_error(): assert "Ambiguous resource file" in str(e.value) -def test_get_loader() -> None: +@pytest.mark.parametrize( + "content, expected_loader_cls", + [ + ("dbName: my_database\n", RawDatabaseLoader), + ("dbName: my_database\ntableName: my_table\n", RawTableLoader), + ], +) +def test_get_loader_database_only_file(content: str, expected_loader_cls: type[ResourceLoader]) -> None: filepath = MagicMock(spec=Path) filepath.name = "filelocation.yaml" filepath.stem = "filelocation" filepath.suffix = ".yaml" - filepath.read_text.return_value = "dbName: my_database\n" - _, loader = get_loader(filepath, "raw", force_pattern=True) + filepath.read_text.return_value = content - assert loader is RawDatabaseLoader + loader, warn = get_loader(filepath, "raw", force_pattern=True) + + assert warn is None + assert loader is expected_loader_cls From 45806174ca1e3a1aa789a680ec9e4ac82ca61752 Mon Sep 17 00:00:00 2001 From: anders-albert Date: Wed, 1 Jan 2025 23:24:38 +0100 Subject: [PATCH 30/33] fix: set correct kind file name --- cognite_toolkit/_cdf_tk/builders/_base.py | 12 ++++++++++-- .../test_cdf_tk/test_builders/test_builder.py | 2 +- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/cognite_toolkit/_cdf_tk/builders/_base.py b/cognite_toolkit/_cdf_tk/builders/_base.py index 333e96799..d32772695 100644 --- a/cognite_toolkit/_cdf_tk/builders/_base.py +++ b/cognite_toolkit/_cdf_tk/builders/_base.py @@ -17,6 +17,7 @@ from cognite_toolkit._cdf_tk.loaders import ( LOADER_BY_FOLDER_NAME, GroupLoader, + RawDatabaseLoader, RawTableLoader, ResourceLoader, ) @@ -136,8 +137,15 @@ def get_loader( ) return None, UnknownResourceTypeWarning(source_path, suggestion) elif len(loaders) > 1 and all(loader.folder_name == "raw" for loader in loaders): - # Multiple raw loaders load from the same file. - return RawTableLoader, None + # Raw files can be ambiguous, so we need to check the content. + # If there is a tableName field, it is a table, otherwise it is a database. + if any( + line.strip().startswith("tableName:") or line.strip().startswith("- tableName:") + for line in source_path.read_text().splitlines() + ): + return RawTableLoader, None + else: + return RawDatabaseLoader, None elif len(loaders) > 1 and all(issubclass(loader, GroupLoader) for loader in loaders): # There are two group loaders, one for resource scoped and one for all scoped. return GroupLoader, None diff --git a/tests/test_unit/test_cdf_tk/test_builders/test_builder.py b/tests/test_unit/test_cdf_tk/test_builders/test_builder.py index d9c9fad50..7285aa47b 100644 --- a/tests/test_unit/test_cdf_tk/test_builders/test_builder.py +++ b/tests/test_unit/test_cdf_tk/test_builders/test_builder.py @@ -29,7 +29,7 @@ def test_get_loader_raises_ambiguous_error(): ("dbName: my_database\ntableName: my_table\n", RawTableLoader), ], ) -def test_get_loader_database_only_file(content: str, expected_loader_cls: type[ResourceLoader]) -> None: +def test_get_loader_raw_loaders(content: str, expected_loader_cls: type[ResourceLoader]) -> None: filepath = MagicMock(spec=Path) filepath.name = "filelocation.yaml" filepath.stem = "filelocation" From 0e19035abb6844a2a125b5f061f40bdb1c0f1574 Mon Sep 17 00:00:00 2001 From: anders-albert Date: Wed, 1 Jan 2025 23:41:42 +0100 Subject: [PATCH 31/33] fix: added missing verbose flags --- CHANGELOG.cdf-tk.md | 2 ++ cognite_toolkit/_cdf_tk/apps/_auth_app.py | 16 ++++++++++ cognite_toolkit/_cdf_tk/apps/_modules_app.py | 24 +++++++++++++++ cognite_toolkit/_cdf_tk/apps/_run.py | 32 ++++++++++++++++++++ 4 files changed, 74 insertions(+) diff --git a/CHANGELOG.cdf-tk.md b/CHANGELOG.cdf-tk.md index 6752b615d..55b854893 100644 --- a/CHANGELOG.cdf-tk.md +++ b/CHANGELOG.cdf-tk.md @@ -36,6 +36,8 @@ Changes are grouped as follows: the required value no longer triggers a redeploy. - `LocationFilters` now parses the `version` key of `View` and `DataModel` correctly as a string. - `LocationFilters` now converts an empty string of `dataSetExternalId` to `0` instead of ignoring it. +- All commands now has a verbose flag `--verbose`. Not all commands have an `--verbose` outupt, but + error handling has a more verbose output which applies to all commands. ## [0.3.23] - 2024-12-13 diff --git a/cognite_toolkit/_cdf_tk/apps/_auth_app.py b/cognite_toolkit/_cdf_tk/apps/_auth_app.py index c45cd6902..35fbc859d 100644 --- a/cognite_toolkit/_cdf_tk/apps/_auth_app.py +++ b/cognite_toolkit/_cdf_tk/apps/_auth_app.py @@ -37,6 +37,14 @@ def init( help="If you verify, and you pass this flag no changes to CDF will be made.", ), ] = False, + verbose: Annotated[ + bool, + typer.Option( + "--verbose", + "-v", + help="Turn on to get more verbose output when running the command", + ), + ] = False, ) -> None: """Sets the OIDC parameters required to authenticate and authorize the Cognite Toolkit in Cognite Data Fusion. @@ -74,6 +82,14 @@ def verify( "If you include this flag, the execution will stop if the user or service principal does not have the required capabilities.", ), ] = False, + verbose: Annotated[ + bool, + typer.Option( + "--verbose", + "-v", + help="Turn on to get more verbose output when running the command", + ), + ] = False, ) -> None: """Verify that the current user or service principal has the required capabilities to run the CDF Toolkit commands.""" cmd = AuthCommand() diff --git a/cognite_toolkit/_cdf_tk/apps/_modules_app.py b/cognite_toolkit/_cdf_tk/apps/_modules_app.py index 8ecd242de..df2a36a79 100644 --- a/cognite_toolkit/_cdf_tk/apps/_modules_app.py +++ b/cognite_toolkit/_cdf_tk/apps/_modules_app.py @@ -53,6 +53,14 @@ def init( help="Clean target directory if it exists", ), ] = False, + verbose: Annotated[ + bool, + typer.Option( + "--verbose", + "-v", + help="Turn on to get more verbose output when running the command", + ), + ] = False, ) -> None: """Initialize or upgrade a new CDF project with templates interactively.""" @@ -100,6 +108,14 @@ def add( help="Path to project directory with the modules. This is used to search for available functions.", ), ] = CDF_TOML.cdf.default_organization_dir, + verbose: Annotated[ + bool, + typer.Option( + "--verbose", + "-v", + help="Turn on to get more verbose output when running the command", + ), + ] = False, ) -> None: """Add one or more new module(s) to the project.""" cmd = ModulesCommand() @@ -179,6 +195,14 @@ def list( help="Build environment to use.", ), ] = CDF_TOML.cdf.default_env, + verbose: Annotated[ + bool, + typer.Option( + "--verbose", + "-v", + help="Turn on to get more verbose output when running the command", + ), + ] = False, ) -> None: """List all available modules in the project.""" cmd = ModulesCommand() diff --git a/cognite_toolkit/_cdf_tk/apps/_run.py b/cognite_toolkit/_cdf_tk/apps/_run.py index 0a0bf7288..e6df9295d 100644 --- a/cognite_toolkit/_cdf_tk/apps/_run.py +++ b/cognite_toolkit/_cdf_tk/apps/_run.py @@ -43,6 +43,14 @@ def run_transformation( help="External id of the transformation to run.", ), ], + verbose: Annotated[ + bool, + typer.Option( + "--verbose", + "-v", + help="Turn on to get more verbose output when running the command", + ), + ] = False, ) -> None: """This command will run the specified transformation using a one-time session.""" cmd = RunTransformationCommand() @@ -93,6 +101,14 @@ def run_workflow( help="Whether to wait for the workflow to complete.", ), ] = False, + verbose: Annotated[ + bool, + typer.Option( + "--verbose", + "-v", + help="Turn on to get more verbose output when running the command", + ), + ] = False, ) -> None: """This command will run the specified workflow.""" cmd = RunWorkflowCommand() @@ -158,6 +174,14 @@ def run_local( help="Whether to rebuild the environment before running the function.", ), ] = False, + verbose: Annotated[ + bool, + typer.Option( + "--verbose", + "-v", + help="Turn on to get more verbose output when running the command", + ), + ] = False, ) -> None: """This command will run the specified function locally.""" cmd = RunFunctionCommand() @@ -214,6 +238,14 @@ def run_cdf( help="Whether to wait for the function to complete.", ), ] = False, + verbose: Annotated[ + bool, + typer.Option( + "--verbose", + "-v", + help="Turn on to get more verbose output when running the command", + ), + ] = False, ) -> None: """This command will run the specified function (assuming it is deployed) in CDF.""" cmd = RunFunctionCommand() From 0cd1c1f46e67a4e24b02f6c314fe6f6edb3990c1 Mon Sep 17 00:00:00 2001 From: anders-albert Date: Thu, 2 Jan 2025 07:02:46 +0100 Subject: [PATCH 32/33] fix: added asset external ID lookup in sequence --- CHANGELOG.cdf-tk.md | 1 + .../_cdf_tk/loaders/_resource_loaders/classic_loaders.py | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/CHANGELOG.cdf-tk.md b/CHANGELOG.cdf-tk.md index 55b854893..7ef7da8ad 100644 --- a/CHANGELOG.cdf-tk.md +++ b/CHANGELOG.cdf-tk.md @@ -38,6 +38,7 @@ Changes are grouped as follows: - `LocationFilters` now converts an empty string of `dataSetExternalId` to `0` instead of ignoring it. - All commands now has a verbose flag `--verbose`. Not all commands have an `--verbose` outupt, but error handling has a more verbose output which applies to all commands. +- Toolkit now looks up the `assetExternalId` in the `Sequence` resource when deploying. ## [0.3.23] - 2024-12-13 diff --git a/cognite_toolkit/_cdf_tk/loaders/_resource_loaders/classic_loaders.py b/cognite_toolkit/_cdf_tk/loaders/_resource_loaders/classic_loaders.py index 9b7dd4379..e9002d70f 100644 --- a/cognite_toolkit/_cdf_tk/loaders/_resource_loaders/classic_loaders.py +++ b/cognite_toolkit/_cdf_tk/loaders/_resource_loaders/classic_loaders.py @@ -274,12 +274,16 @@ def get_required_capability( def load_resource(self, resource: dict[str, Any], is_dry_run: bool = False) -> SequenceWrite: if ds_external_id := resource.pop("dataSetExternalId", None): resource["dataSetId"] = self.client.lookup.data_sets.id(ds_external_id, is_dry_run) + if asset_external_id := resource.pop("assetExternalId", None): + resource["assetId"] = self.client.lookup.assets.id(asset_external_id) return SequenceWrite._load(resource) def dump_resource(self, resource: Sequence, local: dict[str, Any]) -> dict[str, Any]: dumped = resource.as_write().dump() if data_set_id := dumped.pop("dataSetId", None): dumped["dataSetExternalId"] = self.client.lookup.data_sets.external_id(data_set_id) + if asset_id := dumped.pop("assetId", None): + dumped["assetExternalId"] = self.client.lookup.assets.external_id(asset_id) if not dumped.get("metadata") and "metadata" not in local: dumped.pop("metadata", None) local_col_by_id = {col["externalId"]: col for col in local.get("columns", []) if "externalId" in col} From 58fe86ef19820e5c9e55511e523cd774e5cccbca Mon Sep 17 00:00:00 2001 From: Anders Albert <60234212+doctrino@users.noreply.github.com> Date: Mon, 6 Jan 2025 12:01:04 +0100 Subject: [PATCH 33/33] Update CHANGELOG.cdf-tk.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Pål Rønning --- CHANGELOG.cdf-tk.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.cdf-tk.md b/CHANGELOG.cdf-tk.md index 55b854893..2b3459d13 100644 --- a/CHANGELOG.cdf-tk.md +++ b/CHANGELOG.cdf-tk.md @@ -36,7 +36,7 @@ Changes are grouped as follows: the required value no longer triggers a redeploy. - `LocationFilters` now parses the `version` key of `View` and `DataModel` correctly as a string. - `LocationFilters` now converts an empty string of `dataSetExternalId` to `0` instead of ignoring it. -- All commands now has a verbose flag `--verbose`. Not all commands have an `--verbose` outupt, but +- All commands now has a verbose flag `--verbose`. Not all commands have an `--verbose` output, but error handling has a more verbose output which applies to all commands. ## [0.3.23] - 2024-12-13