[CDF-23263] 🤖 Add verbose to all cmds

CHANGELOG.cdf-tk.md

@@ -23,6 +23,7 @@ Changes are grouped as follows:
   with the flag `--offline`.
 - [alpha feature] New subcommand `cdf modules pull` which pulls the configuration from the CDF project to the local
   modules directory.
+- Support for property `dataModelingType` in `LocationFilter` resources.
 
 ### Fixed
 
@@ -31,6 +32,12 @@ Changes are grouped as follows:
 - Cognite Toolkit has improved resources that have server set default values that can lead to redeploy even when
   unchanged. This includes `Sequences`, `Containers`, `DataSets`, `Views`, `Nodes`, `Edges`, `ExtractionPipelines`,
   `CogniteFiles`, `HostedExtractorJobs`, `Relationships`, `RobotMaps`, and `WorkflowVersions`.
+- On CDF deployed on Azure and AWS clouds, setting the `CPU` and `memory` of a CogniteFiles to lower than
+  the required value no longer triggers a redeploy.
+- `LocationFilters` now parses the `version` key of `View` and `DataModel` correctly as a string.
+- `LocationFilters` now converts an empty string of `dataSetExternalId` to `0` instead of ignoring it.
+- All commands now has a verbose flag `--verbose`. Not all commands have an `--verbose` outupt, but
+  error handling has a more verbose output which applies to all commands.
 
 ## [0.3.23] - 2024-12-13
 

cognite_toolkit/_cdf_tk/apps/_auth_app.py

@@ -37,6 +37,14 @@ def init(
                 help="If you verify, and you pass this flag no changes to CDF will be made.",
             ),
         ] = False,
+        verbose: Annotated[
+            bool,
+            typer.Option(
+                "--verbose",
+                "-v",
+                help="Turn on to get more verbose output when running the command",
+            ),
+        ] = False,
     ) -> None:
         """Sets the OIDC parameters required to authenticate and authorize the Cognite Toolkit in Cognite Data Fusion.
 
@@ -74,6 +82,14 @@ def verify(
                 "If you include this flag, the execution will stop if the user or service principal does not have the required capabilities.",
             ),
         ] = False,
+        verbose: Annotated[
+            bool,
+            typer.Option(
+                "--verbose",
+                "-v",
+                help="Turn on to get more verbose output when running the command",
+            ),
+        ] = False,
     ) -> None:
         """Verify that the current user or service principal has the required capabilities to run the CDF Toolkit commands."""
         cmd = AuthCommand()

cognite_toolkit/_cdf_tk/apps/_modules_app.py

@@ -53,6 +53,14 @@ def init(
                 help="Clean target directory if it exists",
             ),
         ] = False,
+        verbose: Annotated[
+            bool,
+            typer.Option(
+                "--verbose",
+                "-v",
+                help="Turn on to get more verbose output when running the command",
+            ),
+        ] = False,
     ) -> None:
         """Initialize or upgrade a new CDF project with templates interactively."""
 
@@ -100,6 +108,14 @@ def add(
                 help="Path to project directory with the modules. This is used to search for available functions.",
             ),
         ] = CDF_TOML.cdf.default_organization_dir,
+        verbose: Annotated[
+            bool,
+            typer.Option(
+                "--verbose",
+                "-v",
+                help="Turn on to get more verbose output when running the command",
+            ),
+        ] = False,
     ) -> None:
         """Add one or more new module(s) to the project."""
         cmd = ModulesCommand()
@@ -179,6 +195,14 @@ def list(
                 help="Build environment to use.",
             ),
         ] = CDF_TOML.cdf.default_env,
+        verbose: Annotated[
+            bool,
+            typer.Option(
+                "--verbose",
+                "-v",
+                help="Turn on to get more verbose output when running the command",
+            ),
+        ] = False,
     ) -> None:
         """List all available modules in the project."""
         cmd = ModulesCommand()

cognite_toolkit/_cdf_tk/apps/_run.py

@@ -43,6 +43,14 @@ def run_transformation(
                 help="External id of the transformation to run.",
             ),
         ],
+        verbose: Annotated[
+            bool,
+            typer.Option(
+                "--verbose",
+                "-v",
+                help="Turn on to get more verbose output when running the command",
+            ),
+        ] = False,
     ) -> None:
         """This command will run the specified transformation using a one-time session."""
         cmd = RunTransformationCommand()
@@ -93,6 +101,14 @@ def run_workflow(
                     help="Whether to wait for the workflow to complete.",
                 ),
             ] = False,
+            verbose: Annotated[
+                bool,
+                typer.Option(
+                    "--verbose",
+                    "-v",
+                    help="Turn on to get more verbose output when running the command",
+                ),
+            ] = False,
         ) -> None:
             """This command will run the specified workflow."""
             cmd = RunWorkflowCommand()
@@ -158,6 +174,14 @@ def run_local(
                 help="Whether to rebuild the environment before running the function.",
             ),
         ] = False,
+        verbose: Annotated[
+            bool,
+            typer.Option(
+                "--verbose",
+                "-v",
+                help="Turn on to get more verbose output when running the command",
+            ),
+        ] = False,
     ) -> None:
         """This command will run the specified function locally."""
         cmd = RunFunctionCommand()
@@ -214,6 +238,14 @@ def run_cdf(
                 help="Whether to wait for the function to complete.",
             ),
         ] = False,
+        verbose: Annotated[
+            bool,
+            typer.Option(
+                "--verbose",
+                "-v",
+                help="Turn on to get more verbose output when running the command",
+            ),
+        ] = False,
     ) -> None:
         """This command will run the specified function (assuming it is deployed) in CDF."""
         cmd = RunFunctionCommand()

cognite_toolkit/_cdf_tk/builders/_base.py

@@ -17,6 +17,7 @@
 from cognite_toolkit._cdf_tk.loaders import (
     LOADER_BY_FOLDER_NAME,
     GroupLoader,
+    RawDatabaseLoader,
     RawTableLoader,
     ResourceLoader,
 )
@@ -103,7 +104,9 @@ def _get_loader(self, source_path: Path) -> tuple[None, ToolkitWarning] | tuple[
 
 
 def get_loader(
-    source_path: Path, resource_folder: str
+    source_path: Path,
+    resource_folder: str,
+    force_pattern: bool = False,
 ) -> tuple[None, ToolkitWarning] | tuple[type[ResourceLoader], None]:
     folder_loaders = LOADER_BY_FOLDER_NAME.get(resource_folder, [])
     if not folder_loaders:
@@ -112,7 +115,9 @@ def get_loader(
             details=f"Available resources are: {', '.join(LOADER_BY_FOLDER_NAME.keys())}",
         )
 
-    loaders = [loader for loader in folder_loaders if loader.is_supported_file(source_path)]
+    loaders = [
+        loader for loader in folder_loaders if loader.is_supported_file(source_path, force_pattern=force_pattern)
+    ]
     if len(loaders) == 0:
         suggestion: str | None = None
         if "." in source_path.stem:
@@ -132,8 +137,15 @@ def get_loader(
                 )
         return None, UnknownResourceTypeWarning(source_path, suggestion)
     elif len(loaders) > 1 and all(loader.folder_name == "raw" for loader in loaders):
-        # Multiple raw loaders load from the same file.
-        return RawTableLoader, None
+        # Raw files can be ambiguous, so we need to check the content.
+        # If there is a tableName field, it is a table, otherwise it is a database.
+        if any(
+            line.strip().startswith("tableName:") or line.strip().startswith("- tableName:")
+            for line in source_path.read_text().splitlines()
+        ):
+            return RawTableLoader, None
+        else:
+            return RawDatabaseLoader, None
     elif len(loaders) > 1 and all(issubclass(loader, GroupLoader) for loader in loaders):
         # There are two group loaders, one for resource scoped and one for all scoped.
         return GroupLoader, None

cognite_toolkit/_cdf_tk/client/__init__.py

@@ -1,3 +1,3 @@
-from ._toolkit_client import ToolkitClient
+from ._toolkit_client import ToolkitClient, ToolkitClientConfig
 
-__all__ = ["ToolkitClient"]
+__all__ = ["ToolkitClient", "ToolkitClientConfig"]

cognite_toolkit/_cdf_tk/client/_toolkit_client.py

@@ -1,5 +1,7 @@
 from __future__ import annotations
 
+from typing import Literal, cast
+
 from cognite.client import ClientConfig, CogniteClient
 
 from .api.dml import DMLAPI
@@ -9,11 +11,41 @@
 from .api.verify import VerifyAPI
 
 
+class ToolkitClientConfig(ClientConfig):
+    @property
+    def cloud_provider(self) -> Literal["azure", "aws", "gcp", "unknown"]:
+        cdf_cluster = self.cdf_cluster
+        if cdf_cluster is None:
+            return "unknown"
+        elif cdf_cluster.startswith("az-") or cdf_cluster in {"azure-dev", "bluefield", "westeurope-1"}:
+            return "azure"
+        elif cdf_cluster.startswith("aws-") or cdf_cluster in {"orangefield"}:
+            return "aws"
+        elif cdf_cluster.startswith("gc-") or cdf_cluster in {
+            "greenfield",
+            "asia-northeast1-1",
+            "cognitedata-development",
+            "cognitedata-production",
+        }:
+            return "gcp"
+        else:
+            return "unknown"
+
+
 class ToolkitClient(CogniteClient):
-    def __init__(self, config: ClientConfig | None = None) -> None:
+    def __init__(self, config: ToolkitClientConfig | None = None) -> None:
         super().__init__(config=config)
         self.location_filters = LocationFiltersAPI(self._config, self._API_VERSION, self)
         self.robotics = RoboticsAPI(self._config, self._API_VERSION, self)
         self.dml = DMLAPI(self._config, self._API_VERSION, self)
         self.verify = VerifyAPI(self._config, self._API_VERSION, self)
         self.lookup = LookUpGroup(self._config, self._API_VERSION, self)
+
+    @property
+    def config(self) -> ToolkitClientConfig:
+        """Returns a config object containing the configuration for the current client.
+
+        Returns:
+            ToolkitClientConfig: The configuration object.
+        """
+        return cast(ToolkitClientConfig, self._config)

cognite_toolkit/_cdf_tk/client/api/lookup.py

@@ -36,14 +36,22 @@ def resource_name(self) -> str:
         return type(self).__name__.removesuffix("LookUpAPI")
 
     @overload
-    def id(self, external_id: str, is_dry_run: bool = False) -> int: ...
+    def id(self, external_id: str, is_dry_run: bool = False, allow_empty: bool = False) -> int: ...
 
     @overload
-    def id(self, external_id: SequenceNotStr[str], is_dry_run: bool = False) -> list[int]: ...
+    def id(
+        self, external_id: SequenceNotStr[str], is_dry_run: bool = False, allow_empty: bool = False
+    ) -> list[int]: ...
 
-    def id(self, external_id: str | SequenceNotStr[str], is_dry_run: bool = False) -> int | list[int]:
+    def id(
+        self, external_id: str | SequenceNotStr[str], is_dry_run: bool = False, allow_empty: bool = False
+    ) -> int | list[int]:
         ids = [external_id] if isinstance(external_id, str) else external_id
         missing = [id for id in ids if id not in self._cache]
+        if allow_empty and "" in missing:
+            # Note we do not want to put empty string in the cache. It is a special case that
+            # as of 01/02/2025 only applies to LocationFilters
+            missing.remove("")
         if missing:
             try:
                 lookup = self._id(missing)
@@ -63,14 +71,19 @@ def id(self, external_id: str | SequenceNotStr[str], is_dry_run: bool = False) -
                 raise ResourceRetrievalError(
                     f"Failed to retrieve {self.resource_name} with external_id {missing}." "Have you created it?"
                 )
-        if is_dry_run:
-            return (
-                self._cache.get(external_id, self.dry_run_id)
-                if isinstance(external_id, str)
-                else [self._cache.get(id, self.dry_run_id) for id in ids]
-            )
+        return (
+            self._get_id_from_cache(external_id, is_dry_run, allow_empty)
+            if isinstance(external_id, str)
+            else [self._get_id_from_cache(id, is_dry_run, allow_empty) for id in ids]
+        )
 
-        return self._cache[external_id] if isinstance(external_id, str) else [self._cache[id] for id in ids]
+    def _get_id_from_cache(self, external_id: str, is_dry_run: bool = False, allow_empty: bool = False) -> int:
+        if allow_empty and external_id == "":
+            return 0
+        elif is_dry_run:
+            return self._cache.get(external_id, self.dry_run_id)
+        else:
+            return self._cache[external_id]
 
     @overload
     def external_id(self, id: int) -> str: ...
@@ -84,6 +97,8 @@ def external_id(
     ) -> str | list[str]:
         ids = [id] if isinstance(id, int) else id
         missing = [id_ for id_ in ids if id_ not in self._reverse_cache]
+        if 0 in missing:
+            missing.remove(0)
         if missing:
             try:
                 lookup = self._external_id(missing)
@@ -103,7 +118,17 @@ def external_id(
                 raise ResourceRetrievalError(
                     f"Failed to retrieve {self.resource_name} with id {missing}." "Have you created it?"
                 )
-        return self._reverse_cache[id] if isinstance(id, int) else [self._reverse_cache[id] for id in ids]
+        return (
+            self._get_external_id_from_cache(id)
+            if isinstance(id, int)
+            else [self._get_external_id_from_cache(id) for id in ids]
+        )
+
+    def _get_external_id_from_cache(self, id: int) -> str:
+        if id == 0:
+            # Reverse of looking up an empty string.
+            return ""
+        return self._reverse_cache[id]
 
     @abstractmethod
     def _id(self, external_id: SequenceNotStr[str]) -> dict[str, int]: