Release 0.2.3 (#679)

# Description Please describe the change you have made. ## Checklist - [ ] Tests added/updated. - [ ] Run Demo Job Locally. - [ ] Documentation updated. - [ ] Changelogs updated in [CHANGELOG.cdf-tk.md](https://github.com/cognitedata/toolkit/blob/main/CHANGELOG.cdf-tk.md). - [ ] Template changelogs updated in [CHANGELOG.templates.md](https://github.com/cognitedata/toolkit/blob/main/CHANGELOG.templates.md). - [ ] Version bumped. [_version.py](https://github.com/cognitedata/toolkit/blob/main/cognite/cognite_toolkit/_version.py) and [pyproject.toml](https://github.com/cognitedata/toolkit/blob/main/pyproject.toml) per [semantic versioning](https://semver.org/).
cognitedata · Jun 20, 2024 · 56a3561 · 56a3561
2 parents 42a09ea + 2bb0e85
commit 56a3561
Show file tree

Hide file tree

Showing 64 changed files with 830 additions and 793 deletions.
diff --git a/.gitignore b/.gitignore
@@ -286,12 +286,12 @@ my_project/
 
 # Hide environments used for migration testing
 **/.venv*
-tests_migrations/tmp*
+module_upgrade/tmp*
 
 cognite_toolkit/config.local.yaml
 **/functions/**/common
 **/.venv*
 .venv.*
 build.*
 cognite_toolkit/.env.*
-tests_migrations/project_inits
+module_upgrade/project_inits
diff --git a/.vscode/launch.json b/.vscode/launch.json
@@ -2,6 +2,7 @@
   // Use IntelliSense to learn about possible attributes.
   // Hover to view descriptions of existing attributes.
   // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+{
   "version": "0.2.0",
   "configurations": [
     {
@@ -158,7 +159,7 @@
       "type": "debugpy",
       "request": "launch",
       "program": "./cdf-tk-dev.py",
-      "args": ["modules", "init", "new_project"],
+      "args": ["modules", "init"],
       "console": "integratedTerminal",
       "justMyCode": false
     },

diff --git a/CHANGELOG.cdf-tk.md b/CHANGELOG.cdf-tk.md
@@ -15,6 +15,25 @@ Changes are grouped as follows:
 - `Fixed` for any bug fixes.
 - `Security` in case of vulnerabilities.
 
+## [0.2.3] - 2024-06-20
+
+### Improved
+
+- When running `cdf-tk build` and missing `CDF_PROJECT` environment variable, the user will now get a more informative
+  error message.
+
+### Fixed
+
+- The variable `type` in the `environment` section of the `config.[env].yaml` now raises an error if it is not
+  set to `dev`, `staging`, or `prod`.
+
+### Added
+
+- The preview feature `IMPORT_CMD` added. This enables you to import a `transformation-cli` manifest into
+  resource configuration files compatible with the `cognite-toolkit`. Activate by running
+  `cdf-tk features set IMPORT_CMD --enable`, and deactivate by running `cdf-tk features set IMPORT_CMD --disable`.
+  Run `cdf-tk import transformation-cli --help` for more information about the import command.
+
 ## [0.2.2] - 2024-06-18
 
 ### Improved

diff --git a/CHANGELOG.templates.md b/CHANGELOG.templates.md
@@ -15,6 +15,10 @@ Changes are grouped as follows:
 - `Fixed` for any bug fixes.
 - `Security` in case of vulnerabilities.
 
+## [0.2.3] - 2024-06-20
+
+No changes to templates.
+
 ## [0.2.2] - 2024-06-18
 
 No changes to templates.

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -108,8 +108,8 @@ To release a new version of the `cdf-tk` tool and the templates, you need to do
       - `tests/tests_unit/test_cdf_tk/project_for_test/cognite_modules/_system.yaml`
    4. Run `poetry lock` to update the `poetry.lock` file.
    5. Run `pytest tests` locally to ensure that tests pass.
-   6. Follow the guide in [tests_migration](tests_migration/README.md) do detect breaking changes, and
-      update the migration instructions in `cognite_toolkit/templates/_migration.py` if necessary.
+   6. Run `python module_upgrade/run_check.py` to ensure that the `cdf-tk modules upgrade` command works as expected.
+      against previous versions. See [Module Upgrade](module_upgrade/README.md) for more information.
 2. Get approval to squash merge the branch into `main`:
    1. Verify that all Github actions pass.
 3. Create a release branch: `release-x.y.z` from `main`:

diff --git a/cognite_toolkit/_api/data_classes.py b/cognite_toolkit/_api/data_classes.py
@@ -20,7 +20,7 @@
 _DUMMY_ENVIRONMENT = Environment(
     name="not used",
     project="not used",
-    build_type="not used",
+    build_type="dev",
     selected=[],
 )
 

diff --git a/cognite_toolkit/_api/modules_api.py b/cognite_toolkit/_api/modules_api.py
@@ -5,7 +5,7 @@
 import tempfile
 from collections.abc import Sequence
 from pathlib import Path
-from typing import Any, overload
+from typing import Any, Literal, cast, overload
 from unittest.mock import MagicMock
 
 import typer
@@ -78,7 +78,7 @@ def _build(self, modules: Sequence[ModuleMeta], verbose: bool) -> None:
             environment=Environment(
                 name=self._build_env,
                 project=self._project_name,
-                build_type=self._build_env,
+                build_type=cast(Literal["dev"], self._build_env),
                 selected=[module.name for module in modules],
             ),
             filepath=Path(""),

diff --git a/cognite_toolkit/_cdf.py b/cognite_toolkit/_cdf.py
@@ -107,6 +107,11 @@ def app() -> NoReturn:
         else:
             _app.command("init")(main_init)
 
+        if FeatureFlag.is_enabled(Flags.IMPORT_CMD):
+            from cognite_toolkit._cdf_tk.prototypes.import_app import import_app
+
+            _app.add_typer(import_app, name="import")
+
         _app()
     except ToolkitError as err:
         print(f"  [bold red]ERROR ([/][red]{type(err).__name__}[/][bold red]):[/] {err}")
@@ -869,7 +874,9 @@ def feature_flag_main(ctx: typer.Context) -> None:
                 "\nDo not enable a flag unless you are familiar with what it does.[/]"
             )
         )
-        print("Use [bold yellow]cdf-tk feature list[/] or [bold yellow]cdf-tk feature --[flag] --enabled=True|False[/]")
+        print(
+            "Use [bold yellow]cdf-tk features list[/] or [bold yellow]cdf-tk features set <flag> --enabled/--disabled[/]"
+        )
     return None
 
 

diff --git a/cognite_toolkit/_cdf_tk/_migration.yaml b/cognite_toolkit/_cdf_tk/_migration.yaml
@@ -1,8 +1,13 @@
-- version: 0.2.2
+- version: 0.2.3
   cognite_modules: {}
   resources: {}
   tool: {}
   cognite_modules_hash: ""
+- version: 0.2.2
+  cognite_modules: {}
+  resources: {}
+  tool: {}
+  cognite_modules_hash: "e3ef5a27540847e955f04c3dad4e296f5d6f646008a7a5490cddf81e79772c1d"
 - version: 0.2.1
   cognite_modules: {}
   resources: {}

diff --git a/cognite_toolkit/_cdf_tk/commands/auth.py b/cognite_toolkit/_cdf_tk/commands/auth.py
@@ -24,6 +24,8 @@
 from cognite.client.data_classes.capabilities import (
     Capability,
     FunctionsAcl,
+    GroupsAcl,
+    ProjectsAcl,
 )
 from cognite.client.data_classes.iam import Group, GroupList, GroupWrite, TokenInspection
 from cognite.client.exceptions import CogniteAPIError
@@ -231,14 +233,20 @@ def check_has_group_access(self, ToolGlobals: CDFToolConfig) -> None:
             "(projectsAcl: LIST, READ and groupsAcl: LIST, READ, CREATE, UPDATE, DELETE)..."
         )
         try:
-            ToolGlobals.verify_client(
-                capabilities={
-                    "projectsAcl": [
-                        "LIST",
-                        "READ",
-                    ],
-                    "groupsAcl": ["LIST", "READ", "CREATE", "UPDATE", "DELETE"],
-                }
+            ToolGlobals.verify_authorization(
+                [
+                    ProjectsAcl([ProjectsAcl.Action.List, ProjectsAcl.Action.Read], ProjectsAcl.Scope.All()),
+                    GroupsAcl(
+                        [
+                            GroupsAcl.Action.Read,
+                            GroupsAcl.Action.List,
+                            GroupsAcl.Action.Create,
+                            GroupsAcl.Action.Update,
+                            GroupsAcl.Action.Delete,
+                        ],
+                        GroupsAcl.Scope.All(),
+                    ),
+                ]
             )
             print("  [bold green]OK[/]")
         except Exception:
@@ -250,11 +258,11 @@ def check_has_group_access(self, ToolGlobals: CDFToolConfig) -> None:
             )
             print("Checking basic group read access rights (projectsAcl: LIST, READ and groupsAcl: LIST, READ)...")
             try:
-                ToolGlobals.verify_client(
-                    capabilities={
-                        "projectsAcl": ["LIST", "READ"],
-                        "groupsAcl": ["LIST", "READ"],
-                    }
+                ToolGlobals.verify_authorization(
+                    capabilities=[
+                        ProjectsAcl([ProjectsAcl.Action.List, ProjectsAcl.Action.Read], ProjectsAcl.Scope.All()),
+                        GroupsAcl([GroupsAcl.Action.Read, GroupsAcl.Action.List], GroupsAcl.Scope.All()),
+                    ]
                 )
                 print("  [bold green]OK[/] - can continue with checks.")
             except Exception:

diff --git a/cognite_toolkit/_cdf_tk/commands/build.py b/cognite_toolkit/_cdf_tk/commands/build.py
@@ -94,7 +94,7 @@ def execute(self, verbose: bool, source_path: Path, build_dir: Path, build_env_n
         print(
             Panel(
                 f"Building {directory_name}:\n  - Toolkit Version '{__version__!s}'\n"
-                f"  - Environment {build_env_name!r}\n"
+                f"  - Environment name {build_env_name!r}, type {config.environment.build_type!r}.\n"
                 f"  - Config '{config.filepath!s}'"
                 f"\n{module_locations}",
                 expand=False,
@@ -136,7 +136,8 @@ def build_config(
         else:
             build_dir.mkdir(exist_ok=True)
 
-        config.validate_environment()
+        if issue := config.validate_environment():
+            self.warn(issue)
 
         module_parts_by_name: dict[str, list[tuple[str, ...]]] = defaultdict(list)
         available_modules: set[str | tuple[str, ...]] = set()

diff --git a/cognite_toolkit/_cdf_tk/commands/clean.py b/cognite_toolkit/_cdf_tk/commands/clean.py
@@ -79,7 +79,7 @@ def clean_resources(
 
         capabilities = loader.get_required_capability(loaded_resources)
         if capabilities:
-            ToolGlobals.verify_capabilities(capabilities)
+            ToolGlobals.verify_authorization(capabilities, action=f"clean {loader.display_name}")
 
         nr_of_items = len(loaded_resources)
         if nr_of_items == 0:

diff --git a/cognite_toolkit/_cdf_tk/commands/deploy.py b/cognite_toolkit/_cdf_tk/commands/deploy.py
@@ -190,6 +190,9 @@ def _deploy_resources(
         verbose: bool = False,
     ) -> ResourceDeployResult | None:
         filepaths = loader.find_files()
+        if not filepaths:
+            self.warn(LowSeverityWarning(f"No {loader.display_name} files found. Skipping..."))
+            return None
 
         def sort_key(p: Path) -> int:
             if result := re.findall(r"^(\d+)", p.stem):
@@ -214,7 +217,7 @@ def sort_key(p: Path) -> int:
 
         capabilities = loader.get_required_capability(loaded_resources)
         if capabilities:
-            ToolGlobals.verify_capabilities(capabilities)
+            ToolGlobals.verify_authorization(capabilities, action=f"deploy {loader.display_name}")
 
         nr_of_items = len(loaded_resources)
         if nr_of_items == 0:

diff --git a/cognite_toolkit/_cdf_tk/commands/describe.py b/cognite_toolkit/_cdf_tk/commands/describe.py
@@ -3,6 +3,7 @@
 import datetime
 
 from cognite.client.data_classes.aggregations import Count
+from cognite.client.data_classes.capabilities import DataModelInstancesAcl, DataModelsAcl
 from cognite.client.data_classes.data_modeling import (
     DirectRelation,
     DirectRelationReference,
@@ -29,11 +30,14 @@ def execute(self, ToolGlobals: CDFToolConfig, space_name: str, model_name: str |
         else:
             print(f"Describing data model {model_name} in space {space_name} in project {ToolGlobals.project}...")
         print("Verifying access rights...")
-        client = ToolGlobals.verify_client(
-            capabilities={
-                "dataModelsAcl": ["READ", "WRITE"],
-                "dataModelInstancesAcl": ["READ", "WRITE"],
-            }
+        client = ToolGlobals.verify_authorization(
+            [
+                DataModelsAcl([DataModelsAcl.Action.Read, DataModelsAcl.Action.Write], DataModelsAcl.Scope.All()),
+                DataModelInstancesAcl(
+                    [DataModelInstancesAcl.Action.Read, DataModelInstancesAcl.Action.Write],
+                    DataModelInstancesAcl.Scope.All(),
+                ),
+            ]
         )
         table = Table(title=f"Space {space_name}")
         table.add_column("Info", justify="left")

diff --git a/cognite_toolkit/_cdf_tk/commands/dump.py b/cognite_toolkit/_cdf_tk/commands/dump.py
@@ -4,6 +4,7 @@
 
 import yaml
 from cognite.client import data_modeling as dm
+from cognite.client.data_classes.capabilities import DataModelsAcl
 from cognite.client.data_classes.data_modeling import DataModelId
 from rich import print
 from rich.panel import Panel
@@ -26,7 +27,9 @@ def execute(
     ) -> None:
         print(f"Dumping {data_model_id} from project {ToolGlobals.project}...")
         print("Verifying access rights...")
-        client = ToolGlobals.verify_client(capabilities={"dataModelsAcl": ["READ", "WRITE"]})
+        client = ToolGlobals.verify_authorization(
+            DataModelsAcl([DataModelsAcl.Action.Read], DataModelsAcl.Scope.All()),
+        )
 
         data_models = client.data_modeling.data_models.retrieve(data_model_id, inline_views=True)
         if not data_models:

diff --git a/cognite_toolkit/_cdf_tk/commands/featureflag.py b/cognite_toolkit/_cdf_tk/commands/featureflag.py
@@ -17,6 +17,7 @@
 class Flags(Enum):
     MODULES_CMD: ClassVar[dict[str, Any]] = {"visible": True, "description": "Enables the modules management subapp"}
     INTERNAL: ClassVar[dict[str, Any]] = {"visible": False, "description": "Does nothing"}
+    IMPORT_CMD: ClassVar[dict[str, Any]] = {"visible": True, "description": "Enables the import sup application"}
 
 
 class FeatureFlag:
@@ -61,7 +62,7 @@ def is_enabled(flag: str | Flags) -> bool:
     @lru_cache
     def to_flag(flag: str) -> Flags | None:
         try:
-            return Flags[flag.upper()]
+            return Flags[flag.upper().replace("-", "_")]
         except KeyError:
             return None
 

diff --git a/cognite_toolkit/_cdf_tk/constants.py b/cognite_toolkit/_cdf_tk/constants.py
@@ -38,3 +38,9 @@
 COGNITE_MODULES_PATH = ROOT_PATH / COGNITE_MODULES
 
 SUPPORT_MODULE_UPGRADE_FROM_VERSION = "0.1.0"
+
+
+class URL:
+    configure_access = "https://docs.cognite.com/cdf/deploy/cdf_deploy/cdf_deploy_access_management"
+    auth_toolkit = "https://docs.cognite.com/cdf/deploy/cdf_toolkit/guides/configure_deploy_modules#configure-the-cdf-toolkit-authentication"
+    docs = "https://docs.cognite.com/"