Release 0.2.0a3 (#474)

# Description Please describe the change you have made. ## Checklist - [ ] Tests added/updated. - [ ] Run Demo Job Locally. - [ ] Documentation updated. - [ ] Changelogs updated in [CHANGELOG.cdf-tk.md](https://github.com/cognitedata/cdf-project-templates/blob/main/CHANGELOG.cdf-tk.md). - [ ] Template changelogs updated in [CHANGELOG.templates.md](https://github.com/cognitedata/cdf-project-templates/blob/main/CHANGELOG.templates.md). - [ ] Version bumped. [_version.py](https://github.com/cognitedata/cdf-project-templates/blob/main/cognite/cognite_toolkit/_version.py) and [pyproject.toml](https://github.com/cognitedata/cdf-project-templates/blob/main/pyproject.toml) per [semantic versioning](https://semver.org/).
cognitedata · Apr 23, 2024 · be9c5cf · be9c5cf
2 parents a140440 + b3d5af8
commit be9c5cf
Show file tree

Hide file tree

Showing 108 changed files with 1,476 additions and 507 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -2,15 +2,15 @@
 name: Lint and test
 on:
   pull_request:
-    branches: [main, release]
+
 env:
   PYTHON_VERSION: '3.9'
 jobs:
   lint:
     name: Run linters
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4
+      - uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f # v4
       - uses: actions/setup-python@v5
         with:
           python-version: ${{ env.PYTHON_VERSION }}
@@ -41,7 +41,7 @@ jobs:
           # Skipping 3.10 and 3.11 as we assume it is covered by 3.9 and 3.12
           - "3.12"
     steps:
-      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4
+      - uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f # v4
       - uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
@@ -68,7 +68,7 @@ jobs:
       IDP_CLIENT_SECRET: ${{ secrets.IDP_CLIENT_SECRET }}
       IDP_TOKEN_URL: ${{ secrets.IDP_TOKEN_URL }}
     steps:
-      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4
+      - uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f # v4
       - uses: actions/setup-python@v5
         with:
           python-version: 3.11

diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml
@@ -10,7 +10,7 @@ jobs:
     name: Run linters
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4
+      - uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f # v4
       - uses: actions/setup-python@v5
         with:
           python-version: ${{ env.PYTHON_VERSION }}
@@ -34,7 +34,7 @@ jobs:
       IDP_CLIENT_SECRET: ${{ secrets.IDP_CLIENT_SECRET }}
       IDP_TOKEN_URL: ${{ secrets.IDP_TOKEN_URL }}
     steps:
-      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4
+      - uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f # v4
       - uses: actions/setup-python@v5
         with:
           python-version: ${{ env.PYTHON_VERSION }}

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -1,6 +1,6 @@
 ---
 repos:
-  - repo: https://github.com/charliermarsh/ruff-pre-commit
+  - repo: https://github.com/astral-sh/ruff-pre-commit
     hooks:
       - id: ruff
         args:
@@ -12,22 +12,13 @@ repos:
           - --select=E,W,F,I,T,RUF,TID,UP
           - --fixable=E,W,F,I,T,RUF,TID,UP
           - --target-version=py39
-    rev: v0.3.4
-
-  - repo: https://github.com/psf/black
-    rev: 24.3.0
-    hooks:
-      - id: black
+      - id: ruff-format
         args:
           - --line-length=120
-          - --target-version=py38
-          - --target-version=py39
-          - --target-version=py310
-          - --target-version=py311
-          - --include=\.py$
+    rev: v0.4.1
 
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.5.0
+    rev: v4.6.0
     hooks:
       - id: debug-statements
       - id: check-docstring-first

diff --git a/.vscode/launch.json b/.vscode/launch.json
@@ -14,7 +14,6 @@
                 "--override-env",
                 "build",
                 "--build-dir=build",
-                "--clean",
                 "--env=local",
                 "./cognite_toolkit/"
             ],
@@ -52,6 +51,19 @@
             "console": "integratedTerminal",
             "justMyCode": false
         },
+        {
+            "name": "Python: auth verify",
+            "type": "debugpy",
+            "request": "launch",
+            "program": "./cdf-tk-dev.py",
+            "args": [
+                "--override-env",
+                "auth",
+                "verify",
+            ],
+            "console": "integratedTerminal",
+            "justMyCode": false
+        },
         {
             "name": "Python: cdf.py",
             "type": "debugpy",

diff --git a/CHANGELOG.cdf-tk.md b/CHANGELOG.cdf-tk.md
@@ -15,6 +15,35 @@ Changes are grouped as follows:
 - `Fixed` for any bug fixes.
 - `Security` in case of vulnerabilities.
 
+## [0.2.0a3] - 2024-04-23
+
+### Added
+
+- Support for the Workflow and WorkflowVersion resource type
+- Support for specifying `selected_modules_and_packages` as paths and parent paths. For example, you can
+  now write `cognite_modules/core/cdf_apm_base` instead of just `cdf_apm_base`. This is to support
+  modules that have the same name but are in different parent directories. In addition, this also better reflects
+  the structure of the `cognite_modules` and `custom_modules` folder better.
+
+### Fixed
+
+- Functions that are deployed with schedules no longer uses a short-lived session (before: failed after ~an hour).
+
+### Changed
+
+- [BREAKING] The `cdf-tk build` will now clean the build directory by default before building the modules to avoid
+  unwanted side effects from previous builds. To stop this behavior, use the `--no-clean` flag.
+- [BREAKING] The `_system.yaml` is now required to be on the root level when running any `cdf-tk` command. This
+  means that the `_system.yaml` file must be in the same directory as the `config.[env].yaml` files. This is to
+  support running `cdf-tk` without the `cognite_modules` folder.
+- [BREAKING] In the `config.[env].yaml` files, the `modules` section is now renamed to `variables`. This is to
+  better reflect the content of this section, which is variables that can be used in the resource files.
+- In addition to `cognite_modules` and `custom_modules`, the `cognite-toolkit` now also support `modules` as the
+  top-level folder for modules. This together with the two changes above, is to have a better support for running
+  the `cognite-toolkit` as a standalone CLI without the `cognite_modules`.
+- The `.gitignore` file you get by running `cdf-tk init` now ignores the `/build` by default.
+- The dependency `cognite-sdk` must now be `>=7.37.0` to use the `cdf-tk`.
+
 ## [0.2.0a2] - 2024-04-03
 
 ### Added

diff --git a/CHANGELOG.templates.md b/CHANGELOG.templates.md
@@ -15,6 +15,23 @@ Changes are grouped as follows:
 - `Fixed` for any bug fixes.
 - `Security` in case of vulnerabilities.
 
+## [0.2.0a3] - 2024-04-23
+
+### Fixed
+
+- Align tag name in asset hierarchy between the 2 example transformations
+- Added default root asset ID to documents for initial annotation
+- Aligned use of asset external ID across contextualization functions
+- Annotation logic with local time stamp for when to reprocess P&ID files
+- Input to P&ID annotation based on list of synonyms for tag
+- Updated module `apm_simple_data_model` for `cognite-sdk>=7.37`, i.e., container properties
+  of type `direct` have now `list: false` explicitly set.
+
+### Added
+
+- Added Transformation for WorkOrder and WorkItems to OID testdata template
+- Added Workflow with a Function and a Transformation to the custom module example
+
 ## [0.2.0a2] - 2024-04-03
 
 No changes to templates.

diff --git a/cdf-tk-dev.py b/cdf-tk-dev.py
@@ -6,23 +6,23 @@
 # However, when you run the file in Visual Studio Code, the module should not be installed in your
 # python virtual environment, but rather be found in the root of the repo.
 # This workaround allows you to run cdf.py in Visual Studio Code like this:
-"""         {
-            "name": "Python: build",
-            "type": "python",
-            "request": "launch",
-            "program": "./cdf-tk-dev.py",
-            "args": [
-                "--verbose",
-                "--override-env",
-                "build",
-                "--build-dir=build",
-                "--clean",
-                "--env=local",
-                "./cognite_toolkit/"
-            ],
-            "console": "integratedTerminal",
-            "justMyCode": false
-        },
+"""{
+    "name": "Python: build",
+    "type": "python",
+    "request": "launch",
+    "program": "./cdf-tk-dev.py",
+    "args": [
+        "--verbose",
+        "--override-env",
+        "build",
+        "--build-dir=build",
+        "--clean",
+        "--env=local",
+        "./cognite_toolkit/"
+    ],
+    "console": "integratedTerminal",
+    "justMyCode": false
+},
 """
 
 import os

diff --git a/cognite_toolkit/.gitignore b/cognite_toolkit/.gitignore
@@ -273,4 +273,5 @@ $RECYCLE.BIN/
 # Custom rules (everything added below won't be overriden by 'Generate .gitignore File' if you use 'Update' option)
 */.venv.*
 .venv.*
-build.*
+build.*
+build/
diff --git a/cognite_toolkit/_api/data_classes.py b/cognite_toolkit/_api/data_classes.py
@@ -78,7 +78,7 @@ def _load(cls, module_path: Path, default_variables: dict[tuple[str, ...], Confi
             variable_module_key = [*module_key_path, "first_pop"]
             while variable_module_key:
                 variable_module_key.pop()
-                key_path = (InitConfigYAML._modules, *variable_module_key, variable_name)
+                key_path = (InitConfigYAML._variables, *variable_module_key, variable_name)
                 if default := default_variables.get(key_path):
                     value.default_value = default.default_value
                     value.default_comment = default.default_comment

diff --git a/cognite_toolkit/_api/modules_api.py b/cognite_toolkit/_api/modules_api.py
@@ -37,7 +37,7 @@ def _source_dir(self) -> Path:
     def _load_modules(self) -> None:
         source_dir = self._source_dir()
 
-        system_yaml = SystemYAML.load_from_directory(source_dir, self._build_env)
+        system_yaml = SystemYAML.load_from_directory(source_dir.parent, self._build_env)
         default_config = InitConfigYAML(_DUMMY_ENVIRONMENT).load_defaults(source_dir)
 
         for module, _ in iterate_modules(source_dir):
@@ -76,13 +76,13 @@ def _build(self, modules: Sequence[ModuleMeta], verbose: bool) -> None:
                 common_function_code="./common_function_code",
             ),
             filepath=Path(""),
-            modules=variables,
+            variables=variables,
         )
         build_config(
             self._build_dir,
             self._source_dir().parent,
             config,
-            system_config=SystemYAML.load_from_directory(self._source_dir(), self._build_env),
+            system_config=SystemYAML.load_from_directory(self._source_dir().parent, self._build_env),
             clean=True,
             verbose=verbose,
         )

diff --git a/cognite_toolkit/_cdf.py b/cognite_toolkit/_cdf.py
@@ -10,14 +10,12 @@
 from pathlib import Path
 from typing import Annotated, Optional, Union, cast
 
-import sentry_sdk
 import typer
 from cognite.client.data_classes.data_modeling import DataModelId, NodeId
 from dotenv import load_dotenv
 from rich import print
 from rich.panel import Panel
 
-from cognite_toolkit import _version
 from cognite_toolkit._cdf_tk import bootstrap
 from cognite_toolkit._cdf_tk.constants import _RUNNING_IN_BROWSER
 from cognite_toolkit._cdf_tk.describe import describe_datamodel
@@ -49,8 +47,11 @@
 from cognite_toolkit._version import __version__ as current_version
 
 if "pytest" not in sys.modules and os.environ.get("SENTRY_ENABLED", "true").lower() == "true":
+    import sentry_sdk
+
     sentry_sdk.init(
         dsn="https://[email protected]/4506429021093888",
+        release=current_version,
         # Set traces_sample_rate to 1.0 to capture 100%
         # of transactions for performance monitoring.
         traces_sample_rate=1.0,
@@ -111,7 +112,7 @@ class Common:
 
 def _version_callback(value: bool) -> None:
     if value:
-        typer.echo(f"CDF-Toolkit version: {_version.__version__}.")
+        typer.echo(f"CDF-Toolkit version: {current_version}.")
         raise typer.Exit()
 
 
@@ -233,12 +234,10 @@ def build(
             help="Build environment to build for",
         ),
     ] = "dev",
-    clean: Annotated[
+    no_clean: Annotated[
         bool,
         typer.Option(
-            "--clean",
-            "-c",
-            help="Delete the build directory before building the configurations",
+            "--no-clean", "-c", help="Whether not to delete the build directory before building the configurations"
         ),
     ] = False,
 ) -> None:
@@ -247,9 +246,7 @@ def build(
     if not source_path.is_dir():
         print(f"  [bold red]ERROR:[/] {source_path} does not exist")
         exit(1)
-    cognite_modules_path = source_path / COGNITE_MODULES
-
-    system_config = SystemYAML.load_from_directory(cognite_modules_path, build_env)
+    system_config = SystemYAML.load_from_directory(source_path, build_env)
     config = BuildConfigYAML.load_from_directory(source_path, build_env)
     print(
         Panel(
@@ -264,7 +261,7 @@ def build(
         source_dir=source_path,
         config=config,
         system_config=system_config,
-        clean=clean,
+        clean=not no_clean,
         verbose=ctx.obj.verbose,
     )
 

diff --git a/cognite_toolkit/_cdf_tk/load/__init__.py b/cognite_toolkit/_cdf_tk/load/__init__.py
@@ -68,6 +68,7 @@
     "extraction_pipelines",
     "functions",
     "raw",
+    "workflows",
 ]
 
 __all__ = [
@@ -98,4 +99,6 @@
     "DeployResult",
     "DeployResults",
     "ResourceTypes",
+    "WorkflowLoader",
+    "WorkflowVersionLoader",
 ]
diff --git a/cognite_toolkit/_cdf_tk/load/_base_loaders.py b/cognite_toolkit/_cdf_tk/load/_base_loaders.py
@@ -10,6 +10,7 @@
 from typing import Any, Generic, TypeVar, Union
 
 from cognite.client import CogniteClient
+from cognite.client.data_classes import WorkflowVersionId
 from cognite.client.data_classes._base import (
     T_CogniteResourceList,
     T_WritableCogniteResource,
@@ -35,7 +36,10 @@
     UploadDeployResult,
 )
 
-T_ID = TypeVar("T_ID", bound=Union[str, int, DataModelingId, InstanceId, VersionedDataModelingId, RawDatabaseTable])
+T_ID = TypeVar(
+    "T_ID",
+    bound=Union[str, int, DataModelingId, InstanceId, VersionedDataModelingId, RawDatabaseTable, WorkflowVersionId],
+)
 T_WritableCogniteResourceList = TypeVar("T_WritableCogniteResourceList", bound=WriteableCogniteResourceList)
-Original file line number
+Diff line change
@@ Expand Up / @@ -273,4 +273,5 @@ $RECYCLE.BIN/ @@
     # Custom rules (everything added below won't be overriden by 'Generate .gitignore File' if you use 'Update' option)
     */.venv.*
     .venv.*
-    build.*
+    build.*
+    build/