Release 0.3.13 (#1235)

# Description

Please describe the change you have made.

## Checklist

- [ ] Tests added/updated.
- [ ] Run Demo Job Locally.
- [ ] Documentation updated.
- [ ] Changelogs updated in
[CHANGELOG.cdf-tk.md](https://github.com/cognitedata/toolkit/blob/main/CHANGELOG.cdf-tk.md).
- [ ] Template changelogs updated in
[CHANGELOG.templates.md](https://github.com/cognitedata/toolkit/blob/main/CHANGELOG.templates.md).
- [ ] Version bumped.

[_version.py](https://github.com/cognitedata/toolkit/blob/main/cognite/cognite_toolkit/_version.py)
and

[pyproject.toml](https://github.com/cognitedata/toolkit/blob/main/pyproject.toml)
per [semantic versioning](https://semver.org/).

.pre-commit-config.yaml

@@ -12,7 +12,7 @@ repos:
           - --fixable=E,W,F,I,T,RUF,TID,UP
           - --target-version=py39
       - id: ruff-format
-    rev: v0.7.4
+    rev: v0.8.0
 
   - repo: https://github.com/igorshubovych/markdownlint-cli
     rev: v0.42.0

CHANGELOG.cdf-tk.md

@@ -15,6 +15,15 @@ Changes are grouped as follows:
 - `Fixed` for any bug fixes.
 - `Security` in case of vulnerabilities.
 
+## [0.3.13] - 2024-11-26
+
+### Fixed
+
+- When using the commands `cdf deploy/clean`, the Toolkit no longer warns about unrelated resources
+  that are not part of the deployment.
+- When using `cdf deploy/clean/build, the Toolkit no longer warns about duplicated RAW databases.
+- Changes to Streamlit app code were not deployed. This is now fixed.
+
 ## [0.3.12] - 2024-11-21
 
 ### Added

CHANGELOG.templates.md

@@ -15,6 +15,17 @@ Changes are grouped as follows:
 - `Fixed` for any bug fixes.
 - `Security` in case of vulnerabilities.
 
+## [0.3.13] - 2024-11-26
+
+## Changed
+
+- The modules `cdf_idm_extension` and `cdf_idm_extension_full` are renamed to `cdf_process_industry_extension` and
+  `cdf_process_industry_extension_full` respectively.
+- The Source System data pipelines now requires `cdf_process_industry_extension` and writes to the extension model
+  instead of the Core model.
+- The modules in Source System, Contextualization, and Industrial Tool packages now use shared resources
+  in `cdf_common`.
+
 ## [0.3.12] - 2024-11-21
 
 ### Added

cdf.toml

@@ -20,4 +20,4 @@ dump = true
 [modules]
 # This is the version of the modules. It should not be changed manually.
 # It will be updated by the 'cdf module upgrade' command.
-version = "0.3.12"
+version = "0.3.13"

cognite_toolkit/_api/data_classes.py

@@ -4,7 +4,7 @@
 from collections import UserDict, UserList
 from dataclasses import dataclass
 
-__all__ = ["Variable", "Variables", "ModuleMeta", "ModuleMetaList"]
+__all__ = ["ModuleMeta", "ModuleMetaList", "Variable", "Variables"]
 
 from pathlib import Path
 from typing import Any, cast

cognite_toolkit/_builtin_modules/cdf.toml

@@ -4,7 +4,7 @@ default_env = "<DEFAULT_ENV_PLACEHOLDER>"
 [modules]
 # This is the version of the modules. It should not be changed manually.
 # It will be updated by the 'cdf module upgrade' command.
-version = "0.3.12"
+version = "0.3.13"
 
 
 [plugins]

cognite_toolkit/_builtin_modules/cdf_common/data_sets/demo.DataSet.yaml

@@ -0,0 +1,3 @@
+name: 'Ingestion'
+externalId: {{ dataset }}
+description: This dataset contains Transformations, Functions, and Workflows for ingesting data into Cognite Data Fusion.

cognite_toolkit/_builtin_modules/cdf_common/default.config.yaml

@@ -0,0 +1,8 @@
+dataset: ingestion
+instanceSpace: springfield_instances
+directRelationJob: ctx_files_direct_relation_write
+rawSourceDatabase: ingestion
+rawStateDatabase: contextualizationState
+rawStateTable: diagramParsing
+schemaSpace: cdf_cdm
+annotationSpace: springfield_instances

cognite_toolkit/_builtin_modules/cdf_common/functions/write.Function.yaml

@@ -0,0 +1,7 @@
+name: Annotation to Direct Relations Writer
+externalId: contextualization_connection_writer
+owner: 'Anonymous'
+description: Writes all approved Annotations as direct relations connections.
+functionPath: 'handler.py'
+# Data set id for the zip file with the code that is uploaded.
+dataSetExternalId: {{ dataset }}

cognite_toolkit/_builtin_modules/cdf_common/module.toml

@@ -0,0 +1,7 @@
+[module]
+title = "cdf_common"
+is_selected_by_default = false
+
+[packages]
+tags = [
+]

cognite_toolkit/_builtin_modules/contextualization/cdf_p_and_id_parser/raw/state.Table.yaml → cognite_toolkit/_builtin_modules/cdf_common/raw/contextualization_state.DataBase.yaml

@@ -1,2 +1 @@
 dbName: {{ rawStateDatabase }}
-tableName: {{ rawStateTable }}

cognite_toolkit/_builtin_modules/cdf_common/raw/source.DataBase.yaml

@@ -0,0 +1 @@
+dbName: {{ rawSourceDatabase }}

cognite_toolkit/_builtin_modules/cdf_ingestion/auth/user.Group.yaml

@@ -0,0 +1,108 @@
+name: gp_user_{{ workflow }}
+members:
+  {{ runWorkflowUserIds }}
+metadata:
+  origin: cognite-toolkit
+capabilities:
+  # minimum runtime capabilities
+  - projectsAcl:
+      actions:
+        - READ
+        - LIST
+      scope:
+        all: {}
+  - groupsAcl:
+      actions:
+        - LIST
+      scope:
+        all: {}
+  - sessionsAcl:
+      actions:
+        - CREATE
+      scope:
+        all: {}
+  # query-specific capabilities
+  - transformationsAcl:
+      actions:
+        - READ
+        - WRITE
+      scope:
+        datasetScope:
+          ids:
+            - {{ dataset }}
+  - functionsAcl:
+      actions:
+          - READ
+          - WRITE
+      scope:
+          all: {}
+  - rawAcl:
+      actions:
+        - READ
+      scope:
+        tableScope:
+          dbsToTables:
+            {{ rawSourceDatabase }}: {}
+            {{ rawStateDatabase  }}: {}
+  - dataModelInstancesAcl:
+      actions:
+        - READ
+        - WRITE
+        - WRITE_PROPERTIES
+      scope:
+        spaceIdScope:
+          spaceIds:
+            {{ instanceSpaces }}
+  - dataModelsAcl:
+      actions:
+        - READ
+      scope:
+        spaceIdScope:
+          spaceIds:
+            - {{ schemaSpace }}
+            - {{ schemaSpace2 }}
+            - {{ schemaSpace3 }}
+            - cdf_cdm_units
+  # Needed to write CogniteAsset.parent
+  - dataModelInstancesAcl:
+      actions:
+        - READ
+      scope:
+        spaceIdScope:
+          spaceIds:
+            - cdf_cdm
+  - annotationsAcl:
+      actions:
+        - READ
+        - WRITE
+        - SUGGEST
+        - REVIEW
+      scope:
+        all: {}
+  - entitymatchingAcl:
+      actions:
+        - READ
+        - WRITE
+      scope:
+        all: {}
+  - extractionConfigsAcl:
+      actions:
+          - READ
+      scope:
+        datasetScope:
+          ids:
+            - {{ dataset }}
+  - extractionRunsAcl:
+      actions:
+          - READ
+          - WRITE
+      scope:
+        datasetScope:
+          ids:
+            - {{ dataset }}
+# Bug in /context/diagram/detect/ endpoint that requires this
+  - assetsAcl:
+      actions:
+        - READ
+      scope:
+        all: {}

cognite_toolkit/_builtin_modules/cdf_ingestion/auth/workflow.Group.yaml

@@ -60,8 +60,15 @@ capabilities:
           spaceIds:
             - {{ schemaSpace }}
             - {{ schemaSpace2 }}
+            - {{ schemaSpace3 }}
             - cdf_cdm_units
-            # Needed to write CogniteAsset.parent
+  # Needed to write CogniteAsset.parent
+  - dataModelInstancesAcl:
+      actions:
+        - READ
+      scope:
+        spaceIdScope:
+          spaceIds:
             - cdf_cdm
   - annotationsAcl:
       actions:

cognite_toolkit/_builtin_modules/cdf_ingestion/default.config.yaml

@@ -19,12 +19,16 @@ timeSeriesToEquipmentTransformationExternalId: timeseries_to_equipment
 pandidContextualizationFunction: contextualization_p_and_id_annotater
 contextualization_connection_writer: contextualization_connection_writer
 
-
 dataset: ingestion
 rawSourceDatabase: ingestion
 rawStateDatabase: contextualizationState
-schemaSpace: cdf_cdm
-schemaSpace2: cdf_idm
+schemaSpace: sp_enterprise_process_industry
+schemaSpace2: cdf_cdm
+schemaSpace3: cdf_idm
+
 instanceSpaces:
   - springfield_instances
   - cdf_cdm_units
+
+runWorkflowUserIds:
+  - <your user id>

cognite_toolkit/_builtin_modules/contextualization/cdf_connection_sql/module.toml

@@ -9,4 +9,11 @@ tags = [
 ]
 
 [dependencies]
-modules = []
+modules = []
+
+[[extra_resources]]
+location = "cdf_common/data_sets/demo.DataSet.yaml"
+
+[[extra_resources]]
+location = "cdf_common/data_models/demo.Space.yaml"
+

cognite_toolkit/_builtin_modules/contextualization/cdf_entity_matching/functions/functions.Function.yaml

@@ -5,10 +5,3 @@
   functionPath: 'handler.py'
   # Data set id for the zip file with the code that is uploaded.
   dataSetExternalId: {{ dataset }}
-- name: Annotation to Direct Relations Writer
-  externalId: contextualization_connection_writer_duplicate
-  owner: 'Anonymous'
-  description: Writes all approved Annotations as direct relations connections.
-  functionPath: 'handler.py'
-  # Data set id for the zip file with the code that is uploaded.
-  dataSetExternalId: {{ dataset }}

cognite_toolkit/_builtin_modules/contextualization/cdf_entity_matching/module.toml

@@ -5,3 +5,24 @@ title = "Contextualization Entity Matching"
 tags = [
     "contextualization",
 ]
+
+[[extra_resources]]
+location = "cdf_common/data_sets/demo.DataSet.yaml"
+
+[[extra_resources]]
+location = "cdf_common/data_models/demo.Space.yaml"
+
+[[extra_resources]]
+location = "cdf_common/extraction_pipelines/ctx_files_direct_relation_write.config.yaml"
+
+[[extra_resources]]
+location = "cdf_common/extraction_pipelines/ctx_files_direct_relation_write.ExtractionPipeline.yaml"
+
+[[extra_resources]]
+location = "cdf_common/functions/contextualization_connection_writer"
+
+[[extra_resources]]
+location = "cdf_common/functions/write.Function.yaml"
+
+[[extra_resources]]
+location = "cdf_common/raw/contextualization_state.DataBase.yaml"

cognite_toolkit/_builtin_modules/contextualization/cdf_p_and_id_parser/default.config.yaml

@@ -1,10 +1,7 @@
 contextualizationId: p_and_id_parser
 contextualizationName: P&ID Parser
 annotatorJob: ctx_files_pandid_annotater
-directRelationJob: ctx_files_direct_relation_write
 dataset: ingestion
-rawStateDatabase: contextualizationState
-rawStateTable: diagramParsing
 schemaSpace: cdf_cdm
 annotationSpace: springfield_instances
 instanceSpaces: