From 818854980325e4644b41fd79fc88241b4e588dea Mon Sep 17 00:00:00 2001 From: Jan Inge Bergseth <31886431+BergsethCognite@users.noreply.github.com> Date: Thu, 4 Apr 2024 11:25:49 +0200 Subject: [PATCH 01/34] div bug fixes and alignment with oid test data --- .../tr_asset_oid_workmate_asset_hierarchy.sql | 8 ++--- .../default.config.yaml | 1 + ...id-fileshare-pandid_annotation.config.yaml | 2 +- ...files-oid-fileshare-pandid_annotation.yaml | 2 +- .../pipeline.py | 35 ++++++++++++------- .../tr_files_oid_fileshare_file_metadata.sql | 8 ++++- .../default.config.yaml | 1 + ...ctx_timeseries_oid_opcua_asset.config.yaml | 4 +-- .../ctx_timeseries_oid_opcua_asset.yaml | 2 +- .../pipeline.py | 3 ++ ...t_oid_workmate_asset_hierarchy_example.sql | 2 +- cognite_toolkit/config.yaml | 4 +++ .../cdf_data_pipeline_asset_valhall.yaml | 10 +++--- .../cdf_data_pipeline_files_valhall.yaml | 26 +++++++------- .../cdf_data_pipeline_timeseries_valhall.yaml | 14 ++++---- .../cdf_oid_example_data.yaml | 4 +-- .../auth_auth_verify_happypath_windows.txt | 6 ++-- ...th_auth_verify_no_capabilities_windows.txt | 6 ++-- .../auth_auth_verify_two_groups_windows.txt | 6 ++-- ...auth_verify_wrong_capabilities_windows.txt | 6 ++-- 20 files changed, 85 insertions(+), 65 deletions(-) diff --git a/cognite_toolkit/cognite_modules/examples/cdf_data_pipeline_asset_valhall/transformations/tr_asset_oid_workmate_asset_hierarchy.sql b/cognite_toolkit/cognite_modules/examples/cdf_data_pipeline_asset_valhall/transformations/tr_asset_oid_workmate_asset_hierarchy.sql index be59d2abf..c3f2ab6e2 100644 --- a/cognite_toolkit/cognite_modules/examples/cdf_data_pipeline_asset_valhall/transformations/tr_asset_oid_workmate_asset_hierarchy.sql +++ b/cognite_toolkit/cognite_modules/examples/cdf_data_pipeline_asset_valhall/transformations/tr_asset_oid_workmate_asset_hierarchy.sql @@ -8,16 +8,16 @@ -- All metadata expect selected fileds are added to metadata -- SELECT - sourceDb || ':' || tag as externalId, - if(parentTag is null, + externalId as externalId, + if(parentExternalId is null, '', - sourceDb || ':' ||parentTag) as parentExternalId, + parentExternalId) as parentExternalId, tag as name, sourceDb as source, description, dataset_id('{{asset_dataset}}') as dataSetId, to_metadata_except( - array("sourceDb", "parentTag", "description"), *) + array("sourceDb", "parentExternalId", "description"), *) as metadata FROM `{{asset_raw_input_db}}`.`{{asset_raw_input_table}}` diff --git a/cognite_toolkit/cognite_modules/examples/cdf_data_pipeline_files_valhall/default.config.yaml b/cognite_toolkit/cognite_modules/examples/cdf_data_pipeline_files_valhall/default.config.yaml index a1782f78b..78ee8be3b 100644 --- a/cognite_toolkit/cognite_modules/examples/cdf_data_pipeline_files_valhall/default.config.yaml +++ b/cognite_toolkit/cognite_modules/examples/cdf_data_pipeline_files_valhall/default.config.yaml @@ -16,6 +16,7 @@ files_dataset: ds_files_oid pause_transformations: true files_raw_input_db: files_oid_fileshare files_raw_input_table: files_metadata +external_root_id_asset: WMT:VAL # source ID from Azure AD for the corresponding groups, ex 'c74797ce-9191-4a4a-9186-8fe21c54c3de' files_location_extractor_group_source_id: diff --git a/cognite_toolkit/cognite_modules/examples/cdf_data_pipeline_files_valhall/extraction_pipelines/ctx_files-oid-fileshare-pandid_annotation.config.yaml b/cognite_toolkit/cognite_modules/examples/cdf_data_pipeline_files_valhall/extraction_pipelines/ctx_files-oid-fileshare-pandid_annotation.config.yaml index 99ef08f4b..e47db3400 100644 --- a/cognite_toolkit/cognite_modules/examples/cdf_data_pipeline_files_valhall/extraction_pipelines/ctx_files-oid-fileshare-pandid_annotation.config.yaml +++ b/cognite_toolkit/cognite_modules/examples/cdf_data_pipeline_files_valhall/extraction_pipelines/ctx_files-oid-fileshare-pandid_annotation.config.yaml @@ -9,6 +9,6 @@ config: docDataSetExtId: "{{files_dataset}}" docTypeMetaCol: "doc_type" pAndIdDocType: "PNID" - assetRootExtIds: [workmate:VAL] + assetRootExtIds: [{{external_root_id_asset}}] matchThreshold: 0.85 diff --git a/cognite_toolkit/cognite_modules/examples/cdf_data_pipeline_files_valhall/extraction_pipelines/ctx_files-oid-fileshare-pandid_annotation.yaml b/cognite_toolkit/cognite_modules/examples/cdf_data_pipeline_files_valhall/extraction_pipelines/ctx_files-oid-fileshare-pandid_annotation.yaml index 795fa1fd8..9d0bdf56b 100644 --- a/cognite_toolkit/cognite_modules/examples/cdf_data_pipeline_files_valhall/extraction_pipelines/ctx_files-oid-fileshare-pandid_annotation.yaml +++ b/cognite_toolkit/cognite_modules/examples/cdf_data_pipeline_files_valhall/extraction_pipelines/ctx_files-oid-fileshare-pandid_annotation.yaml @@ -30,7 +30,7 @@ documentation: > # Document type for P&ID type documents pAndIdDocType: "PNID" # List of externalId for root assets to be used for annotation of documents. Documents are also extracted based on the root asset ID - assetRootExtIds: [workmate:VAL] + assetRootExtIds: [{{external_root_id_asset}}] # Number between 0 and 1, indication of strict the matching should be 1 = only exact matches matchTreshold: 0.85 ``` diff --git a/cognite_toolkit/cognite_modules/examples/cdf_data_pipeline_files_valhall/functions/fn_context_files_oid_fileshare_annotation/pipeline.py b/cognite_toolkit/cognite_modules/examples/cdf_data_pipeline_files_valhall/functions/fn_context_files_oid_fileshare_annotation/pipeline.py index 77ede07a5..cb1611332 100644 --- a/cognite_toolkit/cognite_modules/examples/cdf_data_pipeline_files_valhall/functions/fn_context_files_oid_fileshare_annotation/pipeline.py +++ b/cognite_toolkit/cognite_modules/examples/cdf_data_pipeline_files_valhall/functions/fn_context_files_oid_fileshare_annotation/pipeline.py @@ -6,7 +6,7 @@ import traceback from collections import defaultdict from dataclasses import dataclass -from datetime import datetime, timedelta, timezone +from datetime import datetime, timedelta from pathlib import Path from typing import Any, Optional @@ -64,7 +64,7 @@ def annotate_pnid(client: CogniteClient, config: AnnotationConfig) -> None: """ Read configuration and start P&ID annotation process by 1. Reading files to annotate - 2. Get file entities to be matched aganst files in P&ID + 2. Get file entities to be matched against files in P&ID 3. Read existing annotations for the found files 4. Get assets and put it into the list of entities to be found in the P&ID 5. Process file: @@ -79,15 +79,19 @@ def annotate_pnid(client: CogniteClient, config: AnnotationConfig) -> None: for asset_root_xid in config.asset_root_xids: try: all_files, files_to_process = get_files(client, asset_root_xid, config) - entities = get_files_entities(all_files) - annotation_list = get_existing_annotations(client, entities) if entities else {} - error_count, annotated_count = 0, 0 - if files_to_process: - append_asset_entities(entities, client, asset_root_xid) - annotated_count, error_count = process_files( - client, entities, files_to_process, annotation_list, config - ) + + # if no files to annotate - continue to next asset + if len(files_to_process) > 0: + + entities = get_files_entities(all_files) + annotation_list = get_existing_annotations(client, entities) if entities else {} + + if files_to_process: + append_asset_entities(entities, client, asset_root_xid) + annotated_count, error_count = process_files( + client, entities, files_to_process, annotation_list, config + ) msg = ( f"Annotated P&ID files for asset: {asset_root_xid} number of files annotated: {annotated_count}, " f"file not annotated due to errors: {error_count}" @@ -111,6 +115,9 @@ def update_extpipe_run(client, xid, status, message): def get_file_list(client: CogniteClient, asset_root_xid: str, config: AnnotationConfig) -> FileMetadataList: + """ + Get list of files based on doc_type and mime_type to find P&ID files + """ return client.files.list( metadata={config.doc_type_meta_col: config.pnid_doc_type}, data_set_external_ids=[config.doc_data_set_xid], @@ -276,6 +283,9 @@ def append_asset_entities(entities: list[Entity], client: CogniteClient, asset_r if split_name[0].isnumeric(): names.append(name[len(split_name[0]) + 1 :]) + if split_name[0] + ":" in name: + names.append(name[len(split_name[0]) + 1 :]) + # add wildcards as second element to tag names.append(f"{split_name[0]}-xxx-{name[len(split_name[0])+1:]}") @@ -283,7 +293,7 @@ def append_asset_entities(entities: list[Entity], client: CogniteClient, asset_r if split_name[-1].isnumeric() and len(split_name[-1]) > 3: names.append(name[0 : (len(name) - 3)] + "x" + name[(len(name) - 2) :]) - entities.append(Entity(external_id=asset.external_id, org_name=name, name=name, id=asset.id, type="asset")) + entities.append(Entity(external_id=asset.external_id, org_name=name, name=names, id=asset.id, type="asset")) except Exception as e: print( @@ -349,7 +359,8 @@ def process_files( annotated_count += 1 # Note: add a minute to make sure annotation time is larger than last update time: - timestamp = (datetime.now(timezone.utc) + timedelta(minutes=1)).strftime(ISO_8601) + # using local time, since file update time also uses local time + timestamp = (datetime.now() + timedelta(minutes=1)).strftime(ISO_8601) my_update = ( FileMetadataUpdate(id=file.id) .asset_ids.set(asset_ids_list) diff --git a/cognite_toolkit/cognite_modules/examples/cdf_data_pipeline_files_valhall/transformations/tr_files_oid_fileshare_file_metadata.sql b/cognite_toolkit/cognite_modules/examples/cdf_data_pipeline_files_valhall/transformations/tr_files_oid_fileshare_file_metadata.sql index 44b4a15cf..d93a97778 100644 --- a/cognite_toolkit/cognite_modules/examples/cdf_data_pipeline_files_valhall/transformations/tr_files_oid_fileshare_file_metadata.sql +++ b/cognite_toolkit/cognite_modules/examples/cdf_data_pipeline_files_valhall/transformations/tr_files_oid_fileshare_file_metadata.sql @@ -3,16 +3,22 @@ -- -- Input data from RAW DB table (using example data) and uploaded files -- +With + root_id AS ( + Select id from _cdf.asset where externalId = '{{external_root_id_asset}}' + ) SELECT file.id as id, file.name as name, '{{location_name}}_'|| file.name as externalId, meta.source as source, meta.`mime_type` as mimeType, + array(root_id.id) as assetIds, dataset_id('{{files_dataset}}') as dataSetId, to_metadata(doc_type) as metadata FROM `{{files_raw_input_db}}`.`{{files_raw_input_table}}` meta, - _cdf.files file + _cdf.files file, + root_id WHERE file.name = meta.name diff --git a/cognite_toolkit/cognite_modules/examples/cdf_data_pipeline_timeseries_valhall/default.config.yaml b/cognite_toolkit/cognite_modules/examples/cdf_data_pipeline_timeseries_valhall/default.config.yaml index e09bef8ab..100801934 100644 --- a/cognite_toolkit/cognite_modules/examples/cdf_data_pipeline_timeseries_valhall/default.config.yaml +++ b/cognite_toolkit/cognite_modules/examples/cdf_data_pipeline_timeseries_valhall/default.config.yaml @@ -13,6 +13,7 @@ location_name: oid # specify the name of the source making it possible to identify where the data originates from, ex: 'workmate', 'sap', 'oracle',.. source_name: opcua timeseries_dataset: ds_timeseries_oid +external_root_id_asset: WMT:VAL # configuration for local OPC-UA simulator ( not part of CDF) opcua_endpoint_url: opc.tcp://DESKTOP-18JE0L8:53530/OPCUA/SimulationServer diff --git a/cognite_toolkit/cognite_modules/examples/cdf_data_pipeline_timeseries_valhall/extraction_pipelines/ctx_timeseries_oid_opcua_asset.config.yaml b/cognite_toolkit/cognite_modules/examples/cdf_data_pipeline_timeseries_valhall/extraction_pipelines/ctx_timeseries_oid_opcua_asset.config.yaml index 07af08843..aa0358bb0 100644 --- a/cognite_toolkit/cognite_modules/examples/cdf_data_pipeline_timeseries_valhall/extraction_pipelines/ctx_timeseries_oid_opcua_asset.config.yaml +++ b/cognite_toolkit/cognite_modules/examples/cdf_data_pipeline_timeseries_valhall/extraction_pipelines/ctx_timeseries_oid_opcua_asset.config.yaml @@ -11,5 +11,5 @@ config: rawTableManual: 'contextualization_manual_input' timeSeriesPrefix: [pi_, opc-ua] timeSeriesDataSetExtId: 'ds_timeseries_{{location_name}}' - assetRootExtIds: [workmate:VAL] - matchThreshold: 0.85 + assetRootExtIds: [{{external_root_id_asset}}] + matchThreshold: 0.75 diff --git a/cognite_toolkit/cognite_modules/examples/cdf_data_pipeline_timeseries_valhall/extraction_pipelines/ctx_timeseries_oid_opcua_asset.yaml b/cognite_toolkit/cognite_modules/examples/cdf_data_pipeline_timeseries_valhall/extraction_pipelines/ctx_timeseries_oid_opcua_asset.yaml index fed4fe77b..e4ecfa835 100644 --- a/cognite_toolkit/cognite_modules/examples/cdf_data_pipeline_timeseries_valhall/extraction_pipelines/ctx_timeseries_oid_opcua_asset.yaml +++ b/cognite_toolkit/cognite_modules/examples/cdf_data_pipeline_timeseries_valhall/extraction_pipelines/ctx_timeseries_oid_opcua_asset.yaml @@ -49,7 +49,7 @@ documentation: > # Prefix used for all time series that is used in contextualization timeSeriesPrefix: [pi_, opc-ua] # List of externalId for root assets to be used for contextualization - assetRootExtIds: [workmate:VAL] + assetRootExtIds: [{{external_root_id_asset}}] # Number between 0 and 1, indication of strict the matching should be 1 = only exact matches matchThreshold: 0.85 diff --git a/cognite_toolkit/cognite_modules/examples/cdf_data_pipeline_timeseries_valhall/functions/fn_context_timeseries_oid_opcua_asset/pipeline.py b/cognite_toolkit/cognite_modules/examples/cdf_data_pipeline_timeseries_valhall/functions/fn_context_timeseries_oid_opcua_asset/pipeline.py index c78dab76e..92eb83cce 100644 --- a/cognite_toolkit/cognite_modules/examples/cdf_data_pipeline_timeseries_valhall/functions/fn_context_timeseries_oid_opcua_asset/pipeline.py +++ b/cognite_toolkit/cognite_modules/examples/cdf_data_pipeline_timeseries_valhall/functions/fn_context_timeseries_oid_opcua_asset/pipeline.py @@ -64,6 +64,9 @@ def contextualize_ts_and_asset(client: CogniteClient, config: ContextConfig) -> # If there is any TS to be contextualized if len(ts_entities) > 0: asset_entities = get_assets(client, asset_root_ext_id, numAsset) + if not asset_entities: + print(f"WARNING: No assets found for root asset: {asset_root_ext_id}") + continue match_results = get_matches(client, asset_entities, ts_entities) good_matches, bad_matches = select_and_apply_matches(client, config, match_results, ts_meta_dict) diff --git a/cognite_toolkit/cognite_modules/examples/cdf_oid_example_data/transformations/tr_asset_oid_workmate_asset_hierarchy_example.sql b/cognite_toolkit/cognite_modules/examples/cdf_oid_example_data/transformations/tr_asset_oid_workmate_asset_hierarchy_example.sql index 1990b11cd..d79206bed 100644 --- a/cognite_toolkit/cognite_modules/examples/cdf_oid_example_data/transformations/tr_asset_oid_workmate_asset_hierarchy_example.sql +++ b/cognite_toolkit/cognite_modules/examples/cdf_oid_example_data/transformations/tr_asset_oid_workmate_asset_hierarchy_example.sql @@ -1,6 +1,6 @@ select cast(`externalId` as STRING) as externalId, - cast(`externalId` as STRING) as name, + cast(`tag` as STRING) as name, cast(`description` as STRING) as description, cast(`sourceDb` as STRING) as source, cast(`parentExternalId` as STRING) as parentExternalId diff --git a/cognite_toolkit/config.yaml b/cognite_toolkit/config.yaml index fa75f1242..67e9e85c3 100644 --- a/cognite_toolkit/config.yaml +++ b/cognite_toolkit/config.yaml @@ -80,6 +80,8 @@ cognite_modules: pause_transformations: true files_raw_input_db: files_oid_fileshare files_raw_input_table: files_metadata + external_root_id_asset: WMT:VAL + # source ID from Azure AD for the corresponding groups, ex 'c74797ce-9191-4a4a-9186-8fe21c54c3de' files_location_extractor_group_source_id: @@ -95,6 +97,8 @@ cognite_modules: # specify the name of the source making it possible to identify where the data originates from, ex: 'workmate', 'sap', 'oracle',.. source_name: opcua timeseries_dataset: ds_timeseries_oid + external_root_id_asset: WMT:VAL + # configuration for local OPC-UA simulator ( not part of CDF) opcua_endpoint_url: opc.tcp://DESKTOP-18JE0L8:53530/OPCUA/SimulationServer diff --git a/tests/tests_unit/test_approval_modules_snapshots/cdf_data_pipeline_asset_valhall.yaml b/tests/tests_unit/test_approval_modules_snapshots/cdf_data_pipeline_asset_valhall.yaml index 0508b7e9e..1c38dbcd5 100644 --- a/tests/tests_unit/test_approval_modules_snapshots/cdf_data_pipeline_asset_valhall.yaml +++ b/tests/tests_unit/test_approval_modules_snapshots/cdf_data_pipeline_asset_valhall.yaml @@ -129,11 +129,11 @@ Transformation: query: "--\n-- Create Asset Hierarchy using Transformation\n--\n-- Input data from\ \ RAW DB table (using example data)\n--\n-- Root node has parentExternal id =\ \ ''\n-- Transformation is connected to asset data set\n-- All metadata expect\ - \ selected fileds are added to metadata\n--\nSELECT \n sourceDb || ':' || tag\ - \ as externalId,\n if(parentTag is null, \n '', \n sourceDb\ - \ || ':' ||parentTag) as parentExternalId,\n tag \ - \ as name,\n sourceDb as source,\n description,\n dataset_id('ds_asset_oid')\ - \ as dataSetId,\n to_metadata_except(\n array(\"sourceDb\", \"parentTag\"\ + \ selected fileds are added to metadata\n--\nSELECT \n externalId \ + \ as externalId,\n if(parentExternalId is null, \n '', \n parentExternalId)\ + \ as parentExternalId,\n tag as name,\n\ + \ sourceDb as source,\n description,\n dataset_id('ds_asset_oid')\ + \ as dataSetId,\n to_metadata_except(\n array(\"sourceDb\", \"parentExternalId\"\ , \"description\"), *) \n as metadata\nFROM \n\ \ `asset_oid_workmate`.`assets`\n" sourceOidcCredentials: diff --git a/tests/tests_unit/test_approval_modules_snapshots/cdf_data_pipeline_files_valhall.yaml b/tests/tests_unit/test_approval_modules_snapshots/cdf_data_pipeline_files_valhall.yaml index dd904cccb..794496452 100644 --- a/tests/tests_unit/test_approval_modules_snapshots/cdf_data_pipeline_files_valhall.yaml +++ b/tests/tests_unit/test_approval_modules_snapshots/cdf_data_pipeline_files_valhall.yaml @@ -27,7 +27,7 @@ ExtractionPipeline: \ document metadata, where is location of property for P&ID type documents\n docTypeMetaCol:\ \ \"doc_type\"\n # Document type for P&ID type documents\n pAndIdDocType: \"PNID\"\ \n # List of externalId for root assets to be used for annotation of documents.\ - \ Documents are also extracted based on the root asset ID\n assetRootExtIds: [workmate:VAL]\n\ + \ Documents are also extracted based on the root asset ID\n assetRootExtIds: [WMT:VAL]\n\ \ # Number between 0 and 1, indication of strict the matching should be 1 = only\ \ exact matches\n matchTreshold: 0.85\n```\n" externalId: ep_ctx_files_oid_fileshare_pandid_annotation @@ -53,9 +53,9 @@ ExtractionPipeline: tableName: files_metadata source: fileshare ExtractionPipelineConfig: -- config: "data:\n assetRootExtIds:\n - workmate:VAL\n debug: false\n \ - \ docDataSetExtId: ds_files_oid\n docLimit: -1\n docTypeMetaCol: doc_type\n\ - \ matchThreshold: 0.85\n pAndIdDocType: PNID\n runAll: false\n" +- config: "data:\n assetRootExtIds:\n - WMT:VAL\n debug: false\n docDataSetExtId:\ + \ ds_files_oid\n docLimit: -1\n docTypeMetaCol: doc_type\n matchThreshold:\ + \ 0.85\n pAndIdDocType: PNID\n runAll: false\n" externalId: ep_ctx_files_oid_fileshare_pandid_annotation - config: "cognite:\n data-set:\n external-id: ds-files:valhall\nfiles:\n\ \ extensions:\n - .pdf\n file-provider:\n path: c:/tmp/files\n\ @@ -76,7 +76,7 @@ Function: externalId: fn_context_files_oid_fileshare_annotation functionPath: handler.py metadata: - cdf-toolkit-function-hash: c8e88f94d5a311c2b5f56f4af42ffa22b774f769b91ef48108122cdea205bd1f + cdf-toolkit-function-hash: 2cc9704f01bac8c7f6fa17f7fc916747431253398cb71a312e53c1d3163bf375 version: 0.0.1 name: context:files:oid:fileshare:annotation owner: Anonymous @@ -246,13 +246,15 @@ Transformation: isPublic: true name: files:oid:fileshare:file_metadata query: "--\n-- Update file metdata using Transformation\n--\n-- Input data from\ - \ RAW DB table (using example data) and uploaded files\n--\nSELECT\n file.id\ - \ as id,\n file.name as name,\n\ - \ 'oid_'|| file.name as externalId,\n meta.source as source,\n\ - \ meta.`mime_type` as mimeType,\n dataset_id('ds_files_oid')\ - \ as dataSetId,\n to_metadata(doc_type) as metadata\nFROM \n `files_oid_fileshare`.`files_metadata`\ - \ meta,\n _cdf.files file \nWHERE \n file.name\ - \ = meta.name\n" + \ RAW DB table (using example data) and uploaded files\n--\nWith \n root_id AS\ + \ (\n Select id from _cdf.asset where externalId = 'WMT:VAL'\n )\nSELECT\n\ + \ file.id as id,\n file.name \ + \ as name,\n 'oid_'|| file.name as externalId,\n meta.source \ + \ as source,\n meta.`mime_type` as mimeType,\n array(root_id.id)\ + \ as assetIds,\n dataset_id('ds_files_oid') as dataSetId,\n \ + \ to_metadata(doc_type) as metadata\nFROM \n `files_oid_fileshare`.`files_metadata`\ + \ meta,\n _cdf.files file,\n root_id\nWHERE \n \ + \ file.name = meta.name\n" sourceOidcCredentials: audience: https://bluefield.cognitedata.com cdfProjectName: pytest-project diff --git a/tests/tests_unit/test_approval_modules_snapshots/cdf_data_pipeline_timeseries_valhall.yaml b/tests/tests_unit/test_approval_modules_snapshots/cdf_data_pipeline_timeseries_valhall.yaml index 440d36375..1b7518579 100644 --- a/tests/tests_unit/test_approval_modules_snapshots/cdf_data_pipeline_timeseries_valhall.yaml +++ b/tests/tests_unit/test_approval_modules_snapshots/cdf_data_pipeline_timeseries_valhall.yaml @@ -36,7 +36,7 @@ ExtractionPipeline: \ # RAW table used to store all mapping that NOT was automatically approved\n\ \ rawTableBad: 'contextualization_bad'\n # Prefix used for all time series that\ \ is used in contextualization\n timeSeriesPrefix: [pi_, opc-ua]\n # List of externalId\ - \ for root assets to be used for contextualization\n assetRootExtIds: [workmate:VAL]\n\ + \ for root assets to be used for contextualization\n assetRootExtIds: [WMT:VAL]\n\ \ # Number between 0 and 1, indication of strict the matching should be 1 = only\ \ exact matches\n matchThreshold: 0.85\n\n```\n" externalId: ep_ctx_timeseries_oid_opcua_asset @@ -99,11 +99,11 @@ ExtractionPipeline: name: src:timeseries:oid:opcua source: opcua ExtractionPipelineConfig: -- config: "data:\n assetRootExtIds:\n - workmate:VAL\n debug: false\n \ - \ matchThreshold: 0.85\n rawTableBad: contextualization_bad\n rawTableGood:\ - \ contextualization_good\n rawTableManual: contextualization_manual_input\n\ - \ rawdb: timeseries_oid_opcua\n runAll: false\n timeSeriesDataSetExtId:\ - \ ds_timeseries_oid\n timeSeriesPrefix:\n - pi_\n - opc-ua\n" +- config: "data:\n assetRootExtIds:\n - WMT:VAL\n debug: false\n matchThreshold:\ + \ 0.75\n rawTableBad: contextualization_bad\n rawTableGood: contextualization_good\n\ + \ rawTableManual: contextualization_manual_input\n rawdb: timeseries_oid_opcua\n\ + \ runAll: false\n timeSeriesDataSetExtId: ds_timeseries_oid\n timeSeriesPrefix:\n\ + \ - pi_\n - opc-ua\n" externalId: ep_ctx_timeseries_oid_opcua_asset - config: "cognite:\n cdf-chunking:\n data-point-time-series: 500\n \ \ data-points: 10000\n data-set-external-id: ds_timeseries_oid\n debug:\ @@ -134,7 +134,7 @@ Function: externalId: fn_context_timeseries_oid_opcua_asset functionPath: ./handler.py metadata: - cdf-toolkit-function-hash: b7107f42a646e392c8212900ed30333d5f70d421cd30602f2c27681a7358d032 + cdf-toolkit-function-hash: 84240c2fa9bbb676b53b4a6f8cd3ceaabbfa646282d7ebf5fc6d12d324a753ce version: 0.0.1 name: context:timeseries:oid:opcua:asset owner: Anonymous diff --git a/tests/tests_unit/test_approval_modules_snapshots/cdf_oid_example_data.yaml b/tests/tests_unit/test_approval_modules_snapshots/cdf_oid_example_data.yaml index 9ddb7a207..58df63bcf 100644 --- a/tests/tests_unit/test_approval_modules_snapshots/cdf_oid_example_data.yaml +++ b/tests/tests_unit/test_approval_modules_snapshots/cdf_oid_example_data.yaml @@ -695,8 +695,8 @@ Transformation: ignoreNullFields: true isPublic: true name: asset:oid:workmate:asset_hierarchy:example - query: "select\n cast(`externalId` as STRING) as externalId,\n cast(`externalId`\ - \ as STRING) as name,\n cast(`description` as STRING) as description,\n cast(`sourceDb`\ + query: "select\n cast(`externalId` as STRING) as externalId,\n cast(`tag` as STRING)\ + \ as name,\n cast(`description` as STRING) as description,\n cast(`sourceDb`\ \ as STRING) as source,\n cast(`parentExternalId` as STRING) as parentExternalId\n\ from\n `asset_oid_workmate`.`assets`;\n" sourceOidcCredentials: diff --git a/tests/tests_unit/test_cdf_tk/auth_data_snapshots/auth_auth_verify_happypath_windows.txt b/tests/tests_unit/test_cdf_tk/auth_data_snapshots/auth_auth_verify_happypath_windows.txt index a1ec58c3c..da82e7b4f 100644 --- a/tests/tests_unit/test_cdf_tk/auth_data_snapshots/auth_auth_verify_happypath_windows.txt +++ b/tests/tests_unit/test_cdf_tk/auth_data_snapshots/auth_auth_verify_happypath_windows.txt @@ -1,10 +1,8 @@ Checking current service principal/application and environment configurations... - WARNING: CDF_URL is set to https://greenfield.cognitedata.com, are you sure +WARNING: CDF URL is set to https://westeurope-1.cognitedata.com, are you sure it shouldn't be https://bluefield.cognitedata.com? - WARNING: IDP_SCOPES is set to https://greenfield.cognitedata.com/.default, -are you sure it shouldn't be https://bluefield.cognitedata.com/.default? - + OK Checking basic project configuration... OK Checking projects that the service principal/application has access to... diff --git a/tests/tests_unit/test_cdf_tk/auth_data_snapshots/auth_auth_verify_no_capabilities_windows.txt b/tests/tests_unit/test_cdf_tk/auth_data_snapshots/auth_auth_verify_no_capabilities_windows.txt index 3f0946657..e6a969fa6 100644 --- a/tests/tests_unit/test_cdf_tk/auth_data_snapshots/auth_auth_verify_no_capabilities_windows.txt +++ b/tests/tests_unit/test_cdf_tk/auth_data_snapshots/auth_auth_verify_no_capabilities_windows.txt @@ -1,10 +1,8 @@ Checking current service principal/application and environment configurations... - WARNING: CDF_URL is set to https://greenfield.cognitedata.com, are you sure +WARNING: CDF URL is set to https://westeurope-1.cognitedata.com, are you sure it shouldn't be https://bluefield.cognitedata.com? - WARNING: IDP_SCOPES is set to https://greenfield.cognitedata.com/.default, -are you sure it shouldn't be https://bluefield.cognitedata.com/.default? - + OK Checking basic project configuration... OK Checking projects that the service principal/application has access to... diff --git a/tests/tests_unit/test_cdf_tk/auth_data_snapshots/auth_auth_verify_two_groups_windows.txt b/tests/tests_unit/test_cdf_tk/auth_data_snapshots/auth_auth_verify_two_groups_windows.txt index 8807e4360..c603b42fc 100644 --- a/tests/tests_unit/test_cdf_tk/auth_data_snapshots/auth_auth_verify_two_groups_windows.txt +++ b/tests/tests_unit/test_cdf_tk/auth_data_snapshots/auth_auth_verify_two_groups_windows.txt @@ -1,10 +1,8 @@ Checking current service principal/application and environment configurations... - WARNING: CDF_URL is set to https://greenfield.cognitedata.com, are you sure +WARNING: CDF URL is set to https://westeurope-1.cognitedata.com, are you sure it shouldn't be https://bluefield.cognitedata.com? - WARNING: IDP_SCOPES is set to https://greenfield.cognitedata.com/.default, -are you sure it shouldn't be https://bluefield.cognitedata.com/.default? - + OK Checking basic project configuration... OK Checking projects that the service principal/application has access to... diff --git a/tests/tests_unit/test_cdf_tk/auth_data_snapshots/auth_auth_verify_wrong_capabilities_windows.txt b/tests/tests_unit/test_cdf_tk/auth_data_snapshots/auth_auth_verify_wrong_capabilities_windows.txt index 301e642ca..c30c3e599 100644 --- a/tests/tests_unit/test_cdf_tk/auth_data_snapshots/auth_auth_verify_wrong_capabilities_windows.txt +++ b/tests/tests_unit/test_cdf_tk/auth_data_snapshots/auth_auth_verify_wrong_capabilities_windows.txt @@ -1,10 +1,8 @@ Checking current service principal/application and environment configurations... - WARNING: CDF_URL is set to https://greenfield.cognitedata.com, are you sure +WARNING: CDF URL is set to https://westeurope-1.cognitedata.com, are you sure it shouldn't be https://bluefield.cognitedata.com? - WARNING: IDP_SCOPES is set to https://greenfield.cognitedata.com/.default, -are you sure it shouldn't be https://bluefield.cognitedata.com/.default? - + OK Checking basic project configuration... OK Checking projects that the service principal/application has access to... From 6df8cc68934bb6324bdbaca136495872cd802cb7 Mon Sep 17 00:00:00 2001 From: Jan Inge Bergseth <31886431+BergsethCognite@users.noreply.github.com> Date: Thu, 4 Apr 2024 11:38:03 +0200 Subject: [PATCH 02/34] Update CHANGELOG.templates.md --- CHANGELOG.templates.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.templates.md b/CHANGELOG.templates.md index 23dadd6f6..8a84877c2 100644 --- a/CHANGELOG.templates.md +++ b/CHANGELOG.templates.md @@ -15,6 +15,14 @@ Changes are grouped as follows: - `Fixed` for any bug fixes. - `Security` in case of vulnerabilities. +## [TBD] - TBD + +- Fixed: align tag name in asset hierarchy between the 2 example transformations +- Fixed: added default root asset ID to documents for initial annotation +- Fixed: alined use of asset external ID across contextualization functions +- Fixed: annotation logic with local time stamp for when to reprocess P&ID files +- Fixed: input to P&ID annotation based on list of synonyms for tag + ## [0.2.0a1] - 2024-03-20 - Added functionality for wildcard detection of tags in P&ID From d58fb2053ff8756d06213202098dc7d9e05b210f Mon Sep 17 00:00:00 2001 From: Jan Inge Bergseth <31886431+BergsethCognite@users.noreply.github.com> Date: Thu, 4 Apr 2024 13:49:37 +0200 Subject: [PATCH 03/34] adding config for external_root_id_asset --- demo/config.demo.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/demo/config.demo.yaml b/demo/config.demo.yaml index 6ea5c0f77..f048ffdf0 100644 --- a/demo/config.demo.yaml +++ b/demo/config.demo.yaml @@ -88,6 +88,7 @@ modules: version: 0.0.1 # location based on OID example data location_name: oid + external_root_id_asset: WMT:VAL # specify the name of the source making it possible to identify where the data originates from, ex: 'workmate', 'sap', 'oracle',.. source_name: fileshare files_dataset: ds_files_oid @@ -106,6 +107,7 @@ modules: # specify the name of the source making it possible to identify where the data originates from, ex: 'workmate', 'sap', 'oracle',.. source_name: opcua timeseries_dataset: ds_timeseries_oid + external_root_id_asset: WMT:VAL # configuration for local OPC-UA simulator ( not part of CDF) opcua_endpoint_url: opc.tcp://DESKTOP-18JE0L8:53530/OPCUA/SimulationServer opcua_id_prefix: "opc-ua:" From 198315b815f36e020267e0ad623b35dac3334b59 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?H=C3=A5kon=20V=2E=20Treider?= Date: Tue, 9 Apr 2024 09:59:19 +0200 Subject: [PATCH 04/34] function schedules should not use `oneshotTokenExchange` (#444) --- CHANGELOG.cdf-tk.md | 6 ++ CHANGELOG.templates.md | 2 +- .../_cdf_tk/load/_resource_loaders.py | 61 ++++++------------- tests/tests_unit/approval_client/client.py | 8 +++ tests/tests_unit/approval_client/config.py | 18 +++--- .../cdf_data_pipeline_files_valhall.yaml | 6 ++ .../cdf_data_pipeline_timeseries_valhall.yaml | 6 ++ .../cdf_functions_dummy.yaml | 15 +++++ 8 files changed, 69 insertions(+), 53 deletions(-) diff --git a/CHANGELOG.cdf-tk.md b/CHANGELOG.cdf-tk.md index a8e2c0dbc..942cf1546 100644 --- a/CHANGELOG.cdf-tk.md +++ b/CHANGELOG.cdf-tk.md @@ -15,6 +15,12 @@ Changes are grouped as follows: - `Fixed` for any bug fixes. - `Security` in case of vulnerabilities. +## TBD + +### Fixed + +- Functions that are deployed with schedules no longer uses a short-lived session (before: failed after ~an hour). + ## [0.2.0a2] - 2024-04-03 ### Added diff --git a/CHANGELOG.templates.md b/CHANGELOG.templates.md index 6b91d2005..e432daac7 100644 --- a/CHANGELOG.templates.md +++ b/CHANGELOG.templates.md @@ -19,7 +19,7 @@ Changes are grouped as follows: - Fixed: align tag name in asset hierarchy between the 2 example transformations - Fixed: added default root asset ID to documents for initial annotation -- Fixed: alined use of asset external ID across contextualization functions +- Fixed: aligned use of asset external ID across contextualization functions - Fixed: annotation logic with local time stamp for when to reprocess P&ID files - Fixed: input to P&ID annotation based on list of synonyms for tag diff --git a/cognite_toolkit/_cdf_tk/load/_resource_loaders.py b/cognite_toolkit/_cdf_tk/load/_resource_loaders.py index 0930842ab..efb59a96c 100644 --- a/cognite_toolkit/_cdf_tk/load/_resource_loaders.py +++ b/cognite_toolkit/_cdf_tk/load/_resource_loaders.py @@ -26,8 +26,8 @@ import yaml from cognite.client import CogniteClient -from cognite.client.credentials import OAuthClientCredentials from cognite.client.data_classes import ( + ClientCredentials, DatapointsList, DataSet, DataSetList, @@ -122,7 +122,6 @@ from cognite_toolkit._cdf_tk.utils import ( CDFToolConfig, calculate_directory_hash, - get_oneshot_session, load_yaml_inject_variables, retrieve_view_ancestors, ) @@ -638,50 +637,26 @@ def retrieve(self, ids: SequenceNotStr[str]) -> FunctionSchedulesList: def create(self, items: FunctionScheduleWriteList) -> FunctionSchedulesList: items = self._resolve_functions_ext_id(items) - (_, bearer) = self.client.config.credentials.authorization_header() - created = FunctionSchedulesList([]) + created = [] for item in items: - if ( - authentication := self.extra_configs.get(f"{item.function_external_id}:{item.cron_expression}", {}).get( - "authentication" - ) - ) is not None and len(authentication) > 0: - new_tool_config = CDFToolConfig() - old_credentials = cast(OAuthClientCredentials, new_tool_config.client.config.credentials) - new_tool_config.client.config.credentials = OAuthClientCredentials( - client_id=authentication.get("clientId"), - client_secret=authentication.get("clientSecret"), - scopes=old_credentials.scopes, - token_url=old_credentials.token_url, - ) - session = get_oneshot_session(new_tool_config.client) + key = f"{item.function_external_id}:{item.cron_expression}" + auth_config = self.extra_configs.get(key, {}).get("authentication", {}) + if "clientId" in auth_config and "clientSecret" in auth_config: + client_credentials = ClientCredentials(auth_config["clientId"], auth_config["clientSecret"]) else: - session = get_oneshot_session(self.client) - nonce = session.nonce if session is not None else "" - try: - ret = self.client.post( - url=f"/api/v1/projects/{self.client.config.project}/functions/schedules", - json={ - "items": [ - { - "name": item.name, - "description": item.description, - "cronExpression": item.cron_expression, - "functionId": item.function_id, - "data": item.data, - "nonce": nonce, - } - ], - }, - headers={"Authorization": bearer}, + client_credentials = None + + created.append( + self.client.functions.schedules.create( + name=item.name or "", + description=item.description or "", + cron_expression=cast(str, item.cron_expression), + function_id=cast(int, item.function_id), + data=item.data, + client_credentials=client_credentials, ) - except CogniteAPIError as e: - if e.code == 400 and "Failed to bind session" in e.message: - print(" [bold yellow]WARNING:[/] Failed to bind session because function is not ready.") - continue - if ret.status_code == 201: - created.append(FunctionSchedule.load(ret.json()["items"][0])) - return created + ) + return FunctionSchedulesList(created) def delete(self, ids: SequenceNotStr[str]) -> int: schedules = self.retrieve(ids) diff --git a/tests/tests_unit/approval_client/client.py b/tests/tests_unit/approval_client/client.py index 249f4becb..67b6d8613 100644 --- a/tests/tests_unit/approval_client/client.py +++ b/tests/tests_unit/approval_client/client.py @@ -21,6 +21,8 @@ FileMetadata, Function, FunctionCall, + FunctionSchedule, + FunctionScheduleWrite, FunctionWrite, Group, GroupList, @@ -428,6 +430,11 @@ def create_function_api(**kwargs) -> Function: created_resources[resource_cls.__name__].append(created) return Function.load(created.dump(camel_case=True)) + def create_function_schedule_api(**kwargs) -> FunctionSchedule: + created = FunctionScheduleWrite.load({to_camel_case(k): v for k, v in kwargs.items()}) + created_resources[resource_cls.__name__].append(created) + return FunctionSchedule.load(created.dump(camel_case=True)) + available_create_methods = { fn.__name__: fn for fn in [ @@ -438,6 +445,7 @@ def create_function_api(**kwargs) -> Function: create_extraction_pipeline_config, upload_bytes_files_api, create_function_api, + create_function_schedule_api, ] } if mock_method not in available_create_methods: diff --git a/tests/tests_unit/approval_client/config.py b/tests/tests_unit/approval_client/config.py index 7204a99e5..7ed7f04bf 100644 --- a/tests/tests_unit/approval_client/config.py +++ b/tests/tests_unit/approval_client/config.py @@ -24,6 +24,10 @@ FileMetadataWriteList, Function, FunctionList, + FunctionSchedule, + FunctionSchedulesList, + FunctionScheduleWrite, + FunctionScheduleWriteList, FunctionWrite, FunctionWriteList, Group, @@ -197,16 +201,12 @@ ), APIResource( api_name="functions.schedules", - resource_cls=Function, - _write_cls=FunctionWrite, - list_cls=FunctionList, - _write_list_cls=FunctionWriteList, + resource_cls=FunctionSchedule, + _write_cls=FunctionScheduleWrite, + list_cls=FunctionSchedulesList, + _write_list_cls=FunctionScheduleWriteList, methods={ - "create": [Method(api_class_method="create", mock_name="create_function_api")], - "delete": [Method(api_class_method="delete", mock_name="delete_id_external_id")], - "retrieve": [ - Method(api_class_method="list", mock_name="return_values"), - ], + "create": [Method(api_class_method="create", mock_name="create_function_schedule_api")], }, ), APIResource( diff --git a/tests/tests_unit/test_approval_modules_snapshots/cdf_data_pipeline_files_valhall.yaml b/tests/tests_unit/test_approval_modules_snapshots/cdf_data_pipeline_files_valhall.yaml index 794496452..548caa7a6 100644 --- a/tests/tests_unit/test_approval_modules_snapshots/cdf_data_pipeline_files_valhall.yaml +++ b/tests/tests_unit/test_approval_modules_snapshots/cdf_data_pipeline_files_valhall.yaml @@ -81,6 +81,12 @@ Function: name: context:files:oid:fileshare:annotation owner: Anonymous runtime: py311 +FunctionSchedule: +- cronExpression: 0,30 * * * * + data: + ExtractionPipelineExtId: ep_ctx_files_oid_fileshare_pandid_annotation + description: Run every 30 minute + name: every 30 min Group: - capabilities: - rawAcl: diff --git a/tests/tests_unit/test_approval_modules_snapshots/cdf_data_pipeline_timeseries_valhall.yaml b/tests/tests_unit/test_approval_modules_snapshots/cdf_data_pipeline_timeseries_valhall.yaml index 1b7518579..3d4c57d74 100644 --- a/tests/tests_unit/test_approval_modules_snapshots/cdf_data_pipeline_timeseries_valhall.yaml +++ b/tests/tests_unit/test_approval_modules_snapshots/cdf_data_pipeline_timeseries_valhall.yaml @@ -139,6 +139,12 @@ Function: name: context:timeseries:oid:opcua:asset owner: Anonymous runtime: py311 +FunctionSchedule: +- cronExpression: 0,30 * * * * + data: + ExtractionPipelineExtId: ep_ctx_timeseries_oid_opcua_asset + description: Run every 30 minute + name: daily-every-30-min Group: - capabilities: - rawAcl: diff --git a/tests/tests_unit/test_approval_modules_snapshots/cdf_functions_dummy.yaml b/tests/tests_unit/test_approval_modules_snapshots/cdf_functions_dummy.yaml index 2667326ef..7df433ca4 100644 --- a/tests/tests_unit/test_approval_modules_snapshots/cdf_functions_dummy.yaml +++ b/tests/tests_unit/test_approval_modules_snapshots/cdf_functions_dummy.yaml @@ -27,6 +27,21 @@ Function: metadata: cdf-toolkit-function-hash: e8c92f1b4438f2e6c362b83928e2edd07744cdbc58e6bfd996b13517c464d5c6 name: test2 +FunctionSchedule: +- cronExpression: 0 8 * * * + data: + breakfast: 'today: peanut butter sandwich and coffee' + dinner: 'today: steak and red wine' + lunch: 'today: greek salad and water' + description: Run every day at 8am UTC + name: daily-8am-utc +- cronExpression: 0 20 * * * + data: + breakfast: 'tomorrow: peanut butter sandwich and coffee' + dinner: 'tomorrow: steak and red wine' + lunch: 'tomorrow: greek salad and water' + description: Run every day at 8pm UTC + name: daily-8pm-utc deleted: Function: - externalId: fn_example_repeater From bd7797fc53c62159b7015ff41de004c18ed57066 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Fri, 12 Apr 2024 11:05:48 +0200 Subject: [PATCH 05/34] chore(deps): lock file maintenance (#446) --- poetry.lock | 98 +++++++++++++++++++++++++---------------------------- 1 file changed, 47 insertions(+), 51 deletions(-) diff --git a/poetry.lock b/poetry.lock index bad3b4fce..996bab520 100644 --- a/poetry.lock +++ b/poetry.lock @@ -71,6 +71,21 @@ azure-core = ">=1.29.5,<2.0.0" isodate = ">=0.6.1" typing-extensions = ">=4.0.1" +[[package]] +name = "backports-tarfile" +version = "1.0.0" +description = "Backport of CPython tarfile module" +optional = false +python-versions = ">=3.8" +files = [ + {file = "backports.tarfile-1.0.0-py3-none-any.whl", hash = "sha256:bcd36290d9684beb524d3fe74f4a2db056824c47746583f090b8e55daf0776e4"}, + {file = "backports.tarfile-1.0.0.tar.gz", hash = "sha256:2688f159c21afd56a07b75f01306f9f52c79aebcc5f4a117fb8fbb4445352c75"}, +] + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9.3)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] +testing = ["pytest (>=6,!=8.1.1)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)"] + [[package]] name = "certifi" version = "2024.2.2" @@ -311,13 +326,13 @@ python-json-logger = ">=2.0.7,<3.0.0" [[package]] name = "cognite-sdk" -version = "7.32.4" +version = "7.33.0" description = "Cognite Python SDK" optional = false python-versions = "<4.0,>=3.8" files = [ - {file = "cognite_sdk-7.32.4-py3-none-any.whl", hash = "sha256:1a657da4abf770b0a37208a10dd9a2912c99b7d20ab837678dc1130b676066ed"}, - {file = "cognite_sdk-7.32.4.tar.gz", hash = "sha256:9e983a77a1f62ddc323de3aad951e9732193e5810a4cf0c6112f7832208d10cb"}, + {file = "cognite_sdk-7.33.0-py3-none-any.whl", hash = "sha256:787b4f785621aba964fec3e9f6c3b5e177a0f278790f91aaf850faae79f6fee4"}, + {file = "cognite_sdk-7.33.0.tar.gz", hash = "sha256:7776bc0980ea5b514c25521f5faea5356ee7a71327e1049093264aa3e5952eb3"}, ] [package.dependencies] @@ -677,18 +692,21 @@ testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-ena [[package]] name = "jaraco-context" -version = "4.3.0" -description = "Context managers by jaraco" +version = "5.3.0" +description = "Useful decorators and context managers" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "jaraco.context-4.3.0-py3-none-any.whl", hash = "sha256:5d9e95ca0faa78943ed66f6bc658dd637430f16125d86988e77844c741ff2f11"}, - {file = "jaraco.context-4.3.0.tar.gz", hash = "sha256:4dad2404540b936a20acedec53355bdaea223acb88fd329fa6de9261c941566e"}, + {file = "jaraco.context-5.3.0-py3-none-any.whl", hash = "sha256:3e16388f7da43d384a1a7cd3452e72e14732ac9fe459678773a3608a812bf266"}, + {file = "jaraco.context-5.3.0.tar.gz", hash = "sha256:c2f67165ce1f9be20f32f650f25d8edfc1646a8aeee48ae06fb35f90763576d2"}, ] +[package.dependencies] +"backports.tarfile" = {version = "*", markers = "python_version < \"3.12\""} + [package.extras] -docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] -testing = ["flake8 (<5)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)"] +docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] +testing = ["portend", "pytest (>=6,!=8.1.1)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-mypy", "pytest-ruff (>=0.2.1)"] [[package]] name = "jaraco-functools" @@ -1524,6 +1542,7 @@ files = [ {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, + {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, @@ -1531,8 +1550,16 @@ files = [ {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, + {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, + {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, + {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, + {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, @@ -1549,6 +1576,7 @@ files = [ {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, + {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, @@ -1556,6 +1584,7 @@ files = [ {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, + {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, @@ -1811,54 +1840,21 @@ urllib3 = ">=1.26.0" [[package]] name = "typer" -version = "0.12.0" -description = "Typer, build great CLIs. Easy to code. Based on Python type hints." -optional = false -python-versions = ">=3.7" -files = [ - {file = "typer-0.12.0-py3-none-any.whl", hash = "sha256:0441a0bb8962fb4383b8537ada9f7eb2d0deda0caa2cfe7387cc221290f617e4"}, - {file = "typer-0.12.0.tar.gz", hash = "sha256:900fe786ce2d0ea44653d3c8ee4594a22a496a3104370ded770c992c5e3c542d"}, -] - -[package.dependencies] -typer-cli = "0.12.0" -typer-slim = {version = "0.12.0", extras = ["standard"]} - -[[package]] -name = "typer-cli" -version = "0.12.0" +version = "0.12.2" description = "Typer, build great CLIs. Easy to code. Based on Python type hints." optional = false python-versions = ">=3.7" files = [ - {file = "typer_cli-0.12.0-py3-none-any.whl", hash = "sha256:7b7e2dd49f59974bb5a869747045d5444b17bffb851e006cd424f602d3578104"}, - {file = "typer_cli-0.12.0.tar.gz", hash = "sha256:603ed3d5a278827bd497e4dc73a39bb714b230371c8724090b0de2abdcdd9f6e"}, -] - -[package.dependencies] -typer-slim = {version = "0.12.0", extras = ["standard"]} - -[[package]] -name = "typer-slim" -version = "0.12.0" -description = "Typer, build great CLIs. Easy to code. Based on Python type hints." -optional = false -python-versions = ">=3.7" -files = [ - {file = "typer_slim-0.12.0-py3-none-any.whl", hash = "sha256:ddd7042b29a32140528caa415750bcae54113ba0c32270ca11a6f64069ddadf9"}, - {file = "typer_slim-0.12.0.tar.gz", hash = "sha256:3e8a3f17286b173d76dca0fd4e02651c9a2ce1467b3754876b1ac4bd72572beb"}, + {file = "typer-0.12.2-py3-none-any.whl", hash = "sha256:e1accbaa7e2b2350753acec896ac30493ac573211a8d4603e88f8356217e01f7"}, + {file = "typer-0.12.2.tar.gz", hash = "sha256:977929604fde12aeada011852ad9c64370501be6ac2eac248f3161cdc9eeb7c9"}, ] [package.dependencies] click = ">=8.0.0" -rich = {version = ">=10.11.0", optional = true, markers = "extra == \"standard\""} -shellingham = {version = ">=1.3.0", optional = true, markers = "extra == \"standard\""} +rich = ">=10.11.0" +shellingham = ">=1.3.0" typing-extensions = ">=3.7.4.3" -[package.extras] -all = ["rich (>=10.11.0)", "shellingham (>=1.3.0)"] -standard = ["rich (>=10.11.0)", "shellingham (>=1.3.0)"] - [[package]] name = "types-python-dateutil" version = "2.9.0.20240316" @@ -1883,13 +1879,13 @@ files = [ [[package]] name = "typing-extensions" -version = "4.10.0" +version = "4.11.0" description = "Backported and Experimental Type Hints for Python 3.8+" optional = false python-versions = ">=3.8" files = [ - {file = "typing_extensions-4.10.0-py3-none-any.whl", hash = "sha256:69b1a937c3a517342112fb4c6df7e72fc39a38e7891a5730ed4985b5214b5475"}, - {file = "typing_extensions-4.10.0.tar.gz", hash = "sha256:b0abd7c89e8fb96f98db18d86106ff1d90ab692004eb746cf6eda2682f91b3cb"}, + {file = "typing_extensions-4.11.0-py3-none-any.whl", hash = "sha256:c1f94d72897edaf4ce775bb7558d5b79d8126906a14ea5ed1635921406c0387a"}, + {file = "typing_extensions-4.11.0.tar.gz", hash = "sha256:83f085bd5ca59c80295fc2a82ab5dac679cbe02b9f33f7d83af68e241bea51b0"}, ] [[package]] From 4834cabac3d6f42e7a968bac2baa930558d7d12b Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Fri, 12 Apr 2024 11:09:28 +0200 Subject: [PATCH 06/34] fix(deps): update dependency typer to v0.12.3 (#445) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- poetry.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/poetry.lock b/poetry.lock index 996bab520..65338cbcd 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1840,13 +1840,13 @@ urllib3 = ">=1.26.0" [[package]] name = "typer" -version = "0.12.2" +version = "0.12.3" description = "Typer, build great CLIs. Easy to code. Based on Python type hints." optional = false python-versions = ">=3.7" files = [ - {file = "typer-0.12.2-py3-none-any.whl", hash = "sha256:e1accbaa7e2b2350753acec896ac30493ac573211a8d4603e88f8356217e01f7"}, - {file = "typer-0.12.2.tar.gz", hash = "sha256:977929604fde12aeada011852ad9c64370501be6ac2eac248f3161cdc9eeb7c9"}, + {file = "typer-0.12.3-py3-none-any.whl", hash = "sha256:070d7ca53f785acbccba8e7d28b08dcd88f79f1fbda035ade0aecec71ca5c914"}, + {file = "typer-0.12.3.tar.gz", hash = "sha256:49e73131481d804288ef62598d97a1ceef3058905aa536a1134f90891ba35482"}, ] [package.dependencies] From ec3a574c7fe84a98ae8672b337967f07fb5bc455 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Fri, 12 Apr 2024 11:15:31 +0200 Subject: [PATCH 07/34] chore(deps): update pre-commit hook pre-commit/pre-commit-hooks to v4.6.0 (#447) --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index afdab8ebc..fb3bc99ac 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -27,7 +27,7 @@ repos: - --include=\.py$ - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.5.0 + rev: v4.6.0 hooks: - id: debug-statements - id: check-docstring-first From 812e8ea026dfc480aad5166e14597436b2f23a06 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Fri, 12 Apr 2024 11:25:37 +0200 Subject: [PATCH 08/34] chore(deps): update pre-commit hook charliermarsh/ruff-pre-commit to v0.3.5 (#443) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index fb3bc99ac..bc6b77f55 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -12,7 +12,7 @@ repos: - --select=E,W,F,I,T,RUF,TID,UP - --fixable=E,W,F,I,T,RUF,TID,UP - --target-version=py39 - rev: v0.3.4 + rev: v0.3.5 - repo: https://github.com/psf/black rev: 24.3.0 From 4c4260ea328b9e8b9cbd4b1d67c7107cd685fa19 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Fri, 12 Apr 2024 11:46:35 +0200 Subject: [PATCH 09/34] fix(deps): update dependency cognite-sdk to v7.34.0 (#427) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- poetry.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/poetry.lock b/poetry.lock index 65338cbcd..f3d9506f4 100644 --- a/poetry.lock +++ b/poetry.lock @@ -326,13 +326,13 @@ python-json-logger = ">=2.0.7,<3.0.0" [[package]] name = "cognite-sdk" -version = "7.33.0" +version = "7.34.0" description = "Cognite Python SDK" optional = false python-versions = "<4.0,>=3.8" files = [ - {file = "cognite_sdk-7.33.0-py3-none-any.whl", hash = "sha256:787b4f785621aba964fec3e9f6c3b5e177a0f278790f91aaf850faae79f6fee4"}, - {file = "cognite_sdk-7.33.0.tar.gz", hash = "sha256:7776bc0980ea5b514c25521f5faea5356ee7a71327e1049093264aa3e5952eb3"}, + {file = "cognite_sdk-7.34.0-py3-none-any.whl", hash = "sha256:f0387ff0558c06d4cdfde0a88409904932887799ab71d55eb479779aaba0b2ae"}, + {file = "cognite_sdk-7.34.0.tar.gz", hash = "sha256:1eeb629df867e112c4b081255084250675c0f911ba7d69153ba6d61b06371e47"}, ] [package.dependencies] From ca6889d22903ee998b92b232c155b75547843bf9 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Tue, 16 Apr 2024 10:52:59 +0200 Subject: [PATCH 10/34] chore(deps): lock file maintenance (#451) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- poetry.lock | 114 ++++++++++++++++++++++++++-------------------------- 1 file changed, 56 insertions(+), 58 deletions(-) diff --git a/poetry.lock b/poetry.lock index f3d9506f4..21bf0d54d 100644 --- a/poetry.lock +++ b/poetry.lock @@ -40,20 +40,20 @@ aio = ["aiohttp (>=3.0)"] [[package]] name = "azure-identity" -version = "1.15.0" +version = "1.16.0" description = "Microsoft Azure Identity Library for Python" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "azure-identity-1.15.0.tar.gz", hash = "sha256:4c28fc246b7f9265610eb5261d65931183d019a23d4b0e99357facb2e6c227c8"}, - {file = "azure_identity-1.15.0-py3-none-any.whl", hash = "sha256:a14b1f01c7036f11f148f22cd8c16e05035293d714458d6b44ddf534d93eb912"}, + {file = "azure-identity-1.16.0.tar.gz", hash = "sha256:6ff1d667cdcd81da1ceab42f80a0be63ca846629f518a922f7317a7e3c844e1b"}, + {file = "azure_identity-1.16.0-py3-none-any.whl", hash = "sha256:722fdb60b8fdd55fa44dc378b8072f4b419b56a5e54c0de391f644949f3a826f"}, ] [package.dependencies] -azure-core = ">=1.23.0,<2.0.0" +azure-core = ">=1.23.0" cryptography = ">=2.5" -msal = ">=1.24.0,<2.0.0" -msal-extensions = ">=0.3.0,<2.0.0" +msal = ">=1.24.0" +msal-extensions = ">=0.3.0" [[package]] name = "azure-keyvault-secrets" @@ -523,13 +523,13 @@ files = [ [[package]] name = "docutils" -version = "0.20.1" +version = "0.21.1" description = "Docutils -- Python Documentation Utilities" optional = false -python-versions = ">=3.7" +python-versions = ">=3.9" files = [ - {file = "docutils-0.20.1-py3-none-any.whl", hash = "sha256:96f387a2c5562db4476f09f13bbab2192e764cac08ebbf3a34a95d9b1e4a59d6"}, - {file = "docutils-0.20.1.tar.gz", hash = "sha256:f08a4e276c3a1583a86dce3e34aba3fe04d02bba2dd51ed16106244e8a923e3b"}, + {file = "docutils-0.21.1-py3-none-any.whl", hash = "sha256:14c8d34a55b46c88f9f714adb29cefbdd69fb82f3fef825e59c5faab935390d8"}, + {file = "docutils-0.21.1.tar.gz", hash = "sha256:65249d8a5345bc95e0f40f280ba63c98eb24de35c6c8f5b662e3e8948adea83f"}, ] [[package]] @@ -548,13 +548,13 @@ test = ["pytest (>=6)"] [[package]] name = "filelock" -version = "3.13.3" +version = "3.13.4" description = "A platform independent file lock." optional = false python-versions = ">=3.8" files = [ - {file = "filelock-3.13.3-py3-none-any.whl", hash = "sha256:5ffa845303983e7a0b7ae17636509bc97997d58afeafa72fb141a17b152284cb"}, - {file = "filelock-3.13.3.tar.gz", hash = "sha256:a79895a25bbefdf55d1a2a0a80968f7dbb28edcd6d4234a0afb3f37ecde4b546"}, + {file = "filelock-3.13.4-py3-none-any.whl", hash = "sha256:404e5e9253aa60ad457cae1be07c0f0ca90a63931200a47d9b6a6af84fd7b45f"}, + {file = "filelock-3.13.4.tar.gz", hash = "sha256:d13f466618bfde72bd2c18255e269f72542c6e70e7bac83a0232d6b1cc5c8cf4"}, ] [package.extras] @@ -619,13 +619,13 @@ license = ["ukkonen"] [[package]] name = "idna" -version = "3.6" +version = "3.7" description = "Internationalized Domain Names in Applications (IDNA)" optional = false python-versions = ">=3.5" files = [ - {file = "idna-3.6-py3-none-any.whl", hash = "sha256:c05567e9c24a6b9faaa835c4821bad0590fbb9d5779e7caa6e1cc4978e7eb24f"}, - {file = "idna-3.6.tar.gz", hash = "sha256:9ecdbbd083b06798ae1e86adcbfe8ab1479cf864e4ee30fe4e46a003d12491ca"}, + {file = "idna-3.7-py3-none-any.whl", hash = "sha256:82fee1fc78add43492d3a1898bfa6d8a904cc97d8427f683ed8e798d07761aa0"}, + {file = "idna-3.7.tar.gz", hash = "sha256:028ff3aadf0609c1fd278d8ea3089299412a7a8b9bd005dd08b9f8285bcb5cfc"}, ] [[package]] @@ -1032,47 +1032,45 @@ files = [ [[package]] name = "pandas" -version = "2.2.1" +version = "2.2.2" description = "Powerful data structures for data analysis, time series, and statistics" optional = false python-versions = ">=3.9" files = [ - {file = "pandas-2.2.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:8df8612be9cd1c7797c93e1c5df861b2ddda0b48b08f2c3eaa0702cf88fb5f88"}, - {file = "pandas-2.2.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:0f573ab277252ed9aaf38240f3b54cfc90fff8e5cab70411ee1d03f5d51f3944"}, - {file = "pandas-2.2.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f02a3a6c83df4026e55b63c1f06476c9aa3ed6af3d89b4f04ea656ccdaaaa359"}, - {file = "pandas-2.2.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c38ce92cb22a4bea4e3929429aa1067a454dcc9c335799af93ba9be21b6beb51"}, - {file = "pandas-2.2.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:c2ce852e1cf2509a69e98358e8458775f89599566ac3775e70419b98615f4b06"}, - {file = "pandas-2.2.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:53680dc9b2519cbf609c62db3ed7c0b499077c7fefda564e330286e619ff0dd9"}, - {file = "pandas-2.2.1-cp310-cp310-win_amd64.whl", hash = "sha256:94e714a1cca63e4f5939cdce5f29ba8d415d85166be3441165edd427dc9f6bc0"}, - {file = "pandas-2.2.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f821213d48f4ab353d20ebc24e4faf94ba40d76680642fb7ce2ea31a3ad94f9b"}, - {file = "pandas-2.2.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c70e00c2d894cb230e5c15e4b1e1e6b2b478e09cf27cc593a11ef955b9ecc81a"}, - {file = "pandas-2.2.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e97fbb5387c69209f134893abc788a6486dbf2f9e511070ca05eed4b930b1b02"}, - {file = "pandas-2.2.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:101d0eb9c5361aa0146f500773395a03839a5e6ecde4d4b6ced88b7e5a1a6403"}, - {file = "pandas-2.2.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:7d2ed41c319c9fb4fd454fe25372028dfa417aacb9790f68171b2e3f06eae8cd"}, - {file = "pandas-2.2.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:af5d3c00557d657c8773ef9ee702c61dd13b9d7426794c9dfeb1dc4a0bf0ebc7"}, - {file = "pandas-2.2.1-cp311-cp311-win_amd64.whl", hash = "sha256:06cf591dbaefb6da9de8472535b185cba556d0ce2e6ed28e21d919704fef1a9e"}, - {file = "pandas-2.2.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:88ecb5c01bb9ca927ebc4098136038519aa5d66b44671861ffab754cae75102c"}, - {file = "pandas-2.2.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:04f6ec3baec203c13e3f8b139fb0f9f86cd8c0b94603ae3ae8ce9a422e9f5bee"}, - {file = "pandas-2.2.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a935a90a76c44fe170d01e90a3594beef9e9a6220021acfb26053d01426f7dc2"}, - {file = "pandas-2.2.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c391f594aae2fd9f679d419e9a4d5ba4bce5bb13f6a989195656e7dc4b95c8f0"}, - {file = "pandas-2.2.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:9d1265545f579edf3f8f0cb6f89f234f5e44ba725a34d86535b1a1d38decbccc"}, - {file = "pandas-2.2.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:11940e9e3056576ac3244baef2fedade891977bcc1cb7e5cc8f8cc7d603edc89"}, - {file = "pandas-2.2.1-cp312-cp312-win_amd64.whl", hash = "sha256:4acf681325ee1c7f950d058b05a820441075b0dd9a2adf5c4835b9bc056bf4fb"}, - {file = "pandas-2.2.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9bd8a40f47080825af4317d0340c656744f2bfdb6819f818e6ba3cd24c0e1397"}, - {file = "pandas-2.2.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:df0c37ebd19e11d089ceba66eba59a168242fc6b7155cba4ffffa6eccdfb8f16"}, - {file = "pandas-2.2.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:739cc70eaf17d57608639e74d63387b0d8594ce02f69e7a0b046f117974b3019"}, - {file = "pandas-2.2.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f9d3558d263073ed95e46f4650becff0c5e1ffe0fc3a015de3c79283dfbdb3df"}, - {file = "pandas-2.2.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:4aa1d8707812a658debf03824016bf5ea0d516afdea29b7dc14cf687bc4d4ec6"}, - {file = "pandas-2.2.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:76f27a809cda87e07f192f001d11adc2b930e93a2b0c4a236fde5429527423be"}, - {file = "pandas-2.2.1-cp39-cp39-win_amd64.whl", hash = "sha256:1ba21b1d5c0e43416218db63037dbe1a01fc101dc6e6024bcad08123e48004ab"}, - {file = "pandas-2.2.1.tar.gz", hash = "sha256:0ab90f87093c13f3e8fa45b48ba9f39181046e8f3317d3aadb2fffbb1b978572"}, + {file = "pandas-2.2.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:90c6fca2acf139569e74e8781709dccb6fe25940488755716d1d354d6bc58bce"}, + {file = "pandas-2.2.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4abfe0be0d7221be4f12552995e58723c7422c80a659da13ca382697de830c08"}, + {file = "pandas-2.2.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8635c16bf3d99040fdf3ca3db669a7250ddf49c55dc4aa8fe0ae0fa8d6dcc1f0"}, + {file = "pandas-2.2.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:40ae1dffb3967a52203105a077415a86044a2bea011b5f321c6aa64b379a3f51"}, + {file = "pandas-2.2.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:8e5a0b00e1e56a842f922e7fae8ae4077aee4af0acb5ae3622bd4b4c30aedf99"}, + {file = "pandas-2.2.2-cp310-cp310-win_amd64.whl", hash = "sha256:ddf818e4e6c7c6f4f7c8a12709696d193976b591cc7dc50588d3d1a6b5dc8772"}, + {file = "pandas-2.2.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:696039430f7a562b74fa45f540aca068ea85fa34c244d0deee539cb6d70aa288"}, + {file = "pandas-2.2.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8e90497254aacacbc4ea6ae5e7a8cd75629d6ad2b30025a4a8b09aa4faf55151"}, + {file = "pandas-2.2.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:58b84b91b0b9f4bafac2a0ac55002280c094dfc6402402332c0913a59654ab2b"}, + {file = "pandas-2.2.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6d2123dc9ad6a814bcdea0f099885276b31b24f7edf40f6cdbc0912672e22eee"}, + {file = "pandas-2.2.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:2925720037f06e89af896c70bca73459d7e6a4be96f9de79e2d440bd499fe0db"}, + {file = "pandas-2.2.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:0cace394b6ea70c01ca1595f839cf193df35d1575986e484ad35c4aeae7266c1"}, + {file = "pandas-2.2.2-cp311-cp311-win_amd64.whl", hash = "sha256:873d13d177501a28b2756375d59816c365e42ed8417b41665f346289adc68d24"}, + {file = "pandas-2.2.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:9dfde2a0ddef507a631dc9dc4af6a9489d5e2e740e226ad426a05cabfbd7c8ef"}, + {file = "pandas-2.2.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:e9b79011ff7a0f4b1d6da6a61aa1aa604fb312d6647de5bad20013682d1429ce"}, + {file = "pandas-2.2.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1cb51fe389360f3b5a4d57dbd2848a5f033350336ca3b340d1c53a1fad33bcad"}, + {file = "pandas-2.2.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eee3a87076c0756de40b05c5e9a6069c035ba43e8dd71c379e68cab2c20f16ad"}, + {file = "pandas-2.2.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:3e374f59e440d4ab45ca2fffde54b81ac3834cf5ae2cdfa69c90bc03bde04d76"}, + {file = "pandas-2.2.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:43498c0bdb43d55cb162cdc8c06fac328ccb5d2eabe3cadeb3529ae6f0517c32"}, + {file = "pandas-2.2.2-cp312-cp312-win_amd64.whl", hash = "sha256:d187d355ecec3629624fccb01d104da7d7f391db0311145817525281e2804d23"}, + {file = "pandas-2.2.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:0ca6377b8fca51815f382bd0b697a0814c8bda55115678cbc94c30aacbb6eff2"}, + {file = "pandas-2.2.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:001910ad31abc7bf06f49dcc903755d2f7f3a9186c0c040b827e522e9cef0863"}, + {file = "pandas-2.2.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:66b479b0bd07204e37583c191535505410daa8df638fd8e75ae1b383851fe921"}, + {file = "pandas-2.2.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:a77e9d1c386196879aa5eb712e77461aaee433e54c68cf253053a73b7e49c33a"}, + {file = "pandas-2.2.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:92fd6b027924a7e178ac202cfbe25e53368db90d56872d20ffae94b96c7acc57"}, + {file = "pandas-2.2.2-cp39-cp39-win_amd64.whl", hash = "sha256:640cef9aa381b60e296db324337a554aeeb883ead99dc8f6c18e81a93942f5f4"}, + {file = "pandas-2.2.2.tar.gz", hash = "sha256:9e79019aba43cb4fda9e4d983f8e88ca0373adbb697ae9c6c43093218de28b54"}, ] [package.dependencies] numpy = [ - {version = ">=1.22.4,<2", markers = "python_version < \"3.11\""}, - {version = ">=1.23.2,<2", markers = "python_version == \"3.11\""}, - {version = ">=1.26.0,<2", markers = "python_version >= \"3.12\""}, + {version = ">=1.22.4", markers = "python_version < \"3.11\""}, + {version = ">=1.23.2", markers = "python_version == \"3.11\""}, + {version = ">=1.26.0", markers = "python_version >= \"3.12\""}, ] python-dateutil = ">=2.8.2" pytz = ">=2020.1" @@ -1711,13 +1709,13 @@ jeepney = ">=0.6" [[package]] name = "sentry-sdk" -version = "1.44.1" +version = "1.45.0" description = "Python client for Sentry (https://sentry.io)" optional = false python-versions = "*" files = [ - {file = "sentry-sdk-1.44.1.tar.gz", hash = "sha256:24e6a53eeabffd2f95d952aa35ca52f0f4201d17f820ac9d3ff7244c665aaf68"}, - {file = "sentry_sdk-1.44.1-py2.py3-none-any.whl", hash = "sha256:5f75eb91d8ab6037c754a87b8501cc581b2827e923682f593bed3539ce5b3999"}, + {file = "sentry-sdk-1.45.0.tar.gz", hash = "sha256:509aa9678c0512344ca886281766c2e538682f8acfa50fd8d405f8c417ad0625"}, + {file = "sentry_sdk-1.45.0-py2.py3-none-any.whl", hash = "sha256:1ce29e30240cc289a027011103a8c83885b15ef2f316a60bcc7c5300afa144f1"}, ] [package.dependencies] @@ -1758,18 +1756,18 @@ tornado = ["tornado (>=5)"] [[package]] name = "setuptools" -version = "69.2.0" +version = "69.5.1" description = "Easily download, build, install, upgrade, and uninstall Python packages" optional = false python-versions = ">=3.8" files = [ - {file = "setuptools-69.2.0-py3-none-any.whl", hash = "sha256:c21c49fb1042386df081cb5d86759792ab89efca84cf114889191cd09aacc80c"}, - {file = "setuptools-69.2.0.tar.gz", hash = "sha256:0ff4183f8f42cd8fa3acea16c45205521a4ef28f73c6391d8a25e92893134f2e"}, + {file = "setuptools-69.5.1-py3-none-any.whl", hash = "sha256:c636ac361bc47580504644275c9ad802c50415c7522212252c033bd15f301f32"}, + {file = "setuptools-69.5.1.tar.gz", hash = "sha256:6c1fccdac05a97e598fb0ae3bbed5904ccb317337a51139dcd51453611bbb987"}, ] [package.extras] -docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (<7.2.5)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] -testing = ["build[virtualenv]", "filelock (>=3.4.0)", "importlib-metadata", "ini2toml[lite] (>=0.9)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "mypy (==1.9)", "packaging (>=23.2)", "pip (>=19.1)", "pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff (>=0.2.1)", "pytest-timeout", "pytest-xdist (>=3)", "tomli", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] +testing = ["build[virtualenv]", "filelock (>=3.4.0)", "importlib-metadata", "ini2toml[lite] (>=0.9)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "mypy (==1.9)", "packaging (>=23.2)", "pip (>=19.1)", "pytest (>=6,!=8.1.1)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy", "pytest-perf", "pytest-ruff (>=0.2.1)", "pytest-timeout", "pytest-xdist (>=3)", "tomli", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] testing-integration = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "packaging (>=23.2)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] [[package]] From 272e132dc0c2210cf1649b1718513227c5d08ec9 Mon Sep 17 00:00:00 2001 From: Jan Inge Bergseth <31886431+BergsethCognite@users.noreply.github.com> Date: Tue, 16 Apr 2024 15:46:33 +0200 Subject: [PATCH 11/34] added transformations for workorder & workitems to oid test data module --- .../cdf_oid_example_data/default.config.yaml | 2 + ...id_workmate_workitem_example.schedule.yaml | 3 + ...tr_event_oid_workmate_workitem_example.sql | 34 ++++++++ ...r_event_oid_workmate_workitem_example.yaml | 19 +++++ ...d_workmate_workorder_example.schedule.yaml | 3 + ...r_event_oid_workmate_workorder_example.sql | 34 ++++++++ ..._event_oid_workmate_workorder_example.yaml | 19 +++++ demo/config.demo.yaml | 1 + .../cdf_oid_example_data.yaml | 79 +++++++++++++++++++ .../cdf_oid_example_data.yaml | 2 + .../auth_auth_verify_happypath_windows.txt | 4 +- ...th_auth_verify_no_capabilities_windows.txt | 4 +- .../auth_auth_verify_two_groups_windows.txt | 4 +- ...auth_verify_wrong_capabilities_windows.txt | 4 +- 14 files changed, 204 insertions(+), 8 deletions(-) create mode 100644 cognite_toolkit/cognite_modules/examples/cdf_oid_example_data/transformations/tr_event_oid_workmate_workitem_example.schedule.yaml create mode 100644 cognite_toolkit/cognite_modules/examples/cdf_oid_example_data/transformations/tr_event_oid_workmate_workitem_example.sql create mode 100644 cognite_toolkit/cognite_modules/examples/cdf_oid_example_data/transformations/tr_event_oid_workmate_workitem_example.yaml create mode 100644 cognite_toolkit/cognite_modules/examples/cdf_oid_example_data/transformations/tr_event_oid_workmate_workorder_example.schedule.yaml create mode 100644 cognite_toolkit/cognite_modules/examples/cdf_oid_example_data/transformations/tr_event_oid_workmate_workorder_example.sql create mode 100644 cognite_toolkit/cognite_modules/examples/cdf_oid_example_data/transformations/tr_event_oid_workmate_workorder_example.yaml diff --git a/cognite_toolkit/cognite_modules/examples/cdf_oid_example_data/default.config.yaml b/cognite_toolkit/cognite_modules/examples/cdf_oid_example_data/default.config.yaml index fe1139623..d22cb6baa 100644 --- a/cognite_toolkit/cognite_modules/examples/cdf_oid_example_data/default.config.yaml +++ b/cognite_toolkit/cognite_modules/examples/cdf_oid_example_data/default.config.yaml @@ -12,6 +12,8 @@ # and can be used to control access. default_location: oid source_asset: workmate +source_event: workmate source_workorder: workmate source_files: fileshare source_timeseries: pi + diff --git a/cognite_toolkit/cognite_modules/examples/cdf_oid_example_data/transformations/tr_event_oid_workmate_workitem_example.schedule.yaml b/cognite_toolkit/cognite_modules/examples/cdf_oid_example_data/transformations/tr_event_oid_workmate_workitem_example.schedule.yaml new file mode 100644 index 000000000..4118582ff --- /dev/null +++ b/cognite_toolkit/cognite_modules/examples/cdf_oid_example_data/transformations/tr_event_oid_workmate_workitem_example.schedule.yaml @@ -0,0 +1,3 @@ +externalId: tr_event_{{default_location}}_{{source_asset}}_workitem_example +interval: '{{scheduleHourly}}' +isPaused: false \ No newline at end of file diff --git a/cognite_toolkit/cognite_modules/examples/cdf_oid_example_data/transformations/tr_event_oid_workmate_workitem_example.sql b/cognite_toolkit/cognite_modules/examples/cdf_oid_example_data/transformations/tr_event_oid_workmate_workitem_example.sql new file mode 100644 index 000000000..42892423e --- /dev/null +++ b/cognite_toolkit/cognite_modules/examples/cdf_oid_example_data/transformations/tr_event_oid_workmate_workitem_example.sql @@ -0,0 +1,34 @@ +with assetid ( +select + first(wa.sourceExternalId) as workitemid, + collect_list(ai.id) as assetIds +from + `workorder_oid_workmate`.`workitem2assets` wa, + _cdf.assets ai +where + ai.externalId = wa.targetExternalId +GROUP BY sourceExternalId) +select + wo.externalId, + wo.itemInfo, + dataset_id('ds_transformations_oid') as dataSetId, + cast(from_unixtime(double(wo.`startTime`)/1000) + as TIMESTAMP) as startTime, + cast(from_unixtime(double(wo.`endTime`)/1000) + as TIMESTAMP) as endTime, + ai.assetIds as assetIds, + "workitem" as type, + "OID - workmate" as source, + to_metadata_except( + array("key", + "startTime", + "endTime", + "externalId", + "itemInfo", + "assetIds"), *) as metadata +from + `workorder_oid_workmate`.`workitems` wo, + assetid ai +where + ai.workitemid = wo.externalId + and wo.`endTime` > wo.`startTime` \ No newline at end of file diff --git a/cognite_toolkit/cognite_modules/examples/cdf_oid_example_data/transformations/tr_event_oid_workmate_workitem_example.yaml b/cognite_toolkit/cognite_modules/examples/cdf_oid_example_data/transformations/tr_event_oid_workmate_workitem_example.yaml new file mode 100644 index 000000000..60578efcc --- /dev/null +++ b/cognite_toolkit/cognite_modules/examples/cdf_oid_example_data/transformations/tr_event_oid_workmate_workitem_example.yaml @@ -0,0 +1,19 @@ +externalId: tr_event_{{default_location}}_{{source_event}}_workitem_example +name: event:{{default_location}}:{{source_event}}:workitem:example +destination: + type: "events" +ignoreNullFields: true +isPublic: true +conflictMode: upsert +dataSetExternalId: ds_transformations_{{default_location}} +# Specify credentials separately like this: +# You can also use different credentials for the running transformations than the ones you use to deploy +authentication: + clientId: {{cicd_clientId}} + clientSecret: {{cicd_clientSecret}} + tokenUri: {{cicd_tokenUri}} + # Optional: If idP requires providing the cicd_scopes + cdfProjectName: {{cdfProjectName}} + scopes: {{cicd_scopes}} + # Optional: If idP requires providing the cicd_audience + audience: {{cicd_audience}} \ No newline at end of file diff --git a/cognite_toolkit/cognite_modules/examples/cdf_oid_example_data/transformations/tr_event_oid_workmate_workorder_example.schedule.yaml b/cognite_toolkit/cognite_modules/examples/cdf_oid_example_data/transformations/tr_event_oid_workmate_workorder_example.schedule.yaml new file mode 100644 index 000000000..1b83b3aff --- /dev/null +++ b/cognite_toolkit/cognite_modules/examples/cdf_oid_example_data/transformations/tr_event_oid_workmate_workorder_example.schedule.yaml @@ -0,0 +1,3 @@ +externalId: tr_event_{{default_location}}_{{source_asset}}_workorder_example +interval: '{{scheduleHourly}}' +isPaused: false \ No newline at end of file diff --git a/cognite_toolkit/cognite_modules/examples/cdf_oid_example_data/transformations/tr_event_oid_workmate_workorder_example.sql b/cognite_toolkit/cognite_modules/examples/cdf_oid_example_data/transformations/tr_event_oid_workmate_workorder_example.sql new file mode 100644 index 000000000..e8f86c727 --- /dev/null +++ b/cognite_toolkit/cognite_modules/examples/cdf_oid_example_data/transformations/tr_event_oid_workmate_workorder_example.sql @@ -0,0 +1,34 @@ +with assetid ( +select + first(wa.sourceExternalId) as workorderid, + collect_list(ai.id) as assetIds +from + `workorder_oid_workmate`.`workorder2assets` wa, + _cdf.assets ai +where + ai.externalId = wa.targetExternalId +GROUP BY sourceExternalId) +select + wo.externalId, + wo.description, + dataset_id('ds_transformations_oid') as dataSetId, + cast(from_unixtime(double(wo.`startTime`)/1000) + as TIMESTAMP) as startTime, + cast(from_unixtime(double(wo.`endTime`)/1000) + as TIMESTAMP) as endTime, + ai.assetIds as assetIds, + "workorder" as type, + "OID - workmate" as source, + to_metadata_except( + array("key", + "startTime", + "endTime", + "externalId", + "wo.description", + "assetIds"), *) as metadata +from + `workorder_oid_workmate`.`workorders` wo, + assetid ai +where + ai.workorderid = wo.workOrderNumber + and wo.`endTime` > wo.`startTime` diff --git a/cognite_toolkit/cognite_modules/examples/cdf_oid_example_data/transformations/tr_event_oid_workmate_workorder_example.yaml b/cognite_toolkit/cognite_modules/examples/cdf_oid_example_data/transformations/tr_event_oid_workmate_workorder_example.yaml new file mode 100644 index 000000000..be6e90522 --- /dev/null +++ b/cognite_toolkit/cognite_modules/examples/cdf_oid_example_data/transformations/tr_event_oid_workmate_workorder_example.yaml @@ -0,0 +1,19 @@ +externalId: tr_event_{{default_location}}_{{source_event}}_workorder_example +name: event:{{default_location}}:{{source_event}}:workorder:example +destination: + type: "events" +ignoreNullFields: true +isPublic: true +conflictMode: upsert +dataSetExternalId: ds_transformations_{{default_location}} +# Specify credentials separately like this: +# You can also use different credentials for the running transformations than the ones you use to deploy +authentication: + clientId: {{cicd_clientId}} + clientSecret: {{cicd_clientSecret}} + tokenUri: {{cicd_tokenUri}} + # Optional: If idP requires providing the cicd_scopes + cdfProjectName: {{cdfProjectName}} + scopes: {{cicd_scopes}} + # Optional: If idP requires providing the cicd_audience + audience: {{cicd_audience}} \ No newline at end of file diff --git a/demo/config.demo.yaml b/demo/config.demo.yaml index f048ffdf0..dd44799f0 100644 --- a/demo/config.demo.yaml +++ b/demo/config.demo.yaml @@ -138,6 +138,7 @@ modules: # and can be used to control access. default_location: oid source_asset: workmate + source_event: workmate source_workorder: workmate source_files: fileshare source_timeseries: pi diff --git a/tests/tests_unit/test_approval_modules_snapshots/cdf_oid_example_data.yaml b/tests/tests_unit/test_approval_modules_snapshots/cdf_oid_example_data.yaml index 58df63bcf..11482654a 100644 --- a/tests/tests_unit/test_approval_modules_snapshots/cdf_oid_example_data.yaml +++ b/tests/tests_unit/test_approval_modules_snapshots/cdf_oid_example_data.yaml @@ -706,10 +706,89 @@ Transformation: clientSecret: dummy scopes: https://bluefield.cognitedata.com/.default tokenUri: dummy +- conflictMode: upsert + dataSetId: 42 + destination: + type: events + destinationOidcCredentials: + audience: https://bluefield.cognitedata.com + cdfProjectName: pytest-project + clientId: dummy + clientSecret: dummy + scopes: https://bluefield.cognitedata.com/.default + tokenUri: dummy + externalId: tr_event_oid_workmate_workitem_example + ignoreNullFields: true + isPublic: true + name: event:oid:workmate:workitem:example + query: "with assetid (\nselect\n first(wa.sourceExternalId) as workitemid,\n collect_list(ai.id)\ + \ as assetIds\nfrom\n `workorder_oid_workmate`.`workitem2assets` wa,\n\ + \ _cdf.assets ai\nwhere\n ai.externalId = wa.targetExternalId\n\ + GROUP BY sourceExternalId)\nselect\n wo.externalId,\n wo.itemInfo,\n dataset_id('ds_transformations_oid')\ + \ as dataSetId,\n cast(from_unixtime(double(wo.`startTime`)/1000)\n \ + \ as TIMESTAMP) as startTime,\n cast(from_unixtime(double(wo.`endTime`)/1000)\n\ + \ as TIMESTAMP) as endTime,\n ai.assetIds \ + \ as assetIds,\n \"workitem\" as type,\n\ + \ \"OID - workmate\" as source,\n to_metadata_except(\n\ + \ array(\"key\",\n \"startTime\",\n \"endTime\",\n \ + \ \"externalId\",\n \"itemInfo\",\n \"assetIds\"), *) \ + \ as metadata\nfrom \n `workorder_oid_workmate`.`workitems` wo,\n \ + \ assetid ai\nwhere\n ai.workitemid = wo.externalId\n and wo.`endTime` > wo.`startTime`" + sourceOidcCredentials: + audience: https://bluefield.cognitedata.com + cdfProjectName: pytest-project + clientId: dummy + clientSecret: dummy + scopes: https://bluefield.cognitedata.com/.default + tokenUri: dummy +- conflictMode: upsert + dataSetId: 42 + destination: + type: events + destinationOidcCredentials: + audience: https://bluefield.cognitedata.com + cdfProjectName: pytest-project + clientId: dummy + clientSecret: dummy + scopes: https://bluefield.cognitedata.com/.default + tokenUri: dummy + externalId: tr_event_oid_workmate_workorder_example + ignoreNullFields: true + isPublic: true + name: event:oid:workmate:workorder:example + query: "with assetid (\nselect\n first(wa.sourceExternalId) as workorderid,\n \ + \ collect_list(ai.id) as assetIds\nfrom\n `workorder_oid_workmate`.`workorder2assets`\ + \ wa,\n _cdf.assets ai\nwhere\n ai.externalId\ + \ = wa.targetExternalId\nGROUP BY sourceExternalId)\nselect\n wo.externalId,\n\ + \ wo.description,\n dataset_id('ds_transformations_oid') as dataSetId,\n cast(from_unixtime(double(wo.`startTime`)/1000)\n\ + \ as TIMESTAMP) as startTime,\n cast(from_unixtime(double(wo.`endTime`)/1000)\n\ + \ as TIMESTAMP) as endTime,\n ai.assetIds \ + \ as assetIds,\n \"workorder\" as type,\n\ + \ \"OID - workmate\" as source,\n to_metadata_except(\n\ + \ array(\"key\",\n \"startTime\",\n \"endTime\",\n \ + \ \"externalId\",\n \"wo.description\",\n \"assetIds\"),\ + \ *) as metadata\nfrom\n `workorder_oid_workmate`.`workorders` wo,\n\ + \ assetid ai\nwhere\n ai.workorderid = wo.workOrderNumber\n and wo.`endTime`\ + \ > wo.`startTime`\n" + sourceOidcCredentials: + audience: https://bluefield.cognitedata.com + cdfProjectName: pytest-project + clientId: dummy + clientSecret: dummy + scopes: https://bluefield.cognitedata.com/.default + tokenUri: dummy TransformationSchedule: - externalId: tr_asset_oid_workmate_asset_hierarchy_example interval: 7 * * * * isPaused: false +- externalId: tr_event_oid_workmate_workitem_example + interval: 7 * * * * + isPaused: false +- externalId: tr_event_oid_workmate_workorder_example + interval: 7 * * * * + isPaused: false deleted: TransformationSchedule: - externalId: tr_asset_oid_workmate_asset_hierarchy_example + - externalId: tr_event_oid_workmate_workitem_example + - externalId: tr_event_oid_workmate_workorder_example diff --git a/tests/tests_unit/test_approval_modules_snapshots_clean/cdf_oid_example_data.yaml b/tests/tests_unit/test_approval_modules_snapshots_clean/cdf_oid_example_data.yaml index 4f0c6c207..42a71fe92 100644 --- a/tests/tests_unit/test_approval_modules_snapshots_clean/cdf_oid_example_data.yaml +++ b/tests/tests_unit/test_approval_modules_snapshots_clean/cdf_oid_example_data.yaml @@ -36,3 +36,5 @@ deleted: - workorders TransformationSchedule: - externalId: tr_asset_oid_workmate_asset_hierarchy_example + - externalId: tr_event_oid_workmate_workitem_example + - externalId: tr_event_oid_workmate_workorder_example diff --git a/tests/tests_unit/test_cdf_tk/auth_data_snapshots/auth_auth_verify_happypath_windows.txt b/tests/tests_unit/test_cdf_tk/auth_data_snapshots/auth_auth_verify_happypath_windows.txt index da82e7b4f..89fd217c2 100644 --- a/tests/tests_unit/test_cdf_tk/auth_data_snapshots/auth_auth_verify_happypath_windows.txt +++ b/tests/tests_unit/test_cdf_tk/auth_data_snapshots/auth_auth_verify_happypath_windows.txt @@ -1,7 +1,7 @@ Checking current service principal/application and environment configurations... -WARNING: CDF URL is set to https://westeurope-1.cognitedata.com, are you sure -it shouldn't be https://bluefield.cognitedata.com? +WARNING: CDF URL is set to https://greenfield.cognitedata.com, are you sure it +shouldn't be https://bluefield.cognitedata.com? OK Checking basic project configuration... OK diff --git a/tests/tests_unit/test_cdf_tk/auth_data_snapshots/auth_auth_verify_no_capabilities_windows.txt b/tests/tests_unit/test_cdf_tk/auth_data_snapshots/auth_auth_verify_no_capabilities_windows.txt index e6a969fa6..a5cb57314 100644 --- a/tests/tests_unit/test_cdf_tk/auth_data_snapshots/auth_auth_verify_no_capabilities_windows.txt +++ b/tests/tests_unit/test_cdf_tk/auth_data_snapshots/auth_auth_verify_no_capabilities_windows.txt @@ -1,7 +1,7 @@ Checking current service principal/application and environment configurations... -WARNING: CDF URL is set to https://westeurope-1.cognitedata.com, are you sure -it shouldn't be https://bluefield.cognitedata.com? +WARNING: CDF URL is set to https://greenfield.cognitedata.com, are you sure it +shouldn't be https://bluefield.cognitedata.com? OK Checking basic project configuration... OK diff --git a/tests/tests_unit/test_cdf_tk/auth_data_snapshots/auth_auth_verify_two_groups_windows.txt b/tests/tests_unit/test_cdf_tk/auth_data_snapshots/auth_auth_verify_two_groups_windows.txt index c603b42fc..dcd84fce9 100644 --- a/tests/tests_unit/test_cdf_tk/auth_data_snapshots/auth_auth_verify_two_groups_windows.txt +++ b/tests/tests_unit/test_cdf_tk/auth_data_snapshots/auth_auth_verify_two_groups_windows.txt @@ -1,7 +1,7 @@ Checking current service principal/application and environment configurations... -WARNING: CDF URL is set to https://westeurope-1.cognitedata.com, are you sure -it shouldn't be https://bluefield.cognitedata.com? +WARNING: CDF URL is set to https://greenfield.cognitedata.com, are you sure it +shouldn't be https://bluefield.cognitedata.com? OK Checking basic project configuration... OK diff --git a/tests/tests_unit/test_cdf_tk/auth_data_snapshots/auth_auth_verify_wrong_capabilities_windows.txt b/tests/tests_unit/test_cdf_tk/auth_data_snapshots/auth_auth_verify_wrong_capabilities_windows.txt index c30c3e599..6aed522bd 100644 --- a/tests/tests_unit/test_cdf_tk/auth_data_snapshots/auth_auth_verify_wrong_capabilities_windows.txt +++ b/tests/tests_unit/test_cdf_tk/auth_data_snapshots/auth_auth_verify_wrong_capabilities_windows.txt @@ -1,7 +1,7 @@ Checking current service principal/application and environment configurations... -WARNING: CDF URL is set to https://westeurope-1.cognitedata.com, are you sure -it shouldn't be https://bluefield.cognitedata.com? +WARNING: CDF URL is set to https://greenfield.cognitedata.com, are you sure it +shouldn't be https://bluefield.cognitedata.com? OK Checking basic project configuration... OK From af8eae06b0d281d8b30dd25d6ae789325452e121 Mon Sep 17 00:00:00 2001 From: Jan Inge Bergseth <31886431+BergsethCognite@users.noreply.github.com> Date: Tue, 16 Apr 2024 15:49:25 +0200 Subject: [PATCH 12/34] Update CHANGELOG.templates.md --- CHANGELOG.templates.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.templates.md b/CHANGELOG.templates.md index e432daac7..4d82490c5 100644 --- a/CHANGELOG.templates.md +++ b/CHANGELOG.templates.md @@ -22,6 +22,7 @@ Changes are grouped as follows: - Fixed: aligned use of asset external ID across contextualization functions - Fixed: annotation logic with local time stamp for when to reprocess P&ID files - Fixed: input to P&ID annotation based on list of synonyms for tag +- Added Transformation for WorkOrder and WorkItems to OID testdata template ## [0.2.0a2] - 2024-04-03 From 977e1406cd22c1989b7a77cab161b640a29a5cae Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Wed, 17 Apr 2024 08:58:36 +0200 Subject: [PATCH 13/34] chore(deps): update pre-commit hook charliermarsh/ruff-pre-commit to v0.3.7 (#450) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index bc6b77f55..d1aaf43e0 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -12,7 +12,7 @@ repos: - --select=E,W,F,I,T,RUF,TID,UP - --fixable=E,W,F,I,T,RUF,TID,UP - --target-version=py39 - rev: v0.3.5 + rev: v0.3.7 - repo: https://github.com/psf/black rev: 24.3.0 From b0ded389147e362431724202e628bb570c08078c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A5l=20R=C3=B8nning?= Date: Wed, 17 Apr 2024 10:29:13 +0200 Subject: [PATCH 14/34] Fixed tests that failed when local env vars interfered (#455) * Fixed tests that failed when local env vars interfered * Removed dummies * Update tests/tests_unit/test_cdf_tk/test_utils.py Co-authored-by: Anders Albert <60234212+doctrino@users.noreply.github.com> * Update tests/tests_unit/test_approval_modules_snapshots/cdf_oid_example_data.yaml Co-authored-by: Anders Albert <60234212+doctrino@users.noreply.github.com> * Reverted to cleaner assert --------- Co-authored-by: Anders Albert <60234212+doctrino@users.noreply.github.com> --- tests/tests_integration/conftest.py | 2 +- tests/tests_unit/test_cdf_tk/test_utils.py | 77 ++++++++++++++-------- 2 files changed, 50 insertions(+), 29 deletions(-) diff --git a/tests/tests_integration/conftest.py b/tests/tests_integration/conftest.py index 1c0b5646e..71f2495d2 100644 --- a/tests/tests_integration/conftest.py +++ b/tests/tests_integration/conftest.py @@ -10,7 +10,7 @@ @pytest.fixture(scope="session") def cognite_client() -> CogniteClient: - load_dotenv(REPO_ROOT / ".env") + load_dotenv(REPO_ROOT / ".env", override=True) cdf_cluster = os.environ["CDF_CLUSTER"] credentials = OAuthClientCredentials( token_url=os.environ["IDP_TOKEN_URL"], diff --git a/tests/tests_unit/test_cdf_tk/test_utils.py b/tests/tests_unit/test_cdf_tk/test_utils.py index d0a145991..b0ecc8067 100644 --- a/tests/tests_unit/test_cdf_tk/test_utils.py +++ b/tests/tests_unit/test_cdf_tk/test_utils.py @@ -3,6 +3,7 @@ import tempfile from pathlib import Path from typing import Any +from unittest import mock from unittest.mock import MagicMock, Mock, patch import pytest @@ -225,18 +226,23 @@ def test_initialize_interactive_login(self): CDF_URL=https://my_cluster.cognitedata.com IDP_SCOPES=https://my_cluster.cognitedata.com/.default IDP_AUTHORITY_URL=https://login.microsoftonline.com/{tenant}""" - with MonkeyPatch.context() as mp: - mp.setenv("LOGIN_FLOW", "interactive") - mp.setenv("CDF_CLUSTER", "my_cluster") - mp.setenv("CDF_PROJECT", "my_project") - mp.setenv("IDP_TENANT_ID", "{tenant}") - mp.setenv("IDP_CLIENT_ID", "7890") - mp.setattr("cognite_toolkit._cdf_tk.utils.OAuthInteractive", MagicMock(spec=OAuthInteractive)) - with monkeypatch_cognite_client() as _: - config = CDFToolConfig() - env_file = AuthVariables.from_env(config._environ).create_dotenv_file() - not_equal = set(env_file.splitlines()) ^ set(expected.splitlines()) - assert not not_equal, "Found differences in the generated .env file: \n" + "\n".join(not_equal) + + envs = { + "LOGIN_FLOW": "interactive", + "CDF_CLUSTER": "my_cluster", + "CDF_PROJECT": "my_project", + "IDP_TENANT_ID": "{tenant}", + "IDP_CLIENT_ID": "7890", + } + + with mock.patch.dict(os.environ, envs, clear=True): + with MonkeyPatch.context() as mp: + mp.setattr("cognite_toolkit._cdf_tk.utils.OAuthInteractive", MagicMock(spec=OAuthInteractive)) + with monkeypatch_cognite_client() as _: + config = CDFToolConfig() + env_file = AuthVariables.from_env(config._environ).create_dotenv_file() + not_equal = set(env_file.splitlines()) ^ set(expected.splitlines()) + assert not not_equal, "Found differences in the generated .env file: \n" + "\n".join(not_equal) @pytest.mark.skipif( os.environ.get("IS_GITHUB_ACTIONS") == "true", @@ -256,19 +262,25 @@ def test_initialize_client_credentials_login(self): CDF_URL=https://my_cluster.cognitedata.com IDP_SCOPES=https://my_cluster.cognitedata.com/.default IDP_AUDIENCE=https://my_cluster.cognitedata.com""" - with MonkeyPatch.context() as mp: - mp.setenv("LOGIN_FLOW", "client_credentials") - mp.setenv("CDF_CLUSTER", "my_cluster") - mp.setenv("CDF_PROJECT", "my_project") - mp.setenv("IDP_TENANT_ID", "12345") - mp.setenv("IDP_CLIENT_ID", "7890") - mp.setenv("IDP_CLIENT_SECRET", "12345") - mp.setattr("cognite_toolkit._cdf_tk.utils.OAuthClientCredentials", MagicMock(spec=OAuthClientCredentials)) - with monkeypatch_cognite_client() as _: - config = CDFToolConfig() - env_file = AuthVariables.from_env(config._environ).create_dotenv_file() - not_equal = set(env_file.splitlines()) ^ set(expected.splitlines()) - assert not not_equal, "Found differences in the generated .env file: \n" + "\n".join(not_equal) + + envs = { + "LOGIN_FLOW": "client_credentials", + "CDF_CLUSTER": "my_cluster", + "CDF_PROJECT": "my_project", + "IDP_TENANT_ID": "12345", + "IDP_CLIENT_ID": "7890", + "IDP_CLIENT_SECRET": "12345", + } + with mock.patch.dict(os.environ, envs, clear=True): + with MonkeyPatch.context() as mp: + mp.setattr( + "cognite_toolkit._cdf_tk.utils.OAuthClientCredentials", MagicMock(spec=OAuthClientCredentials) + ) + with monkeypatch_cognite_client() as _: + config = CDFToolConfig() + env_file = AuthVariables.from_env(config._environ).create_dotenv_file() + not_equal = set(env_file.splitlines()) ^ set(expected.splitlines()) + assert not not_equal, "Found differences in the generated .env file: \n" + "\n".join(not_equal) def auth_variables_validate_test_cases(): @@ -358,6 +370,7 @@ def auth_variables_validate_test_cases(): yield pytest.param( { "CDF_CLUSTER": "my_cluster", + "CDF_PROJECT": "", }, False, "error", @@ -367,6 +380,15 @@ def auth_variables_validate_test_cases(): ) +class TestEnvironmentVariables: + def test_env_variable(self): + with patch.dict(os.environ, {"MY_ENV_VAR": "test_value"}): + # Inside this block, MY_ENV_VAR is set to 'test_value' + assert os.environ["MY_ENV_VAR"] == "test_value" + + assert os.environ.get("MY_ENV_VAR") is None + + class TestAuthVariables: @pytest.mark.skipif( os.environ.get("IS_GITHUB_ACTIONS") == "true", @@ -384,9 +406,8 @@ def test_validate( expected_messages: list[str], expected_vars: dict[str, str], ) -> None: - with MonkeyPatch.context() as mp: - for key, value in environment_variables.items(): - mp.setenv(key, value) + + with mock.patch.dict(os.environ, environment_variables, clear=True): auth_var = AuthVariables.from_env() results = auth_var.validate(verbose) From 5b068f4b7ed7def8e5d4909473d1326e16e30491 Mon Sep 17 00:00:00 2001 From: Anders Albert <60234212+doctrino@users.noreply.github.com> Date: Wed, 17 Apr 2024 11:13:12 +0200 Subject: [PATCH 15/34] Replace black with ruff (#456) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * build: replace plack with ruff format * refactor: ruff-format * Ãstyle: pre-commit * refactor: update toolkit function hash --- .pre-commit-config.yaml | 13 ++----- cdf-tk-dev.py | 34 +++++++++---------- .../_cdf_tk/load/_resource_loaders.py | 2 -- .../pipeline.py | 3 -- .../cdf_data_pipeline_files_valhall.yaml | 2 +- tests/tests_unit/test_cdf_tk/test_load.py | 13 ------- tests/tests_unit/test_cdf_tk/test_utils.py | 1 - 7 files changed, 20 insertions(+), 48 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d1aaf43e0..69cde3a97 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -12,19 +12,10 @@ repos: - --select=E,W,F,I,T,RUF,TID,UP - --fixable=E,W,F,I,T,RUF,TID,UP - --target-version=py39 - rev: v0.3.7 - - - repo: https://github.com/psf/black - rev: 24.3.0 - hooks: - - id: black + - id: ruff-format args: - --line-length=120 - - --target-version=py38 - - --target-version=py39 - - --target-version=py310 - - --target-version=py311 - - --include=\.py$ + rev: v0.3.7 - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.6.0 diff --git a/cdf-tk-dev.py b/cdf-tk-dev.py index 112532da0..c26887cb5 100755 --- a/cdf-tk-dev.py +++ b/cdf-tk-dev.py @@ -6,23 +6,23 @@ # However, when you run the file in Visual Studio Code, the module should not be installed in your # python virtual environment, but rather be found in the root of the repo. # This workaround allows you to run cdf.py in Visual Studio Code like this: -""" { - "name": "Python: build", - "type": "python", - "request": "launch", - "program": "./cdf-tk-dev.py", - "args": [ - "--verbose", - "--override-env", - "build", - "--build-dir=build", - "--clean", - "--env=local", - "./cognite_toolkit/" - ], - "console": "integratedTerminal", - "justMyCode": false - }, +"""{ + "name": "Python: build", + "type": "python", + "request": "launch", + "program": "./cdf-tk-dev.py", + "args": [ + "--verbose", + "--override-env", + "build", + "--build-dir=build", + "--clean", + "--env=local", + "./cognite_toolkit/" + ], + "console": "integratedTerminal", + "justMyCode": false +}, """ import os diff --git a/cognite_toolkit/_cdf_tk/load/_resource_loaders.py b/cognite_toolkit/_cdf_tk/load/_resource_loaders.py index efb59a96c..1ac759a62 100644 --- a/cognite_toolkit/_cdf_tk/load/_resource_loaders.py +++ b/cognite_toolkit/_cdf_tk/load/_resource_loaders.py @@ -231,7 +231,6 @@ def load_resource( raw = [raw] for group in raw: - is_resource_scoped = any( any(scope_name in capability.get(acl, {}).get("scope", {}) for scope_name in self.resource_scope_names) for capability in group.get("capabilities", []) @@ -1014,7 +1013,6 @@ def load_resource( transformations = TransformationWriteList([]) for resource in resources: - source_oidc_credentials = ( resource.get("authentication", {}).get("read") or resource.get("authentication") or None ) diff --git a/cognite_toolkit/cognite_modules/examples/cdf_data_pipeline_files_valhall/functions/fn_context_files_oid_fileshare_annotation/pipeline.py b/cognite_toolkit/cognite_modules/examples/cdf_data_pipeline_files_valhall/functions/fn_context_files_oid_fileshare_annotation/pipeline.py index cb1611332..219c2d8c4 100644 --- a/cognite_toolkit/cognite_modules/examples/cdf_data_pipeline_files_valhall/functions/fn_context_files_oid_fileshare_annotation/pipeline.py +++ b/cognite_toolkit/cognite_modules/examples/cdf_data_pipeline_files_valhall/functions/fn_context_files_oid_fileshare_annotation/pipeline.py @@ -83,7 +83,6 @@ def annotate_pnid(client: CogniteClient, config: AnnotationConfig) -> None: # if no files to annotate - continue to next asset if len(files_to_process) > 0: - entities = get_files_entities(all_files) annotation_list = get_existing_annotations(client, entities) if entities else {} @@ -353,7 +352,6 @@ def process_files( asset_ids_list = asset_ids_list[:1000] if config.debug: - print(f"[INFO] Assets found: {asset_names}") continue @@ -413,7 +411,6 @@ def detect_create_annotation( detected_system_num, detected_count = get_sys_nums(job.result["items"][0]["annotations"], detected_count) for item in job.result["items"][0]["annotations"]: - textRegion = get_coordinates(item["region"]["vertices"]) for entity in item["entities"]: diff --git a/tests/tests_unit/test_approval_modules_snapshots/cdf_data_pipeline_files_valhall.yaml b/tests/tests_unit/test_approval_modules_snapshots/cdf_data_pipeline_files_valhall.yaml index 548caa7a6..7eac78f00 100644 --- a/tests/tests_unit/test_approval_modules_snapshots/cdf_data_pipeline_files_valhall.yaml +++ b/tests/tests_unit/test_approval_modules_snapshots/cdf_data_pipeline_files_valhall.yaml @@ -76,7 +76,7 @@ Function: externalId: fn_context_files_oid_fileshare_annotation functionPath: handler.py metadata: - cdf-toolkit-function-hash: 2cc9704f01bac8c7f6fa17f7fc916747431253398cb71a312e53c1d3163bf375 + cdf-toolkit-function-hash: c5fddc5cb0ccfcac02205cfb3eccfef878e73a75762671ebc6bfeb050e6dc86e version: 0.0.1 name: context:files:oid:fileshare:annotation owner: Anonymous diff --git a/tests/tests_unit/test_cdf_tk/test_load.py b/tests/tests_unit/test_cdf_tk/test_load.py index d2efbec6c..bfbf8f2ee 100644 --- a/tests/tests_unit/test_cdf_tk/test_load.py +++ b/tests/tests_unit/test_cdf_tk/test_load.py @@ -74,9 +74,7 @@ def test_loader_class( class TestFunctionLoader: - def test_load_functions(self, cognite_client_approval: ApprovalCogniteClient): - cdf_tool = MagicMock(spec=CDFToolConfig) cdf_tool.verify_client.return_value = cognite_client_approval.mock_client cdf_tool.verify_capabilities.return_value = cognite_client_approval.mock_client @@ -88,7 +86,6 @@ def test_load_functions(self, cognite_client_approval: ApprovalCogniteClient): assert len(loaded) == 2 def test_load_function(self, cognite_client_approval: ApprovalCogniteClient): - cdf_tool = MagicMock(spec=CDFToolConfig) cdf_tool.verify_client.return_value = cognite_client_approval.mock_client cdf_tool.verify_capabilities.return_value = cognite_client_approval.mock_client @@ -231,9 +228,7 @@ def test_update_data_model_random_view_order(self, cognite_client_approval: Appr class TestAuthLoader: - def test_load_all(self, cdf_tool_config: CDFToolConfig, monkeypatch: MonkeyPatch): - loader = AuthLoader.create_loader(cdf_tool_config, "all") loaded = loader.load_resource( @@ -254,7 +249,6 @@ def test_load_all(self, cdf_tool_config: CDFToolConfig, monkeypatch: MonkeyPatch assert caps["SessionsAcl"].scope._scope_name == "all" def test_load_all_scoped_only(self, cdf_tool_config: CDFToolConfig, monkeypatch: MonkeyPatch): - loader = AuthLoader.create_loader(cdf_tool_config, "all_scoped_only") loaded = loader.load_resource( DATA_FOLDER / "auth" / "1.my_group_unscoped.yaml", cdf_tool_config, skip_validation=False @@ -267,7 +261,6 @@ def test_load_all_scoped_only(self, cdf_tool_config: CDFToolConfig, monkeypatch: assert loaded is None def test_load_resource_scoped_only(self, cdf_tool_config: CDFToolConfig, monkeypatch: MonkeyPatch): - loader = AuthLoader.create_loader(cdf_tool_config, "resource_scoped_only") loaded = loader.load_resource( DATA_FOLDER / "auth" / "1.my_group_unscoped.yaml", cdf_tool_config, skip_validation=False @@ -289,7 +282,6 @@ def test_load_resource_scoped_only(self, cdf_tool_config: CDFToolConfig, monkeyp assert caps["SessionsAcl"].scope._scope_name == "all" def test_load_group_list_all(self, cdf_tool_config: CDFToolConfig, monkeypatch: MonkeyPatch): - loader = AuthLoader.create_loader(cdf_tool_config, "all") loaded = loader.load_resource( DATA_FOLDER / "auth" / "1.my_group_list_combined.yaml", cdf_tool_config, skip_validation=True @@ -299,7 +291,6 @@ def test_load_group_list_all(self, cdf_tool_config: CDFToolConfig, monkeypatch: assert len(loaded) == 2 def test_load_group_list_resource_scoped_only(self, cdf_tool_config: CDFToolConfig, monkeypatch: MonkeyPatch): - loader = AuthLoader.create_loader(cdf_tool_config, "resource_scoped_only") loaded = loader.load_resource( DATA_FOLDER / "auth" / "1.my_group_list_combined.yaml", cdf_tool_config, skip_validation=True @@ -309,7 +300,6 @@ def test_load_group_list_resource_scoped_only(self, cdf_tool_config: CDFToolConf assert loaded.name == "scoped_group_name" def test_load_group_list_all_scoped_only(self, cdf_tool_config: CDFToolConfig, monkeypatch: MonkeyPatch): - loader = AuthLoader.create_loader(cdf_tool_config, "all_scoped_only") loaded = loader.load_resource( DATA_FOLDER / "auth" / "1.my_group_list_combined.yaml", cdf_tool_config, skip_validation=True @@ -321,7 +311,6 @@ def test_load_group_list_all_scoped_only(self, cdf_tool_config: CDFToolConfig, m def test_unchanged_new_group( self, cdf_tool_config: CDFToolConfig, cognite_client_approval: ApprovalCogniteClient, monkeypatch: MonkeyPatch ): - loader = AuthLoader.create_loader(cdf_tool_config, "all") loaded = loader.load_resource( DATA_FOLDER / "auth" / "1.my_group_scoped.yaml", cdf_tool_config, skip_validation=True @@ -353,7 +342,6 @@ def test_unchanged_new_group( def test_upsert_group( self, cdf_tool_config: CDFToolConfig, cognite_client_approval: ApprovalCogniteClient, monkeypatch: MonkeyPatch ): - loader = AuthLoader.create_loader(cdf_tool_config, "all") loaded = loader.load_resource( DATA_FOLDER / "auth" / "1.my_group_scoped.yaml", cdf_tool_config, skip_validation=True @@ -503,7 +491,6 @@ def test_loader_takes_dict( def test_loader_takes_list( self, Loader: type[ResourceLoader], cdf_tool_config: CDFToolConfig, monkeypatch: MonkeyPatch ): - loader = Loader.create_loader(cdf_tool_config) if loader.resource_cls in [Transformation, FileMetadata]: diff --git a/tests/tests_unit/test_cdf_tk/test_utils.py b/tests/tests_unit/test_cdf_tk/test_utils.py index b0ecc8067..d039b4faa 100644 --- a/tests/tests_unit/test_cdf_tk/test_utils.py +++ b/tests/tests_unit/test_cdf_tk/test_utils.py @@ -406,7 +406,6 @@ def test_validate( expected_messages: list[str], expected_vars: dict[str, str], ) -> None: - with mock.patch.dict(os.environ, environment_variables, clear=True): auth_var = AuthVariables.from_env() results = auth_var.validate(verbose) From 04db26cadca7df5e953d220774448b64cf42299f Mon Sep 17 00:00:00 2001 From: Anders Albert <60234212+doctrino@users.noreply.github.com> Date: Wed, 17 Apr 2024 16:20:02 +0200 Subject: [PATCH 16/34] Ignore build folder and default to clean (#457) * ci: fix ignore build folder * feat: changed default of build command * build; changelog * fix; syntax error * refactor: uptade test * refactor; update message --- CHANGELOG.cdf-tk.md | 6 ++++++ cognite_toolkit/.gitignore | 3 ++- cognite_toolkit/_cdf.py | 8 +++----- cognite_toolkit/_cdf_tk/templates/_templates.py | 4 +++- tests/tests_unit/test_approval_modules.py | 6 +++--- tests/tests_unit/test_behavior.py | 4 ++-- tests/tests_unit/test_cdf_tk/test_build_files.py | 2 +- 7 files changed, 20 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.cdf-tk.md b/CHANGELOG.cdf-tk.md index 942cf1546..ea5f71385 100644 --- a/CHANGELOG.cdf-tk.md +++ b/CHANGELOG.cdf-tk.md @@ -21,6 +21,12 @@ Changes are grouped as follows: - Functions that are deployed with schedules no longer uses a short-lived session (before: failed after ~an hour). +### Changed + +- The `cdf-tk build` will now clean the build directory by default before building the modules. To stop this behavior, + use the `--no-clean` flag. +- The `.gitignore` file you get by running `cdf-tk init` now ignores the `/build` by default. + ## [0.2.0a2] - 2024-04-03 ### Added diff --git a/cognite_toolkit/.gitignore b/cognite_toolkit/.gitignore index 45633067b..395e4564f 100644 --- a/cognite_toolkit/.gitignore +++ b/cognite_toolkit/.gitignore @@ -273,4 +273,5 @@ $RECYCLE.BIN/ # Custom rules (everything added below won't be overriden by 'Generate .gitignore File' if you use 'Update' option) */.venv.* .venv.* -build.* \ No newline at end of file +build.* +build/ diff --git a/cognite_toolkit/_cdf.py b/cognite_toolkit/_cdf.py index 7fb55605e..dcf4dfdfe 100755 --- a/cognite_toolkit/_cdf.py +++ b/cognite_toolkit/_cdf.py @@ -233,12 +233,10 @@ def build( help="Build environment to build for", ), ] = "dev", - clean: Annotated[ + no_clean: Annotated[ bool, typer.Option( - "--clean", - "-c", - help="Delete the build directory before building the configurations", + "--no-clean", "-c", help="Whether not to delete the build directory before building the configurations" ), ] = False, ) -> None: @@ -264,7 +262,7 @@ def build( source_dir=source_path, config=config, system_config=system_config, - clean=clean, + clean=not no_clean, verbose=ctx.obj.verbose, ) diff --git a/cognite_toolkit/_cdf_tk/templates/_templates.py b/cognite_toolkit/_cdf_tk/templates/_templates.py index 381dc1040..2e8916063 100644 --- a/cognite_toolkit/_cdf_tk/templates/_templates.py +++ b/cognite_toolkit/_cdf_tk/templates/_templates.py @@ -46,7 +46,9 @@ def build_config( if not _RUNNING_IN_BROWSER: print(f" [bold green]INFO:[/] Cleaned existing build directory {build_dir!s}.") elif is_populated and not _RUNNING_IN_BROWSER: - print(" [bold yellow]WARNING:[/] Build directory is not empty. Use --clean to remove existing files.") + print( + " [bold yellow]WARNING:[/] Build directory is not empty. Run without --no-clean to remove existing files." + ) elif build_dir.exists() and not _RUNNING_IN_BROWSER: print(" [bold green]INFO:[/] Build directory does already exist and is empty. No need to create it.") else: diff --git a/tests/tests_unit/test_approval_modules.py b/tests/tests_unit/test_approval_modules.py index fe4aa779d..6b0fb2e4b 100644 --- a/tests/tests_unit/test_approval_modules.py +++ b/tests/tests_unit/test_approval_modules.py @@ -69,7 +69,7 @@ def test_deploy_module_approval( source_dir=str(init_project), build_dir=str(local_tmp_path), build_env="dev", - clean=True, + no_clean=False, ) deploy( typer_context, @@ -114,7 +114,7 @@ def test_deploy_dry_run_module_approval( source_dir=str(init_project), build_dir=str(local_tmp_path), build_env="dev", - clean=True, + no_clean=False, ) deploy( typer_context, @@ -160,7 +160,7 @@ def test_clean_module_approval( source_dir=str(local_tmp_project_path), build_dir=str(local_tmp_path), build_env="dev", - clean=True, + no_clean=False, ) clean( typer_context, diff --git a/tests/tests_unit/test_behavior.py b/tests/tests_unit/test_behavior.py index 2fca32e95..bb7104d08 100644 --- a/tests/tests_unit/test_behavior.py +++ b/tests/tests_unit/test_behavior.py @@ -44,7 +44,7 @@ def test_inject_custom_environmental_variables( source_dir=str(init_project), build_dir=str(local_tmp_path), build_env="dev", - clean=True, + no_clean=False, ) deploy( typer_context, @@ -252,7 +252,7 @@ def test_build_custom_project( source_dir=str(CUSTOM_PROJECT), build_dir=str(local_tmp_path), build_env="dev", - clean=True, + no_clean=False, ) actual_resources = {path.name for path in local_tmp_path.iterdir() if path.is_dir()} diff --git a/tests/tests_unit/test_cdf_tk/test_build_files.py b/tests/tests_unit/test_cdf_tk/test_build_files.py index d9a24d03c..6d2701ec8 100644 --- a/tests/tests_unit/test_cdf_tk/test_build_files.py +++ b/tests/tests_unit/test_cdf_tk/test_build_files.py @@ -41,7 +41,7 @@ def test_files_templates( source_dir=str(init_project), build_dir=str(local_tmp_path / "files_build"), build_env="dev", - clean=True, + no_clean=False, ) assert Path(local_tmp_path / "files_build").exists() From 670cca9eab1d9856fb325aab16e8cb2c0304cea4 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Sun, 21 Apr 2024 14:18:03 +0200 Subject: [PATCH 17/34] fix(deps): update dependency cognite-sdk to v7.37.1 (#461) * fix(deps): update dependency cognite-sdk to v7.35.0 * refactor: Updated apm_simple to 7.37 SDK * tests: updated test * build: cognite-sdk to 7.37 * build; changelog * tests: updated test data * tests: updated test data 2 --------- Co-authored-by: anders-albert --- CHANGELOG.cdf-tk.md | 1 + CHANGELOG.templates.md | 17 ++++++++++++----- .../data_models/1.Asset.container.yaml | 1 + .../data_models/3.WorkItem.container.yaml | 1 + poetry.lock | 8 +++++--- pyproject.toml | 2 +- .../cdf_apm_simple_data_model.yaml | 2 ++ .../data_models/1.Asset.container.yaml | 1 + .../data_models/3.WorkItem.container.yaml | 1 + .../describe_data/data_models/4.Asset.view.yaml | 1 + .../data_models/5.WorkItem.view.yaml | 1 + 11 files changed, 27 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.cdf-tk.md b/CHANGELOG.cdf-tk.md index ea5f71385..f0d2edaa3 100644 --- a/CHANGELOG.cdf-tk.md +++ b/CHANGELOG.cdf-tk.md @@ -26,6 +26,7 @@ Changes are grouped as follows: - The `cdf-tk build` will now clean the build directory by default before building the modules. To stop this behavior, use the `--no-clean` flag. - The `.gitignore` file you get by running `cdf-tk init` now ignores the `/build` by default. +- The dependency `cognite-sdk` must now be `>=7.37.0` to use the `cdf-tk`. ## [0.2.0a2] - 2024-04-03 diff --git a/CHANGELOG.templates.md b/CHANGELOG.templates.md index 4d82490c5..f18ca6a59 100644 --- a/CHANGELOG.templates.md +++ b/CHANGELOG.templates.md @@ -17,11 +17,18 @@ Changes are grouped as follows: ## [TBD] - TBD -- Fixed: align tag name in asset hierarchy between the 2 example transformations -- Fixed: added default root asset ID to documents for initial annotation -- Fixed: aligned use of asset external ID across contextualization functions -- Fixed: annotation logic with local time stamp for when to reprocess P&ID files -- Fixed: input to P&ID annotation based on list of synonyms for tag +### Fixed + +- Align tag name in asset hierarchy between the 2 example transformations +- Added default root asset ID to documents for initial annotation +- Aligned use of asset external ID across contextualization functions +- Annotation logic with local time stamp for when to reprocess P&ID files +- Input to P&ID annotation based on list of synonyms for tag +- Updated module `apm_simple_data_model` for `cognite-sdk>=7.37`, i.e., container properties + of type `direct` have now `list: false` explicitly set. + +### Added + - Added Transformation for WorkOrder and WorkItems to OID testdata template ## [0.2.0a2] - 2024-04-03 diff --git a/cognite_toolkit/cognite_modules/examples/cdf_apm_simple_data_model/data_models/1.Asset.container.yaml b/cognite_toolkit/cognite_modules/examples/cdf_apm_simple_data_model/data_models/1.Asset.container.yaml index 9c52c3fab..7bf0d9bc2 100644 --- a/cognite_toolkit/cognite_modules/examples/cdf_apm_simple_data_model/data_models/1.Asset.container.yaml +++ b/cognite_toolkit/cognite_modules/examples/cdf_apm_simple_data_model/data_models/1.Asset.container.yaml @@ -82,6 +82,7 @@ properties: nullable: true type: type: direct + list: false sourceDb: autoIncrement: false defaultValue: null diff --git a/cognite_toolkit/cognite_modules/examples/cdf_apm_simple_data_model/data_models/3.WorkItem.container.yaml b/cognite_toolkit/cognite_modules/examples/cdf_apm_simple_data_model/data_models/3.WorkItem.container.yaml index 87610f0d9..7b9ea2aff 100644 --- a/cognite_toolkit/cognite_modules/examples/cdf_apm_simple_data_model/data_models/3.WorkItem.container.yaml +++ b/cognite_toolkit/cognite_modules/examples/cdf_apm_simple_data_model/data_models/3.WorkItem.container.yaml @@ -88,5 +88,6 @@ properties: type: container: null type: direct + list: false space: '{{space}}' usedFor: node diff --git a/poetry.lock b/poetry.lock index 21bf0d54d..8dfa039db 100644 --- a/poetry.lock +++ b/poetry.lock @@ -326,13 +326,13 @@ python-json-logger = ">=2.0.7,<3.0.0" [[package]] name = "cognite-sdk" -version = "7.34.0" +version = "7.37.3" description = "Cognite Python SDK" optional = false python-versions = "<4.0,>=3.8" files = [ - {file = "cognite_sdk-7.34.0-py3-none-any.whl", hash = "sha256:f0387ff0558c06d4cdfde0a88409904932887799ab71d55eb479779aaba0b2ae"}, - {file = "cognite_sdk-7.34.0.tar.gz", hash = "sha256:1eeb629df867e112c4b081255084250675c0f911ba7d69153ba6d61b06371e47"}, + {file = "cognite_sdk-7.37.3-py3-none-any.whl", hash = "sha256:1a118eac621ba6d31ab4f8e5d4dc8710fff23c94a257accd1a160010ebfd239c"}, + {file = "cognite_sdk-7.37.3.tar.gz", hash = "sha256:da20f6aec63fa5bdd8bc2f71c0e4c499e6668a080f5761f202fb62b155dcd808"}, ] [package.dependencies] @@ -1038,6 +1038,7 @@ optional = false python-versions = ">=3.9" files = [ {file = "pandas-2.2.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:90c6fca2acf139569e74e8781709dccb6fe25940488755716d1d354d6bc58bce"}, + {file = "pandas-2.2.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c7adfc142dac335d8c1e0dcbd37eb8617eac386596eb9e1a1b77791cf2498238"}, {file = "pandas-2.2.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4abfe0be0d7221be4f12552995e58723c7422c80a659da13ca382697de830c08"}, {file = "pandas-2.2.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8635c16bf3d99040fdf3ca3db669a7250ddf49c55dc4aa8fe0ae0fa8d6dcc1f0"}, {file = "pandas-2.2.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:40ae1dffb3967a52203105a077415a86044a2bea011b5f321c6aa64b379a3f51"}, @@ -1058,6 +1059,7 @@ files = [ {file = "pandas-2.2.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:43498c0bdb43d55cb162cdc8c06fac328ccb5d2eabe3cadeb3529ae6f0517c32"}, {file = "pandas-2.2.2-cp312-cp312-win_amd64.whl", hash = "sha256:d187d355ecec3629624fccb01d104da7d7f391db0311145817525281e2804d23"}, {file = "pandas-2.2.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:0ca6377b8fca51815f382bd0b697a0814c8bda55115678cbc94c30aacbb6eff2"}, + {file = "pandas-2.2.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:9057e6aa78a584bc93a13f0a9bf7e753a5e9770a30b4d758b8d5f2a62a9433cd"}, {file = "pandas-2.2.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:001910ad31abc7bf06f49dcc903755d2f7f3a9186c0c040b827e522e9cef0863"}, {file = "pandas-2.2.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:66b479b0bd07204e37583c191535505410daa8df638fd8e75ae1b383851fe921"}, {file = "pandas-2.2.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:a77e9d1c386196879aa5eb712e77461aaee433e54c68cf253053a73b7e49c33a"}, diff --git a/pyproject.toml b/pyproject.toml index 45b0dcbb7..1bd55c986 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,7 +11,7 @@ include = ["cognite_toolkit/*", "cognite_toolkit/**/*"] [tool.poetry.dependencies] python = "^3.9" python-dotenv = "^1.0.0" -cognite-sdk = {version = "^7.26", extras = ["pandas"]} +cognite-sdk = {version = "^7.37", extras = ["pandas"]} cognite-extractor-utils = ">=7" pandas = ">=1.5.3, <3.0" pyyaml = "^6.0.1" diff --git a/tests/tests_unit/test_approval_modules_snapshots/cdf_apm_simple_data_model.yaml b/tests/tests_unit/test_approval_modules_snapshots/cdf_apm_simple_data_model.yaml index 62546b897..ccd79adfb 100644 --- a/tests/tests_unit/test_approval_modules_snapshots/cdf_apm_simple_data_model.yaml +++ b/tests/tests_unit/test_approval_modules_snapshots/cdf_apm_simple_data_model.yaml @@ -66,6 +66,7 @@ Container: name: parent nullable: true type: + list: false type: direct sourceDb: autoIncrement: false @@ -164,6 +165,7 @@ Container: name: workOrder nullable: true type: + list: false type: direct space: sp_apm_simple usedFor: node diff --git a/tests/tests_unit/test_cdf_tk/describe_data/data_models/1.Asset.container.yaml b/tests/tests_unit/test_cdf_tk/describe_data/data_models/1.Asset.container.yaml index c2efc0784..af36a0a85 100644 --- a/tests/tests_unit/test_cdf_tk/describe_data/data_models/1.Asset.container.yaml +++ b/tests/tests_unit/test_cdf_tk/describe_data/data_models/1.Asset.container.yaml @@ -85,6 +85,7 @@ properties: nullable: true type: type: direct + list: false sourceDb: autoIncrement: false defaultValue: null diff --git a/tests/tests_unit/test_cdf_tk/describe_data/data_models/3.WorkItem.container.yaml b/tests/tests_unit/test_cdf_tk/describe_data/data_models/3.WorkItem.container.yaml index 3aff80f4e..699bb8e01 100644 --- a/tests/tests_unit/test_cdf_tk/describe_data/data_models/3.WorkItem.container.yaml +++ b/tests/tests_unit/test_cdf_tk/describe_data/data_models/3.WorkItem.container.yaml @@ -91,5 +91,6 @@ properties: type: container: null type: direct + list: false space: test usedFor: node diff --git a/tests/tests_unit/test_cdf_tk/describe_data/data_models/4.Asset.view.yaml b/tests/tests_unit/test_cdf_tk/describe_data/data_models/4.Asset.view.yaml index 7403b3ce7..9488028e7 100644 --- a/tests/tests_unit/test_cdf_tk/describe_data/data_models/4.Asset.view.yaml +++ b/tests/tests_unit/test_cdf_tk/describe_data/data_models/4.Asset.view.yaml @@ -126,6 +126,7 @@ properties: name: parent type: type: direct + list: false nullable: true autoIncrement: false sourceDb: diff --git a/tests/tests_unit/test_cdf_tk/describe_data/data_models/5.WorkItem.view.yaml b/tests/tests_unit/test_cdf_tk/describe_data/data_models/5.WorkItem.view.yaml index f3e183a96..5b34fcec7 100644 --- a/tests/tests_unit/test_cdf_tk/describe_data/data_models/5.WorkItem.view.yaml +++ b/tests/tests_unit/test_cdf_tk/describe_data/data_models/5.WorkItem.view.yaml @@ -132,5 +132,6 @@ properties: type: container: null type: direct + list: false nullable: true autoIncrement: false \ No newline at end of file From 3fef43d5496c1fc42c33240c47bb1bdfbeba1c0a Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Sun, 21 Apr 2024 14:24:36 +0200 Subject: [PATCH 18/34] fix(deps): update dependency cognite-extractor-utils to v7.1.2 (#462) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- poetry.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/poetry.lock b/poetry.lock index 8dfa039db..963a12554 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1954,4 +1954,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p [metadata] lock-version = "2.0" python-versions = "^3.9" -content-hash = "ad7912f3f19a32c1be9ea8452e3f3d5d121d5339858edaa0ded45cce2c45254f" +content-hash = "7dea395f55601ab7207a340eb53b6a984f5c1aa1de54f71a4686483c41ea17e2" From f5f393e5fb3e962284b1dc07cce38fa6bf85298b Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 22 Apr 2024 14:12:32 +0200 Subject: [PATCH 19/34] chore(deps): lock file maintenance (#464) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- poetry.lock | 42 +++++++++++++++++++++--------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/poetry.lock b/poetry.lock index 963a12554..254541f67 100644 --- a/poetry.lock +++ b/poetry.lock @@ -73,18 +73,18 @@ typing-extensions = ">=4.0.1" [[package]] name = "backports-tarfile" -version = "1.0.0" +version = "1.1.0" description = "Backport of CPython tarfile module" optional = false python-versions = ">=3.8" files = [ - {file = "backports.tarfile-1.0.0-py3-none-any.whl", hash = "sha256:bcd36290d9684beb524d3fe74f4a2db056824c47746583f090b8e55daf0776e4"}, - {file = "backports.tarfile-1.0.0.tar.gz", hash = "sha256:2688f159c21afd56a07b75f01306f9f52c79aebcc5f4a117fb8fbb4445352c75"}, + {file = "backports.tarfile-1.1.0-py3-none-any.whl", hash = "sha256:b2f4df351db942d094db94588bbf2c6938697a5f190f44c934acc697da56008b"}, + {file = "backports_tarfile-1.1.0.tar.gz", hash = "sha256:91d59138ea401ee2a95e8b839c1e2f51f3e9ca76bdba8b6a29f8d773564686a8"}, ] [package.extras] docs = ["furo", "jaraco.packaging (>=9.3)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] -testing = ["pytest (>=6,!=8.1.1)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)"] +testing = ["jaraco.test", "pytest (>=6,!=8.1.1)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)"] [[package]] name = "certifi" @@ -534,13 +534,13 @@ files = [ [[package]] name = "exceptiongroup" -version = "1.2.0" +version = "1.2.1" description = "Backport of PEP 654 (exception groups)" optional = false python-versions = ">=3.7" files = [ - {file = "exceptiongroup-1.2.0-py3-none-any.whl", hash = "sha256:4bfd3996ac73b41e9b9628b04e079f193850720ea5945fc96a08633c66912f14"}, - {file = "exceptiongroup-1.2.0.tar.gz", hash = "sha256:91f5c769735f051a4290d52edd0858999b57e5876e9f85937691bd4c9fa3ed68"}, + {file = "exceptiongroup-1.2.1-py3-none-any.whl", hash = "sha256:5258b9ed329c5bbdd31a309f53cbfb0b155341807f6ff7606a1e801a891b29ad"}, + {file = "exceptiongroup-1.2.1.tar.gz", hash = "sha256:a4785e48b045528f5bfe627b6ad554ff32def154f42372786903b7abcfe1aa16"}, ] [package.extras] @@ -605,13 +605,13 @@ files = [ [[package]] name = "identify" -version = "2.5.35" +version = "2.5.36" description = "File identification library for Python" optional = false python-versions = ">=3.8" files = [ - {file = "identify-2.5.35-py2.py3-none-any.whl", hash = "sha256:c4de0081837b211594f8e877a6b4fad7ca32bbfc1a9307fdd61c28bfe923f13e"}, - {file = "identify-2.5.35.tar.gz", hash = "sha256:10a7ca245cfcd756a554a7288159f72ff105ad233c7c4b9c6f0f4d108f5f6791"}, + {file = "identify-2.5.36-py2.py3-none-any.whl", hash = "sha256:37d93f380f4de590500d9dba7db359d0d3da95ffe7f9de1753faa159e71e7dfa"}, + {file = "identify-2.5.36.tar.gz", hash = "sha256:e5e00f54165f9047fbebeb4a560f9acfb8af4c88232be60a488e9b68d122745d"}, ] [package.extras] @@ -710,13 +710,13 @@ testing = ["portend", "pytest (>=6,!=8.1.1)", "pytest-checkdocs (>=2.4)", "pytes [[package]] name = "jaraco-functools" -version = "4.0.0" +version = "4.0.1" description = "Functools like those found in stdlib" optional = false python-versions = ">=3.8" files = [ - {file = "jaraco.functools-4.0.0-py3-none-any.whl", hash = "sha256:daf276ddf234bea897ef14f43c4e1bf9eefeac7b7a82a4dd69228ac20acff68d"}, - {file = "jaraco.functools-4.0.0.tar.gz", hash = "sha256:c279cb24c93d694ef7270f970d499cab4d3813f4e08273f95398651a634f0925"}, + {file = "jaraco.functools-4.0.1-py3-none-any.whl", hash = "sha256:3b24ccb921d6b593bdceb56ce14799204f473976e2a9d4b15b04d0f2c2326664"}, + {file = "jaraco_functools-4.0.1.tar.gz", hash = "sha256:d33fa765374c0611b52f8b3a795f8900869aa88c84769d4d1746cd68fb28c3e8"}, ] [package.dependencies] @@ -724,7 +724,7 @@ more-itertools = "*" [package.extras] docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (<7.2.5)", "sphinx (>=3.5)", "sphinx-lint"] -testing = ["jaraco.classes", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-mypy (>=0.9.1)", "pytest-ruff"] +testing = ["jaraco.classes", "pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-mypy", "pytest-ruff (>=0.2.1)"] [[package]] name = "jeepney" @@ -1148,13 +1148,13 @@ test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.4.3)", "pytest- [[package]] name = "pluggy" -version = "1.4.0" +version = "1.5.0" description = "plugin and hook calling mechanisms for python" optional = false python-versions = ">=3.8" files = [ - {file = "pluggy-1.4.0-py3-none-any.whl", hash = "sha256:7db9f7b503d67d1c5b95f59773ebb58a8c1c288129a88665838012cfb07b8981"}, - {file = "pluggy-1.4.0.tar.gz", hash = "sha256:8c85c2876142a764e5b7548e7d9a0e0ddb46f5185161049a79b7e974454223be"}, + {file = "pluggy-1.5.0-py3-none-any.whl", hash = "sha256:44e1ad92c8ca002de6377e165f3e0f1be63266ab4d554740532335b9d75ea669"}, + {file = "pluggy-1.5.0.tar.gz", hash = "sha256:2cffa88e94fdc978c4c574f15f9e59b7f4201d439195c3715ca9e2486f1d0cf1"}, ] [package.extras] @@ -1918,13 +1918,13 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "virtualenv" -version = "20.25.1" +version = "20.25.3" description = "Virtual Python Environment builder" optional = false python-versions = ">=3.7" files = [ - {file = "virtualenv-20.25.1-py3-none-any.whl", hash = "sha256:961c026ac520bac5f69acb8ea063e8a4f071bcc9457b9c1f28f6b085c511583a"}, - {file = "virtualenv-20.25.1.tar.gz", hash = "sha256:e08e13ecdca7a0bd53798f356d5831434afa5b07b93f0abdf0797b7a06ffe197"}, + {file = "virtualenv-20.25.3-py3-none-any.whl", hash = "sha256:8aac4332f2ea6ef519c648d0bc48a5b1d324994753519919bddbb1aff25a104e"}, + {file = "virtualenv-20.25.3.tar.gz", hash = "sha256:7bb554bbdfeaacc3349fa614ea5bff6ac300fc7c335e9facf3a3bcfc703f45be"}, ] [package.dependencies] @@ -1933,7 +1933,7 @@ filelock = ">=3.12.2,<4" platformdirs = ">=3.9.1,<5" [package.extras] -docs = ["furo (>=2023.7.26)", "proselint (>=0.13)", "sphinx (>=7.1.2)", "sphinx-argparse (>=0.4)", "sphinxcontrib-towncrier (>=0.2.1a0)", "towncrier (>=23.6)"] +docs = ["furo (>=2023.7.26)", "proselint (>=0.13)", "sphinx (>=7.1.2,!=7.3)", "sphinx-argparse (>=0.4)", "sphinxcontrib-towncrier (>=0.2.1a0)", "towncrier (>=23.6)"] test = ["covdefaults (>=2.3)", "coverage (>=7.2.7)", "coverage-enable-subprocess (>=1)", "flaky (>=3.7)", "packaging (>=23.1)", "pytest (>=7.4)", "pytest-env (>=0.8.2)", "pytest-freezer (>=0.4.8)", "pytest-mock (>=3.11.1)", "pytest-randomly (>=3.12)", "pytest-timeout (>=2.1)", "setuptools (>=68)", "time-machine (>=2.10)"] [[package]] From c73b84f9cead2877ced0233fbddfa0c522b6d88b Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 22 Apr 2024 14:35:45 +0200 Subject: [PATCH 20/34] chore(deps): update pre-commit hook charliermarsh/ruff-pre-commit to v0.4.0 (#463) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 69cde3a97..62bbfbf06 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -15,7 +15,7 @@ repos: - id: ruff-format args: - --line-length=120 - rev: v0.3.7 + rev: v0.4.0 - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.6.0 From 87559a667867c383800cea24c00506059a13e0d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A5l=20R=C3=B8nning?= Date: Mon, 22 Apr 2024 14:55:32 +0200 Subject: [PATCH 21/34] Workflow loader (#453) * Started added loader for Workflow * Savepoint * Savepoint * Deployment works * mypy caught something * Savepoint workflow tests * Ready for review * Update cognite_toolkit/_cdf_tk/load/_resource_loaders.py Co-authored-by: Anders Albert <60234212+doctrino@users.noreply.github.com> * Update cognite_toolkit/_cdf_tk/load/_resource_loaders.py Co-authored-by: Anders Albert <60234212+doctrino@users.noreply.github.com> * Update cognite_toolkit/_cdf_tk/load/_resource_loaders.py Co-authored-by: Anders Albert <60234212+doctrino@users.noreply.github.com> * Update tests/tests_unit/test_approval_modules_snapshots/cdf_oid_example_data.yaml Co-authored-by: Anders Albert <60234212+doctrino@users.noreply.github.com> * build clean is obsolete * Restored my_example_module * mypy * regen'ed tests * wrangling dataframe value --------- Co-authored-by: Anders Albert <60234212+doctrino@users.noreply.github.com> --- .vscode/launch.json | 1 - CHANGELOG.cdf-tk.md | 4 + cognite_toolkit/_cdf_tk/load/__init__.py | 3 + cognite_toolkit/_cdf_tk/load/_base_loaders.py | 6 +- .../_cdf_tk/load/_resource_loaders.py | 122 ++++++++++++++++++ .../_cdf_tk/templates/_templates.py | 11 ++ cognite_toolkit/_cdf_tk/utils.py | 3 +- .../auth/admin.readonly.group.yaml | 5 + .../auth/admin.readwrite.group.yaml | 6 + .../processing_pipeline/default.config.yaml | 1 + .../workflows/my.Workflow.yaml | 2 + .../workflows/my.WorkflowVersion.yaml | 30 +++++ tests/tests_unit/approval_client/client.py | 41 ++++++ tests/tests_unit/approval_client/config.py | 43 ++++++ .../cdf_auth_readwrite_all.yaml | 11 ++ .../processing_pipeline.yaml | 28 ++++ .../processing_pipeline.yaml | 1 + 17 files changed, 315 insertions(+), 3 deletions(-) create mode 100644 cognite_toolkit/cognite_modules/experimental/processing_pipeline/default.config.yaml create mode 100644 cognite_toolkit/cognite_modules/experimental/processing_pipeline/workflows/my.Workflow.yaml create mode 100644 cognite_toolkit/cognite_modules/experimental/processing_pipeline/workflows/my.WorkflowVersion.yaml create mode 100644 tests/tests_unit/test_approval_modules_snapshots/processing_pipeline.yaml create mode 100644 tests/tests_unit/test_approval_modules_snapshots_clean/processing_pipeline.yaml diff --git a/.vscode/launch.json b/.vscode/launch.json index cdc1b8c2e..3eb41d24a 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -14,7 +14,6 @@ "--override-env", "build", "--build-dir=build", - "--clean", "--env=local", "./cognite_toolkit/" ], diff --git a/CHANGELOG.cdf-tk.md b/CHANGELOG.cdf-tk.md index f0d2edaa3..e58e2e00b 100644 --- a/CHANGELOG.cdf-tk.md +++ b/CHANGELOG.cdf-tk.md @@ -17,6 +17,10 @@ Changes are grouped as follows: ## TBD +### Added + +- Support for the Workflow and WorkflowVersion resource type + ### Fixed - Functions that are deployed with schedules no longer uses a short-lived session (before: failed after ~an hour). diff --git a/cognite_toolkit/_cdf_tk/load/__init__.py b/cognite_toolkit/_cdf_tk/load/__init__.py index 38f12d47f..83cd0e217 100644 --- a/cognite_toolkit/_cdf_tk/load/__init__.py +++ b/cognite_toolkit/_cdf_tk/load/__init__.py @@ -68,6 +68,7 @@ "extraction_pipelines", "functions", "raw", + "workflows", ] __all__ = [ @@ -98,4 +99,6 @@ "DeployResult", "DeployResults", "ResourceTypes", + "WorkflowLoader", + "WorkflowVersionLoader", ] diff --git a/cognite_toolkit/_cdf_tk/load/_base_loaders.py b/cognite_toolkit/_cdf_tk/load/_base_loaders.py index 5f7540e2b..d3bcb3a48 100644 --- a/cognite_toolkit/_cdf_tk/load/_base_loaders.py +++ b/cognite_toolkit/_cdf_tk/load/_base_loaders.py @@ -10,6 +10,7 @@ from typing import Any, Generic, TypeVar, Union from cognite.client import CogniteClient +from cognite.client.data_classes import WorkflowVersionId from cognite.client.data_classes._base import ( T_CogniteResourceList, T_WritableCogniteResource, @@ -35,7 +36,10 @@ UploadDeployResult, ) -T_ID = TypeVar("T_ID", bound=Union[str, int, DataModelingId, InstanceId, VersionedDataModelingId, RawDatabaseTable]) +T_ID = TypeVar( + "T_ID", + bound=Union[str, int, DataModelingId, InstanceId, VersionedDataModelingId, RawDatabaseTable, WorkflowVersionId], +) T_WritableCogniteResourceList = TypeVar("T_WritableCogniteResourceList", bound=WriteableCogniteResourceList) diff --git a/cognite_toolkit/_cdf_tk/load/_resource_loaders.py b/cognite_toolkit/_cdf_tk/load/_resource_loaders.py index 1ac759a62..65081dc8b 100644 --- a/cognite_toolkit/_cdf_tk/load/_resource_loaders.py +++ b/cognite_toolkit/_cdf_tk/load/_resource_loaders.py @@ -61,6 +61,15 @@ TransformationScheduleWriteList, TransformationWrite, TransformationWriteList, + Workflow, + WorkflowList, + WorkflowUpsert, + WorkflowUpsertList, + WorkflowVersion, + WorkflowVersionId, + WorkflowVersionList, + WorkflowVersionUpsert, + WorkflowVersionUpsertList, capabilities, filters, ) @@ -77,6 +86,7 @@ SessionsAcl, TimeSeriesAcl, TransformationsAcl, + WorkflowOrchestrationAcl, ) from cognite.client.data_classes.data_modeling import ( Container, @@ -1899,3 +1909,115 @@ def count(self, ids: SequenceNotStr[NodeId]) -> int: def drop_data(self, ids: SequenceNotStr[NodeId]) -> int: # Nodes will be deleted in .delete call. return 0 + + +@final +class WorkflowLoader(ResourceLoader[str, WorkflowUpsert, Workflow, WorkflowUpsertList, WorkflowList]): + api_name = "workflows" + folder_name = "workflows" + filename_pattern = r"\.Workflow$" + resource_cls = Workflow + resource_write_cls = WorkflowUpsert + list_cls = WorkflowList + list_write_cls = WorkflowUpsertList + + @classmethod + def get_required_capability(cls, items: WorkflowUpsertList) -> Capability: + return WorkflowOrchestrationAcl( + [WorkflowOrchestrationAcl.Action.Read, WorkflowOrchestrationAcl.Action.Write], + WorkflowOrchestrationAcl.Scope.All(), + ) + + @classmethod + def get_id(cls, item: Workflow | WorkflowUpsert) -> str: + if item.external_id is None: + raise ValueError("Workflow must have external_id set.") + return item.external_id + + def load_resource(self, filepath: Path, ToolGlobals: CDFToolConfig, skip_validation: bool) -> WorkflowUpsertList: + resource = load_yaml_inject_variables(filepath, {}) + + workflows = [resource] if isinstance(resource, dict) else resource + return WorkflowUpsertList.load(workflows) + + def retrieve(self, ids: SequenceNotStr[str]) -> WorkflowList: + workflows = [] + for ext_id in ids: + workflow = self.client.workflows.retrieve(external_id=ext_id) + if workflow: + workflows.append(workflow) + return WorkflowList(workflows) + + def _upsert(self, items: WorkflowUpsert | WorkflowUpsertList) -> WorkflowList: + upserts = [items] if isinstance(items, WorkflowUpsert) else items + return WorkflowList([self.client.workflows.upsert(upsert) for upsert in upserts]) + + def create(self, items: WorkflowUpsert | WorkflowUpsertList) -> WorkflowList: + return self._upsert(items) + + def update(self, items: WorkflowUpsertList) -> WorkflowList: + return self._upsert(items) + + +@final +class WorkflowVersionLoader( + ResourceLoader[ + WorkflowVersionId, WorkflowVersionUpsert, WorkflowVersion, WorkflowVersionUpsertList, WorkflowVersionList + ] +): + api_name = "workflows.versions" + folder_name = "workflows" + filename_pattern = r"^.*\.?(WorkflowVersion)$" + resource_cls = WorkflowVersion + resource_write_cls = WorkflowVersionUpsert + list_cls = WorkflowVersionList + list_write_cls = WorkflowVersionUpsertList + dependencies = frozenset({WorkflowLoader}) + + @classmethod + def get_required_capability(cls, items: WorkflowVersionUpsertList) -> Capability: + return WorkflowOrchestrationAcl( + [WorkflowOrchestrationAcl.Action.Read, WorkflowOrchestrationAcl.Action.Write], + WorkflowOrchestrationAcl.Scope.All(), + ) + + @classmethod + def get_id(cls, item: WorkflowVersion | WorkflowVersionUpsert) -> WorkflowVersionId: + return item.as_id() + + def load_resource( + self, filepath: Path, ToolGlobals: CDFToolConfig, skip_validation: bool + ) -> WorkflowVersionUpsertList: + resource = load_yaml_inject_variables(filepath, {}) + + workflowversions = [resource] if isinstance(resource, dict) else resource + return WorkflowVersionUpsertList.load(workflowversions) + + def retrieve(self, ids: SequenceNotStr[WorkflowVersionId]) -> WorkflowVersionList: + return self.client.workflows.versions.list(list(ids)) + + def _upsert(self, items: WorkflowVersionUpsertList) -> WorkflowVersionList: + return WorkflowVersionList([self.client.workflows.versions.upsert(item) for item in items]) + + def create(self, items: WorkflowVersionUpsertList) -> WorkflowVersionList: + upserted = [] + for item in items: + upserted.append(self.client.workflows.versions.upsert(item)) + return WorkflowVersionList(upserted) + + def update(self, items: WorkflowVersionUpsertList) -> WorkflowVersionList: + updated = [] + for item in items: + updated.append(self.client.workflows.versions.upsert(item)) + return WorkflowVersionList(updated) + + def delete(self, ids: SequenceNotStr[WorkflowVersionId]) -> int: + successes = 0 + for id in ids: + try: + self.client.workflows.versions.delete(id) + except CogniteNotFoundError: + print(f" [bold yellow]WARNING:[/] WorkflowVersion {id} does not exist, skipping delete.") + else: + successes += 1 + return successes diff --git a/cognite_toolkit/_cdf_tk/templates/_templates.py b/cognite_toolkit/_cdf_tk/templates/_templates.py index 2e8916063..5801d1f2c 100644 --- a/cognite_toolkit/_cdf_tk/templates/_templates.py +++ b/cognite_toolkit/_cdf_tk/templates/_templates.py @@ -158,6 +158,17 @@ def check_yaml_semantics(parsed: dict | list, filepath_src: Path, filepath_build else: print(f" [bold red]:[/] Raw file {filepath_src} has invalid dataformat.") exit(1) + elif resource_type == "workflows": + if isinstance(parsed, dict): + if "version" in filepath_src.stem.lower(): + ext_id = parsed.get("workflowExternalId") + ext_id_type = "workflowExternalId" + else: + ext_id = parsed.get("externalId") or parsed.get("external_id") + ext_id_type = "externalId" + else: + print(f" [bold red]:[/] Multiple Workflows in one file {filepath_src} is not supported .") + exit(1) else: if isinstance(parsed, list): print(f" [bold red]:[/] Multiple {resource_type} in one file {filepath_src} is not supported .") diff --git a/cognite_toolkit/_cdf_tk/utils.py b/cognite_toolkit/_cdf_tk/utils.py index 829b25076..8d6dd80e9 100644 --- a/cognite_toolkit/_cdf_tk/utils.py +++ b/cognite_toolkit/_cdf_tk/utils.py @@ -666,7 +666,8 @@ def verify_client( def verify_capabilities(self, capability: Capability | Sequence[Capability]) -> CogniteClient: missing_capabilities = self.client.iam.verify_capabilities(capability) if len(missing_capabilities) > 0: - raise CogniteAuthError(f"Missing capabilities: {missing_capabilities}") + print(f"Missing necessary CDF access capabilities: {missing_capabilities}") + return self.client def verify_dataset(self, data_set_external_id: str, skip_validation: bool = False) -> int: diff --git a/cognite_toolkit/cognite_modules/common/cdf_auth_readwrite_all/auth/admin.readonly.group.yaml b/cognite_toolkit/cognite_modules/common/cdf_auth_readwrite_all/auth/admin.readonly.group.yaml index d07f70332..383c571b4 100644 --- a/cognite_toolkit/cognite_modules/common/cdf_auth_readwrite_all/auth/admin.readonly.group.yaml +++ b/cognite_toolkit/cognite_modules/common/cdf_auth_readwrite_all/auth/admin.readonly.group.yaml @@ -122,3 +122,8 @@ capabilities: - READ scope: all: {} + - workflowOrchestrationAcl: + actions: + - READ + scope: + all: {} diff --git a/cognite_toolkit/cognite_modules/common/cdf_auth_readwrite_all/auth/admin.readwrite.group.yaml b/cognite_toolkit/cognite_modules/common/cdf_auth_readwrite_all/auth/admin.readwrite.group.yaml index 8f232ad72..29ed62272 100644 --- a/cognite_toolkit/cognite_modules/common/cdf_auth_readwrite_all/auth/admin.readwrite.group.yaml +++ b/cognite_toolkit/cognite_modules/common/cdf_auth_readwrite_all/auth/admin.readwrite.group.yaml @@ -94,6 +94,12 @@ capabilities: scope: all: {} - transformationsAcl: + actions: + - READ + - WRITE + scope: + all: {} + - workflowOrchestrationAcl: actions: - READ - WRITE diff --git a/cognite_toolkit/cognite_modules/experimental/processing_pipeline/default.config.yaml b/cognite_toolkit/cognite_modules/experimental/processing_pipeline/default.config.yaml new file mode 100644 index 000000000..1df4b28f4 --- /dev/null +++ b/cognite_toolkit/cognite_modules/experimental/processing_pipeline/default.config.yaml @@ -0,0 +1 @@ +workflow_external_id: \ No newline at end of file diff --git a/cognite_toolkit/cognite_modules/experimental/processing_pipeline/workflows/my.Workflow.yaml b/cognite_toolkit/cognite_modules/experimental/processing_pipeline/workflows/my.Workflow.yaml new file mode 100644 index 000000000..796954410 --- /dev/null +++ b/cognite_toolkit/cognite_modules/experimental/processing_pipeline/workflows/my.Workflow.yaml @@ -0,0 +1,2 @@ +externalId: wf_{{ workflow_external_id }} +decription: A workflow for processing data \ No newline at end of file diff --git a/cognite_toolkit/cognite_modules/experimental/processing_pipeline/workflows/my.WorkflowVersion.yaml b/cognite_toolkit/cognite_modules/experimental/processing_pipeline/workflows/my.WorkflowVersion.yaml new file mode 100644 index 000000000..d1d956fca --- /dev/null +++ b/cognite_toolkit/cognite_modules/experimental/processing_pipeline/workflows/my.WorkflowVersion.yaml @@ -0,0 +1,30 @@ +workflowExternalId: wf_{{ workflow_external_id }} +version: '1' +workflowDefinition: + description: 'Execute tasks in sequence' + tasks: + - externalId: '{{ workflow_external_id }}_function_task' + type: 'function' + parameters: + function: + externalId: 'fn_{{ workflow_external_id }}_function' + data: {} + isAsyncComplete: false + name: task_1 + decription: Task Ipsum lorem dolor sit amet + retries: 3 + timeout: 3600 + onFailure: 'abortWorkflow' + - externalId: '{{ workflow_external_id }}_transformation_task' + type: 'transformation' + parameters: + transformation: + externalId: 'tr_{{ workflow_external_id }}_transformation' + concurrencyPolicy: fail + name: task_2 + decription: Task Ipsum lorem dolor sit amet + retries: 3 + timeout: 3600 + onFailure: 'skipTask' + dependsOn: + - externalId: 'fn_{{ workflow_external_id }}_function' diff --git a/tests/tests_unit/approval_client/client.py b/tests/tests_unit/approval_client/client.py index 67b6d8613..d89b604ee 100644 --- a/tests/tests_unit/approval_client/client.py +++ b/tests/tests_unit/approval_client/client.py @@ -26,6 +26,8 @@ FunctionWrite, Group, GroupList, + Workflow, + WorkflowVersion, capabilities, ) from cognite.client.data_classes._base import CogniteResource, T_CogniteResource @@ -120,6 +122,8 @@ def __init__(self, mock_client: CogniteClientMock): "retrieve": self._create_retrieve_method, "inspect": self._create_inspect_method, "post": self._create_post_method, + "upsert": self._create_retrieve_method, + "update": self._create_post_method, }[method_type] method_dict = { "create": self._create_methods, @@ -127,6 +131,8 @@ def __init__(self, mock_client: CogniteClientMock): "retrieve": self._retrieve_methods, "inspect": self._inspect_methods, "post": self._post_methods, + "upsert": self._retrieve_methods, + "update": self._post_methods, }[method_type] for mock_method in methods: if not hasattr(mock_api, mock_method.api_class_method): @@ -313,6 +319,40 @@ def create(*args, **kwargs) -> Any: cognite_client=client, ) + def upsert(*args, **kwargs) -> Any: + upserted = [] + for value in itertools.chain(args, kwargs.values()): + if isinstance(value, write_resource_cls): + upserted.append(value) + elif isinstance(value, Sequence) and all(isinstance(v, write_resource_cls) for v in value): + upserted.extend(value) + created_resources[resource_cls.__name__].extend(upserted) + + if resource_cls is Workflow: + return resource_cls.load( + {"lastUpdatedTime": 0, "createdTime": 0, **upserted[0].dump(camel_case=True)}, cognite_client=client + ) + + if resource_cls is WorkflowVersion: + resource = {"lastUpdatedTime": 0, "createdTime": 0, **upserted[0].dump(camel_case=True)} + + resource.get("workflowDefinition")["hash"] = "123" + + return resource_cls.load(resource, cognite_client=client) + + return resource_list_cls.load( + [ + { + "isGlobal": False, + "lastUpdatedTime": 0, + "createdTime": 0, + **c.dump(camel_case=True), + } + for c in upserted + ], + cognite_client=client, + ) + def insert_dataframe(*args, **kwargs) -> None: args = list(args) kwargs = dict(kwargs) @@ -441,6 +481,7 @@ def create_function_schedule_api(**kwargs) -> FunctionSchedule: create, insert_dataframe, upload, + upsert, create_instances, create_extraction_pipeline_config, upload_bytes_files_api, diff --git a/tests/tests_unit/approval_client/config.py b/tests/tests_unit/approval_client/config.py index 7ed7f04bf..f385ecbf3 100644 --- a/tests/tests_unit/approval_client/config.py +++ b/tests/tests_unit/approval_client/config.py @@ -54,7 +54,20 @@ TransformationScheduleWriteList, TransformationWrite, TransformationWriteList, + Workflow, + WorkflowList, + WorkflowUpsert, + WorkflowUpsertList, + WorkflowVersion, + WorkflowVersionList, + WorkflowVersionUpsert, + WorkflowVersionUpsertList, ) + +# WorkflowVersion, +# WorkflowVersionList, +# WorkflowVersionUpsert, +# WorkflowVersionUpsertList, from cognite.client.data_classes.data_modeling import ( Container, ContainerApply, @@ -377,4 +390,34 @@ ], }, ), + APIResource( + api_name="workflows", + resource_cls=Workflow, + list_cls=WorkflowList, + _write_cls=WorkflowUpsert, + _write_list_cls=WorkflowUpsertList, + methods={ + "create": [Method(api_class_method="upsert", mock_name="upsert")], + # "update": [Method(api_class_method="upsert", mock_name="upsert")], + # "delete": [Method(api_class_method="delete", mock_name="delete_id_external_id")], + "retrieve": [ + Method(api_class_method="retrieve", mock_name="return_value"), + ], + }, + ), + APIResource( + api_name="workflows.versions", + resource_cls=WorkflowVersion, + list_cls=WorkflowVersionList, + _write_cls=WorkflowVersionUpsert, + _write_list_cls=WorkflowVersionUpsertList, + methods={ + "create": [Method(api_class_method="upsert", mock_name="upsert")], + # "update": [Method(api_class_method="upsert", mock_name="upsert")], + # "delete": [Method(api_class_method="delete", mock_name="delete")], + "retrieve": [ + Method(api_class_method="retrieve", mock_name="return_value"), + ], + }, + ), ] diff --git a/tests/tests_unit/test_approval_modules_snapshots/cdf_auth_readwrite_all.yaml b/tests/tests_unit/test_approval_modules_snapshots/cdf_auth_readwrite_all.yaml index b81051a80..237f7d25d 100644 --- a/tests/tests_unit/test_approval_modules_snapshots/cdf_auth_readwrite_all.yaml +++ b/tests/tests_unit/test_approval_modules_snapshots/cdf_auth_readwrite_all.yaml @@ -96,6 +96,12 @@ Group: - WRITE scope: all: {} + - workflowOrchestrationAcl: + actions: + - READ + - WRITE + scope: + all: {} metadata: origin: cdf-project-templates name: gp_admin_read_write @@ -220,6 +226,11 @@ Group: - READ scope: all: {} + - workflowOrchestrationAcl: + actions: + - READ + scope: + all: {} metadata: origin: cdf-project-templates name: gp_admin_readonly diff --git a/tests/tests_unit/test_approval_modules_snapshots/processing_pipeline.yaml b/tests/tests_unit/test_approval_modules_snapshots/processing_pipeline.yaml new file mode 100644 index 000000000..df0d6bd81 --- /dev/null +++ b/tests/tests_unit/test_approval_modules_snapshots/processing_pipeline.yaml @@ -0,0 +1,28 @@ +WorkflowVersion: +- version: '1' + workflowDefinition: + description: Execute tasks in sequence + tasks: + - externalId: _function_task + name: task_1 + onFailure: abortWorkflow + parameters: + function: + externalId: fn__function + isAsyncComplete: false + retries: 3 + timeout: 3600 + type: function + - dependsOn: + - externalId: fn__function + externalId: _transformation_task + name: task_2 + onFailure: skipTask + parameters: + transformation: + concurrencyPolicy: fail + externalId: tr__transformation + retries: 3 + timeout: 3600 + type: transformation + workflowExternalId: wf_ diff --git a/tests/tests_unit/test_approval_modules_snapshots_clean/processing_pipeline.yaml b/tests/tests_unit/test_approval_modules_snapshots_clean/processing_pipeline.yaml new file mode 100644 index 000000000..0967ef424 --- /dev/null +++ b/tests/tests_unit/test_approval_modules_snapshots_clean/processing_pipeline.yaml @@ -0,0 +1 @@ +{} From 20bcb368d29027b23da83a5a5b20a2bb585532ed Mon Sep 17 00:00:00 2001 From: Anders Albert <60234212+doctrino@users.noreply.github.com> Date: Mon, 22 Apr 2024 15:37:55 +0200 Subject: [PATCH 22/34] Tests: Migration (#458) * tests: Added test to check that migration yaml is updated * build: updated changelog * tests: upgraded migration test to compare version correctly * tests: one of the last two * refator: fix logic mistake * refactor: updated migration.yaml * refactor: review feedback --- CHANGELOG.cdf-tk.md | 4 +-- .../_cdf_tk/templates/_migration.yaml | 11 +++++- tests/tests_unit/test_build.py | 35 +++++++++++++++++++ tests_migrations/constants.py | 10 +----- tests_migrations/migrations.py | 11 +++--- tests_migrations/tests_migrations.py | 6 ++-- 6 files changed, 58 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.cdf-tk.md b/CHANGELOG.cdf-tk.md index e58e2e00b..ba05385c7 100644 --- a/CHANGELOG.cdf-tk.md +++ b/CHANGELOG.cdf-tk.md @@ -27,8 +27,8 @@ Changes are grouped as follows: ### Changed -- The `cdf-tk build` will now clean the build directory by default before building the modules. To stop this behavior, - use the `--no-clean` flag. +- [BREAKING] The `cdf-tk build` will now clean the build directory by default before building the modules to avoid + unwanted side effects from previous builds. To stop this behavior, use the `--no-clean` flag. - The `.gitignore` file you get by running `cdf-tk init` now ignores the `/build` by default. - The dependency `cognite-sdk` must now be `>=7.37.0` to use the `cdf-tk`. diff --git a/cognite_toolkit/_cdf_tk/templates/_migration.yaml b/cognite_toolkit/_cdf_tk/templates/_migration.yaml index 48be41dba..b80c7239a 100644 --- a/cognite_toolkit/_cdf_tk/templates/_migration.yaml +++ b/cognite_toolkit/_cdf_tk/templates/_migration.yaml @@ -1,8 +1,17 @@ +- version: 0.2.0a3 + cognite_modules: {} + resources: {} + tool: + - title: '`cdf-tk build` command no longer supports the `--clean` flag' + steps: + - "The build command now cleans the build directory by default to avoid unwanted side effects from previous builds" + - "To avoid cleaning the build directory use the `--no-clean` flag." + cognite_modules_hash: "" - version: 0.2.0a2 cognite_modules: {} resources: {} tool: {} - cognite_modules_hash: "" + cognite_modules_hash: "07825ba8911c45a52881cf4a7536f9b2919c5f96fa8f6570e61406bc1666d329" - version: 0.2.0a1 cognite_modules: {} resources: {} diff --git a/tests/tests_unit/test_build.py b/tests/tests_unit/test_build.py index 6da23c1e2..1745a4ad2 100644 --- a/tests/tests_unit/test_build.py +++ b/tests/tests_unit/test_build.py @@ -3,8 +3,11 @@ from collections.abc import Iterator from datetime import datetime from re import Match +from typing import Any import pytest +import yaml +from packaging import version as version_package from packaging.version import Version from cognite_toolkit._version import __version__ @@ -73,6 +76,38 @@ def test_changelog_entry_date(changelog_name: str) -> None: assert True +@pytest.fixture(scope="session") +def migrations() -> list[dict[Any]]: + migration_yaml = REPO_ROOT / "cognite_toolkit" / "_cdf_tk" / "templates" / "_migration.yaml" + assert migration_yaml.exists(), f"{migration_yaml} does not exist. It has been moved" + + migrations = yaml.safe_load(migration_yaml.read_text()) + assert isinstance(migrations, list), f"{migration_yaml} should contain a list of migrations" + return migrations + + +def test_migration_hash_updated(migrations) -> None: + missing_hash = { + version_str + for entry in migrations + if version_package.parse(version_str := entry["version"]) < version_package.parse(__version__) + and not entry.get("cognite_modules_hash") + } + + # Check the instructions REPO_ROOT / "tests_migrations" / "README.md for more + # information on how to update the hash" + assert not missing_hash, f"Missing hash for the following versions: {missing_hash}" + + +def test_migration_entry_added(migrations) -> None: + last_entry = migrations[0] + second_last_entry = migrations[1] + + assert ( + last_entry["version"] == __version__ or second_last_entry["version"] == __version__ + ), "The latest entry in _migration.yaml should match the current version" + + def _parse_changelog(changelog: str) -> Iterator[Match[str]]: changelog = (REPO_ROOT / changelog).read_text(encoding="utf-8") return re.finditer(r"##\s\[(\d+\.\d+\.\d+([ab]\d+)?)\]\s-\s(\d+-\d+-\d+)", changelog) diff --git a/tests_migrations/constants.py b/tests_migrations/constants.py index 41448d293..7f4f0587a 100644 --- a/tests_migrations/constants.py +++ b/tests_migrations/constants.py @@ -6,19 +6,11 @@ TEST_DIR_ROOT = Path(__file__).resolve().parent SUPPORTED_TOOLKIT_VERSIONS = [ - "0.1.0b1", - "0.1.0b2", - "0.1.0b3", - "0.1.0b4", - "0.1.0b5", - "0.1.0b6", - "0.1.0b7", - "0.1.0b8", - "0.1.0b9", "0.1.0", "0.1.1", "0.1.2", "0.2.0a1", + "0.2.0a2", ] diff --git a/tests_migrations/migrations.py b/tests_migrations/migrations.py index b98da7f2a..ceef33376 100644 --- a/tests_migrations/migrations.py +++ b/tests_migrations/migrations.py @@ -4,6 +4,7 @@ from typing import Any import yaml +from packaging import version from cognite_toolkit._cdf_tk.templates import iterate_modules from cognite_toolkit._version import __version__ @@ -34,17 +35,17 @@ def modify_environment_to_run_all_modules(project_path: Path) -> None: config_dev_file.write_text(yaml.dump(config_dev)) -def get_migration(previous_version: str, current_version: str) -> Callable[[Path], None]: - previous_version = _version_str_to_tuple(previous_version) +def get_migration(previous_version_str: str, current_version: str) -> Callable[[Path], None]: + previous_version = version.parse(previous_version_str) changes = Changes() - if (0, 1, 0, "b", 7) <= previous_version: + if version.parse("0.1.0b7") <= previous_version: changes.append(_update_system_yaml) - if previous_version <= (0, 1, 0, "b", 4): + if previous_version <= version.parse("0.1.0b4"): changes.append(_add_name_to_file_configs) changes.append(_add_ignore_null_fields_to_transformation_configs) - if previous_version <= (0, 1, 0, "b", 6): + if previous_version <= version.parse("0.1.0b6"): changes.append(_to_config_env_yaml) return changes diff --git a/tests_migrations/tests_migrations.py b/tests_migrations/tests_migrations.py index 8d390d342..2f66022ff 100644 --- a/tests_migrations/tests_migrations.py +++ b/tests_migrations/tests_migrations.py @@ -6,6 +6,7 @@ from pathlib import Path import pytest +from packaging import version as version_package from cognite_toolkit._version import __version__ from tests_migrations.constants import SUPPORTED_TOOLKIT_VERSIONS, TEST_DIR_ROOT, chdir @@ -65,11 +66,12 @@ def tests_init_migrate_build_deploy( for cmd in [ [previous_version, "--version"], [previous_version, "init", project_name, "--clean"], - [previous_version, "build", project_name, "--env", "dev", "--clean"], + [previous_version, "build", project_name, "--env", "dev"] + + (["--clean"] if version_package.parse(old_version) < version_package.parse("0.2.0a3") else []), [previous_version, "deploy", "--env", "dev", "--dry-run"], # This runs the cdf-tk command from the cognite_toolkit package in the ROOT of the repo. ["cdf-tk", "--version"], - ["cdf-tk", "build", project_name, "--env", "dev", "--build-dir", build_name, "--clean"], + ["cdf-tk", "build", project_name, "--env", "dev", "--build-dir", build_name], ["cdf-tk", "deploy", "--env", "dev", "--dry-run"], ]: if cmd[0] == "cdf-tk" and is_upgrade: From ca24e97d169acf61f44af1191e5585452492b87b Mon Sep 17 00:00:00 2001 From: Anders Albert <60234212+doctrino@users.noreply.github.com> Date: Mon, 22 Apr 2024 15:47:05 +0200 Subject: [PATCH 23/34] [CDF-21255] Standalone CLI (#460) * tests: Added test to check that migration yaml is updated * build: updated changelog * tests: upgraded migration test to compare version correctly * tests: one of the last two * refator: fix logic mistake * refactor: updated migration.yaml * feat: Add modules as root module * refactor: moded '_system.yaml' up one level * refactor: update to _system.yaml on root level * tests: updated tests * tests: Activated test for custom module named "modules" * refactor: migration guide * refactor: review feedback * refactor; rename modules to variables * refactor: Rename "modules" to "variable" in init config * refactor: Rename section "modules" to "variables in config files * tests: updated test to use variables instead of modules section * ci: Run quality control on all PRs * build; changelog + migration guide * refactor: update demo --- .github/workflows/build.yml | 2 +- CHANGELOG.cdf-tk.md | 8 ++++ cognite_toolkit/_api/data_classes.py | 2 +- cognite_toolkit/_api/modules_api.py | 6 +-- cognite_toolkit/_cdf.py | 4 +- cognite_toolkit/_cdf_tk/pull.py | 9 +--- cognite_toolkit/_cdf_tk/run.py | 3 +- cognite_toolkit/_cdf_tk/templates/__init__.py | 2 + .../_cdf_tk/templates/_constants.py | 6 ++- .../_cdf_tk/templates/_migration.yaml | 6 +++ .../_cdf_tk/templates/_templates.py | 4 +- .../templates/data_classes/_config_yaml.py | 32 ++++++++------ .../data_classes/_project_directory.py | 1 + .../{cognite_modules => }/_system.yaml | 0 demo/config.demo.yaml | 2 +- tests/tests_unit/test_behavior.py | 3 +- .../{cognite_modules => }/_system.yaml | 0 .../project_for_test/config.dev.yaml | 2 +- .../project_no_cognite_modules/_system.yaml | 6 +++ .../config.dev.yaml | 2 +- .../project_with_duplicates/config.dev.yaml | 2 +- .../{cognite_modules => }/_system.yaml | 0 .../test_cdf_tk/run_data/config.dev.yaml | 2 +- tests/tests_unit/test_cdf_tk/test_load.py | 3 +- .../tests_unit/test_cdf_tk/test_templates.py | 43 +++++++++---------- 25 files changed, 83 insertions(+), 67 deletions(-) rename cognite_toolkit/{cognite_modules => }/_system.yaml (100%) rename tests/tests_unit/test_cdf_tk/project_for_test/{cognite_modules => }/_system.yaml (100%) create mode 100644 tests/tests_unit/test_cdf_tk/project_no_cognite_modules/_system.yaml rename tests/tests_unit/test_cdf_tk/run_data/{cognite_modules => }/_system.yaml (100%) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 61760122f..7b1d357bc 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -2,7 +2,7 @@ name: Lint and test on: pull_request: - branches: [main, release] + env: PYTHON_VERSION: '3.9' jobs: diff --git a/CHANGELOG.cdf-tk.md b/CHANGELOG.cdf-tk.md index ba05385c7..1f021cf4b 100644 --- a/CHANGELOG.cdf-tk.md +++ b/CHANGELOG.cdf-tk.md @@ -29,6 +29,14 @@ Changes are grouped as follows: - [BREAKING] The `cdf-tk build` will now clean the build directory by default before building the modules to avoid unwanted side effects from previous builds. To stop this behavior, use the `--no-clean` flag. +- [BREAKING] The `_system.yaml` is now required to be on the root level when running any `cdf-tk` command. This + means that the `_system.yaml` file must be in the same directory as the `config.[env].yaml` files. This is to + support running `cdf-tk` without the `cognite_modules` folder. +- [BREAKING] In the `config.[env].yaml` files, the `modules` section is now renamed to `variables`. This is to + better reflect the content of this section, which is variables that can be used in the resource files. +- In addition to `cognite_modules` and `custom_modules`, the `cognite-toolkit` now also support `modules` as the + top-level folder for modules. This together with the two changes above, is to have a better support for running + the `cognite-toolkit` as a standalone CLI without the `cognite_modules`. - The `.gitignore` file you get by running `cdf-tk init` now ignores the `/build` by default. - The dependency `cognite-sdk` must now be `>=7.37.0` to use the `cdf-tk`. diff --git a/cognite_toolkit/_api/data_classes.py b/cognite_toolkit/_api/data_classes.py index bfe873c3f..d914ab914 100644 --- a/cognite_toolkit/_api/data_classes.py +++ b/cognite_toolkit/_api/data_classes.py @@ -78,7 +78,7 @@ def _load(cls, module_path: Path, default_variables: dict[tuple[str, ...], Confi variable_module_key = [*module_key_path, "first_pop"] while variable_module_key: variable_module_key.pop() - key_path = (InitConfigYAML._modules, *variable_module_key, variable_name) + key_path = (InitConfigYAML._variables, *variable_module_key, variable_name) if default := default_variables.get(key_path): value.default_value = default.default_value value.default_comment = default.default_comment diff --git a/cognite_toolkit/_api/modules_api.py b/cognite_toolkit/_api/modules_api.py index a41b7e0ee..f8b624ca2 100644 --- a/cognite_toolkit/_api/modules_api.py +++ b/cognite_toolkit/_api/modules_api.py @@ -37,7 +37,7 @@ def _source_dir(self) -> Path: def _load_modules(self) -> None: source_dir = self._source_dir() - system_yaml = SystemYAML.load_from_directory(source_dir, self._build_env) + system_yaml = SystemYAML.load_from_directory(source_dir.parent, self._build_env) default_config = InitConfigYAML(_DUMMY_ENVIRONMENT).load_defaults(source_dir) for module, _ in iterate_modules(source_dir): @@ -76,13 +76,13 @@ def _build(self, modules: Sequence[ModuleMeta], verbose: bool) -> None: common_function_code="./common_function_code", ), filepath=Path(""), - modules=variables, + variables=variables, ) build_config( self._build_dir, self._source_dir().parent, config, - system_config=SystemYAML.load_from_directory(self._source_dir(), self._build_env), + system_config=SystemYAML.load_from_directory(self._source_dir().parent, self._build_env), clean=True, verbose=verbose, ) diff --git a/cognite_toolkit/_cdf.py b/cognite_toolkit/_cdf.py index dcf4dfdfe..6b89c74d8 100755 --- a/cognite_toolkit/_cdf.py +++ b/cognite_toolkit/_cdf.py @@ -245,9 +245,7 @@ def build( if not source_path.is_dir(): print(f" [bold red]ERROR:[/] {source_path} does not exist") exit(1) - cognite_modules_path = source_path / COGNITE_MODULES - - system_config = SystemYAML.load_from_directory(cognite_modules_path, build_env) + system_config = SystemYAML.load_from_directory(source_path, build_env) config = BuildConfigYAML.load_from_directory(source_path, build_env) print( Panel( diff --git a/cognite_toolkit/_cdf_tk/pull.py b/cognite_toolkit/_cdf_tk/pull.py index 4830f0472..5426a1e68 100644 --- a/cognite_toolkit/_cdf_tk/pull.py +++ b/cognite_toolkit/_cdf_tk/pull.py @@ -20,7 +20,6 @@ from cognite_toolkit._cdf_tk.load import ResourceLoader from cognite_toolkit._cdf_tk.load._base_loaders import T_ID, T_WritableCogniteResourceList from cognite_toolkit._cdf_tk.templates import ( - COGNITE_MODULES, build_config, ) from cognite_toolkit._cdf_tk.templates.data_classes import BuildConfigYAML, SystemYAML @@ -378,15 +377,9 @@ def pull_command( if not source_path.is_dir(): print(f" [bold red]ERROR:[/] {source_path} does not exist") exit(1) - if not (source_path / COGNITE_MODULES).is_dir(): - print( - f" [bold red]ERROR:[/] {source_path} does not contain a {COGNITE_MODULES} directory. " - f"Did you pass in a valid source directory?" - ) - exit(1) build_dir = Path(tempfile.mkdtemp(prefix="build.", suffix=".tmp", dir=Path.cwd())) - system_config = SystemYAML.load_from_directory(source_path / COGNITE_MODULES, env) + system_config = SystemYAML.load_from_directory(source_path, env) config = BuildConfigYAML.load_from_directory(source_path, env) config.set_environment_variables() config.environment.selected_modules_and_packages = config.available_modules diff --git a/cognite_toolkit/_cdf_tk/run.py b/cognite_toolkit/_cdf_tk/run.py index 2817b8777..cf0247ecf 100644 --- a/cognite_toolkit/_cdf_tk/run.py +++ b/cognite_toolkit/_cdf_tk/run.py @@ -20,7 +20,6 @@ from cognite_toolkit._cdf_tk.constants import _RUNNING_IN_BROWSER from cognite_toolkit._cdf_tk.load import FunctionLoader, FunctionScheduleLoader from cognite_toolkit._cdf_tk.templates import ( - COGNITE_MODULES, build_config, module_from_path, ) @@ -181,7 +180,7 @@ def run_local_function( return False raise - system_config = SystemYAML.load_from_directory(source_path / COGNITE_MODULES, build_env) + system_config = SystemYAML.load_from_directory(source_path, build_env) config = BuildConfigYAML.load_from_directory(source_path, build_env) print(f"[bold]Building for environment {build_env} using {source_path!s} as sources...[/bold]") config.set_environment_variables() diff --git a/cognite_toolkit/_cdf_tk/templates/__init__.py b/cognite_toolkit/_cdf_tk/templates/__init__.py index 5829b0b70..0533f598e 100644 --- a/cognite_toolkit/_cdf_tk/templates/__init__.py +++ b/cognite_toolkit/_cdf_tk/templates/__init__.py @@ -4,6 +4,7 @@ COGNITE_MODULES_PATH, CUSTOM_MODULES, ROOT_MODULES, + ROOT_PATH, ) from ._templates import build_config, check_yaml_semantics, create_local_config, split_config from ._utils import flatten_dict, iterate_modules, module_from_path, resource_folder_from_path @@ -14,6 +15,7 @@ "resource_folder_from_path", "COGNITE_MODULES", "CUSTOM_MODULES", + "ROOT_PATH", "ROOT_MODULES", "COGNITE_MODULES_PATH", "build_config", diff --git a/cognite_toolkit/_cdf_tk/templates/_constants.py b/cognite_toolkit/_cdf_tk/templates/_constants.py index f67c0f42d..3842e734c 100644 --- a/cognite_toolkit/_cdf_tk/templates/_constants.py +++ b/cognite_toolkit/_cdf_tk/templates/_constants.py @@ -11,8 +11,9 @@ COGNITE_MODULES = "cognite_modules" CUSTOM_MODULES = "custom_modules" +ALT_CUSTOM_MODULES = "modules" -ROOT_MODULES = [COGNITE_MODULES, CUSTOM_MODULES] +ROOT_MODULES = [COGNITE_MODULES, CUSTOM_MODULES, ALT_CUSTOM_MODULES] # Add any other files below that should be included in a build EXCL_FILES = ["README.md", DEFAULT_CONFIG_FILE] @@ -23,4 +24,5 @@ # Which suffixes to process for template variable replacement PROC_TMPL_VARS_SUFFIX = frozenset([".yaml", ".yml", ".sql", ".csv", ".parquet", ".json", ".txt", ".md", ".html", ".py"]) -COGNITE_MODULES_PATH = Path(__file__).parent.parent.parent / COGNITE_MODULES +ROOT_PATH = Path(__file__).parent.parent.parent +COGNITE_MODULES_PATH = ROOT_PATH / COGNITE_MODULES diff --git a/cognite_toolkit/_cdf_tk/templates/_migration.yaml b/cognite_toolkit/_cdf_tk/templates/_migration.yaml index b80c7239a..afffa3055 100644 --- a/cognite_toolkit/_cdf_tk/templates/_migration.yaml +++ b/cognite_toolkit/_cdf_tk/templates/_migration.yaml @@ -6,6 +6,12 @@ steps: - "The build command now cleans the build directory by default to avoid unwanted side effects from previous builds" - "To avoid cleaning the build directory use the `--no-clean` flag." + - title: 'The `_system.yaml` file moved one level up from `cognite_modules`' + steps: + - "Go into the `cognite_modules` folder and move the `_system.yaml` file one level up." + - title: "In all `config..yaml` the section 'modules' is renamed to 'variables" + steps: + - For each `config..yaml` file, rename the section `modules` to `variables`. cognite_modules_hash: "" - version: 0.2.0a2 cognite_modules: {} diff --git a/cognite_toolkit/_cdf_tk/templates/_templates.py b/cognite_toolkit/_cdf_tk/templates/_templates.py index 5801d1f2c..ab1a873bc 100644 --- a/cognite_toolkit/_cdf_tk/templates/_templates.py +++ b/cognite_toolkit/_cdf_tk/templates/_templates.py @@ -72,7 +72,7 @@ def build_config( selected_modules = config.get_selected_modules(system_config.packages, available_modules, verbose) - warnings = validate_modules_variables(config.modules, config.filepath) + warnings = validate_modules_variables(config.variables, config.filepath) if warnings: print(f" [bold yellow]WARNING:[/] Found the following warnings in config.{config.environment.name}.yaml:") for warning in warnings: @@ -431,7 +431,7 @@ def process_config_files( source_by_build_path: dict[Path, Path] = {} printed_function_warning = False environment = config.environment - configs = split_config(config.modules) + configs = split_config(config.variables) modules_by_variables = defaultdict(list) for module_path, variables in configs.items(): for variable in variables: diff --git a/cognite_toolkit/_cdf_tk/templates/data_classes/_config_yaml.py b/cognite_toolkit/_cdf_tk/templates/data_classes/_config_yaml.py index 0eff48d76..4df97b1a6 100644 --- a/cognite_toolkit/_cdf_tk/templates/data_classes/_config_yaml.py +++ b/cognite_toolkit/_cdf_tk/templates/data_classes/_config_yaml.py @@ -71,12 +71,12 @@ class ConfigYAMLCore(ABC): class BuildConfigYAML(ConfigCore, ConfigYAMLCore): """This is the config.[env].yaml file used in the cdf-tk build command.""" - modules: dict[str, Any] + variables: dict[str, Any] @property def available_modules(self) -> list[str]: available_modules: list[str] = [] - to_check = [self.modules] + to_check = [self.variables] while to_check: current = to_check.pop() for key, value in current.items(): @@ -117,13 +117,17 @@ def validate_environment(self) -> None: @classmethod def load(cls, data: dict[str, Any], build_env: str, filepath: Path) -> BuildConfigYAML: - try: - environment = Environment.load(data["environment"], build_env) - modules = data["modules"] - except KeyError: - print(f" [bold red]ERROR:[/] Missing 'environment' or 'modules' in {filepath!s}") + if missing := [section for section in ["environment", "variables"] if section not in data]: + print(f" [bold red]ERROR:[/] Expected {', '.join(missing)} section(s) in {filepath!s}. ") + if "modules" in data and "variables" in missing: + print( + " [bold yellow]WARNING:[/] 'modules' section is deprecated and " + "has been renamed to 'variables' instead." + ) exit(1) - return cls(environment=environment, modules=modules, filepath=filepath) + environment = Environment.load(data["environment"], build_env) + variables = data["variables"] + return cls(environment=environment, variables=variables, filepath=filepath) def create_build_environment(self) -> BuildEnvironment: return BuildEnvironment( @@ -300,7 +304,7 @@ class InitConfigYAML(YAMLWithComments[tuple[str, ...], ConfigEntry], ConfigYAMLC # Top level keys _environment = "environment" - _modules = "modules" + _variables = "variables" def __init__(self, environment: Environment, entries: dict[tuple[str, ...], ConfigEntry] | None = None): self.environment = environment @@ -327,7 +331,7 @@ def load_defaults(self, cognite_root_module: Path) -> InitConfigYAML: file_comments = self._extract_comments(raw_file, key_prefix=tuple(parts)) file_data = yaml.safe_load(raw_file) for key, value in file_data.items(): - key_path = (self._modules, *parts, key) + key_path = (self._variables, *parts, key) local_file_path = (*parts, key) if key_path in self: self[key_path].default_value = value @@ -363,10 +367,10 @@ def load_existing(cls, existing_config_yaml: str, build_env: str = "dev") -> Ini else: raise ValueError(f"Missing environment in {existing_config_yaml!s}") - modules = config[cls._modules] if cls._modules in config else config + modules = config[cls._variables] if cls._variables in config else config entries: dict[tuple[str, ...], ConfigEntry] = {} for key_path, value in flatten_dict(modules).items(): - full_key_path = (cls._modules, *key_path) + full_key_path = (cls._variables, *key_path) if full_key_path in entries: entries[full_key_path].current_value = value entries[full_key_path].current_comment = comments.get(full_key_path) @@ -427,12 +431,12 @@ def _load_variables( key_parent_list = key_parent_list[:i] break key_parent = tuple(key_parent_list) - key_path = (self._modules, *key_parent, variable) + key_path = (self._variables, *key_parent, variable) if key_path in self: continue # Search for the first parent that match. for i in range(1, len(key_parent)): - alt_key_path = (self._modules, *key_parent[:i], variable) + alt_key_path = (self._variables, *key_parent[:i], variable) if alt_key_path in self: break else: diff --git a/cognite_toolkit/_cdf_tk/templates/data_classes/_project_directory.py b/cognite_toolkit/_cdf_tk/templates/data_classes/_project_directory.py index 565fcb2d7..32cad246d 100644 --- a/cognite_toolkit/_cdf_tk/templates/data_classes/_project_directory.py +++ b/cognite_toolkit/_cdf_tk/templates/data_classes/_project_directory.py @@ -38,6 +38,7 @@ class ProjectDirectory: "README.md", ".gitignore", ".env.tmpl", + "_system.yaml", ] _directories_to_copy: ClassVar[list[str]] = [ "common_function_code", diff --git a/cognite_toolkit/cognite_modules/_system.yaml b/cognite_toolkit/_system.yaml similarity index 100% rename from cognite_toolkit/cognite_modules/_system.yaml rename to cognite_toolkit/_system.yaml diff --git a/demo/config.demo.yaml b/demo/config.demo.yaml index dd44799f0..c2fa9e384 100644 --- a/demo/config.demo.yaml +++ b/demo/config.demo.yaml @@ -11,7 +11,7 @@ environment: - cdf_data_pipeline_asset_valhall common_function_code: ./common_function_code -modules: +variables: cognite_modules: # Globally defined variables are available to all submodules # of the cognite_modules. diff --git a/tests/tests_unit/test_behavior.py b/tests/tests_unit/test_behavior.py index bb7104d08..55574a525 100644 --- a/tests/tests_unit/test_behavior.py +++ b/tests/tests_unit/test_behavior.py @@ -27,7 +27,7 @@ def test_inject_custom_environmental_variables( init_project: Path, ) -> None: config_yaml = yaml.safe_load((init_project / "config.dev.yaml").read_text()) - config_yaml["modules"]["cognite_modules"]["cicd_clientId"] = "${MY_ENVIRONMENT_VARIABLE}" + config_yaml["variables"]["cognite_modules"]["cicd_clientId"] = "${MY_ENVIRONMENT_VARIABLE}" # Selecting a module with a transformation that uses the cicd_clientId variable config_yaml["environment"]["selected_modules_and_packages"] = ["cdf_infield_location"] config_yaml["environment"]["project"] = "pytest" @@ -241,7 +241,6 @@ def test_dump_datamodel( assert len(child_loaded.properties) == 1 -@pytest.mark.skip("This functionality is not yet implemented") def test_build_custom_project( local_tmp_path: Path, typer_context: typer.Context, diff --git a/tests/tests_unit/test_cdf_tk/project_for_test/cognite_modules/_system.yaml b/tests/tests_unit/test_cdf_tk/project_for_test/_system.yaml similarity index 100% rename from tests/tests_unit/test_cdf_tk/project_for_test/cognite_modules/_system.yaml rename to tests/tests_unit/test_cdf_tk/project_for_test/_system.yaml diff --git a/tests/tests_unit/test_cdf_tk/project_for_test/config.dev.yaml b/tests/tests_unit/test_cdf_tk/project_for_test/config.dev.yaml index 352b0cb09..abdfc9b6a 100644 --- a/tests/tests_unit/test_cdf_tk/project_for_test/config.dev.yaml +++ b/tests/tests_unit/test_cdf_tk/project_for_test/config.dev.yaml @@ -6,7 +6,7 @@ environment: - cdf_demo_infield - cdf_oid_example_data -modules: +variables: cognite_modules: top_variable: a_module: diff --git a/tests/tests_unit/test_cdf_tk/project_no_cognite_modules/_system.yaml b/tests/tests_unit/test_cdf_tk/project_no_cognite_modules/_system.yaml new file mode 100644 index 000000000..f3577b854 --- /dev/null +++ b/tests/tests_unit/test_cdf_tk/project_no_cognite_modules/_system.yaml @@ -0,0 +1,6 @@ +# DO NOT EDIT THIS FILE! +# This is the system configuration for the Cognite Toolkit. +packages: {} + +# This part is used by cdf-toolkit to keep track of the version and help you upgrade. +cdf_toolkit_version: 0.2.0a2 diff --git a/tests/tests_unit/test_cdf_tk/project_no_cognite_modules/config.dev.yaml b/tests/tests_unit/test_cdf_tk/project_no_cognite_modules/config.dev.yaml index ce0c6dcc9..3f6f032ff 100644 --- a/tests/tests_unit/test_cdf_tk/project_no_cognite_modules/config.dev.yaml +++ b/tests/tests_unit/test_cdf_tk/project_no_cognite_modules/config.dev.yaml @@ -6,7 +6,7 @@ environment: - a_module - another_module -modules: +variables: modules: a_module: ds_timeseries: ds_timeseries diff --git a/tests/tests_unit/test_cdf_tk/project_with_duplicates/config.dev.yaml b/tests/tests_unit/test_cdf_tk/project_with_duplicates/config.dev.yaml index d1dfaf14b..e9aa856d3 100644 --- a/tests/tests_unit/test_cdf_tk/project_with_duplicates/config.dev.yaml +++ b/tests/tests_unit/test_cdf_tk/project_with_duplicates/config.dev.yaml @@ -5,7 +5,7 @@ environment: selected_modules_and_packages: - module1 -modules: +variables: cognite_modules: examples: module1: diff --git a/tests/tests_unit/test_cdf_tk/run_data/cognite_modules/_system.yaml b/tests/tests_unit/test_cdf_tk/run_data/_system.yaml similarity index 100% rename from tests/tests_unit/test_cdf_tk/run_data/cognite_modules/_system.yaml rename to tests/tests_unit/test_cdf_tk/run_data/_system.yaml diff --git a/tests/tests_unit/test_cdf_tk/run_data/config.dev.yaml b/tests/tests_unit/test_cdf_tk/run_data/config.dev.yaml index 5677ff3eb..c4fba26a7 100644 --- a/tests/tests_unit/test_cdf_tk/run_data/config.dev.yaml +++ b/tests/tests_unit/test_cdf_tk/run_data/config.dev.yaml @@ -6,7 +6,7 @@ environment: - cdf_functions_dummy common_function_code: ../../../../cognite_toolkit/common_function_code -modules: +variables: cognite_modules: # Globally defined variables are available to all submodules # of the cognite_modules. diff --git a/tests/tests_unit/test_cdf_tk/test_load.py b/tests/tests_unit/test_cdf_tk/test_load.py index bfbf8f2ee..063f80ba0 100644 --- a/tests/tests_unit/test_cdf_tk/test_load.py +++ b/tests/tests_unit/test_cdf_tk/test_load.py @@ -32,7 +32,6 @@ ViewLoader, ) from cognite_toolkit._cdf_tk.templates import ( - COGNITE_MODULES, build_config, ) from cognite_toolkit._cdf_tk.templates.data_classes import ( @@ -422,7 +421,7 @@ def test_load_skip_validation_with_preexisting_dataset( class TestDeployResources: def test_deploy_resource_order(self, cognite_client_approval: ApprovalCogniteClient): build_env = "dev" - system_config = SystemYAML.load_from_directory(PYTEST_PROJECT / COGNITE_MODULES, build_env) + system_config = SystemYAML.load_from_directory(PYTEST_PROJECT, build_env) config = BuildConfigYAML.load_from_directory(PYTEST_PROJECT, build_env) config.environment.selected_modules_and_packages = ["another_module"] build_config(BUILD_DIR, PYTEST_PROJECT, config=config, system_config=system_config, clean=True, verbose=False) diff --git a/tests/tests_unit/test_cdf_tk/test_templates.py b/tests/tests_unit/test_cdf_tk/test_templates.py index b78f0d8a2..0f5f5613a 100644 --- a/tests/tests_unit/test_cdf_tk/test_templates.py +++ b/tests/tests_unit/test_cdf_tk/test_templates.py @@ -9,7 +9,6 @@ from cognite_toolkit._cdf_tk.load import LOADER_BY_FOLDER_NAME from cognite_toolkit._cdf_tk.templates import ( - COGNITE_MODULES, build_config, check_yaml_semantics, create_local_config, @@ -59,7 +58,7 @@ def test_producing_correct_keys(self, config_yaml: str, dummy_environment: Envir expected_keys = set(flatten_dict(yaml.safe_load(config_yaml))) # Custom keys are not loaded from the module folder. # This custom key is added o the dev.config.yaml for other tests. - expected_keys.remove(("modules", "custom_modules", "my_example_module", "transformation_is_paused")) + expected_keys.remove(("variables", "custom_modules", "my_example_module", "transformation_is_paused")) # Skip all environment variables expected_keys = {k for k in expected_keys if not k[0] == "environment"} @@ -73,16 +72,16 @@ def test_producing_correct_keys(self, config_yaml: str, dummy_environment: Envir def test_extract_extract_config_yaml_comments(self, config_yaml: str) -> None: expected_comments = { - ("modules", "cognite_modules", "a_module", "readonly_source_id"): YAMLComment( + ("variables", "cognite_modules", "a_module", "readonly_source_id"): YAMLComment( above=["This is a comment in the middle of the file"], after=[] ), - ("modules", "cognite_modules", "another_module", "default_location"): YAMLComment( + ("variables", "cognite_modules", "another_module", "default_location"): YAMLComment( above=["This is a comment at the beginning of the module."] ), - ("modules", "cognite_modules", "another_module", "source_asset"): YAMLComment( + ("variables", "cognite_modules", "another_module", "source_asset"): YAMLComment( after=["This is an extra comment added to the config only 'lore ipsum'"] ), - ("modules", "cognite_modules", "another_module", "source_files"): YAMLComment( + ("variables", "cognite_modules", "another_module", "source_files"): YAMLComment( after=["This is a comment after a variable"] ), } @@ -128,35 +127,35 @@ def test_persist_variable_with_comment(self, config_yaml: str) -> None: dumped = config.dump_yaml_with_comments() loaded = yaml.safe_load(dumped) - assert loaded["modules"]["cognite_modules"]["another_module"]["source_asset"] == "my_new_workmate" + assert loaded["variables"]["cognite_modules"]["another_module"]["source_asset"] == "my_new_workmate" assert custom_comment in dumped def test_added_and_removed_variables(self, config_yaml: str) -> None: existing_config_yaml = yaml.safe_load(config_yaml) # Added = Exists in the BUILD_CONFIG directory default.config.yaml files but not in config.yaml - existing_config_yaml["modules"]["cognite_modules"]["another_module"].pop("source_asset") + existing_config_yaml["variables"]["cognite_modules"]["another_module"].pop("source_asset") # Removed = Exists in config.yaml but not in the BUILD_CONFIG directory default.config.yaml files - existing_config_yaml["modules"]["cognite_modules"]["another_module"]["removed_variable"] = "old_value" + existing_config_yaml["variables"]["cognite_modules"]["another_module"]["removed_variable"] = "old_value" config = InitConfigYAML.load_existing(yaml.safe_dump(existing_config_yaml)).load_defaults(PYTEST_PROJECT) removed = [v for v in config.values() if v.default_value is None] # There is already a custom variable in the config.yaml file assert len(removed) == 2 - assert ("modules", "cognite_modules", "another_module", "removed_variable") in [v.key_path for v in removed] + assert ("variables", "cognite_modules", "another_module", "removed_variable") in [v.key_path for v in removed] added = [v for v in config.values() if v.current_value is None] assert len(added) == 1 - assert added[0].key_path == ("modules", "cognite_modules", "another_module", "source_asset") + assert added[0].key_path == ("variables", "cognite_modules", "another_module", "source_asset") def test_load_variables(self, dummy_environment: Environment) -> None: expected = { - ("modules", "cognite_modules", "a_module", "readonly_source_id"), + ("variables", "cognite_modules", "a_module", "readonly_source_id"), # default_location is used in two modules and is moved to the top level - ("modules", "cognite_modules", "default_location"), - ("modules", "cognite_modules", "another_module", "source_files"), - ("modules", "cognite_modules", "another_module", "model_space"), - ("modules", "cognite_modules", "parent_module", "child_module", "source_asset"), + ("variables", "cognite_modules", "default_location"), + ("variables", "cognite_modules", "another_module", "source_files"), + ("variables", "cognite_modules", "another_module", "model_space"), + ("variables", "cognite_modules", "parent_module", "child_module", "source_asset"), } config = InitConfigYAML(dummy_environment).load_variables(PYTEST_PROJECT, propagate_reused_variables=True) @@ -170,8 +169,8 @@ def test_load_parent_variables(self, dummy_environment: Environment) -> None: config = InitConfigYAML( dummy_environment, { - ("modules", "cognite_modules", "infield", "shared_variable"): ConfigEntry( - key_path=("modules", "cognite_modules", "infield", "shared_variable"), + ("variables", "cognite_modules", "infield", "shared_variable"): ConfigEntry( + key_path=("variables", "cognite_modules", "infield", "shared_variable"), default_value="shared_value", ) }, @@ -179,8 +178,8 @@ def test_load_parent_variables(self, dummy_environment: Environment) -> None: config._load_variables({"shared_variable": {("cognite_modules", "infield", "cdf_infield_common")}}) - assert ("modules", "cognite_modules", "infield", "shared_variable") in config.keys() - assert ("modules", "cognite_modules", "infield", "cdf_infield_common", "shared_variable") not in config.keys() + assert ("variables", "cognite_modules", "infield", "shared_variable") in config.keys() + assert ("variables", "cognite_modules", "infield", "cdf_infield_common", "shared_variable") not in config.keys() @pytest.mark.parametrize( @@ -326,7 +325,7 @@ def test_module_from_path(self, path: Path, expected: str): class TestBuildConfigYAML: def test_build_config_create_valid_build_folder(self, config_yaml: str) -> None: build_env = "dev" - system_config = SystemYAML.load_from_directory(PYTEST_PROJECT / COGNITE_MODULES, build_env) + system_config = SystemYAML.load_from_directory(PYTEST_PROJECT, build_env) config = BuildConfigYAML.load_from_directory(PYTEST_PROJECT, build_env) available_modules = {module.name for module, _ in iterate_modules(PYTEST_PROJECT)} config.environment.selected_modules_and_packages = list(available_modules) @@ -368,6 +367,6 @@ def test_build_config_create_valid_build_folder(self, config_yaml: str) -> None: def test_available_modules( self, modules: dict[str, Any], expected_available_modules: list[str], dummy_environment: Environment ) -> None: - config = BuildConfigYAML(dummy_environment, filepath=Path("dummy"), modules=modules) + config = BuildConfigYAML(dummy_environment, filepath=Path("dummy"), variables=modules) assert sorted(config.available_modules) == sorted(expected_available_modules) From 68ab175fb5c2e7375c7977654f19352c51f47908 Mon Sep 17 00:00:00 2001 From: Anders Albert <60234212+doctrino@users.noreply.github.com> Date: Mon, 22 Apr 2024 16:17:17 +0200 Subject: [PATCH 24/34] New ruff location (#466) build: new ruff locatin --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 62bbfbf06..cb80002e2 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ --- repos: - - repo: https://github.com/charliermarsh/ruff-pre-commit + - repo: https://github.com/astral-sh/ruff-pre-commit hooks: - id: ruff args: From cdc695edcd8f9a60b867ef59a063b8bc2939bf42 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A5l=20R=C3=B8nning?= Date: Mon, 22 Apr 2024 17:32:04 +0200 Subject: [PATCH 25/34] Example with workflow (#465) * Started added loader for Workflow * Savepoint * Savepoint * Deployment works * mypy caught something * Savepoint workflow tests * Ready for review * Update cognite_toolkit/_cdf_tk/load/_resource_loaders.py Co-authored-by: Anders Albert <60234212+doctrino@users.noreply.github.com> * Update cognite_toolkit/_cdf_tk/load/_resource_loaders.py Co-authored-by: Anders Albert <60234212+doctrino@users.noreply.github.com> * Update cognite_toolkit/_cdf_tk/load/_resource_loaders.py Co-authored-by: Anders Albert <60234212+doctrino@users.noreply.github.com> * Update tests/tests_unit/test_approval_modules_snapshots/cdf_oid_example_data.yaml Co-authored-by: Anders Albert <60234212+doctrino@users.noreply.github.com> * Restored and expanded custom example module * build clean is obsolete * Restored my_example_module * mypy * regen'ed tests * changelog * wrangling dataframe value * Experiment almost snuck in * Trying to fix tests * Added missing key in yaml * restoring loader * fixing tests * tests * updating to 'variables' in config file * tests * Update cognite_toolkit/custom_modules/my_example_module/functions/fn_first_function/requirements.txt Co-authored-by: Anders Albert <60234212+doctrino@users.noreply.github.com> --------- Co-authored-by: Anders Albert <60234212+doctrino@users.noreply.github.com> --- .vscode/launch.json | 13 ++++++ CHANGELOG.templates.md | 1 + .../_cdf_tk/load/_resource_loaders.py | 2 +- .../processing_pipeline/default.config.yaml | 1 - .../my_example_module/default.config.yaml | 4 ++ .../my_example_module/functions/README.md | 4 ++ .../functions/first.function.schedule.yaml | 14 ++++++ .../functions/first.function.yaml | 23 ++++++++++ .../functions/fn_first_function/handler.py | 26 +++++++++++ .../fn_first_function/requirements.txt | 1 + .../fn_first_function/src/__init__.py | 0 .../fn_first_function/src/handler.py | 13 ++++++ .../first.transformation.schedule.yaml | 3 ++ .../transformations/first.transformation.sql | 0 .../transformations/first.transformation.yaml | 9 ++++ .../workflows/first.Workflow.yaml} | 0 .../workflows/first.WorkflowVersion.yaml} | 14 +++--- demo/config.demo.yaml | 44 +++++++++++-------- .../processing_pipeline.yaml | 28 ------------ .../processing_pipeline.yaml | 1 - 20 files changed, 145 insertions(+), 56 deletions(-) delete mode 100644 cognite_toolkit/cognite_modules/experimental/processing_pipeline/default.config.yaml create mode 100644 cognite_toolkit/custom_modules/my_example_module/default.config.yaml create mode 100644 cognite_toolkit/custom_modules/my_example_module/functions/README.md create mode 100644 cognite_toolkit/custom_modules/my_example_module/functions/first.function.schedule.yaml create mode 100644 cognite_toolkit/custom_modules/my_example_module/functions/first.function.yaml create mode 100644 cognite_toolkit/custom_modules/my_example_module/functions/fn_first_function/handler.py create mode 100644 cognite_toolkit/custom_modules/my_example_module/functions/fn_first_function/requirements.txt create mode 100644 cognite_toolkit/custom_modules/my_example_module/functions/fn_first_function/src/__init__.py create mode 100644 cognite_toolkit/custom_modules/my_example_module/functions/fn_first_function/src/handler.py create mode 100644 cognite_toolkit/custom_modules/my_example_module/transformations/first.transformation.schedule.yaml create mode 100644 cognite_toolkit/custom_modules/my_example_module/transformations/first.transformation.sql create mode 100644 cognite_toolkit/custom_modules/my_example_module/transformations/first.transformation.yaml rename cognite_toolkit/{cognite_modules/experimental/processing_pipeline/workflows/my.Workflow.yaml => custom_modules/my_example_module/workflows/first.Workflow.yaml} (100%) rename cognite_toolkit/{cognite_modules/experimental/processing_pipeline/workflows/my.WorkflowVersion.yaml => custom_modules/my_example_module/workflows/first.WorkflowVersion.yaml} (65%) delete mode 100644 tests/tests_unit/test_approval_modules_snapshots/processing_pipeline.yaml delete mode 100644 tests/tests_unit/test_approval_modules_snapshots_clean/processing_pipeline.yaml diff --git a/.vscode/launch.json b/.vscode/launch.json index 3eb41d24a..bd25c0057 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -51,6 +51,19 @@ "console": "integratedTerminal", "justMyCode": false }, + { + "name": "Python: auth verify", + "type": "debugpy", + "request": "launch", + "program": "./cdf-tk-dev.py", + "args": [ + "--override-env", + "auth", + "verify", + ], + "console": "integratedTerminal", + "justMyCode": false + }, { "name": "Python: cdf.py", "type": "debugpy", diff --git a/CHANGELOG.templates.md b/CHANGELOG.templates.md index f18ca6a59..dc1fe8b4c 100644 --- a/CHANGELOG.templates.md +++ b/CHANGELOG.templates.md @@ -30,6 +30,7 @@ Changes are grouped as follows: ### Added - Added Transformation for WorkOrder and WorkItems to OID testdata template +- Added Workflow with a Function and a Transformation to the custom module example ## [0.2.0a2] - 2024-04-03 diff --git a/cognite_toolkit/_cdf_tk/load/_resource_loaders.py b/cognite_toolkit/_cdf_tk/load/_resource_loaders.py index 65081dc8b..dc294e325 100644 --- a/cognite_toolkit/_cdf_tk/load/_resource_loaders.py +++ b/cognite_toolkit/_cdf_tk/load/_resource_loaders.py @@ -1915,7 +1915,7 @@ def drop_data(self, ids: SequenceNotStr[NodeId]) -> int: class WorkflowLoader(ResourceLoader[str, WorkflowUpsert, Workflow, WorkflowUpsertList, WorkflowList]): api_name = "workflows" folder_name = "workflows" - filename_pattern = r"\.Workflow$" + filename_pattern = r"^.*\.Workflow$" resource_cls = Workflow resource_write_cls = WorkflowUpsert list_cls = WorkflowList diff --git a/cognite_toolkit/cognite_modules/experimental/processing_pipeline/default.config.yaml b/cognite_toolkit/cognite_modules/experimental/processing_pipeline/default.config.yaml deleted file mode 100644 index 1df4b28f4..000000000 --- a/cognite_toolkit/cognite_modules/experimental/processing_pipeline/default.config.yaml +++ /dev/null @@ -1 +0,0 @@ -workflow_external_id: \ No newline at end of file diff --git a/cognite_toolkit/custom_modules/my_example_module/default.config.yaml b/cognite_toolkit/custom_modules/my_example_module/default.config.yaml new file mode 100644 index 000000000..9d1326d88 --- /dev/null +++ b/cognite_toolkit/custom_modules/my_example_module/default.config.yaml @@ -0,0 +1,4 @@ + my_example_module: + example_variable: foo + workflow_external_id: baz + example_secret: '${ENVIRONMENT_SECRET}' diff --git a/cognite_toolkit/custom_modules/my_example_module/functions/README.md b/cognite_toolkit/custom_modules/my_example_module/functions/README.md new file mode 100644 index 000000000..af4a612bd --- /dev/null +++ b/cognite_toolkit/custom_modules/my_example_module/functions/README.md @@ -0,0 +1,4 @@ +# cdf_functions_dummy + +This module is an example for how to define a simple function as a template in a tookit module. +The functions can be found below the 'functions' folder, with one folder per function. diff --git a/cognite_toolkit/custom_modules/my_example_module/functions/first.function.schedule.yaml b/cognite_toolkit/custom_modules/my_example_module/functions/first.function.schedule.yaml new file mode 100644 index 000000000..90ab3eccd --- /dev/null +++ b/cognite_toolkit/custom_modules/my_example_module/functions/first.function.schedule.yaml @@ -0,0 +1,14 @@ +- name: "daily-8am-utc" + functionExternalId: 'fn_first_function' + description: "Run every day at 8am UTC" + cronExpression: "0 8 * * *" + data: + breakfast: "today: peanut butter sandwich and coffee" + lunch: "today: greek salad and water" + dinner: "today: steak and red wine" + authentication: + # Credentials to use to run the function in this schedule. + # In this example, we just use the main deploy credentials, so the result is the same, but use a different set of + # credentials (env variables) if you want to run the function with different permissions. + clientId: {{cicd_clientId}} + clientSecret: {{cicd_clientSecret}} diff --git a/cognite_toolkit/custom_modules/my_example_module/functions/first.function.yaml b/cognite_toolkit/custom_modules/my_example_module/functions/first.function.yaml new file mode 100644 index 000000000..62e966c07 --- /dev/null +++ b/cognite_toolkit/custom_modules/my_example_module/functions/first.function.yaml @@ -0,0 +1,23 @@ +# The dir with the function code should have the same name +# and externalId as the function itself as defined below. +- name: 'first:example:function' + externalId: 'fn_first_function' + fileId: + owner: 'Anonymous' + description: 'Returns the input data, secrets, and function info.' + metadata: + version: '1' + secrets: + mysecret: '{{example_secret}}' + envVars: + # The below two environment variables are set by the Toolkit + ENV_TYPE: '${CDF_BUILD_TYPE}' + CDF_ENV: '${CDF_ENVIRON}' + # Number of cores, not available in Azure + #cpu: 0.25 + # Not available in Azure + #memory: 1 + runtime: 'py311' + functionPath: './src/handler.py' + # Data set id for the zip file with the code that is uploaded. + externalDataSetId: 'ds_timeseries_{{example_variable}}' \ No newline at end of file diff --git a/cognite_toolkit/custom_modules/my_example_module/functions/fn_first_function/handler.py b/cognite_toolkit/custom_modules/my_example_module/functions/fn_first_function/handler.py new file mode 100644 index 000000000..075572b81 --- /dev/null +++ b/cognite_toolkit/custom_modules/my_example_module/functions/fn_first_function/handler.py @@ -0,0 +1,26 @@ +from cognite.client import CogniteClient +from cognite.client.data_classes.capabilities import FunctionsAcl + +# You can import from common.tool and get a CDFClientTool instance +# that can be used to run the function locally and verify capabilities. +from common.tool import CDFClientTool + + +def handle(data: dict, client: CogniteClient, secrets: dict, function_call_info: dict) -> dict: + tool = CDFClientTool(client=client) + # This will fail unless the function has the specified capabilities. + tool.verify_capabilities( + [ + FunctionsAcl([FunctionsAcl.Action.Read, FunctionsAcl.Action.Write], FunctionsAcl.Scope.All()), + ] + ) + print(tool) + return { + "data": data, + "secrets": mask_secrets(secrets), + "functionInfo": function_call_info, + } + + +def mask_secrets(secrets: dict) -> dict: + return {k: "***" for k in secrets} diff --git a/cognite_toolkit/custom_modules/my_example_module/functions/fn_first_function/requirements.txt b/cognite_toolkit/custom_modules/my_example_module/functions/fn_first_function/requirements.txt new file mode 100644 index 000000000..88dab18b2 --- /dev/null +++ b/cognite_toolkit/custom_modules/my_example_module/functions/fn_first_function/requirements.txt @@ -0,0 +1 @@ +cognite-sdk>=7.37.0 diff --git a/cognite_toolkit/custom_modules/my_example_module/functions/fn_first_function/src/__init__.py b/cognite_toolkit/custom_modules/my_example_module/functions/fn_first_function/src/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/cognite_toolkit/custom_modules/my_example_module/functions/fn_first_function/src/handler.py b/cognite_toolkit/custom_modules/my_example_module/functions/fn_first_function/src/handler.py new file mode 100644 index 000000000..cd08a0a5c --- /dev/null +++ b/cognite_toolkit/custom_modules/my_example_module/functions/fn_first_function/src/handler.py @@ -0,0 +1,13 @@ +from cognite.client import CogniteClient + + +def handle(data: dict, client: CogniteClient, secrets: dict, function_call_info: dict) -> dict: + return { + "data": data, + "secrets": mask_secrets(secrets), + "functionInfo": function_call_info, + } + + +def mask_secrets(secrets: dict) -> dict: + return {k: "***" for k in secrets} diff --git a/cognite_toolkit/custom_modules/my_example_module/transformations/first.transformation.schedule.yaml b/cognite_toolkit/custom_modules/my_example_module/transformations/first.transformation.schedule.yaml new file mode 100644 index 000000000..3095c7221 --- /dev/null +++ b/cognite_toolkit/custom_modules/my_example_module/transformations/first.transformation.schedule.yaml @@ -0,0 +1,3 @@ +externalId: tr_first_transformation +interval: '{{scheduleHourly}}' +isPaused: true \ No newline at end of file diff --git a/cognite_toolkit/custom_modules/my_example_module/transformations/first.transformation.sql b/cognite_toolkit/custom_modules/my_example_module/transformations/first.transformation.sql new file mode 100644 index 000000000..e69de29bb diff --git a/cognite_toolkit/custom_modules/my_example_module/transformations/first.transformation.yaml b/cognite_toolkit/custom_modules/my_example_module/transformations/first.transformation.yaml new file mode 100644 index 000000000..13f8ded8e --- /dev/null +++ b/cognite_toolkit/custom_modules/my_example_module/transformations/first.transformation.yaml @@ -0,0 +1,9 @@ +externalId: tr_first_transformation +name: 'example:first:transformation' +interval: '{{scheduleHourly}}' +isPaused: true +destination: + type: None +ignoreNullFields: true +isPublic: true +conflictMode: upsert \ No newline at end of file diff --git a/cognite_toolkit/cognite_modules/experimental/processing_pipeline/workflows/my.Workflow.yaml b/cognite_toolkit/custom_modules/my_example_module/workflows/first.Workflow.yaml similarity index 100% rename from cognite_toolkit/cognite_modules/experimental/processing_pipeline/workflows/my.Workflow.yaml rename to cognite_toolkit/custom_modules/my_example_module/workflows/first.Workflow.yaml diff --git a/cognite_toolkit/cognite_modules/experimental/processing_pipeline/workflows/my.WorkflowVersion.yaml b/cognite_toolkit/custom_modules/my_example_module/workflows/first.WorkflowVersion.yaml similarity index 65% rename from cognite_toolkit/cognite_modules/experimental/processing_pipeline/workflows/my.WorkflowVersion.yaml rename to cognite_toolkit/custom_modules/my_example_module/workflows/first.WorkflowVersion.yaml index d1d956fca..798a2582d 100644 --- a/cognite_toolkit/cognite_modules/experimental/processing_pipeline/workflows/my.WorkflowVersion.yaml +++ b/cognite_toolkit/custom_modules/my_example_module/workflows/first.WorkflowVersion.yaml @@ -7,11 +7,11 @@ workflowDefinition: type: 'function' parameters: function: - externalId: 'fn_{{ workflow_external_id }}_function' + externalId: 'fn_first_function' data: {} isAsyncComplete: false - name: task_1 - decription: Task Ipsum lorem dolor sit amet + name: 'Task One' + decription: Task One Ipsum lorem dolor sit amet retries: 3 timeout: 3600 onFailure: 'abortWorkflow' @@ -19,12 +19,12 @@ workflowDefinition: type: 'transformation' parameters: transformation: - externalId: 'tr_{{ workflow_external_id }}_transformation' + externalId: 'tr_first_transformation' concurrencyPolicy: fail - name: task_2 - decription: Task Ipsum lorem dolor sit amet + name: 'Task Two' + decription: Task Two Ipsum lorem dolor sit amet retries: 3 timeout: 3600 onFailure: 'skipTask' dependsOn: - - externalId: 'fn_{{ workflow_external_id }}_function' + - externalId: '{{ workflow_external_id }}_function_task' diff --git a/demo/config.demo.yaml b/demo/config.demo.yaml index c2fa9e384..b71bc9602 100644 --- a/demo/config.demo.yaml +++ b/demo/config.demo.yaml @@ -6,31 +6,36 @@ environment: # You can have multiple deploy commands. # Order is important. selected_modules_and_packages: + - cdf_auth_readwrite_all - cdf_demo_infield - cdf_oid_example_data - cdf_data_pipeline_asset_valhall + - my_example_module + common_function_code: ./common_function_code variables: - cognite_modules: - # Globally defined variables are available to all submodules + + # Globally defined variables are available to all submodules # of the cognite_modules. - cdf_cluster: ${CDF_CLUSTER} - cicd_clientId: ${IDP_CLIENT_ID} - cicd_clientSecret: ${IDP_CLIENT_SECRET} - cicd_tokenUri: ${IDP_TOKEN_URL} - cdfProjectName: ${CDF_PROJECT} - # Optional: If idP requires providing the scopes - cicd_scopes: - - ${IDP_SCOPES} - # Optional: If idP requires providing the audience - cicd_audience: ${IDP_AUDIENCE} - # Seven minutes past each hour - scheduleHourly: 7 * * * * - # Every fifteen minutes - scheduleQuarterly: 0/15 * * * * - # Daily at 1:35 AM - scheduleDaily: 35 1 * * * + cdf_cluster: ${CDF_CLUSTER} + cicd_clientId: ${IDP_CLIENT_ID} + cicd_clientSecret: ${IDP_CLIENT_SECRET} + cicd_tokenUri: ${IDP_TOKEN_URL} + cdfProjectName: ${CDF_PROJECT} + # Optional: If idP requires providing the scopes + cicd_scopes: + - ${IDP_SCOPES} + # Optional: If idP requires providing the audience + cicd_audience: ${IDP_AUDIENCE} + # Seven minutes past each hour + scheduleHourly: 7 * * * * + # Every fifteen minutes + scheduleQuarterly: 0/15 * * * * + # Daily at 1:35 AM + scheduleDaily: 35 1 * * * + + cognite_modules: common: cdf_auth_readwrite_all: readonly_source_id: 7bdcb20c-3e6a-400c-b9ef-cf835f64f05e @@ -232,5 +237,8 @@ variables: custom_modules: + my_example_module: example_variable: demo_dataset # This is the variable that will be used in the module + workflow_external_id: demo_workflow + example_secret: '${ENVIRONMENT_SECRET}' diff --git a/tests/tests_unit/test_approval_modules_snapshots/processing_pipeline.yaml b/tests/tests_unit/test_approval_modules_snapshots/processing_pipeline.yaml deleted file mode 100644 index df0d6bd81..000000000 --- a/tests/tests_unit/test_approval_modules_snapshots/processing_pipeline.yaml +++ /dev/null @@ -1,28 +0,0 @@ -WorkflowVersion: -- version: '1' - workflowDefinition: - description: Execute tasks in sequence - tasks: - - externalId: _function_task - name: task_1 - onFailure: abortWorkflow - parameters: - function: - externalId: fn__function - isAsyncComplete: false - retries: 3 - timeout: 3600 - type: function - - dependsOn: - - externalId: fn__function - externalId: _transformation_task - name: task_2 - onFailure: skipTask - parameters: - transformation: - concurrencyPolicy: fail - externalId: tr__transformation - retries: 3 - timeout: 3600 - type: transformation - workflowExternalId: wf_ diff --git a/tests/tests_unit/test_approval_modules_snapshots_clean/processing_pipeline.yaml b/tests/tests_unit/test_approval_modules_snapshots_clean/processing_pipeline.yaml deleted file mode 100644 index 0967ef424..000000000 --- a/tests/tests_unit/test_approval_modules_snapshots_clean/processing_pipeline.yaml +++ /dev/null @@ -1 +0,0 @@ -{} From bf350895df050bb5fd1aec9cb98c6e9a50afa424 Mon Sep 17 00:00:00 2001 From: Anders Albert <60234212+doctrino@users.noreply.github.com> Date: Tue, 23 Apr 2024 09:21:31 +0200 Subject: [PATCH 26/34] Migration tests updated (#470) * tests; added in migration changes hardcoded * refactor: backwards compatability for containers --- .../_cdf_tk/load/_resource_loaders.py | 21 +++++++++++------ tests_migrations/migrations.py | 23 ++++++++++++++++++- 2 files changed, 36 insertions(+), 8 deletions(-) diff --git a/cognite_toolkit/_cdf_tk/load/_resource_loaders.py b/cognite_toolkit/_cdf_tk/load/_resource_loaders.py index dc294e325..ea63eb9b0 100644 --- a/cognite_toolkit/_cdf_tk/load/_resource_loaders.py +++ b/cognite_toolkit/_cdf_tk/load/_resource_loaders.py @@ -1565,10 +1565,17 @@ def get_id(cls, item: ContainerApply | Container) -> ContainerId: def load_resource( self, filepath: Path, ToolGlobals: CDFToolConfig, skip_validation: bool ) -> ContainerApply | ContainerApplyList | None: - loaded = super().load_resource(filepath, ToolGlobals, skip_validation) - if loaded is None: - return None - items = loaded if isinstance(loaded, ContainerApplyList) else [loaded] + raw_yaml = load_yaml_inject_variables(filepath, ToolGlobals.environment_variables()) + if not isinstance(raw_yaml, list): + raw_yaml = [raw_yaml] + # When upgrading to SDK 7.37.0 there was a breaking change in the SDK requiring 'list' for direct relations. + # This patches the yaml to include the list key for direct relations if it is missing. + for raw_instance in raw_yaml: + for prop in raw_instance.get("properties", {}).values(): + type_ = prop.get("type", {}) + if type_.get("type") == "direct" and "list" not in type_: + type_["list"] = False + items = ContainerApplyList.load(raw_yaml) if not skip_validation: ToolGlobals.verify_spaces(list({item.space for item in items})) for item in items: @@ -1576,11 +1583,11 @@ def load_resource( for prop_name in item.properties.keys(): prop_dumped = item.properties[prop_name].dump() if prop_dumped.get("nullable") is None: - prop_dumped["nullable"] = False + prop_dumped["nullable"] = False # type: ignore[assignment] if prop_dumped.get("autoIncrement") is None: - prop_dumped["autoIncrement"] = False + prop_dumped["autoIncrement"] = False # type: ignore[assignment] item.properties[prop_name] = ContainerProperty.load(prop_dumped) - return loaded + return items def create(self, items: Sequence[ContainerApply]) -> ContainerList: return self.client.data_modeling.containers.apply(items) diff --git a/tests_migrations/migrations.py b/tests_migrations/migrations.py index ceef33376..c9a9d633e 100644 --- a/tests_migrations/migrations.py +++ b/tests_migrations/migrations.py @@ -38,6 +38,10 @@ def modify_environment_to_run_all_modules(project_path: Path) -> None: def get_migration(previous_version_str: str, current_version: str) -> Callable[[Path], None]: previous_version = version.parse(previous_version_str) changes = Changes() + if previous_version < version.parse("0.2.0a3"): + changes.append(_move_system_yaml_to_root) + changes.append(_rename_modules_section_to_variables_in_config_yamls) + if version.parse("0.1.0b7") <= previous_version: changes.append(_update_system_yaml) @@ -63,10 +67,27 @@ def __call__(self, project_path: Path) -> None: change(project_path) -def _update_system_yaml(project_path: Path) -> None: +def _rename_modules_section_to_variables_in_config_yamls(project_path: Path) -> None: + for config_yaml in project_path.glob("config.*.yaml"): + data = yaml.safe_load(config_yaml.read_text()) + if "modules" in data: + data["variables"] = data.pop("modules") + config_yaml.write_text(yaml.dump(data)) + + +def _move_system_yaml_to_root(project_path: Path) -> None: system_yaml = project_path / "cognite_modules" / "_system.yaml" if not system_yaml.exists(): raise FileNotFoundError(f"Could not find _system.yaml in {project_path}") + system_yaml.rename(project_path / "_system.yaml") + + +def _update_system_yaml(project_path: Path) -> None: + old_system_yaml = project_path / "cognite_modules" / "_system.yaml" + new_system_yaml = project_path / "_system.yaml" + if not old_system_yaml.exists() and not new_system_yaml.exists(): + raise FileNotFoundError(f"Could not find _system.yaml in {project_path}") + system_yaml = old_system_yaml if old_system_yaml.exists() else new_system_yaml data = yaml.safe_load(system_yaml.read_text()) data["cdf_toolkit_version"] = __version__ system_yaml.write_text(yaml.dump(data)) From 20e45f4571b6e7926d58e9b626601ed651b32c1f Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Tue, 23 Apr 2024 09:29:14 +0200 Subject: [PATCH 27/34] chore(deps): update pre-commit hook astral-sh/ruff-pre-commit to v0.4.1 (#468) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index cb80002e2..58f9b8224 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -15,7 +15,7 @@ repos: - id: ruff-format args: - --line-length=120 - rev: v0.4.0 + rev: v0.4.1 - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.6.0 From f89b8807c8212689f7970ee90c9a8fa599da760c Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Tue, 23 Apr 2024 11:16:45 +0200 Subject: [PATCH 28/34] chore(deps): update actions/checkout digest to 1d96c77 (#467) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- .github/workflows/build.yml | 6 +++--- .github/workflows/release.yaml | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 7b1d357bc..64c19dbed 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -10,7 +10,7 @@ jobs: name: Run linters runs-on: ubuntu-latest steps: - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 + - uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f # v4 - uses: actions/setup-python@v5 with: python-version: ${{ env.PYTHON_VERSION }} @@ -41,7 +41,7 @@ jobs: # Skipping 3.10 and 3.11 as we assume it is covered by 3.9 and 3.12 - "3.12" steps: - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 + - uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f # v4 - uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} @@ -68,7 +68,7 @@ jobs: IDP_CLIENT_SECRET: ${{ secrets.IDP_CLIENT_SECRET }} IDP_TOKEN_URL: ${{ secrets.IDP_TOKEN_URL }} steps: - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 + - uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f # v4 - uses: actions/setup-python@v5 with: python-version: 3.11 diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 9c54ce3e3..e61eaf071 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -10,7 +10,7 @@ jobs: name: Run linters runs-on: ubuntu-latest steps: - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 + - uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f # v4 - uses: actions/setup-python@v5 with: python-version: ${{ env.PYTHON_VERSION }} @@ -34,7 +34,7 @@ jobs: IDP_CLIENT_SECRET: ${{ secrets.IDP_CLIENT_SECRET }} IDP_TOKEN_URL: ${{ secrets.IDP_TOKEN_URL }} steps: - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 + - uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f # v4 - uses: actions/setup-python@v5 with: python-version: ${{ env.PYTHON_VERSION }} From d5557003d3be31a99c9edb4e1b3532e2fb21754e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?H=C3=A5kon=20V=2E=20Treider?= Date: Tue, 23 Apr 2024 11:21:50 +0200 Subject: [PATCH 29/34] set release in `sentry_sdk.init` (#471) some QOL tweaks for sentry_sdk --- cognite_toolkit/_cdf.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/cognite_toolkit/_cdf.py b/cognite_toolkit/_cdf.py index 6b89c74d8..494bfe5b5 100755 --- a/cognite_toolkit/_cdf.py +++ b/cognite_toolkit/_cdf.py @@ -10,14 +10,12 @@ from pathlib import Path from typing import Annotated, Optional, Union, cast -import sentry_sdk import typer from cognite.client.data_classes.data_modeling import DataModelId, NodeId from dotenv import load_dotenv from rich import print from rich.panel import Panel -from cognite_toolkit import _version from cognite_toolkit._cdf_tk import bootstrap from cognite_toolkit._cdf_tk.constants import _RUNNING_IN_BROWSER from cognite_toolkit._cdf_tk.describe import describe_datamodel @@ -49,8 +47,11 @@ from cognite_toolkit._version import __version__ as current_version if "pytest" not in sys.modules and os.environ.get("SENTRY_ENABLED", "true").lower() == "true": + import sentry_sdk + sentry_sdk.init( dsn="https://ea8b03f98a675ce080056f1583ed9ce7@o124058.ingest.sentry.io/4506429021093888", + release=current_version, # Set traces_sample_rate to 1.0 to capture 100% # of transactions for performance monitoring. traces_sample_rate=1.0, @@ -111,7 +112,7 @@ class Common: def _version_callback(value: bool) -> None: if value: - typer.echo(f"CDF-Toolkit version: {_version.__version__}.") + typer.echo(f"CDF-Toolkit version: {current_version}.") raise typer.Exit() From 9aa155b68b749eecad44b0b32e53bd39534bd60a Mon Sep 17 00:00:00 2001 From: Anders Albert <60234212+doctrino@users.noreply.github.com> Date: Tue, 23 Apr 2024 14:43:04 +0200 Subject: [PATCH 30/34] [CDF-21405] Modules selected by path (#469) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * refactor: switch to full paths in selected modules and packages * refactor: allow string as well * refactor; allow both tuple[str] and str for selected modules * refactor; happy mypy * feat: Support for specifing modules by path * tests: Added failing test for deploying parent module * refactor; extend validation to support parents * feat; Support parent modules * build: changelog * fix: bug when having duplicated module names * refactor: more readable * tests: updated test * build; better language * Printing module as path, not tuple * More helpful hints * Dealing with module potentially being a str * More pretty --------- Co-authored-by: Pål Rønning --- CHANGELOG.cdf-tk.md | 4 ++ .../_cdf_tk/templates/_constants.py | 1 + .../_cdf_tk/templates/_templates.py | 49 ++++++++++++------- .../templates/data_classes/_config_yaml.py | 37 ++++++++++---- .../templates/data_classes/_system_yaml.py | 23 +++++++-- tests/tests_unit/test_behavior.py | 36 ++++++++++++-- .../project_for_test/config.dev.yaml | 3 +- .../config.dev.yaml | 4 +- 8 files changed, 117 insertions(+), 40 deletions(-) diff --git a/CHANGELOG.cdf-tk.md b/CHANGELOG.cdf-tk.md index 1f021cf4b..75088a7f5 100644 --- a/CHANGELOG.cdf-tk.md +++ b/CHANGELOG.cdf-tk.md @@ -20,6 +20,10 @@ Changes are grouped as follows: ### Added - Support for the Workflow and WorkflowVersion resource type +- Support for specifying `selected_modules_and_packages` as paths and parent paths. For example, you can + now write `cognite_modules/core/cdf_apm_base` instead of just `cdf_apm_base`. This is to support + modules that have the same name but are in different parent directories. In addition, this also better reflects + the structure of the `cognite_modules` and `custom_modules` folder better. ### Fixed diff --git a/cognite_toolkit/_cdf_tk/templates/_constants.py b/cognite_toolkit/_cdf_tk/templates/_constants.py index 3842e734c..084622fe6 100644 --- a/cognite_toolkit/_cdf_tk/templates/_constants.py +++ b/cognite_toolkit/_cdf_tk/templates/_constants.py @@ -14,6 +14,7 @@ ALT_CUSTOM_MODULES = "modules" ROOT_MODULES = [COGNITE_MODULES, CUSTOM_MODULES, ALT_CUSTOM_MODULES] +MODULE_PATH_SEP = "/" # Add any other files below that should be included in a build EXCL_FILES = ["README.md", DEFAULT_CONFIG_FILE] diff --git a/cognite_toolkit/_cdf_tk/templates/_templates.py b/cognite_toolkit/_cdf_tk/templates/_templates.py index ab1a873bc..defe25042 100644 --- a/cognite_toolkit/_cdf_tk/templates/_templates.py +++ b/cognite_toolkit/_cdf_tk/templates/_templates.py @@ -22,11 +22,7 @@ from cognite_toolkit._cdf_tk.load import LOADER_BY_FOLDER_NAME, FunctionLoader, Loader, ResourceLoader from cognite_toolkit._cdf_tk.utils import validate_case_raw, validate_data_set_is_set, validate_modules_variables -from ._constants import ( - EXCL_INDEX_SUFFIX, - PROC_TMPL_VARS_SUFFIX, - ROOT_MODULES, -) +from ._constants import EXCL_INDEX_SUFFIX, MODULE_PATH_SEP, PROC_TMPL_VARS_SUFFIX, ROOT_MODULES from ._utils import iterate_functions, iterate_modules, module_from_path, resource_folder_from_path from .data_classes import BuildConfigYAML, SystemYAML @@ -56,18 +52,31 @@ def build_config( config.validate_environment() - module_name_by_path = defaultdict(list) + module_parts_by_name: dict[str, list[tuple[str, ...]]] = defaultdict(list) + available_modules: set[str | tuple[str, ...]] = set() for module, _ in iterate_modules(source_dir): - module_name_by_path[module.name].append(module.relative_to(source_dir)) + available_modules.add(module.name) + module_parts = module.relative_to(source_dir).parts + for i in range(1, len(module_parts) + 1): + available_modules.add(module_parts[:i]) + + module_parts_by_name[module.name].append(module.relative_to(source_dir).parts) + if duplicate_modules := { - module_name: paths for module_name, paths in module_name_by_path.items() if len(paths) > 1 + module_name: paths + for module_name, paths in module_parts_by_name.items() + if len(paths) > 1 and module_name in config.environment.selected_modules_and_packages }: - print(f" [bold red]ERROR:[/] Found the following duplicated module names in {source_dir.name}:") + print(f" [bold red]ERROR:[/] Ambiguous module selected in config.{config.environment.name}.yaml:") for module_name, paths in duplicate_modules.items(): - print(f" {module_name}: {paths}") + locations = "\n ".join([MODULE_PATH_SEP.join(path) for path in paths]) + print(f" {module_name} exists in:\n {locations}") + print( + " You can use the path syntax to disambiguate between modules with the same name. For example " + "'cognite_modules/core/cdf_apm_base' instead of 'cdf_apm_base'." + ) exit(1) - available_modules = set(module_name_by_path.keys()) system_config.validate_modules(available_modules, config.environment.selected_modules_and_packages) selected_modules = config.get_selected_modules(system_config.packages, available_modules, verbose) @@ -423,7 +432,7 @@ def process_files_directory( def process_config_files( project_config_dir: Path, - selected_modules: list[str], + selected_modules: list[str | tuple[str, ...]], build_dir: Path, config: BuildConfigYAML, verbose: bool = False, @@ -439,7 +448,12 @@ def process_config_files( number_by_resource_type: dict[str, int] = defaultdict(int) for module_dir, filepaths in iterate_modules(project_config_dir): - if module_dir.name not in selected_modules: + module_parts = module_dir.relative_to(project_config_dir).parts + is_in_selected_modules = module_dir.name in selected_modules or module_parts in selected_modules + is_parent_in_selected_modules = any( + parent in selected_modules for parent in (module_parts[:i] for i in range(1, len(module_parts))) + ) + if not is_in_selected_modules and not is_parent_in_selected_modules: continue if verbose: print(f" [bold green]INFO:[/] Processing module {module_dir.name}") @@ -612,11 +626,10 @@ def validate(content: str, destination: Path, source_path: Path, modules_by_vari if modules := modules_by_variable.get(variable): module_str = f"{modules[0]!r}" if len(modules) == 1 else (", ".join(modules[:-1]) + f" or {modules[-1]}") print( - f" [bold green]Hint:[/] The variables in 'config.[ENV].yaml' are defined in a tree structure, i.e., " - "variables defined at a higher level can be used in lower levels." - f"\n The variable {variable!r} is defined in the following module{'s' if len(modules) > 1 else ''}: {module_str}." - f"\n It needs to be moved up in the config structure to be used" - f"in {module!r}." + f" [bold green]Hint:[/] The variables in 'config.[ENV].yaml' need to be organised in a tree structure following" + f"\n the folder structure of the template modules, but can also be moved up the config hierarchy to be shared between modules." + f"\n The variable {variable!r} is defined in the variable section{'s' if len(modules) > 1 else ''} {module_str}." + f"\n Check that {'these paths reflect' if len(modules) > 1 else 'this path reflects'} the location of {module}." ) if destination.suffix in {".yaml", ".yml"}: diff --git a/cognite_toolkit/_cdf_tk/templates/data_classes/_config_yaml.py b/cognite_toolkit/_cdf_tk/templates/data_classes/_config_yaml.py index 4df97b1a6..c6fe3982e 100644 --- a/cognite_toolkit/_cdf_tk/templates/data_classes/_config_yaml.py +++ b/cognite_toolkit/_cdf_tk/templates/data_classes/_config_yaml.py @@ -18,6 +18,7 @@ from cognite_toolkit._cdf_tk.templates._constants import ( BUILD_ENVIRONMENT_FILE, DEFAULT_CONFIG_FILE, + MODULE_PATH_SEP, SEARCH_VARIABLES_SUFFIX, ) from cognite_toolkit._cdf_tk.templates._utils import flatten_dict @@ -32,7 +33,7 @@ class Environment: name: str project: str build_type: str - selected_modules_and_packages: list[str] + selected_modules_and_packages: list[str | tuple[str, ...]] common_function_code: str @classmethod @@ -42,7 +43,12 @@ def load(cls, data: dict[str, Any], build_env: str) -> Environment: name=data["name"], project=data["project"], build_type=data["type"], - selected_modules_and_packages=data["selected_modules_and_packages"], + selected_modules_and_packages=[ + tuple([part for part in selected.split(MODULE_PATH_SEP) if part]) + if MODULE_PATH_SEP in selected + else selected + for selected in data["selected_modules_and_packages"] + ], common_function_code=data.get("common_function_code", "./common_function_code"), ) except KeyError: @@ -57,7 +63,10 @@ def dump(self) -> dict[str, Any]: "name": self.name, "project": self.project, "type": self.build_type, - "selected_modules_and_packages": self.selected_modules_and_packages, + "selected_modules_and_packages": [ + MODULE_PATH_SEP.join(selected) if isinstance(selected, tuple) else selected + for selected in self.selected_modules_and_packages + ], "common_function_code": self.common_function_code, } @@ -74,8 +83,8 @@ class BuildConfigYAML(ConfigCore, ConfigYAMLCore): variables: dict[str, Any] @property - def available_modules(self) -> list[str]: - available_modules: list[str] = [] + def available_modules(self) -> list[str | tuple[str, ...]]: + available_modules: list[str | tuple[str, ...]] = [] to_check = [self.variables] while to_check: current = to_check.pop() @@ -140,13 +149,20 @@ def create_build_environment(self) -> BuildEnvironment: ) def get_selected_modules( - self, modules_by_package: dict[str, list[str]], available_modules: set[str], verbose: bool - ) -> list[str]: + self, + modules_by_package: dict[str, list[str | tuple[str, ...]]], + available_modules: set[str | tuple[str, ...]], + verbose: bool, + ) -> list[str | tuple[str, ...]]: selected_packages = [ - package for package in self.environment.selected_modules_and_packages if package in modules_by_package + package + for package in self.environment.selected_modules_and_packages + if package in modules_by_package and isinstance(package, str) ] if verbose: print(" [bold green]INFO:[/] Selected packages:") + if len(selected_packages) == 0: + print(" None") for package in selected_packages: print(f" {package}") @@ -164,7 +180,10 @@ def get_selected_modules( if verbose: print(" [bold green]INFO:[/] Selected modules:") for module in selected_modules: - print(f" {module}") + if isinstance(module, str): + print(f" {module}") + else: + print(f" {MODULE_PATH_SEP.join(module)!s}") if not selected_modules: print( f" [bold yellow]WARNING:[/] Found no defined modules in {self.filepath!s}, have you configured the environment ({self.environment.name})?" diff --git a/cognite_toolkit/_cdf_tk/templates/data_classes/_system_yaml.py b/cognite_toolkit/_cdf_tk/templates/data_classes/_system_yaml.py index 21a942528..426ae64dd 100644 --- a/cognite_toolkit/_cdf_tk/templates/data_classes/_system_yaml.py +++ b/cognite_toolkit/_cdf_tk/templates/data_classes/_system_yaml.py @@ -6,6 +6,7 @@ from rich import print +from cognite_toolkit._cdf_tk.templates._constants import MODULE_PATH_SEP from cognite_toolkit._cdf_tk.templates.data_classes._base import ConfigCore, _load_version_variable @@ -13,7 +14,7 @@ class SystemYAML(ConfigCore): file_name: ClassVar[str] = "_system.yaml" cdf_toolkit_version: str - packages: dict[str, list[str]] = field(default_factory=dict) + packages: dict[str, list[str | tuple[str, ...]]] = field(default_factory=dict) @classmethod def _file_name(cls, build_env: str) -> str: @@ -28,11 +29,25 @@ def load(cls, data: dict[str, Any], build_env: str, filepath: Path) -> SystemYAM return cls( filepath=filepath, cdf_toolkit_version=version, - packages=packages, + packages={ + name: [ + tuple([part for part in entry.split(MODULE_PATH_SEP) if part]) + if MODULE_PATH_SEP in entry + else entry + for entry in package + ] + for name, package in packages.items() + }, ) - def validate_modules(self, available_modules: set[str], selected_modules_and_packages: list[str]) -> None: - selected_packages = {package for package in selected_modules_and_packages if package in self.packages} + def validate_modules( + self, available_modules: set[str | tuple[str, ...]], selected_modules_and_packages: list[str | tuple[str, ...]] + ) -> None: + selected_packages = { + package + for package in selected_modules_and_packages + if package in self.packages and isinstance(package, str) + } for package, modules in self.packages.items(): if package not in selected_packages: # We do not check packages that are not selected. diff --git a/tests/tests_unit/test_behavior.py b/tests/tests_unit/test_behavior.py index 55574a525..4bfe39db8 100644 --- a/tests/tests_unit/test_behavior.py +++ b/tests/tests_unit/test_behavior.py @@ -11,10 +11,10 @@ from cognite_toolkit._cdf import build, deploy, dump_datamodel_cmd, pull_transformation_cmd from cognite_toolkit._cdf_tk.load import TransformationLoader from cognite_toolkit._cdf_tk.templates import build_config -from cognite_toolkit._cdf_tk.templates.data_classes import BuildConfigYAML, SystemYAML +from cognite_toolkit._cdf_tk.templates.data_classes import BuildConfigYAML, Environment, SystemYAML from cognite_toolkit._cdf_tk.utils import CDFToolConfig from tests.tests_unit.approval_client import ApprovalCogniteClient -from tests.tests_unit.test_cdf_tk.constants import CUSTOM_PROJECT, PROJECT_WITH_DUPLICATES +from tests.tests_unit.test_cdf_tk.constants import CUSTOM_PROJECT, PROJECT_WITH_DUPLICATES, PYTEST_PROJECT from tests.tests_unit.utils import PrintCapture, mock_read_yaml_file @@ -61,16 +61,20 @@ def test_inject_custom_environmental_variables( def test_duplicated_modules(local_tmp_path: Path, typer_context: typer.Context, capture_print: PrintCapture) -> None: + config = MagicMock(spec=BuildConfigYAML) + config.environment = MagicMock(spec=Environment) + config.environment.name = "dev" + config.environment.selected_modules_and_packages = ["module1"] with pytest.raises(SystemExit): build_config( build_dir=local_tmp_path, source_dir=PROJECT_WITH_DUPLICATES, - config=MagicMock(spec=BuildConfigYAML), + config=config, system_config=MagicMock(spec=SystemYAML), ) # Check that the error message is printed - assert "module1" in capture_print.messages[-1] - assert "duplicated module names" in capture_print.messages[-2] + assert "module1" in capture_print.messages[-2] + assert "Ambiguous module selected in config.dev.yaml:" in capture_print.messages[-3] def test_pull_transformation( @@ -261,3 +265,25 @@ def test_build_custom_project( extra_resources = actual_resources - expected_resources assert not extra_resources, f"Extra resources: {extra_resources}" + + +def test_build_project_selecting_parent_path( + local_tmp_path, + typer_context, +) -> None: + expected_resources = {"auth", "data_models", "files", "transformations"} + build( + typer_context, + source_dir=str(PYTEST_PROJECT), + build_dir=str(local_tmp_path), + build_env="dev", + no_clean=False, + ) + + actual_resources = {path.name for path in local_tmp_path.iterdir() if path.is_dir()} + + missing_resources = expected_resources - actual_resources + assert not missing_resources, f"Missing resources: {missing_resources}" + + extra_resources = actual_resources - expected_resources + assert not extra_resources, f"Extra resources: {extra_resources}" diff --git a/tests/tests_unit/test_cdf_tk/project_for_test/config.dev.yaml b/tests/tests_unit/test_cdf_tk/project_for_test/config.dev.yaml index abdfc9b6a..db91a0e58 100644 --- a/tests/tests_unit/test_cdf_tk/project_for_test/config.dev.yaml +++ b/tests/tests_unit/test_cdf_tk/project_for_test/config.dev.yaml @@ -3,8 +3,7 @@ environment: project: type: dev selected_modules_and_packages: - - cdf_demo_infield - - cdf_oid_example_data + - cognite_modules/ variables: cognite_modules: diff --git a/tests/tests_unit/test_cdf_tk/project_no_cognite_modules/config.dev.yaml b/tests/tests_unit/test_cdf_tk/project_no_cognite_modules/config.dev.yaml index 3f6f032ff..a3f777f64 100644 --- a/tests/tests_unit/test_cdf_tk/project_no_cognite_modules/config.dev.yaml +++ b/tests/tests_unit/test_cdf_tk/project_no_cognite_modules/config.dev.yaml @@ -3,8 +3,8 @@ environment: project: some-project type: dev selected_modules_and_packages: - - a_module - - another_module + - modules/a_module + - modules/another_module variables: modules: From ea40610a12cc4cf1ee3dfa587a4c645e624edffd Mon Sep 17 00:00:00 2001 From: Anders Albert <60234212+doctrino@users.noreply.github.com> Date: Tue, 23 Apr 2024 14:48:16 +0200 Subject: [PATCH 31/34] Migration: Handle moved system.yaml (#472) refactor: handle moved system yaml --- .../_cdf_tk/templates/data_classes/_base.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/cognite_toolkit/_cdf_tk/templates/data_classes/_base.py b/cognite_toolkit/_cdf_tk/templates/data_classes/_base.py index 615b37c29..a2b03e54c 100644 --- a/cognite_toolkit/_cdf_tk/templates/data_classes/_base.py +++ b/cognite_toolkit/_cdf_tk/templates/data_classes/_base.py @@ -28,7 +28,18 @@ def load_from_directory(cls: type[T_BuildConfig], source_path: Path, build_env: file_name = cls._file_name(build_env) filepath = source_path / file_name filepath = filepath if filepath.is_file() else Path.cwd() / file_name - if not filepath.is_file(): + if ( + (old_filepath := (source_path / "cognite_modules" / file_name)).is_file() + and not filepath.is_file() + and file_name == "_system.yaml" + ): + # This is a fallback for the old location of the system file + print( + f" [bold yellow]Warning:[/] {filepath.name!r} does not exist. " + f"Using 'cognite_toolkit/{old_filepath.name}' instead." + ) + filepath = old_filepath + elif not filepath.is_file(): print(f" [bold red]ERROR:[/] {filepath.name!r} does not exist") exit(1) return cls.load(read_yaml_file(filepath), build_env, filepath) From 9196337cc75f0d8d0cf418ad40bb438c8f21db14 Mon Sep 17 00:00:00 2001 From: Anders Albert <60234212+doctrino@users.noreply.github.com> Date: Tue, 23 Apr 2024 14:59:12 +0200 Subject: [PATCH 32/34] Prepare 0.2.0a3 Release (#473) build: bump --- CHANGELOG.cdf-tk.md | 2 +- CHANGELOG.templates.md | 2 +- cognite_toolkit/_system.yaml | 2 +- cognite_toolkit/_version.py | 2 +- pyproject.toml | 2 +- tests/tests_unit/test_cdf_tk/project_for_test/_system.yaml | 2 +- .../test_cdf_tk/project_no_cognite_modules/_system.yaml | 2 +- tests/tests_unit/test_cdf_tk/run_data/_system.yaml | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.cdf-tk.md b/CHANGELOG.cdf-tk.md index 75088a7f5..b404ed5c4 100644 --- a/CHANGELOG.cdf-tk.md +++ b/CHANGELOG.cdf-tk.md @@ -15,7 +15,7 @@ Changes are grouped as follows: - `Fixed` for any bug fixes. - `Security` in case of vulnerabilities. -## TBD +## [0.2.0a3] - 2024-04-23 ### Added diff --git a/CHANGELOG.templates.md b/CHANGELOG.templates.md index dc1fe8b4c..54647f558 100644 --- a/CHANGELOG.templates.md +++ b/CHANGELOG.templates.md @@ -15,7 +15,7 @@ Changes are grouped as follows: - `Fixed` for any bug fixes. - `Security` in case of vulnerabilities. -## [TBD] - TBD +## [0.2.0a3] - 2024-04-23 ### Fixed diff --git a/cognite_toolkit/_system.yaml b/cognite_toolkit/_system.yaml index 57a02908a..d3adce4e2 100644 --- a/cognite_toolkit/_system.yaml +++ b/cognite_toolkit/_system.yaml @@ -25,4 +25,4 @@ packages: - example_pump_data_model # This part is used by cdf-toolkit to keep track of the version and help you upgrade. -cdf_toolkit_version: 0.2.0a2 +cdf_toolkit_version: 0.2.0a3 diff --git a/cognite_toolkit/_version.py b/cognite_toolkit/_version.py index 9e180df5e..b1b2be8ec 100644 --- a/cognite_toolkit/_version.py +++ b/cognite_toolkit/_version.py @@ -1 +1 @@ -__version__ = "0.2.0a2" +__version__ = "0.2.0a3" diff --git a/pyproject.toml b/pyproject.toml index 1bd55c986..c8fcdb0d0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "cognite_toolkit" -version = "0.2.0a2" +version = "0.2.0a3" description = "Official Cognite Data Fusion tool for project templates and configuration deployment" authors = ["Cognite AS "] license = "Apache-2" diff --git a/tests/tests_unit/test_cdf_tk/project_for_test/_system.yaml b/tests/tests_unit/test_cdf_tk/project_for_test/_system.yaml index cc797839d..e8b9cc6be 100644 --- a/tests/tests_unit/test_cdf_tk/project_for_test/_system.yaml +++ b/tests/tests_unit/test_cdf_tk/project_for_test/_system.yaml @@ -4,4 +4,4 @@ packages: - child_module # This part is used by cdf-toolkit to keep track of the version and help you upgrade. -cdf_toolkit_version: 0.2.0a2 +cdf_toolkit_version: 0.2.0a3 diff --git a/tests/tests_unit/test_cdf_tk/project_no_cognite_modules/_system.yaml b/tests/tests_unit/test_cdf_tk/project_no_cognite_modules/_system.yaml index f3577b854..3bd49fcf8 100644 --- a/tests/tests_unit/test_cdf_tk/project_no_cognite_modules/_system.yaml +++ b/tests/tests_unit/test_cdf_tk/project_no_cognite_modules/_system.yaml @@ -3,4 +3,4 @@ packages: {} # This part is used by cdf-toolkit to keep track of the version and help you upgrade. -cdf_toolkit_version: 0.2.0a2 +cdf_toolkit_version: 0.2.0a3 diff --git a/tests/tests_unit/test_cdf_tk/run_data/_system.yaml b/tests/tests_unit/test_cdf_tk/run_data/_system.yaml index 57a02908a..d3adce4e2 100644 --- a/tests/tests_unit/test_cdf_tk/run_data/_system.yaml +++ b/tests/tests_unit/test_cdf_tk/run_data/_system.yaml @@ -25,4 +25,4 @@ packages: - example_pump_data_model # This part is used by cdf-toolkit to keep track of the version and help you upgrade. -cdf_toolkit_version: 0.2.0a2 +cdf_toolkit_version: 0.2.0a3 From 4a1c773bdc0fee4389e2f9746fe71faf2a4d70e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A5l=20R=C3=B8nning?= Date: Tue, 23 Apr 2024 15:13:15 +0200 Subject: [PATCH 33/34] metadata is dict, not list (#475) --- .../my_example_module/timeseries/timeseries.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cognite_toolkit/custom_modules/my_example_module/timeseries/timeseries.yaml b/cognite_toolkit/custom_modules/my_example_module/timeseries/timeseries.yaml index c93ede30a..df604037a 100644 --- a/cognite_toolkit/custom_modules/my_example_module/timeseries/timeseries.yaml +++ b/cognite_toolkit/custom_modules/my_example_module/timeseries/timeseries.yaml @@ -3,7 +3,7 @@ dataSetExternalId: ds_timeseries_{{example_variable}} isString: false metadata: - - foo: 'bar' + foo: 'bar' isStep: false description: This is an example timeseries used to demonstrate how to create a custom module in the Cognite Data Fusion Toolkit. @@ -11,7 +11,7 @@ name: 'CDF Toolkit Example Timeseries 2' dataSetExternalId: ds_timeseries_{{example_variable}} metadata: - - bar: 'foo' + bar: 'foo' isString: false isStep: false description: This is an example timeseries used to demonstrate how to create a custom module From b3d5af87582649f624f990749e737b416c551ef1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A5l=20R=C3=B8nning?= Date: Tue, 23 Apr 2024 16:19:41 +0200 Subject: [PATCH 34/34] Fix demo ts (#476) * metadata is dict, not list * Fixing deploy for example module * Moving custom example to examples * Syncing tests * hacking tests --- .../examples}/my_example_module/README.md | 0 .../data_sets/data_sets.yaml | 0 .../my_example_module/default.config.yaml | 3 + .../my_example_module/functions/README.md | 0 .../functions/first.function.schedule.yaml | 0 .../functions/first.function.yaml | 0 .../functions/fn_first_function/handler.py | 0 .../fn_first_function/requirements.txt | 0 .../fn_first_function/src/__init__.py | 0 .../fn_first_function/src/handler.py | 0 .../my_example_module/raw/example.yaml | 2 + .../timeseries/timeseries.yaml | 0 .../first.transformation.schedule.yaml | 0 .../transformations/first.transformation.sql | 4 + .../transformations/first.transformation.yaml | 16 +++ .../workflows/first.Workflow.yaml | 0 .../workflows/first.WorkflowVersion.yaml | 0 .../my_example_module/default.config.yaml | 4 - .../transformations/first.transformation.sql | 0 .../transformations/first.transformation.yaml | 9 -- .../my_example_module.yaml | 118 ++++++++++++++++++ .../my_example_module.yaml | 11 ++ 22 files changed, 154 insertions(+), 13 deletions(-) rename cognite_toolkit/{custom_modules => cognite_modules/examples}/my_example_module/README.md (100%) rename cognite_toolkit/{custom_modules => cognite_modules/examples}/my_example_module/data_sets/data_sets.yaml (100%) create mode 100644 cognite_toolkit/cognite_modules/examples/my_example_module/default.config.yaml rename cognite_toolkit/{custom_modules => cognite_modules/examples}/my_example_module/functions/README.md (100%) rename cognite_toolkit/{custom_modules => cognite_modules/examples}/my_example_module/functions/first.function.schedule.yaml (100%) rename cognite_toolkit/{custom_modules => cognite_modules/examples}/my_example_module/functions/first.function.yaml (100%) rename cognite_toolkit/{custom_modules => cognite_modules/examples}/my_example_module/functions/fn_first_function/handler.py (100%) rename cognite_toolkit/{custom_modules => cognite_modules/examples}/my_example_module/functions/fn_first_function/requirements.txt (100%) rename cognite_toolkit/{custom_modules => cognite_modules/examples}/my_example_module/functions/fn_first_function/src/__init__.py (100%) rename cognite_toolkit/{custom_modules => cognite_modules/examples}/my_example_module/functions/fn_first_function/src/handler.py (100%) create mode 100644 cognite_toolkit/cognite_modules/examples/my_example_module/raw/example.yaml rename cognite_toolkit/{custom_modules => cognite_modules/examples}/my_example_module/timeseries/timeseries.yaml (100%) rename cognite_toolkit/{custom_modules => cognite_modules/examples}/my_example_module/transformations/first.transformation.schedule.yaml (100%) create mode 100644 cognite_toolkit/cognite_modules/examples/my_example_module/transformations/first.transformation.sql create mode 100644 cognite_toolkit/cognite_modules/examples/my_example_module/transformations/first.transformation.yaml rename cognite_toolkit/{custom_modules => cognite_modules/examples}/my_example_module/workflows/first.Workflow.yaml (100%) rename cognite_toolkit/{custom_modules => cognite_modules/examples}/my_example_module/workflows/first.WorkflowVersion.yaml (100%) delete mode 100644 cognite_toolkit/custom_modules/my_example_module/default.config.yaml delete mode 100644 cognite_toolkit/custom_modules/my_example_module/transformations/first.transformation.sql delete mode 100644 cognite_toolkit/custom_modules/my_example_module/transformations/first.transformation.yaml create mode 100644 tests/tests_unit/test_approval_modules_snapshots/my_example_module.yaml create mode 100644 tests/tests_unit/test_approval_modules_snapshots_clean/my_example_module.yaml diff --git a/cognite_toolkit/custom_modules/my_example_module/README.md b/cognite_toolkit/cognite_modules/examples/my_example_module/README.md similarity index 100% rename from cognite_toolkit/custom_modules/my_example_module/README.md rename to cognite_toolkit/cognite_modules/examples/my_example_module/README.md diff --git a/cognite_toolkit/custom_modules/my_example_module/data_sets/data_sets.yaml b/cognite_toolkit/cognite_modules/examples/my_example_module/data_sets/data_sets.yaml similarity index 100% rename from cognite_toolkit/custom_modules/my_example_module/data_sets/data_sets.yaml rename to cognite_toolkit/cognite_modules/examples/my_example_module/data_sets/data_sets.yaml diff --git a/cognite_toolkit/cognite_modules/examples/my_example_module/default.config.yaml b/cognite_toolkit/cognite_modules/examples/my_example_module/default.config.yaml new file mode 100644 index 000000000..a3acf3abf --- /dev/null +++ b/cognite_toolkit/cognite_modules/examples/my_example_module/default.config.yaml @@ -0,0 +1,3 @@ +example_variable: foo +workflow_external_id: baz +example_secret: '${ENVIRONMENT_SECRET}' diff --git a/cognite_toolkit/custom_modules/my_example_module/functions/README.md b/cognite_toolkit/cognite_modules/examples/my_example_module/functions/README.md similarity index 100% rename from cognite_toolkit/custom_modules/my_example_module/functions/README.md rename to cognite_toolkit/cognite_modules/examples/my_example_module/functions/README.md diff --git a/cognite_toolkit/custom_modules/my_example_module/functions/first.function.schedule.yaml b/cognite_toolkit/cognite_modules/examples/my_example_module/functions/first.function.schedule.yaml similarity index 100% rename from cognite_toolkit/custom_modules/my_example_module/functions/first.function.schedule.yaml rename to cognite_toolkit/cognite_modules/examples/my_example_module/functions/first.function.schedule.yaml diff --git a/cognite_toolkit/custom_modules/my_example_module/functions/first.function.yaml b/cognite_toolkit/cognite_modules/examples/my_example_module/functions/first.function.yaml similarity index 100% rename from cognite_toolkit/custom_modules/my_example_module/functions/first.function.yaml rename to cognite_toolkit/cognite_modules/examples/my_example_module/functions/first.function.yaml diff --git a/cognite_toolkit/custom_modules/my_example_module/functions/fn_first_function/handler.py b/cognite_toolkit/cognite_modules/examples/my_example_module/functions/fn_first_function/handler.py similarity index 100% rename from cognite_toolkit/custom_modules/my_example_module/functions/fn_first_function/handler.py rename to cognite_toolkit/cognite_modules/examples/my_example_module/functions/fn_first_function/handler.py diff --git a/cognite_toolkit/custom_modules/my_example_module/functions/fn_first_function/requirements.txt b/cognite_toolkit/cognite_modules/examples/my_example_module/functions/fn_first_function/requirements.txt similarity index 100% rename from cognite_toolkit/custom_modules/my_example_module/functions/fn_first_function/requirements.txt rename to cognite_toolkit/cognite_modules/examples/my_example_module/functions/fn_first_function/requirements.txt diff --git a/cognite_toolkit/custom_modules/my_example_module/functions/fn_first_function/src/__init__.py b/cognite_toolkit/cognite_modules/examples/my_example_module/functions/fn_first_function/src/__init__.py similarity index 100% rename from cognite_toolkit/custom_modules/my_example_module/functions/fn_first_function/src/__init__.py rename to cognite_toolkit/cognite_modules/examples/my_example_module/functions/fn_first_function/src/__init__.py diff --git a/cognite_toolkit/custom_modules/my_example_module/functions/fn_first_function/src/handler.py b/cognite_toolkit/cognite_modules/examples/my_example_module/functions/fn_first_function/src/handler.py similarity index 100% rename from cognite_toolkit/custom_modules/my_example_module/functions/fn_first_function/src/handler.py rename to cognite_toolkit/cognite_modules/examples/my_example_module/functions/fn_first_function/src/handler.py diff --git a/cognite_toolkit/cognite_modules/examples/my_example_module/raw/example.yaml b/cognite_toolkit/cognite_modules/examples/my_example_module/raw/example.yaml new file mode 100644 index 000000000..1099423e6 --- /dev/null +++ b/cognite_toolkit/cognite_modules/examples/my_example_module/raw/example.yaml @@ -0,0 +1,2 @@ +dbName: db_{{example_variable}} +tableName: table_{{example_variable}} diff --git a/cognite_toolkit/custom_modules/my_example_module/timeseries/timeseries.yaml b/cognite_toolkit/cognite_modules/examples/my_example_module/timeseries/timeseries.yaml similarity index 100% rename from cognite_toolkit/custom_modules/my_example_module/timeseries/timeseries.yaml rename to cognite_toolkit/cognite_modules/examples/my_example_module/timeseries/timeseries.yaml diff --git a/cognite_toolkit/custom_modules/my_example_module/transformations/first.transformation.schedule.yaml b/cognite_toolkit/cognite_modules/examples/my_example_module/transformations/first.transformation.schedule.yaml similarity index 100% rename from cognite_toolkit/custom_modules/my_example_module/transformations/first.transformation.schedule.yaml rename to cognite_toolkit/cognite_modules/examples/my_example_module/transformations/first.transformation.schedule.yaml diff --git a/cognite_toolkit/cognite_modules/examples/my_example_module/transformations/first.transformation.sql b/cognite_toolkit/cognite_modules/examples/my_example_module/transformations/first.transformation.sql new file mode 100644 index 000000000..69c73b54c --- /dev/null +++ b/cognite_toolkit/cognite_modules/examples/my_example_module/transformations/first.transformation.sql @@ -0,0 +1,4 @@ +select + cast(`externalId` as STRING) as externalId +from + `db_{{example_variable}}`.`table_{{example_variable}}`; diff --git a/cognite_toolkit/cognite_modules/examples/my_example_module/transformations/first.transformation.yaml b/cognite_toolkit/cognite_modules/examples/my_example_module/transformations/first.transformation.yaml new file mode 100644 index 000000000..baa8c11d5 --- /dev/null +++ b/cognite_toolkit/cognite_modules/examples/my_example_module/transformations/first.transformation.yaml @@ -0,0 +1,16 @@ +externalId: tr_first_transformation +name: 'example:first:transformation' +interval: '{{scheduleHourly}}' +isPaused: true +destination: + type: 'assets' +ignoreNullFields: true +isPublic: true +conflictMode: upsert +authentication: + clientId: {{cicd_clientId}} + clientSecret: {{cicd_clientSecret}} + tokenUri: {{cicd_tokenUri}} + cdfProjectName: {{cdfProjectName}} + scopes: {{cicd_scopes}} + audience: {{cicd_audience}} \ No newline at end of file diff --git a/cognite_toolkit/custom_modules/my_example_module/workflows/first.Workflow.yaml b/cognite_toolkit/cognite_modules/examples/my_example_module/workflows/first.Workflow.yaml similarity index 100% rename from cognite_toolkit/custom_modules/my_example_module/workflows/first.Workflow.yaml rename to cognite_toolkit/cognite_modules/examples/my_example_module/workflows/first.Workflow.yaml diff --git a/cognite_toolkit/custom_modules/my_example_module/workflows/first.WorkflowVersion.yaml b/cognite_toolkit/cognite_modules/examples/my_example_module/workflows/first.WorkflowVersion.yaml similarity index 100% rename from cognite_toolkit/custom_modules/my_example_module/workflows/first.WorkflowVersion.yaml rename to cognite_toolkit/cognite_modules/examples/my_example_module/workflows/first.WorkflowVersion.yaml diff --git a/cognite_toolkit/custom_modules/my_example_module/default.config.yaml b/cognite_toolkit/custom_modules/my_example_module/default.config.yaml deleted file mode 100644 index 9d1326d88..000000000 --- a/cognite_toolkit/custom_modules/my_example_module/default.config.yaml +++ /dev/null @@ -1,4 +0,0 @@ - my_example_module: - example_variable: foo - workflow_external_id: baz - example_secret: '${ENVIRONMENT_SECRET}' diff --git a/cognite_toolkit/custom_modules/my_example_module/transformations/first.transformation.sql b/cognite_toolkit/custom_modules/my_example_module/transformations/first.transformation.sql deleted file mode 100644 index e69de29bb..000000000 diff --git a/cognite_toolkit/custom_modules/my_example_module/transformations/first.transformation.yaml b/cognite_toolkit/custom_modules/my_example_module/transformations/first.transformation.yaml deleted file mode 100644 index 13f8ded8e..000000000 --- a/cognite_toolkit/custom_modules/my_example_module/transformations/first.transformation.yaml +++ /dev/null @@ -1,9 +0,0 @@ -externalId: tr_first_transformation -name: 'example:first:transformation' -interval: '{{scheduleHourly}}' -isPaused: true -destination: - type: None -ignoreNullFields: true -isPublic: true -conflictMode: upsert \ No newline at end of file diff --git a/tests/tests_unit/test_approval_modules_snapshots/my_example_module.yaml b/tests/tests_unit/test_approval_modules_snapshots/my_example_module.yaml new file mode 100644 index 000000000..f9978d1f3 --- /dev/null +++ b/tests/tests_unit/test_approval_modules_snapshots/my_example_module.yaml @@ -0,0 +1,118 @@ +DataSet: +- description: This is an example dataset used to demonstrate how to create a custom + module in the Cognite Data Fusion Toolkit. + externalId: ds_timeseries_foo + metadata: {} + name: Example dataset. + writeProtected: false +FileMetadata: +- data_set_id: 42 + external_id: fn_first_function + name: fn_first_function.zip + overwrite: true +Function: +- description: Returns the input data, secrets, and function info. + envVars: + CDF_ENV: dev + ENV_TYPE: dev + externalId: fn_first_function + functionPath: ./src/handler.py + metadata: + cdf-toolkit-function-hash: 239c3468e1a821a4e1c72cacca5d4bfd774017f8aee322c832ff93ec101cfda9 + version: '1' + name: first:example:function + owner: Anonymous + runtime: py311 + secrets: + mysecret: ${ENVIRONMENT_SECRET} +FunctionSchedule: +- cronExpression: 0 8 * * * + data: + breakfast: 'today: peanut butter sandwich and coffee' + dinner: 'today: steak and red wine' + lunch: 'today: greek salad and water' + description: Run every day at 8am UTC + name: daily-8am-utc +TimeSeries: +- dataSetId: 42 + description: This is an example timeseries used to demonstrate how to create a custom + module in the Cognite Data Fusion Toolkit. + externalId: cdf_tooklit:example_timeseries + isStep: false + isString: false + metadata: + foo: bar + name: CDF Toolkit Example Timeseries + securityCategories: [] +- dataSetId: 42 + description: This is an example timeseries used to demonstrate how to create a custom + module in the Cognite Data Fusion Toolkit. + externalId: cdf_tooklit:example_timeseries:2 + isStep: false + isString: false + metadata: + bar: foo + name: CDF Toolkit Example Timeseries 2 + securityCategories: [] +Transformation: +- conflictMode: upsert + destination: + type: assets + destinationOidcCredentials: + audience: https://bluefield.cognitedata.com + cdfProjectName: pytest-project + clientId: dummy + clientSecret: dummy + scopes: https://bluefield.cognitedata.com/.default + tokenUri: dummy + externalId: tr_first_transformation + ignoreNullFields: true + isPublic: true + name: example:first:transformation + query: "select\n cast(`externalId` as STRING) as externalId\nfrom\n `db_foo`.`table_foo`;\n" + sourceOidcCredentials: + audience: https://bluefield.cognitedata.com + cdfProjectName: pytest-project + clientId: dummy + clientSecret: dummy + scopes: https://bluefield.cognitedata.com/.default + tokenUri: dummy +TransformationSchedule: +- externalId: tr_first_transformation + interval: 7 * * * * + isPaused: true +Workflow: +- externalId: wf_baz +WorkflowVersion: +- version: '1' + workflowDefinition: + description: Execute tasks in sequence + tasks: + - externalId: baz_function_task + name: Task One + onFailure: abortWorkflow + parameters: + function: + externalId: fn_first_function + isAsyncComplete: false + retries: 3 + timeout: 3600 + type: function + - dependsOn: + - externalId: baz_function_task + externalId: baz_transformation_task + name: Task Two + onFailure: skipTask + parameters: + transformation: + concurrencyPolicy: fail + externalId: tr_first_transformation + retries: 3 + timeout: 3600 + type: transformation + workflowExternalId: wf_baz +deleted: + Function: + - externalId: fn_first_function + TransformationSchedule: + - externalId: tr_first_transformation diff --git a/tests/tests_unit/test_approval_modules_snapshots_clean/my_example_module.yaml b/tests/tests_unit/test_approval_modules_snapshots_clean/my_example_module.yaml new file mode 100644 index 000000000..d3167c3df --- /dev/null +++ b/tests/tests_unit/test_approval_modules_snapshots_clean/my_example_module.yaml @@ -0,0 +1,11 @@ +deleted: + Database: + - db_name: db_foo + Function: + - externalId: fn_first_function + Table: + - db_name: db_foo + name: + - table_foo + TransformationSchedule: + - externalId: tr_first_transformation