From 10b2b103b4572a46122d68086df909b442ab37d8 Mon Sep 17 00:00:00 2001 From: Megan Davidson Date: Wed, 22 May 2024 13:20:24 +1200 Subject: [PATCH 1/7] feat: new workflow takes a target workaround for historic imagery --- .../raster/publish-odr-historic-imagery.yaml | 105 ++++++++++++++++++ 1 file changed, 105 insertions(+) create mode 100644 workflows/raster/publish-odr-historic-imagery.yaml diff --git a/workflows/raster/publish-odr-historic-imagery.yaml b/workflows/raster/publish-odr-historic-imagery.yaml new file mode 100644 index 000000000..341341178 --- /dev/null +++ b/workflows/raster/publish-odr-historic-imagery.yaml @@ -0,0 +1,105 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/argoproj/argo-workflows/v3.5.5/api/jsonschema/schema.json + +apiVersion: argoproj.io/v1alpha1 +kind: WorkflowTemplate +metadata: + name: publish-odr-historic-imagery + labels: + linz.govt.nz/category: raster + linz.govt.nz/data-type: raster +spec: + parallelism: 50 + nodeSelector: + karpenter.sh/capacity-type: 'spot' + entrypoint: main + synchronization: + semaphore: + configMapKeyRef: + name: semaphores + key: bulkcopy + workflowMetadata: + labelsFrom: + linz.govt.nz/ticket: + expression: workflow.parameters.ticket + linz.govt.nz/region: + expression: workflow.parameters.region + arguments: + parameters: + - name: version_argo_tasks + value: 'v4' + - name: ticket + description: Ticket ID e.g. 'AIP-55' + value: '' + - name: region + description: Region of the dataset + value: 'new-zealand' + enum: + - 'antarctica' + - 'auckland' + - 'bay-of-plenty' + - 'canterbury' + - 'gisborne' + - 'global' + - 'hawkes-bay' + - 'manawatu-whanganui' + - 'marlborough' + - 'nelson' + - 'new-zealand' + - 'northland' + - 'otago' + - 'pacific-islands' + - 'southland' + - 'taranaki' + - 'tasman' + - 'waikato' + - 'wellington' + - 'west-coast' + - name: source + value: 's3://linz-imagery-staging/test/sample/' + - name: target + value: 's3://target/path/' + - name: target_bucket_name + value: '' + enum: + - 'nz-elevation' + - 'nz-imagery' + - '' + - name: copy_option + value: '--no-clobber' + enum: + - '--no-clobber' + - '--force' + - '--force-no-clobber' + templateDefaults: + container: + imagePullPolicy: Always + image: '' + templates: + - name: main + inputs: + parameters: + - name: copy_option + - name: source + - name: target_bucket_name + - name: ticket + - name: target + dag: + tasks: + - name: push-to-github + templateRef: + name: tpl-push-to-github + template: main + arguments: + parameters: + - name: source + value: '{{inputs.parameters.source}}' + - name: target + value: '{{inputs.parameters.target}}' + - name: version_argo_tasks + value: '{{workflow.parameters.version_argo_tasks}}' + - name: repository + value: "{{=sprig.trimPrefix('nz-', inputs.parameters.target_bucket_name)}}" + - name: ticket + value: '{{=sprig.trim(inputs.parameters.ticket)}}' + - name: copy_option + value: '{{inputs.parameters.copy_option}}' From 21eafedfcf7c03f1ad9b5af9c78cac408dfdd156 Mon Sep 17 00:00:00 2001 From: Megan Davidson Date: Wed, 22 May 2024 14:20:43 +1200 Subject: [PATCH 2/7] docs: update readme --- workflows/raster/README.md | 39 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/workflows/raster/README.md b/workflows/raster/README.md index 6eb8522ea..2ddc5b3ad 100644 --- a/workflows/raster/README.md +++ b/workflows/raster/README.md @@ -3,6 +3,7 @@ - [Standardising](#Standardising) - [copy](#copy) - [publish-odr](#Publish-odr) +- [publish-odr-historic-imagery](#Publish-odr-historic-imagery) - [tests](#Tests) # Standardising @@ -278,6 +279,44 @@ graph TD; See the [copy template](#copy) for more information. +# Publish-odr-historic-imagery + +## Workflow Description + +This is a copy of the publish-odr workflow with the addition of a `target` parameter. This workflow allows data managers to bypass the `generate-path` step as it has not yet been implemented for historic imagery. + +> [!IMPORTANT] +> This workflow should only be used when the target path automation is not implemented, for all other instances us `publish-odr`. + +```mermaid +graph TD; + push-to-github; +``` + +## Workflow Input Parameters + +| Parameter | Type | Default | Description | +| ------------------ | ---- | -------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --- | +| ticket | str | | Ticket ID e.g. 'AIP-55' | +| region | enum | | Region of the dataset | +| source | str | s3://linz-imagery-staging/test/sample/ | The URIs (paths) to the s3 source location | +| target | str | s3://target/path/ | the target path where the data will be stored in the ODR | +| target_bucket_name | enum | | The bucket name of the target location | | +| copy_option | enum | --no-clobber |
`--no-clobber`
Skip overwriting existing files.
`--force`
Overwrite all files.
`--force-no-clobber`
Overwrite only changed files, skip unchanged files.
| + +## Examples + +### Publish: + +**source:** `s3://linz-workflow-artifacts/2022-11/15-imagery-standardising-v0.2.0-56-x7699/flat/` + +**target:** `s3://nz-imagery/auckland/auckland_sn1100_1994-1995_0.5m/rgb/2193/` + +**target_bucket_name:** `nz-imagery` + +**copy_option:** `--no-clobber` + +See the [copy template](#copy) for more information. # Tests From 8a7de8b4d2d0e53441208e3b8f17b3360f74443e Mon Sep 17 00:00:00 2001 From: Megan Davidson <33814653+MDavidson17@users.noreply.github.com> Date: Wed, 22 May 2024 14:36:26 +1200 Subject: [PATCH 3/7] Update workflows/raster/README.md Co-authored-by: paulfouquet <86932794+paulfouquet@users.noreply.github.com> --- workflows/raster/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workflows/raster/README.md b/workflows/raster/README.md index 2ddc5b3ad..a0543fab1 100644 --- a/workflows/raster/README.md +++ b/workflows/raster/README.md @@ -286,7 +286,7 @@ See the [copy template](#copy) for more information. This is a copy of the publish-odr workflow with the addition of a `target` parameter. This workflow allows data managers to bypass the `generate-path` step as it has not yet been implemented for historic imagery. > [!IMPORTANT] -> This workflow should only be used when the target path automation is not implemented, for all other instances us `publish-odr`. +> This workflow should only be used when the target path automation is not implemented, for all other instances use `publish-odr`. ```mermaid graph TD; From ad196d4153316dd783f24a9d8d3b748ff369b4a8 Mon Sep 17 00:00:00 2001 From: Megan Davidson Date: Fri, 24 May 2024 09:01:35 +1200 Subject: [PATCH 4/7] fix: remove unnecessary graph --- workflows/raster/README.md | 5 ----- 1 file changed, 5 deletions(-) diff --git a/workflows/raster/README.md b/workflows/raster/README.md index a0543fab1..0cfaa6926 100644 --- a/workflows/raster/README.md +++ b/workflows/raster/README.md @@ -288,11 +288,6 @@ This is a copy of the publish-odr workflow with the addition of a `target` param > [!IMPORTANT] > This workflow should only be used when the target path automation is not implemented, for all other instances use `publish-odr`. -```mermaid -graph TD; - push-to-github; -``` - ## Workflow Input Parameters | Parameter | Type | Default | Description | From ea92b5037ccae1e5f825df312577da07e3ddd951 Mon Sep 17 00:00:00 2001 From: Megan Davidson <33814653+MDavidson17@users.noreply.github.com> Date: Fri, 24 May 2024 09:04:53 +1200 Subject: [PATCH 5/7] docs: fix table in readme --- workflows/raster/README.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/workflows/raster/README.md b/workflows/raster/README.md index 0cfaa6926..c48dd680a 100644 --- a/workflows/raster/README.md +++ b/workflows/raster/README.md @@ -290,14 +290,14 @@ This is a copy of the publish-odr workflow with the addition of a `target` param ## Workflow Input Parameters -| Parameter | Type | Default | Description | -| ------------------ | ---- | -------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --- | -| ticket | str | | Ticket ID e.g. 'AIP-55' | -| region | enum | | Region of the dataset | -| source | str | s3://linz-imagery-staging/test/sample/ | The URIs (paths) to the s3 source location | +| Parameter | Type | Default | Description | +| ------------------ | ----- | ----------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| ticket | str | | Ticket ID e.g. 'AIP-55' | +| region | enum | | Region of the dataset | +| source | str | s3://linz-imagery-staging/test/sample/ | The URIs (paths) to the s3 source location | +| target_bucket_name | enum | | The bucket name of the target location | | +| copy_option | enum | --no-clobber |
`--no-clobber`
Skip overwriting existing files.
`--force`
Overwrite all files.
`--force-no-clobber`
Overwrite only changed files, skip unchanged files.
| | target | str | s3://target/path/ | the target path where the data will be stored in the ODR | -| target_bucket_name | enum | | The bucket name of the target location | | -| copy_option | enum | --no-clobber |
`--no-clobber`
Skip overwriting existing files.
`--force`
Overwrite all files.
`--force-no-clobber`
Overwrite only changed files, skip unchanged files.
| ## Examples From 21f21fd0b8e1def6bce2fd3a1d2aee6925cdc823 Mon Sep 17 00:00:00 2001 From: Megan Davidson Date: Fri, 24 May 2024 10:40:11 +1200 Subject: [PATCH 6/7] docs: fix title --- workflows/raster/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/workflows/raster/README.md b/workflows/raster/README.md index c48dd680a..3dc97f823 100644 --- a/workflows/raster/README.md +++ b/workflows/raster/README.md @@ -3,7 +3,7 @@ - [Standardising](#Standardising) - [copy](#copy) - [publish-odr](#Publish-odr) -- [publish-odr-historic-imagery](#Publish-odr-historic-imagery) +- [Publish ODR Historic Imagery](#Publish-odr-historic-imagery) - [tests](#Tests) # Standardising @@ -279,7 +279,7 @@ graph TD; See the [copy template](#copy) for more information. -# Publish-odr-historic-imagery +# Publish ODR Historic Imagery ## Workflow Description From 39d2ff6517f10574c4aa04d18816e672bb0ba98d Mon Sep 17 00:00:00 2001 From: Megan Davidson <33814653+MDavidson17@users.noreply.github.com> Date: Fri, 24 May 2024 11:07:13 +1200 Subject: [PATCH 7/7] Update workflows/raster/README.md Co-authored-by: Victor Engmark --- workflows/raster/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workflows/raster/README.md b/workflows/raster/README.md index 3dc97f823..f60088a87 100644 --- a/workflows/raster/README.md +++ b/workflows/raster/README.md @@ -283,7 +283,7 @@ See the [copy template](#copy) for more information. ## Workflow Description -This is a copy of the publish-odr workflow with the addition of a `target` parameter. This workflow allows data managers to bypass the `generate-path` step as it has not yet been implemented for historic imagery. +_publish-odr-historic-imagery_ is a copy of the publish-odr workflow with the addition of a `target` parameter. This workflow allows data managers to bypass the `generate-path` step as it has not yet been implemented for historic imagery. > [!IMPORTANT] > This workflow should only be used when the target path automation is not implemented, for all other instances use `publish-odr`.