diff --git a/workflows/raster/README.md b/workflows/raster/README.md index 6eb8522ea..f60088a87 100644 --- a/workflows/raster/README.md +++ b/workflows/raster/README.md @@ -3,6 +3,7 @@ - [Standardising](#Standardising) - [copy](#copy) - [publish-odr](#Publish-odr) +- [Publish ODR Historic Imagery](#Publish-odr-historic-imagery) - [tests](#Tests) # Standardising @@ -278,6 +279,39 @@ graph TD; See the [copy template](#copy) for more information. +# Publish ODR Historic Imagery + +## Workflow Description + +_publish-odr-historic-imagery_ is a copy of the publish-odr workflow with the addition of a `target` parameter. This workflow allows data managers to bypass the `generate-path` step as it has not yet been implemented for historic imagery. + +> [!IMPORTANT] +> This workflow should only be used when the target path automation is not implemented, for all other instances use `publish-odr`. + +## Workflow Input Parameters + +| Parameter | Type | Default | Description | +| ------------------ | ----- | ----------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| ticket | str | | Ticket ID e.g. 'AIP-55' | +| region | enum | | Region of the dataset | +| source | str | s3://linz-imagery-staging/test/sample/ | The URIs (paths) to the s3 source location | +| target_bucket_name | enum | | The bucket name of the target location | | +| copy_option | enum | --no-clobber |
`--no-clobber`
Skip overwriting existing files.
`--force`
Overwrite all files.
`--force-no-clobber`
Overwrite only changed files, skip unchanged files.
| +| target | str | s3://target/path/ | the target path where the data will be stored in the ODR | + +## Examples + +### Publish: + +**source:** `s3://linz-workflow-artifacts/2022-11/15-imagery-standardising-v0.2.0-56-x7699/flat/` + +**target:** `s3://nz-imagery/auckland/auckland_sn1100_1994-1995_0.5m/rgb/2193/` + +**target_bucket_name:** `nz-imagery` + +**copy_option:** `--no-clobber` + +See the [copy template](#copy) for more information. # Tests diff --git a/workflows/raster/publish-odr-historic-imagery.yaml b/workflows/raster/publish-odr-historic-imagery.yaml new file mode 100644 index 000000000..341341178 --- /dev/null +++ b/workflows/raster/publish-odr-historic-imagery.yaml @@ -0,0 +1,105 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/argoproj/argo-workflows/v3.5.5/api/jsonschema/schema.json + +apiVersion: argoproj.io/v1alpha1 +kind: WorkflowTemplate +metadata: + name: publish-odr-historic-imagery + labels: + linz.govt.nz/category: raster + linz.govt.nz/data-type: raster +spec: + parallelism: 50 + nodeSelector: + karpenter.sh/capacity-type: 'spot' + entrypoint: main + synchronization: + semaphore: + configMapKeyRef: + name: semaphores + key: bulkcopy + workflowMetadata: + labelsFrom: + linz.govt.nz/ticket: + expression: workflow.parameters.ticket + linz.govt.nz/region: + expression: workflow.parameters.region + arguments: + parameters: + - name: version_argo_tasks + value: 'v4' + - name: ticket + description: Ticket ID e.g. 'AIP-55' + value: '' + - name: region + description: Region of the dataset + value: 'new-zealand' + enum: + - 'antarctica' + - 'auckland' + - 'bay-of-plenty' + - 'canterbury' + - 'gisborne' + - 'global' + - 'hawkes-bay' + - 'manawatu-whanganui' + - 'marlborough' + - 'nelson' + - 'new-zealand' + - 'northland' + - 'otago' + - 'pacific-islands' + - 'southland' + - 'taranaki' + - 'tasman' + - 'waikato' + - 'wellington' + - 'west-coast' + - name: source + value: 's3://linz-imagery-staging/test/sample/' + - name: target + value: 's3://target/path/' + - name: target_bucket_name + value: '' + enum: + - 'nz-elevation' + - 'nz-imagery' + - '' + - name: copy_option + value: '--no-clobber' + enum: + - '--no-clobber' + - '--force' + - '--force-no-clobber' + templateDefaults: + container: + imagePullPolicy: Always + image: '' + templates: + - name: main + inputs: + parameters: + - name: copy_option + - name: source + - name: target_bucket_name + - name: ticket + - name: target + dag: + tasks: + - name: push-to-github + templateRef: + name: tpl-push-to-github + template: main + arguments: + parameters: + - name: source + value: '{{inputs.parameters.source}}' + - name: target + value: '{{inputs.parameters.target}}' + - name: version_argo_tasks + value: '{{workflow.parameters.version_argo_tasks}}' + - name: repository + value: "{{=sprig.trimPrefix('nz-', inputs.parameters.target_bucket_name)}}" + - name: ticket + value: '{{=sprig.trim(inputs.parameters.ticket)}}' + - name: copy_option + value: '{{inputs.parameters.copy_option}}'