docs: argo workflows workshop TDE-700 (#120)

* wip: draft docs * fix: remove unneeded image * docs: wip training course * docs: training wip * docs: training wip * docs: training wip * docs: training wip * docs: parallelism and outputs example * docs: monitoring running workflows in argo workflows * docs: fix formatting * docs: add examples DAG and outputs * docs: check examples and diagrams * docs: tidy up * fix: typo --------- Co-authored-by: paul <[email protected]>
linz · Mar 30, 2023 · 71b6876 · 71b6876
1 parent 3c9a798
commit 71b6876
Show file tree

Hide file tree

Showing 11 changed files with 715 additions and 0 deletions.
diff --git a/docs/training/ARGO_WORKFLOWS.md b/docs/training/ARGO_WORKFLOWS.md
diff --git a/docs/training/argo_ui_show_logs.png b/docs/training/argo_ui_show_logs.png
diff --git a/docs/training/pods.png b/docs/training/pods.png
diff --git a/docs/training/standardising_structure.png b/docs/training/standardising_structure.png
diff --git a/docs/training/wf_helloworld.yaml b/docs/training/wf_helloworld.yaml
@@ -0,0 +1,19 @@
+---
+apiVersion: argoproj.io/v1alpha1
+kind: Workflow
+metadata:
+  generateName: test-name-hello-world-
+  namespace: argo
+spec:
+  entrypoint: main
+  templates:
+    - name: main
+      dag:
+        tasks:
+          - name: say-hello-step
+            template: say-hello-template
+    - name: say-hello-template
+      container:
+        image: "019359803926.dkr.ecr.ap-southeast-2.amazonaws.com/eks:topo-imagery-v1"
+        command: [echo]
+        args: ["hello world"]
diff --git a/docs/training/wf_helloworld_args.yaml b/docs/training/wf_helloworld_args.yaml
@@ -0,0 +1,30 @@
+---
+apiVersion: argoproj.io/v1alpha1
+kind: Workflow
+metadata:
+  generateName: test-name-hello-world-args-
+  namespace: argo
+spec:
+  entrypoint: main
+  arguments:
+    parameters:
+      - name: message
+        value: "hello world"
+  templates:
+    - name: main
+      dag:
+        tasks:
+          - name: say-hello-task
+            template: say-hello-template
+            arguments:
+              parameters:
+                - name: message
+                  value: "{{workflow.parameters.message}}"
+    - name: say-hello-template
+      inputs:
+        parameters:
+          - name: message
+      container:
+        image: "019359803926.dkr.ecr.ap-southeast-2.amazonaws.com/eks:topo-imagery-v1"
+        command: [echo]
+        args: ["{{inputs.parameters.message}}"]
diff --git a/docs/training/wf_helloworld_args_tasks.yaml b/docs/training/wf_helloworld_args_tasks.yaml
@@ -0,0 +1,38 @@
+---
+apiVersion: argoproj.io/v1alpha1
+kind: Workflow
+metadata:
+  generateName: test-name-hello-world-
+  namespace: argo
+spec:
+  entrypoint: main
+  arguments:
+    parameters:
+      - name: message1
+        value: "hello world 1"
+      - name: message2
+        value: "hello world 2"
+  templates:
+    - name: main
+      dag:
+        tasks:
+          - name: say-hello-task1
+            template: say-hello-template
+            arguments:
+              parameters:
+                - name: message
+                  value: "{{workflow.parameters.message1}}"
+          - name: say-hello-task2
+            template: say-hello-template
+            arguments:
+              parameters:
+                - name: message
+                  value: "{{workflow.parameters.message2}}"
+    - name: say-hello-template
+      inputs:
+        parameters:
+          - name: message
+      container:
+        image: "019359803926.dkr.ecr.ap-southeast-2.amazonaws.com/eks:topo-imagery-v1"
+        command: [echo]
+        args: ["{{inputs.parameters.message}}"]
diff --git a/docs/training/wf_helloworld_dag.yaml b/docs/training/wf_helloworld_dag.yaml
@@ -0,0 +1,53 @@
+---
+apiVersion: argoproj.io/v1alpha1
+kind: Workflow
+metadata:
+  generateName: test-name-hello-world-dag-
+  namespace: argo
+spec:
+  entrypoint: main
+  arguments:
+    parameters:
+      - name: message1
+        value: "hello world 1"
+      - name: message2
+        value: "hello world 2"
+  templates:
+    - name: main
+      dag:
+        tasks:
+          - name: say-hello-task1
+            template: say-hello-template
+            arguments:
+              parameters:
+                - name: message
+                  value: "{{workflow.parameters.message1}}"
+          - name: say-hello-task2
+            template: say-hello-template
+            arguments:
+              parameters:
+                - name: message
+                  value: "{{workflow.parameters.message2}}"
+            depends: "say-hello-task1"
+          - name: say-hello-task3
+            template: say-hello-template
+            arguments:
+              parameters:
+                - name: message
+                  value: "say hello task 3"
+            depends: "say-hello-task1"
+          - name: say-hello-task4
+            template: say-hello-template
+            arguments:
+              parameters:
+                - name: message
+                  value: "say hello task 4"
+            depends: "say-hello-task2 && say-hello-task3"
+    - name: say-hello-template
+      inputs:
+        parameters:
+          - name: message
+      container:
+        image: "019359803926.dkr.ecr.ap-southeast-2.amazonaws.com/eks:topo-imagery-v1"
+        command: [echo]
+        args: ["{{inputs.parameters.message}}"]
diff --git a/docs/training/wf_output_parallel.yaml b/docs/training/wf_output_parallel.yaml
@@ -0,0 +1,80 @@
+---
+apiVersion: argoproj.io/v1alpha1
+kind: Workflow
+metadata:
+  generateName: test-output-parallel-
+spec:
+  parallelism: 20
+  nodeSelector:
+    karpenter.sh/capacity-type: "spot"
+  serviceAccountName: workflow-runner-sa
+  entrypoint: main
+  arguments:
+    parameters:
+      - name: uri
+        value: "s3://linz-imagery-staging/test/stac-validate/"
+  templateDefaults:
+    container:
+      imagePullPolicy: Always
+  templates:
+    - name: main
+      dag:
+        tasks:
+          - name: aws-list
+            template: aws-list
+            arguments:
+              parameters:
+                - name: uri
+                  value: "{{workflow.parameters.uri}}"
+                - name: include
+                  value: "json$"
+          - name: stac-print-path
+            template: stac-print-path
+            arguments:
+              parameters:
+                - name: file
+                  value: "{{item}}"
+            depends: "aws-list"
+            withParam: "{{tasks.aws-list.outputs.parameters.files}}"
+    - name: aws-list
+      inputs:
+        parameters:
+          - name: uri
+          - name: include
+      container:
+        image: "019359803926.dkr.ecr.ap-southeast-2.amazonaws.com/eks:argo-tasks-v2"
+        command: [node, /app/index.js]
+        env:
+          - name: AWS_ROLE_CONFIG_PATH
+            value: s3://linz-bucket-config/config.json
+        args:
+          [
+            "list",
+            "--verbose",
+            "--include",
+            "{{inputs.parameters.include}}",
+            "--group",
+            "4",
+            "--output",
+            "/tmp/file_list.json",
+            "{{inputs.parameters.uri}}",
+          ]
+      outputs:
+        parameters:
+          - name: files
+            valueFrom:
+              path: /tmp/file_list.json
+    - name: stac-print-path
+      inputs:
+        parameters:
+          - name: file
+      script:
+        image: "019359803926.dkr.ecr.ap-southeast-2.amazonaws.com/eks:argo-tasks-v2"
+        env:
+          - name: AWS_ROLE_CONFIG_PATH
+            value: s3://linz-bucket-config/config.json
+        command:
+          - "bash"
+        source: |
+          PATH_OUT=$(echo "{{inputs.parameters.file}}" | sed 's/,/ /g; s/\[/ /g; s/\]/ /g')
+          echo $PATH_OUT
diff --git a/docs/training/wf_structure.png b/docs/training/wf_structure.png
diff --git a/docs/training/workflow_ui.png b/docs/training/workflow_ui.png