From 2be90072dca673b31b91262e82a89b19d0c70f17 Mon Sep 17 00:00:00 2001 From: Tony Worm Date: Sun, 26 May 2024 14:08:31 -0400 Subject: [PATCH] more work on getting started & workflow docs --- README.md | 2 +- docs/code/cmd-help/def | 32 ++++++++ docs/code/cmd-help/dev | 7 ++ docs/code/cmd-help/eval | 41 ++++++++++ docs/code/cmd-help/export | 31 +++++++ docs/code/cmd-help/vet | 35 ++++++++ .../getting-started/task-engine/vertex.cue | 80 +++++++++++++++++++ docs/content/getting-started/command-help.md | 75 ++++++++++++++--- .../content/getting-started/hof-attributes.md | 5 +- docs/content/getting-started/task-engine.md | 5 +- docs/content/task-engine/_index.md | 11 ++- docs/hack/make/extern.inc | 4 + 12 files changed, 313 insertions(+), 15 deletions(-) create mode 100644 docs/code/cmd-help/def create mode 100644 docs/code/cmd-help/dev create mode 100644 docs/code/cmd-help/eval create mode 100644 docs/code/cmd-help/export create mode 100644 docs/code/cmd-help/vet create mode 100644 docs/code/getting-started/task-engine/vertex.cue diff --git a/README.md b/README.md index 3ee7d1145..cc3b75c08 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ __`hof` is a CLI tool you *add* to your workflow.__ | Core Features | | |:--- |:-- | | __code generation__ | Data + templates = _ (anything), technology agnostic | -| __data model__ | Define and manage data models, life-cycle features | +| __data modeling__ | Define, checkpoint, and diff data models | | __task engine__ | Extensible task and DAG workflow engine | | __CUE cmds__ | Core def, eval, export, and vet commands | | __creators__ | bootstraping and starter kits from any repo | diff --git a/docs/code/cmd-help/def b/docs/code/cmd-help/def new file mode 100644 index 000000000..a3e715634 --- /dev/null +++ b/docs/code/cmd-help/def @@ -0,0 +1,32 @@ +print consolidated CUE definitions + +Usage: + hof def [flags] + +Flags: + -a, --attributes diplay field attributes + -C, --comments include comments in output + -e, --expression stringArray evaluate these expressions only + -h, --help help for def + --inline-imports expand references to non-core imports + --list concatenate multiple objects into a list + --out string output data format, when detection does not work + -o, --outfile string filename or - for stdout with optional file prefix + --simplify simplify CUE statements where possible + +Global Flags: + -E, --all-errors print all available errors + -i, --ignore-errors turn off output and assume defaults at prompts + -D, --include-data auto include all data files found with cue files + -V, --inject-env inject all ENV VARs as default tag vars + -I, --input stringArray extra data to unify into the root value + -p, --package string the Cue package context to use during execution + -l, --path stringArray CUE expression for single path component when placing data files + -q, --quiet turn off output and assume defaults at prompts + -d, --schema stringArray expression to select schema to apply to data files + --stats print generator statistics + -0, --stdin-empty A flag that ensure stdin is zero and does not block + -t, --tags stringArray @tags() to be injected into CUE code + -U, --user-files stringArray file globs to embed into the root value (=), use % as slash to trim before + -v, --verbosity int set the verbosity of output + --with-context add extra context for data files, usable in the -l/path flag diff --git a/docs/code/cmd-help/dev b/docs/code/cmd-help/dev new file mode 100644 index 000000000..e7dca1d35 --- /dev/null +++ b/docs/code/cmd-help/dev @@ -0,0 +1,7 @@ +unknown command "dev" for "hof" + +Did you mean this? + gen + def + vet + diff --git a/docs/code/cmd-help/eval b/docs/code/cmd-help/eval new file mode 100644 index 000000000..3af6e441d --- /dev/null +++ b/docs/code/cmd-help/eval @@ -0,0 +1,41 @@ +evaluate and print CUE configuration + +Usage: + hof eval [flags] + +Flags: + -a, --all show optional and hidden fields + -A, --attributes display field attributes + -C, --comments include comments in output + -c, --concrete require the evaluation to be concrete + --defaults use default values if not set + -S, --definitions display defintions (default true) + --escape use HTLM escaping + -e, --expression stringArray evaluate these expressions only + --final finalize the value (default true) + -h, --help help for eval + -H, --hidden display hidden fields + --inline-imports expand references to non-core imports + --list concatenate multiple objects into a list + -O, --optional display optional fields + --out string output data format, when detection does not work + -o, --outfile string filename or - for stdout with optional file prefix + --resolve resolve references in value + --simplify simplify CUE statements where possible + +Global Flags: + -E, --all-errors print all available errors + -i, --ignore-errors turn off output and assume defaults at prompts + -D, --include-data auto include all data files found with cue files + -V, --inject-env inject all ENV VARs as default tag vars + -I, --input stringArray extra data to unify into the root value + -p, --package string the Cue package context to use during execution + -l, --path stringArray CUE expression for single path component when placing data files + -q, --quiet turn off output and assume defaults at prompts + -d, --schema stringArray expression to select schema to apply to data files + --stats print generator statistics + -0, --stdin-empty A flag that ensure stdin is zero and does not block + -t, --tags stringArray @tags() to be injected into CUE code + -U, --user-files stringArray file globs to embed into the root value (=), use % as slash to trim before + -v, --verbosity int set the verbosity of output + --with-context add extra context for data files, usable in the -l/path flag diff --git a/docs/code/cmd-help/export b/docs/code/cmd-help/export new file mode 100644 index 000000000..81ac01517 --- /dev/null +++ b/docs/code/cmd-help/export @@ -0,0 +1,31 @@ +output data in a standard format + +Usage: + hof export [flags] + +Flags: + -C, --comments include comments in output + --escape use HTLM escaping + -e, --expression stringArray evaluate these expressions only + -h, --help help for export + --list concatenate multiple objects into a list + --out string output data format, when detection does not work + -o, --outfile string filename or - for stdout with optional file prefix + --simplify simplify CUE statements where possible + +Global Flags: + -E, --all-errors print all available errors + -i, --ignore-errors turn off output and assume defaults at prompts + -D, --include-data auto include all data files found with cue files + -V, --inject-env inject all ENV VARs as default tag vars + -I, --input stringArray extra data to unify into the root value + -p, --package string the Cue package context to use during execution + -l, --path stringArray CUE expression for single path component when placing data files + -q, --quiet turn off output and assume defaults at prompts + -d, --schema stringArray expression to select schema to apply to data files + --stats print generator statistics + -0, --stdin-empty A flag that ensure stdin is zero and does not block + -t, --tags stringArray @tags() to be injected into CUE code + -U, --user-files stringArray file globs to embed into the root value (=), use % as slash to trim before + -v, --verbosity int set the verbosity of output + --with-context add extra context for data files, usable in the -l/path flag diff --git a/docs/code/cmd-help/vet b/docs/code/cmd-help/vet new file mode 100644 index 000000000..56b569b84 --- /dev/null +++ b/docs/code/cmd-help/vet @@ -0,0 +1,35 @@ +validate data with CUE + +Usage: + hof vet [flags] + +Flags: + -A, --attributes display field attributes + -C, --comments include comments in output + -c, --concrete require the evaluation to be concrete + -S, --definitions display defintions (default true) + -e, --expression stringArray evaluate these expressions only + -h, --help help for vet + -H, --hidden display hidden fields + --list concatenate multiple objects into a list + -O, --optional display optional fields + --out string output data format, when detection does not work + -o, --outfile string filename or - for stdout with optional file prefix + --simplify simplify CUE statements where possible + +Global Flags: + -E, --all-errors print all available errors + -i, --ignore-errors turn off output and assume defaults at prompts + -D, --include-data auto include all data files found with cue files + -V, --inject-env inject all ENV VARs as default tag vars + -I, --input stringArray extra data to unify into the root value + -p, --package string the Cue package context to use during execution + -l, --path stringArray CUE expression for single path component when placing data files + -q, --quiet turn off output and assume defaults at prompts + -d, --schema stringArray expression to select schema to apply to data files + --stats print generator statistics + -0, --stdin-empty A flag that ensure stdin is zero and does not block + -t, --tags stringArray @tags() to be injected into CUE code + -U, --user-files stringArray file globs to embed into the root value (=), use % as slash to trim before + -v, --verbosity int set the verbosity of output + --with-context add extra context for data files, usable in the -l/path flag diff --git a/docs/code/getting-started/task-engine/vertex.cue b/docs/code/getting-started/task-engine/vertex.cue new file mode 100644 index 000000000..49c156da7 --- /dev/null +++ b/docs/code/getting-started/task-engine/vertex.cue @@ -0,0 +1,80 @@ +package examples + +import "strings" + +vertex_chat: { + @flow() // define a flow + + steps: { + + // task: get auth from external command + gcp: { + @task(os.Exec) + cmd: ["gcloud", "auth", "print-access-token"] + stdout: string + key: strings.TrimSpace(stdout) + } + + // task: api call via reusable task + call: _gemini & { + apikey: gcp.key + + msg: "What is the CUE language?" + + resp: body: _ + } + + // task: print text to std output + out: { + @task(os.Stdout) + text: call.final.text + } + + } +} + +// reusable task +_gemini: { + @task(api.Call) + + model: string | *"gemini-1.0-pro-002:generateContent" + + msg: string + apikey: string + prompt: string | *"You are a model which is direct and concise when responding." + + req: { + host: "https://us-central1-aiplatform.googleapis.com" + path: "/v1/projects/hof-io--develop/locations/us-central1/publishers/google/models/\(model)" + headers: { + "Content-Type": "application/json" + Authorization: "Bearer \(apikey)" + } + data: { + systemInstruction: { + role: "MODEL" + parts: [{ + text: prompt + }] + } + + contents: [{ + role: "USER" + parts: [{ + text: msg + }] + }] + } + method: "POST" + } + + resp: { + body: _ + } + + // task-local ETL + final: { + cand: resp.body.candidates[0] + text: cand.content.parts[0].text + } +} \ No newline at end of file diff --git a/docs/content/getting-started/command-help.md b/docs/content/getting-started/command-help.md index 36eed1bb7..3a8c82a42 100644 --- a/docs/content/getting-started/command-help.md +++ b/docs/content/getting-started/command-help.md @@ -1,7 +1,7 @@ --- title: "Command Help" description: "Help text for hof's main commands" -brief: "of the hof CLI" +brief: "Help text for hof's main commands" weight: 100 --- @@ -20,29 +20,31 @@ Top level commands and help message -### hof / datamodel +### hof / gen -Used for data model management (dm for short) +Declarative code generation for directories and files from data, CUE, and templates. +Build with adhoc one-liners or use composable generators to create reusable blueprints. -See the [data modeling section](/data-modeling/) for details. +See [the code generation section](/code-generation/) to learn more
-hof help datamodel -{{}} +hof help flow +{{}}
-### hof / gen -Create one-liners to generate files with data, CUE, and templates -or use composable generators to build out advanced applications. +### hof / create -See [the code generation section](/code-generation/) to learn more +Generate boilerplate from any git repository using hof. + +See [the creators section](/code-generation/creators/) to learn more
hof help flow {{}}
+ ### hof / flow Build workflows and scripts with CUE and a DAG engine @@ -55,6 +57,18 @@ See [the task engine section](/task-engine/) to learn more +### hof / datamodel + +Used for data model management (dm for short) + +See the [data modeling section](/data-modeling/) for details. + +
+hof help datamodel +{{}} +
+ + ### hof / mod Manage a MVS (Golang) style modules and dependencies. @@ -75,3 +89,44 @@ hof mod vendor cue {{}} + +### hof / fmt + +Format many languages at once with good defaults. + +See the [formatting](/code-generation/formatting) for details. + +
+hof help fmt +{{}} +
+ + +### hof / eval + +CUE eval embedded in hof. + +
+hof help eval +{{}} +
+ + +### hof / export + +CUE export embedded in hof. + +
+hof help export +{{}} +
+ + +### hof / vet + +CUE vet embedded in hof. + +
+hof help vet +{{}} +
diff --git a/docs/content/getting-started/hof-attributes.md b/docs/content/getting-started/hof-attributes.md index af6ad39f9..710f5033b 100644 --- a/docs/content/getting-started/hof-attributes.md +++ b/docs/content/getting-started/hof-attributes.md @@ -1,5 +1,6 @@ --- title: "#hof & @attributes" +brief: "Special annotations for hof." weight: 70 --- @@ -17,10 +18,10 @@ Hof turns these CUE attributes into `#hof` configuration. - `@gen()` - the root of a generator - `@datamodel()` - the root of a datamodel -- `@flow()` - the root of a workflow or a task type +- `@flow()` - the root of a workflow Datamodels and workflows have a few more attributes -that can be specified under their root. +that can be used under their root. They are covered in the respective sections on each. ### Schema diff --git a/docs/content/getting-started/task-engine.md b/docs/content/getting-started/task-engine.md index ba7e8aff7..17599dd84 100644 --- a/docs/content/getting-started/task-engine.md +++ b/docs/content/getting-started/task-engine.md @@ -23,7 +23,10 @@ Hof's task engine is an extension of cue/flow with {{}} -## Overview +### Example + + +{{}} ### Command diff --git a/docs/content/task-engine/_index.md b/docs/content/task-engine/_index.md index 02e0dc91a..744d44678 100644 --- a/docs/content/task-engine/_index.md +++ b/docs/content/task-engine/_index.md @@ -14,9 +14,18 @@ as they are ready or needed. {{}} +{{< childpages >}} -{{< childpages >}} +### Attributes + +`@flow()`: useful for a single, unnamed workflow + +`@flow()`: write and call one or more flows by name + +`@task()`: a workflow step of type `` + +`@print()`: output values to log for debugging ### Command Help diff --git a/docs/hack/make/extern.inc b/docs/hack/make/extern.inc index 5cc99fd75..afe4c4bf4 100644 --- a/docs/hack/make/extern.inc +++ b/docs/hack/make/extern.inc @@ -17,4 +17,8 @@ cmdhelp: @hof create -h > code/cmd-help/create @hof chat -h > code/cmd-help/chat @hof tui -h > code/cmd-help/tui + @hof eval -h > code/cmd-help/eval + @hof export -h > code/cmd-help/export + @hof vet -h > code/cmd-help/vet + @hof def -h > code/cmd-help/def