Merge branch 'release/1.0.2'

.gitignore

@@ -15,3 +15,12 @@
 /publish
 /VERSION
 /build
+/logs/workflow.json
+/logs/trapper.json
+/logs/sleep.json
+/logs/piper.json
+/logs/false.json
+/logs/fail.json
+/logs/env.json
+/logs/dropcheck.json
+/logs/arger.json

README.md

@@ -160,6 +160,8 @@ The following attributes can be set for the workflow:
 | version  | Workflow format version | `1` |
 | version  | Any metadata for the workflow | None |
 | steps  | List of all workflow steps (See below) | [] |
+| logger | Workflow Logger | Default Logger (see below) |
+| SessionID | Auto generated 8 digit value for each run of the workflow | |
 
 ## Step Attributes
 
@@ -180,6 +182,7 @@ The following attributes can be set for each step:
 | show_command  | Shows the command and arguments for this step before running it | `false` |
 | disabled | Disables the step (doesn't run it). This can be used for debugging or other selective workflow manipulations | `false` |
 | env | Environment variables specific to this step | [] |
+| logger | Step logger | Workflow logger (see below) |
 
 ## Trackman CLI
 
@@ -191,6 +194,9 @@ The CLI supports the following global options:
 |---|---|---|
 | config  | Config file | $HOME/.trackman.yaml |
 | log-level  | Log level | `info` |
+| log-type  | Log Type. Valid options are `stdout`, `stderr`, `discard`, `file` | `stdout` |
+| log-format  | Log Format. Valid options are `text` and `json` | `text` |
+| log-file  | Log File. If `file` is used as `log-type` then this is used as the filename (path can be included) |  |
 | no-update  | Don't update trackman CLI automatically | `false` |
 
 ### Run
@@ -212,6 +218,58 @@ Run command supports the following options
 | concurrency  | Number of concurrent steps to run | Number of CPUs - 1 |
 | yes, y  | Answer Yes to all `ask_to_proceed` questions | false |
 
+### Logging
+
+By default, trackman logs all output to `stdout` and at the `info` level. All logs from all steps are also combined and shown together as they are produced.
+
+You can specify log configuration at the workflow level or for each individual step. If a step has no specific log configuration, it will inherit the configuration of the workflow. Preflight and Probes use the same log configuration as their step.
+
+Log configuration can be defined with the following options:
+
+| Option  | Description  | Default  |
+|---|---|---|
+| type  | Logger Type. Valid options are `stdout`, `stderr`, `discard` and `file` | `stdout` |
+| level  | Log level. Valid values are `error`, `warn`, `info` and `debug` | `info` |
+| format  | Log Format. Valid options are `text` and `json` | `text` |
+| destination  | Log file (can include path). If type is `file` this is used as the file name. If no path is provided, the current directory is used. | |
+
+Here is an example:
+
+```yaml
+version: 1
+logger:
+  type: "file"
+  format: "json"
+  destination: "workflow.json"
+steps:
+  - name: step1
+    logger:
+      type: "stdout"
+  - name: step2
+```
+
+In the example above, the workflow and step2 share a file called `workflow.json` for logging. Step1 however will log text to `stdout`.
+
+You can use any attribute from Workflow and Step in naming your log file. Here is an example:
+
+```yaml
+version: 1
+logger:
+  type: "file"
+  format: "json"
+  destination: "logs/{{ if .Step }}{{.Step.Name}}{{ else }}workflow{{ end }}.json"
+```
+
+The example above, will use `workflow.json` for workflow logs but a file named after the step name for each step. You can use Golang templates for this feature. The template is rendered with a context of `Workflow` and `Step` (in the example above, `.Step` is used)
+
+Another example is to use the Workflow SessionID as log file name:
+
+```yaml
+version: 1
+logger:
+  type: "file"
+  destination: "logs/{{.Workflow.SessionID}}.log"
+```
 
 ### Update
 

cmd/root.go

@@ -29,9 +29,15 @@ func init() {
 	cobra.OnInitialize(initConfig)
 	rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file (default is $HOME/.trackman.yaml)")
 	rootCmd.PersistentFlags().String("log-level", "info", "log level. Use debug to see process output")
+	rootCmd.PersistentFlags().String("log-type", "stdout", "log type. Valid values are stdout, stderr, discard and file")
+	rootCmd.PersistentFlags().String("log-format", "text", "log format. Valid values are text and json")
+	rootCmd.PersistentFlags().String("log-file", "trackman.log", "file path for logs. Only used when log-type is file")
 	rootCmd.PersistentFlags().Bool("no-update", false, "turn off auto update")
 
 	_ = viper.BindPFlag("log-level", rootCmd.PersistentFlags().Lookup("log-level"))
+	_ = viper.BindPFlag("log-file", rootCmd.PersistentFlags().Lookup("log-file"))
+	_ = viper.BindPFlag("log-type", rootCmd.PersistentFlags().Lookup("log-type"))
+	_ = viper.BindPFlag("log-format", rootCmd.PersistentFlags().Lookup("log-format"))
 	_ = viper.BindPFlag("no-update", rootCmd.PersistentFlags().Lookup("no-update"))
 }
 

cmd/run.go

@@ -10,7 +10,6 @@ import (
 
 	"github.com/cloud66-oss/trackman/notifiers"
 	"github.com/cloud66-oss/trackman/utils"
-	"github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 )
@@ -40,14 +39,6 @@ func init() {
 
 func runExec(cmd *cobra.Command, args []string) {
 	ctx := context.Background()
-	level, err := logrus.ParseLevel(viper.GetString("log-level"))
-	if err != nil {
-		fmt.Println(err)
-		os.Exit(1)
-	}
-
-	ctx = context.WithValue(ctx, utils.CtxLogLevel, level)
-	logger, ctx := utils.LoggerContext(ctx)
 
 	options := &utils.WorkflowOptions{
 		Notifier:    notifiers.ConsoleNotify,
@@ -61,6 +52,12 @@ func runExec(cmd *cobra.Command, args []string) {
 		os.Exit(1)
 	}
 
+	logger, err := utils.NewLogger(workflow.Logger, utils.NewLoggingContext(workflow, nil))
+	if err != nil {
+		fmt.Println(err)
+		os.Exit(1)
+	}
+
 	err, stepErrors := workflow.Run(ctx)
 	if err != nil {
 		logger.Error(err)

main.go

@@ -5,10 +5,13 @@ import (
 	"time"
 
 	"github.com/cloud66-oss/trackman/cmd"
+	"github.com/cloud66-oss/trackman/utils"
 )
 
 func main() {
 	defer func() {
+		utils.CloseAllFiles()
+
 		c := make(chan struct{})
 		go func() {
 			defer close(c)

notifiers/console_notifier.go

@@ -4,12 +4,11 @@ import (
 	"context"
 
 	"github.com/cloud66-oss/trackman/utils"
+	"github.com/sirupsen/logrus"
 )
 
 // ConsoleNotify writes notifications to console
-func ConsoleNotify(ctx context.Context, event *utils.Event) error {
-	logger, _ := utils.LoggerContext(ctx)
-
+func ConsoleNotify(ctx context.Context, logger *logrus.Logger, event *utils.Event) error {
 	switch event.Name {
 	case utils.EventRunRequested:
 		logger.WithField(utils.FldStep, event.Payload.Spinner.Name).Info("Starting")

samples/logging.yml

@@ -0,0 +1,54 @@
+version: 1
+metadata:
+  cloud66.com/uuid: ff97e4c6
+  cloud66.com/test: 123
+logger:
+  type: "file"
+  format: "json"
+  destination: "logs/{{ if .Step }}{{.Step.Name}}{{ else }}workflow{{ end }}.json"
+steps:
+  - name: list
+    logger:
+      type: "stdout"
+    command: ls -la
+    metadata:
+      cloud66.com/uuid: 8e2b4a31
+    preflights:
+      - command: true
+        message: "Oh nose!"
+  - name: env
+    command: echo $USER
+  - name: false
+    command: false
+    continue_on_fail: true
+  - name: fail
+    command: fail
+    metadata:
+      cloud66.com/uuid: 8e2b4a3e
+    continue_on_fail: true
+  - name: sleep
+    command: sleep 30
+    timeout: 1s
+    metadata:
+      cloud66.com/uuid: 547c8a3c
+    continue_on_fail: true
+  - name: trapper
+    workdir: "$HOME/work/go/src/github.com/cloud66-oss/trackman/samples"
+    command: ruby trapper.rb
+    timeout: 1s
+    continue_on_fail: true
+  - name: piper
+    workdir: "$HOME/work/go/src/github.com/cloud66-oss/trackman/samples"
+    command: ruby piper.rb
+  - name: dropcheck
+    timeout: 3s
+    workdir: "$HOME/work/go/src/github.com/cloud66-oss/trackman/samples"
+    command: ruby filer.rb
+    probe:
+      command: ruby probe.rb
+    continue_on_fail: true
+  - name: arger
+    metadata:
+      cloud66.com/uuid: arger-123
+    workdir: "$HOME/work/go/src/github.com/cloud66-oss/trackman/samples"
+    command: "ruby arger.rb hello {{ index .MergedMetadata \"cloud66.com/test\" }}"

utils/context.go

@@ -1,47 +1,9 @@
 package utils
 
-import (
-	"context"
-
-	"github.com/sirupsen/logrus"
-)
-
 // CtxKey is a context key
 type CtxKey struct{ int }
 
 var (
 	// CtxSpinner is the key to a spinner on the context
 	CtxSpinner = CtxKey{1}
-	// CtxLogger is the key to a logger on the context
-	CtxLogger = CtxKey{2}
-	// CtxLogLevel holds the desired log level on the context
-	CtxLogLevel = CtxKey{3}
 )
-
-// GetLogger returns a new or existing logger from the context
-func getLogger(ctx context.Context) *logrus.Logger {
-	var logger *logrus.Logger
-	var logLevel logrus.Level
-	if ctx.Value(CtxLogLevel) == nil {
-		logLevel = logrus.DebugLevel
-	} else {
-		logLevel = ctx.Value(CtxLogLevel).(logrus.Level)
-	}
-	if ctx.Value(CtxLogger) == nil {
-		logger = logrus.New()
-		logger.SetLevel(logLevel)
-	} else {
-		logger = ctx.Value(CtxLogger).(*logrus.Logger)
-	}
-
-	return logger
-}
-
-// LoggerContext checks the context for a logger and creates a new one and puts it
-// on the context if not there
-func LoggerContext(ctx context.Context) (*logrus.Logger, context.Context) {
-	logger := getLogger(ctx)
-	ctx = context.WithValue(ctx, CtxLogger, logger)
-
-	return logger, ctx
-}