fix before/after insert into doc

harden update command

.env.example

@@ -1,12 +1,11 @@
 #!/bin/bash
 # -*- mode: bash -*-
 # vi: ft=bash
-
+# shellcheck disable=SC2034,SC2148,SC2016
+#
 # Use Dottie (https://github.com/jippi/dottie) to manage this .env file easier!
 #
 # @dottie/source .env.example
-#
-# shellcheck disable=SC2034,SC2148
 
 ################################################################################
 # app
@@ -30,7 +29,7 @@ APP_DOMAIN="example.com"
 # @dottie/validate required,http_url
 APP_URL="https://${APP_DOMAIN}"
 
-# Application domains used for routing.
+# Application domains used for routing
 #
 # @see https://docs.pixelfed.org/technical-documentation/config/#admin_domain
 # @dottie/validate required,fqdn

cmd/set/set.go

@@ -24,80 +24,7 @@ func Command() *cobra.Command {
 			WithSuffixIsLiteral(true).
 			WithHandlers(render.ExcludeDisabledAssignments).
 			Get(),
-		RunE: func(cmd *cobra.Command, args []string) error {
-			filename := cmd.Flag("file").Value.String()
-
-			env, err := pkg.Load(filename)
-			if err != nil {
-				return err
-			}
-
-			if len(args) == 0 {
-				return errors.New("Missing required argument: KEY=VALUE")
-			}
-
-			comments, _ := cmd.Flags().GetStringArray("comment")
-			options := ast.UpsertOptions{
-				InsertBefore:   shared.StringFlag(cmd.Flags(), "before"),
-				Comments:       comments,
-				ErrorIfMissing: shared.BoolFlag(cmd.Flags(), "error-if-missing"),
-				Group:          shared.StringFlag(cmd.Flags(), "group"),
-				SkipValidation: !shared.BoolWithInverseValue(cmd.Flags(), "validate"),
-			}
-
-			for _, stringPair := range args {
-				pairSlice := strings.SplitN(stringPair, "=", 2)
-				if len(pairSlice) != 2 {
-					return errors.New("expected KEY=VALUE pair, missing '='")
-				}
-
-				key := pairSlice[0]
-				value := pairSlice[1]
-
-				assignment := &ast.Assignment{
-					Name:    key,
-					Literal: value,
-					// by default we take the user input and assume its interpolated,
-					// it will be interpolated inside (*Document).Set if applicable
-					Interpolated: value,
-					Active:       !shared.BoolFlag(cmd.Flags(), "disabled"),
-					Quote:        token.QuoteFromString(shared.StringFlag(cmd.Flags(), "quote-style")),
-				}
-
-				//
-				// Upsert key
-				//
-
-				assignment, err := env.Upsert(assignment, options)
-				if err != nil {
-					fmt.Fprintln(os.Stderr, validation.Explain(env, validation.NewError(assignment, err), false, true))
-
-					return fmt.Errorf("failed to upsert the key/value pair [%s]", key)
-				}
-
-				if validationErrors := validation.ValidateSingleAssignment(env, assignment.Name, nil, nil); len(validationErrors) > 0 {
-					for _, errIsh := range validationErrors {
-						fmt.Fprintln(os.Stderr, validation.Explain(env, errIsh, false, false))
-					}
-
-					return errors.New("validation failed")
-				}
-
-				tui.Theme.Success.StderrPrinter().Printfln("Key [%s] was successfully upserted", key)
-			}
-
-			//
-			// Save file
-			//
-
-			if err := pkg.Save(shared.StringFlag(cmd.Flags(), "file"), env); err != nil {
-				return fmt.Errorf("failed to save file: %w", err)
-			}
-
-			tui.Theme.Success.StderrPrinter().Println("File was successfully saved")
-
-			return nil
-		},
+		RunE: runE,
 	}
 
 	shared.BoolWithInverse(cmd, "validate", true, "Validate the VALUE input before saving the file", "Do not validate the VALUE input before saving the file")
@@ -110,5 +37,112 @@ func Command() *cobra.Command {
 	cmd.Flags().String("quote-style", "double", "The quote style to use (single, double, none)")
 	cmd.Flags().StringSlice("comment", nil, "Set one or multiple lines of comments to the KEY=VALUE pair")
 
+	cmd.MarkFlagsMutuallyExclusive("before", "after", "group")
+
 	return cmd
 }
+
+func runE(cmd *cobra.Command, args []string) error {
+	filename := cmd.Flag("file").Value.String()
+
+	env, err := pkg.Load(filename)
+	if err != nil {
+		return err
+	}
+
+	if len(args) == 0 {
+		return errors.New("Missing required argument: KEY=VALUE")
+	}
+
+	comments, _ := cmd.Flags().GetStringArray("comment")
+
+	options := ast.UpsertOptions{
+		UpsertPlacementType: ast.UpsertLast,
+		Comments:            comments,
+		ErrorIfMissing:      shared.BoolFlag(cmd.Flags(), "error-if-missing"),
+		Group:               shared.StringFlag(cmd.Flags(), "group"),
+		SkipValidation:      !shared.BoolWithInverseValue(cmd.Flags(), "validate"),
+	}
+
+	// If we want placement *BEFORE* another statement
+	if before, _ := cmd.Flags().GetString("before"); len(before) > 0 {
+		other := env.Get(before)
+		if other == nil {
+			return fmt.Errorf("The key [%s] does not exists in [%s]. Can't be used in [--before]", before, filename)
+		}
+
+		options.UpsertPlacementType = ast.UpsertBefore
+		options.UpsertPlacementValue = before
+
+		if other.Group != nil {
+			options.Group = other.Group.String()
+		}
+	}
+
+	// If we want placement *AFTER* another statement
+	if after, _ := cmd.Flags().GetString("after"); len(after) > 0 {
+		other := env.Get(after)
+		if other == nil {
+			return fmt.Errorf("The key [%s] does not exists in [%s]. Can't be used in [--after]", after, filename)
+		}
+
+		options.UpsertPlacementType = ast.UpsertAfter
+		options.UpsertPlacementValue = after
+
+		if other.Group != nil {
+			options.Group = other.Group.String()
+		}
+	}
+
+	// Loop arguments and place them
+	for _, stringPair := range args {
+		pairSlice := strings.SplitN(stringPair, "=", 2)
+		if len(pairSlice) != 2 {
+			return errors.New("expected KEY=VALUE pair, missing '='")
+		}
+
+		key := pairSlice[0]
+		value := pairSlice[1]
+
+		assignment := &ast.Assignment{
+			Name:         key,
+			Literal:      value,
+			Interpolated: value,
+			Active:       !shared.BoolFlag(cmd.Flags(), "disabled"),
+			Quote:        token.QuoteFromString(shared.StringFlag(cmd.Flags(), "quote-style")),
+		}
+
+		//
+		// Upsert key
+		//
+
+		assignment, err := env.Upsert(assignment, options)
+		if err != nil {
+			fmt.Fprintln(os.Stderr, validation.Explain(env, validation.NewError(assignment, err), false, true))
+
+			return fmt.Errorf("failed to upsert the key/value pair [%s]", key)
+		}
+
+		if validationErrors := validation.ValidateSingleAssignment(env, assignment.Name, nil, nil); len(validationErrors) > 0 {
+			for _, errIsh := range validationErrors {
+				fmt.Fprintln(os.Stderr, validation.Explain(env, errIsh, false, false))
+			}
+
+			return errors.New("validation failed")
+		}
+
+		tui.Theme.Success.StderrPrinter().Printfln("Key [%s] was successfully upserted", key)
+	}
+
+	//
+	// Save file
+	//
+
+	if err := pkg.Save(shared.StringFlag(cmd.Flags(), "file"), env); err != nil {
+		return fmt.Errorf("failed to save file: %w", err)
+	}
+
+	tui.Theme.Success.StderrPrinter().Println("File was successfully saved")
+
+	return nil
+}

cmd/update/update.go

@@ -10,6 +10,7 @@ import (
 	"github.com/hashicorp/go-getter"
 	"github.com/jippi/dottie/pkg"
 	"github.com/jippi/dottie/pkg/ast"
+	"github.com/jippi/dottie/pkg/cli/shared"
 	"github.com/jippi/dottie/pkg/tui"
 	"github.com/jippi/dottie/pkg/validation"
 	"github.com/spf13/cobra"
@@ -23,6 +24,7 @@ func Command() *cobra.Command {
 	}
 
 	cmd.Flags().String("source", "", "URL or local file path to the upstream source file. This will take precedence over any [@dottie/source] annotation in the file")
+	shared.BoolWithInverse(cmd, "error-on-missing-key", true, "Error if a KEY in FILE is missing from SOURCE", "Add KEY to FILE if missing from SOURCE")
 
 	return cmd
 }
@@ -94,7 +96,7 @@ func runE(cmd *cobra.Command, args []string) error {
 	// Load the soon-to-be-merged file
 	dark.Println("Loading and parsing source")
 
-	mergedEnv, err := pkg.Load(tmp.Name())
+	sourceDoc, err := pkg.Load(tmp.Name())
 	if err != nil {
 		return err
 	}
@@ -108,15 +110,83 @@ func runE(cmd *cobra.Command, args []string) error {
 
 	sawError := false
 	lastWasError := false
+	counter := 0
 
-	for _, stmt := range env.Assignments() {
+	for _, stmt := range env.AllAssignments() {
 		if !stmt.Active {
 			continue
 		}
 
-		changed, err := mergedEnv.Upsert(stmt, ast.UpsertOptions{SkipIfSame: true, ErrorIfMissing: true})
+		options := ast.UpsertOptions{
+			SkipIfSame:     true,
+			ErrorIfMissing: shared.BoolWithInverseValue(cmd.Flags(), "error-on-missing-key"),
+		}
+
+		// If the KEY does not exists in the SOURCE doc
+		if sourceDoc.Get(stmt.Name) == nil {
+			// Copy comments if the KEY doesn't exist in the SOURCE document
+			options.Comments = stmt.CommentsSlice()
+
+			// Try to find positioning in the statement list for the new KEY pair
+			var parent ast.StatementCollection = env
+			if stmt.Group != nil {
+				parent = stmt.Group
+			}
+
+			idx, _ := parent.GetAssignmentIndex(stmt.Name)
+
+			// Try to keep the position of the KEY around where it was before
+			switch {
+			// If we can't find any placement, put us last in the list
+			case idx == -1:
+				options.UpsertPlacementType = ast.UpsertLast
+
+				// Retain the group name if its still present in the SOURCE doc
+				if stmt.Group != nil && sourceDoc.HasGroup(stmt.Group.String()) {
+					options.Group = stmt.Group.String()
+				}
+
+			// If we were first in the FILE doc, make sure we're first again
+			case idx == 0:
+				options.UpsertPlacementType = ast.UpsertFirst
+
+				// Retain the group name if its still present in the SOURCE doc
+				if stmt.Group != nil && sourceDoc.HasGroup(stmt.Group.String()) {
+					options.Group = stmt.Group.String()
+				}
+
+			// If we were not first, then put us behind the key that was
+			// just before us in the FILE doc
+			case idx > 0:
+				before := parent.Assignments()[idx-1]
+
+				options.UpsertPlacementType = ast.UpsertAfter
+				options.UpsertPlacementValue = before.Name
+
+				if before.Group != nil && sourceDoc.HasGroup(before.Group.String()) {
+					options.Group = before.Group.String()
+				}
+			}
+		}
+
+		changed, err := sourceDoc.Upsert(stmt, options)
 		if err != nil {
-			danger.Println("  ERROR", err.Error())
+			sawError = true
+			lastWasError = true
+
+			if counter > 0 {
+				dark.Println()
+			}
+
+			dark.Print("  ")
+			dangerEmphasis.Print(stmt.Name)
+			dark.Print(" could not be set to ")
+			primary.Print(stmt.Literal)
+			dark.Println(" due to error:")
+
+			danger.Println(" ", strings.Repeat(" ", len(stmt.Name)), err.Error())
+
+			counter++
 
 			continue
 		}
@@ -125,7 +195,10 @@ func runE(cmd *cobra.Command, args []string) error {
 			sawError = true
 			lastWasError = true
 
-			dark.Println()
+			if counter > 0 {
+				dark.Println()
+			}
+
 			dark.Print("  ")
 			dangerEmphasis.Print(stmt.Name)
 			dark.Print(" could not be set to ")
@@ -136,10 +209,14 @@ func runE(cmd *cobra.Command, args []string) error {
 				danger.Println(" ", strings.Repeat(" ", len(stmt.Name)), strings.TrimSpace(validation.Explain(env, errIsh, false, false)))
 			}
 
+			counter++
+
 			continue
 		}
 
 		if changed != nil {
+			counter++
+
 			if lastWasError {
 				danger.Println()
 			}
@@ -175,7 +252,7 @@ func runE(cmd *cobra.Command, args []string) error {
 
 	dark.Println("Saving the new", primary.Sprint(filename))
 
-	if err := pkg.Save(filename, mergedEnv); err != nil {
+	if err := pkg.Save(filename, sourceDoc); err != nil {
 		danger.Println("  ERROR", err.Error())
 
 		return err

pkg/ast/assignment.go

@@ -113,3 +113,13 @@ func (a *Assignment) Disable() {
 func (a *Assignment) Enable() {
 	a.Active = true
 }
+
+func (a *Assignment) CommentsSlice() []string {
+	res := []string{}
+
+	for _, comment := range a.Comments {
+		res = append(res, comment.CleanString())
+	}
+
+	return res
+}

pkg/ast/comment.go

@@ -2,6 +2,7 @@ package ast
 
 import (
 	"reflect"
+	"strings"
 
 	"github.com/jippi/dottie/pkg/token"
 )
@@ -50,3 +51,7 @@ func (c *Comment) statementNode() {
 func (c Comment) String() string {
 	return c.Value
 }
+
+func (c Comment) CleanString() string {
+	return strings.TrimPrefix(c.Value, "# ")
+}