fix(scanner): support multi-line variables (#41)

Fixes #40

cmd/print/tests/multi-line.env

@@ -0,0 +1,17 @@
+multi_double="first
+second"
+
+multi_single='first
+second'
+
+multi_double_literal_newline="first\n
+second"
+
+multi_single_literal_newline='first\n
+second'
+
+multi_double_literal_newline_with_tab="first\n
+\tsecond"
+
+multi_single_literal_newline_with_tab='first\n
+\tsecond'

cmd/print/tests/multi-line.run

@@ -0,0 +1 @@
+--no-color --pretty

cmd/print/tests/multi-line/stderr.golden

@@ -0,0 +1,6 @@
+--------------------------------------------------------------------------------
+- Output of command from line 1 in [tests/multi-line.run]:
+- [print --no-color --pretty]
+--------------------------------------------------------------------------------
+
+(no output to stderr)

cmd/print/tests/multi-line/stdout.golden

@@ -0,0 +1,20 @@
+--------------------------------------------------------------------------------
+- Output of command from line 1 in [tests/multi-line.run]:
+- [print --no-color --pretty]
+--------------------------------------------------------------------------------
+
+multi_double="first
+second"
+multi_single='first
+second'
+multi_double_literal_newline="first
+
+second"
+multi_single_literal_newline='first\n
+second'
+multi_double_literal_newline_with_tab="first
+
+    second"
+multi_single_literal_newline_with_tab='first\n
+\tsecond'
+

go.mod

@@ -1,6 +1,6 @@
 module github.com/jippi/dottie
 
-go 1.22.2
+go 1.22.3
 
 replace github.com/go-playground/validator/v10 => github.com/jippi/go-validator/v10 v10.0.0-20240202193343-be965b89f3aa
 

pkg/scanner/scanner.go

@@ -282,11 +282,13 @@ func (s *Scanner) scanQuotedValue(_ context.Context, tType token.Type, quote tok
 	start := s.offset
 
 	escapes := 0
+	foundEndQuote := false
 
 	for {
 		escapingPrevious := escapes == 1
 
-		if isEOF(s.rune) || isNewLine(s.rune) {
+		// If we reach EOF before any closing rune; its a bad syntax
+		if isEOF(s.rune) {
 			tType = token.Illegal
 
 			break
@@ -295,6 +297,8 @@ func (s *Scanner) scanQuotedValue(_ context.Context, tType token.Type, quote tok
 		// Break parsing if we hit our quote style,
 		// and the previous token IS NOT an escape sequence
 		if quote.Is(s.rune) && !escapingPrevious {
+			foundEndQuote = true
+
 			break
 		}
 
@@ -311,6 +315,11 @@ func (s *Scanner) scanQuotedValue(_ context.Context, tType token.Type, quote tok
 		s.next()
 	}
 
+	// If we did not get an end quote while parsing, then its a syntax error
+	if !foundEndQuote {
+		tType = token.Illegal
+	}
+
 	offset := s.offset
 	lit := s.input[start:offset]
 

pkg/scanner/scanner_test.go

@@ -212,6 +212,24 @@ func TestScanner_NextToken_Valid_Identifier(t *testing.T) {
 			expectedTokenType: token.Identifier,
 			expectedLiteral:   "一个类型",
 		},
+		{
+			name:              "mixed quotes should fail",
+			input:             `'valid value \n"`,
+			expectedTokenType: token.Illegal,
+			expectedLiteral:   `valid value \n"`,
+		},
+		{
+			name:              "multiline single quote with double quote inside OK",
+			input:             `'valid value \n"'`,
+			expectedTokenType: token.RawValue,
+			expectedLiteral:   `valid value \n"`,
+		},
+		{
+			name:              "multiline double quote with single quote inside OK",
+			input:             `"valid value \n'"`,
+			expectedTokenType: token.Value,
+			expectedLiteral:   "valid value \n'",
+		},
 	}
 	for _, tt := range tests {
 		tt := tt
@@ -222,7 +240,7 @@ func TestScanner_NextToken_Valid_Identifier(t *testing.T) {
 			sc := scanner.New(tt.input)
 
 			actual := sc.NextToken(context.TODO())
-			assert.Equal(t, tt.expectedTokenType, actual.Type)
+			assert.Equal(t, tt.expectedTokenType.String(), actual.Type.String())
 			assert.Equal(t, tt.expectedLiteral, actual.Literal)
 		})
 	}
@@ -274,7 +292,7 @@ func TestScanner_NextToken_Naked_Value(t *testing.T) {
 			assert.Equal(t, token.Assign.String(), assign.Literal)
 
 			actual := sc.NextToken(context.TODO())
-			assert.Equal(t, tt.expectedTokenType, actual.Type)
+			assert.Equal(t, tt.expectedTokenType.String(), actual.Type.String())
 			assert.Equal(t, tt.expectedLiteral, actual.Literal)
 		})
 	}
@@ -298,11 +316,16 @@ func TestScanner_NextToken_Illegal(t *testing.T) {
 			input:           `"quotes must be closed`,
 			expectedLiteral: "quotes must be closed",
 		},
+		{
+			name:            "not-paired mixed quotes",
+			input:           `"quotes should not be mixed'`,
+			expectedLiteral: "quotes should not be mixed'",
+		},
 		{
 			name: "not-paired double quotes with new line",
 			input: `"quotes must be closed
 `,
-			expectedLiteral: "quotes must be closed",
+			expectedLiteral: "quotes must be closed\n",
 		},
 		{
 			name:            "not-paired single quotes",
@@ -313,7 +336,7 @@ func TestScanner_NextToken_Illegal(t *testing.T) {
 			name: "not-paired single quotes with new line",
 			input: `'quotes must be closed
 `,
-			expectedLiteral: "quotes must be closed",
+			expectedLiteral: "quotes must be closed\n",
 		},
 	}
 	for _, tt := range tests {
@@ -325,7 +348,7 @@ func TestScanner_NextToken_Illegal(t *testing.T) {
 			sc := scanner.New(tt.input)
 
 			actual := sc.NextToken(context.TODO())
-			assert.Equal(t, token.Illegal, actual.Type)
+			assert.Equal(t, token.Illegal.String(), actual.Type.String())
 			assert.Equal(t, tt.expectedLiteral, actual.Literal)
 		})
 	}
@@ -340,14 +363,24 @@ func TestScanner_NextToken(t *testing.T) {
 		expected []token.Token
 	}{
 		{
-			name: "illegal value",
+			name: "illegal value quote double quotes",
 			input: `x="yxc
 `,
 			expected: []token.Token{
 				{Type: token.Identifier, Literal: "x"},
 				{Type: token.Assign, Literal: token.Assign.String()},
-				{Type: token.Illegal, Literal: "yxc"},
-				{Type: token.NewLine, Literal: "\n"},
+				{Type: token.Illegal, Literal: "yxc\n"},
+				{Type: token.EOF, Literal: token.EOF.String()},
+			},
+		},
+		{
+			name: "illegal value quote single quotes",
+			input: `x='yxc
+`,
+			expected: []token.Token{
+				{Type: token.Identifier, Literal: "x"},
+				{Type: token.Assign, Literal: token.Assign.String()},
+				{Type: token.Illegal, Literal: "yxc\n"},
 				{Type: token.EOF, Literal: token.EOF.String()},
 			},
 		},