From e691d14f5c0245c6aeaffe564e499b2ab68c18ee Mon Sep 17 00:00:00 2001 From: Christian Winther Date: Sun, 19 May 2024 21:33:24 +0200 Subject: [PATCH] fix(scanner): support multi-line variables (#41) Fixes https://github.com/jippi/dottie/issues/40 --- cmd/print/tests/multi-line.env | 17 ++++++++ cmd/print/tests/multi-line.run | 1 + cmd/print/tests/multi-line/stderr.golden | 6 +++ cmd/print/tests/multi-line/stdout.golden | 20 ++++++++++ go.mod | 2 +- pkg/scanner/scanner.go | 11 +++++- pkg/scanner/scanner_test.go | 49 ++++++++++++++++++++---- 7 files changed, 96 insertions(+), 10 deletions(-) create mode 100644 cmd/print/tests/multi-line.env create mode 100644 cmd/print/tests/multi-line.run create mode 100644 cmd/print/tests/multi-line/stderr.golden create mode 100644 cmd/print/tests/multi-line/stdout.golden diff --git a/cmd/print/tests/multi-line.env b/cmd/print/tests/multi-line.env new file mode 100644 index 0000000..cc2c42c --- /dev/null +++ b/cmd/print/tests/multi-line.env @@ -0,0 +1,17 @@ +multi_double="first +second" + +multi_single='first +second' + +multi_double_literal_newline="first\n +second" + +multi_single_literal_newline='first\n +second' + +multi_double_literal_newline_with_tab="first\n +\tsecond" + +multi_single_literal_newline_with_tab='first\n +\tsecond' diff --git a/cmd/print/tests/multi-line.run b/cmd/print/tests/multi-line.run new file mode 100644 index 0000000..13e33d6 --- /dev/null +++ b/cmd/print/tests/multi-line.run @@ -0,0 +1 @@ +--no-color --pretty diff --git a/cmd/print/tests/multi-line/stderr.golden b/cmd/print/tests/multi-line/stderr.golden new file mode 100644 index 0000000..a397039 --- /dev/null +++ b/cmd/print/tests/multi-line/stderr.golden @@ -0,0 +1,6 @@ +-------------------------------------------------------------------------------- +- Output of command from line 1 in [tests/multi-line.run]: +- [print --no-color --pretty] +-------------------------------------------------------------------------------- + +(no output to stderr) diff --git a/cmd/print/tests/multi-line/stdout.golden b/cmd/print/tests/multi-line/stdout.golden new file mode 100644 index 0000000..f2c69c4 --- /dev/null +++ b/cmd/print/tests/multi-line/stdout.golden @@ -0,0 +1,20 @@ +-------------------------------------------------------------------------------- +- Output of command from line 1 in [tests/multi-line.run]: +- [print --no-color --pretty] +-------------------------------------------------------------------------------- + +multi_double="first +second" +multi_single='first +second' +multi_double_literal_newline="first + +second" +multi_single_literal_newline='first\n +second' +multi_double_literal_newline_with_tab="first + + second" +multi_single_literal_newline_with_tab='first\n +\tsecond' + diff --git a/go.mod b/go.mod index 8077c56..d2e8b3e 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/jippi/dottie -go 1.22.2 +go 1.22.3 replace github.com/go-playground/validator/v10 => github.com/jippi/go-validator/v10 v10.0.0-20240202193343-be965b89f3aa diff --git a/pkg/scanner/scanner.go b/pkg/scanner/scanner.go index efbeba0..8fe2a33 100644 --- a/pkg/scanner/scanner.go +++ b/pkg/scanner/scanner.go @@ -282,11 +282,13 @@ func (s *Scanner) scanQuotedValue(_ context.Context, tType token.Type, quote tok start := s.offset escapes := 0 + foundEndQuote := false for { escapingPrevious := escapes == 1 - if isEOF(s.rune) || isNewLine(s.rune) { + // If we reach EOF before any closing rune; its a bad syntax + if isEOF(s.rune) { tType = token.Illegal break @@ -295,6 +297,8 @@ func (s *Scanner) scanQuotedValue(_ context.Context, tType token.Type, quote tok // Break parsing if we hit our quote style, // and the previous token IS NOT an escape sequence if quote.Is(s.rune) && !escapingPrevious { + foundEndQuote = true + break } @@ -311,6 +315,11 @@ func (s *Scanner) scanQuotedValue(_ context.Context, tType token.Type, quote tok s.next() } + // If we did not get an end quote while parsing, then its a syntax error + if !foundEndQuote { + tType = token.Illegal + } + offset := s.offset lit := s.input[start:offset] diff --git a/pkg/scanner/scanner_test.go b/pkg/scanner/scanner_test.go index c2e3bd5..73ec771 100644 --- a/pkg/scanner/scanner_test.go +++ b/pkg/scanner/scanner_test.go @@ -212,6 +212,24 @@ func TestScanner_NextToken_Valid_Identifier(t *testing.T) { expectedTokenType: token.Identifier, expectedLiteral: "一个类型", }, + { + name: "mixed quotes should fail", + input: `'valid value \n"`, + expectedTokenType: token.Illegal, + expectedLiteral: `valid value \n"`, + }, + { + name: "multiline single quote with double quote inside OK", + input: `'valid value \n"'`, + expectedTokenType: token.RawValue, + expectedLiteral: `valid value \n"`, + }, + { + name: "multiline double quote with single quote inside OK", + input: `"valid value \n'"`, + expectedTokenType: token.Value, + expectedLiteral: "valid value \n'", + }, } for _, tt := range tests { tt := tt @@ -222,7 +240,7 @@ func TestScanner_NextToken_Valid_Identifier(t *testing.T) { sc := scanner.New(tt.input) actual := sc.NextToken(context.TODO()) - assert.Equal(t, tt.expectedTokenType, actual.Type) + assert.Equal(t, tt.expectedTokenType.String(), actual.Type.String()) assert.Equal(t, tt.expectedLiteral, actual.Literal) }) } @@ -274,7 +292,7 @@ func TestScanner_NextToken_Naked_Value(t *testing.T) { assert.Equal(t, token.Assign.String(), assign.Literal) actual := sc.NextToken(context.TODO()) - assert.Equal(t, tt.expectedTokenType, actual.Type) + assert.Equal(t, tt.expectedTokenType.String(), actual.Type.String()) assert.Equal(t, tt.expectedLiteral, actual.Literal) }) } @@ -298,11 +316,16 @@ func TestScanner_NextToken_Illegal(t *testing.T) { input: `"quotes must be closed`, expectedLiteral: "quotes must be closed", }, + { + name: "not-paired mixed quotes", + input: `"quotes should not be mixed'`, + expectedLiteral: "quotes should not be mixed'", + }, { name: "not-paired double quotes with new line", input: `"quotes must be closed `, - expectedLiteral: "quotes must be closed", + expectedLiteral: "quotes must be closed\n", }, { name: "not-paired single quotes", @@ -313,7 +336,7 @@ func TestScanner_NextToken_Illegal(t *testing.T) { name: "not-paired single quotes with new line", input: `'quotes must be closed `, - expectedLiteral: "quotes must be closed", + expectedLiteral: "quotes must be closed\n", }, } for _, tt := range tests { @@ -325,7 +348,7 @@ func TestScanner_NextToken_Illegal(t *testing.T) { sc := scanner.New(tt.input) actual := sc.NextToken(context.TODO()) - assert.Equal(t, token.Illegal, actual.Type) + assert.Equal(t, token.Illegal.String(), actual.Type.String()) assert.Equal(t, tt.expectedLiteral, actual.Literal) }) } @@ -340,14 +363,24 @@ func TestScanner_NextToken(t *testing.T) { expected []token.Token }{ { - name: "illegal value", + name: "illegal value quote double quotes", input: `x="yxc `, expected: []token.Token{ {Type: token.Identifier, Literal: "x"}, {Type: token.Assign, Literal: token.Assign.String()}, - {Type: token.Illegal, Literal: "yxc"}, - {Type: token.NewLine, Literal: "\n"}, + {Type: token.Illegal, Literal: "yxc\n"}, + {Type: token.EOF, Literal: token.EOF.String()}, + }, + }, + { + name: "illegal value quote single quotes", + input: `x='yxc +`, + expected: []token.Token{ + {Type: token.Identifier, Literal: "x"}, + {Type: token.Assign, Literal: token.Assign.String()}, + {Type: token.Illegal, Literal: "yxc\n"}, {Type: token.EOF, Literal: token.EOF.String()}, }, },