From b18206d8d5923776a43f42ab1870c3d24fdc87b2 Mon Sep 17 00:00:00 2001 From: Mithrandie Date: Sat, 18 May 2019 04:57:25 +0900 Subject: [PATCH 1/5] Refactor string escaping. --- lib/cmd/utils.go | 2 -- lib/cmd/utils_test.go | 2 +- lib/parser/ast.go | 10 +--------- lib/parser/ast_test.go | 5 +++-- lib/query/built_in_command.go | 2 +- 5 files changed, 6 insertions(+), 15 deletions(-) diff --git a/lib/cmd/utils.go b/lib/cmd/utils.go index 29045db3..9959775a 100644 --- a/lib/cmd/utils.go +++ b/lib/cmd/utils.go @@ -33,8 +33,6 @@ func EscapeString(s string) string { buf.WriteString("\\t") case '\v': buf.WriteString("\\v") - case '"': - buf.WriteString("\\\"") case '\'': buf.WriteString("\\'") case '\\': diff --git a/lib/cmd/utils_test.go b/lib/cmd/utils_test.go index bfe3ed89..17b9b7c3 100644 --- a/lib/cmd/utils_test.go +++ b/lib/cmd/utils_test.go @@ -9,7 +9,7 @@ import ( func TestEscapeString(t *testing.T) { str := "fo\\o\a\b\f\n\r\t\v\\\\'\"bar\\" - expect := "fo\\\\o\\a\\b\\f\\n\\r\\t\\v\\\\\\\\\\'\\\"bar\\\\" + expect := "fo\\\\o\\a\\b\\f\\n\\r\\t\\v\\\\\\\\\\'\"bar\\\\" unescaped := EscapeString(str) if unescaped != expect { t.Errorf("escaped string = %q, want %q", unescaped, expect) diff --git a/lib/parser/ast.go b/lib/parser/ast.go index e02df152..08db192c 100644 --- a/lib/parser/ast.go +++ b/lib/parser/ast.go @@ -152,7 +152,7 @@ func (e PrimitiveType) String() string { if 0 < len(e.Literal) { switch e.Value.(type) { case *value.String, *value.Datetime: - return quoteString(e.Literal) + return cmd.QuoteString(e.Literal) default: return e.Literal } @@ -1587,11 +1587,3 @@ func listQueryExpressions(exprs []QueryExpression) string { } return strings.Join(s, ", ") } - -func quoteString(s string) string { - return "'" + s + "'" -} - -func quoteIdentifier(s string) string { - return "`" + s + "`" -} diff --git a/lib/parser/ast_test.go b/lib/parser/ast_test.go index abd96fd7..38ff0137 100644 --- a/lib/parser/ast_test.go +++ b/lib/parser/ast_test.go @@ -5,6 +5,7 @@ import ( "testing" "time" + "github.com/mithrandie/csvq/lib/cmd" "github.com/mithrandie/csvq/lib/value" "github.com/mithrandie/ternary" @@ -127,8 +128,8 @@ func TestIdentifier_String(t *testing.T) { s = "abcde" e = Identifier{Literal: s, Quoted: true} - if e.String() != quoteIdentifier(s) { - t.Errorf("string = %q, want %q for %#v", e.String(), quoteIdentifier(s), e) + if e.String() != cmd.QuoteIdentifier(s) { + t.Errorf("string = %q, want %q for %#v", e.String(), cmd.QuoteIdentifier(s), e) } } diff --git a/lib/query/built_in_command.go b/lib/query/built_in_command.go index ddc65b26..6e40e402 100644 --- a/lib/query/built_in_command.go +++ b/lib/query/built_in_command.go @@ -789,7 +789,7 @@ func writeFields(w *ObjectWriter, fields []string) { w.BeginSubBlock() lastIdx := len(fields) - 1 for i, f := range fields { - escaped := cmd.EscapeString(f) + escaped := cmd.EscapeIdentifier(f) if i < lastIdx && !w.FitInLine(escaped+", ") { w.NewLine() } From e70e4beb07c97a1bc10582a3e4637fe7c8b4b6ef Mon Sep 17 00:00:00 2001 From: Mithrandie Date: Sat, 18 May 2019 06:17:51 +0900 Subject: [PATCH 2/5] Fix bugs of string escaping. - Double backspaches in a string cannot be parsed correctly. - Escaping in external commands cannot be parsed correctly. --- lib/excmd/args_splitter.go | 3 ++- lib/excmd/args_splitter_test.go | 8 ++++---- lib/excmd/argument_scanner.go | 4 ++-- lib/excmd/argument_scanner_test.go | 6 +++--- lib/parser/scanner.go | 1 + lib/parser/scanner_test.go | 20 ++++++++++++++++++++ lib/query/eval.go | 2 +- lib/query/eval_test.go | 16 ++++++++++++++++ 8 files changed, 49 insertions(+), 11 deletions(-) diff --git a/lib/excmd/args_splitter.go b/lib/excmd/args_splitter.go index 7dd8304b..07d22005 100644 --- a/lib/excmd/args_splitter.go +++ b/lib/excmd/args_splitter.go @@ -131,6 +131,7 @@ func (s *ArgsSplitter) scanQuotedString(quote rune) { if ch == '\\' { switch s.peek() { case '\\', quote: + s.text.WriteRune(ch) ch = s.next() } } @@ -162,7 +163,7 @@ func (s *ArgsSplitter) scanExternalCommand() { if ch == '\\' { switch s.peek() { - case '\\', parser.BeginExpression, parser.EndExpression: + case parser.BeginExpression, parser.EndExpression: ch = s.next() } } diff --git a/lib/excmd/args_splitter_test.go b/lib/excmd/args_splitter_test.go index df0aa492..9a683ff8 100644 --- a/lib/excmd/args_splitter_test.go +++ b/lib/excmd/args_splitter_test.go @@ -17,8 +17,8 @@ var argsSplitterScanTests = []struct { "-opt", "arg1", "arg 2", - "arg'3", - "arg\\4", + "arg\\'3", + "arg\\\\4", }, }, { @@ -78,11 +78,11 @@ var argsSplitterScanTests = []struct { }, }, { - Args: "sh -c ${format('echo \"${\\}\"')}", + Args: "sh -c ${format('echo \"${\\\\\\}\"')}", Expect: []string{ "sh", "-c", - "${format('echo \"${}\"')}", + "${format('echo \"${\\\\}\"')}", }, }, { diff --git a/lib/excmd/argument_scanner.go b/lib/excmd/argument_scanner.go index 2db7c6be..a1d4cf44 100644 --- a/lib/excmd/argument_scanner.go +++ b/lib/excmd/argument_scanner.go @@ -139,7 +139,7 @@ func (s *ArgumentScanner) scanString() { s.next() switch s.peek() { - case parser.VariableSign, parser.ExternalCommandSign, '\\': + case parser.VariableSign, parser.ExternalCommandSign: s.text.WriteRune(s.next()) default: s.text.WriteRune('\\') @@ -171,7 +171,7 @@ func (s *ArgumentScanner) scanCsvqExpression() { if ch == '\\' { switch s.peek() { - case '\\', parser.BeginExpression, parser.EndExpression: + case parser.BeginExpression, parser.EndExpression: ch = s.next() } } diff --git a/lib/excmd/argument_scanner_test.go b/lib/excmd/argument_scanner_test.go index 1599fab5..992cb015 100644 --- a/lib/excmd/argument_scanner_test.go +++ b/lib/excmd/argument_scanner_test.go @@ -40,7 +40,7 @@ var argumentScannerScanTests = []struct { { Input: "arg\\@arg\\\\\\$arg\\arg", Expect: []argumentScannerResult{ - {Text: "arg@arg\\$arg\\arg", ElementType: FixedString}, + {Text: "arg@arg\\\\$arg\\arg", ElementType: FixedString}, }, }, { @@ -74,9 +74,9 @@ var argumentScannerScanTests = []struct { }, }, { - Input: "${print 'a\\{bc\\}de'}", + Input: "${print 'a\\{bc\\\\\\}de'}", Expect: []argumentScannerResult{ - {Text: "print 'a{bc}de'", ElementType: CsvqExpression}, + {Text: "print 'a{bc\\\\}de'", ElementType: CsvqExpression}, }, }, { diff --git a/lib/parser/scanner.go b/lib/parser/scanner.go index c652ed6c..5363fccf 100644 --- a/lib/parser/scanner.go +++ b/lib/parser/scanner.go @@ -327,6 +327,7 @@ func (s *Scanner) scanString(quote rune) error { if ch == '\\' { switch s.peek() { case '\\', quote: + s.literal.WriteRune(ch) ch = s.next() } } diff --git a/lib/parser/scanner_test.go b/lib/parser/scanner_test.go index 1ecd6350..73b8d490 100644 --- a/lib/parser/scanner_test.go +++ b/lib/parser/scanner_test.go @@ -72,6 +72,26 @@ var scanTests = []struct { }, }, }, + { + Name: "QuotedString Escape Mark", + Input: "\"string\\t\"", + Output: []scanResult{ + { + Token: STRING, + Literal: "string\t", + }, + }, + }, + { + Name: "QuotedString Double Escape Mark", + Input: "\"string\\\\t\"", + Output: []scanResult{ + { + Token: STRING, + Literal: "string\\t", + }, + }, + }, { Name: "Integer", Input: "1", diff --git a/lib/query/eval.go b/lib/query/eval.go index 4b9ca2a8..fc70c479 100644 --- a/lib/query/eval.go +++ b/lib/query/eval.go @@ -1143,7 +1143,7 @@ func EvaluateEmbeddedString(ctx context.Context, scope *ReferenceScope, embedded for scanner.Scan() { switch scanner.ElementType() { case excmd.FixedString: - buf.WriteString(scanner.Text()) + buf.WriteString(cmd.UnescapeString(scanner.Text())) case excmd.Variable: if err = writeEmbeddedExpression(ctx, scope, buf, parser.Variable{Name: scanner.Text()}); err != nil { return buf.String(), err diff --git a/lib/query/eval_test.go b/lib/query/eval_test.go index fbd57dcb..a3f93a2a 100644 --- a/lib/query/eval_test.go +++ b/lib/query/eval_test.go @@ -4122,6 +4122,14 @@ var evaluateEmbeddedStringTests = []struct { Input: "str", Expect: "str", }, + { + Input: "str\\tstr", + Expect: "str\tstr", + }, + { + Input: "str\\\\tstr", + Expect: "str\\tstr", + }, { Input: "@var", Expect: "1", @@ -4142,6 +4150,14 @@ var evaluateEmbeddedStringTests = []struct { Input: "abc${@var}def", Expect: "abc1def", }, + { + Input: "abc${'a\\tb'}def", + Expect: "abca\tbdef", + }, + { + Input: "abc${'a\\\\tb'}def", + Expect: "abca\\tbdef", + }, { Input: "@notexist", Error: "variable @notexist is undeclared", From 09cdc6e34d556f5bf9dfd9e2b97dc0cafa316487 Mon Sep 17 00:00:00 2001 From: Mithrandie Date: Sun, 19 May 2019 05:05:54 +0900 Subject: [PATCH 3/5] Enable quotations to be escaped with double quotations. --- lib/cmd/flags.go | 4 --- lib/cmd/utils.go | 48 ++++++++++++++++++++++++--------- lib/cmd/utils_test.go | 16 +++++------ lib/excmd/args_splitter.go | 13 ++++++--- lib/excmd/args_splitter_test.go | 24 ++++++++++++++--- lib/parser/scanner.go | 13 ++++++--- lib/parser/scanner_test.go | 10 +++++++ lib/query/eval.go | 13 ++++++++- lib/query/eval_test.go | 12 +++++++++ lib/query/view.go | 2 +- 10 files changed, 117 insertions(+), 38 deletions(-) diff --git a/lib/cmd/flags.go b/lib/cmd/flags.go index db7be30d..e3a16a33 100644 --- a/lib/cmd/flags.go +++ b/lib/cmd/flags.go @@ -337,8 +337,6 @@ func (f *Flags) SetDelimiterPositions(s string) error { if len(s) < 1 { return nil } - s = UnescapeString(s) - delimiterPositions, singleLine, err := ParseDelimiterPositions(s) if err != nil { return err @@ -441,8 +439,6 @@ func (f *Flags) SetWriteDelimiterPositions(s string) error { if len(s) < 1 { return nil } - s = UnescapeString(s) - delimiterPositions, singleLine, err := ParseDelimiterPositions(s) if err != nil { return errors.New(fmt.Sprintf("write-delimiter-positions must be %q or a JSON array of integers", DelimitAutomatically)) diff --git a/lib/cmd/utils.go b/lib/cmd/utils.go index 9959775a..45fc48bb 100644 --- a/lib/cmd/utils.go +++ b/lib/cmd/utils.go @@ -44,12 +44,22 @@ func EscapeString(s string) string { return buf.String() } -func UnescapeString(s string) string { +func UnescapeString(s string, quote rune) string { runes := []rune(s) var buf bytes.Buffer escaped := false + quoteRune := rune(0) for _, r := range runes { + if 0 < quoteRune { + buf.WriteRune(quoteRune) + if r == quoteRune { + quoteRune = 0 + continue + } + quoteRune = 0 + } + if escaped { switch r { case 'a': @@ -76,12 +86,14 @@ func UnescapeString(s string) string { continue } - if r == '\\' { + switch r { + case '\\': escaped = true - continue + case quote: + quoteRune = r + default: + buf.WriteRune(r) } - - buf.WriteRune(r) } if escaped { buf.WriteRune('\\') @@ -121,12 +133,22 @@ func EscapeIdentifier(s string) string { return buf.String() } -func UnescapeIdentifier(s string) string { +func UnescapeIdentifier(s string, quote rune) string { runes := []rune(s) var buf bytes.Buffer escaped := false + quoteRune := rune(0) for _, r := range runes { + if 0 < quoteRune { + buf.WriteRune(quoteRune) + if r == quoteRune { + quoteRune = 0 + continue + } + quoteRune = 0 + } + if escaped { switch r { case 'a': @@ -153,12 +175,14 @@ func UnescapeIdentifier(s string) string { continue } - if r == '\\' { + switch r { + case '\\': escaped = true - continue + case quote: + quoteRune = r + default: + buf.WriteRune(r) } - - buf.WriteRune(r) } if escaped { buf.WriteRune('\\') @@ -279,7 +303,7 @@ func ParseLineBreak(s string) (text.LineBreak, error) { return lb, err } func ParseDelimiter(s string) (rune, error) { - r := []rune(UnescapeString(s)) + r := []rune(UnescapeString(s, '\'')) if len(r) != 1 { return 0, errors.New("delimiter must be one character") } @@ -287,7 +311,7 @@ func ParseDelimiter(s string) (rune, error) { } func ParseDelimiterPositions(s string) ([]int, bool, error) { - s = UnescapeString(s) + s = UnescapeString(s, '\'') var delimiterPositions []int = nil singleLine := false diff --git a/lib/cmd/utils_test.go b/lib/cmd/utils_test.go index 17b9b7c3..73da9404 100644 --- a/lib/cmd/utils_test.go +++ b/lib/cmd/utils_test.go @@ -17,9 +17,9 @@ func TestEscapeString(t *testing.T) { } func TestUnescapeString(t *testing.T) { - str := "fo\\o\\a\\b\\f\\n\\r\\t\\v\\\\\\\\'\\\"bar\\" - expect := "fo\\o\a\b\f\n\r\t\v\\\\'\"bar\\" - unescaped := UnescapeString(str) + str := "fo\\o\\a\\b\\f\\n\\r\\t\\v\\\\\\\\'\\\"bar''\"\"\\" + expect := "fo\\o\a\b\f\n\r\t\v\\\\'\"bar'\"\"\\" + unescaped := UnescapeString(str, '\'') if unescaped != expect { t.Errorf("unescaped string = %q, want %q", unescaped, expect) } @@ -35,9 +35,9 @@ func TestEscapeIdentifier(t *testing.T) { } func TestUnescapeIdentifier(t *testing.T) { - str := "fo\\o\\a\\b\\f\\n\\r\\t\\v\\\\\\\\`bar\\" - expect := "fo\\o\a\b\f\n\r\t\v\\\\`bar\\" - unescaped := UnescapeIdentifier(str) + str := "fo\\o\\a\\b\\f\\n\\r\\t\\v\\\\\\\\`bar``\\" + expect := "fo\\o\a\b\f\n\r\t\v\\\\`bar`\\" + unescaped := UnescapeIdentifier(str, '`') if unescaped != expect { t.Errorf("unescaped identifier = %q, want %q", unescaped, expect) } @@ -317,12 +317,12 @@ var unescapeStringBenchString2 = "abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrst func BenchmarkUnescapeString(b *testing.B) { for i := 0; i < b.N; i++ { - _ = UnescapeString(unescapeStringBenchString) + _ = UnescapeString(unescapeStringBenchString, '\'') } } func BenchmarkUnescapeString2(b *testing.B) { for i := 0; i < b.N; i++ { - _ = UnescapeString(unescapeStringBenchString2) + _ = UnescapeString(unescapeStringBenchString2, '\'') } } diff --git a/lib/excmd/args_splitter.go b/lib/excmd/args_splitter.go index 07d22005..c7809442 100644 --- a/lib/excmd/args_splitter.go +++ b/lib/excmd/args_splitter.go @@ -84,6 +84,7 @@ func (s *ArgsSplitter) Scan() bool { s.scanExternalCommand() } case '"', '\'': + s.text.WriteRune(ch) s.scanQuotedString(ch) default: s.text.WriteRune(ch) @@ -124,17 +125,21 @@ func (s *ArgsSplitter) scanQuotedString(quote rune) { break } - if ch == quote { - break - } - if ch == '\\' { switch s.peek() { case '\\', quote: s.text.WriteRune(ch) ch = s.next() } + } else if ch == quote { + s.text.WriteRune(ch) + if s.peek() == quote { + ch = s.next() + } else { + break + } } + s.text.WriteRune(ch) } } diff --git a/lib/excmd/args_splitter_test.go b/lib/excmd/args_splitter_test.go index 9a683ff8..8b63b3de 100644 --- a/lib/excmd/args_splitter_test.go +++ b/lib/excmd/args_splitter_test.go @@ -16,9 +16,9 @@ var argsSplitterScanTests = []struct { "cmd", "-opt", "arg1", - "arg 2", - "arg\\'3", - "arg\\\\4", + "'arg 2'", + "'arg\\'3'", + "\"arg\\\\4\"", }, }, { @@ -34,7 +34,15 @@ var argsSplitterScanTests = []struct { Expect: []string{ "cmd", "arg1", - "arg 2", + "'arg 2'", + }, + }, + { + Args: "cmd arg1 'arg''2'", + Expect: []string{ + "cmd", + "arg1", + "'arg''2'", }, }, { @@ -69,6 +77,14 @@ var argsSplitterScanTests = []struct { "@#var", }, }, + { + Args: "cmd arg1 ${'arg''2'}", + Expect: []string{ + "cmd", + "arg1", + "${'arg''2'}", + }, + }, { Args: "sh -c ${format('echo %s | wc', @%HOME)}", Expect: []string{ diff --git a/lib/parser/scanner.go b/lib/parser/scanner.go index 5363fccf..d0176474 100644 --- a/lib/parser/scanner.go +++ b/lib/parser/scanner.go @@ -263,7 +263,7 @@ func (s *Scanner) Scan() (Token, error) { if token == ENVIRONMENT_VARIABLE && s.peek() == '`' { err = s.scanString(s.next()) - literal = cmd.UnescapeIdentifier(s.literal.String()) + literal = cmd.UnescapeIdentifier(s.literal.String(), '`') quoted = true } else { if s.isIdentRune(s.peek()) { @@ -293,7 +293,7 @@ func (s *Scanner) Scan() (Token, error) { break case '"', '\'': err = s.scanString(ch) - literal = cmd.UnescapeString(s.literal.String()) + literal = cmd.UnescapeString(s.literal.String(), ch) if _, ok := value.StrToTime(literal, s.datetimeFormats); ok { token = DATETIME } else { @@ -301,7 +301,7 @@ func (s *Scanner) Scan() (Token, error) { } case '`': err = s.scanString(ch) - literal = cmd.UnescapeIdentifier(s.literal.String()) + literal = cmd.UnescapeIdentifier(s.literal.String(), ch) token = IDENTIFIER quoted = true } @@ -321,7 +321,12 @@ func (s *Scanner) scanString(quote rune) error { } if ch == quote { - break + if s.peek() == quote { + s.literal.WriteRune(ch) + ch = s.next() + } else { + break + } } if ch == '\\' { diff --git a/lib/parser/scanner_test.go b/lib/parser/scanner_test.go index 73b8d490..0e1da444 100644 --- a/lib/parser/scanner_test.go +++ b/lib/parser/scanner_test.go @@ -92,6 +92,16 @@ var scanTests = []struct { }, }, }, + { + Name: "QuotedString Double Quotation Mark", + Input: "\"string\"\"string\"", + Output: []scanResult{ + { + Token: STRING, + Literal: "string\"string", + }, + }, + }, { Name: "Integer", Input: "1", diff --git a/lib/query/eval.go b/lib/query/eval.go index fc70c479..e8a3854b 100644 --- a/lib/query/eval.go +++ b/lib/query/eval.go @@ -1136,6 +1136,17 @@ func evalJsonQueryParameters(ctx context.Context, scope *ReferenceScope, expr pa } func EvaluateEmbeddedString(ctx context.Context, scope *ReferenceScope, embedded string) (string, error) { + enclosure := rune(0) + if 1 < len(embedded) { + if embedded[0] == '\'' && embedded[len(embedded)-1] == '\'' { + embedded = embedded[1 : len(embedded)-1] + enclosure = '\'' + } else if embedded[0] == '"' && embedded[len(embedded)-1] == '"' { + embedded = embedded[1 : len(embedded)-1] + enclosure = '"' + } + } + scanner := new(excmd.ArgumentScanner).Init(embedded) buf := &bytes.Buffer{} var err error @@ -1143,7 +1154,7 @@ func EvaluateEmbeddedString(ctx context.Context, scope *ReferenceScope, embedded for scanner.Scan() { switch scanner.ElementType() { case excmd.FixedString: - buf.WriteString(cmd.UnescapeString(scanner.Text())) + buf.WriteString(cmd.UnescapeString(scanner.Text(), enclosure)) case excmd.Variable: if err = writeEmbeddedExpression(ctx, scope, buf, parser.Variable{Name: scanner.Text()}); err != nil { return buf.String(), err diff --git a/lib/query/eval_test.go b/lib/query/eval_test.go index a3f93a2a..dc85a893 100644 --- a/lib/query/eval_test.go +++ b/lib/query/eval_test.go @@ -4130,6 +4130,18 @@ var evaluateEmbeddedStringTests = []struct { Input: "str\\\\tstr", Expect: "str\\tstr", }, + { + Input: "str''str", + Expect: "str''str", + }, + { + Input: "'str''str'", + Expect: "str'str", + }, + { + Input: "\"str\"\"str\"", + Expect: "str\"str", + }, { Input: "@var", Expect: "1", diff --git a/lib/query/view.go b/lib/query/view.go index df51aa35..49082ceb 100644 --- a/lib/query/view.go +++ b/lib/query/view.go @@ -157,7 +157,7 @@ func loadView(ctx context.Context, scope *ReferenceScope, tableExpr parser.Query if value.IsNull(felem) { return nil, NewTableObjectInvalidDelimiterError(tableObject, tableObject.FormatElement.String()) } - s := cmd.UnescapeString(felem.(*value.String).Raw()) + s := felem.(*value.String).Raw() d := []rune(s) if 1 != len(d) { return nil, NewTableObjectInvalidDelimiterError(tableObject, tableObject.FormatElement.String()) From 7105a6ce283b4698004632e5dc042336afaaee3f Mon Sep 17 00:00:00 2001 From: Mithrandie Date: Sun, 19 May 2019 17:51:15 +0900 Subject: [PATCH 4/5] Implement --ansi-quotes option. --- lib/action/calc.go | 2 +- lib/action/run.go | 4 ++-- lib/cmd/flags.go | 8 ++++++++ lib/cmd/flags_test.go | 11 ++++++++++- lib/cmd/utils.go | 2 +- lib/parser/parser.go | 4 ++-- lib/parser/parser.y | 4 ++-- lib/parser/parser_test.go | 3 ++- lib/parser/scanner.go | 11 +++++------ lib/parser/scanner_test.go | 19 ++++++++++++++++--- lib/query/built_in_command.go | 15 ++++++++------- lib/query/built_in_command_test.go | 21 +++++++++++++++++++++ lib/query/completer_readline.go | 4 ++-- lib/query/eval.go | 2 +- lib/query/main_test.go | 1 + lib/query/prepared_statement.go | 2 +- lib/query/terminal.go | 2 +- lib/query/transaction.go | 8 ++++++++ main.go | 7 +++++++ 19 files changed, 99 insertions(+), 31 deletions(-) diff --git a/lib/action/calc.go b/lib/action/calc.go index 541e1418..f0a1cb66 100644 --- a/lib/action/calc.go +++ b/lib/action/calc.go @@ -14,7 +14,7 @@ func Calc(ctx context.Context, proc *query.Processor, expr string) error { _ = proc.Tx.SetFlag(cmd.NoHeaderFlag, true) q := "SELECT " + expr + " FROM STDIN" - program, _, err := parser.Parse(q, "", proc.Tx.Flags.DatetimeFormat, false) + program, _, err := parser.Parse(q, "", proc.Tx.Flags.DatetimeFormat, false, proc.Tx.Flags.AnsiQuotes) if err != nil { e := err.(*parser.SyntaxError) e.SourceFile = "" diff --git a/lib/action/run.go b/lib/action/run.go index 8f788a3a..6be459f7 100644 --- a/lib/action/run.go +++ b/lib/action/run.go @@ -25,7 +25,7 @@ func Run(ctx context.Context, proc *query.Processor, input string, sourceFile st showStats(ctx, proc, start) }() - statements, _, err := parser.Parse(input, sourceFile, proc.Tx.Flags.DatetimeFormat, false) + statements, _, err := parser.Parse(input, sourceFile, proc.Tx.Flags.DatetimeFormat, false, proc.Tx.Flags.AnsiQuotes) if err != nil { return query.NewSyntaxError(err.(*parser.SyntaxError)) } @@ -134,7 +134,7 @@ func LaunchInteractiveShell(ctx context.Context, proc *query.Processor) error { proc.LogError(e.Error()) } - statements, _, e := parser.Parse(strings.Join(lines, "\n"), "", proc.Tx.Flags.DatetimeFormat, false) + statements, _, e := parser.Parse(strings.Join(lines, "\n"), "", proc.Tx.Flags.DatetimeFormat, false, proc.Tx.Flags.AnsiQuotes) if e != nil { if e = query.NewSyntaxError(e.(*parser.SyntaxError)); e != nil { proc.LogError(e.Error()) diff --git a/lib/cmd/flags.go b/lib/cmd/flags.go index e3a16a33..3a24f347 100644 --- a/lib/cmd/flags.go +++ b/lib/cmd/flags.go @@ -26,6 +26,7 @@ const ( RepositoryFlag = "REPOSITORY" TimezoneFlag = "TIMEZONE" DatetimeFormatFlag = "DATETIME_FORMAT" + AnsiQuotesFlag = "ANSI_QUOTES" WaitTimeoutFlag = "WAIT_TIMEOUT" ImportFormatFlag = "IMPORT_FORMAT" DelimiterFlag = "DELIMITER" @@ -57,6 +58,7 @@ var FlagList = []string{ RepositoryFlag, TimezoneFlag, DatetimeFormatFlag, + AnsiQuotesFlag, WaitTimeoutFlag, ImportFormatFlag, DelimiterFlag, @@ -148,6 +150,7 @@ type Flags struct { Repository string Location string DatetimeFormat []string + AnsiQuotes bool // Must be updated from Transaction WaitTimeout float64 @@ -210,6 +213,7 @@ func NewFlags(env *Environment) *Flags { Repository: "", Location: "Local", DatetimeFormat: datetimeFormat, + AnsiQuotes: false, WaitTimeout: 10, Color: false, ImportFormat: CSV, @@ -295,6 +299,10 @@ func (f *Flags) SetDatetimeFormat(s string) { } } +func (f *Flags) SetAnsiQuotes(b bool) { + f.AnsiQuotes = b +} + func (f *Flags) SetWaitTimeout(t float64) { if t < 0 { t = 0 diff --git a/lib/cmd/flags_test.go b/lib/cmd/flags_test.go index 4508c7ba..6a177acf 100644 --- a/lib/cmd/flags_test.go +++ b/lib/cmd/flags_test.go @@ -105,6 +105,15 @@ func TestFlags_SetDatetimeFormat(t *testing.T) { } } +func TestFlags_SetAnsiQuotes(t *testing.T) { + flags := NewFlags(nil) + + flags.SetAnsiQuotes(true) + if !flags.AnsiQuotes { + t.Errorf("ansi_quotes = %t, expect to set %t", flags.AnsiQuotes, true) + } +} + func TestFlags_SetWaitTimeout(t *testing.T) { flags := NewFlags(nil) @@ -574,7 +583,7 @@ func TestFlags_SetQuiet(t *testing.T) { flags.SetQuiet(true) if !flags.Quiet { - t.Errorf("silent = %t, expect to set %t", flags.Quiet, true) + t.Errorf("quiet = %t, expect to set %t", flags.Quiet, true) } } diff --git a/lib/cmd/utils.go b/lib/cmd/utils.go index 45fc48bb..31d14619 100644 --- a/lib/cmd/utils.go +++ b/lib/cmd/utils.go @@ -165,7 +165,7 @@ func UnescapeIdentifier(s string, quote rune) string { buf.WriteRune('\t') case 'v': buf.WriteRune('\v') - case '`', '\\': + case '"', '`', '\\': buf.WriteRune(r) default: buf.WriteRune('\\') diff --git a/lib/parser/parser.go b/lib/parser/parser.go index 138c19ef..859a02cc 100644 --- a/lib/parser/parser.go +++ b/lib/parser/parser.go @@ -376,9 +376,9 @@ func SetDebugLevel(level int, verbose bool) { yyErrorVerbose = verbose } -func Parse(s string, sourceFile string, datetimeFormats []string, forPrepared bool) ([]Statement, int, error) { +func Parse(s string, sourceFile string, datetimeFormats []string, forPrepared bool, ansiQuotes bool) ([]Statement, int, error) { l := new(Lexer) - l.Init(s, sourceFile, datetimeFormats, forPrepared) + l.Init(s, sourceFile, datetimeFormats, forPrepared, ansiQuotes) yyParse(l) return l.program, l.HolderNumber(), l.err } diff --git a/lib/parser/parser.y b/lib/parser/parser.y index 971e3e4f..ade2548d 100644 --- a/lib/parser/parser.y +++ b/lib/parser/parser.y @@ -2572,9 +2572,9 @@ func SetDebugLevel(level int, verbose bool) { yyErrorVerbose = verbose } -func Parse(s string, sourceFile string, datetimeFormats []string, forPrepared bool) ([]Statement, int, error) { +func Parse(s string, sourceFile string, datetimeFormats []string, forPrepared bool, ansiQuotes bool) ([]Statement, int, error) { l := new(Lexer) - l.Init(s, sourceFile, datetimeFormats, forPrepared) + l.Init(s, sourceFile, datetimeFormats, forPrepared, ansiQuotes) yyParse(l) return l.program, l.HolderNumber(), l.err } \ No newline at end of file diff --git a/lib/parser/parser_test.go b/lib/parser/parser_test.go index 0f9fdc5f..1fa66354 100644 --- a/lib/parser/parser_test.go +++ b/lib/parser/parser_test.go @@ -10,6 +10,7 @@ import ( var parseTests = []struct { Input string ForPrepared bool + AnsiQuotes bool Output []Statement SourceFile string HolderNum int @@ -6154,7 +6155,7 @@ var parseTests = []struct { func TestParse(t *testing.T) { for _, v := range parseTests { - prog, holderNum, err := Parse(v.Input, v.SourceFile, nil, v.ForPrepared) + prog, holderNum, err := Parse(v.Input, v.SourceFile, nil, v.ForPrepared, v.AnsiQuotes) if err != nil { if len(v.Error) < 1 { t.Errorf("unexpected error %q for %q", err, v.Input) diff --git a/lib/parser/scanner.go b/lib/parser/scanner.go index d0176474..1e806a0e 100644 --- a/lib/parser/scanner.go +++ b/lib/parser/scanner.go @@ -102,13 +102,14 @@ type Scanner struct { datetimeFormats []string forPrepared bool + ansiQuotes bool holderOrdinal int holderNames []string holderNumber int } -func (s *Scanner) Init(src string, sourceFile string, datetimeFormats []string, forPrepared bool) *Scanner { +func (s *Scanner) Init(src string, sourceFile string, datetimeFormats []string, forPrepared bool, ansiQuotes bool) *Scanner { s.src = []rune(src) s.srcPos = 0 s.line = 1 @@ -116,6 +117,7 @@ func (s *Scanner) Init(src string, sourceFile string, datetimeFormats []string, s.sourceFile = sourceFile s.datetimeFormats = datetimeFormats s.forPrepared = forPrepared + s.ansiQuotes = ansiQuotes s.holderOrdinal = 0 s.holderNames = make([]string, 0, 10) s.holderNumber = 0 @@ -288,10 +290,7 @@ func (s *Scanner) Scan() (Token, error) { s.scanLineComment() return s.Scan() default: - switch ch { - case EOF: - break - case '"', '\'': + if ch == '\'' || (!s.ansiQuotes && ch == '"') { err = s.scanString(ch) literal = cmd.UnescapeString(s.literal.String(), ch) if _, ok := value.StrToTime(literal, s.datetimeFormats); ok { @@ -299,7 +298,7 @@ func (s *Scanner) Scan() (Token, error) { } else { token = STRING } - case '`': + } else if ch == '`' || (s.ansiQuotes && ch == '"') { err = s.scanString(ch) literal = cmd.UnescapeIdentifier(s.literal.String(), ch) token = IDENTIFIER diff --git a/lib/parser/scanner_test.go b/lib/parser/scanner_test.go index 0e1da444..1b30a36b 100644 --- a/lib/parser/scanner_test.go +++ b/lib/parser/scanner_test.go @@ -18,6 +18,7 @@ var scanTests = []struct { Input string DTFormats []string ForPrepared bool + AnsiQuotes bool Output []scanResult Error string }{ @@ -33,11 +34,11 @@ var scanTests = []struct { }, { Name: "QuotedIdentifier", - Input: "`id\\enti\\`fier`", + Input: "`id\\enti\\`fier```", Output: []scanResult{ { Token: IDENTIFIER, - Literal: "id\\enti`fier", + Literal: "id\\enti`fier`", Quoted: true, }, }, @@ -102,6 +103,18 @@ var scanTests = []struct { }, }, }, + { + Name: "AnsiQuotes", + Input: "\"identifier\"", + AnsiQuotes: true, + Output: []scanResult{ + { + Token: IDENTIFIER, + Literal: "identifier", + Quoted: true, + }, + }, + }, { Name: "Integer", Input: "1", @@ -567,7 +580,7 @@ var scanTests = []struct { func TestScanner_Scan(t *testing.T) { for _, v := range scanTests { - s := new(Scanner).Init(v.Input, "", v.DTFormats, v.ForPrepared) + s := new(Scanner).Init(v.Input, "", v.DTFormats, v.ForPrepared, v.AnsiQuotes) tokenCount := 0 for { diff --git a/lib/query/built_in_command.go b/lib/query/built_in_command.go index 6e40e402..ec93c708 100644 --- a/lib/query/built_in_command.go +++ b/lib/query/built_in_command.go @@ -159,7 +159,7 @@ func LoadStatementsFromFile(ctx context.Context, tx *Transaction, fpath parser.I return nil, err } - statements, _, err = parser.Parse(content, fpath.Literal, tx.Flags.DatetimeFormat, false) + statements, _, err = parser.Parse(content, fpath.Literal, tx.Flags.DatetimeFormat, false, tx.Flags.AnsiQuotes) if err != nil { err = NewSyntaxError(err.(*parser.SyntaxError)) } @@ -191,7 +191,7 @@ func ParseExecuteStatements(ctx context.Context, scope *ReferenceScope, expr par if err != nil { return nil, NewReplaceValueLengthError(expr, err.(Error).Message()) } - statements, _, err := parser.Parse(input, fmt.Sprintf("(L:%d C:%d) EXECUTE", expr.Line(), expr.Char()), scope.Tx.Flags.DatetimeFormat, false) + statements, _, err := parser.Parse(input, fmt.Sprintf("(L:%d C:%d) EXECUTE", expr.Line(), expr.Char()), scope.Tx.Flags.DatetimeFormat, false, scope.Tx.Flags.AnsiQuotes) if err != nil { err = NewSyntaxError(err.(*parser.SyntaxError)) } @@ -222,7 +222,7 @@ func SetFlag(ctx context.Context, scope *ReferenceScope, expr parser.SetFlag) er return NewFlagValueNotAllowedFormatError(expr) } val = p.(*value.String).Raw() - case cmd.NoHeaderFlag, cmd.WithoutNullFlag, cmd.WithoutHeaderFlag, cmd.EncloseAllFlag, cmd.PrettyPrintFlag, + case cmd.AnsiQuotesFlag, cmd.NoHeaderFlag, cmd.WithoutNullFlag, cmd.WithoutHeaderFlag, cmd.EncloseAllFlag, cmd.PrettyPrintFlag, cmd.EastAsianEncodingFlag, cmd.CountDiacriticalSignFlag, cmd.CountFormatCodeFlag, cmd.ColorFlag, cmd.QuietFlag, cmd.StatsFlag: p = value.ToBoolean(v) if value.IsNull(p) { @@ -262,8 +262,9 @@ func AddFlagElement(ctx context.Context, scope *ReferenceScope, expr parser.AddF Value: expr.Value, } return SetFlag(ctx, scope, e) - case cmd.RepositoryFlag, cmd.TimezoneFlag, cmd.DelimiterFlag, cmd.JsonQueryFlag, cmd.EncodingFlag, - cmd.WriteEncodingFlag, cmd.FormatFlag, cmd.WriteDelimiterFlag, cmd.LineBreakFlag, cmd.JsonEscapeFlag, + case cmd.RepositoryFlag, cmd.TimezoneFlag, cmd.AnsiQuotesFlag, + cmd.ImportFormatFlag, cmd.DelimiterFlag, cmd.DelimiterPositionsFlag, cmd.JsonQueryFlag, cmd.EncodingFlag, + cmd.WriteEncodingFlag, cmd.FormatFlag, cmd.WriteDelimiterFlag, cmd.WriteDelimiterPositionsFlag, cmd.LineBreakFlag, cmd.JsonEscapeFlag, cmd.NoHeaderFlag, cmd.WithoutNullFlag, cmd.WithoutHeaderFlag, cmd.EncloseAllFlag, cmd.PrettyPrintFlag, cmd.EastAsianEncodingFlag, cmd.CountDiacriticalSignFlag, cmd.CountFormatCodeFlag, cmd.ColorFlag, cmd.QuietFlag, cmd.StatsFlag, cmd.WaitTimeoutFlag, @@ -307,7 +308,7 @@ func RemoveFlagElement(ctx context.Context, scope *ReferenceScope, expr parser.R } else { return NewInvalidFlagValueToBeRemovedError(expr) } - case cmd.RepositoryFlag, cmd.TimezoneFlag, + case cmd.RepositoryFlag, cmd.TimezoneFlag, cmd.AnsiQuotesFlag, cmd.ImportFormatFlag, cmd.DelimiterFlag, cmd.DelimiterPositionsFlag, cmd.JsonQueryFlag, cmd.EncodingFlag, cmd.WriteEncodingFlag, cmd.FormatFlag, cmd.WriteDelimiterFlag, cmd.WriteDelimiterPositionsFlag, cmd.LineBreakFlag, cmd.JsonEscapeFlag, cmd.NoHeaderFlag, cmd.WithoutNullFlag, cmd.WithoutHeaderFlag, cmd.EncloseAllFlag, cmd.PrettyPrintFlag, @@ -444,7 +445,7 @@ func showFlag(tx *Transaction, flagName string) (string, bool) { s = tx.Palette.Render(cmd.NumberEffect, val.(*value.Integer).String()) case cmd.WaitTimeoutFlag: s = tx.Palette.Render(cmd.NumberEffect, val.(*value.Float).String()) - case cmd.NoHeaderFlag, cmd.WithoutNullFlag, cmd.ColorFlag, cmd.QuietFlag, cmd.StatsFlag: + case cmd.AnsiQuotesFlag, cmd.NoHeaderFlag, cmd.WithoutNullFlag, cmd.ColorFlag, cmd.QuietFlag, cmd.StatsFlag: s = tx.Palette.Render(cmd.BooleanEffect, val.(*value.Boolean).String()) } diff --git a/lib/query/built_in_command_test.go b/lib/query/built_in_command_test.go index b2a2f4b9..66597a5b 100644 --- a/lib/query/built_in_command_test.go +++ b/lib/query/built_in_command_test.go @@ -402,6 +402,13 @@ var setFlagTests = []struct { Value: parser.NewStringValue("%Y%m%d"), }, }, + { + Name: "Set AnsiQuotes", + Expr: parser.SetFlag{ + Flag: parser.Flag{Name: "ansi_quotes"}, + Value: parser.NewTernaryValueFromString("true"), + }, + }, { Name: "Set WaitTimeout", Expr: parser.SetFlag{ @@ -897,6 +904,19 @@ var showFlagTests = []struct { }, Result: "\033[34;1m@@DATETIME_FORMAT:\033[0m \033[32m[\"%Y%m%d\", \"%Y%m%d %H%i%s\"]\033[0m", }, + { + Name: "Show AnsiQuotes", + Expr: parser.ShowFlag{ + Flag: parser.Flag{Name: "ansi_quotes"}, + }, + SetExprs: []parser.SetFlag{ + { + Flag: parser.Flag{Name: "ansi_quotes"}, + Value: parser.NewTernaryValueFromString("true"), + }, + }, + Result: "\033[34;1m@@ANSI_QUOTES:\033[0m \033[33;1mtrue\033[0m", + }, { Name: "Show WaitTimeout", Expr: parser.ShowFlag{ @@ -2103,6 +2123,7 @@ var showObjectsTests = []struct { " @@REPOSITORY: .\n" + " @@TIMEZONE: UTC\n" + " @@DATETIME_FORMAT: (not set)\n" + + " @@ANSI_QUOTES: false\n" + " @@WAIT_TIMEOUT: 15\n" + " @@IMPORT_FORMAT: CSV\n" + " @@DELIMITER: ','\n" + diff --git a/lib/query/completer_readline.go b/lib/query/completer_readline.go index cd33b0f8..1db09036 100644 --- a/lib/query/completer_readline.go +++ b/lib/query/completer_readline.go @@ -1716,7 +1716,7 @@ func (c *Completer) SetArgs(line string, origLine string, index int) readline.Ca return nil, c.candidateList(c.encodingList(), false), true case cmd.WriteEncodingFlag: return nil, c.candidateList(exportEncodingsCandidates, false), true - case cmd.NoHeaderFlag, cmd.WithoutNullFlag, cmd.WithoutHeaderFlag, cmd.EncloseAllFlag, cmd.PrettyPrintFlag, + case cmd.AnsiQuotesFlag, cmd.NoHeaderFlag, cmd.WithoutNullFlag, cmd.WithoutHeaderFlag, cmd.EncloseAllFlag, cmd.PrettyPrintFlag, cmd.EastAsianEncodingFlag, cmd.CountDiacriticalSignFlag, cmd.CountFormatCodeFlag, cmd.ColorFlag, cmd.QuietFlag, cmd.StatsFlag: return nil, c.candidateList([]string{ternary.TRUE.String(), ternary.FALSE.String()}, false), true @@ -2340,7 +2340,7 @@ func (c *Completer) completeArgs( func (c *Completer) UpdateTokens(line string, origLine string) { c.tokens = c.tokens[:0] s := new(parser.Scanner) - s.Init(origLine, "", c.scope.Tx.Flags.DatetimeFormat, false) + s.Init(origLine, "", c.scope.Tx.Flags.DatetimeFormat, false, c.scope.Tx.Flags.AnsiQuotes) for { t, _ := s.Scan() if t.Token == parser.EOF { diff --git a/lib/query/eval.go b/lib/query/eval.go index e8a3854b..cf4c883f 100644 --- a/lib/query/eval.go +++ b/lib/query/eval.go @@ -1168,7 +1168,7 @@ func EvaluateEmbeddedString(ctx context.Context, scope *ReferenceScope, embedded case excmd.CsvqExpression: expr := scanner.Text() if 0 < len(expr) { - statements, _, err := parser.Parse(expr, expr, scope.Tx.Flags.DatetimeFormat, false) + statements, _, err := parser.Parse(expr, expr, scope.Tx.Flags.DatetimeFormat, false, scope.Tx.Flags.AnsiQuotes) if err != nil { if syntaxErr, ok := err.(*parser.SyntaxError); ok { err = NewSyntaxError(syntaxErr) diff --git a/lib/query/main_test.go b/lib/query/main_test.go index f291bc5f..72945706 100644 --- a/lib/query/main_test.go +++ b/lib/query/main_test.go @@ -202,6 +202,7 @@ func initFlag(flags *cmd.Flags) { flags.Repository = "." flags.Location = TestLocation flags.DatetimeFormat = []string{} + flags.AnsiQuotes = false flags.WaitTimeout = 15 flags.ImportFormat = cmd.CSV flags.Delimiter = ',' diff --git a/lib/query/prepared_statement.go b/lib/query/prepared_statement.go index e5acb8c5..0610b8e3 100644 --- a/lib/query/prepared_statement.go +++ b/lib/query/prepared_statement.go @@ -72,7 +72,7 @@ type PreparedStatement struct { } func NewPreparedStatement(flags *cmd.Flags, expr parser.StatementPreparation) (*PreparedStatement, error) { - statements, holderNum, err := parser.Parse(expr.Statement.Raw(), expr.Name.Literal, flags.DatetimeFormat, true) + statements, holderNum, err := parser.Parse(expr.Statement.Raw(), expr.Name.Literal, flags.DatetimeFormat, true, flags.AnsiQuotes) if err != nil { return nil, NewPreparedStatementSyntaxError(err.(*parser.SyntaxError)) } diff --git a/lib/query/terminal.go b/lib/query/terminal.go index 7ef2e7dc..7c21ebde 100644 --- a/lib/query/terminal.go +++ b/lib/query/terminal.go @@ -151,7 +151,7 @@ func (p *Prompt) Render(ctx context.Context, sequence []PromptElement) (string, case excmd.CsvqExpression: if 0 < len(element.Text) { command := element.Text - statements, _, err := parser.Parse(command, "", p.scope.Tx.Flags.DatetimeFormat, false) + statements, _, err := parser.Parse(command, "", p.scope.Tx.Flags.DatetimeFormat, false, p.scope.Tx.Flags.AnsiQuotes) if err != nil { syntaxErr := err.(*parser.SyntaxError) return "", NewPromptEvaluationError(syntaxErr.Message) diff --git a/lib/query/transaction.go b/lib/query/transaction.go index 5e1118f7..95a336fd 100644 --- a/lib/query/transaction.go +++ b/lib/query/transaction.go @@ -352,6 +352,12 @@ func (tx *Transaction) setFlag(key string, value interface{}, outFile string) er } else { err = errNotAllowdFlagFormat } + case cmd.AnsiQuotesFlag: + if b, ok := value.(bool); ok { + tx.Flags.SetAnsiQuotes(b) + } else { + err = errNotAllowdFlagFormat + } case cmd.WaitTimeoutFlag: if f, ok := value.(float64); ok { tx.UpdateWaitTimeout(f, file.DefaultRetryDelay) @@ -531,6 +537,8 @@ func (tx *Transaction) GetFlag(key string) (value.Primary, bool) { s = "[" + strings.Join(list, ", ") + "]" } val = value.NewString(s) + case cmd.AnsiQuotesFlag: + val = value.NewBoolean(tx.Flags.AnsiQuotes) case cmd.WaitTimeoutFlag: val = value.NewFloat(tx.Flags.WaitTimeout) case cmd.ImportFormatFlag: diff --git a/main.go b/main.go index 74b7b4b9..d77fc507 100644 --- a/main.go +++ b/main.go @@ -45,6 +45,10 @@ func main() { Name: "datetime-format, t", Usage: "datetime format to parse strings", }, + cli.BoolFlag{ + Name: "ansi-quotes, k", + Usage: "use double quotation mark as identifier enclosure", + }, cli.Float64Flag{ Name: "wait-timeout, w", Value: 10, @@ -335,6 +339,9 @@ func overwriteFlags(c *cli.Context, tx *query.Transaction) error { if c.GlobalIsSet("datetime-format") { _ = tx.SetFlag(cmd.DatetimeFormatFlag, c.GlobalString("datetime-format")) } + if c.GlobalIsSet("ansi-quotes") { + _ = tx.SetFlag(cmd.AnsiQuotesFlag, c.GlobalBool("ansi-quotes")) + } if c.GlobalIsSet("wait-timeout") { _ = tx.SetFlag(cmd.WaitTimeoutFlag, c.GlobalFloat64("wait-timeout")) From b2f772d47197ca8c1b3fd3a87a831ca9cf4b9e7c Mon Sep 17 00:00:00 2001 From: Mithrandie Date: Sun, 19 May 2019 18:33:18 +0900 Subject: [PATCH 5/5] Update docs. --- docs/_posts/2006-01-02-command.md | 5 ++++- docs/_posts/2006-01-02-flag.md | 1 + docs/_posts/2006-01-02-statement.md | 25 +++++++++++++++++-------- docs/sitemap.xml | 6 +++--- lib/syntax/syntax.go | 22 ++++++++++++++-------- 5 files changed, 39 insertions(+), 20 deletions(-) diff --git a/docs/_posts/2006-01-02-command.md b/docs/_posts/2006-01-02-command.md index 51d42f7d..e512fb8e 100644 --- a/docs/_posts/2006-01-02-command.md +++ b/docs/_posts/2006-01-02-command.md @@ -80,6 +80,9 @@ csvq > This option can be specified multiple formats using JSON array of strings. +--ansi-quotes, -k +: Use double quotation mark (U+0022 `"`) as identifier enclosure. + --wait-timeout value, -w value : Limit of the waiting time in seconds to wait for locked files to be released. The default is 10. @@ -516,7 +519,7 @@ In command parameters and statements, following strings represent special charac | \r | U+000D Carriage Return | | \t | U+0009 Horizontal Tab | | \v | U+000b Vertical Tab | -| \\" | U+0022 Double Quote (in strings only) | +| \\" | U+0022 Double Quote | | \\' | U+0027 Single Quote (in strings only) | | \\\` | U+0060 Grave Accent (in identifiers only) | | \\\\ | U+005c Backslash | diff --git a/docs/_posts/2006-01-02-flag.md b/docs/_posts/2006-01-02-flag.md index cb2bc4f0..1f72a6a6 100644 --- a/docs/_posts/2006-01-02-flag.md +++ b/docs/_posts/2006-01-02-flag.md @@ -15,6 +15,7 @@ A flag is a representation of a [command option]({{ '/reference/command.html#opt | @@REPOSITORY | string | Directory path where files are located | | @@TIMEZONE | string | Default TimeZone | | @@DATETIME_FORMAT | string | Datetime Format to parse strings | +| @@ANSI_QUOTES | boolean | Use double quotation mark as identifier enclosure | | @@WAIT_TIMEOUT | float | Limit of the waiting time in seconds to wait for locked files to be released | | @@IMPORT_FORMAT | string | Default format to load files | | @@DELIMITER | string | Field delimiter for CSV | diff --git a/docs/_posts/2006-01-02-statement.md b/docs/_posts/2006-01-02-statement.md index 32f904e0..d8fc26e1 100644 --- a/docs/_posts/2006-01-02-statement.md +++ b/docs/_posts/2006-01-02-statement.md @@ -88,18 +88,18 @@ csvq > EXIT; You can use following types in statements. Identifier -: A identifier is a word starting with any unicode letter or a Low Line(U+005F `_`) and followed by a character string that contains any unicode letters, any digits or Low Lines(U+005F `_`). - You cannot use [reserved words](#reserved_words) as a identifier. +: An identifier is a word starting with any unicode letter or a Low Line(U+005F `_`) and followed by a character string that contains any unicode letters, any digits or Low Lines(U+005F `_`). + You cannot use [reserved words](#reserved_words) as an identifier. - Notwithstanding above naming restriction, you can use most character strings as a identifier by enclosing in Grave Accents(U+0060 ` ). - Back quotes are escaped by back slashes. + Notwithstanding above naming restriction, you can use most character strings as an identifier by enclosing in Grave Accents(U+0060 \` ) or Quotation Marks(U+0022 `"`) if [--ansi-quotes]({{ '/reference/command.html#options' | relative_url }}) is specified. + Enclosure characters are escaped by back slashes or double enclosures. Identifiers represent tables, columns, functions or cursors. Character case is insensitive except file paths, and whether file paths are case insensitive or not depends on your file system. String -: A string is a character string enclosed in Apostrophes(U+0027 `'`) or Quotation Marks(U+0022 `"`). - In a string, single quotes or double quotes are escaped by back slashes. +: A string is a character string enclosed in Apostrophes(U+0027 `'`) or Quotation Marks(U+0022 `"`) if [--ansi-quotes]({{ '/reference/command.html#options' | relative_url }}) is not specified. + In a string, enclosure characters are escaped by back slashes or double enclosures. Integer : An integer is a word that contains only \[0-9\]. @@ -159,9 +159,10 @@ Runtime Information ```sql abcde -- identifier 識別子 -- identifier -`ab+c\`de` -- identifier +`abc\`de` -- identifier +`abc``de` -- identifier 'abcd\'e' -- string -"abcd\"e" -- string +'abcd''e' -- string 123 -- integer 123.456 -- float true -- ternary @@ -172,6 +173,14 @@ null -- null @%ENV_VAR -- environment variable @%`ENV_VAR` -- environment variable @#INFO -- runtime information + +/* if --ansi-quotes is specified */ +"abcd\"e" -- identifier +"abcd""e" -- identifier + +/* if --ansi-quotes is not specified */ +"abcd\"e" -- string +"abcd""e" -- string ``` ## Comments diff --git a/docs/sitemap.xml b/docs/sitemap.xml index 282c8691..b493764f 100644 --- a/docs/sitemap.xml +++ b/docs/sitemap.xml @@ -18,11 +18,11 @@ https://mithrandie.github.io/csvq/reference/command.html - 2019-05-04T12:09:52+00:00 + 2019-05-19T09:32:34+00:00 https://mithrandie.github.io/csvq/reference/statement.html - 2018-11-24T06:47:39+00:00 + 2019-05-19T09:32:34+00:00 https://mithrandie.github.io/csvq/reference/value.html @@ -102,7 +102,7 @@ https://mithrandie.github.io/csvq/reference/flag.html - 2018-11-17T22:33:20+00:00 + 2019-05-19T09:32:34+00:00 https://mithrandie.github.io/csvq/reference/environment-variable.html diff --git a/lib/syntax/syntax.go b/lib/syntax/syntax.go index cb9f8b37..98cf2bc6 100644 --- a/lib/syntax/syntax.go +++ b/lib/syntax/syntax.go @@ -993,6 +993,8 @@ var CsvqSyntax = []Expression{ "%s \n" + " > Datetime Format to parse strings.\n" + "%s \n" + + " > Use double quotation mark(U+0022 \") as identifier enclosure.\n" + + "%s \n" + " > Limit of the waiting time in seconds to wait for locked files to be released.\n" + "%s \n" + " > Default format to load files.\n" + @@ -1045,6 +1047,7 @@ var CsvqSyntax = []Expression{ Flag("@@REPOSITORY"), String("string"), Flag("@@TIMEZONE"), String("string"), Link("Timezone"), Flag("@@DATETIME_FORMAT"), String("string"), + Flag("@@ANSI_QUOTES"), String("boolean"), Flag("@@WAIT_TIMEOUT"), Float("float"), Flag("@@IMPORT_FORMAT"), String("string"), Flag("@@DELIMITER"), String("string"), @@ -2675,14 +2678,16 @@ var CsvqSyntax = []Expression{ Description: Description{ Template: "" + "%s\n" + - " > A identifier is a word starting with any unicode letter or" + + " > An identifier is a word starting with any unicode letter or" + " a Low Line(U+005F _) and followed by a character string that" + - " contains any unicode letters, any digits or Low Lines(U+005F _)." + - " You cannot use %s as a identifier.\n" + + " contains any unicode letters, any digits or Low Lines(U+005F _)." + + " You cannot use %s as an identifier.\n" + "\n" + " > Notwithstanding above naming restriction, you can use most" + - " character strings as a identifier by enclosing in" + - " Grave Accents(U+0060 `). Back quotes are escaped by back slashes.\n" + + " character strings as an identifier by enclosing in" + + " Grave Accents(U+0060 `) or Quotation Marks(U+0022 \") if" + + " --ansi-quotes is specified. Enclosure characters are escaped by" + + " back slashes or double enclosures.\n" + "\n" + " > Identifiers represent tables, columns, functions or cursors." + " Character case is insensitive except file paths, and whether file" + @@ -2690,8 +2695,9 @@ var CsvqSyntax = []Expression{ "\n" + "%s\n" + " > A string is a character string enclosed in Apostrophes(U+0027 ') or" + - " Quotation Marks(U+0022 \"). In a string, single quotes or double" + - " quotes are escaped by back slashes.\n" + + " Quotation Marks(U+0022 \") if --ansi-quotes is not specified." + + " In a string, enclosure characters are escaped by back slashes or" + + " double enclosures.\n" + "\n" + "%s\n" + " > An integer is a word that contains only [0-9].\n" + @@ -2774,7 +2780,7 @@ var CsvqSyntax = []Expression{ " | \\r | U+000D Carriage Return |\n" + " | \\t | U+0009 Horizontal Tab |\n" + " | \\v | U+000b Vertical Tab |\n" + - " | \\\" | U+0022 Double Quote (in strings only) |\n" + + " | \\\" | U+0022 Double Quote |\n" + " | \\' | U+0027 Single Quote (in strings only) |\n" + " | \\` | U+0060 Grave Accent (in identifiers only) |\n" + " | \\\\ | U+005c Backslash |\n" +