diff --git a/modules/caddyhttp/celmatcher_test.go b/modules/caddyhttp/celmatcher_test.go index 3604562b312..c98aa16ac42 100644 --- a/modules/caddyhttp/celmatcher_test.go +++ b/modules/caddyhttp/celmatcher_test.go @@ -60,6 +60,15 @@ eqp31wM9il1n+guTNyxJd+FzVAH+hCZE5K+tCgVDdVFUlDEHHbS/wqb2PSIoouLV urlTarget: "https://example.com/foo", wantResult: true, }, + { + name: "boolean matches succeed for placeholder default", + expression: &MatchExpression{ + Expr: "{unset:default} == 'default'", + }, + clientCertificate: clientCert, + urlTarget: "https://example.com/foo", + wantResult: true, + }, { name: "header matches (MatchHeader)", expression: &MatchExpression{ diff --git a/modules/caddyhttp/replacer.go b/modules/caddyhttp/replacer.go index c58b56ed307..2560ba7f94a 100644 --- a/modules/caddyhttp/replacer.go +++ b/modules/caddyhttp/replacer.go @@ -62,19 +62,17 @@ func addHTTPVarsToReplacer(repl *caddy.Replacer, req *http.Request, w http.Respo if req != nil { // query string parameters if strings.HasPrefix(key, reqURIQueryReplPrefix) { - vals := req.URL.Query()[key[len(reqURIQueryReplPrefix):]] + vals, found := req.URL.Query()[key[len(reqURIQueryReplPrefix):]] // always return true, since the query param might // be present only in some requests - return strings.Join(vals, ","), true + return strings.Join(vals, ","), found } // request header fields if strings.HasPrefix(key, reqHeaderReplPrefix) { field := key[len(reqHeaderReplPrefix):] - vals := req.Header[textproto.CanonicalMIMEHeaderKey(field)] - // always return true, since the header field might - // be present only in some requests - return strings.Join(vals, ","), true + vals, found := req.Header[textproto.CanonicalMIMEHeaderKey(field)] + return strings.Join(vals, ","), found } // cookies @@ -82,11 +80,10 @@ func addHTTPVarsToReplacer(repl *caddy.Replacer, req *http.Request, w http.Respo name := key[len(reqCookieReplPrefix):] for _, cookie := range req.Cookies() { if strings.EqualFold(name, cookie.Name) { - // always return true, since the cookie might - // be present only in some requests return cookie.Value, true } } + return nil, false } // http.request.tls.* @@ -161,7 +158,7 @@ func addHTTPVarsToReplacer(repl *caddy.Replacer, req *http.Request, w http.Respo return id.String(), true case "http.request.body": if req.Body == nil { - return "", true + return "", false } // normally net/http will close the body for us, but since we // are replacing it with a fake one, we have to ensure we close @@ -219,11 +216,11 @@ func addHTTPVarsToReplacer(repl *caddy.Replacer, req *http.Request, w http.Respo // convert to integer then compute prefix bits, err := strconv.Atoi(bitsStr) if err != nil { - return "", true + return "", false } prefix, err := addr.Prefix(bits) if err != nil { - return "", true + return "", false } return prefix.String(), true } @@ -293,7 +290,7 @@ func addHTTPVarsToReplacer(repl *caddy.Replacer, req *http.Request, w http.Respo raw := GetVar(req.Context(), varName) // variables can be dynamic, so always return true // even when it may not be set; treat as empty then - return raw, true + return raw, raw != nil } } @@ -301,10 +298,8 @@ func addHTTPVarsToReplacer(repl *caddy.Replacer, req *http.Request, w http.Respo // response header fields if strings.HasPrefix(key, respHeaderReplPrefix) { field := key[len(respHeaderReplPrefix):] - vals := w.Header()[textproto.CanonicalMIMEHeaderKey(field)] - // always return true, since the header field might - // be present only in some responses - return strings.Join(vals, ","), true + vals, found := w.Header()[textproto.CanonicalMIMEHeaderKey(field)] + return strings.Join(vals, ","), found } } diff --git a/modules/caddyhttp/replacer_test.go b/modules/caddyhttp/replacer_test.go index 44c6f2a89a7..a03a4f0f81c 100644 --- a/modules/caddyhttp/replacer_test.go +++ b/modules/caddyhttp/replacer_test.go @@ -107,10 +107,6 @@ eqp31wM9il1n+guTNyxJd+FzVAH+hCZE5K+tCgVDdVFUlDEHHbS/wqb2PSIoouLV get: "http.request.remote.host/24,32", expect: "192.168.159.0/24", }, - { - get: "http.request.remote.host/999", - expect: "", - }, { get: "http.request.remote.port", expect: "1234", @@ -207,7 +203,7 @@ eqp31wM9il1n+guTNyxJd+FzVAH+hCZE5K+tCgVDdVFUlDEHHbS/wqb2PSIoouLV } { actual, got := repl.GetString(tc.get) if !got { - t.Errorf("Test %d: Expected to recognize the placeholder name, but didn't", i) + t.Errorf("Test %d: Expected to recognize the placeholder name %q, but didn't", i, tc.get) } if actual != tc.expect { t.Errorf("Test %d: Expected %s to be '%s' but got '%s'", diff --git a/replacer.go b/replacer.go index 7f97f34723d..138f03a68ee 100644 --- a/replacer.go +++ b/replacer.go @@ -16,6 +16,7 @@ package caddy import ( "fmt" + "math" "os" "path/filepath" "runtime" @@ -101,14 +102,15 @@ func (r *Replacer) fromStatic(key string) (any, bool) { // that are empty or not recognized will cause an error to // be returned. func (r *Replacer) ReplaceOrErr(input string, errOnEmpty, errOnUnknown bool) (string, error) { - return r.replace(input, "", false, errOnEmpty, errOnUnknown, nil) + out, _, err := r.replace(input, "", false, errOnEmpty, errOnUnknown, nil) + return out, err } // ReplaceKnown is like ReplaceAll but only replaces // placeholders that are known (recognized). Unrecognized // placeholders will remain in the output. func (r *Replacer) ReplaceKnown(input, empty string) string { - out, _ := r.replace(input, empty, false, false, false, nil) + out, _, _ := r.replace(input, empty, false, false, false, nil) return out } @@ -117,7 +119,7 @@ func (r *Replacer) ReplaceKnown(input, empty string) string { // whether they are recognized or not. Values that are empty // string will be substituted with empty. func (r *Replacer) ReplaceAll(input, empty string) string { - out, _ := r.replace(input, empty, true, false, false, nil) + out, _, _ := r.replace(input, empty, true, false, false, nil) return out } @@ -125,83 +127,160 @@ func (r *Replacer) ReplaceAll(input, empty string) string { // replacement to be made, in case f wants to change or inspect // the replacement. func (r *Replacer) ReplaceFunc(input string, f ReplacementFunc) (string, error) { - return r.replace(input, "", true, false, false, f) + out, _, err := r.replace(input, "", true, false, false, f) + return out, err } -func (r *Replacer) replace(input, empty string, - treatUnknownAsEmpty, errOnEmpty, errOnUnknown bool, - f ReplacementFunc) (string, error) { - if !strings.Contains(input, string(phOpen)) { - return input, nil +func (r *Replacer) replace(input, empty string, treatUnknownAsEmpty, errOnEmpty, errOnUnknown bool, f ReplacementFunc) (string, bool, error) { + // exit early in case of strings without escapes and opening braces, which are common and won't be changed by this function + potentialPlaceholderFound := false + for _, byte := range input { + if byte == '\\' || byte == '{' { + potentialPlaceholderFound = true + break + } + } + if !potentialPlaceholderFound { + return input, true, nil } - var sb strings.Builder - - // it is reasonable to assume that the output - // will be approximately as long as the input - sb.Grow(len(input)) + // it is reasonable to assume that the output will be approximately as long as the input + var result strings.Builder + result.Grow(len(input)) + allPlaceholdersFound := true // iterate the input to find each placeholder var lastWriteCursor int - - // fail fast if too many placeholders are unclosed - var unclosedCount int - -scan: - for i := 0; i < len(input); i++ { - // check for escaped braces - if i > 0 && input[i-1] == phEscape && (input[i] == phClose || input[i] == phOpen) { - sb.WriteString(input[lastWriteCursor : i-1]) - lastWriteCursor = i + skipOpeningBraces := 0 + for placeholderStart := 0; placeholderStart < len(input); placeholderStart++ { + switch input[placeholderStart] { + case phOpen: + if skipOpeningBraces > 0 { + skipOpeningBraces-- + continue + } + // process possible placeholder in remaining loop (do not drop into default) + case phEscape: + // escape character at the end of the input or next character not a brace or escape character + if placeholderStart+1 == len(input) || (input[placeholderStart+1] != phOpen && input[placeholderStart+1] != phClose && input[placeholderStart+1] != phEscape) { + continue + } + // if there's anything to copy (until the escape character), do so + if placeholderStart > lastWriteCursor { + result.WriteString(input[lastWriteCursor:placeholderStart]) + } + // skip handling escaped character, get it copied with the next special character + placeholderStart++ + lastWriteCursor = placeholderStart continue - } - - if input[i] != phOpen { + default: + // just copy anything else continue } - // our iterator is now on an unescaped open brace (start of placeholder) - - // too many unclosed placeholders in absolutely ridiculous input can be extremely slow (issue #4170) - if unclosedCount > 100 { - return "", fmt.Errorf("too many unclosed placeholders") + // our iterator is now on an unescaped open brace (start of placeholder), find matching closing brace + var placeholderEnd int + skipOpeningBraces = math.MaxInt + unclosedBraces := 1 // first unclosed brace already at `placeholderStart`, we start scanning immediately after this + placeHolderEndFound := false + placeholderEndScanner: + for placeholderEnd = placeholderStart + 1; placeholderEnd < len(input); placeholderEnd++ { + switch input[placeholderEnd] { + case phOpen: + unclosedBraces++ + case phClose: + unclosedBraces-- + // Remember the minimum level of unclosed braces, we can skip that many opening braces later. Idea behind this + // optimization: given we have an input like + // + // a{b{c{d{e}f{g} + // 1 2 3 4 3 4 3 (unclosedBraces counter) + // + // This will remember a maximumm of two braces (3 minus the one we're already handling right now) to skip during + // the next iteration. After dismissing the opening brace between a and b for not having a paired closing + // brace, it will skip those between b and d and the next placeholder search will continue at the `{e}` + // placeholder. + if unclosedBraces < skipOpeningBraces { + skipOpeningBraces = unclosedBraces - 1 + } + if unclosedBraces > 0 { + continue + } + placeHolderEndFound = true + break placeholderEndScanner + case phEscape: + // skip escaped character + placeholderEnd++ + default: + } } - - // find the end of the placeholder - end := strings.Index(input[i:], string(phClose)) + i - if end < i { - unclosedCount++ + // no matching closing brace found, this is not a complete placeholder, continue search + if !placeHolderEndFound { + // remember the minimum level of unclosed braces, we can skip that many opening braces later + if unclosedBraces < skipOpeningBraces { + skipOpeningBraces = unclosedBraces - 1 + } continue } - // if necessary look for the first closing brace that is not escaped - for end > 0 && end < len(input)-1 && input[end-1] == phEscape { - nextEnd := strings.Index(input[end+1:], string(phClose)) - if nextEnd < 0 { - unclosedCount++ - continue scan + // write the substring from the last cursor to this point + result.WriteString(input[lastWriteCursor:placeholderStart]) + + // split to key and default (if exists), allowing for escaped colons + var keyBuilder strings.Builder + var key, defaultKey string + // both key and defaultKey are bound to placeholder length + keyBuilder.Grow(placeholderEnd - placeholderStart) + defaultKeyExists := false + keyScanner: + for dividerScanner := placeholderStart + 1; dividerScanner < placeholderEnd; dividerScanner++ { + switch input[dividerScanner] { + case phColon: + defaultKeyExists = true + // default key will be parsed recursively + defaultKey = input[dividerScanner+1 : placeholderEnd] + break keyScanner + case phEscape: + // skip escape character, then copy escaped character + dividerScanner++ + fallthrough + default: + keyBuilder.WriteByte(input[dividerScanner]) } - end += nextEnd + 1 } - - // write the substring from the last cursor to this point - sb.WriteString(input[lastWriteCursor:i]) - - // trim opening bracket - key := input[i+1 : end] + key = keyBuilder.String() + unescapedPlaceholder := key // try to get a value for this key, handle empty values accordingly val, found := r.Get(key) + // try to replace with variable default, if one is defined; if key contains a quote, consider it JSON and do not apply defaulting + if !found && defaultKeyExists && !strings.Contains(key, string(phQuote)) { + var err error + defaultVal, defaultFound, err := r.replace(defaultKey, empty, treatUnknownAsEmpty, errOnEmpty, errOnUnknown, f) + if err != nil { + return "", false, err + } + if !defaultFound { + allPlaceholdersFound = false + unescapedPlaceholder = unescapedPlaceholder + ":" + defaultVal + } else { + found = true + val = defaultVal + } + } + + // if placeholder is still unknown (unrecognized); see if we need to error out or skip the placeholder if !found { - // placeholder is unknown (unrecognized); handle accordingly if errOnUnknown { - return "", fmt.Errorf("unrecognized placeholder %s%s%s", + return "", false, fmt.Errorf("unrecognized placeholder %s%s%s", string(phOpen), key, string(phClose)) - } else if !treatUnknownAsEmpty { - // if treatUnknownAsEmpty is true, we'll handle an empty - // val later; so only continue otherwise - lastWriteCursor = i - continue + } + // if not supposed to treat unknown placeholders as empty values, print the unescaped copy + if !treatUnknownAsEmpty { + allPlaceholdersFound = false + result.WriteByte(phOpen) + result.WriteString(unescapedPlaceholder) + result.WriteByte(phClose) } } @@ -210,35 +289,36 @@ scan: var err error val, err = f(key, val) if err != nil { - return "", err + return "", false, err } } // convert val to a string as efficiently as possible valStr := ToString(val) - // write the value; if it's empty, either return - // an error or write a default value - if valStr == "" { + // write the value; if it's empty, either return an error or write a default value + if valStr == "" && (found || treatUnknownAsEmpty) { if errOnEmpty { - return "", fmt.Errorf("evaluated placeholder %s%s%s is empty", + return "", false, fmt.Errorf("evaluated placeholder %s%s%s is empty", string(phOpen), key, string(phClose)) - } else if empty != "" { - sb.WriteString(empty) } - } else { - sb.WriteString(valStr) + if empty != "" { + valStr = empty + } } + result.WriteString(valStr) // advance cursor to end of placeholder - i = end - lastWriteCursor = i + 1 + placeholderStart = placeholderEnd + lastWriteCursor = placeholderStart + 1 } // flush any unwritten remainder - sb.WriteString(input[lastWriteCursor:]) + if lastWriteCursor < len(input) { + result.WriteString(input[lastWriteCursor:]) + } - return sb.String(), nil + return result.String(), allPlaceholdersFound, nil } // ToString returns val as a string, as efficiently as possible. @@ -344,4 +424,4 @@ var nowFunc = time.Now // ReplacerCtxKey is the context key for a replacer. const ReplacerCtxKey CtxKey = "replacer" -const phOpen, phClose, phEscape = '{', '}', '\\' +const phOpen, phClose, phEscape, phQuote, phColon = '{', '}', '\\', '"', ':' diff --git a/replacer_test.go b/replacer_test.go index 41ada7d6da0..b1b9b2c6a24 100644 --- a/replacer_test.go +++ b/replacer_test.go @@ -31,6 +31,10 @@ func TestReplacer(t *testing.T) { // ReplaceAll for i, tc := range []testCase{ + { + input: `\`, + expect: `\`, + }, { input: "{", expect: "{", @@ -69,7 +73,7 @@ func TestReplacer(t *testing.T) { }, { input: `\}`, - expect: `\}`, + expect: `}`, }, { input: "{}", @@ -80,8 +84,13 @@ func TestReplacer(t *testing.T) { expect: `{}`, }, { + input: `{"json"\: "object"}`, + expect: ``, + }, + { // JSON-compatible interpretation (keys may not have quotes), interpreted as non-existing placeholder name input: `{"json": "object"}`, - expect: "", + empty: `-`, + expect: `-`, }, { input: `\{"json": "object"}`, @@ -120,12 +129,12 @@ func TestReplacer(t *testing.T) { expect: "{{", }, { - input: `{{}`, - expect: "", + input: `a{b{c}d`, + expect: "a{bd", }, { input: `{"json": "object"\}`, - expect: "", + expect: `{"json": "object"}`, }, { input: `{unknown}`, @@ -138,7 +147,7 @@ func TestReplacer(t *testing.T) { }, { input: `double back\\slashes`, - expect: `double back\\slashes`, + expect: `double back\slashes`, }, { input: `placeholder {with \{ brace} in name`, @@ -156,9 +165,27 @@ func TestReplacer(t *testing.T) { input: `\{'group':'default','max_age':3600,'endpoints':[\{'url':'https://some.domain.local/a/d/g'\}],'include_subdomains':true\}`, expect: `{'group':'default','max_age':3600,'endpoints':[{'url':'https://some.domain.local/a/d/g'}],'include_subdomains':true}`, }, + { // escapes in braces get dropped, escapes outside remain + input: `{}{}{}\\\\{\\\\}`, + expect: `\\`, + }, + { + input: `{{{{{{{}`, + expect: `{{{{{{`, + }, + { + input: `{:}`, + empty: `-`, + expect: `-`, + }, { - input: `{}{}{}{\\\\}\\\\`, - expect: `{\\\}\\\\`, + input: `{\:default}`, + empty: `-`, + expect: `-`, + }, + { + input: `{:default}`, + expect: `default`, }, { input: string([]byte{0x26, 0x00, 0x83, 0x7B, 0x84, 0x07, 0x5C, 0x7D, 0x84}), @@ -258,6 +285,8 @@ func TestReplacerReplaceKnown(t *testing.T) { return "1.2.3.4", true case "testEmpty": return "", true + case "with:colon": + return "colon:value", true default: return "NOOO", false } @@ -265,41 +294,131 @@ func TestReplacerReplaceKnown(t *testing.T) { }, } + const empty = "EMPTY" for _, tc := range []struct { - testInput string - expected string + testInput string + expectedReplaceKnown string + expectedReplaceAll string }{ { // test vars without space - testInput: "{test1}{asdf}{äöü}{1}{with space}{mySuper_IP}", - expected: "val1123öö_äütest-123space value1.2.3.4", + testInput: "{test1}{asdf}{äöü}{1}{with space}{mySuper_IP}", + expectedReplaceKnown: "val1123öö_äütest-123space value1.2.3.4", + expectedReplaceAll: "val1123öö_äütest-123space value1.2.3.4", }, { // test vars with space - testInput: "{test1} {asdf} {äöü} {1} {with space} {mySuper_IP} ", - expected: "val1 123 öö_äü test-123 space value 1.2.3.4 ", + testInput: "{test1} {asdf} {äöü} {1} {with space} {mySuper_IP} ", + expectedReplaceKnown: "val1 123 öö_äü test-123 space value 1.2.3.4 ", + expectedReplaceAll: "val1 123 öö_äü test-123 space value 1.2.3.4 ", }, { // test with empty val - testInput: "{test1} {testEmpty} {asdf} {1} ", - expected: "val1 EMPTY 123 test-123 ", + testInput: "{test1} {testEmpty} {asdf} {1} ", + expectedReplaceKnown: fmt.Sprintf("val1 %s 123 test-123 ", empty), + expectedReplaceAll: fmt.Sprintf("val1 %s 123 test-123 ", empty), }, { // test vars with not finished placeholders - testInput: "{te{test1}{as{{df{1}", - expected: "{teval1{as{{dftest-123", + testInput: "{te{test1}{as{{df{1}", + expectedReplaceKnown: "{teval1{as{{dftest-123", + expectedReplaceAll: "{teval1{as{{dftest-123", }, { // test with non existing vars - testInput: "{test1} {nope} {1} ", - expected: "val1 {nope} test-123 ", + testInput: "{test1} {nope} {1} ", + expectedReplaceKnown: "val1 {nope} test-123 ", + expectedReplaceAll: fmt.Sprintf("val1 %s test-123 ", empty), + }, + { + // test with default + testInput: "{nope} {nope:default} {test1:default}", + expectedReplaceKnown: "{nope} default val1", + expectedReplaceAll: fmt.Sprintf("%s default val1", empty), + }, + { + // test with empty default + testInput: "{nope:}", + expectedReplaceKnown: empty, + expectedReplaceAll: empty, + }, + { + // should chain variable expands + testInput: "{nope:foo {test1}bar}", + expectedReplaceKnown: "foo val1bar", + expectedReplaceAll: "foo val1bar", + }, + { + // should chain variable expands + testInput: "{nope:{nope:{test1:default}}}", + expectedReplaceKnown: "val1", + expectedReplaceAll: "val1", + }, + { + // should chain variable expands + testInput: `{unknown\:with\:colon}`, + expectedReplaceKnown: "{unknown:with:colon}", + expectedReplaceAll: empty, + }, + { + // should chain variable expands + testInput: `{with\:colon}`, + expectedReplaceKnown: "colon:value", + expectedReplaceAll: "colon:value", + }, + { + testInput: `{nope:foo:test1}`, + expectedReplaceKnown: "foo:test1", + expectedReplaceAll: "foo:test1", + }, + { + testInput: `{nope\:foo:test1}`, + expectedReplaceKnown: "test1", + expectedReplaceAll: "test1", + }, + { + testInput: `{nope:{foo\:test1}}`, + expectedReplaceKnown: "{nope:{foo:test1}}", + expectedReplaceAll: empty, + }, + { + testInput: `{nope:foo\}bar}`, + expectedReplaceKnown: "foo}bar", + expectedReplaceAll: "foo}bar", + }, + { + testInput: `{nope:foo\{bar}`, + expectedReplaceKnown: "foo{bar", + expectedReplaceAll: "foo{bar", + }, + { + testInput: `{nope:{foo\{bar}}`, + expectedReplaceKnown: `{nope:{foo{bar}}`, + expectedReplaceAll: empty, + }, + { + testInput: `{nope:{nope2}}`, + expectedReplaceKnown: `{nope:{nope2}}`, + expectedReplaceAll: empty, + }, + { + testInput: `a{b{c{test1}`, + expectedReplaceKnown: "a{b{cval1", + expectedReplaceAll: "a{b{cval1", }, } { - actual := rep.ReplaceKnown(tc.testInput, "EMPTY") + actual := rep.ReplaceKnown(tc.testInput, empty) // test if all are replaced as expected - if actual != tc.expected { - t.Errorf("Expected '%s' got '%s' for '%s'", tc.expected, actual, tc.testInput) + if actual != tc.expectedReplaceKnown { + t.Errorf("Expected '%s' got '%s' for '%s' (ReplaceKnown)", tc.expectedReplaceKnown, actual, tc.testInput) + } + + actual = rep.ReplaceAll(tc.testInput, empty) + + // test if all are replaced as expected + if actual != tc.expectedReplaceAll { + t.Errorf("Expected '%s' got '%s' for '%s' (ReplaceAll)", tc.expectedReplaceAll, actual, tc.testInput) } } } @@ -446,6 +565,10 @@ func BenchmarkReplacer(b *testing.B) { name: "escaped placeholder", input: `\{"json": \{"nested": "{bar}"\}\}`, }, + { + name: "many unclosed braces", + input: `{{{{{{{{{{{{{{{{{{{{{{{{{`, + }, } { b.Run(bm.name, func(b *testing.B) { for i := 0; i < b.N; i++ {