diff --git a/docs/request_rewriters.md b/docs/request_rewriters.md index 5e91312eb..8fc34b497 100644 --- a/docs/request_rewriters.md +++ b/docs/request_rewriters.md @@ -33,6 +33,34 @@ In a Rule's `case` configurations, provide `req_rewriter_name`. If there is a Ru In a Request Rewriter instruction using the `chain` instruction type. Provide the Rewriter Name as the third argument in the instruction as follows: `[ 'chain', 'exec', '$rewriter_name']`. See more information [below](#chain). +## Regex Capture Tokens + +An `rmatch` rule can expose its regular expression matches to request rewriters. Numeric tokens use `${0}` for the complete match and `${1}`, `${2}`, etc. for capture groups. A named capture such as `(?P[a-z0-9]{3})` is also available as `${tenant}`. + +Captures are available only to the matched case rewriter and the current rule's egress rewriter. The ingress rewriter runs before matching and cannot use captures from its own rule, and captures are cleared before the request enters the next route. If the regular expression does not match, no capture tokens are available to the no-match or egress rewriter. Undefined tokens are left unchanged, while an optional capture group that did not participate in a successful match expands to an empty string. + +Token expansion is supported in setter and appender values for headers and +parameters, and in configured values for path, method, host, hostname, port and +scheme instructions. Header and parameter replace/delete instructions also +expand their key, search and replacement fields. Chained rewriters receive the +same captures. For example: + +```yaml +request_rewriters: + tenant-host: + instructions: + - [ 'hostname', 'set', '${tenant}.writer.example.com' ] + - [ 'header', 'set', 'X-Tenant', '${1}' ] +``` + +Captured values are inserted without additional validation or escaping. Use +restrictive regular expressions for values written to a hostname, path, header +or query parameter. In particular, a client-controlled authority token can +route a request, including configured upstream credentials, to an unintended +host if the expression is too broad. + +When Trickster constructs the final upstream URL, only host components explicitly changed by a request rewriter override the configured backend `origin_url`. A `hostname` rewrite preserves the `origin_url` port, while `host` replaces both hostname and port. The inbound request's host never overrides `origin_url` by itself. + ## Instruction Construction Guide ### header diff --git a/docs/rule.md b/docs/rule.md index 1a893a7c4..a45aec7cf 100644 --- a/docs/rule.md +++ b/docs/rule.md @@ -142,3 +142,42 @@ backends: path_routing_disabled: true # restrict routing to this backend via rule only, so # users cannot directly access via /example-writer-cluster/ ``` + +## Example Rule - Rewrite a Hostname From a Regex Capture + +An `rmatch` rule makes its numeric and named capture groups available to the matched case and egress request rewriters. This example routes a request containing a three-character tenant label and prefixes the destination hostname with that tenant. + +```yaml +request_rewriters: + tenant-host: + instructions: + - [ 'hostname', 'set', '${tenant}.writer.example.com' ] + +rules: + tenant-router: + next_route: example-reader-cluster + input_source: path + input_type: string + operation: rmatch + operation_arg: '\{mylabel="(?P[a-z0-9]{3})"\}' + cases: + - matches: [ 'true' ] + req_rewriter_name: tenant-host + next_route: example-writer-cluster + +backends: + example: + provider: rule + rule_name: tenant-router + + example-reader-cluster: + provider: rpc + origin_url: 'http://reader-cluster.example.com' + + example-writer-cluster: + provider: rpc + origin_url: 'http://writer-cluster.example.com' + path_routing_disabled: true +``` + +For `input_source: path`, matching uses Go's decoded `URL.Path`. For example, `%7Bmylabel%3D%22abc%22%7D` is matched as `{mylabel="abc"}` and `${tenant}` expands to `abc`. `${0}` represents the complete regex match, while `${1}` represents the first capture group. See [Request Rewriters](./request_rewriters.md#regex-capture-tokens) for token lifetime and safety details. diff --git a/pkg/backends/rule/parse.go b/pkg/backends/rule/parse.go index d991c7d83..68b47415c 100644 --- a/pkg/backends/rule/parse.go +++ b/pkg/backends/rule/parse.go @@ -156,6 +156,7 @@ func (c *Client) parseOptions(o *ro.Options, rwi rewriter.InstructionsLookup) er return err } compiledRegexes[r.operationArg] = re + r.regex = re } if len(o.CaseOptions) > 0 { @@ -205,6 +206,16 @@ func (c *Client) parseOptions(o *ro.Options, rwi rewriter.InstructionsLookup) er } } + r.hasCaptureTokens = r.egressReqRewriter.HasTokens() + if !r.hasCaptureTokens { + for _, c := range r.cases { + if c.rewriter.HasTokens() { + r.hasCaptureTokens = true + break + } + } + } + c.rule = r return nil } diff --git a/pkg/backends/rule/regex_captures_test.go b/pkg/backends/rule/regex_captures_test.go new file mode 100644 index 000000000..c9880502a --- /dev/null +++ b/pkg/backends/rule/regex_captures_test.go @@ -0,0 +1,298 @@ +/* + * Copyright 2018 The Trickster Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package rule + +import ( + "context" + "fmt" + "net" + "net/http" + "net/http/httptest" + "net/url" + "regexp" + "testing" + + "github.com/trickstercache/trickster/v2/pkg/backends" + bo "github.com/trickstercache/trickster/v2/pkg/backends/options" + ro "github.com/trickstercache/trickster/v2/pkg/backends/rule/options" + "github.com/trickstercache/trickster/v2/pkg/proxy/engines" + po "github.com/trickstercache/trickster/v2/pkg/proxy/paths/options" + "github.com/trickstercache/trickster/v2/pkg/proxy/request" + "github.com/trickstercache/trickster/v2/pkg/proxy/request/rewriter" + rwo "github.com/trickstercache/trickster/v2/pkg/proxy/request/rewriter/options" + "github.com/trickstercache/trickster/v2/pkg/proxy/urls" +) + +func newRegexCaptureRule(t *testing.T) *rule { + t.Helper() + + rwi, err := rewriter.ProcessConfigs(rwo.Lookup{ + "capture-host": { + Instructions: rwo.RewriteList{ + {"hostname", "set", "${tenant}.writer.example.com"}, + }, + }, + "capture-header": { + Instructions: rwo.RewriteList{ + {"header", "set", "X-Egress-Captures", "${1}:${tenant}:${0}:${missing}"}, + }, + }, + "no-match": { + Instructions: rwo.RewriteList{ + {"header", "set", "X-No-Match-Capture", "${tenant}"}, + }, + }, + }) + if err != nil { + t.Fatal(err) + } + + destination, err := NewClient("destination", nil, http.NotFoundHandler(), nil, nil, nil) + if err != nil { + t.Fatal(err) + } + clients := backends.Backends{"destination": destination} + backendClient, err := NewClient("capture-rule", bo.New(), nil, nil, clients, nil) + if err != nil { + t.Fatal(err) + } + c := backendClient.(*Client) + err = c.parseOptions(&ro.Options{ + Name: "capture-rule", + InputType: "string", + InputSource: "path", + Operation: "rmatch", + OperationArg: `\{mylabel="(?P[a-z0-9]{3})"\}`, + NextRoute: "destination", + EgressReqRewriterName: "capture-header", + NoMatchReqRewriterName: "no-match", + CaseOptions: ro.CaseOptionsList{ + { + Matches: []string{trueValue}, + ReqRewriterName: "capture-host", + NextRoute: "destination", + }, + }, + }, rwi) + if err != nil { + t.Fatal(err) + } + if !c.rule.hasCaptureTokens { + t.Fatal("rule did not detect capture token use") + } + return c.rule +} + +func TestRegexCaptureTokensIncludeOptionalGroups(t *testing.T) { + re := regexp.MustCompile(`(?Pfoo)?(?Pbar)`) + tokens := regexCaptureTokens(re, re.FindStringSubmatch("bar")) + wants := map[string]string{ + "0": "bar", + "1": "", + "2": "bar", + "optional": "", + "tenant": "bar", + } + for key, want := range wants { + if got := tokens[key]; got != want { + t.Errorf("token %q = %q, want %q", key, got, want) + } + } +} + +func TestRegexCapturesRewriteRequest(t *testing.T) { + rule := newRegexCaptureRule(t) + req := httptest.NewRequest(http.MethodGet, + `/query/%7Bmylabel%3D%22abc%22%7D`, nil) + if req.URL.Host != "" { + t.Fatalf("inbound URL host = %q, want empty", req.URL.Host) + } + + _, rewritten, err := rule.EvaluateOpArg(req) + if err != nil { + t.Fatal(err) + } + if got, want := rewritten.URL.Hostname(), "abc.writer.example.com"; got != want { + t.Fatalf("hostname = %q, want %q", got, want) + } + if got, want := rewritten.Header.Get("X-Egress-Captures"), + `abc:abc:{mylabel="abc"}:${missing}`; got != want { + t.Fatalf("X-Egress-Captures = %q, want %q", got, want) + } + base, err := url.Parse("http://writer.example.com:9090/base") + if err != nil { + t.Fatal(err) + } + upstream := urls.BuildUpstreamURL(rewritten, base) + if got, want := upstream.Host, "abc.writer.example.com:9090"; got != want { + t.Fatalf("upstream host = %q, want %q", got, want) + } + + downstream, err := rewriter.ParseRewriteList(rwo.RewriteList{ + {"header", "set", "X-Downstream-Capture", "${tenant}"}, + }) + if err != nil { + t.Fatal(err) + } + downstream.Execute(rewritten) + if got, want := rewritten.Header.Get("X-Downstream-Capture"), "${tenant}"; got != want { + t.Fatalf("downstream capture = %q, want %q", got, want) + } +} + +func TestRegexCaptureRewritesFinalUpstreamRequest(t *testing.T) { + type upstreamRequest struct { + host string + path string + } + requests := make(chan upstreamRequest, 1) + origin := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests <- upstreamRequest{host: r.Host, path: r.URL.Path} + w.WriteHeader(http.StatusNoContent) + })) + defer origin.Close() + + originURL, err := url.Parse(origin.URL) + if err != nil { + t.Fatal(err) + } + transport := &http.Transport{ + DialContext: func(ctx context.Context, network, _ string) (net.Conn, error) { + return (&net.Dialer{}).DialContext(ctx, network, originURL.Host) + }, + } + t.Cleanup(transport.CloseIdleConnections) + + rule := newRegexCaptureRule(t) + req, err := http.NewRequest(http.MethodGet, + `http://trickster.example/query/%7Bmylabel%3D%22abc%22%7D`, nil) + if err != nil { + t.Fatal(err) + } + _, req, err = rule.EvaluateOpArg(req) + if err != nil { + t.Fatal(err) + } + + backendOptions := bo.New() + backendOptions.Name = "writer" + backendOptions.HTTPClient = &http.Client{Transport: transport} + pathOptions := po.New() + req = request.SetResources(req, + request.NewResources(backendOptions, pathOptions, nil, nil, nil, nil)) + base, err := url.Parse("http://writer.example.com:9090/base") + if err != nil { + t.Fatal(err) + } + req.URL = urls.BuildUpstreamURL(req, base) + + reader, resp, _ := engines.PrepareFetchReader(req) + if reader != nil { + defer reader.Close() + } + if resp.StatusCode != http.StatusNoContent { + t.Fatalf("status = %d, want %d", resp.StatusCode, http.StatusNoContent) + } + upstream := <-requests + if got, want := upstream.host, "abc.writer.example.com:9090"; got != want { + t.Errorf("origin Host = %q, want %q", got, want) + } + if got, want := upstream.path, `/base/query/{mylabel="abc"}`; got != want { + t.Errorf("origin path = %q, want %q", got, want) + } +} + +func TestRegexCaptureTokensDoNotLeakBetweenRequests(t *testing.T) { + rule := newRegexCaptureRule(t) + results := make(chan error, 100) + + for i := range 100 { + go func() { + tenant := fmt.Sprintf("a%02d", i) + req, err := http.NewRequest(http.MethodGet, + fmt.Sprintf(`http://trickster.example/query/%%7Bmylabel%%3D%%22%s%%22%%7D`, tenant), nil) + if err != nil { + results <- err + return + } + _, rewritten, err := rule.EvaluateOpArg(req) + if err != nil { + results <- err + return + } + if got, want := rewritten.URL.Hostname(), tenant+".writer.example.com"; got != want { + results <- fmt.Errorf("hostname = %q, want %q", got, want) + return + } + results <- nil + }() + } + + for range 100 { + if err := <-results; err != nil { + t.Error(err) + } + } +} + +func TestRegexCaptureTokensAreClearedOnNoMatch(t *testing.T) { + rule := newRegexCaptureRule(t) + req, err := http.NewRequest(http.MethodGet, + `http://trickster.example/query/%7Bmylabel%3D%22abc%22%7D`, nil) + if err != nil { + t.Fatal(err) + } + + _, rewritten, err := rule.EvaluateOpArg(req) + if err != nil { + t.Fatal(err) + } + rewritten.URL.Path = "/query/no-label" + _, rewritten, err = rule.EvaluateOpArg(rewritten) + if err != nil { + t.Fatal(err) + } + if got, want := rewritten.Header.Get("X-No-Match-Capture"), "${tenant}"; got != want { + t.Fatalf("X-No-Match-Capture = %q, want %q", got, want) + } + if got, want := rewritten.Header.Get("X-Egress-Captures"), + "${1}:${tenant}:${0}:${missing}"; got != want { + t.Fatalf("X-Egress-Captures = %q, want %q", got, want) + } +} + +func TestRegexCapturesAreUnavailableWithoutMatchingCase(t *testing.T) { + rule := newRegexCaptureRule(t) + rule.cases[0].matchValue = falseValue + req, err := http.NewRequest(http.MethodGet, + `http://trickster.example/query/%7Bmylabel%3D%22abc%22%7D`, nil) + if err != nil { + t.Fatal(err) + } + + _, rewritten, err := rule.EvaluateOpArg(req) + if err != nil { + t.Fatal(err) + } + if got, want := rewritten.Header.Get("X-No-Match-Capture"), "${tenant}"; got != want { + t.Fatalf("X-No-Match-Capture = %q, want %q", got, want) + } + if got, want := rewritten.Header.Get("X-Egress-Captures"), + "${1}:${tenant}:${0}:${missing}"; got != want { + t.Fatalf("X-Egress-Captures = %q, want %q", got, want) + } +} diff --git a/pkg/backends/rule/rule.go b/pkg/backends/rule/rule.go index d8acac0a2..17eae3f90 100644 --- a/pkg/backends/rule/rule.go +++ b/pkg/backends/rule/rule.go @@ -18,6 +18,8 @@ package rule import ( "net/http" + "regexp" + "strconv" "github.com/trickstercache/trickster/v2/pkg/proxy/context" "github.com/trickstercache/trickster/v2/pkg/proxy/handlers/trickster/failures" @@ -57,8 +59,10 @@ type rule struct { cases caseList - extractionArg string - operationArg string + extractionArg string + operationArg string + regex *regexp.Regexp + hasCaptureTokens bool defaultRedirectURL string defaultRedirectCode int @@ -85,6 +89,7 @@ type evaluatorFunc func(*http.Request) (http.Handler, *http.Request, error) var badRequestHandler = http.HandlerFunc(failures.HandleBadRequestResponse) func (r *rule) EvaluateOpArg(hr *http.Request) (http.Handler, *http.Request, error) { + hr = rewriter.WithoutTokens(hr) currentHops, maxHops := context.Hops(hr.Context()) if r.maxRuleExecutions < maxHops { maxHops = r.maxRuleExecutions @@ -100,13 +105,21 @@ func (r *rule) EvaluateOpArg(hr *http.Request) (http.Handler, *http.Request, err } h := r.defaultRouter - res := r.operationFunc(r.extractionFunc(hr, r.extractionArg), - r.operationArg, r.negateOpResult) + input := r.extractionFunc(hr, r.extractionArg) + res := r.operationFunc(input, r.operationArg, r.negateOpResult) + var captureTokens map[string]string + if r.regex != nil && r.hasCaptureTokens { + matches := r.regex.FindStringSubmatch(input) + captureTokens = regexCaptureTokens(r.regex, matches) + } var nonDefault bool for _, c := range r.cases { if c.matchValue == res { nonDefault = true + if len(captureTokens) > 0 { + hr = rewriter.WithTokens(hr, captureTokens) + } h, hr = handleMatchedCase(c, hr) } } @@ -125,11 +138,30 @@ func (r *rule) EvaluateOpArg(hr *http.Request) (http.Handler, *http.Request, err r.defaultRedirectCode, r.defaultRedirectURL)) } + hr = rewriter.WithoutTokens(hr) hr = hr.WithContext(context.WithHops(hr.Context(), currentHops+1, maxHops)) return h, hr, nil } +func regexCaptureTokens(re *regexp.Regexp, matches []string) map[string]string { + if len(matches) == 0 { + return nil + } + tokens := make(map[string]string, len(matches)*2) + names := re.SubexpNames() + for i, match := range matches { + tokens[strconv.Itoa(i)] = match + if i >= len(names) || names[i] == "" { + continue + } + if _, ok := tokens[names[i]]; !ok { + tokens[names[i]] = match + } + } + return tokens +} + func (r *rule) EvaluateCaseArg(hr *http.Request) (http.Handler, *http.Request, error) { currentHops, maxHops := context.Hops(hr.Context()) if r.maxRuleExecutions < maxHops { diff --git a/pkg/proxy/engines/key.go b/pkg/proxy/engines/key.go index 6730a6392..31c6ed0cb 100644 --- a/pkg/proxy/engines/key.go +++ b/pkg/proxy/engines/key.go @@ -30,6 +30,7 @@ import ( "github.com/trickstercache/trickster/v2/pkg/proxy/headers" "github.com/trickstercache/trickster/v2/pkg/proxy/methods" "github.com/trickstercache/trickster/v2/pkg/proxy/params" + proxyurls "github.com/trickstercache/trickster/v2/pkg/proxy/urls" "github.com/trickstercache/trickster/v2/pkg/util/sets" ) @@ -46,9 +47,11 @@ func ComposeCacheKey(name, prefix, engine, suffix string) string { // DeriveCacheKey calculates a query-specific keyname based on the user request func (pr *proxyRequest) DeriveCacheKey(extra string) string { pc := pr.rsc.PathConfig + upstreamKeyPart := pr.upstreamURLRewriteCacheKey() if pc == nil { - return md5.Checksum(pr.URL.Path + pr.corsCacheKeyPart(pr.Request) + extra) + return md5.Checksum(pr.URL.Path + upstreamKeyPart + + pr.corsCacheKeyPart(pr.Request) + extra) } var qp url.Values @@ -80,8 +83,9 @@ func (pr *proxyRequest) DeriveCacheKey(extra string) string { if pc.KeyHasher != nil { key := pc.KeyHasher(r.URL.Path, qp, r.Header, b, trq, extra) - if corsPart := pr.corsCacheKeyPart(r); corsPart != "" { - return md5.Checksum(key + corsPart) + keyPart := upstreamKeyPart + pr.corsCacheKeyPart(r) + if keyPart != "" { + return md5.Checksum(key + keyPart) } return key } @@ -189,9 +193,19 @@ func (pr *proxyRequest) DeriveCacheKey(extra string) string { vals = vals[:k] slices.Sort(vals) return md5.Checksum(pr.URL.Path + "." + strings.Join(vals, "") + + upstreamKeyPart + pr.corsCacheKeyPart(r) + extra) } +func (pr *proxyRequest) upstreamURLRewriteCacheKey() string { + if pr == nil || pr.rsc == nil || pr.rsc.BackendOptions == nil { + return "" + } + o := pr.rsc.BackendOptions + base := proxyurls.FromParts(o.Scheme, o.Host, "", "", "") + return proxyurls.UpstreamURLRewriteCacheKey(pr.Request, base) +} + func (pr *proxyRequest) corsCacheKeyPart(r *http.Request) string { if pr == nil || pr.rsc == nil || pr.rsc.FrontendCORS == nil || !pr.rsc.FrontendCORS.PreservesOrigin() || r == nil { diff --git a/pkg/proxy/engines/key_test.go b/pkg/proxy/engines/key_test.go index 7c340d279..bc3f3b778 100644 --- a/pkg/proxy/engines/key_test.go +++ b/pkg/proxy/engines/key_test.go @@ -37,6 +37,7 @@ import ( "github.com/trickstercache/trickster/v2/pkg/proxy/headers" po "github.com/trickstercache/trickster/v2/pkg/proxy/paths/options" "github.com/trickstercache/trickster/v2/pkg/proxy/request" + proxyurls "github.com/trickstercache/trickster/v2/pkg/proxy/urls" tu "github.com/trickstercache/trickster/v2/pkg/testutil" "github.com/trickstercache/trickster/v2/pkg/timeseries" ) @@ -205,6 +206,106 @@ func TestDeriveCacheKey(t *testing.T) { } } +func TestDeriveCacheKeySeparatesRewrittenUpstreams(t *testing.T) { + path := po.New() + path.CacheKeyParams = []string{"query"} + + derive := func(host string, rewritten bool) string { + t.Helper() + rsc := request.NewResources(&bo.Options{ + Scheme: "http", + Host: "origin.example.com:9090", + }, path, nil, nil, nil, nil) + r := httptest.NewRequest(http.MethodGet, + "http://"+host+"/data?query=value", nil) + r = request.SetResources(r, rsc) + if rewritten { + proxyurls.SetUpstreamHost(r, host) + } + return newProxyRequest(r, nil).DeriveCacheKey("") + } + + unmarkedA := derive("one.example.com", false) + unmarkedB := derive("two.example.com", false) + if unmarkedA != unmarkedB { + t.Errorf("inbound hosts unexpectedly changed cache key: %s != %s", unmarkedA, unmarkedB) + } + + markedA := derive("one.example.com", true) + markedB := derive("two.example.com", true) + if markedA == markedB { + t.Errorf("rewritten upstreams share cache key %s", markedA) + } + if markedA == unmarkedA { + t.Errorf("rewritten and default upstream share cache key %s", markedA) + } + if got := derive("one.example.com", true); got != markedA { + t.Errorf("same rewritten upstream produced unstable keys: %s != %s", got, markedA) + } + + path.KeyHasher = exampleKeyHasher + customA := derive("one.example.com", true) + customB := derive("two.example.com", true) + if customA == customB { + t.Errorf("custom hasher reused a key across rewritten upstreams: %s", customA) + } + if got := derive("one.example.com", false); got != "test-key" { + t.Errorf("unrewritten custom key = %q, want %q", got, "test-key") + } + + deriveWithoutPath := func(host string) string { + rsc := request.NewResources(&bo.Options{ + Scheme: "http", + Host: "origin.example.com:9090", + }, nil, nil, nil, nil, nil) + r := httptest.NewRequest(http.MethodGet, "http://frontend.example.com/data", nil) + r = request.SetResources(r, rsc) + proxyurls.SetUpstreamHost(r, host) + return newProxyRequest(r, nil).DeriveCacheKey("") + } + if first, second := deriveWithoutPath("one.example.com"), + deriveWithoutPath("two.example.com"); first == second { + t.Errorf("nil path config reused a key across rewritten upstreams: %s", first) + } +} + +func TestDeriveCacheKeyUsesFinalRewrittenUpstream(t *testing.T) { + path := po.New() + path.CacheKeyParams = []string{"query"} + backendOptions := &bo.Options{ + Scheme: "http", + Host: "origin.example.com:9090", + } + + derive := func(frontendURL string, rewrite func(*http.Request)) string { + t.Helper() + rsc := request.NewResources(backendOptions, path, nil, nil, nil, nil) + r := httptest.NewRequest(http.MethodGet, frontendURL+"/data?query=value", nil) + r = request.SetResources(r, rsc) + if rewrite != nil { + rewrite(r) + } + return newProxyRequest(r, nil).DeriveCacheKey("") + } + + setTenant := func(r *http.Request) { + proxyurls.SetUpstreamHostname(r, "tenant.example.com") + } + first := derive("http://frontend.example.com:8480", setTenant) + second := derive("https://other-frontend.example.com:443", setTenant) + if first != second { + t.Errorf("same final upstream produced different keys: %s != %s", first, second) + } + + defaultKey := derive("http://frontend.example.com:8480", nil) + noOpRewrite := derive("http://frontend.example.com:8480", func(r *http.Request) { + proxyurls.SetUpstreamHost(r, backendOptions.Host) + }) + if defaultKey != noOpRewrite { + t.Errorf("no-op upstream rewrite changed key: %s != %s", defaultKey, noOpRewrite) + } +} + func TestDeriveCacheKeyUsesCanonicalTimeRangeQuery(t *testing.T) { canonical := func(tenant string) string { return "SELECT toStartOfMinute(ts) AS t, count() FROM events WHERE tenant = '" + tenant + diff --git a/pkg/proxy/request/rewriter/options/options_extended_test.go b/pkg/proxy/request/rewriter/options/options_extended_test.go index d8f9584ae..940ed921a 100644 --- a/pkg/proxy/request/rewriter/options/options_extended_test.go +++ b/pkg/proxy/request/rewriter/options/options_extended_test.go @@ -86,7 +86,7 @@ func TestUnmarshalYAML(t *testing.T) { request_rewriters: rewrite-host: instructions: - - [set, Host, example.com] + - [hostname, set, '${tenant}.example.com'] ` type doc struct { Rewriters Lookup `yaml:"request_rewriters"` @@ -96,7 +96,7 @@ request_rewriters: t.Fatalf("yaml.Unmarshal: %v", err) } o := d.Rewriters["rewrite-host"] - if o == nil || len(o.Instructions) != 1 || o.Instructions[0][2] != "example.com" { + if o == nil || len(o.Instructions) != 1 || o.Instructions[0][2] != "${tenant}.example.com" { t.Fatalf("unexpected rewriter: %+v", o) } } diff --git a/pkg/proxy/request/rewriter/rewrite_instructions.go b/pkg/proxy/request/rewriter/rewrite_instructions.go index c48902ab7..9d5ed555c 100644 --- a/pkg/proxy/request/rewriter/rewrite_instructions.go +++ b/pkg/proxy/request/rewriter/rewrite_instructions.go @@ -18,6 +18,7 @@ package rewriter import ( "fmt" + "net" "net/http" "net/url" "slices" @@ -26,6 +27,7 @@ import ( "github.com/trickstercache/trickster/v2/pkg/proxy/context" "github.com/trickstercache/trickster/v2/pkg/proxy/request/rewriter/options" + proxyurls "github.com/trickstercache/trickster/v2/pkg/proxy/urls" ) type rewriteInstruction interface { @@ -131,6 +133,7 @@ var scalarSets = map[string]scalarSetFunc{ "scheme": func(r *http.Request, v string) { if r != nil && r.URL != nil { r.URL.Scheme = v + proxyurls.SetUpstreamScheme(r, v) } }, "params": func(r *http.Request, v string) { @@ -146,34 +149,35 @@ var scalarSets = map[string]scalarSetFunc{ "host": func(r *http.Request, v string) { if r != nil && r.URL != nil { r.URL.Host = v + proxyurls.SetUpstreamHost(r, v) } }, "hostname": func(r *http.Request, v string) { if r != nil && r.URL != nil { - h := r.URL.Host - var port string - if i := strings.Index(h, ":"); i > 0 { - port = h[i:] - } - r.URL.Host = v + port + r.URL.Host = joinHostnamePort(v, r.URL.Port()) + proxyurls.SetUpstreamHostname(r, v) } }, "port": func(r *http.Request, v string) { if r == nil || r.URL == nil { return } - h := r.URL.Host - var port string - if i := strings.Index(h, ":"); i > 0 { - h = h[:i] - } - if v != "" { - port = ":" + v - } - r.URL.Host = h + port + r.URL.Host = joinHostnamePort(r.URL.Hostname(), v) + proxyurls.SetUpstreamPort(r, v) }, } +func joinHostnamePort(hostname, port string) string { + hostname = strings.TrimPrefix(strings.TrimSuffix(hostname, "]"), "[") + if port != "" { + return net.JoinHostPort(hostname, port) + } + if strings.Contains(hostname, ":") { + return "[" + hostname + "]" + } + return hostname +} + func (ris RewriteInstructions) String() string { l := make([]string, len(ris)) for i, instr := range ris { @@ -189,14 +193,21 @@ func (ris RewriteInstructions) Execute(r *http.Request) { } } -func checkTokens(input string) bool { - i := strings.Index(input, "${") - if i > -1 && strings.Index(input, "}") > i { - return true +// HasTokens returns true when an instruction consumes rewrite tokens. +func (ris RewriteInstructions) HasTokens() bool { + for _, instr := range ris { + if instr.HasTokens() { + return true + } } return false } +func checkTokens(input string) bool { + _, after, ok := strings.Cut(input, "${") + return ok && strings.IndexByte(after, '}') > -1 +} + // parseKeyBasedInstruction parses a 4-part key-based instruction func parseKeyBasedInstruction(parts []string, dict *dictFunc, key *string, value *string, hasTokens *bool) error { if len(parts) != 4 { @@ -229,7 +240,11 @@ func (ri *rwiKeyBasedSetter) Parse(parts []string) error { func (ri *rwiKeyBasedSetter) Execute(r *http.Request) { dict := ri.dict(r) - dict.Set(ri.key, ri.value) + value := ri.value + if ri.hasTokens { + value = expandTokens(r, value) + } + dict.Set(ri.key, value) if qp, ok := dict.(url.Values); ok { r.URL.RawQuery = qp.Encode() } @@ -258,6 +273,10 @@ type mappable map[string][]string func (ri *rwiKeyBasedAppender) Execute(r *http.Request) { dict := ri.dict(r) + value := ri.value + if ri.hasTokens { + value = expandTokens(r, value) + } var m mappable var ok bool var h http.Header @@ -276,7 +295,7 @@ func (ri *rwiKeyBasedAppender) Execute(r *http.Request) { vals, ok = m[ri.key] // key does not exist, so set value instead of appending if !ok { - dict.Set(ri.key, ri.value) + dict.Set(ri.key, value) if q != nil { r.URL.RawQuery = q.Encode() } @@ -285,11 +304,11 @@ func (ri *rwiKeyBasedAppender) Execute(r *http.Request) { // appending to url param value if q != nil { - if slices.Contains(vals, ri.value) { + if slices.Contains(vals, value) { // the desired value is already in the query, do nothing return } - m[ri.key] = append(vals, ri.value) + m[ri.key] = append(vals, value) r.URL.RawQuery = q.Encode() return } @@ -297,11 +316,11 @@ func (ri *rwiKeyBasedAppender) Execute(r *http.Request) { // appending to header value var subkey string - j := strings.Index(ri.value, "=") + j := strings.Index(value, "=") if j > 0 { - subkey = ri.value[:j] + subkey = value[:j] } else { - subkey = ri.value + subkey = value } // this might look redundant, but it normalizes something like: @@ -311,19 +330,19 @@ func (ri *rwiKeyBasedAppender) Execute(r *http.Request) { var found bool for i, part := range parts { - if part == ri.value { + if part == value { // value exists in header already, nothing to do return } if strings.HasPrefix(part, subkey+"=") { // a right-subkey=wrong-value exists, set it to the right value - parts[i] = ri.value + parts[i] = value found = true } } if !found { - parts = append(parts, ri.value) + parts = append(parts, value) } h.Set(ri.key, strings.Join(parts, ", ")) @@ -361,8 +380,15 @@ func (ri *rwiKeyBasedReplacer) Parse(parts []string) error { } func (ri *rwiKeyBasedReplacer) Execute(r *http.Request) { - if ri.depth == 0 { - ri.depth = -1 + key, search, replacement := ri.key, ri.search, ri.replacement + if ri.hasTokens { + key = expandTokens(r, key) + search = expandTokens(r, search) + replacement = expandTokens(r, replacement) + } + depth := ri.depth + if depth == 0 { + depth = -1 } dict := ri.dict(r) @@ -381,15 +407,15 @@ func (ri *rwiKeyBasedReplacer) Execute(r *http.Request) { m = mappable(q) } - vals, ok = m[ri.key] + vals, ok = m[key] if !ok { return } for i := range vals { - vals[i] = strings.Replace(vals[i], ri.search, ri.replacement, ri.depth) + vals[i] = strings.Replace(vals[i], search, replacement, depth) } - m[ri.key] = vals + m[key] = vals if q != nil { r.URL.RawQuery = q.Encode() @@ -431,9 +457,14 @@ func (ri *rwiKeyBasedDeleter) Parse(parts []string) error { func (ri *rwiKeyBasedDeleter) Execute(r *http.Request) { dict := ri.dict(r) + key, value := ri.key, ri.value + if ri.hasTokens { + key = expandTokens(r, key) + value = expandTokens(r, value) + } - if ri.value == "" { - dict.Del(ri.key) + if value == "" { + dict.Del(key) if qp, ok := dict.(url.Values); ok { r.URL.RawQuery = qp.Encode() } @@ -443,15 +474,15 @@ func (ri *rwiKeyBasedDeleter) Execute(r *http.Request) { found := -1 // url params if qp, ok := dict.(url.Values); ok { - if vals, ok1 := qp[ri.key]; ok1 { + if vals, ok1 := qp[key]; ok1 { for i, v := range vals { - if v == ri.value { + if v == value { found = i break } } if found > -1 { - qp[ri.key] = append(vals[:found], vals[found+1:]...) + qp[key] = append(vals[:found], vals[found+1:]...) r.URL.RawQuery = qp.Encode() } } @@ -459,10 +490,10 @@ func (ri *rwiKeyBasedDeleter) Execute(r *http.Request) { } // headers - val := dict.Get(ri.key) + val := dict.Get(key) parts := strings.Split(val, ", ") for i, part := range parts { - if strings.HasPrefix(part, ri.value+"=") || part == ri.value { + if strings.HasPrefix(part, value+"=") || part == value { found = i break } @@ -470,7 +501,7 @@ func (ri *rwiKeyBasedDeleter) Execute(r *http.Request) { if found > -1 { parts = append(parts[:found], parts[found+1:]...) - dict.Set(ri.key, strings.Join(parts, ", ")) + dict.Set(key, strings.Join(parts, ", ")) } } @@ -514,21 +545,25 @@ func (ri *rwiPathSetter) HasTokens() bool { } func (ri *rwiPathSetter) Execute(r *http.Request) { + value := ri.value + if ri.hasTokens { + value = expandTokens(r, value) + } if ri.depth > -1 { r.URL.Path = strings.TrimPrefix(r.URL.Path, "/") parts := strings.Split(r.URL.Path, "/") if len(parts) >= ri.depth { - parts[ri.depth] = ri.value + parts[ri.depth] = value r.URL.Path = "/" + strings.Join(parts, "/") } return } - if !strings.HasPrefix(ri.value, "/") { - ri.value = "/" + ri.value + if !strings.HasPrefix(value, "/") { + value = "/" + value } - r.URL.Path = ri.value + r.URL.Path = value } type rwiPathReplacer struct { @@ -564,7 +599,12 @@ func (ri *rwiPathReplacer) Parse(parts []string) error { } func (ri *rwiPathReplacer) Execute(r *http.Request) { - r.URL.Path = strings.Replace(r.URL.Path, ri.search, ri.replacement, ri.depth) + search, replacement := ri.search, ri.replacement + if ri.hasTokens { + search = expandTokens(r, search) + replacement = expandTokens(r, replacement) + } + r.URL.Path = strings.Replace(r.URL.Path, search, replacement, ri.depth) } func (ri *rwiPathReplacer) HasTokens() bool { @@ -599,7 +639,11 @@ func (ri *rwiBasicSetter) Parse(parts []string) error { } func (ri *rwiBasicSetter) Execute(r *http.Request) { - ri.setter(r, ri.value) + value := ri.value + if ri.hasTokens { + value = expandTokens(r, value) + } + ri.setter(r, value) } func (ri *rwiBasicSetter) HasTokens() bool { @@ -648,9 +692,16 @@ func (ri *rwiBasicReplacer) Parse(parts []string) error { } func (ri *rwiBasicReplacer) Execute(r *http.Request) { - val := ri.getter(r) - val = strings.Replace(val, ri.search, ri.replacement, ri.depth) - ri.setter(r, val) + search, replacement := ri.search, ri.replacement + if ri.hasTokens { + search = expandTokens(r, search) + replacement = expandTokens(r, replacement) + } + current := ri.getter(r) + value := strings.Replace(current, search, replacement, ri.depth) + if value != current { + ri.setter(r, value) + } } func (ri *rwiBasicReplacer) HasTokens() bool { @@ -669,11 +720,8 @@ func (ri *rwiPortDeleter) Parse([]string) error { func (ri *rwiPortDeleter) Execute(r *http.Request) { if r != nil && r.URL != nil { - h := r.URL.Host - if i := strings.Index(h, ":"); i > 0 { - h = h[:i] - } - r.URL.Host = h + r.URL.Host = joinHostnamePort(r.URL.Hostname(), "") + proxyurls.SetUpstreamPort(r, "") } } @@ -684,6 +732,7 @@ func (ri *rwiPortDeleter) HasTokens() bool { type rwiChainExecutor struct { rewriterName string rewriter RewriteInstructions + hasTokens bool } func (ri *rwiChainExecutor) String() string { @@ -715,5 +764,5 @@ func (ri *rwiChainExecutor) Execute(r *http.Request) { } func (ri *rwiChainExecutor) HasTokens() bool { - return false + return ri.hasTokens } diff --git a/pkg/proxy/request/rewriter/rewrite_instructions_test.go b/pkg/proxy/request/rewriter/rewrite_instructions_test.go index cb5919270..c5190a4e6 100644 --- a/pkg/proxy/request/rewriter/rewrite_instructions_test.go +++ b/pkg/proxy/request/rewriter/rewrite_instructions_test.go @@ -402,7 +402,7 @@ func TestNilRequestGetters(t *testing.T) { } func TestMiscRequestGetters(t *testing.T) { - r := &http.Request{Method: "GET", URL: testURL} + r := &http.Request{Method: "GET", URL: urls.Clone(testURL)} fm := scalarGets["method"] fh := scalarGets["hostname"] @@ -418,7 +418,7 @@ func TestMiscRequestGetters(t *testing.T) { } func TestMiscRequestSetters(t *testing.T) { - r := &http.Request{Method: "GET", URL: testURL} + r := &http.Request{Method: "GET", URL: urls.Clone(testURL)} fp := scalarSets["port"] fh := scalarSets["hostname"] diff --git a/pkg/proxy/request/rewriter/rewriter.go b/pkg/proxy/request/rewriter/rewriter.go index d1fa1b2d7..48ea481f8 100644 --- a/pkg/proxy/request/rewriter/rewriter.go +++ b/pkg/proxy/request/rewriter/rewriter.go @@ -54,6 +54,24 @@ func ProcessConfigs(rwl options.Lookup) (InstructionsLookup, error) { } } + // Propagate token use through chains without recursively walking cycles. + for range len(crw) { + var changed bool + for _, ri := range crw { + for _, instr := range ri { + ce, ok := instr.(*rwiChainExecutor) + if !ok || ce.hasTokens || !ce.rewriter.HasTokens() { + continue + } + ce.hasTokens = true + changed = true + } + } + if !changed { + break + } + } + return crw, nil } diff --git a/pkg/proxy/request/rewriter/tokens.go b/pkg/proxy/request/rewriter/tokens.go new file mode 100644 index 000000000..3dd01836a --- /dev/null +++ b/pkg/proxy/request/rewriter/tokens.go @@ -0,0 +1,82 @@ +/* + * Copyright 2018 The Trickster Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package rewriter + +import ( + "context" + "maps" + "net/http" + "strings" +) + +type tokenContextKey struct{} + +// WithTokens returns a shallow copy of r carrying request-scoped rewrite tokens. +func WithTokens(r *http.Request, tokens map[string]string) *http.Request { + if r == nil { + return nil + } + ctx := context.WithValue(r.Context(), tokenContextKey{}, maps.Clone(tokens)) + return r.WithContext(ctx) +} + +// WithoutTokens returns a shallow copy of r without rewrite tokens. +func WithoutTokens(r *http.Request) *http.Request { + if len(tokensFromRequest(r)) == 0 { + return r + } + return WithTokens(r, nil) +} + +func tokensFromRequest(r *http.Request) map[string]string { + if r == nil { + return nil + } + tokens, _ := r.Context().Value(tokenContextKey{}).(map[string]string) + return tokens +} + +func expandTokens(r *http.Request, input string) string { + tokens := tokensFromRequest(r) + if len(tokens) == 0 || !checkTokens(input) { + return input + } + + var output strings.Builder + for len(input) > 0 { + start := strings.Index(input, "${") + if start < 0 { + output.WriteString(input) + break + } + output.WriteString(input[:start]) + end := strings.IndexByte(input[start+2:], '}') + if end < 0 { + output.WriteString(input[start:]) + break + } + end += start + 2 + name := input[start+2 : end] + if value, ok := tokens[name]; ok { + output.WriteString(value) + } else { + output.WriteString(input[start : end+1]) + } + input = input[end+1:] + } + return output.String() +} diff --git a/pkg/proxy/request/rewriter/tokens_test.go b/pkg/proxy/request/rewriter/tokens_test.go new file mode 100644 index 000000000..773ac5b75 --- /dev/null +++ b/pkg/proxy/request/rewriter/tokens_test.go @@ -0,0 +1,310 @@ +/* + * Copyright 2018 The Trickster Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package rewriter + +import ( + "net/http" + "net/url" + "slices" + "testing" + + "github.com/trickstercache/trickster/v2/pkg/proxy/request/rewriter/options" + proxyurls "github.com/trickstercache/trickster/v2/pkg/proxy/urls" +) + +func TestRewriteTokensAcrossInstructionTypes(t *testing.T) { + instructions, err := ParseRewriteList(options.RewriteList{ + {"header", "set", "X-Tenant", "${value}:${unknown}"}, + {"param", "set", "tenant", "${value}"}, + {"path", "set", "/tenants/${value}"}, + {"hostname", "set", "${value}.example.com"}, + }) + if err != nil { + t.Fatal(err) + } + req, err := http.NewRequest(http.MethodGet, "http://old.example.com/original", nil) + if err != nil { + t.Fatal(err) + } + req = WithTokens(req, map[string]string{"name": "Tenant", "value": "abc"}) + + instructions.Execute(req) + + if got, want := req.Header.Get("X-Tenant"), "abc:${unknown}"; got != want { + t.Errorf("X-Tenant = %q, want %q", got, want) + } + if got, want := req.URL.Query().Get("tenant"), "abc"; got != want { + t.Errorf("tenant param = %q, want %q", got, want) + } + if got, want := req.URL.Path, "/tenants/abc"; got != want { + t.Errorf("path = %q, want %q", got, want) + } + if got, want := req.URL.Hostname(), "abc.example.com"; got != want { + t.Errorf("hostname = %q, want %q", got, want) + } +} + +func TestRewriteTokensInKeyBasedInstructions(t *testing.T) { + instructions, err := ParseRewriteList(options.RewriteList{ + {"header", "set", "X-Tenant", "before-${value}"}, + {"header", "replace", "X-${name}", "${value}", "after"}, + {"header", "append", "X-Tenant", "scope=${value}"}, + {"header", "set", "X-Delete", "scope=abc, keep=true"}, + {"header", "delete", "X-Delete", "scope=${value}"}, + {"param", "set", "Tenant", "before-${value}"}, + {"param", "replace", "${name}", "${value}", "after"}, + {"param", "append", "Tenant", "extra-${value}"}, + {"param", "set", "delete-Tenant", "${value}"}, + {"param", "delete", "delete-${name}", "${value}"}, + }) + if err != nil { + t.Fatal(err) + } + tokens := map[string]string{"name": "Tenant", "value": "abc"} + req, err := http.NewRequest(http.MethodGet, "http://example.com/", nil) + if err != nil { + t.Fatal(err) + } + req = WithTokens(req, tokens) + tokens["value"] = "changed" + + instructions.Execute(req) + + if got, want := req.Header.Get("X-Tenant"), "before-after, scope=abc"; got != want { + t.Errorf("X-Tenant = %q, want %q", got, want) + } + if got, want := req.Header.Get("X-Delete"), "keep=true"; got != want { + t.Errorf("X-Delete = %q, want %q", got, want) + } + if got, want := req.URL.Query()["Tenant"], []string{"before-after", "extra-abc"}; !slices.Equal(got, want) { + t.Errorf("Tenant params = %q, want %q", got, want) + } + if req.URL.Query().Has("delete-Tenant") { + t.Errorf("delete param remains in %q", req.URL.RawQuery) + } +} + +func TestRewriteTokensInScalarAndPathInstructions(t *testing.T) { + t.Run("path", func(t *testing.T) { + instructions, err := ParseRewriteList(options.RewriteList{ + {"path", "set", "/tenants/${value}/old"}, + {"path", "set", "${value}", "1"}, + {"path", "replace", "${value}", "${name}", "1"}, + }) + if err != nil { + t.Fatal(err) + } + req, err := http.NewRequest(http.MethodGet, "http://example.com/original", nil) + if err != nil { + t.Fatal(err) + } + req = WithTokens(req, map[string]string{"name": "Tenant", "value": "abc"}) + + instructions.Execute(req) + + if got, want := req.URL.Path, "/tenants/Tenant/old"; got != want { + t.Errorf("path = %q, want %q", got, want) + } + }) + + t.Run("scalars", func(t *testing.T) { + instructions, err := ParseRewriteList(options.RewriteList{ + {"params", "set", "tenant=${value}&state=old"}, + {"params", "replace", "old", "${name}"}, + {"method", "set", "${method}"}, + {"host", "set", "${value}.example.com:${port}"}, + {"host", "replace", "${value}", "${name}"}, + {"hostname", "replace", "${name}", "${value}"}, + {"port", "replace", "${port}", "8080"}, + {"scheme", "set", "${scheme}"}, + }) + if err != nil { + t.Fatal(err) + } + req, err := http.NewRequest(http.MethodGet, "http://old.example.com/original", nil) + if err != nil { + t.Fatal(err) + } + req = WithTokens(req, map[string]string{ + "method": "POST", + "name": "Tenant", + "port": "9090", + "scheme": "https", + "value": "abc", + }) + + instructions.Execute(req) + + if got, want := req.Method, http.MethodPost; got != want { + t.Errorf("method = %q, want %q", got, want) + } + if got, want := req.URL.String(), + "https://abc.example.com:8080/original?tenant=abc&state=Tenant"; got != want { + t.Errorf("URL = %q, want %q", got, want) + } + }) +} + +func TestRewriteTokensPassThroughChains(t *testing.T) { + lookup, err := ProcessConfigs(options.Lookup{ + "parent": { + Instructions: options.RewriteList{{"chain", "exec", "child"}}, + }, + "child": { + Instructions: options.RewriteList{{"header", "set", "X-Chain", "${value}"}}, + }, + }) + if err != nil { + t.Fatal(err) + } + req, err := http.NewRequest(http.MethodGet, "http://example.com/", nil) + if err != nil { + t.Fatal(err) + } + req = WithTokens(req, map[string]string{"value": "abc"}) + + lookup["parent"].Execute(req) + + if got, want := req.Header.Get("X-Chain"), "abc"; got != want { + t.Errorf("X-Chain = %q, want %q", got, want) + } + if !lookup["parent"].HasTokens() { + t.Error("parent chain did not propagate token use") + } +} + +func TestUnmatchedAuthorityReplacementDoesNotOverrideUpstream(t *testing.T) { + instructions, err := ParseRewriteList(options.RewriteList{ + {"hostname", "replace", "missing.example.com", "other.example.com"}, + }) + if err != nil { + t.Fatal(err) + } + req, err := http.NewRequest(http.MethodGet, "http://frontend.example.com/path", nil) + if err != nil { + t.Fatal(err) + } + + instructions.Execute(req) + + base, err := url.Parse("http://origin.example.com:9090") + if err != nil { + t.Fatal(err) + } + if got, want := proxyurls.BuildUpstreamURL(req, base).Host, + "origin.example.com:9090"; got != want { + t.Errorf("upstream host = %q, want %q", got, want) + } +} + +func TestAuthorityRewritersSupportIPv6(t *testing.T) { + instructions, err := ParseRewriteList(options.RewriteList{ + {"hostname", "set", "2001:db8::2"}, + {"port", "set", "8443"}, + }) + if err != nil { + t.Fatal(err) + } + req, err := http.NewRequest(http.MethodGet, "http://[2001:db8::1]:9090/path", nil) + if err != nil { + t.Fatal(err) + } + + instructions.Execute(req) + + if got, want := req.URL.Host, "[2001:db8::2]:8443"; got != want { + t.Errorf("request host = %q, want %q", got, want) + } + base, err := url.Parse("http://[2001:db8::3]:7070") + if err != nil { + t.Fatal(err) + } + if got, want := proxyurls.BuildUpstreamURL(req, base).Host, + "[2001:db8::2]:8443"; got != want { + t.Errorf("upstream host = %q, want %q", got, want) + } + + deletePort, err := ParseRewriteList(options.RewriteList{{"port", "delete"}}) + if err != nil { + t.Fatal(err) + } + deletePort.Execute(req) + if got, want := req.URL.Host, "[2001:db8::2]"; got != want { + t.Errorf("request host after port delete = %q, want %q", got, want) + } +} + +func TestWithTokensNilRequest(t *testing.T) { + if WithTokens(nil, map[string]string{"value": "abc"}) != nil { + t.Fatal("expected nil request") + } + if WithoutTokens(nil) != nil { + t.Fatal("expected nil request") + } +} + +func TestWithoutTokens(t *testing.T) { + req, err := http.NewRequest(http.MethodGet, "http://example.com/", nil) + if err != nil { + t.Fatal(err) + } + req = WithTokens(req, map[string]string{"value": "abc"}) + req = WithoutTokens(req) + if got, want := expandTokens(req, "${value}"), "${value}"; got != want { + t.Fatalf("expanded value = %q, want %q", got, want) + } +} + +func TestWithoutTokensReturnsUnchangedRequestWhenEmpty(t *testing.T) { + req, err := http.NewRequest(http.MethodGet, "http://example.com/", nil) + if err != nil { + t.Fatal(err) + } + if got := WithoutTokens(req); got != req { + t.Fatal("request without tokens was cloned") + } +} + +func TestExpandTokens(t *testing.T) { + req, err := http.NewRequest(http.MethodGet, "http://example.com/", nil) + if err != nil { + t.Fatal(err) + } + req = WithTokens(req, map[string]string{ + "empty": "", + "nested": "${value}", + "value": "abc", + }) + tests := []struct { + name, input, want string + }{ + {"single", "prefix-${value}-suffix", "prefix-abc-suffix"}, + {"multiple", "${value}/${empty}/${value}", "abc//abc"}, + {"leading closing brace", "prefix}${value}", "prefix}abc"}, + {"unknown", "${unknown}", "${unknown}"}, + {"not recursive", "${nested}", "${value}"}, + {"unterminated", "${value", "${value"}, + {"plain", "value", "value"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if got := expandTokens(req, test.input); got != test.want { + t.Errorf("expandTokens(%q) = %q, want %q", test.input, got, test.want) + } + }) + } +} diff --git a/pkg/proxy/urls/upstream_rewrites.go b/pkg/proxy/urls/upstream_rewrites.go new file mode 100644 index 000000000..4a9bd0dfa --- /dev/null +++ b/pkg/proxy/urls/upstream_rewrites.go @@ -0,0 +1,132 @@ +/* + * Copyright 2018 The Trickster Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package urls + +import ( + "context" + "net" + "net/http" + "net/url" + "slices" + "strings" +) + +type upstreamURLComponent uint8 + +const ( + upstreamScheme upstreamURLComponent = iota + upstreamHost + upstreamHostname + upstreamPort +) + +type upstreamURLRewrite struct { + component upstreamURLComponent + value string +} + +type upstreamURLRewritesKey struct{} + +// SetUpstreamScheme records a request rewriter's explicit upstream scheme. +func SetUpstreamScheme(r *http.Request, scheme string) { + setUpstreamURLRewrite(r, upstreamScheme, scheme) +} + +// SetUpstreamHost records a request rewriter's explicit upstream host and port. +func SetUpstreamHost(r *http.Request, host string) { + setUpstreamURLRewrite(r, upstreamHost, host) +} + +// SetUpstreamHostname records a request rewriter's explicit upstream hostname. +func SetUpstreamHostname(r *http.Request, hostname string) { + setUpstreamURLRewrite(r, upstreamHostname, hostname) +} + +// SetUpstreamPort records a request rewriter's explicit upstream port. +func SetUpstreamPort(r *http.Request, port string) { + setUpstreamURLRewrite(r, upstreamPort, port) +} + +func setUpstreamURLRewrite(r *http.Request, component upstreamURLComponent, value string) { + if r == nil { + return + } + rewrites, _ := r.Context().Value(upstreamURLRewritesKey{}).([]upstreamURLRewrite) + rewrites = append(slices.Clone(rewrites), upstreamURLRewrite{ + component: component, + value: value, + }) + *r = *r.WithContext(context.WithValue(r.Context(), upstreamURLRewritesKey{}, rewrites)) +} + +func applyUpstreamURLRewrites(r *http.Request, u *url.URL) { + if r == nil || u == nil { + return + } + rewrites, _ := r.Context().Value(upstreamURLRewritesKey{}).([]upstreamURLRewrite) + for _, rewrite := range rewrites { + switch rewrite.component { + case upstreamScheme: + u.Scheme = rewrite.value + case upstreamHost: + u.Host = rewrite.value + case upstreamHostname: + u.Host = replaceHostname(u.Host, rewrite.value) + case upstreamPort: + u.Host = replacePort(u.Host, rewrite.value) + } + } +} + +// UpstreamURLRewriteCacheKey returns a cache key component when a request +// rewriter changed the configured upstream authority. +func UpstreamURLRewriteCacheKey(r *http.Request, base *url.URL) string { + if r == nil || base == nil { + return "" + } + rewrites, _ := r.Context().Value(upstreamURLRewritesKey{}).([]upstreamURLRewrite) + if len(rewrites) == 0 { + return "" + } + rewritten := Clone(base) + applyUpstreamURLRewrites(r, rewritten) + if rewritten.Scheme == base.Scheme && rewritten.Host == base.Host { + return "" + } + return "\x00upstream=" + rewritten.Scheme + "://" + rewritten.Host + "\x00" +} + +func replaceHostname(host, hostname string) string { + port := (&url.URL{Host: host}).Port() + return joinHostPort(hostname, port) +} + +func replacePort(host, port string) string { + hostname := (&url.URL{Host: host}).Hostname() + return joinHostPort(hostname, port) +} + +func joinHostPort(hostname, port string) string { + hostname = strings.TrimPrefix(strings.TrimSuffix(hostname, "]"), "[") + if port != "" { + return net.JoinHostPort(hostname, port) + } + if strings.Contains(hostname, ":") { + return "[" + hostname + "]" + } + return hostname +} diff --git a/pkg/proxy/urls/upstream_rewrites_test.go b/pkg/proxy/urls/upstream_rewrites_test.go new file mode 100644 index 000000000..1478f7bcb --- /dev/null +++ b/pkg/proxy/urls/upstream_rewrites_test.go @@ -0,0 +1,163 @@ +/* + * Copyright 2018 The Trickster Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package urls + +import ( + "net/http" + "net/url" + "testing" +) + +func TestBuildUpstreamURLAppliesExplicitRewrites(t *testing.T) { + base, err := url.Parse("http://origin.example.com:9090/base") + if err != nil { + t.Fatal(err) + } + + tests := []struct { + name string + rewrite func(*http.Request) + wantURL string + }{ + { + name: "inbound host is ignored without a rewrite", + wantURL: "http://origin.example.com:9090/base/path?query=value", + }, + { + name: "scheme", + rewrite: func(r *http.Request) { + SetUpstreamScheme(r, "https") + }, + wantURL: "https://origin.example.com:9090/base/path?query=value", + }, + { + name: "host", + rewrite: func(r *http.Request) { + SetUpstreamHost(r, "other.example.com:7070") + }, + wantURL: "http://other.example.com:7070/base/path?query=value", + }, + { + name: "hostname preserves origin port", + rewrite: func(r *http.Request) { + SetUpstreamHostname(r, "tenant.origin.example.com") + }, + wantURL: "http://tenant.origin.example.com:9090/base/path?query=value", + }, + { + name: "port preserves origin hostname", + rewrite: func(r *http.Request) { + SetUpstreamPort(r, "8443") + }, + wantURL: "http://origin.example.com:8443/base/path?query=value", + }, + { + name: "later host supersedes component rewrites", + rewrite: func(r *http.Request) { + SetUpstreamHostname(r, "tenant.origin.example.com") + SetUpstreamPort(r, "8443") + SetUpstreamHost(r, "final.example.com:6060") + }, + wantURL: "http://final.example.com:6060/base/path?query=value", + }, + { + name: "components update a prior host rewrite", + rewrite: func(r *http.Request) { + SetUpstreamHost(r, "first.example.com:7070") + SetUpstreamHostname(r, "final.example.com") + SetUpstreamPort(r, "6060") + }, + wantURL: "http://final.example.com:6060/base/path?query=value", + }, + { + name: "empty port removes origin port", + rewrite: func(r *http.Request) { + SetUpstreamPort(r, "") + }, + wantURL: "http://origin.example.com/base/path?query=value", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + r, err := http.NewRequest(http.MethodGet, + "http://trickster.example.com:8480/path?query=value", nil) + if err != nil { + t.Fatal(err) + } + if test.rewrite != nil { + test.rewrite(r) + } + + got := BuildUpstreamURL(r, base) + if got.String() != test.wantURL { + t.Errorf("URL = %q, want %q", got.String(), test.wantURL) + } + if got := base.String(); got != "http://origin.example.com:9090/base" { + t.Errorf("base URL was modified: %q", got) + } + }) + } +} + +func TestUpstreamURLRewritesAreIsolatedAfterClone(t *testing.T) { + base, err := url.Parse("http://origin.example.com:9090") + if err != nil { + t.Fatal(err) + } + r, err := http.NewRequest(http.MethodGet, "http://trickster.example.com/path", nil) + if err != nil { + t.Fatal(err) + } + SetUpstreamHostname(r, "parent.example.com") + + clone := r.Clone(r.Context()) + SetUpstreamHostname(clone, "clone.example.com") + + if got, want := BuildUpstreamURL(r, base).Host, "parent.example.com:9090"; got != want { + t.Errorf("parent host = %q, want %q", got, want) + } + if got, want := BuildUpstreamURL(clone, base).Host, "clone.example.com:9090"; got != want { + t.Errorf("clone host = %q, want %q", got, want) + } +} + +func TestUpstreamURLRewriteSupportsIPv6(t *testing.T) { + base, err := url.Parse("http://[2001:db8::1]:9090") + if err != nil { + t.Fatal(err) + } + r, err := http.NewRequest(http.MethodGet, "http://trickster.example.com/path", nil) + if err != nil { + t.Fatal(err) + } + SetUpstreamHostname(r, "2001:db8::2") + + if got, want := BuildUpstreamURL(r, base).Host, "[2001:db8::2]:9090"; got != want { + t.Errorf("host = %q, want %q", got, want) + } +} + +func TestSetUpstreamURLRewriteWithNilRequest(t *testing.T) { + SetUpstreamScheme(nil, "https") + SetUpstreamHost(nil, "example.com") + SetUpstreamHostname(nil, "example.com") + SetUpstreamPort(nil, "443") + if got := UpstreamURLRewriteCacheKey(nil, nil); got != "" { + t.Errorf("cache key = %q, want empty", got) + } +} diff --git a/pkg/proxy/urls/url.go b/pkg/proxy/urls/url.go index 86bf52dec..9b99df6ed 100644 --- a/pkg/proxy/urls/url.go +++ b/pkg/proxy/urls/url.go @@ -53,6 +53,7 @@ func FromParts(scheme, host, path, query, fragment string) *url.URL { // to construct the full upstream URL func BuildUpstreamURL(r *http.Request, u *url.URL) *url.URL { u2 := Clone(u) + applyUpstreamURLRewrites(r, u2) u2.Path += r.URL.Path u2.RawQuery = r.URL.RawQuery u2.Fragment = r.URL.Fragment