diff --git a/plugin/output/elasticsearch/README.md b/plugin/output/elasticsearch/README.md index bcf034035..00df09dc5 100755 --- a/plugin/output/elasticsearch/README.md +++ b/plugin/output/elasticsearch/README.md @@ -170,5 +170,18 @@ Process ES response and report errors, if any.
+**`ban_period`** *`cfg.Duration`* *`default=10s`* + +Period for which addresses will be banned in case of unavailability. +If set to 0, circuit breaker is disabled. + +
+ +**`reconnect_interval`** *`cfg.Duration`* *`default=5s`* + +Interval for reconnecting to addresses that are unavailable during initialization. + +
+
*Generated using [__insane-doc__](https://github.com/vitkovskii/insane-doc)* \ No newline at end of file diff --git a/plugin/output/elasticsearch/elasticsearch.go b/plugin/output/elasticsearch/elasticsearch.go index 4bdaea55e..5daccf124 100644 --- a/plugin/output/elasticsearch/elasticsearch.go +++ b/plugin/output/elasticsearch/elasticsearch.go @@ -203,6 +203,19 @@ type Config struct { // > // > Process ES response and report errors, if any. ProcessResponse bool `json:"process_response" default:"true"` // * + + // > @3@4@5@6 + // > + // > Period for which addresses will be banned in case of unavailability. + // > If set to 0, circuit breaker is disabled. + BanPeriod cfg.Duration `json:"ban_period" default:"10s" parse:"duration"` // * + BanPeriod_ time.Duration + + // > @3@4@5@6 + // > + // > Interval for reconnecting to addresses that are unavailable during initialization. + ReconnectInterval cfg.Duration `json:"reconnect_interval" default:"5s" parse:"duration"` // * + ReconnectInterval_ time.Duration } type KeepAliveConfig struct { @@ -243,8 +256,17 @@ func (p *Plugin) Start(config pipeline.AnyConfig, params *pipeline.OutputPluginP if len(p.config.IndexValues) == 0 { p.config.IndexValues = append(p.config.IndexValues, "@time") } + if p.config.ReconnectInterval_ < 1 { + p.logger.Fatal("'reconnect_interval' can't be <1") + } + if p.config.BanPeriod_ < 0 { + p.logger.Fatal("'ban_period' cant't be <0") + } + + ctx, cancel := context.WithCancel(context.Background()) + p.cancel = cancel - p.prepareClient() + p.prepareClient(ctx) p.maintenance(nil) @@ -295,9 +317,6 @@ func (p *Plugin) Start(config pipeline.AnyConfig, params *pipeline.OutputPluginP onError, ) - ctx, cancel := context.WithCancel(context.Background()) - p.cancel = cancel - p.batcher.Start(ctx) } @@ -315,11 +334,13 @@ func (p *Plugin) registerMetrics(ctl *metric.Ctl) { p.indexingErrorsMetric = ctl.RegisterCounter("output_elasticsearch_index_error_total", "Number of elasticsearch indexing errors") } -func (p *Plugin) prepareClient() { +func (p *Plugin) prepareClient(ctx context.Context) { config := &xhttp.ClientConfig{ Endpoints: prepareEndpoints(p.config.Endpoints, p.config.IngestPipeline), ConnectionTimeout: p.config.ConnectionTimeout_ * 2, AuthHeader: p.getAuthHeader(), + BanPeriod: p.config.BanPeriod_, + ReconnectInterval: p.config.ReconnectInterval_, KeepAlive: &xhttp.ClientKeepAliveConfig{ MaxConnDuration: p.config.KeepAlive.MaxConnDuration_, MaxIdleConnDuration: p.config.KeepAlive.MaxIdleConnDuration_, @@ -335,7 +356,7 @@ func (p *Plugin) prepareClient() { } var err error - p.client, err = xhttp.NewClient(config) + p.client, err = xhttp.NewClient(ctx, config) if err != nil { p.logger.Fatal("can't create http client", zap.Error(err)) } diff --git a/plugin/output/http/README.md b/plugin/output/http/README.md index 3c3810aaa..79d6ea6af 100755 --- a/plugin/output/http/README.md +++ b/plugin/output/http/README.md @@ -144,4 +144,17 @@ After a non-retryable write error, fall with a non-zero exit code or not
+**`ban_period`** *`cfg.Duration`* *`default=10s`* + +Period for which addresses will be banned in case of unavailability. +If set to 0, circuit breaker is disabled. + +
+ +**`reconnect_interval`** *`cfg.Duration`* *`default=5s`* + +Interval for reconnecting to addresses that are unavailable during initialization. + +
+
*Generated using [__insane-doc__](https://github.com/vitkovskii/insane-doc)* \ No newline at end of file diff --git a/plugin/output/http/http.go b/plugin/output/http/http.go index 03edf9c51..edb9f9cfb 100644 --- a/plugin/output/http/http.go +++ b/plugin/output/http/http.go @@ -176,6 +176,19 @@ type Config struct { // > // > After a non-retryable write error, fall with a non-zero exit code or not Strict bool `json:"strict" default:"false"` // * + + // > @3@4@5@6 + // > + // > Period for which addresses will be banned in case of unavailability. + // > If set to 0, circuit breaker is disabled. + BanPeriod cfg.Duration `json:"ban_period" default:"10s" parse:"duration"` // * + BanPeriod_ time.Duration + + // > @3@4@5@6 + // > + // > Interval for reconnecting to addresses that are unavailable during initialization. + ReconnectInterval cfg.Duration `json:"reconnect_interval" default:"5s" parse:"duration"` // * + ReconnectInterval_ time.Duration } type KeepAliveConfig struct { @@ -212,13 +225,23 @@ func (p *Plugin) Start(config pipeline.AnyConfig, params *pipeline.OutputPluginP p.registerMetrics(params.MetricCtl) p.mu = &sync.Mutex{} + if p.config.ReconnectInterval_ < 1 { + p.logger.Fatal("'reconnect_interval' can't be <1") + } + if p.config.BanPeriod_ < 0 { + p.logger.Fatal("'ban_period' cant't be <0") + } + var err error p.encoder, err = NewEncoder(p.config.Encoding) if err != nil { p.logger.Fatal("can't create encoder", zap.Error(err)) } - p.prepareClient() + ctx, cancel := context.WithCancel(context.Background()) + p.cancel = cancel + + p.prepareClient(ctx) p.logger.Info("starting batcher", zap.Duration("timeout", p.config.BatchFlushTimeout_)) @@ -267,9 +290,6 @@ func (p *Plugin) Start(config pipeline.AnyConfig, params *pipeline.OutputPluginP onError, ) - ctx, cancel := context.WithCancel(context.Background()) - p.cancel = cancel - p.batcher.Start(ctx) } @@ -286,11 +306,13 @@ func (p *Plugin) registerMetrics(ctl *metric.Ctl) { p.sendErrorMetric = ctl.RegisterCounterVec("output_http_send_error_total", "Total HTTP send errors", "status_code") } -func (p *Plugin) prepareClient() { +func (p *Plugin) prepareClient(ctx context.Context) { config := &xhttp.ClientConfig{ Endpoints: p.prepareEndpoints(), ConnectionTimeout: p.config.ConnectionTimeout_ * 2, AuthHeader: p.getAuthHeader(), + BanPeriod: p.config.BanPeriod_, + ReconnectInterval: p.config.ReconnectInterval_, KeepAlive: &xhttp.ClientKeepAliveConfig{ MaxConnDuration: p.config.KeepAlive.MaxConnDuration_, MaxIdleConnDuration: p.config.KeepAlive.MaxIdleConnDuration_, @@ -306,7 +328,7 @@ func (p *Plugin) prepareClient() { } var err error - p.client, err = xhttp.NewClient(config) + p.client, err = xhttp.NewClient(ctx, config) if err != nil { p.logger.Fatal("can't create http client", zap.Error(err)) } diff --git a/plugin/output/loki/README.md b/plugin/output/loki/README.md index 1ce47cee1..b08ced5d4 100644 --- a/plugin/output/loki/README.md +++ b/plugin/output/loki/README.md @@ -149,5 +149,18 @@ Multiplier for exponential increase of retention between retries
+**`ban_period`** *`cfg.Duration`* *`default=10s`* + +Period for which addresses will be banned in case of unavailability. +If set to 0, circuit breaker is disabled. + +
+ +**`reconnect_interval`** *`cfg.Duration`* *`default=5s`* + +Interval for reconnecting to addresses that are unavailable during initialization. + +
+
*Generated using [__insane-doc__](https://github.com/vitkovskii/insane-doc)* \ No newline at end of file diff --git a/plugin/output/loki/loki.go b/plugin/output/loki/loki.go index 160054675..b5753c5db 100644 --- a/plugin/output/loki/loki.go +++ b/plugin/output/loki/loki.go @@ -178,6 +178,19 @@ type Config struct { // > // > Multiplier for exponential increase of retention between retries RetentionExponentMultiplier int `json:"retention_exponentially_multiplier" default:"2"` // * + + // > @3@4@5@6 + // > + // > Period for which addresses will be banned in case of unavailability. + // > If set to 0, circuit breaker is disabled. + BanPeriod cfg.Duration `json:"ban_period" default:"10s" parse:"duration"` // * + BanPeriod_ time.Duration + + // > @3@4@5@6 + // > + // > Interval for reconnecting to addresses that are unavailable during initialization. + ReconnectInterval cfg.Duration `json:"reconnect_interval" default:"5s" parse:"duration"` // * + ReconnectInterval_ time.Duration } type AuthStrategy byte @@ -259,7 +272,18 @@ func (p *Plugin) Start(config pipeline.AnyConfig, params *pipeline.OutputPluginP p.labels = p.parseLabels() - p.prepareClient() + if p.config.ReconnectInterval_ < 1 { + p.logger.Fatal("'reconnect_interval' can't be <1") + } + if p.config.BanPeriod_ < 0 { + p.logger.Fatal("'ban_period' cant't be <0") + } + + ctx, cancel := context.WithCancel(context.Background()) + p.ctx = ctx + p.cancel = cancel + + p.prepareClient(ctx) batcherOpts := &pipeline.BatcherOptions{ PipelineName: params.PipelineName, @@ -303,10 +327,6 @@ func (p *Plugin) Start(config pipeline.AnyConfig, params *pipeline.OutputPluginP onError, ) - ctx, cancel := context.WithCancel(context.Background()) - p.ctx = ctx - p.cancel = cancel - p.batcher.Start(ctx) } @@ -430,12 +450,14 @@ func (p *Plugin) registerMetrics(ctl *metric.Ctl) { p.sendErrorMetric = ctl.RegisterCounterVec("output_loki_send_error_total", "Total Loki send errors", "status_code") } -func (p *Plugin) prepareClient() { +func (p *Plugin) prepareClient(ctx context.Context) { config := &xhttp.ClientConfig{ Endpoints: []string{fmt.Sprintf("%s/loki/api/v1/push", p.config.Address)}, ConnectionTimeout: p.config.ConnectionTimeout_ * 2, AuthHeader: p.getAuthHeader(), CustomHeaders: p.getCustomHeaders(), + BanPeriod: p.config.BanPeriod_, + ReconnectInterval: p.config.ReconnectInterval_, KeepAlive: &xhttp.ClientKeepAliveConfig{ MaxConnDuration: p.config.KeepAlive.MaxConnDuration_, MaxIdleConnDuration: p.config.KeepAlive.MaxIdleConnDuration_, @@ -443,7 +465,7 @@ func (p *Plugin) prepareClient() { } var err error - p.client, err = xhttp.NewClient(config) + p.client, err = xhttp.NewClient(ctx, config) if err != nil { p.logger.Fatal("can't create http client", zap.Error(err)) } diff --git a/plugin/output/splunk/README.md b/plugin/output/splunk/README.md index b9df5013f..55aa91e39 100755 --- a/plugin/output/splunk/README.md +++ b/plugin/output/splunk/README.md @@ -153,5 +153,18 @@ or the "event" key with any of its subkeys.
+**`ban_period`** *`cfg.Duration`* *`default=10s`* + +Period for which addresses will be banned in case of unavailability. +If set to 0, circuit breaker is disabled. + +
+ +**`reconnect_interval`** *`cfg.Duration`* *`default=5s`* + +Interval for reconnecting to addresses that are unavailable during initialization. + +
+
*Generated using [__insane-doc__](https://github.com/vitkovskii/insane-doc)* \ No newline at end of file diff --git a/plugin/output/splunk/splunk.go b/plugin/output/splunk/splunk.go index 090ae5048..742eddb32 100644 --- a/plugin/output/splunk/splunk.go +++ b/plugin/output/splunk/splunk.go @@ -202,6 +202,19 @@ type Config struct { // > Supports copying whole original event, but does not allow to copy directly to the output root // > or the "event" key with any of its subkeys. CopyFields []CopyField `json:"copy_fields" slice:"true"` // * + + // > @3@4@5@6 + // > + // > Period for which addresses will be banned in case of unavailability. + // > If set to 0, circuit breaker is disabled. + BanPeriod cfg.Duration `json:"ban_period" default:"10s" parse:"duration"` // * + BanPeriod_ time.Duration + + // > @3@4@5@6 + // > + // > Interval for reconnecting to addresses that are unavailable during initialization. + ReconnectInterval cfg.Duration `json:"reconnect_interval" default:"5s" parse:"duration"` // * + ReconnectInterval_ time.Duration } type KeepAliveConfig struct { @@ -235,7 +248,18 @@ func (p *Plugin) Start(config pipeline.AnyConfig, params *pipeline.OutputPluginP p.avgEventSize = params.PipelineSettings.AvgEventSize p.config = config.(*Config) p.registerMetrics(params.MetricCtl) - p.prepareClient() + + if p.config.ReconnectInterval_ < 1 { + p.logger.Fatal("'reconnect_interval' can't be <1") + } + if p.config.BanPeriod_ < 0 { + p.logger.Fatal("'ban_period' cant't be <0") + } + + ctx, cancel := context.WithCancel(context.Background()) + p.cancel = cancel + + p.prepareClient(ctx) for _, cf := range p.config.CopyFields { if cf.To == "" { @@ -296,9 +320,6 @@ func (p *Plugin) Start(config pipeline.AnyConfig, params *pipeline.OutputPluginP onError, ) - ctx, cancel := context.WithCancel(context.Background()) - p.cancel = cancel - p.batcher.Start(ctx) } @@ -319,11 +340,13 @@ func (p *Plugin) registerMetrics(ctl *metric.Ctl) { ) } -func (p *Plugin) prepareClient() { +func (p *Plugin) prepareClient(ctx context.Context) { config := &xhttp.ClientConfig{ Endpoints: []string{p.config.Endpoint}, ConnectionTimeout: p.config.RequestTimeout_, AuthHeader: "Splunk " + p.config.Token, + BanPeriod: p.config.BanPeriod_, + ReconnectInterval: p.config.ReconnectInterval_, KeepAlive: &xhttp.ClientKeepAliveConfig{ MaxConnDuration: p.config.KeepAlive.MaxConnDuration_, MaxIdleConnDuration: p.config.KeepAlive.MaxIdleConnDuration_, @@ -338,7 +361,7 @@ func (p *Plugin) prepareClient() { } var err error - p.client, err = xhttp.NewClient(config) + p.client, err = xhttp.NewClient(ctx, config) if err != nil { p.logger.Fatal("can't create http client", zap.Error(err)) } diff --git a/plugin/output/splunk/splunk_test.go b/plugin/output/splunk/splunk_test.go index 40626e076..fec6de177 100644 --- a/plugin/output/splunk/splunk_test.go +++ b/plugin/output/splunk/splunk_test.go @@ -1,6 +1,7 @@ package splunk import ( + "context" "io" "net/http" "net/http/httptest" @@ -54,7 +55,7 @@ func TestSplunk(t *testing.T) { }, logger: zap.NewExample().Sugar(), } - plugin.prepareClient() + plugin.prepareClient(context.Background()) batch := pipeline.NewPreparedBatch([]*pipeline.Event{ {Root: input}, @@ -185,7 +186,7 @@ func TestCopyFields(t *testing.T) { copyFieldsPaths: tt.copyFields, logger: zap.NewExample().Sugar(), } - plugin.prepareClient() + plugin.prepareClient(context.Background()) batch := pipeline.NewPreparedBatch([]*pipeline.Event{ {Root: input}, diff --git a/xhttp/circuit_breaker.go b/xhttp/circuit_breaker.go new file mode 100644 index 000000000..cf7e0a42c --- /dev/null +++ b/xhttp/circuit_breaker.go @@ -0,0 +1,98 @@ +package xhttp + +import ( + "context" + "math/rand" + "sync" + "time" + + "github.com/ozontech/file.d/xtime" + "github.com/valyala/fasthttp" +) + +type endpoint struct { + uri *fasthttp.URI + banUntil time.Time +} + +type circuitBreaker struct { + endpoints []endpoint + idxByURI map[string]int + banPeriod time.Duration + mu sync.RWMutex +} + +func newCircuitBreaker(ctx context.Context, uris []*fasthttp.URI, banPeriod, reconnectInterval time.Duration) *circuitBreaker { + if banPeriod <= 0 { + return nil + } + + cb := &circuitBreaker{ + endpoints: make([]endpoint, 0, len(uris)), + idxByURI: make(map[string]int, len(uris)), + banPeriod: banPeriod, + } + + for i, uri := range uris { + cb.endpoints = append(cb.endpoints, endpoint{uri: uri}) + cb.idxByURI[uri.String()] = i + } + + go cb.checkBannedEndpoints(ctx, reconnectInterval) + + return cb +} + +func (cb *circuitBreaker) getEndpoint() *fasthttp.URI { + cb.mu.RLock() + defer cb.mu.RUnlock() + + now := xtime.GetInaccurateTime() + activeEndpoints := make([]*fasthttp.URI, 0, len(cb.endpoints)) + for i := range cb.endpoints { + e := cb.endpoints[i] + if e.banUntil.IsZero() || now.After(e.banUntil) { + activeEndpoints = append(activeEndpoints, e.uri) + } + } + + if len(activeEndpoints) == 0 { + return nil + } + return activeEndpoints[rand.Intn(len(activeEndpoints))] +} + +func (cb *circuitBreaker) banEndpoint(uri *fasthttp.URI) { + cb.mu.Lock() + defer cb.mu.Unlock() + + idx := cb.idxByURI[uri.String()] + cb.endpoints[idx].banUntil = xtime.GetInaccurateTime().Add(cb.banPeriod) +} + +func (cb *circuitBreaker) restoreBannedEndpoints() { + cb.mu.Lock() + defer cb.mu.Unlock() + + now := xtime.GetInaccurateTime() + for i := range cb.endpoints { + e := &cb.endpoints[i] + if !e.banUntil.IsZero() && now.After(e.banUntil) { + e.banUntil = time.Time{} + } + } +} + +func (cb *circuitBreaker) checkBannedEndpoints(ctx context.Context, reconnectInterval time.Duration) { + ticker := time.NewTicker(reconnectInterval) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + cb.restoreBannedEndpoints() + } + } +} diff --git a/xhttp/client.go b/xhttp/client.go index f9ea6e6a9..ec9bb9880 100644 --- a/xhttp/client.go +++ b/xhttp/client.go @@ -1,6 +1,7 @@ package xhttp import ( + "context" "fmt" "math/rand" "net/http" @@ -30,17 +31,20 @@ type ClientConfig struct { GzipCompressionLevel string TLS *ClientTLSConfig KeepAlive *ClientKeepAliveConfig + BanPeriod time.Duration + ReconnectInterval time.Duration } type Client struct { client *fasthttp.Client endpoints []*fasthttp.URI + cb *circuitBreaker authHeader string customHeaders map[string]string gzipCompressionLevel int } -func NewClient(cfg *ClientConfig) (*Client, error) { +func NewClient(ctx context.Context, cfg *ClientConfig) (*Client, error) { client := &fasthttp.Client{ ReadTimeout: cfg.ConnectionTimeout, WriteTimeout: cfg.ConnectionTimeout, @@ -72,6 +76,7 @@ func NewClient(cfg *ClientConfig) (*Client, error) { return &Client{ client: client, endpoints: endpoints, + cb: newCircuitBreaker(ctx, endpoints, cfg.BanPeriod, cfg.ReconnectInterval), authHeader: cfg.AuthHeader, customHeaders: cfg.CustomHeaders, gzipCompressionLevel: parseGzipCompressionLevel(cfg.GzipCompressionLevel), @@ -89,16 +94,15 @@ func (c *Client) DoTimeout( resp := fasthttp.AcquireResponse() defer fasthttp.ReleaseResponse(resp) - var endpoint *fasthttp.URI - if len(c.endpoints) == 1 { - endpoint = c.endpoints[0] - } else { - endpoint = c.endpoints[rand.Int()%len(c.endpoints)] + endpoint := c.getEndpoint() + if endpoint == nil { + return 0, fmt.Errorf("no available endpoints") } c.prepareRequest(req, endpoint, method, contentType, body) if err := c.client.DoTimeout(req, resp, timeout); err != nil { + c.banEndpoint(endpoint) return 0, fmt.Errorf("can't send request to %s: %w", endpoint.String(), err) } @@ -106,6 +110,9 @@ func (c *Client) DoTimeout( statusCode := resp.Header.StatusCode() if !(http.StatusOK <= statusCode && statusCode <= http.StatusAccepted) { + if shouldBanEndpoint(statusCode) { + c.banEndpoint(endpoint) + } return statusCode, fmt.Errorf("response status from %s isn't OK: status=%d, body=%s", endpoint.String(), statusCode, string(respContent)) } @@ -168,3 +175,32 @@ func parseGzipCompressionLevel(level string) int { return -1 } } + +func (c *Client) getEndpoint() *fasthttp.URI { + if c.cb != nil { + return c.cb.getEndpoint() + } + + if len(c.endpoints) == 0 { + return nil + } + return c.endpoints[rand.Intn(len(c.endpoints))] +} + +func (c *Client) banEndpoint(endpoint *fasthttp.URI) { + if c.cb != nil { + c.cb.banEndpoint(endpoint) + } +} + +func shouldBanEndpoint(statusCode int) bool { + switch statusCode { + case http.StatusBadGateway, + http.StatusServiceUnavailable, + http.StatusGatewayTimeout, + http.StatusTooManyRequests: + return true + default: + return false + } +}