From 68d08ae6667be3d573775cc672395d3326bdb4ff Mon Sep 17 00:00:00 2001 From: Ben Moskovitz Date: Thu, 30 Apr 2026 17:49:11 +1000 Subject: [PATCH 1/3] Replace dogstatsd metrics with opentelemetry metrics --- agent/run_job.go | 10 +- clicommand/agent_start.go | 34 +-- go.mod | 8 +- go.sum | 43 +-- metrics/metrics.go | 250 +++++++++++++----- .../share/buildkite-agent/buildkite-agent.cfg | 13 +- 6 files changed, 216 insertions(+), 142 deletions(-) diff --git a/agent/run_job.go b/agent/run_job.go index a656d748b2..015c094225 100644 --- a/agent/run_job.go +++ b/agent/run_job.go @@ -433,14 +433,8 @@ func (r *JobRunner) cleanup(ctx context.Context, wg *sync.WaitGroup, exit core.P // Write some metrics about the job run jobMetrics := r.conf.MetricsScope.With(metrics.Tags{"exit_code": strconv.Itoa(exit.Status)}) - - if exit.Status == 0 { - jobMetrics.Timing("jobs.duration.success", finishedAt.Sub(r.startedAt)) - jobMetrics.Count("jobs.success", 1) - } else { - jobMetrics.Timing("jobs.duration.error", finishedAt.Sub(r.startedAt)) - jobMetrics.Count("jobs.failed", 1) - } + jobMetrics.Timing("jobs.duration", finishedAt.Sub(r.startedAt)) + jobMetrics.Count("jobs.finished", 1) // Finish the build in the Buildkite Agent API // Once we tell the API we're finished it might assign us new work, so make sure everything else is done first. diff --git a/clicommand/agent_start.go b/clicommand/agent_start.go index d61e733c79..79b7a331a5 100644 --- a/clicommand/agent_start.go +++ b/clicommand/agent_start.go @@ -188,10 +188,8 @@ type AgentStartConfig struct { HealthCheckAddr string `cli:"health-check-addr"` - // Datadog statsd metrics config - MetricsDatadog bool `cli:"metrics-datadog"` - MetricsDatadogHost string `cli:"metrics-datadog-host"` - MetricsDatadogDistributions bool `cli:"metrics-datadog-distributions"` + // Metrics config + OpenTelemetryMetrics bool `cli:"opentelemetry-metrics"` // Tracing config TracingBackend string `cli:"tracing-backend"` @@ -289,8 +287,8 @@ func (asc AgentStartConfig) Features(ctx context.Context) []string { features = append(features, "env-godebug") } - if asc.MetricsDatadog { - features = append(features, "datadog-metrics") + if asc.OpenTelemetryMetrics { + features = append(features, "opentelemetry-metrics") } return features @@ -622,20 +620,9 @@ var AgentStartCommand = &cli.Command{ Sources: cli.EnvVars("BUILDKITE_ALLOWED_PLUGINS"), }, &cli.BoolFlag{ - Name: "metrics-datadog", - Usage: "Send metrics to DogStatsD for Datadog (default: false)", - Sources: cli.EnvVars("BUILDKITE_METRICS_DATADOG"), - }, - &cli.StringFlag{ - Name: "metrics-datadog-host", - Usage: "The dogstatsd instance to send metrics to using udp", - Sources: cli.EnvVars("BUILDKITE_METRICS_DATADOG_HOST"), - Value: "127.0.0.1:8125", - }, - &cli.BoolFlag{ - Name: "metrics-datadog-distributions", - Usage: "Use Datadog Distributions for Timing metrics (default: false)", - Sources: cli.EnvVars("BUILDKITE_METRICS_DATADOG_DISTRIBUTIONS"), + Name: "opentelemetry-metrics", + Usage: "Enable agent metrics export over OpenTelemetry OTLP. Configure OTLP with standard OTEL_EXPORTER_OTLP_* env vars (default: false)", + Sources: cli.EnvVars("BUILDKITE_OPENTELEMETRY_METRICS"), }, &cli.StringFlag{ Name: "log-format", @@ -676,7 +663,7 @@ var AgentStartCommand = &cli.Command{ }, &cli.StringFlag{ Name: "tracing-service-name", - Usage: "Service name to use when reporting traces.", + Usage: "Service name to use when reporting telemetry.", Sources: cli.EnvVars("BUILDKITE_TRACING_SERVICE_NAME"), Value: "buildkite-agent", }, @@ -892,9 +879,8 @@ var AgentStartCommand = &cli.Command{ } mc := metrics.NewCollector(l, metrics.CollectorConfig{ - Datadog: cfg.MetricsDatadog, - DatadogHost: cfg.MetricsDatadogHost, - DatadogDistributions: cfg.MetricsDatadogDistributions, + Enabled: cfg.OpenTelemetryMetrics, + ServiceName: cfg.TracingServiceName, }) // Sense check supported tracing backends, we don't want bootstrapped jobs to silently have no tracing diff --git a/go.mod b/go.mod index 6e72463bd4..1bd5ef2f92 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,6 @@ require ( drjosh.dev/zzglob v0.4.3 github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.14.0 github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.8.0 - github.com/DataDog/datadog-go/v5 v5.8.3 github.com/Khan/genqlient v0.8.1 github.com/aws/aws-sdk-go-v2 v1.42.0 github.com/aws/aws-sdk-go-v2/config v1.32.25 @@ -55,9 +54,13 @@ require ( go.opentelemetry.io/contrib/propagators/jaeger v1.44.0 go.opentelemetry.io/contrib/propagators/ot v1.44.0 go.opentelemetry.io/otel v1.44.0 + go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.39.0 + go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.39.0 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.44.0 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.44.0 + go.opentelemetry.io/otel/metric v1.44.0 go.opentelemetry.io/otel/sdk v1.44.0 + go.opentelemetry.io/otel/sdk/metric v1.44.0 go.opentelemetry.io/otel/trace v1.44.0 golang.org/x/net v0.56.0 golang.org/x/oauth2 v0.36.0 @@ -79,7 +82,6 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/azcore v1.22.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0 // indirect github.com/AzureAD/microsoft-authentication-library-for-go v1.7.2 // indirect - github.com/Microsoft/go-winio v0.6.2 // indirect github.com/agnivade/levenshtein v1.2.1 // indirect github.com/alexflint/go-arg v1.5.1 // indirect github.com/alexflint/go-scalar v1.2.0 // indirect @@ -136,13 +138,11 @@ require ( github.com/rivo/uniseg v0.4.7 // indirect github.com/saracen/zipextra v0.0.0-20250129175152-f1aa42d25216 // indirect github.com/segmentio/asm v1.2.1 // indirect - github.com/stretchr/objx v0.5.2 // indirect github.com/vektah/gqlparser/v2 v2.5.32 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.68.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.68.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.44.0 // indirect - go.opentelemetry.io/otel/metric v1.44.0 // indirect go.opentelemetry.io/proto/otlp v1.10.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.yaml.in/yaml/v2 v2.4.4 // indirect diff --git a/go.sum b/go.sum index 037f800515..e703a439de 100644 --- a/go.sum +++ b/go.sum @@ -36,13 +36,8 @@ github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1 h1:WJ github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mod h1:tCcJZ0uHAmvjsVYzEFivsRTN00oz5BEsRgQHu5JZ9WE= github.com/AzureAD/microsoft-authentication-library-for-go v1.7.2 h1:RHK7bS+HQMslb1sZpAokUt+zTVmue0hKSs2C791hhzU= github.com/AzureAD/microsoft-authentication-library-for-go v1.7.2/go.mod h1:HKpQxkWaGLJ+D/5H8QRpyQXA1eKjxkFlOMwck5+33Jk= -github.com/DataDog/datadog-go/v5 v5.8.3 h1:s58CUJ9s8lezjhTNJO/SxkPBv2qZjS3ktpRSqGF5n0s= -github.com/DataDog/datadog-go/v5 v5.8.3/go.mod h1:K9kcYBlxkcPP8tvvjZZKs/m1edNAUFzBbdpTUKfCsuw= github.com/Khan/genqlient v0.8.1 h1:wtOCc8N9rNynRLXN3k3CnfzheCUNKBcvXmVv5zt6WCs= github.com/Khan/genqlient v0.8.1/go.mod h1:R2G6DzjBvCbhjsEajfRjbWdVglSH/73kSivC9TLWVjU= -github.com/Microsoft/go-winio v0.5.0/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= -github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= -github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= github.com/agnivade/levenshtein v1.2.1 h1:EHBY3UOn1gwdy/VbFwgo4cxecRznFk7fKWN1KOX7eoM= github.com/agnivade/levenshtein v1.2.1/go.mod h1:QVVI16kDrtSuwcpd0p1+xMC6Z/VfhtCyDIjcwga4/DU= github.com/alexflint/go-arg v1.5.1 h1:nBuWUCpuRy0snAG+uIJ6N0UvYxpxA0/ghA/AaHxlT8Y= @@ -175,7 +170,6 @@ github.com/gofrs/flock v0.13.0 h1:95JolYOvGMqeH31+FC7D2+uULf6mG61mEZ/A8dRYMzw= github.com/gofrs/flock v0.13.0/go.mod h1:jxeyy9R1auM5S6JYDBhDt+E2TCo7DkratH4Pgi8P+Z0= github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY= github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= -github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= @@ -241,7 +235,6 @@ github.com/pborman/uuid v1.2.1 h1:+ZZIw58t/ozdjRaXh/3awHfmWRbzYxJoAdNJxe/3pvw= github.com/pborman/uuid v1.2.1/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU= -github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo= github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -273,19 +266,14 @@ github.com/segmentio/asm v1.2.1/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8= github.com/sergi/go-diff v1.3.1/go.mod h1:aMJSSKb2lpPvRNec0+w3fl7LP9IOFzdc9Pa4NFbPK1I= -github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= -github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= -github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= -github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/urfave/cli/v3 v3.8.0 h1:XqKPrm0q4P0q5JpoclYoCAv0/MIvH/jZ2umzuf8pNTI= @@ -294,7 +282,6 @@ github.com/vektah/gqlparser/v2 v2.5.32 h1:k9QPJd4sEDTL+qB4ncPLflqTJ3MmjB9SrVzJra github.com/vektah/gqlparser/v2 v2.5.32/go.mod h1:c1I28gSOVNzlfc4WuDlqU7voQnsqI6OG2amkBAFmgts= github.com/wolfeidau/quickzip v1.0.2 h1:QPc4CVE8ECYng87o63C4t6+6ihZk1howyrZWwklKjq8= github.com/wolfeidau/quickzip v1.0.2/go.mod h1:ZDvxJMzI2iVp6CavOqxFq1tORhRyuxCDFw4HO25RbuA= -github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.68.0 h1:0Qx7VGBacMm9ZENQ7TnNObTYI4ShC+lHI16seduaxZo= @@ -311,6 +298,10 @@ go.opentelemetry.io/contrib/propagators/ot v1.44.0 h1:JLTPenzmPtLp5ODPntAA5JhxVu go.opentelemetry.io/contrib/propagators/ot v1.44.0/go.mod h1:8zr0bHgwkoQXucBK39/H4QphmLf1lSen1Z7FPDZD5Uc= go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU= go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhOPIAyYc= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.39.0 h1:cEf8jF6WbuGQWUVcqgyWtTR0kOOAWY1DYZ+UhvdmQPw= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.39.0/go.mod h1:k1lzV5n5U3HkGvTCJHraTAGJ7MqsgL1wrGwTj1Isfiw= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.39.0 h1:nKP4Z2ejtHn3yShBb+2KawiXgpn8In5cT7aO2wXuOTE= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.39.0/go.mod h1:NwjeBbNigsO4Aj9WgM0C+cKIrxsZUaRmZUO7A8I7u8o= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.44.0 h1:4YsVu3B8+3qtWYYrsUYgn0OG78pN0rnNPRGX4SbokQI= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.44.0/go.mod h1:+wnlSn0mD1ADVMe3v9Z/WIaiz6q6gL2J/ejaAmdmv80= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.44.0 h1:qazEJlUOQzhCpzQpFETGby7EdqjI1wsd0W+6Gg1SCTU= @@ -319,6 +310,8 @@ go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.44.0 h1:lgh3P go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.44.0/go.mod h1:5Cnhth3m/AgOeTgE3ex12pPmiu/gGtZit03kSzx9X7s= go.opentelemetry.io/otel/metric v1.44.0 h1:1w0gILTcHdr3YI+ixLyjemwrVnsMURbTZFrSYCdDdmc= go.opentelemetry.io/otel/metric v1.44.0/go.mod h1:8O7hanEPBNgEMmybD3s2VBKcgWOCsA6tzHBPODAiquo= +go.opentelemetry.io/otel/metric/x v0.66.0 h1:YkCrx1zLOChi9ZcZ6euupOcsgzbVlec7D/xoEU1+cTA= +go.opentelemetry.io/otel/metric/x v0.66.0/go.mod h1:d1+BDj9t96do0/1LoU1ayfCv79ZgNE41qbhBvnMOBZk= go.opentelemetry.io/otel/sdk v1.44.0 h1:nHYwb9lK+fJPU/dnT6s7W7Z8itMWyqrnVfbheVYrZ58= go.opentelemetry.io/otel/sdk v1.44.0/go.mod h1:Osuydd3Se74nqjAKxid74N5eC+jfEqfTegHRnq58oK0= go.opentelemetry.io/otel/sdk/metric v1.44.0 h1:3LlKgI+VjbVsjNRFZJZAJ30WjXC5VkNRks6si09iEfI= @@ -333,52 +326,28 @@ go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ= go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.53.0 h1:QZ4Muo8THX6CizN2vPPd5fBGHyogrdK9fG4wLPFUsto= golang.org/x/crypto v0.53.0/go.mod h1:DNLU434OwVakk9PzuwV8w62mAJpRJL3vsgcfp4Qnsio= -golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.36.0 h1:JJjpVx6myfUsUdAzZuOSTTmRE0PfZeNWzzvKrP7amb4= golang.org/x/mod v0.36.0/go.mod h1:moc6ELqsWcOw5Ef3xVprK5ul/MvtVvkIXLziUOICjUQ= -golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.56.0 h1:Rw8j/hFzGvJUZwNBXnAtf5sVDVt+65SK2C7IxCxZt5o= golang.org/x/net v0.56.0/go.mod h1:D3Ku6r+V6JROoZK144D2XfMHFcMq/0zSfLelVTCFKec= golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM= golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw= golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.44.0 h1:0rLvDRCtNj0gZkyIXhCyOb2OAzEhLVqc4B+hrsBhrmc= golang.org/x/term v0.44.0/go.mod h1:7ze4MdzUzLXpSAoFP1H0bOI9aXDqveSvatT5vKcFh2Y= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.38.0 h1:sXmwo9DwP3OK9EZ7PqAdaooSGozfl/3a6/xJcbzPRhE= golang.org/x/text v0.38.0/go.mod h1:YXZt3QhHUKYT53r2lLKFIVi6Ao1jdzrTR/KQ09qyxF4= golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= -golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.45.0 h1:18qN3FAooORvApf5XjCXgsuayZOEtXf6JK18I3+ONa8= golang.org/x/tools v0.45.0/go.mod h1:LuUGqqaXcXMEFEruIVJVm5mgDD8vww/z/SR1gQ4uE/0= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= google.golang.org/api v0.284.0 h1:i+cKTgeQRcRySkP7QTl5PDO7/pAm8EcMFIUMlNbk4Vc= diff --git a/metrics/metrics.go b/metrics/metrics.go index 02a3454bc8..e21db3f1f2 100644 --- a/metrics/metrics.go +++ b/metrics/metrics.go @@ -1,77 +1,170 @@ -// Package metrics provides a wrapper around Datadog metrics collection. +// Package metrics provides a wrapper around OpenTelemetry metrics collection. // // It is intended for internal use by buildkite-agent only. package metrics import ( + "context" + "errors" "fmt" + "os" "regexp" "sort" - "strings" + "sync" "time" - "github.com/DataDog/datadog-go/v5/statsd" "github.com/buildkite/agent/v4/logger" + "github.com/buildkite/agent/v4/version" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc" + "go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp" + otelmetric "go.opentelemetry.io/otel/metric" + sdkmetric "go.opentelemetry.io/otel/sdk/metric" + "go.opentelemetry.io/otel/sdk/resource" + semconv "go.opentelemetry.io/otel/semconv/v1.4.0" ) const ( - // Number of statsd commands that are buffered before - // being sent to statsd - statsdBufferLen = 10 - - // The default port for dogstatsd - defaultDogStatsdPort = 8125 + defaultOTLPProtocol = "grpc" + defaultServiceName = "buildkite-agent" ) type Collector struct { config CollectorConfig logger logger.Logger - client *statsd.Client + + mu sync.Mutex + started int + provider *sdkmetric.MeterProvider + meter otelmetric.Meter + counters map[string]otelmetric.Int64Counter + histograms map[string]otelmetric.Float64Histogram } type CollectorConfig struct { - Datadog bool - DatadogHost string - DatadogDistributions bool + Enabled bool + ServiceName string } func NewCollector(l logger.Logger, c CollectorConfig) *Collector { + if c.ServiceName == "" { + c.ServiceName = defaultServiceName + } + return &Collector{ - config: c, - logger: l, + config: c, + logger: l, + counters: make(map[string]otelmetric.Int64Counter), + histograms: make(map[string]otelmetric.Float64Histogram), } } -var portSuffixRegexp = regexp.MustCompile(`:\d+$`) - func (c *Collector) Start() error { - if c.config.Datadog { - if !portSuffixRegexp.MatchString(c.config.DatadogHost) { - c.config.DatadogHost += fmt.Sprintf(":%d", defaultDogStatsdPort) - } + c.mu.Lock() + defer c.mu.Unlock() - c.logger.Infof("Starting datadog metrics collection to %s", c.config.DatadogHost) + c.started++ + if c.started > 1 { + return nil + } - var err error - c.client, err = statsd.New(c.config.DatadogHost, - statsd.WithMaxMessagesPerPayload(statsdBufferLen), - statsd.WithNamespace("buildkite."), - ) - if err != nil { - return err - } + if !c.config.Enabled { + return nil } + + protocol := otlpProtocol() + c.logger.Infof("Starting OpenTelemetry metrics collection using OTLP/%s", protocol) + + provider, err := c.newMeterProvider(context.Background(), protocol) + if err != nil { + c.started-- + return err + } + + c.provider = provider + c.meter = provider.Meter( + "buildkite-agent", + otelmetric.WithInstrumentationVersion(version.Version()), + otelmetric.WithSchemaURL(semconv.SchemaURL), + ) + c.counters = make(map[string]otelmetric.Int64Counter) + c.histograms = make(map[string]otelmetric.Float64Histogram) return nil } func (c *Collector) Stop() error { - if c.config.Datadog && c.client != nil { + c.mu.Lock() + if c.started == 0 { + c.mu.Unlock() + return nil + } + + c.started-- + if c.started > 0 { + c.mu.Unlock() + return nil + } + + provider := c.provider + c.provider = nil + c.meter = nil + c.counters = make(map[string]otelmetric.Int64Counter) + c.histograms = make(map[string]otelmetric.Float64Histogram) + c.mu.Unlock() + + if provider != nil { c.logger.Infof("Stopping metrics collection") - return c.client.Close() + + ctx := context.Background() + flushErr := provider.ForceFlush(ctx) + shutdownErr := provider.Shutdown(ctx) + return errors.Join(flushErr, shutdownErr) } + return nil } +func (c *Collector) newMeterProvider(ctx context.Context, protocol string) (*sdkmetric.MeterProvider, error) { + var ( + exporter sdkmetric.Exporter + err error + ) + + switch protocol { + case "grpc": + exporter, err = otlpmetricgrpc.New(ctx) + case "http/protobuf", "http": + exporter, err = otlpmetrichttp.New(ctx) + default: + return nil, fmt.Errorf("unsupported OTLP protocol %q", protocol) + } + if err != nil { + return nil, err + } + + resources := resource.NewWithAttributes( + semconv.SchemaURL, + semconv.ServiceNameKey.String(c.config.ServiceName), + semconv.ServiceVersionKey.String(version.Version()), + semconv.DeploymentEnvironmentKey.String("ci"), + ) + + return sdkmetric.NewMeterProvider( + sdkmetric.WithReader(sdkmetric.NewPeriodicReader(exporter)), + sdkmetric.WithResource(resources), + ), nil +} + +func otlpProtocol() string { + if protocol := os.Getenv("OTEL_EXPORTER_OTLP_METRICS_PROTOCOL"); protocol != "" { + return protocol + } + if protocol := os.Getenv("OTEL_EXPORTER_OTLP_PROTOCOL"); protocol != "" { + return protocol + } + return defaultOTLPProtocol +} + func (c *Collector) Scope(tags Tags) *Scope { return &Scope{ Tags: tags, @@ -86,28 +179,14 @@ type Scope struct { // Timing sends timing information in milliseconds. func (s *Scope) Timing(name string, value time.Duration, tags ...Tags) { - if s.c.client == nil { + histogram, ok := s.c.histogram(name) + if !ok { return } - mergedTags := s.mergeTags(tags...).StringSlice() - s.c.logger.Debugf("Metrics timing %s=%v %v", name, value, mergedTags) - - var err error - if s.c.config.DatadogDistributions { - // Datadog recommends that, as distributions are a new distinct metric, - // they belong to a new metric name. We handle this by just slamming - // .distribution to end of all metrics that we submit this way - if !strings.HasSuffix(name, ".distribution") { - name = name + ".distribution" - } - err = s.c.client.Distribution(name, float64(value.Milliseconds()), mergedTags, 1) - } else { - err = s.c.client.Timing(name, value, mergedTags, 1) - } - if err != nil { - s.c.logger.Errorf("Metrics timing failed: %v", err) - } + mergedTags := s.mergeTags(tags...) + s.c.logger.Debugf("Metrics timing %s=%v %v", name, value, mergedTags.StringSlice()) + histogram.Record(context.Background(), float64(value.Milliseconds()), otelmetric.WithAttributes(mergedTags.Attributes()...)) } // With returns a scope with more tags added @@ -118,18 +197,62 @@ func (s *Scope) With(tags Tags) *Scope { } } -// Count tracks how many times something happened per second. +// Count tracks how many times something happened. func (s *Scope) Count(name string, value int64, tags ...Tags) { - if s.c.client == nil { + counter, ok := s.c.counter(name) + if !ok { return } - mergedTags := s.mergeTags(tags...).StringSlice() - s.c.logger.Debugf("Metrics count %s=%v %v", name, value, mergedTags) + mergedTags := s.mergeTags(tags...) + s.c.logger.Debugf("Metrics count %s=%v %v", name, value, mergedTags.StringSlice()) + counter.Add(context.Background(), value, otelmetric.WithAttributes(mergedTags.Attributes()...)) +} + +func (c *Collector) counter(name string) (otelmetric.Int64Counter, bool) { + metricName := formatName(name) + + c.mu.Lock() + defer c.mu.Unlock() + + if c.meter == nil { + return nil, false + } + if counter, ok := c.counters[metricName]; ok { + return counter, true + } + + counter, err := c.meter.Int64Counter(metricName) + if err != nil { + c.logger.Errorf("Metrics counter creation failed: %v", err) + return nil, false + } + + c.counters[metricName] = counter + return counter, true +} + +func (c *Collector) histogram(name string) (otelmetric.Float64Histogram, bool) { + metricName := formatName(name) + + c.mu.Lock() + defer c.mu.Unlock() - if err := s.c.client.Count(name, value, mergedTags, 1); err != nil { - s.c.logger.Errorf("Metrics count failed: %v", err) + if c.meter == nil { + return nil, false + } + if histogram, ok := c.histograms[metricName]; ok { + return histogram, true + } + + histogram, err := c.meter.Float64Histogram(metricName, otelmetric.WithUnit("ms")) + if err != nil { + c.logger.Errorf("Metrics histogram creation failed: %v", err) + return nil, false } + + c.histograms[metricName] = histogram + return histogram, true } func (s *Scope) mergeTags(tagsSlice ...Tags) Tags { @@ -147,6 +270,16 @@ func (s *Scope) mergeTags(tagsSlice ...Tags) Tags { type Tags map[string]string +func (tags Tags) Attributes() []attribute.KeyValue { + attrs := make([]attribute.KeyValue, 0, len(tags)) + for k, v := range tags { + if k != "" && v != "" { + attrs = append(attrs, attribute.String(formatName(k), formatName(v))) + } + } + return attrs +} + func (tags Tags) StringSlice() []string { var stringSlice []string for k, v := range tags { @@ -158,8 +291,7 @@ func (tags Tags) StringSlice() []string { return stringSlice } -// Datadog allows '.', '_' and alphas only. -// If we don't validate this here then the datadog error logs can fill up disk really quickly +// Keep metric names and tag keys portable across OpenTelemetry exporters. var nameRegex = regexp.MustCompile(`[^\._a-zA-Z0-9]+`) func formatName(name string) string { diff --git a/packaging/linux/root/usr/share/buildkite-agent/buildkite-agent.cfg b/packaging/linux/root/usr/share/buildkite-agent/buildkite-agent.cfg index e27338ff26..f68c78b84a 100644 --- a/packaging/linux/root/usr/share/buildkite-agent/buildkite-agent.cfg +++ b/packaging/linux/root/usr/share/buildkite-agent/buildkite-agent.cfg @@ -66,16 +66,9 @@ plugins-path="/etc/buildkite-agent/plugins" # Don't show colors in logging # no-color=true -# The next two options are relevant to the Datadog integration, available 3.7.0 and on -# See https://buildkite.com/docs/agent/self-hosted/configure#metrics-datadog -# Send metrics to DogStatsD running on metrics-datadog-host -# metrics-datadog=true - -# Host to collect Buildkite metrics -# datadog-agent will need to run DogStatsD, presumed on port 8125. -# See https://buildkite.com/docs/agent/self-hosted/configure#metrics-datadog-host -# Specify port below like my-host:8126 if not using 8125 -# metrics-datadog-host=127.0.0.1 +# Enable OpenTelemetry metrics export over OTLP. +# Configure the endpoint and protocol with the standard OTEL_EXPORTER_OTLP_* env vars. +# opentelemetry-metrics=true # If set and valid, the given tracing backend will be enabled. Eg: datadog, opentelemetry # tracing-backend="" From f99a879df63c79453deffb6531642d71e63b6b19 Mon Sep 17 00:00:00 2001 From: Ben Moskovitz Date: Thu, 30 Apr 2026 21:24:39 +1000 Subject: [PATCH 2/3] Flatten tracing config to be a single bool flag --- agent/agent_configuration.go | 2 +- agent/job_runner.go | 8 ++++---- clicommand/agent_start.go | 28 ++++++++-------------------- clicommand/agent_start_test.go | 11 +++++++++++ clicommand/bootstrap.go | 21 +++++++++++++-------- 5 files changed, 37 insertions(+), 33 deletions(-) diff --git a/agent/agent_configuration.go b/agent/agent_configuration.go index 4b36a5bb34..4fba514998 100644 --- a/agent/agent_configuration.go +++ b/agent/agent_configuration.go @@ -70,7 +70,7 @@ type AgentConfiguration struct { Profile string RedactedVars []string AcquireJob string - TracingBackend string + OpenTelemetryTracing bool TracingServiceName string TracingPropagateTraceparent bool DisableWarningsFor []string diff --git a/agent/job_runner.go b/agent/job_runner.go index a4276abf6c..2ac011aa52 100644 --- a/agent/job_runner.go +++ b/agent/job_runner.go @@ -391,7 +391,7 @@ func (r *JobRunner) createEnvironment(ctx context.Context) ([]string, error) { if r.envShellFile != nil { // Note that some variables in this list might not be defined later, // when something comes to read the file. See below where they are - // added conditionally, e.g. BUILDKITE_TRACING_BACKEND. + // added conditionally, e.g. BUILDKITE_OPENTELEMETRY_TRACING. // Docker in particular tolerates undefined vars in an env file // without complaints. const agentCfgVars = `BUILDKITE_GIT_CHECKOUT_FLAGS @@ -416,7 +416,7 @@ BUILDKITE_SHELL BUILDKITE_HOOKS_SHELL BUILDKITE_SSH_KEYSCAN BUILDKITE_STRICT_SINGLE_HOOKS -BUILDKITE_TRACING_BACKEND +BUILDKITE_OPENTELEMETRY_TRACING BUILDKITE_TRACING_SERVICE_NAME BUILDKITE_TRACING_TRACEPARENT BUILDKITE_TRACING_TRACESTATE @@ -638,8 +638,8 @@ BUILDKITE_AGENT_JWKS_KEY_ID` } setEnv("BUILDKITE_PLUGIN_VALIDATION", fmt.Sprint(enablePluginValidation)) - if r.conf.AgentConfiguration.TracingBackend != "" { - setEnv("BUILDKITE_TRACING_BACKEND", r.conf.AgentConfiguration.TracingBackend) + if r.conf.AgentConfiguration.OpenTelemetryTracing { + setEnv("BUILDKITE_OPENTELEMETRY_TRACING", "true") setEnv("BUILDKITE_TRACING_SERVICE_NAME", r.conf.AgentConfiguration.TracingServiceName) // Buildkite backend can provide a traceparent property on the job diff --git a/clicommand/agent_start.go b/clicommand/agent_start.go index 79b7a331a5..8251c5adee 100644 --- a/clicommand/agent_start.go +++ b/clicommand/agent_start.go @@ -6,7 +6,6 @@ import ( "errors" "fmt" "io" - "maps" "net/url" "os" "os/signal" @@ -39,7 +38,6 @@ import ( "github.com/buildkite/agent/v4/logger" "github.com/buildkite/agent/v4/metrics" "github.com/buildkite/agent/v4/status" - "github.com/buildkite/agent/v4/tracetools" "github.com/buildkite/agent/v4/version" "github.com/buildkite/shellwords" "github.com/lestrrat-go/jwx/v2/jwk" @@ -192,7 +190,7 @@ type AgentStartConfig struct { OpenTelemetryMetrics bool `cli:"opentelemetry-metrics"` // Tracing config - TracingBackend string `cli:"tracing-backend"` + OpenTelemetryTracing bool `cli:"opentelemetry-tracing"` TracingServiceName string `cli:"tracing-service-name"` TracingPropagateTraceparent bool `cli:"tracing-propagate-traceparent"` @@ -235,7 +233,7 @@ func (asc AgentStartConfig) Features(ctx context.Context) []string { features = append(features, "acquire-job") } - if asc.TracingBackend == tracetools.BackendOpenTelemetry { + if asc.OpenTelemetryTracing { features = append(features, "opentelemetry-tracing") } @@ -650,15 +648,14 @@ var AgentStartCommand = &cli.Command{ }, cancelSignalFlag, cancelCleanupTimeoutFlag, - &cli.StringFlag{ - Name: "tracing-backend", - Usage: `Enable tracing for build jobs by specifying a backend. Currently only "opentelemetry" (or empty) is supported`, - Sources: cli.EnvVars("BUILDKITE_TRACING_BACKEND"), - Value: "", + &cli.BoolFlag{ + Name: "opentelemetry-tracing", + Usage: "Enable tracing for build jobs with OpenTelemetry OTLP. Configure OTLP with standard OTEL_EXPORTER_OTLP_* env vars (default: false)", + Sources: cli.EnvVars("BUILDKITE_OPENTELEMETRY_TRACING"), }, &cli.BoolFlag{ Name: "tracing-propagate-traceparent", - Usage: `Enable accepting traceparent context from Buildkite control plane (only supported for OpenTelemetry backend) (default: false)`, + Usage: "Enable accepting traceparent context from Buildkite control plane. Requires --opentelemetry-tracing (default: false)", Sources: cli.EnvVars("BUILDKITE_TRACING_PROPAGATE_TRACEPARENT"), }, &cli.StringFlag{ @@ -883,15 +880,6 @@ var AgentStartCommand = &cli.Command{ ServiceName: cfg.TracingServiceName, }) - // Sense check supported tracing backends, we don't want bootstrapped jobs to silently have no tracing - if _, has := tracetools.ValidTracingBackends[cfg.TracingBackend]; !has { - return fmt.Errorf( - "the given tracing backend %q is not supported. Valid backends are: %q", - cfg.TracingBackend, - slices.Collect(maps.Keys(tracetools.ValidTracingBackends)), - ) - } - if experiments.IsEnabled(ctx, experiments.AgentAPI) { shutdown, err := runAgentAPI(ctx, l, cfg.SocketsPath) if err != nil { @@ -1020,7 +1008,7 @@ var AgentStartCommand = &cli.Command{ HooksShell: cfg.HooksShell, RedactedVars: cfg.RedactedVars, AcquireJob: cfg.AcquireJob, - TracingBackend: cfg.TracingBackend, + OpenTelemetryTracing: cfg.OpenTelemetryTracing, TracingServiceName: cfg.TracingServiceName, TracingPropagateTraceparent: cfg.TracingPropagateTraceparent, AllowMultipartArtifactUpload: !cfg.NoMultipartArtifactUpload, diff --git a/clicommand/agent_start_test.go b/clicommand/agent_start_test.go index cf49c4cb49..2889f5125b 100644 --- a/clicommand/agent_start_test.go +++ b/clicommand/agent_start_test.go @@ -1,10 +1,12 @@ package clicommand import ( + "context" "errors" "os" "path/filepath" "runtime" + "slices" "testing" "github.com/buildkite/agent/v4/core" @@ -13,6 +15,15 @@ import ( "github.com/urfave/cli/v3" ) +func TestAgentStartFeatures_OpenTelemetryTracing(t *testing.T) { + t.Parallel() + + features := AgentStartConfig{OpenTelemetryTracing: true}.Features(context.Background()) + if !slices.Contains(features, "opentelemetry-tracing") { + t.Fatalf("Features() = %v, want opentelemetry-tracing", features) + } +} + func setupHooksPath(t *testing.T) (string, func()) { t.Helper() diff --git a/clicommand/bootstrap.go b/clicommand/bootstrap.go index 36d7789e1d..553cb9409c 100644 --- a/clicommand/bootstrap.go +++ b/clicommand/bootstrap.go @@ -15,6 +15,7 @@ import ( "github.com/buildkite/agent/v4/internal/process" "github.com/buildkite/agent/v4/internal/self" "github.com/buildkite/agent/v4/logger" + "github.com/buildkite/agent/v4/tracetools" "github.com/urfave/cli/v3" ) @@ -108,7 +109,7 @@ type BootstrapConfig struct { CancelSignalTimeout time.Duration `cli:"cancel-signal-timeout"` CancelCleanupTimeout time.Duration `cli:"cancel-cleanup-timeout"` RedactedVars []string `cli:"redacted-vars" normalize:"list"` - TracingBackend string `cli:"tracing-backend"` + OpenTelemetryTracing bool `cli:"opentelemetry-tracing"` TracingServiceName string `cli:"tracing-service-name"` TracingTraceParent string `cli:"tracing-traceparent"` TracingTraceState string `cli:"tracing-tracestate"` @@ -336,11 +337,10 @@ var BootstrapCommand = &cli.Command{ Usage: "The specific phases to execute. The order they're defined is irrelevant.", Sources: cli.EnvVars("BUILDKITE_BOOTSTRAP_PHASES"), }, - &cli.StringFlag{ - Name: "tracing-backend", - Usage: "The name of the tracing backend to use.", - Sources: cli.EnvVars("BUILDKITE_TRACING_BACKEND"), - Value: "", + &cli.BoolFlag{ + Name: "opentelemetry-tracing", + Usage: "Enable tracing for build jobs with OpenTelemetry OTLP. Configure OTLP with standard OTEL_EXPORTER_OTLP_* env vars (default: false)", + Sources: cli.EnvVars("BUILDKITE_OPENTELEMETRY_TRACING"), }, &cli.StringFlag{ Name: "tracing-service-name", @@ -362,7 +362,7 @@ var BootstrapCommand = &cli.Command{ }, &cli.BoolFlag{ Name: "tracing-propagate-traceparent", - Usage: "Accept traceparent from Buildkite control plane (default: false)", + Usage: "Accept traceparent from Buildkite control plane. Requires --opentelemetry-tracing (default: false)", Sources: cli.EnvVars("BUILDKITE_TRACING_PROPAGATE_TRACEPARENT"), }, @@ -428,6 +428,11 @@ var BootstrapCommand = &cli.Command{ return fmt.Errorf("failed to parse cancel-signal: %w", err) } + tracingBackend := tracetools.BackendNone + if cfg.OpenTelemetryTracing { + tracingBackend = tracetools.BackendOpenTelemetry + } + // Configure the bootstraper bootstrap := job.New(job.ExecutorConfig{ AgentName: cfg.AgentName, @@ -486,7 +491,7 @@ var BootstrapCommand = &cli.Command{ HooksShell: cfg.HooksShell, StrictSingleHooks: cfg.StrictSingleHooks, Tag: cfg.Tag, - TracingBackend: cfg.TracingBackend, + TracingBackend: tracingBackend, TracingServiceName: cfg.TracingServiceName, TracingTraceParent: cfg.TracingTraceParent, TracingTraceState: cfg.TracingTraceState, From df7e0af23ca7df0fb3eb84e0728de3026fc41060 Mon Sep 17 00:00:00 2001 From: Ben Moskovitz Date: Fri, 1 May 2026 15:52:37 +1000 Subject: [PATCH 3/3] Always enable traceparent propagation from the backend if otel tracing is enabled --- agent/agent_configuration.go | 3 +-- agent/job_runner.go | 8 ++----- clicommand/agent_start.go | 23 +++++-------------- clicommand/agent_start_test.go | 3 +-- clicommand/bootstrap.go | 15 ++++-------- clicommand/global.go | 4 ++-- internal/job/config.go | 5 +--- internal/job/tracing.go | 9 ++------ .../docker/alpine-k8s/buildkite-agent.cfg | 5 ++-- packaging/docker/alpine/buildkite-agent.cfg | 5 ++-- packaging/docker/sidecar/buildkite-agent.cfg | 5 ++-- .../docker/ubuntu-20.04/buildkite-agent.cfg | 5 ++-- .../docker/ubuntu-22.04/buildkite-agent.cfg | 5 ++-- .../docker/ubuntu-24.04/buildkite-agent.cfg | 5 ++-- packaging/github/linux/buildkite-agent.cfg | 5 ++-- packaging/github/windows/buildkite-agent.cfg | 5 ++-- .../share/buildkite-agent/buildkite-agent.cfg | 3 --- 17 files changed, 43 insertions(+), 70 deletions(-) diff --git a/agent/agent_configuration.go b/agent/agent_configuration.go index 4fba514998..98aaeec1d1 100644 --- a/agent/agent_configuration.go +++ b/agent/agent_configuration.go @@ -71,8 +71,7 @@ type AgentConfiguration struct { RedactedVars []string AcquireJob string OpenTelemetryTracing bool - TracingServiceName string - TracingPropagateTraceparent bool + TelemetryServiceName string DisableWarningsFor []string AllowMultipartArtifactUpload bool ArtifactUploadConcurrency int diff --git a/agent/job_runner.go b/agent/job_runner.go index 2ac011aa52..fba31f60c5 100644 --- a/agent/job_runner.go +++ b/agent/job_runner.go @@ -417,10 +417,9 @@ BUILDKITE_HOOKS_SHELL BUILDKITE_SSH_KEYSCAN BUILDKITE_STRICT_SINGLE_HOOKS BUILDKITE_OPENTELEMETRY_TRACING -BUILDKITE_TRACING_SERVICE_NAME +BUILDKITE_TELEMETRY_SERVICE_NAME BUILDKITE_TRACING_TRACEPARENT BUILDKITE_TRACING_TRACESTATE -BUILDKITE_TRACING_PROPAGATE_TRACEPARENT BUILDKITE_AGENT_AWS_KMS_KEY BUILDKITE_AGENT_GCP_KMS_KEY BUILDKITE_AGENT_JWKS_FILE @@ -640,7 +639,7 @@ BUILDKITE_AGENT_JWKS_KEY_ID` if r.conf.AgentConfiguration.OpenTelemetryTracing { setEnv("BUILDKITE_OPENTELEMETRY_TRACING", "true") - setEnv("BUILDKITE_TRACING_SERVICE_NAME", r.conf.AgentConfiguration.TracingServiceName) + setEnv("BUILDKITE_TELEMETRY_SERVICE_NAME", r.conf.AgentConfiguration.TelemetryServiceName) // Buildkite backend can provide a traceparent property on the job // which can be propagated to the job tracing if OpenTelemetry is used @@ -656,9 +655,6 @@ BUILDKITE_AGENT_JWKS_KEY_ID` if r.conf.Job.TraceState != "" { setEnv("BUILDKITE_TRACING_TRACESTATE", r.conf.Job.TraceState) } - if r.conf.AgentConfiguration.TracingPropagateTraceparent { - setEnv("BUILDKITE_TRACING_PROPAGATE_TRACEPARENT", "true") - } } setEnv("BUILDKITE_AGENT_DISABLE_WARNINGS_FOR", strings.Join(r.conf.AgentConfiguration.DisableWarningsFor, ",")) diff --git a/clicommand/agent_start.go b/clicommand/agent_start.go index 8251c5adee..1ed50e7f9e 100644 --- a/clicommand/agent_start.go +++ b/clicommand/agent_start.go @@ -190,9 +190,8 @@ type AgentStartConfig struct { OpenTelemetryMetrics bool `cli:"opentelemetry-metrics"` // Tracing config - OpenTelemetryTracing bool `cli:"opentelemetry-tracing"` - TracingServiceName string `cli:"tracing-service-name"` - TracingPropagateTraceparent bool `cli:"tracing-propagate-traceparent"` + OpenTelemetryTracing bool `cli:"opentelemetry-tracing"` + TelemetryServiceName string `cli:"telemetry-service-name"` // Other shared flags StrictSingleHooks bool `cli:"strict-single-hooks"` @@ -237,10 +236,6 @@ func (asc AgentStartConfig) Features(ctx context.Context) []string { features = append(features, "opentelemetry-tracing") } - if asc.TracingPropagateTraceparent { - features = append(features, "propagate-traceparent") - } - if asc.DisconnectAfterJob { features = append(features, "disconnect-after-job") } @@ -653,15 +648,10 @@ var AgentStartCommand = &cli.Command{ Usage: "Enable tracing for build jobs with OpenTelemetry OTLP. Configure OTLP with standard OTEL_EXPORTER_OTLP_* env vars (default: false)", Sources: cli.EnvVars("BUILDKITE_OPENTELEMETRY_TRACING"), }, - &cli.BoolFlag{ - Name: "tracing-propagate-traceparent", - Usage: "Enable accepting traceparent context from Buildkite control plane. Requires --opentelemetry-tracing (default: false)", - Sources: cli.EnvVars("BUILDKITE_TRACING_PROPAGATE_TRACEPARENT"), - }, &cli.StringFlag{ - Name: "tracing-service-name", + Name: "telemetry-service-name", Usage: "Service name to use when reporting telemetry.", - Sources: cli.EnvVars("BUILDKITE_TRACING_SERVICE_NAME"), + Sources: cli.EnvVars("BUILDKITE_TELEMETRY_SERVICE_NAME"), Value: "buildkite-agent", }, &cli.StringFlag{ @@ -877,7 +867,7 @@ var AgentStartCommand = &cli.Command{ mc := metrics.NewCollector(l, metrics.CollectorConfig{ Enabled: cfg.OpenTelemetryMetrics, - ServiceName: cfg.TracingServiceName, + ServiceName: cfg.TelemetryServiceName, }) if experiments.IsEnabled(ctx, experiments.AgentAPI) { @@ -1009,8 +999,7 @@ var AgentStartCommand = &cli.Command{ RedactedVars: cfg.RedactedVars, AcquireJob: cfg.AcquireJob, OpenTelemetryTracing: cfg.OpenTelemetryTracing, - TracingServiceName: cfg.TracingServiceName, - TracingPropagateTraceparent: cfg.TracingPropagateTraceparent, + TelemetryServiceName: cfg.TelemetryServiceName, AllowMultipartArtifactUpload: !cfg.NoMultipartArtifactUpload, ArtifactUploadConcurrency: cfg.ArtifactUploadConcurrency, KubernetesExec: cfg.KubernetesExec, diff --git a/clicommand/agent_start_test.go b/clicommand/agent_start_test.go index 2889f5125b..f2bcffe34a 100644 --- a/clicommand/agent_start_test.go +++ b/clicommand/agent_start_test.go @@ -1,7 +1,6 @@ package clicommand import ( - "context" "errors" "os" "path/filepath" @@ -18,7 +17,7 @@ import ( func TestAgentStartFeatures_OpenTelemetryTracing(t *testing.T) { t.Parallel() - features := AgentStartConfig{OpenTelemetryTracing: true}.Features(context.Background()) + features := AgentStartConfig{OpenTelemetryTracing: true}.Features(t.Context()) if !slices.Contains(features, "opentelemetry-tracing") { t.Fatalf("Features() = %v, want opentelemetry-tracing", features) } diff --git a/clicommand/bootstrap.go b/clicommand/bootstrap.go index 553cb9409c..86d2de8702 100644 --- a/clicommand/bootstrap.go +++ b/clicommand/bootstrap.go @@ -110,10 +110,9 @@ type BootstrapConfig struct { CancelCleanupTimeout time.Duration `cli:"cancel-cleanup-timeout"` RedactedVars []string `cli:"redacted-vars" normalize:"list"` OpenTelemetryTracing bool `cli:"opentelemetry-tracing"` - TracingServiceName string `cli:"tracing-service-name"` + TelemetryServiceName string `cli:"telemetry-service-name"` TracingTraceParent string `cli:"tracing-traceparent"` TracingTraceState string `cli:"tracing-tracestate"` - TracingPropagateTraceparent bool `cli:"tracing-propagate-traceparent"` NoJobAPI bool `cli:"no-job-api"` DisableWarningsFor []string `cli:"disable-warnings-for" normalize:"list"` CheckoutAttempts int `cli:"checkout-attempts"` @@ -343,9 +342,9 @@ var BootstrapCommand = &cli.Command{ Sources: cli.EnvVars("BUILDKITE_OPENTELEMETRY_TRACING"), }, &cli.StringFlag{ - Name: "tracing-service-name", + Name: "telemetry-service-name", Usage: "Service name to use when reporting traces.", - Sources: cli.EnvVars("BUILDKITE_TRACING_SERVICE_NAME"), + Sources: cli.EnvVars("BUILDKITE_TELEMETRY_SERVICE_NAME"), Value: "buildkite-agent", }, &cli.StringFlag{ @@ -360,11 +359,6 @@ var BootstrapCommand = &cli.Command{ Sources: cli.EnvVars("BUILDKITE_TRACING_TRACESTATE"), Value: "", }, - &cli.BoolFlag{ - Name: "tracing-propagate-traceparent", - Usage: "Accept traceparent from Buildkite control plane. Requires --opentelemetry-tracing (default: false)", - Sources: cli.EnvVars("BUILDKITE_TRACING_PROPAGATE_TRACEPARENT"), - }, &cli.BoolFlag{ Name: "no-job-api", @@ -492,10 +486,9 @@ var BootstrapCommand = &cli.Command{ StrictSingleHooks: cfg.StrictSingleHooks, Tag: cfg.Tag, TracingBackend: tracingBackend, - TracingServiceName: cfg.TracingServiceName, + TelemetryServiceName: cfg.TelemetryServiceName, TracingTraceParent: cfg.TracingTraceParent, TracingTraceState: cfg.TracingTraceState, - TracingPropagateTraceparent: cfg.TracingPropagateTraceparent, JobAPI: !cfg.NoJobAPI, DisabledWarnings: cfg.DisableWarningsFor, Secrets: cfg.Secrets, diff --git a/clicommand/global.go b/clicommand/global.go index 06e1408c3e..9856a6d893 100644 --- a/clicommand/global.go +++ b/clicommand/global.go @@ -584,8 +584,8 @@ func setupLoggerAndConfig[T any](ctx context.Context, c *cli.Command, opts ...co } if tracingBackend == tracetools.BackendOpenTelemetry { - serviceName := os.Getenv("BUILDKITE_TRACING_SERVICE_NAME") - if sn, err := reflections.GetField(cfg, "TracingServiceName"); err == nil { + serviceName := os.Getenv("BUILDKITE_TELEMETRY_SERVICE_NAME") + if sn, err := reflections.GetField(cfg, "TelemetryServicename"); err == nil { if snStr, ok := sn.(string); ok && snStr != "" { serviceName = snStr } diff --git a/internal/job/config.go b/internal/job/config.go index 72c3d8d10d..ad5834dc0c 100644 --- a/internal/job/config.go +++ b/internal/job/config.go @@ -196,7 +196,7 @@ type ExecutorConfig struct { TracingBackend string // Service name to use when reporting traces. - TracingServiceName string + TelemetryServiceName string // Traceing context information TracingTraceParent string @@ -207,9 +207,6 @@ type ExecutorConfig struct { // enabled (same opt-in gate as TracingTraceParent). TracingTraceState string - // Accept traceparent context from Buildkite control plane - TracingPropagateTraceparent bool - // Whether to start the JobAPI JobAPI bool diff --git a/internal/job/tracing.go b/internal/job/tracing.go index f11951627a..57eedb0ad1 100644 --- a/internal/job/tracing.go +++ b/internal/job/tracing.go @@ -127,7 +127,7 @@ func (e *Executor) startTracingOpenTelemetry(ctx context.Context) (trace.Span, c trace.WithSchemaURL(semconv.SchemaURL), ) - ctx = e.contextWithTraceparentIfEnabled(ctx) + ctx = e.contextWithTraceparentIfPresent(ctx) spanAttrs := append(extras, attribute.String("analytics.event", "true")) ctx, span := tracer.Start(ctx, e.otRootSpanName(), trace.WithAttributes(spanAttrs...), @@ -148,13 +148,8 @@ func (e *Executor) startTracingOpenTelemetry(ctx context.Context) (trace.Span, c // accepting traceparent from Buildkite control plane is an opt-in feature as its // technically a breaking change to the behaviour, and if the server-side tracing // isn't set up correctly, agent traces may end up without root spans to link to -func (e *Executor) contextWithTraceparentIfEnabled(ctx context.Context) context.Context { - if !e.TracingPropagateTraceparent { - return ctx - } - +func (e *Executor) contextWithTraceparentIfPresent(ctx context.Context) context.Context { if e.TracingTraceParent == "" { - e.shell.Warningf("tracing-propagate-traceparent enabled, but no traceparent provided by server") return ctx } diff --git a/packaging/docker/alpine-k8s/buildkite-agent.cfg b/packaging/docker/alpine-k8s/buildkite-agent.cfg index 3340abf933..40c4b0b83f 100644 --- a/packaging/docker/alpine-k8s/buildkite-agent.cfg +++ b/packaging/docker/alpine-k8s/buildkite-agent.cfg @@ -66,5 +66,6 @@ plugins-path="/buildkite/plugins" # Don't show colors in logging # no-color=true -# If set and valid, the given tracing backend will be enabled. Eg: datadog -# tracing-backend="" +# Enable OpenTelemetry metrics export over OTLP. +# Configure the endpoint and protocol with the standard OTEL_EXPORTER_OTLP_* env vars. +# opentelemetry-metrics=true diff --git a/packaging/docker/alpine/buildkite-agent.cfg b/packaging/docker/alpine/buildkite-agent.cfg index 3340abf933..40c4b0b83f 100644 --- a/packaging/docker/alpine/buildkite-agent.cfg +++ b/packaging/docker/alpine/buildkite-agent.cfg @@ -66,5 +66,6 @@ plugins-path="/buildkite/plugins" # Don't show colors in logging # no-color=true -# If set and valid, the given tracing backend will be enabled. Eg: datadog -# tracing-backend="" +# Enable OpenTelemetry metrics export over OTLP. +# Configure the endpoint and protocol with the standard OTEL_EXPORTER_OTLP_* env vars. +# opentelemetry-metrics=true diff --git a/packaging/docker/sidecar/buildkite-agent.cfg b/packaging/docker/sidecar/buildkite-agent.cfg index 3340abf933..40c4b0b83f 100644 --- a/packaging/docker/sidecar/buildkite-agent.cfg +++ b/packaging/docker/sidecar/buildkite-agent.cfg @@ -66,5 +66,6 @@ plugins-path="/buildkite/plugins" # Don't show colors in logging # no-color=true -# If set and valid, the given tracing backend will be enabled. Eg: datadog -# tracing-backend="" +# Enable OpenTelemetry metrics export over OTLP. +# Configure the endpoint and protocol with the standard OTEL_EXPORTER_OTLP_* env vars. +# opentelemetry-metrics=true diff --git a/packaging/docker/ubuntu-20.04/buildkite-agent.cfg b/packaging/docker/ubuntu-20.04/buildkite-agent.cfg index 3340abf933..40c4b0b83f 100644 --- a/packaging/docker/ubuntu-20.04/buildkite-agent.cfg +++ b/packaging/docker/ubuntu-20.04/buildkite-agent.cfg @@ -66,5 +66,6 @@ plugins-path="/buildkite/plugins" # Don't show colors in logging # no-color=true -# If set and valid, the given tracing backend will be enabled. Eg: datadog -# tracing-backend="" +# Enable OpenTelemetry metrics export over OTLP. +# Configure the endpoint and protocol with the standard OTEL_EXPORTER_OTLP_* env vars. +# opentelemetry-metrics=true diff --git a/packaging/docker/ubuntu-22.04/buildkite-agent.cfg b/packaging/docker/ubuntu-22.04/buildkite-agent.cfg index 3340abf933..40c4b0b83f 100644 --- a/packaging/docker/ubuntu-22.04/buildkite-agent.cfg +++ b/packaging/docker/ubuntu-22.04/buildkite-agent.cfg @@ -66,5 +66,6 @@ plugins-path="/buildkite/plugins" # Don't show colors in logging # no-color=true -# If set and valid, the given tracing backend will be enabled. Eg: datadog -# tracing-backend="" +# Enable OpenTelemetry metrics export over OTLP. +# Configure the endpoint and protocol with the standard OTEL_EXPORTER_OTLP_* env vars. +# opentelemetry-metrics=true diff --git a/packaging/docker/ubuntu-24.04/buildkite-agent.cfg b/packaging/docker/ubuntu-24.04/buildkite-agent.cfg index 3340abf933..40c4b0b83f 100644 --- a/packaging/docker/ubuntu-24.04/buildkite-agent.cfg +++ b/packaging/docker/ubuntu-24.04/buildkite-agent.cfg @@ -66,5 +66,6 @@ plugins-path="/buildkite/plugins" # Don't show colors in logging # no-color=true -# If set and valid, the given tracing backend will be enabled. Eg: datadog -# tracing-backend="" +# Enable OpenTelemetry metrics export over OTLP. +# Configure the endpoint and protocol with the standard OTEL_EXPORTER_OTLP_* env vars. +# opentelemetry-metrics=true diff --git a/packaging/github/linux/buildkite-agent.cfg b/packaging/github/linux/buildkite-agent.cfg index 3475513027..a1c916e6f0 100644 --- a/packaging/github/linux/buildkite-agent.cfg +++ b/packaging/github/linux/buildkite-agent.cfg @@ -66,5 +66,6 @@ plugins-path="$HOME/.buildkite-agent/plugins" # Don't show colors in logging # no-color=true -# If set and valid, the given tracing backend will be enabled. Eg: datadog -# tracing-backend="" +# Enable OpenTelemetry metrics export over OTLP. +# Configure the endpoint and protocol with the standard OTEL_EXPORTER_OTLP_* env vars. +# opentelemetry-metrics=true diff --git a/packaging/github/windows/buildkite-agent.cfg b/packaging/github/windows/buildkite-agent.cfg index 3acbc668c6..069cef3693 100644 --- a/packaging/github/windows/buildkite-agent.cfg +++ b/packaging/github/windows/buildkite-agent.cfg @@ -48,5 +48,6 @@ plugins-path="C:\buildkite-agent\plugins" # Enable debug mode # debug=true -# If set and valid, the given tracing backend will be enabled. Eg: datadog -# tracing-backend="" +# Enable OpenTelemetry metrics export over OTLP. +# Configure the endpoint and protocol with the standard OTEL_EXPORTER_OTLP_* env vars. +# opentelemetry-metrics=true diff --git a/packaging/linux/root/usr/share/buildkite-agent/buildkite-agent.cfg b/packaging/linux/root/usr/share/buildkite-agent/buildkite-agent.cfg index f68c78b84a..368c83e7fd 100644 --- a/packaging/linux/root/usr/share/buildkite-agent/buildkite-agent.cfg +++ b/packaging/linux/root/usr/share/buildkite-agent/buildkite-agent.cfg @@ -69,6 +69,3 @@ plugins-path="/etc/buildkite-agent/plugins" # Enable OpenTelemetry metrics export over OTLP. # Configure the endpoint and protocol with the standard OTEL_EXPORTER_OTLP_* env vars. # opentelemetry-metrics=true - -# If set and valid, the given tracing backend will be enabled. Eg: datadog, opentelemetry -# tracing-backend=""