diff --git a/benchmark/auto.go b/benchmark/auto.go index 0f6f3a1..9f3adfc 100644 --- a/benchmark/auto.go +++ b/benchmark/auto.go @@ -618,6 +618,46 @@ func (m cacheMetrics) DisplayHitRate() float64 { return m.hitRate } +// noCacheDataObserved reports whether, over a large-enough window, neither +// cache signal has ever fired: the TTFT heuristic saw no warm request AND the +// server never reported cached_tokens. In that case DisplayHitRate's 0% is a +// real absence of data — the server may not support prompt caching, or (for +// vLLM/SGLang) may simply not be configured to report it — rather than a +// rounding artifact of an otherwise-working cache. Requiring minCount records +// avoids flagging a run that just hasn't warmed up yet. +func noCacheDataObserved(m cacheMetrics, minCount int) bool { + return m.count >= minCount && m.hitRate == 0 && !m.serverReported +} + +// cacheWarningMessage tailors the cacheWarning explanation to what the model +// spec's type= actually names, when it names one of the two backends known +// to gate cached_tokens reporting behind a server-launch flag: vLLM's +// --enable-prompt-tokens-details and SGLang's --enable-cache-report. Both +// require the client to ask per-request (already done for type=openai_sglang +// — see llm/chat_clients.go and replay_router_wire.go) AND the server to be +// launched with the matching flag; wekai controls only the former; a +// deployment missing the latter looks from here exactly like a server that +// genuinely never caches — the message says both are possible rather than +// asserting whichever cause a client can never observe from the API alone. +func cacheWarningMessage(modelSpec string) string { + generic := "Server may not support prompt caching, or is not configured to report it" + if !llm.IsDynamicModel(modelSpec) { + return generic + } + dyn, err := llm.ParseDynamicModel(modelSpec) + if err != nil { + return generic + } + switch dyn.Type { + case "openai_vllm": + return generic + " (vLLM must be launched with --enable-prompt-tokens-details)" + case "openai_sglang": + return generic + " (SGLang must be launched with --enable-cache-report)" + default: + return generic + } +} + // GlobalLocalCacheRate returns the all-time fraction of warm input tokens among all input tokens. // O(1): reads two running counters maintained in Add(), never scans history. // Purely local: a request is "cached" when its series already submitted this prefix before. @@ -1437,7 +1477,7 @@ func printAutoSummary(res autoBenchmarkResult, cfg AutoBenchmarkConfig) { fmt.Printf(" Cache hit rate : %.1f%%\n", res.cacheHitRate*100) fmt.Printf(" Tok/s in/out : %s / %s\n", formatKilo(res.inputTokPerSec), formatKilo(res.outputTokPerSec)) if res.cacheWarning { - fmt.Println(" ⚠ Server may not support prompt caching") + fmt.Printf(" ⚠ %s\n", cacheWarningMessage(cfg.Model)) } fmt.Println(strings.Repeat("-", 62)) totalInput := res.totalInputCold + res.totalInputWarm @@ -2079,6 +2119,9 @@ func runSingleModelBenchmark( // waits for the goroutine — no sample write can race the file close. defer sampler.stop() } + if sampler := startSGLangMetricsSampler(benchCtx, cfg.Model, rdw); sampler != nil { + defer sampler.stop() + } if cfg.HotSeriesConcurrency > 0 { st.hotGate = newConcurrencyGate(cfg.HotSeriesConcurrency*hotGateFanoutMultiplier, !cfg.FIFOGateOrder) } @@ -2836,6 +2879,14 @@ func runSingleModelBenchmark( cm2 := st.stream.CacheMetrics(cfg.CacheWindowSize, cfg.MinStabilization) hitRate := cm2.hitRate + // Latch cacheWarning once neither signal has ever shown a hit over a + // large-enough window — see noCacheDataObserved. + if noCacheDataObserved(cm2, cfg.MinStabilization) { + st.mu.Lock() + st.cacheWarning = true + st.mu.Unlock() + } + if cfg.VerboseCache && cm2.count > 0 && math.Abs(hitRate-lastVerboseHitRate) >= 0.03 { lastVerboseHitRate = hitRate frozenBaseline := st.earlyColdStartTTFT() diff --git a/benchmark/replay_router_charsize_test.go b/benchmark/replay_router_charsize_test.go index 6409871..b40c354 100644 --- a/benchmark/replay_router_charsize_test.go +++ b/benchmark/replay_router_charsize_test.go @@ -129,7 +129,7 @@ func TestBuildOpenAIChatCompletionsBodyCharsPerToken(t *testing.T) { return msg["content"].(string) } - byteBody, _, err := buildOpenAIChatCompletionsBody(req, docs, "model", "", 0, false, 0, nil) + byteBody, _, err := buildOpenAIChatCompletionsBody(req, docs, "model", "", 0, false, 0, nil, false) if err != nil { t.Fatalf("build (charsPerToken=0): %v", err) } @@ -140,7 +140,7 @@ func TestBuildOpenAIChatCompletionsBodyCharsPerToken(t *testing.T) { const charsPerToken = 3.4 wantLen := int(math.Round(150 * charsPerToken)) - tokBody, _, err := buildOpenAIChatCompletionsBody(req, docs, "model", "", 0, false, charsPerToken, nil) + tokBody, _, err := buildOpenAIChatCompletionsBody(req, docs, "model", "", 0, false, charsPerToken, nil, false) if err != nil { t.Fatalf("build (charsPerToken=3.4): %v", err) } diff --git a/benchmark/replay_router_post.go b/benchmark/replay_router_post.go index a13c0e7..b552594 100644 --- a/benchmark/replay_router_post.go +++ b/benchmark/replay_router_post.go @@ -48,7 +48,7 @@ type replayPoster struct { epMu sync.Mutex epResolved string // latched endpoint; "" until the first success epFellBack bool - apiType string // "anthropic", "openai", or "openai_vllm" + apiType string // "anthropic", "openai", "openai_vllm", or "openai_sglang" client *http.Client // retryBudget is the total time a request may spend waiting out 429s // before the shed stands as an error. Defaults to retry429Budget; a field @@ -207,18 +207,20 @@ func newReplayPoster(modelSpec string, keys llm.APIKeys, endpointOverride string return nil, fmt.Errorf("parse model spec: %w", err) } - // Accepted target types: anthropic (original), openai, openai_vllm. - // openai_vllm is treated identically to openai in the replay path — both - // use /v1/chat/completions. The distinction matters for the Chat path - // (max_tokens vs max_completion_tokens) but replay requests carry their - // own max_tokens so it's irrelevant here. + // Accepted target types: anthropic (original), openai, openai_vllm, + // openai_sglang. openai_vllm and openai_sglang are treated identically to + // openai in the replay path — all three use /v1/chat/completions. The + // distinction matters for the Chat path (max_tokens vs + // max_completion_tokens, and SGLang's return_cached_tokens_details) but + // replay requests carry their own max_tokens and opt into cached-token + // reporting explicitly below, so the type only selects that behavior. switch dyn.Type { case "anthropic": // OK — existing behaviour. - case "openai", "openai_vllm": + case "openai", "openai_vllm", "openai_sglang": // OK — new path. default: - return nil, fmt.Errorf("router-replay supports type=anthropic, type=openai, or type=openai_vllm (got %q)", dyn.Type) + return nil, fmt.Errorf("router-replay supports type=anthropic, type=openai, type=openai_vllm, or type=openai_sglang (got %q)", dyn.Type) } base := "" @@ -235,7 +237,7 @@ func newReplayPoster(modelSpec string, keys llm.APIKeys, endpointOverride string // local endpoints); OpenAI targets use Bearer auth with the OpenAI key // (or dummy-key for local endpoints). apiKey := keys.Anthropic - if dyn.Type == "openai" || dyn.Type == "openai_vllm" { + if dyn.Type == "openai" || dyn.Type == "openai_vllm" || dyn.Type == "openai_sglang" { apiKey = keys.OpenAI } if apiKey == "" { @@ -246,7 +248,7 @@ func newReplayPoster(modelSpec string, keys llm.APIKeys, endpointOverride string // The primary attempt appends it to the operator's base verbatim; the // fallback inserts /v1 (see the struct comment for the contract). leaf := "/messages" - if dyn.Type == "openai" || dyn.Type == "openai_vllm" { + if dyn.Type == "openai" || dyn.Type == "openai_vllm" || dyn.Type == "openai_sglang" { leaf = "/chat/completions" } epPrimary := base + leaf @@ -581,8 +583,8 @@ func (p *replayPoster) do( var canonical string var err error switch p.apiType { - case "openai", "openai_vllm": - bodyBytes, canonical, err = buildOpenAIChatCompletionsBody(req, docs, p.model, stampFor(p, req), p.outputRatio, p.forceVolume, p.replayCharsPerToken, inj) + case "openai", "openai_vllm", "openai_sglang": + bodyBytes, canonical, err = buildOpenAIChatCompletionsBody(req, docs, p.model, stampFor(p, req), p.outputRatio, p.forceVolume, p.replayCharsPerToken, inj, p.apiType == "openai_sglang") default: bodyBytes, canonical, err = buildAnthropicMessagesBody(req, docs, p.model, stampFor(p, req), p.outputRatio, p.forceVolume, p.replayCharsPerToken, inj) } @@ -742,7 +744,7 @@ func (p *replayPoster) do( // consumers compute must describe the attempt the server actually ran, so // that backoff cannot make a healthy fleet look slow. The client-side wait // is added back into TotalResponseTime below, where it belongs. - if p.apiType == "openai" || p.apiType == "openai_vllm" { + if p.apiType == "openai" || p.apiType == "openai_vllm" || p.apiType == "openai_sglang" { if req.Stream { consumeOpenAISSE(respReader, attemptStart, &m) } else { @@ -1267,8 +1269,8 @@ func (p *replayPoster) dryDo( inj := p.buildInjection(req, su) var canonical string switch p.apiType { - case "openai", "openai_vllm": - _, canonical, _ = buildOpenAIChatCompletionsBody(req, docs, p.model, stampFor(p, req), p.outputRatio, p.forceVolume, p.replayCharsPerToken, inj) + case "openai", "openai_vllm", "openai_sglang": + _, canonical, _ = buildOpenAIChatCompletionsBody(req, docs, p.model, stampFor(p, req), p.outputRatio, p.forceVolume, p.replayCharsPerToken, inj, p.apiType == "openai_sglang") default: _, canonical, _ = buildAnthropicMessagesBody(req, docs, p.model, stampFor(p, req), p.outputRatio, p.forceVolume, p.replayCharsPerToken, inj) } diff --git a/benchmark/replay_router_post_test.go b/benchmark/replay_router_post_test.go index e02e552..80e8de7 100644 --- a/benchmark/replay_router_post_test.go +++ b/benchmark/replay_router_post_test.go @@ -214,6 +214,13 @@ func TestNewReplayPoster_OpenAI(t *testing.T) { wantPrimary: "http://127.0.0.1:8000/v1/chat/completions", wantFallback: "http://127.0.0.1:8000/v1/v1/chat/completions", }, + { + name: "openai_sglang type, /v1 base", + modelSpec: "dynamic/http://127.0.0.1:8000/v1,type=openai_sglang,model=my-model", + wantType: "openai_sglang", + wantPrimary: "http://127.0.0.1:8000/v1/chat/completions", + wantFallback: "http://127.0.0.1:8000/v1/v1/chat/completions", + }, { name: "anthropic type, /v1 base", modelSpec: "dynamic/http://127.0.0.1:8000/v1,type=anthropic,model=claude", @@ -467,6 +474,70 @@ func TestOpenAIReplayEndToEnd(t *testing.T) { } } +// TestOpenAISGLangReplaySetsReturnCachedTokensDetails confirms +// type=openai_sglang is accepted by newReplayPoster's type allow-list and +// that the wire body it builds carries return_cached_tokens_details — SGLang's +// per-request opt-in for usage.prompt_tokens_details.cached_tokens (vLLM's +// equivalent is the server-launch flag --enable-prompt-tokens-details, so +// type=openai_vllm needs no such field — see TestOpenAIReplayEndToEnd, which +// exercises type=openai without it). Without this, cache hit rate reads as 0 +// against every SGLang response, same failure mode as an unflagged vLLM +// server. +func TestOpenAISGLangReplaySetsReturnCachedTokensDetails(t *testing.T) { + var receivedBody map[string]interface{} + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/v1/chat/completions" { + w.WriteHeader(404) + return + } + if err := json.NewDecoder(r.Body).Decode(&receivedBody); err != nil { + t.Errorf("failed to decode request body: %v", err) + w.WriteHeader(400) + return + } + w.Header().Set("Content-Type", "text/event-stream") + w.WriteHeader(200) + fmt.Fprintf(w, `data: {"id":"c","object":"chat.completion.chunk","created":1,"model":"test-model","choices":[{"index":0,"delta":{"content":"ok"},"finish_reason":null}]}`+"\n\n") + if flusher, ok := w.(http.Flusher); ok { + flusher.Flush() + } + fmt.Fprintf(w, `data: {"id":"c","object":"chat.completion.chunk","created":1,"model":"test-model","choices":[{"index":0,"delta":{},"finish_reason":"stop"}],"usage":{"prompt_tokens":10,"completion_tokens":1,"total_tokens":11,"prompt_tokens_details":{"cached_tokens":5}}}`+"\n\n") + fmt.Fprintf(w, "data: [DONE]\n\n") + })) + defer ts.Close() + + modelSpec := fmt.Sprintf("dynamic/%s,type=openai_sglang,model=test-model", ts.URL) + keys := llm.APIKeys{OpenAI: "sk-test-123"} + p, err := newReplayPoster(modelSpec, keys, "", "", false, 0, 0, 0, nil, nil) + if err != nil { + t.Fatalf("newReplayPoster: %v", err) + } + if p.apiType != "openai_sglang" { + t.Fatalf("apiType = %q, want openai_sglang", p.apiType) + } + + req := RouterReplayRequest{ + RequestID: 1, + Stream: true, + OutputTokens: 100, + SystemBlocks: []RouterReplaySystemBlock{{Hash: "syshash", Bytes: 250}}, + Messages: []RouterReplayMessage{ + {Role: "user", Hash: "msghash1", Bytes: 60, BlockTypes: []string{"text"}}, + }, + } + st := &autoState{stream: newCompletionStream(200)} + metrics := p.do(context.Background(), req, strings.Repeat("x", 300), 1, "session-1", "instance-1", 1, st, nil) + if metrics.Error != nil { + t.Fatalf("unexpected error: %v", metrics.Error) + } + if v, ok := receivedBody["return_cached_tokens_details"]; !ok || v != true { + t.Errorf("return_cached_tokens_details = %v (present=%v), want true", v, ok) + } + if metrics.UsageData.CachedTokens.Count != 5 { + t.Errorf("cached tokens = %d, want 5", metrics.UsageData.CachedTokens.Count) + } +} + // TestOpenAIReplayToolTranslation verifies that assistant messages with // tool_use blocks are translated into proper OpenAI tool_calls (not flattened). func TestOpenAIReplayToolTranslation(t *testing.T) { @@ -485,7 +556,7 @@ func TestOpenAIReplayToolTranslation(t *testing.T) { }, } - body, _, err := buildOpenAIChatCompletionsBody(req, docs, "test-model", "", 0, false, 0, nil) + body, _, err := buildOpenAIChatCompletionsBody(req, docs, "test-model", "", 0, false, 0, nil, false) if err != nil { t.Fatalf("buildOpenAIChatCompletionsBody: %v", err) } @@ -548,7 +619,7 @@ func TestOpenAIBodyBuilderExtra(t *testing.T) { {Role: "user", Hash: "h1", Bytes: 50, BlockTypes: []string{"text"}}, }, } - body, _, err := buildOpenAIChatCompletionsBody(req, docs, "test-model", "run-42", 0, false, 0, nil) + body, _, err := buildOpenAIChatCompletionsBody(req, docs, "test-model", "run-42", 0, false, 0, nil, false) if err != nil { t.Fatalf("build: %v", err) } @@ -577,7 +648,7 @@ func TestOpenAIBodyBuilderExtra(t *testing.T) { {Role: "user", Hash: "h1", Bytes: 50, BlockTypes: []string{"text"}}, }, } - body, _, err := buildOpenAIChatCompletionsBody(req, docs, "test-model", "", 0, false, 0, nil) + body, _, err := buildOpenAIChatCompletionsBody(req, docs, "test-model", "", 0, false, 0, nil, false) if err != nil { t.Fatalf("build: %v", err) } @@ -605,7 +676,7 @@ func TestOpenAIBodyBuilderExtra(t *testing.T) { {Role: "user", Hash: "h1", Bytes: 50, BlockTypes: []string{"text"}}, }, } - body, _, err := buildOpenAIChatCompletionsBody(req, docs, "test-model", "", 0, false, 0, nil) + body, _, err := buildOpenAIChatCompletionsBody(req, docs, "test-model", "", 0, false, 0, nil, false) if err != nil { t.Fatalf("build: %v", err) } @@ -631,7 +702,7 @@ func TestOpenAIBodyBuilderExtra(t *testing.T) { Stream: true, OutputTokens: 100, } - body, _, err := buildOpenAIChatCompletionsBody(req, docs, "test-model", "", 0, false, 0, nil) + body, _, err := buildOpenAIChatCompletionsBody(req, docs, "test-model", "", 0, false, 0, nil, false) if err != nil { t.Fatalf("build: %v", err) } diff --git a/benchmark/replay_router_uuid_test.go b/benchmark/replay_router_uuid_test.go index db6d87d..5bbb2f8 100644 --- a/benchmark/replay_router_uuid_test.go +++ b/benchmark/replay_router_uuid_test.go @@ -436,7 +436,7 @@ func TestWireInjectionDeterminism(t *testing.T) { var body []byte var err error if kind == "openai" { - body, _, err = buildOpenAIChatCompletionsBody(r, docs, "model", "", 0, false, 0, inj) + body, _, err = buildOpenAIChatCompletionsBody(r, docs, "model", "", 0, false, 0, inj, false) } else { body, _, err = buildAnthropicMessagesBody(r, docs, "model", "", 0, false, 0, inj) } diff --git a/benchmark/replay_router_wire.go b/benchmark/replay_router_wire.go index 65a7db2..26819f5 100644 --- a/benchmark/replay_router_wire.go +++ b/benchmark/replay_router_wire.go @@ -593,7 +593,17 @@ func buildOpenAITools(spec *RouterReplayToolsSpec, docs string, charsPerToken fl // // inj carries the UUID cache-coherency injection (--verify, // router path — see replay_router_uuid.go); nil means "no injection". -func buildOpenAIChatCompletionsBody(req RouterReplayRequest, docs string, modelName string, runID string, outputRatio float64, forceVolume bool, charsPerToken float64, inj *uuidInjection) ([]byte, string, error) { +// +// sglang requests cached-token reporting per-request via +// return_cached_tokens_details. That is necessary but not sufficient: the +// server must ALSO be launched with --enable-cache-report +// (sglang/srt/entrypoints/openai/serving_chat.py checks both the request +// field and tokenizer_manager.server_args.enable_cache_report) — the same +// two-sided requirement as vLLM's --enable-prompt-tokens-details flag. +// Without either half, usage.prompt_tokens_details is simply absent from +// every SGLang response, and cache hit rate always reads as 0 — see +// cacheWarningMessage in auto.go, which surfaces this in the run summary. +func buildOpenAIChatCompletionsBody(req RouterReplayRequest, docs string, modelName string, runID string, outputRatio float64, forceVolume bool, charsPerToken float64, inj *uuidInjection, sglang bool) ([]byte, string, error) { var stampByHash map[string]turnStamp if inj != nil { stampByHash = inj.StampByHash @@ -603,6 +613,9 @@ func buildOpenAIChatCompletionsBody(req RouterReplayRequest, docs string, modelN "max_tokens": pickMaxTokens(req, outputRatio), "stream": req.Stream, } + if sglang { + body["return_cached_tokens_details"] = true + } if req.Temperature != nil { body["temperature"] = *req.Temperature } diff --git a/benchmark/replay_router_wire_test.go b/benchmark/replay_router_wire_test.go index 789f9f6..b26c623 100644 --- a/benchmark/replay_router_wire_test.go +++ b/benchmark/replay_router_wire_test.go @@ -172,11 +172,14 @@ func TestEffectiveSystemBlocksSkipsHeader(t *testing.T) { }, } hdrText := synthText("uniq-header-per-req", 106, "") + openaiBuilder := func(r RouterReplayRequest, docs, model, runID string, outputRatio float64, forceVolume bool, charsPerToken float64, inj *uuidInjection) ([]byte, string, error) { + return buildOpenAIChatCompletionsBody(r, docs, model, runID, outputRatio, forceVolume, charsPerToken, inj, false) + } for _, builder := range []struct { name string fn func(RouterReplayRequest, string, string, string, float64, bool, float64, *uuidInjection) ([]byte, string, error) }{ - {"openai", buildOpenAIChatCompletionsBody}, + {"openai", openaiBuilder}, {"anthropic", buildAnthropicMessagesBody}, } { body, canonical, err := builder.fn(req, "", "m", "", 0, false, 0, nil) @@ -333,7 +336,7 @@ func TestBuildOpenAIChatCompletionsBodyForceOutput(t *testing.T) { // force-output off: no ignore_eos, but the instruction still rides — the // modes differ ONLY by engine enforcement. - body, _, err := buildOpenAIChatCompletionsBody(req, docs, "model", "", 0, false, 0, nil) + body, _, err := buildOpenAIChatCompletionsBody(req, docs, "model", "", 0, false, 0, nil, false) if err != nil { t.Fatalf("build (force-output off): %v", err) } @@ -349,7 +352,7 @@ func TestBuildOpenAIChatCompletionsBodyForceOutput(t *testing.T) { } // force-output on (default): ignore_eos=true AND the instruction is present. - body, _, err = buildOpenAIChatCompletionsBody(req, docs, "model", "", 0, true, 0, nil) + body, _, err = buildOpenAIChatCompletionsBody(req, docs, "model", "", 0, true, 0, nil, false) if err != nil { t.Fatalf("build (force-output on): %v", err) } @@ -384,7 +387,7 @@ func TestBuildAnthropicMessagesBodyOutputRatioMaxTokens(t *testing.T) { t.Errorf("anthropic max_tokens = %v, want %v", got, want) } - openaiBody, _, err := buildOpenAIChatCompletionsBody(req, docs, "model", "", 0.25, false, 0, nil) + openaiBody, _, err := buildOpenAIChatCompletionsBody(req, docs, "model", "", 0.25, false, 0, nil, false) if err != nil { t.Fatalf("openai build: %v", err) } diff --git a/benchmark/replay_router_wire_tools_test.go b/benchmark/replay_router_wire_tools_test.go index e0680c4..357048d 100644 --- a/benchmark/replay_router_wire_tools_test.go +++ b/benchmark/replay_router_wire_tools_test.go @@ -44,7 +44,7 @@ func TestOpenAIVsAnthropicBodySize(t *testing.T) { if err != nil { t.Fatalf("buildAnthropicMessagesBody: %v", err) } - openaiBody, _, err := buildOpenAIChatCompletionsBody(req, docs, "model-a", "", 0, false, 0, nil) + openaiBody, _, err := buildOpenAIChatCompletionsBody(req, docs, "model-a", "", 0, false, 0, nil, false) if err != nil { t.Fatalf("buildOpenAIChatCompletionsBody: %v", err) } @@ -128,7 +128,7 @@ func TestOpenAIToolUseConversion(t *testing.T) { }, } - body, _, err := buildOpenAIChatCompletionsBody(req, docs, "model-x", "", 0, false, 0, nil) + body, _, err := buildOpenAIChatCompletionsBody(req, docs, "model-x", "", 0, false, 0, nil, false) if err != nil { t.Fatalf("buildOpenAIChatCompletionsBody: %v", err) } @@ -221,7 +221,7 @@ func TestOpenAIToolUseConversion(t *testing.T) { }, }, } - orphanBody, _, err := buildOpenAIChatCompletionsBody(reqOrphan, docs, "model-x", "", 0, false, 0, nil) + orphanBody, _, err := buildOpenAIChatCompletionsBody(reqOrphan, docs, "model-x", "", 0, false, 0, nil, false) if err != nil { t.Fatalf("buildOpenAIChatCompletionsBody orphan: %v", err) } diff --git a/benchmark/sglang_metrics.go b/benchmark/sglang_metrics.go new file mode 100644 index 0000000..186e070 --- /dev/null +++ b/benchmark/sglang_metrics.go @@ -0,0 +1,312 @@ +package benchmark + +// SGLang metrics collector: during `benchmark auto` runs against a +// type=openai_sglang endpoint, a per-model sampler goroutine polls the +// server's Prometheus /metrics endpoint every sglangMetricsSampleInterval and +// persists the current cache-hit-rate gauge into the same +// --save-request-data JSONL stream as the request rows, as records of their +// own type ("sglang_metrics_sample"). Sampling is strictly best-effort and +// can never affect the benchmark itself. +// +// This deliberately does NOT reuse vllmMetricsSampler's delta-accumulation +// scheme. vLLM's vllm:prompt_tokens_by_source is a monotonic counter, so a +// fleet total is the sum of per-endpoint DELTAS (see vllm_metrics.go). +// SGLang's sglang:cache_hit_rate is a GAUGE already expressed as a ratio in +// [0,1] — it is a point-in-time reading, not something that accumulates. +// Delta-summing it would produce a number with no meaning; the only correct +// thing to do with a gauge is read its current value. +// +// Unlike vLLM eligibility (which speculatively samples any chat/completions +// endpoint on the theory that it might be vLLM behind the scenes — see +// vllmMetricsEndpoints), SGLang sampling only ever starts when the spec says +// type=openai_sglang outright: there is no bare-host/default shape in this +// codebase that plausibly resolves to SGLang, so guessing would only add +// failed-probe traffic against every non-SGLang endpoint. A sampler that +// starts therefore keeps polling for the life of the run — the operator +// asserted the server is SGLang, so a server still loading weights or +// briefly unreachable must not cost the rest of the run's samples. +// +// The gauge family is sglang:cache_hit_rate (sglang/srt/metrics/collector.py), +// labeled by model_name (and, on a multi-GPU deployment, engine/rank labels +// this code does not care about). Multiple label instances are averaged, not +// summed — each is already a ratio in [0,1], so averaging is the only +// combination that keeps the result in range. + +import ( + "bufio" + "context" + "fmt" + "io" + "net/http" + "os" + "strconv" + "strings" + "time" + + "github.com/weka/wekai/llm" +) + +const ( + recordTypeSGLangMetricsSample = "sglang_metrics_sample" + + sglangCacheHitRateFamily = "sglang:cache_hit_rate" + + sglangMetricsSampleInterval = 60 * time.Second + sglangMetricsFetchTimeout = 5 * time.Second +) + +// sglangMetricsSample is one periodic sample persisted into the request-data +// JSONL alongside requestDataRecord rows. record_type distinguishes it from +// request rows (which carry no record_type field) and from vllmMetricsSample. +type sglangMetricsSample struct { + RecordType string `json:"record_type"` + TS time.Time `json:"ts"` + Model string `json:"model"` + + // CacheHitRate is the average of sglang:cache_hit_rate across all + // endpoints and label instances that answered this round — a current + // reading, not an accumulated delta (see package comment). + CacheHitRate float64 `json:"cache_hit_rate"` + + // EndpointsOK of EndpointsTotal answered this round. Without them a flat + // interval and an unobserved one look identical. + EndpointsOK int `json:"endpoints_ok"` + EndpointsTotal int `json:"endpoints_total"` +} + +// parseCacheHitRateGauge scans Prometheus text exposition for the +// sglang:cache_hit_rate gauge and returns the average value across every +// label-set instance found (e.g. one per model_name). Returns ok=false when +// the family is absent from the scrape. +func parseCacheHitRateGauge(r io.Reader) (avg float64, ok bool, err error) { + sc := bufio.NewScanner(r) + sc.Buffer(make([]byte, 0, 64*1024), 16*1024*1024) + var sum float64 + var n int + for sc.Scan() { + line := sc.Text() + if line == "" || line[0] == '#' { + continue + } + rest, matched := strings.CutPrefix(line, sglangCacheHitRateFamily) + if !matched { + continue + } + // The next byte must open a label set or separate the name from the + // value with whitespace — reject sibling series sharing the prefix + // (there are none known today, but the family-name-not-followed-by- + // '{'-or-space check mirrors parsePromptTokensBySource's guard). + if len(rest) == 0 { + continue + } + if rest[0] != '{' && rest[0] != ' ' { + continue + } + valueField := rest + if rest[0] == '{' { + closeIdx := strings.LastIndex(rest, "}") + if closeIdx < 0 { + continue + } + valueField = rest[closeIdx+1:] + } + fields := strings.Fields(valueField) + if len(fields) == 0 { + continue + } + v, perr := strconv.ParseFloat(fields[0], 64) + if perr != nil { + continue + } + sum += v + n++ + } + if err := sc.Err(); err != nil { + return 0, false, err + } + if n == 0 { + return 0, false, nil + } + return sum / float64(n), true, nil +} + +// sglangMetricsEndpoints derives the Prometheus /metrics URLs for a dynamic +// model spec pointing at an SGLang endpoint (the /v1 API suffix is stripped +// to reach the server root, where SGLang mounts /metrics — same convention +// as vLLM). Returns nil unless the spec says type=openai_sglang outright: +// see the package comment for why this is never speculative. +func sglangMetricsEndpoints(model string) []string { + if !llm.IsDynamicModel(model) { + return nil + } + dyn, err := llm.ParseDynamicModel(model) + if err != nil || dyn.Type != "openai_sglang" { + return nil + } + out := make([]string, 0, len(dyn.BaseURLs)) + for _, u := range dyn.BaseURLs { + u = strings.TrimRight(u, "/") + u = strings.TrimSuffix(u, "/v1") + out = append(out, u+"/metrics") + } + return out +} + +// sglangMetricsSampler polls one model's endpoints and writes samples to rdw. +type sglangMetricsSampler struct { + model string + urls []string + rdw *requestDataWriter + interval time.Duration + client *http.Client + now func() time.Time + logf func(format string, args ...any) + + everSucceeded bool + + cancel context.CancelFunc + done chan struct{} +} + +// startSGLangMetricsSampler launches the sampler goroutine for cfg.Model when +// the spec is type=openai_sglang and request data is being saved. Returns nil +// when sampling doesn't apply. Callers must invoke stop() before closing rdw. +func startSGLangMetricsSampler(ctx context.Context, model string, rdw *requestDataWriter) *sglangMetricsSampler { + if rdw == nil { + return nil + } + urls := sglangMetricsEndpoints(model) + if len(urls) == 0 { + return nil + } + runCtx, cancel := context.WithCancel(ctx) + s := &sglangMetricsSampler{ + model: model, + urls: urls, + rdw: rdw, + interval: sglangMetricsSampleInterval, + client: &http.Client{}, + now: time.Now, + logf: func(f string, a ...any) { fmt.Fprintf(os.Stderr, "[sglang-metrics] "+f+"\n", a...) }, + cancel: cancel, + done: make(chan struct{}), + } + go s.run(runCtx) + return s +} + +// stop terminates the sampler and waits for the goroutine to exit, so no +// write can race the rdw close that follows. +func (s *sglangMetricsSampler) stop() { + s.cancel() + <-s.done +} + +func (s *sglangMetricsSampler) run(ctx context.Context) { + defer close(s.done) + if !s.sampleOnce(ctx) { + return + } + ticker := time.NewTicker(s.interval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + if !s.sampleOnce(ctx) { + return + } + } + } +} + +// sampleOnce fetches every endpoint and writes one sample of the AVERAGE +// current gauge value across whichever endpoints answered. It reports +// whether the sampler should keep polling; ctx cancellation is the only case +// that stops it — the operator asserted type=openai_sglang, so a transient +// failure (server still loading weights, momentary timeout) is ridden out +// rather than treated as evidence of a bad guess (there is no guess here). +func (s *sglangMetricsSampler) sampleOnce(ctx context.Context) (keepPolling bool) { + var sum float64 + ok := 0 + var lastErr error + var lastBadURL string + for _, u := range s.urls { + v, err := s.fetchOne(ctx, u) + if err != nil { + if ctx.Err() != nil { + return false + } + lastErr, lastBadURL = err, u + continue + } + sum += v + ok++ + } + + rate := 0.0 + if ok > 0 { + rate = sum / float64(ok) + s.noteSuccess() + } else if lastErr != nil { + s.log("%s unavailable (%v) — skipping this sample, still polling every %s", lastBadURL, lastErr, s.interval) + } + s.write(rate, ok) + return true +} + +func (s *sglangMetricsSampler) noteSuccess() { + if s.everSucceeded { + return + } + s.everSucceeded = true + s.log("%s serves %s — sampling every %s", strings.Join(s.urls, ", "), sglangCacheHitRateFamily, s.interval) +} + +func (s *sglangMetricsSampler) write(rate float64, endpointsOK int) { + if s.rdw == nil { + return + } + rec := sglangMetricsSample{ + RecordType: recordTypeSGLangMetricsSample, + TS: s.now(), + Model: s.model, + CacheHitRate: rate, + EndpointsOK: endpointsOK, + EndpointsTotal: len(s.urls), + } + // Write errors are swallowed: sampling must never affect the benchmark. + _ = s.rdw.writeAny(rec) +} + +func (s *sglangMetricsSampler) log(format string, args ...any) { + if s.logf != nil { + s.logf(format, args...) + } +} + +func (s *sglangMetricsSampler) fetchOne(ctx context.Context, url string) (float64, error) { + reqCtx, cancel := context.WithTimeout(ctx, sglangMetricsFetchTimeout) + defer cancel() + req, err := http.NewRequestWithContext(reqCtx, http.MethodGet, url, nil) + if err != nil { + return 0, err + } + resp, err := s.client.Do(req) + if err != nil { + return 0, err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return 0, fmt.Errorf("metrics fetch: status %d", resp.StatusCode) + } + avg, ok, err := parseCacheHitRateGauge(resp.Body) + if err != nil { + return 0, err + } + if !ok { + return 0, fmt.Errorf("metrics fetch: family %s not found", sglangCacheHitRateFamily) + } + return avg, nil +} diff --git a/benchmark/sglang_metrics_test.go b/benchmark/sglang_metrics_test.go new file mode 100644 index 0000000..a4697d1 --- /dev/null +++ b/benchmark/sglang_metrics_test.go @@ -0,0 +1,214 @@ +package benchmark + +import ( + "bufio" + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +const sglangPromFixture = `# HELP sglang:cache_hit_rate The cache hit rate. +# TYPE sglang:cache_hit_rate gauge +sglang:cache_hit_rate{model_name="m",engine_type="unified"} 0.75 +sglang:num_running_reqs{model_name="m"} 3 +` + +func TestParseCacheHitRateGauge(t *testing.T) { + avg, ok, err := parseCacheHitRateGauge(strings.NewReader(sglangPromFixture)) + if err != nil { + t.Fatalf("parse: %v", err) + } + if !ok { + t.Fatal("expected ok=true") + } + if avg != 0.75 { + t.Fatalf("got %v, want 0.75", avg) + } +} + +func TestParseCacheHitRateGaugeAveragesMultipleInstances(t *testing.T) { + in := `sglang:cache_hit_rate{model_name="m",engine="0"} 0.6 +sglang:cache_hit_rate{model_name="m",engine="1"} 0.4 +` + avg, ok, err := parseCacheHitRateGauge(strings.NewReader(in)) + if err != nil { + t.Fatalf("parse: %v", err) + } + if !ok { + t.Fatal("expected ok=true") + } + if avg != 0.5 { + t.Fatalf("got %v, want 0.5", avg) + } +} + +func TestParseCacheHitRateGaugeAbsent(t *testing.T) { + avg, ok, err := parseCacheHitRateGauge(strings.NewReader("vllm:prompt_tokens_by_source_total{source=\"local_compute\"} 7\n")) + if err != nil { + t.Fatalf("parse should tolerate an unrelated family: %v", err) + } + if ok { + t.Fatalf("expected ok=false, got avg=%v", avg) + } +} + +func TestParseCacheHitRateGaugeGarbage(t *testing.T) { + avg, ok, err := parseCacheHitRateGauge(strings.NewReader("not prometheus\n")) + if err != nil { + t.Fatalf("parse should tolerate garbage: %v", err) + } + if ok { + t.Fatalf("expected ok=false, got avg=%v", avg) + } +} + +func TestSGLangMetricsEndpoints(t *testing.T) { + cases := []struct { + spec string + want []string + }{ + {"dynamic/http://localhost:8000/v1,type=openai_sglang,alias=x", []string{"http://localhost:8000/metrics"}}, + {"dynamic/http://a:8000/v1|http://b:8001/v1,type=openai_sglang", []string{"http://a:8000/metrics", "http://b:8001/metrics"}}, + // Unlike vLLM, SGLang sampling is never speculative: plain "openai" or + // the default type must NOT start a sampler. + {"dynamic/http://localhost:8000/v1,type=openai", nil}, + {"dynamic/http://localhost:8000/v1", nil}, + {"dynamic/http://localhost:8000/v1,type=openai_vllm", nil}, + // Not a dynamic model at all. + {"gpt-4", nil}, + } + for _, c := range cases { + got := sglangMetricsEndpoints(c.spec) + if len(got) != len(c.want) { + t.Errorf("spec %q: got %v, want %v", c.spec, got, c.want) + continue + } + for i := range got { + if got[i] != c.want[i] { + t.Errorf("spec %q: got %v, want %v", c.spec, got, c.want) + break + } + } + } +} + +// TestSGLangMetricsSamplerDoesNotStartForNonSGLang confirms +// startSGLangMetricsSampler stays inert for the vLLM/default eligibility +// shapes — SGLang sampling is additive, never a second guess at the same +// endpoint. +func TestSGLangMetricsSamplerDoesNotStartForNonSGLang(t *testing.T) { + dir := t.TempDir() + rdw, err := newRequestDataWriter(dir, "no_sglang_model", time.Now()) + if err != nil { + t.Fatalf("newRequestDataWriter: %v", err) + } + defer func() { _ = rdw.close() }() + for _, spec := range []string{ + "dynamic/http://localhost:8000/v1,type=openai", + "dynamic/http://localhost:8000/v1,type=openai_vllm", + "dynamic/http://localhost:8000/v1", + } { + if s := startSGLangMetricsSampler(context.Background(), spec, rdw); s != nil { + s.stop() + t.Errorf("spec %q: expected no sampler, got one", spec) + } + } +} + +// TestSGLangMetricsSamplerCollects drives a real sampler against a fake +// SGLang /metrics endpoint and confirms it writes a sample carrying the +// scraped gauge average. +func TestSGLangMetricsSamplerCollects(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/metrics" { + http.NotFound(w, r) + return + } + w.Header().Set("Content-Type", "text/plain") + _, _ = w.Write([]byte(sglangPromFixture)) + })) + defer srv.Close() + + dir := t.TempDir() + rdw, err := newRequestDataWriter(dir, "sglang_sampler_test_model", time.Now()) + if err != nil { + t.Fatalf("newRequestDataWriter: %v", err) + } + + // Drive sampleOnce synchronously (same pattern as + // TestVLLMMetricsSamplerCollects) rather than starting the background + // goroutine and mutating fields on it afterward — the latter races with + // the goroutine under -race. + s := &sglangMetricsSampler{ + model: "dynamic/" + srv.URL + "/v1,type=openai_sglang,model=m", + urls: []string{srv.URL + "/metrics"}, + rdw: rdw, + client: srv.Client(), + now: func() time.Time { return time.Unix(1721000000, 0) }, + } + if !s.sampleOnce(context.Background()) { + t.Fatal("sampleOnce reported it should stop, want keepPolling=true") + } + if err := rdw.close(); err != nil { + t.Fatalf("close rdw: %v", err) + } + + // readJSONLFile (used by vllm_metrics_test.go's equivalent) deliberately + // skips record types it doesn't recognize — sglang_metrics_sample isn't + // wired into the shared reader/report path yet (see sglang_metrics.go's + // package comment), so read the raw line directly instead. + samples := readSGLangSamples(t, filepath.Join(dir, "sglang_sampler_test_model.jsonl")) + if len(samples) != 1 { + t.Fatalf("got %d sglang samples, want 1: %v", len(samples), samples) + } + got := samples[0] + if got.RecordType != recordTypeSGLangMetricsSample { + t.Errorf("record_type = %q", got.RecordType) + } + if got.CacheHitRate != 0.75 { + t.Errorf("cache_hit_rate = %v, want 0.75", got.CacheHitRate) + } + if got.EndpointsOK != 1 || got.EndpointsTotal != 1 { + t.Errorf("coverage = %d/%d, want 1/1", got.EndpointsOK, got.EndpointsTotal) + } +} + +// readSGLangSamples reads only the sglang_metrics_sample rows from a +// request-data JSONL file, mirroring readJSONLFile's record_type dispatch for +// the one record type it doesn't (yet) recognize. +func readSGLangSamples(t *testing.T, path string) []sglangMetricsSample { + t.Helper() + f, err := os.Open(path) + if err != nil { + t.Fatalf("open %s: %v", path, err) + } + defer f.Close() + var out []sglangMetricsSample + sc := bufio.NewScanner(f) + for sc.Scan() { + var probe struct { + RecordType string `json:"record_type"` + } + if err := json.Unmarshal(sc.Bytes(), &probe); err != nil { + continue + } + if probe.RecordType != recordTypeSGLangMetricsSample { + continue + } + var s sglangMetricsSample + if err := json.Unmarshal(sc.Bytes(), &s); err != nil { + t.Fatalf("unmarshal sglang sample: %v", err) + } + out = append(out, s) + } + if err := sc.Err(); err != nil { + t.Fatalf("scan %s: %v", path, err) + } + return out +} diff --git a/benchmark/throughput_test.go b/benchmark/throughput_test.go index 8b5faa4..9a2e882 100644 --- a/benchmark/throughput_test.go +++ b/benchmark/throughput_test.go @@ -2,6 +2,7 @@ package benchmark import ( "math" + "strings" "testing" "time" ) @@ -89,6 +90,60 @@ func TestDisplayHitRateServerCache(t *testing.T) { }) } +// TestNoCacheDataObserved locks down the distinction noCacheDataObserved +// exists to draw: a genuinely absent cache signal (server not caching, or not +// reporting it) versus a cache that is working but happens to show 0% right +// now (e.g. still warming up, or a heuristic miss offset by a nonzero server +// report). +func TestNoCacheDataObserved(t *testing.T) { + cases := []struct { + name string + cm cacheMetrics + min int + want bool + }{ + {"neither signal ever fired, enough records", cacheMetrics{count: 20, hitRate: 0, serverReported: false}, 10, true}, + {"below minCount — too early to tell", cacheMetrics{count: 5, hitRate: 0, serverReported: false}, 10, false}, + {"server reported even though heuristic is 0", cacheMetrics{count: 20, hitRate: 0, serverReported: true}, 10, false}, + {"heuristic found hits", cacheMetrics{count: 20, hitRate: 0.4, serverReported: false}, 10, false}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + if got := noCacheDataObserved(c.cm, c.min); got != c.want { + t.Errorf("noCacheDataObserved(%+v, %d) = %v, want %v", c.cm, c.min, got, c.want) + } + }) + } +} + +// TestCacheWarningMessage locks down that the warning names the actual +// server-launch flag for vLLM/SGLang, rather than a generic message that +// leaves the operator to guess between "server doesn't cache at all" and +// "server caches but wasn't launched with the flag that reports it" — the +// distinction that prompted this message in the first place (see +// cacheWarningMessage's doc comment for the exact flags and why both a +// client-side opt-in and a server-side flag are required). +func TestCacheWarningMessage(t *testing.T) { + cases := []struct { + name string + spec string + wantContains string + }{ + {"vllm names its flag", "dynamic/http://h:1/v1,type=openai_vllm,model=m", "--enable-prompt-tokens-details"}, + {"sglang names its flag", "dynamic/http://h:1/v1,type=openai_sglang,model=m", "--enable-cache-report"}, + {"plain openai gets the generic message", "dynamic/http://h:1/v1,type=openai,model=m", "may not support"}, + {"non-dynamic model gets the generic message", "gpt-4", "may not support"}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + got := cacheWarningMessage(c.spec) + if !strings.Contains(got, c.wantContains) { + t.Errorf("cacheWarningMessage(%q) = %q, want it to contain %q", c.spec, got, c.wantContains) + } + }) + } +} + // TestWarmTokensIncludeServerCache reproduces the multi-backend anomaly where an // aggressively-caching backend reported near-zero "warm" tokens despite serving // more requests. Providers subtract server-cached tokens out of prompt_tokens, so diff --git a/benchmark/visualize.go b/benchmark/visualize.go index 629ec7b..16d0d74 100644 --- a/benchmark/visualize.go +++ b/benchmark/visualize.go @@ -442,6 +442,21 @@ var vizTemplate = template.Must(template.New("viz").Parse(` #summaryTable tbody tr:hover td, #summaryTable tbody tr:hover .vcol { background: #1E2429; } #summaryTable tbody tr.row-hidden { opacity: 0.32; } #summaryTable .err-hot { color: #FF6B6B; } + /* Hover-help affordance, shared by summary headers and control labels: the + label TEXT itself is the hover target -- a dotted underline in a muted + colour plus a help cursor, the conventional "more info on hover" + affordance -- rather than a separate "?" glyph, which costs zero + horizontal space (the summary table has 8 columns to fit). Opens the + shared #helpTip custom tooltip; see its rules next to #tooltip below and + the wiring near the bottom of this script. */ + .help-label { border-bottom: 1px dotted #8a9096; cursor: help; padding-bottom: 2px; } + /* Right-axis title for the Totals (ingest) layer: a real DOM node + positioned over the canvas (see totalsAxisLabel / drawTotalsAxis in the + script) rather than ctx.fillText, purely so it can carry the same + .help-label hover-tooltip affordance as everything else here. Centered + at (left, top) via the translate(-50%,-50%) trick, then rotated about + that same center -- see drawTotalsAxis for how left/top are computed. */ + .totals-axis-label { position: fixed; font: 12px sans-serif; color: #C9C9C9; white-space: nowrap; z-index: 2; } /* Ratio-to-baseline sits BELOW its value, right-aligned under it, so the value column stays in one straight line under its header — inline, the ratio pushed each value left by its own width and the numbers no longer @@ -476,6 +491,35 @@ var vizTemplate = template.Must(template.New("viz").Parse(` .legend-ctx { color: #C79FF1; font-size: 0.85em; } canvas { background: #171C20; border-radius: 8px; display: block; cursor: crosshair; } #tooltip { position: fixed; background: #1E2429; border: 1px solid #42464A; border-radius: 6px; padding: 8px 10px; font-size: 0.8em; pointer-events: none; display: none; z-index: 100; max-width: 300px; line-height: 1.5; } + /* Help tooltip for the .help-label affordance above: a second, independent + tooltip deliberately matching #tooltip's look (surface, border, radius, + padding, font) so the report reads as one system, but its own element + with its own lifecycle so the two can never fight over position or + visibility. opacity+visibility (not display) so the fade/translate + transition has something to animate. Always fixed + appended at the + body level (see the markup near #tooltip) so the summary panel's own + scroll container (.summary-wrap, overflow: auto) can never clip it -- + an ancestor with overflow set COULD clip an absolutely-positioned + descendant, so fixed positioning sidesteps the question entirely + (matching #tooltip's own position: fixed). A small caret + (::after, rotated square) points at whichever trigger is active; its + side flips with .caret-top/.caret-bottom depending on whether the tip + landed below or above the trigger. */ + .help-tip { + position: fixed; z-index: 150; background: #1E2429; border: 1px solid #42464A; + border-radius: 6px; padding: 8px 10px; font-size: 0.8em; line-height: 1.45; + max-width: 280px; color: #C9C9C9; box-shadow: 0 4px 14px rgba(0,0,0,0.4); + opacity: 0; visibility: hidden; transform: translateY(4px); + transition: opacity 120ms ease-out, transform 120ms ease-out, visibility 120ms; + pointer-events: none; + } + .help-tip.visible { opacity: 1; visibility: visible; transform: translateY(0); } + .help-tip::after { + content: ""; position: absolute; width: 8px; height: 8px; background: #1E2429; + left: var(--caret-x, 16px); margin-left: -4px; transform: rotate(45deg); + } + .help-tip.caret-top::after { top: -5px; border-left: 1px solid #42464A; border-top: 1px solid #42464A; } + .help-tip.caret-bottom::after { bottom: -5px; border-right: 1px solid #42464A; border-bottom: 1px solid #42464A; } .modal-backdrop { position: fixed; inset: 0; background: rgba(0,0,0,0.55); z-index: 200; display: none; } .modal { background: #1E2429; border: 1px solid #42464A; border-radius: 8px; padding: 16px; width: 440px; max-height: 72vh; overflow-y: auto; margin: 10vh auto 0; } .modal h2 { font-size: 1em; font-weight: 500; color: #F2F2EB; margin-bottom: 10px; } @@ -508,21 +552,20 @@ var vizTemplate = template.Must(template.New("viz").Parse(`
Controls
- - - - - - - - + + + + + +
- + +
@@ -537,9 +580,14 @@ var vizTemplate = template.Must(template.New("viz").Parse(`
+