Feat/sglang endpoint support

benchmark/auto.go

@@ -618,6 +618,46 @@ func (m cacheMetrics) DisplayHitRate() float64 {
 	return m.hitRate
 }
 
+// noCacheDataObserved reports whether, over a large-enough window, neither
+// cache signal has ever fired: the TTFT heuristic saw no warm request AND the
+// server never reported cached_tokens. In that case DisplayHitRate's 0% is a
+// real absence of data — the server may not support prompt caching, or (for
+// vLLM/SGLang) may simply not be configured to report it — rather than a
+// rounding artifact of an otherwise-working cache. Requiring minCount records
+// avoids flagging a run that just hasn't warmed up yet.
+func noCacheDataObserved(m cacheMetrics, minCount int) bool {
+	return m.count >= minCount && m.hitRate == 0 && !m.serverReported
+}
+
+// cacheWarningMessage tailors the cacheWarning explanation to what the model
+// spec's type= actually names, when it names one of the two backends known
+// to gate cached_tokens reporting behind a server-launch flag: vLLM's
+// --enable-prompt-tokens-details and SGLang's --enable-cache-report. Both
+// require the client to ask per-request (already done for type=openai_sglang
+// — see llm/chat_clients.go and replay_router_wire.go) AND the server to be
+// launched with the matching flag; wekai controls only the former; a
+// deployment missing the latter looks from here exactly like a server that
+// genuinely never caches — the message says both are possible rather than
+// asserting whichever cause a client can never observe from the API alone.
+func cacheWarningMessage(modelSpec string) string {
+	generic := "Server may not support prompt caching, or is not configured to report it"
+	if !llm.IsDynamicModel(modelSpec) {
+		return generic
+	}
+	dyn, err := llm.ParseDynamicModel(modelSpec)
+	if err != nil {
+		return generic
+	}
+	switch dyn.Type {
+	case "openai_vllm":
+		return generic + " (vLLM must be launched with --enable-prompt-tokens-details)"
+	case "openai_sglang":
+		return generic + " (SGLang must be launched with --enable-cache-report)"
+	default:
+		return generic
+	}
+}
+
 // GlobalLocalCacheRate returns the all-time fraction of warm input tokens among all input tokens.
 // O(1): reads two running counters maintained in Add(), never scans history.
 // Purely local: a request is "cached" when its series already submitted this prefix before.
@@ -1437,7 +1477,7 @@ func printAutoSummary(res autoBenchmarkResult, cfg AutoBenchmarkConfig) {
 	fmt.Printf(" Cache hit rate     : %.1f%%\n", res.cacheHitRate*100)
 	fmt.Printf(" Tok/s in/out       : %s / %s\n", formatKilo(res.inputTokPerSec), formatKilo(res.outputTokPerSec))
 	if res.cacheWarning {
-		fmt.Println(" ⚠  Server may not support prompt caching")
+		fmt.Printf(" ⚠  %s\n", cacheWarningMessage(cfg.Model))
 	}
 	fmt.Println(strings.Repeat("-", 62))
 	totalInput := res.totalInputCold + res.totalInputWarm
@@ -2079,6 +2119,9 @@ func runSingleModelBenchmark(
 		// waits for the goroutine — no sample write can race the file close.
 		defer sampler.stop()
 	}
+	if sampler := startSGLangMetricsSampler(benchCtx, cfg.Model, rdw); sampler != nil {
+		defer sampler.stop()
+	}
 	if cfg.HotSeriesConcurrency > 0 {
 		st.hotGate = newConcurrencyGate(cfg.HotSeriesConcurrency*hotGateFanoutMultiplier, !cfg.FIFOGateOrder)
 	}
@@ -2836,6 +2879,14 @@ func runSingleModelBenchmark(
 		cm2 := st.stream.CacheMetrics(cfg.CacheWindowSize, cfg.MinStabilization)
 		hitRate := cm2.hitRate
 
+		// Latch cacheWarning once neither signal has ever shown a hit over a
+		// large-enough window — see noCacheDataObserved.
+		if noCacheDataObserved(cm2, cfg.MinStabilization) {
+			st.mu.Lock()
+			st.cacheWarning = true
+			st.mu.Unlock()
+		}
+
 		if cfg.VerboseCache && cm2.count > 0 && math.Abs(hitRate-lastVerboseHitRate) >= 0.03 {
 			lastVerboseHitRate = hitRate
 			frozenBaseline := st.earlyColdStartTTFT()

benchmark/replay_router_charsize_test.go

@@ -129,7 +129,7 @@ func TestBuildOpenAIChatCompletionsBodyCharsPerToken(t *testing.T) {
 		return msg["content"].(string)
 	}
 
-	byteBody, _, err := buildOpenAIChatCompletionsBody(req, docs, "model", "", 0, false, 0, nil)
+	byteBody, _, err := buildOpenAIChatCompletionsBody(req, docs, "model", "", 0, false, 0, nil, false)
 	if err != nil {
 		t.Fatalf("build (charsPerToken=0): %v", err)
 	}
@@ -140,7 +140,7 @@ func TestBuildOpenAIChatCompletionsBodyCharsPerToken(t *testing.T) {
 
 	const charsPerToken = 3.4
 	wantLen := int(math.Round(150 * charsPerToken))
-	tokBody, _, err := buildOpenAIChatCompletionsBody(req, docs, "model", "", 0, false, charsPerToken, nil)
+	tokBody, _, err := buildOpenAIChatCompletionsBody(req, docs, "model", "", 0, false, charsPerToken, nil, false)
 	if err != nil {
 		t.Fatalf("build (charsPerToken=3.4): %v", err)
 	}

benchmark/replay_router_post.go

@@ -48,7 +48,7 @@ type replayPoster struct {
 	epMu       sync.Mutex
 	epResolved string // latched endpoint; "" until the first success
 	epFellBack bool
-	apiType    string // "anthropic", "openai", or "openai_vllm"
+	apiType    string // "anthropic", "openai", "openai_vllm", or "openai_sglang"
 	client     *http.Client
 	// retryBudget is the total time a request may spend waiting out 429s
 	// before the shed stands as an error. Defaults to retry429Budget; a field
@@ -207,18 +207,20 @@ func newReplayPoster(modelSpec string, keys llm.APIKeys, endpointOverride string
 		return nil, fmt.Errorf("parse model spec: %w", err)
 	}
 
-	// Accepted target types: anthropic (original), openai, openai_vllm.
-	// openai_vllm is treated identically to openai in the replay path — both
-	// use /v1/chat/completions. The distinction matters for the Chat path
-	// (max_tokens vs max_completion_tokens) but replay requests carry their
-	// own max_tokens so it's irrelevant here.
+	// Accepted target types: anthropic (original), openai, openai_vllm,
+	// openai_sglang. openai_vllm and openai_sglang are treated identically to
+	// openai in the replay path — all three use /v1/chat/completions. The
+	// distinction matters for the Chat path (max_tokens vs
+	// max_completion_tokens, and SGLang's return_cached_tokens_details) but
+	// replay requests carry their own max_tokens and opt into cached-token
+	// reporting explicitly below, so the type only selects that behavior.
 	switch dyn.Type {
 	case "anthropic":
 		// OK — existing behaviour.
-	case "openai", "openai_vllm":
+	case "openai", "openai_vllm", "openai_sglang":
 		// OK — new path.
 	default:
-		return nil, fmt.Errorf("router-replay supports type=anthropic, type=openai, or type=openai_vllm (got %q)", dyn.Type)
+		return nil, fmt.Errorf("router-replay supports type=anthropic, type=openai, type=openai_vllm, or type=openai_sglang (got %q)", dyn.Type)
 	}
 
 	base := ""
@@ -235,7 +237,7 @@ func newReplayPoster(modelSpec string, keys llm.APIKeys, endpointOverride string
 	// local endpoints); OpenAI targets use Bearer auth with the OpenAI key
 	// (or dummy-key for local endpoints).
 	apiKey := keys.Anthropic
-	if dyn.Type == "openai" || dyn.Type == "openai_vllm" {
+	if dyn.Type == "openai" || dyn.Type == "openai_vllm" || dyn.Type == "openai_sglang" {
 		apiKey = keys.OpenAI
 	}
 	if apiKey == "" {
@@ -246,7 +248,7 @@ func newReplayPoster(modelSpec string, keys llm.APIKeys, endpointOverride string
 	// The primary attempt appends it to the operator's base verbatim; the
 	// fallback inserts /v1 (see the struct comment for the contract).
 	leaf := "/messages"
-	if dyn.Type == "openai" || dyn.Type == "openai_vllm" {
+	if dyn.Type == "openai" || dyn.Type == "openai_vllm" || dyn.Type == "openai_sglang" {
 		leaf = "/chat/completions"
 	}
 	epPrimary := base + leaf
@@ -581,8 +583,8 @@ func (p *replayPoster) do(
 	var canonical string
 	var err error
 	switch p.apiType {
-	case "openai", "openai_vllm":
-		bodyBytes, canonical, err = buildOpenAIChatCompletionsBody(req, docs, p.model, stampFor(p, req), p.outputRatio, p.forceVolume, p.replayCharsPerToken, inj)
+	case "openai", "openai_vllm", "openai_sglang":
+		bodyBytes, canonical, err = buildOpenAIChatCompletionsBody(req, docs, p.model, stampFor(p, req), p.outputRatio, p.forceVolume, p.replayCharsPerToken, inj, p.apiType == "openai_sglang")
 	default:
 		bodyBytes, canonical, err = buildAnthropicMessagesBody(req, docs, p.model, stampFor(p, req), p.outputRatio, p.forceVolume, p.replayCharsPerToken, inj)
 	}
@@ -742,7 +744,7 @@ func (p *replayPoster) do(
 	// consumers compute must describe the attempt the server actually ran, so
 	// that backoff cannot make a healthy fleet look slow. The client-side wait
 	// is added back into TotalResponseTime below, where it belongs.
-	if p.apiType == "openai" || p.apiType == "openai_vllm" {
+	if p.apiType == "openai" || p.apiType == "openai_vllm" || p.apiType == "openai_sglang" {
 		if req.Stream {
 			consumeOpenAISSE(respReader, attemptStart, &m)
 		} else {
@@ -1267,8 +1269,8 @@ func (p *replayPoster) dryDo(
 	inj := p.buildInjection(req, su)
 	var canonical string
 	switch p.apiType {
-	case "openai", "openai_vllm":
-		_, canonical, _ = buildOpenAIChatCompletionsBody(req, docs, p.model, stampFor(p, req), p.outputRatio, p.forceVolume, p.replayCharsPerToken, inj)
+	case "openai", "openai_vllm", "openai_sglang":
+		_, canonical, _ = buildOpenAIChatCompletionsBody(req, docs, p.model, stampFor(p, req), p.outputRatio, p.forceVolume, p.replayCharsPerToken, inj, p.apiType == "openai_sglang")
 	default:
 		_, canonical, _ = buildAnthropicMessagesBody(req, docs, p.model, stampFor(p, req), p.outputRatio, p.forceVolume, p.replayCharsPerToken, inj)
 	}

benchmark/replay_router_post_test.go

@@ -214,6 +214,13 @@ func TestNewReplayPoster_OpenAI(t *testing.T) {
 			wantPrimary:  "http://127.0.0.1:8000/v1/chat/completions",
 			wantFallback: "http://127.0.0.1:8000/v1/v1/chat/completions",
 		},
+		{
+			name:         "openai_sglang type, /v1 base",
+			modelSpec:    "dynamic/http://127.0.0.1:8000/v1,type=openai_sglang,model=my-model",
+			wantType:     "openai_sglang",
+			wantPrimary:  "http://127.0.0.1:8000/v1/chat/completions",
+			wantFallback: "http://127.0.0.1:8000/v1/v1/chat/completions",
+		},
 		{
 			name:         "anthropic type, /v1 base",
 			modelSpec:    "dynamic/http://127.0.0.1:8000/v1,type=anthropic,model=claude",
@@ -467,6 +474,70 @@ func TestOpenAIReplayEndToEnd(t *testing.T) {
 	}
 }
 
+// TestOpenAISGLangReplaySetsReturnCachedTokensDetails confirms
+// type=openai_sglang is accepted by newReplayPoster's type allow-list and
+// that the wire body it builds carries return_cached_tokens_details — SGLang's
+// per-request opt-in for usage.prompt_tokens_details.cached_tokens (vLLM's
+// equivalent is the server-launch flag --enable-prompt-tokens-details, so
+// type=openai_vllm needs no such field — see TestOpenAIReplayEndToEnd, which
+// exercises type=openai without it). Without this, cache hit rate reads as 0
+// against every SGLang response, same failure mode as an unflagged vLLM
+// server.
+func TestOpenAISGLangReplaySetsReturnCachedTokensDetails(t *testing.T) {
+	var receivedBody map[string]interface{}
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path != "/v1/chat/completions" {
+			w.WriteHeader(404)
+			return
+		}
+		if err := json.NewDecoder(r.Body).Decode(&receivedBody); err != nil {
+			t.Errorf("failed to decode request body: %v", err)
+			w.WriteHeader(400)
+			return
+		}
+		w.Header().Set("Content-Type", "text/event-stream")
+		w.WriteHeader(200)
+		fmt.Fprintf(w, `data: {"id":"c","object":"chat.completion.chunk","created":1,"model":"test-model","choices":[{"index":0,"delta":{"content":"ok"},"finish_reason":null}]}`+"\n\n")
+		if flusher, ok := w.(http.Flusher); ok {
+			flusher.Flush()
+		}
+		fmt.Fprintf(w, `data: {"id":"c","object":"chat.completion.chunk","created":1,"model":"test-model","choices":[{"index":0,"delta":{},"finish_reason":"stop"}],"usage":{"prompt_tokens":10,"completion_tokens":1,"total_tokens":11,"prompt_tokens_details":{"cached_tokens":5}}}`+"\n\n")
+		fmt.Fprintf(w, "data: [DONE]\n\n")
+	}))
+	defer ts.Close()
+
+	modelSpec := fmt.Sprintf("dynamic/%s,type=openai_sglang,model=test-model", ts.URL)
+	keys := llm.APIKeys{OpenAI: "sk-test-123"}
+	p, err := newReplayPoster(modelSpec, keys, "", "", false, 0, 0, 0, nil, nil)
+	if err != nil {
+		t.Fatalf("newReplayPoster: %v", err)
+	}
+	if p.apiType != "openai_sglang" {
+		t.Fatalf("apiType = %q, want openai_sglang", p.apiType)
+	}
+
+	req := RouterReplayRequest{
+		RequestID:    1,
+		Stream:       true,
+		OutputTokens: 100,
+		SystemBlocks: []RouterReplaySystemBlock{{Hash: "syshash", Bytes: 250}},
+		Messages: []RouterReplayMessage{
+			{Role: "user", Hash: "msghash1", Bytes: 60, BlockTypes: []string{"text"}},
+		},
+	}
+	st := &autoState{stream: newCompletionStream(200)}
+	metrics := p.do(context.Background(), req, strings.Repeat("x", 300), 1, "session-1", "instance-1", 1, st, nil)
+	if metrics.Error != nil {
+		t.Fatalf("unexpected error: %v", metrics.Error)
+	}
+	if v, ok := receivedBody["return_cached_tokens_details"]; !ok || v != true {
+		t.Errorf("return_cached_tokens_details = %v (present=%v), want true", v, ok)
+	}
+	if metrics.UsageData.CachedTokens.Count != 5 {
+		t.Errorf("cached tokens = %d, want 5", metrics.UsageData.CachedTokens.Count)
+	}
+}
+
 // TestOpenAIReplayToolTranslation verifies that assistant messages with
 // tool_use blocks are translated into proper OpenAI tool_calls (not flattened).
 func TestOpenAIReplayToolTranslation(t *testing.T) {
@@ -485,7 +556,7 @@ func TestOpenAIReplayToolTranslation(t *testing.T) {
 		},
 	}
 
-	body, _, err := buildOpenAIChatCompletionsBody(req, docs, "test-model", "", 0, false, 0, nil)
+	body, _, err := buildOpenAIChatCompletionsBody(req, docs, "test-model", "", 0, false, 0, nil, false)
 	if err != nil {
 		t.Fatalf("buildOpenAIChatCompletionsBody: %v", err)
 	}
@@ -548,7 +619,7 @@ func TestOpenAIBodyBuilderExtra(t *testing.T) {
 				{Role: "user", Hash: "h1", Bytes: 50, BlockTypes: []string{"text"}},
 			},
 		}
-		body, _, err := buildOpenAIChatCompletionsBody(req, docs, "test-model", "run-42", 0, false, 0, nil)
+		body, _, err := buildOpenAIChatCompletionsBody(req, docs, "test-model", "run-42", 0, false, 0, nil, false)
 		if err != nil {
 			t.Fatalf("build: %v", err)
 		}
@@ -577,7 +648,7 @@ func TestOpenAIBodyBuilderExtra(t *testing.T) {
 				{Role: "user", Hash: "h1", Bytes: 50, BlockTypes: []string{"text"}},
 			},
 		}
-		body, _, err := buildOpenAIChatCompletionsBody(req, docs, "test-model", "", 0, false, 0, nil)
+		body, _, err := buildOpenAIChatCompletionsBody(req, docs, "test-model", "", 0, false, 0, nil, false)
 		if err != nil {
 			t.Fatalf("build: %v", err)
 		}
@@ -605,7 +676,7 @@ func TestOpenAIBodyBuilderExtra(t *testing.T) {
 				{Role: "user", Hash: "h1", Bytes: 50, BlockTypes: []string{"text"}},
 			},
 		}
-		body, _, err := buildOpenAIChatCompletionsBody(req, docs, "test-model", "", 0, false, 0, nil)
+		body, _, err := buildOpenAIChatCompletionsBody(req, docs, "test-model", "", 0, false, 0, nil, false)
 		if err != nil {
 			t.Fatalf("build: %v", err)
 		}
@@ -631,7 +702,7 @@ func TestOpenAIBodyBuilderExtra(t *testing.T) {
 			Stream:       true,
 			OutputTokens: 100,
 		}
-		body, _, err := buildOpenAIChatCompletionsBody(req, docs, "test-model", "", 0, false, 0, nil)
+		body, _, err := buildOpenAIChatCompletionsBody(req, docs, "test-model", "", 0, false, 0, nil, false)
 		if err != nil {
 			t.Fatalf("build: %v", err)
 		}

benchmark/replay_router_uuid_test.go

@@ -436,7 +436,7 @@ func TestWireInjectionDeterminism(t *testing.T) {
 			var body []byte
 			var err error
 			if kind == "openai" {
-				body, _, err = buildOpenAIChatCompletionsBody(r, docs, "model", "", 0, false, 0, inj)
+				body, _, err = buildOpenAIChatCompletionsBody(r, docs, "model", "", 0, false, 0, inj, false)
 			} else {
 				body, _, err = buildAnthropicMessagesBody(r, docs, "model", "", 0, false, 0, inj)
 			}

benchmark/replay_router_wire.go

@@ -593,7 +593,17 @@ func buildOpenAITools(spec *RouterReplayToolsSpec, docs string, charsPerToken fl
 //
 // inj carries the UUID cache-coherency injection (--verify,
 // router path — see replay_router_uuid.go); nil means "no injection".
-func buildOpenAIChatCompletionsBody(req RouterReplayRequest, docs string, modelName string, runID string, outputRatio float64, forceVolume bool, charsPerToken float64, inj *uuidInjection) ([]byte, string, error) {
+//
+// sglang requests cached-token reporting per-request via
+// return_cached_tokens_details. That is necessary but not sufficient: the
+// server must ALSO be launched with --enable-cache-report
+// (sglang/srt/entrypoints/openai/serving_chat.py checks both the request
+// field and tokenizer_manager.server_args.enable_cache_report) — the same
+// two-sided requirement as vLLM's --enable-prompt-tokens-details flag.
+// Without either half, usage.prompt_tokens_details is simply absent from
+// every SGLang response, and cache hit rate always reads as 0 — see
+// cacheWarningMessage in auto.go, which surfaces this in the run summary.
+func buildOpenAIChatCompletionsBody(req RouterReplayRequest, docs string, modelName string, runID string, outputRatio float64, forceVolume bool, charsPerToken float64, inj *uuidInjection, sglang bool) ([]byte, string, error) {
 	var stampByHash map[string]turnStamp
 	if inj != nil {
 		stampByHash = inj.StampByHash
@@ -603,6 +613,9 @@ func buildOpenAIChatCompletionsBody(req RouterReplayRequest, docs string, modelN
 		"max_tokens": pickMaxTokens(req, outputRatio),
 		"stream":     req.Stream,
 	}
+	if sglang {
+		body["return_cached_tokens_details"] = true
+	}
 	if req.Temperature != nil {
 		body["temperature"] = *req.Temperature
 	}

benchmark/replay_router_wire_test.go

@@ -172,11 +172,14 @@ func TestEffectiveSystemBlocksSkipsHeader(t *testing.T) {
 		},
 	}
 	hdrText := synthText("uniq-header-per-req", 106, "")
+	openaiBuilder := func(r RouterReplayRequest, docs, model, runID string, outputRatio float64, forceVolume bool, charsPerToken float64, inj *uuidInjection) ([]byte, string, error) {
+		return buildOpenAIChatCompletionsBody(r, docs, model, runID, outputRatio, forceVolume, charsPerToken, inj, false)
+	}
 	for _, builder := range []struct {
 		name string
 		fn   func(RouterReplayRequest, string, string, string, float64, bool, float64, *uuidInjection) ([]byte, string, error)
 	}{
-		{"openai", buildOpenAIChatCompletionsBody},
+		{"openai", openaiBuilder},
 		{"anthropic", buildAnthropicMessagesBody},
 	} {
 		body, canonical, err := builder.fn(req, "", "m", "", 0, false, 0, nil)
@@ -333,7 +336,7 @@ func TestBuildOpenAIChatCompletionsBodyForceOutput(t *testing.T) {
 
 	// force-output off: no ignore_eos, but the instruction still rides — the
 	// modes differ ONLY by engine enforcement.
-	body, _, err := buildOpenAIChatCompletionsBody(req, docs, "model", "", 0, false, 0, nil)
+	body, _, err := buildOpenAIChatCompletionsBody(req, docs, "model", "", 0, false, 0, nil, false)
 	if err != nil {
 		t.Fatalf("build (force-output off): %v", err)
 	}
@@ -349,7 +352,7 @@ func TestBuildOpenAIChatCompletionsBodyForceOutput(t *testing.T) {
 	}
 
 	// force-output on (default): ignore_eos=true AND the instruction is present.
-	body, _, err = buildOpenAIChatCompletionsBody(req, docs, "model", "", 0, true, 0, nil)
+	body, _, err = buildOpenAIChatCompletionsBody(req, docs, "model", "", 0, true, 0, nil, false)
 	if err != nil {
 		t.Fatalf("build (force-output on): %v", err)
 	}
@@ -384,7 +387,7 @@ func TestBuildAnthropicMessagesBodyOutputRatioMaxTokens(t *testing.T) {
 		t.Errorf("anthropic max_tokens = %v, want %v", got, want)
 	}
 
-	openaiBody, _, err := buildOpenAIChatCompletionsBody(req, docs, "model", "", 0.25, false, 0, nil)
+	openaiBody, _, err := buildOpenAIChatCompletionsBody(req, docs, "model", "", 0.25, false, 0, nil, false)
 	if err != nil {
 		t.Fatalf("openai build: %v", err)
 	}