From 7e8bd3a7082abad43d673a4d426e834d035dc8ad Mon Sep 17 00:00:00 2001 From: yueming-yuan Date: Wed, 15 Jul 2026 15:39:01 -0700 Subject: [PATCH 1/2] Support router manual policy: mint session ids and attach routing key under manual too --- miles/rollout/sglang_rollout.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/miles/rollout/sglang_rollout.py b/miles/rollout/sglang_rollout.py index 64662ed522..cd8b8de58f 100644 --- a/miles/rollout/sglang_rollout.py +++ b/miles/rollout/sglang_rollout.py @@ -194,7 +194,7 @@ async def generate(args: Namespace, sample: Sample, sampling_params: dict[str, A # Use session_id for consistent hashing routing if router uses consistent_hashing policy headers = None - if args.sglang_router_policy == "consistent_hashing" and sample.session_id: + if args.sglang_router_policy in ("consistent_hashing", "manual") and sample.session_id: headers = {"X-SMG-Routing-Key": sample.session_id} output = await post(url, payload, headers=headers) @@ -314,7 +314,7 @@ async def generate_and_rm_group( return group # Generate a unique session_id for each sample in the group (consistent hashing only) - if args.sglang_router_policy == "consistent_hashing": + if args.sglang_router_policy in ("consistent_hashing", "manual"): for sample in group: if sample.session_id is None: sample.session_id = str(uuid.uuid4()) From ff40ec5a0ff7d3a7cca8a3719efcf018e1ac03fe Mon Sep 17 00:00:00 2001 From: yueming-yuan Date: Wed, 15 Jul 2026 16:17:26 -0700 Subject: [PATCH 2/2] Default router policy to manual+min_load when session server is enabled --- miles/backends/sglang_utils/arguments.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/miles/backends/sglang_utils/arguments.py b/miles/backends/sglang_utils/arguments.py index 71e3f48fb4..153286b1ee 100644 --- a/miles/backends/sglang_utils/arguments.py +++ b/miles/backends/sglang_utils/arguments.py @@ -147,6 +147,13 @@ def validate_args(args): if args.sglang_dp_size > 1: assert args.sglang_enable_dp_attention + from miles.utils.environ import enable_experimental_rollout_refactor + + if args.sglang_router_policy is None and args.use_session_server and not enable_experimental_rollout_refactor(): + args.sglang_router_policy = "manual" + if args.router_assignment_mode == "random": + args.router_assignment_mode = "min_load" + if args.sglang_router_policy: from miles.utils.environ import enable_experimental_rollout_refactor