diff --git a/miles/backends/sglang_utils/arguments.py b/miles/backends/sglang_utils/arguments.py index 71e3f48fb4..153286b1ee 100644 --- a/miles/backends/sglang_utils/arguments.py +++ b/miles/backends/sglang_utils/arguments.py @@ -147,6 +147,13 @@ def validate_args(args): if args.sglang_dp_size > 1: assert args.sglang_enable_dp_attention + from miles.utils.environ import enable_experimental_rollout_refactor + + if args.sglang_router_policy is None and args.use_session_server and not enable_experimental_rollout_refactor(): + args.sglang_router_policy = "manual" + if args.router_assignment_mode == "random": + args.router_assignment_mode = "min_load" + if args.sglang_router_policy: from miles.utils.environ import enable_experimental_rollout_refactor diff --git a/miles/rollout/sglang_rollout.py b/miles/rollout/sglang_rollout.py index 64662ed522..cd8b8de58f 100644 --- a/miles/rollout/sglang_rollout.py +++ b/miles/rollout/sglang_rollout.py @@ -194,7 +194,7 @@ async def generate(args: Namespace, sample: Sample, sampling_params: dict[str, A # Use session_id for consistent hashing routing if router uses consistent_hashing policy headers = None - if args.sglang_router_policy == "consistent_hashing" and sample.session_id: + if args.sglang_router_policy in ("consistent_hashing", "manual") and sample.session_id: headers = {"X-SMG-Routing-Key": sample.session_id} output = await post(url, payload, headers=headers) @@ -314,7 +314,7 @@ async def generate_and_rm_group( return group # Generate a unique session_id for each sample in the group (consistent hashing only) - if args.sglang_router_policy == "consistent_hashing": + if args.sglang_router_policy in ("consistent_hashing", "manual"): for sample in group: if sample.session_id is None: sample.session_id = str(uuid.uuid4())