feat(DATAPF-2790): Upgrade vLLM to 0.20.0 with RunAI streamer s…

python/huggingface_server.Dockerfile

@@ -52,9 +52,8 @@ WORKDIR ${WORKSPACE_DIR}
 FROM base AS build
 
 ARG WORKSPACE_DIR
-ARG VLLM_VERSION=0.15.1
-ARG LMCACHE_VERSION=0.3.9
-ARG FLASHINFER_VERSION=0.6.1
+ARG VLLM_VERSION=0.20.0
+ARG LMCACHE_VERSION=0.4.4
 
 WORKDIR ${WORKSPACE_DIR}
 
@@ -87,22 +86,10 @@ RUN --mount=type=cache,target=/root/.cache/uv cd huggingfaceserver && uv sync --
 # Install vllm
 # https://docs.vllm.ai/en/latest/models/extensions/runai_model_streamer.html, https://docs.vllm.ai/en/latest/models/extensions/tensorizer.html
 # https://docs.vllm.ai/en/latest/models/extensions/fastsafetensor.html
-RUN --mount=type=cache,target=/root/.cache/pip pip install vllm[runai,tensorizer,fastsafetensors]==${VLLM_VERSION}
+RUN --mount=type=cache,target=/root/.cache/uv uv pip install vllm[runai,tensorizer,fastsafetensors]==${VLLM_VERSION}
 
 # Install lmcache
-RUN --mount=type=cache,target=/root/.cache/pip pip install lmcache==${LMCACHE_VERSION}
-
-# Use Bash with `-o pipefail` so we can leverage Bash-specific features (like `[[ … ]]` for glob tests)
-# and ensure that failures in any part of a piped command cause the build to fail immediately.
-SHELL ["/bin/bash", "-o", "pipefail", "-c"]
-
-# Install flashinfer
-# https://docs.flashinfer.ai/installation.html
-RUN --mount=type=cache,target=/root/.cache/pip \
-    pip install flashinfer-cubin==${FLASHINFER_VERSION} && \
-    pip install flashinfer-jit-cache==${FLASHINFER_VERSION} \
-        --extra-index-url https://flashinfer.ai/whl/cu$(echo ${CUDA_VERSION} | cut -d. -f1,2 | tr -d '.') && \
-    flashinfer show-config
+RUN --mount=type=cache,target=/root/.cache/uv uv pip install lmcache==${LMCACHE_VERSION}
 
 # Generate third-party licenses
 COPY pyproject.toml pyproject.toml
@@ -162,6 +149,10 @@ ENV VLLM_NCCL_SO_PATH="/lib/x86_64-linux-gnu/libnccl.so.2"
 # Set the multiprocess method to spawn to avoid issues with cuda initialization for `mp` executor backend.
 ENV VLLM_WORKER_MULTIPROC_METHOD="spawn"
 
+ENV LD_LIBRARY_PATH=/usr/local/nvidia/lib64:/usr/local/cuda/lib64:/usr/local/cuda/targets/x86_64-linux/lib:$(LD_LIBRARY_PATH)
+# Default AWS region when using runai to download from S3 bucket. TODO: pass this at runtime.
+ENV AWS_REGION=eu-west-1
+
 USER 1000
 ENV PYTHONPATH=${WORKSPACE_DIR}/huggingfaceserver
 ENTRYPOINT ["python3", "-m", "huggingfaceserver"]

python/huggingface_server_cpu.Dockerfile

@@ -83,7 +83,7 @@ RUN cd huggingfaceserver && \
     rm -rf ~/.cache/uv
 
 # install vllm
-ARG VLLM_VERSION=0.15.1
+ARG VLLM_VERSION=0.20.0
 ARG VLLM_CPU_DISABLE_AVX512=true
 ENV VLLM_CPU_DISABLE_AVX512=${VLLM_CPU_DISABLE_AVX512}
 ARG VLLM_CPU_AVX512BF16=1

python/huggingfaceserver/huggingfaceserver/__main__.py

@@ -50,22 +50,37 @@ def list_of_strings(arg):
     return arg.split(",")
 
 
+_REMOTE_URI_PREFIXES = (
+    "s3://", "gs://", "az://", "azure://",
+    "http://", "https://", "ftp://", "hdfs://",
+)
+
+
 def get_model_id_or_path(args: argparse.Namespace) -> Union[str, Path]:
     # If --model_id is specified then pass model_id to HF API, otherwise load the model from /mnt/models
     if args.model_id:
         return cast(str, args.model_id)
+    # For remote URIs (s3://, gs://, etc.), pass directly to vLLM which handles
+    # streaming via load format (e.g., runai_streamer). Skip Storage.download
+    # to avoid attempting a full local download.
+    if isinstance(args.model_dir, str) and args.model_dir.startswith(_REMOTE_URI_PREFIXES):
+        return args.model_dir
     return Path(Storage.download(args.model_dir))
 
 
 def is_vllm_backend_enabled(
     args: argparse.Namespace, model_id_or_path: Union[str, Path]
 ) -> bool:
+    # When backend is explicitly set to vllm, skip architecture validation.
+    # This allows using custom load formats (like runai_streamer) with remote URIs.
+    force_vllm = args.backend == Backend.vllm
     return (
         (args.backend == Backend.vllm or args.backend == Backend.auto)
         and vllm_available()
         and infer_vllm_supported_from_model_architecture(
             model_id_or_path,
             trust_remote_code=args.trust_remote_code,
+            force_vllm=force_vllm,
         )
     )
 
@@ -140,6 +155,12 @@ def is_vllm_backend_enabled(
 parser.add_argument(
     "--return_token_type_ids", action="store_true", help="Return token type ids"
 )
+parser.add_argument(
+    "--return_offsets_mapping",
+    action="store_true",
+    default=False,
+    help="Return start/end character offsets for each token (token_classification only).",
+)
 
 # Create a mutually exclusive group for output format options
 # This group allows the user to choose between returning probabilities or disabling postprocessing.
@@ -312,6 +333,7 @@ def load_model():
                 tensor_input_names=kwargs.get("tensor_input_names", None),
                 return_token_type_ids=kwargs.get("return_token_type_ids", None),
                 request_logger=request_logger,
+                return_offsets_mapping=kwargs.get("return_offsets_mapping", False),
                 return_probabilities=kwargs.get("return_probabilities", False),
                 return_raw_logits=kwargs.get("return_raw_logits", False),
             )

python/huggingfaceserver/huggingfaceserver/vllm/utils.py

@@ -41,18 +41,47 @@ def vllm_available() -> bool:
 def infer_vllm_supported_from_model_architecture(
     model_config_path: Union[Path, str],
     trust_remote_code: bool = False,
+    force_vllm: bool = False,
 ) -> bool:
     if not _vllm:
         return False
 
-    model_config = AutoConfig.from_pretrained(
-        model_config_path, trust_remote_code=trust_remote_code
-    )
-    for architecture in model_config.architectures:
-        if architecture not in ModelRegistry.get_supported_archs():
-            logger.info("not a supported model by vLLM")
-            return False
-    return True
+    model_path_str = str(model_config_path)
+
+    # Skip architecture validation for remote URIs (S3, GCS, Azure, HTTP).
+    # These are used with custom load formats like runai_streamer that bypass
+    # standard HuggingFace model loading.
+    is_remote_uri = model_path_str.startswith((
+        "s3://", "gs://", "az://", "azure://",
+        "http://", "https://", "ftp://", "hdfs://",
+    ))
+
+    if is_remote_uri:
+        logger.info(
+            f"Skipping architecture validation for remote URI: {model_path_str}. "
+            "Assuming vLLM can handle this with appropriate load-format settings."
+        )
+        return True
+
+    if force_vllm:
+        logger.info("vLLM backend explicitly requested, skipping architecture validation.")
+        return True
+
+    try:
+        model_config = AutoConfig.from_pretrained(
+            model_config_path, trust_remote_code=trust_remote_code
+        )
+        for architecture in model_config.architectures:
+            if architecture not in ModelRegistry.get_supported_archs():
+                logger.info("not a supported model by vLLM")
+                return False
+        return True
+    except Exception as e:
+        logger.warning(
+            f"Failed to load model config from {model_path_str}: {e}. "
+            "If using a remote URI with custom load format, consider setting --backend=vllm explicitly."
+        )
+        return False
 
 
 def maybe_add_vllm_cli_parser(parser: ArgumentParser) -> ArgumentParser:
@@ -70,26 +99,16 @@ def build_vllm_engine_args(args) -> "AsyncEngineArgs":
 @asynccontextmanager
 async def build_async_engine_client_from_engine_args(
     engine_args: AsyncEngineArgs,
-    disable_frontend_multiprocessing: bool = False,
 ) -> AsyncIterator[EngineClient]:
     """
-    Create EngineClient, either:
-        - V1 AsyncLLM (default)
-        - V0 AsyncLLMEngine (legacy)
+    Create V1 AsyncLLM EngineClient.
 
     Returns the Client or None if the creation failed.
     """
 
-    # Create the EngineConfig (determines if we can use V1).
     usage_context = UsageContext.OPENAI_API_SERVER
     vllm_config = engine_args.create_engine_config(usage_context=usage_context)
 
-    if disable_frontend_multiprocessing:
-        logger.warning(
-            "V1 is enabled, but got --disable-frontend-multiprocessing. "
-            "To disable frontend multiprocessing, set VLLM_USE_V1=0."
-        )
-
     from vllm.v1.engine.async_llm import AsyncLLM
 
     async_llm: Optional[AsyncLLM] = None

python/huggingfaceserver/huggingfaceserver/vllm/vllm_model.py

@@ -23,8 +23,8 @@
 from vllm.engine.protocol import EngineClient
 from vllm.entrypoints.openai.completion.serving import OpenAIServingCompletion
 from vllm.entrypoints.openai.chat_completion.serving import OpenAIServingChat
-from vllm.entrypoints.pooling.embed.serving import OpenAIServingEmbedding
-from vllm.entrypoints.pooling.score.serving import ServingScores
+from vllm.entrypoints.pooling.embed.serving import ServingEmbedding
+from vllm.entrypoints.pooling.scoring.serving import ServingScores
 from vllm.tool_parsers import ToolParserManager
 from vllm.entrypoints.openai.models.protocol import BaseModelPath
 from vllm.entrypoints.openai.models.serving import OpenAIServingModels
@@ -60,7 +60,7 @@ class VLLMModel(OpenAIEncoderModel, OpenAIGenerativeModel):  # pylint:disable=c-
     openai_serving_models: Optional[OpenAIServingModels] = None
     openai_serving_completion: Optional[OpenAIServingCompletion] = None
     openai_serving_chat: Optional[OpenAIServingChat] = None
-    openai_serving_embedding: Optional[OpenAIServingEmbedding] = None
+    openai_serving_embedding: Optional[ServingEmbedding] = None
     serving_reranking: Optional[ServingScores] = None
 
     def __init__(
@@ -107,7 +107,7 @@ async def start_engine(self):
             self.vllm_engine_args.tensor_parallel_size = torch.cuda.device_count()
 
         async with build_async_engine_client_from_engine_args(
-            self.vllm_engine_args, self.args.disable_frontend_multiprocessing
+            self.vllm_engine_args,
         ) as engine_client:
             self.engine_client = engine_client
             vllm_config = self.engine_client.vllm_config
@@ -137,11 +137,29 @@ async def start_engine(self):
             )
             await self.openai_serving_models.init_static_loras()
 
+            from vllm.entrypoints.serve.render.serving import OpenAIServingRender
+
+            openai_serving_render = OpenAIServingRender(
+                model_config=vllm_config.model_config,
+                renderer=self.engine_client.renderer,
+                model_registry=self.openai_serving_models.registry,
+                request_logger=self.request_logger,
+                chat_template=resolved_chat_template,
+                chat_template_content_format=self.args.chat_template_content_format,
+                trust_request_chat_template=self.args.trust_request_chat_template,
+                enable_auto_tools=self.args.enable_auto_tool_choice,
+                exclude_tools_when_tool_choice_none=self.args.exclude_tools_when_tool_choice_none,
+                tool_parser=self.args.tool_call_parser,
+                reasoning_parser=self.args.structured_outputs_config.reasoning_parser,
+                log_error_stack=self.args.log_error_stack,
+            )
+
             self.openai_serving_chat = (
                 OpenAIServingChat(
                     self.engine_client,
                     self.openai_serving_models,
                     self.args.response_role,
+                    openai_serving_render=openai_serving_render,
                     request_logger=self.request_logger,
                     chat_template=resolved_chat_template,
                     chat_template_content_format=self.args.chat_template_content_format,
@@ -154,7 +172,6 @@ async def start_engine(self):
                     enable_prompt_tokens_details=self.args.enable_prompt_tokens_details,
                     enable_force_include_usage=self.args.enable_force_include_usage,
                     enable_log_outputs=self.args.enable_log_outputs,
-                    log_error_stack=self.args.log_error_stack,
                 )
                 if "generate" in supported_tasks
                 else None
@@ -164,18 +181,18 @@ async def start_engine(self):
                 OpenAIServingCompletion(
                     self.engine_client,
                     self.openai_serving_models,
+                    openai_serving_render=openai_serving_render,
                     request_logger=self.request_logger,
                     return_tokens_as_token_ids=self.args.return_tokens_as_token_ids,
                     enable_prompt_tokens_details=self.args.enable_prompt_tokens_details,
                     enable_force_include_usage=self.args.enable_force_include_usage,
-                    log_error_stack=self.args.log_error_stack,
                 )
                 if "generate" in supported_tasks
                 else None
             )
 
             self.openai_serving_embedding = (
-                OpenAIServingEmbedding(
+                ServingEmbedding(
                     self.engine_client,
                     self.openai_serving_models,
                     request_logger=self.request_logger,
@@ -195,7 +212,7 @@ async def start_engine(self):
                     request_logger=self.request_logger,
                     log_error_stack=self.args.log_error_stack,
                 )
-                if ("embed" in supported_tasks or "score" in supported_tasks)
+                if ("embed" in supported_tasks or "classify" in supported_tasks)
                 else None
             )
 
@@ -281,9 +298,7 @@ async def create_embedding(
                 message="The model does not support Embeddings API",
                 status_code=HTTPStatus.BAD_REQUEST,
             )
-        response = await self.openai_serving_embedding.create_embedding(
-            request, raw_request
-        )
+        response = await self.openai_serving_embedding(request, raw_request)
 
         if isinstance(response, engineError):
             return create_error_response(
@@ -306,7 +321,7 @@ async def create_rerank(
                 message="The model does not support Rerank API",
                 status_code=HTTPStatus.BAD_REQUEST,
             )
-        response = await self.serving_reranking.do_rerank(request, raw_request)
+        response = await self.serving_reranking(request, raw_request)
 
         if isinstance(response, engineError):
             return create_error_response(

python/huggingfaceserver/pyproject.toml

@@ -14,10 +14,16 @@ dependencies = [
     "setuptools>=70.0.0",
 ]
 name = "huggingfaceserver"
-version = "0.17.0"
+version = "0.18.0"
 description = "Model Server implementation for huggingface. Not intended for use outside KServe Frameworks Images."
 readme = "README.md"
 
+# fixes CVE-2026-40192
+[tool.uv]
+override-dependencies = [
+    "pillow>=12.2.0",
+]
+
 [dependency-groups]
 test = [
     "pytest<8.0.0,>=7.4.4",
@@ -26,7 +32,7 @@ test = [
     "pytest-asyncio<1.0.0,>=0.23.4",
     "pytest-httpx<1.0.0,>=0.30.0",
     "einops<1.0.0,>=0.8.0",
-    "openai<2.0.0,>=1.59.9",
+    "openai>=2.0.0",
 ]
 dev = [
     "black[colorama]~=24.3.0",

python/kserve/kserve/protocol/rest/openai/types/__init__.py

@@ -43,7 +43,7 @@
     EmbeddingResponseData,
     EmbeddingCompletionRequest,
 )
-from vllm.entrypoints.pooling.score.protocol import (
+from vllm.entrypoints.pooling.scoring.protocol import (
     RerankRequest,
     RerankResponse as Rerank,
 )

python/kserve/pyproject.toml

@@ -69,7 +69,7 @@ ray = [
     "ray[serve]>=2.43.0",
 ]
 llm = [
-    "vllm==0.15.1",
+    "vllm==0.18.0",
 ]
 
 [dependency-groups]