feat(DATAPF-2790): Upgrade vLLM to 0.20.0 with RunAI streamer s…

python/huggingface_server.Dockerfile

@@ -52,9 +52,9 @@ WORKDIR ${WORKSPACE_DIR}
 FROM base AS build
 
 ARG WORKSPACE_DIR
-ARG VLLM_VERSION=0.15.1
-ARG LMCACHE_VERSION=0.3.9
-ARG FLASHINFER_VERSION=0.6.1
+ARG VLLM_VERSION=0.18.0
+ARG LMCACHE_VERSION=0.4.2
+ARG FLASHINFER_VERSION=0.6.6
 
 WORKDIR ${WORKSPACE_DIR}
 
@@ -162,6 +162,10 @@ ENV VLLM_NCCL_SO_PATH="/lib/x86_64-linux-gnu/libnccl.so.2"
 # Set the multiprocess method to spawn to avoid issues with cuda initialization for `mp` executor backend.
 ENV VLLM_WORKER_MULTIPROC_METHOD="spawn"
 
+ENV LD_LIBRARY_PATH=/usr/local/nvidia/lib64:/usr/local/cuda/lib64:/usr/local/cuda/targets/x86_64-linux/lib:$(LD_LIBRARY_PATH)
+# Default AWS region when using runai to download from S3 bucket. TODO: pass this at runtime.
+ENV AWS_REGION=eu-west-1
+
 USER 1000
 ENV PYTHONPATH=${WORKSPACE_DIR}/huggingfaceserver
 ENTRYPOINT ["python3", "-m", "huggingfaceserver"]

python/huggingface_server_cpu.Dockerfile

@@ -83,7 +83,7 @@ RUN cd huggingfaceserver && \
     rm -rf ~/.cache/uv
 
 # install vllm
-ARG VLLM_VERSION=0.15.1
+ARG VLLM_VERSION=0.18.0
 ARG VLLM_CPU_DISABLE_AVX512=true
 ENV VLLM_CPU_DISABLE_AVX512=${VLLM_CPU_DISABLE_AVX512}
 ARG VLLM_CPU_AVX512BF16=1

python/huggingfaceserver/huggingfaceserver/__main__.py

@@ -50,22 +50,37 @@ def list_of_strings(arg):
     return arg.split(",")
 
 
+_REMOTE_URI_PREFIXES = (
+    "s3://", "gs://", "az://", "azure://",
+    "http://", "https://", "ftp://", "hdfs://",
+)
+
+
 def get_model_id_or_path(args: argparse.Namespace) -> Union[str, Path]:
     # If --model_id is specified then pass model_id to HF API, otherwise load the model from /mnt/models
     if args.model_id:
         return cast(str, args.model_id)
+    # For remote URIs (s3://, gs://, etc.), pass directly to vLLM which handles
+    # streaming via load format (e.g., runai_streamer). Skip Storage.download
+    # to avoid attempting a full local download.
+    if isinstance(args.model_dir, str) and args.model_dir.startswith(_REMOTE_URI_PREFIXES):
+        return args.model_dir
     return Path(Storage.download(args.model_dir))
 
 
 def is_vllm_backend_enabled(
     args: argparse.Namespace, model_id_or_path: Union[str, Path]
 ) -> bool:
+    # When backend is explicitly set to vllm, skip architecture validation.
+    # This allows using custom load formats (like runai_streamer) with remote URIs.
+    force_vllm = args.backend == Backend.vllm
     return (
         (args.backend == Backend.vllm or args.backend == Backend.auto)
         and vllm_available()
         and infer_vllm_supported_from_model_architecture(
             model_id_or_path,
             trust_remote_code=args.trust_remote_code,
+            force_vllm=force_vllm,
         )
     )
 

python/huggingfaceserver/huggingfaceserver/vllm/utils.py

@@ -41,18 +41,47 @@ def vllm_available() -> bool:
 def infer_vllm_supported_from_model_architecture(
     model_config_path: Union[Path, str],
     trust_remote_code: bool = False,
+    force_vllm: bool = False,
 ) -> bool:
     if not _vllm:
         return False
 
-    model_config = AutoConfig.from_pretrained(
-        model_config_path, trust_remote_code=trust_remote_code
-    )
-    for architecture in model_config.architectures:
-        if architecture not in ModelRegistry.get_supported_archs():
-            logger.info("not a supported model by vLLM")
-            return False
-    return True
+    model_path_str = str(model_config_path)
+
+    # Skip architecture validation for remote URIs (S3, GCS, Azure, HTTP).
+    # These are used with custom load formats like runai_streamer that bypass
+    # standard HuggingFace model loading.
+    is_remote_uri = model_path_str.startswith((
+        "s3://", "gs://", "az://", "azure://",
+        "http://", "https://", "ftp://", "hdfs://",
+    ))
+
+    if is_remote_uri:
+        logger.info(
+            f"Skipping architecture validation for remote URI: {model_path_str}. "
+            "Assuming vLLM can handle this with appropriate load-format settings."
+        )
+        return True
+
+    if force_vllm:
+        logger.info("vLLM backend explicitly requested, skipping architecture validation.")
+        return True
+
+    try:
+        model_config = AutoConfig.from_pretrained(
+            model_config_path, trust_remote_code=trust_remote_code
+        )
+        for architecture in model_config.architectures:
+            if architecture not in ModelRegistry.get_supported_archs():
+                logger.info("not a supported model by vLLM")
+                return False
+        return True
+    except Exception as e:
+        logger.warning(
+            f"Failed to load model config from {model_path_str}: {e}. "
+            "If using a remote URI with custom load format, consider setting --backend=vllm explicitly."
+        )
+        return False
 
 
 def maybe_add_vllm_cli_parser(parser: ArgumentParser) -> ArgumentParser:

python/huggingfaceserver/huggingfaceserver/vllm/vllm_model.py

@@ -23,7 +23,7 @@
 from vllm.engine.protocol import EngineClient
 from vllm.entrypoints.openai.completion.serving import OpenAIServingCompletion
 from vllm.entrypoints.openai.chat_completion.serving import OpenAIServingChat
-from vllm.entrypoints.pooling.embed.serving import OpenAIServingEmbedding
+from vllm.entrypoints.pooling.embed.serving import ServingEmbedding
 from vllm.entrypoints.pooling.score.serving import ServingScores
 from vllm.tool_parsers import ToolParserManager
 from vllm.entrypoints.openai.models.protocol import BaseModelPath
@@ -60,7 +60,7 @@ class VLLMModel(OpenAIEncoderModel, OpenAIGenerativeModel):  # pylint:disable=c-
     openai_serving_models: Optional[OpenAIServingModels] = None
     openai_serving_completion: Optional[OpenAIServingCompletion] = None
     openai_serving_chat: Optional[OpenAIServingChat] = None
-    openai_serving_embedding: Optional[OpenAIServingEmbedding] = None
+    openai_serving_embedding: Optional[ServingEmbedding] = None
     serving_reranking: Optional[ServingScores] = None
 
     def __init__(
@@ -175,7 +175,7 @@ async def start_engine(self):
             )
 
             self.openai_serving_embedding = (
-                OpenAIServingEmbedding(
+                ServingEmbedding(
                     self.engine_client,
                     self.openai_serving_models,
                     request_logger=self.request_logger,

python/kserve/pyproject.toml

@@ -69,7 +69,7 @@ ray = [
     "ray[serve]>=2.43.0",
 ]
 llm = [
-    "vllm==0.15.1",
+    "vllm==0.18.0",
 ]
 
 [dependency-groups]