add FastVLM

README.md

@@ -132,7 +132,12 @@ PaddleFormers 是基于百度深度学习框架 PaddlePaddle 搭建的 Transform
     </tr>
     <!-- VLM 分类 - 跨行合并开始 -->
     <tr>
-      <td rowspan="4" style="vertical-align: top;">VLM</td>
+      <td rowspan="5" style="vertical-align: top;">VLM</td>
+      <td>FastVLM</td>
+      <td>apple/FastVLM-0.5B</td>
+      <td>jinja</td>
+    </tr>
+    <tr>
       <td>🏛️ERNIE-4.5-VL</td>
       <td>baidu/ERNIE-4.5-VL-28B-A3B-Base-PT、baidu/ERNIE-4.5-VL-28B-A3B-PT、baidu/ERNIE-4.5-VL-424B-A47B-Base-PT、baidu/ERNIE-4.5-VL-424B-A47B-PT、baidu/ERNIE-4.5-VL-28B-A3B-Thinking</td>
       <td>ernie_vl、ernie_vl_nothink</td>

docs/zh/model_capability.md

@@ -13,6 +13,7 @@
 |Qwen2|✓|✓|✓|✓|✓|
 |Qwen3|✓|✓|✓|✓|✓|
 |Qwen3-Next|✓|✓|✓|✓|✓|
+|FastVLM|x|✓|x|x|x|
 |🏛️ERNIE-4.5-VL|x|✓|✓|x|x|
 |🏛️PaddleOCR-VL|x|✓|✓|x|x|
 |Qwen2.5-VL|x|✓|✓|x|x|
@@ -33,6 +34,7 @@
 |Qwen2|✓|✓|x|x|✓|✓|
 |Qwen3|✓|✓|✓|✓|✓|✓|
 |Qwen3-Next|✓|✓|✓|x|✓|✓|
+|FastVLM|x|x|-|x|x|x|
 |🏛️ERNIE-4.5-VL|✓|✓|✓|x|✓|✓|
 |🏛️PaddleOCR-VL|x|x|-|x|✓|✓|
 |Qwen2.5-VL|✓|x|-|x|✓|✓|

paddleformers/cli/utils/llm_utils.py

@@ -109,7 +109,7 @@ def get_lora_target_modules(model):
             ".*mlp.w2.*",
             ".*mlp.c_proj.*",
         ]
-    elif model.config.model_type == "qwen2":
+    elif model.config.model_type in ["qwen2", "llava_qwen2"]:
         target_modules = [
             ".*qkv_proj.*",
             ".*up_gate_proj.*",

paddleformers/cli/utils/mllm_utils.py

@@ -25,6 +25,7 @@
 
 
 class MLLMModelMapping:
+    fastvlm = "llava_qwen2"
     qwen2_5_vl = "qwen2_5_vl"
     qwen3_vl = "qwen3_vl"
     qwen3_vl_moe = "qwen3_vl_moe"
@@ -188,6 +189,14 @@ def freeze_model_parameters(model, freeze_config):
         logger.info("\n".join(log_info))
 
 
+register_multimodel_keys(
+    MultiModelKeys(
+        model_dtype=MLLMModelMapping.fastvlm,
+        aligner="model.mm_projector",
+        llm=["model.embed_tokens", "model.layers", "model.norm", "lm_head"],
+        vision="model.vision_tower",
+    )
+)
 register_multimodel_keys(
     MultiModelKeys(
         model_dtype=MLLMModelMapping.qwen2_5_vl,

paddleformers/transformers/__init__.py

@@ -94,6 +94,10 @@
     "auto.processing": ["AutoProcessor"],
     "paligemma2.image_processor": ["PaliGemmaImageProcessor"],
     "paligemma2.processor": ["PaliGemmaProcessor"],
+    "fastvlm.configuration": ["FastVLMConfig"],
+    "fastvlm.image_processor": ["FastVLMImageProcessor"],
+    "fastvlm.processor": ["FastVLMProcessor"],
+    "fastvlm.modeling": ["FastVLMModel", "FastVLMForCausalLM", "FastVLMForConditionalGeneration"],
     "auto.tokenizer": ["AutoTokenizer", "TOKENIZER_MAPPING"],
     "auto.video_processing": ["AutoVideoProcessor", "VIDEO_PROCESSOR_MAPPING"],
     "auto.feature_extraction": ["AutoFeatureExtractor"],

paddleformers/transformers/auto/configuration.py

@@ -37,6 +37,7 @@
         ("deepseek_v32", "DeepseekV32Config"),
         ("paligemma", "PaliGemma2Config"),
         ("paligemma2", "PaliGemma2Config"),
+        ("llava_qwen2", "FastVLMConfig"),
         ("diff_transformer", "DiffTransformerConfig"),
         ("ernie4_5", "Ernie4_5Config"),
         ("ernie4_5_moe", "Ernie4_5_MoeConfig"),
@@ -92,6 +93,7 @@
         ("deepseek_v3", "DeepseekV3"),
         ("paligemma", "PaliGemma2"),
         ("paligemma2", "PaliGemma2"),
+        ("llava_qwen2", "FastVLM"),
         ("diff_transformer", "DiffTransformer"),
         ("ernie4_5", "Ernie4_5"),
         ("ernie4_5_moe", "Ernie4_5_Moe"),
@@ -139,6 +141,7 @@
 SPECIAL_MODEL_TYPE_TO_MODULE_NAME = OrderedDict(
     [
         ("paligemma", "paligemma2"),
+        ("llava_qwen2", "fastvlm"),
         ("gemma3", "gemma3"),
         ("qwen2_5_vl_text", "qwen2_5_vl"),
         ("qwen3_vl_text", "qwen3_vl"),

paddleformers/transformers/auto/image_processing.py

@@ -62,6 +62,7 @@
         "qwen3_vl": ("Qwen3VLImageProcessor", "Qwen3VLImageProcessorFast"),
         "glm_ocr": ("Glm46VImageProcessor"),
         "paligemma": ("PaliGemmaImageProcessor"),
+        "llava_qwen2": ("FastVLMImageProcessor"),
     }
 )
 

paddleformers/transformers/auto/modeling.py

@@ -57,6 +57,7 @@
         ("DeepseekV3", "deepseek_v3"),
         ("DeepseekV32", "deepseek_v32"),
         ("PaliGemma2", "paligemma2"),
+        ("FastVLM", "fastvlm"),
         ("DiffTransformer", "diff_transformer"),
         ("Ernie4_5", "ernie4_5"),
         ("Ernie4_5_Moe", "ernie4_5_moe"),
@@ -214,6 +215,9 @@ def _get_model_class_from_config(
         if init_class == "PaliGemmaForConditionalGeneration":
             import_class = importlib.import_module("paddleformers.transformers.paligemma2.modeling")
             return getattr(import_class, "PaliGemma2ForConditionalGeneration")
+        if init_class == "LlavaQwen2ForCausalLM":
+            import_class = importlib.import_module("paddleformers.transformers.fastvlm.modeling")
+            return getattr(import_class, "FastVLMForConditionalGeneration")
         if model_name is None:
             # Try to get model class from config class
             if not isinstance(config, PretrainedConfig) and pretrained_model_name_or_path is not None:

paddleformers/transformers/auto/processing.py

@@ -61,6 +61,7 @@
         ("glm4v_moe", "Glm4vProcessor"),
         ("glm_ocr", "Glm46VProcessor"),
         ("paligemma2", "PaliGemmaProcessor"),
+        ("llava_qwen2", "FastVLMProcessor"),
     ]
 )
 
@@ -193,6 +194,13 @@ def from_pretrained(cls, pretrained_model_name_or_path, **kwargs):
             if hasattr(config, "auto_map") and "AutoProcessor" in config.auto_map:
                 processor_auto_map = config.auto_map["AutoProcessor"]
 
+            # Some upstream multimodal checkpoints only declare a model type and
+            # do not ship a processor_class or preprocessor_config.json. Resolve
+            # PaddleFormers-native processor registrations by model type before
+            # falling back to the Transformers config-class mapping.
+            if processor_class is None:
+                processor_class = PROCESSOR_MAPPING_NAMES.get(getattr(config, "model_type", None))
+
         if processor_class is not None:
             processor_class = processor_class_from_name(processor_class)
 

paddleformers/transformers/fastvlm/__init__.py

@@ -0,0 +1,30 @@
+# Copyright (c) 2026 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+
+import sys
+from typing import TYPE_CHECKING
+
+from ...utils.lazy_import import _LazyModule
+
+import_structure = {
+    "configuration": ["FastVLMConfig"],
+    "image_processor": ["FastVLMImageProcessor"],
+    "processor": ["FastVLMProcessor"],
+    "modeling": ["FastVLMModel", "FastVLMForCausalLM", "FastVLMForConditionalGeneration"],
+    "modeling_vision": ["FastVLMVisionModel"],
+}
+
+if TYPE_CHECKING:
+    from .configuration import FastVLMConfig
+    from .image_processor import FastVLMImageProcessor
+    from .modeling import (
+        FastVLMForCausalLM,
+        FastVLMForConditionalGeneration,
+        FastVLMModel,
+    )
+    from .modeling_vision import FastVLMVisionModel
+    from .processor import FastVLMProcessor
+else:
+    sys.modules[__name__] = _LazyModule(__name__, globals()["__file__"], import_structure, module_spec=__spec__)

paddleformers/transformers/fastvlm/configuration.py

@@ -0,0 +1,52 @@
+# Copyright (c) 2026 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""FastVLM configuration."""
+
+from ..qwen2.configuration import Qwen2Config
+
+
+class FastVLMConfig(Qwen2Config):
+    """Configuration for Apple FastVLM checkpoints based on LLaVA-Qwen2."""
+
+    model_type = "llava_qwen2"
+
+    def __init__(
+        self,
+        mm_vision_tower="mobileclip_l_1024",
+        mm_hidden_size=3072,
+        mm_projector_type="mlp2x_gelu",
+        mm_patch_merge_type="flat",
+        image_aspect_ratio="pad",
+        image_token_index=-200,
+        unfreeze_mm_vision_tower=True,
+        tokenizer_model_max_length=8192,
+        tokenizer_padding_side="right",
+        vision_config=None,
+        **kwargs,
+    ):
+        super().__init__(**kwargs)
+        self.mm_vision_tower = mm_vision_tower
+        self.mm_hidden_size = mm_hidden_size
+        self.mm_projector_type = mm_projector_type
+        self.mm_patch_merge_type = mm_patch_merge_type
+        self.image_aspect_ratio = image_aspect_ratio
+        self.image_token_index = image_token_index
+        self.unfreeze_mm_vision_tower = unfreeze_mm_vision_tower
+        self.tokenizer_model_max_length = tokenizer_model_max_length
+        self.tokenizer_padding_side = tokenizer_padding_side
+        self.vision_config = vision_config
+
+
+__all__ = ["FastVLMConfig"]

paddleformers/transformers/fastvlm/image_processor.py

@@ -0,0 +1,26 @@
+# Copyright (c) 2026 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+
+"""Image preprocessing for FastVLM."""
+
+from transformers import CLIPImageProcessor
+
+from ..image_processing_utils import PaddleImageProcessingMixin
+
+
+class FastVLMImageProcessor(PaddleImageProcessingMixin, CLIPImageProcessor):
+    def __init__(self, image_size=1024, **kwargs):
+        kwargs.setdefault("do_resize", True)
+        kwargs.setdefault("size", {"shortest_edge": image_size})
+        kwargs.setdefault("do_center_crop", True)
+        kwargs.setdefault("crop_size", {"height": image_size, "width": image_size})
+        kwargs.setdefault("do_rescale", True)
+        kwargs.setdefault("rescale_factor", 1 / 255)
+        kwargs.setdefault("do_normalize", True)
+        kwargs.setdefault("image_mean", [0.0, 0.0, 0.0])
+        kwargs.setdefault("image_std", [1.0, 1.0, 1.0])
+        super().__init__(**kwargs)
+
+
+__all__ = ["FastVLMImageProcessor"]