add FastVLM

paddleformers/transformers/__init__.py

@@ -94,6 +94,10 @@
     "auto.processing": ["AutoProcessor"],
     "paligemma2.image_processor": ["PaliGemmaImageProcessor"],
     "paligemma2.processor": ["PaliGemmaProcessor"],
+    "fastvlm.configuration": ["FastVLMConfig"],
+    "fastvlm.image_processor": ["FastVLMImageProcessor"],
+    "fastvlm.processor": ["FastVLMProcessor"],
+    "fastvlm.modeling": ["FastVLMModel", "FastVLMForCausalLM", "FastVLMForConditionalGeneration"],
     "auto.tokenizer": ["AutoTokenizer", "TOKENIZER_MAPPING"],
     "auto.video_processing": ["AutoVideoProcessor", "VIDEO_PROCESSOR_MAPPING"],
     "auto.feature_extraction": ["AutoFeatureExtractor"],

paddleformers/transformers/auto/configuration.py

@@ -37,6 +37,7 @@
         ("deepseek_v32", "DeepseekV32Config"),
         ("paligemma", "PaliGemma2Config"),
         ("paligemma2", "PaliGemma2Config"),
+        ("llava_qwen2", "FastVLMConfig"),
         ("diff_transformer", "DiffTransformerConfig"),
         ("ernie4_5", "Ernie4_5Config"),
         ("ernie4_5_moe", "Ernie4_5_MoeConfig"),
@@ -90,6 +91,7 @@
         ("deepseek_v3", "DeepseekV3"),
         ("paligemma", "PaliGemma2"),
         ("paligemma2", "PaliGemma2"),
+        ("llava_qwen2", "FastVLM"),
         ("diff_transformer", "DiffTransformer"),
         ("ernie4_5", "Ernie4_5"),
         ("ernie4_5_moe", "Ernie4_5_Moe"),
@@ -135,6 +137,7 @@
 SPECIAL_MODEL_TYPE_TO_MODULE_NAME = OrderedDict(
     [
         ("paligemma", "paligemma2"),
+        ("llava_qwen2", "fastvlm"),
         ("gemma3", "gemma3"),
         ("qwen2_5_vl_text", "qwen2_5_vl"),
         ("qwen3_vl_text", "qwen3_vl"),

paddleformers/transformers/auto/image_processing.py

@@ -62,6 +62,7 @@
         "qwen3_vl": ("Qwen3VLImageProcessor", "Qwen3VLImageProcessorFast"),
         "glm_ocr": ("Glm46VImageProcessor"),
         "paligemma": ("PaliGemmaImageProcessor"),
+        "llava_qwen2": ("FastVLMImageProcessor"),
     }
 )
 

paddleformers/transformers/auto/modeling.py

@@ -56,6 +56,7 @@
         ("DeepseekV3", "deepseek_v3"),
         ("DeepseekV32", "deepseek_v32"),
         ("PaliGemma2", "paligemma2"),
+        ("FastVLM", "fastvlm"),
         ("DiffTransformer", "diff_transformer"),
         ("Ernie4_5", "ernie4_5"),
         ("Ernie4_5_Moe", "ernie4_5_moe"),
@@ -202,6 +203,9 @@ def _get_model_class_from_config(cls, pretrained_model_name_or_path, config_file
         if init_class == "PaliGemmaForConditionalGeneration":
             import_class = importlib.import_module("paddleformers.transformers.paligemma2.modeling")
             return getattr(import_class, "PaliGemma2ForConditionalGeneration")
+        if init_class == "LlavaQwen2ForCausalLM":
+            import_class = importlib.import_module("paddleformers.transformers.fastvlm.modeling")
+            return getattr(import_class, "FastVLMForConditionalGeneration")
         if model_name is None:
             # Try to get model class from config class
             if not isinstance(config, PretrainedConfig) and pretrained_model_name_or_path is not None:

paddleformers/transformers/auto/processing.py

@@ -61,6 +61,7 @@
         ("glm4v_moe", "Glm4vProcessor"),
         ("glm_ocr", "Glm46VProcessor"),
         ("paligemma2", "PaliGemmaProcessor"),
+        ("llava_qwen2", "FastVLMProcessor"),
     ]
 )
 

paddleformers/transformers/fastvlm/__init__.py

@@ -0,0 +1,30 @@
+# Copyright (c) 2026 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+
+import sys
+from typing import TYPE_CHECKING
+
+from ...utils.lazy_import import _LazyModule
+
+import_structure = {
+    "configuration": ["FastVLMConfig"],
+    "image_processor": ["FastVLMImageProcessor"],
+    "processor": ["FastVLMProcessor"],
+    "modeling": ["FastVLMModel", "FastVLMForCausalLM", "FastVLMForConditionalGeneration"],
+    "modeling_vision": ["FastVLMVisionModel"],
+}
+
+if TYPE_CHECKING:
+    from .configuration import FastVLMConfig
+    from .image_processor import FastVLMImageProcessor
+    from .modeling import (
+        FastVLMForCausalLM,
+        FastVLMForConditionalGeneration,
+        FastVLMModel,
+    )
+    from .modeling_vision import FastVLMVisionModel
+    from .processor import FastVLMProcessor
+else:
+    sys.modules[__name__] = _LazyModule(__name__, globals()["__file__"], import_structure, module_spec=__spec__)

paddleformers/transformers/fastvlm/configuration.py

@@ -0,0 +1,52 @@
+# Copyright (c) 2026 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""FastVLM configuration."""
+
+from ..qwen2.configuration import Qwen2Config
+
+
+class FastVLMConfig(Qwen2Config):
+    """Configuration for Apple FastVLM checkpoints based on LLaVA-Qwen2."""
+
+    model_type = "llava_qwen2"
+
+    def __init__(
+        self,
+        mm_vision_tower="mobileclip_l_1024",
+        mm_hidden_size=3072,
+        mm_projector_type="mlp2x_gelu",
+        mm_patch_merge_type="flat",
+        image_aspect_ratio="pad",
+        image_token_index=-200,
+        unfreeze_mm_vision_tower=True,
+        tokenizer_model_max_length=8192,
+        tokenizer_padding_side="right",
+        vision_config=None,
+        **kwargs,
+    ):
+        super().__init__(**kwargs)
+        self.mm_vision_tower = mm_vision_tower
+        self.mm_hidden_size = mm_hidden_size
+        self.mm_projector_type = mm_projector_type
+        self.mm_patch_merge_type = mm_patch_merge_type
+        self.image_aspect_ratio = image_aspect_ratio
+        self.image_token_index = image_token_index
+        self.unfreeze_mm_vision_tower = unfreeze_mm_vision_tower
+        self.tokenizer_model_max_length = tokenizer_model_max_length
+        self.tokenizer_padding_side = tokenizer_padding_side
+        self.vision_config = vision_config
+
+
+__all__ = ["FastVLMConfig"]

paddleformers/transformers/fastvlm/image_processor.py

@@ -0,0 +1,26 @@
+# Copyright (c) 2026 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+
+"""Image preprocessing for FastVLM."""
+
+from transformers import CLIPImageProcessor
+
+from ..image_processing_utils import PaddleImageProcessingMixin
+
+
+class FastVLMImageProcessor(PaddleImageProcessingMixin, CLIPImageProcessor):
+    def __init__(self, image_size=1024, **kwargs):
+        kwargs.setdefault("do_resize", True)
+        kwargs.setdefault("size", {"shortest_edge": image_size})
+        kwargs.setdefault("do_center_crop", True)
+        kwargs.setdefault("crop_size", {"height": image_size, "width": image_size})
+        kwargs.setdefault("do_rescale", True)
+        kwargs.setdefault("rescale_factor", 1 / 255)
+        kwargs.setdefault("do_normalize", True)
+        kwargs.setdefault("image_mean", [0.0, 0.0, 0.0])
+        kwargs.setdefault("image_std", [1.0, 1.0, 1.0])
+        super().__init__(**kwargs)
+
+
+__all__ = ["FastVLMImageProcessor"]