Add Llama4 text model

README.md

@@ -55,7 +55,7 @@ PaddleFormers 是基于百度深度学习框架 PaddlePaddle 搭建的 Transform
   <tbody>
     <!-- LLM 分类 - 跨行合并开始 -->
     <tr>
-      <td rowspan="15" style="vertical-align: top;">LLM</td>
+      <td rowspan="16" style="vertical-align: top;">LLM</td>
       <td>DeepSeekv3</td>
       <td>deepseek-ai/DeepSeek-V3-Base、deepseek-ai/DeepSeek-V3、deepseek-ai/DeepSeek-V3-0324</td>
       <td>deepseek3</td>
@@ -100,6 +100,11 @@ PaddleFormers 是基于百度深度学习框架 PaddlePaddle 搭建的 Transform
       <td>meta-llama/Meta-Llama-3-8B、meta-llama/Meta-Llama-3-8B-Instruct、meta-llama/Meta-Llama-3-70B、meta-llama/Meta-Llama-3-70B-Instruct、meta-llama/Llama-3.1-8B、meta-llama/Llama-3.1-8B-Instruct、meta-llama/Llama-3.1-70B、meta-llama/Llama-3.1-70B-Instruct、meta-llama/Llama-3.1-405B、meta-llama/Llama-3.1-405B-Instruct、meta-llama/Llama-3.2-1B、meta-llama/Llama-3.2-1B-Instruct、meta-llama/Llama-3.2-3B、meta-llama/Llama-3.2-3B-Instruct、meta-llama/Llama-3.3-70B-Instruct</td>
       <td>llama3</td>
     </tr>
+    <tr>
+      <td>Llama-4-Text</td>
+      <td>meta-llama/Llama-4-Scout-17B-16E、meta-llama/Llama-4-Maverick-17B-128E</td>
+      <td>llama4</td>
+    </tr>
     <tr>
       <td>OLMo2</td>
       <td>allenai/OLMo-2-0425-1B、allenai/OLMo-2-0425-1B-SFT、allenai/OLMo-2-0425-1B-DPO、allenai/OLMo-2-0425-1B-Instruct、allenai/OLMo-2-1124-7B、allenai/OLMo-2-1124-7B-SFT、allenai/OLMo-2-1124-7B-DPO、allenai/OLMo-2-1124-7B-Instruct、allenai/OLMo-2-1124-13B、allenai/OLMo-2-1124-13B-SFT、allenai/OLMo-2-1124-13B-DPO、allenai/OLMo-2-1124-13B-Instruct、allenai/OLMo-2-0325-32B、allenai/OLMo-2-0325-32B-SFT、allenai/OLMo-2-0325-32B-DPO、allenai/OLMo-2-0325-32B-Instruct</td>
@@ -132,7 +137,12 @@ PaddleFormers 是基于百度深度学习框架 PaddlePaddle 搭建的 Transform
     </tr>
     <!-- VLM 分类 - 跨行合并开始 -->
     <tr>
-      <td rowspan="4" style="vertical-align: top;">VLM</td>
+      <td rowspan="5" style="vertical-align: top;">VLM</td>
+      <td>Llama Guard 4</td>
+      <td>meta-llama/Llama-Guard-4-12B</td>
+      <td>llama4</td>
+    </tr>
+    <tr>
       <td>🏛️ERNIE-4.5-VL</td>
       <td>baidu/ERNIE-4.5-VL-28B-A3B-Base-PT、baidu/ERNIE-4.5-VL-28B-A3B-PT、baidu/ERNIE-4.5-VL-424B-A47B-Base-PT、baidu/ERNIE-4.5-VL-424B-A47B-PT、baidu/ERNIE-4.5-VL-28B-A3B-Thinking</td>
       <td>ernie_vl、ernie_vl_nothink</td>

docs/zh/model_capability.md

@@ -8,11 +8,13 @@
 |GPT-OSS|✓|✓|✓|x|x|
 |Granite|✓|✓|✓|✓|✓|
 |LLaMA3|✓|✓|✓|✓|✓|
+|LLaMA4-Text|✓|✓|✓|x|x|
 |OLMo2|✓|✓|✓|✓|✓|
 |Phi4|✓|✓|✓|✓|✓|
 |Qwen2|✓|✓|✓|✓|✓|
 |Qwen3|✓|✓|✓|✓|✓|
 |Qwen3-Next|✓|✓|✓|✓|✓|
+|Llama Guard 4|x|✓|✓|x|x|
 |🏛️ERNIE-4.5-VL|x|✓|✓|x|x|
 |🏛️PaddleOCR-VL|x|✓|✓|x|x|
 |Qwen2.5-VL|x|✓|✓|x|x|
@@ -28,11 +30,13 @@
 |GPT-OSS|✓|✓|x|x|✓|✓|
 |Granite|✓|✓|-|x|✓|✓|
 |LLaMA3|✓|✓|-|x|✓|✓|
+|LLaMA4-Text|x|x|x|x|✓|✓|
 |OLMo2|✓|✓|-|x|✓|✓|
 |Phi4|✓|✓|-|x|✓|✓|
 |Qwen2|✓|✓|x|x|✓|✓|
 |Qwen3|✓|✓|✓|✓|✓|✓|
 |Qwen3-Next|✓|✓|✓|x|✓|✓|
+|Llama Guard 4|x|x|-|x|x|x|
 |🏛️ERNIE-4.5-VL|✓|✓|✓|x|✓|✓|
 |🏛️PaddleOCR-VL|x|x|-|x|✓|✓|
 |Qwen2.5-VL|✓|x|-|x|✓|✓|

paddleformers/cli/utils/llm_utils.py

@@ -77,7 +77,7 @@ def get_lora_target_modules(model):
         ]
     elif model.config.model_type == "bloom":
         target_modules = [".*query_key_value.*", ".*dense.*", ".*dense_h_to_4h.*", ".*dense_4h_to_h.*"]
-    elif model.config.model_type in ["llama", "jamba", "olmo2"]:
+    elif model.config.model_type in ["llama", "llama4_text", "jamba", "olmo2"]:
         target_modules = [
             ".*q_proj.*",
             ".*v_proj.*",

paddleformers/datasets/template/__init__.py

@@ -42,6 +42,7 @@
         "Qwen3VLPlugin",
         "GLM4VPlugin",
         "Gemma3Plugin",
+        "Llama4Plugin",
         "register_mm_plugin",
         "get_mm_plugin",
     ],

paddleformers/datasets/template/mm_plugin.py

@@ -1402,6 +1402,44 @@ def get_mm_inputs(
         return mm_inputs
 
 
+@dataclass
+class Llama4Plugin(BasePlugin):
+    masked_tokens: Optional[list[str]] = None
+
+    @override
+    def process_messages(
+        self,
+        messages,
+        images,
+        videos,
+        audios,
+        mm_inputs,
+        processor,
+    ):
+        self._validate_input(processor, images, videos, audios)
+        self._validate_messages(messages, images, videos, audios)
+        if not images:
+            return messages
+
+        aspect_ratios = mm_inputs.get("aspect_ratios")
+        if aspect_ratios is None or len(aspect_ratios) != len(images):
+            raise ValueError("Llama 4 requires one aspect ratio for each input image.")
+
+        messages = deepcopy(messages)
+        image_index = 0
+        num_patches_per_chunk = processor._num_patches_per_chunk()
+        for message in messages:
+            content = message["content"]
+            while IMAGE_PLACEHOLDER in content:
+                image_string = processor._prompt_split_image(aspect_ratios[image_index], num_patches_per_chunk)
+                if not self.expand_mm_tokens:
+                    image_string = self.image_token
+                content = content.replace(IMAGE_PLACEHOLDER, image_string, 1)
+                image_index += 1
+            message["content"] = content
+        return messages
+
+
 @dataclass
 class GlmOcrPlugin(BasePlugin):
     """
@@ -1608,6 +1646,7 @@ def _get_mm_inputs(
     "qwen3_vl": Qwen3VLPlugin,
     "glm4v": GLM4VPlugin,
     "gemma3": Gemma3Plugin,
+    "llama4": Llama4Plugin,
     "qwen2_omni": Qwen2OmniPlugin,
     "glm_ocr": GlmOcrPlugin,
     "kimi_k3": KimiK3Plugin,

paddleformers/datasets/template/template.py

@@ -955,6 +955,42 @@ def _get_gpt_oss_prefix():
     chat_sep="<|eot_id|>",
 )
 
+register_template(
+    name="llama4",
+    format_user=StringFormatter(
+        slots=[
+            ("<|header_start|>user<|header_end|>\n\n{{content}}<|eot|>" "<|header_start|>assistant<|header_end|>\n\n")
+        ]
+    ),
+    format_assistant=StringFormatter(slots=["{{content}}"]),
+    format_system=StringFormatter(slots=["<|header_start|>system<|header_end|>\n\n{{content}}<|eot|>"]),
+    format_function=FunctionFormatter(slots=["{{content}}"], tool_format="llama4"),
+    format_observation=StringFormatter(
+        slots=[
+            (
+                "<|header_start|>ipython<|header_end|>\n\n{{content}}<|eot|>"
+                "<|header_start|>assistant<|header_end|>\n\n"
+            )
+        ]
+    ),
+    format_tools=ToolFormatter(tool_format="llama4"),
+    format_prefix=EmptyFormatter(slots=[{"bos_token"}]),
+    suffix=["<|eot|>"],
+    chat_sep="<|eot|>",
+    mm_plugin=get_mm_plugin(
+        name="llama4",
+        image_token="<|image|>",
+        masked_tokens=[
+            "<|image|>",
+            "<|image_start|>",
+            "<|image_end|>",
+            "<|patch|>",
+            "<|tile_x_separator|>",
+            "<|tile_y_separator|>",
+        ],
+    ),
+)
+
 
 # copied from gemma template
 register_template(

paddleformers/datasets/template/tool_utils.py

@@ -90,6 +90,15 @@ class FunctionCall(NamedTuple):
     "Do not use variables.\n\n{tool_text}"
 )
 
+LLAMA4_TOOL_PROMPT = (
+    "You are a helpful assistant and an expert in function composition. You can answer general questions using your "
+    "internal knowledge OR invoke functions when necessary.\n"
+    "Here is a list of functions in JSON format that you can invoke:\n{tool_text}\n"
+    "Should you decide to return the function call(s), put them in the format of "
+    "[func1(param_name=param_value, ...), func2(...)].\n"
+    "Do not include any other text in the response when returning function calls."
+)
+
 
 @dataclass
 class ToolUtils(ABC):
@@ -288,6 +297,33 @@ def function_formatter(functions: list["FunctionCall"]) -> str:
         return json.dumps(function_objects[0] if len(function_objects) == 1 else function_objects, ensure_ascii=False)
 
 
+class Llama4ToolUtils(ToolUtils):
+    r"""Llama 4 zero-shot tool calling format.
+
+    Reference: https://github.com/meta-llama/llama-models/blob/main/models/llama4/prompt_format.md
+    """
+
+    @override
+    @staticmethod
+    def tool_formatter(tools: list[dict[str, Any]]) -> str:
+        normalized_tools = [tool.get("function", {}) if tool.get("type") == "function" else tool for tool in tools]
+        tool_text = json.dumps(normalized_tools, indent=4, ensure_ascii=False)
+        return LLAMA4_TOOL_PROMPT.format(tool_text=tool_text)
+
+    @override
+    @staticmethod
+    def function_formatter(functions: list["FunctionCall"]) -> str:
+        function_texts = []
+        for name, arguments in functions:
+            parsed_arguments = json.loads(arguments)
+            argument_text = ", ".join(
+                f"{key}={json.dumps(value, ensure_ascii=False)}" for key, value in parsed_arguments.items()
+            )
+            function_texts.append(f"{name}({argument_text})")
+
+        return "[" + ", ".join(function_texts) + "]"
+
+
 class ERNIEToolUtils(ToolUtils):
     r"""ERNIE 4.5 tool using template."""
 
@@ -346,6 +382,7 @@ def function_formatter(functions: list["FunctionCall"]) -> str:
     "glm4_moe": GLM4MOEToolUtils(),
     "glm_moe_dsa": GLM_5ToolUtils(),
     "llama3": Llama3ToolUtils(),
+    "llama4": Llama4ToolUtils(),
 }
 
 

paddleformers/transformers/__init__.py

@@ -240,6 +240,17 @@
     "llama.modeling": ["LlamaForCausalLM", "LlamaModel", "LlamaForCausalLMPipe", "LlamaRotaryEmbedding"],
     "llama.tokenizer": ["LlamaTokenizer", "Llama3Tokenizer"],
     "llama.tokenizer_fast": ["LlamaTokenizerFast"],
+    "llama4.configuration": ["Llama4Config", "Llama4TextConfig", "Llama4VisionConfig"],
+    "llama4.image_processor": ["Llama4ImageProcessor", "Llama4ImageProcessorKwargs"],
+    "llama4.image_processor_fast": ["Llama4ImageProcessorFast"],
+    "llama4.modeling": [
+        "Llama4ForCausalLM",
+        "Llama4TextDecoderLayer",
+        "Llama4TextModel",
+        "Llama4TextPretrainedModel",
+    ],
+    "llama4.multimodal": ["Llama4ForConditionalGeneration", "Llama4VisionEncoderLayer", "Llama4VisionModel"],
+    "llama4.processor": ["Llama4Processor", "Llama4ProcessorKwargs"],
     "optimization": [
         "LinearDecayWithWarmup",
         "ConstScheduleWithWarmup",
@@ -351,6 +362,7 @@
         "Qwen3NextPretrainingCriterion",
     ],
     "llama": [],
+    "llama4": [],
     "qwen2": [],
     "glm_ocr": [],
     "qwen3": [],

paddleformers/transformers/auto/configuration.py

@@ -43,6 +43,8 @@
         ("ernie4_5_moe_vl", "Ernie4_5_VLConfig"),
         ("paddleocr_vl", "PaddleOCRVLConfig"),
         ("llama", "LlamaConfig"),
+        ("llama4", "Llama4Config"),
+        ("llama4_text", "Llama4TextConfig"),
         ("kimi_k2", "KimiK2Config"),
         ("kimi_k3", "KimiK3Config"),
         ("qwen2", "Qwen2Config"),
@@ -99,6 +101,8 @@
         ("ernie4_5_moe_vl", "Ernie4_5_VLMoeForConditionalGeneration"),
         ("paddleocr_vl", "PaddleOCRVLForConditionalGeneration"),
         ("llama", "Llama"),
+        ("llama4", "Llama4ForConditionalGeneration"),
+        ("llama4_text", "Llama4TextModel"),
         ("kimi_k3", "KimiK3"),
         ("qwen2", "Qwen2"),
         ("qwen2_5_vl", "Qwen2_5_VL"),
@@ -140,6 +144,7 @@
 
 SPECIAL_MODEL_TYPE_TO_MODULE_NAME = OrderedDict(
     [
+        ("llama4_text", "llama4"),
         ("paligemma", "paligemma2"),
         ("gemma3", "gemma3"),
         ("qwen2_5_vl_text", "qwen2_5_vl"),

paddleformers/transformers/auto/image_processing.py

@@ -57,6 +57,7 @@
         "kimi_k3": ("KimiK3VisionProcessor"),
         "paddleocr_vl": ("PaddleOCRVLImageProcessor"),
         "gemma3": ("Gemma3ImageProcessor", "Gemma3ImageProcessorFast"),
+        "llama4": ("Llama4ImageProcessor", "Llama4ImageProcessorFast"),
         "qwen2_5_vl": ("Qwen2VLImageProcessor", "Qwen2VLImageProcessorFast"),
         "qwen2_vl": ("Qwen2VLImageProcessor", "Qwen2VLImageProcessorFast"),
         "qwen3_vl": ("Qwen3VLImageProcessor", "Qwen3VLImageProcessorFast"),

paddleformers/transformers/auto/modeling.py

@@ -63,6 +63,7 @@
         ("Ernie4_5_VLMoe", "ernie4_5_moe_vl"),
         ("PaddleOCRVL", "paddleocr_vl"),
         ("Llama", "llama"),
+        ("Llama4", "llama4"),
         ("KimiK2", "kimi_k2"),
         ("KimiK3", "kimi_k3"),
         ("Qwen2", "qwen2"),
@@ -97,7 +98,13 @@
     ]
 )
 
-MAPPING_SPACIAL_KEY = OrderedDict([("Gemma3", "Gemma3"), ("Ernie4_5_VLMoe", "Ernie4_5_VLMoeForConditionalGeneration")])
+MAPPING_SPACIAL_KEY = OrderedDict(
+    [
+        ("Gemma3", "Gemma3"),
+        ("Ernie4_5_VLMoe", "Ernie4_5_VLMoeForConditionalGeneration"),
+        ("Llama4", "Llama4Text"),
+    ]
+)
 CONFIGURATION_MODEL_MAPPING = OrderedDict([((), "Gemma3ForConditionalGeneration")])
 
 MAPPING_TASKS = OrderedDict(
@@ -203,9 +210,9 @@ def _get_model_class_from_config(
                 if model_flag in init_class:
                     model_name = model_flag + "Model"
                     break
-            if model_name is None and init_class == "PaliGemmaForConditionalGeneration":
-                model_name = "PaliGemma2ForConditionalGenerationModel"
-        elif model_name is None:
+        if model_name is None and init_class == "PaliGemmaForConditionalGeneration":
+            model_name = "PaliGemma2ForConditionalGenerationModel"
+        if model_name is None and init_class is None:
             # From pretrained_model_name_or_path
             for model_flag, name in SORTED_MAPPING_NAMES.items():
                 if type(pretrained_model_name_or_path) is str and name in pretrained_model_name_or_path.lower():
@@ -214,6 +221,9 @@ def _get_model_class_from_config(
         if init_class == "PaliGemmaForConditionalGeneration":
             import_class = importlib.import_module("paddleformers.transformers.paligemma2.modeling")
             return getattr(import_class, "PaliGemma2ForConditionalGeneration")
+        if init_class == "Llama4ForConditionalGeneration":
+            import_class = importlib.import_module("paddleformers.transformers.llama4.multimodal")
+            return getattr(import_class, "Llama4ForConditionalGeneration")
         if model_name is None:
             # Try to get model class from config class
             if not isinstance(config, PretrainedConfig) and pretrained_model_name_or_path is not None:

paddleformers/transformers/auto/processing.py

@@ -57,6 +57,7 @@
         ("paddleocr_vl", "PaddleOCRVLProcessor"),
         ("ernie4_5_moe_vl", "Ernie4_5_VLProcessor"),
         ("gemma3", "Gemma3Processor"),
+        ("llama4", "Llama4Processor"),
         ("shieldgemma2", "ShieldGemma2Processor"),
         ("glm4v_moe", "Glm4vProcessor"),
         ("glm_ocr", "Glm46VProcessor"),