Add Llama4 text model

README.md

@@ -55,7 +55,7 @@ PaddleFormers 是基于百度深度学习框架 PaddlePaddle 搭建的 Transform
   <tbody>
     <!-- LLM 分类 - 跨行合并开始 -->
     <tr>
-      <td rowspan="15" style="vertical-align: top;">LLM</td>
+      <td rowspan="16" style="vertical-align: top;">LLM</td>
       <td>DeepSeekv3</td>
       <td>deepseek-ai/DeepSeek-V3-Base、deepseek-ai/DeepSeek-V3、deepseek-ai/DeepSeek-V3-0324</td>
       <td>deepseek3</td>
@@ -100,6 +100,11 @@ PaddleFormers 是基于百度深度学习框架 PaddlePaddle 搭建的 Transform
       <td>meta-llama/Meta-Llama-3-8B、meta-llama/Meta-Llama-3-8B-Instruct、meta-llama/Meta-Llama-3-70B、meta-llama/Meta-Llama-3-70B-Instruct、meta-llama/Llama-3.1-8B、meta-llama/Llama-3.1-8B-Instruct、meta-llama/Llama-3.1-70B、meta-llama/Llama-3.1-70B-Instruct、meta-llama/Llama-3.1-405B、meta-llama/Llama-3.1-405B-Instruct、meta-llama/Llama-3.2-1B、meta-llama/Llama-3.2-1B-Instruct、meta-llama/Llama-3.2-3B、meta-llama/Llama-3.2-3B-Instruct、meta-llama/Llama-3.3-70B-Instruct</td>
       <td>llama3</td>
     </tr>
+    <tr>
+      <td>Llama-4-Text</td>
+      <td>meta-llama/Llama-4-Scout-17B-16E、meta-llama/Llama-4-Maverick-17B-128E</td>
+      <td>llama4</td>
+    </tr>
     <tr>
       <td>OLMo2</td>
       <td>allenai/OLMo-2-0425-1B、allenai/OLMo-2-0425-1B-SFT、allenai/OLMo-2-0425-1B-DPO、allenai/OLMo-2-0425-1B-Instruct、allenai/OLMo-2-1124-7B、allenai/OLMo-2-1124-7B-SFT、allenai/OLMo-2-1124-7B-DPO、allenai/OLMo-2-1124-7B-Instruct、allenai/OLMo-2-1124-13B、allenai/OLMo-2-1124-13B-SFT、allenai/OLMo-2-1124-13B-DPO、allenai/OLMo-2-1124-13B-Instruct、allenai/OLMo-2-0325-32B、allenai/OLMo-2-0325-32B-SFT、allenai/OLMo-2-0325-32B-DPO、allenai/OLMo-2-0325-32B-Instruct</td>

docs/zh/model_capability.md

@@ -8,6 +8,7 @@
 |GPT-OSS|✓|✓|✓|x|x|
 |Granite|✓|✓|✓|✓|✓|
 |LLaMA3|✓|✓|✓|✓|✓|
+|LLaMA4-Text|✓|✓|✓|x|x|
 |OLMo2|✓|✓|✓|✓|✓|
 |Phi4|✓|✓|✓|✓|✓|
 |Qwen2|✓|✓|✓|✓|✓|
@@ -28,6 +29,7 @@
 |GPT-OSS|✓|✓|x|x|✓|✓|
 |Granite|✓|✓|-|x|✓|✓|
 |LLaMA3|✓|✓|-|x|✓|✓|
+|LLaMA4-Text|x|x|x|x|✓|✓|
 |OLMo2|✓|✓|-|x|✓|✓|
 |Phi4|✓|✓|-|x|✓|✓|
 |Qwen2|✓|✓|x|x|✓|✓|

paddleformers/cli/utils/llm_utils.py

@@ -77,7 +77,7 @@ def get_lora_target_modules(model):
         ]
     elif model.config.model_type == "bloom":
         target_modules = [".*query_key_value.*", ".*dense.*", ".*dense_h_to_4h.*", ".*dense_4h_to_h.*"]
-    elif model.config.model_type in ["llama", "jamba", "olmo2"]:
+    elif model.config.model_type in ["llama", "llama4_text", "jamba", "olmo2"]:
         target_modules = [
             ".*q_proj.*",
             ".*v_proj.*",

paddleformers/datasets/template/template.py

@@ -949,6 +949,27 @@ def _get_gpt_oss_prefix():
     chat_sep="<|eot_id|>",
 )
 
+register_template(
+    name="llama4",
+    format_user=StringFormatter(
+        slots=[
+            ("<|header_start|>user<|header_end|>\n\n{{content}}<|eot|>" "<|header_start|>assistant<|header_end|>\n\n")
+        ]
+    ),
+    format_assistant=StringFormatter(slots=["{{content}}"]),
+    format_system=StringFormatter(slots=["<|header_start|>system<|header_end|>\n\n{{content}}<|eot|>"]),
+    format_function=FunctionFormatter(slots=["{{content}}"], tool_format="llama4"),
+    format_observation=StringFormatter(
+        slots=[
+            ("<|header_start|>user<|header_end|>\n\n{{content}}<|eot|>" "<|header_start|>assistant<|header_end|>\n\n")
+        ]
+    ),
+    format_tools=ToolFormatter(tool_format="llama4"),
+    format_prefix=EmptyFormatter(slots=[{"bos_token"}]),
+    suffix=["<|eot|>"],
+    chat_sep="<|eot|>",
+)
+
 
 # copied from gemma template
 register_template(

paddleformers/datasets/template/tool_utils.py

@@ -90,6 +90,15 @@ class FunctionCall(NamedTuple):
     "Do not use variables.\n\n{tool_text}"
 )
 
+LLAMA4_TOOL_PROMPT = (
+    "You are a helpful assistant and an expert in function composition. You can answer general questions using your "
+    "internal knowledge OR invoke functions when necessary.\n"
+    "Here is a list of functions in JSON format that you can invoke:\n{tool_text}\n"
+    "Should you decide to return the function call(s), put them in the format of "
+    "[func1(param_name=param_value, ...), func2(...)].\n"
+    "Do not include any other text in the response when returning function calls."
+)
+
 
 @dataclass
 class ToolUtils(ABC):
@@ -288,6 +297,33 @@ def function_formatter(functions: list["FunctionCall"]) -> str:
         return json.dumps(function_objects[0] if len(function_objects) == 1 else function_objects, ensure_ascii=False)
 
 
+class Llama4ToolUtils(ToolUtils):
+    r"""Llama 4 zero-shot tool calling format.
+
+    Reference: https://github.com/meta-llama/llama-models/blob/main/models/llama4/prompt_format.md
+    """
+
+    @override
+    @staticmethod
+    def tool_formatter(tools: list[dict[str, Any]]) -> str:
+        normalized_tools = [tool.get("function", {}) if tool.get("type") == "function" else tool for tool in tools]
+        tool_text = json.dumps(normalized_tools, indent=4, ensure_ascii=False)
+        return LLAMA4_TOOL_PROMPT.format(tool_text=tool_text)
+
+    @override
+    @staticmethod
+    def function_formatter(functions: list["FunctionCall"]) -> str:
+        function_texts = []
+        for name, arguments in functions:
+            parsed_arguments = json.loads(arguments)
+            argument_text = ", ".join(
+                f"{key}={json.dumps(value, ensure_ascii=False)}" for key, value in parsed_arguments.items()
+            )
+            function_texts.append(f"{name}({argument_text})")
+
+        return "[" + ", ".join(function_texts) + "]"
+
+
 class ERNIEToolUtils(ToolUtils):
     r"""ERNIE 4.5 tool using template."""
 
@@ -346,6 +382,7 @@ def function_formatter(functions: list["FunctionCall"]) -> str:
     "glm4_moe": GLM4MOEToolUtils(),
     "glm_moe_dsa": GLM_5ToolUtils(),
     "llama3": Llama3ToolUtils(),
+    "llama4": Llama4ToolUtils(),
 }
 
 

paddleformers/transformers/__init__.py

@@ -217,6 +217,13 @@
     "llama.modeling": ["LlamaForCausalLM", "LlamaModel", "LlamaForCausalLMPipe", "LlamaRotaryEmbedding"],
     "llama.tokenizer": ["LlamaTokenizer", "Llama3Tokenizer"],
     "llama.tokenizer_fast": ["LlamaTokenizerFast"],
+    "llama4.configuration": ["Llama4TextConfig"],
+    "llama4.modeling": [
+        "Llama4ForCausalLM",
+        "Llama4TextDecoderLayer",
+        "Llama4TextModel",
+        "Llama4TextPretrainedModel",
+    ],
     "optimization": [
         "LinearDecayWithWarmup",
         "ConstScheduleWithWarmup",
@@ -328,6 +335,7 @@
         "Qwen3NextPretrainingCriterion",
     ],
     "llama": [],
+    "llama4": [],
     "qwen2": [],
     "glm_ocr": [],
     "qwen3": [],

paddleformers/transformers/auto/configuration.py

@@ -43,6 +43,7 @@
         ("ernie4_5_moe_vl", "Ernie4_5_VLConfig"),
         ("paddleocr_vl", "PaddleOCRVLConfig"),
         ("llama", "LlamaConfig"),
+        ("llama4_text", "Llama4TextConfig"),
         ("kimi_k2", "KimiK2Config"),
         ("qwen2", "Qwen2Config"),
         ("qwen2_5_vl", "Qwen2_5_VLConfig"),
@@ -95,6 +96,7 @@
         ("ernie4_5_moe_vl", "Ernie4_5_VLMoeForConditionalGeneration"),
         ("paddleocr_vl", "PaddleOCRVLForConditionalGeneration"),
         ("llama", "Llama"),
+        ("llama4_text", "Llama4TextModel"),
         ("qwen2", "Qwen2"),
         ("qwen2_5_vl", "Qwen2_5_VL"),
         ("qwen2_5_vl_text", "Qwen2_5_VL"),
@@ -132,6 +134,7 @@
 
 SPECIAL_MODEL_TYPE_TO_MODULE_NAME = OrderedDict(
     [
+        ("llama4_text", "llama4"),
         ("paligemma", "paligemma2"),
         ("gemma3", "gemma3"),
         ("qwen2_5_vl_text", "qwen2_5_vl"),

paddleformers/transformers/auto/modeling.py

@@ -62,6 +62,7 @@
         ("Ernie4_5_VLMoe", "ernie4_5_moe_vl"),
         ("PaddleOCRVL", "paddleocr_vl"),
         ("Llama", "llama"),
+        ("Llama4", "llama4"),
         ("KimiK2", "kimi_k2"),
         ("Qwen2", "qwen2"),
         ("Qwen2_5_VL", "qwen2_5_vl"),
@@ -93,7 +94,13 @@
     ]
 )
 
-MAPPING_SPACIAL_KEY = OrderedDict([("Gemma3", "Gemma3"), ("Ernie4_5_VLMoe", "Ernie4_5_VLMoeForConditionalGeneration")])
+MAPPING_SPACIAL_KEY = OrderedDict(
+    [
+        ("Gemma3", "Gemma3"),
+        ("Ernie4_5_VLMoe", "Ernie4_5_VLMoeForConditionalGeneration"),
+        ("Llama4", "Llama4Text"),
+    ]
+)
 CONFIGURATION_MODEL_MAPPING = OrderedDict([((), "Gemma3ForConditionalGeneration")])
 
 MAPPING_TASKS = OrderedDict(

paddleformers/transformers/llama4/__init__.py

@@ -0,0 +1,43 @@
+# Copyright (c) 2026 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import sys
+from typing import TYPE_CHECKING
+
+from ...utils.lazy_import import _LazyModule
+
+import_structure = {
+    "configuration": ["Llama4TextConfig"],
+    "modeling": [
+        "Llama4TextDecoderLayer",
+        "Llama4TextModel",
+        "Llama4ForCausalLM",
+        "Llama4TextPretrainedModel",
+    ],
+}
+
+if TYPE_CHECKING:
+    from .configuration import Llama4TextConfig
+    from .modeling import (
+        Llama4ForCausalLM,
+        Llama4TextDecoderLayer,
+        Llama4TextModel,
+        Llama4TextPretrainedModel,
+    )
+else:
+    sys.modules[__name__] = _LazyModule(
+        __name__,
+        globals()["__file__"],
+        import_structure,
+    )

paddleformers/transformers/llama4/configuration.py

@@ -0,0 +1,127 @@
+# Copyright (c) 2026 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from ..configuration_utils import PretrainedConfig
+from ..modeling_rope_utils import rope_config_validation, standardize_rope_params
+
+
+class Llama4TextConfig(PretrainedConfig):
+    model_type = "llama4_text"
+
+    def __init__(
+        self,
+        vocab_size=202048,
+        hidden_size=5120,
+        intermediate_size=8192,
+        intermediate_size_mlp=16384,
+        num_hidden_layers=48,
+        num_attention_heads=40,
+        num_key_value_heads=8,
+        head_dim=128,
+        hidden_act="silu",
+        max_position_embeddings=131072,
+        initializer_range=0.02,
+        rms_norm_eps=1e-5,
+        use_cache=True,
+        pad_token_id=None,
+        bos_token_id=1,
+        eos_token_id=2,
+        tie_word_embeddings=False,
+        attention_dropout=0.0,
+        attention_bias=False,
+        num_experts_per_tok=1,
+        num_local_experts=16,
+        interleave_moe_layer_step=1,
+        use_qk_norm=True,
+        output_router_logits=False,
+        router_aux_loss_coef=0.001,
+        router_jitter_noise=0.0,
+        no_rope_layer_interval=4,
+        attention_chunk_size=8192,
+        attn_temperature_tuning=True,
+        floor_scale=8192,
+        attn_scale=0.1,
+        fuse_rms_norm=False,
+        **kwargs,
+    ):
+        self.vocab_size = vocab_size
+        self.hidden_size = hidden_size
+        self.intermediate_size = intermediate_size
+        self.intermediate_size_mlp = intermediate_size_mlp
+        self.num_hidden_layers = num_hidden_layers
+        self.num_attention_heads = num_attention_heads
+        self.num_key_value_heads = num_key_value_heads
+        self.head_dim = head_dim
+        self.hidden_act = hidden_act
+        self.max_position_embeddings = max_position_embeddings
+        self.initializer_range = initializer_range
+        self.rms_norm_eps = rms_norm_eps
+        self.use_cache = use_cache
+        self.tie_word_embeddings = tie_word_embeddings
+        self.attention_dropout = attention_dropout
+        self.attention_bias = attention_bias
+        self.num_experts_per_tok = num_experts_per_tok
+        self.num_local_experts = num_local_experts
+        self.interleave_moe_layer_step = interleave_moe_layer_step
+        self.use_qk_norm = use_qk_norm
+        self.output_router_logits = output_router_logits
+        self.router_aux_loss_coef = router_aux_loss_coef
+        self.router_jitter_noise = router_jitter_noise
+        self.no_rope_layer_interval = no_rope_layer_interval
+        self.attention_chunk_size = attention_chunk_size
+        self.attn_temperature_tuning = attn_temperature_tuning
+        self.floor_scale = floor_scale
+        self.attn_scale = attn_scale
+        self.fuse_rms_norm = fuse_rms_norm
+
+        self.moe_layers = kwargs.pop("moe_layers", None)
+        if self.moe_layers is None:
+            self.moe_layers = list(
+                range(
+                    interleave_moe_layer_step - 1,
+                    num_hidden_layers,
+                    interleave_moe_layer_step,
+                )
+            )
+
+        self.no_rope_layers = kwargs.pop("no_rope_layers", None)
+        if not self.no_rope_layers:
+            self.no_rope_layers = [int((i + 1) % no_rope_layer_interval != 0) for i in range(num_hidden_layers)]
+
+        self.layer_types = kwargs.pop("layer_types", None)
+        if self.layer_types is None:
+            self.layer_types = [
+                "chunked_attention" if self.no_rope_layers[i] else "full_attention" for i in range(num_hidden_layers)
+            ]
+
+        self.rope_theta = kwargs.get("rope_theta", 500000.0)
+        self.rope_scaling = kwargs.pop("rope_scaling", None)
+        rope_parameters = kwargs.pop("rope_parameters", None)
+        if rope_parameters is None:
+            rope_parameters = self.rope_scaling.copy() if isinstance(self.rope_scaling, dict) else {}
+            rope_parameters.setdefault("rope_type", rope_parameters.pop("type", "default"))
+            rope_parameters.setdefault("rope_theta", self.rope_theta)
+        self.rope_parameters = rope_parameters
+        standardize_rope_params(self, rope_theta=self.rope_theta)
+        rope_config_validation(self)
+
+        kwargs.setdefault("_attn_implementation", "eager")
+
+        super().__init__(
+            pad_token_id=pad_token_id,
+            bos_token_id=bos_token_id,
+            eos_token_id=eos_token_id,
+            tie_word_embeddings=tie_word_embeddings,
+            **kwargs,
+        )