From 5c2e7ed3f41352bd210055fbc1fd3d4fd001054d Mon Sep 17 00:00:00 2001 From: Matthew Charles Busel Date: Thu, 19 Nov 2026 17:20:52 -0500 Subject: [PATCH] fix: don't retry LLM calls the provider rejected (400/401/403/404/422) WriteCode.write_code, WriteCodeReview.write_code_review_and_rewrite and ActionNode._aask_v1 are decorated with @retry(stop=stop_after_attempt(6)) and no retry= filter, so any exception, a 400 context_length_exceeded or a 401 bad key included, re-sent the same prompt up to 6 times. Under them, acompletion_text retries connection errors 6 times and the OpenAI SDK twice more, so one bad request could become dozens of paid calls. Adds metagpt.utils.common.is_retryable_llm_error: an error carrying a 4xx status_code other than 408/409/429 is not retried; parse failures of the model's answer, timeouts, 429 and 5xx are retried as before. --- metagpt/actions/action_node.py | 14 ++- metagpt/actions/write_code.py | 19 ++++- metagpt/actions/write_code_review.py | 15 +++- metagpt/utils/common.py | 22 +++++ .../metagpt/utils/test_retryable_llm_error.py | 85 +++++++++++++++++++ 5 files changed, 147 insertions(+), 8 deletions(-) create mode 100644 tests/metagpt/utils/test_retryable_llm_error.py diff --git a/metagpt/actions/action_node.py b/metagpt/actions/action_node.py index 7109f287e2..8c83666738 100644 --- a/metagpt/actions/action_node.py +++ b/metagpt/actions/action_node.py @@ -15,7 +15,12 @@ from typing import Any, Dict, List, Optional, Tuple, Type, Union from pydantic import BaseModel, Field, create_model, model_validator -from tenacity import retry, stop_after_attempt, wait_random_exponential +from tenacity import ( + retry, + retry_if_exception, + stop_after_attempt, + wait_random_exponential, +) from metagpt.actions.action_outcls_registry import register_action_outcls from metagpt.const import MARKDOWN_TITLE_PREFIX, USE_CONFIG_TIMEOUT @@ -24,7 +29,11 @@ from metagpt.llm import BaseLLM from metagpt.logs import logger from metagpt.provider.postprocess.llm_output_postprocess import llm_output_postprocess -from metagpt.utils.common import OutputParser, general_after_log +from metagpt.utils.common import ( + OutputParser, + general_after_log, + is_retryable_llm_error, +) from metagpt.utils.human_interaction import HumanInteraction from metagpt.utils.sanitize import sanitize @@ -423,6 +432,7 @@ def compile(self, context, schema="json", mode="children", template=SIMPLE_TEMPL @retry( wait=wait_random_exponential(min=1, max=20), stop=stop_after_attempt(6), + retry=retry_if_exception(is_retryable_llm_error), after=general_after_log(logger), ) async def _aask_v1( diff --git a/metagpt/actions/write_code.py b/metagpt/actions/write_code.py index da25fe621c..e8e3257966 100644 --- a/metagpt/actions/write_code.py +++ b/metagpt/actions/write_code.py @@ -20,14 +20,23 @@ from typing import Optional from pydantic import BaseModel, Field -from tenacity import retry, stop_after_attempt, wait_random_exponential +from tenacity import ( + retry, + retry_if_exception, + stop_after_attempt, + wait_random_exponential, +) from metagpt.actions.action import Action from metagpt.actions.project_management_an import REFINED_TASK_LIST, TASK_LIST from metagpt.actions.write_code_plan_and_change_an import REFINED_TEMPLATE from metagpt.logs import logger from metagpt.schema import CodingContext, Document, RunCodeResult -from metagpt.utils.common import CodeParser, get_markdown_code_block_type +from metagpt.utils.common import ( + CodeParser, + get_markdown_code_block_type, + is_retryable_llm_error, +) from metagpt.utils.project_repo import ProjectRepo from metagpt.utils.report import EditorReporter @@ -91,7 +100,11 @@ class WriteCode(Action): repo: Optional[ProjectRepo] = Field(default=None, exclude=True) input_args: Optional[BaseModel] = Field(default=None, exclude=True) - @retry(wait=wait_random_exponential(min=1, max=60), stop=stop_after_attempt(6)) + @retry( + wait=wait_random_exponential(min=1, max=60), + stop=stop_after_attempt(6), + retry=retry_if_exception(is_retryable_llm_error), + ) async def write_code(self, prompt) -> str: code_rsp = await self._aask(prompt) code = CodeParser.parse_code(text=code_rsp) diff --git a/metagpt/actions/write_code_review.py b/metagpt/actions/write_code_review.py index 209e4e8ac4..36961dd437 100644 --- a/metagpt/actions/write_code_review.py +++ b/metagpt/actions/write_code_review.py @@ -13,14 +13,19 @@ from typing import Optional from pydantic import BaseModel, Field -from tenacity import retry, stop_after_attempt, wait_random_exponential +from tenacity import ( + retry, + retry_if_exception, + stop_after_attempt, + wait_random_exponential, +) from metagpt.actions import WriteCode from metagpt.actions.action import Action from metagpt.logs import logger from metagpt.schema import CodingContext, Document from metagpt.tools.tool_registry import register_tool -from metagpt.utils.common import CodeParser, aread, awrite +from metagpt.utils.common import CodeParser, aread, awrite, is_retryable_llm_error from metagpt.utils.project_repo import ProjectRepo from metagpt.utils.report import EditorReporter @@ -144,7 +149,11 @@ class WriteCodeReview(Action): repo: Optional[ProjectRepo] = Field(default=None, exclude=True) input_args: Optional[BaseModel] = Field(default=None, exclude=True) - @retry(wait=wait_random_exponential(min=1, max=60), stop=stop_after_attempt(6)) + @retry( + wait=wait_random_exponential(min=1, max=60), + stop=stop_after_attempt(6), + retry=retry_if_exception(is_retryable_llm_error), + ) async def write_code_review_and_rewrite(self, context_prompt, cr_prompt, doc): filename = doc.filename cr_rsp = await self._aask(context_prompt + cr_prompt) diff --git a/metagpt/utils/common.py b/metagpt/utils/common.py index cf2aa58bab..cd440f762d 100644 --- a/metagpt/utils/common.py +++ b/metagpt/utils/common.py @@ -536,6 +536,28 @@ def remove_affix(text, affix: Literal["brace", "url", "none"] = "brace"): return decoder(text) +# 4xx statuses worth sending again: request timeout, conflict, rate limit. +_RETRYABLE_CLIENT_STATUS = frozenset({408, 409, 429}) + + +def is_retryable_llm_error(exception: BaseException) -> bool: + """False for an API error the provider will return again for the same request. + + A 400 (context too long, bad parameter), 401, 403, 404 or 422 is rejected the same + way on every attempt, so a retry only pays for the prompt again. Every other error, + including parse failures of the model's answer and 429/5xx, stays retryable. + Works with any SDK whose errors carry an integer ``status_code`` (openai, anthropic, + httpx-based clients). + """ + status = getattr(exception, "status_code", None) + if not isinstance(status, int): + response = getattr(exception, "response", None) + status = getattr(response, "status_code", None) + if isinstance(status, int) and 400 <= status < 500: + return status in _RETRYABLE_CLIENT_STATUS + return True + + def general_after_log(i: "loguru.Logger", sec_format: str = "%0.3f") -> Callable[["RetryCallState"], None]: """ Generates a logging function to be used after a call is retried. diff --git a/tests/metagpt/utils/test_retryable_llm_error.py b/tests/metagpt/utils/test_retryable_llm_error.py new file mode 100644 index 0000000000..6d0e7ee73c --- /dev/null +++ b/tests/metagpt/utils/test_retryable_llm_error.py @@ -0,0 +1,85 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +"""A request the provider rejects as malformed or unauthorized is not sent again.""" +import httpx +import openai +import pytest +from tenacity import retry, retry_if_exception, stop_after_attempt, wait_none + +from metagpt.utils.common import is_retryable_llm_error + + +def _openai_error(cls, status: int): + request = httpx.Request("POST", "https://api.openai.com/v1/chat/completions") + return cls("rejected", response=httpx.Response(status, request=request), body=None) + + +@pytest.mark.parametrize( + ("cls", "status"), + [ + (openai.BadRequestError, 400), + (openai.AuthenticationError, 401), + (openai.PermissionDeniedError, 403), + (openai.NotFoundError, 404), + (openai.UnprocessableEntityError, 422), + ], +) +def test_client_errors_are_not_retryable(cls, status): + assert is_retryable_llm_error(_openai_error(cls, status)) is False + + +@pytest.mark.parametrize( + ("cls", "status"), + [(openai.RateLimitError, 429), (openai.InternalServerError, 500), (openai.ConflictError, 409)], +) +def test_transient_status_errors_are_retryable(cls, status): + assert is_retryable_llm_error(_openai_error(cls, status)) is True + + +@pytest.mark.parametrize( + "exc", + [ + ValueError("could not parse the code block"), + openai.APITimeoutError(request=httpx.Request("POST", "https://api.openai.com")), + KeyError("Code Review Result"), + ], +) +def test_parse_and_network_errors_stay_retryable(exc): + assert is_retryable_llm_error(exc) is True + + +def test_httpx_status_error_uses_response_status(): + request = httpx.Request("POST", "https://example.com") + exc = httpx.HTTPStatusError("bad", request=request, response=httpx.Response(401, request=request)) + assert is_retryable_llm_error(exc) is False + + +@pytest.mark.asyncio +async def test_bad_request_is_sent_once_under_the_action_retry_policy(): + calls = 0 + + @retry(wait=wait_none(), stop=stop_after_attempt(6), retry=retry_if_exception(is_retryable_llm_error)) + async def aask(): + nonlocal calls + calls += 1 + raise _openai_error(openai.BadRequestError, 400) + + with pytest.raises(openai.BadRequestError): + await aask() + assert calls == 1 + + +@pytest.mark.asyncio +async def test_parse_failure_is_still_retried_under_the_action_retry_policy(): + calls = 0 + + @retry(wait=wait_none(), stop=stop_after_attempt(6), retry=retry_if_exception(is_retryable_llm_error)) + async def aask(): + nonlocal calls + calls += 1 + if calls < 3: + raise ValueError("no code block") + return "ok" + + assert await aask() == "ok" + assert calls == 3