Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 12 additions & 2 deletions metagpt/actions/action_node.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,12 @@
from typing import Any, Dict, List, Optional, Tuple, Type, Union

from pydantic import BaseModel, Field, create_model, model_validator
from tenacity import retry, stop_after_attempt, wait_random_exponential
from tenacity import (
retry,
retry_if_exception,
stop_after_attempt,
wait_random_exponential,
)

from metagpt.actions.action_outcls_registry import register_action_outcls
from metagpt.const import MARKDOWN_TITLE_PREFIX, USE_CONFIG_TIMEOUT
Expand All @@ -24,7 +29,11 @@
from metagpt.llm import BaseLLM
from metagpt.logs import logger
from metagpt.provider.postprocess.llm_output_postprocess import llm_output_postprocess
from metagpt.utils.common import OutputParser, general_after_log
from metagpt.utils.common import (
OutputParser,
general_after_log,
is_retryable_llm_error,
)
from metagpt.utils.human_interaction import HumanInteraction
from metagpt.utils.sanitize import sanitize

Expand Down Expand Up @@ -423,6 +432,7 @@ def compile(self, context, schema="json", mode="children", template=SIMPLE_TEMPL
@retry(
wait=wait_random_exponential(min=1, max=20),
stop=stop_after_attempt(6),
retry=retry_if_exception(is_retryable_llm_error),
after=general_after_log(logger),
)
async def _aask_v1(
Expand Down
19 changes: 16 additions & 3 deletions metagpt/actions/write_code.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,14 +20,23 @@
from typing import Optional

from pydantic import BaseModel, Field
from tenacity import retry, stop_after_attempt, wait_random_exponential
from tenacity import (
retry,
retry_if_exception,
stop_after_attempt,
wait_random_exponential,
)

from metagpt.actions.action import Action
from metagpt.actions.project_management_an import REFINED_TASK_LIST, TASK_LIST
from metagpt.actions.write_code_plan_and_change_an import REFINED_TEMPLATE
from metagpt.logs import logger
from metagpt.schema import CodingContext, Document, RunCodeResult
from metagpt.utils.common import CodeParser, get_markdown_code_block_type
from metagpt.utils.common import (
CodeParser,
get_markdown_code_block_type,
is_retryable_llm_error,
)
from metagpt.utils.project_repo import ProjectRepo
from metagpt.utils.report import EditorReporter

Expand Down Expand Up @@ -91,7 +100,11 @@ class WriteCode(Action):
repo: Optional[ProjectRepo] = Field(default=None, exclude=True)
input_args: Optional[BaseModel] = Field(default=None, exclude=True)

@retry(wait=wait_random_exponential(min=1, max=60), stop=stop_after_attempt(6))
@retry(
wait=wait_random_exponential(min=1, max=60),
stop=stop_after_attempt(6),
retry=retry_if_exception(is_retryable_llm_error),
)
async def write_code(self, prompt) -> str:
code_rsp = await self._aask(prompt)
code = CodeParser.parse_code(text=code_rsp)
Expand Down
15 changes: 12 additions & 3 deletions metagpt/actions/write_code_review.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,14 +13,19 @@
from typing import Optional

from pydantic import BaseModel, Field
from tenacity import retry, stop_after_attempt, wait_random_exponential
from tenacity import (
retry,
retry_if_exception,
stop_after_attempt,
wait_random_exponential,
)

from metagpt.actions import WriteCode
from metagpt.actions.action import Action
from metagpt.logs import logger
from metagpt.schema import CodingContext, Document
from metagpt.tools.tool_registry import register_tool
from metagpt.utils.common import CodeParser, aread, awrite
from metagpt.utils.common import CodeParser, aread, awrite, is_retryable_llm_error
from metagpt.utils.project_repo import ProjectRepo
from metagpt.utils.report import EditorReporter

Expand Down Expand Up @@ -144,7 +149,11 @@ class WriteCodeReview(Action):
repo: Optional[ProjectRepo] = Field(default=None, exclude=True)
input_args: Optional[BaseModel] = Field(default=None, exclude=True)

@retry(wait=wait_random_exponential(min=1, max=60), stop=stop_after_attempt(6))
@retry(
wait=wait_random_exponential(min=1, max=60),
stop=stop_after_attempt(6),
retry=retry_if_exception(is_retryable_llm_error),
)
async def write_code_review_and_rewrite(self, context_prompt, cr_prompt, doc):
filename = doc.filename
cr_rsp = await self._aask(context_prompt + cr_prompt)
Expand Down
22 changes: 22 additions & 0 deletions metagpt/utils/common.py
Original file line number Diff line number Diff line change
Expand Up @@ -536,6 +536,28 @@ def remove_affix(text, affix: Literal["brace", "url", "none"] = "brace"):
return decoder(text)


# 4xx statuses worth sending again: request timeout, conflict, rate limit.
_RETRYABLE_CLIENT_STATUS = frozenset({408, 409, 429})


def is_retryable_llm_error(exception: BaseException) -> bool:
"""False for an API error the provider will return again for the same request.

A 400 (context too long, bad parameter), 401, 403, 404 or 422 is rejected the same
way on every attempt, so a retry only pays for the prompt again. Every other error,
including parse failures of the model's answer and 429/5xx, stays retryable.
Works with any SDK whose errors carry an integer ``status_code`` (openai, anthropic,
httpx-based clients).
"""
status = getattr(exception, "status_code", None)
if not isinstance(status, int):
response = getattr(exception, "response", None)
status = getattr(response, "status_code", None)
if isinstance(status, int) and 400 <= status < 500:
return status in _RETRYABLE_CLIENT_STATUS
return True


def general_after_log(i: "loguru.Logger", sec_format: str = "%0.3f") -> Callable[["RetryCallState"], None]:
"""
Generates a logging function to be used after a call is retried.
Expand Down
85 changes: 85 additions & 0 deletions tests/metagpt/utils/test_retryable_llm_error.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,85 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""A request the provider rejects as malformed or unauthorized is not sent again."""
import httpx
import openai
import pytest
from tenacity import retry, retry_if_exception, stop_after_attempt, wait_none

from metagpt.utils.common import is_retryable_llm_error


def _openai_error(cls, status: int):
request = httpx.Request("POST", "https://api.openai.com/v1/chat/completions")
return cls("rejected", response=httpx.Response(status, request=request), body=None)


@pytest.mark.parametrize(
("cls", "status"),
[
(openai.BadRequestError, 400),
(openai.AuthenticationError, 401),
(openai.PermissionDeniedError, 403),
(openai.NotFoundError, 404),
(openai.UnprocessableEntityError, 422),
],
)
def test_client_errors_are_not_retryable(cls, status):
assert is_retryable_llm_error(_openai_error(cls, status)) is False


@pytest.mark.parametrize(
("cls", "status"),
[(openai.RateLimitError, 429), (openai.InternalServerError, 500), (openai.ConflictError, 409)],
)
def test_transient_status_errors_are_retryable(cls, status):
assert is_retryable_llm_error(_openai_error(cls, status)) is True


@pytest.mark.parametrize(
"exc",
[
ValueError("could not parse the code block"),
openai.APITimeoutError(request=httpx.Request("POST", "https://api.openai.com")),
KeyError("Code Review Result"),
],
)
def test_parse_and_network_errors_stay_retryable(exc):
assert is_retryable_llm_error(exc) is True


def test_httpx_status_error_uses_response_status():
request = httpx.Request("POST", "https://example.com")
exc = httpx.HTTPStatusError("bad", request=request, response=httpx.Response(401, request=request))
assert is_retryable_llm_error(exc) is False


@pytest.mark.asyncio
async def test_bad_request_is_sent_once_under_the_action_retry_policy():
calls = 0

@retry(wait=wait_none(), stop=stop_after_attempt(6), retry=retry_if_exception(is_retryable_llm_error))
async def aask():
nonlocal calls
calls += 1
raise _openai_error(openai.BadRequestError, 400)

with pytest.raises(openai.BadRequestError):
await aask()
assert calls == 1


@pytest.mark.asyncio
async def test_parse_failure_is_still_retried_under_the_action_retry_policy():
calls = 0

@retry(wait=wait_none(), stop=stop_after_attempt(6), retry=retry_if_exception(is_retryable_llm_error))
async def aask():
nonlocal calls
calls += 1
if calls < 3:
raise ValueError("no code block")
return "ok"

assert await aask() == "ok"
assert calls == 3
Loading