feat(slack): Reply to @-mentions when Seer runs out of credits

src/sentry/seer/entrypoints/slack/entrypoint.py

@@ -21,6 +21,7 @@
 from sentry.notifications.platform.service import NotificationService
 from sentry.notifications.platform.templates.seer import (
     SeerAgentError,
+    SeerAgentOutOfCredits,
     SeerAgentResponse,
     SeerAgentWriteApproval,
     SeerAutofixError,
@@ -737,6 +738,22 @@ def on_agent_update(
                     },
                 )
 
+        for pr in _get_repo_pr_states(state):
+            if not _is_pr_out_of_credits(pr):
+                continue
+            key = f"pr_out_of_credits:{pr.repo_name}:{pr.pr_number}:{pr.commit_sha}"
+            if key in replies:
+                continue
+            message_ts = send_thread_update(
+                install=install,
+                thread=thread,
+                data=SeerAgentOutOfCredits(organization_id=organization_id),
+            )
+            if message_ts:
+                SlackSeerAgentReplyCache.set(
+                    run_id=run_id, key=key, reply=SlackSeerAgentReply(message_ts=message_ts)
+                )
+
         replies = SlackSeerAgentReplyCache.get(run_id=run_id)
         answers = [answer for turn in state.get_turns() if (answer := turn.answer) is not None]
         latest = next(
@@ -786,6 +803,10 @@ def _get_repo_pr_states(state: SeerRunState) -> list[RepoPRState]:
     return [state.repo_pr_states[repo] for repo in sorted(state.repo_pr_states)]
 
 
+def _is_pr_out_of_credits(pr: RepoPRState) -> bool:
+    return pr.pr_creation_status == "error" and pr.pr_creation_error_code == OUT_OF_CREDITS
+
+
 def _hash_pr_status(prs: Sequence[RepoPRState]) -> str:
     payload = orjson.dumps(
         [
@@ -804,7 +825,7 @@ def _next_agent_reply(
     cache_payload: SlackAgentCachePayload,
     state: SeerRunState,
     replies: Mapping[str, SlackSeerAgentReply],
-) -> tuple[str, SeerAgentError | SeerAgentResponse] | None:
+) -> tuple[str, SeerAgentError | SeerAgentResponse | SeerAgentOutOfCredits] | None:
     """The reply to post and its cache key, or None if the thread is up to date.
 
     Sends and records the write approval DM when a new approval is pending.
@@ -861,6 +882,13 @@ def _next_agent_reply(
             "I sent you a message."
         )
         repo_pr_states: list[RepoPRState] = []
+    elif state.failure_reason == OUT_OF_CREDITS:
+        # Seer ends the turn with its own plain-text notice; reply with the agent out-of-credits
+        # notification instead. Keyed on the last block so each stopped turn replies once.
+        key = f"out_of_credits:{state.blocks[-1].id}" if state.blocks else "out_of_credits"
+        if key in replies:
+            return None
+        return key, SeerAgentOutOfCredits(organization_id=organization_id)
     else:
         turn = next((turn for turn in reversed(state.get_turns()) if turn.answer), None)
         if turn is None or turn.answer is None:

src/sentry/seer/entrypoints/slack/tasks.py

@@ -14,6 +14,7 @@
 from sentry.integrations.services.integration.service import integration_service
 from sentry.models.organization import Organization
 from sentry.notifications.platform.slack.provider import SlackRenderable
+from sentry.notifications.platform.templates.seer import SeerAgentOutOfCredits
 from sentry.seer.credits import SeerOperation, can_run_seer_operation
 from sentry.seer.entrypoints.metrics import (
     SlackEntrypointEventLifecycleMetric,
@@ -25,7 +26,11 @@
     SlackSeerAgentResponded,
 )
 from sentry.seer.entrypoints.slack.cache import SlackSeerAgentMessageCache
-from sentry.seer.entrypoints.slack.entrypoint import EntrypointSetupError, SlackAgentEntrypoint
+from sentry.seer.entrypoints.slack.entrypoint import (
+    EntrypointSetupError,
+    SlackAgentEntrypoint,
+    SlackThreadDetails,
+)
 from sentry.seer.entrypoints.slack.mention import (
     SlackMessageLink,
     build_linked_messages_context,
@@ -37,6 +42,7 @@
 from sentry.seer.entrypoints.slack.messaging import (
     send_halt_message,
     send_not_org_member_message,
+    send_thread_update,
 )
 from sentry.seer.entrypoints.slack.metrics import (
     ProcessMentionFailureReason,
@@ -161,9 +167,15 @@ def process_mention_for_slack(
             )
             return
 
-        # Check credits before fetching context or starting a run.
+        # Check credits before fetching context or starting a run, so an org that can't pay
+        # gets an answer right away instead of a run that fails partway through.
         if not can_run_seer_operation(organization, SeerOperation.AGENT_CHAT_MESSAGE):
             lifecycle.record_halt(ProcessMentionHaltReason.OUT_OF_SEER_CREDITS)
+            _send_out_of_credits_message(
+                entrypoint=entrypoint,
+                organization=organization,
+                thread_ts=thread_ts or ts,
+            )
             # The webhook set "is thinking..." before queueing this task; clear it.
             entrypoint.install.set_thread_status(
                 channel_id=channel_id,
@@ -406,6 +418,24 @@ def _resolve_linked_messages(
     )
 
 
+def _send_out_of_credits_message(
+    *,
+    entrypoint: SlackAgentEntrypoint,
+    organization: Organization,
+    thread_ts: str,
+) -> None:
+    """Tell the user, in the thread and only to them, that the org is out of Seer credits."""
+    try:
+        send_thread_update(
+            install=entrypoint.install,
+            thread=SlackThreadDetails(thread_ts=thread_ts, channel_id=entrypoint.channel_id),
+            data=SeerAgentOutOfCredits(organization_id=organization.id),
+            ephemeral_user_id=entrypoint.slack_user_id,
+        )
+    except Exception as e:
+        _logger.warning("seer.slack.process_mention.send_out_of_credits_failed", exc_info=e)
+
+
 def _send_inaccessible_links_prompt(
     *,
     entrypoint: SlackAgentEntrypoint,

tests/sentry/seer/entrypoints/slack/test_slack.py

@@ -26,6 +26,7 @@
 from sentry.notifications.platform.templates.seer import (
     AUTOFIX_OUT_OF_CREDITS_MESSAGE,
     SeerAgentError,
+    SeerAgentOutOfCredits,
     SeerAgentResponse,
     SeerAgentWriteApproval,
     SeerAutofixUpdate,
@@ -90,6 +91,11 @@
 from sentry.testutils.helpers.options import override_options
 from sentry.utils.locking import UnableToAcquireLock
 
+SEER_OUT_OF_CREDITS_TEXT = (
+    "Your organization doesn't have enough Seer credits for this, so I stopped here. "
+    "Add credits to continue."
+)
+
 
 class SlackAutofixEntrypointTest(TestCase):
     def setUp(self) -> None:
@@ -1381,6 +1387,58 @@ def test_on_agent_update_missing_scope_url_cached_per_thread(
         second_data = mock_send.call_args.kwargs["data"]
         assert second_data.missing_scope_settings_url is None
 
+    @patch("sentry.seer.entrypoints.slack.entrypoint.send_thread_update", return_value="1.1")
+    def test_on_agent_update_out_of_credits_replies_once(self, mock_send: MagicMock) -> None:
+        cache_payload = self._get_entrypoint().create_agent_cache_payload()
+        state = _agent_state(SEER_OUT_OF_CREDITS_TEXT)
+        state.status = "error"
+        state.failure_reason = "out_of_credits"
+
+        SlackAgentEntrypoint.on_agent_update(cache_payload=cache_payload, run_id=12345, state=state)
+        SlackAgentEntrypoint.on_agent_update(cache_payload=cache_payload, run_id=12345, state=state)
+
+        mock_send.assert_called_once()
+        data = mock_send.call_args.kwargs["data"]
+        assert isinstance(data, SeerAgentOutOfCredits)
+
+    @patch("sentry.seer.entrypoints.slack.entrypoint.send_thread_update", return_value="1.1")
+    def test_on_agent_update_pr_out_of_credits_posts_notice_once(
+        self, mock_send: MagicMock
+    ) -> None:
+        cache_payload = self._get_entrypoint().create_agent_cache_payload()
+        state = _agent_state("I tried to open a PR.")
+        pr = RepoPRState(
+            repo_name="acme/web",
+            pr_creation_status="error",
+            pr_creation_error_code="out_of_credits",
+        )
+        state.repo_pr_states[pr.repo_name] = pr
+
+        SlackAgentEntrypoint.on_agent_update(cache_payload=cache_payload, run_id=12345, state=state)
+        SlackAgentEntrypoint.on_agent_update(cache_payload=cache_payload, run_id=12345, state=state)
+
+        sent = [call.kwargs["data"] for call in mock_send.call_args_list]
+        assert len(sent) == 2
+        answer, notice = sent
+        assert isinstance(answer, SeerAgentResponse)
+        assert answer.summary == "I tried to open a PR."
+        assert answer.repo_pr_states == [pr]
+        assert isinstance(notice, SeerAgentOutOfCredits)
+
+    @patch("sentry.seer.entrypoints.slack.entrypoint.send_thread_update", return_value="1.1")
+    def test_on_agent_update_pr_error_without_credits_posts_no_notice(
+        self, mock_send: MagicMock
+    ) -> None:
+        cache_payload = self._get_entrypoint().create_agent_cache_payload()
+        state = _agent_state("I tried to open a PR.")
+        pr = RepoPRState(repo_name="acme/web", pr_creation_status="error")
+        state.repo_pr_states[pr.repo_name] = pr
+
+        SlackAgentEntrypoint.on_agent_update(cache_payload=cache_payload, run_id=12345, state=state)
+
+        mock_send.assert_called_once()
+        assert isinstance(mock_send.call_args.kwargs["data"], SeerAgentResponse)
+
     @patch("sentry.seer.entrypoints.slack.entrypoint.send_thread_update", return_value="1.1")
     def test_on_agent_update_does_not_duplicate_answers(self, mock_send: MagicMock) -> None:
         cache_payload = self._get_entrypoint().create_agent_cache_payload()

tests/sentry/seer/entrypoints/slack/test_tasks.py

@@ -8,6 +8,7 @@
 from django.test import override_settings
 
 from sentry.integrations.messaging.metrics import SeerSlackHaltReason
+from sentry.notifications.platform.templates.seer import AGENT_OUT_OF_CREDITS_MESSAGE
 from sentry.seer.credits import SeerOperation
 from sentry.seer.entrypoints.slack.analytics import (
     SlackSeerAgentConversation,
@@ -26,6 +27,7 @@
     process_mention_for_slack,
     process_reaction_for_slack,
 )
+from sentry.shared_integrations.exceptions import IntegrationError
 from sentry.testutils.asserts import assert_failure_metric, assert_halt_metric
 from sentry.testutils.cases import TestCase
 from sentry.testutils.helpers.analytics import (
@@ -416,12 +418,21 @@ def _run_task(
         mock_agent_cls.return_value = self.mock_entrypoint
         process_mention_for_slack(**{**TASK_KWARGS, "organization_id": self.organization.id})
 
+    def _sent_text(self) -> str:
+        send = self.mock_entrypoint.install.send_threaded_ephemeral_message
+        send.assert_called_once()
+        call_kwargs = send.call_args.kwargs
+        assert call_kwargs["slack_user_id"] == self.mock_entrypoint.slack_user_id
+        assert call_kwargs["channel_id"] == self.mock_entrypoint.channel_id
+        assert call_kwargs["thread_ts"] == "1234567890.123456"
+        return call_kwargs["renderable"]["text"]
+
     @patch("sentry.integrations.utils.metrics.EventLifecycle.record_event")
     @patch("sentry.seer.entrypoints.slack.tasks.can_run_seer_operation", return_value=False)
     @patch("sentry.seer.entrypoints.slack.tasks.SeerAgentOperator")
     @patch("sentry.seer.entrypoints.slack.tasks.SlackAgentEntrypoint")
     @patch("sentry.seer.entrypoints.slack.tasks._resolve_user")
-    def test_out_of_credits_halts_without_running(
+    def test_out_of_credits_replies_instead_of_running(
         self,
         mock_resolve_user: MagicMock,
         mock_agent_cls: MagicMock,
@@ -438,13 +449,43 @@ def test_out_of_credits_halts_without_running(
         mock_can_run.assert_called_once_with(self.organization, SeerOperation.AGENT_CHAT_MESSAGE)
         mock_operator_cls.assert_not_called()
         self.mock_entrypoint.install.get_thread_history.assert_not_called()
+
+        assert self._sent_text() == AGENT_OUT_OF_CREDITS_MESSAGE
+
         self.mock_entrypoint.install.set_thread_status.assert_called_once_with(
             channel_id="C1234567890",
             thread_ts="1234567890.123456",
             status="",
         )
         assert_halt_metric(mock_record, ProcessMentionHaltReason.OUT_OF_SEER_CREDITS)
 
+    @patch("sentry.seer.entrypoints.slack.tasks.can_run_seer_operation", return_value=False)
+    @patch("sentry.seer.entrypoints.slack.tasks.SeerAgentOperator")
+    @patch("sentry.seer.entrypoints.slack.tasks.SlackAgentEntrypoint")
+    @patch("sentry.seer.entrypoints.slack.tasks._resolve_user")
+    def test_out_of_credits_still_clears_status_when_reply_fails(
+        self,
+        mock_resolve_user: MagicMock,
+        mock_agent_cls: MagicMock,
+        mock_operator_cls: MagicMock,
+        mock_can_run: MagicMock,
+    ) -> None:
+        mock_operator_cls.has_access.return_value = True
+        self.mock_entrypoint.install.send_threaded_ephemeral_message.side_effect = IntegrationError(
+            "slack down"
+        )
+
+        self._run_task(
+            user_id=self.user.id, mock_agent_cls=mock_agent_cls, mock_resolve_user=mock_resolve_user
+        )
+
+        mock_operator_cls.assert_not_called()
+        self.mock_entrypoint.install.set_thread_status.assert_called_once_with(
+            channel_id="C1234567890",
+            thread_ts="1234567890.123456",
+            status="",
+        )
+
     @patch("sentry.seer.credits.seer_credit.backend.check_seer_credits")
     @patch("sentry.seer.entrypoints.slack.tasks.SeerAgentOperator")
     @patch("sentry.seer.entrypoints.slack.tasks.SlackAgentEntrypoint")
@@ -471,6 +512,7 @@ def test_credit_check_flag_on_with_enough_credits_runs(
 
         mock_check.assert_called_once()
         mock_operator_cls.return_value.trigger_agent.assert_called_once()
+        self.mock_entrypoint.install.send_threaded_ephemeral_message.assert_not_called()
 
     @patch("sentry.seer.credits.seer_credit.backend.check_seer_credits")
     @patch("sentry.seer.entrypoints.slack.tasks.SeerAgentOperator")
@@ -495,8 +537,8 @@ def test_credit_check_flag_on_without_enough_credits_halts(
                 mock_resolve_user=mock_resolve_user,
             )
 
-        mock_check.assert_called_once()
         mock_operator_cls.assert_not_called()
+        assert self._sent_text() == AGENT_OUT_OF_CREDITS_MESSAGE
 
 
 class LinkedMessagesContextTest(TestCase):