From cbdde3a0098a58a2743926f9d2ff063e214802d2 Mon Sep 17 00:00:00 2001 From: srest2021 Date: Fri, 6 Nov 2026 10:42:25 -0800 Subject: [PATCH 01/25] feat(seer): Gate agent chat messages on Seer credits Return a 402 before sending a chat message when a token-billed org does not have enough Seer credits. Dashboard generation runs are not gated. can_run_seer_operation now passes orgs that are not token-billed. Refs CW-3061 --- src/sentry/seer/credits.py | 14 +++- .../endpoints/organization_seer_agent_chat.py | 18 +++- .../test_organization_seer_agent_chat.py | 84 +++++++++++++++++++ .../test_organization_seer_credit_check.py | 4 +- tests/sentry/seer/test_credits.py | 4 +- 5 files changed, 115 insertions(+), 9 deletions(-) diff --git a/src/sentry/seer/credits.py b/src/sentry/seer/credits.py index 6908b5ba4d85..5b47405b5243 100644 --- a/src/sentry/seer/credits.py +++ b/src/sentry/seer/credits.py @@ -60,8 +60,11 @@ class SeerOperationConfig: } -def can_run_seer_operation(organization: Organization, operation: SeerOperation) -> bool: - """Whether the organization has enough Seer credits to run the operation.""" +def has_enough_seer_credits(organization: Organization, operation: SeerOperation) -> bool: + """ + Checks whether the organization has enough Seer credits to run the operation + for callers that already checked the org is token-billed. + """ if not features.has("organizations:seer-credit-checks", organization): return True @@ -75,3 +78,10 @@ def can_run_seer_operation(organization: Organization, operation: SeerOperation) ) ) return response.has_enough_credits + + +def can_run_seer_operation(organization: Organization, operation: SeerOperation) -> bool: + """Whether the organization is token-billed and has enough Seer credits to run the operation.""" + return not features.has( + "organizations:token-billed-seer-enabled", organization + ) or has_enough_seer_credits(organization, operation) diff --git a/src/sentry/seer/endpoints/organization_seer_agent_chat.py b/src/sentry/seer/endpoints/organization_seer_agent_chat.py index ae787654d804..269705ba3b88 100644 --- a/src/sentry/seer/endpoints/organization_seer_agent_chat.py +++ b/src/sentry/seer/endpoints/organization_seer_agent_chat.py @@ -28,6 +28,7 @@ has_seer_agent_access_with_detail, snapshot_to_markdown, ) +from sentry.seer.credits import SeerOperation, can_run_seer_operation from sentry.seer.endpoints.utils import ResolvedSeerRun, resolve_seer_run from sentry.seer.models import SeerApiError, SeerPermissionError from sentry.seer.models.run import SeerAgentRun @@ -356,10 +357,21 @@ def post( return result resolved = result - if not has_access and not ( - resolved is not None and _is_dashboard_generate_run(organization, resolved) - ): + is_dashboard_generate_run = resolved is not None and _is_dashboard_generate_run( + organization, resolved + ) + if not has_access and not is_dashboard_generate_run: raise PermissionDenied(error) + # Do not bill for dashboard generation. + if not is_dashboard_generate_run and not can_run_seer_operation( + organization, SeerOperation.AGENT_CHAT_MESSAGE + ): + return Response( + { + "detail": "Your organization does not have enough credits left for you to message Seer." + }, + status=402, + ) try: # Seer only exposes write_pull_request when coding and bash mode are also on. diff --git a/tests/sentry/seer/endpoints/test_organization_seer_agent_chat.py b/tests/sentry/seer/endpoints/test_organization_seer_agent_chat.py index f9cf4b9eadf8..1b33d749ef45 100644 --- a/tests/sentry/seer/endpoints/test_organization_seer_agent_chat.py +++ b/tests/sentry/seer/endpoints/test_organization_seer_agent_chat.py @@ -4,6 +4,9 @@ import pytest from django.test import override_settings +from sentry_protos.billing.v1.services.seer_feature_billing.v1.endpoint_check_seer_credits_pb2 import ( + CheckSeerCreditsResponse, +) from sentry.seer.agent.client_models import ( MemoryBlock, @@ -493,6 +496,87 @@ def test_post_continue_with_uuid_resolves(self, mock_client_class: MagicMock) -> assert response.data == {"run_id": 555, "sentry_run_id": str(run.uuid)} assert mock_client.continue_run.call_args.kwargs["run_id"] == 555 + @with_feature(["organizations:seer-credit-checks", "organizations:token-billed-seer-enabled"]) + @patch( + "sentry.seer_credit.backend.check_seer_credits", + return_value=CheckSeerCreditsResponse(has_enough_credits=False), + ) + @patch("sentry.seer.endpoints.organization_seer_agent_chat.SeerAgentClient") + def test_post_new_conversation_without_credits_returns_402( + self, mock_client_class: MagicMock, mock_check_seer_credits: MagicMock + ) -> None: + response = self.client.post(self.url, {"query": "What is this error about?"}, format="json") + + assert response.status_code == 402 + assert response.data == { + "detail": "Your organization does not have enough credits left for you to message Seer." + } + mock_client_class.assert_not_called() + + @with_feature(["organizations:seer-credit-checks", "organizations:token-billed-seer-enabled"]) + @patch( + "sentry.seer_credit.backend.check_seer_credits", + return_value=CheckSeerCreditsResponse(has_enough_credits=False), + ) + @patch("sentry.seer.endpoints.organization_seer_agent_chat.SeerAgentClient") + def test_post_continue_without_credits_returns_402( + self, mock_client_class: MagicMock, mock_check_seer_credits: MagicMock + ) -> None: + run = self.create_seer_run( + organization=self.organization, seer_run_state_id=555, user_id=self.user.id + ) + + response = self.client.post(f"{self.url}{run.uuid}/", {"query": "More"}, format="json") + + assert response.status_code == 402 + assert response.data == { + "detail": "Your organization does not have enough credits left for you to message Seer." + } + mock_client_class.assert_not_called() + + @with_feature(["organizations:seer-credit-checks", "organizations:token-billed-seer-enabled"]) + @patch( + "sentry.seer_credit.backend.check_seer_credits", + return_value=CheckSeerCreditsResponse(has_enough_credits=False), + ) + @patch("sentry.seer.endpoints.organization_seer_agent_chat.SeerAgentClient") + def test_post_continue_dashboard_run_without_credits_calls_client( + self, mock_client_class: MagicMock, mock_check_seer_credits: MagicMock + ) -> None: + """Dashboard generation isn't gated on credits, so its follow-ups aren't either.""" + run = self.create_seer_run( + organization=self.organization, seer_run_state_id=789, user_id=self.user.id + ) + self.create_seer_agent_run(run=run, source="dashboard_generate") + mock_client = MagicMock() + mock_client.continue_run.return_value = 789 + mock_client_class.return_value = mock_client + + response = self.client.post( + f"{self.url}{run.seer_run_state_id}/", {"query": "Follow up question"}, format="json" + ) + + assert response.status_code == 200 + assert response.data == {"run_id": 789, "sentry_run_id": str(run.uuid)} + + @with_feature(["organizations:seer-credit-checks", "organizations:token-billed-seer-enabled"]) + @patch( + "sentry.seer_credit.backend.check_seer_credits", + return_value=CheckSeerCreditsResponse(has_enough_credits=True), + ) + @patch("sentry.seer.endpoints.organization_seer_agent_chat.SeerAgentClient") + def test_post_with_enough_credits_calls_client( + self, mock_client_class: MagicMock, mock_check_seer_credits: MagicMock + ) -> None: + mock_client = MagicMock() + mock_client.start_run.return_value = MagicMock(seer_run_state_id=456, uuid=uuid.uuid4()) + mock_client_class.return_value = mock_client + + response = self.client.post(self.url, {"query": "What is this error about?"}, format="json") + + assert response.status_code == 200 + mock_client.start_run.assert_called_once() + @patch("sentry.seer.endpoints.organization_seer_agent_chat.SeerAgentClient") def test_post_continuing_another_users_conversation_is_denied( self, mock_client_class: MagicMock diff --git a/tests/sentry/seer/endpoints/test_organization_seer_credit_check.py b/tests/sentry/seer/endpoints/test_organization_seer_credit_check.py index 199a106443b3..90aaf513233d 100644 --- a/tests/sentry/seer/endpoints/test_organization_seer_credit_check.py +++ b/tests/sentry/seer/endpoints/test_organization_seer_credit_check.py @@ -19,7 +19,7 @@ def setUp(self) -> None: super().setUp() self.login_as(user=self.user) - @with_feature("organizations:seer-credit-checks") + @with_feature(["organizations:seer-credit-checks", "organizations:token-billed-seer-enabled"]) @patch( "sentry.seer_credit.backend.check_seer_credits", return_value=CheckSeerCreditsResponse(has_enough_credits=True), @@ -39,7 +39,7 @@ def test_get_returns_has_enough_credits_when_billing_has_credits( ) ) - @with_feature("organizations:seer-credit-checks") + @with_feature(["organizations:seer-credit-checks", "organizations:token-billed-seer-enabled"]) @patch( "sentry.seer_credit.backend.check_seer_credits", return_value=CheckSeerCreditsResponse(has_enough_credits=False), diff --git a/tests/sentry/seer/test_credits.py b/tests/sentry/seer/test_credits.py index ba587cd18881..afd37295c106 100644 --- a/tests/sentry/seer/test_credits.py +++ b/tests/sentry/seer/test_credits.py @@ -27,7 +27,7 @@ def test_can_run_seer_operation_skips_billing_when_credit_checks_disabled( assert can_run_seer_operation(self.organization, SeerOperation.AUTOFIX_ROOT_CAUSE) mock_check_seer_credits.assert_not_called() - @with_feature("organizations:seer-credit-checks") + @with_feature(["organizations:seer-credit-checks", "organizations:token-billed-seer-enabled"]) @patch( "sentry.seer_credit.backend.check_seer_credits", return_value=CheckSeerCreditsResponse(has_enough_credits=False), @@ -45,7 +45,7 @@ def test_can_run_seer_operation_returns_false_when_billing_has_no_credits( ) ) - @with_feature("organizations:seer-credit-checks") + @with_feature(["organizations:seer-credit-checks", "organizations:token-billed-seer-enabled"]) @patch( "sentry.seer_credit.backend.check_seer_credits", return_value=CheckSeerCreditsResponse(has_enough_credits=True), From c2473282edb707762f2ba74673fc5f8c36b06b98 Mon Sep 17 00:00:00 2001 From: srest2021 Date: Fri, 6 Nov 2026 10:43:12 -0800 Subject: [PATCH 02/25] feat(seer): Check Seer credits before Autofix steps and Open PR Token-billed orgs are checked for Seer credits before each Autofix step and before opening a PR, and get SeerOutOfCreditsException when they can't pay. Other orgs keep the seat-based quota check. Agentic triage runs are not gated. Refs CW-3061 --- src/sentry/seer/autofix/autofix_agent.py | 50 ++++--- src/sentry/seer/autofix/exceptions.py | 4 + src/sentry/seer/autofix/feature/dispatch.py | 90 ++++++++++-- src/sentry/seer/autofix/steps.py | 14 ++ .../seer/autofix/feature/test_dispatch.py | 86 ++++++++++- .../sentry/seer/autofix/test_autofix_agent.py | 136 +++++++++++++++++- 6 files changed, 342 insertions(+), 38 deletions(-) diff --git a/src/sentry/seer/autofix/autofix_agent.py b/src/sentry/seer/autofix/autofix_agent.py index 6515cef5ab52..7ce38f75c09a 100644 --- a/src/sentry/seer/autofix/autofix_agent.py +++ b/src/sentry/seer/autofix/autofix_agent.py @@ -9,7 +9,7 @@ from scm.types import GetBranchProtocol, GetRepositoryProtocol from sentry_sdk import traces -from sentry import features, quotas +from sentry import features from sentry.analytics.events.autofix_events import ( AiAutofixAgentHandoffEvent, AiAutofixCodeChangesCompletedEvent, @@ -23,7 +23,7 @@ AiAutofixSolutionCompletedEvent, AiAutofixSolutionStartedEvent, ) -from sentry.constants import ENABLE_SEER_CODING_DEFAULT, DataCategory +from sentry.constants import ENABLE_SEER_CODING_DEFAULT from sentry.integrations.services.integration import integration_service from sentry.models.group import Group from sentry.seer.agent.client import SeerAgentClient @@ -31,10 +31,11 @@ from sentry.seer.autofix.analytics import record_autofix_event from sentry.seer.autofix.commit_author import SeerCommitAuthor from sentry.seer.autofix.constants import AutofixReferrer -from sentry.seer.autofix.exceptions import NoSeerQuotaException +from sentry.seer.autofix.exceptions import NoSeerQuotaException, SeerOutOfCreditsException from sentry.seer.autofix.feature.dispatch import ( AutofixFeatureArgs, autofix_run_targets_other_issue, + check_autofix_billing, require_autofix_run_for_group, trigger_autofix_feature, ) @@ -62,6 +63,7 @@ is_free_cohort_org, read_preference_from_sentry_db, ) +from sentry.seer.credits import SeerOperation from sentry.seer.entrypoints.operator import ( SeerActivityAttribution, SeerAutofixOperator, @@ -460,19 +462,13 @@ def trigger_autofix_agent( """ outcome = "success" try: - # check billing quota for triggering a new autofix run - # Free cohort orgs bypass quota only when called from agentic triage - # (allow_free_cohort=True). The API endpoint never sets this flag, - # so manual triggers still require quota. - if run_id is None: - skip_quota_check = allow_free_cohort and is_free_cohort_org(group.organization) - if not skip_quota_check: - has_budget: bool = quotas.backend.check_seer_quota( - org_id=group.organization.id, - data_category=DataCategory.SEER_AUTOFIX, - ) - if not has_budget: - raise NoSeerQuotaException() + check_autofix_billing( + group, + step.seer_operation, + referrer, + is_new_run=run_id is None, + skip_quota_for_free_cohort=allow_free_cohort, + ) if run_id is not None: _assert_existing_run_belongs_to_group(group, run_id) @@ -547,7 +543,12 @@ def trigger_autofix_agent( actor_user_id=actor_user_id, ) return feature_run - except (NoSeerQuotaException, SeerPermissionError, PrIterationNoPullRequestException): + except ( + NoSeerQuotaException, + SeerOutOfCreditsException, + SeerPermissionError, + PrIterationNoPullRequestException, + ): outcome = "declined" raise except BaseException: @@ -834,6 +835,17 @@ def trigger_push_changes( ): raise PermissionDenied("Code generation is disabled for this organization") + seer_run = SeerRun.objects.filter( + organization_id=group.organization.id, seer_run_state_id=run_id + ).first() + # Open PR always continues a run, so it never uses quota. + check_autofix_billing( + group, + SeerOperation.AUTOFIX_OPEN_PR, + seer_run.referrer if seer_run else None, + is_new_run=False, + ) + client = get_autofix_agent_client(group, user=user) if state is None: @@ -851,10 +863,6 @@ def trigger_push_changes( ) ) - seer_run = SeerRun.objects.filter( - organization_id=group.organization.id, seer_run_state_id=run_id - ).first() - # Draft when review-request/CI-green is enabled; mark ready once green fires. client.push_changes( run_id, diff --git a/src/sentry/seer/autofix/exceptions.py b/src/sentry/seer/autofix/exceptions.py index 74332026f4dc..7ee853708745 100644 --- a/src/sentry/seer/autofix/exceptions.py +++ b/src/sentry/seer/autofix/exceptions.py @@ -2,6 +2,10 @@ class NoSeerQuotaException(Exception): """Raised when an Autofix run cannot consume Seer quota.""" +class SeerOutOfCreditsException(Exception): + """Raised when the organization doesn't have enough Seer credits for an Autofix operation.""" + + class IssueSummaryUnavailable(Exception): """Raised when an issue summary cannot be generated for an expected reason.""" diff --git a/src/sentry/seer/autofix/feature/dispatch.py b/src/sentry/seer/autofix/feature/dispatch.py index 5ec273daaaec..02b7f3f5c49b 100644 --- a/src/sentry/seer/autofix/feature/dispatch.py +++ b/src/sentry/seer/autofix/feature/dispatch.py @@ -22,7 +22,7 @@ ) from sentry.seer.agent.on_completion_hook import extract_hook_definition from sentry.seer.autofix.constants import AutofixReferrer -from sentry.seer.autofix.exceptions import NoSeerQuotaException +from sentry.seer.autofix.exceptions import NoSeerQuotaException, SeerOutOfCreditsException from sentry.seer.autofix.feature.models import ( FEATURE_ID, LEGACY_FEATURE_ID, @@ -41,6 +41,7 @@ get_automation_handoff, is_free_cohort_org, ) +from sentry.seer.credits import SeerOperation, has_enough_seer_credits from sentry.seer.models.run import SeerAgentRun, SeerRun from sentry.seer.models.seer_api_models import UNKNOWN_RUN_ID_FOR_GROUP, SeerPermissionError from sentry.users.models.user import User @@ -140,18 +141,42 @@ def _single_run_pr_internal_settings( ) -def trigger_autofix_feature( - group: Group, - args: AutofixFeatureArgs, -) -> SeerRun: - # Avoid a circular import through the legacy Autofix dispatcher. - from sentry.seer.autofix.on_completion_hook import AutofixOnCompletionHook +def _check_autofix_credits(group: Group, operation: SeerOperation, referrer: str | None) -> None: + """ + Raise if the organization can't afford the operation under token-based + credits. Agentic triage runs stay free. + """ + if referrer == AutofixReferrer.AGENTIC_TRIAGE: + return + if not has_enough_seer_credits(group.organization, operation): + logger.info( + "autofix_feature.dispatch.out_of_credits", + extra={ + "group_id": group.id, + "organization_id": group.organization.id, + "operation": operation.value, + "referrer": referrer, + }, + ) + raise SeerOutOfCreditsException() - is_new_run = args.existing_run_id is None + +def _check_autofix_quota( + group: Group, + referrer: str | None, + *, + is_new_run: bool, + allow_free_cohort: bool, +) -> bool: + """ + Raise if the organization has no seat-based Seer quota left. Returns whether + the step uses quota, which is recorded once the run starts. + """ # Free cohort orgs bypass quota only when called from agentic triage # (allow_free_cohort=True). Not exposed via the API. - skip_quota = is_new_run and args.allow_free_cohort and is_free_cohort_org(group.organization) - if is_new_run and not skip_quota: + skip_quota = is_new_run and allow_free_cohort and is_free_cohort_org(group.organization) + uses_quota = is_new_run and not skip_quota + if uses_quota: has_budget: bool = quotas.backend.check_seer_quota( org_id=group.organization.id, data_category=DataCategory.SEER_AUTOFIX, @@ -162,11 +187,50 @@ def trigger_autofix_feature( extra={ "group_id": group.id, "organization_id": group.organization.id, - "referrer": args.referrer.value, + "referrer": referrer, }, ) raise NoSeerQuotaException() + return uses_quota + + +def check_autofix_billing( + group: Group, + operation: SeerOperation, + referrer: str | None, + *, + is_new_run: bool, + skip_quota_for_free_cohort: bool = False, +) -> bool: + """ + Raise if the organization can't pay for the operation. Token-billed orgs are + checked for credits, everyone else for seat-based quota. Returns whether the + operation uses quota. + """ + if features.has("organizations:token-billed-seer-enabled", group.organization): + _check_autofix_credits(group, operation, referrer) + return False + return _check_autofix_quota( + group, referrer, is_new_run=is_new_run, allow_free_cohort=skip_quota_for_free_cohort + ) + + +def trigger_autofix_feature( + group: Group, + args: AutofixFeatureArgs, +) -> SeerRun: + # Avoid a circular import through the legacy Autofix dispatcher. + from sentry.seer.autofix.on_completion_hook import AutofixOnCompletionHook + + uses_quota = check_autofix_billing( + group, + args.step.seer_operation, + args.referrer, + is_new_run=args.existing_run_id is None, + skip_quota_for_free_cohort=args.allow_free_cohort, + ) + single_run = should_use_single_run( group, args.step, @@ -259,7 +323,7 @@ def trigger_autofix_feature( extras["stopping_point"] = stopping_point.value user_org_context = collect_user_org_context(args.user, group.organization) - if is_new_run: + if args.existing_run_id is None: run = client.start_feature_run( feature_id=SINGLE_RUN_FEATURE_ID if single_run else FEATURE_ID, payload=payload.dict(), @@ -285,7 +349,7 @@ def trigger_autofix_feature( else: raise Exception("Unhandled run_id branch, this should never happen") - if is_new_run and not skip_quota: + if uses_quota: quotas.backend.record_seer_run( group.organization.id, group.project.id, DataCategory.SEER_AUTOFIX ) diff --git a/src/sentry/seer/autofix/steps.py b/src/sentry/seer/autofix/steps.py index b35a0d86ebea..8ec531cfde9e 100644 --- a/src/sentry/seer/autofix/steps.py +++ b/src/sentry/seer/autofix/steps.py @@ -3,6 +3,7 @@ from enum import StrEnum from sentry.seer.autofix.utils import AutofixStoppingPoint +from sentry.seer.credits import SeerOperation class AutofixStep(StrEnum): @@ -13,6 +14,19 @@ class AutofixStep(StrEnum): CODE_CHANGES = "code_changes" PR_ITERATION = "pr_iteration" + @property + def seer_operation(self) -> SeerOperation: + """The Seer credit check operation corresponding to this step.""" + match self: + case AutofixStep.ROOT_CAUSE: + return SeerOperation.AUTOFIX_ROOT_CAUSE + case AutofixStep.SOLUTION: + return SeerOperation.AUTOFIX_SOLUTION + case AutofixStep.CODE_CHANGES: + return SeerOperation.AUTOFIX_CODE_CHANGES + case AutofixStep.PR_ITERATION: + return SeerOperation.AUTOFIX_PR_ITERATION + @staticmethod def from_autofix_stopping_point( autofix_stopping_point: AutofixStoppingPoint, diff --git a/tests/sentry/seer/autofix/feature/test_dispatch.py b/tests/sentry/seer/autofix/feature/test_dispatch.py index 06ddaf9ca71b..9827183d33c6 100644 --- a/tests/sentry/seer/autofix/feature/test_dispatch.py +++ b/tests/sentry/seer/autofix/feature/test_dispatch.py @@ -2,10 +2,13 @@ from unittest.mock import MagicMock, patch import pytest +from sentry_protos.billing.v1.services.seer_feature_billing.v1.endpoint_check_seer_credits_pb2 import ( + CheckSeerCreditsResponse, +) from sentry.constants import DataCategory from sentry.seer.autofix.constants import AutofixReferrer -from sentry.seer.autofix.exceptions import NoSeerQuotaException +from sentry.seer.autofix.exceptions import NoSeerQuotaException, SeerOutOfCreditsException from sentry.seer.autofix.feature.dispatch import AutofixFeatureArgs, trigger_autofix_feature from sentry.seer.autofix.feature.models import ( FEATURE_ID, @@ -21,6 +24,7 @@ from sentry.seer.autofix.utils import AutofixStoppingPoint from sentry.seer.models import SeerPermissionError from sentry.testutils.cases import TestCase +from sentry.testutils.helpers.features import with_feature from sentry.testutils.pytest.fixtures import django_db_all @@ -275,6 +279,86 @@ def test_free_cohort_skips_quota_check_and_usage_recording(self) -> None: mock_quotas.backend.check_seer_quota.assert_not_called() mock_quotas.backend.record_seer_run.assert_not_called() + @with_feature(["organizations:seer-credit-checks", "organizations:token-billed-seer-enabled"]) + @patch( + "sentry.seer_credit.backend.check_seer_credits", + return_value=CheckSeerCreditsResponse(has_enough_credits=False), + ) + def test_raises_when_out_of_credits(self, mock_check_seer_credits: MagicMock) -> None: + with ( + patch("sentry.seer.autofix.feature.dispatch.SeerAgentClient") as MockClient, + patch("sentry.seer.autofix.feature.dispatch.quotas") as mock_quotas, + ): + mock_quotas.backend.check_seer_quota.return_value = True + + with pytest.raises(SeerOutOfCreditsException): + trigger_autofix_feature( + self.group, + AutofixFeatureArgs( + referrer=AutofixReferrer.GROUP_AUTOFIX_ENDPOINT, + step=AutofixStep.ROOT_CAUSE, + step_args=RCAStepArgs(), + ), + ) + + MockClient.return_value.start_feature_run.assert_not_called() + mock_quotas.backend.check_seer_quota.assert_not_called() + mock_quotas.backend.record_seer_run.assert_not_called() + + @with_feature(["organizations:seer-credit-checks", "organizations:token-billed-seer-enabled"]) + @patch( + "sentry.seer_credit.backend.check_seer_credits", + return_value=CheckSeerCreditsResponse(has_enough_credits=False), + ) + def test_raises_when_out_of_credits_for_a_continued_step( + self, mock_check_seer_credits: MagicMock + ) -> None: + """Unlike quota, credits are checked for every step, not only new runs.""" + with ( + patch("sentry.seer.autofix.feature.dispatch.SeerAgentClient") as MockClient, + patch("sentry.seer.autofix.feature.dispatch.quotas") as mock_quotas, + ): + with pytest.raises(SeerOutOfCreditsException): + trigger_autofix_feature( + self.group, + AutofixFeatureArgs( + referrer=AutofixReferrer.GROUP_AUTOFIX_ENDPOINT, + step=AutofixStep.SOLUTION, + existing_run_id=123, + step_args=SolutionStepArgs(), + ), + ) + + mock_quotas.backend.check_seer_quota.assert_not_called() + MockClient.return_value.continue_feature_run.assert_not_called() + + @with_feature(["organizations:seer-credit-checks", "organizations:token-billed-seer-enabled"]) + @patch( + "sentry.seer_credit.backend.check_seer_credits", + return_value=CheckSeerCreditsResponse(has_enough_credits=False), + ) + def test_agentic_triage_skips_credit_check(self, mock_check_seer_credits: MagicMock) -> None: + fake_run = self.create_seer_run(organization=self.organization, type="feature_run") + + with ( + patch("sentry.seer.autofix.feature.dispatch.SeerAgentClient") as MockClient, + patch("sentry.seer.autofix.feature.dispatch.quotas") as mock_quotas, + ): + mock_quotas.backend.check_seer_quota.return_value = True + MockClient.return_value.start_feature_run.return_value = fake_run + + run = trigger_autofix_feature( + self.group, + AutofixFeatureArgs( + referrer=AutofixReferrer.AGENTIC_TRIAGE, + step=AutofixStep.ROOT_CAUSE, + step_args=RCAStepArgs(), + ), + ) + + assert run is fake_run + mock_check_seer_credits.assert_not_called() + def test_allows_async_dispatch(self) -> None: fake_run = self.create_seer_run(organization=self.organization, type="feature_run") diff --git a/tests/sentry/seer/autofix/test_autofix_agent.py b/tests/sentry/seer/autofix/test_autofix_agent.py index 477b8bf89ffa..dd598ac0a01e 100644 --- a/tests/sentry/seer/autofix/test_autofix_agent.py +++ b/tests/sentry/seer/autofix/test_autofix_agent.py @@ -4,6 +4,9 @@ import pytest from django.test import override_settings from rest_framework.exceptions import PermissionDenied +from sentry_protos.billing.v1.services.seer_feature_billing.v1.endpoint_check_seer_credits_pb2 import ( + CheckSeerCreditsResponse, +) from sentry.analytics.events.autofix_events import AiAutofixSolutionCompletedEvent from sentry.constants import DataCategory @@ -27,7 +30,7 @@ ) from sentry.seer.autofix.commit_author import SeerCommitAuthor from sentry.seer.autofix.constants import AutofixReferrer, SeerAutomationSource -from sentry.seer.autofix.exceptions import NoSeerQuotaException +from sentry.seer.autofix.exceptions import NoSeerQuotaException, SeerOutOfCreditsException from sentry.seer.autofix.feature.models import SINGLE_RUN_FEATURE_ID from sentry.seer.autofix.issue_summary import referrer_map from sentry.seer.autofix.pr_iteration.feedback import Feedback @@ -38,6 +41,7 @@ from sentry.seer.models.run import SeerAgentRun, SeerRun from sentry.sentry_apps.utils.webhooks import SeerActionType from sentry.testutils.cases import TestCase +from sentry.testutils.helpers.features import with_feature from sentry.types.activity import ActivityType from sentry.utils import json @@ -720,6 +724,30 @@ def test_when_no_quota(self, mock_check_quota: MagicMock, mock_feature: MagicMoc ) mock_feature.assert_not_called() + @with_feature(["organizations:seer-credit-checks", "organizations:token-billed-seer-enabled"]) + @patch( + "sentry.seer_credit.backend.check_seer_credits", + return_value=CheckSeerCreditsResponse(has_enough_credits=False), + ) + @patch("sentry.seer.autofix.autofix_agent.trigger_autofix_feature") + @patch("sentry.quotas.backend.check_seer_quota", return_value=True) + def test_when_out_of_credits( + self, + mock_check_quota: MagicMock, + mock_feature: MagicMock, + mock_check_seer_credits: MagicMock, + ) -> None: + with pytest.raises(SeerOutOfCreditsException): + trigger_autofix_agent( + group=self.group, + step=AutofixStep.CODE_CHANGES, + referrer=AutofixReferrer.UNKNOWN, + run_id=None, + ) + mock_check_seer_credits.assert_called_once() + mock_check_quota.assert_not_called() + mock_feature.assert_not_called() + @patch("sentry.quotas.backend.check_seer_quota", return_value=True) @patch("sentry.seer.autofix.autofix_agent.broadcast_webhooks_for_organization.delay") @patch("sentry.seer.autofix.autofix_agent.trigger_autofix_feature") @@ -732,7 +760,7 @@ def test_free_cohort_skips_quota_check( with ( patch("sentry.seer.autofix.autofix_agent.features.has", return_value=False), - patch("sentry.seer.autofix.autofix_agent.is_free_cohort_org", return_value=True), + patch("sentry.seer.autofix.feature.dispatch.is_free_cohort_org", return_value=True), ): trigger_autofix_agent( group=self.group, @@ -756,7 +784,7 @@ def test_allow_free_cohort_still_checks_quota_for_non_cohort_org( with ( patch("sentry.seer.autofix.autofix_agent.features.has", return_value=False), - patch("sentry.seer.autofix.autofix_agent.is_free_cohort_org", return_value=False), + patch("sentry.seer.autofix.feature.dispatch.is_free_cohort_org", return_value=False), ): trigger_autofix_agent( group=self.group, @@ -767,6 +795,35 @@ def test_allow_free_cohort_still_checks_quota_for_non_cohort_org( mock_check_quota.assert_called_once() + @with_feature(["organizations:seer-credit-checks", "organizations:token-billed-seer-enabled"]) + @patch( + "sentry.seer_credit.backend.check_seer_credits", + return_value=CheckSeerCreditsResponse(has_enough_credits=False), + ) + @patch("sentry.quotas.backend.check_seer_quota", return_value=True) + @patch("sentry.seer.autofix.autofix_agent.broadcast_webhooks_for_organization.delay") + @patch("sentry.seer.autofix.autofix_agent.trigger_autofix_feature") + def test_agentic_triage_skips_credit_check( + self, + mock_feature: MagicMock, + mock_broadcast: MagicMock, + mock_check_quota: MagicMock, + mock_check_seer_credits: MagicMock, + ) -> None: + mock_feature.return_value = self.create_seer_run( + organization=self.group.organization, seer_run_state_id=12345 + ) + + trigger_autofix_agent( + group=self.group, + step=AutofixStep.CODE_CHANGES, + referrer=AutofixReferrer.AGENTIC_TRIAGE, + ) + + mock_check_seer_credits.assert_not_called() + mock_check_quota.assert_not_called() + mock_feature.assert_called_once() + @patch("sentry.quotas.backend.check_seer_quota", return_value=False) @patch("sentry.seer.autofix.autofix_agent.broadcast_webhooks_for_organization.delay") @patch("sentry.seer.autofix.autofix_agent.trigger_autofix_feature") @@ -975,6 +1032,35 @@ def test_counts_declined_when_no_quota( } ] + @with_feature(["organizations:seer-credit-checks", "organizations:token-billed-seer-enabled"]) + @patch( + "sentry.seer_credit.backend.check_seer_credits", + return_value=CheckSeerCreditsResponse(has_enough_credits=False), + ) + def test_counts_declined_when_out_of_credits( + self, + mock_check_seer_credits: MagicMock, + mock_client_class: MagicMock, + mock_incr: MagicMock, + mock_check_quota: MagicMock, + mock_record_run: MagicMock, + mock_broadcast: MagicMock, + ) -> None: + with pytest.raises(SeerOutOfCreditsException): + trigger_autofix_agent( + self.group, + AutofixStep.ROOT_CAUSE, + AutofixReferrer.ISSUE_SUMMARY_POST_PROCESS_FIXABILITY, + ) + + assert _trigger_attempts(mock_incr) == [ + { + "referrer": "issue_summary.post_process_fixability", + "step": "root_cause", + "outcome": "declined", + } + ] + @patch("sentry.seer.autofix.autofix_agent.trigger_autofix_feature") def test_counts_error_when_seer_rejects_run( self, @@ -1674,6 +1760,50 @@ def test_raises_permission_denied_when_coding_disabled(self) -> None: referrer=AutofixReferrer.UNKNOWN, ) + @with_feature(["organizations:seer-credit-checks", "organizations:token-billed-seer-enabled"]) + @patch( + "sentry.seer_credit.backend.check_seer_credits", + return_value=CheckSeerCreditsResponse(has_enough_credits=False), + ) + @patch("sentry.seer.agent.client.make_agent_update_request") + def test_raises_when_out_of_credits( + self, mock_post: MagicMock, mock_check_seer_credits: MagicMock + ) -> None: + with pytest.raises(SeerOutOfCreditsException): + trigger_push_changes( + group=self.group, + run_id=123, + referrer=AutofixReferrer.UNKNOWN, + ) + + mock_post.assert_not_called() + + @patch( + "sentry.seer_credit.backend.check_seer_credits", + return_value=CheckSeerCreditsResponse(has_enough_credits=False), + ) + @patch("sentry.seer.agent.client.make_agent_update_request") + def test_agentic_triage_run_skips_credit_check( + self, mock_post: MagicMock, mock_check_seer_credits: MagicMock + ) -> None: + # The push is called with another referrer; the run's own referrer decides. + self.create_seer_run( + organization=self.organization, + seer_run_state_id=123, + referrer=AutofixReferrer.AGENTIC_TRIAGE.value, + ) + + self._push( + mock_post, + features={ + "organizations:seer-credit-checks": True, + "organizations:token-billed-seer-enabled": True, + }, + ) + + mock_check_seer_credits.assert_not_called() + mock_post.assert_called_once() + @patch("sentry.seer.agent.client.make_agent_update_request") def test_passes_pr_description_suffix_ready_state_and_author( self, mock_post: MagicMock From e2855ba1be5c9a28d483661f1ea99b9c1bc7cbef Mon Sep 17 00:00:00 2001 From: srest2021 Date: Fri, 6 Nov 2026 10:48:24 -0800 Subject: [PATCH 03/25] ref(seer): Keep the token-billed check inside can_run_seer_operation --- src/sentry/seer/credits.py | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/src/sentry/seer/credits.py b/src/sentry/seer/credits.py index 5b47405b5243..17ae7eb017a6 100644 --- a/src/sentry/seer/credits.py +++ b/src/sentry/seer/credits.py @@ -60,11 +60,10 @@ class SeerOperationConfig: } -def has_enough_seer_credits(organization: Organization, operation: SeerOperation) -> bool: - """ - Checks whether the organization has enough Seer credits to run the operation - for callers that already checked the org is token-billed. - """ +def can_run_seer_operation(organization: Organization, operation: SeerOperation) -> bool: + """Whether the organization is token-billed and has enough Seer credits to run the operation.""" + if not features.has("organizations:token-billed-seer-enabled", organization): + return True if not features.has("organizations:seer-credit-checks", organization): return True @@ -78,10 +77,3 @@ def has_enough_seer_credits(organization: Organization, operation: SeerOperation ) ) return response.has_enough_credits - - -def can_run_seer_operation(organization: Organization, operation: SeerOperation) -> bool: - """Whether the organization is token-billed and has enough Seer credits to run the operation.""" - return not features.has( - "organizations:token-billed-seer-enabled", organization - ) or has_enough_seer_credits(organization, operation) From d70d6cbddd51750df5c0dd1aac152f630622a332 Mon Sep 17 00:00:00 2001 From: srest2021 Date: Fri, 6 Nov 2026 10:49:40 -0800 Subject: [PATCH 04/25] ref(seer): Split has_enough_seer_credits out of can_run_seer_operation Autofix billing checks the token-billed flag itself, so it only needs the credit check. --- src/sentry/seer/credits.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/sentry/seer/credits.py b/src/sentry/seer/credits.py index 17ae7eb017a6..5b47405b5243 100644 --- a/src/sentry/seer/credits.py +++ b/src/sentry/seer/credits.py @@ -60,10 +60,11 @@ class SeerOperationConfig: } -def can_run_seer_operation(organization: Organization, operation: SeerOperation) -> bool: - """Whether the organization is token-billed and has enough Seer credits to run the operation.""" - if not features.has("organizations:token-billed-seer-enabled", organization): - return True +def has_enough_seer_credits(organization: Organization, operation: SeerOperation) -> bool: + """ + Checks whether the organization has enough Seer credits to run the operation + for callers that already checked the org is token-billed. + """ if not features.has("organizations:seer-credit-checks", organization): return True @@ -77,3 +78,10 @@ def can_run_seer_operation(organization: Organization, operation: SeerOperation) ) ) return response.has_enough_credits + + +def can_run_seer_operation(organization: Organization, operation: SeerOperation) -> bool: + """Whether the organization is token-billed and has enough Seer credits to run the operation.""" + return not features.has( + "organizations:token-billed-seer-enabled", organization + ) or has_enough_seer_credits(organization, operation) From 58a1b631b942cf732754ee84b6e5037efbc25268 Mon Sep 17 00:00:00 2001 From: srest2021 Date: Fri, 6 Nov 2026 10:54:24 -0800 Subject: [PATCH 05/25] ref(seer): Combine the credit check flag conditions --- src/sentry/seer/credits.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/sentry/seer/credits.py b/src/sentry/seer/credits.py index 17ae7eb017a6..ebf4c3fc1411 100644 --- a/src/sentry/seer/credits.py +++ b/src/sentry/seer/credits.py @@ -62,9 +62,9 @@ class SeerOperationConfig: def can_run_seer_operation(organization: Organization, operation: SeerOperation) -> bool: """Whether the organization is token-billed and has enough Seer credits to run the operation.""" - if not features.has("organizations:token-billed-seer-enabled", organization): - return True - if not features.has("organizations:seer-credit-checks", organization): + if not features.has( + "organizations:token-billed-seer-enabled", organization + ) or not features.has("organizations:seer-credit-checks", organization): return True config = SEER_OPERATIONS[operation] From 0ab3b109a98367d1f3cee0159b7a89d71a3656bc Mon Sep 17 00:00:00 2001 From: srest2021 Date: Fri, 6 Nov 2026 11:21:40 -0800 Subject: [PATCH 06/25] feat(seer): Check Seer credits before replying to an Autofix single run Chat replies to single runs continue the run through continue_single_run, which skipped billing. The run's own referrer decides, so agentic triage runs stay free. --- src/sentry/seer/autofix/feature/dispatch.py | 8 +++++ src/sentry/seer/credits.py | 4 +++ .../seer/autofix/feature/test_dispatch.py | 36 +++++++++++++++++++ 3 files changed, 48 insertions(+) diff --git a/src/sentry/seer/autofix/feature/dispatch.py b/src/sentry/seer/autofix/feature/dispatch.py index 1e9e5496062b..46d533585a58 100644 --- a/src/sentry/seer/autofix/feature/dispatch.py +++ b/src/sentry/seer/autofix/feature/dispatch.py @@ -160,6 +160,14 @@ def continue_single_run( ) -> SeerRun: from sentry.seer.autofix.on_completion_hook import AutofixOnCompletionHook + # A reply always continues a run, so it never uses quota. + check_autofix_billing( + group, + SeerOperation.AUTOFIX_SINGLE_RUN_REPLY, + agent_run.run.referrer, + is_new_run=False, + ) + enable_coding = _org_allows_coding(group) open_pr = enable_coding and can_publish payload = AutofixSingleRunFeaturePayload( diff --git a/src/sentry/seer/credits.py b/src/sentry/seer/credits.py index 5b47405b5243..a4915a46067e 100644 --- a/src/sentry/seer/credits.py +++ b/src/sentry/seer/credits.py @@ -19,6 +19,7 @@ class SeerOperation(StrEnum): AUTOFIX_CODE_CHANGES = "autofix_code_changes" AUTOFIX_OPEN_PR = "autofix_open_pr" AUTOFIX_PR_ITERATION = "autofix_pr_iteration" + AUTOFIX_SINGLE_RUN_REPLY = "autofix_single_run_reply" class SeerFeature(StrEnum): @@ -57,6 +58,9 @@ class SeerOperationConfig: SeerOperation.AUTOFIX_PR_ITERATION: SeerOperationConfig( SeerFeature.AUTOFIX, lambda: PLACEHOLDER_CREDIT_ESTIMATE ), + SeerOperation.AUTOFIX_SINGLE_RUN_REPLY: SeerOperationConfig( + SeerFeature.AUTOFIX, lambda: PLACEHOLDER_CREDIT_ESTIMATE + ), } diff --git a/tests/sentry/seer/autofix/feature/test_dispatch.py b/tests/sentry/seer/autofix/feature/test_dispatch.py index 7264c3e2dbc1..ab47678c838f 100644 --- a/tests/sentry/seer/autofix/feature/test_dispatch.py +++ b/tests/sentry/seer/autofix/feature/test_dispatch.py @@ -692,3 +692,39 @@ def test_respects_org_coding_opt_out(self) -> None: assert kwargs["agent_run_options"]["enable_coding"] is False assert kwargs["agent_run_options"]["enable_write_pull_request"] is False assert kwargs["payload"]["pr_internal_settings"] is None + + @with_feature(["organizations:seer-credit-checks", "organizations:token-billed-seer-enabled"]) + @patch( + "sentry.seer_credit.backend.check_seer_credits", + return_value=CheckSeerCreditsResponse(has_enough_credits=False), + ) + def test_raises_when_out_of_credits(self, mock_check_seer_credits: MagicMock) -> None: + with ( + patch("sentry.seer.autofix.feature.dispatch.SeerAgentClient") as mock_client_cls, + pytest.raises(SeerOutOfCreditsException), + ): + continue_single_run( + self.agent_run, + self.group, + "make a code fix", + user=self.user, + can_publish=True, + referrer=AutofixReferrer.SEER_EXPLORER, + ) + + mock_client_cls.return_value.continue_feature_run.assert_not_called() + + @with_feature(["organizations:seer-credit-checks", "organizations:token-billed-seer-enabled"]) + @patch( + "sentry.seer_credit.backend.check_seer_credits", + return_value=CheckSeerCreditsResponse(has_enough_credits=False), + ) + def test_agentic_triage_run_skips_credit_check( + self, mock_check_seer_credits: MagicMock + ) -> None: + """The run's own referrer decides, not the reply's.""" + self.seer_run.update(referrer=AutofixReferrer.AGENTIC_TRIAGE) + + self._reply() + + mock_check_seer_credits.assert_not_called() From a3dae0d0298e80374d49e598e1a4e129766c4a4d Mon Sep 17 00:00:00 2001 From: srest2021 Date: Fri, 6 Nov 2026 11:38:52 -0800 Subject: [PATCH 07/25] ref(seer): Check the credit check flag first --- src/sentry/seer/credits.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/sentry/seer/credits.py b/src/sentry/seer/credits.py index ebf4c3fc1411..ac40567af5cd 100644 --- a/src/sentry/seer/credits.py +++ b/src/sentry/seer/credits.py @@ -62,9 +62,9 @@ class SeerOperationConfig: def can_run_seer_operation(organization: Organization, operation: SeerOperation) -> bool: """Whether the organization is token-billed and has enough Seer credits to run the operation.""" - if not features.has( + if not features.has("organizations:seer-credit-checks", organization) or not features.has( "organizations:token-billed-seer-enabled", organization - ) or not features.has("organizations:seer-credit-checks", organization): + ): return True config = SEER_OPERATIONS[operation] From 94b979c0fadb6da8ca292f4165cf2e0dc5b2fe89 Mon Sep 17 00:00:00 2001 From: srest2021 Date: Fri, 6 Nov 2026 11:54:15 -0800 Subject: [PATCH 08/25] test(seer): Align credit check tests with their files Cover orgs that aren't token-billed, group the chat credit tests after the access tests, and match the neighboring tests' names and setup. --- .../test_organization_seer_agent_chat.py | 165 +++++++++--------- .../test_organization_seer_credit_check.py | 13 ++ tests/sentry/seer/test_credits.py | 12 ++ 3 files changed, 109 insertions(+), 81 deletions(-) diff --git a/tests/sentry/seer/endpoints/test_organization_seer_agent_chat.py b/tests/sentry/seer/endpoints/test_organization_seer_agent_chat.py index 46b6c73afedf..26bd0eb24494 100644 --- a/tests/sentry/seer/endpoints/test_organization_seer_agent_chat.py +++ b/tests/sentry/seer/endpoints/test_organization_seer_agent_chat.py @@ -647,87 +647,6 @@ def test_post_continue_with_uuid_resolves(self, mock_client_class: MagicMock) -> assert response.data == {"run_id": 555, "sentry_run_id": str(run.uuid)} assert mock_client.continue_run.call_args.kwargs["run_id"] == 555 - @with_feature(["organizations:seer-credit-checks", "organizations:token-billed-seer-enabled"]) - @patch( - "sentry.seer_credit.backend.check_seer_credits", - return_value=CheckSeerCreditsResponse(has_enough_credits=False), - ) - @patch("sentry.seer.endpoints.organization_seer_agent_chat.SeerAgentClient") - def test_post_new_conversation_without_credits_returns_402( - self, mock_client_class: MagicMock, mock_check_seer_credits: MagicMock - ) -> None: - response = self.client.post(self.url, {"query": "What is this error about?"}, format="json") - - assert response.status_code == 402 - assert response.data == { - "detail": "Your organization does not have enough credits left for you to message Seer." - } - mock_client_class.assert_not_called() - - @with_feature(["organizations:seer-credit-checks", "organizations:token-billed-seer-enabled"]) - @patch( - "sentry.seer_credit.backend.check_seer_credits", - return_value=CheckSeerCreditsResponse(has_enough_credits=False), - ) - @patch("sentry.seer.endpoints.organization_seer_agent_chat.SeerAgentClient") - def test_post_continue_without_credits_returns_402( - self, mock_client_class: MagicMock, mock_check_seer_credits: MagicMock - ) -> None: - run = self.create_seer_run( - organization=self.organization, seer_run_state_id=555, user_id=self.user.id - ) - - response = self.client.post(f"{self.url}{run.uuid}/", {"query": "More"}, format="json") - - assert response.status_code == 402 - assert response.data == { - "detail": "Your organization does not have enough credits left for you to message Seer." - } - mock_client_class.assert_not_called() - - @with_feature(["organizations:seer-credit-checks", "organizations:token-billed-seer-enabled"]) - @patch( - "sentry.seer_credit.backend.check_seer_credits", - return_value=CheckSeerCreditsResponse(has_enough_credits=False), - ) - @patch("sentry.seer.endpoints.organization_seer_agent_chat.SeerAgentClient") - def test_post_continue_dashboard_run_without_credits_calls_client( - self, mock_client_class: MagicMock, mock_check_seer_credits: MagicMock - ) -> None: - """Dashboard generation isn't gated on credits, so its follow-ups aren't either.""" - run = self.create_seer_run( - organization=self.organization, seer_run_state_id=789, user_id=self.user.id - ) - self.create_seer_agent_run(run=run, source="dashboard_generate") - mock_client = MagicMock() - mock_client.continue_run.return_value = 789 - mock_client_class.return_value = mock_client - - response = self.client.post( - f"{self.url}{run.seer_run_state_id}/", {"query": "Follow up question"}, format="json" - ) - - assert response.status_code == 200 - assert response.data == {"run_id": 789, "sentry_run_id": str(run.uuid)} - - @with_feature(["organizations:seer-credit-checks", "organizations:token-billed-seer-enabled"]) - @patch( - "sentry.seer_credit.backend.check_seer_credits", - return_value=CheckSeerCreditsResponse(has_enough_credits=True), - ) - @patch("sentry.seer.endpoints.organization_seer_agent_chat.SeerAgentClient") - def test_post_with_enough_credits_calls_client( - self, mock_client_class: MagicMock, mock_check_seer_credits: MagicMock - ) -> None: - mock_client = MagicMock() - mock_client.start_run.return_value = MagicMock(seer_run_state_id=456, uuid=uuid.uuid4()) - mock_client_class.return_value = mock_client - - response = self.client.post(self.url, {"query": "What is this error about?"}, format="json") - - assert response.status_code == 200 - mock_client.start_run.assert_called_once() - @patch("sentry.seer.endpoints.organization_seer_agent_chat.SeerAgentClient") def test_post_continuing_another_users_conversation_is_denied( self, mock_client_class: MagicMock @@ -1007,6 +926,90 @@ def test_get_denied_without_seer_access(self) -> None: assert response.status_code == 403 + @with_feature(["organizations:seer-credit-checks", "organizations:token-billed-seer-enabled"]) + @patch( + "sentry.seer_credit.backend.check_seer_credits", + return_value=CheckSeerCreditsResponse(has_enough_credits=False), + ) + @patch("sentry.seer.endpoints.organization_seer_agent_chat.SeerAgentClient") + def test_post_new_conversation_without_credits_returns_402( + self, mock_client_class: MagicMock, mock_check_seer_credits: MagicMock + ) -> None: + response = self.client.post(self.url, {"query": "What is this error about?"}, format="json") + + assert response.status_code == 402 + assert response.data == { + "detail": "Your organization does not have enough credits left for you to message Seer." + } + mock_client_class.assert_not_called() + + @with_feature(["organizations:seer-credit-checks", "organizations:token-billed-seer-enabled"]) + @patch( + "sentry.seer_credit.backend.check_seer_credits", + return_value=CheckSeerCreditsResponse(has_enough_credits=True), + ) + @patch("sentry.seer.endpoints.organization_seer_agent_chat.SeerAgentClient") + def test_post_new_conversation_with_enough_credits_calls_client( + self, mock_client_class: MagicMock, mock_check_seer_credits: MagicMock + ) -> None: + mock_client = MagicMock() + mock_client.start_run.return_value = MagicMock(seer_run_state_id=456, uuid=uuid.uuid4()) + mock_client_class.return_value = mock_client + + response = self.client.post(self.url, {"query": "What is this error about?"}, format="json") + + assert response.status_code == 200 + mock_client.start_run.assert_called_once() + + @with_feature(["organizations:seer-credit-checks", "organizations:token-billed-seer-enabled"]) + @patch( + "sentry.seer_credit.backend.check_seer_credits", + return_value=CheckSeerCreditsResponse(has_enough_credits=False), + ) + @patch("sentry.seer.endpoints.organization_seer_agent_chat.SeerAgentClient") + def test_post_continue_without_credits_returns_402( + self, mock_client_class: MagicMock, mock_check_seer_credits: MagicMock + ) -> None: + run = self.create_seer_run( + organization=self.organization, seer_run_state_id=789, user_id=self.user.id + ) + + response = self.client.post( + f"{self.url}{run.seer_run_state_id}/", {"query": "Follow up question"}, format="json" + ) + + assert response.status_code == 402 + assert response.data == { + "detail": "Your organization does not have enough credits left for you to message Seer." + } + mock_client_class.assert_not_called() + + @with_feature(["organizations:seer-credit-checks", "organizations:token-billed-seer-enabled"]) + @patch( + "sentry.seer_credit.backend.check_seer_credits", + return_value=CheckSeerCreditsResponse(has_enough_credits=False), + ) + @patch("sentry.seer.endpoints.organization_seer_agent_chat.SeerAgentClient") + def test_post_continue_dashboard_run_without_credits_calls_client( + self, mock_client_class: MagicMock, mock_check_seer_credits: MagicMock + ) -> None: + """Dashboard generation isn't gated on credits, so its follow-ups aren't either.""" + run = self.create_seer_run( + organization=self.organization, seer_run_state_id=789, user_id=self.user.id + ) + self.create_seer_agent_run(run=run, source="dashboard_generate") + mock_client = MagicMock() + mock_client.continue_run.return_value = 789 + mock_client_class.return_value = mock_client + + response = self.client.post( + f"{self.url}{run.seer_run_state_id}/", {"query": "Follow up question"}, format="json" + ) + + assert response.status_code == 200 + assert response.data == {"run_id": 789, "sentry_run_id": str(run.uuid)} + mock_check_seer_credits.assert_not_called() + @patch("sentry.seer.endpoints.organization_seer_agent_chat.SeerAgentClient") def test_post_json_on_page_context_converted_to_markdown( self, mock_client_class: MagicMock diff --git a/tests/sentry/seer/endpoints/test_organization_seer_credit_check.py b/tests/sentry/seer/endpoints/test_organization_seer_credit_check.py index 90aaf513233d..9a7303e5b8d3 100644 --- a/tests/sentry/seer/endpoints/test_organization_seer_credit_check.py +++ b/tests/sentry/seer/endpoints/test_organization_seer_credit_check.py @@ -57,6 +57,19 @@ def test_get_returns_has_enough_credits_when_credit_checks_disabled(self) -> Non assert response.data == {"hasEnoughCredits": True} + @with_feature("organizations:seer-credit-checks") + @patch( + "sentry.seer_credit.backend.check_seer_credits", + return_value=CheckSeerCreditsResponse(has_enough_credits=False), + ) + def test_get_returns_has_enough_credits_when_not_token_billed( + self, mock_check_seer_credits: MagicMock + ) -> None: + response = self.get_success_response(self.organization.slug, operation="autofix_solution") + + assert response.data == {"hasEnoughCredits": True} + mock_check_seer_credits.assert_not_called() + def test_get_returns_400_when_operation_missing(self) -> None: self.get_error_response(self.organization.slug, status_code=400) diff --git a/tests/sentry/seer/test_credits.py b/tests/sentry/seer/test_credits.py index afd37295c106..0b730e6d390a 100644 --- a/tests/sentry/seer/test_credits.py +++ b/tests/sentry/seer/test_credits.py @@ -17,6 +17,7 @@ def test_every_operation_has_a_credit_estimate(self) -> None: for operation in SeerOperation: assert SEER_OPERATIONS[operation].estimate() > 0 + @with_feature("organizations:token-billed-seer-enabled") @patch( "sentry.seer_credit.backend.check_seer_credits", return_value=CheckSeerCreditsResponse(has_enough_credits=False), @@ -27,6 +28,17 @@ def test_can_run_seer_operation_skips_billing_when_credit_checks_disabled( assert can_run_seer_operation(self.organization, SeerOperation.AUTOFIX_ROOT_CAUSE) mock_check_seer_credits.assert_not_called() + @with_feature("organizations:seer-credit-checks") + @patch( + "sentry.seer_credit.backend.check_seer_credits", + return_value=CheckSeerCreditsResponse(has_enough_credits=False), + ) + def test_can_run_seer_operation_skips_billing_when_not_token_billed( + self, mock_check_seer_credits: MagicMock + ) -> None: + assert can_run_seer_operation(self.organization, SeerOperation.AUTOFIX_ROOT_CAUSE) + mock_check_seer_credits.assert_not_called() + @with_feature(["organizations:seer-credit-checks", "organizations:token-billed-seer-enabled"]) @patch( "sentry.seer_credit.backend.check_seer_credits", From 6f7cb6f9492f6d1332b90b4d05a7fc34213af13a Mon Sep 17 00:00:00 2001 From: srest2021 Date: Fri, 6 Nov 2026 12:16:01 -0800 Subject: [PATCH 09/25] ref(seer): Leave single run replies out of credit gating Single runs are behind organizations:autofix-single-run and need their own estimate, so they're out of scope here. --- src/sentry/seer/autofix/feature/dispatch.py | 8 ----- src/sentry/seer/credits.py | 4 --- .../seer/autofix/feature/test_dispatch.py | 36 ------------------- 3 files changed, 48 deletions(-) diff --git a/src/sentry/seer/autofix/feature/dispatch.py b/src/sentry/seer/autofix/feature/dispatch.py index 82b9a0b31dc8..408b573b403e 100644 --- a/src/sentry/seer/autofix/feature/dispatch.py +++ b/src/sentry/seer/autofix/feature/dispatch.py @@ -160,14 +160,6 @@ def continue_single_run( ) -> SeerRun: from sentry.seer.autofix.on_completion_hook import AutofixOnCompletionHook - # A reply always continues a run, so it never uses quota. - check_autofix_billing( - group, - SeerOperation.AUTOFIX_SINGLE_RUN_REPLY, - agent_run.run.referrer, - is_new_run=False, - ) - enable_coding = _org_allows_coding(group) open_pr = enable_coding and can_publish payload = AutofixSingleRunFeaturePayload( diff --git a/src/sentry/seer/credits.py b/src/sentry/seer/credits.py index b945dfb0ea83..5a894919c0a3 100644 --- a/src/sentry/seer/credits.py +++ b/src/sentry/seer/credits.py @@ -19,7 +19,6 @@ class SeerOperation(StrEnum): AUTOFIX_CODE_CHANGES = "autofix_code_changes" AUTOFIX_OPEN_PR = "autofix_open_pr" AUTOFIX_PR_ITERATION = "autofix_pr_iteration" - AUTOFIX_SINGLE_RUN_REPLY = "autofix_single_run_reply" class SeerFeature(StrEnum): @@ -58,9 +57,6 @@ class SeerOperationConfig: SeerOperation.AUTOFIX_PR_ITERATION: SeerOperationConfig( SeerFeature.AUTOFIX, lambda: PLACEHOLDER_CREDIT_ESTIMATE ), - SeerOperation.AUTOFIX_SINGLE_RUN_REPLY: SeerOperationConfig( - SeerFeature.AUTOFIX, lambda: PLACEHOLDER_CREDIT_ESTIMATE - ), } diff --git a/tests/sentry/seer/autofix/feature/test_dispatch.py b/tests/sentry/seer/autofix/feature/test_dispatch.py index ab47678c838f..7264c3e2dbc1 100644 --- a/tests/sentry/seer/autofix/feature/test_dispatch.py +++ b/tests/sentry/seer/autofix/feature/test_dispatch.py @@ -692,39 +692,3 @@ def test_respects_org_coding_opt_out(self) -> None: assert kwargs["agent_run_options"]["enable_coding"] is False assert kwargs["agent_run_options"]["enable_write_pull_request"] is False assert kwargs["payload"]["pr_internal_settings"] is None - - @with_feature(["organizations:seer-credit-checks", "organizations:token-billed-seer-enabled"]) - @patch( - "sentry.seer_credit.backend.check_seer_credits", - return_value=CheckSeerCreditsResponse(has_enough_credits=False), - ) - def test_raises_when_out_of_credits(self, mock_check_seer_credits: MagicMock) -> None: - with ( - patch("sentry.seer.autofix.feature.dispatch.SeerAgentClient") as mock_client_cls, - pytest.raises(SeerOutOfCreditsException), - ): - continue_single_run( - self.agent_run, - self.group, - "make a code fix", - user=self.user, - can_publish=True, - referrer=AutofixReferrer.SEER_EXPLORER, - ) - - mock_client_cls.return_value.continue_feature_run.assert_not_called() - - @with_feature(["organizations:seer-credit-checks", "organizations:token-billed-seer-enabled"]) - @patch( - "sentry.seer_credit.backend.check_seer_credits", - return_value=CheckSeerCreditsResponse(has_enough_credits=False), - ) - def test_agentic_triage_run_skips_credit_check( - self, mock_check_seer_credits: MagicMock - ) -> None: - """The run's own referrer decides, not the reply's.""" - self.seer_run.update(referrer=AutofixReferrer.AGENTIC_TRIAGE) - - self._reply() - - mock_check_seer_credits.assert_not_called() From 99eb2acddfff3a71f38b84b997d3d98c300fbf0b Mon Sep 17 00:00:00 2001 From: srest2021 Date: Fri, 6 Nov 2026 12:43:19 -0800 Subject: [PATCH 10/25] fix(seer): Don't let API callers claim the agentic triage referrer Agentic triage runs skip the Seer credit check, and the Autofix endpoint accepted any referrer from the request body. Treat a client-supplied agentic triage referrer as unknown. --- src/sentry/seer/endpoints/group_ai_autofix.py | 7 ++++++- .../seer/endpoints/test_group_ai_autofix.py | 19 +++++++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/src/sentry/seer/endpoints/group_ai_autofix.py b/src/sentry/seer/endpoints/group_ai_autofix.py index 4767dde92f13..93010d501f0e 100644 --- a/src/sentry/seer/endpoints/group_ai_autofix.py +++ b/src/sentry/seer/endpoints/group_ai_autofix.py @@ -165,10 +165,15 @@ def _parse_autofix_referrer(raw: str | None, request: Request) -> AutofixReferre return AutofixReferrer.MCP return AutofixReferrer.GROUP_AUTOFIX_ENDPOINT try: - return AutofixReferrer(raw) + referrer = AutofixReferrer(raw) except ValueError: logger.warning("group_ai_autofix.unknown_referrer", extra={"referrer": raw}) return AutofixReferrer.UNKNOWN + # Agentic triage runs skip the credit check, so callers can't claim to be it. + if referrer == AutofixReferrer.AGENTIC_TRIAGE: + logger.warning("group_ai_autofix.internal_referrer", extra={"referrer": raw}) + return AutofixReferrer.UNKNOWN + return referrer def _record_user_context_length(request_data: Any) -> None: diff --git a/tests/sentry/seer/endpoints/test_group_ai_autofix.py b/tests/sentry/seer/endpoints/test_group_ai_autofix.py index 10c804a02408..f4ab9f832b8c 100644 --- a/tests/sentry/seer/endpoints/test_group_ai_autofix.py +++ b/tests/sentry/seer/endpoints/test_group_ai_autofix.py @@ -1198,6 +1198,25 @@ def test_post_seer_explorer_referrer_is_recognized( assert response.status_code == 202, response.data assert mock_trigger_explorer.call_args.kwargs["referrer"] == AutofixReferrer.SEER_EXPLORER + @patch("sentry.seer.endpoints.group_ai_autofix.trigger_autofix_agent") + def test_post_agentic_triage_referrer_is_unknown( + self, mock_trigger_explorer: MagicMock + ) -> None: + """Agentic triage runs skip the credit check, so the API can't claim to be one.""" + group = self.create_group() + run = self.create_seer_run(organization=self.organization, seer_run_state_id=123) + mock_trigger_explorer.return_value = run + + self.login_as(user=self.user) + response = self.client.post( + self._get_url(group.id), + data={"step": "root_cause", "referrer": AutofixReferrer.AGENTIC_TRIAGE.value}, + format="json", + ) + + assert response.status_code == 202, response.data + assert mock_trigger_explorer.call_args.kwargs["referrer"] == AutofixReferrer.UNKNOWN + @patch("sentry.seer.endpoints.group_ai_autofix.trigger_autofix_agent") def test_stopping_point(self, mock_trigger_explorer: MagicMock) -> None: """Stopping point forces the step to be root_cause""" From e8f415e822486ba13a99ce8f40d9a8a0def89f58 Mon Sep 17 00:00:00 2001 From: srest2021 Date: Fri, 6 Nov 2026 12:58:04 -0800 Subject: [PATCH 11/25] ref(seer): Clarify why the endpoint ignores the agentic triage referrer --- src/sentry/seer/endpoints/group_ai_autofix.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/sentry/seer/endpoints/group_ai_autofix.py b/src/sentry/seer/endpoints/group_ai_autofix.py index 93010d501f0e..4b6bfb155757 100644 --- a/src/sentry/seer/endpoints/group_ai_autofix.py +++ b/src/sentry/seer/endpoints/group_ai_autofix.py @@ -169,7 +169,8 @@ def _parse_autofix_referrer(raw: str | None, request: Request) -> AutofixReferre except ValueError: logger.warning("group_ai_autofix.unknown_referrer", extra={"referrer": raw}) return AutofixReferrer.UNKNOWN - # Agentic triage runs skip the credit check, so callers can't claim to be it. + # Real agentic triage runs never come through this endpoint, so don't trust a + # request claiming to be one. Agentic triage runs skip the Seer credit check. if referrer == AutofixReferrer.AGENTIC_TRIAGE: logger.warning("group_ai_autofix.internal_referrer", extra={"referrer": raw}) return AutofixReferrer.UNKNOWN From 396779d7c000946eb2fba43bd4360e90f3824eb2 Mon Sep 17 00:00:00 2001 From: srest2021 Date: Fri, 6 Nov 2026 12:59:06 -0800 Subject: [PATCH 12/25] feat(seer): Check Seer credits before replying to an Autofix single run Restores the single run reply check, using the placeholder estimate for now. --- src/sentry/seer/autofix/feature/dispatch.py | 8 +++++ src/sentry/seer/credits.py | 4 +++ .../seer/autofix/feature/test_dispatch.py | 36 +++++++++++++++++++ 3 files changed, 48 insertions(+) diff --git a/src/sentry/seer/autofix/feature/dispatch.py b/src/sentry/seer/autofix/feature/dispatch.py index 408b573b403e..82b9a0b31dc8 100644 --- a/src/sentry/seer/autofix/feature/dispatch.py +++ b/src/sentry/seer/autofix/feature/dispatch.py @@ -160,6 +160,14 @@ def continue_single_run( ) -> SeerRun: from sentry.seer.autofix.on_completion_hook import AutofixOnCompletionHook + # A reply always continues a run, so it never uses quota. + check_autofix_billing( + group, + SeerOperation.AUTOFIX_SINGLE_RUN_REPLY, + agent_run.run.referrer, + is_new_run=False, + ) + enable_coding = _org_allows_coding(group) open_pr = enable_coding and can_publish payload = AutofixSingleRunFeaturePayload( diff --git a/src/sentry/seer/credits.py b/src/sentry/seer/credits.py index 5a894919c0a3..b945dfb0ea83 100644 --- a/src/sentry/seer/credits.py +++ b/src/sentry/seer/credits.py @@ -19,6 +19,7 @@ class SeerOperation(StrEnum): AUTOFIX_CODE_CHANGES = "autofix_code_changes" AUTOFIX_OPEN_PR = "autofix_open_pr" AUTOFIX_PR_ITERATION = "autofix_pr_iteration" + AUTOFIX_SINGLE_RUN_REPLY = "autofix_single_run_reply" class SeerFeature(StrEnum): @@ -57,6 +58,9 @@ class SeerOperationConfig: SeerOperation.AUTOFIX_PR_ITERATION: SeerOperationConfig( SeerFeature.AUTOFIX, lambda: PLACEHOLDER_CREDIT_ESTIMATE ), + SeerOperation.AUTOFIX_SINGLE_RUN_REPLY: SeerOperationConfig( + SeerFeature.AUTOFIX, lambda: PLACEHOLDER_CREDIT_ESTIMATE + ), } diff --git a/tests/sentry/seer/autofix/feature/test_dispatch.py b/tests/sentry/seer/autofix/feature/test_dispatch.py index 7264c3e2dbc1..ab47678c838f 100644 --- a/tests/sentry/seer/autofix/feature/test_dispatch.py +++ b/tests/sentry/seer/autofix/feature/test_dispatch.py @@ -692,3 +692,39 @@ def test_respects_org_coding_opt_out(self) -> None: assert kwargs["agent_run_options"]["enable_coding"] is False assert kwargs["agent_run_options"]["enable_write_pull_request"] is False assert kwargs["payload"]["pr_internal_settings"] is None + + @with_feature(["organizations:seer-credit-checks", "organizations:token-billed-seer-enabled"]) + @patch( + "sentry.seer_credit.backend.check_seer_credits", + return_value=CheckSeerCreditsResponse(has_enough_credits=False), + ) + def test_raises_when_out_of_credits(self, mock_check_seer_credits: MagicMock) -> None: + with ( + patch("sentry.seer.autofix.feature.dispatch.SeerAgentClient") as mock_client_cls, + pytest.raises(SeerOutOfCreditsException), + ): + continue_single_run( + self.agent_run, + self.group, + "make a code fix", + user=self.user, + can_publish=True, + referrer=AutofixReferrer.SEER_EXPLORER, + ) + + mock_client_cls.return_value.continue_feature_run.assert_not_called() + + @with_feature(["organizations:seer-credit-checks", "organizations:token-billed-seer-enabled"]) + @patch( + "sentry.seer_credit.backend.check_seer_credits", + return_value=CheckSeerCreditsResponse(has_enough_credits=False), + ) + def test_agentic_triage_run_skips_credit_check( + self, mock_check_seer_credits: MagicMock + ) -> None: + """The run's own referrer decides, not the reply's.""" + self.seer_run.update(referrer=AutofixReferrer.AGENTIC_TRIAGE) + + self._reply() + + mock_check_seer_credits.assert_not_called() From 90c5db5f8f301e460de7cd339f3e3358bf6c2ba6 Mon Sep 17 00:00:00 2001 From: srest2021 Date: Fri, 6 Nov 2026 12:40:39 -0800 Subject: [PATCH 13/25] cleaning up diff --- src/sentry/seer/autofix/feature/dispatch.py | 5 +++-- src/sentry/seer/credits.py | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/sentry/seer/autofix/feature/dispatch.py b/src/sentry/seer/autofix/feature/dispatch.py index 82b9a0b31dc8..326785f2f95c 100644 --- a/src/sentry/seer/autofix/feature/dispatch.py +++ b/src/sentry/seer/autofix/feature/dispatch.py @@ -295,11 +295,12 @@ def trigger_autofix_feature( # Avoid a circular import through the legacy Autofix dispatcher. from sentry.seer.autofix.on_completion_hook import AutofixOnCompletionHook + is_new_run = args.existing_run_id is None uses_quota = check_autofix_billing( group, args.step.seer_operation, args.referrer, - is_new_run=args.existing_run_id is None, + is_new_run=is_new_run, skip_quota_for_free_cohort=args.allow_free_cohort, ) @@ -391,7 +392,7 @@ def trigger_autofix_feature( extras["stopping_point"] = stopping_point.value user_org_context = collect_user_org_context(args.user, group.organization) - if args.existing_run_id is None: + if is_new_run: run = client.start_feature_run( feature_id=SINGLE_RUN_FEATURE_ID if single_run else FEATURE_ID, payload=payload.dict(), diff --git a/src/sentry/seer/credits.py b/src/sentry/seer/credits.py index b945dfb0ea83..d6cff3e7f32d 100644 --- a/src/sentry/seer/credits.py +++ b/src/sentry/seer/credits.py @@ -66,7 +66,7 @@ class SeerOperationConfig: def has_enough_seer_credits(organization: Organization, operation: SeerOperation) -> bool: """ - Checks whether the organization has enough Seer credits to run the operation + Checks whether the organization has enough Seer credits to run the operation, for callers that already checked the credit check and token-billed flags. """ config = SEER_OPERATIONS[operation] @@ -82,7 +82,7 @@ def has_enough_seer_credits(organization: Organization, operation: SeerOperation def can_run_seer_operation(organization: Organization, operation: SeerOperation) -> bool: - """Whether the organization is token-billed and has enough Seer credits to run the operation.""" + """Checks whether the organization is token-billed and has enough Seer credits to run the operation.""" if not features.has("organizations:seer-credit-checks", organization) or not features.has( "organizations:token-billed-seer-enabled", organization ): From ff4faaef48b3f2ac633a425603f81f04d85996bd Mon Sep 17 00:00:00 2001 From: srest2021 Date: Fri, 6 Nov 2026 13:05:09 -0800 Subject: [PATCH 14/25] ref(seer): Restore the billing comment in trigger_autofix_agent --- src/sentry/seer/autofix/autofix_agent.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/sentry/seer/autofix/autofix_agent.py b/src/sentry/seer/autofix/autofix_agent.py index 588cec9a4705..275a475ee4d5 100644 --- a/src/sentry/seer/autofix/autofix_agent.py +++ b/src/sentry/seer/autofix/autofix_agent.py @@ -463,6 +463,11 @@ def trigger_autofix_agent( """ outcome = "success" try: + # check billing quota for triggering a new autofix run + # Free cohort orgs bypass quota only when called from agentic triage + # (allow_free_cohort=True). The API endpoint never sets this flag, + # so manual triggers still require quota. + # Token-billed orgs are checked for credits on every step instead. check_autofix_billing( group, step.seer_operation, From b86335d8d96a012ea9b3d57d971a624168cabad9 Mon Sep 17 00:00:00 2001 From: srest2021 Date: Fri, 6 Nov 2026 13:18:02 -0800 Subject: [PATCH 15/25] cleaning up --- src/sentry/seer/autofix/autofix_agent.py | 3 +-- src/sentry/seer/autofix/feature/dispatch.py | 1 - src/sentry/seer/endpoints/group_ai_autofix.py | 4 ++-- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/src/sentry/seer/autofix/autofix_agent.py b/src/sentry/seer/autofix/autofix_agent.py index 275a475ee4d5..f7a2f66c5f2b 100644 --- a/src/sentry/seer/autofix/autofix_agent.py +++ b/src/sentry/seer/autofix/autofix_agent.py @@ -463,7 +463,7 @@ def trigger_autofix_agent( """ outcome = "success" try: - # check billing quota for triggering a new autofix run + # Seat-based orgs are checked for billing quota when triggering a new autofix run. # Free cohort orgs bypass quota only when called from agentic triage # (allow_free_cohort=True). The API endpoint never sets this flag, # so manual triggers still require quota. @@ -848,7 +848,6 @@ def trigger_push_changes( seer_run = SeerRun.objects.filter( organization_id=group.organization.id, seer_run_state_id=run_id ).first() - # Open PR always continues a run, so it never uses quota. check_autofix_billing( group, SeerOperation.AUTOFIX_OPEN_PR, diff --git a/src/sentry/seer/autofix/feature/dispatch.py b/src/sentry/seer/autofix/feature/dispatch.py index 326785f2f95c..e2b7fda54543 100644 --- a/src/sentry/seer/autofix/feature/dispatch.py +++ b/src/sentry/seer/autofix/feature/dispatch.py @@ -160,7 +160,6 @@ def continue_single_run( ) -> SeerRun: from sentry.seer.autofix.on_completion_hook import AutofixOnCompletionHook - # A reply always continues a run, so it never uses quota. check_autofix_billing( group, SeerOperation.AUTOFIX_SINGLE_RUN_REPLY, diff --git a/src/sentry/seer/endpoints/group_ai_autofix.py b/src/sentry/seer/endpoints/group_ai_autofix.py index 4b6bfb155757..1d8af3cdc4cf 100644 --- a/src/sentry/seer/endpoints/group_ai_autofix.py +++ b/src/sentry/seer/endpoints/group_ai_autofix.py @@ -169,8 +169,8 @@ def _parse_autofix_referrer(raw: str | None, request: Request) -> AutofixReferre except ValueError: logger.warning("group_ai_autofix.unknown_referrer", extra={"referrer": raw}) return AutofixReferrer.UNKNOWN - # Real agentic triage runs never come through this endpoint, so don't trust a - # request claiming to be one. Agentic triage runs skip the Seer credit check. + # Agentic triage is not exposed via the API. Don't trust such a referrer since + # it may have token-based billing implications. if referrer == AutofixReferrer.AGENTIC_TRIAGE: logger.warning("group_ai_autofix.internal_referrer", extra={"referrer": raw}) return AutofixReferrer.UNKNOWN From 245aca91af2f299185fcdf5ee22924c413b5457a Mon Sep 17 00:00:00 2001 From: srest2021 Date: Fri, 6 Nov 2026 13:19:02 -0800 Subject: [PATCH 16/25] fix ordering --- src/sentry/seer/autofix/feature/dispatch.py | 120 ++++++++++---------- 1 file changed, 60 insertions(+), 60 deletions(-) diff --git a/src/sentry/seer/autofix/feature/dispatch.py b/src/sentry/seer/autofix/feature/dispatch.py index e2b7fda54543..112efc9ade08 100644 --- a/src/sentry/seer/autofix/feature/dispatch.py +++ b/src/sentry/seer/autofix/feature/dispatch.py @@ -149,66 +149,6 @@ def _org_allows_coding(group: Group) -> bool: ) -def continue_single_run( - agent_run: SeerAgentRun, - group: Group, - reply: str, - *, - user: User | RpcUser, - can_publish: bool, - referrer: AutofixReferrer, -) -> SeerRun: - from sentry.seer.autofix.on_completion_hook import AutofixOnCompletionHook - - check_autofix_billing( - group, - SeerOperation.AUTOFIX_SINGLE_RUN_REPLY, - agent_run.run.referrer, - is_new_run=False, - ) - - enable_coding = _org_allows_coding(group) - open_pr = enable_coding and can_publish - payload = AutofixSingleRunFeaturePayload( - group_id=group.id, - project_id=group.project_id, - short_id=group.qualified_short_id or str(group.id), - title=group.title or "Unknown error", - culprit=group.culprit or "unknown", - on_completion_hook=extract_hook_definition(AutofixOnCompletionHook, call_on_failure=True), - should_run_repo_checks=True, - existing_run_id=agent_run.run.seer_run_state_id, - reply=reply, - pr_internal_settings=( - _single_run_pr_internal_settings(group, referrer) if open_pr else None - ), - ) - agent_run_options = AgentRunOptions( - is_context_engine_enabled=False, - enable_frontend_code_search=False, - enable_coding=enable_coding, - enable_pr_context_tools=False, - enable_write_pull_request=open_pr, - ) - - client = SeerAgentClient( - organization=group.organization, - project=group.project, - group=group, - user=user, - enable_coding=enable_coding, - enable_bash_mode=True, - ) - return client.continue_feature_run( - existing_agent_run=agent_run, - payload=payload.dict(), - referrer=referrer.value, - user_org_context=collect_user_org_context(user, group.organization), - proxy_headers=get_proxy_headers(), - agent_run_options=agent_run_options, - ) - - def _check_autofix_credits(group: Group, operation: SeerOperation, referrer: str | None) -> None: """ Raise if the organization can't afford the operation under token-based @@ -287,6 +227,66 @@ def check_autofix_billing( ) +def continue_single_run( + agent_run: SeerAgentRun, + group: Group, + reply: str, + *, + user: User | RpcUser, + can_publish: bool, + referrer: AutofixReferrer, +) -> SeerRun: + from sentry.seer.autofix.on_completion_hook import AutofixOnCompletionHook + + check_autofix_billing( + group, + SeerOperation.AUTOFIX_SINGLE_RUN_REPLY, + agent_run.run.referrer, + is_new_run=False, + ) + + enable_coding = _org_allows_coding(group) + open_pr = enable_coding and can_publish + payload = AutofixSingleRunFeaturePayload( + group_id=group.id, + project_id=group.project_id, + short_id=group.qualified_short_id or str(group.id), + title=group.title or "Unknown error", + culprit=group.culprit or "unknown", + on_completion_hook=extract_hook_definition(AutofixOnCompletionHook, call_on_failure=True), + should_run_repo_checks=True, + existing_run_id=agent_run.run.seer_run_state_id, + reply=reply, + pr_internal_settings=( + _single_run_pr_internal_settings(group, referrer) if open_pr else None + ), + ) + agent_run_options = AgentRunOptions( + is_context_engine_enabled=False, + enable_frontend_code_search=False, + enable_coding=enable_coding, + enable_pr_context_tools=False, + enable_write_pull_request=open_pr, + ) + + client = SeerAgentClient( + organization=group.organization, + project=group.project, + group=group, + user=user, + enable_coding=enable_coding, + enable_bash_mode=True, + ) + return client.continue_feature_run( + existing_agent_run=agent_run, + payload=payload.dict(), + referrer=referrer.value, + user_org_context=collect_user_org_context(user, group.organization), + proxy_headers=get_proxy_headers(), + agent_run_options=agent_run_options, + ) + + def trigger_autofix_feature( group: Group, args: AutofixFeatureArgs, From 80bba2393003f19a88955e554663a45d757ebc2a Mon Sep 17 00:00:00 2001 From: srest2021 Date: Fri, 6 Nov 2026 13:21:07 -0800 Subject: [PATCH 17/25] Restore comment --- src/sentry/seer/autofix/feature/dispatch.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/sentry/seer/autofix/feature/dispatch.py b/src/sentry/seer/autofix/feature/dispatch.py index 112efc9ade08..419de6744f38 100644 --- a/src/sentry/seer/autofix/feature/dispatch.py +++ b/src/sentry/seer/autofix/feature/dispatch.py @@ -300,6 +300,8 @@ def trigger_autofix_feature( args.step.seer_operation, args.referrer, is_new_run=is_new_run, + # Free cohort orgs bypass quota only when called from agentic triage + # (allow_free_cohort=True). Not exposed via the API. skip_quota_for_free_cohort=args.allow_free_cohort, ) From 914a60cd07bc47d7d50df3f05db2a6f47a1ffda2 Mon Sep 17 00:00:00 2001 From: srest2021 Date: Fri, 6 Nov 2026 13:24:39 -0800 Subject: [PATCH 18/25] ref(seer): Keep all billing skip rules in check_autofix_billing --- src/sentry/seer/autofix/feature/dispatch.py | 89 +++++++++------------ 1 file changed, 36 insertions(+), 53 deletions(-) diff --git a/src/sentry/seer/autofix/feature/dispatch.py b/src/sentry/seer/autofix/feature/dispatch.py index 419de6744f38..ad74466886e0 100644 --- a/src/sentry/seer/autofix/feature/dispatch.py +++ b/src/sentry/seer/autofix/feature/dispatch.py @@ -149,61 +149,22 @@ def _org_allows_coding(group: Group) -> bool: ) -def _check_autofix_credits(group: Group, operation: SeerOperation, referrer: str | None) -> None: - """ - Raise if the organization can't afford the operation under token-based - credits. Agentic triage runs stay free. - """ - if ( - not features.has("organizations:seer-credit-checks", group.organization) - or referrer == AutofixReferrer.AGENTIC_TRIAGE - ): - return - if not has_enough_seer_credits(group.organization, operation): - logger.info( - "autofix_feature.dispatch.out_of_credits", +def _check_autofix_quota(group: Group, referrer: str | None) -> None: + """Raise if the organization has no seat-based Seer quota left.""" + has_budget: bool = quotas.backend.check_seer_quota( + org_id=group.organization.id, + data_category=DataCategory.SEER_AUTOFIX, + ) + if not has_budget: + logger.warning( + "autofix_feature.dispatch.quota_denied", extra={ "group_id": group.id, "organization_id": group.organization.id, - "operation": operation.value, "referrer": referrer, }, ) - raise SeerOutOfCreditsException() - - -def _check_autofix_quota( - group: Group, - referrer: str | None, - *, - is_new_run: bool, - allow_free_cohort: bool, -) -> bool: - """ - Raise if the organization has no seat-based Seer quota left. Returns whether - the step uses quota, which is recorded once the run starts. - """ - # Free cohort orgs bypass quota only when called from agentic triage - # (allow_free_cohort=True). Not exposed via the API. - skip_quota = is_new_run and allow_free_cohort and is_free_cohort_org(group.organization) - uses_quota = is_new_run and not skip_quota - if uses_quota: - has_budget: bool = quotas.backend.check_seer_quota( - org_id=group.organization.id, - data_category=DataCategory.SEER_AUTOFIX, - ) - if not has_budget: - logger.warning( - "autofix_feature.dispatch.quota_denied", - extra={ - "group_id": group.id, - "organization_id": group.organization.id, - "referrer": referrer, - }, - ) - raise NoSeerQuotaException() - - return uses_quota + raise NoSeerQuotaException() def check_autofix_billing( @@ -219,12 +180,34 @@ def check_autofix_billing( checked for credits, everyone else for seat-based quota. Returns whether the operation uses quota. """ - if features.has("organizations:token-billed-seer-enabled", group.organization): - _check_autofix_credits(group, operation, referrer) + organization = group.organization + if features.has("organizations:token-billed-seer-enabled", organization): + if ( + features.has("organizations:seer-credit-checks", organization) + and referrer != AutofixReferrer.AGENTIC_TRIAGE # Agentic triage runs stay free. + and not has_enough_seer_credits(organization, operation) + ): + logger.info( + "autofix_feature.dispatch.out_of_credits", + extra={ + "group_id": group.id, + "organization_id": organization.id, + "operation": operation.value, + "referrer": referrer, + }, + ) + raise SeerOutOfCreditsException() return False - return _check_autofix_quota( - group, referrer, is_new_run=is_new_run, allow_free_cohort=skip_quota_for_free_cohort + + # Quota counts runs, so only new runs use it. + # Free cohort orgs bypass quota only when called from agentic triage + # (skip_quota_for_free_cohort=True). Agentic triage is not exposed via the API. + uses_quota = is_new_run and not ( + skip_quota_for_free_cohort and is_free_cohort_org(organization) ) + if uses_quota: + _check_autofix_quota(group, referrer) + return uses_quota def continue_single_run( From 28e59907cf76b2a9b874161de47b5bfaf9825327 Mon Sep 17 00:00:00 2001 From: srest2021 Date: Fri, 6 Nov 2026 13:33:09 -0800 Subject: [PATCH 19/25] refactor billing helper --- src/sentry/seer/autofix/feature/dispatch.py | 50 ++++++++++----------- 1 file changed, 23 insertions(+), 27 deletions(-) diff --git a/src/sentry/seer/autofix/feature/dispatch.py b/src/sentry/seer/autofix/feature/dispatch.py index ad74466886e0..4c650cca6ad9 100644 --- a/src/sentry/seer/autofix/feature/dispatch.py +++ b/src/sentry/seer/autofix/feature/dispatch.py @@ -149,24 +149,6 @@ def _org_allows_coding(group: Group) -> bool: ) -def _check_autofix_quota(group: Group, referrer: str | None) -> None: - """Raise if the organization has no seat-based Seer quota left.""" - has_budget: bool = quotas.backend.check_seer_quota( - org_id=group.organization.id, - data_category=DataCategory.SEER_AUTOFIX, - ) - if not has_budget: - logger.warning( - "autofix_feature.dispatch.quota_denied", - extra={ - "group_id": group.id, - "organization_id": group.organization.id, - "referrer": referrer, - }, - ) - raise NoSeerQuotaException() - - def check_autofix_billing( group: Group, operation: SeerOperation, @@ -177,10 +159,12 @@ def check_autofix_billing( ) -> bool: """ Raise if the organization can't pay for the operation. Token-billed orgs are - checked for credits, everyone else for seat-based quota. Returns whether the - operation uses quota. + checked for credits, and everyone else is checked for quota once per run. + Returns whether the operation uses quota. """ organization = group.organization + + # Check credits for token-billed orgs. if features.has("organizations:token-billed-seer-enabled", organization): if ( features.has("organizations:seer-credit-checks", organization) @@ -199,14 +183,26 @@ def check_autofix_billing( raise SeerOutOfCreditsException() return False - # Quota counts runs, so only new runs use it. - # Free cohort orgs bypass quota only when called from agentic triage - # (skip_quota_for_free_cohort=True). Agentic triage is not exposed via the API. - uses_quota = is_new_run and not ( - skip_quota_for_free_cohort and is_free_cohort_org(organization) - ) + # Check quota for non-token-billed orgs once per run. + # Free cohort orgs skip quota when the caller passes skip_quota_for_free_cohort=True. + # Only agentic triage does, and it is not exposed via API. + skips_quota = skip_quota_for_free_cohort and is_free_cohort_org(organization) + uses_quota = is_new_run and not skips_quota if uses_quota: - _check_autofix_quota(group, referrer) + has_budget: bool = quotas.backend.check_seer_quota( + org_id=group.organization.id, + data_category=DataCategory.SEER_AUTOFIX, + ) + if not has_budget: + logger.warning( + "autofix_feature.dispatch.quota_denied", + extra={ + "group_id": group.id, + "organization_id": group.organization.id, + "referrer": referrer, + }, + ) + raise NoSeerQuotaException() return uses_quota From 766bc10bc13f1811b71349e821d3726b331b7815 Mon Sep 17 00:00:00 2001 From: srest2021 Date: Fri, 6 Nov 2026 13:39:40 -0800 Subject: [PATCH 20/25] move logs out of helper --- src/sentry/seer/autofix/feature/dispatch.py | 69 +++++++++++---------- 1 file changed, 36 insertions(+), 33 deletions(-) diff --git a/src/sentry/seer/autofix/feature/dispatch.py b/src/sentry/seer/autofix/feature/dispatch.py index 4c650cca6ad9..f636c20c0b88 100644 --- a/src/sentry/seer/autofix/feature/dispatch.py +++ b/src/sentry/seer/autofix/feature/dispatch.py @@ -171,15 +171,6 @@ def check_autofix_billing( and referrer != AutofixReferrer.AGENTIC_TRIAGE # Agentic triage runs stay free. and not has_enough_seer_credits(organization, operation) ): - logger.info( - "autofix_feature.dispatch.out_of_credits", - extra={ - "group_id": group.id, - "organization_id": organization.id, - "operation": operation.value, - "referrer": referrer, - }, - ) raise SeerOutOfCreditsException() return False @@ -188,21 +179,11 @@ def check_autofix_billing( # Only agentic triage does, and it is not exposed via API. skips_quota = skip_quota_for_free_cohort and is_free_cohort_org(organization) uses_quota = is_new_run and not skips_quota - if uses_quota: - has_budget: bool = quotas.backend.check_seer_quota( - org_id=group.organization.id, - data_category=DataCategory.SEER_AUTOFIX, - ) - if not has_budget: - logger.warning( - "autofix_feature.dispatch.quota_denied", - extra={ - "group_id": group.id, - "organization_id": group.organization.id, - "referrer": referrer, - }, - ) - raise NoSeerQuotaException() + if uses_quota and not quotas.backend.check_seer_quota( + org_id=organization.id, + data_category=DataCategory.SEER_AUTOFIX, + ): + raise NoSeerQuotaException() return uses_quota @@ -274,15 +255,37 @@ def trigger_autofix_feature( from sentry.seer.autofix.on_completion_hook import AutofixOnCompletionHook is_new_run = args.existing_run_id is None - uses_quota = check_autofix_billing( - group, - args.step.seer_operation, - args.referrer, - is_new_run=is_new_run, - # Free cohort orgs bypass quota only when called from agentic triage - # (allow_free_cohort=True). Not exposed via the API. - skip_quota_for_free_cohort=args.allow_free_cohort, - ) + try: + uses_quota = check_autofix_billing( + group, + args.step.seer_operation, + args.referrer, + is_new_run=is_new_run, + # Free cohort orgs bypass quota only when called from agentic triage + # (allow_free_cohort=True). Not exposed via the API. + skip_quota_for_free_cohort=args.allow_free_cohort, + ) + except NoSeerQuotaException: + logger.warning( + "autofix_feature.dispatch.quota_denied", + extra={ + "group_id": group.id, + "organization_id": group.organization.id, + "referrer": args.referrer.value, + }, + ) + raise + except SeerOutOfCreditsException: + logger.info( + "autofix_feature.dispatch.out_of_credits", + extra={ + "group_id": group.id, + "organization_id": group.organization.id, + "operation": args.step.seer_operation.value, + "referrer": args.referrer.value, + }, + ) + raise single_run = should_use_single_run( group, From 6c0043dddb83fe8e502df025c1719775a275a7be Mon Sep 17 00:00:00 2001 From: srest2021 Date: Fri, 6 Nov 2026 13:45:45 -0800 Subject: [PATCH 21/25] small test fix: --- tests/sentry/seer/autofix/feature/test_dispatch.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/sentry/seer/autofix/feature/test_dispatch.py b/tests/sentry/seer/autofix/feature/test_dispatch.py index ab47678c838f..2bf923774c12 100644 --- a/tests/sentry/seer/autofix/feature/test_dispatch.py +++ b/tests/sentry/seer/autofix/feature/test_dispatch.py @@ -348,7 +348,6 @@ def test_agentic_triage_skips_credit_check(self, mock_check_seer_credits: MagicM patch("sentry.seer.autofix.feature.dispatch.SeerAgentClient") as MockClient, patch("sentry.seer.autofix.feature.dispatch.quotas") as mock_quotas, ): - mock_quotas.backend.check_seer_quota.return_value = True MockClient.return_value.start_feature_run.return_value = fake_run run = trigger_autofix_feature( @@ -362,6 +361,8 @@ def test_agentic_triage_skips_credit_check(self, mock_check_seer_credits: MagicM assert run is fake_run mock_check_seer_credits.assert_not_called() + mock_quotas.backend.check_seer_quota.assert_not_called() + mock_quotas.backend.record_seer_run.assert_not_called() def test_allows_async_dispatch(self) -> None: fake_run = self.create_seer_run(organization=self.organization, type="feature_run") From 1c7c1906056cd11fc5818b36256c423ed39a38eb Mon Sep 17 00:00:00 2001 From: srest2021 Date: Fri, 6 Nov 2026 13:49:40 -0800 Subject: [PATCH 22/25] cleaning up tests --- tests/sentry/seer/autofix/feature/test_dispatch.py | 7 ++----- tests/sentry/seer/autofix/test_autofix_agent.py | 2 -- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/tests/sentry/seer/autofix/feature/test_dispatch.py b/tests/sentry/seer/autofix/feature/test_dispatch.py index 2bf923774c12..751f07dd2818 100644 --- a/tests/sentry/seer/autofix/feature/test_dispatch.py +++ b/tests/sentry/seer/autofix/feature/test_dispatch.py @@ -320,7 +320,7 @@ def test_raises_when_out_of_credits_for_a_continued_step( """Unlike quota, credits are checked for every step, not only new runs.""" with ( patch("sentry.seer.autofix.feature.dispatch.SeerAgentClient") as MockClient, - patch("sentry.seer.autofix.feature.dispatch.quotas") as mock_quotas, + patch("sentry.seer.autofix.feature.dispatch.quotas"), ): with pytest.raises(SeerOutOfCreditsException): trigger_autofix_feature( @@ -333,7 +333,6 @@ def test_raises_when_out_of_credits_for_a_continued_step( ), ) - mock_quotas.backend.check_seer_quota.assert_not_called() MockClient.return_value.continue_feature_run.assert_not_called() @with_feature(["organizations:seer-credit-checks", "organizations:token-billed-seer-enabled"]) @@ -346,7 +345,7 @@ def test_agentic_triage_skips_credit_check(self, mock_check_seer_credits: MagicM with ( patch("sentry.seer.autofix.feature.dispatch.SeerAgentClient") as MockClient, - patch("sentry.seer.autofix.feature.dispatch.quotas") as mock_quotas, + patch("sentry.seer.autofix.feature.dispatch.quotas"), ): MockClient.return_value.start_feature_run.return_value = fake_run @@ -361,8 +360,6 @@ def test_agentic_triage_skips_credit_check(self, mock_check_seer_credits: MagicM assert run is fake_run mock_check_seer_credits.assert_not_called() - mock_quotas.backend.check_seer_quota.assert_not_called() - mock_quotas.backend.record_seer_run.assert_not_called() def test_allows_async_dispatch(self) -> None: fake_run = self.create_seer_run(organization=self.organization, type="feature_run") diff --git a/tests/sentry/seer/autofix/test_autofix_agent.py b/tests/sentry/seer/autofix/test_autofix_agent.py index dd598ac0a01e..af9eb7ef1951 100644 --- a/tests/sentry/seer/autofix/test_autofix_agent.py +++ b/tests/sentry/seer/autofix/test_autofix_agent.py @@ -821,8 +821,6 @@ def test_agentic_triage_skips_credit_check( ) mock_check_seer_credits.assert_not_called() - mock_check_quota.assert_not_called() - mock_feature.assert_called_once() @patch("sentry.quotas.backend.check_seer_quota", return_value=False) @patch("sentry.seer.autofix.autofix_agent.broadcast_webhooks_for_organization.delay") From 451f3bda8485afed7a78612d1894e2aeb4f5cc05 Mon Sep 17 00:00:00 2001 From: "sentry-junior[bot]" <264270552+sentry-junior[bot]@users.noreply.github.com> Date: Mon, 9 Nov 2026 19:28:49 +0000 Subject: [PATCH 23/25] ref(seer-billing): Pass organization to check_autofix_billing Co-Authored-By: Sofia Rest --- src/sentry/seer/autofix/autofix_agent.py | 4 ++-- src/sentry/seer/autofix/feature/dispatch.py | 9 ++++----- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/src/sentry/seer/autofix/autofix_agent.py b/src/sentry/seer/autofix/autofix_agent.py index f7a2f66c5f2b..9c138164c6f1 100644 --- a/src/sentry/seer/autofix/autofix_agent.py +++ b/src/sentry/seer/autofix/autofix_agent.py @@ -469,7 +469,7 @@ def trigger_autofix_agent( # so manual triggers still require quota. # Token-billed orgs are checked for credits on every step instead. check_autofix_billing( - group, + group.organization, step.seer_operation, referrer, is_new_run=run_id is None, @@ -849,7 +849,7 @@ def trigger_push_changes( organization_id=group.organization.id, seer_run_state_id=run_id ).first() check_autofix_billing( - group, + group.organization, SeerOperation.AUTOFIX_OPEN_PR, seer_run.referrer if seer_run else None, is_new_run=False, diff --git a/src/sentry/seer/autofix/feature/dispatch.py b/src/sentry/seer/autofix/feature/dispatch.py index f636c20c0b88..e5fc4c3ca0af 100644 --- a/src/sentry/seer/autofix/feature/dispatch.py +++ b/src/sentry/seer/autofix/feature/dispatch.py @@ -13,6 +13,7 @@ DataCategory, ) from sentry.models.group import Group +from sentry.models.organization import Organization from sentry.models.project import Project from sentry.seer.agent.client import SeerAgentClient from sentry.seer.agent.client_utils import ( @@ -150,7 +151,7 @@ def _org_allows_coding(group: Group) -> bool: def check_autofix_billing( - group: Group, + organization: Organization, operation: SeerOperation, referrer: str | None, *, @@ -162,8 +163,6 @@ def check_autofix_billing( checked for credits, and everyone else is checked for quota once per run. Returns whether the operation uses quota. """ - organization = group.organization - # Check credits for token-billed orgs. if features.has("organizations:token-billed-seer-enabled", organization): if ( @@ -199,7 +198,7 @@ def continue_single_run( from sentry.seer.autofix.on_completion_hook import AutofixOnCompletionHook check_autofix_billing( - group, + group.organization, SeerOperation.AUTOFIX_SINGLE_RUN_REPLY, agent_run.run.referrer, is_new_run=False, @@ -257,7 +256,7 @@ def trigger_autofix_feature( is_new_run = args.existing_run_id is None try: uses_quota = check_autofix_billing( - group, + group.organization, args.step.seer_operation, args.referrer, is_new_run=is_new_run, From ab0d0aa2223dd6df6afab58d1c8809a448508ae9 Mon Sep 17 00:00:00 2001 From: srest2021 Date: Mon, 9 Nov 2026 13:08:32 -0800 Subject: [PATCH 24/25] ref(seer-billing): Check Autofix billing only in dispatch trigger_autofix_agent checked billing, then called trigger_autofix_feature, which checked it again with the same arguments. Keep only the dispatch check, and test a token-billed org with credit checks off. --- src/sentry/seer/autofix/autofix_agent.py | 13 --- .../seer/autofix/feature/test_dispatch.py | 52 +++++++++++ .../sentry/seer/autofix/test_autofix_agent.py | 91 ------------------- 3 files changed, 52 insertions(+), 104 deletions(-) diff --git a/src/sentry/seer/autofix/autofix_agent.py b/src/sentry/seer/autofix/autofix_agent.py index 9c138164c6f1..1b8a2a6d609b 100644 --- a/src/sentry/seer/autofix/autofix_agent.py +++ b/src/sentry/seer/autofix/autofix_agent.py @@ -463,19 +463,6 @@ def trigger_autofix_agent( """ outcome = "success" try: - # Seat-based orgs are checked for billing quota when triggering a new autofix run. - # Free cohort orgs bypass quota only when called from agentic triage - # (allow_free_cohort=True). The API endpoint never sets this flag, - # so manual triggers still require quota. - # Token-billed orgs are checked for credits on every step instead. - check_autofix_billing( - group.organization, - step.seer_operation, - referrer, - is_new_run=run_id is None, - skip_quota_for_free_cohort=allow_free_cohort, - ) - if run_id is not None: _assert_existing_run_belongs_to_group(group, run_id) diff --git a/tests/sentry/seer/autofix/feature/test_dispatch.py b/tests/sentry/seer/autofix/feature/test_dispatch.py index 751f07dd2818..809188a99de6 100644 --- a/tests/sentry/seer/autofix/feature/test_dispatch.py +++ b/tests/sentry/seer/autofix/feature/test_dispatch.py @@ -283,6 +283,58 @@ def test_free_cohort_skips_quota_check_and_usage_recording(self) -> None: mock_quotas.backend.check_seer_quota.assert_not_called() mock_quotas.backend.record_seer_run.assert_not_called() + def test_non_free_cohort_org_checks_quota_and_records_usage(self) -> None: + fake_run = self.create_seer_run(organization=self.organization, type="feature_run") + + with ( + patch("sentry.seer.autofix.feature.dispatch.SeerAgentClient") as MockClient, + patch("sentry.seer.autofix.feature.dispatch.quotas") as mock_quotas, + patch("sentry.seer.autofix.feature.dispatch.is_free_cohort_org", return_value=False), + ): + mock_quotas.backend.check_seer_quota.return_value = True + MockClient.return_value.start_feature_run.return_value = fake_run + + run = trigger_autofix_feature( + self.group, + AutofixFeatureArgs( + referrer=AutofixReferrer.AGENTIC_TRIAGE, + step=AutofixStep.ROOT_CAUSE, + step_args=RCAStepArgs(), + allow_free_cohort=True, + ), + ) + + assert run is fake_run + mock_quotas.backend.check_seer_quota.assert_called_once() + mock_quotas.backend.record_seer_run.assert_called_once() + + @with_feature("organizations:token-billed-seer-enabled") + @patch("sentry.seer_credit.backend.check_seer_credits") + def test_token_billed_without_credit_checks_skips_quota_and_usage_recording( + self, mock_check_seer_credits: MagicMock + ) -> None: + fake_run = self.create_seer_run(organization=self.organization, type="feature_run") + + with ( + patch("sentry.seer.autofix.feature.dispatch.SeerAgentClient") as MockClient, + patch("sentry.seer.autofix.feature.dispatch.quotas") as mock_quotas, + ): + MockClient.return_value.start_feature_run.return_value = fake_run + + run = trigger_autofix_feature( + self.group, + AutofixFeatureArgs( + referrer=AutofixReferrer.GROUP_AUTOFIX_ENDPOINT, + step=AutofixStep.ROOT_CAUSE, + step_args=RCAStepArgs(), + ), + ) + + assert run is fake_run + mock_check_seer_credits.assert_not_called() + mock_quotas.backend.check_seer_quota.assert_not_called() + mock_quotas.backend.record_seer_run.assert_not_called() + @with_feature(["organizations:seer-credit-checks", "organizations:token-billed-seer-enabled"]) @patch( "sentry.seer_credit.backend.check_seer_credits", diff --git a/tests/sentry/seer/autofix/test_autofix_agent.py b/tests/sentry/seer/autofix/test_autofix_agent.py index af9eb7ef1951..95e41640240d 100644 --- a/tests/sentry/seer/autofix/test_autofix_agent.py +++ b/tests/sentry/seer/autofix/test_autofix_agent.py @@ -9,7 +9,6 @@ ) from sentry.analytics.events.autofix_events import AiAutofixSolutionCompletedEvent -from sentry.constants import DataCategory from sentry.models.activity import Activity from sentry.models.group import Group from sentry.seer.agent.client_models import ( @@ -708,46 +707,6 @@ def test_pr_iteration_requires_existing_pr( mock_feature.assert_not_called() - @patch("sentry.seer.autofix.autofix_agent.trigger_autofix_feature") - @patch("sentry.quotas.backend.check_seer_quota", return_value=False) - def test_when_no_quota(self, mock_check_quota: MagicMock, mock_feature: MagicMock) -> None: - with pytest.raises(NoSeerQuotaException): - trigger_autofix_agent( - group=self.group, - step=AutofixStep.CODE_CHANGES, - referrer=AutofixReferrer.UNKNOWN, - run_id=None, - ) - mock_check_quota.assert_called_once_with( - org_id=self.group.organization.id, - data_category=DataCategory.SEER_AUTOFIX, - ) - mock_feature.assert_not_called() - - @with_feature(["organizations:seer-credit-checks", "organizations:token-billed-seer-enabled"]) - @patch( - "sentry.seer_credit.backend.check_seer_credits", - return_value=CheckSeerCreditsResponse(has_enough_credits=False), - ) - @patch("sentry.seer.autofix.autofix_agent.trigger_autofix_feature") - @patch("sentry.quotas.backend.check_seer_quota", return_value=True) - def test_when_out_of_credits( - self, - mock_check_quota: MagicMock, - mock_feature: MagicMock, - mock_check_seer_credits: MagicMock, - ) -> None: - with pytest.raises(SeerOutOfCreditsException): - trigger_autofix_agent( - group=self.group, - step=AutofixStep.CODE_CHANGES, - referrer=AutofixReferrer.UNKNOWN, - run_id=None, - ) - mock_check_seer_credits.assert_called_once() - mock_check_quota.assert_not_called() - mock_feature.assert_not_called() - @patch("sentry.quotas.backend.check_seer_quota", return_value=True) @patch("sentry.seer.autofix.autofix_agent.broadcast_webhooks_for_organization.delay") @patch("sentry.seer.autofix.autofix_agent.trigger_autofix_feature") @@ -772,56 +731,6 @@ def test_free_cohort_skips_quota_check( mock_check_quota.assert_not_called() assert mock_feature.call_args.args[1].allow_free_cohort is True - @patch("sentry.quotas.backend.check_seer_quota", return_value=True) - @patch("sentry.seer.autofix.autofix_agent.broadcast_webhooks_for_organization.delay") - @patch("sentry.seer.autofix.autofix_agent.trigger_autofix_feature") - def test_allow_free_cohort_still_checks_quota_for_non_cohort_org( - self, mock_feature: MagicMock, mock_broadcast: MagicMock, mock_check_quota: MagicMock - ) -> None: - mock_feature.return_value = self.create_seer_run( - organization=self.group.organization, seer_run_state_id=12345 - ) - - with ( - patch("sentry.seer.autofix.autofix_agent.features.has", return_value=False), - patch("sentry.seer.autofix.feature.dispatch.is_free_cohort_org", return_value=False), - ): - trigger_autofix_agent( - group=self.group, - step=AutofixStep.CODE_CHANGES, - referrer=AutofixReferrer.AGENTIC_TRIAGE, - allow_free_cohort=True, - ) - - mock_check_quota.assert_called_once() - - @with_feature(["organizations:seer-credit-checks", "organizations:token-billed-seer-enabled"]) - @patch( - "sentry.seer_credit.backend.check_seer_credits", - return_value=CheckSeerCreditsResponse(has_enough_credits=False), - ) - @patch("sentry.quotas.backend.check_seer_quota", return_value=True) - @patch("sentry.seer.autofix.autofix_agent.broadcast_webhooks_for_organization.delay") - @patch("sentry.seer.autofix.autofix_agent.trigger_autofix_feature") - def test_agentic_triage_skips_credit_check( - self, - mock_feature: MagicMock, - mock_broadcast: MagicMock, - mock_check_quota: MagicMock, - mock_check_seer_credits: MagicMock, - ) -> None: - mock_feature.return_value = self.create_seer_run( - organization=self.group.organization, seer_run_state_id=12345 - ) - - trigger_autofix_agent( - group=self.group, - step=AutofixStep.CODE_CHANGES, - referrer=AutofixReferrer.AGENTIC_TRIAGE, - ) - - mock_check_seer_credits.assert_not_called() - @patch("sentry.quotas.backend.check_seer_quota", return_value=False) @patch("sentry.seer.autofix.autofix_agent.broadcast_webhooks_for_organization.delay") @patch("sentry.seer.autofix.autofix_agent.trigger_autofix_feature") From 89bdfdcc00ade7ac8fa9ff52c0b5b42c78ce0e5f Mon Sep 17 00:00:00 2001 From: srest2021 Date: Mon, 9 Nov 2026 15:25:38 -0800 Subject: [PATCH 25/25] ref(seer-billing): Drop the agentic triage referrer exemption Credit checks no longer skip runs by referrer. Agentic triage runs are checked like any other until a follow-up marks them by feature instead. Credit checks are behind seer-credit-checks, so nothing changes in production. Refs CW-3061 --- src/sentry/seer/autofix/autofix_agent.py | 14 +++---- src/sentry/seer/autofix/feature/dispatch.py | 11 ++--- src/sentry/seer/endpoints/group_ai_autofix.py | 8 +--- .../seer/autofix/feature/test_dispatch.py | 41 ------------------- .../sentry/seer/autofix/test_autofix_agent.py | 26 ------------ .../seer/endpoints/test_group_ai_autofix.py | 19 --------- 6 files changed, 9 insertions(+), 110 deletions(-) diff --git a/src/sentry/seer/autofix/autofix_agent.py b/src/sentry/seer/autofix/autofix_agent.py index 1b8a2a6d609b..af99e4c3e05f 100644 --- a/src/sentry/seer/autofix/autofix_agent.py +++ b/src/sentry/seer/autofix/autofix_agent.py @@ -832,15 +832,7 @@ def trigger_push_changes( ): raise PermissionDenied("Code generation is disabled for this organization") - seer_run = SeerRun.objects.filter( - organization_id=group.organization.id, seer_run_state_id=run_id - ).first() - check_autofix_billing( - group.organization, - SeerOperation.AUTOFIX_OPEN_PR, - seer_run.referrer if seer_run else None, - is_new_run=False, - ) + check_autofix_billing(group.organization, SeerOperation.AUTOFIX_OPEN_PR, is_new_run=False) client = get_autofix_agent_client(group, user=user) @@ -859,6 +851,10 @@ def trigger_push_changes( ) ) + seer_run = SeerRun.objects.filter( + organization_id=group.organization.id, seer_run_state_id=run_id + ).first() + # Draft when review-request/CI-green is enabled; mark ready once green fires. client.push_changes( run_id, diff --git a/src/sentry/seer/autofix/feature/dispatch.py b/src/sentry/seer/autofix/feature/dispatch.py index e5fc4c3ca0af..be25d1b1c476 100644 --- a/src/sentry/seer/autofix/feature/dispatch.py +++ b/src/sentry/seer/autofix/feature/dispatch.py @@ -153,7 +153,6 @@ def _org_allows_coding(group: Group) -> bool: def check_autofix_billing( organization: Organization, operation: SeerOperation, - referrer: str | None, *, is_new_run: bool, skip_quota_for_free_cohort: bool = False, @@ -165,11 +164,9 @@ def check_autofix_billing( """ # Check credits for token-billed orgs. if features.has("organizations:token-billed-seer-enabled", organization): - if ( - features.has("organizations:seer-credit-checks", organization) - and referrer != AutofixReferrer.AGENTIC_TRIAGE # Agentic triage runs stay free. - and not has_enough_seer_credits(organization, operation) - ): + if features.has( + "organizations:seer-credit-checks", organization + ) and not has_enough_seer_credits(organization, operation): raise SeerOutOfCreditsException() return False @@ -200,7 +197,6 @@ def continue_single_run( check_autofix_billing( group.organization, SeerOperation.AUTOFIX_SINGLE_RUN_REPLY, - agent_run.run.referrer, is_new_run=False, ) @@ -258,7 +254,6 @@ def trigger_autofix_feature( uses_quota = check_autofix_billing( group.organization, args.step.seer_operation, - args.referrer, is_new_run=is_new_run, # Free cohort orgs bypass quota only when called from agentic triage # (allow_free_cohort=True). Not exposed via the API. diff --git a/src/sentry/seer/endpoints/group_ai_autofix.py b/src/sentry/seer/endpoints/group_ai_autofix.py index 1d8af3cdc4cf..4767dde92f13 100644 --- a/src/sentry/seer/endpoints/group_ai_autofix.py +++ b/src/sentry/seer/endpoints/group_ai_autofix.py @@ -165,16 +165,10 @@ def _parse_autofix_referrer(raw: str | None, request: Request) -> AutofixReferre return AutofixReferrer.MCP return AutofixReferrer.GROUP_AUTOFIX_ENDPOINT try: - referrer = AutofixReferrer(raw) + return AutofixReferrer(raw) except ValueError: logger.warning("group_ai_autofix.unknown_referrer", extra={"referrer": raw}) return AutofixReferrer.UNKNOWN - # Agentic triage is not exposed via the API. Don't trust such a referrer since - # it may have token-based billing implications. - if referrer == AutofixReferrer.AGENTIC_TRIAGE: - logger.warning("group_ai_autofix.internal_referrer", extra={"referrer": raw}) - return AutofixReferrer.UNKNOWN - return referrer def _record_user_context_length(request_data: Any) -> None: diff --git a/tests/sentry/seer/autofix/feature/test_dispatch.py b/tests/sentry/seer/autofix/feature/test_dispatch.py index 809188a99de6..62f8d3c9e03c 100644 --- a/tests/sentry/seer/autofix/feature/test_dispatch.py +++ b/tests/sentry/seer/autofix/feature/test_dispatch.py @@ -387,32 +387,6 @@ def test_raises_when_out_of_credits_for_a_continued_step( MockClient.return_value.continue_feature_run.assert_not_called() - @with_feature(["organizations:seer-credit-checks", "organizations:token-billed-seer-enabled"]) - @patch( - "sentry.seer_credit.backend.check_seer_credits", - return_value=CheckSeerCreditsResponse(has_enough_credits=False), - ) - def test_agentic_triage_skips_credit_check(self, mock_check_seer_credits: MagicMock) -> None: - fake_run = self.create_seer_run(organization=self.organization, type="feature_run") - - with ( - patch("sentry.seer.autofix.feature.dispatch.SeerAgentClient") as MockClient, - patch("sentry.seer.autofix.feature.dispatch.quotas"), - ): - MockClient.return_value.start_feature_run.return_value = fake_run - - run = trigger_autofix_feature( - self.group, - AutofixFeatureArgs( - referrer=AutofixReferrer.AGENTIC_TRIAGE, - step=AutofixStep.ROOT_CAUSE, - step_args=RCAStepArgs(), - ), - ) - - assert run is fake_run - mock_check_seer_credits.assert_not_called() - def test_allows_async_dispatch(self) -> None: fake_run = self.create_seer_run(organization=self.organization, type="feature_run") @@ -763,18 +737,3 @@ def test_raises_when_out_of_credits(self, mock_check_seer_credits: MagicMock) -> ) mock_client_cls.return_value.continue_feature_run.assert_not_called() - - @with_feature(["organizations:seer-credit-checks", "organizations:token-billed-seer-enabled"]) - @patch( - "sentry.seer_credit.backend.check_seer_credits", - return_value=CheckSeerCreditsResponse(has_enough_credits=False), - ) - def test_agentic_triage_run_skips_credit_check( - self, mock_check_seer_credits: MagicMock - ) -> None: - """The run's own referrer decides, not the reply's.""" - self.seer_run.update(referrer=AutofixReferrer.AGENTIC_TRIAGE) - - self._reply() - - mock_check_seer_credits.assert_not_called() diff --git a/tests/sentry/seer/autofix/test_autofix_agent.py b/tests/sentry/seer/autofix/test_autofix_agent.py index 95e41640240d..555ef5924e20 100644 --- a/tests/sentry/seer/autofix/test_autofix_agent.py +++ b/tests/sentry/seer/autofix/test_autofix_agent.py @@ -1685,32 +1685,6 @@ def test_raises_when_out_of_credits( mock_post.assert_not_called() - @patch( - "sentry.seer_credit.backend.check_seer_credits", - return_value=CheckSeerCreditsResponse(has_enough_credits=False), - ) - @patch("sentry.seer.agent.client.make_agent_update_request") - def test_agentic_triage_run_skips_credit_check( - self, mock_post: MagicMock, mock_check_seer_credits: MagicMock - ) -> None: - # The push is called with another referrer; the run's own referrer decides. - self.create_seer_run( - organization=self.organization, - seer_run_state_id=123, - referrer=AutofixReferrer.AGENTIC_TRIAGE.value, - ) - - self._push( - mock_post, - features={ - "organizations:seer-credit-checks": True, - "organizations:token-billed-seer-enabled": True, - }, - ) - - mock_check_seer_credits.assert_not_called() - mock_post.assert_called_once() - @patch("sentry.seer.agent.client.make_agent_update_request") def test_passes_pr_description_suffix_ready_state_and_author( self, mock_post: MagicMock diff --git a/tests/sentry/seer/endpoints/test_group_ai_autofix.py b/tests/sentry/seer/endpoints/test_group_ai_autofix.py index f4ab9f832b8c..10c804a02408 100644 --- a/tests/sentry/seer/endpoints/test_group_ai_autofix.py +++ b/tests/sentry/seer/endpoints/test_group_ai_autofix.py @@ -1198,25 +1198,6 @@ def test_post_seer_explorer_referrer_is_recognized( assert response.status_code == 202, response.data assert mock_trigger_explorer.call_args.kwargs["referrer"] == AutofixReferrer.SEER_EXPLORER - @patch("sentry.seer.endpoints.group_ai_autofix.trigger_autofix_agent") - def test_post_agentic_triage_referrer_is_unknown( - self, mock_trigger_explorer: MagicMock - ) -> None: - """Agentic triage runs skip the credit check, so the API can't claim to be one.""" - group = self.create_group() - run = self.create_seer_run(organization=self.organization, seer_run_state_id=123) - mock_trigger_explorer.return_value = run - - self.login_as(user=self.user) - response = self.client.post( - self._get_url(group.id), - data={"step": "root_cause", "referrer": AutofixReferrer.AGENTIC_TRIAGE.value}, - format="json", - ) - - assert response.status_code == 202, response.data - assert mock_trigger_explorer.call_args.kwargs["referrer"] == AutofixReferrer.UNKNOWN - @patch("sentry.seer.endpoints.group_ai_autofix.trigger_autofix_agent") def test_stopping_point(self, mock_trigger_explorer: MagicMock) -> None: """Stopping point forces the step to be root_cause"""