Resolve database/GitHub task status disconnect

packit_service/worker/reporting/reporters/base.py

@@ -6,6 +6,7 @@
 from typing import Callable, Optional, Union
 
 from ogr.abstract import GitProject, PullRequest
+from ogr.exceptions import GithubAPIException, GitlabAPIException, PagureAPIException
 from ogr.services.github import GithubProject
 from ogr.services.gitlab import GitlabProject
 from ogr.services.pagure import PagureProject
@@ -29,6 +30,7 @@ def __init__(
         packit_user: str,
         project_event_id: Optional[int] = None,
         pr_id: Optional[int] = None,
+        reraise_transient_errors: bool = False,
     ):
         logger.debug(
             f"Status reporter will report for {project}, commit={commit_sha}, pr={pr_id}",
@@ -41,6 +43,7 @@ def __init__(
         self.project_event_id: int = project_event_id
         self.pr_id: Optional[int] = pr_id
         self._pull_request_object: Optional[PullRequest] = None
+        self.reraise_transient_errors: bool = reraise_transient_errors
 
     @classmethod
     def get_instance(
@@ -50,6 +53,7 @@ def get_instance(
         packit_user: str,
         project_event_id: Optional[int] = None,
         pr_id: Optional[int] = None,
+        reraise_transient_errors: bool = False,
     ) -> "StatusReporter":
         """
         Get the StatusReporter instance.
@@ -67,7 +71,9 @@ def get_instance(
             reporter = StatusReporterGitlab
         elif isinstance(project, PagureProject):
             reporter = StatusReporterPagure
-        return reporter(project, commit_sha, packit_user, project_event_id, pr_id)
+        return reporter(
+            project, commit_sha, packit_user, project_event_id, pr_id, reraise_transient_errors
+        )
 
     @property
     def project_with_commit(self) -> GitProject:
@@ -97,6 +103,32 @@ def get_commit_status(state: BaseCommitStatus):
     def get_check_run(state: BaseCommitStatus):
         return MAP_TO_CHECK_RUN[state]
 
+    @staticmethod
+    def is_transient_error(
+        exception: Union[GithubAPIException, GitlabAPIException, PagureAPIException],
+    ) -> bool:
+        """
+        Check if an API exception represents a transient error that should be retried.
+
+        Transient errors include:
+        - Network errors (no response_code attribute)
+        - Rate limiting (HTTP 429)
+        - Server errors (HTTP 5xx)
+
+        Args:
+            exception: An API exception from ogr
+
+        Returns:
+            True if the error is transient and should be retried, False otherwise
+        """
+        response_code = getattr(exception, "response_code", None)
+
+        if response_code is None:
+            # Network errors (no response code) are transient
+            return True
+
+        return response_code == 429 or (500 <= response_code < 600)
+
     def set_status(
         self,
         state: BaseCommitStatus,

packit_service/worker/reporting/reporters/github.py

@@ -58,6 +58,12 @@ def set_status(
                 trim=True,
             )
         except GithubAPIException as e:
+            if self.is_transient_error(e) and self.reraise_transient_errors:
+                logger.debug(
+                    f"Re-raising transient GitHub API error when setting "
+                    f"status for '{check_name}': {e}."
+                )
+                raise
             self._comment_as_set_status_fallback(e, state, description, check_name, url)
 
 
@@ -137,6 +143,12 @@ def set_status(
                 output=create_github_check_run_output(description, summary),
             )
         except GithubAPIException as e:
+            if self.is_transient_error(e) and self.reraise_transient_errors:
+                logger.debug(
+                    f"Re-raising transient GitHub API error when setting "
+                    f"status for '{check_name}': {e}."
+                )
+                raise
             logger.debug(
                 f"Failed to set status check, setting status as a fallback: {e!s}",
             )

packit_service/worker/reporting/reporters/gitlab.py

@@ -61,11 +61,15 @@ def set_status(
             )
         except GitlabAPIException as e:
             logger.debug(f"Failed to set the status: {e}. Response code: {e.response_code}")
-            # Ignoring Gitlab error regarding reporting a status of the same state
+
+            # Special case: Ignore "Cannot transition status" errors
             # https://github.com/packit-service/packit-service/issues/741
-            if e.response_code != 400 or "Cannot transition status" not in str(e):
-                # 403: No permissions to set status, falling back to comment
-                # 404: Commit has not been found, e.g. used target project on GitLab
-                self._comment_as_set_status_fallback(e, state, description, check_name, url)
-            if e.response_code not in {400, 403, 404}:
+            if e.response_code == 400 and "Cannot transition status" in str(e):
+                return
+
+            # Check if error is transient and reraise is enabled
+            if self.is_transient_error(e) and self.reraise_transient_errors:
                 raise
+
+            # Fall back to comment for all other errors
+            self._comment_as_set_status_fallback(e, state, description, check_name, url)