Custom logger for Swarming & Removes circular dependency at logs.py

src/clusterfuzz/_internal/bot/tasks/utasks/__init__.py

@@ -454,7 +454,7 @@ def uworker_main(input_download_url) -> None:
     _start_web_server_if_needed(uworker_input.job_type)
 
     utask_module = get_utask_module(uworker_input.module_name)
-    execution_mode = Mode.SWARMING if environment.is_swarming_bot(
+    execution_mode = Mode.SWARMING if environment.is_running_on_swarming(
     ) else Mode.BATCH
     recorder.set_task_details(
         utask_module, uworker_input.job_type, execution_mode,

src/clusterfuzz/_internal/metrics/logs.py

@@ -99,9 +99,10 @@ def _file_logging_enabled():
   This is disabled if we are running in app engine or kubernetes as these have
     their dedicated loggers, see configure_appengine() and configure_k8s().
   """
+  from clusterfuzz._internal.system import environment
   return bool(os.getenv(
-      'LOG_TO_FILE',
-      'True')) and not _is_running_on_app_engine() and not _is_running_on_k8s()
+      'LOG_TO_FILE', 'True')) and not _is_running_on_app_engine() and (
+          not _is_running_on_k8s() or environment.is_running_on_swarming())
 
 
 def _cloud_logging_enabled():
@@ -110,9 +111,11 @@ def _cloud_logging_enabled():
   This is disabled for local development and if we are running in a app engine
     or kubernetes as these have their dedicated loggers, see
     configure_appengine() and configure_k8s()."""
+  from clusterfuzz._internal.system import environment
   return (bool(os.getenv('LOG_TO_GCP', 'True')) and
           not os.getenv("PY_UNITTESTS") and not _is_local() and
-          not _is_running_on_app_engine() and not _is_running_on_k8s())
+          not _is_running_on_app_engine() and
+          (not _is_running_on_k8s() or environment.is_running_on_swarming()))
 
 
 def suppress_unwanted_warnings():
@@ -554,12 +557,42 @@ def cloud_label_filter(record):
   logging.getLogger().addHandler(handler)
 
 
+def configure_swarming(name, extras=None):
+  """Configure logging for swarming bots."""
+  if extras is None:
+    extras = {}
+  extras['task_id'] = os.getenv('TASK_ID')
+  extras['instance_id'] = os.getenv('BOT_NAME')
+  extras['platform'] = 'swarming'
+
+  global _default_extras
+  _default_extras = extras
+
+  if _console_logging_enabled():
+    logging.basicConfig(level=logging.INFO)
+  if _file_logging_enabled():
+    config.dictConfig(get_logging_config_dict(name))
+  if _cloud_logging_enabled():
+    configure_cloud_logging()
+
+  logger = logging.getLogger(name)
+  logger.setLevel(logging.INFO)
+  set_logger(logger)
+
+  sys.excepthook = uncaught_exception_handler
+
+
 def configure(name, extras=None):
   """Set logger. See the list of loggers in bot/config/logging.yaml.
   Also configures the process to log any uncaught exceptions as an error.
   |extras| will be included by emit() in log messages."""
   suppress_unwanted_warnings()
 
+  from clusterfuzz._internal.system import environment
+  if environment.is_running_on_swarming():
+    configure_swarming(name, extras)
+    return
+
   if _is_running_on_k8s():
     configure_k8s()
     return

src/clusterfuzz/_internal/system/environment.py

@@ -749,9 +749,10 @@ def get_runtime() -> UtaskMainRuntime:
   return UtaskMainRuntime.INSTANCE_GROUP
 
 
-def is_swarming_bot():
+def is_running_on_swarming():
   """Return whether or not the current bot is a swarming bot."""
-  return get_value('SWARMING_BOT')
+  value = get_value('SWARMING_BOT')
+  return value is True or value == 'true'
 
 
 def is_running_on_app_engine():

src/clusterfuzz/_internal/tests/core/bot/tasks/utasks/utasks_test.py

@@ -199,7 +199,7 @@ def setUp(self):
         'clusterfuzz._internal.bot.tasks.utasks.uworker_io.download_and_deserialize_uworker_input',
         'clusterfuzz._internal.bot.tasks.utasks.uworker_io.serialize_and_upload_uworker_output',
         'clusterfuzz._internal.bot.tasks.utasks.get_utask_module',
-        'clusterfuzz._internal.system.environment.is_swarming_bot',
+        'clusterfuzz._internal.system.environment.is_running_on_swarming',
         'clusterfuzz._internal.metrics.events.emit',
     ])
     self.module = mock.MagicMock(__name__='tasks.analyze_task')
@@ -208,12 +208,13 @@ def setUp(self):
   @parameterized.parameterized.expand([utasks.Mode.BATCH, utasks.Mode.SWARMING])
   def test_uworker_main(self, execution_mode: utasks.Mode):
     """Tests that uworker_main works as intended."""
+    # pylint: disable=protected-access
     start_time_ns = time.time_ns()
 
     if execution_mode == utasks.Mode.SWARMING:
-      self.mock.is_swarming_bot.return_value = True  # pylint: disable=protected-access
+      self.mock.is_running_on_swarming.return_value = True
     else:
-      self.mock.is_swarming_bot.return_value = False
+      self.mock.is_running_on_swarming.return_value = False
 
     preprocess_start_time_ns = start_time_ns - 42 * 10**9  # In the past.
     preprocess_start_timestamp = timestamp_pb2.Timestamp()

src/clusterfuzz/_internal/tests/core/metrics/logs_test.py

@@ -521,6 +521,85 @@ def test_configure_appengine(self):
     logs.configure('test')
     self.assertEqual(0, self.mock.dictConfig.call_count)
 
+  def test_configure_swarming(self):
+    """Test configure for swarming bot."""
+    # pylint: disable=protected-access
+    os.environ['SWARMING_BOT'] = 'true'
+    os.environ['TASK_ID'] = 'task-123'
+    os.environ['BOT_NAME'] = 'bot-123'
+
+    helpers.patch(
+        self,
+        ['clusterfuzz._internal.system.environment.is_running_on_swarming'])
+    self.mock.is_running_on_swarming.return_value = True
+
+    logger = mock.MagicMock()
+    self.mock.getLogger.return_value = logger
+
+    logs.configure('test')
+
+    self.assertEqual(logs._default_extras['task_id'], 'task-123')
+    self.assertEqual(logs._default_extras['instance_id'], 'bot-123')
+    self.assertEqual(logs._default_extras['platform'], 'swarming')
+
+
+class LoggingEnabledTest(unittest.TestCase):
+  """Test _cloud_logging_enabled and _file_logging_enabled."""
+
+  # pylint: disable=protected-access
+
+  def setUp(self):
+    helpers.patch_environ(self)
+    helpers.patch(self, [
+        'clusterfuzz._internal.metrics.logs._is_running_on_app_engine',
+        'clusterfuzz._internal.metrics.logs._is_running_on_k8s',
+        'clusterfuzz._internal.metrics.logs._is_local',
+        'clusterfuzz._internal.system.environment.is_running_on_swarming',
+    ])
+
+  def test_cloud_logging_enabled_swarming_k8s(self):
+    """Test cloud logging enabled for swarming bot on K8S."""
+    self.mock._is_running_on_app_engine.return_value = False
+    self.mock._is_running_on_k8s.return_value = True
+    self.mock._is_local.return_value = False
+    self.mock.is_running_on_swarming.return_value = True
+
+    os.environ['LOG_TO_GCP'] = 'True'
+    os.environ['PY_UNITTESTS'] = ''
+
+    self.assertTrue(logs._cloud_logging_enabled())
+
+  def test_cloud_logging_disabled_non_swarming_k8s(self):
+    """Test cloud logging disabled for non-swarming bot on K8S."""
+    self.mock._is_running_on_app_engine.return_value = False
+    self.mock._is_running_on_k8s.return_value = True
+    self.mock._is_local.return_value = False
+    self.mock.is_running_on_swarming.return_value = False
+
+    os.environ['LOG_TO_GCP'] = 'True'
+
+    self.assertFalse(logs._cloud_logging_enabled())
+
+  def test_file_logging_enabled_swarming_k8s(self):
+    """Test file logging enabled for swarming bot on K8S."""
+    self.mock._is_running_on_app_engine.return_value = False
+    self.mock._is_running_on_k8s.return_value = True
+    self.mock.is_running_on_swarming.return_value = True
+
+    os.environ['LOG_TO_FILE'] = 'True'
+
+    self.assertTrue(logs._file_logging_enabled())
+
+  def test_file_logging_disabled_non_swarming_k8s(self):
+    """Test file logging disabled for non-swarming bot on K8S."""
+    self.mock._is_running_on_app_engine.return_value = False
+    self.mock._is_running_on_k8s.return_value = True
+    self.mock.is_running_on_swarming.return_value = False
+
+    os.environ['LOG_TO_FILE'] = 'True'
+
+    self.assertFalse(logs._file_logging_enabled())
+
 
 @test_utils.with_cloud_emulators('datastore')
 class EmitTest(unittest.TestCase):