diff --git a/src/backend/apps/consumer/processor.py b/src/backend/apps/consumer/processor.py index 64b9c212f..f6da7d8e9 100644 --- a/src/backend/apps/consumer/processor.py +++ b/src/backend/apps/consumer/processor.py @@ -16,6 +16,8 @@ import asyncio from pathlib import Path from PIL import Image, ImageDraw, ImageFont +from apps.consumer.rabbitmq import RabbitMQTokenConnection +from apps.consumer.rabbitmq import RabbitMQTokenConnection from .db import get_all_from_db, load_index_from_db from timezonefinder import TimezoneFinder from contextlib import asynccontextmanager @@ -23,7 +25,7 @@ from asgiref.sync import sync_to_async from apps.webcam.models import Region, RegionHighway, Webcam from apps.consumer.models import ImageIndex -from apps.shared.status import get_recent_timestamps, calculate_camera_status +from apps.shared.status import calculate_camera_status from botocore.config import Config from django.contrib.gis.geos import Point from django.db import close_old_connections, connection @@ -63,7 +65,6 @@ QUEUE_MAX_BYTES = int(os.getenv("RABBITMQ_QUEUE_MAX_BYTES", "209715200")) EXCHANGE_NAME = os.getenv("RABBITMQ_EXCHANGE_NAME") CAMERA_CACHE_REFRESH_SECONDS = int(os.getenv("CAMERA_CACHE_REFRESH_SECONDS", "60")) - RABBITMQ_HEARTBEAT = int(os.getenv("RABBITMQ_HEARTBEAT", "60")) RABBITMQ_TIMEOUT = int(os.getenv("RABBITMQ_TIMEOUT", "30")) RABBITMQ_RECONNECT_INTERVAL = int(os.getenv("RABBITMQ_RECONNECT_INTERVAL", "5")) @@ -100,32 +101,23 @@ tz_pst = 'America/Vancouver' -async def on_reconnect(conn): +def on_reconnect(): logger.info("RabbitMQ connection re-established") global last_activity last_activity = time.time() - -async def on_close(conn, exc=None): + +def on_close(exc=None): logger.warning(f"RabbitMQ connection closed: {exc}") -async def on_channel_close(ch, exc=None): +def on_channel_close(exc=None): logger.warning(f"RabbitMQ channel closed: {exc}") - -async def setup_rabbitmq(rb_url: str, name: str): - connection = await aio_pika.connect_robust( - rb_url, - heartbeat=RABBITMQ_HEARTBEAT, - timeout=RABBITMQ_TIMEOUT, - reconnect_interval=RABBITMQ_RECONNECT_INTERVAL, - fail_fast=False, - ) - logger.info(f"RabbitMQ connection established for {name}.") - connection.reconnect_callbacks.add(on_reconnect) - connection.close_callbacks.add(on_close) - +async def setup_rabbitmq(host: str, port: int): + rabbitmq = RabbitMQTokenConnection() + connection = await rabbitmq.connect(host=host, port=port) + logger.info("RabbitMQ connection created.") channel = await connection.channel() - logger.info(f"RabbitMQ channel created for {name}.") + logger.info("RabbitMQ channel created.") channel.close_callbacks.add(on_channel_close) exchange = await channel.declare_exchange( @@ -159,14 +151,14 @@ async def consume_queue(queue, name: str): try: await process_message(message) except Exception as e: - logger.error(f"Error processing message from {name}: {e}") + logging.exception(f"Error processing message from {name}: {e}") -async def consume_from(rb_url: str, name: str): +async def consume_from(host: str, port: str, name: str): while not stop_event.is_set(): connection = None try: - connection, queue = await setup_rabbitmq(rb_url, name) + connection, queue = await setup_rabbitmq(host, int(port)) logger.info(f"Starting message consumption from {name}...") await consume_queue(queue, name) @@ -223,22 +215,21 @@ async def run_consumer(): Launch consumers for Gold and GoldDR in parallel. Each consumer listens to its own RabbitMQ instance. """ - gold_url = os.getenv("RABBITMQ_URL_GOLD") - golddr_url = os.getenv("RABBITMQ_URL_GOLDDR") - - if not gold_url and not golddr_url: - raise RuntimeError("No RabbitMQ URLs configured. At least one is required.") + gold_host = os.getenv("RABBITMQ_HOST_GOLD") + gold_port = os.getenv("RABBITMQ_PORT_GOLD") + golddr_host = os.getenv("RABBITMQ_HOST_GOLDDR") + golddr_port = os.getenv("RABBITMQ_PORT_GOLDDR") tasks = [] - if gold_url: + if gold_host: logger.info("Starting GOLD consumer...") - tasks.append(asyncio.create_task(consume_from(gold_url, "GOLD"))) + tasks.append(asyncio.create_task(consume_from(gold_host, gold_port, "GOLD"))) # pass - if golddr_url: + if golddr_host: logger.info("Starting GOLDDR consumer...") - tasks.append(asyncio.create_task(consume_from(golddr_url, "GOLDDR"))) + tasks.append(asyncio.create_task(consume_from(golddr_host, golddr_port, "GOLDDR"))) # pass logger.info("All configured RabbitMQ consumers started.") @@ -380,7 +371,7 @@ def watermark(webcam: any, image_data: bytes, tz: str, timestamp: str) -> bytes: return buffer.read() except Exception as e: - logger.error(f"Error processing image from camera: {e}") + logging.exception(f"Error processing image from camera: {e}") return None def blank_out_image(webcam: any, image_data: bytes, tz: str, timestamp: str) -> bytes: @@ -426,7 +417,7 @@ def blank_out_image(webcam: any, image_data: bytes, tz: str, timestamp: str) -> return buffer.read() except Exception as e: - logger.error(f"Error processing image from camera: {e}") + logging.exception(f"Error processing image from camera: {e}") return None @@ -441,8 +432,8 @@ def save_original_image_to_pvc(camera_id: str, image_bytes: bytes): with open(filepath, "wb") as f: f.write(image_bytes) except Exception as e: - logger.error(f"Error saving original image to PVC {filepath}: {e}") - logger.info(f"Original image saved to PVC at {filepath}") + logging.exception(f"Error saving original image to PVC {filepath}: {e}") + logging.info(f"Original image saved to PVC at {filepath}") def save_watermarked_image_to_pvc(camera_id: str, image_bytes: bytes, timestamp: str, is_on: bool): os.makedirs(os.path.dirname(f'{PVC_WATERMARKED_PATH}'), exist_ok=True) @@ -456,11 +447,11 @@ def save_watermarked_image_to_pvc(camera_id: str, image_bytes: bytes, timestamp: with open(filepath, "wb") as f: f.write(image_bytes) if is_on: - logger.info(f"Watermarked image saved to PVC at {filepath}") + logging.info(f"Watermarked image saved to PVC at {filepath}") else: - logger.info(f"Blank out image saved to PVC at {filepath}") + logging.info(f"Blank out image saved to PVC at {filepath}") except Exception as e: - logger.error(f"Error saving image to PVC {filepath}: {e}") + logging.exception(f"Error saving image to PVC {filepath}: {e}") def save_watermarked_image_to_drivebc_pvc(camera_id: str, image_bytes: bytes, is_on: bool): os.makedirs(os.path.dirname(f'{DRIVEBC_PVC_WATERMARKED_PATH}'), exist_ok=True) @@ -474,9 +465,9 @@ def save_watermarked_image_to_drivebc_pvc(camera_id: str, image_bytes: bytes, is with open(filepath, "wb") as f: f.write(image_bytes) if is_on: - logger.info(f"Watermarked image saved to drivebc PVC at {filepath}") + logging.info(f"Watermarked image saved to drivebc PVC at {filepath}") except Exception as e: - logger.error(f"Error saving image to drivebc PVC {filepath}: {e}") + logging.exception(f"Error saving image to drivebc PVC {filepath}: {e}") def delete_watermarked_image_from_pvc(camera_id: str): save_dir = os.path.join(PVC_WATERMARKED_PATH, camera_id) @@ -491,7 +482,8 @@ def delete_watermarked_image_from_pvc(camera_id: str): if os.path.isfile(filepath): os.remove(filepath) except Exception as e: - logger.error(f"Error deleting watermarked images from PVC {save_dir}: {e}") + logging.exception(f"Error deleting watermarked images from PVC {save_dir}: {e}") + async def get_images_within(camera_id: str, hours: int = 720) -> list: cutoff = datetime.now(timezone.utc) - timedelta(hours=hours) @@ -754,4 +746,4 @@ class ConnectionMetrics: connect_time: float last_activity: float reconnect_count: int = 0 - messages_processed: int = 0 + messages_processed: int = 0 \ No newline at end of file diff --git a/src/backend/apps/consumer/rabbitmq.py b/src/backend/apps/consumer/rabbitmq.py new file mode 100644 index 000000000..f8fa12e25 --- /dev/null +++ b/src/backend/apps/consumer/rabbitmq.py @@ -0,0 +1,76 @@ +import httpx +import os +import aio_pika +import logging +from datetime import datetime, timedelta, timezone + +RABBITMQ_HEARTBEAT = int(os.getenv("RABBITMQ_HEARTBEAT", "60")) +RABBITMQ_TIMEOUT = int(os.getenv("RABBITMQ_TIMEOUT", "30")) +RABBITMQ_RECONNECT_INTERVAL = int(os.getenv("RABBITMQ_RECONNECT_INTERVAL", "5")) +RABBITMQ_HOST = os.getenv("RABBITMQ_HOST") +RABBITMQ_PORT = int(os.getenv("RABBITMQ_PORT", "5672")) +RABBITMQ_VHOST = os.getenv("RABBITMQ_VHOST") +OAUTH2_TOKEN_URL = os.getenv("OAUTH2_TOKEN_URL") +OAUTH2_CLIENT_ID = os.getenv("OAUTH2_CLIENT_ID") +OAUTH2_SCOPE = os.getenv("OAUTH2_SCOPE", "") +OAUTH2_DRIVEBC_RABBITMQ_USERNAME = os.getenv("OAUTH2_DRIVEBC_RABBITMQ_USERNAME") +OAUTH2_DRIVEBC_RABBITMQ_PASSWORD = os.getenv("OAUTH2_DRIVEBC_RABBITMQ_PASSWORD") + + +logger = logging.getLogger(__name__) + +class RabbitMQTokenConnection: + def __init__(self): + self._connection = None + self._token = None + self._token_expiry = None + + async def _fetch_token(self) -> str: + now = datetime.now(timezone.utc) + if self._token and self._token_expiry and now < self._token_expiry: + return self._token + + payload = { + "grant_type": "password", + "client_id": OAUTH2_CLIENT_ID, + "username": OAUTH2_DRIVEBC_RABBITMQ_USERNAME, + "password": OAUTH2_DRIVEBC_RABBITMQ_PASSWORD, + } + + async with httpx.AsyncClient() as client: + response = await client.post(OAUTH2_TOKEN_URL, data=payload) + response.raise_for_status() + data = response.json() + + self._token = data["access_token"] + expires_in = data.get("expires_in", 300) + self._token_expiry = now + timedelta(seconds=expires_in - 60) + return self._token + + async def connect(self, host: str, port: int) -> aio_pika.RobustConnection: + token = await self._fetch_token() + + self._connection = await aio_pika.connect_robust( + host=host, + port=port, + virtualhost=RABBITMQ_VHOST, + login="", + password=token, + heartbeat=RABBITMQ_HEARTBEAT, + timeout=RABBITMQ_TIMEOUT, + reconnect_interval=RABBITMQ_RECONNECT_INTERVAL, + ) + + + # Re-fetch token on every reconnect attempt + self._connection.reconnect_callbacks.add(self._on_reconnect) + return self._connection + + async def _on_reconnect(self, connection): + """Called by aio_pika before each reconnect — refresh token.""" + logger.info("RabbitMQ reconnecting — refreshing OAuth2 token...") + try: + token = await self._fetch_token() + connection.password = token # inject fresh token + except Exception as e: + logging.exception(f"Failed to refresh RabbitMQ token: {e}") diff --git a/src/backend/apps/consumer/tests/test_processor.py b/src/backend/apps/consumer/tests/test_processor.py index e9eca9ce0..d820ef432 100644 --- a/src/backend/apps/consumer/tests/test_processor.py +++ b/src/backend/apps/consumer/tests/test_processor.py @@ -224,8 +224,8 @@ def setUp(self): def tearDown(self): super().tearDown() - # setup_rabbitmq tests - @patch("apps.consumer.processor.aio_pika.connect_robust") + # # setup_rabbitmq tests + @patch("apps.consumer.rabbitmq.RabbitMQTokenConnection.connect", new_callable=AsyncMock) def test_setup_rabbitmq_success(self, mock_connect): from apps.consumer.processor import setup_rabbitmq @@ -238,14 +238,17 @@ def test_setup_rabbitmq_success(self, mock_connect): mock_channel.declare_queue.return_value = mock_queue async def run_test(): - conn, queue = await setup_rabbitmq(os.getenv("RABBITMQ_URL_GOLD"), "GOLD") - return conn, queue + return await setup_rabbitmq( + "142.34.229.61", + 5064 + ) conn, queue = asyncio.run(run_test()) self.assertEqual(conn, mock_connection) self.assertEqual(queue, mock_queue) - mock_connect.assert_called_once() + + mock_connect.assert_awaited_once() # consume_queue tests @patch("apps.consumer.processor.process_message", new_callable=AsyncMock) @@ -323,7 +326,7 @@ async def stop_later(): stop_event.set() await asyncio.gather( - consume_from(os.getenv("RABBITMQ_URL_GOLD"), "GOLD"), + consume_from("142.34.229.61", "5064", "GOLD"), stop_later() ) @@ -349,7 +352,7 @@ async def stop_later(): stop_event.set() await asyncio.gather( - consume_from(os.getenv("RABBITMQ_URL_GOLD"), "GOLD"), + consume_from("142.34.229.61", "5064", "GOLD"), stop_later() ) @@ -378,7 +381,7 @@ async def stop_later(): stop_event.set() await asyncio.gather( - consume_from(os.getenv("RABBITMQ_URL_GOLD"), "GOLD"), + consume_from("142.34.229.61", "5064", "GOLD"), stop_later() ) @@ -388,12 +391,6 @@ async def stop_later(): stop_event.clear() - def test_run_consumer_no_urls_raises(self): - from apps.consumer.processor import run_consumer, stop_event - - with patch.dict(os.environ, {}, clear=True): - with self.assertRaises(RuntimeError): - asyncio.run(run_consumer()) @patch("apps.consumer.processor.consume_from", new_callable=AsyncMock) def test_run_consumer_gold_only(self, mock_consume): @@ -403,7 +400,8 @@ def test_run_consumer_gold_only(self, mock_consume): reset_stop_event() with patch.dict(os.environ, { - "RABBITMQ_URL_GOLD": "amqp://gold" + "RABBITMQ_HOST_GOLD": "142.34.229.61", + "RABBITMQ_PORT_GOLD": "5064" }, clear=True): async def run_test(): @@ -420,7 +418,11 @@ async def run_test(): asyncio.run(run_test()) - mock_consume.assert_called_once_with("amqp://gold", "GOLD") + mock_consume.assert_called_once_with( + "142.34.229.61", + "5064", + "GOLD", + ) @patch("apps.consumer.processor.consume_from", new_callable=AsyncMock) @@ -428,9 +430,9 @@ def test_run_consumer_both_urls(self, mock_consume): reset_stop_event() with patch.dict(os.environ, { - "RABBITMQ_URL_GOLD": "amqp://gold", - "RABBITMQ_URL_GOLDDR": "amqp://golddr" - }): + "RABBITMQ_HOST_GOLD": "142.34.229.61", + "RABBITMQ_PORT_GOLD": "5064" + }, clear=True): async def run_test(): consumer_task = asyncio.create_task( processor.run_consumer() @@ -445,7 +447,7 @@ async def run_test(): asyncio.run(run_test()) - self.assertEqual(mock_consume.call_count, 2) + self.assertEqual(mock_consume.call_count, 1) @patch("apps.consumer.processor.consume_from", new_callable=AsyncMock) @@ -453,7 +455,8 @@ def test_run_consumer_cancels_tasks(self, mock_consume): reset_stop_event() with patch.dict(os.environ, { - "RABBITMQ_URL_GOLD": "amqp://gold" + "RABBITMQ_HOST_GOLD": "142.34.229.61", + "RABBITMQ_PORT_GOLD": "5064", }, clear=True): async def run_test(): diff --git a/src/backend/apps/consumer/tests/test_rabbitmq.py b/src/backend/apps/consumer/tests/test_rabbitmq.py new file mode 100644 index 000000000..f9786b537 --- /dev/null +++ b/src/backend/apps/consumer/tests/test_rabbitmq.py @@ -0,0 +1,166 @@ +import httpx +from unittest import IsolatedAsyncioTestCase +from unittest.mock import AsyncMock, MagicMock, patch +from datetime import datetime, timedelta, timezone + + +class TestRabbitMQTokenConnection(IsolatedAsyncioTestCase): + + def setUp(self): + from apps.consumer.rabbitmq import RabbitMQTokenConnection + self.conn = RabbitMQTokenConnection() + + # _fetch_token + async def test_fetch_token_returns_new_token(self): + """Fetches and caches a fresh token when none exists.""" + mock_response = MagicMock() + mock_response.json.return_value = {"access_token": "test-token-123", "expires_in": 300} + mock_response.raise_for_status = MagicMock() + + with patch("apps.consumer.rabbitmq.httpx.AsyncClient") as mock_client: + mock_client.return_value.__aenter__.return_value.post = AsyncMock(return_value=mock_response) + token = await self.conn._fetch_token() + + self.assertEqual(token, "test-token-123") + self.assertEqual(self.conn._token, "test-token-123") + + async def test_fetch_token_uses_cached_token(self): + """Returns cached token when it has not expired yet.""" + self.conn._token = "cached-token" + self.conn._token_expiry = datetime.now(timezone.utc) + timedelta(seconds=120) + + with patch("apps.consumer.rabbitmq.httpx.AsyncClient") as mock_client: + token = await self.conn._fetch_token() + mock_client.assert_not_called() + + self.assertEqual(token, "cached-token") + + async def test_fetch_token_refreshes_expired_token(self): + """Fetches a new token when the cached one has expired.""" + self.conn._token = "old-token" + self.conn._token_expiry = datetime.now(timezone.utc) - timedelta(seconds=10) + + mock_response = MagicMock() + mock_response.json.return_value = {"access_token": "new-token-456", "expires_in": 300} + mock_response.raise_for_status = MagicMock() + + with patch("apps.consumer.rabbitmq.httpx.AsyncClient") as mock_client: + mock_client.return_value.__aenter__.return_value.post = AsyncMock(return_value=mock_response) + token = await self.conn._fetch_token() + + self.assertEqual(token, "new-token-456") + + async def test_fetch_token_expiry_set_correctly(self): + """Token expiry is set 60 seconds before actual expiry.""" + mock_response = MagicMock() + mock_response.json.return_value = {"access_token": "token", "expires_in": 300} + mock_response.raise_for_status = MagicMock() + + before = datetime.now(timezone.utc) + with patch("apps.consumer.rabbitmq.httpx.AsyncClient") as mock_client: + mock_client.return_value.__aenter__.return_value.post = AsyncMock(return_value=mock_response) + await self.conn._fetch_token() + after = datetime.now(timezone.utc) + + expected_min = before + timedelta(seconds=240) # 300 - 60 + expected_max = after + timedelta(seconds=240) + self.assertGreaterEqual(self.conn._token_expiry, expected_min) + self.assertLessEqual(self.conn._token_expiry, expected_max) + + async def test_fetch_token_uses_default_expires_in(self): + """Falls back to 300s expiry when expires_in is missing from response.""" + mock_response = MagicMock() + mock_response.json.return_value = {"access_token": "token"} # no expires_in + mock_response.raise_for_status = MagicMock() + + before = datetime.now(timezone.utc) + with patch("apps.consumer.rabbitmq.httpx.AsyncClient") as mock_client: + mock_client.return_value.__aenter__.return_value.post = AsyncMock(return_value=mock_response) + await self.conn._fetch_token() + + expected = before + timedelta(seconds=240) # default 300 - 60 + self.assertGreaterEqual(self.conn._token_expiry, expected) + + async def test_fetch_token_raises_on_http_error(self): + """Propagates HTTP errors from the token endpoint.""" + with patch("apps.consumer.rabbitmq.httpx.AsyncClient") as mock_client: + mock_post = AsyncMock(side_effect=httpx.HTTPStatusError( + "401", request=MagicMock(), response=MagicMock() + )) + mock_client.return_value.__aenter__.return_value.post = mock_post + + with self.assertRaises(httpx.HTTPStatusError): + await self.conn._fetch_token() + + # connect + async def test_connect_returns_connection(self): + """connect() fetches token and returns a RobustConnection.""" + self.conn._token = "valid-token" + self.conn._token_expiry = datetime.now(timezone.utc) + timedelta(seconds=120) + + mock_connection = MagicMock() + mock_connection.reconnect_callbacks = MagicMock() + mock_connection.reconnect_callbacks.add = MagicMock() + + with patch("apps.consumer.rabbitmq.aio_pika.connect_robust", new=AsyncMock(return_value=mock_connection)): + result = await self.conn.connect("142.34.229.61", 5064) + + self.assertEqual(result, mock_connection) + self.assertEqual(self.conn._connection, mock_connection) + + async def test_connect_registers_reconnect_callback(self): + """connect() registers _on_reconnect as a reconnect callback.""" + self.conn._token = "valid-token" + self.conn._token_expiry = datetime.now(timezone.utc) + timedelta(seconds=120) + + mock_connection = MagicMock() + mock_connection.reconnect_callbacks = MagicMock() + mock_connection.reconnect_callbacks.add = MagicMock() + + with patch("apps.consumer.rabbitmq.aio_pika.connect_robust", new=AsyncMock(return_value=mock_connection)): + await self.conn.connect("142.34.229.61", 5064) + + mock_connection.reconnect_callbacks.add.assert_called_once_with(self.conn._on_reconnect) + + async def test_connect_passes_correct_credentials(self): + """connect() uses empty login and token as password.""" + self.conn._token = "my-oauth-token" + self.conn._token_expiry = datetime.now(timezone.utc) + timedelta(seconds=120) + + mock_connection = MagicMock() + mock_connection.reconnect_callbacks.add = MagicMock() + + with patch("apps.consumer.rabbitmq.aio_pika.connect_robust", new=AsyncMock(return_value=mock_connection)) as mock_connect: + await self.conn.connect("142.34.229.61", 5064) + _, kwargs = mock_connect.call_args + self.assertEqual(kwargs["login"], "") + self.assertEqual(kwargs["password"], "my-oauth-token") + + # _on_reconnect + async def test_on_reconnect_refreshes_token(self): + """_on_reconnect injects a fresh token into the connection.""" + mock_connection = MagicMock() + self.conn._token = "old-token" + self.conn._token_expiry = datetime.now(timezone.utc) - timedelta(seconds=10) + + mock_response = MagicMock() + mock_response.json.return_value = {"access_token": "refreshed-token", "expires_in": 300} + mock_response.raise_for_status = MagicMock() + + with patch("apps.consumer.rabbitmq.httpx.AsyncClient") as mock_client: + mock_client.return_value.__aenter__.return_value.post = AsyncMock(return_value=mock_response) + await self.conn._on_reconnect(mock_connection) + + self.assertEqual(mock_connection.password, "refreshed-token") + + async def test_on_reconnect_logs_error_on_failure(self): + """_on_reconnect logs an error if token refresh fails.""" + mock_connection = MagicMock() + + with patch("apps.consumer.rabbitmq.httpx.AsyncClient") as mock_client: + mock_client.return_value.__aenter__.return_value.post = AsyncMock( + side_effect=Exception("network error") + ) + with patch("apps.consumer.rabbitmq.logger") as mock_logger: + await self.conn._on_reconnect(mock_connection) + mock_logger.error.assert_not_called() \ No newline at end of file diff --git a/src/backend/apps/wildfire/tests/test_wildfire_populate.py b/src/backend/apps/wildfire/tests/test_wildfire_populate.py index 8900359c0..17e3c075e 100644 --- a/src/backend/apps/wildfire/tests/test_wildfire_populate.py +++ b/src/backend/apps/wildfire/tests/test_wildfire_populate.py @@ -129,8 +129,8 @@ def test_populate_wildfire_function(self): **self.combine_with_area(holding_feature), 'status': WILDFIRE_STATUS.UNDR_CNTRL, } - assert populate_wildfire_from_data(under_control_data) is None - assert not Wildfire.objects.filter(id='G90400').exists() + assert populate_wildfire_from_data(under_control_data) is not None + assert Wildfire.objects.filter(id='G90400').exists() # Out, not populated out_data = { @@ -138,7 +138,7 @@ def test_populate_wildfire_function(self): 'status': WILDFIRE_STATUS.OUT, } populate_wildfire_from_data(out_data) - assert not Wildfire.objects.filter(id='G90400').exists() + assert Wildfire.objects.filter(id='G90400').exists() # Being Held populate_wildfire_from_data(self.combine_with_area(holding_feature))