diff --git a/app/connectors_service/NOTICE.txt b/app/connectors_service/NOTICE.txt index ffb93b9b9..244b23cb0 100644 --- a/app/connectors_service/NOTICE.txt +++ b/app/connectors_service/NOTICE.txt @@ -4241,7 +4241,7 @@ Apache Software License google-auth -2.56.0 +2.56.2 Apache Software License Apache License Version 2.0, January 2004 @@ -4475,7 +4475,7 @@ SOFTWARE. greenlet -3.5.3 +3.5.4 MIT AND PSF-2.0 The following files are derived from Stackless Python and are subject to the same license as Stackless Python: @@ -7616,7 +7616,7 @@ made under the terms of *both* these licenses. soupsieve -2.9 +2.9.1 MIT MIT License diff --git a/app/connectors_service/connectors/sources/microsoft_teams/client.py b/app/connectors_service/connectors/sources/microsoft_teams/client.py index beb259e5e..913cc36ae 100644 --- a/app/connectors_service/connectors/sources/microsoft_teams/client.py +++ b/app/connectors_service/connectors/sources/microsoft_teams/client.py @@ -3,39 +3,49 @@ # or more contributor license agreements. Licensed under the Elastic License 2.0; # you may not use this file except in compliance with the Elastic License 2.0. # +"""Microsoft Teams client. + +The connector uses application-only authentication (client secret or certificate) +and the least-privilege permission model: + +- Enumerating teams uses the tenant-wide `Team.ReadBasic.All` application + permission. +- Enumerating channels uses `Channel.ReadBasic.All` (or the resource-specific + `ChannelSettings.Read.Group`). +- Reading channel messages/replies uses the resource-specific consent (RSC) + permission `ChannelMessage.Read.Group`, granted when the connector's Teams app + is installed in a team. +- Reading team members uses the RSC permission `TeamMember.Read.Group`. +- Enumerating and reading chats uses `Chat.ReadBasic.WhereInstalled` and + `Chat.Read.WhereInstalled`, which scope results to chats where the connector's + Teams app is installed. +- Downloading attachment content requires `Files.Read.All`. + +Because RSC/WhereInstalled permissions are only granted where the app is +installed, per-resource calls that hit a resource the app is not installed in +respond with 403/404. Those are swallowed so a partially-installed tenant still +syncs whatever it has access to. +""" + import os -from datetime import datetime, timedelta +from collections import Counter +from collections.abc import AsyncIterator from enum import Enum import aiohttp -from aiohttp import ClientResponseError from connectors_sdk.logger import logger -from msal import ConfidentialClientApplication - -from connectors.utils import ( - CacheWithTimeout, - CancellableSleeps, - RetryStrategy, - retryable, - url_encode, + +from connectors.sources.shared.microsoft.graph import ( + EntraAPIToken, + GraphAPIToken, + MicrosoftAPISession, + NotFound, + PermissionsMissing, ) +from connectors.utils import url_encode -SCOPE = [ - "User.Read.All", - "TeamMember.Read.All", - "ChannelMessage.Read.All", - "Chat.Read", - "Chat.ReadBasic", - "Calendars.Read", -] -FILE_WRITE_CHUNK_SIZE = 1024 * 64 # 64KB default SSD page size -TOKEN_EXPIRES = 3599 -RETRY_COUNT = 3 -RETRY_SECONDS = 30 -RETRY_INTERVAL = 2 -RUNNING_FTEST = ( - "RUNNING_FTEST" in os.environ -) # Flag to check if a connector is run for ftest or not. +GRAPH_ACQUIRE_TOKEN_URL = "https://graph.microsoft.com/.default" # noqa S105 +DEFAULT_PARALLEL_CONNECTION_COUNT = 10 if "OVERRIDE_URL" in os.environ: logger.warning("x" * 50) @@ -44,138 +54,100 @@ ) logger.warning("IT'S SUPPOSED TO BE USED ONLY FOR TESTING") logger.warning("x" * 50) - override_url = os.environ["OVERRIDE_URL"] - BASE_URL = override_url - GRAPH_API_AUTH_URL = override_url - GRAPH_ACQUIRE_TOKEN_URL = override_url + BASE_URL = os.environ["OVERRIDE_URL"] else: - GRAPH_API_AUTH_URL = "https://login.microsoftonline.com" - GRAPH_ACQUIRE_TOKEN_URL = "https://graph.microsoft.com/.default" # noqa S105 BASE_URL = "https://graph.microsoft.com/v1.0" +CHAT_FILES_FOLDER_NAME = "microsoft teams chat files" -class UserEndpointName(Enum): - PING = "Ping" - USERS = "Users" - CHAT = "User Chat" - CHATS_MESSAGE = "User Chat Messages" - MEETING_RECORDING = "User Chat Meeting Recording" - TABS = "User Chat Tabs" - DRIVE = "User Drive" - DRIVE_CHILDREN = "User Drive Children" - ATTACHMENT = "User Chat Attachment" - MEETING = "User Meeting" +class TeamsObjectType(Enum): + """Document `type` values emitted by the connector.""" -class TeamEndpointName(Enum): - TEAMS = "Teams" + TEAM = "Teams" + TEAM_MEMBER = "Team Member" CHANNEL = "Team Channel" - TAB = "Channel Tab" - MESSAGE = "Channel Message" - MEETING = "Channel Meeting" - ATTACHMENT = "Channel Attachment" - FILE = "File" - ROOT_DRIVE_CHILDREN = "Root Drive Children" + CHANNEL_MESSAGE = "Channel Message" + CHANNEL_ATTACHMENT = "Channel Attachment" + CHAT = "Chat" + CHAT_MESSAGE = "Chat Message" + CHAT_ATTACHMENT = "Chat Attachment" class EndSignal(Enum): - USER_CHAT_TASK_FINISHED = "USER_CHAT_TASK_FINISHED" + ENUMERATION_FINISHED = "ENUMERATION_FINISHED" TEAM_TASK_FINISHED = "TEAM_TASK_FINISHED" - CHANNEL_TASK_FINISHED = "CHANNEL_TASK_FINISHED" - CALENDAR_TASK_FINISHED = "CALENDAR_TASK_FINISHED" - - -URLS = { - UserEndpointName.PING.value: "{base_url}/me", - UserEndpointName.USERS.value: "{base_url}/users?$top=999", - UserEndpointName.CHAT.value: "{base_url}/chats?$expand=members&$top=50", - UserEndpointName.CHATS_MESSAGE.value: "{base_url}/chats/{chat_id}/messages?$top=50", - UserEndpointName.TABS.value: "{base_url}/chats/{chat_id}/tabs", - UserEndpointName.DRIVE.value: "{base_url}/users/{sender_id}/drive", - UserEndpointName.DRIVE_CHILDREN.value: "{base_url}/drives/{drive_id}/items/root/children", - UserEndpointName.ATTACHMENT.value: "{base_url}/drives/{drive_id}/items/{child_id}/children?$filter=name eq '{attachment_name}'", - UserEndpointName.MEETING.value: "{base_url}/users/{user_id}/events?$top=50", - TeamEndpointName.TEAMS.value: "{base_url}/teams?$top=999", - TeamEndpointName.CHANNEL.value: "{base_url}/teams/{team_id}/channels", - TeamEndpointName.TAB.value: "{base_url}/teams/{team_id}/channels/{channel_id}/tabs", - TeamEndpointName.MESSAGE.value: "{base_url}/teams/{team_id}/channels/{channel_id}/messages?$expand=replies", - TeamEndpointName.FILE.value: "{base_url}/teams/{team_id}/channels/{channel_id}/filesFolder", - TeamEndpointName.ROOT_DRIVE_CHILDREN.value: "{base_url}/drives/{drive_id}/items/{item_id}/children?$top=5000", -} + CHAT_TASK_FINISHED = "CHAT_TASK_FINISHED" class Schema: - def chat_messages(self): + """Maps Elasticsearch document fields to Microsoft Graph fields.""" + + def team(self): return { "_id": "id", - "_timestamp": "lastModifiedDateTime", + "title": "displayName", + "description": "description", "creation_time": "createdDateTime", - "webUrl": "webUrl", - "title": "title", - "chatType": "chatType", - "sender": "sender", - "message": "message", } - def chat_tabs(self): + def team_member(self): return { "_id": "id", "title": "displayName", + "email": "email", } - def chat_attachments(self): + def channel(self): return { "_id": "id", - "name": "name", - "weburl": "webUrl", - "size_in_bytes": "size", - "_timestamp": "lastModifiedDateTime", + "url": "webUrl", + "title": "displayName", + "description": "description", "creation_time": "createdDateTime", } - def meeting(self): + def channel_message(self): return { "_id": "id", - "creation_time": "createdDateTime", + "url": "webUrl", "_timestamp": "lastModifiedDateTime", - "title": "subject", - "Cancelled_status": "isCancelled", - "web_link": "webLink", - "reminder": "isReminderOn", - "reminder_time": "reminderMinutesBeforeStart", - "categories": "categories", - "original_start_timezone": "originalStartTimeZone", - "original_end_timezone": "originalEndTimeZone", + "creation_time": "createdDateTime", } - def teams(self): + def channel_attachment(self): return { "_id": "id", - "title": "displayName", - "description": "description", + "name": "name", + "weburl": "webUrl", + "size_in_bytes": "size", + "_timestamp": "lastModifiedDateTime", + "creation_time": "createdDateTime", } - def channel(self): + def chat(self): return { "_id": "id", + "title": "topic", "url": "webUrl", - "title": "displayName", - "description": "description", + "chatType": "chatType", + "_timestamp": "lastUpdatedDateTime", "creation_time": "createdDateTime", } - def channel_tab(self): - return {"_id": "id", "title": "displayName", "url": "webUrl"} - - def channel_message(self): + def chat_message(self): return { "_id": "id", - "url": "webUrl", "_timestamp": "lastModifiedDateTime", "creation_time": "createdDateTime", + "webUrl": "webUrl", + "title": "title", + "chatType": "chatType", + "sender": "sender", + "message": "message", } - def channel_attachment(self): + def chat_attachment(self): return { "_id": "id", "name": "name", @@ -186,164 +158,20 @@ def channel_attachment(self): } -class NotFound(Exception): - """Internal exception class to handle 404s from the API that has a meaning, that collection - for specific object is empty. - - It's not an exception for us, we just want to return [], and this exception class facilitates it. - """ - - pass - - -class ThrottledError(Exception): - """Internal exception class to indicate that request was throttled by the API""" - - pass - - -class InternalServerError(Exception): - """Exception class to indicate that something went wrong on the server side.""" - - pass - - -class PermissionsMissing(Exception): - """Exception class to notify that specific Application Permission is missing for the credentials used. - See: https://learn.microsoft.com/en-us/graph/permissions-reference - """ - - pass - - -class TokenFetchFailed(Exception): - """Exception class to notify that connector was unable to fetch authentication token from either - Microsoft Teams Graph API. - - Error message will indicate human-readable reason. - """ - - pass - - -class GraphAPIToken: - """Class for handling access token for Microsoft Graph APIs""" - - def __init__(self, tenant_id, client_id, client_secret, username, password): - """Initializer. - - Args: - tenant_id (str): Azure AD Tenant Id - client_id (str): Azure App Client Id - client_secret (str): Azure App Client Secret Value - username (str): Username of the Azure account to fetch the access_token - password (str): Password of the Azure account to fetch the access_token""" - - self.tenant_id = tenant_id - self.client_id = client_id - self.client_secret = client_secret - self.username = username - self.password = password - - self._token_cache_with_client = CacheWithTimeout() - self._token_cache_with_username = CacheWithTimeout() - - async def get_with_client(self): - """Get bearer token for provided credentials. - - If token has been retrieved, it'll be taken from the cache. - Otherwise, call to `_fetch_token` is made to fetch the token - from 3rd-party service. - - Returns: - str: bearer token for one of Microsoft services""" - if RUNNING_FTEST: - return - - cached_value = self._token_cache_with_client.get_value() - - if cached_value: - return cached_value - - # We measure now before request to be on a pessimistic side - now = datetime.utcnow() - access_token, expires_in = await self._fetch_token(is_acquire_for_client=True) - - self._token_cache_with_client.set_value( - access_token, now + timedelta(seconds=expires_in) - ) - - return access_token - - async def get_with_username_password(self): - """Get bearer token for provided credentials. - - If token has been retrieved, it'll be taken from the cache. - Otherwise, call to `_fetch_token` is made to fetch the token - from 3rd-party service. - - Returns: - str: bearer token for one of Microsoft services""" - if RUNNING_FTEST: - return - - cached_value = self._token_cache_with_username.get_value() - if cached_value: - return cached_value - - # We measure now before request to be on a pessimistic side - now = datetime.utcnow() - access_token, expires_in = await self._fetch_token() - - self._token_cache_with_username.set_value( - access_token, now + timedelta(seconds=expires_in) - ) - - return access_token - - @retryable( - retries=RETRY_COUNT, - interval=RETRY_INTERVAL, - strategy=RetryStrategy.EXPONENTIAL_BACKOFF, - ) - async def _fetch_token(self, is_acquire_for_client=False): - """Generate API token for usage with Graph API - Args: - is_acquire_for_client (boolean): True if token needs be generated using client. Default to false - - Returns: - (str, int) - a tuple containing access token as a string and number of seconds it will be valid for as an integer - """ - authority = f"{GRAPH_API_AUTH_URL}/{self.tenant_id}" - - auth_context = ConfidentialClientApplication( - client_id=self.client_id, - client_credential=self.client_secret, - authority=authority, - ) - if is_acquire_for_client: - token_metadata = auth_context.acquire_token_for_client( - scopes=[GRAPH_ACQUIRE_TOKEN_URL] - ) - else: - token_metadata = auth_context.acquire_token_by_username_password( - username=self.username, password=self.password, scopes=SCOPE - ) - if not token_metadata.get("access_token"): - msg = f"Failed to authorize to Graph API. Please verify, that provided details are valid. Error: {token_metadata.get('error_description')}" - raise TokenFetchFailed(msg) - - access_token = token_metadata.get("access_token") - expires_in = int(token_metadata.get("expires_in", TOKEN_EXPIRES)) - return access_token, expires_in - - class MicrosoftTeamsClient: - """Client Class for API calls to Microsoft Teams""" - - def __init__(self, tenant_id, client_id, client_secret, username, password): - self._sleeps = CancellableSleeps() + """Client Class for API calls to Microsoft Teams via Microsoft Graph.""" + + def __init__( + self, + tenant_id, + client_id, + client_secret=None, + certificate=None, + private_key=None, + ): + tcp_connector = aiohttp.TCPConnector(limit=DEFAULT_PARALLEL_CONNECTION_COUNT) self._http_session = aiohttp.ClientSession( + connector=tcp_connector, headers={ "accept": "application/json", "content-type": "application/json", @@ -351,250 +179,194 @@ def __init__(self, tenant_id, client_id, client_secret, username, password): timeout=aiohttp.ClientTimeout(total=None), raise_for_status=True, ) - self._api_token = GraphAPIToken( - tenant_id=tenant_id, - client_id=client_id, - client_secret=client_secret, - username=username, - password=password, - ) + + if client_secret and not certificate and not private_key: + self.graph_api_token = GraphAPIToken( + self._http_session, tenant_id, None, client_id, client_secret + ) + elif certificate and private_key: + self.graph_api_token = EntraAPIToken( + self._http_session, + tenant_id, + None, + client_id, + certificate, + private_key, + GRAPH_ACQUIRE_TOKEN_URL, + ) + else: + msg = "Unexpected authentication: either a client_secret or certificate+private_key should be provided" + raise Exception(msg) self._logger = logger + self._skipped = Counter() + self._graph_api_client = MicrosoftAPISession( + self._http_session, self.graph_api_token, "@odata.nextLink", self._logger + ) def set_logger(self, logger_): self._logger = logger_ + self._graph_api_client.set_logger(self._logger) - async def fetch(self, url): - return await self._get_json(absolute_url=url) + def log_skip_summary(self): + """Emit an aggregate warning for resources skipped during the sync. - async def pipe(self, url, stream): - try: - async for response in self._get(absolute_url=url, use_token=False): - async for data in response.content.iter_chunked(FILE_WRITE_CHUNK_SIZE): - await stream.write(data) - except Exception as exception: - self._logger.warning( - f"Data for {url} is being skipped. Error: {exception}." - ) + Per-resource 403/404s are logged at debug to avoid noise; this surfaces the + totals so an under-installed tenant (missing RSC / Teams app not installed) + is visible instead of producing a quiet, near-empty "successful" sync. + """ + if not self._skipped: + return - async def scroll(self, url): - scroll_url = url + details = ", ".join( + f"{count} {resource}" for resource, count in sorted(self._skipped.items()) + ) + self._logger.warning( + f"Skipped some resources because the connector's Teams app is not " + f"installed there or the required permissions are missing: {details}. " + f"Content in those teams/chats was not indexed." + ) + self._skipped.clear() - while True: - if graph_data := await self._get_json(scroll_url): - # We're yielding the whole page here, not one item - yield graph_data.get("value", []) + async def ping(self): + return await self._graph_api_client.fetch(f"{BASE_URL}/teams?$top=1") - if not graph_data.get("@odata.nextLink"): - break - scroll_url = graph_data.get("@odata.nextLink") - else: - break + async def get_teams(self): + async for teams in self._graph_api_client.scroll(f"{BASE_URL}/teams?$top=999"): + yield teams - async def _get_json(self, absolute_url): + async def get_team_members(self, team_id): try: - async for response in self._get(absolute_url=absolute_url): - return await response.json() - except Exception as exception: - self._logger.warning( - f"Data for {absolute_url} is being skipped. Error: {exception}." + async for members in self._graph_api_client.scroll( + f"{BASE_URL}/teams/{team_id}/members" + ): + yield members + except (NotFound, PermissionsMissing): + self._skipped["teams' members"] += 1 + self._logger.debug( + f"Skipping members for team '{team_id}': the connector's Teams app is not installed there or 'TeamMember.Read.Group' is missing." ) + return - @retryable( - retries=RETRY_COUNT, - interval=RETRY_INTERVAL, - strategy=RetryStrategy.EXPONENTIAL_BACKOFF, - skipped_exceptions=[NotFound, PermissionsMissing], - ) - async def _get(self, absolute_url, use_token=True): + async def get_team_channels(self, team_id): try: - if use_token: - if any( - substring in absolute_url - for substring in [ - "/drive", - "/items/root/children", - "/children?$filter=name eq", - "/events", - ] - ): - token = await self._api_token.get_with_client() - else: - token = await self._api_token.get_with_username_password() - self._logger.debug(f"Calling Microsoft Teams Endpoint: {absolute_url}") - async with self._http_session.get( - url=absolute_url, - headers={"authorization": f"Bearer {token}"}, - ) as resp: - yield resp - else: - async with self._http_session.get( - url=absolute_url, - ) as resp: - yield resp - except aiohttp.client_exceptions.ClientOSError: - self._logger.error( - "Graph API dropped the connection. It might indicate, that connector makes too many requests - decrease concurrency settings, otherwise Graph API can block this app." - ) - raise - except ClientResponseError as e: - await self._handle_client_response_error(absolute_url, e) - - async def _handle_client_response_error(self, absolute_url, e): - if e.status == 429 or e.status == 503: - response_headers = e.headers or {} - updated_response_headers = { - key.lower(): value for key, value in response_headers.items() - } - retry_seconds = int( - updated_response_headers.get("retry-after", RETRY_SECONDS) - ) + async for channels in self._graph_api_client.scroll( + f"{BASE_URL}/teams/{team_id}/channels" + ): + yield channels + except (NotFound, PermissionsMissing): + self._skipped["teams' channels"] += 1 + self._logger.debug(f"Skipping channels for team '{team_id}'.") + return + + async def get_channel_messages(self, team_id, channel_id): + try: + async for messages in self._graph_api_client.scroll( + f"{BASE_URL}/teams/{team_id}/channels/{channel_id}/messages?$expand=replies&$top=50" + ): + yield messages + except (NotFound, PermissionsMissing): + self._skipped["channels' messages"] += 1 self._logger.debug( - f"Rate limited by Microsoft Teams. Retrying after {retry_seconds} seconds" - ) - await self._sleeps.sleep(retry_seconds) - msg = f"Service is throttled because too many requests have been made to {absolute_url}: Exception: {e}" - raise ThrottledError(msg) from e - elif e.status == 403: - msg = f"Received Unauthorized response for {absolute_url}.\nVerify that the correct Graph API and Microsoft Teams permissions are granted to the app and admin consent is given. If the permissions and consent are correct, wait for several minutes and try again." - raise PermissionsMissing(msg) from e - elif e.status == 404: - raise NotFound from e - elif e.status == 500: - msg = ( - f"Received InternalServerError error for {absolute_url}: Exception: {e}" + f"Skipping messages for channel '{channel_id}' in team '{team_id}': the connector's Teams app is not installed there or 'ChannelMessage.Read.Group' is missing." ) - raise InternalServerError(msg) from e - else: - raise - - async def ping(self): - return await self.fetch( - url=URLS[UserEndpointName.PING.value].format(base_url=BASE_URL) - ) - - async def users(self): - async for users in self.scroll( - url=URLS[UserEndpointName.USERS.value].format(base_url=BASE_URL) - ): - yield users - - async def get_user_chats(self): - async for chats in self.scroll( - url=URLS[UserEndpointName.CHAT.value].format(base_url=BASE_URL) - ): - yield chats + return - async def get_user_chat_messages(self, chat_id): - async for messages in self.scroll( - url=URLS[UserEndpointName.CHATS_MESSAGE.value].format( - base_url=BASE_URL, chat_id=chat_id + async def get_channel_file(self, team_id, channel_id): + try: + return await self._graph_api_client.fetch( + f"{BASE_URL}/teams/{team_id}/channels/{channel_id}/filesFolder" ) - ): - yield messages - - async def get_user_chat_tabs(self, chat_id): - async for tabs in self.scroll( - url=URLS[UserEndpointName.TABS.value].format( - base_url=BASE_URL, chat_id=chat_id + except (NotFound, PermissionsMissing): + self._logger.debug( + f"Skipping files folder for channel '{channel_id}' in team '{team_id}'." ) - ): - yield tabs - - async def get_user_drive(self, sender_id): - return await self.fetch( - url=URLS[UserEndpointName.DRIVE.value].format( - base_url=BASE_URL, sender_id=sender_id - ), - ) - - async def get_user_drive_root_children(self, drive_id): - async for root_children_data in self.scroll( - url=URLS[UserEndpointName.DRIVE_CHILDREN.value].format( - base_url=BASE_URL, drive_id=drive_id - ), - ): - for child in root_children_data: - if child["name"].lower() == "microsoft teams chat files": - return child - - async def get_user_chat_attachments(self, sender_id, attachment_name): - drive = await self.get_user_drive(sender_id=sender_id) - child = await self.get_user_drive_root_children(drive_id=drive.get("id")) - async for attachment in self.scroll( - url=URLS[UserEndpointName.ATTACHMENT.value].format( - base_url=BASE_URL, - drive_id=drive.get("id"), - child_id=child.get("id"), - attachment_name=url_encode(attachment_name).replace("'", "''"), - ), - ): - yield attachment - - async def get_calendars(self, user_id): - async for events in self.scroll( - url=URLS[UserEndpointName.MEETING.value].format( - base_url=BASE_URL, user_id=user_id - ), - ): - for event in events: - yield event + return None - async def download_item(self, url, async_buffer): - await self.pipe(url=url, stream=async_buffer) + async def get_channel_drive_children( + self, drive_id, item_id + ) -> AsyncIterator[dict]: + try: + async for children in self._graph_api_client.scroll( + f"{BASE_URL}/drives/{drive_id}/items/{item_id}/children?$top=5000" + ): + for child in children: + if child.get("folder"): + async for descendant in self.get_channel_drive_children( + drive_id, child["id"] + ): + yield descendant + yield child + except (NotFound, PermissionsMissing): + self._logger.debug( + f"Skipping drive children for item '{item_id}' in drive '{drive_id}'." + ) + return - async def get_teams(self): - async for teams in self.scroll( - url=URLS[TeamEndpointName.TEAMS.value].format(base_url=BASE_URL) + async def get_chats(self): + async for chats in self._graph_api_client.scroll( + f"{BASE_URL}/chats?$expand=members&$top=50" ): - yield teams + yield chats - async def get_team_channels(self, team_id): - async for channels in self.scroll( - url=URLS[TeamEndpointName.CHANNEL.value].format( - base_url=BASE_URL, team_id=team_id + async def get_chat_messages(self, chat_id): + try: + async for messages in self._graph_api_client.scroll( + f"{BASE_URL}/chats/{chat_id}/messages?$top=50" + ): + yield messages + except (NotFound, PermissionsMissing): + self._skipped["chats' messages"] += 1 + self._logger.debug( + f"Skipping messages for chat '{chat_id}': 'Chat.Read.WhereInstalled' is missing or the connector's Teams app is not installed there." ) - ): - yield channels + return - async def get_channel_tabs(self, team_id, channel_id): - async for channel_tabs in self.scroll( - url=URLS[TeamEndpointName.TAB.value].format( - base_url=BASE_URL, team_id=team_id, channel_id=channel_id + async def _get_user_drive(self, user_id): + try: + return await self._graph_api_client.fetch( + f"{BASE_URL}/users/{user_id}/drive" ) - ): - yield channel_tabs + except (NotFound, PermissionsMissing): + return None - async def get_channel_messages(self, team_id, channel_id): - async for channel_messages in self.scroll( - url=URLS[TeamEndpointName.MESSAGE.value].format( - base_url=BASE_URL, team_id=team_id, channel_id=channel_id - ) - ): - yield channel_messages + async def _get_chat_files_folder(self, drive_id): + try: + async for children in self._graph_api_client.scroll( + f"{BASE_URL}/drives/{drive_id}/items/root/children" + ): + for child in children: + if child.get("name", "").lower() == CHAT_FILES_FOLDER_NAME: + return child + except (NotFound, PermissionsMissing): + return None + return None + + async def get_chat_attachments(self, sender_id, attachment_name): + """Resolve a chat attachment (a reference to a file in the sender's OneDrive). + + Requires the `Files.Read.All` application permission. + """ + drive = await self._get_user_drive(sender_id) + if not drive: + return + folder = await self._get_chat_files_folder(drive.get("id")) + if not folder: + return + escaped_name = url_encode(attachment_name).replace("'", "''") + try: + async for attachments in self._graph_api_client.scroll( + f"{BASE_URL}/drives/{drive.get('id')}/items/{folder.get('id')}/children?$filter=name eq '{escaped_name}'" + ): + yield attachments + except (NotFound, PermissionsMissing): + return - async def get_channel_file(self, team_id, channel_id): - file = await self.fetch( - url=URLS[TeamEndpointName.FILE.value].format( - base_url=BASE_URL, team_id=team_id, channel_id=channel_id - ) + async def download_drive_item(self, drive_id, item_id, async_buffer): + await self._graph_api_client.pipe( + f"{BASE_URL}/drives/{drive_id}/items/{item_id}/content", async_buffer ) - return file - - async def get_channel_drive_childrens(self, drive_id, item_id): - async for root_childrens in self.scroll( - url=URLS[TeamEndpointName.ROOT_DRIVE_CHILDREN.value].format( - base_url=BASE_URL, drive_id=drive_id, item_id=item_id - ) - ): - for child in root_childrens: - if child.get("folder"): - async for documents in self.get_channel_drive_childrens( # pyright: ignore - drive_id=drive_id, item_id=child["id"] - ): - yield documents - yield child async def close(self): - self._sleeps.cancel() + self._graph_api_client.close() await self._http_session.close() diff --git a/app/connectors_service/connectors/sources/microsoft_teams/datasource.py b/app/connectors_service/connectors/sources/microsoft_teams/datasource.py index 42e1c4da9..c7e97e5cd 100644 --- a/app/connectors_service/connectors/sources/microsoft_teams/datasource.py +++ b/app/connectors_service/connectors/sources/microsoft_teams/datasource.py @@ -5,30 +5,25 @@ # """Microsoft Teams source module responsible to fetch documents from Microsoft Teams.""" -import asyncio import os -from datetime import datetime from functools import cached_property, partial -import aiofiles -from aiofiles.os import remove -from aiofiles.tempfile import NamedTemporaryFile -from connectors_sdk.content_extraction import ( - TIKA_SUPPORTED_FILETYPES, -) -from connectors_sdk.source import BaseDataSource -from connectors_sdk.utils import ( - convert_to_b64, -) +from connectors_sdk.source import BaseDataSource, ConfigurableFieldValueError +from connectors_sdk.utils import iso_zulu +from connectors.access_control import ( + ACCESS_CONTROL, + es_access_control_query, + prefix_identity, +) from connectors.sources.microsoft_teams.client import ( EndSignal, MicrosoftTeamsClient, Schema, - TeamEndpointName, - UserEndpointName, + TeamsObjectType, ) from connectors.sources.microsoft_teams.formatter import MicrosoftTeamsFormatter +from connectors.sources.shared.microsoft.graph import PermissionsMissing from connectors.utils import ( ConcurrentTasks, MemQueue, @@ -37,7 +32,14 @@ QUEUE_MEM_SIZE = 5 * 1024 * 1024 # Size in Megabytes MAX_CONCURRENCY = 80 -MAX_FILE_SIZE = 10485760 + + +def _prefix_user_id(user_id): + return prefix_identity("user_id", user_id) + + +def _prefix_email(email): + return prefix_identity("email", email) class MicrosoftTeamsDataSource(BaseDataSource): @@ -45,16 +47,20 @@ class MicrosoftTeamsDataSource(BaseDataSource): name = "Microsoft Teams" service_type = "microsoft_teams" - incremental_sync_enabled = True + incremental_sync_enabled = False + dls_enabled = True def __init__(self, configuration): - """Set up the connection to the Microsoft Teams. + """Set up the connection to Microsoft Teams. Args: configuration (DataSourceConfiguration): Object of DataSourceConfiguration class. """ super().__init__(configuration=configuration) self.tasks = 0 + self._teams_enumeration_failed = False + self._chats_enumeration_failed = False + self._enumeration_error: Exception | None = None self.queue = MemQueue(maxmemsize=QUEUE_MEM_SIZE, refresh_timeout=120) self.fetchers = ConcurrentTasks(max_concurrency=MAX_CONCURRENCY) self.schema = Schema() @@ -67,73 +73,118 @@ def _set_internal_logger(self): def client(self): tenant_id = self.configuration["tenant_id"] client_id = self.configuration["client_id"] - client_secret = self.configuration["secret_value"] - username = self.configuration["username"] - password = self.configuration["password"] + auth_method = self.configuration["auth_method"] + + if auth_method == "certificate": + return MicrosoftTeamsClient( + tenant_id, + client_id, + certificate=self.configuration["certificate"], + private_key=self.configuration["private_key"], + ) return MicrosoftTeamsClient( - tenant_id, client_id, client_secret, username, password + tenant_id, + client_id, + client_secret=self.configuration["secret_value"], ) - async def _consumer(self): - """Async generator to process entries of the queue + @classmethod + def get_default_configuration(cls): + """Get the default configuration for Microsoft Teams. - Yields: - dictionary: Documents from Microsoft Teams. + Returns: + dictionary: Default configuration. """ - while self.tasks > 0: - _, item = await self.queue.get() - - if isinstance(item, EndSignal): - self.tasks -= 1 - else: - yield item - - def verify_filename_for_extraction(self, filename): - attachment_extension = os.path.splitext(filename)[-1] - if attachment_extension == "": - self._logger.debug( - f"Files without extension are not supported, skipping {filename}." - ) - return - if attachment_extension.lower() not in TIKA_SUPPORTED_FILETYPES: - self._logger.debug( - f"Files with the extension {attachment_extension} are not supported, skipping {filename}." - ) - return - return True - - async def _download_content_for_attachment(self, download_func, original_filename): - attachment = None - source_file_name = "" - - try: - async with NamedTemporaryFile(mode="wb", delete=False) as async_buffer: - source_file_name = async_buffer.name - await download_func(async_buffer) - - self._logger.debug( - f"Download completed for file: {original_filename}. Calling convert_to_b64" - ) - await asyncio.to_thread( - convert_to_b64, - source=source_file_name, - ) - async with aiofiles.open(file=source_file_name, mode="r") as target_file: - # base64 on macOS will add a EOL, so we strip() here - attachment = (await target_file.read()).strip() - - finally: - if source_file_name: - await remove(str(source_file_name)) - - return attachment + return { + "tenant_id": { + "label": "Tenant ID", + "order": 1, + "type": "str", + }, + "client_id": { + "label": "Client ID", + "order": 2, + "type": "str", + }, + "auth_method": { + "label": "Authentication Method", + "order": 3, + "type": "str", + "display": "dropdown", + "options": [ + {"label": "Client Secret", "value": "secret"}, + {"label": "Certificate", "value": "certificate"}, + ], + "value": "secret", + }, + "secret_value": { + "label": "Secret value", + "order": 4, + "sensitive": True, + "type": "str", + "depends_on": [{"field": "auth_method", "value": "secret"}], + }, + "certificate": { + "label": "Content of certificate file", + "display": "textarea", + "sensitive": True, + "order": 5, + "type": "str", + "depends_on": [{"field": "auth_method", "value": "certificate"}], + }, + "private_key": { + "label": "Content of private key file", + "display": "textarea", + "sensitive": True, + "order": 6, + "type": "str", + "depends_on": [{"field": "auth_method", "value": "certificate"}], + }, + "fetch_attachment_content": { + "display": "toggle", + "label": "Fetch attachment content", + "order": 7, + "tooltip": "Enable to fetch the content of channel and chat attachments. Requires the 'Files.Read.All' application permission.", + "type": "bool", + "value": True, + }, + "use_text_extraction_service": { + "display": "toggle", + "label": "Use text extraction service", + "order": 8, + "tooltip": "Requires a separate deployment of the Elastic Text Extraction Service. Requires that pipeline settings disable text extraction.", + "type": "bool", + "ui_restrictions": ["advanced"], + "value": False, + }, + "use_document_level_security": { + "display": "toggle", + "label": "Enable document level security", + "order": 9, + "tooltip": "Document level security ensures identities and permissions set in Microsoft Teams are maintained in Elasticsearch. This enables you to restrict and personalize read-access users and groups have to documents in this index. Access control syncs ensure this metadata is kept up to date in your Elasticsearch documents.", + "type": "bool", + "value": False, + }, + } async def validate_config(self): await super().validate_config() - # Check that we can log in into Graph API - await self.client._api_token.get_with_username_password() + auth_method = self.configuration["auth_method"] + if auth_method == "certificate": + if ( + not self.configuration["certificate"] + or not self.configuration["private_key"] + ): + msg = "Both 'Content of certificate file' and 'Content of private key file' are required when the authentication method is 'Certificate'." + raise ConfigurableFieldValueError(msg) + elif not self.configuration["secret_value"]: + msg = "'Secret value' is required when the authentication method is 'Client Secret'." + raise ConfigurableFieldValueError(msg) + + # Check that we can obtain a Graph API token with the provided credentials + await self.client.graph_api_token.get() async def ping(self): """Verify the connection with Microsoft Teams""" @@ -144,402 +195,470 @@ async def ping(self): self._logger.exception("Error while connecting to Microsoft Teams") raise - async def update_user_chat_attachments(self, **kwargs): - async for attachments in self.client.get_user_chat_attachments( - sender_id=kwargs["sender_id"], - attachment_name=kwargs["attachment_name"], - ): - for attachment in attachments: - format_attachment = self.formatter.format_doc( - item=attachment, - document_type=self.schema.chat_attachments, - document={ - "type": UserEndpointName.ATTACHMENT.value, - "members": kwargs.get("members", ""), - }, - ) - download_url = attachment.get("@microsoft.graph.downloadUrl") - await self.queue.put( - ( - format_attachment, - partial( - self.get_content, - user_attachment=format_attachment, - download_url=download_url, - ), - ) - ) + async def close(self): + """Closes unclosed client session""" + await self.client.close() + + # -- Document level security ------------------------------------------- + + def _dls_enabled(self): + if self._features is None: + return False + + if not self._features.document_level_security_enabled(): + return False + + return self.configuration["use_document_level_security"] + + def access_control_query(self, access_control): + return es_access_control_query(access_control) + + def _decorate_with_access_control(self, document, access_control): + if self._dls_enabled(): + document[ACCESS_CONTROL] = sorted( + set(document.get(ACCESS_CONTROL, []) + access_control) + ) + return document + + def _access_control_for_members(self, members): + access_control = set() + for member in members or []: + user_id = member.get("userId") or member.get("id") + if user_id: + access_control.add(_prefix_user_id(user_id)) + email = member.get("email") + if email: + access_control.add(_prefix_email(email)) + return list(access_control) + + def _user_access_control_doc(self, member): + user_id = member.get("userId") or member.get("id") + if not user_id: + return None + + email = member.get("email") + prefixed_user_id = _prefix_user_id(user_id) + prefixed_email = _prefix_email(email) if email else None + + access_control = [prefixed_user_id] + if prefixed_email: + access_control.append(prefixed_email) + + return { + "_id": user_id, + "identity": { + "user_id": prefixed_user_id, + "email": prefixed_email, + }, + "created_at": iso_zulu(), + } | self.access_control_query(access_control) + + async def get_access_control(self): + """Yields an access control document for every user participating in a synced team or chat.""" + if not self._dls_enabled(): + self._logger.warning("DLS is not enabled. Skipping access control sync.") + return + + seen = set() + + try: + async for teams in self.client.get_teams(): + for team in teams: + async for members in self.client.get_team_members(team["id"]): + for member in members: + doc = self._user_access_control_doc(member) + if doc and doc["_id"] not in seen: + seen.add(doc["_id"]) + yield doc + except PermissionsMissing: + self._logger.warning( + "Unable to enumerate teams for access control. Verify the 'Team.ReadBasic.All' application permission is granted." + ) + + try: + async for chats in self.client.get_chats(): + for chat in chats: + for member in chat.get("members", []): + doc = self._user_access_control_doc(member) + if doc and doc["_id"] not in seen: + seen.add(doc["_id"]) + yield doc + except PermissionsMissing: + self._logger.warning( + "Unable to enumerate chats for access control. Verify the 'Chat.ReadBasic.WhereInstalled' application permission is granted and the connector's Teams app is installed." + ) + + # -- Content extraction ------------------------------------------------ async def get_content( - self, user_attachment, download_url, timestamp=None, doit=False + self, attachment, drive_id, item_id, timestamp=None, doit=False ): """Extracts the content for allowed file types. Args: - user_attachment (dictionary): Attachment object dictionary - download_url (str): Attachment downloadable url - timestamp (timestamp, optional): Timestamp of attachment last modified. Defaults to None. - doit (boolean, optional): Boolean value for whether to get content or not. Defaults to False. + attachment (dict): Attachment document (already mapped by the formatter). + drive_id (str): The drive id the item belongs to. + item_id (str): The drive item id. + timestamp (str, optional): Unused, kept for interface compatibility. + doit (bool, optional): Whether to actually fetch the content. Returns: - dictionary: Content document with _id, _timestamp and attachment content + dict: Content document with `_id`, `_timestamp` and the attachment content. """ - document_size = int(user_attachment["size_in_bytes"]) - - if not (doit and document_size): + file_size = int(attachment["size_in_bytes"] or 0) + if not (doit and file_size): return - filename = user_attachment["name"] - if not self.verify_filename_for_extraction(filename=filename): - return - if user_attachment["size_in_bytes"] > MAX_FILE_SIZE: - self._logger.warning( - f"File size {document_size} of file {filename} is larger than {MAX_FILE_SIZE} bytes. Discarding file content" - ) + + filename = attachment["name"] + file_extension = self.get_file_extension(filename) + if not self.can_file_be_downloaded(file_extension, filename, file_size): return + document = { - "_id": user_attachment["id"], - "_timestamp": user_attachment["_timestamp"], + "_id": attachment["_id"], + "_timestamp": attachment["_timestamp"], } - self._logger.debug(f"Downloading {filename}") - attachment_content = await self._download_content_for_attachment( - partial(self.client.download_item, download_url), - original_filename=filename, - ) - document["_attachment"] = attachment_content + async with self.create_temp_file(file_extension) as async_buffer: + temp_filename = async_buffer.name + await self.client.download_drive_item(drive_id, item_id, async_buffer) + await async_buffer.close() + document = await self.handle_file_content_extraction( + document, filename, temp_filename + ) return document - async def get_messages( - self, message, document_type=None, chat=None, channel_name=None, members=None - ): - if not message.get("deletedDateTime") and ( - "unknownFutureValue" not in message.get("messageType") - ): - if message_content := html_to_text( - html=message.get("body", {}).get("content") - ): - if document_type == UserEndpointName.CHATS_MESSAGE.value: - message_document = await self.formatter.format_user_chat_messages( - chat=chat, - message=message, - message_content=message_content, - members=members, - ) - await self.queue.put( - ( - self.formatter.format_doc( - item=message_document, - document_type=self.schema.chat_messages, - document={"type": document_type}, - ), - None, - ) - ) - else: - await self.queue.put( - ( - self.formatter.format_channel_message( - item=message, - channel_name=channel_name, - message_content=message_content, - ), - None, - ) - ) + def get_file_extension(self, filename): + return os.path.splitext(filename)[-1] - async def user_chat_meeting_recording(self, message): - if ( - message.get("eventDetail") - and message["eventDetail"].get("@odata.type") - == "#microsoft.graph.callRecordingEventMessageDetail" - ): - url = message["eventDetail"].get("callRecordingUrl") + # -- Document producers ------------------------------------------------ - if url and ".sharepoint.com" in url: - await self.queue.put( - ( - self.formatter.format_user_chat_meeting_recording( - item=message, url=url - ), - None, - ) - ) + async def _consumer(self): + """Async generator to process entries of the queue. - def get_chat_members(self, members): - return ",".join( - member.get("displayName") - for member in members - if member.get("displayName", "") + Yields: + dictionary: Documents from Microsoft Teams. + """ + while self.tasks > 0: + _, item = await self.queue.get() + + if isinstance(item, EndSignal): + self.tasks -= 1 + else: + yield item + + def _attachments_enabled(self): + return self.configuration["fetch_attachment_content"] + + async def _process_channel_message(self, message, channel_name, access_control): + if message.get("deletedDateTime"): + return + if "unknownFutureValue" in (message.get("messageType") or ""): + return + message_content = html_to_text(html=(message.get("body") or {}).get("content")) + if not message_content and not message.get("attachments"): + return + document = self.formatter.format_channel_message( + item=message, channel_name=channel_name, message_content=message_content + ) + await self.queue.put( + (self._decorate_with_access_control(document, access_control), None) ) - async def user_chat_producer(self, chat): - members = self.get_chat_members(chat.get("members", [])) - async for messages in self.client.get_user_chat_messages(chat_id=chat["id"]): + async def _process_channel_attachment( + self, child, team_name, channel_name, access_control + ): + drive_id = child.get("parentReference", {}).get("driveId") + if not drive_id: + return + document = self.formatter.format_doc( + item=child, + document_type=self.schema.channel_attachment, + document={ + "type": TeamsObjectType.CHANNEL_ATTACHMENT.value, + "team_name": team_name, + "channel_name": channel_name, + }, + ) + document = self._decorate_with_access_control(document, access_control) + await self.queue.put( + ( + document, + partial( + self.get_content, + attachment=document, + drive_id=drive_id, + item_id=child["id"], + ), + ) + ) + + async def _produce_channel(self, channel, team_id, team_name, access_control): + channel_id = channel.get("id") + channel_name = channel.get("displayName") + + channel_document = self.formatter.format_doc( + item=channel, + document_type=self.schema.channel, + document={ + "type": TeamsObjectType.CHANNEL.value, + "_timestamp": iso_zulu(), + "team_name": team_name, + }, + ) + await self.queue.put( + (self._decorate_with_access_control(channel_document, access_control), None) + ) + + async for messages in self.client.get_channel_messages(team_id, channel_id): for message in messages: - await self.get_messages( - message=message, - document_type=UserEndpointName.CHATS_MESSAGE.value, - chat=chat, - members=members, + await self._process_channel_message( + message, channel_name, access_control ) + for reply in message.get("replies", []): + await self._process_channel_message( + reply, channel_name, access_control + ) - await self.user_chat_meeting_recording(message=message) - - if message.get("from") and message["from"].get("user"): - for attachment in message.get("attachments", []): - if ( - attachment.get("name") - and attachment.get("contentType") == "reference" - ): - await self.update_user_chat_attachments( - sender_id=message["from"]["user"]["id"], - attachment_name=attachment["name"], - members=members, + if self._attachments_enabled(): + files_folder = await self.client.get_channel_file(team_id, channel_id) + if files_folder: + drive_id = files_folder.get("parentReference", {}).get("driveId") + item_id = files_folder.get("id") + if drive_id and item_id: + async for child in self.client.get_channel_drive_children( + drive_id, item_id + ): + if child.get("file"): + await self._process_channel_attachment( + child, team_name, channel_name, access_control ) - async for tabs in self.client.get_user_chat_tabs(chat_id=chat["id"]): - for tab in tabs: - await self.queue.put( - ( - self.formatter.format_doc( - item=tab, - document_type=self.schema.chat_tabs, - document={ - "type": UserEndpointName.TABS.value, - "url": tab.get("configuration", {}).get("websiteUrl"), - "_timestamp": chat["lastUpdatedDateTime"], - "members": members, - }, - ), - None, - ) - ) - await self.queue.put(EndSignal.USER_CHAT_TASK_FINISHED) + async def team_producer(self, team): + team_id = team.get("id") + team_name = team.get("displayName") - async def get_channel_messages(self, message, channel_name): - await self.get_messages(message=message, channel_name=channel_name) - meeting_document = {} - for reply in message.get("replies", []): - message_content = html_to_text(html=reply.get("body", {}).get("content")) - if ( - not reply.get("deletedDateTime") - and ("unknownFutureValue" not in reply.get("messageType")) - and message_content - ): - await self.queue.put( - ( - self.formatter.format_channel_message( - item=reply, - channel_name=channel_name, - message_content=message_content, - ), - None, - ) + try: + members = [] + async for member_page in self.client.get_team_members(team_id): + members.extend(member_page) + + access_control = self._access_control_for_members(members) + + team_document = self.formatter.format_doc( + item=team, + document_type=self.schema.team, + document={ + "type": TeamsObjectType.TEAM.value, + "_timestamp": iso_zulu(), + }, + ) + await self.queue.put( + ( + self._decorate_with_access_control(team_document, access_control), + None, ) - - elif reply.get("eventDetail"): - call_id = reply["eventDetail"].get("callId") - if call_id not in meeting_document: - meeting_document[call_id] = {} - document = self.formatter.format_channel_meeting(reply=reply) - meeting_document[call_id].update(document) - for document in meeting_document.values(): - await self.queue.put((document, None)) - - async def team_channel_producer(self, channel, team_id, team_name): - channel_name = channel.get("displayName") - await self.queue.put( - ( - self.formatter.format_doc( - item=channel, - document_type=self.schema.channel, - document={ - "type": TeamEndpointName.CHANNEL.value, - "_timestamp": datetime.utcnow(), - "team_name": team_name, - }, - ), - None, ) - ) - async for tabs in self.client.get_channel_tabs( - team_id=team_id, channel_id=channel.get("id") - ): - for tab in tabs: + for member in members: + member_document = self.formatter.format_team_member( + member, team_id, team_name + ) + member_document["_timestamp"] = iso_zulu() await self.queue.put( ( - self.formatter.format_doc( - item=tab, - document_type=self.schema.channel_tab, - document={ - "type": TeamEndpointName.TAB.value, - "_timestamp": datetime.utcnow(), - "team_name": team_name, - "channel_name": channel_name, - }, + self._decorate_with_access_control( + member_document, access_control ), None, ) ) - async for messages in self.client.get_channel_messages( - team_id=team_id, channel_id=channel.get("id") - ): - for message in messages: - await self.get_channel_messages( - message=message, channel_name=channel_name - ) + # Channels are processed inline (not scheduled as separate pool tasks) + # to avoid a nested fetchers.put that could deadlock the bounded pool. + async for channels in self.client.get_team_channels(team_id): + for channel in channels: + await self._produce_channel( + channel, team_id, team_name, access_control + ) + finally: + await self.queue.put(EndSignal.TEAM_TASK_FINISHED) - file = await self.client.get_channel_file( - team_id=team_id, channel_id=channel.get("id") - ) - drive_id = file.get("parentReference", {}).get("driveId") - await self.get_channel_drive_producer( - drive_id=drive_id, - item_id=file.get("id"), - team_name=team_name, - channel_name=channel_name, + def _chat_member_names(self, members): + return ",".join( + member.get("displayName") + for member in members + if member.get("displayName", "") ) - await self.queue.put(EndSignal.CHANNEL_TASK_FINISHED) + async def _process_chat_message(self, chat, message, member_names, access_control): + if message.get("deletedDateTime"): + return + if "unknownFutureValue" in (message.get("messageType") or ""): + return + message_content = html_to_text(html=(message.get("body") or {}).get("content")) + if not message_content and not message.get("attachments"): + return + document = self.formatter.format_chat_message( + chat=chat, + message=message, + message_content=message_content, + members=member_names, + ) + await self.queue.put( + (self._decorate_with_access_control(document, access_control), None) + ) - async def get_channel_drive_producer( - self, drive_id, item_id, team_name, channel_name + async def _process_chat_attachment( + self, sender_id, attachment_name, member_names, access_control ): - async for drive_child in self.client.get_channel_drive_childrens( - drive_id=drive_id, - item_id=item_id, + async for attachments in self.client.get_chat_attachments( + sender_id, attachment_name ): - if drive_child.get("file"): - format_attachment = self.formatter.format_doc( - item=drive_child, - document_type=self.schema.channel_attachment, + for attachment in attachments: + if not attachment.get("file"): + continue + drive_id = attachment.get("parentReference", {}).get("driveId") + if not drive_id: + continue + document = self.formatter.format_doc( + item=attachment, + document_type=self.schema.chat_attachment, document={ - "type": TeamEndpointName.ATTACHMENT.value, - "team_name": team_name, - "channel_name": channel_name, + "type": TeamsObjectType.CHAT_ATTACHMENT.value, + "members": member_names, }, ) + document = self._decorate_with_access_control(document, access_control) await self.queue.put( ( - format_attachment, + document, partial( self.get_content, - user_attachment=format_attachment, - download_url=drive_child.get( - "@microsoft.graph.downloadUrl" - ), + attachment=document, + drive_id=drive_id, + item_id=attachment["id"], ), ) ) - async def teams_producer(self, team): - team_id = team.get("id") - team_name = team.get("displayName") - await self.queue.put( - ( - self.formatter.format_doc( - item=team, - document_type=self.schema.teams, - document={ - "type": TeamEndpointName.TEAMS.value, - "_timestamp": datetime.utcnow(), - }, - ), - None, - ) - ) + async def chat_producer(self, chat): + chat_id = chat.get("id") + members = chat.get("members", []) + access_control = self._access_control_for_members(members) + member_names = self._chat_member_names(members) - async for channels in self.client.get_team_channels(team_id=team["id"]): - for channel in channels: - await self.fetchers.put( - partial(self.team_channel_producer, channel, team_id, team_name) + try: + chat_document = self.formatter.format_doc( + item=chat, + document_type=self.schema.chat, + document={"type": TeamsObjectType.CHAT.value}, + ) + if not chat_document.get("title"): + chat_document["title"] = member_names + await self.queue.put( + ( + self._decorate_with_access_control(chat_document, access_control), + None, ) - self.tasks += 1 - - await self.queue.put(EndSignal.TEAM_TASK_FINISHED) + ) - async def calendars_producer(self, user): - async for event in self.client.get_calendars(user_id=user["id"]): - if event and not event.get("isCancelled"): - await self.queue.put( - ( - self.formatter.format_user_calendars( - item=event, - ), - None, + async for messages in self.client.get_chat_messages(chat_id): + for message in messages: + await self._process_chat_message( + chat, message, member_names, access_control ) + + if self._attachments_enabled() and ( + message.get("from") and message["from"].get("user") + ): + for attachment in message.get("attachments", []): + if ( + attachment.get("name") + and attachment.get("contentType") == "reference" + ): + await self._process_chat_attachment( + message["from"]["user"]["id"], + attachment["name"], + member_names, + access_control, + ) + finally: + await self.queue.put(EndSignal.CHAT_TASK_FINISHED) + + async def _enumerate_producers(self): + """Enumerates teams and chats and schedules their producers. + + Runs as a single top-level task so that ``_consumer`` can drain the queue + concurrently. Enumerating everything up front (before consuming) can stall + on a large tenant: producers fill the bounded ``MemQueue`` while nothing is + draining it, and ``fetchers`` blocks once ``MAX_CONCURRENCY`` is reached. + """ + self._teams_enumeration_failed = False + self._chats_enumeration_failed = False + self._enumeration_error = None + try: + try: + async for teams in self.client.get_teams(): + for team in teams: + await self.fetchers.put(partial(self.team_producer, team)) + self.tasks += 1 + except PermissionsMissing: + self._teams_enumeration_failed = True + self._logger.warning( + "Unable to enumerate teams. Verify the 'Team.ReadBasic.All' application permission is granted." ) - await self.queue.put(EndSignal.CALENDAR_TASK_FINISHED) + + try: + async for chats in self.client.get_chats(): + for chat in chats: + await self.fetchers.put(partial(self.chat_producer, chat)) + self.tasks += 1 + except PermissionsMissing: + self._chats_enumeration_failed = True + self._logger.warning( + "Unable to enumerate chats. Verify the 'Chat.ReadBasic.WhereInstalled' application permission is granted and the connector's Teams app is installed." + ) + except Exception as exc: + # An unexpected (non-permission) error is treated as a connection-wide + # failure: record it and abort so get_docs can re-raise it. The finally + # below still emits ENUMERATION_FINISHED so the consumer never hangs. + self._enumeration_error = exc + self._logger.error( + "Unexpected error while enumerating Microsoft Teams resources; aborting sync.", + exc_info=exc, + ) + finally: + await self.queue.put(EndSignal.ENUMERATION_FINISHED) async def get_docs(self, filtering=None): - """Executes the logic to fetch Microsoft Teams objects in async manner + """Executes the logic to fetch Microsoft Teams objects in an async manner. Args: - filtering (Filtering): Object of class Filtering + filtering (Filtering): Object of class Filtering. Yields: - dictionary: dictionary containing meta-data of the files. + tuple: A document mapping and an optional coroutine to fetch its content. """ - async for chats in self.client.get_user_chats(): - for chat in chats: - await self.fetchers.put(partial(self.user_chat_producer, chat)) - self.tasks += 1 - - async for users in self.client.users(): - for user in users: - if user.get("mail"): - await self.fetchers.put(partial(self.calendars_producer, user)) - self.tasks += 1 - - async for teams in self.client.get_teams(): - for team in teams: - await self.fetchers.put(partial(self.teams_producer, team)) - self.tasks += 1 + await self.fetchers.put(partial(self._enumerate_producers)) + self.tasks += 1 async for item in self._consumer(): yield item await self.fetchers.join() - async def close(self): - """Closes unclosed client session""" - await self.client.close() + self.client.log_skip_summary() - @classmethod - def get_default_configuration(cls): - """Get the default configuration for Microsoft Teams. + if self._enumeration_error is not None: + raise self._enumeration_error - Returns: - dictionary: Default configuration. - """ - return { - "tenant_id": { - "label": "Tenant ID", - "order": 1, - "type": "str", - }, - "client_id": { - "label": "Client ID", - "order": 2, - "type": "str", - }, - "secret_value": { - "label": "Secret value", - "order": 3, - "sensitive": True, - "type": "str", - }, - "username": { - "label": "Username", - "order": 4, - "type": "str", - }, - "password": { - "label": "Password", - "order": 5, - "sensitive": True, - "type": "str", - }, - } + if self._teams_enumeration_failed and self._chats_enumeration_failed: + msg = ( + "Both team and chat enumeration failed due to missing permissions. " + "Refusing to report a successful sync, as this would delete previously " + "indexed documents. Verify 'Team.ReadBasic.All' and " + "'Chat.ReadBasic.WhereInstalled' are granted and the connector's Teams " + "app is installed." + ) + raise PermissionsMissing(msg) diff --git a/app/connectors_service/connectors/sources/microsoft_teams/formatter.py b/app/connectors_service/connectors/sources/microsoft_teams/formatter.py index 360080708..d0c2fb534 100644 --- a/app/connectors_service/connectors/sources/microsoft_teams/formatter.py +++ b/app/connectors_service/connectors/sources/microsoft_teams/formatter.py @@ -3,205 +3,69 @@ # or more contributor license agreements. Licensed under the Elastic License 2.0; # you may not use this file except in compliance with the Elastic License 2.0. # -from calendar import month_name -from datetime import datetime - -from connectors.sources.microsoft_teams.client import TeamEndpointName, UserEndpointName +from connectors.sources.microsoft_teams.client import TeamsObjectType class MicrosoftTeamsFormatter: - """Format documents""" + """Format Microsoft Graph objects into Elasticsearch documents.""" def __init__(self, schema): self.schema = schema - def map_document_with_schema( - self, - document, - item, - document_type, - ): - """Prepare key mappings for documents + def map_document_with_schema(self, document, item, document_type): + """Prepare key mappings for documents. Args: - document(dictionary): Modified document - item (dictionary): Document from Microsoft Teams. - document_type(string): Name of function to be called for fetching the mapping. - - Returns: - dictionary: Modified document with the help of adapter schema. + document (dict): Document being built. + item (dict): Object returned by Microsoft Graph. + document_type (callable): Schema method returning the field mapping. """ - for elasticsearch_field, sharepoint_field in document_type().items(): - document[elasticsearch_field] = item.get(sharepoint_field) + for elasticsearch_field, graph_field in document_type().items(): + document[elasticsearch_field] = item.get(graph_field) - def format_doc(self, item, document_type, **kwargs): - document = {} - for elasticsearch_field, sharepoint_field in kwargs["document"].items(): - document[elasticsearch_field] = sharepoint_field + def format_doc(self, item, document_type, document): + result = dict(document) self.map_document_with_schema( - document=document, item=item, document_type=document_type - ) - return document - - def format_user_chat_meeting_recording(self, item, url): - document = {"type": UserEndpointName.MEETING_RECORDING.value} - document.update( - { - "_id": item.get("eventDetail", {}).get("callId"), - "title": item.get("eventDetail", {}).get("callRecordingDisplayName"), - "url": url, - "_timestamp": item.get("lastModifiedDateTime"), - } - ) - return document - - def get_calendar_detail(self, calendar): - body = "" - organizer = calendar.get("organizer", {}).get("emailAddress").get("name") - calendar_recurrence = calendar.get("recurrence") - if calendar_recurrence: - recurrence_range = calendar_recurrence.get("range") - pattern = calendar_recurrence.get("pattern") - occurrence = f"{pattern['interval']}" if pattern.get("interval") else "" - pattern_type = pattern.get("type", "") - - # In case type of meeting is daily so body will be: Recurrence: Occurs every 1 day starting {startdate} - # until {enddate} - if pattern_type == "daily": - days = f"{occurrence} day" - - # If type of meeting is yearly so body will be: Recurrence: Occurs every year on day 5 of march starting - # {date} until {enddate} - elif pattern_type in ["absoluteYearly", "relativeYearly"]: - day_pattern = ( - f"on day {pattern['dayOfMonth']}" - if pattern.get("dayOfMonth") - else f"on {pattern['index']} {','.join(pattern['daysOfWeek'])}" - ) - days = f"year {day_pattern} of {month_name[pattern['month']]}" - - # If type of meeting is monthly so body will be: Recurrence: Occurs every month on day 5 of march - # starting {date} until {enddate} - elif pattern_type in ["absoluteMonthly", "relativeMonthly"]: - days_pattern = ( - f"on day {pattern['dayOfMonth']}" - if pattern.get("dayOfMonth") - else f"on {pattern['index']} {','.join(pattern['daysOfWeek'])}" - ) - days = f"{occurrence} month {days_pattern}" - - # Else goes in weekly situation where body will be: Recurrence: Occurs Every 3 week on monday,tuesday, - # wednesday starting {date} until {enddate} - else: - week = ",".join(pattern.get("daysOfWeek")) - days = f"{occurrence} week on {week}" - - date = ( - f"{recurrence_range.get('startDate')}" - if recurrence_range.get("type", "") == "noEnd" - else f"{recurrence_range.get('startDate')} until {recurrence_range.get('endDate')}" - ) - recurrence = f"Occurs Every {days} starting {date}" - body = f"Recurrence: {recurrence} Organizer: {organizer}" - - else: - start_time = datetime.strptime( - calendar["start"]["dateTime"][:-4], USER_MEETING_DATETIME_FORMAT - ).strftime("%d %b, %Y at %H:%M") - end_time = datetime.strptime( - calendar["end"]["dateTime"][:-4], USER_MEETING_DATETIME_FORMAT - ).strftime("%d %b, %Y at %H:%M") - body = f"Schedule: {start_time} to {end_time} Organizer: {organizer}" - return body - - def format_user_calendars(self, item): - document = {"type": UserEndpointName.MEETING.value} - attendee_list = ( - [ - f"{attendee.get('emailAddress', {}).get('name')}({attendee.get('emailAddress', {}).get('address')})" - for attendee in item["attendees"] - ] - if item.get("attendees") - else [] - ) - document.update( - { # pyright: ignore - "attendees": attendee_list, - "online_meeting_url": item["onlineMeeting"].get("joinUrl") - if item.get("onlineMeeting") - else "", - "description": item.get("bodyPreview"), - "meeting_detail": self.get_calendar_detail(calendar=item), - "location": [ - f"{location['displayName']}" for location in item["locations"] - ] - if item.get("locations") - else [], - } + document=result, item=item, document_type=document_type ) + return result + + def format_team_member(self, member, team_id, team_name): + document = { + "type": TeamsObjectType.TEAM_MEMBER.value, + "team_name": team_name, + "user_id": member.get("userId"), + "roles": ", ".join(member.get("roles", []) or []), + } self.map_document_with_schema( - document=document, item=item, document_type=self.schema.meeting + document=document, item=member, document_type=self.schema.team_member ) + # The membership id is not globally unique; prefix it with the team id. + document["_id"] = f"{team_id}-{member.get('id')}" return document def format_channel_message(self, item, channel_name, message_content): - document = {"type": TeamEndpointName.MESSAGE.value} - document.update( - { # pyright: ignore - "sender": item["from"]["user"].get("displayName") - if item.get("from") and item["from"].get("user") - else "", - "channel": channel_name, - "message": message_content, - "attached_documents": self.format_attachment_names( - attachments=item.get("attachments") - ), - } - ) + document = { + "type": TeamsObjectType.CHANNEL_MESSAGE.value, + "sender": item["from"]["user"].get("displayName") + if item.get("from") and item["from"].get("user") + else "", + "channel": channel_name, + "message": message_content, + "attached_documents": self.format_attachment_names( + attachments=item.get("attachments") + ), + } self.map_document_with_schema( document=document, item=item, document_type=self.schema.channel_message ) return document - def format_channel_meeting(self, reply): - document = {"type": TeamEndpointName.MEETING.value} - event = reply["eventDetail"] - if event.get("@odata.type") == "#microsoft.graph.callEndedEventMessageDetail": - participant_list = [] - for participant in event.get("callParticipants", []): - user = participant.get("participant", {}).get("user") - if user: - participant_list.append(user.get("displayName")) - participant_names = ", ".join(participant_list) - document.update( - { - "_id": event.get("callId"), - "_timestamp": reply.get("lastModifiedDateTime"), - "participants": participant_names, - } - ) - elif ( - event.get("@odata.type") - == "#microsoft.graph.callRecordingEventMessageDetail" - ): - if event.get("callRecordingUrl") and ( - ".sharepoint.com" in event["callRecordingUrl"] - ): - document.update( - { - "title": event.get("callRecordingDisplayName"), - "recording_url": event.get("callRecordingUrl"), - } - ) - return document - - async def format_user_chat_messages(self, chat, message, message_content, members): - if chat.get("topic"): - message.update({"title": chat["topic"]}) - else: - message.update({"title": members}) - message.update( + def format_chat_message(self, chat, message, message_content, members): + augmented = dict(message) + augmented.update( { + "title": chat.get("topic") or members, "webUrl": chat.get("webUrl"), "chatType": chat.get("chatType"), "sender": message["from"]["user"].get("displayName") @@ -210,7 +74,16 @@ async def format_user_chat_messages(self, chat, message, message_content, member "message": message_content, } ) - return message + document = { + "type": TeamsObjectType.CHAT_MESSAGE.value, + "attached_documents": self.format_attachment_names( + attachments=message.get("attachments") + ), + } + self.map_document_with_schema( + document=document, item=augmented, document_type=self.schema.chat_message + ) + return document def format_attachment_names(self, attachments): if not attachments: @@ -221,6 +94,3 @@ def format_attachment_names(self, attachments): for attachment in attachments if attachment.get("name", "") ) - - -USER_MEETING_DATETIME_FORMAT = "%Y-%m-%dT%H:%M:%S.%f" diff --git a/app/connectors_service/connectors/sources/microsoft_teams/teams_app_manifest/README.md b/app/connectors_service/connectors/sources/microsoft_teams/teams_app_manifest/README.md new file mode 100644 index 000000000..a47a46469 --- /dev/null +++ b/app/connectors_service/connectors/sources/microsoft_teams/teams_app_manifest/README.md @@ -0,0 +1,99 @@ +# Microsoft Teams connector setup + +The Microsoft Teams connector authenticates with **application-only** credentials +(no user sign-in) and follows the least-privilege permission model. Because +Microsoft only exposes read access to Teams messages through either broad +`*.All` "protected" permissions or the narrower **resource-specific consent +(RSC)** / **`WhereInstalled`** permissions, this connector uses the latter. That +means the connector's Entra (Azure AD) application must be packaged as a Teams +app and **installed into the teams and chats** you want to sync. + +## 1. Register an Entra (Azure AD) application + +1. In the [Microsoft Entra admin center](https://entra.microsoft.com), register a + new application (a confidential client). +2. Record the **Directory (tenant) ID** and **Application (client) ID**. +3. Create either: + - a **client secret** (Certificates & secrets -> New client secret), or + - a **certificate** (upload a certificate and keep the matching private key). + +Use these values for the connector's `Tenant ID`, `Client ID`, and +`Secret value` (or `Certificate` + `Private key`) configuration fields. + +## 2. Grant tenant-wide application permissions (Entra) + +Add and grant **admin consent** for these Microsoft Graph **application** +permissions. They are used to enumerate resources and to read chats where the +app is installed: + +| Permission | Why | +| --- | --- | +| `Team.ReadBasic.All` | Enumerate all teams in the tenant. | +| `Channel.ReadBasic.All` | List channels of each team. | +| `Chat.ReadBasic.WhereInstalled` | List chats where the app is installed. | +| `Chat.Read.WhereInstalled` | Read messages of chats where the app is installed. | +| `ChatMember.Read.WhereInstalled` | Read members of chats where the app is installed. | +| `Files.Read.All` (optional) | Download channel/chat attachment content. Only needed when "Fetch attachment content" is enabled. | + +> There is no `WhereInstalled` variant for reading channel messages or team +> members; those use the RSC permissions declared in the Teams app manifest +> (step 3) and are only granted for teams where the app is installed. + +> **On the two `.All` permissions:** `Team.ReadBasic.All` and +> `Channel.ReadBasic.All` are unavoidable. Microsoft Graph offers no narrower +> (`.Selected` / `.WhereInstalled` / RSC) way to *enumerate* the teams and +> channels of a tenant, and both are metadata-only (names/descriptions, not +> message content). `Files.Read.All` is the only application permission that can +> download attachment bytes and is required only when "Fetch attachment content" +> is enabled. + +## 3. Package and install the Teams app (RSC) + +1. Edit [`manifest.json`](./manifest.json): + - Set `id` to a new GUID (the Teams app id). + - Set `webApplicationInfo.id` to your Entra **Application (client) ID**. + - Set `webApplicationInfo.resource` to an Application ID URI on your tenant. +2. Add `color.png` (192x192) and `outline.png` (32x32) icons next to the + manifest and zip the three files together into an app package. +3. Upload the package to your organization's Teams app catalog + (Teams admin center -> Teams apps -> Manage apps -> Upload). +4. **Install** the app into the teams and chats you want to sync. To cover the + whole tenant, use an + [app setup policy](https://learn.microsoft.com/en-us/microsoftteams/teams-app-setup-policies) + to auto-install it broadly. + +The RSC permissions declared in the manifest +(`ChannelMessage.Read.Group`, `ChannelSettings.Read.Group`, +`TeamMember.Read.Group`, `ChatMessage.Read.Chat`, `ChatMember.Read.Chat`) are +granted per resource when the app is installed there. + +## What gets synced + +- **Teams** (from groups that are teams) +- **Channels** and their messages, message **replies**, and attachments +- **Team members** +- **Chats** and their messages and attachments + +### Coverage notes and limitations + +- **Chat coverage is limited to chats where the connector's Teams app is + installed** (`Chat.*.WhereInstalled`), not "all chats of all team members." + Full-tenant chat coverage would require `Chat.Read.All`, which is a broad, + "protected" permission that this connector intentionally does not use in order + to stay least-privilege. Install the app in the chats you need indexed. +- **Chats have no threaded replies in Microsoft Graph.** Reply threading exists + only for *channel* messages; chats are linear, so every chat message is + already captured by the flat message listing (a chat "reply" is just another + message). Only channel messages expand `replies`. + +Anything the app is not installed in is invisible to the connector and is +skipped. Skipped resources are counted and reported as an aggregate warning at +the end of each sync, so an under-installed tenant is visible in the logs rather +than producing a quiet, near-empty sync. + +## Document level security (DLS) + +When "Enable document level security" is on, access to each document is +restricted to the members of its team/chat. The access-control sync only creates +identities for users that participate in synced teams/chats, so `Users.Read.All` +is never required. diff --git a/app/connectors_service/connectors/sources/microsoft_teams/teams_app_manifest/manifest.json b/app/connectors_service/connectors/sources/microsoft_teams/teams_app_manifest/manifest.json new file mode 100644 index 000000000..cbcedea0d --- /dev/null +++ b/app/connectors_service/connectors/sources/microsoft_teams/teams_app_manifest/manifest.json @@ -0,0 +1,56 @@ +{ + "$schema": "https://developer.microsoft.com/en-us/json-schemas/teams/v1.16/MicrosoftTeams.schema.json", + "manifestVersion": "1.16", + "version": "1.0.0", + "id": "REPLACE_WITH_A_NEW_GUID", + "packageName": "co.elastic.connectors.teams", + "developer": { + "name": "Elastic", + "websiteUrl": "https://www.elastic.co", + "privacyUrl": "https://www.elastic.co/legal/privacy-statement", + "termsOfUseUrl": "https://www.elastic.co/legal/terms-of-service" + }, + "name": { + "short": "Elastic Teams Connector", + "full": "Elastic Microsoft Teams Connector" + }, + "description": { + "short": "Indexes Microsoft Teams content into Elasticsearch.", + "full": "Grants the Elastic Microsoft Teams connector resource-specific read access to teams and chats where it is installed so their content can be indexed into Elasticsearch." + }, + "icons": { + "color": "color.png", + "outline": "outline.png" + }, + "accentColor": "#FFFFFF", + "webApplicationInfo": { + "id": "REPLACE_WITH_ENTRA_APPLICATION_CLIENT_ID", + "resource": "https://REPLACE_WITH_YOUR_TENANT.onmicrosoft.com/teams-connector" + }, + "authorization": { + "permissions": { + "resourceSpecific": [ + { + "name": "ChannelMessage.Read.Group", + "type": "Application" + }, + { + "name": "ChannelSettings.Read.Group", + "type": "Application" + }, + { + "name": "TeamMember.Read.Group", + "type": "Application" + }, + { + "name": "ChatMessage.Read.Chat", + "type": "Application" + }, + { + "name": "ChatMember.Read.Chat", + "type": "Application" + } + ] + } + } +} diff --git a/app/connectors_service/connectors/sources/shared/microsoft/__init__.py b/app/connectors_service/connectors/sources/shared/microsoft/__init__.py new file mode 100644 index 000000000..1fa99ac6a --- /dev/null +++ b/app/connectors_service/connectors/sources/shared/microsoft/__init__.py @@ -0,0 +1,5 @@ +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License 2.0; +# you may not use this file except in compliance with the Elastic License 2.0. +# diff --git a/app/connectors_service/connectors/sources/shared/microsoft/graph.py b/app/connectors_service/connectors/sources/shared/microsoft/graph.py new file mode 100644 index 000000000..cc4bd3e00 --- /dev/null +++ b/app/connectors_service/connectors/sources/shared/microsoft/graph.py @@ -0,0 +1,464 @@ +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License 2.0; +# you may not use this file except in compliance with the Elastic License 2.0. +# +"""Shared Microsoft Graph / Entra building blocks. + +This module hosts the reusable authentication and HTTP plumbing used to talk to +Microsoft Graph. It is shared between connectors that integrate with Microsoft +services (for example SharePoint Online and Microsoft Teams) so that token +management, pagination, throttling and retry behavior are implemented once. +""" + +import os +from contextlib import asynccontextmanager +from datetime import datetime, timedelta +from functools import wraps + +import aiohttp +from aiohttp.client_exceptions import ClientPayloadError, ClientResponseError +from aiohttp.client_reqrep import RequestInfo +from azure.identity.aio import CertificateCredential +from connectors_sdk.logger import logger +from connectors_sdk.utils import nested_get_from_dict + +from connectors.utils import ( + CacheWithTimeout, + CancellableSleeps, + retryable, +) + +if "OVERRIDE_URL" in os.environ: + logger.warning("x" * 50) + logger.warning( + f"MICROSOFT GRAPH CALLS ARE REDIRECTED TO {os.environ['OVERRIDE_URL']}" + ) + logger.warning("IT'S SUPPOSED TO BE USED ONLY FOR TESTING") + logger.warning("x" * 50) + GRAPH_API_AUTH_URL = os.environ["OVERRIDE_URL"] +else: + GRAPH_API_AUTH_URL = "https://login.microsoftonline.com" + +DEFAULT_RETRY_COUNT = 5 +DEFAULT_RETRY_SECONDS = 30 +DEFAULT_BACKOFF_MULTIPLIER = 5 +FILE_WRITE_CHUNK_SIZE = 1024 * 64 # 64KB default SSD page size + +# Microsoft Graph API pagination constants +# https://learn.microsoft.com/en-us/graph/delta-query-overview +DELTA_NEXT_LINK_KEY = "@odata.nextLink" +DELTA_LINK_KEY = "@odata.deltaLink" + + +class NotFound(Exception): + """Internal exception class to handle 404s from the API that has a meaning, that collection + for specific object is empty. + + For example List Items API from Sharepoint REST API returns 404 if list has no items. + + It's not an exception for us, we just want to return [], and this exception class facilitates it. + """ + + pass + + +class BadRequestError(Exception): + """Internal exception class to handle 400's from the API. + + Similar to the NotFound exception, this allows us to catch edge-case responses that should + be translated as empty results, and let us return [].""" + + pass + + +class InternalServerError(Exception): + """Exception class to indicate that something went wrong on the server side.""" + + pass + + +class ThrottledError(Exception): + """Internal exception class to indicate that request was throttled by the API""" + + pass + + +class TokenFetchFailed(Exception): + """Exception class to notify that connector was unable to fetch authentication token from either + Sharepoint REST API or Graph API. + + Error message will indicate human-readable reason. + """ + + pass + + +class PermissionsMissing(Exception): + """Exception class to notify that specific Application Permission is missing for the credentials used. + See: https://learn.microsoft.com/en-us/graph/permissions-reference + """ + + pass + + +class MicrosoftSecurityToken: + """Abstract token for connecting to one of Microsoft Azure services. + + This class is an abstract base class for getting auth token. + + It takes care of caching the token and asking for new token once the + token expires. + + Classes that inherit from this class need to implement `async def _fetch_token(self)` method + that needs to return a tuple: access_token and expires_in. + + To read more about tenants and authentication, see: + - https://learn.microsoft.com/en-us/azure/active-directory/develop/quickstart-create-new-tenant + - https://learn.microsoft.com/en-us/azure/active-directory/develop/quickstart-register-app + """ + + def __init__(self, http_session, tenant_id, tenant_name, client_id): + """Initializer. + + Args: + http_session (aiohttp.ClientSession): HTTP Client Session + tenant_id (str): Azure AD Tenant Id + tenant_name (str): Azure AD Tenant Name + client_id (str): Azure App Client Id + client_secret (str): Azure App Client Secret Value""" + + self._http_session = http_session + self._tenant_id = tenant_id + self._tenant_name = tenant_name + self._client_id = client_id + + self._token_cache = CacheWithTimeout() + + async def get(self): + """Get bearer token for provided credentials. + + If token has been retrieved, it'll be taken from the cache. + Otherwise, call to `_fetch_token` is made to fetch the token + from 3rd-party service. + + Returns: + str: bearer token for one of Microsoft services""" + + cached_value = self._token_cache.get_value() + + if cached_value: + return cached_value + + try: + access_token, expires_at = await self._fetch_token() + except ClientResponseError as e: + # Both Graph API and REST API return error codes that indicate different problems happening when authenticating. + # Error Code serves as a good starting point classifying these errors, see the messages below: + match e.status: + case 400: + msg = "Failed to authorize to Microsoft services. Please verify, that provided Tenant Id, Tenant Name and Client ID are valid." + raise TokenFetchFailed(msg) from e + case 401: + msg = "Failed to authorize to Microsoft services. Please verify, that provided Secret Value is valid." + raise TokenFetchFailed(msg) from e + case _: + msg = f"Failed to authorize to Microsoft services. Response Status: {e.status}, Message: {e.message}" + raise TokenFetchFailed(msg) from e + + self._token_cache.set_value(access_token, expires_at) + + return access_token + + async def _fetch_token(self): + """Fetch token from Microsoft service. + + This method needs to be implemented in the class that inherits MicrosoftSecurityToken. + + Returns: + (str, int) - a tuple containing access token as a string and number of seconds it will be valid for as an integer + """ + + raise NotImplementedError + + +class SecretAPIToken(MicrosoftSecurityToken): + def __init__(self, http_session, tenant_id, tenant_name, client_id, client_secret): + super().__init__(http_session, tenant_id, tenant_name, client_id) + self._client_secret = client_secret + + async def _fetch_token(self): + return await super()._fetch_token() + + +class GraphAPIToken(SecretAPIToken): + """Token to connect to Microsoft Graph API endpoints.""" + + @retryable(retries=3) + async def _fetch_token(self): + """Fetch API token for usage with Graph API + + Returns: + (str, int) - a tuple containing access token as a string and number of seconds it will be valid for as an integer + """ + + url = f"{GRAPH_API_AUTH_URL}/{self._tenant_id}/oauth2/v2.0/token" + headers = {"Content-Type": "application/x-www-form-urlencoded"} + data = f"client_id={self._client_id}&scope=https://graph.microsoft.com/.default&client_secret={self._client_secret}&grant_type=client_credentials" + + # We measure now before request to be on a pessimistic side + now = datetime.utcnow() + async with self._http_session.post(url, headers=headers, data=data) as resp: + json_response = await resp.json() + access_token = json_response["access_token"] + expires_in = int(json_response["expires_in"]) + + return access_token, now + timedelta(seconds=expires_in) + + +class EntraAPIToken(MicrosoftSecurityToken): + """Token to connect to Microsoft Graph API endpoints using certificate credentials.""" + + def __init__( + self, + http_session, + tenant_id, + tenant_name, + client_id, + certificate, + private_key, + scope, + ): + super().__init__(http_session, tenant_id, tenant_name, client_id) + self._certificate = certificate + self._private_key = private_key + self._scope = scope + + @retryable(retries=3) + async def _fetch_token(self): + """Fetch API token for usage with Graph API + + Returns: + (str, int) - a tuple containing access token as a string and number of seconds it will be valid for as an integer + """ + + secrets_concat = self._certificate + "\n" + self._private_key + + credentials = CertificateCredential( + self._tenant_id, self._client_id, certificate_data=secrets_concat.encode() + ) + + token = await credentials.get_token(self._scope) + + await credentials.close() + + return token.token, datetime.utcfromtimestamp(token.expires_on) + + +def retryable_aiohttp_call(retries): + # TODO: improve utils.retryable to allow custom logic + # that can help choose what to retry + def wrapper(func): + @wraps(func) + async def wrapped(*args, **kwargs): + retry = 1 + while retry <= retries: + try: + async for item in func(*args, **kwargs, retry_count=retry): + yield item + break + except (NotFound, BadRequestError): + raise + except Exception: + if retry >= retries: + raise + retry += 1 + + return wrapped + + return wrapper + + +class MicrosoftAPISession: + def __init__(self, http_session, api_token, scroll_field, logger_): + self._http_session = http_session + self._api_token = api_token + + # Graph API and Sharepoint API scroll over slightly different fields: + # - odata.nextPage for Sharepoint REST API uses + # - @odata.nextPage for Graph API uses - notice the @ glyph + # Therefore for flexibility I made it a field passed in the initializer, + # but this abstraction can be better. + self._scroll_field = scroll_field + self._sleeps = CancellableSleeps() + self._logger = logger_ + + def set_logger(self, logger_): + self._logger = logger_ + + def close(self): + self._sleeps.cancel() + + async def fetch(self, url): + return await self._get_json(url) + + async def post(self, url, payload): + self._logger.debug(f"Post to url: '{url}' with body: {payload}") + async with self._post(url, payload) as resp: + return await resp.json() + + async def pipe(self, url, stream): + async with self._get(url) as resp: + async for data in resp.content.iter_chunked(FILE_WRITE_CHUNK_SIZE): + await stream.write(data) + + async def scroll(self, url): + scroll_url = url + + while True: + graph_data = await self._get_json(scroll_url) + # We're yielding the whole page here, not one item + yield graph_data["value"] + + if self._scroll_field in graph_data: + scroll_url = graph_data[self._scroll_field] + else: + break + + async def scroll_delta_url(self, url): + scroll_url = url + + while True: + graph_data = await self._get_json(scroll_url) + + yield graph_data + + if DELTA_NEXT_LINK_KEY in graph_data: + scroll_url = graph_data[DELTA_NEXT_LINK_KEY] + else: + break + + async def _get_json(self, absolute_url): + self._logger.debug(f"Fetching url: {absolute_url}") + async with self._get(absolute_url) as resp: + return await resp.json() + + @asynccontextmanager + @retryable_aiohttp_call(retries=DEFAULT_RETRY_COUNT) + async def _post(self, absolute_url, payload=None, retry_count=0): + try: + token = await self._api_token.get() + headers = {"authorization": f"Bearer {token}"} + self._logger.debug(f"Posting to Microsoft Endpoint: {absolute_url}.") + + async with self._http_session.post( + absolute_url, headers=headers, json=payload + ) as resp: + if absolute_url.endswith("/$batch"): # response code of $batch lies + await self._check_batch_items_for_errors(absolute_url, resp) + yield resp + except aiohttp.client_exceptions.ClientOSError: + self._logger.warning( + "The Microsoft Graph API dropped the connection. It might indicate that the connector is making too many requests. Decrease concurrency settings, otherwise the Graph API may block this app." + ) + raise + except ClientResponseError as e: + await self._handle_client_response_error(absolute_url, e, retry_count) + except ClientPayloadError as e: + await self._handle_client_payload_error(e, retry_count) + + async def _check_batch_items_for_errors(self, url, batch_resp): + body = await batch_resp.json() + responses = body.get("responses", []) + for response in responses: + status = response.get("status", 200) + if status != 200: + self._logger.warning(f"Batch request item failed with: {response}") + headers = response.get("headers", {}) + req_info = RequestInfo(url=url, method="POST", headers=headers) + raise ClientResponseError( + request_info=req_info, + headers=headers, + status=status, + message=nested_get_from_dict( + response, ["body", "error", "message"] + ), + history=(batch_resp), + ) + + @asynccontextmanager + @retryable_aiohttp_call(retries=DEFAULT_RETRY_COUNT) + async def _get(self, absolute_url, retry_count=0): + try: + token = await self._api_token.get() + headers = {"authorization": f"Bearer {token}"} + self._logger.debug(f"Calling Microsoft Endpoint: {absolute_url}") + + async with self._http_session.get( + absolute_url, + headers=headers, + ) as resp: + yield resp + except aiohttp.client_exceptions.ClientOSError: + self._logger.warning( + "Graph API dropped the connection. It might indicate, that connector makes too many requests - decrease concurrency settings, otherwise Graph API can block this app." + ) + raise + except ClientResponseError as e: + await self._handle_client_response_error(absolute_url, e, retry_count) + except ClientPayloadError as e: + await self._handle_client_payload_error(e, retry_count) + + async def _handle_client_payload_error(self, e, retry_count): + await self._sleeps.sleep( + self._compute_retry_after( + DEFAULT_RETRY_SECONDS, retry_count, DEFAULT_BACKOFF_MULTIPLIER + ) + ) + + raise e + + async def _handle_client_response_error(self, absolute_url, e, retry_count): + if e.status == 429 or e.status == 503: + response_headers = e.headers or {} + + if "Retry-After" in response_headers: + retry_after = int(response_headers["Retry-After"]) + else: + self._logger.warning( + f"Response Code from Microsoft Graph is {e.status} but Retry-After header is not found, using default retry time: {DEFAULT_RETRY_SECONDS} seconds" + ) + retry_after = DEFAULT_RETRY_SECONDS + + retry_seconds = self._compute_retry_after( + retry_after, retry_count, DEFAULT_BACKOFF_MULTIPLIER + ) + + self._logger.warning( + f"Rate Limited by Microsoft: a new attempt will be performed in {retry_seconds} seconds (retry-after header: {retry_after}, retry_count: {retry_count}, backoff: {DEFAULT_BACKOFF_MULTIPLIER})" + ) + + await self._sleeps.sleep(retry_seconds) + raise ThrottledError from e + elif ( + e.status == 403 or e.status == 401 + ): # Might work weird, but Graph returns 403 and REST returns 401 + msg = f"Received Unauthorized response for {absolute_url}.\nVerify that the correct Graph API and Microsoft permissions are granted to the app and admin consent is given. If the permissions and consent are correct, wait for several minutes and try again." + raise PermissionsMissing(msg) from e + elif e.status == 404: + raise NotFound from e # We wanna catch it in the code that uses this and ignore in some cases + elif e.status == 500: + raise InternalServerError from e + elif e.status == 400: + self._logger.warning(f"Received 400 response from {absolute_url}") + raise BadRequestError from e + else: + raise + + def _compute_retry_after(self, retry_after, retry_count, backoff): + # Wait for what the API asks after the first failure. + # Apply backoff if API is still not available. + if retry_count <= 1: + return retry_after + else: + return retry_after + retry_count * backoff diff --git a/app/connectors_service/connectors/sources/sharepoint/sharepoint_online/client.py b/app/connectors_service/connectors/sources/sharepoint/sharepoint_online/client.py index 64d275559..7b3f1addf 100644 --- a/app/connectors_service/connectors/sources/sharepoint/sharepoint_online/client.py +++ b/app/connectors_service/connectors/sources/sharepoint/sharepoint_online/client.py @@ -7,26 +7,32 @@ import os import re from collections.abc import Iterable, Sized -from contextlib import asynccontextmanager from datetime import datetime, timedelta -from functools import wraps import aiohttp -from aiohttp.client_exceptions import ClientPayloadError, ClientResponseError -from aiohttp.client_reqrep import RequestInfo -from azure.identity.aio import CertificateCredential from connectors_sdk.logger import logger -from connectors_sdk.utils import nested_get_from_dict +# Shared Microsoft Graph / Entra building blocks. Re-exported below so that +# existing imports from this module keep working. +from connectors.sources.shared.microsoft.graph import ( # noqa: F401 + BadRequestError, + EntraAPIToken, + GraphAPIToken, + InternalServerError, + MicrosoftAPISession, + MicrosoftSecurityToken, + NotFound, + PermissionsMissing, + SecretAPIToken, + ThrottledError, + TokenFetchFailed, + retryable_aiohttp_call, +) from connectors.sources.sharepoint.sharepoint_online.constants import ( - DEFAULT_BACKOFF_MULTIPLIER, DEFAULT_PARALLEL_CONNECTION_COUNT, DEFAULT_RETRY_COUNT, - DEFAULT_RETRY_SECONDS, DELTA_LINK_KEY, - DELTA_NEXT_LINK_KEY, DRIVE_ITEMS_FIELDS, - FILE_WRITE_CHUNK_SIZE, GRAPH_API_AUTH_URL, GRAPH_API_URL, REST_API_AUTH_URL, @@ -36,184 +42,17 @@ _is_excluded_sharepoint_url, ) from connectors.utils import ( - CacheWithTimeout, - CancellableSleeps, retryable, url_encode, ) -class NotFound(Exception): - """Internal exception class to handle 404s from the API that has a meaning, that collection - for specific object is empty. - - For example List Items API from Sharepoint REST API returns 404 if list has no items. - - It's not an exception for us, we just want to return [], and this exception class facilitates it. - """ - - pass - - -class BadRequestError(Exception): - """Internal exception class to handle 400's from the API. - - Similar to the NotFound exception, this allows us to catch edge-case responses that should - be translated as empty resutls, and let us return [].""" - - pass - - -class InternalServerError(Exception): - """Exception class to indicate that something went wrong on the server side.""" - - pass - - -class ThrottledError(Exception): - """Internal exception class to indicate that request was throttled by the API""" - - pass - - class InvalidSharepointTenant(Exception): """Exception class to notify that tenant name is invalid or does not match tenant id provided""" pass -class TokenFetchFailed(Exception): - """Exception class to notify that connector was unable to fetch authentication token from either - Sharepoint REST API or Graph API. - - Error message will indicate human-readable reason. - """ - - pass - - -class PermissionsMissing(Exception): - """Exception class to notify that specific Application Permission is missing for the credentials used. - See: https://learn.microsoft.com/en-us/graph/permissions-reference - """ - - pass - - -class MicrosoftSecurityToken: - """Abstract token for connecting to one of Microsoft Azure services. - - This class is an abstract base class for getting auth token. - - It takes care of caching the token and asking for new token once the - token expires. - - Classes that inherit from this class need to implement `async def _fetch_token(self)` method - that needs to return a tuple: access_token and expires_in. - - To read more about tenants and authentication, see: - - https://learn.microsoft.com/en-us/azure/active-directory/develop/quickstart-create-new-tenant - - https://learn.microsoft.com/en-us/azure/active-directory/develop/quickstart-register-app - """ - - def __init__(self, http_session, tenant_id, tenant_name, client_id): - """Initializer. - - Args: - http_session (aiohttp.ClientSession): HTTP Client Session - tenant_id (str): Azure AD Tenant Id - tenant_name (str): Azure AD Tenant Name - client_id (str): Azure App Client Id - client_secret (str): Azure App Client Secret Value""" - - self._http_session = http_session - self._tenant_id = tenant_id - self._tenant_name = tenant_name - self._client_id = client_id - - self._token_cache = CacheWithTimeout() - - async def get(self): - """Get bearer token for provided credentials. - - If token has been retrieved, it'll be taken from the cache. - Otherwise, call to `_fetch_token` is made to fetch the token - from 3rd-party service. - - Returns: - str: bearer token for one of Microsoft services""" - - cached_value = self._token_cache.get_value() - - if cached_value: - return cached_value - - try: - access_token, expires_at = await self._fetch_token() - except ClientResponseError as e: - # Both Graph API and REST API return error codes that indicate different problems happening when authenticating. - # Error Code serves as a good starting point classifying these errors, see the messages below: - match e.status: - case 400: - msg = "Failed to authorize to Sharepoint REST API. Please verify, that provided Tenant Id, Tenant Name and Client ID are valid." - raise TokenFetchFailed(msg) from e - case 401: - msg = "Failed to authorize to Sharepoint REST API. Please verify, that provided Secret Value is valid." - raise TokenFetchFailed(msg) from e - case _: - msg = f"Failed to authorize to Sharepoint REST API. Response Status: {e.status}, Message: {e.message}" - raise TokenFetchFailed(msg) from e - - self._token_cache.set_value(access_token, expires_at) - - return access_token - - async def _fetch_token(self): - """Fetch token from Microsoft service. - - This method needs to be implemented in the class that inherits MicrosoftSecurityToken. - - Returns: - (str, int) - a tuple containing access token as a string and number of seconds it will be valid for as an integer - """ - - raise NotImplementedError - - -class SecretAPIToken(MicrosoftSecurityToken): - def __init__(self, http_session, tenant_id, tenant_name, client_id, client_secret): - super().__init__(http_session, tenant_id, tenant_name, client_id) - self._client_secret = client_secret - - async def _fetch_token(self): - return await super()._fetch_token() - - -class GraphAPIToken(SecretAPIToken): - """Token to connect to Microsoft Graph API endpoints.""" - - @retryable(retries=3) - async def _fetch_token(self): - """Fetch API token for usage with Graph API - - Returns: - (str, int) - a tuple containing access token as a string and number of seconds it will be valid for as an integer - """ - - url = f"{GRAPH_API_AUTH_URL}/{self._tenant_id}/oauth2/v2.0/token" - headers = {"Content-Type": "application/x-www-form-urlencoded"} - data = f"client_id={self._client_id}&scope=https://graph.microsoft.com/.default&client_secret={self._client_secret}&grant_type=client_credentials" - - # We measure now before request to be on a pessimistic side - now = datetime.utcnow() - async with self._http_session.post(url, headers=headers, data=data) as resp: - json_response = await resp.json() - access_token = json_response["access_token"] - expires_in = int(json_response["expires_in"]) - - return access_token, now + timedelta(seconds=expires_in) - - class SharepointRestAPIToken(SecretAPIToken): """Token to connect to Sharepoint REST API endpoints.""" @@ -245,254 +84,6 @@ async def _fetch_token(self): return access_token, now + timedelta(seconds=expires_in) -class EntraAPIToken(MicrosoftSecurityToken): - """Token to connect to Microsoft Graph API endpoints.""" - - def __init__( - self, - http_session, - tenant_id, - tenant_name, - client_id, - certificate, - private_key, - scope, - ): - super().__init__(http_session, tenant_id, tenant_name, client_id) - self._certificate = certificate - self._private_key = private_key - self._scope = scope - - @retryable(retries=3) - async def _fetch_token(self): - """Fetch API token for usage with Graph API - - Returns: - (str, int) - a tuple containing access token as a string and number of seconds it will be valid for as an integer - """ - - secrets_concat = self._certificate + "\n" + self._private_key - - credentials = CertificateCredential( - self._tenant_id, self._client_id, certificate_data=secrets_concat.encode() - ) - - token = await credentials.get_token(self._scope) - - await credentials.close() - - return token.token, datetime.utcfromtimestamp(token.expires_on) - - -def retryable_aiohttp_call(retries): - # TODO: improve utils.retryable to allow custom logic - # that can help choose what to retry - def wrapper(func): - @wraps(func) - async def wrapped(*args, **kwargs): - retry = 1 - while retry <= retries: - try: - async for item in func(*args, **kwargs, retry_count=retry): - yield item - break - except (NotFound, BadRequestError): - raise - except Exception: - if retry >= retries: - raise - retry += 1 - - return wrapped - - return wrapper - - -class MicrosoftAPISession: - def __init__(self, http_session, api_token, scroll_field, logger_): - self._http_session = http_session - self._api_token = api_token - - # Graph API and Sharepoint API scroll over slightly different fields: - # - odata.nextPage for Sharepoint REST API uses - # - @odata.nextPage for Graph API uses - notice the @ glyph - # Therefore for flexibility I made it a field passed in the initializer, - # but this abstraction can be better. - self._scroll_field = scroll_field - self._sleeps = CancellableSleeps() - self._logger = logger_ - - def set_logger(self, logger_): - self._logger = logger_ - - def close(self): - self._sleeps.cancel() - - async def fetch(self, url): - return await self._get_json(url) - - async def post(self, url, payload): - self._logger.debug(f"Post to url: '{url}' with body: {payload}") - async with self._post(url, payload) as resp: - return await resp.json() - - async def pipe(self, url, stream): - async with self._get(url) as resp: - async for data in resp.content.iter_chunked(FILE_WRITE_CHUNK_SIZE): - await stream.write(data) - - async def scroll(self, url): - scroll_url = url - - while True: - graph_data = await self._get_json(scroll_url) - # We're yielding the whole page here, not one item - yield graph_data["value"] - - if self._scroll_field in graph_data: - scroll_url = graph_data[self._scroll_field] - else: - break - - async def scroll_delta_url(self, url): - scroll_url = url - - while True: - graph_data = await self._get_json(scroll_url) - - yield graph_data - - if DELTA_NEXT_LINK_KEY in graph_data: - scroll_url = graph_data[DELTA_NEXT_LINK_KEY] - else: - break - - async def _get_json(self, absolute_url): - self._logger.debug(f"Fetching url: {absolute_url}") - async with self._get(absolute_url) as resp: - return await resp.json() - - @asynccontextmanager - @retryable_aiohttp_call(retries=DEFAULT_RETRY_COUNT) - async def _post(self, absolute_url, payload=None, retry_count=0): - try: - token = await self._api_token.get() - headers = {"authorization": f"Bearer {token}"} - self._logger.debug(f"Posting to Sharepoint Endpoint: {absolute_url}.") - - async with self._http_session.post( - absolute_url, headers=headers, json=payload - ) as resp: - if absolute_url.endswith("/$batch"): # response code of $batch lies - await self._check_batch_items_for_errors(absolute_url, resp) - yield resp - except aiohttp.client_exceptions.ClientOSError: - self._logger.warning( - "The Microsoft Graph API dropped the connection. It might indicate that the connector is making too many requests. Decrease concurrency settings, otherwise the Graph API may block this app." - ) - raise - except ClientResponseError as e: - await self._handle_client_response_error(absolute_url, e, retry_count) - except ClientPayloadError as e: - await self._handle_client_payload_error(e, retry_count) - - async def _check_batch_items_for_errors(self, url, batch_resp): - body = await batch_resp.json() - responses = body.get("responses", []) - for response in responses: - status = response.get("status", 200) - if status != 200: - self._logger.warning(f"Batch request item failed with: {response}") - headers = response.get("headers", {}) - req_info = RequestInfo(url=url, method="POST", headers=headers) - raise ClientResponseError( - request_info=req_info, - headers=headers, - status=status, - message=nested_get_from_dict( - response, ["body", "error", "message"] - ), - history=(batch_resp), - ) - - @asynccontextmanager - @retryable_aiohttp_call(retries=DEFAULT_RETRY_COUNT) - async def _get(self, absolute_url, retry_count=0): - try: - token = await self._api_token.get() - headers = {"authorization": f"Bearer {token}"} - self._logger.debug(f"Calling Sharepoint Endpoint: {absolute_url}") - - async with self._http_session.get( - absolute_url, - headers=headers, - ) as resp: - yield resp - except aiohttp.client_exceptions.ClientOSError: - self._logger.warning( - "Graph API dropped the connection. It might indicate, that connector makes too many requests - decrease concurrency settings, otherwise Graph API can block this app." - ) - raise - except ClientResponseError as e: - await self._handle_client_response_error(absolute_url, e, retry_count) - except ClientPayloadError as e: - await self._handle_client_payload_error(e, retry_count) - - async def _handle_client_payload_error(self, e, retry_count): - await self._sleeps.sleep( - self._compute_retry_after( - DEFAULT_RETRY_SECONDS, retry_count, DEFAULT_BACKOFF_MULTIPLIER - ) - ) - - raise e - - async def _handle_client_response_error(self, absolute_url, e, retry_count): - if e.status == 429 or e.status == 503: - response_headers = e.headers or {} - - if "Retry-After" in response_headers: - retry_after = int(response_headers["Retry-After"]) - else: - self._logger.warning( - f"Response Code from Sharepoint Online is {e.status} but Retry-After header is not found, using default retry time: {DEFAULT_RETRY_SECONDS} seconds" - ) - retry_after = DEFAULT_RETRY_SECONDS - - retry_seconds = self._compute_retry_after( - retry_after, retry_count, DEFAULT_BACKOFF_MULTIPLIER - ) - - self._logger.warning( - f"Rate Limited by Sharepoint: a new attempt will be performed in {retry_seconds} seconds (retry-after header: {retry_after}, retry_count: {retry_count}, backoff: {DEFAULT_BACKOFF_MULTIPLIER})" - ) - - await self._sleeps.sleep(retry_seconds) - raise ThrottledError from e - elif ( - e.status == 403 or e.status == 401 - ): # Might work weird, but Graph returns 403 and REST returns 401 - msg = f"Received Unauthorized response for {absolute_url}.\nVerify that the correct Graph API and Sharepoint permissions are granted to the app and admin consent is given. If the permissions and consent are correct, wait for several minutes and try again." - raise PermissionsMissing(msg) from e - elif e.status == 404: - raise NotFound from e # We wanna catch it in the code that uses this and ignore in some cases - elif e.status == 500: - raise InternalServerError from e - elif e.status == 400: - self._logger.warning(f"Received 400 response from {absolute_url}") - raise BadRequestError from e - else: - raise - - def _compute_retry_after(self, retry_after, retry_count, backoff): - # Wait for what Sharepoint API asks after the first failure. - # Apply backoff if API is still not available. - if retry_count <= 1: - return retry_after - else: - return retry_after + retry_count * backoff - - class DriveItemsPage(Iterable, Sized): """ Container for Microsoft Graph API DriveItem response diff --git a/app/connectors_service/tests/sources/fixtures/microsoft_teams/connector.json b/app/connectors_service/tests/sources/fixtures/microsoft_teams/connector.json index f19c00020..8146efa9d 100644 --- a/app/connectors_service/tests/sources/fixtures/microsoft_teams/connector.json +++ b/app/connectors_service/tests/sources/fixtures/microsoft_teams/connector.json @@ -27,11 +27,29 @@ "options": [], "validations": [], "value": "QQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQ", - "order": 4, + "order": 2, "ui_restrictions": [] }, - "secret_value": { + "auth_method": { "depends_on": [], + "display": "dropdown", + "tooltip": null, + "default_value": null, + "label": "Authentication Method", + "sensitive": false, + "type": "str", + "required": true, + "options": [ + {"label": "Client Secret", "value": "secret"}, + {"label": "Certificate", "value": "certificate"} + ], + "validations": [], + "value": "secret", + "order": 3, + "ui_restrictions": [] + }, + "secret_value": { + "depends_on": [{"field": "auth_method", "value": "secret"}], "display": "text", "tooltip": null, "default_value": null, @@ -42,38 +60,83 @@ "options": [], "validations": [], "value": "00000000-0000-0000-0000-000000000000", - "order": 3, + "order": 4, "ui_restrictions": [] }, - "username": { - "depends_on": [], - "display": "text", + "certificate": { + "depends_on": [{"field": "auth_method", "value": "certificate"}], + "display": "textarea", "tooltip": null, "default_value": null, - "label": "Username", + "label": "Content of certificate file", "sensitive": true, "type": "str", "required": true, "options": [], "validations": [], - "value": "dummy", - "order": 4, + "value": "", + "order": 5, "ui_restrictions": [] }, - "password": { - "depends_on": [], - "display": "text", + "private_key": { + "depends_on": [{"field": "auth_method", "value": "certificate"}], + "display": "textarea", "tooltip": null, "default_value": null, - "label": "Password", + "label": "Content of private key file", "sensitive": true, "type": "str", "required": true, "options": [], "validations": [], - "value": "changeme", - "order": 5, + "value": "", + "order": 6, + "ui_restrictions": [] + }, + "fetch_attachment_content": { + "depends_on": [], + "display": "toggle", + "tooltip": "Enable to fetch the content of channel and chat attachments. Requires the 'Files.Read.All' application permission.", + "default_value": null, + "label": "Fetch attachment content", + "sensitive": false, + "type": "bool", + "required": true, + "options": [], + "validations": [], + "value": true, + "order": 7, + "ui_restrictions": [] + }, + "use_text_extraction_service": { + "depends_on": [], + "display": "toggle", + "tooltip": "Requires a separate deployment of the Elastic Text Extraction Service. Requires that pipeline settings disable text extraction.", + "default_value": null, + "label": "Use text extraction service", + "sensitive": false, + "type": "bool", + "required": true, + "options": [], + "validations": [], + "value": false, + "order": 8, + "ui_restrictions": ["advanced"] + }, + "use_document_level_security": { + "depends_on": [], + "display": "toggle", + "tooltip": "Document level security ensures identities and permissions set in Microsoft Teams are maintained in Elasticsearch.", + "default_value": null, + "label": "Enable document level security", + "sensitive": false, + "type": "bool", + "required": true, + "options": [], + "validations": [], + "value": false, + "order": 9, "ui_restrictions": [] } } -} \ No newline at end of file +} diff --git a/app/connectors_service/tests/sources/fixtures/microsoft_teams/fixture.py b/app/connectors_service/tests/sources/fixtures/microsoft_teams/fixture.py index f5ba82750..92b4a5151 100644 --- a/app/connectors_service/tests/sources/fixtures/microsoft_teams/fixture.py +++ b/app/connectors_service/tests/sources/fixtures/microsoft_teams/fixture.py @@ -15,126 +15,151 @@ fake_provider = WeightedFakeProvider() -app = Flask(__name__) - - THROTTLING = os.environ.get("THROTTLING", False) -PRE_REQUEST_SLEEP = float(os.environ.get("PRE_REQUEST_SLEEP", "0.05")) - -if THROTTLING: - limiter = Limiter( - get_remote_address, - app=app, - storage_uri="memory://", - application_limits=[ - "6000 per minute", - "6000000 per day", - ], # Microsoft 50k+ licences limits - retry_after="delta-seconds", - headers_enabled=True, - header_name_mapping={ - HEADERS.LIMIT: "RateLimit-Limit", - HEADERS.RESET: "RateLimit-Reset", - HEADERS.REMAINING: "RateLimit-Remaining", - }, - ) - - limiter.init_app(app) - -SIZES = { - "small": 100000, - "medium": 500000, - "large": 1000000, -} DOC_ID_SIZE = 36 - -def adjust_document_id_size(doc_id): - """ - This methods make sure that all the documemts ids are min 36 bytes - """ - - bytesize = len(doc_id) - - if bytesize >= DOC_ID_SIZE: - return doc_id - - addition = "".join(["0" for _ in range(DOC_ID_SIZE - bytesize - 1)]) - return f"{doc_id}-{addition}" - - DATA_SIZE = os.environ.get("DATA_SIZE", "medium").lower() match DATA_SIZE: case "small": - MESSAGES = 25 - EVENTS = 50 - CHANNEL = 3 - FILES = 10 - CHANNEL_MESSAGE = 50 + TEAMS = 2 + MEMBERS = 3 + CHANNELS = 2 + CHANNEL_MESSAGES = 10 + REPLIES = 1 + FILES = 3 + CHATS = 3 + CHAT_MESSAGES = 10 case "medium": - MESSAGES = 50 - EVENTS = 50 - CHANNEL = 3 - FILES = 50 - CHANNEL_MESSAGE = 500 + TEAMS = 3 + MEMBERS = 5 + CHANNELS = 3 + CHANNEL_MESSAGES = 50 + REPLIES = 2 + FILES = 10 + CHATS = 10 + CHAT_MESSAGES = 50 case "large": - MESSAGES = 250 - EVENTS = 150 - CHANNEL = 5 - FILES = 150 - CHANNEL_MESSAGE = 1000 + TEAMS = 5 + MEMBERS = 10 + CHANNELS = 5 + CHANNEL_MESSAGES = 100 + REPLIES = 3 + FILES = 20 + CHATS = 20 + CHAT_MESSAGES = 100 case _: msg = f"Unknown DATA_SIZE: {DATA_SIZE}. Expecting 'small', 'medium' or 'large'" raise Exception(msg) -MESSAGES_TO_DELETE = 10 -EVENTS_TO_DELETE = 1 -ROOT = os.environ.get("OVERRIDE_URL", "http://127.0.0.1:10971") +def adjust_document_id_size(doc_id): + """Ensure document ids are at least DOC_ID_SIZE bytes.""" + bytesize = len(doc_id) + if bytesize >= DOC_ID_SIZE: + return doc_id + addition = "".join(["0" for _ in range(DOC_ID_SIZE - bytesize - 1)]) + return f"{doc_id}-{addition}" + + +def expected_docs_count(): + teams = TEAMS + team_members = TEAMS * MEMBERS + channels = TEAMS * CHANNELS + channel_messages = TEAMS * CHANNELS * CHANNEL_MESSAGES + channel_replies = TEAMS * CHANNELS * CHANNEL_MESSAGES * REPLIES + channel_attachments = TEAMS * CHANNELS * FILES + chats = CHATS + chat_messages = CHATS * CHAT_MESSAGES + # every chat message carries a single reference attachment resolving to one file + chat_attachments = CHATS * CHAT_MESSAGES + return ( + teams + + team_members + + channels + + channel_messages + + channel_replies + + channel_attachments + + chats + + chat_messages + + chat_attachments + ) def get_num_docs(): - # I tried to do the maths, but it's not possible without diving too deep into the connector - # Therefore, doing naive way - just ran connector and took the number from the test - expected_count = 0 - match DATA_SIZE: - case "small": - expected_count = 508 - case "medium": - expected_count = 4613 - case "large": - expected_count = 15435 + print(expected_docs_count()) + + +def _message(message_id, with_replies=False): + replies = [] + if with_replies: + for reply in range(REPLIES): + replies.append(_message(f"{message_id}-reply-{reply}", with_replies=False)) + return { + "id": adjust_document_id_size(message_id), + "messageType": "message", + "createdDateTime": "2023-08-16T04:47:55.794Z", + "lastModifiedDateTime": "2023-08-16T04:47:55.794Z", + "deletedDateTime": None, + "subject": None, + "summary": None, + "webUrl": f"https://teams.microsoft.com/l/message/{message_id}", + "eventDetail": None, + "from": {"user": {"id": "sender-1", "displayName": "Dummy"}}, + "body": {"contentType": "html", "content": fake_provider.get_html()}, + "attachments": [], + "replies": replies, + } + + +def _file(file_id, drive_id): + return { + "id": adjust_document_id_size(file_id), + "name": "list.txt", + "webUrl": f"{ROOT}/download/{file_id}", + "size": 45441, + "createdDateTime": "2023-08-16T04:47:26Z", + "lastModifiedDateTime": "2023-08-16T04:47:29Z", + "file": {"mimeType": "text/plain"}, + "parentReference": {"driveId": drive_id, "driveType": "documentLibrary"}, + } + - print(expected_count) +ROOT = os.environ.get("OVERRIDE_URL", "http://127.0.0.1:10971") class MicrosoftTeamsAPI: def __init__(self): self.app = Flask(__name__) - self.app.route("/me", methods=["GET"])(self.get_myself) - self.app.route("/chats", methods=["GET"])(self.get_user_chats) - self.app.route("/chats//messages", methods=["GET"])( - self.get_user_chat_messages - ) - self.app.route("/chats//tabs", methods=["GET"])( - self.get_user_chat_tabs - ) + if THROTTLING: + limiter = Limiter( + get_remote_address, + app=self.app, + storage_uri="memory://", + application_limits=["6000 per minute", "6000000 per day"], + retry_after="delta-seconds", + headers_enabled=True, + header_name_mapping={ + HEADERS.LIMIT: "RateLimit-Limit", + HEADERS.RESET: "RateLimit-Reset", + HEADERS.REMAINING: "RateLimit-Remaining", + }, + ) + limiter.init_app(self.app) - self.app.route("/users", methods=["GET"])(self.get_users) - self.app.route("/users//events", methods=["GET"])( - self.get_events + self.app.route("//oauth2/v2.0/token", methods=["POST"])( + self.get_token ) self.app.route("/teams", methods=["GET"])(self.get_teams) + self.app.route("/teams//members", methods=["GET"])( + self.get_team_members + ) self.app.route("/teams//channels", methods=["GET"])( self.get_channels ) - self.app.route( - "/teams//channels//tabs", methods=["GET"] - )(self.get_channel_tabs) self.app.route( "/teams//channels//messages", methods=["GET"], @@ -142,284 +167,153 @@ def __init__(self): self.app.route( "/teams//channels//filesFolder", methods=["GET"], - )(self.get_teams_filefolder) + )(self.get_channel_files_folder) + + self.app.route("/chats", methods=["GET"])(self.get_chats) + self.app.route("/chats//messages", methods=["GET"])( + self.get_chat_messages + ) + + self.app.route("/users//drive", methods=["GET"])( + self.get_user_drive + ) self.app.route( "/drives//items//children", methods=["GET"], - )(self.get_teams_file) - - self.app.route("/sites/list.txt", methods=["GET"])(self.download_file) - - def get_myself(self): - return { - "displayName": "Alex Wilber", - "givenName": "Alex", - "mail": "Alex@3hr2.onmicrosoft.com", - "userPrincipalName": "Alex@3hr2.onmicrosoft.com", - "id": adjust_document_id_size("me-1"), - } + )(self.get_drive_children) + self.app.route( + "/drives//items//content", + methods=["GET"], + )(self.download_file) - def get_user_chats(self): + def get_token(self, tenant_id): return { - "value": [ - { - "id": adjust_document_id_size("1"), - "topic": None, - "createdDateTime": "2023-07-21T21:24:18.338Z", - "lastUpdatedDateTime": "2023-07-21T21:24:18.338Z", - "chatType": "oneOnOne", - "webUrl": "https://teams.microsoft.com/l/chat/1", - "members": [ - { - "@odata.type": "#microsoft.graph.aadUserConversationMember", - "displayName": "Cervantes, Andres", - "userId": "123abc", - "email": "ACervantes@mlock.com", - }, - ], - } - ] + "access_token": "fake-access-token", + "token_type": "Bearer", + "expires_in": 3600, } - def get_user_chat_messages(self, chat_id): - global MESSAGES - message_data = [] - top = int(request.args.get("$top")) - for message in range(MESSAGES): - message_data.append( + def get_teams(self): + teams = [] + for team in range(TEAMS): + teams.append( { - "id": adjust_document_id_size(f"user-chat-{message}-{MESSAGES}"), - "messageType": "onetoone", - "createdDateTime": "2023-07-21T21:24:18.726Z", - "lastModifiedDateTime": "2023-07-21T21:24:18.726Z", - "deletedDateTime": None, - "subject": None, - "summary": None, - "webUrl": None, - "from": None, - "body": {"contentType": "html", "content": "

dummy data

"}, - "attachments": [], - "eventDetail": None, + "id": adjust_document_id_size(f"team-{team}"), + "displayName": f"team{team}", + "description": f"team {team} description", + "createdDateTime": "2023-08-16T04:46:53.056Z", + "webUrl": f"https://teams.microsoft.com/l/team/{team}", } ) - response = { - "@odata.context": "https://graph.microsoft.com/v1.0/$metadata#Chats)", - "value": message_data, - } - - if len(message_data) == top: - response["@odata.nextLink"] = f"{ROOT}/chats/{chat_id}/messages?$top=50" - MESSAGES -= MESSAGES_TO_DELETE # performs deletion and pagination - return response - - def get_user_chat_tabs(self, chat_id): - return { - "value": [ - { - "id": adjust_document_id_size("tab-1"), - "displayName": "Notes", - "configuration": {"websiteUrl": "https://onenote.com"}, - } - ] - } - - def get_users(self): - return { - "value": [ - { - "mail": "AdeleV@3hmnr2.onmicrosoft.com", - "id": adjust_document_id_size("user-1"), - } - ] - } + return {"value": teams} - def get_events(self, user_id): - global EVENTS - event_data = [] - top = int(request.args.get("$top")) - for event in range(EVENTS): - event_data.append( + def get_team_members(self, team_id): + members = [] + for member in range(MEMBERS): + members.append( { - "id": adjust_document_id_size(f"event-{user_id}"), - "createdDateTime": "2023-08-10T08:22:14.5296951Z", - "lastModifiedDateTime": "2023-08-10T08:25:29.3693436Z", - "categories": [], - "originalStartTimeZone": "India Standard Time", - "originalEndTimeZone": "India Standard Time", - "reminderMinutesBeforeStart": 15, - "isReminderOn": True, - "subject": "new meet", - "bodyPreview": "Body prebiew dummy", - "importance": "normal", - "isAllDay": False, - "isCancelled": False, - "showAs": "busy", - "webLink": f"https://outlook.office365.com/calendar/item/{event}", - "body": {"contentType": "html", "content": "dummy"}, - "start": { - "dateTime": "2023-08-10T08:00:00.0000000", - "timeZone": "UTC", - }, - "end": { - "dateTime": "2023-08-10T08:30:00.0000000", - "timeZone": "UTC", - }, - "locations": [ - { - "displayName": "Microsoft Teams Meeting", - } - ], - "recurrence": None, - "attendees": [], - "organizer": { - "emailAddress": { - "name": "Dummy", - "address": "dummy@mnr.onmicrosoft.com", - } - }, - "onlineMeeting": {"joinUrl": "https://teams.microsoft.com/meet"}, + "id": f"membership-{member}-{team_id}", + "displayName": f"Member {member}", + "userId": f"user-{member}", + "email": f"member{member}@example.onmicrosoft.com", + "roles": ["owner"] if member == 0 else [], } ) - response = { - "@odata.context": "https://graph.microsoft.com/v1.0/$metadata#events)", - "value": event_data, - } - - if len(event_data) == top: - response["@odata.nextLink"] = f"{ROOT}/users/{user_id}/events?$top=50" - EVENTS -= EVENTS_TO_DELETE - return response - - def get_teams(self): - return { - "@odata.context": "https://graph.microsoft.com/v1.0/$metadata#teams)", - "value": [ - { - "id": adjust_document_id_size("team-1"), - "createdDateTime": None, - "displayName": "team1", - "description": "team1", - "webUrl": None, - "summary": None, - }, - { - "id": adjust_document_id_size("team-2"), - "createdDateTime": None, - "displayName": "team2", - "description": "team2", - "webUrl": None, - "summary": None, - }, - { - "id": adjust_document_id_size("team-3"), - "createdDateTime": None, - "displayName": "team3", - "description": "team3", - "webUrl": None, - "summary": None, - }, - ], - } + return {"value": members} def get_channels(self, team_id): - channel_list = [] - for channel in range(CHANNEL): - channel_list.append( + channels = [] + for channel in range(CHANNELS): + channels.append( { "id": adjust_document_id_size(f"channel-{channel}-{team_id}"), - "createdDateTime": "2023-08-16T04:46:53.056Z", "displayName": f"General-{channel}", "description": "channel", "webUrl": f"https://teams.microsoft.com/l/channel/{channel}/{team_id}", + "createdDateTime": "2023-08-16T04:46:53.056Z", } ) - return { - "@odata.context": "https://graph.microsoft.com/v1.0/$metadata#teams)", - "value": channel_list, - } + return {"value": channels} def get_channel_messages(self, team_id, channel_id): - message_list = [] - for message in range(CHANNEL_MESSAGE): - message_list.append( - { - "id": adjust_document_id_size( - f"message-{team_id}-{channel_id}-{message}" - ), - "messageType": "message", - "createdDateTime": "2023-08-16T04:47:55.794Z", - "lastModifiedDateTime": "2023-08-16T04:47:55.794Z", - "deletedDateTime": None, - "subject": "", - "summary": None, - "webUrl": f"https://teams.microsoft.com/l/message/{team_id}/{channel_id}/{message}", - "policyViolation": None, - "eventDetail": None, - "from": { - "user": { - "displayName": "Dummy", - } - }, - "body": { - "contentType": "html", - "content": "
I added a tab at the top of this channel. Check it out!
", - }, - "attachments": [], - } + messages = [] + for message in range(CHANNEL_MESSAGES): + messages.append( + _message(f"message-{team_id}-{channel_id}-{message}", with_replies=True) ) - return { - "@odata.context": "https://graph.microsoft.com/v1.0/$metadata#message)", - "value": message_list, - } + return {"value": messages} - def get_channel_tabs(self, team_id, channel_id): + def get_channel_files_folder(self, team_id, channel_id): + drive_id = f"cdrive-{team_id}-{channel_id}" return { - "@odata.context": "https://graph.microsoft.com/v1.0/$metadata#tabs)", - "value": [ - { - "id": adjust_document_id_size(f"tabs-{team_id}-{channel_id}"), - "displayName": "Notes", - "webUrl": f"https://teams.microsoft.com/l/entity/tab/{team_id}/{channel_id}", - "configuration": {"websiteUrl": "https://onenote.com"}, - } - ], - } - - def get_teams_filefolder(self, team_id, channel_id): - return { - "id": "filfolder-1", - "createdDateTime": "0001-01-01T00:00:00Z", - "lastModifiedDateTime": "2023-09-21T10:23:48Z", + "id": f"cfolder-{team_id}-{channel_id}", "name": "root", - "size": 351660, - "parentReference": { - "driveId": "driveid-123", - "driveType": "documentLibrary", - }, + "parentReference": {"driveId": drive_id, "driveType": "documentLibrary"}, } - def get_teams_file(self, drive_id, item_id): - files_list = [] - for file_data in range(FILES): - files_list.append( + def get_chats(self): + chats = [] + for chat in range(CHATS): + chats.append( { - "@microsoft.graph.downloadUrl": f"{ROOT}/sites/list.txt", - "createdDateTime": "2023-08-16T04:47:26Z", - "id": adjust_document_id_size( - f"file-{file_data}-{drive_id}-{item_id}" - ), - "lastModifiedDateTime": "2023-08-16T04:47:29Z", - "name": "list.txt", - "size": 45441, - "webUrl": f"{ROOT}/sites/list.html", - "file": { - "mimeType": "text/plain", - }, + "id": adjust_document_id_size(f"chat-{chat}"), + "topic": f"Chat {chat}", + "chatType": "group", + "webUrl": f"https://teams.microsoft.com/l/chat/{chat}", + "createdDateTime": "2023-07-21T21:24:18.338Z", + "lastUpdatedDateTime": "2023-07-21T21:24:18.338Z", + "members": [ + { + "id": f"cm-{member}-{chat}", + "displayName": f"Chat member {member}", + "userId": f"user-{member}", + "email": f"member{member}@example.onmicrosoft.com", + } + for member in range(MEMBERS) + ], } ) - return {"value": files_list} + return {"value": chats} - def download_file(self): + def get_chat_messages(self, chat_id): + messages = [] + for message in range(CHAT_MESSAGES): + doc = _message(f"chat-message-{chat_id}-{message}", with_replies=False) + doc["attachments"] = [ + { + "id": f"att-{chat_id}-{message}", + "name": "chat.txt", + "contentType": "reference", + } + ] + messages.append(doc) + return {"value": messages} + + def get_user_drive(self, user_id): + return {"id": f"udrive-{user_id}"} + + def get_drive_children(self, drive_id, item_id): + if item_id == "root": + # OneDrive "Microsoft Teams Chat Files" folder lookup + return { + "value": [ + { + "id": f"chatfolder-{drive_id}", + "name": "Microsoft Teams Chat Files", + "folder": {"childCount": 1}, + } + ] + } + + if request.args.get("$filter"): + # Chat attachment resolution by name + return {"value": [_file(f"chatfile-{drive_id}-{item_id}", drive_id)]} + + # Channel drive children + files = [_file(f"file-{i}-{drive_id}", drive_id) for i in range(FILES)] + return {"value": files} + + def download_file(self, drive_id, item_id): return io.BytesIO(bytes(fake_provider.get_html(), encoding="utf-8")) diff --git a/app/connectors_service/tests/sources/test_microsoft_teams.py b/app/connectors_service/tests/sources/test_microsoft_teams.py index ac8008b37..06cde549f 100644 --- a/app/connectors_service/tests/sources/test_microsoft_teams.py +++ b/app/connectors_service/tests/sources/test_microsoft_teams.py @@ -3,1217 +3,934 @@ # or more contributor license agreements. Licensed under the Elastic License 2.0; # you may not use this file except in compliance with the Elastic License 2.0. # -import base64 -from io import BytesIO +import asyncio +from contextlib import asynccontextmanager from unittest.mock import AsyncMock, MagicMock, Mock, patch import pytest -import pytest_asyncio -from aiohttp.client_exceptions import ClientOSError, ClientResponseError -from connectors_sdk.logger import logger -from connectors_sdk.source import ConfigurableFieldValueError, DataSourceConfiguration +from connectors_sdk.source import BaseDataSource, ConfigurableFieldValueError +from connectors.access_control import ACCESS_CONTROL from connectors.sources.microsoft_teams import ( MicrosoftTeamsClient, MicrosoftTeamsDataSource, ) from connectors.sources.microsoft_teams.client import ( + EndSignal, + Schema, + TeamsObjectType, +) +from connectors.sources.microsoft_teams.formatter import MicrosoftTeamsFormatter +from connectors.sources.shared.microsoft.graph import ( + EntraAPIToken, GraphAPIToken, - InternalServerError, NotFound, PermissionsMissing, ) +from connectors.utils import ConcurrentTasks from tests.commons import AsyncIterator from tests.sources.support import create_source -USER_CHATS = [ +TEAMS = [ { - "id": "19:2ea91886", - "topic": "topic1", - "createdDateTime": "2023-08-03T12:19:27.57Z", - "lastUpdatedDateTime": "2023-08-09T07:17:26.482Z", - "chatType": "oneOnOne", - "webUrl": "https://teams.microsoft.com/l/chat/19:2ea91886", - "tenantId": "a57a7700", - "members": [ - { - "id": "123-3=", - "displayName": "Duumy3", - "userId": "82e32463", - "email": "dummy3@3hm.onmicrosoft.com", - }, - { - "id": "123-2=", - "displayName": "Dummy2", - "userId": "2ea91886", - "email": "dummy2@3hm.onmicrosoft.com", - }, - ], + "id": "team-1", + "displayName": "Team One", + "description": "First team", + "createdDateTime": "2023-08-16T04:46:53.056Z", + } +] + +TEAM_MEMBERS = [ + { + "id": "membership-1", + "displayName": "Alice", + "userId": "user-alice", + "email": "alice@example.com", + "roles": ["owner"], }, { - "id": "19:2ea918861", - "topic": "dummy chat", - "createdDateTime": "2023-08-03T12:19:27.57Z", - "lastUpdatedDateTime": "2023-08-09T07:17:26.482Z", - "chatType": "oneOnOne", - "webUrl": "https://teams.microsoft.com/l/chat/19:2ea918861", - "tenantId": "a57a7700", - "members": [ - { - "id": "123-1=", - "displayName": "Duumy1", - "userId": "82e32462", - "email": "dummy1@3hm.onmicrosoft.com", - }, - { - "id": "123-2=", - "displayName": "Dummy2", - "userId": "2ea91886", - "email": "dummy2@3hm.onmicrosoft.com", - }, - ], + "id": "membership-2", + "displayName": "Bob", + "userId": "user-bob", + "email": "bob@example.com", + "roles": [], }, ] -MESSAGES = [ +CHANNELS = [ { - "id": "1691582610121", - "messageType": "message", - "createdDateTime": "2023-08-09T12:03:30.121Z", - "lastModifiedDateTime": "2023-08-09T12:03:30.121Z", - "deletedDateTime": None, - "subject": None, - "summary": None, - "eventDetail": None, - "webUrl": "https://3hm-my.sharepoint.com/11_dummy", - "from": { - "user": { - "@odata.type": "#microsoft.graph.teamworkUserIdentity", - "id": "82e32462", - "displayName": "Dummy", - "userIdentityType": "aadUser", - "tenantId": "a57a7700", - } - }, - "body": { - "contentType": "html", - "content": '', - }, - "attachments": [ - { - "id": "c5fddd7c", - "contentType": "reference", - "contentUrl": "https://3hmn-my.sharepoint.com//Microsoft%20Teams%20Chat%20Files/~%60!@$%5E8()_+-=%5B%5D%7B%7D;%27,k.txt", - "content": None, - "name": "~`!@$^8()_+-=[]{};',k.txt", - } - ], - }, + "id": "channel-1", + "displayName": "General", + "description": "General channel", + "webUrl": "https://teams.microsoft.com/l/channel/1", + "createdDateTime": "2023-08-16T04:46:53.056Z", + } +] + +CHANNEL_MESSAGES = [ { - "id": "1691565463404", - "messageType": "unknownFutureValue", - "createdDateTime": "2023-08-09T07:17:43.404Z", - "lastModifiedDateTime": "2023-08-09T07:17:44.668Z", + "id": "message-1", + "messageType": "message", + "createdDateTime": "2023-08-16T04:47:55.794Z", + "lastModifiedDateTime": "2023-08-16T04:47:55.794Z", "deletedDateTime": None, - "subject": None, - "summary": None, - "chatId": "19:2ea91886", - "webUrl": None, - "from": None, - "body": {"contentType": "html", "content": ""}, + "webUrl": "https://teams.microsoft.com/l/message/1", + "from": {"user": {"displayName": "Alice"}}, + "body": {"contentType": "html", "content": "
Hello channel
"}, "attachments": [], - "eventDetail": { - "@odata.type": "#microsoft.graph.callRecordingEventMessageDetail", - "callId": "7ed8a6cb", - "callRecordingDisplayName": "Call with alex wilber-20230809_124714-Meeting Recording.mp4", - "callRecordingUrl": "https://3hm-my.sharepoint.com/:v:/g/personal/dummy_3hm_onmicrosoft_com/123", - "meetingOrganizer": None, - "initiator": { - "application": None, - "device": None, - "user": { - "@odata.type": "#microsoft.graph.teamworkUserIdentity", - "id": "82e32462-782f-4988-b5f1-60f14b09a33c", - "displayName": None, - "userIdentityType": "aadUser", - }, - }, - }, "replies": [ { - "id": "1691565463406", - "messageType": "dummy", - "createdDateTime": "2023-08-09T07:17:43.404Z", - "lastModifiedDateTime": "2023-08-09T07:17:44.668Z", - "deletedDateTime": "2023-08-09T07:17:44.668Z", - "subject": None, - "summary": None, - "chatId": "19:2ea91886", - "webUrl": None, - "from": None, - "body": {"contentType": "html", "content": ""}, + "id": "reply-1", + "messageType": "message", + "createdDateTime": "2023-08-16T04:48:55.794Z", + "lastModifiedDateTime": "2023-08-16T04:48:55.794Z", + "deletedDateTime": None, + "webUrl": "https://teams.microsoft.com/l/message/1/reply/1", + "from": {"user": {"displayName": "Bob"}}, + "body": {"contentType": "html", "content": "
Hi Alice
"}, "attachments": [], - "eventDetail": { - "@odata.type": "#microsoft.graph.callEndedEventMessageDetail", - "callId": "7ed8a6cb", - "callRecordingDisplayName": "Call with alex wilber-20230809_124714-Meeting Recording.mp4", - "meetingOrganizer": None, - "callParticipants": [ - { - "participant": { - "user": { - "id": "2ea91886-", - "displayName": "alex wilber", - "userIdentityType": "aadUser", - } - } - }, - ], - }, } ], }, -] - -ATTACHMENTS = [ { - "@microsoft.graph.downloadUrl": "https://3hm-my.sharepoint.com/personal/_layouts/15/download.aspx?UniqueId=c5fddd7c", - "createdDateTime": "2023-08-09T12:03:07Z", - "id": "01EL4RL6L43X64K4ATD5H34UUQPOMMTRQX", - "lastModifiedDateTime": "2023-08-09T12:03:20Z", - "name": "~`!@$^8()_+-=[]{};',k.txt", - "webUrl": "https://3hm-my.sharepoint.com/personal/~%60!@$()_+-=%5B%5D%7B%7D;%27,k.txt", - "size": 10914302891111, - "file": { - "mimeType": "text/plain", - "hashes": {"quickXorHash": "6eO+6QCH+yCKU+k+tDEuDazGwkU="}, - }, - } + "id": "message-2-deleted", + "messageType": "message", + "createdDateTime": "2023-08-16T04:49:55.794Z", + "lastModifiedDateTime": "2023-08-16T04:49:55.794Z", + "deletedDateTime": "2023-08-16T04:50:55.794Z", + "webUrl": "https://teams.microsoft.com/l/message/2", + "from": {"user": {"displayName": "Alice"}}, + "body": {"contentType": "html", "content": "
deleted
"}, + "attachments": [], + "replies": [], + }, ] -TABS = [ - { - "id": "400960d1", - "displayName": "testing Whiteboard", - "webUrl": "https://teams.microsoft.com/l/entity/95de633a?webUrl=app.whiteboard.microsoft.com", - "configuration": { - "websiteUrl": "https://app.whiteboard.microsoft.com/tabappboard" - }, - } -] +CHANNEL_FILES_FOLDER = { + "id": "root-folder", + "parentReference": {"driveId": "drive-123"}, +} -USERS = [ +CHANNEL_DRIVE_CHILDREN = [ { - "displayName": "Adele Vance", - "givenName": "Adele", - "jobTitle": "Retail Manager", - "mail": "AdeleV@w076v.onmicrosoft.com", - "officeLocation": "18/2111", - "preferredLanguage": "en-US", - "id": "3fada5d6-125c-4aaa-bc23-09b5d301b2a7", - }, - { - "displayName": "Alex Wilber", - "givenName": "Alex", - "jobTitle": "Marketing Assistant", - "mail": "AlexW@w076v.onmicrosoft.com", - "officeLocation": "131/1104", - "preferredLanguage": "en-US", - "id": "8e083933-1720-4d2f-80d0-3976669b40ee", - }, + "id": "file-1", + "name": "report.txt", + "webUrl": "https://example.com/report.txt", + "size": 42, + "lastModifiedDateTime": "2023-08-16T04:47:29Z", + "createdDateTime": "2023-08-16T04:47:26Z", + "file": {"mimeType": "text/plain"}, + "parentReference": {"driveId": "drive-123"}, + } ] -EVENTS = [ +CHATS = [ { - "id": "AAMkADkyYTNmOTcw", - "createdDateTime": "2023-08-10T05:47:41.5466652Z", - "lastModifiedDateTime": "2023-08-10T05:48:53.6618239Z", - "originalStartTimeZone": "India Standard Time", - "originalEndTimeZone": "India Standard Time", - "reminderMinutesBeforeStart": 15, - "isReminderOn": True, - "hasAttachments": False, - "subject": "Transitive connector meeting", - "bodyPreview": "Hey all attend the meeting without a fail...", - "isAllDay": True, - "isCancelled": False, - "isOrganizer": True, - "webLink": "https://outlook.office365.com/owa/?itemid=AAMkADkyYTNm?path=/calendar/item", - "onlineMeetingUrl": None, - "isOnlineMeeting": True, - "categories": ["Red category"], - "body": {"contentType": "html", "content": "dummy"}, - "start": {"dateTime": "2023-08-10T06:30:00.0000000", "timeZone": "UTC"}, - "end": {"dateTime": "2023-08-10T07:00:00.0000000", "timeZone": "UTC"}, - "locations": [ - { - "displayName": "Microsoft Teams Meeting", - } - ], - "recurrence": None, - "attendees": [ - { - "emailAddress": { - "name": "alex wilber", - "address": "alexwilber@3hm.onmicrosoft.com", - } - } - ], - "organizer": { - "emailAddress": {"name": "Dummy", "address": "dummy@3hm.onmicrosoft.com"} - }, - "onlineMeeting": {"joinUrl": "https://teams.microsoft.com/l/meetup-join/123"}, - }, - { - "id": "AAMkADkyYTNmOTcw1", - "createdDateTime": "2023-08-10T05:47:41.5466652Z", - "lastModifiedDateTime": "2023-08-10T05:48:53.6618239Z", - "originalStartTimeZone": "India Standard Time", - "originalEndTimeZone": "India Standard Time", - "reminderMinutesBeforeStart": 15, - "isReminderOn": True, - "hasAttachments": False, - "subject": "Transitive connector meeting", - "bodyPreview": "Hey all attend the meeting without a fail...", - "isAllDay": False, - "isCancelled": False, - "isOrganizer": True, - "webLink": "https://outlook.office365.com/owa/?itemid=AAMkADkyYTNm?path=/calendar/item", - "onlineMeetingUrl": None, - "isOnlineMeeting": True, - "categories": ["Red category"], - "body": {"contentType": "html", "content": "dummy"}, - "start": {"dateTime": "2023-08-10T06:30:00.0000000", "timeZone": "UTC"}, - "end": {"dateTime": "2023-08-10T07:00:00.0000000", "timeZone": "UTC"}, - "locations": [ + "id": "chat-1", + "topic": "Project chat", + "chatType": "group", + "webUrl": "https://teams.microsoft.com/l/chat/1", + "createdDateTime": "2023-07-21T21:24:18.338Z", + "lastUpdatedDateTime": "2023-07-21T21:24:18.338Z", + "members": [ { - "displayName": "Microsoft Teams Meeting", - } - ], - "recurrence": { - "pattern": { - "type": "relativeMonthly", - "interval": 1, - "month": 0, - "dayOfMonth": 0, - "daysOfWeek": ["thursday"], - "firstDayOfWeek": "sunday", - "index": "second", - }, - "range": { - "type": "noEnd", - "startDate": "2023-08-10", - "endDate": "0001-01-01", - "recurrenceTimeZone": "India Standard Time", - "numberOfOccurrences": 0, + "id": "cm-1", + "displayName": "Alice", + "userId": "user-alice", + "email": "alice@example.com", }, - }, - "attendees": [ { - "emailAddress": { - "name": "alex wilber", - "address": "alexwilber@3hm.onmicrosoft.com", - } - } - ], - "organizer": { - "emailAddress": {"name": "Dummy", "address": "dummy@3hm.onmicrosoft.com"} - }, - "onlineMeeting": {"joinUrl": "https://teams.microsoft.com/l/meetup-join/123"}, - }, - { - "id": "AAMkADkyYTNmOTcw2", - "createdDateTime": "2023-08-10T05:47:41.5466652Z", - "lastModifiedDateTime": "2023-08-10T05:48:53.6618239Z", - "originalStartTimeZone": "India Standard Time", - "originalEndTimeZone": "India Standard Time", - "reminderMinutesBeforeStart": 15, - "isReminderOn": True, - "hasAttachments": False, - "subject": "Transitive connector meeting", - "bodyPreview": "Hey all attend the meeting without a fail...", - "isAllDay": False, - "isCancelled": False, - "isOrganizer": True, - "webLink": "https://outlook.office365.com/owa/?itemid=AAMkADkyYTNm?path=/calendar/item", - "onlineMeetingUrl": None, - "isOnlineMeeting": True, - "categories": ["Red category"], - "body": {"contentType": "html", "content": "dummy"}, - "start": {"dateTime": "2023-08-10T06:30:00.0000000", "timeZone": "UTC"}, - "end": {"dateTime": "2023-08-10T07:00:00.0000000", "timeZone": "UTC"}, - "locations": [ - { - "displayName": "Microsoft Teams Meeting", - } - ], - "recurrence": { - "pattern": { - "type": "absoluteYearly", - "interval": 1, - "month": 0, - "dayOfMonth": 0, - "daysOfWeek": ["thursday"], - "firstDayOfWeek": "sunday", - "index": "second", - }, - "range": { - "type": "noEnd", - "startDate": "2023-08-10", - "endDate": "0001-01-01", - "recurrenceTimeZone": "India Standard Time", - "numberOfOccurrences": 0, + "id": "cm-2", + "displayName": "Bob", + "userId": "user-bob", + "email": "bob@example.com", }, - }, - "attendees": [ - { - "emailAddress": { - "name": "alex wilber", - "address": "alexwilber@3hm.onmicrosoft.com", - } - } ], - "organizer": { - "emailAddress": {"name": "Dummy", "address": "dummy@3hm.onmicrosoft.com"} - }, - "onlineMeeting": {"joinUrl": "https://teams.microsoft.com/l/meetup-join/123"}, - }, + } ] -TEAMS = [ +CHAT_MESSAGES = [ { - "id": "25ab782d", - "createdDateTime": None, - "displayName": "team1", - "description": "Welcome to the team that we've assembled to create the Mark 8.", - "visibility": "public", + "id": "chat-message-1", + "messageType": "message", + "createdDateTime": "2023-07-21T21:24:18.726Z", + "lastModifiedDateTime": "2023-07-21T21:24:18.726Z", + "deletedDateTime": None, "webUrl": None, - "summary": None, + "from": {"user": {"id": "user-alice", "displayName": "Alice"}}, + "body": {"contentType": "html", "content": "

chat body

"}, + "attachments": [ + { + "id": "att-1", + "name": "doc.txt", + "contentType": "reference", + } + ], } ] -CHANNELS = [ +CHAT_ATTACHMENTS = [ { - "id": "19:36b3f1125", - "createdDateTime": "2023-08-08T08:23:13.984Z", - "displayName": "channel2", - "description": None, - "tenantId": "a57a7700", - "webUrl": "https://teams.microsoft.com/l/channel/19%3A36b3f1125c82456", - "membershipType": "standard", + "id": "chat-file-1", + "name": "doc.txt", + "webUrl": "https://example.com/doc.txt", + "size": 10, + "lastModifiedDateTime": "2023-07-21T21:24:18.726Z", + "createdDateTime": "2023-07-21T21:24:18.726Z", + "file": {"mimeType": "text/plain"}, + "parentReference": {"driveId": "drive-chat"}, } ] -MOCK_ATTACHMENT = { - "created_at": "2023-05-01T09:09:31Z", - "id": "123", - "_timestamp": "2023-05-01T09:10:21Z", - "name": "Document.docx", - "type": "file", - "size_in_bytes": 10484, -} -MOCK_ATTACHMENT_WITHOUT_EXTENSION = { - "created_at": "2023-05-01T09:09:31Z", - "id": "123", - "_timestamp": "2023-05-01T09:10:21Z", - "name": "Document", - "type": "file", - "size_in_bytes": 10484, -} -MOCK_ATTACHMENT_WITH_LARGE_DATA = { - "created_at": "2023-05-01T09:09:31Z", - "id": "123", - "_timestamp": "2023-05-01T09:10:21Z", - "name": "Document.docx", - "type": "file", - "size_in_bytes": 23000000, -} -MOCK_ATTACHMENT_WITH_UNSUPPORTED_EXTENSION = { - "created_at": "2023-05-01T09:09:31Z", - "id": "123", - "_timestamp": "2023-05-01T09:10:21Z", - "name": "Document.xyz", - "type": "file", - "size_in_bytes": 10484, -} -MOCK_ATTACHMENT_WITH_ZERO_SIZE = { - "created_at": "2023-05-01T09:09:31Z", - "id": "123", - "_timestamp": "2023-05-01T09:10:21Z", - "name": "Document.xyz", - "type": "file", - "size_in_bytes": 0, -} -DOWNLOAD_URL = "https://attachment.com" -CHANNEL_MESSAGE = { - "id": "1691588610121", - "messageType": "message", - "createdDateTime": "2023-08-09T12:03:30.121Z", - "lastModifiedDateTime": "2023-08-09T12:03:30.121Z", - "deletedDateTime": None, - "subject": None, - "summary": None, - "eventDetail": None, - "webUrl": "https://3hm-my.sharepoint.com/11_dummy", - "from": { - "user": { - "@odata.type": "#microsoft.graph.teamworkUserIdentity", - "id": "82e32462", - "displayName": "Dummy", - "userIdentityType": "aadUser", - "tenantId": "a57a7700", - } - }, - "body": { - "contentType": "html", - "content": "
hello
", - }, - "attachments": [], - "replies": [ - { - "id": "1691588680121", - "messageType": "message", - "createdDateTime": "2023-08-09T12:03:30.121Z", - "lastModifiedDateTime": "2023-08-09T12:03:30.121Z", - "deletedDateTime": None, - "subject": None, - "summary": None, - "eventDetail": None, - "webUrl": "https://3hm-my.sharepoint.com/11_dummy", - "from": { - "user": { - "@odata.type": "#microsoft.graph.teamworkUserIdentity", - "id": "82e32462", - "displayName": "Dummy", - "userIdentityType": "aadUser", - "tenantId": "a57a7700", - } - }, - "body": { - "contentType": "html", - "content": "
hello
", - }, - "attachments": [], - } - ], -} -FILESFOLDER = { - "id": "123", - "name": "root", - "size": 351660, - "parentReference": { - "driveId": "b!NJX42spQh0CsSMeITDaWt", - "driveType": "documentLibrary", - }, -} +@asynccontextmanager +async def create_teams_source( + auth_method="secret", + use_document_level_security=False, + fetch_attachment_content=True, +): + async with create_source( + MicrosoftTeamsDataSource, + tenant_id="tenant-id", + client_id="client-id", + auth_method=auth_method, + secret_value="secret", + certificate="certificate", + private_key="private-key", + use_document_level_security=use_document_level_security, + fetch_attachment_content=fetch_attachment_content, + ) as source: + yield source + + +class FakeGraphSession: + """Minimal MicrosoftAPISession stand-in for client tests.""" + + def __init__(self, pages=None, fetches=None, raises=None): + self._pages = pages or {} + self._fetches = fetches or {} + self._raises = raises or {} + self.set_logger = Mock() + self.close = Mock() + + async def scroll(self, url): + for key, exc in self._raises.items(): + if key in url: + raise exc + for key, pages in self._pages.items(): + if key in url: + for page in pages: + yield page + return + return + + async def fetch(self, url): + for key, exc in self._raises.items(): + if key in url: + raise exc + for key, value in self._fetches.items(): + if key in url: + return value + return None + + +def build_client(): + client = MicrosoftTeamsClient("tenant-id", "client-id", client_secret="secret") + return client + + +@pytest.mark.asyncio +async def test_get_default_configuration_has_new_auth_fields(): + config = MicrosoftTeamsDataSource.get_default_configuration() + assert set(config.keys()) == { + "tenant_id", + "client_id", + "auth_method", + "secret_value", + "certificate", + "private_key", + "fetch_attachment_content", + "use_text_extraction_service", + "use_document_level_security", + } + # legacy username/password auth is gone + assert "username" not in config + assert "password" not in config -class StubAPIToken: - async def get_with_username_password(self): - return "something" +@pytest.mark.asyncio +async def test_client_uses_graph_token_for_secret_auth(): + async with create_teams_source(auth_method="secret") as source: + assert isinstance(source.client.graph_api_token, GraphAPIToken) + await source.client.close() -@pytest_asyncio.fixture -async def microsoft_client(): - yield MicrosoftTeamsClient(None, None, None, None, None) +@pytest.mark.asyncio +async def test_client_uses_entra_token_for_certificate_auth(): + async with create_teams_source(auth_method="certificate") as source: + assert isinstance(source.client.graph_api_token, EntraAPIToken) + await source.client.close() -class ClientErrorException: - real_url = "" +@pytest.mark.asyncio +async def test_client_rejects_missing_credentials(): + with pytest.raises(Exception): + MicrosoftTeamsClient("tenant-id", "client-id") -@pytest_asyncio.fixture -async def patch_scroll(): - with patch.object( - MicrosoftTeamsClient, "scroll", return_value=AsyncMock() - ) as scroll: - yield scroll +@pytest.mark.asyncio +async def test_client_get_teams(): + client = build_client() + client._graph_api_client = FakeGraphSession(pages={"/teams?$top=999": [TEAMS]}) + result = [] + async for page in client.get_teams(): + result.extend(page) + await client.close() + assert result == TEAMS -@pytest_asyncio.fixture -async def client(): - client = MicrosoftTeamsClient( - "tenant_id", "client_id", "client_secret", "username", "password" +@pytest.mark.asyncio +async def test_client_get_team_members_swallows_permissions_missing(): + client = build_client() + client._graph_api_client = FakeGraphSession( + raises={"/members": PermissionsMissing()} ) + result = [] + async for page in client.get_team_members("team-1"): + result.extend(page) + await client.close() + assert result == [] - yield client +@pytest.mark.asyncio +async def test_client_get_channel_messages_swallows_not_found(): + client = build_client() + client._graph_api_client = FakeGraphSession(raises={"/messages": NotFound()}) + result = [] + async for page in client.get_channel_messages("team-1", "channel-1"): + result.extend(page) + await client.close() + assert result == [] -class ClientSession: - """Mock Client Session Class""" - async def close(self): - """Close method of Mock Client Session Class""" - pass +@pytest.mark.asyncio +async def test_client_get_channel_drive_children_recurses_into_folders(): + client = build_client() + folder = { + "id": "folder-1", + "folder": {"childCount": 1}, + "parentReference": {"driveId": "d"}, + } + nested_file = {"id": "nested", "file": {}, "parentReference": {"driveId": "d"}} + top_file = {"id": "top", "file": {}, "parentReference": {"driveId": "d"}} + pages = { + "/items/root-folder/children": [[folder, top_file]], + "/items/folder-1/children": [[nested_file]], + } + client._graph_api_client = FakeGraphSession(pages=pages) -async def create_fake_coroutine(item): - """create a method for returning fake coroutine value for - Args: - item: Value for converting into coroutine - """ - return item + seen = [] + async for child in client.get_channel_drive_children("drive-123", "root-folder"): + seen.append(child["id"]) + await client.close() + assert "nested" in seen + assert "top" in seen -def test_get_configuration(): - config = DataSourceConfiguration( - config=MicrosoftTeamsDataSource.get_default_configuration() - ) - assert config["client_id"] == "" +@pytest.mark.asyncio +async def test_client_ping_calls_teams_endpoint(): + client = build_client() + client._graph_api_client = MagicMock() + client._graph_api_client.fetch = AsyncMock(return_value={"value": []}) + await client.ping() + await client.close() + client._graph_api_client.fetch.assert_awaited_once() + assert "/teams" in client._graph_api_client.fetch.await_args.args[0] @pytest.mark.asyncio +async def test_validate_config_fetches_token(): + async with create_teams_source() as source: + source.client.graph_api_token.get = AsyncMock(return_value="token") + await source.validate_config() + source.client.graph_api_token.get.assert_awaited() + + +@pytest.mark.asyncio +async def test_ping_raises_on_error(): + async with create_teams_source() as source: + source.client.ping = AsyncMock(side_effect=Exception("boom")) + with pytest.raises(Exception): + await source.ping() + + +# -- Formatter --------------------------------------------------------------- + + +def test_formatter_team_member_id_is_prefixed_with_team(): + formatter = MicrosoftTeamsFormatter(Schema()) + doc = formatter.format_team_member(TEAM_MEMBERS[0], "team-1", "Team One") + assert doc["_id"] == "team-1-membership-1" + assert doc["type"] == TeamsObjectType.TEAM_MEMBER.value + assert doc["title"] == "Alice" + assert doc["email"] == "alice@example.com" + + +def test_formatter_channel_message(): + formatter = MicrosoftTeamsFormatter(Schema()) + doc = formatter.format_channel_message( + item=CHANNEL_MESSAGES[0], + channel_name="General", + message_content="Hello channel", + ) + assert doc["type"] == TeamsObjectType.CHANNEL_MESSAGE.value + assert doc["sender"] == "Alice" + assert doc["channel"] == "General" + assert doc["message"] == "Hello channel" + assert doc["_id"] == "message-1" + + +def test_formatter_chat_message_uses_topic_as_title(): + formatter = MicrosoftTeamsFormatter(Schema()) + doc = formatter.format_chat_message( + chat=CHATS[0], + message=CHAT_MESSAGES[0], + message_content="chat body", + members="Alice,Bob", + ) + assert doc["type"] == TeamsObjectType.CHAT_MESSAGE.value + assert doc["title"] == "Project chat" + assert doc["sender"] == "Alice" + assert doc["message"] == "chat body" + + +def test_formatter_chat_message_falls_back_to_members(): + formatter = MicrosoftTeamsFormatter(Schema()) + chat = dict(CHATS[0]) + chat["topic"] = None + doc = formatter.format_chat_message( + chat=chat, + message=CHAT_MESSAGES[0], + message_content="chat body", + members="Alice,Bob", + ) + assert doc["title"] == "Alice,Bob" + + +# -- DLS --------------------------------------------------------------------- + + @pytest.mark.parametrize( - "extras", + "feature_flag, config_value, expected", [ - ( - { - "client_id": "", - "client_secret": "", - "tenant_id": "", - } - ), + (True, True, True), + (True, False, False), + (False, True, False), ], ) -async def test_validate_configuration_with_invalid_fields_raises_error( - extras, -): - async with create_source(MicrosoftTeamsDataSource, **extras) as source: - with pytest.raises(ConfigurableFieldValueError): - await source.validate_config() +@pytest.mark.asyncio +async def test_dls_enabled(feature_flag, config_value, expected): + async with create_teams_source(use_document_level_security=config_value) as source: + source._features = Mock() + source._features.document_level_security_enabled = Mock( + return_value=feature_flag + ) + assert source._dls_enabled() == expected @pytest.mark.asyncio -async def test_ping_for_successful_connection(): - async with create_source(MicrosoftTeamsDataSource) as source: - DUMMY_RESPONSE = {} - source.client.fetch = Mock( - return_value=create_fake_coroutine(item=DUMMY_RESPONSE) - ) - await source.ping() +async def test_dls_disabled_when_features_missing(): + async with create_teams_source(use_document_level_security=True) as source: + source._features = None + assert not source._dls_enabled() @pytest.mark.asyncio -@patch("aiohttp.ClientSession.get") -async def test_ping_for_failed_connection_exception(mock_get): - async with create_source(MicrosoftTeamsDataSource) as source: - with patch.object( - MicrosoftTeamsClient, - "fetch", - side_effect=Exception("Something went wrong"), - ): - with pytest.raises(Exception): - await source.ping() +async def test_access_control_for_members(): + async with create_teams_source() as source: + acl = source._access_control_for_members(TEAM_MEMBERS) + assert "user_id:user-alice" in acl + assert "email:alice@example.com" in acl + assert "user_id:user-bob" in acl @pytest.mark.asyncio -async def test_get_docs_for_user_chats(): - async with create_source(MicrosoftTeamsDataSource) as source: - source.client.get_user_chats = Mock(return_value=AsyncIterator([USER_CHATS])) - source.client.get_user_chat_messages = Mock( - return_value=AsyncIterator([MESSAGES]) - ) - source.client.get_user_chat_attachments = Mock( - return_value=AsyncIterator([ATTACHMENTS]) - ) - source.client.get_user_chat_tabs = Mock(return_value=AsyncIterator([TABS])) - source.client.users = Mock(return_value=AsyncIterator([])) - source.client.get_teams = Mock(return_value=AsyncIterator([])) - async for item, _ in source.get_docs(): - assert item["_id"] in [ - "1691582610121", - "7ed8a6cb", - "400960d1", - "01EL4RL6L43X64K4ATD5H34UUQPOMMTRQX", - ] +async def test_decorate_with_access_control_noop_when_dls_disabled(): + async with create_teams_source(use_document_level_security=False) as source: + source._features = None + doc = source._decorate_with_access_control({"_id": "x"}, ["user_id:1"]) + assert ACCESS_CONTROL not in doc @pytest.mark.asyncio -async def test_get_docs_for_events(): - async with create_source(MicrosoftTeamsDataSource) as source: - source.client.get_user_chats = Mock(return_value=AsyncIterator([])) - source.client.get_teams = Mock(return_value=AsyncIterator([])) - source.client.users = Mock(return_value=AsyncIterator([USERS])) - source.client.get_calendars = Mock(return_value=AsyncIterator(EVENTS)) - - async for item, _ in source.get_docs(): - assert item["_id"] in [ - "AAMkADkyYTNmOTcw", - "AAMkADkyYTNmOTcw1", - "AAMkADkyYTNmOTcw2", - ] - assert item["title"] == "Transitive connector meeting" +async def test_decorate_with_access_control_when_dls_enabled(): + async with create_teams_source(use_document_level_security=True) as source: + source._features = Mock() + source._features.document_level_security_enabled = Mock(return_value=True) + doc = source._decorate_with_access_control({"_id": "x"}, ["user_id:1"]) + assert doc[ACCESS_CONTROL] == ["user_id:1"] @pytest.mark.asyncio -async def test_get_docs_for_teams(): - async with create_source(MicrosoftTeamsDataSource) as source: - source.client.get_user_chats = Mock(return_value=AsyncIterator([])) - source.client.users = Mock(return_value=AsyncIterator([])) - source.client.get_teams = Mock(return_value=AsyncIterator([TEAMS])) - source.client.get_team_channels = Mock(return_value=AsyncIterator([CHANNELS])) - source.client.get_channel_tabs = Mock(return_value=AsyncIterator([TABS])) - source.client.get_channel_messages = Mock( - return_value=AsyncIterator([MESSAGES]) - ) - source.client.get_channel_file = Mock( - return_value=create_fake_coroutine(item=FILESFOLDER) - ) - source.client.get_channel_drive_parent_children = Mock( - return_value=AsyncIterator(ATTACHMENTS) - ) - source.client.get_channel_drive_childrens = Mock( - return_value=AsyncIterator(ATTACHMENTS) +async def test_get_access_control_yields_unique_identities(): + async with create_teams_source(use_document_level_security=True) as source: + source._features = Mock() + source._features.document_level_security_enabled = Mock(return_value=True) + source.client.get_teams = MagicMock(return_value=AsyncIterator([TEAMS])) + source.client.get_team_members = MagicMock( + return_value=AsyncIterator([TEAM_MEMBERS]) ) + source.client.get_chats = MagicMock(return_value=AsyncIterator([CHATS])) - async for item, _ in source.get_docs(): - assert item["_id"] in [ - "400960d1", - "7ed8a6cb", - "1691565463406", - "19:36b3f1125", - "25ab782d", - "1691582610121", - "1691565463404", - "01EL4RL6L43X64K4ATD5H34UUQPOMMTRQX", - ] - if item.get("type") == "Channel Message": - assert item["attached_documents"] == "~`!@$^8()_+-=[]{};',k.txt" + ids = [] + async for doc in source.get_access_control(): + ids.append(doc["_id"]) + + # alice and bob appear in both team and chat, but must be deduped + assert sorted(ids) == ["user-alice", "user-bob"] @pytest.mark.asyncio -async def test_get_content(): - message = b"This is content of attachment" +async def test_get_access_control_skips_when_disabled(): + async with create_teams_source(use_document_level_security=False) as source: + source._features = None + docs = [doc async for doc in source.get_access_control()] + assert docs == [] - async def download_func(url, async_buffer): - await async_buffer.write(message) - async with create_source(MicrosoftTeamsDataSource) as source: - source.client.download_item = download_func - download_result = await source.get_content( - MOCK_ATTACHMENT, DOWNLOAD_URL, doit=True - ) - assert download_result["_attachment"] == base64.b64encode(message).decode() +# -- get_docs ---------------------------------------------------------------- + + +def _mock_client_for_get_docs(source, with_attachments=True): + source.client.get_teams = MagicMock(return_value=AsyncIterator([TEAMS])) + source.client.get_team_members = MagicMock( + return_value=AsyncIterator([TEAM_MEMBERS]) + ) + source.client.get_team_channels = MagicMock(return_value=AsyncIterator([CHANNELS])) + source.client.get_channel_messages = MagicMock( + return_value=AsyncIterator([CHANNEL_MESSAGES]) + ) + source.client.get_channel_file = AsyncMock(return_value=CHANNEL_FILES_FOLDER) + source.client.get_channel_drive_children = MagicMock( + return_value=AsyncIterator(CHANNEL_DRIVE_CHILDREN) + ) + source.client.get_chats = MagicMock(return_value=AsyncIterator([CHATS])) + source.client.get_chat_messages = MagicMock( + return_value=AsyncIterator([CHAT_MESSAGES]) + ) + source.client.get_chat_attachments = MagicMock( + return_value=AsyncIterator([CHAT_ATTACHMENTS]) + ) -@pytest.mark.parametrize( - "attachment, download_url", - [ - (MOCK_ATTACHMENT_WITHOUT_EXTENSION, DOWNLOAD_URL), - (MOCK_ATTACHMENT_WITH_LARGE_DATA, DOWNLOAD_URL), - (MOCK_ATTACHMENT_WITH_UNSUPPORTED_EXTENSION, DOWNLOAD_URL), - (MOCK_ATTACHMENT_WITH_ZERO_SIZE, DOWNLOAD_URL), - ], -) @pytest.mark.asyncio -async def test_get_content_negative(attachment, download_url): - message = b"This is content of attachment" +async def test_get_docs_emits_expected_types(): + async with create_teams_source() as source: + _mock_client_for_get_docs(source) - async def download_func(url, async_buffer): - await async_buffer.write(message) + docs = [] + async for doc, _download in source.get_docs(): + docs.append(doc) - async with create_source(MicrosoftTeamsDataSource) as source: - source.client.download_item = download_func - download_result = await source.get_content(attachment, download_url, doit=True) - assert download_result is None + types = {doc["type"] for doc in docs} + assert types == { + TeamsObjectType.TEAM.value, + TeamsObjectType.TEAM_MEMBER.value, + TeamsObjectType.CHANNEL.value, + TeamsObjectType.CHANNEL_MESSAGE.value, + TeamsObjectType.CHANNEL_ATTACHMENT.value, + TeamsObjectType.CHAT.value, + TeamsObjectType.CHAT_MESSAGE.value, + TeamsObjectType.CHAT_ATTACHMENT.value, + } @pytest.mark.asyncio -async def test_get_channel_file(): - async with create_source( - MicrosoftTeamsDataSource, - ) as source: - source.client.fetch = Mock(return_value=create_fake_coroutine(item=FILESFOLDER)) - returned_items = await source.client.get_channel_file( - team_id="team_id", channel_id="channel_id" - ) +async def test_get_docs_skips_deleted_channel_messages(): + async with create_teams_source() as source: + _mock_client_for_get_docs(source) - assert returned_items["id"] == "123" + message_ids = [] + async for doc, _download in source.get_docs(): + if doc["type"] == TeamsObjectType.CHANNEL_MESSAGE.value: + message_ids.append(doc["_id"]) + + assert "message-1" in message_ids + assert "reply-1" in message_ids + assert "message-2-deleted" not in message_ids @pytest.mark.asyncio -async def test_get_channel_messages(patch_scroll, client): - async with create_source( - MicrosoftTeamsDataSource, - ) as source: - source.client.scroll = AsyncIterator([MESSAGES]) - async for messages in source.client.get_channel_messages( - "team_id", "channel_id" - ): - for message in messages: - assert message["id"] in ["1691582610121", "1691565463404"] +async def test_get_docs_without_attachments_when_disabled(): + async with create_teams_source(fetch_attachment_content=False) as source: + _mock_client_for_get_docs(source) + types = set() + async for doc, _download in source.get_docs(): + types.add(doc["type"]) -@pytest.mark.asyncio -async def test_format_user_chat_messages(patch_scroll, client): - async with create_source( - MicrosoftTeamsDataSource, - ) as source: - message = await source.formatter.format_user_chat_messages( - chat=USER_CHATS[0], message={}, message_content="hello", members="dummy" - ) - assert message["title"] == "topic1" + assert TeamsObjectType.CHANNEL_ATTACHMENT.value not in types + assert TeamsObjectType.CHAT_ATTACHMENT.value not in types @pytest.mark.asyncio -async def test_format_user_chat_messages_for_members(patch_scroll, client): - user_chat_request = { - "id": "19:2ea91886", - "topic": None, - "createdDateTime": "2023-08-03T12:19:27.57Z", - "lastUpdatedDateTime": "2023-08-09T07:17:26.482Z", - "chatType": "oneOnOne", - "webUrl": "https://teams.microsoft.com/l/chat/19:2ea91886", - "tenantId": "a57a7700", - "members": [ - { - "id": "123-3=", - "displayName": "Duumy3", - "userId": "82e32463", - "email": "dummy3@3hm.onmicrosoft.com", - }, - { - "id": "123-2=", - "displayName": "Dummy2", - "userId": "2ea91886", - "email": "dummy2@3hm.onmicrosoft.com", - }, - ], - } - async with create_source( - MicrosoftTeamsDataSource, - ) as source: - message = await source.formatter.format_user_chat_messages( - chat=user_chat_request, - message={}, - message_content="hello", - members="Duumy3,Dummy2", +async def test_get_docs_continues_when_teams_permission_missing(): + async with create_teams_source() as source: + source.client.get_teams = MagicMock(side_effect=PermissionsMissing()) + source.client.get_chats = MagicMock(return_value=AsyncIterator([CHATS])) + source.client.get_chat_messages = MagicMock( + return_value=AsyncIterator([CHAT_MESSAGES]) + ) + source.client.get_chat_attachments = MagicMock( + return_value=AsyncIterator([CHAT_ATTACHMENTS]) ) - assert message["title"] == "Duumy3,Dummy2" + + types = set() + async for doc, _download in source.get_docs(): + types.add(doc["type"]) + + assert TeamsObjectType.CHAT.value in types + assert TeamsObjectType.TEAM.value not in types + + +# -- get_content ------------------------------------------------------------- @pytest.mark.asyncio -async def test_get_channel_messages_for_base_class(patch_scroll, client): - async with create_source( - MicrosoftTeamsDataSource, - ) as source: - await source.get_channel_messages( - message=CHANNEL_MESSAGE, channel_name="channel_name" +async def test_get_content_returns_none_when_not_doit(): + async with create_teams_source() as source: + attachment = { + "_id": "file-1", + "_timestamp": "2023-08-16T04:47:29Z", + "name": "report.txt", + "size_in_bytes": 42, + } + result = await source.get_content( + attachment, drive_id="d", item_id="i", doit=False ) + assert result is None @pytest.mark.asyncio -async def test_get_messages(patch_scroll, client): - async with create_source( - MicrosoftTeamsDataSource, - ) as source: - await source.get_messages( - message=CHANNEL_MESSAGE, - document_type="User Chat Messages", - chat=USER_CHATS[0], - channel_name="channel_name", +async def test_get_content_returns_none_for_zero_size(): + async with create_teams_source() as source: + attachment = { + "_id": "file-1", + "_timestamp": "2023-08-16T04:47:29Z", + "name": "report.txt", + "size_in_bytes": 0, + } + result = await source.get_content( + attachment, drive_id="d", item_id="i", doit=True ) + assert result is None @pytest.mark.asyncio -async def test_get_channel_tabs(): - async with create_source( - MicrosoftTeamsDataSource, - ) as source: - source.client.scroll = AsyncIterator([TABS]) - async for tabs in source.client.get_channel_tabs("team_id", "channel_id"): - for tab in tabs: - assert tab["id"] == "400960d1" +async def test_get_content_downloads_and_extracts(): + async with create_teams_source() as source: + attachment = { + "_id": "file-1", + "_timestamp": "2023-08-16T04:47:29Z", + "name": "report.txt", + "size_in_bytes": 42, + } + source.client.download_drive_item = AsyncMock() + with patch.object( + source, + "download_and_extract_file", + new=AsyncMock(), + ): + # Bypass the real extraction; assert only branching / helper calls + source.can_file_be_downloaded = Mock(return_value=True) + source.create_temp_file = MagicMock() + handle = AsyncMock(return_value={"_id": "file-1", "_attachment": "b64"}) + source.handle_file_content_extraction = handle -@pytest.mark.asyncio -async def test_get_team_channels(): - async with create_source( - MicrosoftTeamsDataSource, - ) as source: - source.client.scroll = AsyncIterator([CHANNELS]) - async for channels in source.client.get_team_channels("team_id"): - for channel in channels: - assert channel["id"] == "19:36b3f1125" + @asynccontextmanager + async def fake_temp(_ext): + buffer = MagicMock() + buffer.name = "/tmp/report.txt" + buffer.close = AsyncMock() + yield buffer + source.create_temp_file = fake_temp -@pytest.mark.asyncio -async def test_get_teams(): - async with create_source( - MicrosoftTeamsDataSource, - ) as source: - source.client.scroll = AsyncIterator([TEAMS]) - async for teams in source.client.get_teams(): - for team in teams: - assert team["id"] == "25ab782d" + result = await source.get_content( + attachment, drive_id="drive-123", item_id="file-1", doit=True + ) + + source.client.download_drive_item.assert_awaited_once() + handle.assert_awaited_once() + assert result["_attachment"] == "b64" @pytest.mark.asyncio -async def test_get_calendars(): - async with create_source( - MicrosoftTeamsDataSource, - ) as source: - source.client.scroll = AsyncIterator([[EVENTS]]) - async for events in source.client.get_calendars("user_id"): - for event in events: - assert event["id"] in [ - "AAMkADkyYTNmOTcw", - "AAMkADkyYTNmOTcw1", - "AAMkADkyYTNmOTcw2", - ] +async def test_consumer_decrements_tasks_on_end_signal(): + async with create_teams_source() as source: + source.tasks = 1 + await source.queue.put(({"_id": "1"}, None)) + await source.queue.put(EndSignal.TEAM_TASK_FINISHED) + collected = [] + async for item in source._consumer(): + collected.append(item) -@pytest.mark.asyncio -async def test_get_user_chat_tabs(): - async with create_source( - MicrosoftTeamsDataSource, - ) as source: - source.client.scroll = AsyncIterator([TABS]) - async for tabs in source.client.get_user_chat_tabs("chat_id"): - for tab in tabs: - assert tab["id"] == "400960d1" + assert collected == [({"_id": "1"}, None)] + assert source.tasks == 0 -@pytest.mark.asyncio -async def test_get_user_chat_messages(): - async with create_source( - MicrosoftTeamsDataSource, - ) as source: - source.client.scroll = AsyncIterator([MESSAGES]) - async for message in source.client.get_user_chat_messages("chat_id"): - assert len(message) == len(MESSAGES) +# -- Message processing edge cases ------------------------------------------- -@pytest.mark.asyncio -async def test_get_user_chats(): - async with create_source( - MicrosoftTeamsDataSource, - ) as source: - source.client.scroll = AsyncIterator([USER_CHATS]) - async for chats in source.client.get_user_chats(): - assert len(chats) == len(USER_CHATS) +def _empty_body_message(with_attachment): + return { + "id": "msg-empty", + "messageType": "message", + "createdDateTime": "2023-08-16T04:47:55.794Z", + "lastModifiedDateTime": "2023-08-16T04:47:55.794Z", + "deletedDateTime": None, + "webUrl": None, + "from": {"user": {"id": "user-alice", "displayName": "Alice"}}, + "body": {"contentType": "html", "content": ""}, + "attachments": [{"id": "a1", "name": "f.txt", "contentType": "reference"}] + if with_attachment + else [], + } @pytest.mark.asyncio -async def test_users(): - async with create_source( - MicrosoftTeamsDataSource, - ) as source: - source.client.scroll = AsyncIterator([USERS]) - async for users in source.client.users(): - assert len(users) == len(USERS) +async def test_process_channel_message_indexes_attachment_only_message(): + async with create_teams_source() as source: + await source._process_channel_message( + _empty_body_message(with_attachment=True), "General", [] + ) + assert not source.queue.empty() + _, (doc, _download) = source.queue.get_nowait() + assert doc["_id"] == "msg-empty" + assert doc["type"] == TeamsObjectType.CHANNEL_MESSAGE.value + assert doc["attached_documents"] == "f.txt" @pytest.mark.asyncio -async def test_set_internal_logger(): - async with create_source( - MicrosoftTeamsDataSource, - ) as source: - source._set_internal_logger() - assert source.client._logger == logger +async def test_process_channel_message_drops_truly_empty_message(): + async with create_teams_source() as source: + await source._process_channel_message( + _empty_body_message(with_attachment=False), "General", [] + ) + assert source.queue.empty() @pytest.mark.asyncio -async def test_scroll(microsoft_client, mock_responses): - url = "http://localhost:1234/url" - first_page = ["1", "2", "3"] - - next_url = "http://localhost:1234/url/page-two" - second_page = ["4", "5", "6"] +async def test_process_chat_message_indexes_attachment_only_message(): + async with create_teams_source() as source: + await source._process_chat_message( + CHATS[0], _empty_body_message(with_attachment=True), "Alice,Bob", [] + ) + assert not source.queue.empty() + _, (doc, _download) = source.queue.get_nowait() + assert doc["_id"] == "msg-empty" + assert doc["type"] == TeamsObjectType.CHAT_MESSAGE.value + assert doc["attached_documents"] == "f.txt" - first_payload = {"value": first_page, "@odata.nextLink": next_url} - second_payload = {"value": second_page} - mock_responses.get(url, payload=first_payload) - mock_responses.get(next_url, payload=second_payload) +@pytest.mark.asyncio +async def test_process_message_handles_null_body_without_raising(): + async with create_teams_source() as source: + null_body = _empty_body_message(with_attachment=False) + null_body["body"] = None + # Should not raise AttributeError; message is dropped (no text, no attachments) + await source._process_channel_message(null_body, "General", []) + await source._process_chat_message(CHATS[0], null_body, "Alice,Bob", []) + assert source.queue.empty() - pages = [] - with patch.object( - GraphAPIToken, - "_fetch_token", - return_value=await create_fake_coroutine(("hello", 15)), - ): - async for page in microsoft_client.scroll(url): - pages.append(page) - assert first_page in pages - assert second_page in pages +# -- Robustness: access control + total-failure safeguard -------------------- @pytest.mark.asyncio -async def test_pipe(microsoft_client, mock_responses): - class AsyncStream: - def __init__(self): - self.stream = BytesIO() - - async def write(self, data): - self.stream.write(data) +async def test_get_access_control_tolerates_permissions_missing_on_teams(): + async with create_teams_source(use_document_level_security=True) as source: + source._features = Mock() + source._features.document_level_security_enabled = Mock(return_value=True) + source.client.get_teams = MagicMock(side_effect=PermissionsMissing()) + source.client.get_chats = MagicMock(return_value=AsyncIterator([CHATS])) - def read(self): - return self.stream.getvalue().decode() + ids = [doc["_id"] async for doc in source.get_access_control()] - url = "http://localhost:1234/download-some-sample-file" - file_content = "hello world, this is content of downloaded file" - stream = AsyncStream() - mock_responses.get(url, body=file_content) - with patch.object( - GraphAPIToken, - "_fetch_token", - return_value=await create_fake_coroutine(("hello", 15)), - ): - await microsoft_client.pipe(url, stream) - - assert stream.read() == file_content + # teams enumeration failed but chats still produced identities + assert sorted(ids) == ["user-alice", "user-bob"] @pytest.mark.asyncio -async def test_call_api_with_403( - microsoft_client, - mock_responses, -): - url = "http://localhost:1234/download-some-sample-file" - - unauthorized_error = ClientResponseError(None, None) - unauthorized_error.status = 403 - unauthorized_error.message = "Something went wrong" - - mock_responses.get(url, exception=unauthorized_error) - mock_responses.get(url, exception=unauthorized_error) - mock_responses.get(url, exception=unauthorized_error) - with patch.object( - GraphAPIToken, - "_fetch_token", - return_value=await create_fake_coroutine(("hello", 15)), - ): - with pytest.raises(PermissionsMissing) as e: - async for _ in microsoft_client._get(url): - pass - - assert e is not None +async def test_get_docs_raises_when_both_enumerations_fail(): + async with create_teams_source() as source: + source.client.get_teams = MagicMock(side_effect=PermissionsMissing()) + source.client.get_chats = MagicMock(side_effect=PermissionsMissing()) - -@pytest.mark.asyncio -async def test_call_api_with_404( - microsoft_client, - mock_responses, -): - url = "http://localhost:1234/download-some-sample-file" - - not_found_error = ClientResponseError(None, None) - not_found_error.status = 404 - not_found_error.message = "Something went wrong" - - mock_responses.get(url, exception=not_found_error) - mock_responses.get(url, exception=not_found_error) - mock_responses.get(url, exception=not_found_error) - with patch.object( - GraphAPIToken, - "_fetch_token", - return_value=await create_fake_coroutine(("hello", 15)), - ): - with pytest.raises(NotFound) as e: - async for _ in microsoft_client._get(url): + with pytest.raises(PermissionsMissing): + async for _doc, _download in source.get_docs(): pass - assert e is not None - @pytest.mark.asyncio -async def test_call_api_with_os_error( - microsoft_client, - mock_responses, - patch_sleep, -): - url = "http://localhost:1234/download-some-sample-file" - - not_found_error = ClientOSError() - not_found_error.message = "Something went wrong" - - mock_responses.get(url, exception=not_found_error) - mock_responses.get(url, exception=not_found_error) - mock_responses.get(url, exception=not_found_error) - with patch.object( - GraphAPIToken, - "_fetch_token", - return_value=await create_fake_coroutine(("hello", 15)), - ): - with pytest.raises(ClientOSError) as e: - async for _ in microsoft_client._get(url): - pass +async def test_get_docs_empty_tenant_does_not_raise(): + async with create_teams_source() as source: + source.client.get_teams = MagicMock(return_value=AsyncIterator([])) + source.client.get_chats = MagicMock(return_value=AsyncIterator([])) - assert e is not None + docs = [doc async for doc, _download in source.get_docs()] + assert docs == [] @pytest.mark.asyncio -async def test_call_api_with_500( - microsoft_client, - mock_responses, - patch_sleep, -): - url = "http://localhost:1234/download-some-sample-file" +async def test_get_docs_raises_on_unexpected_enumeration_error(): + # A non-PermissionsMissing error during enumeration must fail the sync, not + # hang it (the orchestrator runs as a pool task whose exception is swallowed + # by ConcurrentTasks; get_docs re-raises the recorded error instead). + async with create_teams_source() as source: + source.client.get_teams = MagicMock(side_effect=RuntimeError("boom")) + source.client.get_chats = MagicMock(return_value=AsyncIterator([])) - not_found_error = ClientResponseError( - history="history", request_info=ClientErrorException - ) - not_found_error.status = 500 - not_found_error.message = "Something went wrong" - - mock_responses.get(url, exception=not_found_error) - mock_responses.get(url, exception=not_found_error) - mock_responses.get(url, exception=not_found_error) - with patch.object( - GraphAPIToken, - "_fetch_token", - return_value=await create_fake_coroutine(("hello", 15)), - ): - with pytest.raises(InternalServerError) as e: - async for _ in microsoft_client._get(url): + async def drain(): + async for _doc, _download in source.get_docs(): pass - assert e is not None + with pytest.raises(RuntimeError, match="boom"): + await asyncio.wait_for(drain(), timeout=5) + + +@pytest.mark.asyncio +async def test_get_docs_completes_under_low_concurrency(): + # Regression: channels used to be scheduled into the same bounded pool from + # within team_producer. With a small pool and multiple teams that each have a + # channel, that nested scheduling deadlocked. Channels are now processed + # inline, so the sync must complete regardless of pool size. + two_teams = [ + {"id": "team-1", "displayName": "Team One"}, + {"id": "team-2", "displayName": "Team Two"}, + ] + async with create_teams_source(fetch_attachment_content=False) as source: + source.fetchers = ConcurrentTasks(max_concurrency=2) + source.client.get_teams = MagicMock(return_value=AsyncIterator([two_teams])) + source.client.get_team_members = MagicMock( + side_effect=lambda *a, **k: AsyncIterator([TEAM_MEMBERS]) + ) + source.client.get_team_channels = MagicMock( + side_effect=lambda *a, **k: AsyncIterator([CHANNELS]) + ) + source.client.get_channel_messages = MagicMock( + side_effect=lambda *a, **k: AsyncIterator([[]]) + ) + source.client.get_chats = MagicMock(return_value=AsyncIterator([])) + async def drain(): + return [doc async for doc, _download in source.get_docs()] -class JSONAsyncMock(AsyncMock): - def __init__(self, json, *args, **kwargs): - super().__init__(*args, **kwargs) - self._json = json + docs = await asyncio.wait_for(drain(), timeout=5) - async def json(self): - return self._json + types = {doc["type"] for doc in docs} + assert TeamsObjectType.TEAM.value in types + assert TeamsObjectType.CHANNEL.value in types + # both teams' channels were produced inline + channel_ids = [ + doc["_id"] for doc in docs if doc["type"] == TeamsObjectType.CHANNEL.value + ] + assert len(channel_ids) == 2 @pytest.mark.asyncio -@patch("connectors.utils.time_to_sleep_between_retries", Mock(return_value=0)) -async def test_call_api_with_429( - microsoft_client, - mock_responses, -): - initial_response = ClientResponseError(None, None) - initial_response.status = 429 - initial_response.message = "rate-limited" - url = "http://localhost:1234/download-some-sample-file" - - retried_response = AsyncMock() - payload = {"value": "Test rate limit"} +async def test_get_docs_calls_log_skip_summary(): + async with create_teams_source() as source: + _mock_client_for_get_docs(source) + source.client.log_skip_summary = Mock() - retried_response.__aenter__ = AsyncMock(return_value=JSONAsyncMock(payload)) - with patch("connectors.sources.microsoft_teams.client.RETRY_SECONDS", 0.3): - with patch.object( - GraphAPIToken, "get_with_username_password", return_value="abc" - ): - with patch( - "aiohttp.ClientSession.get", - side_effect=[initial_response, retried_response], - ): - async for response in microsoft_client._get(url): - actual_payload = await response.json() - assert actual_payload == payload + async for _doc, _download in source.get_docs(): + pass + source.client.log_skip_summary.assert_called_once() -@pytest.mark.asyncio -@patch("connectors.utils.time_to_sleep_between_retries", Mock(return_value=0)) -async def test_call_api_with_429_with_retry_after( - microsoft_client, - mock_responses, -): - initial_response = ClientResponseError(None, None) - initial_response.status = 429 - initial_response.message = "rate-limited" - initial_response.headers = {"Retry-After": "0"} - url = "http://localhost:1234/download-some-sample-file" - - retried_response = AsyncMock() - payload = {"value": "Test rate limit"} - retried_response.__aenter__ = AsyncMock(return_value=JSONAsyncMock(payload)) - with patch("connectors.sources.microsoft_teams.client.RETRY_SECONDS", 0.3): - with patch.object( - GraphAPIToken, "get_with_username_password", return_value="abc" - ): - with patch( - "aiohttp.ClientSession.get", - side_effect=[initial_response, retried_response], - ): - async for response in microsoft_client._get(url): - actual_payload = await response.json() - assert actual_payload == payload +# -- Skip visibility --------------------------------------------------------- @pytest.mark.asyncio -async def test_call_api_with_unhandled_status( - microsoft_client, mock_responses, patch_sleep -): - url = "http://localhost:1234/download-some-sample-file" - - error_message = "Something went wrong" - - not_found_error = ClientResponseError(MagicMock(), MagicMock()) - not_found_error.status = 420 - not_found_error.message = error_message - - mock_responses.get(url, exception=not_found_error) - mock_responses.get(url, exception=not_found_error) - mock_responses.get(url, exception=not_found_error) - with patch.object( - GraphAPIToken, - "_fetch_token", - return_value=await create_fake_coroutine(("hello", 15)), - ): - with pytest.raises(ClientResponseError) as e: - async for _ in microsoft_client._get(url): - pass - - assert e.match(error_message) +async def test_client_counts_skipped_resources(): + client = build_client() + client._graph_api_client = FakeGraphSession( + raises={"/members": PermissionsMissing()} + ) + async for _ in client.get_team_members("team-1"): + pass + await client.close() + assert client._skipped["teams' members"] == 1 @pytest.mark.asyncio -async def test_get_for_client(): - bearer = "hello" - source = GraphAPIToken( - "tenant_id", "client_id", "client_secret", "username", "password" - ) - source._fetch_token = Mock(return_value=create_fake_coroutine(("hello", 15))) - actual_token = await source.get_with_client() - - assert actual_token == bearer +async def test_log_skip_summary_warns_when_skips_recorded(): + client = build_client() + client._logger = Mock() + client._skipped["teams' messages"] = 3 + client.log_skip_summary() + await client.close() + client._logger.warning.assert_called_once() + # counters are reset after logging + assert not client._skipped @pytest.mark.asyncio -async def test_get_for_username_password(): - bearer = "hello" - source = GraphAPIToken( - "tenant_id", "client_id", "client_secret", "username", "password" - ) - source._fetch_token = Mock(return_value=create_fake_coroutine(("hello", 15))) - actual_token = await source.get_with_username_password() +async def test_log_skip_summary_silent_when_nothing_skipped(): + client = build_client() + client._logger = Mock() + client.log_skip_summary() + await client.close() + client._logger.warning.assert_not_called() - assert actual_token == bearer + +# -- Config validation ------------------------------------------------------- @pytest.mark.asyncio -async def test_fetch(microsoft_client, mock_responses): - with patch.object( - GraphAPIToken, - "_fetch_token", - return_value=await create_fake_coroutine(("hello", 15)), - ): - url = "http://localhost:1234/url" +async def test_incremental_sync_disabled(): + assert MicrosoftTeamsDataSource.incremental_sync_enabled is False + - response = {"displayName": "Dummy", "id": "123"} +@asynccontextmanager +async def _source_with_config(**overrides): + config = { + "tenant_id": "tenant-id", + "client_id": "client-id", + "auth_method": "secret", + "secret_value": "secret", + "certificate": "certificate", + "private_key": "private-key", + "use_document_level_security": False, + "fetch_attachment_content": True, + } + config.update(overrides) + async with create_source(MicrosoftTeamsDataSource, **config) as source: + yield source - mock_responses.get(url, payload=response) - fetch_response = await microsoft_client.fetch(url) - assert response == fetch_response +@pytest.mark.asyncio +async def test_validate_config_requires_secret_value(): + async with _source_with_config(auth_method="secret", secret_value="") as source: + # Prevent the client cached_property from building with invalid creds + source.client = MagicMock(close=AsyncMock()) + with patch.object(BaseDataSource, "validate_config", new=AsyncMock()): + with pytest.raises(ConfigurableFieldValueError): + await source.validate_config() @pytest.mark.asyncio -async def test_get_user_drive_root_children(): - async with create_source( - MicrosoftTeamsDataSource, +async def test_validate_config_requires_certificate_and_key(): + async with _source_with_config( + auth_method="certificate", certificate="cert", private_key="" ) as source: - source.client.scroll = AsyncIterator([[{"name": "microsoft teams chat files"}]]) - child = await source.client.get_user_drive_root_children(drive_id=1) - assert child["name"] == "microsoft teams chat files" + source.client = MagicMock(close=AsyncMock()) + with patch.object(BaseDataSource, "validate_config", new=AsyncMock()): + with pytest.raises(ConfigurableFieldValueError): + await source.validate_config() diff --git a/app/connectors_service/tests/sources/test_sharepoint_online.py b/app/connectors_service/tests/sources/test_sharepoint_online.py index 74bbbc9f5..5e339ef49 100644 --- a/app/connectors_service/tests/sources/test_sharepoint_online.py +++ b/app/connectors_service/tests/sources/test_sharepoint_online.py @@ -546,7 +546,7 @@ async def test_fetch_token(self, token, mock_responses): certificate_credential_mock.close = AsyncMock() with patch( - "connectors.sources.sharepoint.sharepoint_online.client.CertificateCredential", + "connectors.sources.shared.microsoft.graph.CertificateCredential", return_value=certificate_credential_mock, ): actual_token, actual_expires_at = await token._fetch_token() @@ -575,7 +575,7 @@ def effect(*args, **kwargs): certificate_credential_mock.get_token = AsyncMock(side_effect=effect()) with patch( - "connectors.sources.sharepoint.sharepoint_online.client.CertificateCredential", + "connectors.sources.shared.microsoft.graph.CertificateCredential", return_value=certificate_credential_mock, ): actual_token, actual_expires_at = await token._fetch_token()