diff --git a/libraries/microsoft-agents-activity/microsoft_agents/activity/attachment.py b/libraries/microsoft-agents-activity/microsoft_agents/activity/attachment.py index af702f5af..51cfa7091 100644 --- a/libraries/microsoft-agents-activity/microsoft_agents/activity/attachment.py +++ b/libraries/microsoft-agents-activity/microsoft_agents/activity/attachment.py @@ -21,7 +21,7 @@ class Attachment(AgentsModel): """ content_type: NonEmptyString - content_url: NonEmptyString = None + content_url: NonEmptyString | None = None content: object = None - name: NonEmptyString = None + name: NonEmptyString | None = None thumbnail_url: NonEmptyString = None diff --git a/libraries/microsoft-agents-activity/microsoft_agents/activity/channels.py b/libraries/microsoft-agents-activity/microsoft_agents/activity/channels.py index 3896e1547..ae9aa86f1 100644 --- a/libraries/microsoft-agents-activity/microsoft_agents/activity/channels.py +++ b/libraries/microsoft-agents-activity/microsoft_agents/activity/channels.py @@ -79,6 +79,9 @@ class Channels(str, Enum): copilot_studio = "pva-studio" """Microsoft Copilot Studio channel.""" + m365_copilot = f"msteams:COPILOT" + """Microsoft 365 Copilot channel.""" + ms_teams = "msteams" """Deprecated alias for :attr:`msteams`. Kept for backwards compatibility.""" diff --git a/libraries/microsoft-agents-hosting-core/microsoft_agents/hosting/core/app/_utils.py b/libraries/microsoft-agents-hosting-core/microsoft_agents/hosting/core/app/_utils.py new file mode 100644 index 000000000..2bd859082 --- /dev/null +++ b/libraries/microsoft-agents-hosting-core/microsoft_agents/hosting/core/app/_utils.py @@ -0,0 +1,23 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +from email.message import Message + +_CONTENT_TYPE = "Content-Type" + + +def _parse_content_type(content_type: str) -> tuple[str, dict[str, str]] | None: + """Parses the given content type string into its main type and parameters. + + :param content_type: The content type string to parse. + :return: A tuple containing the main content type and a dictionary of parameters, + or None if parsing fails. + """ + email = Message() + email[_CONTENT_TYPE] = content_type + params = email.get_params() + if params is None: + return None + # the first param is the mime-type + # the later ones are the attribtues like "charset" + return params[0][0], dict(params[1:]) diff --git a/libraries/microsoft-agents-hosting-core/microsoft_agents/hosting/core/app/attachment_downloader.py b/libraries/microsoft-agents-hosting-core/microsoft_agents/hosting/core/app/attachment_downloader.py new file mode 100644 index 000000000..a8c8c1eac --- /dev/null +++ b/libraries/microsoft-agents-hosting-core/microsoft_agents/hosting/core/app/attachment_downloader.py @@ -0,0 +1,105 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +import json +from typing import Callable + +import aiohttp + +from microsoft_agents.activity import ( + Attachment, + Channels, + ChannelId, +) + +from microsoft_agents.hosting.core.turn_context import TurnContext +from microsoft_agents.hosting.core.outbound_host_validator import OutboundHostValidator + +from .input_file import InputFileDownloader, InputFile +from ._utils import _parse_content_type + + +class AttachmentDownloader(InputFileDownloader): + + def __init__( + self, + client_factory: Callable[[], aiohttp.ClientSession] | None = None, + host_validator: OutboundHostValidator | None = None, + ): + """Constructor for AttachmentDownloader. + + :param client_factory: A callable that returns an aiohttp.ClientSession instance. + :param host_validator: An optional OutboundHostValidator instance. + """ + + self._client_factory = client_factory or aiohttp.ClientSession + self._host_validator = host_validator + + async def download_files(self, context: TurnContext) -> list[InputFile]: + """Downloads files for the given turn context. + + :param context: The TurnContext instance for the current turn. + """ + if ChannelId.get_channel(context.activity.channel_id) == Channels.ms_teams: + return [] + + if not context.activity.attachments: + return [] + + files: list[InputFile] = [] + for attachment in context.activity.attachments: + file = await self._download_file(attachment) + if file: + files.append(file) + + return files + + async def _download_file(self, attachment: Attachment) -> InputFile | None: + """Downloads a single file from the given attachment. + + :param attachment: The attachment to download. + :return: An InputFile instance if the download is successful, None otherwise. + """ + if attachment.content_url and ( + attachment.content_url.startswith("https://") + or attachment.content_url.startswith("http://localhost") + ): + remote_file_url = attachment.content_url + + if ( + self._host_validator + and self._host_validator.enabled + and not self._host_validator.is_allowed(remote_file_url) + ): + return None + + async with self._client_factory() as client: + async with client.get(remote_file_url) as response: + + if not response.status == 200: + return None + + content_type_val = response.headers.get("Content-Type", "") + result = _parse_content_type(content_type_val) + if result is None: + return None + content_type, _ = result + if content_type.startswith("image/"): + content_type = "image/png" + + res = await response.read() + + return InputFile( + content=res, + content_type=content_type, + content_url=attachment.content_url, + filename=attachment.name, + ) + else: + content = bytes(json.dumps(attachment.content), "utf-8") + return InputFile( + content=content, + content_type=attachment.content_type, + content_url=attachment.content_url, + filename=attachment.name, + ) diff --git a/libraries/microsoft-agents-hosting-core/microsoft_agents/hosting/core/app/input_file.py b/libraries/microsoft-agents-hosting-core/microsoft_agents/hosting/core/app/input_file.py index 1a6589d93..a8fc4e14c 100644 --- a/libraries/microsoft-agents-hosting-core/microsoft_agents/hosting/core/app/input_file.py +++ b/libraries/microsoft-agents-hosting-core/microsoft_agents/hosting/core/app/input_file.py @@ -1,13 +1,10 @@ -""" -Copyright (c) Microsoft Corporation. All rights reserved. -Licensed under the MIT License. -""" +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. from __future__ import annotations from abc import ABC, abstractmethod from dataclasses import dataclass -from typing import Optional from microsoft_agents.hosting.core import TurnContext @@ -26,7 +23,8 @@ class InputFile: content: bytes content_type: str - content_url: Optional[str] + content_url: str | None = None + filename: str | None = None class InputFileDownloader(ABC): diff --git a/libraries/microsoft-agents-hosting-core/microsoft_agents/hosting/core/app/m365_attachment_downloader.py b/libraries/microsoft-agents-hosting-core/microsoft_agents/hosting/core/app/m365_attachment_downloader.py new file mode 100644 index 000000000..80f47d1fa --- /dev/null +++ b/libraries/microsoft-agents-hosting-core/microsoft_agents/hosting/core/app/m365_attachment_downloader.py @@ -0,0 +1,177 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +import json +from typing import Callable, cast, Any + +import aiohttp + +from microsoft_agents.activity import ( + Attachment, + Channels, + ChannelId, +) + +from microsoft_agents.hosting.core.authorization import ( + AccessTokenProviderBase, + Connections, +) +from microsoft_agents.hosting.core.turn_context import TurnContext +from microsoft_agents.hosting.core.outbound_host_validator import OutboundHostValidator + +from .input_file import InputFileDownloader, InputFile +from ._utils import _parse_content_type + + +class M365AttachmentDownloader(InputFileDownloader): + """Downloads attachments from M365/Teams using the configured Token Provider (from Connections).""" + + def __init__( + self, + connections: Connections, + client_factory: Callable[[], aiohttp.ClientSession] | None = None, + host_validator: OutboundHostValidator | None = None, + *, + token_provider_name: str = "", + use_anonymous: bool = False, + scopes: list[str] | None = None, + ): + """Constructor for M365AttachmentDownloader. + + :param connections: A Connections instance. + :param client_factory: A callable that returns an aiohttp.ClientSession instance. + :param host_validator: An optional OutboundHostValidator instance. + :param token_provider_name: The name of the token provider. + :param use_anonymous: Whether to use anonymous access. + :param scopes: A list of scopes for the access token. + :param connections: A Connections instance. + """ + + self._connections = connections + self._client_factory = client_factory or aiohttp.ClientSession + self._host_validator = host_validator + + self._token_provider_name = token_provider_name + self._use_anonymous = use_anonymous + self._scopes = scopes or [] + + async def download_files(self, context: TurnContext) -> list[InputFile]: + """Download files from the given context. + + :param context: The TurnContext instance. + :return: A list of InputFile instances. + """ + + if not context.identity: + raise ValueError("No valid context identity found.") + + outgoing_audience_claim = context.identity.get_outgoing_audience_claim() + if not outgoing_audience_claim: + raise ValueError("No valid outgoing App ID found.") + + if context.activity.channel_id not in ( + Channels.ms_teams, + Channels.m365_copilot, + ): + return [] + + attachments: list[Attachment] + if not context.activity.attachments: + return [] + attachments = [ + att + for att in context.activity.attachments + if not att.content_type.startswith("text/html") + ] + if not attachments: + return [] + + access_token = "" + + if not self._use_anonymous: + token_provider: AccessTokenProviderBase | None = None + if self._token_provider_name: + try: + token_provider = self._connections.get_connection( + self._token_provider_name + ) + except ValueError: + pass + if not token_provider: + token_provider = self._connections.get_token_provider_from_activity( + context.identity, context.activity + ) + if not token_provider: + raise RuntimeError("No valid token provider found.") + + access_token = await token_provider.get_access_token( + outgoing_audience_claim, self._scopes + ) + + files: list[InputFile] = [] + for att in attachments: + file = await self._download_file(att, access_token) + if file: + files.append(file) + + return files + + async def _download_file( + self, attachment: Attachment, access_token: str + ) -> InputFile | None: + """Download a single file from the given attachment. + + :param attachment: The Attachment instance. + :param access_token: The access token for authentication. + :return: An InputFile instance or None if the download fails. + """ + name = attachment.name + + if attachment.content_url and ( + attachment.content_url.startswith("https://") + or attachment.content_url.startswith("http://localhost") + ): + download_url: str + if isinstance(attachment.content, dict): + content_dict = cast(dict[str, Any], attachment.content) + download_url = content_dict.get("downloadUrl", attachment.content_url) + else: + download_url = attachment.content_url + + if ( + self._host_validator is not None + and self._host_validator.enabled + and not self._host_validator.is_allowed(download_url) + ): + return None + + async with self._client_factory() as client: + async with client.get( + download_url, headers={"Authorization": f"Bearer {access_token}"} + ) as response: + if response.status >= 300: + return None + content = await response.read() + result = _parse_content_type( + response.headers.get("Content-Type", "") + ) + if result is None: + return None + content_type, _ = result + if content_type.startswith("image/"): + content_type = "image/png" + + return InputFile( + content=content, + content_type=content_type, + content_url=attachment.content_url, + filename=name, + ) + else: + content = bytes(json.dumps(attachment.content), "utf-8") + return InputFile( + content=content, + content_type=attachment.content_type, + content_url=attachment.content_url, + filename=attachment.name, + ) diff --git a/libraries/microsoft-agents-hosting-core/microsoft_agents/hosting/core/authorization/claims_identity.py b/libraries/microsoft-agents-hosting-core/microsoft_agents/hosting/core/authorization/claims_identity.py index 1067c9eff..9a862e153 100644 --- a/libraries/microsoft-agents-hosting-core/microsoft_agents/hosting/core/authorization/claims_identity.py +++ b/libraries/microsoft-agents-hosting-core/microsoft_agents/hosting/core/authorization/claims_identity.py @@ -79,6 +79,14 @@ def get_outgoing_app_id(self) -> str | None: return app_id + def get_outgoing_audience_claim(self) -> str | None: + """Retrieves the audience for an outgoing token from the given incoming activity.""" + if self.is_agent_claim(): + return f"api://{self.get_outgoing_app_id()}" + if self.is_gov_botframework_claim(): + return AuthenticationConstants.GOV_AGENTS_SDK_TOKEN_ISSUER + return AuthenticationConstants.AGENTS_SDK_SCOPE + def is_agent_claim(self) -> bool: """ Checks if the current claims represents an agent claim (not coming from ABS/SMBA). @@ -116,6 +124,15 @@ def get_token_audience(self) -> str: else AuthenticationConstants.AGENTS_SDK_SCOPE ) + def is_gov_botframework_claim(self) -> bool: + """Determines whether the specified incoming identity represents a government Bot Framework claim.""" + aud = self.claims.get(AuthenticationConstants.AUDIENCE_CLAIM, None) + return ( + aud.lower() == AuthenticationConstants.GOV_AGENTS_SDK_TOKEN_ISSUER.lower() + if aud + else False + ) + def get_token_scope(self) -> list[str]: """ Gets the token scope from current claims. diff --git a/libraries/microsoft-agents-hosting-core/microsoft_agents/hosting/core/connector/attachments_base.py b/libraries/microsoft-agents-hosting-core/microsoft_agents/hosting/core/connector/attachments_base.py index ea9d2b9c8..c26410289 100644 --- a/libraries/microsoft-agents-hosting-core/microsoft_agents/hosting/core/connector/attachments_base.py +++ b/libraries/microsoft-agents-hosting-core/microsoft_agents/hosting/core/connector/attachments_base.py @@ -15,3 +15,7 @@ async def get_attachment_info(self, attachment_id: str) -> AttachmentInfo: @abstractmethod async def get_attachment(self) -> Optional[AsyncIterator[bytes]]: pass + + @abstractmethod + def get_attachment_uri(self, attachment_id: str, view_id: str = "original") -> str: + pass diff --git a/libraries/microsoft-agents-hosting-core/microsoft_agents/hosting/core/connector/client/connector_client.py b/libraries/microsoft-agents-hosting-core/microsoft_agents/hosting/core/connector/client/connector_client.py index 484077553..825da3bc5 100644 --- a/libraries/microsoft-agents-hosting-core/microsoft_agents/hosting/core/connector/client/connector_client.py +++ b/libraries/microsoft-agents-hosting-core/microsoft_agents/hosting/core/connector/client/connector_client.py @@ -8,6 +8,7 @@ from typing import Any, Optional from aiohttp import ClientSession from io import BytesIO +from urllib.parse import quote from microsoft_agents.activity import ( Activity, @@ -146,6 +147,37 @@ async def get_attachment(self, attachment_id: str, view_id: str) -> BytesIO: data = await response.read() return BytesIO(data) + def get_attachment_uri(self, attachment_id: str, view_id: str = "original") -> str: + """ + Gets the URI of an attachment view. + + :param attachment_id: The ID of the attachment. + :param view_id: The ID of the view, defaults to "original". + :return: The URI of the attachment view. + """ + if not attachment_id: + logger.error( + "AttachmentsOperations.get_attachment_uri(): attachmentId is required", + stack_info=True, + ) + raise ValueError("attachmentId is required") + + if not view_id: + logger.error( + "AttachmentsOperations.get_attachment_uri(): viewId is required", + stack_info=True, + ) + raise ValueError("viewId is required") + + base_url = str(self._client._base_url) + if not base_url.endswith("/"): + base_url += "/" + + return ( + f"{base_url}v3/attachments/{quote(attachment_id, safe='')}" + f"/views/{quote(view_id, safe='')}" + ) + class ConversationsOperations(ConversationsBase, _BaseClient): diff --git a/libraries/microsoft-agents-hosting-core/microsoft_agents/hosting/core/connector/mcs/mcs_connector_client.py b/libraries/microsoft-agents-hosting-core/microsoft_agents/hosting/core/connector/mcs/mcs_connector_client.py index 95ddd4d54..707bc5c29 100644 --- a/libraries/microsoft-agents-hosting-core/microsoft_agents/hosting/core/connector/mcs/mcs_connector_client.py +++ b/libraries/microsoft-agents-hosting-core/microsoft_agents/hosting/core/connector/mcs/mcs_connector_client.py @@ -194,6 +194,14 @@ async def get_attachment(self, attachment_id: str, *args, **kwargs) -> bytes: "GetAttachment is not supported for Microsoft Copilot Studio Connector" ) + def get_attachment_uri( + self, attachment_id: str, view_id: str = "original", **kwargs + ) -> str: + """Not supported for MCS Connector.""" + raise NotImplementedError( + "GetAttachmentUri is not supported for Microsoft Copilot Studio Connector" + ) + class MCSConnectorClient(ConnectorClientBase): """ diff --git a/test_samples/handling_attachments/README.md b/test_samples/handling_attachments/README.md new file mode 100644 index 000000000..8c9b4a867 --- /dev/null +++ b/test_samples/handling_attachments/README.md @@ -0,0 +1,39 @@ +# Handling Attachments + +This sample agent demonstrates how to send and receive attachments: + +- **Inline attachments** - an image embedded directly in the activity as a base64 data URI. +- **Internet attachments** - an image referenced by an external HTTP(S) URL. +- **Uploaded attachments** (Microsoft Teams only) - an image uploaded to the channel via the connector client and referenced by its attachment URI. +- **Incoming attachments** - files sent by the user are automatically downloaded (via `AttachmentDownloader` and `M365AttachmentDownloader`) and echoed back as an inline attachment. + +## Setup + +1. **Install dependencies:** + ```bash + pip install -r requirements.txt + ``` + +2. **Install Microsoft Agents libraries** (from the root of the repository): + ```bash + pip install -e libraries/microsoft-agents-activity + pip install -e libraries/microsoft-agents-hosting-core + pip install -e libraries/microsoft-agents-authentication-msal + pip install -e libraries/microsoft-agents-hosting-fastapi + ``` + +3. **Configure environment variables:** + - Copy `env.TEMPLATE` to `.env` + - Fill in the required configuration values (`CLIENTID`, `CLIENTSECRET`, `TENANTID`) + +## Running the sample + +Run from the `handling_attachments` sample directory (not from `src/`), so that the `resources/` folder used by the "Inline Attachment" and "Upload Attachment" options can be resolved relative to the current working directory: + +```bash +python -m src.main +``` + +The agent will start on `http://localhost:3978` by default. You can change the port by setting the `PORT` environment variable. + +Connect to the agent with the [M365 Agents Playground](https://github.com/OfficeDev/microsoft-365-agents-toolkit) or Microsoft Teams and select one of the options presented by the agent. diff --git a/test_samples/handling_attachments/env.TEMPLATE b/test_samples/handling_attachments/env.TEMPLATE new file mode 100644 index 000000000..df82361bd --- /dev/null +++ b/test_samples/handling_attachments/env.TEMPLATE @@ -0,0 +1,5 @@ +CONNECTIONS__SERVICE_CONNECTION__SETTINGS__CLIENTID= +CONNECTIONS__SERVICE_CONNECTION__SETTINGS__CLIENTSECRET= +CONNECTIONS__SERVICE_CONNECTION__SETTINGS__TENANTID= + +LOGGING__LOGLEVEL__microsoft_agents.hosting.core=INFO \ No newline at end of file diff --git a/test_samples/handling_attachments/requirements.txt b/test_samples/handling_attachments/requirements.txt new file mode 100644 index 000000000..0fc6c7849 --- /dev/null +++ b/test_samples/handling_attachments/requirements.txt @@ -0,0 +1,5 @@ +uvicorn +python-dotenv +microsoft-agents-hosting-core +microsoft-agents-hosting-fastapi +microsoft-agents-authentication-msal \ No newline at end of file diff --git a/test_samples/handling_attachments/resources/agents-sdk.png b/test_samples/handling_attachments/resources/agents-sdk.png new file mode 100644 index 000000000..363d24ee0 Binary files /dev/null and b/test_samples/handling_attachments/resources/agents-sdk.png differ diff --git a/test_samples/handling_attachments/resources/build-agents.png b/test_samples/handling_attachments/resources/build-agents.png new file mode 100644 index 000000000..f0e860053 Binary files /dev/null and b/test_samples/handling_attachments/resources/build-agents.png differ diff --git a/test_samples/handling_attachments/resources/introducing-agents-sdk.png b/test_samples/handling_attachments/resources/introducing-agents-sdk.png new file mode 100644 index 000000000..9e6e46f80 Binary files /dev/null and b/test_samples/handling_attachments/resources/introducing-agents-sdk.png differ diff --git a/test_samples/handling_attachments/src/__init__.py b/test_samples/handling_attachments/src/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/test_samples/handling_attachments/src/agent.py b/test_samples/handling_attachments/src/agent.py new file mode 100644 index 000000000..5c1c549ee --- /dev/null +++ b/test_samples/handling_attachments/src/agent.py @@ -0,0 +1,198 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +from dotenv import load_dotenv + +from pathlib import Path + +import os +import base64 + +from microsoft_agents.activity import ( + Activity, + ActionTypes, + Attachment, + AttachmentData, + CardAction, + Channels, + HeroCard, +) + +from microsoft_agents.hosting.core import ( + Authorization, + AgentApplication, + TurnState, + TurnContext, + MemoryStorage, + MessageFactory, + ConnectorClientBase, +) +from microsoft_agents.hosting.core.app.attachment_downloader import ( + AttachmentDownloader, +) +from microsoft_agents.hosting.core.app.m365_attachment_downloader import ( + M365AttachmentDownloader, +) +from microsoft_agents.activity import load_configuration_from_env +from microsoft_agents.authentication.msal import MsalConnectionManager +from microsoft_agents.hosting.fastapi import CloudAdapter + + +# Create the agent application + +load_dotenv() + +agents_sdk_config = load_configuration_from_env(os.environ) + +STORAGE = MemoryStorage() +CONNECTION_MANAGER = MsalConnectionManager(**agents_sdk_config) +ADAPTER = CloudAdapter(connection_manager=CONNECTION_MANAGER) +AUTHORIZATION = Authorization(STORAGE, CONNECTION_MANAGER, **agents_sdk_config) + +AGENT_APP = AgentApplication[TurnState]( + storage=STORAGE, + adapter=ADAPTER, + authorization=AUTHORIZATION, + file_downloaders=[ + AttachmentDownloader(), + M365AttachmentDownloader(connections=CONNECTION_MANAGER), + ], + **agents_sdk_config, +) + + +# Agent handlers +async def _help(context: TurnContext, _state: TurnState): + for member in context.activity.members_added: + if member.id != context.activity.recipient.id: + await context.send_activity( + "Welcome to the HandlingAttachments Agent. " + + "This agent will introduce you to attachments. " + + "Please select an option." + ) + await display_options(context) + +AGENT_APP.conversation_update("membersAdded")(_help) +AGENT_APP.message("/help")(_help) + + +@AGENT_APP.activity("message") +async def on_message(context: TurnContext, state: TurnState): + reply = await process_input(context, state) + if reply is not None: + await context.send_activity(reply) + await display_options(context) + +async def display_options(context: TurnContext) -> None: + card = HeroCard( + text="You can upload an image or select one of the following choices", + buttons=[ + CardAction( + type=ActionTypes.im_back, + title="1. Inline Attachment", + value="1" + ), + CardAction( + type=ActionTypes.im_back, + title="2. Internet Attachment", + value="2" + ) + ] + ) + + if context.activity.channel_id == Channels.ms_teams: + card.buttons.append( + CardAction( + type=ActionTypes.im_back, + title="3. Upload Attachment", + value="3" + ) + ) + + reply = MessageFactory.attachment(card.to_attachment()) + await context.send_activity(reply) + +async def process_input(context: TurnContext, state: TurnState) -> Activity | None: + + reply: Activity | None = None + + if state.temp.input_files: + reply = MessageFactory.text(f"There are {len(state.temp.input_files)} attachments.") + image_data = base64.b64encode(state.temp.input_files[0].content).decode( + "utf-8" + ) + reply.attachments = [ + Attachment( + name=state.temp.input_files[0].filename, + content_type="image/png", + content_url=f"data:image/png;base64,{image_data}" + ) + ] + else: + reply = await handle_outgoing_attachment(context, context.activity) + + return reply + +async def handle_outgoing_attachment(context: TurnContext, activity: Activity) -> Activity | None: + if not activity.text: + return None + + reply: Activity | None = None + + if activity.text.startswith("1"): + reply = MessageFactory.text("This is an inline attachment.") + reply.attachments = [get_inline_attachment()] + elif activity.text.startswith("2"): + reply = MessageFactory.text("This is an attachment from an HTTP URL.") + reply.attachments = [get_internet_attachment()] + elif activity.text.startswith("3"): + reply = MessageFactory.text("This is an uploaded attachment.") + uploaded_attachment = await upload_attachment(context, activity.service_url, activity.conversation.id) + reply.attachments = [uploaded_attachment] + return reply + +def get_inline_attachment() -> Attachment: + image_path = Path(os.getcwd()) / "resources" / "build-agents.png" + image_data = base64.b64encode(image_path.read_bytes()).decode("utf-8") + + return Attachment( + name="resources\\build-agents.png", + content_type="image/png", + content_url=f"data:image/png;base64,{image_data}" + ) + +async def upload_attachment(context: TurnContext, service_url: str, conversation_id: str) -> Attachment: + if not service_url: + raise ValueError("Service URL is required.") + if not conversation_id: + raise ValueError("Conversation ID is required.") + + image_path = Path(os.getcwd()) / "resources" / "agents-sdk.png" + + connector = context.services.get(ConnectorClientBase) + if not connector: + raise RuntimeError("Connector client is required.") + + response = await connector.conversations.upload_attachment( + conversation_id, + AttachmentData( + name="resources\\agents-sdk.png", + type="image/png", + original_base64=image_path.read_bytes() + ) + ) + + attachment_uri = connector.attachments.get_attachment_uri(response.id) + + return Attachment( + name="resources\\agents-sdk.png", + content_type="image/png", + content_url=attachment_uri, + ) + +def get_internet_attachment() -> Attachment: + return Attachment( + name="resources\\introducing-agents-sdk.png", + content_type="image/png", + content_url="https://devblogs.microsoft.com/microsoft365dev/wp-content/uploads/sites/73/2024/11/word-image-23435-1.png" + ) \ No newline at end of file diff --git a/test_samples/handling_attachments/src/main.py b/test_samples/handling_attachments/src/main.py new file mode 100644 index 000000000..16f5d95b6 --- /dev/null +++ b/test_samples/handling_attachments/src/main.py @@ -0,0 +1,35 @@ +import os + +import uvicorn +from fastapi import FastAPI, Request + +from microsoft_agents.hosting.fastapi import ( + start_agent_process, + jwt_authorization_decorator, +) + +from .agent import AGENT_APP, CONNECTION_MANAGER + + +if __name__ == "__main__": + + app = FastAPI(title="Empty Agent Sample", version="1.0.0") + app.state.agent_configuration = ( + CONNECTION_MANAGER.get_default_connection_configuration() + ) + + @app.post("/api/messages") + @jwt_authorization_decorator + async def messages_handler( + request: Request, + ): + """Main endpoint for processing bot messages.""" + + return await start_agent_process( + request, + AGENT_APP, + AGENT_APP.adapter, + ) + + port = int(os.environ.get("PORT", 3978)) + uvicorn.run(app, host="127.0.0.1", port=port) diff --git a/tests/hosting_core/app/test_attachment_downloader.py b/tests/hosting_core/app/test_attachment_downloader.py new file mode 100644 index 000000000..b06e6fbf6 --- /dev/null +++ b/tests/hosting_core/app/test_attachment_downloader.py @@ -0,0 +1,216 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +import json +from unittest.mock import MagicMock + +import pytest + +from microsoft_agents.activity import ( + Activity, + Attachment, + ChannelAccount, + ConversationAccount, +) +from microsoft_agents.hosting.core import TurnContext +from microsoft_agents.hosting.core.app.attachment_downloader import AttachmentDownloader +from microsoft_agents.hosting.core.outbound_host_validator import OutboundHostValidator + + +def _make_context( + channel_id: str = "test", attachments: list[Attachment] | None = None +) -> TurnContext: + kwargs = {} + if attachments is not None: + kwargs["attachments"] = attachments + activity = Activity( + type="message", + id="1234", + channel_id=channel_id, + from_property=ChannelAccount(id="user", name="User Name"), + recipient=ChannelAccount(id="bot", name="Bot Name"), + conversation=ConversationAccount(id="convo", name="Convo Name"), + service_url="https://example.org", + **kwargs, + ) + return TurnContext(MagicMock(), activity) + + +class _FakeResponse: + def __init__(self, status: int, content: bytes, content_type: str): + self.status = status + self._content = content + self.headers = {"Content-Type": content_type} + + async def read(self) -> bytes: + return self._content + + async def __aenter__(self) -> "_FakeResponse": + return self + + async def __aexit__(self, *args) -> bool: + return False + + +class _FakeSession: + def __init__(self, response: _FakeResponse): + self._response = response + self.requested_urls: list[str] = [] + + def get(self, url: str, **kwargs) -> _FakeResponse: + self.requested_urls.append(url) + return self._response + + async def __aenter__(self) -> "_FakeSession": + return self + + async def __aexit__(self, *args) -> bool: + return False + + +class TestAttachmentDownloaderChannelAndEmptyCases: + @pytest.mark.asyncio + async def test_returns_empty_list_for_teams_channel(self): + downloader = AttachmentDownloader() + context = _make_context( + channel_id="msteams", + attachments=[ + Attachment( + content_type="image/png", content_url="https://example.org/a.png" + ) + ], + ) + + assert await downloader.download_files(context) == [] + + @pytest.mark.asyncio + async def test_returns_empty_list_when_no_attachments(self): + downloader = AttachmentDownloader() + context = _make_context(channel_id="test", attachments=None) + + assert await downloader.download_files(context) == [] + + +class TestAttachmentDownloaderInlineContent: + @pytest.mark.asyncio + async def test_downloads_inline_content_as_json_bytes(self): + downloader = AttachmentDownloader() + attachment = Attachment( + content_type="application/vnd.custom", + content={"foo": "bar"}, + name="data.json", + ) + context = _make_context(attachments=[attachment]) + + files = await downloader.download_files(context) + + assert len(files) == 1 + assert files[0].content == bytes(json.dumps({"foo": "bar"}), "utf-8") + assert files[0].content_type == "application/vnd.custom" + assert files[0].filename == "data.json" + + +class TestAttachmentDownloaderRemoteContent: + @pytest.mark.asyncio + async def test_downloads_remote_file_and_returns_input_file(self): + response = _FakeResponse( + status=200, content=b"file-bytes", content_type="text/plain" + ) + session = _FakeSession(response) + downloader = AttachmentDownloader(client_factory=lambda: session) + attachment = Attachment( + content_type="text/plain", + content_url="https://example.org/file.txt", + name="file.txt", + ) + context = _make_context(attachments=[attachment]) + + files = await downloader.download_files(context) + + assert len(files) == 1 + assert files[0].content == b"file-bytes" + assert files[0].content_type == "text/plain" + assert files[0].content_url == "https://example.org/file.txt" + assert files[0].filename == "file.txt" + assert session.requested_urls == ["https://example.org/file.txt"] + + @pytest.mark.asyncio + async def test_allows_http_localhost_urls(self): + response = _FakeResponse( + status=200, content=b"local-bytes", content_type="text/plain" + ) + session = _FakeSession(response) + downloader = AttachmentDownloader(client_factory=lambda: session) + attachment = Attachment( + content_type="text/plain", content_url="http://localhost:3000/file.txt" + ) + context = _make_context(attachments=[attachment]) + + files = await downloader.download_files(context) + + assert len(files) == 1 + + @pytest.mark.asyncio + async def test_normalizes_image_content_type_to_png(self): + response = _FakeResponse( + status=200, content=b"\x89PNG", content_type="image/jpeg" + ) + session = _FakeSession(response) + downloader = AttachmentDownloader(client_factory=lambda: session) + attachment = Attachment( + content_type="image/jpeg", content_url="https://example.org/pic.jpg" + ) + context = _make_context(attachments=[attachment]) + + files = await downloader.download_files(context) + + assert files[0].content_type == "image/png" + + @pytest.mark.asyncio + async def test_returns_none_for_non_success_status(self): + response = _FakeResponse(status=404, content=b"", content_type="text/plain") + session = _FakeSession(response) + downloader = AttachmentDownloader(client_factory=lambda: session) + attachment = Attachment( + content_type="text/plain", content_url="https://example.org/missing.txt" + ) + context = _make_context(attachments=[attachment]) + + assert await downloader.download_files(context) == [] + + @pytest.mark.asyncio + async def test_skips_disallowed_hosts_when_host_validator_enabled(self): + response = _FakeResponse( + status=200, content=b"file-bytes", content_type="text/plain" + ) + session = _FakeSession(response) + host_validator = OutboundHostValidator(enabled=True, hosts=["contoso.com"]) + downloader = AttachmentDownloader( + client_factory=lambda: session, host_validator=host_validator + ) + attachment = Attachment( + content_type="text/plain", content_url="https://evil.example.com/file.txt" + ) + context = _make_context(attachments=[attachment]) + + assert await downloader.download_files(context) == [] + assert session.requested_urls == [] + + @pytest.mark.asyncio + async def test_allows_permitted_hosts_when_host_validator_enabled(self): + response = _FakeResponse( + status=200, content=b"file-bytes", content_type="text/plain" + ) + session = _FakeSession(response) + host_validator = OutboundHostValidator(enabled=True, hosts=["contoso.com"]) + downloader = AttachmentDownloader( + client_factory=lambda: session, host_validator=host_validator + ) + attachment = Attachment( + content_type="text/plain", content_url="https://contoso.com/file.txt" + ) + context = _make_context(attachments=[attachment]) + + files = await downloader.download_files(context) + + assert len(files) == 1 diff --git a/tests/hosting_core/app/test_m365_attachment_downloader.py b/tests/hosting_core/app/test_m365_attachment_downloader.py new file mode 100644 index 000000000..f9b3396f3 --- /dev/null +++ b/tests/hosting_core/app/test_m365_attachment_downloader.py @@ -0,0 +1,389 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +import json +from unittest.mock import MagicMock + +import pytest + +from microsoft_agents.activity import ( + Activity, + Attachment, + ChannelAccount, + ConversationAccount, +) +from microsoft_agents.hosting.core import ( + AccessTokenProviderBase, + ClaimsIdentity, + Connections, + TurnContext, +) +from microsoft_agents.hosting.core.app.m365_attachment_downloader import ( + M365AttachmentDownloader, +) +from microsoft_agents.hosting.core.outbound_host_validator import OutboundHostValidator + + +class _FakeTokenProvider(AccessTokenProviderBase): + def __init__(self, name: str): + self.name = name + self.requested: tuple[str, list[str]] | None = None + + @property + def configuration(self): + return None + + async def get_access_token( + self, resource_url: str, scopes: list[str], force_refresh: bool = False + ) -> str: + self.requested = (resource_url, scopes) + return f"{self.name}-token" + + async def acquire_token_on_behalf_of( + self, scopes: list[str], user_assertion: str + ) -> str: + return f"{self.name}-obo-token" + + +class _FakeConnections(Connections): + def __init__(self, provider: AccessTokenProviderBase | None): + self._provider = provider + + def get_connection(self, connection_name: str) -> AccessTokenProviderBase: + if self._provider is None: + raise ValueError("no connection configured") + return self._provider + + def get_default_connection(self) -> AccessTokenProviderBase: + return self._provider + + def get_token_provider( + self, claims_identity, service_url + ) -> AccessTokenProviderBase: + return self._provider + + def get_token_provider_from_activity( + self, claims_identity, activity + ) -> AccessTokenProviderBase: + return self._provider + + def get_default_connection_configuration(self): + return None + + +def _make_claims_identity() -> ClaimsIdentity: + return ClaimsIdentity(claims={"aud": "test-audience"}, authentication_type="test") + + +def _make_context( + channel_id: str = "msteams", + attachments: list[Attachment] | None = None, + identity: ClaimsIdentity | None = None, +) -> TurnContext: + kwargs = {} + if attachments is not None: + kwargs["attachments"] = attachments + activity = Activity( + type="message", + id="1234", + channel_id=channel_id, + from_property=ChannelAccount(id="user", name="User Name"), + recipient=ChannelAccount(id="bot", name="Bot Name"), + conversation=ConversationAccount(id="convo", name="Convo Name"), + service_url="https://example.org", + **kwargs, + ) + return TurnContext( + MagicMock(), + activity, + identity=identity if identity is not None else _make_claims_identity(), + ) + + +class _FakeResponse: + def __init__(self, status: int, content: bytes, content_type: str): + self.status = status + self._content = content + self.headers = {"Content-Type": content_type} + self.request_headers: dict | None = None + + async def read(self) -> bytes: + return self._content + + async def __aenter__(self) -> "_FakeResponse": + return self + + async def __aexit__(self, *args) -> bool: + return False + + +class _FakeSession: + def __init__(self, response: _FakeResponse): + self._response = response + self.requested_urls: list[str] = [] + + def get(self, url: str, headers=None, **kwargs) -> _FakeResponse: + self.requested_urls.append(url) + self._response.request_headers = headers + return self._response + + async def __aenter__(self) -> "_FakeSession": + return self + + async def __aexit__(self, *args) -> bool: + return False + + +class TestM365AttachmentDownloaderChannelAndValidation: + @pytest.mark.asyncio + async def test_raises_when_context_has_no_identity(self): + downloader = M365AttachmentDownloader(connections=_FakeConnections(None)) + activity = Activity( + type="message", + id="1234", + channel_id="msteams", + from_property=ChannelAccount(id="user", name="User Name"), + recipient=ChannelAccount(id="bot", name="Bot Name"), + conversation=ConversationAccount(id="convo", name="Convo Name"), + service_url="https://example.org", + ) + context = TurnContext(MagicMock(), activity, identity=None) + + with pytest.raises(ValueError): + await downloader.download_files(context) + + @pytest.mark.asyncio + async def test_returns_empty_list_for_non_teams_channel(self): + downloader = M365AttachmentDownloader( + connections=_FakeConnections(_FakeTokenProvider("p")) + ) + context = _make_context( + channel_id="webchat", + attachments=[ + Attachment( + content_type="image/png", content_url="https://example.org/a.png" + ) + ], + ) + + assert await downloader.download_files(context) == [] + + @pytest.mark.asyncio + async def test_allows_m365_copilot_channel(self): + response = _FakeResponse( + status=200, content=b"bytes", content_type="text/plain" + ) + session = _FakeSession(response) + downloader = M365AttachmentDownloader( + connections=_FakeConnections(_FakeTokenProvider("p")), + client_factory=lambda: session, + ) + context = _make_context( + channel_id="msteams:COPILOT", + attachments=[ + Attachment( + content_type="text/plain", content_url="https://example.org/a.txt" + ) + ], + ) + + files = await downloader.download_files(context) + + assert len(files) == 1 + + @pytest.mark.asyncio + async def test_returns_empty_list_when_no_attachments(self): + downloader = M365AttachmentDownloader( + connections=_FakeConnections(_FakeTokenProvider("p")) + ) + context = _make_context(attachments=None) + + assert await downloader.download_files(context) == [] + + @pytest.mark.asyncio + async def test_filters_out_html_attachments(self): + downloader = M365AttachmentDownloader( + connections=_FakeConnections(_FakeTokenProvider("p")) + ) + context = _make_context( + attachments=[Attachment(content_type="text/html", content="

hi

")] + ) + + assert await downloader.download_files(context) == [] + + +class TestM365AttachmentDownloaderInlineContent: + @pytest.mark.asyncio + async def test_downloads_inline_content_as_json_bytes(self): + downloader = M365AttachmentDownloader( + connections=_FakeConnections(_FakeTokenProvider("p")) + ) + attachment = Attachment( + content_type="application/vnd.custom", + content={"foo": "bar"}, + name="data.json", + ) + context = _make_context(attachments=[attachment]) + + files = await downloader.download_files(context) + + assert len(files) == 1 + assert files[0].content == bytes(json.dumps({"foo": "bar"}), "utf-8") + assert files[0].filename == "data.json" + + +class TestM365AttachmentDownloaderRemoteContent: + @pytest.mark.asyncio + async def test_downloads_remote_file_using_download_url_from_content(self): + response = _FakeResponse( + status=200, content=b"file-bytes", content_type="text/plain" + ) + session = _FakeSession(response) + token_provider = _FakeTokenProvider("p") + downloader = M365AttachmentDownloader( + connections=_FakeConnections(token_provider), client_factory=lambda: session + ) + attachment = Attachment( + content_type="text/plain", + content_url="https://example.org/file.txt", + content={"downloadUrl": "https://example.org/real-download"}, + name="file.txt", + ) + context = _make_context(attachments=[attachment]) + + files = await downloader.download_files(context) + + assert len(files) == 1 + assert files[0].content == b"file-bytes" + assert files[0].content_url == "https://example.org/file.txt" + assert session.requested_urls == ["https://example.org/real-download"] + + @pytest.mark.asyncio + async def test_falls_back_to_content_url_when_download_url_missing(self): + response = _FakeResponse( + status=200, content=b"file-bytes", content_type="text/plain" + ) + session = _FakeSession(response) + downloader = M365AttachmentDownloader( + connections=_FakeConnections(_FakeTokenProvider("p")), + client_factory=lambda: session, + ) + attachment = Attachment( + content_type="text/plain", + content_url="https://example.org/file.txt", + content={"someOtherKey": "value"}, + ) + context = _make_context(attachments=[attachment]) + + files = await downloader.download_files(context) + + assert len(files) == 1 + assert session.requested_urls == ["https://example.org/file.txt"] + + @pytest.mark.asyncio + async def test_normalizes_image_content_type_to_png(self): + response = _FakeResponse( + status=200, content=b"\x89PNG", content_type="image/jpeg" + ) + session = _FakeSession(response) + downloader = M365AttachmentDownloader( + connections=_FakeConnections(_FakeTokenProvider("p")), + client_factory=lambda: session, + ) + attachment = Attachment( + content_type="image/jpeg", content_url="https://example.org/pic.jpg" + ) + context = _make_context(attachments=[attachment]) + + files = await downloader.download_files(context) + + assert files[0].content_type == "image/png" + + @pytest.mark.asyncio + async def test_returns_none_entry_for_failed_status(self): + response = _FakeResponse(status=404, content=b"", content_type="text/plain") + session = _FakeSession(response) + downloader = M365AttachmentDownloader( + connections=_FakeConnections(_FakeTokenProvider("p")), + client_factory=lambda: session, + ) + attachment = Attachment( + content_type="text/plain", content_url="https://example.org/missing.txt" + ) + context = _make_context(attachments=[attachment]) + + assert await downloader.download_files(context) == [] + + @pytest.mark.asyncio + async def test_skips_disallowed_hosts_when_host_validator_enabled(self): + response = _FakeResponse( + status=200, content=b"file-bytes", content_type="text/plain" + ) + session = _FakeSession(response) + host_validator = OutboundHostValidator(enabled=True, hosts=["contoso.com"]) + downloader = M365AttachmentDownloader( + connections=_FakeConnections(_FakeTokenProvider("p")), + client_factory=lambda: session, + host_validator=host_validator, + ) + attachment = Attachment( + content_type="text/plain", + content_url="https://example.org/file.txt", + content={"downloadUrl": "https://evil.example.com/relay"}, + ) + context = _make_context(attachments=[attachment]) + + assert await downloader.download_files(context) == [] + assert session.requested_urls == [] + + @pytest.mark.asyncio + async def test_uses_anonymous_mode_without_requesting_token(self): + response = _FakeResponse( + status=200, content=b"file-bytes", content_type="text/plain" + ) + session = _FakeSession(response) + token_provider = _FakeTokenProvider("p") + downloader = M365AttachmentDownloader( + connections=_FakeConnections(token_provider), + client_factory=lambda: session, + use_anonymous=True, + ) + attachment = Attachment( + content_type="text/plain", content_url="https://example.org/file.txt" + ) + context = _make_context(attachments=[attachment]) + + files = await downloader.download_files(context) + + assert len(files) == 1 + assert token_provider.requested is None + + @pytest.mark.asyncio + async def test_uses_named_token_provider_when_configured(self): + named_provider = _FakeTokenProvider("named") + downloader = M365AttachmentDownloader( + connections=_FakeConnections(named_provider), + client_factory=lambda: _FakeSession( + _FakeResponse(status=200, content=b"bytes", content_type="text/plain") + ), + token_provider_name="named-connection", + ) + attachment = Attachment( + content_type="text/plain", content_url="https://example.org/file.txt" + ) + context = _make_context(attachments=[attachment]) + + await downloader.download_files(context) + + assert named_provider.requested is not None + + @pytest.mark.asyncio + async def test_raises_when_no_token_provider_can_be_resolved(self): + downloader = M365AttachmentDownloader(connections=_FakeConnections(None)) + attachment = Attachment( + content_type="text/plain", content_url="https://example.org/file.txt" + ) + context = _make_context(attachments=[attachment]) + + with pytest.raises(RuntimeError): + await downloader.download_files(context) diff --git a/tests/hosting_core/authorization/test_claims_identity.py b/tests/hosting_core/authorization/test_claims_identity.py index c6a1715f0..356f0225b 100644 --- a/tests/hosting_core/authorization/test_claims_identity.py +++ b/tests/hosting_core/authorization/test_claims_identity.py @@ -67,3 +67,36 @@ def test_get_claim_value_returns_matching_claim(): assert identity.get_claim_value("aud") == "app-id" assert identity.get_claim_value("missing") is None + + +@pytest.mark.parametrize( + ("claims", "expected"), + [ + ( + { + "ver": "1.0", + "aud": "target-app-id", + "appid": "calling-app-id", + }, + "api://calling-app-id", + ), + ( + { + "ver": "2.0", + "aud": "target-app-id", + "azp": "calling-app-id", + }, + "api://calling-app-id", + ), + ( + {"aud": "HTTPS://API.BOTFRAMEWORK.US"}, + "https://api.botframework.us", + ), + ({}, "https://api.botframework.com"), + ({"aud": "app-id"}, "https://api.botframework.com"), + ], +) +def test_get_outgoing_audience_claim(claims, expected): + identity = ClaimsIdentity(claims=claims) + + assert identity.get_outgoing_audience_claim() == expected diff --git a/tests/hosting_core/connector/test_connector_client.py b/tests/hosting_core/connector/test_connector_client.py index 013b3d59d..4ae22e199 100644 --- a/tests/hosting_core/connector/test_connector_client.py +++ b/tests/hosting_core/connector/test_connector_client.py @@ -439,6 +439,50 @@ async def content_handler(request): assert info.views == [{"viewId": "original"}] assert content.read() == b"attachment bytes" + @pytest.mark.asyncio + async def test_get_attachment_uri_builds_view_url(self): + client = ConnectorClient("https://example.org/", token="") + try: + uri = client.attachments.get_attachment_uri("attachment-1") + uri_with_view = client.attachments.get_attachment_uri( + "attachment-1", "thumbnail" + ) + finally: + await client.close() + + assert uri == "https://example.org/v3/attachments/attachment-1/views/original" + assert ( + uri_with_view + == "https://example.org/v3/attachments/attachment-1/views/thumbnail" + ) + + @pytest.mark.asyncio + async def test_get_attachment_uri_escapes_special_characters(self): + client = ConnectorClient("https://example.org/", token="") + try: + uri = client.attachments.get_attachment_uri( + "id with space/and#hash?q=1", "a view/with#special?chars" + ) + finally: + await client.close() + + assert ( + uri == "https://example.org/v3/attachments/" + "id%20with%20space%2Fand%23hash%3Fq%3D1" + "/views/a%20view%2Fwith%23special%3Fchars" + ) + + @pytest.mark.asyncio + async def test_get_attachment_uri_requires_attachment_id_and_view_id(self): + client = ConnectorClient("https://example.org/", token="") + try: + with pytest.raises(ValueError): + client.attachments.get_attachment_uri(None) + with pytest.raises(ValueError): + client.attachments.get_attachment_uri("attachment-1", None) + finally: + await client.close() + @pytest.mark.asyncio @pytest.mark.parametrize("status", [302, 400, 500]) async def test_unexpected_response_status_raises_client_response_error(