diff --git a/client/android/api-reference.mdx b/api-reference/client/android/api-reference.mdx similarity index 100% rename from client/android/api-reference.mdx rename to api-reference/client/android/api-reference.mdx diff --git a/client/android/introduction.mdx b/api-reference/client/android/overview.mdx similarity index 86% rename from client/android/introduction.mdx rename to api-reference/client/android/overview.mdx index 9a2e76df..353c15f7 100644 --- a/client/android/introduction.mdx +++ b/api-reference/client/android/overview.mdx @@ -1,5 +1,6 @@ --- -title: "SDK Introduction" +title: "Android SDK Overview" +sidebarTitle: "Overview" description: "Build Android applications with Pipecat's Kotlin client library" --- @@ -48,13 +49,13 @@ client.startBotAndConnect(startBotParams).withCallback { ## Documentation - + SDK API documentation Daily, Gemini, OpenAI WebRTC, and SmallWebRTC transports diff --git a/client/android/transports/daily.mdx b/api-reference/client/android/transports/daily.mdx similarity index 100% rename from client/android/transports/daily.mdx rename to api-reference/client/android/transports/daily.mdx diff --git a/client/android/transports/gemini-websocket.mdx b/api-reference/client/android/transports/gemini-websocket.mdx similarity index 100% rename from client/android/transports/gemini-websocket.mdx rename to api-reference/client/android/transports/gemini-websocket.mdx diff --git a/client/android/transports/openai-webrtc.mdx b/api-reference/client/android/transports/openai-webrtc.mdx similarity index 100% rename from client/android/transports/openai-webrtc.mdx rename to api-reference/client/android/transports/openai-webrtc.mdx diff --git a/client/android/transports/small-webrtc.mdx b/api-reference/client/android/transports/small-webrtc.mdx similarity index 100% rename from client/android/transports/small-webrtc.mdx rename to api-reference/client/android/transports/small-webrtc.mdx diff --git a/client/c++/introduction.mdx b/api-reference/client/cpp/overview.mdx similarity index 93% rename from client/c++/introduction.mdx rename to api-reference/client/cpp/overview.mdx index 693cafa5..97ed6c25 100644 --- a/client/c++/introduction.mdx +++ b/api-reference/client/cpp/overview.mdx @@ -1,5 +1,6 @@ --- -title: "SDK Introduction" +title: "C++ SDK Overview" +sidebarTitle: "Overview" description: "Build native applications with Pipecat’s C++ client library" --- @@ -99,7 +100,7 @@ ninja -C build Complete SDK API documentation - + WebRTC implementation using Daily diff --git a/client/c++/transport.mdx b/api-reference/client/cpp/transport.mdx similarity index 100% rename from client/c++/transport.mdx rename to api-reference/client/cpp/transport.mdx diff --git a/client/ios/api-reference.mdx b/api-reference/client/ios/api-reference.mdx similarity index 100% rename from client/ios/api-reference.mdx rename to api-reference/client/ios/api-reference.mdx diff --git a/client/ios/introduction.mdx b/api-reference/client/ios/overview.mdx similarity index 91% rename from client/ios/introduction.mdx rename to api-reference/client/ios/overview.mdx index 2f7f7196..885f42d6 100644 --- a/client/ios/introduction.mdx +++ b/api-reference/client/ios/overview.mdx @@ -1,5 +1,6 @@ --- -title: "SDK Introduction" +title: "iOS SDK Overview" +sidebarTitle: "Overview" description: "Build iOS applications with Pipecat’s Swift client library" --- @@ -62,13 +63,13 @@ self.pipecatClientIOS?.startBotAndConnect(startBotParams: startBotParams) { (res ## Documentation - + SDK API documentation Daily, Gemini, OpenAI WebRTC, and SmallWebRTC transports diff --git a/client/ios/transports/daily.mdx b/api-reference/client/ios/transports/daily.mdx similarity index 100% rename from client/ios/transports/daily.mdx rename to api-reference/client/ios/transports/daily.mdx diff --git a/client/ios/transports/gemini-websocket.mdx b/api-reference/client/ios/transports/gemini-websocket.mdx similarity index 100% rename from client/ios/transports/gemini-websocket.mdx rename to api-reference/client/ios/transports/gemini-websocket.mdx diff --git a/client/ios/transports/openai-webrtc.mdx b/api-reference/client/ios/transports/openai-webrtc.mdx similarity index 100% rename from client/ios/transports/openai-webrtc.mdx rename to api-reference/client/ios/transports/openai-webrtc.mdx diff --git a/client/ios/transports/small-webrtc.mdx b/api-reference/client/ios/transports/small-webrtc.mdx similarity index 100% rename from client/ios/transports/small-webrtc.mdx rename to api-reference/client/ios/transports/small-webrtc.mdx diff --git a/client/js/api-reference/callbacks.mdx b/api-reference/client/js/callbacks.mdx similarity index 97% rename from client/js/api-reference/callbacks.mdx rename to api-reference/client/js/callbacks.mdx index 1863d563..27e24856 100644 --- a/client/js/api-reference/callbacks.mdx +++ b/api-reference/client/js/callbacks.mdx @@ -51,7 +51,7 @@ pcClient.on(RTVIEvent.BotReady, () => console.log("Bot ready via event")); Provides a `TransportState` string representing the connectivity state of the - local client. See [transports](../transports/transport) for state explanation. + local client. See [transports](./transports/transport) for state explanation. @@ -77,15 +77,20 @@ pcClient.on(RTVIEvent.BotReady, () => console.log("Bot ready via event")); Bot disconnected from the transport. This may occur due to session expiry, a - pipeline error or for any reason the server deems the session over. + pipeline error or for any reason the server deems the session over. By + default, the client will also disconnect when this fires. Set + `disconnectOnBotDisconnect: false` in the constructor to keep the client + connected. - A non-bot participant joined the session. + A participant joined the session. Fires for all participants, including the + bot and the local user. - A non-bot participant left the session. Note: excluded local participant. + A participant left the session. Fires for all participants, including the bot + and the local user. ### Messages and errors @@ -288,10 +293,10 @@ Callback receives a `TranscriptData` object: Search capabilities are currently only supported by Google Gemini. To take advantage of this event, your pipeline must include a - [`GoogleLLMService`](/api-reference/server/services/llm/google) and your pipeline task - should include the - [`GoogleRTVIObserver`](/api-reference/server/rtvi/google-rtvi-observer) in lieu - of the typical `RTVIObserver`. + [`GoogleLLMService`](/api-reference/server/services/llm/google) and your + pipeline task should include the + [`GoogleRTVIObserver`](/api-reference/server/rtvi/google-rtvi-observer) in + lieu of the typical `RTVIObserver`. @@ -370,7 +375,7 @@ Callback receives a `TranscriptData` object: A function call is in progress. This replaces the deprecated `onLLMFunctionCall` callback and is the event that triggers registered - [`FunctionCallHandler`s](/client/js/api-reference/client-methods#registerfunctioncallhandler) + [`FunctionCallHandler`s](/api-reference/client/js/client-methods#registerfunctioncallhandler) when a `function_name` is present. diff --git a/client/js/api-reference/client-constructor.mdx b/api-reference/client/js/client-constructor.mdx similarity index 85% rename from client/js/api-reference/client-constructor.mdx rename to api-reference/client/js/client-constructor.mdx index 47b78cf8..e3757973 100644 --- a/client/js/api-reference/client-constructor.mdx +++ b/api-reference/client/js/client-constructor.mdx @@ -12,9 +12,9 @@ import { PipecatClient } from "@pipecat-ai/client-js"; When initializing the `PipecatClient`, you must provide a transport instance to the constructor for your chosen protocol or provider. See - [Transport](/client/js/transports) for more information. For the purpose of + [Transport](./transports/transport) for more information. For the purpose of this guide, we'll demonstrate using the [Daily WebRTC - transport](/client/js/transports/daily). + transport](/api-reference/client/js/transports/daily). ## Example @@ -101,3 +101,12 @@ const pcClient = new PipecatClient({ Enable user's local screen share. Note: Not all transports support screen sharing. Setting this value in that case will have no effect. + +### Session behavior + + + When `true` (default), the client will automatically disconnect when the bot + disconnects. Set to `false` to keep the client transport connected after the + bot leaves — useful if you want to handle reconnection or show a waiting state + without tearing down the transport. Introduced in client-js 1.7.0. + diff --git a/client/js/api-reference/client-methods.mdx b/api-reference/client/js/client-methods.mdx similarity index 98% rename from client/js/api-reference/client-methods.mdx rename to api-reference/client/js/client-methods.mdx index 26269741..8ba1e39f 100644 --- a/client/js/api-reference/client-methods.mdx +++ b/api-reference/client/js/client-methods.mdx @@ -84,9 +84,9 @@ During the connection process, the transport state will transition through the f Calling `connect()` asynchronously will resolve when the bot and client signal - that they are ready. See [messages and events](./messages). If you want to - call `connect()` without `await`, you can use the `onBotReady` callback or - `BotReady` event to know when you can interact with the bot. + that they are ready. If you want to call `connect()` without `await`, you can use the + `onBotReady` callback or `BotReady` event to know when you can interact with + the bot. @@ -143,7 +143,7 @@ await pcClient.disconnectBot(); Custom messaging between the client and the bot. This is useful for sending data to the bot, triggering specific actions, reacting to server events, or querying the server. -For more, see: [messages and events](./messages). +For more, see the [Custom Messaging](/client/guides/custom-messaging) guide. ### sendClientMessage() diff --git a/client/js/api-reference/errors.mdx b/api-reference/client/js/errors.mdx similarity index 100% rename from client/js/api-reference/errors.mdx rename to api-reference/client/js/errors.mdx diff --git a/client/js/introduction.mdx b/api-reference/client/js/overview.mdx similarity index 88% rename from client/js/introduction.mdx rename to api-reference/client/js/overview.mdx index 271f1f37..c4a817d0 100644 --- a/client/js/introduction.mdx +++ b/api-reference/client/js/overview.mdx @@ -1,5 +1,6 @@ --- -title: "SDK Introduction" +title: "JavaScript SDK Overview" +sidebarTitle: "Overview" description: "Build web applications with Pipecat’s JavaScript client library" --- @@ -59,28 +60,28 @@ pcClient.connect({ Configure your client instance with transport and callbacks Core methods for interacting with your bot Handle bot events, messages, and state changes Daily, SmallWebRTC, WebSocket, and other transports diff --git a/client/js/transports/daily.mdx b/api-reference/client/js/transports/daily.mdx similarity index 97% rename from client/js/transports/daily.mdx rename to api-reference/client/js/transports/daily.mdx index f63c5b21..6c5ac425 100644 --- a/client/js/transports/daily.mdx +++ b/api-reference/client/js/transports/daily.mdx @@ -64,7 +64,7 @@ The `DailyTransportConstructorOptions` extends the `DailyFactoryOptions` type th ### TransportConnectionParams -On `connect()`, the `DailyTransport` optionally takes a set of [`DailyCallOptions`](https://docs.daily.co/reference/daily-js/daily-call-client/methods#dailycalloptions) to connect to a Daily room. This can be provided directly to the `PipecatClient`'s `connect()` method or via a starting endpoint passed to the `PipecatClient`'s `startBotAndConnect()` method. If using an endpoint, your endpoint should return a JSON object matching the `DailyCallOptions` type. See the [client connect()](/client/js/api-reference/client-methods#connect) documentation for more information. +On `connect()`, the `DailyTransport` optionally takes a set of [`DailyCallOptions`](https://docs.daily.co/reference/daily-js/daily-call-client/methods#dailycalloptions) to connect to a Daily room. This can be provided directly to the `PipecatClient`'s `connect()` method or via a starting endpoint passed to the `PipecatClient`'s `startBotAndConnect()` method. If using an endpoint, your endpoint should return a JSON object matching the `DailyCallOptions` type. See the [client connect()](/api-reference/client/js/client-methods#connect) documentation for more information. ```typescript client @@ -119,7 +119,7 @@ For most operations, you will not interact with the transport directly. Most met ## Events -The transport implements the various [`PipecatClient` event handlers](/client/js/api-reference/callbacks). For Daily-specific events, you can attach listeners to the underlying Daily call client. For a list of available events, see the [Daily API Reference](https://docs.daily.co/reference/daily-js/events). +The transport implements the various [`PipecatClient` event handlers](/api-reference/client/js/callbacks). For Daily-specific events, you can attach listeners to the underlying Daily call client. For a list of available events, see the [Daily API Reference](https://docs.daily.co/reference/daily-js/events). ```typescript pcClient.transport.dailyCallClient.on('recording-started', (ev) => {...}); diff --git a/client/js/transports/gemini.mdx b/api-reference/client/js/transports/gemini.mdx similarity index 98% rename from client/js/transports/gemini.mdx rename to api-reference/client/js/transports/gemini.mdx index 5652a246..3522eed1 100644 --- a/client/js/transports/gemini.mdx +++ b/api-reference/client/js/transports/gemini.mdx @@ -93,7 +93,7 @@ The `GeminiLiveWebsocketTransport` does not take connection parameters. It conne ### Events -The GeminiLiveWebSocketTransport implements the various [PipecatClient event handlers](/client/js/api-reference/callbacks). Check out the docs or samples for more info. +The GeminiLiveWebSocketTransport implements the various [PipecatClient event handlers](/api-reference/client/js/callbacks). Check out the docs or samples for more info. ## More Information diff --git a/client/js/transports/openai-webrtc.mdx b/api-reference/client/js/transports/openai-webrtc.mdx similarity index 89% rename from client/js/transports/openai-webrtc.mdx rename to api-reference/client/js/transports/openai-webrtc.mdx index 35e6ca10..aaa4e0d9 100644 --- a/client/js/transports/openai-webrtc.mdx +++ b/api-reference/client/js/transports/openai-webrtc.mdx @@ -4,7 +4,7 @@ title: "OpenAIRealTimeWebRTCTransport" ## Overview -The `OpenAIRealTimeWebRTCTransport` is a fully functional [Pipecat `Transport`](/client/js/transports/transport). It provides a framework for implementing real-time communication directly with the [OpenAI Realtime API using WebRTC](https://platform.openai.com/docs/guides/realtime-webrtc) voice-to-voice service. It handles media device management, audio/video streams, and state management for the connection. +The `OpenAIRealTimeWebRTCTransport` is a fully functional [Pipecat `Transport`](/api-reference/client/js/transports/transport). It provides a framework for implementing real-time communication directly with the [OpenAI Realtime API using WebRTC](https://platform.openai.com/docs/guides/realtime-webrtc) voice-to-voice service. It handles media device management, audio/video streams, and state management for the connection. Transports of this type are designed primarily for development and testing @@ -106,7 +106,7 @@ The `OpenAIRealTimeWebRTCTransport` does not take connection parameters. It conn ### Events -The transport implements the various [`PipecatClient` event handlers](/client/js/api-reference/callbacks). Check out the docs or samples for more info. +The transport implements the various [`PipecatClient` event handlers](/api-reference/client/js/callbacks). Check out the docs or samples for more info. ## More Information diff --git a/client/js/transports/small-webrtc.mdx b/api-reference/client/js/transports/small-webrtc.mdx similarity index 99% rename from client/js/transports/small-webrtc.mdx rename to api-reference/client/js/transports/small-webrtc.mdx index 0a1aa6f3..3d20a703 100644 --- a/client/js/transports/small-webrtc.mdx +++ b/api-reference/client/js/transports/small-webrtc.mdx @@ -162,7 +162,7 @@ transport.setVideoCodec("VP8"); ## Events -The transport implements the various [`PipecatClient` event handlers](/client/js/api-reference/callbacks). +The transport implements the various [`PipecatClient` event handlers](/api-reference/client/js/callbacks). ## Connection Process diff --git a/client/js/transports/transport.mdx b/api-reference/client/js/transports/transport.mdx similarity index 100% rename from client/js/transports/transport.mdx rename to api-reference/client/js/transports/transport.mdx diff --git a/client/js/transports/websocket.mdx b/api-reference/client/js/transports/websocket.mdx similarity index 99% rename from client/js/transports/websocket.mdx rename to api-reference/client/js/transports/websocket.mdx index 0ab87a45..fe350385 100644 --- a/client/js/transports/websocket.mdx +++ b/api-reference/client/js/transports/websocket.mdx @@ -200,7 +200,7 @@ transport.handleUserAudioStream(chunk.data); ## Events -The transport implements the various [`PipecatClient` event handlers](/client/js/api-reference/callbacks). +The transport implements the various [`PipecatClient` event handlers](/api-reference/client/js/callbacks). ## Reconnection Handling diff --git a/client/react-native/api-reference.mdx b/api-reference/client/react-native/api-reference.mdx similarity index 86% rename from client/react-native/api-reference.mdx rename to api-reference/client/react-native/api-reference.mdx index 4b0ffce2..18963ecb 100644 --- a/client/react-native/api-reference.mdx +++ b/api-reference/client/react-native/api-reference.mdx @@ -5,7 +5,7 @@ description: "API reference for the Pipecat React Native SDK" The Pipecat React Native SDK leverages the Pipecat JavaScript SDK for seamless integration with React Native applications. - For detailed information, please reference to the [Javascript SDK docs](/client/js/api-reference/client-constructor). + For detailed information, please reference to the [Javascript SDK docs](/api-reference/client/js/client-constructor). **Just ensure you use the appropriate transport layer for React Native.** diff --git a/client/react-native/introduction.mdx b/api-reference/client/react-native/overview.mdx similarity index 89% rename from client/react-native/introduction.mdx rename to api-reference/client/react-native/overview.mdx index 362ace50..79830be7 100644 --- a/client/react-native/introduction.mdx +++ b/api-reference/client/react-native/overview.mdx @@ -1,9 +1,10 @@ --- -title: "SDK Introduction" +title: "React Native SDK Overview" +sidebarTitle: "Overview" description: "Build React Native applications with Pipecat's React Native client library" --- -The Pipecat React Native SDK leverages the [Pipecat JavaScript SDK](/client/js/introduction) and its `PipecatClient` to provide seamless integration for React Native applications. +The Pipecat React Native SDK leverages the [Pipecat JavaScript SDK](/api-reference/client/js/overview) and its `PipecatClient` to provide seamless integration for React Native applications. Since the JavaScript SDK is designed to work across both web and React Native platforms, the core functionalities remain the same: - Device and media stream management @@ -94,28 +95,28 @@ The Pipecat React Native SDK leverages the Pipecat JavaScript SDK for seamless i React Native-specific API documentation Daily and SmallWebRTC transports for React Native Configure your client instance with transport and callbacks Core methods for interacting with your bot diff --git a/client/react-native/transports/daily.mdx b/api-reference/client/react-native/transports/daily.mdx similarity index 97% rename from client/react-native/transports/daily.mdx rename to api-reference/client/react-native/transports/daily.mdx index c7073dc5..a522fa8c 100644 --- a/client/react-native/transports/daily.mdx +++ b/api-reference/client/react-native/transports/daily.mdx @@ -39,7 +39,7 @@ The `DailyTransportConstructorOptions` extends the `DailyFactoryOptions` type th The Pipecat React Native SDK leverages the Pipecat JavaScript SDK for seamless integration with React Native applications. - For detailed information, please reference to the [Javascript SDK docs](/client/js/transports/daily). + For detailed information, please reference to the [Javascript SDK docs](/api-reference/client/js/transports/daily). **Just ensure you use the appropriate transport layer for React Native.** diff --git a/client/react-native/transports/small-webrtc.mdx b/api-reference/client/react-native/transports/small-webrtc.mdx similarity index 97% rename from client/react-native/transports/small-webrtc.mdx rename to api-reference/client/react-native/transports/small-webrtc.mdx index 01ab399c..193d2bcb 100644 --- a/client/react-native/transports/small-webrtc.mdx +++ b/api-reference/client/react-native/transports/small-webrtc.mdx @@ -41,7 +41,7 @@ await client?.startBotAndConnect(connectParams); The Pipecat React Native SDK leverages the Pipecat JavaScript SDK for seamless integration with React Native applications. - For detailed information, please reference to the [Javascript SDK docs](/client/js/transports/small-webrtc). + For detailed information, please reference to the [Javascript SDK docs](/api-reference/client/js/transports/small-webrtc). **Just ensure you use the appropriate transport layer for React Native.** diff --git a/client/react/components.mdx b/api-reference/client/react/components.mdx similarity index 97% rename from client/react/components.mdx rename to api-reference/client/react/components.mdx index aacc2679..f1b18ae7 100644 --- a/client/react/components.mdx +++ b/api-reference/client/react/components.mdx @@ -7,7 +7,7 @@ The Pipecat React SDK provides several components for handling audio, video, and ## PipecatClientProvider -The root component for providing Pipecat client context to your application. It also includes built-in conversation state management, so any descendant component can use the [`usePipecatConversation`](/client/react/hooks#usepipecatconversation) hook to access messages without adding a separate provider. +The root component for providing Pipecat client context to your application. It also includes built-in conversation state management, so any descendant component can use the [`usePipecatConversation`](/api-reference/client/react/hooks#usepipecatconversation) hook to access messages without adding a separate provider. ```jsx diff --git a/client/react/hooks.mdx b/api-reference/client/react/hooks.mdx similarity index 100% rename from client/react/hooks.mdx rename to api-reference/client/react/hooks.mdx diff --git a/client/react/introduction.mdx b/api-reference/client/react/overview.mdx similarity index 85% rename from client/react/introduction.mdx rename to api-reference/client/react/overview.mdx index aa67a588..cbbc5664 100644 --- a/client/react/introduction.mdx +++ b/api-reference/client/react/overview.mdx @@ -1,5 +1,6 @@ --- -title: "SDK Introduction" +title: "React SDK Overview" +sidebarTitle: "Overview" description: "Build React applications with Pipecat's React client library" --- @@ -88,12 +89,12 @@ function VoiceBot() { ## Explore the SDK - + Ready-to-use components for audio, video, and visualization - + React hooks for accessing client functionality -The Pipecat React SDK builds on top of the [JavaScript SDK](/client/js/introduction) to provide an idiomatic React interface while maintaining compatibility with the RTVI standard. +The Pipecat React SDK builds on top of the [JavaScript SDK](/api-reference/client/js/overview) to provide an idiomatic React interface while maintaining compatibility with the RTVI standard. diff --git a/api-reference/server/rtvi/rtvi-processor.mdx b/api-reference/server/rtvi/rtvi-processor.mdx index 998c34d1..2d951ab1 100644 --- a/api-reference/server/rtvi/rtvi-processor.mdx +++ b/api-reference/server/rtvi/rtvi-processor.mdx @@ -213,7 +213,7 @@ await processor.interrupt_bot() ## Custom Messaging -Server messages let you push unsolicited data from the server to the client at any time — notifications, status updates, real-time results, etc. They are distinct from **server responses**, which reply to a specific client request (see [Requesting Information from the Server](/client/js/api-reference/messages#requesting-information-from-the-server)). +Server messages let you push unsolicited data from the server to the client at any time — notifications, status updates, real-time results, etc. They are distinct from **server responses**, which reply to a specific client request (see [Requesting Information from the Server](/client/guides/custom-messaging#requesting-information-from-the-server)). ### Sending server messages @@ -250,4 +250,4 @@ pcClient.onServerMessage((message) => { }); ``` -See [Handling Custom Messages from the Server](/client/js/api-reference/messages#handling-custom-messages-from-the-server) for more details and examples. +See [Handling Custom Messages from the Server](/client/guides/custom-messaging#handling-custom-messages-from-the-server) for more details and examples. diff --git a/api-reference/server/services/transport/daily.mdx b/api-reference/server/services/transport/daily.mdx index 47b181ba..ad2f6658 100644 --- a/api-reference/server/services/transport/daily.mdx +++ b/api-reference/server/services/transport/daily.mdx @@ -663,4 +663,4 @@ async def on_dtmf_event(transport, data): - [Events Overview](/api-reference/server/events/overview) - Overview of all events in Pipecat - [Daily REST Helper Utility](/api-reference/server/utilities/daily/rest-helper) - [Pipecat Development Runner's Transport Utilities](/api-reference/server/utilities/runner/transport-utils) -- [Client SDK Integration](/client/js/transports/daily) +- [Client SDK Integration](/api-reference/client/js/transports/daily) diff --git a/api-reference/server/services/transport/lemonslice.mdx b/api-reference/server/services/transport/lemonslice.mdx index 333c2464..a9f18d11 100644 --- a/api-reference/server/services/transport/lemonslice.mdx +++ b/api-reference/server/services/transport/lemonslice.mdx @@ -13,7 +13,7 @@ When used, the Pipecat bot connects to a Daily room alongside the LemonSlice ava `LemonSliceTransport` always uses Daily as the underlying WebRTC transport. There is no standalone WebRTC variant. The avatar joins a Daily room, either one you supply via `daily_room_url` or one LemonSlice provisions for you. -**Embedding in a custom UI:** Daily Prebuilt is not required. Use [`@pipecat-ai/client-js`](/client/js/introduction) or [`@pipecat-ai/client-react`](/client/react/introduction) to connect to the session and render the avatar participant's video track inside your own modal or component. The Pipecat bot, the LemonSlice avatar, and your custom UI all connect to the same Daily room. +**Embedding in a custom UI:** Daily Prebuilt is not required. Use [`@pipecat-ai/client-js`](/api-reference/client/js/overview) or [`@pipecat-ai/client-react`](/api-reference/client/react/overview) to connect to the session and render the avatar participant's video track inside your own modal or component. The Pipecat bot, the LemonSlice avatar, and your custom UI all connect to the same Daily room. **Combining with other LLM services (e.g. Bedrock AgentCore):** the avatar transport is independent of your LLM choice. Any LLM service that fits in a Pipecat pipeline can drive a `LemonSliceTransport` bot. diff --git a/api-reference/server/services/transport/small-webrtc.mdx b/api-reference/server/services/transport/small-webrtc.mdx index 6f29fded..67a3c07e 100644 --- a/api-reference/server/services/transport/small-webrtc.mdx +++ b/api-reference/server/services/transport/small-webrtc.mdx @@ -188,4 +188,4 @@ async def on_app_message(transport, message, sender): - [Events Overview](/api-reference/server/events/overview) - Overview of all events in Pipecat - [Pipecat Development Runner's Transport Utilities](/api-reference/server/utilities/runner/transport-utils) -- [Client SDK Integration](/client/js/transports/small-webrtc) +- [Client SDK Integration](/api-reference/client/js/transports/small-webrtc) diff --git a/api-reference/server/services/transport/tavus.mdx b/api-reference/server/services/transport/tavus.mdx index 1393f03f..50300e11 100644 --- a/api-reference/server/services/transport/tavus.mdx +++ b/api-reference/server/services/transport/tavus.mdx @@ -34,7 +34,7 @@ When used, the Pipecat bot connects to a Daily room alongside the Tavus avatar a Connect using Pipecat Client SDK with Daily transport @@ -246,4 +246,4 @@ async def on_client_disconnected(transport, participant): ## Additional Resources - [Events Overview](/api-reference/server/events/overview) - Overview of all events in Pipecat -- [Client SDK Integration](/client/js/transports/daily) +- [Client SDK Integration](/api-reference/client/js/transports/daily) diff --git a/client/concepts/choosing-a-transport.mdx b/client/concepts/choosing-a-transport.mdx new file mode 100644 index 00000000..d3e18fd0 --- /dev/null +++ b/client/concepts/choosing-a-transport.mdx @@ -0,0 +1,162 @@ +--- +title: "Choosing a Transport" +description: "How to pick the right transport for your Pipecat client application." +--- + +A transport is the connection layer between your client and your Pipecat bot. It handles device management, the underlying connection, media streaming, and session state. You choose one transport when creating your `PipecatClient` and it stays for the lifetime of that client instance. + +The choice of transport has one important constraint: **the client transport and server transport must be a matching pair**. If your client uses `DailyTransport`, your Pipecat pipeline must use the server-side `DailyTransport`. See the [server transport docs](/pipecat/learn/transports) for the server side of this picture. + +## Available transports + + + + Serverless peer-to-peer WebRTC. No third-party account needed. + + + WebRTC via Daily's global infrastructure. Recommended for production. + + + Direct WebSocket connection. For server-to-server setups only. + + + Direct connection to Google's Gemini Live API. No Pipecat server needed. + + + Direct connection to OpenAI's Realtime API. No Pipecat server needed. + + + +## WebRTC vs WebSocket + +For any client-to-server voice application, **WebRTC is the right choice**. WebSocket may look simpler, but it's built on TCP, which makes it a poor fit for real-time audio: + +- **Head-of-line blocking**: TCP retransmits lost packets and holds up everything behind them. For audio, a dropped packet is better discarded than waited for — a brief gap sounds far better than a stutter. +- **Opus codec coupling**: Opus's forward error correction and packet loss concealment are designed to work with UDP's delivery model. On TCP, that machinery either doesn't help or actively hurts. +- **No automatic timestamping**: WebRTC handles RTP timestamping and jitter buffering automatically. WebSocket leaves that to you. +- **No built-in echo cancellation or noise suppression**: Browser echo cancellation (AEC) is wired into the WebRTC stack. It's not available to arbitrary WebSocket streams. +- **Reconnection complexity**: WebRTC handles ICE restarts and network changes. WebSocket reconnection on mobile, sleep/wake cycles, or network switches requires you to rebuild that logic yourself. + +WebSocket is appropriate for **server-to-server** communication (where both sides are on stable, controlled networks) or for **text-only** bots with no audio. + +## Serverless WebRTC vs WebRTC cloud + +Once you've settled on WebRTC, the next question is how you route it. There are two approaches: + +**Serverless WebRTC (SmallWebRTC)** establishes a direct peer-to-peer connection between the browser and your Pipecat server. There's no relay infrastructure — the media takes the most direct path. This works well when latency is already low (local dev, same-region deployments) and keeps your stack simple. If you're self-hosting Pipecat, this is the recommended approach. + +**WebRTC cloud (Daily)** routes media through a global network of Points of Presence (PoPs). Instead of a single direct hop, your client connects to the nearest PoP, and Daily's mesh routing finds the best path from there to your bot. Daily's network spans ~75 PoPs with a P50 first-hop latency of ~13ms. For users on degraded networks, distant from your server, or on mobile, the reliability and audio quality advantages are significant. + +A useful way to think about it: if you're self-hosting, use SmallWebRTC — it's simpler than running your own WebRTC infrastructure and avoids a third-party dependency. If you want managed infrastructure that handles global routing, audio processing, and scaling for you, use Daily. + + + For a deeper look at this tradeoff, see the Daily blog post [You don't need a WebRTC server for your voice agents](https://www.daily.co/blog/you-dont-need-a-webrtc-server-for-your-voice-agents/). + + +--- + +## How to choose + +### SmallWebRTC — self-hosted and local development + +SmallWebRTC is the default in all Pipecat quickstart templates. No account needed, no third-party services — just a direct peer-to-peer WebRTC connection between your client and your bot. + +**Use it when:** +- Developing or testing locally +- Running a self-hosted deployment +- Building embedded or edge deployments where simplicity matters + +**Avoid it when:** +- Your users are geographically distributed +- You need built-in echo cancellation, noise reduction, or network resilience at scale +- You're scaling beyond a handful of concurrent sessions + +```tsx +import { SmallWebRTCTransport } from "@pipecat-ai/small-webrtc-transport"; + +const client = new PipecatClient({ + transport: new SmallWebRTCTransport(), + enableMic: true, +}); + +await client.connect({ webrtcUrl: "http://localhost:7860/api/offer" }); +``` + +--- + +### Daily — production web and mobile apps + +Daily provides a global WebRTC network with mesh routing, built-in audio processing (echo cancellation, noise suppression, automatic gain control), and resilience to network changes. It's the recommended choice for anything user-facing in production. + +**Use it when:** +- Shipping a production app to real users +- Your users are on a variety of networks, devices, or locations +- You want managed infrastructure without operating it yourself +- You're deploying to Pipecat Cloud (Daily is included) + +```tsx +import { DailyTransport } from "@pipecat-ai/daily-transport"; + +const client = new PipecatClient({ + transport: new DailyTransport(), + enableMic: true, +}); + +// startBotAndConnect calls your server endpoint, which creates a Daily room +// and returns the connection credentials +await client.startBotAndConnect({ endpoint: "/api/start" }); +``` + +--- + +### WebSocket — server-to-server and text-only + +The WebSocket transport connects to a WebSocket server rather than using WebRTC. It's appropriate for controlled, server-to-server scenarios where both sides are on stable networks, or for text-only bots with no audio requirements. + +**Use it when:** +- Building text-only interactions (no audio) +- Connecting two servers (not a browser client) +- Network constraints make WebRTC impractical in a specific environment + +**Do not use it for browser-to-server voice interactions.** See [WebRTC vs WebSocket](#webrtc-vs-websocket) above. + +--- + +### Gemini Live and OpenAI WebRTC — direct API connections + +These transports connect your client directly to Google's Gemini Live API or OpenAI's Realtime API, bypassing a Pipecat server entirely. They're useful for prototyping or demos when you want speech-to-speech interactions without running a backend. + + + Direct API transports expose your API key in the client. They are appropriate for development and demos, but **not for production** — use a server-side Pipecat pipeline with `DailyTransport` or `SmallWebRTCTransport` to keep API keys secure. + + +--- + +## Swapping transports + +Transports are interchangeable — the rest of your application code stays the same. The only thing that changes is the import and constructor: + +```tsx +// Development / self-hosted +import { SmallWebRTCTransport } from "@pipecat-ai/small-webrtc-transport"; +const transport = new SmallWebRTCTransport(); + +// Production / managed +import { DailyTransport } from "@pipecat-ai/daily-transport"; +const transport = new DailyTransport(); + +// Everything else is identical +const client = new PipecatClient({ transport, enableMic: true }); +``` + +The Pipecat CLI scaffolds a `config.ts` that selects the transport based on an environment variable, so you can run SmallWebRTC locally and Daily in production without changing your app code. + +## Summary + +| Transport | Best for | Requires | +|---|---|---| +| SmallWebRTC | Local dev, self-hosted | Nothing | +| Daily | Production apps, global users | Daily account | +| WebSocket | Text-only, server-to-server | Custom server | +| Gemini Live | Gemini prototypes | Gemini API key | +| OpenAI WebRTC | OpenAI prototypes | OpenAI API key | diff --git a/client/concepts/events-and-callbacks.mdx b/client/concepts/events-and-callbacks.mdx new file mode 100644 index 00000000..c1db900f --- /dev/null +++ b/client/concepts/events-and-callbacks.mdx @@ -0,0 +1,287 @@ +--- +title: "Events & Callbacks" +description: "How to respond to bot and session events in Pipecat client applications." +--- + + + +You are currently viewing the React version of this page. Use the dropdown to the right to customize this page for your client framework. + + + + +You are currently viewing the JavaScript version of this page. Use the dropdown to the right to customize this page for your client framework. + + + +The Pipecat client emits events throughout the session lifecycle — when the bot connects, when the user speaks, when a transcript arrives, and more. + +## Subscribing to events + +### Callbacks + +Pass handlers in the `PipecatClient` constructor. Good for events you always want to handle, defined once at setup. This works the same in all frameworks: + +```tsx +const client = new PipecatClient({ + transport: new DailyTransport(), + callbacks: { + onBotReady: () => console.log("Bot is ready"), + onUserTranscript: (data) => console.log("User said:", data.text), + }, +}); +``` + +### Event listeners + + + +In React, use the `useRTVIClientEvent` hook to subscribe within a component. It handles registration and cleanup automatically when the component mounts and unmounts: + +```tsx +import { RTVIEvent } from "@pipecat-ai/client-js"; +import { useRTVIClientEvent } from "@pipecat-ai/client-react"; + +function TranscriptDisplay() { + useRTVIClientEvent( + RTVIEvent.UserTranscript, + useCallback((data) => { + if (data.final) setTranscript(data.text); + }, []) + ); +} +``` + +You can also use `.on()` directly on the client instance, but `useRTVIClientEvent` is preferred in React since it avoids stale closure issues and cleans up automatically. + + + + + +Add handlers with `.on()` at any point — useful for dynamic subscriptions or when you want to add and remove handlers at runtime: + +```tsx +client.on(RTVIEvent.BotReady, () => console.log("Bot is ready")); +client.on(RTVIEvent.UserTranscript, (data) => console.log("User said:", data.text)); +``` + +Callbacks and event listeners are equivalent — use whichever pattern fits your architecture. + + + +--- + +## Event reference + +### Session and connectivity + +These events track the connection state of the client and bot. See [Session Lifecycle](/client/concepts/session-lifecycle) for the full state progression. + +| Event | Callback | When it fires | +|---|---|---| +| `Connected` | `onConnected` | Client transport connection established | +| `Disconnected` | `onDisconnected` | Client disconnected (intentional or error) | +| `TransportStateChanged` | `onTransportStateChanged` | Any transport state change; receives the new `TransportState` string | +| `BotConnected` | `onBotConnected` | Bot joined the transport; pipeline may still be initializing | +| `BotReady` | `onBotReady` | Bot pipeline is ready; safe to send messages and expect audio | +| `BotDisconnected` | `onBotDisconnected` | Bot left the session; client will also disconnect unless `disconnectOnBotDisconnect: false` | +| `ParticipantConnected` | `onParticipantJoined` | Any participant joined (bot, local, or other) | +| `ParticipantLeft` | `onParticipantLeft` | Any participant left (bot, local, or other) | + + +`BotReady` receives a `BotReadyData` object with a `version` field — the RTVI version the bot is running. You can use this to check compatibility if your client and server may be on different versions. + + +### Voice activity + +These events are driven by the bot's VAD (voice activity detection) model. VAD is smarter than tracking raw audio levels — it understands turn-taking, so it can distinguish between a user who has finished speaking and one who has simply paused or is speaking slowly. + +| Event | Callback | When it fires | +|---|---|---| +| `UserStartedSpeaking` | `onUserStartedSpeaking` | VAD detected the user started speaking | +| `UserStoppedSpeaking` | `onUserStoppedSpeaking` | VAD detected the user stopped speaking | +| `BotStartedSpeaking` | `onBotStartedSpeaking` | Bot started sending audio | +| `BotStoppedSpeaking` | `onBotStoppedSpeaking` | Bot stopped sending audio | +| `LocalAudioLevel` | `onLocalAudioLevel` | Local audio gain level (0–1); fires continuously | +| `RemoteAudioLevel` | `onRemoteAudioLevel` | Remote audio gain level (0–1); fires continuously | +| `UserMuteStarted` | `onUserMuteStarted` | Server started ignoring client audio (server-side mute) | +| `UserMuteStopped` | `onUserMuteStopped` | Server resumed processing client audio | + + +`UserMuteStarted`/`UserMuteStopped` reflect server-side muting — the client continues sending audio, but the bot is ignoring it. Use these to update your UI (e.g., show a muted indicator) without actually stopping the local mic. + + +### Transcription and bot output + +| Event | Callback | Data | When it fires | +|---|---|---|---| +| `UserTranscript` | `onUserTranscript` | `TranscriptData` | User speech transcribed; fires for both partial (`final: false`) and final results | +| `BotOutput` | `onBotOutput` | `BotOutputData` | Bot text output, typically aggregated by sentence or word during TTS synthesis | +| `BotLlmText` | `onBotLlmText` | `BotLLMTextData` | Raw LLM token stream | +| `BotLlmStarted` | `onBotLlmStarted` | — | LLM inference started | +| `BotLlmStopped` | `onBotLlmStopped` | — | LLM inference finished | +| `BotTtsText` | `onBotTtsText` | `BotTTSTextData` | Words from TTS as they are synthesized (streaming TTS only) | +| `BotTtsStarted` | `onBotTtsStarted` | — | TTS synthesis started | +| `BotTtsStopped` | `onBotTtsStopped` | — | TTS synthesis finished | + +`UserTranscript` fires continuously as speech is recognized. Check `data.final` to distinguish committed transcripts from work-in-progress partials: + + + +```tsx +useRTVIClientEvent( + RTVIEvent.UserTranscript, + useCallback((data) => { + if (data.final) { + addMessage(data.text); // committed + } else { + updatePartial(data.text); // still in progress + } + }, []) +); +``` + + + + + +```tsx +client.on(RTVIEvent.UserTranscript, (data) => { + if (data.final) { + addMessage(data.text); // committed + } else { + updatePartial(data.text); // still in progress + } +}); +``` + + + +`BotOutput` is the recommended way to display the bot's response text. It provides the best possible representation of what the bot is saying — supporting interruptions and unspoken responses. By default, Pipecat aggregates output by sentences and words (assuming your TTS supports streaming), but custom aggregation strategies are supported too - like breaking out code snippets or other structured content: + + + +```tsx +useRTVIClientEvent( + RTVIEvent.BotOutput, + useCallback((data) => { + if (data.aggregated_by === "sentence") { + appendSentence(data.text); + } + }, []) +); +``` + + + + + +```tsx +client.on(RTVIEvent.BotOutput, (data) => { + if (data.aggregated_by === "sentence") { + appendSentence(data.text); + } +}); +``` + + + +### Errors + +| Event | Callback | When it fires | +|---|---|---| +| `Error` | `onError` | Bot signalled an error; `data.fatal` is `true` if the session is unrecoverable | +| `MessageError` | `onMessageError` | A client message failed or got an error response | + +Always handle `Error`. If `data.fatal` is `true`, the bot has already disconnected — update your UI accordingly: + + + +```tsx +useRTVIClientEvent( + RTVIEvent.Error, + useCallback(({ data }) => { + if (data.fatal) { + showReconnectPrompt(data.message); + } else { + showToast(data.message); + } + }, []) +); +``` + + + + + +```tsx +client.on(RTVIEvent.Error, ({ data }) => { + if (data.fatal) { + showReconnectPrompt(data.message); + } else { + showToast(data.message); + } +}); +``` + + + +### Devices and tracks + +| Event | Callback | When it fires | +|---|---|---| +| `AvailableMicsUpdated` | `onAvailableMicsUpdated` | Mic list changed or `initDevices()` called | +| `AvailableCamsUpdated` | `onAvailableCamsUpdated` | Camera list changed or `initDevices()` called | +| `AvailableSpeakersUpdated` | `onAvailableSpeakersUpdated` | Speaker list changed or `initDevices()` called | +| `MicUpdated` | `onMicUpdated` | Active microphone changed | +| `CamUpdated` | `onCamUpdated` | Active camera changed | +| `SpeakerUpdated` | `onSpeakerUpdated` | Active speaker changed | +| `DeviceError` | `onDeviceError` | Mic, camera, or permission error | +| `TrackStarted` | `onTrackStarted` | A media track (audio or video) became playable | +| `TrackStopped` | `onTrackStopped` | A media track stopped | + +### Function calling + +These events fire when the bot's LLM makes a function call. If your bot uses function calling, you can use these to track the status of calls and display relevant UI (e.g., a loading spinner while the call is in progress). + +| Event | Callback | When it fires | +|---|---|---| +| `LLMFunctionCallStarted` | `onLLMFunctionCallStarted` | LLM initiated a function call | +| `LLMFunctionCallInProgress` | `onLLMFunctionCallInProgress` | Function call is executing; triggers registered `FunctionCallHandler`s | +| `LLMFunctionCallStopped` | `onLLMFunctionCallStopped` | Function call completed or was cancelled | + +### Other + +| Event | Callback | When it fires | +|---|---|---| +| `ServerMessage` | `onServerMessage` | Custom message sent from the bot to the client | +| `Metrics` | `onMetrics` | Pipeline performance metrics from Pipecat | + +For custom server\<-\>client messaging, see [Custom Messaging](/client/guides/custom-messaging). + +--- + +## API reference + + + + + + `useRTVIClientEvent` and other React-specific event utilities + + + Complete callback signatures, data types, and transport compatibility + + + + + + + + + + Complete callback signatures, data types, and transport compatibility + + + + diff --git a/client/concepts/media-management.mdx b/client/concepts/media-management.mdx new file mode 100644 index 00000000..4fadaa80 --- /dev/null +++ b/client/concepts/media-management.mdx @@ -0,0 +1,365 @@ +--- +title: "Media Management" +description: "Managing microphones, cameras, speakers, and media tracks in Pipecat client applications." +--- + + + +You are currently viewing the React version of this page. Use the dropdown to the right to customize this page for your client framework. + + + + +You are currently viewing the JavaScript version of this page. Use the dropdown to the right to customize this page for your client framework. + + + +The Pipecat client handles media at two levels: **local devices** (the user's mic, camera, and speakers) and **media tracks** (the live audio/video streams flowing between client and bot). This page covers how to work with both. + +## Bot audio output + + + +Drop [`PipecatClientAudio`](/api-reference/client/react/components#pipecatclientaudio) inside your [`PipecatClientProvider`](/api-reference/client/react/components#pipecatclientprovider) and it handles everything — it mounts a hidden ` + + + +You need to attach the bot's audio track to an ` + +--- + +## Microphone + +### Enabling and muting + +Set `enableMic: true` in the [constructor](/api-reference/client/js/client-constructor) to start the session with the mic active. This is the default: + +```tsx +const client = new PipecatClient({ + transport: new DailyTransport(), + enableMic: true, +}); +``` + + + +Use [`usePipecatClientMicControl`](/api-reference/client/react/hooks#usepipecatclientmiccontrol) to mute and unmute within a component: + +```tsx +import { usePipecatClientMicControl } from "@pipecat-ai/client-react"; + +function MicButton() { + const { enableMic, isMicEnabled } = usePipecatClientMicControl(); + + return ( + + ); +} +``` + +Or use the headless [`PipecatClientMicToggle`](/api-reference/client/react/components#pipecatclientmictoggle) component, which provides the same state via a render prop. + + + + + +Call [`enableMic()`](/api-reference/client/js/client-methods#enablemic-enable-boolean) to mute and unmute. Check the current state with [`isMicEnabled`](/api-reference/client/js/client-methods#ismicenabled): + +```ts +client.enableMic(false); // mute +client.enableMic(true); // unmute + +const muted = !client.isMicEnabled; +``` + + + +### Switching microphones + + + +[`usePipecatClientMediaDevices`](/api-reference/client/react/hooks#usepipecatclientmediadevices) gives you a reactive device list and setters: + +```tsx +import { usePipecatClientMediaDevices } from "@pipecat-ai/client-react"; + +function MicSelector() { + const { availableMics, selectedMic, updateMic } = usePipecatClientMediaDevices(); + + return ( + + ); +} +``` + + + + + +Enumerate available devices with [`getAllMics()`](/api-reference/client/js/client-methods#getallmics), then switch by `deviceId` using [`updateMic()`](/api-reference/client/js/client-methods#updatemic): + +```ts +const mics = await client.getAllMics(); +client.updateMic(mics[1].deviceId); +``` + +Listen for `AvailableMicsUpdated` to react when devices are plugged in or removed: + +```ts +client.on(RTVIEvent.AvailableMicsUpdated, (mics) => { + renderMicList(mics); +}); +``` + + + +--- + +## Camera + +Camera is disabled by default. Enable it in the [constructor](/api-reference/client/js/client-constructor): + +```tsx +const client = new PipecatClient({ + transport: new DailyTransport(), + enableCam: true, +}); +``` + + + Not all transports support video. `enableCam` has no effect on transports that + don't support it (e.g. WebSocket). + + + + +Use [`usePipecatClientCamControl`](/api-reference/client/react/hooks#usepipecatclientcamcontrol) to toggle the camera, and [`PipecatClientVideo`](/api-reference/client/react/components#pipecatclientvideo) to render the feed: + +```tsx +import { usePipecatClientCamControl } from "@pipecat-ai/client-react"; + +const { enableCam, isCamEnabled } = usePipecatClientCamControl(); +``` + +```tsx +import { PipecatClientVideo } from "@pipecat-ai/client-react"; + + +``` + +Use `participant="bot"` to render the bot's video feed instead. Switch cameras via `availableCams` / `updateCam` from [`usePipecatClientMediaDevices`](/api-reference/client/react/hooks#usepipecatclientmediadevices). + + + + + +Toggle the camera with [`enableCam()`](/api-reference/client/js/client-methods#enablecam-enable-boolean) and check state with [`isCamEnabled`](/api-reference/client/js/client-methods#iscamenabled): + +```ts +client.enableCam(true); +const isOn = client.isCamEnabled; +``` + +To render the local video feed, attach the track to a ` + +--- + +## Speakers + + + +Enumerate and switch output devices via [`usePipecatClientMediaDevices`](/api-reference/client/react/hooks#usepipecatclientmediadevices): + +```tsx +const { availableSpeakers, selectedSpeaker, updateSpeaker } = usePipecatClientMediaDevices(); +``` + + + + + +Enumerate and switch output devices with [`getAllSpeakers()`](/api-reference/client/js/client-methods#getallspeakers) / [`updateSpeaker()`](/api-reference/client/js/client-methods#updatespeaker): + +```ts +const speakers = await client.getAllSpeakers(); +client.updateSpeaker(speakers[0].deviceId); +``` + + + +--- + +## Device initialization before connecting + +By default, device access is requested when `connect()` is called. If you want to enumerate or test devices before the session starts — for example, to show a device picker pre-call — call [`initDevices()`](/api-reference/client/js/client-methods#initdevices) first: + +```tsx +await client.initDevices(); +// devices are now available; user hasn't connected yet +const mics = await client.getAllMics(); +``` + +--- + +## Media tracks + +For advanced use cases — custom rendering, audio processing, recording — you can access the raw `MediaStreamTrack` objects directly. + + + +Use [`usePipecatClientMediaTrack`](/api-reference/client/react/hooks#usepipecatclientmediatrack): + +```tsx +import { usePipecatClientMediaTrack } from "@pipecat-ai/client-react"; + +function CustomAudioViz() { + const botAudio = usePipecatClientMediaTrack("audio", "bot"); + + useEffect(() => { + if (!botAudio) return; + // attach to Web Audio API, recorder, etc. + }, [botAudio]); +} +``` + + + + + +Use [`tracks()`](/api-reference/client/js/client-methods#tracks) to access tracks synchronously once the session is ready: + +```ts +const { local, bot } = client.tracks(); +// local.audio, local.video, bot.audio, bot.video +``` + +Or listen for [`TrackStarted`](/api-reference/client/js/callbacks#media-events) to be notified as tracks become available: + +```ts +client.on(RTVIEvent.TrackStarted, (track, participant) => { + // attach to Web Audio API, recorder, etc. +}); +``` + + + +--- + +## Audio visualization + + + +The React SDK includes [`VoiceVisualizer`](/api-reference/client/react/components#voicevisualizer), a canvas-based component that renders audio level bars for any participant: + +```tsx +import { VoiceVisualizer } from "@pipecat-ai/client-react"; + + +``` + +Set `participantType="bot"` to visualize the bot's audio instead. + + + + + +Listen for `LocalAudioLevel` and `RemoteAudioLevel` events, which fire continuously with gain values between 0 and 1, and drive your own visualization: + +```ts +const canvas = document.getElementById("viz") as HTMLCanvasElement; +const ctx = canvas.getContext("2d")!; + +client.on(RTVIEvent.LocalAudioLevel, (level) => { + ctx.clearRect(0, 0, canvas.width, canvas.height); + ctx.fillStyle = "#3b82f6"; + ctx.fillRect(0, 0, canvas.width * level, canvas.height); +}); +``` + + + +--- + +## API reference + + + + + + `initDevices`, `getAllMics`, `enableMic`, and more + + + `usePipecatClientMediaDevices`, `usePipecatClientMicControl`, `usePipecatClientCamControl`, `usePipecatClientMediaTrack` + + + `PipecatClientAudio`, `PipecatClientVideo`, `PipecatClientMicToggle`, `PipecatClientCamToggle`, `VoiceVisualizer` + + + + + + + + + + `initDevices`, `getAllMics`, `updateMic`, `enableMic`, `tracks`, and more + + + + diff --git a/client/concepts/session-lifecycle.mdx b/client/concepts/session-lifecycle.mdx new file mode 100644 index 00000000..61812f52 --- /dev/null +++ b/client/concepts/session-lifecycle.mdx @@ -0,0 +1,305 @@ +--- +title: "Session Lifecycle" +description: "How a Pipecat session starts, runs, and ends — and how to handle each phase." +--- + + + +You are currently viewing the React version of this page. Use the Dropdown to the right to customize this page for your client framework. + + + + +You are currently viewing the JavaScript version of this page. Use the Dropdown to the right to customize this page for your client framework. + + + +A session is the span of time from when your client connects to a bot until it disconnects. Understanding the lifecycle helps you build correct connection flows, handle errors gracefully, and clean up reliably. + +## Transport states + +The client exposes a `state` string that tracks where you are in the lifecycle. States progress in order: + +``` +idle → authenticating → authenticated → connecting → connected → ready → disconnecting → disconnected +``` + +| State | What it means | +|---|---| +| `idle` | No connection attempt has started | +| `authenticating` | `startBot()` called; waiting for your server to start the bot | +| `authenticated` | Server responded; bot is starting | +| `connecting` | Transport is establishing the WebRTC/WebSocket connection | +| `connected` | Transport is connected; bot pipeline is initializing | +| `ready` | Bot pipeline is running and ready to receive audio/messages | +| `disconnecting` | Disconnect in progress | +| `disconnected` | Session has ended | + +The `ready` state is the one that matters most — it's the gate before which you should not send messages or expect audio. The `connected` state means the transport is up but the bot's pipeline may still be warming up. + +## Starting a session + +There are three ways to start a session, depending on your architecture. + +### Option 1: `startBotAndConnect()` (recommended) + +The most common pattern. Calls your server endpoint to start the bot, then connects the transport with the credentials returned by your server: + +```tsx +await client.startBotAndConnect({ + endpoint: "/api/start", + requestData: { + // Optional data forwarded to your server + initialPrompt: "You are a helpful assistant.", + }, +}); +// Resolves when the bot reaches 'ready' state +``` + +Use this when you control the server and need to create a session (e.g., a Daily room) before connecting. + +### Option 2: `connect()` with direct params + +Pass connection parameters directly — useful when you've already obtained them out of band, or with SmallWebRTC where the transport URL is known at build time: + +```tsx +await client.connect({ webrtcUrl: "http://localhost:7860/api/offer" }); +``` + +### Option 3: `startBot()` + `connect()` separately + +Gives you explicit control over each phase — useful if you need to inspect the server response before connecting: + +```tsx +const connectionParams = await client.startBot({ endpoint: "/api/start" }); +// inspect or modify connectionParams here +await client.connect(connectionParams); +``` + +All three methods resolve when the bot signals it is `ready`. Wrap them in `try/catch` to handle startup errors. + +## Waiting for ready + +`await client.connect(...)` (and `startBotAndConnect`) resolves only once the bot sends its `BotReady` signal, meaning the pipeline is fully initialized. You don't need to poll or listen for a separate event — the promise resolves at the right time. + +If you prefer event-driven code, or you call `connect()` without `await`: + + + +Use `useRTVIClientEvent` so the handler is properly managed by React's lifecycle: + +```tsx +import { RTVIEvent } from "@pipecat-ai/client-js"; +import { useRTVIClientEvent } from "@pipecat-ai/client-react"; + +function MyComponent() { + useRTVIClientEvent( + RTVIEvent.BotReady, + useCallback((data) => { + console.log("Bot ready, RTVI version:", data.version); + }, []) + ); +} +``` + + + + + +Listen for the `BotReady` event on the client: + +```tsx +import { RTVIEvent } from "@pipecat-ai/client-js"; + +client.on(RTVIEvent.BotReady, (data) => { + console.log("Bot ready, RTVI version:", data.version); +}); +``` + + + +## Ending a session + +### Client-initiated disconnect + +Call `disconnect()` to end the session from the client side. The bot will typically shut down when the client disconnects: + +```tsx +await client.disconnect(); +// Transport state: disconnecting → disconnected +``` + +### Bot-initiated disconnect + +The bot can end the session on its own — due to a pipeline error, session timeout, or intentional shutdown. By default, the client disconnects automatically when the bot disconnects. + +To keep the client connected after the bot disconnects, set `disconnectOnBotDisconnect: false` in the constructor: + +```tsx +const client = new PipecatClient({ + transport: new DailyTransport(), + disconnectOnBotDisconnect: false, +}); +``` + +Listen for `BotDisconnected` to update your UI when this happens: + + + +```tsx +import { RTVIEvent } from "@pipecat-ai/client-js"; +import { useRTVIClientEvent } from "@pipecat-ai/client-react"; + +function MyComponent() { + useRTVIClientEvent( + RTVIEvent.BotDisconnected, + useCallback(() => { + // Update UI to show the session has ended + }, []) + ); +} +``` + + + + + +```tsx +client.on(RTVIEvent.BotDisconnected, () => { + // Update UI to show the session has ended +}); +``` + + + +After a bot disconnect, you can call `connect()` again to start a new session. + +## Error handling + +Two distinct error signals: + +**Startup errors** — thrown by `connect()` / `startBotAndConnect()` if the server can't be reached, credentials are invalid, or the transport can't establish a connection: + +```tsx +try { + await client.startBotAndConnect({ endpoint: "/api/start" }); +} catch (error) { + // Connection failed — update UI accordingly + console.error("Failed to start session:", error); +} +``` + +**Runtime errors** — sent by the bot during an active session. The error includes a `fatal` boolean: if `true`, the bot has already disconnected and the client will clean up automatically: + + + +```tsx +import { RTVIEvent } from "@pipecat-ai/client-js"; +import { useRTVIClientEvent } from "@pipecat-ai/client-react"; + +function MyComponent() { + useRTVIClientEvent( + RTVIEvent.Error, + useCallback((message) => { + const { message: text, fatal } = message.data; + console.error("Bot error:", text); + if (fatal) { + // Session is over — show reconnect UI + } + }, []) + ); +} +``` + + + + + +```tsx +client.on(RTVIEvent.Error, (message) => { + const { message: text, fatal } = message.data; + console.error("Bot error:", text); + if (fatal) { + // Session is over — show reconnect UI + } +}); +``` + + + +## Reconnecting + +The client does not automatically reconnect. After a disconnect (whether client-initiated, bot-initiated, or due to a fatal error), you need to call `startBotAndConnect()` or `connect()` again to start a new session. The same `PipecatClient` instance can be reused. + +```tsx +async function reconnect() { + await client.startBotAndConnect({ endpoint: "/api/start" }); +} +``` + +## Tracking state + + + +The `usePipecatClientTransportState` hook gives you the current state as a reactive value: + +```tsx +import { usePipecatClientTransportState } from "@pipecat-ai/client-react"; + +function StatusBar() { + const transportState = usePipecatClientTransportState(); + + const isReady = transportState === "ready"; + const isConnecting = ["authenticating", "connecting", "connected"].includes(transportState); + + return ( + + {isReady ? "Connected" : isConnecting ? "Connecting…" : "Disconnected"} + + ); +} +``` + + + + + +Listen for `TransportStateChanged` to react to state changes: + +```tsx +import { RTVIEvent } from "@pipecat-ai/client-js"; + +client.on(RTVIEvent.TransportStateChanged, (state) => { + console.log("State:", state); + if (state === "ready") { + // Session is active + } +}); +``` + +You can also read the current state synchronously at any time: + +```tsx +const state = client.state; +``` + + + +## Lifecycle summary + +``` +client.startBotAndConnect() + │ + ├─ authenticating ← hitting your /api/start endpoint + ├─ authenticated ← server responded, bot is starting + ├─ connecting ← transport handshake + ├─ connected ← transport up, pipeline initializing + └─ ready ← bot pipeline running ✓ + │ + │ (session active — audio flows, messages work) + │ + client.disconnect() or bot disconnect + │ + ├─ disconnecting + └─ disconnected +``` diff --git a/client/get-started/quickstart.mdx b/client/get-started/quickstart.mdx new file mode 100644 index 00000000..c5e2354f --- /dev/null +++ b/client/get-started/quickstart.mdx @@ -0,0 +1,250 @@ +--- +title: "Quickstart" +description: "Get a Pipecat voice app running in minutes with the CLI." +--- + + + + You are currently viewing the React version of this page. Use the dropdown + to the right to customize this page for your client framework. + + + + + You are currently viewing the JavaScript version of this page. Use the + dropdown to the right to customize this page for your client framework. + + + + + This quickstart guide will help you build your first Pipecat voice AI bot with + a React front-end and run it locally. You'll create a simple conversational + agent that you connect and talk to in real-time via your browser. + + + This quickstart guide will help you build your first Pipecat voice AI bot with + a vanilla JavaScript front-end and run it locally. You'll create a simple + conversational agent that you connect and talk to in real-time via your + browser. + + +## Prerequisites + +- Python 3.11+ and [uv](https://docs.astral.sh/uv/getting-started/installation/) +- Node.js 18+ + +## Step 1: Scaffold your project + +```bash +# Install the Pipecat CLI +uv tool install pipecat-ai-cli + +# Scaffold the quickstart project +pipecat init +``` + +The CLI will guide you through the setup. Choose the following options: + + + +- **Project name**: `my-voice-app` +- **Bot type**: `Web/Mobile` +- **Client framework**: `React` +- **React dev server**: `Vite` +- **Transport**: `SmallWebRTC` + +The rest is up to you! The CLI generates a complete project with two main directories: + +- `bot/` — a Python Pipecat bot +- `client/` — a React front-end built on the [Voice UI Kit](/client/voice-ui-kit) + + + + + +- **Project name**: `my-voice-app` +- **Bot type**: `Web/Mobile` +- **Client framework**: `Vanilla JS` +- **Transport**: `SmallWebRTC` + +The rest is up to you! The CLI generates a complete project with two main directories: + +- `bot/` — a Python Pipecat bot +- `client/` — a vanilla JavaScript front-end built on the Pipecat JS SDK + + + +## Step 2: Start the bot + +```bash +cd my-voice-app/bot +cp env.example .env # add your API keys +uv sync +uv run bot.py +``` + +The bot starts at `http://localhost:7860`. + +## Step 3: Start the client + +```bash +cd ../client +npm install +npm run dev +``` + +Open `http://localhost:5173`, click **Connect**, allow microphone access, and start talking. + + + **First run note**: The initial startup may take ~20 seconds as Pipecat + downloads required models and imports. Subsequent runs will be much faster. + + +🎉 **Success!** Your bot is running locally. Now let's deploy it to production so others can use it. + +--- + +## Understanding the Quickstart Client + + + +The React app is built on the [Voice UI Kit](/client/voice-ui-kit) — Pipecat's library of pre-built voice UI components. Here's what each file does: + +**`src/config.ts`** — transport configuration. By default it uses SmallWebRTC for local development. To switch to Daily for production, add your Daily credentials and update `AVAILABLE_TRANSPORTS`. + +```ts +const botStartUrl = + import.meta.env.VITE_BOT_START_URL || "http://localhost:7860/start"; + +export const DEFAULT_TRANSPORT: TransportType = "smallwebrtc"; +``` + +**`src/main.tsx`** — entry point. `PipecatAppBase` handles transport creation, connection lifecycle, loading states, and errors, passing a ready `client` down to your app via a render prop. + +```tsx + + {({ client, handleConnect, handleDisconnect, error }) => + !client ? : + error ? {error} : + + } + +``` + +**`src/components/App.tsx`** — the UI. Built entirely from Voice UI Kit components: + +| Component | What it does | +| ------------------- | ---------------------------------------------------------------- | +| `ConnectButton` | Connect/disconnect button with built-in loading and error states | +| `UserAudioControl` | Mic mute/unmute toggle | +| `ConversationPanel` | Live transcript of the conversation | +| `EventsPanel` | Real-time stream of RTVI events — useful during development | + + + + + +The vanilla JS app is built directly on the Pipecat JS SDK. Here's what each file does: + +**`src/config.js`** — transport configuration. By default it uses SmallWebRTC for local development. Also exports a `createTransport()` helper that dynamically imports the correct transport package. To switch to Daily for production, add your Daily credentials and update `AVAILABLE_TRANSPORTS`. + +```js +const botStartUrl = + import.meta.env.VITE_BOT_START_URL || "http://localhost:7860/start"; + +export const DEFAULT_TRANSPORT = "smallwebrtc"; +``` + +**`src/app.js`** — the entire client UI, implemented as a `VoiceChatClient` class. On construction it wires up DOM elements and event listeners. Calling `connect()` creates a `PipecatClient`, sets up the audio track, and starts the bot: + +```js +async connect() { + const transport = await createTransport(this.transportType); + this.client = new PipecatClient({ + transport, + enableMic: true, + callbacks: { + onUserTranscript: (data) => { if (data.final) this.addConversationMessage(data.text, 'user'); }, + onBotTranscript: (data) => { this.addConversationMessage(data.text, 'bot'); }, + // ... + }, + }); + await this.client.startBotAndConnect(TRANSPORT_CONFIG[this.transportType]); +} +``` + +**`index.html`** — the page structure. Contains the transport selector, connect button, mic toggle, conversation log, and events panel that `app.js` references by ID. + +| Element | What it does | +| -------------------------- | ----------------------------------------------------------- | +| `#connect-btn` | Connect/disconnect button | +| `#mic-btn` / `#mic-status` | Mic mute/unmute toggle | +| `#conversation-log` | Live transcript of the conversation | +| `#events-log` | Real-time stream of RTVI events — useful during development | + + + + + +## Deploying to production + +The `env.example` file shows the production configuration. Point `VITE_BOT_START_URL` at your deployed bot's start endpoint: + +```bash +# Pipecat Cloud +VITE_BOT_START_URL="https://api.pipecat.daily.co/v1/public/{agentName}/start" +VITE_BOT_START_PUBLIC_API_KEY="your-public-api-key" +``` + + + The client app is a static site — build it with `npm run build` and deploy to + any static hosting provider (Vercel, Netlify, S3, etc.). + + + + +## Next steps + + + + + + Customize themes, swap components, and extend the generated UI + + + Build a React voice app from scratch — understand the SDK without + abstractions + + + Understand transports, sessions, events, and media + + + Deploy to Pipecat Cloud for production + + + + + + + + + + Build a voice app from scratch — understand the SDK without abstractions + + + Understand transports, sessions, events, and media + + + Deploy to Pipecat Cloud for production + + + + diff --git a/client/guides/building-a-voice-ui.mdx b/client/guides/building-a-voice-ui.mdx new file mode 100644 index 00000000..d94c563f --- /dev/null +++ b/client/guides/building-a-voice-ui.mdx @@ -0,0 +1,318 @@ +--- +title: "Building a Voice UI" +description: "Build a voice application from scratch using the Pipecat client SDK." +--- + + + +You are currently viewing the React version of this page. Use the dropdown to the right to customize this page for your client framework. + + + + +You are currently viewing the JavaScript version of this page. Use the dropdown to the right to customize this page for your client framework. + + + + +This guide walks through building a React voice app without any UI abstractions — just the Pipecat React SDK directly. You'll see exactly how the client, provider, hooks, and audio output fit together, which is useful whether you're building a fully custom UI or want to understand what the [CLI-generated app](/client/get-started/quickstart) is doing under the hood. + + + +This guide walks through building a voice app in vanilla TypeScript — just the Pipecat JavaScript SDK directly, with no framework. You'll see how to set up the client, handle events, and wire up audio output manually. + + +## Prerequisites + +- Node.js 18+ +- A running Pipecat bot + + + Follow the Pipecat server quickstart to get a bot running locally at `http://localhost:7860`. + + +## Installation + + + +Create a new React project and install the SDK, React bindings, and SmallWebRTC transport: + +```bash +npm create vite@latest my-voice-app -- --template react-ts +cd my-voice-app +npm install @pipecat-ai/client-js @pipecat-ai/client-react @pipecat-ai/small-webrtc-transport +``` + + + + + +Create a new TypeScript project and install the SDK and SmallWebRTC transport: + +```bash +npm create vite@latest my-voice-app -- --template vanilla-ts +cd my-voice-app +npm install @pipecat-ai/client-js @pipecat-ai/small-webrtc-transport +``` + + + + + SmallWebRTC is ideal for development — no third-party account needed. + For production, swap in the [Daily transport](/api-reference/client/js/transports/daily), + which provides global infrastructure, echo cancellation, and more. + + +## Step 1: Create the client + + + +Create the `PipecatClient` once at the module level with SmallWebRTC as the transport. Pass it to `PipecatClientProvider` so every component in your app can access it. `PipecatClientAudio` mounts a hidden ` + + + +Create the `PipecatClient` and wire up the bot's audio output manually. Replace `src/main.ts` with: + +```ts +import { PipecatClient, RTVIEvent } from "@pipecat-ai/client-js"; +import { SmallWebRTCTransport } from "@pipecat-ai/small-webrtc-transport"; + +export const client = new PipecatClient({ + transport: new SmallWebRTCTransport(), + enableMic: true, +}); + +// Mount a hidden + +## Step 2: Build the app + + + +Replace `src/App.tsx` with: + +```tsx +import { useCallback } from "react"; +import { RTVIEvent } from "@pipecat-ai/client-js"; +import { + usePipecatClient, + usePipecatClientTransportState, + usePipecatConversation, + useRTVIClientEvent, +} from "@pipecat-ai/client-react"; + +export default function App() { + const client = usePipecatClient(); + const transportState = usePipecatClientTransportState(); + const { messages } = usePipecatConversation(); + + const isConnected = transportState === "ready"; + const isConnecting = ["authenticating", "connecting", "connected"].includes(transportState); + + useRTVIClientEvent( + RTVIEvent.Error, + useCallback((error) => console.error("Bot error:", error), []) + ); + + const handleConnect = async () => { + await client.connect({ webrtcUrl: "http://localhost:7860/api/offer" }); + }; + + return ( +
+

Pipecat Voice App

+ +
+ + + {transportState} + +
+ +
    + {messages.map((msg, i) => ( +
  • + {msg.role === "user" ? "You" : "Bot"}:{" "} + {msg.parts?.map((part, j) => ( + + {typeof part.text === "string" + ? part.text + : part.text.spoken + part.text.unspoken} + + ))} +
    • + ))} +
+
+ ); +} +``` + + + + + +Replace `src/app.ts` with: + +```ts +import { RTVIEvent } from "@pipecat-ai/client-js"; +import { client } from "./main"; + +const button = document.getElementById("connect-btn") as HTMLButtonElement; +const statusEl = document.getElementById("status") as HTMLSpanElement; +const messageList = document.getElementById("messages") as HTMLUListElement; + +client.on(RTVIEvent.TransportStateChanged, (state) => { + const isConnected = state === "ready"; + const isConnecting = ["authenticating", "connecting", "connected"].includes(state); + + statusEl.textContent = state; + button.textContent = isConnected ? "Disconnect" : isConnecting ? "Connecting…" : "Connect"; + button.disabled = isConnecting; +}); + +client.on(RTVIEvent.BotOutput, (data) => { + if (data.aggregated_by === "sentence") { + appendMessage("Bot", data.text); + } +}); + +client.on(RTVIEvent.UserTranscript, (data) => { + if (data.final) appendMessage("You", data.text); +}); + +client.on(RTVIEvent.Error, (error) => console.error("Bot error:", error)); + +button.addEventListener("click", async () => { + if (client.state === "ready") { + await client.disconnect(); + } else { + await client.connect({ webrtcUrl: "http://localhost:7860/api/offer" }); + } +}); + +function appendMessage(role: string, text: string) { + const li = document.createElement("li"); + li.style.marginBottom = "8px"; + li.innerHTML = `${role}: ${text}`; + messageList.appendChild(li); +} +``` + +And update `index.html` to include the elements the script expects: + +```html +
+

Pipecat Voice App

+
+ + +
+
    +
    + +``` + +     + +## Step 3: Run it + +Start your Pipecat bot (if it isn't already running), then start the app: + +```bash +npm run dev +``` + +Open `http://localhost:5173`, click **Connect**, allow microphone access, and start talking. + +## What each piece does + + + +| Piece | Role | +|---|---| +| `PipecatClient` | Manages the connection and session lifecycle | +| `SmallWebRTCTransport` | Handles the WebRTC peer connection to your bot | +| `PipecatClientProvider` | Shares the client instance via React context | +| `PipecatClientAudio` | Mounts a hidden ` + + + +| Piece | Role | +|---|---| +| `PipecatClient` | Manages the connection and session lifecycle | +| `SmallWebRTCTransport` | Handles the WebRTC peer connection to your bot | +| `TrackStarted` event | Notifies when the bot's audio track is ready to play | +| `TransportStateChanged` event | Drives button state and status display | +| `BotOutput` event | Streams the bot's response text sentence by sentence | +| `UserTranscript` event | Receives finalized speech-to-text from the user | + + + +## Next steps + + + + Understand connection states, reconnection, and teardown + + + The full event system — what fires, when, and how to handle it + + + Send and receive custom messages between client and bot + + + All hooks, components, and types + + diff --git a/client/guides/custom-messaging.mdx b/client/guides/custom-messaging.mdx new file mode 100644 index 00000000..3aa218bb --- /dev/null +++ b/client/guides/custom-messaging.mdx @@ -0,0 +1,313 @@ +--- +title: "Custom Messaging" +description: "Send and receive arbitrary messages between your client and Pipecat bot." +--- + + + +You are currently viewing the React version of this page. Use the dropdown to the right to customize this page for your client framework. + + + + +You are currently viewing the JavaScript version of this page. Use the dropdown to the right to customize this page for your client framework. + + + +The Pipecat client can send and receive arbitrary messages to and from the server running the bot. This is useful for passing configuration at startup, triggering actions mid-session, querying server state, and receiving notifications from the bot. + +## Connection-time configuration + +Often you need to pass configuration to the server when starting the bot — a system prompt, preferred language, user preferences, or any other data the server needs before the pipeline runs. Pass it via `requestData` in `startBotAndConnect()`. + + + +```tsx +import { usePipecatClient } from "@pipecat-ai/client-react"; + +function MyComponent() { + const client = usePipecatClient(); + + const handleStart = async () => { + try { + await client.startBotAndConnect({ + endpoint: "/api/start", + requestData: { + initial_prompt: "You are a pirate captain", + preferred_language: "en-US", + }, + }); + } catch (error) { + console.error("Error starting the bot:", error); + } + }; +} +``` + + + + + +```ts +try { + await client.startBotAndConnect({ + endpoint: "/api/start", + requestData: { + initial_prompt: "You are a pirate captain", + preferred_language: "en-US", + }, + }); +} catch (error) { + console.error("Error starting the bot:", error); +} +``` + + + +On the server, your endpoint receives this data as the request body and can forward relevant values to the bot process or use them to configure the pipeline: + + +```python FastAPI endpoint +@app.post("/api/start") +async def start(request: Request) -> Dict[Any, Any]: + body = await request.json() + prompt = body.get("initial_prompt", "You are a pirate captain") + lang = body.get("preferred_language", "en-US") + + room_url, token = await create_room_and_token() + + proc = subprocess.Popen( + [f"python3 -m bot -u {room_url} -t {token} -p {prompt} -l {lang}"], + shell=True, + cwd=os.path.dirname(os.path.abspath(__file__)), + ) + bot_procs[proc.pid] = (proc, room_url) + + return {"url": room_url, "token": token} +``` + +```python bot +def extract_arguments(): + parser = argparse.ArgumentParser() + parser.add_argument("-u", "--room-url", type=str) + parser.add_argument("-t", "--token", type=str) + parser.add_argument("-p", "--prompt", type=str, default="You are a pirate captain") + parser.add_argument("-l", "--language", type=str, default="en-US") + return parser.parse_args() + +async def main(): + args = extract_arguments() + + llm = GeminiLiveLLMService( + api_key=os.getenv("GOOGLE_API_KEY"), + settings=GeminiLiveLLMService.Settings( + system_instruction=args.prompt, + language=args.language, + ), + ) +``` + + +--- + +## Sending messages to the server + +Use `sendClientMessage()` to send a fire-and-forget message to the bot. The server handles it and is not expected to send a direct response. + + + +```tsx +const client = usePipecatClient(); + +client.sendClientMessage("set-language", { language: "en-US" }); +``` + + + + + +```ts +client.sendClientMessage("set-language", { language: "en-US" }); +``` + + + +On the server, handle it via the `on_client_message` event handler or from inside a `FrameProcessor`: + + +```python event handler +@rtvi.event_handler("on_client_message") +async def on_client_message(rtvi, msg): + if msg.type == "set-language": + language = msg.data.get("language", "en-US") + await task.queue_frames([STTUpdateSettingsFrame(language=language)]) +``` + +```python FrameProcessor +class CustomFrameProcessor(FrameProcessor): + async def process_frame(self, frame: Frame, direction: FrameDirection): + await super().process_frame(frame, direction) + if isinstance(frame, RTVIClientMessageFrame): + if frame.type == "set-language": + language = frame.data.get("language", "en-US") + await self.push_frame(STTUpdateSettingsFrame(language=language)) + return + await self.push_frame(frame, direction) +``` + + +--- + +## Requesting data from the server + +Use `sendClientRequest()` to send a message and wait for a response. Useful for querying server state or triggering an action that needs to confirm success or failure. + + + +```tsx +const client = usePipecatClient(); + +try { + const response = await client.sendClientRequest("get-language"); + console.log("Current language:", response.language); +} catch (error) { + console.error("Request failed:", error); +} +``` + + + + + +```ts +try { + const response = await client.sendClientRequest("get-language"); + console.log("Current language:", response.language); +} catch (error) { + console.error("Request failed:", error); +} +``` + + + +On the server, respond with `send_server_response()` or push a `RTVIServerResponseFrame`: + + +```python event handler +@rtvi.event_handler("on_client_message") +async def on_client_message(rtvi, msg): + if msg.type == "get-language": + await rtvi.send_server_response(msg, {"language": get_current_language()}) + else: + await rtvi.send_error_response(msg, "Unknown request type") +``` + +```python FrameProcessor +class CustomFrameProcessor(FrameProcessor): + async def process_frame(self, frame: Frame, direction: FrameDirection): + await super().process_frame(frame, direction) + if isinstance(frame, RTVIClientMessageFrame): + if frame.type == "get-language": + await self.push_frame( + RTVIServerResponseFrame( + client_msg=frame, + data={"language": get_current_language()}, + ) + ) + return + else: + await self.push_frame( + RTVIServerResponseFrame( + client_msg=frame, + error="Unknown request type", + ) + ) + await self.push_frame(frame, direction) +``` + + +--- + +## Receiving messages from the server + +The server can push unsolicited messages to the client at any time — for example, to notify the client that a setting has changed. + + + +Use `useRTVIClientEvent` to subscribe to `ServerMessage` within a component: + +```tsx +import { RTVIEvent } from "@pipecat-ai/client-js"; +import { useRTVIClientEvent } from "@pipecat-ai/client-react"; + +function LanguageListener() { + useRTVIClientEvent( + RTVIEvent.ServerMessage, + useCallback((message) => { + if (message.data.msg === "language-updated") { + console.log("Language updated to:", message.data.language); + } + }, []) + ); +} +``` + + + + + +Listen for the `ServerMessage` event on the client, or pass `onServerMessage` as a constructor callback: + +```ts +client.on(RTVIEvent.ServerMessage, (message) => { + if (message.data.msg === "language-updated") { + console.log("Language updated to:", message.data.language); + } +}); +``` + + + +On the server, send messages via `send_server_message()` or by pushing a `RTVIServerMessageFrame`: + + +```python Observer +class CustomObserver(BaseObserver): + async def on_push_frame(self, data: FramePushed): + if isinstance(data.frame, STTUpdateSettingsFrame): + for key, value in data.frame.settings.items(): + if key == "language": + await rtvi.send_server_message({ + "msg": "language-updated", + "language": value, + }) +``` + +```python FrameProcessor +class CustomFrameProcessor(FrameProcessor): + async def process_frame(self, frame: Frame, direction: FrameDirection): + await super().process_frame(frame, direction) + if isinstance(frame, STTUpdateSettingsFrame): + for key, value in frame.settings.items(): + if key == "language": + await self.push_frame( + RTVIServerMessageFrame( + data={"msg": "language-updated", "language": value} + ) + ) + await self.push_frame(frame, direction) +``` + + +--- + +## API reference + + + + `sendClientMessage`, `sendClientRequest`, `registerFunctionCallHandler` + + + `onServerMessage`, `onMessageError`, and related events + + diff --git a/client/introduction.mdx b/client/introduction.mdx index 8e24b520..355af9bc 100644 --- a/client/introduction.mdx +++ b/client/introduction.mdx @@ -1,315 +1,107 @@ --- -title: "Client SDKs" -description: "Client libraries for building real-time AI applications with Pipecat" +title: "Pipecat Clients" +description: "Client SDKs for building real-time voice and multimodal AI applications with Pipecat." --- - - All Client SDKs have transitioned to v1.0, which uses a new, simpler API - design. For guidance in transitioning to the new API, please refer to the - migration guide for each platform. If you have any questions or need - assistance, please reach out to us on [Discord](https://discord.gg/pipecat). - +Pipecat's client SDKs connect your users to Pipecat agents running on the server. They handle the real-time media layer — microphone and camera access, audio playback, transport connections, and the event stream from your bot — so you can focus on building your application. -Pipecat provides client SDKs for multiple platforms, all implementing the RTVI (Real-Time Voice and Video Inference) standard. These SDKs make it easy to build real-time AI applications that can handle voice, video, and text interactions. + + Build and run your first Pipecat voice application + + +## Available SDKs - Pipecat JS SDK + Core SDK for web applications - Pipecat React SDK + Hooks and components for React apps - Pipecat React Native SDK + iOS and Android via React Native - Pipecat iOS SDK + Native Swift SDK for iOS - Pipecat Android SDK - - - Pipecat C++ SDK + Native Kotlin SDK for Android - - -## Core Functionality - -All Pipecat client SDKs provide: - - - - Handle device inputs and media streams for audio and video - - - Configure and communicate with your Pipecat bot - - - Manage connection state and error handling + + Native SDK for desktop and embedded -## Core Types - -### PipecatClient - -The main class for interacting with Pipecat bots. It is the primary type you will interact with. - -### Transport - -The `PipecatClient` wraps a Transport, which defines and provides the underlying connection mechanism (e.g., WebSocket, WebRTC). Your Pipecat pipeline will contain a corresponding transport. - -### RTVIMessage - -Represents a message sent to or received from a Pipecat bot. - -## Simple Usage Examples - - - - Establish ongoing connections via WebSocket or WebRTC for: - - Live voice conversations - - Real-time video processing - - Continuous interactions - - - - ```javascript javascript - // Example: Establishing a real-time connection - import { RTVIEvent, RTVIMessage, PipecatClient } from "@pipecat-ai/client-js"; - import { DailyTransport } from "@pipecat-ai/daily-transport"; - - const pcClient = new PipecatClient({ - transport: new DailyTransport(), - enableMic: true, - enableCam: false, - enableScreenShare: false, - callbacks: { - onBotConnected: () => { - console.log("[CALLBACK] Bot connected"); - }, - onBotDisconnected: () => { - console.log("[CALLBACK] Bot disconnected"); - }, - onBotReady: () => { - console.log("[CALLBACK] Bot ready to chat!"); - }, - }, - }); +## How It Fits Together - try { - // Below, we use a REST endpoint to fetch connection credentials for our - // Daily Transport. Alternatively, you could provide those credentials - // directly to `connect()`. - await pcClient.startBotAndConnect({ - endpoint: "https://your-connect-end-point-here/connect", - }); - } catch (e) { - console.error(e.message); - } +Pipecat's architecture is split between server and client. The server runs your AI pipeline — speech recognition, LLM, text-to-speech — and the client connects your user to it over a real-time transport. - // Events (alternative approach to constructor-provided callbacks) - pcClient.on(RTVIEvent.Connected, () => { - console.log("[EVENT] User connected"); - }); - pcClient.on(RTVIEvent.Disconnected, () => { - console.log("[EVENT] User disconnected"); - }); - ``` + + + - ```jsx react - // Example: Using PipecatClient in a React component - import { PipecatClient } from "@pipecat-ai/client-js"; - import { - PipecatClientProvider, - PipecatClientAudio, - usePipecatClient, - useRTVIClientEvent, - } from "@pipecat-ai/client-react"; - import { DailyTransport } from "@pipecat-ai/daily-transport"; +The client SDKs implement the [RTVI standard](/client/rtvi-standard) — an open protocol for real-time AI inference — which means they work with any RTVI-compatible server, not just Pipecat. - // Create the client instance - const client = new PipecatClient({ - transport: new DailyTransport(), - enableMic: true, - }); +## What the SDKs Handle - // Root component wraps the app with the provider - function App() { - return ( - - - - - ); - } +- **Transport management** — establishing, maintaining, and tearing down connections +- **Media capture** — microphone and camera device access and streaming +- **Audio playback** — rendering bot audio output +- **Session state** — tracking connection state and bot readiness +- **Event stream** — callbacks and events for transcriptions, bot speaking, errors, and more +- **Messaging** — sending custom messages to your bot and handling responses - // Component using the client - function VoiceBot() { - const client = usePipecatClient(); - - const handleClick = async () => { - await client.startBotAndConnect({ - endpoint: `${process.env.PIPECAT_API_URL || "/api"}/connect` - }); - }; - - return ( - ; - ); - } - - function EventListener() { - useRTVIClientEvent( - RTVIEvent.Connected, - useCallback(() => { - console.log("[EVENT] User connected"); - }, []) - ); - } - ``` - - - - - - Send custom messages and handle responses from your bot. This is useful for: - - Running server-side functionality - - Triggering specific bot actions - - Querying the server - - Responding to server requests - - - -```javascript javascript -import { PipecatClient } from "@pipecat-ai/client-js"; - -const pcClient = new PipecatClient({ - transport: new DailyTransport(), - callbacks: { - onBotConnected: () => { - pcClient - .sendClientRequest("get-language") - .then((response) => { - console.log("[CALLBACK] Bot using language:", response); - if (response !== preferredLanguage) { - pcClient.sendClientMessage("set-language", { - language: preferredLanguage, - }); - } - }) - .catch((error) => { - console.error("[CALLBACK] Error getting language:", error); - }); - }, - onServerMessage: (message) => { - console.log("[CALLBACK] Received message from server:", message); - }, - }, -}); -// Here we have obtained the connection details separately and pass them -// directly to connect(). -// Alternatively, you can use a connection endpoint to fetch these details -// using `startBotAndConnect()`. -await pcClient.connect({ - url: "https://your-daily-room-url", - token: "your-daily-token", -}); -``` - -```jsx react -// Example: Messaging in a React application -import { useCallback } from "react"; -import { RTVIEvent, TransportState } from "@pipecat-ai/client-js"; -import { usePipecatClient, useRTVIClientEvent } from "@pipecat-ai/client-react"; - -function EventListener() { - const pcClient = usePipecatClient(); - useRTVIClientEvent( - RTVIEvent.BotConnected, - useCallback(() => { - pcClient - .sendClientRequest("get-language") - .then((response) => { - console.log("[CALLBACK] Bot using language:", response); - if (response !== preferredLanguage) { - pcClient.sendClientMessage("set-language", { - language: preferredLanguage, - }); - } - }) - .catch((error) => { - console.error("[CALLBACK] Error getting language:", error); - }); - }, []), - ); - useRTVIClientEvent( - RTVIEvent.ServerMessage, - useCallback((data) => { - console.log("[CALLBACK] Received message from server:", data); - }, []), - ); -} -``` - - - - - - -## About RTVI - -Pipecat's client SDKs implement the RTVI (Real-Time Voice and Video Inference) standard, an open specification for real-time AI inference. This means: - -- Your code can work with any RTVI-compatible inference service -- You get battle-tested tooling for real-time multimedia handling -- You can easily set up development and testing environments - -## Next Steps - -Get started by trying out examples: +## Ready to Build? + + Get a voice conversation running in minutes + + + Understand transports, sessions, events, and media + - Complete client-server example with both bot backend (Python) and frontend - implementation (JS, React, React Native, iOS, and Android). + Full reference for all SDK methods, hooks, and callbacks - Explore our full collection of example applications and implementations - across different platforms and use cases. + Complete example applications across all platforms diff --git a/client/js/api-reference/messages.mdx b/client/js/api-reference/messages.mdx deleted file mode 100644 index f1a52b65..00000000 --- a/client/js/api-reference/messages.mdx +++ /dev/null @@ -1,304 +0,0 @@ ---- -title: "Custom Messaging" ---- - -The Pipecat JavaScript client can send and receive arbitrary messages to/from the server running the bot. This page outlines and demonstrates both client and server code for passing and responding to custom messages as well as providing arbitrary data at connection time. - -## Connection-Time Configuration - -Oftentimes clients need to provide configuration data to the server when starting the bot. This can include things like preferred language, user preferences, initial messages, or any other data that the server needs to know about the client. This must occur before the bot is started and therefore is not part of the RTVI standard, but rather a custom implementation. That said, the `PipecatClient` makes it easy to send this data as part of the `startBot()` and `startBotAndConnect()` methods, by passing an API endpoint object with the `requestData` property. Your server endpoint can then handle this data as needed. In the example below, we demonstrate sending an initial prompt and preferred language to the server when connecting. - - -```javascript client -try { - pcClient.startBotAndConnect({ - endpoint: '/api/start', // Your server endpoint to start the bot - requestData: { - initial_prompt: "You are a pirate captain", - preferred_language: 'en-US' - } - }); -} catch (error) { - console.error("Error starting the bot:", error); -} -``` - -```python FastAPI endpoint - -def validate_request_data(body: Dict[str, Any]) -> Tuple[str, str]: - """Validate and extract prompt and language from request data.""" - if not isinstance(body, dict): - raise ValueError("Request body must be a dictionary") - - prompt = body.get("initial_prompt", "You are a pirate captain") - lang = body.get("preferred_language", "en-US") - - if not isinstance(prompt, str) or not isinstance(lang, str): - raise ValueError("Both initial_prompt and preferred_language must be strings") - - return prompt, lang - -@app.post("/api/start") -async def start(request: Request) -> Dict[Any, Any]: - """Startup endpoint that creates a room, starts the bot, and returns - connection credentials. - - This endpoint is called by clients to kick off a session and establish - a connection. - - Returns: - Dict[Any, Any]: Authentication bundle containing room_url and token - - Raises: - HTTPException: If room creation, token generation, or bot startup fails - """ - body = await request.json() - try: - prompt, lang = validate_request_data(body) - except ValueError as e: - raise HTTPException(status_code=400, info=f"Invalid request data: {e}") - - print("Creating room for RTVI connection", body) - room_url, token = await create_room_and_token() - print(f"Room URL: {room_url}") - - # Start the bot process - try: - proc = subprocess.Popen( - [f"python3 -m bot -u {room_url} -t {token} -p {prompt} -l {lang}"], - shell=True, - bufsize=1, - cwd=os.path.dirname(os.path.abspath(__file__)), - ) - bot_procs[proc.pid] = (proc, room_url) - except Exception as e: - raise HTTPException(status_code=500, detail=f"Failed to start subprocess: {e}") - - # Return the authentication bundle in format expected by DailyTransport - return {"url": room_url, "token": token} -``` - -```python bot -import argparse -import asyncio - -def extract_arguments(): - parser = argparse.ArgumentParser(description="Example") - parser.add_argument( - "-u", "--room-url", type=str, default=os.getenv("DAILY_SAMPLE_ROOM_URL", "") - ) - parser.add_argument( - "-t", "--token", type=str, default=os.getenv("DAILY_SAMPLE_ROOM_TOKEN", None) - ) - parser.add_argument( - "-p", "--prompt", type=str, default="You are a pirate captain" - ) - parser.add_argument("-l", "--language", type=str, default="en-US") - return parser.parse_args() - -async def main(): - args = extract_arguments() - print(f"room_url: {args.room_url}") - - daily_transport = DailyTransport( - args.room_url, - args.token, - "Chatbot", - DailyParams( - audio_in_enabled=True, - audio_out_enabled=True, - ), - ) - - llm = GeminiLiveLLMService( - api_key=os.getenv("GOOGLE_API_KEY"), - settings=GeminiLiveLLMService.Settings( - system_instruction=SYSTEM_INSTRUCTION, - voice="Puck", - language=args.language, # Pass preferred language to LLM - ), - ) - - messages = [ { role: "system", content: args.prompt } ] - context = LLMContext(messages) - user_aggregator, assistant_aggregator = LLMContextAggregatorPair( - context, - user_params=LLMUserAggregatorParams( - vad_analyzer=SileroVADAnalyzer(), - ), - ) - - pipeline = Pipeline( - [ - daily_transport.input(), - user_aggregator, - llm, - daily_transport.output(), - assistant_aggregator, - ] - ) - - task = PipelineTask(pipeline) - - @rtvi.event_handler("on_client_ready") - async def on_client_ready(rtvi): - logger.debug("Client ready") - # Kick off the conversation - await task.queue_frames([LLMRunFrame()]) - - @transport.event_handler("on_client_disconnected") - async def on_client_disconnected(transport, client): - logger.debug(f"Client disconnected") - await task.cancel() - - runner = PipelineRunner(handle_sigint=False) - - await runner.run(task) - -if __name__ == "__main__": - asyncio.run(main()) -``` - - - -## Sending Custom Messages to the Server - -Once connected, you can send custom messages to the server using the `sendClientMessage` method. This is useful for triggering specific actions or sending data that the server needs to process. - - -```javascript client -try { - pcClient.sendClientMessage('set-language', { language: 'en-US' }); -} catch (error) { - console.error("Error sending message to server:", error); -} -``` - -```python bot - task = PipelineTask( - pipeline, - params, - observers=[RTVIObserver(rtvi)], - ) - - @rtvi.event_handler("on_client_message") - async def on_client_message(rtvi, msg): - print("RTVI client message:", msg.type, msg.data) - if msg.type == "set-language": - language = msg.data.get("language", "en-US") - await task.queue_frames([STTUpdateSettingsFrame(language=language)]) - -### Alternatively, if your message requires asynchronous processing or storing of -### state, you may want to handle it from inside a FrameProcessor, listen for a -### RTVIClientMessageFrame and push a RTVIServerResponseFrame -class CustomFrameProcessor(FrameProcessor): - async def process_frame(self, frame: Frame, direction: FrameDirection): - await super().process_frame(frame, direction) - if isinstance(frame, RTVIClientMessageFrame): - print("RTVI client message:", frame.msg_id, frame.type, frame.data) - if frame.type == "set-language": - language = frame.data.get("language", "en-US") - await self.push_frame(STTUpdateSettingsFrame(language=language)) - return - await self.push_frame(frame, direction) -``` - - - -## Requesting Information from the Server - -You can also request information from the server using the `sendClientRequest` method. This is useful for querying the server for specific data or triggering and action and getting a success/failure response. - - -```javascript client -try { - const response = await pcClient.sendClientRequest('get-language'); - console.log("Current language:", response.language); -} catch (error) { - console.error("Error requesting data from server:", error); -} -``` - -```python bot -@rtvi.event_handler("on_client_message") -async def on_client_message(rtvi, msg): - print("RTVI client message:", msg.type, msg.data) - if msg.type == "get-language": - await rtvi.send_server_response(msg, {"language": get_current_language()}) - else: - await rtvi.send_error_response(msg, "Unknown request type") - -### Alternatively, if your message requires asynchronous processing or storing of -### state, you may want to handle it from inside a FrameProcessor, listen for a -### RTVIClientMessageFrame and push a RTVIServerResponseFrame -class CustomFrameProcessor(FrameProcessor): - async def process_frame(self, frame: Frame, direction: FrameDirection): - await super().process_frame(frame, direction) - if isinstance(frame, RTVIClientMessageFrame): - print("RTVI client message:", frame.msg_id, frame.type, frame.data) - if frame.type == "get-language": - data = {"language": get_current_language()} - await self.push_frame( - RTVIServerResponseFrame( - client_msg=frame, - data=data, - ), - ) - return - else: - await self.push_frame( - RTVIServerResponseFrame( - client_msg=frame, - error="Unknown request type" - ) - ) - await self.push_frame(frame, direction) -``` - - - -## Handling Custom Messages from the Server - -You can handle custom messages sent from the server using the `onServerMessage` callback. This allows you to process messages that the server sends back to the client, such as notifications or updates. For full details on sending server messages from your bot, see the [RTVIProcessor custom messaging docs](/api-reference/server/rtvi/rtvi-processor#custom-messaging). - - -```javascript client -pcClient.onServerMessage((message) => { - console.log("Received message from server:", message); - if (message.data.msg === 'language-updated') { - console.log("Language updated to:", message.data.language); - } -}); -``` - -```python bot -## From inside an Observer, call `send_server_message` directly on your rtvi instance -class CustomObserver(BaseObserver): - async def on_push_frame(self, data: FramePushed): - if isinstance(frame, STTUpdateSettingsFrame): - for key, value in settings.items(): - if key == "language": - await rtvi.send_server_message({ - "msg": "language-updated", - "language": value - }) - -### Alternatively, from inside a FrameProcessor, push a RTVIServerMessageFrame -class CustomFrameProcessor(FrameProcessor): - async def process_frame(self, frame: Frame, direction: FrameDirection): - await super().process_frame(frame, direction) - if isinstance(frame, STTUpdateSettingsFrame): - for key, value in settings.items(): - if key == "language": - await self.push_frame( - RTVIServerMessageFrame( - data={ - "msg": "language-updated", - "language": value - } - ) - ) - await self.push_frame(frame, direction) -``` - - diff --git a/docs.json b/docs.json index a9dc979b..ca41b54e 100644 --- a/docs.json +++ b/docs.json @@ -171,96 +171,29 @@ "tab": "Pipecat Clients", "groups": [ { - "group": "Introduction", - "pages": ["client/introduction", "client/rtvi-standard"] - }, - { - "group": "JavaScript SDK", - "pages": [ - "client/js/introduction", - { - "group": "API Reference", - "pages": [ - "client/js/api-reference/client-constructor", - "client/js/api-reference/client-methods", - "client/js/api-reference/callbacks", - "client/js/api-reference/messages", - "client/js/api-reference/errors" - ] - }, - { - "group": "Transport Packages", - "pages": [ - "client/js/transports/transport", - "client/js/transports/daily", - "client/js/transports/small-webrtc", - "client/js/transports/websocket", - "client/js/transports/gemini", - "client/js/transports/openai-webrtc" - ] - } - ] - }, - { - "group": "React SDK", - "pages": [ - "client/react/introduction", - { - "group": "API Reference", - "pages": ["client/react/components", "client/react/hooks"] - } - ] - }, - { - "group": "React Native SDK", + "group": "Get Started", "pages": [ - "client/react-native/introduction", - "client/react-native/api-reference", - { - "group": "Transport Packages", - "pages": [ - "client/react-native/transports/daily", - "client/react-native/transports/small-webrtc" - ] - } + "client/introduction", + "client/get-started/quickstart" ] }, { - "group": "iOS SDK", + "group": "Core Concepts", "pages": [ - "client/ios/introduction", - "client/ios/api-reference", - { - "group": "Transport Packages", - "pages": [ - "client/ios/transports/daily", - "client/ios/transports/gemini-websocket", - "client/ios/transports/openai-webrtc", - "client/ios/transports/small-webrtc" - ] - } + "client/rtvi-standard", + "client/concepts/choosing-a-transport", + "client/concepts/session-lifecycle", + "client/concepts/events-and-callbacks", + "client/concepts/media-management" ] }, { - "group": "Android SDK", + "group": "Guides", "pages": [ - "client/android/introduction", - "client/android/api-reference", - { - "group": "Transport Packages", - "pages": [ - "client/android/transports/daily", - "client/android/transports/gemini-websocket", - "client/android/transports/openai-webrtc", - "client/android/transports/small-webrtc" - ] - } + "client/guides/building-a-voice-ui", + "client/guides/custom-messaging" ] }, - { - "group": "C++ SDK", - "pages": ["client/c++/introduction", "client/c++/transport"] - }, { "group": "Voice UI Kit", "pages": ["client/voice-ui-kit"] @@ -726,6 +659,93 @@ } ] }, + { + "group": "Client SDKs", + "pages": [ + { + "group": "JavaScript SDK", + "pages": [ + "api-reference/client/js/overview", + "api-reference/client/js/client-constructor", + "api-reference/client/js/client-methods", + "api-reference/client/js/callbacks", + "api-reference/client/js/errors", + { + "group": "Transports", + "pages": [ + "api-reference/client/js/transports/transport", + "api-reference/client/js/transports/daily", + "api-reference/client/js/transports/small-webrtc", + "api-reference/client/js/transports/websocket", + "api-reference/client/js/transports/gemini", + "api-reference/client/js/transports/openai-webrtc" + ] + } + ] + }, + { + "group": "React SDK", + "pages": [ + "api-reference/client/react/overview", + "api-reference/client/react/components", + "api-reference/client/react/hooks" + ] + }, + { + "group": "React Native SDK", + "pages": [ + "api-reference/client/react-native/overview", + "api-reference/client/react-native/api-reference", + { + "group": "Transports", + "pages": [ + "api-reference/client/react-native/transports/daily", + "api-reference/client/react-native/transports/small-webrtc" + ] + } + ] + }, + { + "group": "iOS SDK", + "pages": [ + "api-reference/client/ios/overview", + "api-reference/client/ios/api-reference", + { + "group": "Transports", + "pages": [ + "api-reference/client/ios/transports/daily", + "api-reference/client/ios/transports/gemini-websocket", + "api-reference/client/ios/transports/openai-webrtc", + "api-reference/client/ios/transports/small-webrtc" + ] + } + ] + }, + { + "group": "Android SDK", + "pages": [ + "api-reference/client/android/overview", + "api-reference/client/android/api-reference", + { + "group": "Transports", + "pages": [ + "api-reference/client/android/transports/daily", + "api-reference/client/android/transports/gemini-websocket", + "api-reference/client/android/transports/openai-webrtc", + "api-reference/client/android/transports/small-webrtc" + ] + } + ] + }, + { + "group": "C++ SDK", + "pages": [ + "api-reference/client/cpp/overview", + "api-reference/client/cpp/transport" + ] + } + ] + }, { "group": "Pipecat Flows", "pages": [ @@ -2178,6 +2198,138 @@ { "source": "/api-reference/server/frameworks/rtvi/google-rtvi-observer", "destination": "/api-reference/server/rtvi/google-rtvi-observer" + }, + { + "source": "/client/js/introduction", + "destination": "/api-reference/client/js/overview" + }, + { + "source": "/client/js/api-reference/callbacks", + "destination": "/api-reference/client/js/callbacks" + }, + { + "source": "/client/js/api-reference/client-constructor", + "destination": "/api-reference/client/js/client-constructor" + }, + { + "source": "/client/js/api-reference/client-methods", + "destination": "/api-reference/client/js/client-methods" + }, + { + "source": "/client/js/api-reference/errors", + "destination": "/api-reference/client/js/errors" + }, + { + "source": "/client/js/api-reference/messages", + "destination": "/client/guides/custom-messaging" + }, + { + "source": "/client/js/transports/daily", + "destination": "/api-reference/client/js/transports/daily" + }, + { + "source": "/client/js/transports/gemini", + "destination": "/api-reference/client/js/transports/gemini" + }, + { + "source": "/client/js/transports/openai-webrtc", + "destination": "/api-reference/client/js/transports/openai-webrtc" + }, + { + "source": "/client/js/transports/small-webrtc", + "destination": "/api-reference/client/js/transports/small-webrtc" + }, + { + "source": "/client/js/transports/transport", + "destination": "/api-reference/client/js/transports/transport" + }, + { + "source": "/client/js/transports/websocket", + "destination": "/api-reference/client/js/transports/websocket" + }, + { + "source": "/client/react/introduction", + "destination": "/api-reference/client/react/overview" + }, + { + "source": "/client/react/hooks", + "destination": "/api-reference/client/react/hooks" + }, + { + "source": "/client/react/components", + "destination": "/api-reference/client/react/components" + }, + { + "source": "/client/react-native/introduction", + "destination": "/api-reference/client/react-native/overview" + }, + { + "source": "/client/react-native/api-reference", + "destination": "/api-reference/client/react-native/api-reference" + }, + { + "source": "/client/react-native/transports/daily", + "destination": "/api-reference/client/react-native/transports/daily" + }, + { + "source": "/client/react-native/transports/small-webrtc", + "destination": "/api-reference/client/react-native/transports/small-webrtc" + }, + { + "source": "/client/ios/introduction", + "destination": "/api-reference/client/ios/overview" + }, + { + "source": "/client/ios/api-reference", + "destination": "/api-reference/client/ios/api-reference" + }, + { + "source": "/client/ios/transports/daily", + "destination": "/api-reference/client/ios/transports/daily" + }, + { + "source": "/client/ios/transports/gemini-websocket", + "destination": "/api-reference/client/ios/transports/gemini-websocket" + }, + { + "source": "/client/ios/transports/openai-webrtc", + "destination": "/api-reference/client/ios/transports/openai-webrtc" + }, + { + "source": "/client/ios/transports/small-webrtc", + "destination": "/api-reference/client/ios/transports/small-webrtc" + }, + { + "source": "/client/android/introduction", + "destination": "/api-reference/client/android/overview" + }, + { + "source": "/client/android/api-reference", + "destination": "/api-reference/client/android/api-reference" + }, + { + "source": "/client/android/transports/daily", + "destination": "/api-reference/client/android/transports/daily" + }, + { + "source": "/client/android/transports/gemini-websocket", + "destination": "/api-reference/client/android/transports/gemini-websocket" + }, + { + "source": "/client/android/transports/openai-webrtc", + "destination": "/api-reference/client/android/transports/openai-webrtc" + }, + { + "source": "/client/android/transports/small-webrtc", + "destination": "/api-reference/client/android/transports/small-webrtc" + }, + { + "source": "/client/c++/introduction", + "destination": "/api-reference/client/cpp/overview" + }, + { + "source": "/client/c++/transport", + "destination": "/api-reference/client/cpp/transport" } ] } diff --git a/images/client-server-architecture.png b/images/client-server-architecture.png new file mode 100644 index 00000000..5d9f58f9 Binary files /dev/null and b/images/client-server-architecture.png differ diff --git a/overview/clients.mdx b/overview/clients.mdx index aa15a526..dff9ad9b 100644 --- a/overview/clients.mdx +++ b/overview/clients.mdx @@ -12,7 +12,7 @@ Pipecat Clients are a family of SDKs that connect users to your Pipecat agents t title="JavaScript" icon="js" color="#f7e014" - href="/client/js/introduction" + href="/api-reference/client/js/overview" > Web applications with vanilla JavaScript     @@ -20,7 +20,7 @@ Pipecat Clients are a family of SDKs that connect users to your Pipecat agents t title="React" icon="react" color="#56c4db" - href="/client/react/introduction" + href="/api-reference/client/react/overview" > React applications with hooks and components @@ -28,7 +28,7 @@ Pipecat Clients are a family of SDKs that connect users to your Pipecat agents t title="React Native" icon="react" color="#56c4db" - href="/client/react-native/introduction" + href="/api-reference/client/react-native/overview" > Cross-platform mobile apps @@ -36,7 +36,7 @@ Pipecat Clients are a family of SDKs that connect users to your Pipecat agents t title="iOS (Swift)" icon="swift" color="#F05138" - href="/client/ios/introduction" + href="/api-reference/client/ios/overview" > Native iOS applications @@ -44,11 +44,11 @@ Pipecat Clients are a family of SDKs that connect users to your Pipecat agents t title="Android (Kotlin)" icon="android" color="#78C257" - href="/client/android/introduction" + href="/api-reference/client/android/overview" > Native Android applications - + High-performance native applications     diff --git a/pipecat/features/gemini-live.mdx b/pipecat/features/gemini-live.mdx index 633681c9..4e32b6e7 100644 --- a/pipecat/features/gemini-live.mdx +++ b/pipecat/features/gemini-live.mdx @@ -142,7 +142,7 @@ Each starter includes a `pcc-deploy.toml` file with sensible defaults for agent > Add external integrations and dynamic responses - + Build custom web interfaces