diff --git a/doc/source/models/builtin/audio/index.rst b/doc/source/models/builtin/audio/index.rst index f88964d76b..141d8fb735 100644 --- a/doc/source/models/builtin/audio/index.rst +++ b/doc/source/models/builtin/audio/index.rst @@ -89,6 +89,10 @@ The following is a list of built-in audio models in Xinference: sensevoicesmall + speech_campplus_sv_zh-cn_16k-common + + speech_campplus_sv_zh_en_16k-common_advanced + voxcpm2 whisper-base @@ -130,4 +134,3 @@ The following is a list of built-in audio models in Xinference: whisper-tiny.en whisper-tiny.en-mlx - \ No newline at end of file diff --git a/doc/source/models/builtin/audio/speech_campplus_sv_zh-cn_16k-common.rst b/doc/source/models/builtin/audio/speech_campplus_sv_zh-cn_16k-common.rst new file mode 100644 index 0000000000..b939feaa41 --- /dev/null +++ b/doc/source/models/builtin/audio/speech_campplus_sv_zh-cn_16k-common.rst @@ -0,0 +1,38 @@ +.. _models_builtin_speech_campplus_sv_zh-cn_16k-common: + +======================================== +speech_campplus_sv_zh-cn_16k-common +======================================== + +- **Model Name:** speech_campplus_sv_zh-cn_16k-common +- **Model Family:** campplus +- **Abilities:** ['speaker_embedding'] +- **Multilingual:** False + +This CAMPPlus speaker-verification model converts a speech sample into a +fixed-length representation of speaker identity. It is intended for Chinese +speech and can be used as the embedding stage in speaker verification and +speaker identification systems. It does not transcribe the spoken content. + +Specifications +^^^^^^^^^^^^^^ + +- **Model ID:** iic/speech_campplus_sv_zh-cn_16k-common +- **Model Hub:** `ModelScope `__ +- **Embedding Dimensions:** 192 +- **Sample Rate:** 16 kHz + +Output and comparison +^^^^^^^^^^^^^^^^^^^^^ + +Each request returns one 192-dimensional vector. Store the vector in your +application and use cosine similarity to compare samples. Select a similarity +threshold using representative recordings from the microphones, speakers, and +acoustic conditions expected in production. + +Execute the following command to launch the model:: + + xinference launch --model-name speech_campplus_sv_zh-cn_16k-common --model-type audio + +See :ref:`audio` for Web UI, cURL, and Python examples for the +``/v1/audio/embeddings`` endpoint. diff --git a/doc/source/models/builtin/audio/speech_campplus_sv_zh_en_16k-common_advanced.rst b/doc/source/models/builtin/audio/speech_campplus_sv_zh_en_16k-common_advanced.rst new file mode 100644 index 0000000000..06b36c70fe --- /dev/null +++ b/doc/source/models/builtin/audio/speech_campplus_sv_zh_en_16k-common_advanced.rst @@ -0,0 +1,38 @@ +.. _models_builtin_speech_campplus_sv_zh_en_16k-common_advanced: + +==================================================== +speech_campplus_sv_zh_en_16k-common_advanced +==================================================== + +- **Model Name:** speech_campplus_sv_zh_en_16k-common_advanced +- **Model Family:** campplus +- **Abilities:** ['speaker_embedding'] +- **Multilingual:** True + +This multilingual CAMPPlus speaker-verification model converts a Chinese or +English speech sample into a fixed-length representation of speaker identity. +It can be used as the embedding stage in speaker verification and speaker +identification systems. It does not transcribe the spoken content. + +Specifications +^^^^^^^^^^^^^^ + +- **Model ID:** iic/speech_campplus_sv_zh_en_16k-common_advanced +- **Model Hub:** `ModelScope `__ +- **Embedding Dimensions:** 192 +- **Sample Rate:** 16 kHz + +Output and comparison +^^^^^^^^^^^^^^^^^^^^^ + +Each request returns one 192-dimensional vector. Store the vector in your +application and use cosine similarity to compare samples. Select a similarity +threshold using representative recordings from the microphones, speakers, and +acoustic conditions expected in production. + +Execute the following command to launch the model:: + + xinference launch --model-name speech_campplus_sv_zh_en_16k-common_advanced --model-type audio + +See :ref:`audio` for Web UI, cURL, and Python examples for the +``/v1/audio/embeddings`` endpoint. diff --git a/doc/source/models/model_abilities/audio.rst b/doc/source/models/model_abilities/audio.rst index 676995eb03..9a9385b879 100644 --- a/doc/source/models/model_abilities/audio.rst +++ b/doc/source/models/model_abilities/audio.rst @@ -4,19 +4,20 @@ Audio ===== -Learn how to turn audio into text or text into audio with Xinference. +Learn how to turn audio into text, text into audio, or audio into speaker embeddings with Xinference. Introduction ================== -The Audio API provides three methods for interacting with audio: +The Audio API provides four methods for interacting with audio: * The transcriptions endpoint transcribes audio into the input language. * The translations endpoint translates audio into English. * The speech endpoint generates audio from the input text. +* The embeddings endpoint extracts a speaker embedding from an audio file. .. list-table:: @@ -35,6 +36,9 @@ The Audio API provides three methods for interacting with audio: * - Speech API - /v1/audio/speech + * - Speaker Embedding API + - /v1/audio/embeddings + Supported models ------------------- @@ -97,6 +101,12 @@ Text to audio (TTS) * :ref:`Kokoro-82M-MLX ` * :ref:`MegaTTS3 ` +Speaker embeddings +~~~~~~~~~~~~~~~~~~ + +* :ref:`speech_campplus_sv_zh-cn_16k-common ` +* :ref:`speech_campplus_sv_zh_en_16k-common_advanced ` + **Models supporting voice cloning** (requires reference audio): * :ref:`CosyVoice-300M ` @@ -118,6 +128,61 @@ For Mac M-series chips only: Quickstart =================== +Speaker Embeddings +-------------------- + +The Speaker Embedding API accepts one audio file and returns one speaker +embedding. The built-in CAMPPlus models return a 192-dimensional vector. The +endpoint is intentionally stateless: applications can store the returned vectors +and use cosine similarity for speaker verification or 1:N speaker identification. +The request uses ``multipart/form-data``: ``model`` is the UID of a running +speaker-embedding model and ``file`` is the audio sample. Unlike the general +``/v1/embeddings`` endpoint, this endpoint returns one embedding object rather +than a list of text embeddings. + +.. tabs:: + + .. tab:: Web UI + + Open **Running Models**, select a running CAMPPlus model, and upload a clear + speech sample in the **Speaker Embedding** panel. Select **Extract embedding** + to inspect the vector and copy it from the results panel. + + .. code-tab:: bash cURL + + curl -X POST \ + 'http://:/v1/audio/embeddings' \ + -H 'accept: application/json' \ + -F 'model=' \ + -F 'file=@speaker.wav' + + .. code-tab:: python Xinference Python Client + + from xinference.client import Client + + client = Client("http://:") + model = client.get_model("") + + with open("speaker.wav", "rb") as audio_file: + result = model.create_embedding(audio_file.read()) + + embedding = result["embedding"] + + .. code-tab:: json output + + { + "object": "embedding", + "model": "", + "dimensions": 192, + "embedding": [0.0123, -0.0456, 0.0789] + } + +ModelScope decodes the input, converts multi-channel audio to one channel, and +resamples it to the model's 16 kHz sample rate. The returned vector preserves +the model output. Use cosine similarity when comparing two vectors; choose a +verification or identification threshold using representative audio from your +own application. + Transcription -------------------- diff --git a/doc/source/user_guide/client_api.rst b/doc/source/user_guide/client_api.rst index cbb7ae836c..3e26ba5b95 100644 --- a/doc/source/user_guide/client_api.rst +++ b/doc/source/user_guide/client_api.rst @@ -301,7 +301,7 @@ Output: Audio ~~~~~ -To list the available built-in image models: +To list the available built-in audio models: .. code-block:: @@ -316,6 +316,8 @@ To list the available built-in image models: audio whisper-medium.en whisper False True audio whisper-tiny whisper True True audio whisper-tiny.en whisper False True + audio speech_campplus_sv_zh-cn_16k-common campplus False True + audio speech_campplus_sv_zh_en_16k-common_advanced campplus True True To initiate an audio model and get text from an audio: @@ -367,6 +369,61 @@ Output: Translation(text=' This list lists the airlines in Hong Kong.') +Speaker Embeddings +================== + +Speaker-embedding audio models extract a fixed-length representation of speaker +identity. Xinference provides two built-in CAMPPlus models: +``speech_campplus_sv_zh-cn_16k-common`` and +``speech_campplus_sv_zh_en_16k-common_advanced``. Both return a 192-dimensional +vector through the ``speaker_embedding`` ability. + +Launch a model and call it with the Xinference client: + +.. code-block:: python + + from xinference.client import Client + + client = Client("http://localhost:9997") + model_uid = client.launch_model( + model_name="speech_campplus_sv_zh-cn_16k-common", + model_type="audio", + ) + model = client.get_model(model_uid) + + with open("speaker.wav", "rb") as audio_file: + result = model.create_embedding(audio_file.read()) + + print(result["dimensions"]) + embedding = result["embedding"] + +The equivalent HTTP request uploads the model UID and audio file as multipart +form fields: + +.. code-block:: bash + + curl -X POST 'http://localhost:9997/v1/audio/embeddings' \ + -H 'accept: application/json' \ + -F 'model=' \ + -F 'file=@speaker.wav' + +The response contains one vector rather than the list-shaped response returned +by the text Embeddings API: + +.. code-block:: json + + { + "object": "embedding", + "model": "", + "dimensions": 192, + "embedding": [0.0123, -0.0456, 0.0789] + } + +Compare vectors with cosine similarity for speaker verification or speaker +identification. See :ref:`audio` for Web UI usage, input processing details, +and the complete Speaker Embedding API example. + + Rerank ~~~~~~ To launch a rerank model and compute the similarity scores: diff --git a/frontend/src/components/pages/running-model-detail/capability-config.tsx b/frontend/src/components/pages/running-model-detail/capability-config.tsx index f90b96d7ce..9fe41c519d 100644 --- a/frontend/src/components/pages/running-model-detail/capability-config.tsx +++ b/frontend/src/components/pages/running-model-detail/capability-config.tsx @@ -25,6 +25,7 @@ import { OcrPanel, EmbedPanel, RerankPanel, + SpeakerEmbeddingPanel, SpeechPanel, TextPromptPanel, TextToImagePanel, @@ -175,6 +176,16 @@ function audioTextFormData(context: TransformContext) { return formData; } +function audioEmbeddingFormData(context: TransformContext) { + const { modelUid, values } = context; + const audio = firstUpload(values, 'file'); + const formData = new FormData(); + + formData.append('model', modelUid); + if (audio) formData.append('file', audio.file); + return formData; +} + export const CAPABILITY_CONFIGS: Partial> = { [ModelAbility.Generate]: { ability: ModelAbility.Generate, @@ -235,6 +246,25 @@ export const CAPABILITY_CONFIGS: Partial> input: stringValue(values.input), }), }, + [ModelAbility.SpeakerEmbedding]: { + ability: ModelAbility.SpeakerEmbedding, + label: 'Speaker Embedding', + icon: Binary, + requestApi: '/v1/audio/embeddings', + codeExample: { + method: 'POST', + contentType: 'form', + fields: [ + { key: 'model', required: true }, + { key: 'file', required: true, type: 'file', value: '/path/to/speaker.wav' }, + ], + }, + initialValues: { file: [] }, + submitLabel: 'Extract embedding', + formPanel: SpeakerEmbeddingPanel, + resultPanel: ResultPanels.Universal, + transformValues: audioEmbeddingFormData, + }, [ModelAbility.Rerank]: { ability: ModelAbility.Rerank, label: 'Rerank', @@ -432,7 +462,7 @@ export const CAPABILITY_CONFIGS: Partial> key: 'kwargs', value: { ...imageKwargsExample, - strength: 0.6 + strength: 0.6, }, stringify: true, comment: 'Optional(other key/value)', @@ -471,7 +501,7 @@ export const CAPABILITY_CONFIGS: Partial> key: 'kwargs', value: { ...imageKwargsExample, - strength: 0.6 + strength: 0.6, }, stringify: true, comment: 'Optional(other key/value)', diff --git a/frontend/src/components/pages/running-model-detail/components/try-api-drawer.tsx b/frontend/src/components/pages/running-model-detail/components/try-api-drawer.tsx index 56d1e66011..9c8ef63c29 100644 --- a/frontend/src/components/pages/running-model-detail/components/try-api-drawer.tsx +++ b/frontend/src/components/pages/running-model-detail/components/try-api-drawer.tsx @@ -6,7 +6,14 @@ import { useRouter } from 'next/navigation'; import { Button } from '@/components/ui/button'; import { JSONSyntaxHighlighter } from '@/components/ui/json-syntax-highlighter'; -import { Sheet, SheetClose, SheetContent, SheetHeader, SheetTitle } from '@/components/ui/sheet'; +import { + Sheet, + SheetClose, + SheetContent, + SheetDescription, + SheetHeader, + SheetTitle, +} from '@/components/ui/sheet'; import { Tabs, TabsContent, TabsList, TabsTrigger } from '@/components/ui/tabs'; import { ModelAbility } from '@/constants'; import { @@ -21,6 +28,7 @@ import { useMenuAuth } from '@/hooks/use-menu-auth'; import { copyToClipboard, getApiUrl } from '@/lib/utils'; import { CAPABILITY_CONFIGS } from '../capability-config'; +import { getPrimaryModelAbilities } from '../utils'; interface TryApiDrawerProps { open: boolean; @@ -470,7 +478,7 @@ function getCodeExample( } export function getTryApiAbility(abilities: ModelAbility[] = []) { - const primaryAbilities = abilities.filter((ability) => !ability.includes('_')); + const primaryAbilities = getPrimaryModelAbilities(abilities); return primaryAbilities.find((ability) => ability === ModelAbility.Chat) || primaryAbilities[0]; } @@ -493,8 +501,11 @@ export function TryApiDrawer({ return ( - - + +
- Try To API +
+ Try To API + + API request examples for the selected model ability. + +
- {modelUid} + {modelUid} } loading={loading} className="gap-5" + headerClassName="flex-col items-stretch gap-3 sm:flex-row sm:items-center" extraContent={ -
+
{!isChat && (