From d1c7f45865b809f0b6fea191229b925af11dc5c3 Mon Sep 17 00:00:00 2001 From: luxiang Date: Tue, 4 Aug 2026 23:35:33 +0800 Subject: [PATCH 1/4] feat(audio): add speaker embedding extraction --- doc/source/models/builtin/audio/index.rst | 5 +- .../speech_campplus_sv_zh-cn_16k-common.rst | 24 ++++ ..._campplus_sv_zh_en_16k-common_advanced.rst | 24 ++++ doc/source/models/model_abilities/audio.rst | 57 +++++++- .../capability-config.tsx | 33 ++++- .../panels/form-panels.tsx | 12 ++ frontend/src/constants/index.ts | 1 + frontend/src/constants/register.ts | 10 +- frontend/src/i18n/locales/en.ts | 4 +- frontend/src/i18n/locales/ja.ts | 1 + frontend/src/i18n/locales/ko.ts | 1 + frontend/src/i18n/locales/zh.ts | 1 + xinference/api/restful_api.py | 26 ++++ xinference/api/routers/audio.py | 8 +- xinference/api/tests/test_audio_embedding.py | 70 +++++++++ .../client/restful/async_restful_client.py | 19 +++ xinference/client/restful/restful_client.py | 18 +++ .../tests/test_audio_embedding_handle.py | 69 +++++++++ xinference/core/model.py | 14 ++ xinference/model/audio/__init__.py | 4 +- xinference/model/audio/core.py | 12 +- xinference/model/audio/model_spec.json | 54 +++++++ xinference/model/audio/speaker_embedding.py | 134 ++++++++++++++++++ .../audio/tests/test_speaker_embedding.py | 124 ++++++++++++++++ xinference/types.py | 7 + 25 files changed, 720 insertions(+), 12 deletions(-) create mode 100644 doc/source/models/builtin/audio/speech_campplus_sv_zh-cn_16k-common.rst create mode 100644 doc/source/models/builtin/audio/speech_campplus_sv_zh_en_16k-common_advanced.rst create mode 100644 xinference/api/tests/test_audio_embedding.py create mode 100644 xinference/client/tests/test_audio_embedding_handle.py create mode 100644 xinference/model/audio/speaker_embedding.py create mode 100644 xinference/model/audio/tests/test_speaker_embedding.py diff --git a/doc/source/models/builtin/audio/index.rst b/doc/source/models/builtin/audio/index.rst index f88964d76b..141d8fb735 100644 --- a/doc/source/models/builtin/audio/index.rst +++ b/doc/source/models/builtin/audio/index.rst @@ -89,6 +89,10 @@ The following is a list of built-in audio models in Xinference: sensevoicesmall + speech_campplus_sv_zh-cn_16k-common + + speech_campplus_sv_zh_en_16k-common_advanced + voxcpm2 whisper-base @@ -130,4 +134,3 @@ The following is a list of built-in audio models in Xinference: whisper-tiny.en whisper-tiny.en-mlx - \ No newline at end of file diff --git a/doc/source/models/builtin/audio/speech_campplus_sv_zh-cn_16k-common.rst b/doc/source/models/builtin/audio/speech_campplus_sv_zh-cn_16k-common.rst new file mode 100644 index 0000000000..b9d4a4b7ed --- /dev/null +++ b/doc/source/models/builtin/audio/speech_campplus_sv_zh-cn_16k-common.rst @@ -0,0 +1,24 @@ +.. _models_builtin_speech_campplus_sv_zh-cn_16k-common: + +======================================== +speech_campplus_sv_zh-cn_16k-common +======================================== + +- **Model Name:** speech_campplus_sv_zh-cn_16k-common +- **Model Family:** campplus +- **Abilities:** ['speaker_embedding'] +- **Multilingual:** False + +Specifications +^^^^^^^^^^^^^^ + +- **Model ID:** iic/speech_campplus_sv_zh-cn_16k-common +- **Model Hub:** `ModelScope `__ +- **Embedding Dimensions:** 192 +- **Sample Rate:** 16 kHz + +Execute the following command to launch the model:: + + xinference launch --model-name speech_campplus_sv_zh-cn_16k-common --model-type audio + +See :ref:`audio` for the Speaker Embedding API usage. diff --git a/doc/source/models/builtin/audio/speech_campplus_sv_zh_en_16k-common_advanced.rst b/doc/source/models/builtin/audio/speech_campplus_sv_zh_en_16k-common_advanced.rst new file mode 100644 index 0000000000..210c1d7536 --- /dev/null +++ b/doc/source/models/builtin/audio/speech_campplus_sv_zh_en_16k-common_advanced.rst @@ -0,0 +1,24 @@ +.. _models_builtin_speech_campplus_sv_zh_en_16k-common_advanced: + +==================================================== +speech_campplus_sv_zh_en_16k-common_advanced +==================================================== + +- **Model Name:** speech_campplus_sv_zh_en_16k-common_advanced +- **Model Family:** campplus +- **Abilities:** ['speaker_embedding'] +- **Multilingual:** True + +Specifications +^^^^^^^^^^^^^^ + +- **Model ID:** iic/speech_campplus_sv_zh_en_16k-common_advanced +- **Model Hub:** `ModelScope `__ +- **Embedding Dimensions:** 192 +- **Sample Rate:** 16 kHz + +Execute the following command to launch the model:: + + xinference launch --model-name speech_campplus_sv_zh_en_16k-common_advanced --model-type audio + +See :ref:`audio` for the Speaker Embedding API usage. diff --git a/doc/source/models/model_abilities/audio.rst b/doc/source/models/model_abilities/audio.rst index 676995eb03..8b02cbeace 100644 --- a/doc/source/models/model_abilities/audio.rst +++ b/doc/source/models/model_abilities/audio.rst @@ -4,19 +4,20 @@ Audio ===== -Learn how to turn audio into text or text into audio with Xinference. +Learn how to turn audio into text, text into audio, or audio into speaker embeddings with Xinference. Introduction ================== -The Audio API provides three methods for interacting with audio: +The Audio API provides four methods for interacting with audio: * The transcriptions endpoint transcribes audio into the input language. * The translations endpoint translates audio into English. * The speech endpoint generates audio from the input text. +* The embeddings endpoint extracts a speaker embedding from an audio file. .. list-table:: @@ -35,6 +36,9 @@ The Audio API provides three methods for interacting with audio: * - Speech API - /v1/audio/speech + * - Speaker Embedding API + - /v1/audio/embeddings + Supported models ------------------- @@ -97,6 +101,12 @@ Text to audio (TTS) * :ref:`Kokoro-82M-MLX ` * :ref:`MegaTTS3 ` +Speaker embeddings +~~~~~~~~~~~~~~~~~~ + +* :ref:`speech_campplus_sv_zh-cn_16k-common ` +* :ref:`speech_campplus_sv_zh_en_16k-common_advanced ` + **Models supporting voice cloning** (requires reference audio): * :ref:`CosyVoice-300M ` @@ -118,6 +128,49 @@ For Mac M-series chips only: Quickstart =================== +Speaker Embeddings +-------------------- + +The Speaker Embedding API accepts one audio file and returns one speaker +embedding. The built-in CAMPPlus models return a 192-dimensional vector. The +endpoint is intentionally stateless: applications can store the returned vectors +and use cosine similarity for speaker verification or 1:N speaker identification. + +.. tabs:: + + .. code-tab:: bash cURL + + curl -X POST \ + 'http://:/v1/audio/embeddings' \ + -H 'accept: application/json' \ + -F 'model=' \ + -F 'file=@speaker.wav' + + .. code-tab:: python Xinference Python Client + + from xinference.client import Client + + client = Client("http://:") + model = client.get_model("") + + with open("speaker.wav", "rb") as audio_file: + result = model.create_embedding(audio_file.read()) + + embedding = result["embedding"] + + .. code-tab:: json output + + { + "object": "embedding", + "model": "", + "dimensions": 192, + "embedding": [0.0123, -0.0456, 0.0789] + } + +ModelScope decodes the input, converts multi-channel audio to one channel, and +resamples it to the model's 16 kHz sample rate. The returned vector preserves +the model output. Use cosine similarity when comparing two vectors. + Transcription -------------------- diff --git a/frontend/src/components/pages/running-model-detail/capability-config.tsx b/frontend/src/components/pages/running-model-detail/capability-config.tsx index f90b96d7ce..44959e13e3 100644 --- a/frontend/src/components/pages/running-model-detail/capability-config.tsx +++ b/frontend/src/components/pages/running-model-detail/capability-config.tsx @@ -25,6 +25,7 @@ import { OcrPanel, EmbedPanel, RerankPanel, + SpeakerEmbeddingPanel, SpeechPanel, TextPromptPanel, TextToImagePanel, @@ -175,6 +176,16 @@ function audioTextFormData(context: TransformContext) { return formData; } +function audioEmbeddingFormData(context: TransformContext) { + const { modelUid, values } = context; + const audio = firstUpload(values, 'file'); + const formData = new FormData(); + + formData.append('model', modelUid); + if (audio) formData.append('file', audio.file); + return formData; +} + export const CAPABILITY_CONFIGS: Partial> = { [ModelAbility.Generate]: { ability: ModelAbility.Generate, @@ -235,6 +246,24 @@ export const CAPABILITY_CONFIGS: Partial> input: stringValue(values.input), }), }, + [ModelAbility.SpeakerEmbedding]: { + ability: ModelAbility.SpeakerEmbedding, + label: 'Speaker Embedding', + icon: Binary, + requestApi: '/v1/audio/embeddings', + codeExample: { + method: 'POST', + contentType: 'form', + fields: [ + { key: 'model', required: true }, + { key: 'file', required: true, type: 'file', value: '/path/to/speaker.wav' }, + ], + }, + initialValues: { file: [] }, + formPanel: SpeakerEmbeddingPanel, + resultPanel: ResultPanels.Universal, + transformValues: audioEmbeddingFormData, + }, [ModelAbility.Rerank]: { ability: ModelAbility.Rerank, label: 'Rerank', @@ -432,7 +461,7 @@ export const CAPABILITY_CONFIGS: Partial> key: 'kwargs', value: { ...imageKwargsExample, - strength: 0.6 + strength: 0.6, }, stringify: true, comment: 'Optional(other key/value)', @@ -471,7 +500,7 @@ export const CAPABILITY_CONFIGS: Partial> key: 'kwargs', value: { ...imageKwargsExample, - strength: 0.6 + strength: 0.6, }, stringify: true, comment: 'Optional(other key/value)', diff --git a/frontend/src/components/pages/running-model-detail/panels/form-panels.tsx b/frontend/src/components/pages/running-model-detail/panels/form-panels.tsx index 1056e964db..ebfaad8bab 100644 --- a/frontend/src/components/pages/running-model-detail/panels/form-panels.tsx +++ b/frontend/src/components/pages/running-model-detail/panels/form-panels.tsx @@ -365,6 +365,18 @@ export function AudioToTextPanel() { ); } +export function SpeakerEmbeddingPanel() { + return ( + + + + ); +} + export function SpeechPanel({ model }: CapabilityFormProps) { const supportsVoiceCloning = model.model_ability.includes(ModelAbility.Text2audioVoiceCloning); diff --git a/frontend/src/constants/index.ts b/frontend/src/constants/index.ts index 08c2297bf8..6c393eedb9 100644 --- a/frontend/src/constants/index.ts +++ b/frontend/src/constants/index.ts @@ -57,6 +57,7 @@ export enum ModelAbility { Image2image = 'image2image', Inpainting = 'inpainting', Audio2text = 'audio2text', + SpeakerEmbedding = 'speaker_embedding', Text2audio = 'text2audio', Audio2audio = 'audio2audio', Text2video = 'text2video', diff --git a/frontend/src/constants/register.ts b/frontend/src/constants/register.ts index 5b75e4af02..eb44f9cb05 100644 --- a/frontend/src/constants/register.ts +++ b/frontend/src/constants/register.ts @@ -226,10 +226,12 @@ export const MODEL_ABILITY_IMAGE_OPTIONS = [ export enum ModelAbilityForAudio { Text2audio = 'text2audio', Audio2text = 'audio2text', + SpeakerEmbedding = 'speaker_embedding', } export const MODEL_ABILITY_AUDIO_OPTIONS = [ { label: 'Text2audio', value: ModelAbilityForAudio.Text2audio }, { label: 'Audio2text', value: ModelAbilityForAudio.Audio2text }, + { label: 'SpeakerEmbedding', value: ModelAbilityForAudio.SpeakerEmbedding }, ]; export const MODEL_ABILITY_OPTIONS_MAP: Partial< Record @@ -256,6 +258,7 @@ export enum ModelFamilyForAudio { MegaTTS = 'MegaTTS', MeloTTS = 'MeloTTS', Funasr = 'funasr', + CampPlus = 'campplus', } export const MODEL_FAMILY_AUDIO_OPTIONS = [ { label: ModelFamilyForAudio.Whisper, value: ModelFamilyForAudio.Whisper }, @@ -268,6 +271,7 @@ export const MODEL_FAMILY_AUDIO_OPTIONS = [ { label: ModelFamilyForAudio.MegaTTS, value: ModelFamilyForAudio.MegaTTS }, { label: ModelFamilyForAudio.MeloTTS, value: ModelFamilyForAudio.MeloTTS }, { label: ModelFamilyForAudio.Funasr, value: ModelFamilyForAudio.Funasr }, + { label: ModelFamilyForAudio.CampPlus, value: ModelFamilyForAudio.CampPlus }, ]; export const MODEL_FAMILY_OPTIONS_MAP: Partial< Record @@ -293,7 +297,7 @@ export enum ModelFormat { AWQ = 'awq', FP8 = 'fp8', MLX = 'mlx', - GGMLV3 ='ggmlv3' + GGMLV3 = 'ggmlv3', } export const MODEL_FORMAT_LLM_OPTIONS = [ { value: ModelFormat.PyTorch, label: 'PyTorch' }, @@ -323,8 +327,8 @@ export enum ControlnetModelFamily { Controlnet = 'controlnet', } export const CONTROLNET_MODEL_FAMILY_OPTIONS = [ - { label: ControlnetModelFamily.Controlnet, value: ControlnetModelFamily.Controlnet} -] + { label: ControlnetModelFamily.Controlnet, value: ControlnetModelFamily.Controlnet }, +]; export const REGISTER_MODEL_INIT_DATA = { [ModelType.LLM]: { version: 2, diff --git a/frontend/src/i18n/locales/en.ts b/frontend/src/i18n/locales/en.ts index df74c1ab54..d0cb16fb76 100644 --- a/frontend/src/i18n/locales/en.ts +++ b/frontend/src/i18n/locales/en.ts @@ -47,6 +47,7 @@ const en = { inpainting: 'Image Inpainting', ocr: 'Optical Character Recognition (OCR)', audio2text: 'Audio to Text', + speaker_embedding: 'Speaker Embedding', text2audio: 'Text to Audio', text2audio_zero_shot: 'Text to Audio (Zero-Shot)', text2audio_voice_cloning: 'Text to Audio (Voice Cloning)', @@ -696,7 +697,8 @@ const en = { }, common: { accessDenied: 'Access Denied', - accessDeniedDescription: 'You do not have the required permission ({{scope}}) to access this page.', + accessDeniedDescription: + 'You do not have the required permission ({{scope}}) to access this page.', valueEmpty: 'Cannot be empty', patternError: 'Invalid format, please check', loginSuccess: 'Login successful', diff --git a/frontend/src/i18n/locales/ja.ts b/frontend/src/i18n/locales/ja.ts index 5b376db278..5911ce85e8 100644 --- a/frontend/src/i18n/locales/ja.ts +++ b/frontend/src/i18n/locales/ja.ts @@ -47,6 +47,7 @@ const ja = { inpainting: '画像修復', ocr: '光学文字認識 (OCR)', audio2text: '音声からテキスト', + speaker_embedding: '話者埋め込み', text2audio: 'テキストから音声', text2audio_zero_shot: 'テキストから音声(ゼロショット)', text2audio_voice_cloning: 'テキストから音声(声のクローン)', diff --git a/frontend/src/i18n/locales/ko.ts b/frontend/src/i18n/locales/ko.ts index 00e3586241..4791d37ef4 100644 --- a/frontend/src/i18n/locales/ko.ts +++ b/frontend/src/i18n/locales/ko.ts @@ -47,6 +47,7 @@ const ko = { inpainting: '이미지 인페인팅', ocr: '광학 문자 인식 (OCR)', audio2text: '오디오에서 텍스트', + speaker_embedding: '화자 임베딩', text2audio: '텍스트에서 오디오', text2audio_zero_shot: '텍스트에서 오디오(제로 샷)', text2audio_voice_cloning: '텍스트에서 오디오(음성 클로닝)', diff --git a/frontend/src/i18n/locales/zh.ts b/frontend/src/i18n/locales/zh.ts index c5d1027e17..42df96c9cd 100644 --- a/frontend/src/i18n/locales/zh.ts +++ b/frontend/src/i18n/locales/zh.ts @@ -47,6 +47,7 @@ const zh = { inpainting: '图像修复', ocr: '光学字符识别', audio2text: '语音转文本', + speaker_embedding: '说话人向量', text2audio: '文本转语音', text2audio_zero_shot: '文本转语音(零样本)', text2audio_voice_cloning: '文本转语音(声音克隆)', diff --git a/xinference/api/restful_api.py b/xinference/api/restful_api.py index 63511e4670..b65c8863a8 100644 --- a/xinference/api/restful_api.py +++ b/xinference/api/restful_api.py @@ -1542,6 +1542,32 @@ async def rerank(self, request: Request) -> Response: self.handle_request_limit_error(e) raise HTTPException(status_code=500, detail=str(e)) + async def create_audio_embedding( + self, + request: Request, + model: str = Form(...), + file: UploadFile = File(media_type="application/octet-stream"), + ) -> Response: + model_uid = model + self._set_trace_model(model_uid) + self._set_trace_model_type("audio") + self._check_model_access(request, model_uid, "audio") + model_ref = await require_model( + self._get_supervisor_ref, model_uid, self._report_error_event + ) + + try: + embedding = await model_ref.create_audio_embedding( + audio=await file.read(), model_uid=model_uid + ) + return Response(content=embedding, media_type="application/json") + except Exception as e: + e = await self._get_model_last_error(model_ref.uid, e) + logger.error(e, exc_info=True) + await self._report_error_event(model_uid, str(e)) + self.handle_request_limit_error(e) + raise HTTPException(status_code=500, detail=str(e)) + async def create_transcriptions( self, request: Request, diff --git a/xinference/api/routers/audio.py b/xinference/api/routers/audio.py index b573007c75..8cb22f72cd 100644 --- a/xinference/api/routers/audio.py +++ b/xinference/api/routers/audio.py @@ -1,4 +1,4 @@ -"""Audio route registration (transcriptions, translations, speech).""" +"""Audio route registration (embeddings, transcriptions, translations, speech).""" from __future__ import annotations @@ -15,6 +15,12 @@ def register_routes(api: "RESTfulAPI") -> None: auth = api._auth_service is_auth = api.is_authenticated() + router.add_api_route( + "/v1/audio/embeddings", + api.create_audio_embedding, + methods=["POST"], + dependencies=([Security(auth, scopes=["models:read"])] if is_auth else None), + ) router.add_api_route( "/v1/audio/transcriptions", api.create_transcriptions, diff --git a/xinference/api/tests/test_audio_embedding.py b/xinference/api/tests/test_audio_embedding.py new file mode 100644 index 0000000000..d0f6e3b2af --- /dev/null +++ b/xinference/api/tests/test_audio_embedding.py @@ -0,0 +1,70 @@ +# Copyright 2022-2026 Xinference Holdings Pte. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from io import BytesIO +from unittest.mock import ANY, AsyncMock, MagicMock, patch + +import pytest +from fastapi import UploadFile + +from xinference.api.routers import audio + + +def test_audio_embedding_route_is_registered(): + captured = {} + + def add_api_route(path, endpoint, methods=None, **kwargs): + captured[(path, tuple(methods or []))] = endpoint + + api = MagicMock() + api._router.add_api_route.side_effect = add_api_route + api.is_authenticated.return_value = False + + audio.register_routes(api) + + assert captured[("/v1/audio/embeddings", ("POST",))] is api.create_audio_embedding + + +@pytest.mark.asyncio +async def test_create_audio_embedding_forwards_audio_and_model_uid(): + from xinference.api import restful_api + + api = MagicMock() + api._get_supervisor_ref = MagicMock() + api._report_error_event = AsyncMock() + model_ref = MagicMock(uid="replica-uid") + model_ref.create_audio_embedding = AsyncMock( + return_value=( + '{"object":"embedding","model":"speaker-model",' + '"dimensions":3,"embedding":[0.1,0.2,0.3]}' + ) + ) + upload = UploadFile(filename="speaker.wav", file=BytesIO(b"encoded-audio")) + + with patch.object(restful_api, "require_model", AsyncMock(return_value=model_ref)): + response = await restful_api.RESTfulAPI.create_audio_embedding( + api, + request=MagicMock(), + model="speaker-model", + file=upload, + ) + + assert response.media_type == "application/json" + assert b'"dimensions":3' in response.body + model_ref.create_audio_embedding.assert_awaited_once_with( + audio=b"encoded-audio", model_uid="speaker-model" + ) + api._set_trace_model.assert_called_once_with("speaker-model") + api._set_trace_model_type.assert_called_once_with("audio") + api._check_model_access.assert_called_once_with(ANY, "speaker-model", "audio") diff --git a/xinference/client/restful/async_restful_client.py b/xinference/client/restful/async_restful_client.py index f6ef90ce0a..5c1d391a23 100644 --- a/xinference/client/restful/async_restful_client.py +++ b/xinference/client/restful/async_restful_client.py @@ -22,6 +22,7 @@ if TYPE_CHECKING: from ...types import ( + AudioEmbedding, ChatCompletion, ChatCompletionChunk, Completion, @@ -916,6 +917,24 @@ async def chat( class AsyncRESTfulAudioModelHandle(AsyncRESTfulModelHandle): + async def create_embedding(self, audio: bytes) -> "AudioEmbedding": + """Create a speaker embedding from encoded audio bytes.""" + url = f"{self._base_url}/v1/audio/embeddings" + data = aiohttp.FormData() + data.add_field("model", self._model_uid) + data.add_field( + "file", audio, filename="file", content_type="application/octet-stream" + ) + response = await self.session.post(url, data=data, headers=self.auth_headers) + if response.status != 200: + raise RuntimeError( + "Failed to create the audio embedding, " + f"detail: {await _get_error_string(response)}" + ) + response_data = await response.json() + await _release_response(response) + return response_data + async def transcriptions( self, audio: bytes, diff --git a/xinference/client/restful/restful_client.py b/xinference/client/restful/restful_client.py index 52b97220ef..bc7480bf2e 100644 --- a/xinference/client/restful/restful_client.py +++ b/xinference/client/restful/restful_client.py @@ -21,6 +21,7 @@ if TYPE_CHECKING: from ...types import ( + AudioEmbedding, ChatCompletion, ChatCompletionChunk, Completion, @@ -831,6 +832,23 @@ def chat( class RESTfulAudioModelHandle(RESTfulModelHandle): + def create_embedding(self, audio: bytes) -> "AudioEmbedding": + """Create a speaker embedding from encoded audio bytes.""" + url = f"{self._base_url}/v1/audio/embeddings" + files = [("file", ("file", audio, "application/octet-stream"))] + response = self.session.post( + url, + data={"model": self._model_uid}, + files=files, + headers=self.auth_headers, + ) + if response.status_code != 200: + raise RuntimeError( + "Failed to create the audio embedding, " + f"detail: {_get_error_string(response)}" + ) + return response.json() + def transcriptions( self, audio: bytes, diff --git a/xinference/client/tests/test_audio_embedding_handle.py b/xinference/client/tests/test_audio_embedding_handle.py new file mode 100644 index 0000000000..b585cb9373 --- /dev/null +++ b/xinference/client/tests/test_audio_embedding_handle.py @@ -0,0 +1,69 @@ +# Copyright 2022-2026 Xinference Holdings Pte. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from unittest.mock import AsyncMock, MagicMock + +import pytest + +from xinference.client.restful.async_restful_client import AsyncRESTfulAudioModelHandle +from xinference.client.restful.restful_client import RESTfulAudioModelHandle + + +def test_sync_audio_handle_create_embedding(): + handle = RESTfulAudioModelHandle.__new__(RESTfulAudioModelHandle) + handle._model_uid = "speaker-model" + handle._base_url = "http://127.0.0.1:9997" + handle.auth_headers = {"Authorization": "Bearer token"} + handle.session = MagicMock() + response = MagicMock(status_code=200) + response.json.return_value = { + "object": "embedding", + "model": "speaker-model", + "dimensions": 3, + "embedding": [0.1, 0.2, 0.3], + } + handle.session.post.return_value = response + + result = handle.create_embedding(b"encoded-audio") + + assert result["embedding"] == [0.1, 0.2, 0.3] + _, kwargs = handle.session.post.call_args + assert kwargs["data"] == {"model": "speaker-model"} + assert kwargs["files"][0][1][1] == b"encoded-audio" + + +@pytest.mark.asyncio +async def test_async_audio_handle_create_embedding(): + handle = AsyncRESTfulAudioModelHandle.__new__(AsyncRESTfulAudioModelHandle) + handle._model_uid = "speaker-model" + handle._base_url = "http://127.0.0.1:9997" + handle.auth_headers = {"Authorization": "Bearer token"} + handle.session = MagicMock() + response = MagicMock(status=200) + response.json = AsyncMock( + return_value={ + "object": "embedding", + "model": "speaker-model", + "dimensions": 3, + "embedding": [0.1, 0.2, 0.3], + } + ) + response.wait_for_close = AsyncMock() + handle.session.post = AsyncMock(return_value=response) + + result = await handle.create_embedding(b"encoded-audio") + + assert result["dimensions"] == 3 + response.release.assert_called_once() + response.wait_for_close.assert_awaited_once() diff --git a/xinference/core/model.py b/xinference/core/model.py index 1330475bbd..aeda6aee02 100644 --- a/xinference/core/model.py +++ b/xinference/core/model.py @@ -930,6 +930,20 @@ async def create_embedding(self, input: Union[str, List[str]], *args, **kwargs): f"Model {self._model.model_spec} is not for creating embedding." ) + @request_limit + @log_async(logger=logger, ignore_kwargs=["audio"]) + async def create_audio_embedding(self, audio: bytes, *args, **kwargs): + self._require_ready() + kwargs.pop("request_id", None) + if hasattr(self._model, "create_embedding"): + return await self._call_wrapper_json( + self._model.create_embedding, audio, *args, **kwargs + ) + + raise AttributeError( + f"Model {self._model.model_spec} is not for creating audio embeddings." + ) + @request_limit @log_async(logger=logger) async def convert_ids_to_tokens( diff --git a/xinference/model/audio/__init__.py b/xinference/model/audio/__init__.py index 751849f730..39cd8dceb9 100644 --- a/xinference/model/audio/__init__.py +++ b/xinference/model/audio/__init__.py @@ -85,7 +85,9 @@ def _install(): # register model description after recording model revision for model_name, model_specs in BUILTIN_AUDIO_MODELS.items(): - model_spec = [x for x in model_specs if x.model_hub == "huggingface"][0] + model_spec = ( + [x for x in model_specs if x.model_hub == "huggingface"] + model_specs + )[0] if model_spec.model_name not in AUDIO_MODEL_DESCRIPTIONS: AUDIO_MODEL_DESCRIPTIONS.update(generate_audio_description(model_spec)) diff --git a/xinference/model/audio/core.py b/xinference/model/audio/core.py index ad7624a836..e2b83046ca 100644 --- a/xinference/model/audio/core.py +++ b/xinference/model/audio/core.py @@ -32,6 +32,7 @@ from .melotts import MeloTTSModel from .qwen3_asr import Qwen3ASRModel from .qwen3_tts import Qwen3TTSModel +from .speaker_embedding import ModelScopeSpeakerEmbeddingModel from .voxcpm import VoxCPMModel from .whisper import WhisperModel from .whisper_mlx import WhisperMLXModel @@ -127,7 +128,10 @@ def match_audio( + [x for x in model_families if x.model_hub == "huggingface"] )[0] else: - return [x for x in model_families if x.model_hub == "huggingface"][0] + return ( + [x for x in model_families if x.model_hub == "huggingface"] + + [x for x in model_families if x.model_hub == "modelscope"] + )[0] else: raise ValueError( @@ -163,6 +167,7 @@ def create_audio_model_instance( Qwen3ASRModel, Qwen3TTSModel, VoxCPMModel, + ModelScopeSpeakerEmbeddingModel, AudioEngineModel, ]: from ..cache_manager import CacheManager @@ -230,6 +235,7 @@ def create_audio_model_instance( Qwen3ASRModel, Qwen3TTSModel, VoxCPMModel, + ModelScopeSpeakerEmbeddingModel, ] if model_spec.model_family == "whisper": if not model_spec.engine: @@ -266,6 +272,10 @@ def create_audio_model_instance( model = Qwen3TTSModel(model_uid, model_path, model_spec, **kwargs) elif model_spec.model_family == "VoxCPM": model = VoxCPMModel(model_uid, model_path, model_spec, **kwargs) + elif model_spec.model_family == "campplus": + model = ModelScopeSpeakerEmbeddingModel( + model_uid, model_path, model_spec, **kwargs + ) else: raise Exception(f"Unsupported audio model family: {model_spec.model_family}") return model diff --git a/xinference/model/audio/model_spec.json b/xinference/model/audio/model_spec.json index 18ceca4dff..65a9e421d2 100644 --- a/xinference/model/audio/model_spec.json +++ b/xinference/model/audio/model_spec.json @@ -1949,6 +1949,60 @@ "featured": false, "updated_at": 1775717207 }, + { + "version": 2, + "model_name": "speech_campplus_sv_zh-cn_16k-common", + "model_family": "campplus", + "model_ability": [ + "speaker_embedding" + ], + "multilingual": false, + "model_src": { + "modelscope": { + "model_id": "iic/speech_campplus_sv_zh-cn_16k-common", + "model_revision": "v1.0.0" + } + }, + "virtualenv": { + "packages": [ + "modelscope>=1.19.0", + "soundfile", + "scikit-learn>=1.3.2", + "#system_torch#", + "#system_torchaudio#", + "#system_numpy#" + ] + }, + "updated_at": 1781811658, + "featured": false + }, + { + "version": 2, + "model_name": "speech_campplus_sv_zh_en_16k-common_advanced", + "model_family": "campplus", + "model_ability": [ + "speaker_embedding" + ], + "multilingual": true, + "model_src": { + "modelscope": { + "model_id": "iic/speech_campplus_sv_zh_en_16k-common_advanced", + "model_revision": "v1.0.0" + } + }, + "virtualenv": { + "packages": [ + "modelscope>=1.19.0", + "soundfile", + "scikit-learn>=1.3.2", + "#system_torch#", + "#system_torchaudio#", + "#system_numpy#" + ] + }, + "updated_at": 1719208861, + "featured": false + }, { "version": 2, "model_name": "VoxCPM2", diff --git a/xinference/model/audio/speaker_embedding.py b/xinference/model/audio/speaker_embedding.py new file mode 100644 index 0000000000..d1e26eff70 --- /dev/null +++ b/xinference/model/audio/speaker_embedding.py @@ -0,0 +1,134 @@ +# Copyright 2022-2026 Xinference Holdings Pte. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import os +import tempfile +from typing import TYPE_CHECKING, Any, Dict, Optional + +import numpy as np + +from ...device_utils import get_available_device, is_device_available + +if TYPE_CHECKING: + from .core import AudioModelFamilyV2 + +logger = logging.getLogger(__name__) + + +class ModelScopeSpeakerEmbeddingModel: + """Extract speaker embeddings with a ModelScope speaker-verification model.""" + + def __init__( + self, + model_uid: str, + model_path: str, + model_spec: "AudioModelFamilyV2", + device: Optional[str] = None, + **kwargs: Any, + ): + self.model_family = model_spec + self._model_uid = model_uid + self._model_path = model_path + self._model_spec = model_spec + self._device = device + self._pipeline = None + self._kwargs = kwargs + + @property + def model_ability(self): + return self._model_spec.model_ability + + def load(self): + try: + from modelscope.pipelines import pipeline + from modelscope.utils.constant import Tasks + except ImportError as e: + raise ImportError( + "Failed to import ModelScope speaker-verification dependencies. " + "Please install `modelscope`, `soundfile`, and `scikit-learn`." + ) from e + + if self._device is None: + self._device = get_available_device() + elif not is_device_available(self._device): + raise ValueError(f"Device {self._device} is not available!") + + logger.debug( + "Loading ModelScope speaker embedding model from %s on %s", + self._model_path, + self._device, + ) + self._pipeline = pipeline( + task=Tasks.speaker_verification, + model=self._model_path, + device=self._device, + **self._kwargs, + ) + + def create_embedding( + self, audio: bytes, model_uid: Optional[str] = None + ) -> Dict[str, Any]: + if not audio: + raise ValueError("Audio input must not be empty.") + if self._pipeline is None: + raise RuntimeError("Speaker embedding model is not loaded.") + + temp_path = None + try: + # ModelScope's speaker-verification pipeline accepts file paths or + # NumPy waveforms, but not encoded audio bytes. Close the temporary + # file before inference so this also works on Windows. + with tempfile.NamedTemporaryFile(delete=False) as f: + f.write(audio) + temp_path = f.name + + result = self._pipeline([temp_path], output_emb=True) + finally: + if temp_path is not None: + try: + os.unlink(temp_path) + except FileNotFoundError: + pass + + if not isinstance(result, dict) or "embs" not in result: + raise RuntimeError( + f"ModelScope returned an invalid speaker embedding result: {result!r}" + ) + + embeddings = result["embs"] + if hasattr(embeddings, "detach"): + embeddings = embeddings.detach() + if hasattr(embeddings, "cpu"): + embeddings = embeddings.cpu() + if hasattr(embeddings, "numpy"): + embeddings = embeddings.numpy() + + embedding_array = np.asarray(embeddings, dtype=np.float32) + if embedding_array.ndim == 2 and embedding_array.shape[0] == 1: + embedding_array = embedding_array[0] + if embedding_array.ndim != 1 or embedding_array.size == 0: + raise RuntimeError( + "ModelScope returned an unexpected speaker embedding shape: " + f"{embedding_array.shape}" + ) + if not np.isfinite(embedding_array).all(): + raise RuntimeError("ModelScope returned a non-finite speaker embedding.") + + return { + "object": "embedding", + "model": model_uid or self._model_uid, + "dimensions": int(embedding_array.size), + "embedding": embedding_array.tolist(), + } diff --git a/xinference/model/audio/tests/test_speaker_embedding.py b/xinference/model/audio/tests/test_speaker_embedding.py new file mode 100644 index 0000000000..f7824262f2 --- /dev/null +++ b/xinference/model/audio/tests/test_speaker_embedding.py @@ -0,0 +1,124 @@ +# Copyright 2022-2026 Xinference Holdings Pte. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import patch + +import numpy as np +import pytest + +from .. import load_model_family_from_json +from ..core import create_audio_model_instance, match_audio +from ..speaker_embedding import ModelScopeSpeakerEmbeddingModel + + +def _new_model(): + spec = SimpleNamespace( + model_name="speech_campplus_sv_zh-cn_16k-common", + model_ability=["speaker_embedding"], + model_family="campplus", + ) + return ModelScopeSpeakerEmbeddingModel("replica-uid", "/unused", spec) + + +def test_create_speaker_embedding_from_audio_bytes(): + model = _new_model() + observed = {} + + def fake_pipeline(inputs, output_emb): + path = Path(inputs[0]) + observed["path"] = path + observed["audio"] = path.read_bytes() + observed["output_emb"] = output_emb + return { + "embs": np.array([[0.25, -0.5, 0.75]], dtype=np.float32), + "outputs": {"text": "No similarity score output"}, + } + + model._pipeline = fake_pipeline + + result = model.create_embedding(b"encoded-audio", model_uid="speaker-model") + + assert result == { + "object": "embedding", + "model": "speaker-model", + "dimensions": 3, + "embedding": [0.25, -0.5, 0.75], + } + assert observed["audio"] == b"encoded-audio" + assert observed["output_emb"] is True + assert not observed["path"].exists() + + +@pytest.mark.parametrize( + "pipeline_result, match", + [ + ({}, "invalid speaker embedding result"), + ({"embs": np.empty((0, 192))}, "unexpected speaker embedding shape"), + ({"embs": np.array([[np.nan]])}, "non-finite speaker embedding"), + ], +) +def test_create_speaker_embedding_validates_pipeline_output(pipeline_result, match): + model = _new_model() + model._pipeline = lambda *_args, **_kwargs: pipeline_result + + with pytest.raises(RuntimeError, match=match): + model.create_embedding(b"encoded-audio") + + +def test_create_speaker_embedding_rejects_empty_audio(): + model = _new_model() + model._pipeline = object() + + with pytest.raises(ValueError, match="must not be empty"): + model.create_embedding(b"") + + +def test_campplus_factory_dispatch(): + spec = SimpleNamespace( + model_name="speaker-model", + model_family="campplus", + model_ability=["speaker_embedding"], + ) + with patch("xinference.model.audio.core.match_audio", return_value=spec): + model = create_audio_model_instance( + "uid", "speaker-model", model_path="/fake/path" + ) + + assert isinstance(model, ModelScopeSpeakerEmbeddingModel) + + +def test_builtin_campplus_specs_are_modelscope_only_and_match_by_default(): + models = {} + load_model_family_from_json("model_spec.json", models) + + expected = { + "speech_campplus_sv_zh-cn_16k-common", + "speech_campplus_sv_zh_en_16k-common_advanced", + } + for model_name in expected: + specs = models[model_name] + assert len(specs) == 1 + assert specs[0].model_hub == "modelscope" + assert specs[0].model_ability == ["speaker_embedding"] + + with ( + patch("xinference.model.audio.BUILTIN_AUDIO_MODELS", models), + patch("xinference.model.audio.custom.get_user_defined_audios", return_value=[]), + patch("xinference.model.utils.download_from_modelscope", return_value=False), + ): + matched = match_audio("speech_campplus_sv_zh-cn_16k-common") + + assert matched.model_hub == "modelscope" diff --git a/xinference/types.py b/xinference/types.py index 33f2a24b7f..b5ebc13919 100644 --- a/xinference/types.py +++ b/xinference/types.py @@ -92,6 +92,13 @@ class Embedding(TypedDict): usage: EmbeddingUsage +class AudioEmbedding(TypedDict): + object: Literal["embedding"] + model: str + dimensions: int + embedding: List[float] + + class Document(TypedDict): text: str From c838b8c8179641272728ad968d5e2b09c5129dca Mon Sep 17 00:00:00 2001 From: leslie2046 <253605712@qq.com> Date: Thu, 6 Aug 2026 16:32:34 +0800 Subject: [PATCH 2/4] fix(audio): install ModelScope speaker dependencies --- xinference/model/audio/model_spec.json | 24 +++++++++++++++++-- xinference/model/audio/speaker_embedding.py | 3 ++- .../audio/tests/test_speaker_embedding.py | 4 ++++ 3 files changed, 28 insertions(+), 3 deletions(-) diff --git a/xinference/model/audio/model_spec.json b/xinference/model/audio/model_spec.json index 65a9e421d2..bd1149fe9f 100644 --- a/xinference/model/audio/model_spec.json +++ b/xinference/model/audio/model_spec.json @@ -1965,7 +1965,17 @@ }, "virtualenv": { "packages": [ - "modelscope>=1.19.0", + "modelscope[framework]>=1.19.0", + "addict", + "attrs", + "datasets>=3,<5", + "einops", + "oss2", + "Pillow", + "python-dateutil>=2.1", + "simplejson>=3.3.0", + "sortedcontainers>=1.5.9", + "transformers", "soundfile", "scikit-learn>=1.3.2", "#system_torch#", @@ -1992,7 +2002,17 @@ }, "virtualenv": { "packages": [ - "modelscope>=1.19.0", + "modelscope[framework]>=1.19.0", + "addict", + "attrs", + "datasets>=3,<5", + "einops", + "oss2", + "Pillow", + "python-dateutil>=2.1", + "simplejson>=3.3.0", + "sortedcontainers>=1.5.9", + "transformers", "soundfile", "scikit-learn>=1.3.2", "#system_torch#", diff --git a/xinference/model/audio/speaker_embedding.py b/xinference/model/audio/speaker_embedding.py index d1e26eff70..692e3db2f7 100644 --- a/xinference/model/audio/speaker_embedding.py +++ b/xinference/model/audio/speaker_embedding.py @@ -57,7 +57,8 @@ def load(self): except ImportError as e: raise ImportError( "Failed to import ModelScope speaker-verification dependencies. " - "Please install `modelscope`, `soundfile`, and `scikit-learn`." + "Please install `modelscope[framework]`, `soundfile`, and " + "`scikit-learn`." ) from e if self._device is None: diff --git a/xinference/model/audio/tests/test_speaker_embedding.py b/xinference/model/audio/tests/test_speaker_embedding.py index f7824262f2..c121977d31 100644 --- a/xinference/model/audio/tests/test_speaker_embedding.py +++ b/xinference/model/audio/tests/test_speaker_embedding.py @@ -113,6 +113,10 @@ def test_builtin_campplus_specs_are_modelscope_only_and_match_by_default(): assert len(specs) == 1 assert specs[0].model_hub == "modelscope" assert specs[0].model_ability == ["speaker_embedding"] + packages = specs[0].virtualenv.packages + assert "modelscope[framework]>=1.19.0" in packages + assert "addict" in packages + assert "datasets>=3,<5" in packages with ( patch("xinference.model.audio.BUILTIN_AUDIO_MODELS", models), From 809581491abfb62ff3e463f12087b365e717bdc9 Mon Sep 17 00:00:00 2001 From: leslie2046 <253605712@qq.com> Date: Fri, 7 Aug 2026 11:37:00 +0800 Subject: [PATCH 3/4] Add speaker embedding Web UI and documentation --- .../speech_campplus_sv_zh-cn_16k-common.rst | 16 +++- ..._campplus_sv_zh_en_16k-common_advanced.rst | 16 +++- doc/source/models/model_abilities/audio.rst | 14 +++- doc/source/user_guide/client_api.rst | 59 ++++++++++++++- .../capability-config.tsx | 1 + .../components/try-api-drawer.tsx | 30 ++++++-- .../pages/running-model-detail/index.tsx | 38 +++++----- .../panels/capability-task-panel.tsx | 17 ++++- .../panels/form-panels.tsx | 6 +- .../panels/result-panels.tsx | 70 +++++++++++++++++- .../pages/running-model-detail/types.ts | 1 + .../pages/running-model-detail/utils.ts | 11 +++ frontend/src/components/ui/page-container.tsx | 14 ++-- frontend/src/types/services.ts | 7 ++ xinference/model/audio/model_spec.json | 74 ------------------- .../audio/tests/test_speaker_embedding.py | 31 +------- 16 files changed, 255 insertions(+), 150 deletions(-) diff --git a/doc/source/models/builtin/audio/speech_campplus_sv_zh-cn_16k-common.rst b/doc/source/models/builtin/audio/speech_campplus_sv_zh-cn_16k-common.rst index b9d4a4b7ed..b939feaa41 100644 --- a/doc/source/models/builtin/audio/speech_campplus_sv_zh-cn_16k-common.rst +++ b/doc/source/models/builtin/audio/speech_campplus_sv_zh-cn_16k-common.rst @@ -9,6 +9,11 @@ speech_campplus_sv_zh-cn_16k-common - **Abilities:** ['speaker_embedding'] - **Multilingual:** False +This CAMPPlus speaker-verification model converts a speech sample into a +fixed-length representation of speaker identity. It is intended for Chinese +speech and can be used as the embedding stage in speaker verification and +speaker identification systems. It does not transcribe the spoken content. + Specifications ^^^^^^^^^^^^^^ @@ -17,8 +22,17 @@ Specifications - **Embedding Dimensions:** 192 - **Sample Rate:** 16 kHz +Output and comparison +^^^^^^^^^^^^^^^^^^^^^ + +Each request returns one 192-dimensional vector. Store the vector in your +application and use cosine similarity to compare samples. Select a similarity +threshold using representative recordings from the microphones, speakers, and +acoustic conditions expected in production. + Execute the following command to launch the model:: xinference launch --model-name speech_campplus_sv_zh-cn_16k-common --model-type audio -See :ref:`audio` for the Speaker Embedding API usage. +See :ref:`audio` for Web UI, cURL, and Python examples for the +``/v1/audio/embeddings`` endpoint. diff --git a/doc/source/models/builtin/audio/speech_campplus_sv_zh_en_16k-common_advanced.rst b/doc/source/models/builtin/audio/speech_campplus_sv_zh_en_16k-common_advanced.rst index 210c1d7536..06b36c70fe 100644 --- a/doc/source/models/builtin/audio/speech_campplus_sv_zh_en_16k-common_advanced.rst +++ b/doc/source/models/builtin/audio/speech_campplus_sv_zh_en_16k-common_advanced.rst @@ -9,6 +9,11 @@ speech_campplus_sv_zh_en_16k-common_advanced - **Abilities:** ['speaker_embedding'] - **Multilingual:** True +This multilingual CAMPPlus speaker-verification model converts a Chinese or +English speech sample into a fixed-length representation of speaker identity. +It can be used as the embedding stage in speaker verification and speaker +identification systems. It does not transcribe the spoken content. + Specifications ^^^^^^^^^^^^^^ @@ -17,8 +22,17 @@ Specifications - **Embedding Dimensions:** 192 - **Sample Rate:** 16 kHz +Output and comparison +^^^^^^^^^^^^^^^^^^^^^ + +Each request returns one 192-dimensional vector. Store the vector in your +application and use cosine similarity to compare samples. Select a similarity +threshold using representative recordings from the microphones, speakers, and +acoustic conditions expected in production. + Execute the following command to launch the model:: xinference launch --model-name speech_campplus_sv_zh_en_16k-common_advanced --model-type audio -See :ref:`audio` for the Speaker Embedding API usage. +See :ref:`audio` for Web UI, cURL, and Python examples for the +``/v1/audio/embeddings`` endpoint. diff --git a/doc/source/models/model_abilities/audio.rst b/doc/source/models/model_abilities/audio.rst index 8b02cbeace..9a9385b879 100644 --- a/doc/source/models/model_abilities/audio.rst +++ b/doc/source/models/model_abilities/audio.rst @@ -135,9 +135,19 @@ The Speaker Embedding API accepts one audio file and returns one speaker embedding. The built-in CAMPPlus models return a 192-dimensional vector. The endpoint is intentionally stateless: applications can store the returned vectors and use cosine similarity for speaker verification or 1:N speaker identification. +The request uses ``multipart/form-data``: ``model`` is the UID of a running +speaker-embedding model and ``file`` is the audio sample. Unlike the general +``/v1/embeddings`` endpoint, this endpoint returns one embedding object rather +than a list of text embeddings. .. tabs:: + .. tab:: Web UI + + Open **Running Models**, select a running CAMPPlus model, and upload a clear + speech sample in the **Speaker Embedding** panel. Select **Extract embedding** + to inspect the vector and copy it from the results panel. + .. code-tab:: bash cURL curl -X POST \ @@ -169,7 +179,9 @@ and use cosine similarity for speaker verification or 1:N speaker identification ModelScope decodes the input, converts multi-channel audio to one channel, and resamples it to the model's 16 kHz sample rate. The returned vector preserves -the model output. Use cosine similarity when comparing two vectors. +the model output. Use cosine similarity when comparing two vectors; choose a +verification or identification threshold using representative audio from your +own application. Transcription -------------------- diff --git a/doc/source/user_guide/client_api.rst b/doc/source/user_guide/client_api.rst index cbb7ae836c..3e26ba5b95 100644 --- a/doc/source/user_guide/client_api.rst +++ b/doc/source/user_guide/client_api.rst @@ -301,7 +301,7 @@ Output: Audio ~~~~~ -To list the available built-in image models: +To list the available built-in audio models: .. code-block:: @@ -316,6 +316,8 @@ To list the available built-in image models: audio whisper-medium.en whisper False True audio whisper-tiny whisper True True audio whisper-tiny.en whisper False True + audio speech_campplus_sv_zh-cn_16k-common campplus False True + audio speech_campplus_sv_zh_en_16k-common_advanced campplus True True To initiate an audio model and get text from an audio: @@ -367,6 +369,61 @@ Output: Translation(text=' This list lists the airlines in Hong Kong.') +Speaker Embeddings +================== + +Speaker-embedding audio models extract a fixed-length representation of speaker +identity. Xinference provides two built-in CAMPPlus models: +``speech_campplus_sv_zh-cn_16k-common`` and +``speech_campplus_sv_zh_en_16k-common_advanced``. Both return a 192-dimensional +vector through the ``speaker_embedding`` ability. + +Launch a model and call it with the Xinference client: + +.. code-block:: python + + from xinference.client import Client + + client = Client("http://localhost:9997") + model_uid = client.launch_model( + model_name="speech_campplus_sv_zh-cn_16k-common", + model_type="audio", + ) + model = client.get_model(model_uid) + + with open("speaker.wav", "rb") as audio_file: + result = model.create_embedding(audio_file.read()) + + print(result["dimensions"]) + embedding = result["embedding"] + +The equivalent HTTP request uploads the model UID and audio file as multipart +form fields: + +.. code-block:: bash + + curl -X POST 'http://localhost:9997/v1/audio/embeddings' \ + -H 'accept: application/json' \ + -F 'model=' \ + -F 'file=@speaker.wav' + +The response contains one vector rather than the list-shaped response returned +by the text Embeddings API: + +.. code-block:: json + + { + "object": "embedding", + "model": "", + "dimensions": 192, + "embedding": [0.0123, -0.0456, 0.0789] + } + +Compare vectors with cosine similarity for speaker verification or speaker +identification. See :ref:`audio` for Web UI usage, input processing details, +and the complete Speaker Embedding API example. + + Rerank ~~~~~~ To launch a rerank model and compute the similarity scores: diff --git a/frontend/src/components/pages/running-model-detail/capability-config.tsx b/frontend/src/components/pages/running-model-detail/capability-config.tsx index 44959e13e3..9fe41c519d 100644 --- a/frontend/src/components/pages/running-model-detail/capability-config.tsx +++ b/frontend/src/components/pages/running-model-detail/capability-config.tsx @@ -260,6 +260,7 @@ export const CAPABILITY_CONFIGS: Partial> ], }, initialValues: { file: [] }, + submitLabel: 'Extract embedding', formPanel: SpeakerEmbeddingPanel, resultPanel: ResultPanels.Universal, transformValues: audioEmbeddingFormData, diff --git a/frontend/src/components/pages/running-model-detail/components/try-api-drawer.tsx b/frontend/src/components/pages/running-model-detail/components/try-api-drawer.tsx index 56d1e66011..9c8ef63c29 100644 --- a/frontend/src/components/pages/running-model-detail/components/try-api-drawer.tsx +++ b/frontend/src/components/pages/running-model-detail/components/try-api-drawer.tsx @@ -6,7 +6,14 @@ import { useRouter } from 'next/navigation'; import { Button } from '@/components/ui/button'; import { JSONSyntaxHighlighter } from '@/components/ui/json-syntax-highlighter'; -import { Sheet, SheetClose, SheetContent, SheetHeader, SheetTitle } from '@/components/ui/sheet'; +import { + Sheet, + SheetClose, + SheetContent, + SheetDescription, + SheetHeader, + SheetTitle, +} from '@/components/ui/sheet'; import { Tabs, TabsContent, TabsList, TabsTrigger } from '@/components/ui/tabs'; import { ModelAbility } from '@/constants'; import { @@ -21,6 +28,7 @@ import { useMenuAuth } from '@/hooks/use-menu-auth'; import { copyToClipboard, getApiUrl } from '@/lib/utils'; import { CAPABILITY_CONFIGS } from '../capability-config'; +import { getPrimaryModelAbilities } from '../utils'; interface TryApiDrawerProps { open: boolean; @@ -470,7 +478,7 @@ function getCodeExample( } export function getTryApiAbility(abilities: ModelAbility[] = []) { - const primaryAbilities = abilities.filter((ability) => !ability.includes('_')); + const primaryAbilities = getPrimaryModelAbilities(abilities); return primaryAbilities.find((ability) => ability === ModelAbility.Chat) || primaryAbilities[0]; } @@ -493,8 +501,11 @@ export function TryApiDrawer({ return ( - - + +
- Try To API +
+ Try To API + + API request examples for the selected model ability. + +
- {modelUid} + {modelUid} } loading={loading} className="gap-5" + headerClassName="flex-col items-stretch gap-3 sm:flex-row sm:items-center" extraContent={ -
+
{!isChat && (