From dd4422ff8be84edd3c5ae261a5f68886de09c6df Mon Sep 17 00:00:00 2001 From: martsforever Date: Sun, 2 Nov 2025 18:28:13 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E5=87=86=E5=A4=87=E7=94=A8=E4=B8=8A=20?= =?UTF-8?q?BaiLianVoiceRecognise?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../add_websocket_voice_recognise_route.py | 12 +- .../BaiLianVoiceRecognise.py | 156 ++++++++++++++++++ .../BaiLianVoiceRecogniseDemo.txt | 54 ++++++ 3 files changed, 219 insertions(+), 3 deletions(-) create mode 100644 app/utils/bailian_voice_recognise/BaiLianVoiceRecognise.py create mode 100644 app/utils/bailian_voice_recognise/BaiLianVoiceRecogniseDemo.txt diff --git a/app/controller/add_websocket_voice_recognise_route.py b/app/controller/add_websocket_voice_recognise_route.py index e956094..c7c650c 100644 --- a/app/controller/add_websocket_voice_recognise_route.py +++ b/app/controller/add_websocket_voice_recognise_route.py @@ -83,7 +83,10 @@ def add_websocket_voice_recognise_route(app: FastAPI): # print("已转发音频数据到百炼", f"PCM二进制数据，长度: {len(pcm_bytes)} 字节") except Exception as e: - print(f"等待前端的消息出错: {e}") + if "CloseCode.NO_STATUS_RCVD" in str(e): + print("前端连接已断开(无状态码)") + else: + print(f"等待前端的消息出错: {e}") async def waiting_msg_from_bailian(): """将百炼识别结果转发到客户端""" @@ -120,8 +123,11 @@ def add_websocket_voice_recognise_route(app: FastAPI): finally: print("finally ===>>>>>>>>>>>>>>>>>>>>>>>>>>>") # 清理连接 - if bailian_socket and bailian_socket.open: - await bailian_socket.close() + if bailian_socket: + try: + await bailian_socket.close() + except: + pass # 忽略关闭时的任何错误 if receive_socket.application_state != WebSocketState.DISCONNECTED: print("关闭websocket") diff --git a/app/utils/bailian_voice_recognise/BaiLianVoiceRecognise.py b/app/utils/bailian_voice_recognise/BaiLianVoiceRecognise.py new file mode 100644 index 0000000..7688b2e --- /dev/null +++ b/app/utils/bailian_voice_recognise/BaiLianVoiceRecognise.py @@ -0,0 +1,156 @@ +import json +from typing import Callable, TypedDict + +import websockets +from websockets.client import ClientConnection + +from app.config.env import env + + +class BaiLianVoiceRecognise: + """ + 通义千问——实时语音识别 + 官方文档：https://help.aliyun.com/zh/model-studio/qwen-real-time-speech-recognition?spm=a2c4g.11186623.0.0.620e5a26eUNz80#0d1b30e0a1glp + 客户端事件：https://help.aliyun.com/zh/model-studio/qwen-asr-realtime-client-events + 服务端事件：https://help.aliyun.com/zh/model-studio/qwen-asr-realtime-server-events?spm=a2c4g.11186623.help-menu-2400256.d_2_6_8_1.620e5acdS0URfG + """ + + def __init__( + self, + # 百炼平台的api key，https://bailian.console.aliyun.com/?spm=5176.28326591.0.0.4d0555e3pzewCB&tab=model#/api-key + api_key: str = env.llm_key_bailian, + # 模型名称，支持“qwen3-asr-flash-realtime”与“qwen3-asr-flash-realtime-2025-10-27” + qwen_model: str = "qwen3-asr-flash-realtime", + # websocket接口地址 + base_url: str = "wss://dashscope.aliyuncs.com/api-ws/v1/realtime", + # 可选的附加头信息 + additional_headers: dict | None = None, + # 是否启用vad模式，否则启用manual模式 + # vad模式：服务端自动检测语音的起点和终点（断句）。开发者只需持续发送音频流，服务端会在检测到一句话结束时自动返回最终识别结果。此模式适用于实时对话、会议记录等场景。 + # vad文档：https://help.aliyun.com/zh/model-studio/qwen-asr-realtime-interaction-process?spm=a2c4g.11186623.0.0.16fe44488GkDZB#9b49887720jcw + # manual模式：由客户端控制断句。客户端需要发送完一整句话的音频后，再发送一个input_audio_buffer.commit事件来通知服务端。此模式适用于客户端能明确判断语句边界的场景，如聊天软件中的发送语音。 + # manual文档：https://help.aliyun.com/zh/model-studio/qwen-asr-realtime-interaction-process?spm=a2c4g.11186623.0.0.16fe44488GkDZB#ee09a3493fsuc + enable_server_vad: bool = True, + + # 监听开始说话动作，参数为item_id + on_speech_started: Callable[[str], None] | None = None, + # 监听正在说话的内容，参数为item_id以及本次说话叠加的完整内容 + on_speech_content: Callable[[str, str], None] | None = None, + # 监听说话结束动作，参数为item_id + on_speech_stopped: Callable[[str], None] | None = None, + # 监听说话完毕动作，参数为item_id以及本次说话万恒内容 + on_speech_completed: Callable[[str, str], None] | None = None + ): + """ + 构造函数，会自动设置连接百炼实时语音识别websocket接口所需要的秘钥，模型名称，websocket接口地址，以及可选的附加头信息 + """ + self.api_key = api_key + self.qwen_model = qwen_model + self.base_url = base_url + self.additional_headers = additional_headers or {} + self.enable_server_vad = enable_server_vad + + self.on_speech_started = on_speech_started + self.on_speech_content = on_speech_content + self.on_speech_stopped = on_speech_stopped + self.on_speech_completed = on_speech_completed + + socket: ClientConnection | None = None + self.socket = socket + + @staticmethod + def log(message): + print("BaiLianVoiceRecognise: " + message) + + async def connect(self): + """ + 连接websocket接口，得到socket对象 + """ + url = f"{self.base_url}?model={self.qwen_model}" + headers = { + "Authorization": f"Bearer {self.api_key}", + "OpenAI-Beta": "realtime=v1", + **self.additional_headers, + } + self.socket = await websockets.connect(url, additional_headers=headers) + + await self.send_session_update_event() + + async def send_session_update_event(self): + """ + 发送session.update事件，通知百炼接口启用vad模式还是manual模式 + """ + if not self.socket: + raise Exception("请先调用connect方法初始化websocket连接") + # 会话更新事件 + event_manual = { + "event_id": "event_123", + "type": "session.update", + "session": { + "modalities": ["text"], + "input_audio_format": "pcm", + "sample_rate": 16000, + "input_audio_transcription": { + # 语种标识，可选，如果有明确的语种信息，建议设置 + "language": "zh" + # 语料，可选，如果有语料，建议设置以增强识别效果 + # "corpus": { + # "text": "" + # } + }, + "turn_detection": None + } + } + event_vad = { + "event_id": "event_123", + "type": "session.update", + "session": { + "modalities": ["text"], + "input_audio_format": "pcm", + "sample_rate": 16000, + "input_audio_transcription": { + "language": "zh" + }, + "turn_detection": { + "type": "server_vad", + "threshold": 0.2, + "silence_duration_ms": 800 + } + } + } + if self.enable_server_vad: + self.log(f"send session.update event: {json.dumps(event_vad, indent=2)}") + self.socket.send(json.dumps(event_vad)) + else: + self.log(f"send session.update event: {json.dumps(event_manual, indent=2)}") + self.socket.send(json.dumps(event_manual)) + + async def waiting_message_from_socket(self): + """将百炼识别结果转发到客户端""" + try: + async for message in self.socket: + data = json.loads(message) + + message_data_type = data.get('type', None) + message_data_item_id = data.get('item_id', None) + + if message_data_type == "input_audio_buffer.speech_started": + self.on_speech_started(message_data_item_id) + if message_data_type == "conversation.item.input_audio_transcription.text": + self.on_speech_content(message_data_item_id, data.get('stash', "")) + if message_data_type == "input_audio_buffer.speech_stopped": + self.on_speech_stopped(message_data_item_id) + if message_data_type == "conversation.item.input_audio_transcription.completed": + self.on_speech_completed(message_data_item_id, data.get('transcript', "")) + + except Exception as e: + print(f"等待百炼的消息出错: {e}") + + +class BaiLianRecogniseMessage(TypedDict): + event_id: str + type: str + item_id: str + content_index: int + text: str + stash: str diff --git a/app/utils/bailian_voice_recognise/BaiLianVoiceRecogniseDemo.txt b/app/utils/bailian_voice_recognise/BaiLianVoiceRecogniseDemo.txt new file mode 100644 index 0000000..5f11407 --- /dev/null +++ b/app/utils/bailian_voice_recognise/BaiLianVoiceRecogniseDemo.txt @@ -0,0 +1,54 @@ +[18:07:02]receive message: {"event_id": "event_Xg4o5eXrwBq3Dv6i16tcL", "type": "session.created", "session": {"object": "realtime.session", "model": "qwen3-asr-flash-realtime", "modalities": ["text"], "input_audio_format": "pcm", "sample_rate": 16000, "input_audio_transcription": {"model": "qwen3-asr-flash-realtime"}, "turn_detection": {"type": "server_vad", "threshold": 0.2, "silence_duration_ms": 800}, "id": "sess_RTmRAcSuEbITu6sua3jOv"}} + +[18:07:02]receive message: {"event_id": "event_RY29NFop4tAIDZpZ1ceiv", "type": "session.updated", "session": {"id": "sess_RTmRAcSuEbITu6sua3jOv", "object": "realtime.session", "model": "qwen3-asr-flash-realtime", "modalities": ["text"], "input_audio_format": "pcm", "input_audio_transcription": {"model": "qwen3-asr-flash-realtime", "language": "zh"}, "turn_detection": {"type": "server_vad", "threshold": 0.2, "silence_duration_ms": 800, "create_response": true, "interrupt_response": true}, "sample_rate": 16000}} + +[18:07:04]receive message: {"event_id": "event_EX2uwu6xLLvLJhCTT5Tya", "type": "input_audio_buffer.speech_started", "audio_start_ms": 1900, "item_id": "item_AEQBaDSactuh6lFHdhLnh"} + +[18:07:05]receive message: {"event_id": "event_WbJ31CaIwPwwnNfJOLRZ7", "type": "conversation.item.created", "item": {"id": "item_AEQBaDSactuh6lFHdhLnh", "object": "realtime.item", "type": "message", "status": "in_progress", "role": "assistant", "content": [{"type": "input_audio"}]}} + +[18:07:05]receive message: {"event_id": "event_LhToqCQmQX3zWIdjEKvlg", "type": "conversation.item.input_audio_transcription.text", "item_id": "item_AEQBaDSactuh6lFHdhLnh", "content_index": 0, "text": "", "stash": "大"} + +[18:07:05]receive message: {"event_id": "event_X9X7zlxIp44pQelwbfRbH", "type": "conversation.item.input_audio_transcription.text", "item_id": "item_AEQBaDSactuh6lFHdhLnh", "content_index": 0, "text": "", "stash": "大漠"} + +[18:07:05]receive message: {"event_id": "event_XLB64gdbKbfAM1uocAYGP", "type": "conversation.item.input_audio_transcription.text", "item_id": "item_AEQBaDSactuh6lFHdhLnh", "content_index": 0, "text": "", "stash": "大漠孤"} + +[18:07:05]receive message: {"event_id": "event_Fa4qnePfwiWTlIRoBsWF1", "type": "conversation.item.input_audio_transcription.text", "item_id": "item_AEQBaDSactuh6lFHdhLnh", "content_index": 0, "text": "", "stash": "大漠孤。"} + +[18:07:07]receive message: {"event_id": "event_YtgfDmpLi8ClAV5wCZeDZ", "type": "input_audio_buffer.speech_stopped", "audio_end_ms": 5240, "item_id": "item_AEQBaDSactuh6lFHdhLnh"} + +[18:07:07]receive message: {"event_id": "event_FUpop8WUSmc1orkna58gi", "type": "input_audio_buffer.committed", "item_id": "item_AEQBaDSactuh6lFHdhLnh"} + +[18:07:07]receive message: {"event_id": "event_WoFtx1fmvIoq7vjpfYez2", "type": "conversation.item.input_audio_transcription.text", "item_id": "item_AEQBaDSactuh6lFHdhLnh", "content_index": 0, "text": "", "stash": "大漠孤烟"} + +[18:07:07]receive message: {"event_id": "event_WDPETKwaVDffRnt22RdLP", "type": "conversation.item.input_audio_transcription.text", "item_id": "item_AEQBaDSactuh6lFHdhLnh", "content_index": 0, "text": "", "stash": "大漠孤烟直"} + +[18:07:07]receive message: {"event_id": "event_O28jlaFUWdxMRDA5WKOp8", "type": "conversation.item.input_audio_transcription.text", "item_id": "item_AEQBaDSactuh6lFHdhLnh", "content_index": 0, "text": "", "stash": "大漠孤烟直。"} + +[18:07:07]receive message: {"event_id": "event_MN9TdqMM13VKHg6gthJSQ", "type": "conversation.item.input_audio_transcription.completed", "item_id": "item_AEQBaDSactuh6lFHdhLnh", "content_index": 0, "transcript": "大漠孤烟直。", "usage": {"duration": 5}} + +[18:07:07]receive message: {"type": "final_transcript", "text": "大漠孤烟直。"} + +[18:07:10]receive message: {"event_id": "event_OOc7k8cX7GKpjNqog71ZK", "type": "input_audio_buffer.speech_started", "audio_start_ms": 8060, "item_id": "item_S997Yu46cT66srf8uqqxj"} + +[18:07:12]receive message: {"event_id": "event_Eo4KtXx5JVHP8V4qtKITT", "type": "conversation.item.created", "previous_item_id": "item_AEQBaDSactuh6lFHdhLnh", "item": {"id": "item_S997Yu46cT66srf8uqqxj", "object": "realtime.item", "type": "message", "status": "in_progress", "role": "assistant", "content": [{"type": "input_audio"}]}} + +[18:07:12]receive message: {"event_id": "event_PHmM2m1scdbo8TjSyf1Fm", "type": "conversation.item.input_audio_transcription.text", "item_id": "item_S997Yu46cT66srf8uqqxj", "content_index": 0, "text": "", "stash": "长"} + +[18:07:12]receive message: {"event_id": "event_QaoimHMAGiBeQbV9Ln1El", "type": "conversation.item.input_audio_transcription.text", "item_id": "item_S997Yu46cT66srf8uqqxj", "content_index": 0, "text": "", "stash": "长河"} + +[18:07:12]receive message: {"event_id": "event_MovuQE8C6JnDrPcMTeIpJ", "type": "conversation.item.input_audio_transcription.text", "item_id": "item_S997Yu46cT66srf8uqqxj", "content_index": 0, "text": "", "stash": "长河落"} + +[18:07:12]receive message: {"event_id": "event_F8fZyacfbtXgwNNizAQux", "type": "conversation.item.input_audio_transcription.text", "item_id": "item_S997Yu46cT66srf8uqqxj", "content_index": 0, "text": "", "stash": "长河落。"} + +[18:07:13]receive message: {"event_id": "event_A9wzZZyznzSkouWsnUP4E", "type": "input_audio_buffer.speech_stopped", "audio_end_ms": 11460, "item_id": "item_S997Yu46cT66srf8uqqxj"} + +[18:07:13]receive message: {"event_id": "event_UZJUdhU1OzbWViVEM6rM6", "type": "input_audio_buffer.committed", "previous_item_id": "item_AEQBaDSactuh6lFHdhLnh", "item_id": "item_S997Yu46cT66srf8uqqxj"} + +[18:07:14]receive message: {"event_id": "event_SouO2xnlII5hbUJNPry5Q", "type": "conversation.item.input_audio_transcription.text", "item_id": "item_S997Yu46cT66srf8uqqxj", "content_index": 0, "text": "", "stash": "长河落日圆"} + +[18:07:14]receive message: {"event_id": "event_V379dqiLrs1kSQExTKXjH", "type": "conversation.item.input_audio_transcription.text", "item_id": "item_S997Yu46cT66srf8uqqxj", "content_index": 0, "text": "", "stash": "长河落日圆。"} + +[18:07:14]receive message: {"event_id": "event_ZmjEdcRtMGaOadgSl6sD8", "type": "conversation.item.input_audio_transcription.completed", "item_id": "item_S997Yu46cT66srf8uqqxj", "content_index": 0, "transcript": "长河落日圆。", "usage": {"duration": 11}} + +[18:07:14]receive message: {"type": "final_transcript", "text": "长河落日圆。"} +