feat: 准备用上 BaiLianVoiceRecognise

app/controller/add_websocket_voice_recognise_route.py

@@ -83,6 +83,9 @@ def add_websocket_voice_recognise_route(app: FastAPI):
             # print("已转发音频数据到百炼", f"PCM二进制数据，长度: {len(pcm_bytes)} 字节")
 
         except Exception as e:
+          if "CloseCode.NO_STATUS_RCVD" in str(e):
+            print("前端连接已断开(无状态码)")
+          else:
             print(f"等待前端的消息出错: {e}")
 
       async def waiting_msg_from_bailian():
@@ -120,8 +123,11 @@ def add_websocket_voice_recognise_route(app: FastAPI):
     finally:
       print("finally ===>>>>>>>>>>>>>>>>>>>>>>>>>>>")
       # 清理连接
-      if bailian_socket and bailian_socket.open:
+      if bailian_socket:
+        try:
           await bailian_socket.close()
+        except:
+          pass  # 忽略关闭时的任何错误
 
       if receive_socket.application_state != WebSocketState.DISCONNECTED:
         print("关闭websocket")

app/utils/bailian_voice_recognise/BaiLianVoiceRecognise.py

@@ -0,0 +1,156 @@
+import json
+from typing import Callable, TypedDict
+
+import websockets
+from websockets.client import ClientConnection
+
+from app.config.env import env
+
+
+class BaiLianVoiceRecognise:
+  """
+  通义千问——实时语音识别
+  官方文档：https://help.aliyun.com/zh/model-studio/qwen-real-time-speech-recognition?spm=a2c4g.11186623.0.0.620e5a26eUNz80#0d1b30e0a1glp
+  客户端事件：https://help.aliyun.com/zh/model-studio/qwen-asr-realtime-client-events
+  服务端事件：https://help.aliyun.com/zh/model-studio/qwen-asr-realtime-server-events?spm=a2c4g.11186623.help-menu-2400256.d_2_6_8_1.620e5acdS0URfG
+  """
+
+  def __init__(
+    self,
+    # 百炼平台的api key，https://bailian.console.aliyun.com/?spm=5176.28326591.0.0.4d0555e3pzewCB&tab=model#/api-key
+    api_key: str = env.llm_key_bailian,
+    # 模型名称，支持“qwen3-asr-flash-realtime”与“qwen3-asr-flash-realtime-2025-10-27”
+    qwen_model: str = "qwen3-asr-flash-realtime",
+    # websocket接口地址
+    base_url: str = "wss://dashscope.aliyuncs.com/api-ws/v1/realtime",
+    # 可选的附加头信息
+    additional_headers: dict | None = None,
+    # 是否启用vad模式，否则启用manual模式
+    # vad模式：服务端自动检测语音的起点和终点（断句）。开发者只需持续发送音频流，服务端会在检测到一句话结束时自动返回最终识别结果。此模式适用于实时对话、会议记录等场景。
+    # vad文档：https://help.aliyun.com/zh/model-studio/qwen-asr-realtime-interaction-process?spm=a2c4g.11186623.0.0.16fe44488GkDZB#9b49887720jcw
+    # manual模式：由客户端控制断句。客户端需要发送完一整句话的音频后，再发送一个input_audio_buffer.commit事件来通知服务端。此模式适用于客户端能明确判断语句边界的场景，如聊天软件中的发送语音。
+    # manual文档：https://help.aliyun.com/zh/model-studio/qwen-asr-realtime-interaction-process?spm=a2c4g.11186623.0.0.16fe44488GkDZB#ee09a3493fsuc
+    enable_server_vad: bool = True,
+
+    # 监听开始说话动作，参数为item_id
+    on_speech_started: Callable[[str], None] | None = None,
+    # 监听正在说话的内容，参数为item_id以及本次说话叠加的完整内容
+    on_speech_content: Callable[[str, str], None] | None = None,
+    # 监听说话结束动作，参数为item_id
+    on_speech_stopped: Callable[[str], None] | None = None,
+    # 监听说话完毕动作，参数为item_id以及本次说话万恒内容
+    on_speech_completed: Callable[[str, str], None] | None = None
+  ):
+    """
+    构造函数，会自动设置连接百炼实时语音识别websocket接口所需要的秘钥，模型名称，websocket接口地址，以及可选的附加头信息
+    """
+    self.api_key = api_key
+    self.qwen_model = qwen_model
+    self.base_url = base_url
+    self.additional_headers = additional_headers or {}
+    self.enable_server_vad = enable_server_vad
+
+    self.on_speech_started = on_speech_started
+    self.on_speech_content = on_speech_content
+    self.on_speech_stopped = on_speech_stopped
+    self.on_speech_completed = on_speech_completed
+
+    socket: ClientConnection | None = None
+    self.socket = socket
+
+  @staticmethod
+  def log(message):
+    print("BaiLianVoiceRecognise: " + message)
+
+  async def connect(self):
+    """
+    连接websocket接口，得到socket对象
+    """
+    url = f"{self.base_url}?model={self.qwen_model}"
+    headers = {
+      "Authorization": f"Bearer {self.api_key}",
+      "OpenAI-Beta": "realtime=v1",
+      **self.additional_headers,
+    }
+    self.socket = await websockets.connect(url, additional_headers=headers)
+
+    await self.send_session_update_event()
+
+  async def send_session_update_event(self):
+    """
+    发送session.update事件，通知百炼接口启用vad模式还是manual模式
+    """
+    if not self.socket:
+      raise Exception("请先调用connect方法初始化websocket连接")
+    # 会话更新事件
+    event_manual = {
+      "event_id": "event_123",
+      "type": "session.update",
+      "session": {
+        "modalities": ["text"],
+        "input_audio_format": "pcm",
+        "sample_rate": 16000,
+        "input_audio_transcription": {
+          # 语种标识，可选，如果有明确的语种信息，建议设置
+          "language": "zh"
+          # 语料，可选，如果有语料，建议设置以增强识别效果
+          # "corpus": {
+          #     "text": ""
+          # }
+        },
+        "turn_detection": None
+      }
+    }
+    event_vad = {
+      "event_id": "event_123",
+      "type": "session.update",
+      "session": {
+        "modalities": ["text"],
+        "input_audio_format": "pcm",
+        "sample_rate": 16000,
+        "input_audio_transcription": {
+          "language": "zh"
+        },
+        "turn_detection": {
+          "type": "server_vad",
+          "threshold": 0.2,
+          "silence_duration_ms": 800
+        }
+      }
+    }
+    if self.enable_server_vad:
+      self.log(f"send session.update event: {json.dumps(event_vad, indent=2)}")
+      self.socket.send(json.dumps(event_vad))
+    else:
+      self.log(f"send session.update event: {json.dumps(event_manual, indent=2)}")
+      self.socket.send(json.dumps(event_manual))
+
+  async def waiting_message_from_socket(self):
+    """将百炼识别结果转发到客户端"""
+    try:
+      async for message in self.socket:
+        data = json.loads(message)
+
+        message_data_type = data.get('type', None)
+        message_data_item_id = data.get('item_id', None)
+
+        if message_data_type == "input_audio_buffer.speech_started":
+          self.on_speech_started(message_data_item_id)
+        if message_data_type == "conversation.item.input_audio_transcription.text":
+          self.on_speech_content(message_data_item_id, data.get('stash', ""))
+        if message_data_type == "input_audio_buffer.speech_stopped":
+          self.on_speech_stopped(message_data_item_id)
+        if message_data_type == "conversation.item.input_audio_transcription.completed":
+          self.on_speech_completed(message_data_item_id, data.get('transcript', ""))
+
+    except Exception as e:
+      print(f"等待百炼的消息出错: {e}")
+
+
+class BaiLianRecogniseMessage(TypedDict):
+  event_id: str
+  type: str
+  item_id: str
+  content_index: int
+  text: str
+  stash: str

app/utils/bailian_voice_recognise/BaiLianVoiceRecogniseDemo.txt

@@ -0,0 +1,54 @@
+[18:07:02]receive message:  {"event_id": "event_Xg4o5eXrwBq3Dv6i16tcL", "type": "session.created", "session": {"object": "realtime.session", "model": "qwen3-asr-flash-realtime", "modalities": ["text"], "input_audio_format": "pcm", "sample_rate": 16000, "input_audio_transcription": {"model": "qwen3-asr-flash-realtime"}, "turn_detection": {"type": "server_vad", "threshold": 0.2, "silence_duration_ms": 800}, "id": "sess_RTmRAcSuEbITu6sua3jOv"}}
+
+[18:07:02]receive message:  {"event_id": "event_RY29NFop4tAIDZpZ1ceiv", "type": "session.updated", "session": {"id": "sess_RTmRAcSuEbITu6sua3jOv", "object": "realtime.session", "model": "qwen3-asr-flash-realtime", "modalities": ["text"], "input_audio_format": "pcm", "input_audio_transcription": {"model": "qwen3-asr-flash-realtime", "language": "zh"}, "turn_detection": {"type": "server_vad", "threshold": 0.2, "silence_duration_ms": 800, "create_response": true, "interrupt_response": true}, "sample_rate": 16000}}
+
+[18:07:04]receive message:  {"event_id": "event_EX2uwu6xLLvLJhCTT5Tya", "type": "input_audio_buffer.speech_started", "audio_start_ms": 1900, "item_id": "item_AEQBaDSactuh6lFHdhLnh"}
+
+[18:07:05]receive message:  {"event_id": "event_WbJ31CaIwPwwnNfJOLRZ7", "type": "conversation.item.created", "item": {"id": "item_AEQBaDSactuh6lFHdhLnh", "object": "realtime.item", "type": "message", "status": "in_progress", "role": "assistant", "content": [{"type": "input_audio"}]}}
+
+[18:07:05]receive message:  {"event_id": "event_LhToqCQmQX3zWIdjEKvlg", "type": "conversation.item.input_audio_transcription.text", "item_id": "item_AEQBaDSactuh6lFHdhLnh", "content_index": 0, "text": "", "stash": "大"}
+
+[18:07:05]receive message:  {"event_id": "event_X9X7zlxIp44pQelwbfRbH", "type": "conversation.item.input_audio_transcription.text", "item_id": "item_AEQBaDSactuh6lFHdhLnh", "content_index": 0, "text": "", "stash": "大漠"}
+
+[18:07:05]receive message:  {"event_id": "event_XLB64gdbKbfAM1uocAYGP", "type": "conversation.item.input_audio_transcription.text", "item_id": "item_AEQBaDSactuh6lFHdhLnh", "content_index": 0, "text": "", "stash": "大漠孤"}
+
+[18:07:05]receive message:  {"event_id": "event_Fa4qnePfwiWTlIRoBsWF1", "type": "conversation.item.input_audio_transcription.text", "item_id": "item_AEQBaDSactuh6lFHdhLnh", "content_index": 0, "text": "", "stash": "大漠孤。"}
+
+[18:07:07]receive message:  {"event_id": "event_YtgfDmpLi8ClAV5wCZeDZ", "type": "input_audio_buffer.speech_stopped", "audio_end_ms": 5240, "item_id": "item_AEQBaDSactuh6lFHdhLnh"}
+
+[18:07:07]receive message:  {"event_id": "event_FUpop8WUSmc1orkna58gi", "type": "input_audio_buffer.committed", "item_id": "item_AEQBaDSactuh6lFHdhLnh"}
+
+[18:07:07]receive message:  {"event_id": "event_WoFtx1fmvIoq7vjpfYez2", "type": "conversation.item.input_audio_transcription.text", "item_id": "item_AEQBaDSactuh6lFHdhLnh", "content_index": 0, "text": "", "stash": "大漠孤烟"}
+
+[18:07:07]receive message:  {"event_id": "event_WDPETKwaVDffRnt22RdLP", "type": "conversation.item.input_audio_transcription.text", "item_id": "item_AEQBaDSactuh6lFHdhLnh", "content_index": 0, "text": "", "stash": "大漠孤烟直"}
+
+[18:07:07]receive message:  {"event_id": "event_O28jlaFUWdxMRDA5WKOp8", "type": "conversation.item.input_audio_transcription.text", "item_id": "item_AEQBaDSactuh6lFHdhLnh", "content_index": 0, "text": "", "stash": "大漠孤烟直。"}
+
+[18:07:07]receive message:  {"event_id": "event_MN9TdqMM13VKHg6gthJSQ", "type": "conversation.item.input_audio_transcription.completed", "item_id": "item_AEQBaDSactuh6lFHdhLnh", "content_index": 0, "transcript": "大漠孤烟直。", "usage": {"duration": 5}}
+
+[18:07:07]receive message:  {"type": "final_transcript", "text": "大漠孤烟直。"}
+
+[18:07:10]receive message:  {"event_id": "event_OOc7k8cX7GKpjNqog71ZK", "type": "input_audio_buffer.speech_started", "audio_start_ms": 8060, "item_id": "item_S997Yu46cT66srf8uqqxj"}
+
+[18:07:12]receive message:  {"event_id": "event_Eo4KtXx5JVHP8V4qtKITT", "type": "conversation.item.created", "previous_item_id": "item_AEQBaDSactuh6lFHdhLnh", "item": {"id": "item_S997Yu46cT66srf8uqqxj", "object": "realtime.item", "type": "message", "status": "in_progress", "role": "assistant", "content": [{"type": "input_audio"}]}}
+
+[18:07:12]receive message:  {"event_id": "event_PHmM2m1scdbo8TjSyf1Fm", "type": "conversation.item.input_audio_transcription.text", "item_id": "item_S997Yu46cT66srf8uqqxj", "content_index": 0, "text": "", "stash": "长"}
+
+[18:07:12]receive message:  {"event_id": "event_QaoimHMAGiBeQbV9Ln1El", "type": "conversation.item.input_audio_transcription.text", "item_id": "item_S997Yu46cT66srf8uqqxj", "content_index": 0, "text": "", "stash": "长河"}
+
+[18:07:12]receive message:  {"event_id": "event_MovuQE8C6JnDrPcMTeIpJ", "type": "conversation.item.input_audio_transcription.text", "item_id": "item_S997Yu46cT66srf8uqqxj", "content_index": 0, "text": "", "stash": "长河落"}
+
+[18:07:12]receive message:  {"event_id": "event_F8fZyacfbtXgwNNizAQux", "type": "conversation.item.input_audio_transcription.text", "item_id": "item_S997Yu46cT66srf8uqqxj", "content_index": 0, "text": "", "stash": "长河落。"}
+
+[18:07:13]receive message:  {"event_id": "event_A9wzZZyznzSkouWsnUP4E", "type": "input_audio_buffer.speech_stopped", "audio_end_ms": 11460, "item_id": "item_S997Yu46cT66srf8uqqxj"}
+
+[18:07:13]receive message:  {"event_id": "event_UZJUdhU1OzbWViVEM6rM6", "type": "input_audio_buffer.committed", "previous_item_id": "item_AEQBaDSactuh6lFHdhLnh", "item_id": "item_S997Yu46cT66srf8uqqxj"}
+
+[18:07:14]receive message:  {"event_id": "event_SouO2xnlII5hbUJNPry5Q", "type": "conversation.item.input_audio_transcription.text", "item_id": "item_S997Yu46cT66srf8uqqxj", "content_index": 0, "text": "", "stash": "长河落日圆"}
+
+[18:07:14]receive message:  {"event_id": "event_V379dqiLrs1kSQExTKXjH", "type": "conversation.item.input_audio_transcription.text", "item_id": "item_S997Yu46cT66srf8uqqxj", "content_index": 0, "text": "", "stash": "长河落日圆。"}
+
+[18:07:14]receive message:  {"event_id": "event_ZmjEdcRtMGaOadgSl6sD8", "type": "conversation.item.input_audio_transcription.completed", "item_id": "item_S997Yu46cT66srf8uqqxj", "content_index": 0, "transcript": "长河落日圆。", "usage": {"duration": 11}}
+
+[18:07:14]receive message:  {"type": "final_transcript", "text": "长河落日圆。"}
+