diff --git a/backend/api/voice_chat_router.py b/backend/api/voice_chat_router.py index c304056..f194266 100644 --- a/backend/api/voice_chat_router.py +++ b/backend/api/voice_chat_router.py @@ -28,7 +28,9 @@ from services.realtime_voice_service import ( Speaker ) from utils.security import get_current_user -from utils.database import get_db +from utils.database import get_db, AsyncSessionLocal +from models.database import Pet +from sqlalchemy import select logger = logging.getLogger(__name__) @@ -54,6 +56,31 @@ class SpeakersResponse(BaseModel): data: dict +async def _load_pet_voice_config(pet_id: Optional[int]) -> Optional[dict]: + """按 pet_id 读取主角的实时语音配置""" + if not pet_id: + return None + + try: + async with AsyncSessionLocal() as db: + result = await db.execute(select(Pet).where(Pet.id == pet_id)) + pet = result.scalar_one_or_none() + + if not pet: + return None + + return { + "speaker": pet.volcano_voice_id, + "model": pet.volcano_model_version, + "bot_name": pet.volcano_bot_name, + "system_role": pet.volcano_system_role, + "character_manifest": pet.volcano_character_manifest + } + except Exception as e: + logger.warning(f"Load pet voice config failed: pet_id={pet_id}, error={str(e)}") + return None + + # ============ REST API ============ @voice_chat_router.get("/speakers", response_model=SpeakersResponse) @@ -268,14 +295,23 @@ async def websocket_realtime_call(websocket: WebSocket, user_id: str): msg_type = message.get("type") if msg_type == "start_call": - # 开始实时通话 + # 开始实时通话（优先使用主角配置） + pet_id = message.get("pet_id") + pet_config = await _load_pet_voice_config(pet_id) + + model_value = (pet_config.get("model") if pet_config else None) or message.get("model", "O") + speaker_value = (pet_config.get("speaker") if pet_config else None) or message.get("speaker", Speaker.YUNZHOU.value) + bot_name_value = (pet_config.get("bot_name") if pet_config else None) or message.get("bot_name", "AI助手") + system_role_value = (pet_config.get("system_role") if pet_config else None) or message.get("system_role", "") + config = SessionConfig( - model=ModelVersion(message.get("model", "O")), - speaker=message.get("speaker", Speaker.YUNZHOU.value), + model=ModelVersion(model_value), + speaker=speaker_value, audio_format=AudioFormat.PCM_S16LE, - bot_name=message.get("bot_name", "AI助手"), - system_role=message.get("system_role", ""), + bot_name=bot_name_value, + system_role=system_role_value, speaking_style=message.get("speaking_style", ""), + character_manifest=(pet_config.get("character_manifest") if pet_config else "") or message.get("character_manifest", ""), end_smooth_window_ms=message.get("end_smooth_window_ms", 1500), input_mod="audio" ) @@ -383,19 +419,28 @@ async def handle_start_session( existing_client: Optional[RealtimeVoiceClient] ) -> Optional[RealtimeVoiceClient]: """处理开始会话请求""" - + # 如果已有客户端，先关闭 if existing_client: await existing_client.close() - - # 解析配置 + + pet_id = message.get("pet_id") + pet_config = await _load_pet_voice_config(pet_id) + + model_value = (pet_config.get("model") if pet_config else None) or message.get("model", "O") + speaker_value = (pet_config.get("speaker") if pet_config else None) or message.get("speaker", Speaker.YUNZHOU.value) + bot_name_value = (pet_config.get("bot_name") if pet_config else None) or message.get("bot_name", "AI助手") + system_role_value = (pet_config.get("system_role") if pet_config else None) or message.get("system_role", "") + + # 解析配置（优先使用主角配置） config = SessionConfig( - model=ModelVersion(message.get("model", "O")), - speaker=message.get("speaker", Speaker.YUNZHOU.value), + model=ModelVersion(model_value), + speaker=speaker_value, audio_format=AudioFormat.PCM_S16LE, - bot_name=message.get("bot_name", "AI助手"), - system_role=message.get("system_role", ""), + bot_name=bot_name_value, + system_role=system_role_value, speaking_style=message.get("speaking_style", ""), + character_manifest=(pet_config.get("character_manifest") if pet_config else "") or message.get("character_manifest", ""), input_mod=message.get("input_mod", "audio"), recv_timeout=message.get("recv_timeout", 30) ) diff --git a/backend/config/settings.py b/backend/config/settings.py index 4e7f5c1..0965924 100644 --- a/backend/config/settings.py +++ b/backend/config/settings.py @@ -44,9 +44,6 @@ class Settings(BaseSettings): WECHAT_APP_ID: str = os.getenv("WECHAT_APP_ID", "") WECHAT_APP_SECRET: str = os.getenv("WECHAT_APP_SECRET", "") - TENCENT_CLOUD_SECRET_ID: str = os.getenv("TENCENT_CLOUD_SECRET_ID", "") - TENCENT_CLOUD_SECRET_KEY: str = os.getenv("TENCENT_CLOUD_SECRET_KEY", "") - TENCENT_CLOUD_REGION: str = os.getenv("TENCENT_CLOUD_REGION", "ap-beijing") VOLCANO_APP_ID: str = os.getenv("VOLCANO_APP_ID", "2661618707") VOLCANO_ACCESS_TOKEN: str = os.getenv("VOLCANO_ACCESS_TOKEN", "Ip4UpS92mfQBzkbTMHyGaIfpGcLUe1ZH") diff --git a/backend/requirements.txt b/backend/requirements.txt index 6fffad9..a50729d 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -14,7 +14,6 @@ python-multipart>=0.0.6 redis>=5.0.1 requests>=2.31.0 websockets>=12.0 -tencentcloud-sdk-python>=3.0.876 python-dotenv>=1.0.0 numpy>=1.24.3 scipy>=1.10.0 diff --git a/backend/services/tts_service.py b/backend/services/tts_service.py deleted file mode 100644 index 8e9a40d..0000000 --- a/backend/services/tts_service.py +++ /dev/null @@ -1,402 +0,0 @@ -""" -腾讯云TTS（语音合成）服务 -实现文字转语音功能 -""" - -import base64 -import json -import time -from datetime import datetime -from typing import Optional, Dict, Any, List -import logging -from tencentcloud.common import credential -from tencentcloud.common.profile.client_profile import ClientProfile -from tencentcloud.common.profile.http_profile import HttpProfile -from tencentcloud.common.exception.tencent_cloud_sdk_exception import TencentCloudSDKException -from tencentcloud.tts.v20190823 import tts_client, models -from config.settings import settings - -logger = logging.getLogger(__name__) - -class TTSService: - """腾讯云TTS语音合成服务""" - - def __init__(self): - self.secret_id = settings.TENCENT_CLOUD_SECRET_ID - self.secret_key = settings.TENCENT_CLOUD_SECRET_KEY - self.region = settings.TENCENT_CLOUD_REGION - - self.client = None - self.is_available = False - - try: - if not self.secret_id or not self.secret_key: - logger.warning("TTS service disabled: TENCENT_CLOUD_SECRET_ID or TENCENT_CLOUD_SECRET_KEY not configured") - self.is_available = False - return - - self.cred = credential.Credential(self.secret_id, self.secret_key) - self.httpProfile = HttpProfile() - self.httpProfile.endpoint = "tts.tencentcloudapi.com" - - self.clientProfile = ClientProfile() - self.clientProfile.httpProfile = self.httpProfile - - self.client = tts_client.TtsClient( - self.cred, - self.region, - self.clientProfile - ) - - self.is_available = True - logger.info("TTS service initialized successfully") - - self.default_voice = "101001" - self.default_speed = 0 - self.default_volume = 0 - self.default_pitch = 0 - self.default_format = "wav" - - except Exception as e: - logger.error(f"TTS service initialization failed: {e}") - self.is_available = False - - self.default_voice = "101001" - self.default_speed = 0 - self.default_volume = 0 - self.default_pitch = 0 - self.default_format = "wav" - - async def synthesize_speech( - self, - text: str, - voice_id: Optional[str] = None, - speed: Optional[int] = None, - volume: Optional[int] = None, - pitch: Optional[int] = None, - format: Optional[str] = None, - sample_rate: int = 16000, - codec: str = "raw" - ) -> Dict[str, Any]: - """ - 语音合成 - - Args: - text: 要合成的文本 - voice_id: 音色ID - speed: 语速 (-6到6，默认0) - volume: 音量 (-6到6，默认0) - pitch: 音调 (-6到6，默认0) - format: 音频格式 (wav, mp3, flac等) - sample_rate: 采样率 - codec: 编码格式 (raw, opus) - - Returns: - 包含音频数据的字典 - """ - if not self.is_available or not self.client: - return { - "success": False, - "audio_data": None, - "text": text, - "error": "TTS服务未初始化", - "message": "腾讯云TTS配置缺失或初始化失败" - } - - try: - # 构建请求 - req = models.TextToSpeechRequest() - req.Text = text - req.VoiceId = voice_id or self.default_voice - req.ModelType = 1 # 1代表标准版 - req.SampleRate = sample_rate - req.Speed = speed if speed is not None else self.default_speed - req.Volume = volume if volume is not None else self.default_volume - req.Pitch = pitch if pitch is not None else self.default_pitch - req.ProjectId = 0 # 默认项目ID - req.Codec = codec - req.Format = format or self.default_format - - # 发送请求 - start_time = time.time() - resp = await self._send_request(req) - duration_ms = int((time.time() - start_time) * 1000) - - if resp.Success: - # 解码音频数据 - audio_base64 = resp.Audio - audio_data = base64.b64decode(audio_base64) - - result = { - "success": True, - "audio_data": audio_data, - "audio_base64": audio_base64, - "text": text, - "voice_id": req.VoiceId, - "sample_rate": req.SampleRate, - "format": req.Format, - "duration_ms": duration_ms, - "audio_length": len(audio_data), - "message": "语音合成成功" - } - - logger.info( - f"TTS synthesis successful: voice={req.VoiceId}, " - f"text_length={len(text)}, audio_length={len(audio_data)}" - ) - else: - result = { - "success": False, - "audio_data": None, - "text": text, - "error": "合成失败", - "message": resp.Message if hasattr(resp, 'Message') else "未知错误", - "duration_ms": duration_ms - } - logger.warning(f"TTS synthesis failed: {result['message']}") - - return result - - except TencentCloudSDKException as e: - logger.error(f"TTS SDK error: {e.code} - {e.message}") - return { - "success": False, - "audio_data": None, - "text": text, - "error": f"SDK错误: {e.code}", - "message": e.message - } - except Exception as e: - logger.error(f"TTS synthesis error: {str(e)}") - return { - "success": False, - "audio_data": None, - "text": text, - "error": str(e), - "message": "合成过程出错" - } - - async def synthesize_streaming( - self, - text_list: List[str], - voice_id: Optional[str] = None, - speed: Optional[int] = None, - **kwargs - ) -> List[Dict[str, Any]]: - """ - 流式语音合成（多段文本） - - Args: - text_list: 文本列表 - voice_id: 音色ID - **kwargs: 其他参数 - - Returns: - 音频数据列表 - """ - results = [] - total_duration = 0 - total_audio_length = 0 - - for i, text in enumerate(text_list): - logger.info(f"Synthesizing chunk {i+1}/{len(text_list)}: {text[:50]}...") - - result = await self.synthesize_speech( - text=text, - voice_id=voice_id, - speed=speed, - **kwargs - ) - - if result["success"]: - total_duration += result["duration_ms"] - total_audio_length += result["audio_length"] - results.append(result) - else: - # 如果某段合成失败，添加错误信息 - results.append({ - "success": False, - "text": text, - "error": result["error"], - "message": result["message"] - }) - logger.error(f"Chunk {i+1} synthesis failed: {result['error']}") - - logger.info( - f"Streaming TTS completed: {len(results)} chunks, " - f"total_duration={total_duration}ms, " - f"total_audio_length={total_audio_length}bytes" - ) - - return results - - async def get_voice_list(self) -> Dict[str, Any]: - """ - 获取可用音色列表 - - Returns: - 音色列表 - """ - try: - # 注意：这是示例，实际API可能不同 - # 腾讯云TTS的音色列表需要通过其他方式获取 - # 这里提供一个预定义的音色列表 - - voices = { - "101001": {"name": "智云", "language": "zh", "gender": "female", "description": "温和女声"}, - "101002": {"name": "智云", "language": "zh", "gender": "male", "description": "沉稳男声"}, - "101003": {"name": "智香", "language": "zh", "gender": "female", "description": "甜美女声"}, - "101004": {"name": "智程", "language": "zh", "gender": "male", "description": "磁性男声"}, - "101005": {"name": "智娜", "language": "zh", "gender": "female", "description": "知性女声"}, - "101006": {"name": "智诚", "language": "zh", "gender": "male", "description": "年轻男声"}, - } - - result = { - "success": True, - "voices": voices, - "total": len(voices), - "message": "获取音色列表成功" - } - - logger.info(f"Retrieved {len(voices)} voices") - return result - - except Exception as e: - logger.error(f"Get voice list error: {str(e)}") - return { - "success": False, - "voices": {}, - "error": str(e), - "message": "获取音色列表失败" - } - - async def batch_synthesize( - self, - texts: List[Dict[str, Any]], - default_voice: Optional[str] = None - ) -> Dict[str, Any]: - """ - 批量语音合成 - - Args: - texts: 文本列表，每个元素包含text和可选参数 - default_voice: 默认音色ID - - Returns: - 批量合成结果 - """ - results = [] - success_count = 0 - failed_count = 0 - total_duration = 0 - total_audio_length = 0 - - for i, text_config in enumerate(texts): - text = text_config.get("text", "") - if not text: - logger.warning(f"Empty text at index {i}") - results.append({ - "success": False, - "text": "", - "error": "empty_text", - "message": "文本为空" - }) - failed_count += 1 - continue - - voice_id = text_config.get("voice_id", default_voice or self.default_voice) - speed = text_config.get("speed", self.default_speed) - - logger.info(f"Batch synthesizing {i+1}/{len(texts)}: {text[:50]}...") - - result = await self.synthesize_speech( - text=text, - voice_id=voice_id, - speed=speed - ) - - result["index"] = i - results.append(result) - - if result["success"]: - success_count += 1 - total_duration += result["duration_ms"] - total_audio_length += result["audio_length"] - else: - failed_count += 1 - - summary = { - "success_count": success_count, - "failed_count": failed_count, - "total": len(texts), - "success_rate": success_count / len(texts) if texts else 0, - "total_duration_ms": total_duration, - "total_audio_length": total_audio_length, - "results": results - } - - logger.info( - f"Batch TTS completed: {success_count}/{len(texts)} successful, " - f"success_rate={summary['success_rate']:.2%}" - ) - - return summary - - async def _send_request(self, req): - """发送请求的异步包装""" - import asyncio - loop = asyncio.get_event_loop() - - def _sync_request(): - return self.client.TextToSpeech(req) - - return await loop.run_in_executor(None, _sync_request) - - async def save_audio_to_file( - self, - audio_data: bytes, - filename: str, - format: str = "wav" - ) -> Dict[str, Any]: - """ - 将音频数据保存到文件 - - Args: - audio_data: 音频二进制数据 - filename: 文件名 - format: 音频格式 - - Returns: - 保存结果 - """ - try: - # 确保文件名有正确的扩展名 - if not filename.endswith(f".{format}"): - filename = f"{filename}.{format}" - - # 写入文件 - with open(filename, "wb") as f: - f.write(audio_data) - - result = { - "success": True, - "filename": filename, - "format": format, - "size": len(audio_data), - "message": f"音频已保存到 {filename}" - } - - logger.info(f"Audio saved to file: {filename}, size={len(audio_data)} bytes") - return result - - except Exception as e: - logger.error(f"Save audio to file error: {str(e)}") - return { - "success": False, - "filename": filename, - "error": str(e), - "message": "保存音频文件失败" - } - -# 创建全局实例 -tts_service = TTSService() diff --git a/frontend/src/stores/chatStore.js b/frontend/src/stores/chatStore.js index 75e25d5..2195b31 100644 --- a/frontend/src/stores/chatStore.js +++ b/frontend/src/stores/chatStore.js @@ -437,9 +437,21 @@ export const useChatStore = defineStore('chat', () => { await connectVoiceWebSocket() } + const selectedPet = JSON.parse(localStorage.getItem('selectedPet') || '{}') + const petConfig = { + pet_id: selectedPet.id || currentPetId.value, + speaker: selectedPet.volcano_voice_id || currentSessionConfig.value.speaker, + model: selectedPet.volcano_model_version || currentSessionConfig.value.model, + bot_name: selectedPet.volcano_bot_name || selectedPet.name || currentSessionConfig.value.bot_name, + system_role: selectedPet.volcano_system_role || selectedPet.global_prompt || currentSessionConfig.value.system_role, + character_manifest: selectedPet.volcano_character_manifest || '' + } + const sessionConfig = { ...currentSessionConfig.value, - ...config + ...petConfig, + ...config, + background_id: currentBackgroundId.value } sendVoiceMessage({ @@ -650,10 +662,22 @@ export const useChatStore = defineStore('chat', () => { await connectRealtimeWebSocket() } + const selectedPet = JSON.parse(localStorage.getItem('selectedPet') || '{}') + const petConfig = { + pet_id: selectedPet.id || currentPetId.value, + speaker: selectedPet.volcano_voice_id || currentSessionConfig.value.speaker, + model: selectedPet.volcano_model_version || currentSessionConfig.value.model, + bot_name: selectedPet.volcano_bot_name || selectedPet.name || currentSessionConfig.value.bot_name, + system_role: selectedPet.volcano_system_role || selectedPet.global_prompt || currentSessionConfig.value.system_role, + character_manifest: selectedPet.volcano_character_manifest || '' + } + const callConfig = { type: 'start_call', ...currentSessionConfig.value, - ...config + ...petConfig, + ...config, + background_id: currentBackgroundId.value } sendRealtimeMessage(callConfig)