优化智能体设置，可以实际引用
This commit is contained in:
parent
71479e2ad1
commit
4a4553eb48
@ -28,7 +28,9 @@ from services.realtime_voice_service import (
|
||||
Speaker
|
||||
)
|
||||
from utils.security import get_current_user
|
||||
from utils.database import get_db
|
||||
from utils.database import get_db, AsyncSessionLocal
|
||||
from models.database import Pet
|
||||
from sqlalchemy import select
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@ -54,6 +56,31 @@ class SpeakersResponse(BaseModel):
|
||||
data: dict
|
||||
|
||||
|
||||
async def _load_pet_voice_config(pet_id: Optional[int]) -> Optional[dict]:
|
||||
"""按 pet_id 读取主角的实时语音配置"""
|
||||
if not pet_id:
|
||||
return None
|
||||
|
||||
try:
|
||||
async with AsyncSessionLocal() as db:
|
||||
result = await db.execute(select(Pet).where(Pet.id == pet_id))
|
||||
pet = result.scalar_one_or_none()
|
||||
|
||||
if not pet:
|
||||
return None
|
||||
|
||||
return {
|
||||
"speaker": pet.volcano_voice_id,
|
||||
"model": pet.volcano_model_version,
|
||||
"bot_name": pet.volcano_bot_name,
|
||||
"system_role": pet.volcano_system_role,
|
||||
"character_manifest": pet.volcano_character_manifest
|
||||
}
|
||||
except Exception as e:
|
||||
logger.warning(f"Load pet voice config failed: pet_id={pet_id}, error={str(e)}")
|
||||
return None
|
||||
|
||||
|
||||
# ============ REST API ============
|
||||
|
||||
@voice_chat_router.get("/speakers", response_model=SpeakersResponse)
|
||||
@ -268,14 +295,23 @@ async def websocket_realtime_call(websocket: WebSocket, user_id: str):
|
||||
msg_type = message.get("type")
|
||||
|
||||
if msg_type == "start_call":
|
||||
# 开始实时通话
|
||||
# 开始实时通话（优先使用主角配置）
|
||||
pet_id = message.get("pet_id")
|
||||
pet_config = await _load_pet_voice_config(pet_id)
|
||||
|
||||
model_value = (pet_config.get("model") if pet_config else None) or message.get("model", "O")
|
||||
speaker_value = (pet_config.get("speaker") if pet_config else None) or message.get("speaker", Speaker.YUNZHOU.value)
|
||||
bot_name_value = (pet_config.get("bot_name") if pet_config else None) or message.get("bot_name", "AI助手")
|
||||
system_role_value = (pet_config.get("system_role") if pet_config else None) or message.get("system_role", "")
|
||||
|
||||
config = SessionConfig(
|
||||
model=ModelVersion(message.get("model", "O")),
|
||||
speaker=message.get("speaker", Speaker.YUNZHOU.value),
|
||||
model=ModelVersion(model_value),
|
||||
speaker=speaker_value,
|
||||
audio_format=AudioFormat.PCM_S16LE,
|
||||
bot_name=message.get("bot_name", "AI助手"),
|
||||
system_role=message.get("system_role", ""),
|
||||
bot_name=bot_name_value,
|
||||
system_role=system_role_value,
|
||||
speaking_style=message.get("speaking_style", ""),
|
||||
character_manifest=(pet_config.get("character_manifest") if pet_config else "") or message.get("character_manifest", ""),
|
||||
end_smooth_window_ms=message.get("end_smooth_window_ms", 1500),
|
||||
input_mod="audio"
|
||||
)
|
||||
@ -383,19 +419,28 @@ async def handle_start_session(
|
||||
existing_client: Optional[RealtimeVoiceClient]
|
||||
) -> Optional[RealtimeVoiceClient]:
|
||||
"""处理开始会话请求"""
|
||||
|
||||
|
||||
# 如果已有客户端，先关闭
|
||||
if existing_client:
|
||||
await existing_client.close()
|
||||
|
||||
# 解析配置
|
||||
|
||||
pet_id = message.get("pet_id")
|
||||
pet_config = await _load_pet_voice_config(pet_id)
|
||||
|
||||
model_value = (pet_config.get("model") if pet_config else None) or message.get("model", "O")
|
||||
speaker_value = (pet_config.get("speaker") if pet_config else None) or message.get("speaker", Speaker.YUNZHOU.value)
|
||||
bot_name_value = (pet_config.get("bot_name") if pet_config else None) or message.get("bot_name", "AI助手")
|
||||
system_role_value = (pet_config.get("system_role") if pet_config else None) or message.get("system_role", "")
|
||||
|
||||
# 解析配置（优先使用主角配置）
|
||||
config = SessionConfig(
|
||||
model=ModelVersion(message.get("model", "O")),
|
||||
speaker=message.get("speaker", Speaker.YUNZHOU.value),
|
||||
model=ModelVersion(model_value),
|
||||
speaker=speaker_value,
|
||||
audio_format=AudioFormat.PCM_S16LE,
|
||||
bot_name=message.get("bot_name", "AI助手"),
|
||||
system_role=message.get("system_role", ""),
|
||||
bot_name=bot_name_value,
|
||||
system_role=system_role_value,
|
||||
speaking_style=message.get("speaking_style", ""),
|
||||
character_manifest=(pet_config.get("character_manifest") if pet_config else "") or message.get("character_manifest", ""),
|
||||
input_mod=message.get("input_mod", "audio"),
|
||||
recv_timeout=message.get("recv_timeout", 30)
|
||||
)
|
||||
|
||||
@ -44,9 +44,6 @@ class Settings(BaseSettings):
|
||||
WECHAT_APP_ID: str = os.getenv("WECHAT_APP_ID", "")
|
||||
WECHAT_APP_SECRET: str = os.getenv("WECHAT_APP_SECRET", "")
|
||||
|
||||
TENCENT_CLOUD_SECRET_ID: str = os.getenv("TENCENT_CLOUD_SECRET_ID", "")
|
||||
TENCENT_CLOUD_SECRET_KEY: str = os.getenv("TENCENT_CLOUD_SECRET_KEY", "")
|
||||
TENCENT_CLOUD_REGION: str = os.getenv("TENCENT_CLOUD_REGION", "ap-beijing")
|
||||
|
||||
VOLCANO_APP_ID: str = os.getenv("VOLCANO_APP_ID", "2661618707")
|
||||
VOLCANO_ACCESS_TOKEN: str = os.getenv("VOLCANO_ACCESS_TOKEN", "Ip4UpS92mfQBzkbTMHyGaIfpGcLUe1ZH")
|
||||
|
||||
@ -14,7 +14,6 @@ python-multipart>=0.0.6
|
||||
redis>=5.0.1
|
||||
requests>=2.31.0
|
||||
websockets>=12.0
|
||||
tencentcloud-sdk-python>=3.0.876
|
||||
python-dotenv>=1.0.0
|
||||
numpy>=1.24.3
|
||||
scipy>=1.10.0
|
||||
|
||||
@ -1,402 +0,0 @@
|
||||
"""
|
||||
腾讯云TTS（语音合成）服务
|
||||
实现文字转语音功能
|
||||
"""
|
||||
|
||||
import base64
|
||||
import json
|
||||
import time
|
||||
from datetime import datetime
|
||||
from typing import Optional, Dict, Any, List
|
||||
import logging
|
||||
from tencentcloud.common import credential
|
||||
from tencentcloud.common.profile.client_profile import ClientProfile
|
||||
from tencentcloud.common.profile.http_profile import HttpProfile
|
||||
from tencentcloud.common.exception.tencent_cloud_sdk_exception import TencentCloudSDKException
|
||||
from tencentcloud.tts.v20190823 import tts_client, models
|
||||
from config.settings import settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
class TTSService:
|
||||
"""腾讯云TTS语音合成服务"""
|
||||
|
||||
def __init__(self):
|
||||
self.secret_id = settings.TENCENT_CLOUD_SECRET_ID
|
||||
self.secret_key = settings.TENCENT_CLOUD_SECRET_KEY
|
||||
self.region = settings.TENCENT_CLOUD_REGION
|
||||
|
||||
self.client = None
|
||||
self.is_available = False
|
||||
|
||||
try:
|
||||
if not self.secret_id or not self.secret_key:
|
||||
logger.warning("TTS service disabled: TENCENT_CLOUD_SECRET_ID or TENCENT_CLOUD_SECRET_KEY not configured")
|
||||
self.is_available = False
|
||||
return
|
||||
|
||||
self.cred = credential.Credential(self.secret_id, self.secret_key)
|
||||
self.httpProfile = HttpProfile()
|
||||
self.httpProfile.endpoint = "tts.tencentcloudapi.com"
|
||||
|
||||
self.clientProfile = ClientProfile()
|
||||
self.clientProfile.httpProfile = self.httpProfile
|
||||
|
||||
self.client = tts_client.TtsClient(
|
||||
self.cred,
|
||||
self.region,
|
||||
self.clientProfile
|
||||
)
|
||||
|
||||
self.is_available = True
|
||||
logger.info("TTS service initialized successfully")
|
||||
|
||||
self.default_voice = "101001"
|
||||
self.default_speed = 0
|
||||
self.default_volume = 0
|
||||
self.default_pitch = 0
|
||||
self.default_format = "wav"
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"TTS service initialization failed: {e}")
|
||||
self.is_available = False
|
||||
|
||||
self.default_voice = "101001"
|
||||
self.default_speed = 0
|
||||
self.default_volume = 0
|
||||
self.default_pitch = 0
|
||||
self.default_format = "wav"
|
||||
|
||||
async def synthesize_speech(
|
||||
self,
|
||||
text: str,
|
||||
voice_id: Optional[str] = None,
|
||||
speed: Optional[int] = None,
|
||||
volume: Optional[int] = None,
|
||||
pitch: Optional[int] = None,
|
||||
format: Optional[str] = None,
|
||||
sample_rate: int = 16000,
|
||||
codec: str = "raw"
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
语音合成
|
||||
|
||||
Args:
|
||||
text: 要合成的文本
|
||||
voice_id: 音色ID
|
||||
speed: 语速 (-6到6，默认0)
|
||||
volume: 音量 (-6到6，默认0)
|
||||
pitch: 音调 (-6到6，默认0)
|
||||
format: 音频格式 (wav, mp3, flac等)
|
||||
sample_rate: 采样率
|
||||
codec: 编码格式 (raw, opus)
|
||||
|
||||
Returns:
|
||||
包含音频数据的字典
|
||||
"""
|
||||
if not self.is_available or not self.client:
|
||||
return {
|
||||
"success": False,
|
||||
"audio_data": None,
|
||||
"text": text,
|
||||
"error": "TTS服务未初始化",
|
||||
"message": "腾讯云TTS配置缺失或初始化失败"
|
||||
}
|
||||
|
||||
try:
|
||||
# 构建请求
|
||||
req = models.TextToSpeechRequest()
|
||||
req.Text = text
|
||||
req.VoiceId = voice_id or self.default_voice
|
||||
req.ModelType = 1 # 1代表标准版
|
||||
req.SampleRate = sample_rate
|
||||
req.Speed = speed if speed is not None else self.default_speed
|
||||
req.Volume = volume if volume is not None else self.default_volume
|
||||
req.Pitch = pitch if pitch is not None else self.default_pitch
|
||||
req.ProjectId = 0 # 默认项目ID
|
||||
req.Codec = codec
|
||||
req.Format = format or self.default_format
|
||||
|
||||
# 发送请求
|
||||
start_time = time.time()
|
||||
resp = await self._send_request(req)
|
||||
duration_ms = int((time.time() - start_time) * 1000)
|
||||
|
||||
if resp.Success:
|
||||
# 解码音频数据
|
||||
audio_base64 = resp.Audio
|
||||
audio_data = base64.b64decode(audio_base64)
|
||||
|
||||
result = {
|
||||
"success": True,
|
||||
"audio_data": audio_data,
|
||||
"audio_base64": audio_base64,
|
||||
"text": text,
|
||||
"voice_id": req.VoiceId,
|
||||
"sample_rate": req.SampleRate,
|
||||
"format": req.Format,
|
||||
"duration_ms": duration_ms,
|
||||
"audio_length": len(audio_data),
|
||||
"message": "语音合成成功"
|
||||
}
|
||||
|
||||
logger.info(
|
||||
f"TTS synthesis successful: voice={req.VoiceId}, "
|
||||
f"text_length={len(text)}, audio_length={len(audio_data)}"
|
||||
)
|
||||
else:
|
||||
result = {
|
||||
"success": False,
|
||||
"audio_data": None,
|
||||
"text": text,
|
||||
"error": "合成失败",
|
||||
"message": resp.Message if hasattr(resp, 'Message') else "未知错误",
|
||||
"duration_ms": duration_ms
|
||||
}
|
||||
logger.warning(f"TTS synthesis failed: {result['message']}")
|
||||
|
||||
return result
|
||||
|
||||
except TencentCloudSDKException as e:
|
||||
logger.error(f"TTS SDK error: {e.code} - {e.message}")
|
||||
return {
|
||||
"success": False,
|
||||
"audio_data": None,
|
||||
"text": text,
|
||||
"error": f"SDK错误: {e.code}",
|
||||
"message": e.message
|
||||
}
|
||||
except Exception as e:
|
||||
logger.error(f"TTS synthesis error: {str(e)}")
|
||||
return {
|
||||
"success": False,
|
||||
"audio_data": None,
|
||||
"text": text,
|
||||
"error": str(e),
|
||||
"message": "合成过程出错"
|
||||
}
|
||||
|
||||
async def synthesize_streaming(
|
||||
self,
|
||||
text_list: List[str],
|
||||
voice_id: Optional[str] = None,
|
||||
speed: Optional[int] = None,
|
||||
**kwargs
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
流式语音合成（多段文本）
|
||||
|
||||
Args:
|
||||
text_list: 文本列表
|
||||
voice_id: 音色ID
|
||||
**kwargs: 其他参数
|
||||
|
||||
Returns:
|
||||
音频数据列表
|
||||
"""
|
||||
results = []
|
||||
total_duration = 0
|
||||
total_audio_length = 0
|
||||
|
||||
for i, text in enumerate(text_list):
|
||||
logger.info(f"Synthesizing chunk {i+1}/{len(text_list)}: {text[:50]}...")
|
||||
|
||||
result = await self.synthesize_speech(
|
||||
text=text,
|
||||
voice_id=voice_id,
|
||||
speed=speed,
|
||||
**kwargs
|
||||
)
|
||||
|
||||
if result["success"]:
|
||||
total_duration += result["duration_ms"]
|
||||
total_audio_length += result["audio_length"]
|
||||
results.append(result)
|
||||
else:
|
||||
# 如果某段合成失败，添加错误信息
|
||||
results.append({
|
||||
"success": False,
|
||||
"text": text,
|
||||
"error": result["error"],
|
||||
"message": result["message"]
|
||||
})
|
||||
logger.error(f"Chunk {i+1} synthesis failed: {result['error']}")
|
||||
|
||||
logger.info(
|
||||
f"Streaming TTS completed: {len(results)} chunks, "
|
||||
f"total_duration={total_duration}ms, "
|
||||
f"total_audio_length={total_audio_length}bytes"
|
||||
)
|
||||
|
||||
return results
|
||||
|
||||
async def get_voice_list(self) -> Dict[str, Any]:
|
||||
"""
|
||||
获取可用音色列表
|
||||
|
||||
Returns:
|
||||
音色列表
|
||||
"""
|
||||
try:
|
||||
# 注意：这是示例，实际API可能不同
|
||||
# 腾讯云TTS的音色列表需要通过其他方式获取
|
||||
# 这里提供一个预定义的音色列表
|
||||
|
||||
voices = {
|
||||
"101001": {"name": "智云", "language": "zh", "gender": "female", "description": "温和女声"},
|
||||
"101002": {"name": "智云", "language": "zh", "gender": "male", "description": "沉稳男声"},
|
||||
"101003": {"name": "智香", "language": "zh", "gender": "female", "description": "甜美女声"},
|
||||
"101004": {"name": "智程", "language": "zh", "gender": "male", "description": "磁性男声"},
|
||||
"101005": {"name": "智娜", "language": "zh", "gender": "female", "description": "知性女声"},
|
||||
"101006": {"name": "智诚", "language": "zh", "gender": "male", "description": "年轻男声"},
|
||||
}
|
||||
|
||||
result = {
|
||||
"success": True,
|
||||
"voices": voices,
|
||||
"total": len(voices),
|
||||
"message": "获取音色列表成功"
|
||||
}
|
||||
|
||||
logger.info(f"Retrieved {len(voices)} voices")
|
||||
return result
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Get voice list error: {str(e)}")
|
||||
return {
|
||||
"success": False,
|
||||
"voices": {},
|
||||
"error": str(e),
|
||||
"message": "获取音色列表失败"
|
||||
}
|
||||
|
||||
async def batch_synthesize(
|
||||
self,
|
||||
texts: List[Dict[str, Any]],
|
||||
default_voice: Optional[str] = None
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
批量语音合成
|
||||
|
||||
Args:
|
||||
texts: 文本列表，每个元素包含text和可选参数
|
||||
default_voice: 默认音色ID
|
||||
|
||||
Returns:
|
||||
批量合成结果
|
||||
"""
|
||||
results = []
|
||||
success_count = 0
|
||||
failed_count = 0
|
||||
total_duration = 0
|
||||
total_audio_length = 0
|
||||
|
||||
for i, text_config in enumerate(texts):
|
||||
text = text_config.get("text", "")
|
||||
if not text:
|
||||
logger.warning(f"Empty text at index {i}")
|
||||
results.append({
|
||||
"success": False,
|
||||
"text": "",
|
||||
"error": "empty_text",
|
||||
"message": "文本为空"
|
||||
})
|
||||
failed_count += 1
|
||||
continue
|
||||
|
||||
voice_id = text_config.get("voice_id", default_voice or self.default_voice)
|
||||
speed = text_config.get("speed", self.default_speed)
|
||||
|
||||
logger.info(f"Batch synthesizing {i+1}/{len(texts)}: {text[:50]}...")
|
||||
|
||||
result = await self.synthesize_speech(
|
||||
text=text,
|
||||
voice_id=voice_id,
|
||||
speed=speed
|
||||
)
|
||||
|
||||
result["index"] = i
|
||||
results.append(result)
|
||||
|
||||
if result["success"]:
|
||||
success_count += 1
|
||||
total_duration += result["duration_ms"]
|
||||
total_audio_length += result["audio_length"]
|
||||
else:
|
||||
failed_count += 1
|
||||
|
||||
summary = {
|
||||
"success_count": success_count,
|
||||
"failed_count": failed_count,
|
||||
"total": len(texts),
|
||||
"success_rate": success_count / len(texts) if texts else 0,
|
||||
"total_duration_ms": total_duration,
|
||||
"total_audio_length": total_audio_length,
|
||||
"results": results
|
||||
}
|
||||
|
||||
logger.info(
|
||||
f"Batch TTS completed: {success_count}/{len(texts)} successful, "
|
||||
f"success_rate={summary['success_rate']:.2%}"
|
||||
)
|
||||
|
||||
return summary
|
||||
|
||||
async def _send_request(self, req):
|
||||
"""发送请求的异步包装"""
|
||||
import asyncio
|
||||
loop = asyncio.get_event_loop()
|
||||
|
||||
def _sync_request():
|
||||
return self.client.TextToSpeech(req)
|
||||
|
||||
return await loop.run_in_executor(None, _sync_request)
|
||||
|
||||
async def save_audio_to_file(
|
||||
self,
|
||||
audio_data: bytes,
|
||||
filename: str,
|
||||
format: str = "wav"
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
将音频数据保存到文件
|
||||
|
||||
Args:
|
||||
audio_data: 音频二进制数据
|
||||
filename: 文件名
|
||||
format: 音频格式
|
||||
|
||||
Returns:
|
||||
保存结果
|
||||
"""
|
||||
try:
|
||||
# 确保文件名有正确的扩展名
|
||||
if not filename.endswith(f".{format}"):
|
||||
filename = f"{filename}.{format}"
|
||||
|
||||
# 写入文件
|
||||
with open(filename, "wb") as f:
|
||||
f.write(audio_data)
|
||||
|
||||
result = {
|
||||
"success": True,
|
||||
"filename": filename,
|
||||
"format": format,
|
||||
"size": len(audio_data),
|
||||
"message": f"音频已保存到 {filename}"
|
||||
}
|
||||
|
||||
logger.info(f"Audio saved to file: {filename}, size={len(audio_data)} bytes")
|
||||
return result
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Save audio to file error: {str(e)}")
|
||||
return {
|
||||
"success": False,
|
||||
"filename": filename,
|
||||
"error": str(e),
|
||||
"message": "保存音频文件失败"
|
||||
}
|
||||
|
||||
# 创建全局实例
|
||||
tts_service = TTSService()
|
||||
@ -437,9 +437,21 @@ export const useChatStore = defineStore('chat', () => {
|
||||
await connectVoiceWebSocket()
|
||||
}
|
||||
|
||||
const selectedPet = JSON.parse(localStorage.getItem('selectedPet') || '{}')
|
||||
const petConfig = {
|
||||
pet_id: selectedPet.id || currentPetId.value,
|
||||
speaker: selectedPet.volcano_voice_id || currentSessionConfig.value.speaker,
|
||||
model: selectedPet.volcano_model_version || currentSessionConfig.value.model,
|
||||
bot_name: selectedPet.volcano_bot_name || selectedPet.name || currentSessionConfig.value.bot_name,
|
||||
system_role: selectedPet.volcano_system_role || selectedPet.global_prompt || currentSessionConfig.value.system_role,
|
||||
character_manifest: selectedPet.volcano_character_manifest || ''
|
||||
}
|
||||
|
||||
const sessionConfig = {
|
||||
...currentSessionConfig.value,
|
||||
...config
|
||||
...petConfig,
|
||||
...config,
|
||||
background_id: currentBackgroundId.value
|
||||
}
|
||||
|
||||
sendVoiceMessage({
|
||||
@ -650,10 +662,22 @@ export const useChatStore = defineStore('chat', () => {
|
||||
await connectRealtimeWebSocket()
|
||||
}
|
||||
|
||||
const selectedPet = JSON.parse(localStorage.getItem('selectedPet') || '{}')
|
||||
const petConfig = {
|
||||
pet_id: selectedPet.id || currentPetId.value,
|
||||
speaker: selectedPet.volcano_voice_id || currentSessionConfig.value.speaker,
|
||||
model: selectedPet.volcano_model_version || currentSessionConfig.value.model,
|
||||
bot_name: selectedPet.volcano_bot_name || selectedPet.name || currentSessionConfig.value.bot_name,
|
||||
system_role: selectedPet.volcano_system_role || selectedPet.global_prompt || currentSessionConfig.value.system_role,
|
||||
character_manifest: selectedPet.volcano_character_manifest || ''
|
||||
}
|
||||
|
||||
const callConfig = {
|
||||
type: 'start_call',
|
||||
...currentSessionConfig.value,
|
||||
...config
|
||||
...petConfig,
|
||||
...config,
|
||||
background_id: currentBackgroundId.value
|
||||
}
|
||||
|
||||
sendRealtimeMessage(callConfig)
|
||||
|
||||
Loading…
Reference in New Issue
Block a user