"""统计服务:日志查询、趋势数据、知识库健康度、Token 成本。""" import csv import io import json from insurance.db.compat import db from sqlalchemy import text class StatsService: """统计报表业务逻辑。""" def _parse_model_name(self, provider: str = "", model_id: str = "", model_json=None) -> str: """Extract provider/model from Dify app model config fields.""" if provider and model_id: return f"{provider}/{model_id}" if not model_json: return "" try: model_data = json.loads(model_json) if isinstance(model_json, str) and model_json.startswith("{") else model_json except (json.JSONDecodeError, TypeError): return "" if not isinstance(model_data, dict): return "" parsed_provider = model_data.get("provider", "") parsed_model = model_data.get("name", "") or model_data.get("model", "") or model_data.get("model_id", "") if parsed_provider and parsed_model and not isinstance(parsed_model, dict): return f"{parsed_provider}/{parsed_model}" nested = model_data.get("model", {}) if isinstance(nested, dict): nested_provider = nested.get("provider", "") or parsed_provider nested_model = nested.get("name", "") or nested.get("model", "") or nested.get("model_id", "") if nested_provider and nested_model: return f"{nested_provider}/{nested_model}" return "" def _get_default_model_name(self) -> str: """Get the latest configured app model name for records saved before model_id was populated.""" try: result = db.session.execute(text(""" SELECT provider, model_id, model FROM app_model_configs ORDER BY updated_at DESC """)).fetchall() for provider, model_id, model_json in result: model_name = self._parse_model_name(provider, model_id, model_json) if model_name: return model_name except Exception: pass return "unknown" def get_chat_logs(self, params: dict) -> dict: """查询对话记录(从本地 insurance_chat_records 表获取)。""" # 构建 WHERE 条件 conditions = ["u.role = 'user'"] bind_params = {} if params.get("user_id"): conditions.append("u.user_id = :user_id") bind_params["user_id"] = params["user_id"] if params.get("start_date"): conditions.append("u.created_at >= :start_date") bind_params["start_date"] = params["start_date"] if params.get("end_date"): conditions.append("u.created_at <= :end_date") bind_params["end_date"] = params["end_date"] + " 23:59:59" if params.get("keyword"): conditions.append(""" (u.content LIKE :keyword OR EXISTS ( SELECT 1 FROM insurance_chat_records a2 WHERE a2.session_id = u.session_id AND a2.role = 'assistant' AND a2.content LIKE :keyword )) """) bind_params["keyword"] = f"%{params['keyword']}%" if params.get("rating"): conditions.append(""" EXISTS ( SELECT 1 FROM insurance_chat_records a3 WHERE a3.session_id = u.session_id AND a3.role = 'assistant' AND a3.rating = :rating ) """) bind_params["rating"] = params["rating"] where_clause = " AND ".join(conditions) # 查询总数(只计算用户消息数,每条用户消息代表一个问答对) count_sql = text(f""" SELECT COUNT(*) as total FROM insurance_chat_records u WHERE {where_clause} """) total_result = db.session.execute(count_sql, bind_params) total = total_result.scalar() or 0 # 查询数据(用户问题 + 助手回答配对) # 使用子查询确保每条用户消息只匹配一条助手消息(兼容 PostgreSQL 和 SQLite) query_sql = text(f""" SELECT u.id, u.user_id as user_id, COALESCE(NULLIF(m.real_name, ''), NULLIF(m.username, ''), u.user_id) as "user", COALESCE(NULLIF(m.real_name, ''), NULLIF(m.username, ''), u.user_id) as user_name, m.username as username, u.content as query, COALESCE( (SELECT a.content FROM insurance_chat_records a WHERE a.session_id = u.session_id AND a.role = 'assistant' AND a.created_at > u.created_at ORDER BY a.created_at ASC LIMIT 1), '' ) as answer, COALESCE( (SELECT a.rating FROM insurance_chat_records a WHERE a.session_id = u.session_id AND a.role = 'assistant' AND a.created_at > u.created_at ORDER BY a.created_at ASC LIMIT 1), '' ) as feedback, COALESCE( (SELECT a.correction FROM insurance_chat_records a WHERE a.session_id = u.session_id AND a.role = 'assistant' AND a.created_at > u.created_at ORDER BY a.created_at ASC LIMIT 1), '' ) as correction, u.created_at FROM insurance_chat_records u LEFT JOIN wecom_user_mapping m ON u.user_id = CAST(m.id AS TEXT) WHERE {where_clause} ORDER BY u.created_at DESC LIMIT :limit OFFSET :offset """) bind_params["limit"] = params["page_size"] bind_params["offset"] = (params["page"] - 1) * params["page_size"] result = db.session.execute(query_sql, bind_params) items = [row._asdict() for row in result] # 格式化时间字段 for item in items: if item.get("created_at"): item["created_at"] = str(item["created_at"]) return {"code": 0, "data": {"items": items, "total": total}} def export_chat_logs(self, params: dict) -> str: """导出问答记录为 CSV(带 UTF-8 BOM,解决 Excel 乱码问题)。""" result = self.get_chat_logs({**params, "page": 1, "page_size": 10000}) output = io.StringIO() # 添加 UTF-8 BOM,让 Excel 正确识别中文编码 output.write('') writer = csv.writer(output) writer.writerow(["ID", "用户", "问题", "回答", "评分", "纠错内容", "时间"]) for item in result.get("data", {}).get("items", []): writer.writerow([ item.get("id", ""), item.get("user", ""), item.get("query", ""), item.get("answer", ""), item.get("feedback", ""), item.get("correction", ""), item.get("created_at", ""), ]) return output.getvalue() def get_system_logs(self, params: dict) -> dict: """查询系统操作日志。""" # 构建WHERE条件 conditions = ["1=1"] bind_params = {} if params.get("action"): conditions.append("action = :action") bind_params["action"] = params["action"] if params.get("user_id"): conditions.append("user_id = :user_id") bind_params["user_id"] = params["user_id"] where_clause = " AND ".join(conditions) # 查询总数 count_sql = text(f""" SELECT COUNT(*) as total FROM system_operation_logs WHERE {where_clause} """) total_result = db.session.execute(count_sql, bind_params) total = total_result.scalar() or 0 # 查询数据 query_sql = text(f""" SELECT id, user_id, action, target_type, target_id, detail, ip, user_agent, created_at FROM system_operation_logs WHERE {where_clause} ORDER BY created_at DESC LIMIT :limit OFFSET :offset """) bind_params["limit"] = params["page_size"] bind_params["offset"] = (params["page"] - 1) * params["page_size"] result = db.session.execute(query_sql, bind_params) items = [row._asdict() for row in result] return {"code": 0, "data": {"total": total, "items": items}} def get_overview(self) -> dict: """使用概览:今日问答数、活跃用户、问答记录总数。""" from datetime import datetime result = { "today_chats": 0, "active_users": 0, "total_documents": 0, } today_start = datetime.now().replace(hour=0, minute=0, second=0, microsecond=0) try: # 今日问答数:统计 insurance_chat_records 表中今日的用户消息数 count_sql = text(""" SELECT COUNT(*) as total FROM insurance_chat_records WHERE role = 'user' AND created_at >= :today_start """) total_result = db.session.execute(count_sql, {"today_start": today_start}) result["today_chats"] = total_result.scalar() or 0 except Exception: pass try: # 活跃用户数:统计今日在 insurance_chat_records 表中有消息的去重用户数 active_sql = text(""" SELECT COUNT(DISTINCT user_id) as total FROM insurance_chat_records WHERE role = 'user' AND created_at >= :today_start """) active_result = db.session.execute(active_sql, {"today_start": today_start}) result["active_users"] = active_result.scalar() or 0 except Exception: pass try: # 文档总数:统计所有历史问答对数量(每条用户消息代表一个问答对) doc_sql = text(""" SELECT COUNT(*) as total FROM insurance_chat_records WHERE role = 'user' """) doc_result = db.session.execute(doc_sql) result["total_documents"] = doc_result.scalar() or 0 except Exception: pass return {"code": 0, "data": result} def get_trend(self, metric: str, start_date: str, end_date: str, granularity: str) -> dict: """趋势数据(折线图数据源)。""" from datetime import datetime, timedelta # 如果未提供日期,默认最近 30 天 now = datetime.now() if not start_date or not end_date: end = now start = now - timedelta(days=30) else: try: start = datetime.strptime(start_date, "%Y-%m-%d") end = datetime.strptime(end_date, "%Y-%m-%d") except ValueError: return {"code": 1001, "message": "日期格式错误,请使用 YYYY-MM-DD 格式", "data": None} # 根据指标选择查询(使用 insurance_chat_records 表) if metric == "chat_count": # 每日问答数(只统计用户消息,每条代表一个问答对) sql = text(""" SELECT DATE(created_at) as date, COUNT(*) as value FROM insurance_chat_records WHERE role = 'user' AND created_at >= :start_date AND created_at <= :end_date GROUP BY DATE(created_at) ORDER BY date DESC """) elif metric == "active_users": # 每日活跃用户数 sql = text(""" SELECT DATE(created_at) as date, COUNT(DISTINCT user_id) as value FROM insurance_chat_records WHERE role = 'user' AND created_at >= :start_date AND created_at <= :end_date GROUP BY DATE(created_at) ORDER BY date DESC """) else: return {"code": 1001, "message": f"不支持的指标: {metric}", "data": None} try: result = db.session.execute(sql, { "start_date": start.strftime("%Y-%m-%d"), "end_date": end.strftime("%Y-%m-%d") + " 23:59:59", }).fetchall() # 转换为字典,方便查找 data_map = {row[0].strftime("%Y-%m-%d") if hasattr(row[0], 'strftime') else str(row[0]): row[1] for row in result} # 补全日期(没有数据的日期填 0) data_points = [] current = start while current <= end: date_str = current.strftime("%Y-%m-%d") data_points.append({ "date": date_str, "value": data_map.get(date_str, 0), }) current += timedelta(days=1) data_points.reverse() except Exception as e: return {"code": 5001, "message": f"获取趋势数据失败: {str(e)}", "data": None} return { "code": 0, "data": { "metric": metric, "granularity": granularity, "data": data_points, }, } def get_kb_health(self, days: int) -> dict: """知识库健康度。""" import requests from flask import current_app base_url = current_app.config.get("BAODAN_API_URL", "http://localhost:5001") api_key = current_app.config.get("BAODAN_CHAT_API_KEY", "") result = { "hot_questions": [], "missed_questions": [], "coverage": {}, } try: # 获取热门问题(从对话日志中统计) resp = requests.get( f"{base_url}/v1/messages", params={"limit": 100, "user": ""}, headers={"Authorization": f"Bearer {api_key}"}, timeout=10, ) if resp.status_code == 200: messages = resp.json().get("data", []) # 统计问题频次 question_counts = {} for msg in messages: query = msg.get("query", "") if query: question_counts[query] = question_counts.get(query, 0) + 1 # 取Top20 sorted_questions = sorted(question_counts.items(), key=lambda x: x[1], reverse=True)[:20] result["hot_questions"] = [ {"question": q, "count": c} for q, c in sorted_questions ] except Exception: pass return {"code": 0, "data": result} def get_token_usage(self, params: dict) -> dict: """Token 使用统计:总量 + 按模型分组 + 按用户分组 + 日趋势。 查询 insurance_chat_records 表。 """ import logging logging.info(f"[TOKEN_USAGE] 收到请求: params={params}") default_model = self._get_default_model_name() conditions = ["1=1"] bind_params = {"default_model": default_model} # 时间范围筛选 if params.get("start_date"): conditions.append("created_at >= :start_date") bind_params["start_date"] = params["start_date"] if params.get("end_date"): conditions.append("created_at <= :end_date") bind_params["end_date"] = params["end_date"] + " 23:59:59" # 模型筛选 if params.get("models"): model_list = params["models"] placeholders = ", ".join([f":model_{i}" for i in range(len(model_list))]) conditions.append(f"COALESCE(NULLIF(model_id, ''), :default_model) IN ({placeholders})") for i, model in enumerate(model_list): bind_params[f"model_{i}"] = model where_clause = " AND ".join(conditions) logging.info(f"[TOKEN_USAGE] SQL WHERE: {where_clause}, params={bind_params}") try: # 1. 总量统计(只统计 assistant 消息的 answer_tokens,user 消息的 message_tokens) total_sql = text(f""" SELECT COALESCE(SUM(CASE WHEN role = 'assistant' THEN answer_tokens ELSE message_tokens END), 0) as total_tokens, COALESCE(SUM(message_tokens), 0) as total_input_tokens, COALESCE(SUM(answer_tokens), 0) as total_output_tokens FROM insurance_chat_records WHERE {where_clause} """) total_result = db.session.execute(total_sql, bind_params).fetchone() logging.info(f"[TOKEN_USAGE] 总量统计: total_tokens={total_result.total_tokens if total_result else 0}, input={total_result.total_input_tokens if total_result else 0}, output={total_result.total_output_tokens if total_result else 0}") # 2. 按模型分组 model_sql = text(f""" SELECT COALESCE(NULLIF(model_id, ''), :default_model) as model, COALESCE(SUM(CASE WHEN role = 'assistant' THEN answer_tokens ELSE message_tokens END), 0) as total_tokens, COUNT(*) as message_count FROM insurance_chat_records WHERE {where_clause} GROUP BY COALESCE(NULLIF(model_id, ''), :default_model) ORDER BY total_tokens DESC """) model_result = db.session.execute(model_sql, bind_params).fetchall() by_model = [row._asdict() for row in model_result] logging.info(f"[TOKEN_USAGE] 按模型统计: {len(by_model)} 个模型, data={by_model[:3]}") # 3. 按用户分组 user_sql = text(f""" SELECT user_id, COALESCE(SUM(CASE WHEN role = 'assistant' THEN answer_tokens ELSE message_tokens END), 0) as total_tokens, COUNT(*) as message_count FROM insurance_chat_records WHERE {where_clause} GROUP BY user_id ORDER BY total_tokens DESC LIMIT 50 """) user_result = db.session.execute(user_sql, bind_params).fetchall() by_user = [row._asdict() for row in user_result] logging.info(f"[TOKEN_USAGE] 按用户统计: {len(by_user)} 个用户, data={by_user[:3]}") # 4. 日趋势(最近30天) trend_sql = text(f""" SELECT DATE(created_at) as date, COALESCE(SUM(CASE WHEN role = 'assistant' THEN answer_tokens ELSE message_tokens END), 0) as total_tokens FROM insurance_chat_records WHERE {where_clause} GROUP BY DATE(created_at) ORDER BY date DESC LIMIT 30 """) trend_result = db.session.execute(trend_sql, bind_params).fetchall() daily_trend = [row._asdict() for row in trend_result] return { "code": 0, "data": { "total_tokens": total_result.total_tokens if total_result else 0, "total_input_tokens": total_result.total_input_tokens if total_result else 0, "total_output_tokens": total_result.total_output_tokens if total_result else 0, "by_model": by_model, "by_user": by_user, "daily_trend": daily_trend, }, } except Exception as e: import logging logging.warning("get_token_usage 查询失败: %s", e) return { "code": 0, "data": { "total_tokens": 0, "total_input_tokens": 0, "total_output_tokens": 0, "by_model": [], "by_user": [], "daily_trend": [], }, } def get_token_models(self) -> dict: """获取可用的模型列表。查询 insurance_chat_records 表。""" try: default_model = self._get_default_model_name() sql = text(""" SELECT DISTINCT COALESCE(NULLIF(model_id, ''), :default_model) as model FROM insurance_chat_records ORDER BY model """) result = db.session.execute(sql, {"default_model": default_model}).fetchall() models = [row[0] for row in result if row[0]] return {"code": 0, "data": models} except Exception: return {"code": 0, "data": []} def get_token_cost(self, start_date: str, end_date: str, group_by: str = "model") -> dict: """Token 成本统计(兼容旧接口)。""" params = { "start_date": start_date, "end_date": end_date, } result = self.get_token_usage(params) return result