wsb1224
caee27b0d3
海报加载失败根因：html-to-image 给 blob: 底图地址追加缓存参数，导致地址失效。现已关闭该行为。 合成失败后再次点击“重新生成”，会复用已有 AI 底图，只重试浏览器合成和上传，避免重复调用 AI。 增加底图加载、图表超时、导出失败、尺寸越界等分阶段错误提示。 修复计划书 (cid:数字) 字体乱码被误判为正常文本的问题，现在会正确转入 OCR。 增加繁体中文 OCR 运行支持。 补齐 SIUL 文件名中的确定字段，并且不会覆盖正文已识别数据。 增加“首期规划保费/償還至形成基金所需保費”等保费标签识别。 修正 IUL 年龄、保额、退保价值、缴费期、公司信息等字段映射。 LLM 返回空对象或缺字段时不再视为成功。 增加错误利益数值和年龄/保单年度错位校验。 修复依赖版本降级导致 API/Worker 无法启动的风险。 真实计划书复验结果： 产品：Manulife SIUL 3 投保年龄：48 岁 性别：女性 吸烟状态：非吸烟 币种：USD 基本保额：3,000,000 年缴保费：80,060 缴费期：5 年 利益演示：识别到 10 行 验证结果： 后端相关回归测试：91 passed 前端生产构建：通过 API、数据库、Redis、存储、数据表健康检查：全部正常 Celery Worker：已重启并连接 Redis 真实 PDF：确认进入 OCR，不再使用 (cid:...) 乱码 前端构建目录由运行容器挂载，修复已生效
460 lines
18 KiB
Python
460 lines
18 KiB
Python
"""归一化模块 — 将 LLM 提取的原始数据转为标准结构。"""
|
||
import re
|
||
import hashlib
|
||
from typing import Optional
|
||
|
||
|
||
def _safe_number(value) -> float:
|
||
"""安全数值转换。"""
|
||
if value is None:
|
||
return 0
|
||
try:
|
||
parsed = float(value)
|
||
return parsed if parsed == parsed else 0 # NaN check
|
||
except (ValueError, TypeError):
|
||
return 0
|
||
|
||
|
||
def _normalize_gender(raw) -> str:
|
||
"""统一性别值为 male/female/unknown。兼容旧数据中的 男/女。"""
|
||
if not raw:
|
||
return "unknown"
|
||
s = str(raw).strip().lower()
|
||
if s in ("male", "m", "男"):
|
||
return "male"
|
||
if s in ("female", "f", "女"):
|
||
return "female"
|
||
return "unknown"
|
||
|
||
|
||
def _normalize_smoker(raw) -> str:
|
||
"""统一吸烟状态为 yes/no/unknown。"""
|
||
if raw is None:
|
||
return "unknown"
|
||
value = str(raw).strip().lower()
|
||
if value in ("yes", "y", "true", "1", "是", "吸烟", "吸煙", "smoker"):
|
||
return "yes"
|
||
if value in ("no", "n", "false", "0", "否", "不吸烟", "不吸煙", "non-smoker", "nonsmoker"):
|
||
return "no"
|
||
return "unknown"
|
||
|
||
|
||
def _normalize_currency(raw) -> Optional[str]:
|
||
"""统一常见币种别名；未知值保持为空，避免伪造 USD。"""
|
||
if raw is None:
|
||
return None
|
||
value = str(raw).strip().upper().replace(" ", "")
|
||
aliases = {
|
||
"US$": "USD",
|
||
"$": "USD",
|
||
"USB": "USD",
|
||
"RMB": "CNY",
|
||
"人民币": "CNY",
|
||
"￥": "CNY",
|
||
"¥": "CNY",
|
||
"HK$": "HKD",
|
||
"S$": "SGD",
|
||
"€": "EUR",
|
||
"£": "GBP",
|
||
}
|
||
normalized = aliases.get(value, value)
|
||
return normalized if normalized in {"USD", "HKD", "CNY", "SGD", "EUR", "GBP"} else None
|
||
|
||
|
||
def _optional_number(value):
|
||
"""可选金额：缺失保留 None，不把未知值伪装成 0。"""
|
||
if value in (None, ""):
|
||
return None
|
||
return _safe_number(value)
|
||
|
||
|
||
def _first_value(mapping: dict, *keys):
|
||
"""从 snake_case/camelCase 字段中读取第一个非空值。"""
|
||
for key in keys:
|
||
value = mapping.get(key)
|
||
if value not in (None, ""):
|
||
return value
|
||
return None
|
||
|
||
|
||
def _normalize_pay_years(raw) -> int:
|
||
"""统一缴费年期为 int。兼容旧数据中的 '5年' 字符串。"""
|
||
if raw is None:
|
||
return 0
|
||
if isinstance(raw, (int, float)):
|
||
return int(raw)
|
||
s = str(raw).strip()
|
||
m = re.search(r'(\d+)', s)
|
||
if m:
|
||
return int(m.group(1))
|
||
if "整付" in s or "趸缴" in s or "single" in s.lower():
|
||
return 1 # 整付视为 1 年
|
||
return 0
|
||
|
||
|
||
def _extract_years(value) -> int:
|
||
"""从字符串提取年数（如 '5年' → 5）。"""
|
||
if value is None:
|
||
return 0
|
||
match = re.search(r"\d+(?:\.\d+)?", str(value))
|
||
return int(float(match.group())) if match else 0
|
||
|
||
|
||
def _sha256(file_path: Optional[str]) -> str:
|
||
"""计算文件 SHA-256。"""
|
||
if not file_path:
|
||
return ""
|
||
try:
|
||
h = hashlib.sha256()
|
||
with open(file_path, "rb") as f:
|
||
for chunk in iter(lambda: f.read(8192), b""):
|
||
h.update(chunk)
|
||
return h.hexdigest()
|
||
except Exception:
|
||
return ""
|
||
|
||
|
||
def normalize_savings_plan(raw: dict, pdf_path: str = None, parser: str = "llm-json") -> dict:
|
||
"""归一化储蓄险提取数据。"""
|
||
insured_age = _safe_number(raw.get("insured", {}).get("age"))
|
||
insured = raw.get("insured", {})
|
||
|
||
# 归一化利益演示行
|
||
benefit_rows = []
|
||
for row in raw.get("benefit_illustration", []):
|
||
if not isinstance(row, dict):
|
||
continue
|
||
policy_year = _safe_number(row.get("policy_year"))
|
||
if policy_year <= 0:
|
||
continue
|
||
age = _safe_number(row.get("age")) or (insured_age + policy_year - 1)
|
||
benefit_rows.append({
|
||
"policyYear": int(policy_year),
|
||
"age": int(age),
|
||
"totalPremiumPaid": _safe_number(row.get("total_premium_paid")),
|
||
"guaranteedCashValue": _safe_number(row.get("guaranteed_cash_value")),
|
||
"reversionaryBonus": _safe_number(row.get("reversionary_bonus")),
|
||
"terminalDividend": _safe_number(row.get("terminal_dividend")),
|
||
"totalSurrenderValue": _safe_number(row.get("total_surrender_value")),
|
||
"deathBenefit": _safe_number(row.get("death_benefit")),
|
||
"sourcePage": int(_safe_number(row.get("source_page"))) if row.get("source_page") else None,
|
||
})
|
||
benefit_rows.sort(key=lambda r: r["policyYear"])
|
||
|
||
# 归一化退保行
|
||
cumulative = 0.0
|
||
withdrawal_rows = []
|
||
for row in raw.get("withdrawal_illustration", []):
|
||
if not isinstance(row, dict):
|
||
continue
|
||
policy_year = _safe_number(row.get("policy_year"))
|
||
if policy_year <= 0:
|
||
continue
|
||
annual = _safe_number(row.get("annual_withdrawal") or row.get("withdrawal_amount"))
|
||
total_withdrawn = _safe_number(row.get("total_withdrawn") or row.get("cumulative_withdrawal"))
|
||
cumulative = total_withdrawn if total_withdrawn > 0 else cumulative + annual
|
||
age = _safe_number(row.get("age")) or (insured_age + policy_year)
|
||
withdrawal_rows.append({
|
||
"policyYear": int(policy_year),
|
||
"age": int(age),
|
||
"totalPremiumPaid": _safe_number(row.get("total_premium_paid")),
|
||
"annualWithdrawal": annual,
|
||
"cumulativeWithdrawal": cumulative,
|
||
"surrenderValueAfter": _safe_number(row.get("surrender_value_after") or row.get("remaining_surrender_value")),
|
||
"guaranteedValueAfter": _safe_number(row.get("guaranteed_value_after")),
|
||
"basicSumInsuredAfter": _safe_number(row.get("basic_sum_insured_after")),
|
||
"sourcePage": int(_safe_number(row.get("source_page"))) if row.get("source_page") else None,
|
||
})
|
||
withdrawal_rows.sort(key=lambda r: r["policyYear"])
|
||
|
||
# 保单信息
|
||
policy = raw.get("policy", {})
|
||
annual_premium = _safe_number(_first_value(policy, "annual_premium", "annualPremium"))
|
||
pay_years = _extract_years(_first_value(policy, "premium_payment_period", "payYears", "paymentPeriod"))
|
||
raw_product_name = _first_value(raw, "product_name", "productName") or _first_value(policy, "product_name", "productName") or ""
|
||
|
||
return {
|
||
"kind": "savings",
|
||
"productName": raw_product_name,
|
||
"rawProductName": raw_product_name,
|
||
"insured": {
|
||
"name": insured.get("name") or "客户",
|
||
"age": int(insured_age),
|
||
"gender": _normalize_gender(insured.get("gender")),
|
||
"smoker": _normalize_smoker(insured.get("smoker")),
|
||
},
|
||
"policy": {
|
||
"currency": _normalize_currency(policy.get("currency")),
|
||
"sumInsured": _safe_number(_first_value(policy, "sum_insured", "sumInsured")),
|
||
"annualPremium": annual_premium,
|
||
"basicPlanAnnualPremium": _optional_number(_first_value(policy, "basic_plan_annual_premium", "basicPlanAnnualPremium")),
|
||
"basicSumInsured": _optional_number(_first_value(policy, "basic_sum_insured", "basicSumInsured")),
|
||
"firstYearAmountDue": _optional_number(_first_value(policy, "first_year_amount_due", "firstYearAmountDue")),
|
||
"annualPremiumWithLevy": policy.get("total_premium_with_levy"),
|
||
"payYears": pay_years,
|
||
"contractualTotalPremium": annual_premium * pay_years,
|
||
"coveragePeriod": policy.get("coverage_period", ""),
|
||
},
|
||
"benefitRows": benefit_rows,
|
||
"withdrawalRows": withdrawal_rows,
|
||
"withdrawalProvenance": "official_extracted" if withdrawal_rows else "missing",
|
||
"source": {
|
||
"pdfHash": _sha256(pdf_path),
|
||
"pdfPath": pdf_path,
|
||
"parser": parser,
|
||
},
|
||
}
|
||
|
||
|
||
def normalize_ci_plan(raw: dict, pdf_path: str = None, parser: str = "llm-json") -> dict:
|
||
"""归一化重疾险提取数据。"""
|
||
insured = raw.get("insured", {})
|
||
policy = raw.get("policy", {})
|
||
insured_age = _safe_number(insured.get("age"))
|
||
|
||
# 保障项目
|
||
coverage_items = []
|
||
for item in raw.get("coverage_items", []):
|
||
if not isinstance(item, dict):
|
||
continue
|
||
name = item.get("name") or item.get("label") or ""
|
||
coverage_items.append({
|
||
"name": name,
|
||
"amount": _safe_number(item.get("amount")),
|
||
"description": item.get("description") or "",
|
||
"sourcePage": int(_safe_number(item.get("source_page"))) if item.get("source_page") else None,
|
||
})
|
||
|
||
# 利益演示行
|
||
benefit_rows = []
|
||
for row in raw.get("benefit_illustration", []):
|
||
if not isinstance(row, dict):
|
||
continue
|
||
policy_year = _safe_number(row.get("policy_year"))
|
||
if policy_year <= 0:
|
||
continue
|
||
death_benefit = _safe_number(row.get("death_benefit"))
|
||
if death_benefit == 0:
|
||
death_benefit = _safe_number(policy.get("sum_insured"))
|
||
benefit_rows.append({
|
||
"policyYear": int(policy_year),
|
||
"totalPremiumPaid": _safe_number(row.get("total_premium_paid")),
|
||
"deathBenefit": death_benefit,
|
||
"totalSurrenderValue": death_benefit, # 渲染器统一字段（CI 用身故赔付作为主值）
|
||
"ciBenefit": _safe_number(row.get("ci_benefit")) if row.get("ci_benefit") else None,
|
||
"sourcePage": int(_safe_number(row.get("source_page"))) if row.get("source_page") else None,
|
||
})
|
||
benefit_rows.sort(key=lambda r: r["policyYear"])
|
||
|
||
annual_premium = _safe_number(_first_value(policy, "annual_premium", "annualPremium"))
|
||
pay_years = _extract_years(_first_value(policy, "premium_payment_period", "payYears", "paymentPeriod"))
|
||
base_sum_insured = _safe_number(
|
||
raw.get("base_sum_insured")
|
||
or policy.get("basic_sum_insured")
|
||
or policy.get("sum_insured")
|
||
)
|
||
|
||
return {
|
||
"kind": "ci",
|
||
"productName": raw.get("product_name", ""),
|
||
"insured": {
|
||
"name": insured.get("name") or "客户",
|
||
"age": int(insured_age),
|
||
"gender": _normalize_gender(insured.get("gender")),
|
||
"smoker": _normalize_smoker(insured.get("smoker")),
|
||
},
|
||
"policy": {
|
||
"currency": _normalize_currency(policy.get("currency")),
|
||
"sumInsured": _safe_number(_first_value(policy, "sum_insured", "sumInsured")),
|
||
"baseSumInsured": base_sum_insured,
|
||
"basicPlanAnnualPremium": _optional_number(policy.get("basic_plan_annual_premium")),
|
||
"basicSumInsured": _optional_number(policy.get("basic_sum_insured")),
|
||
"firstYearAmountDue": _optional_number(policy.get("first_year_amount_due")),
|
||
"upgradeBenefitAmount": _safe_number(raw.get("upgrade_benefit_amount")),
|
||
"upgradeBenefitYears": _safe_number(raw.get("upgrade_benefit_years")),
|
||
"annualPremium": annual_premium,
|
||
"annualPremiumWithLevy": policy.get("total_premium_with_levy"),
|
||
"payYears": pay_years,
|
||
"totalPremium": annual_premium * pay_years,
|
||
"coveragePeriod": policy.get("coverage_period", ""),
|
||
},
|
||
"coverageSummary": {
|
||
"majorCiCount": int(_safe_number(raw.get("major_ci_count"))),
|
||
"earlyCiCount": int(_safe_number(raw.get("early_ci_count"))),
|
||
},
|
||
"coverageItems": coverage_items,
|
||
"icuBenefitRules": raw.get("icu_benefit_rules", []),
|
||
"multiClaimRules": raw.get("multi_claim", []),
|
||
"premiumWaiverRiders": raw.get("premium_waiver_riders", []),
|
||
"benefitRows": benefit_rows,
|
||
"source": {
|
||
"pdfHash": _sha256(pdf_path),
|
||
"pdfPath": pdf_path,
|
||
"parser": parser,
|
||
},
|
||
}
|
||
|
||
|
||
def normalize_iul_plan(raw: dict, pdf_path: str = None, parser: str = "llm-json") -> dict:
|
||
"""归一化 IUL 提取数据。"""
|
||
insured = raw.get("insured", {})
|
||
policy = raw.get("policy", {})
|
||
insured_age = _safe_number(insured.get("age"))
|
||
|
||
# 指数账户
|
||
index_accounts = []
|
||
for acc in raw.get("index_accounts", []):
|
||
if not isinstance(acc, dict):
|
||
continue
|
||
allocation = acc.get("allocation", 0)
|
||
if isinstance(allocation, str):
|
||
try:
|
||
allocation = float(allocation)
|
||
except ValueError:
|
||
allocation = 0
|
||
index_accounts.append({
|
||
"name": acc.get("name", ""),
|
||
"allocation": allocation,
|
||
"assumedRate": acc.get("current_assumed_rate") or acc.get("current_rate") or "",
|
||
"floorRate": acc.get("guaranteed_floor_rate") or acc.get("guaranteed_floor") or "",
|
||
"capRate": acc.get("cap_rate") or "",
|
||
"participationRate": acc.get("participation_rate") or "",
|
||
})
|
||
|
||
# 利益演示行
|
||
benefit_rows = []
|
||
for row in raw.get("benefit_illustration", []):
|
||
if not isinstance(row, dict):
|
||
continue
|
||
policy_year = _safe_number(row.get("policy_year"))
|
||
if policy_year <= 0:
|
||
continue
|
||
age = _safe_number(row.get("age")) or (insured_age + policy_year - 1)
|
||
|
||
# 回填非保证字段
|
||
non_guaranteed_account = _safe_number(row.get("non_guaranteed_account_value") or row.get("account_value"))
|
||
non_guaranteed_cash = _safe_number(row.get("non_guaranteed_cash_value") or row.get("cash_value"))
|
||
total_surrender = _safe_number(
|
||
row.get("total_surrender_value")
|
||
or row.get("totalSurrenderValue")
|
||
or row.get("non_guaranteed_cash_value")
|
||
or row.get("cash_value")
|
||
or row.get("account_value")
|
||
)
|
||
non_guaranteed_death = _safe_number(row.get("non_guaranteed_death_benefit") or row.get("death_benefit"))
|
||
|
||
benefit_rows.append({
|
||
"policyYear": int(policy_year),
|
||
"age": int(age),
|
||
"totalPremiumPaid": _safe_number(row.get("total_premium_paid")),
|
||
"guaranteedCashValue": _safe_number(row.get("guaranteed_cash_value")),
|
||
"nonGuaranteedCashValue": non_guaranteed_cash,
|
||
"totalSurrenderValue": total_surrender,
|
||
"guaranteedDeathBenefit": _safe_number(row.get("guaranteed_death_benefit")),
|
||
"nonGuaranteedDeathBenefit": non_guaranteed_death,
|
||
"sourcePage": int(_safe_number(row.get("source_page"))) if row.get("source_page") else None,
|
||
})
|
||
benefit_rows.sort(key=lambda r: r["policyYear"])
|
||
|
||
annual_premium = _safe_number(_first_value(policy, "annual_premium", "annualPremium"))
|
||
payment_period = _first_value(policy, "premium_payment_period", "payYears", "paymentPeriod") or ""
|
||
pay_years = _extract_years(payment_period)
|
||
|
||
return {
|
||
"kind": "iul",
|
||
"productName": _first_value(raw, "product_name", "productName") or "",
|
||
"insured": {
|
||
"name": insured.get("name") or "客户",
|
||
"age": int(insured_age),
|
||
"gender": _normalize_gender(insured.get("gender")),
|
||
"smoker": _normalize_smoker(insured.get("smoker")),
|
||
},
|
||
"policy": {
|
||
"currency": _normalize_currency(policy.get("currency")),
|
||
"sumInsured": _safe_number(_first_value(policy, "sum_insured", "sumInsured")),
|
||
"initialPremium": _safe_number(policy.get("initial_premium")),
|
||
"annualPremium": annual_premium,
|
||
"targetPremium": _optional_number(policy.get("target_premium")),
|
||
"minimumPremium": _optional_number(policy.get("minimum_premium")),
|
||
"basicPlanAnnualPremium": _optional_number(_first_value(policy, "basic_plan_annual_premium", "basicPlanAnnualPremium")),
|
||
"basicSumInsured": _optional_number(_first_value(policy, "basic_sum_insured", "basicSumInsured")),
|
||
"firstYearAmountDue": _optional_number(_first_value(policy, "first_year_amount_due", "firstYearAmountDue")),
|
||
"payYears": pay_years,
|
||
"totalPremium": annual_premium * pay_years,
|
||
"paymentPeriod": str(payment_period),
|
||
"coveragePeriod": policy.get("coverage_period", ""),
|
||
},
|
||
"indexAccounts": index_accounts,
|
||
"benefitRows": benefit_rows,
|
||
"source": {
|
||
"pdfHash": _sha256(pdf_path),
|
||
"pdfPath": pdf_path,
|
||
"parser": parser,
|
||
},
|
||
}
|
||
|
||
|
||
def map_savings_metrics(data: dict) -> dict:
|
||
"""提取储蓄险关键指标。"""
|
||
insured = data.get("insured", {})
|
||
policy = data.get("policy", {})
|
||
benefit_rows = data.get("benefit_illustration", [])
|
||
if not isinstance(benefit_rows, list):
|
||
benefit_rows = []
|
||
|
||
annual_premium = _safe_number(policy.get("annual_premium"))
|
||
pay_years = _extract_years(policy.get("premium_payment_period"))
|
||
total_premium = annual_premium * pay_years
|
||
|
||
# 回本年度
|
||
breakeven_year = None
|
||
for row in benefit_rows:
|
||
if not isinstance(row, dict):
|
||
continue
|
||
tp = _safe_number(row.get("total_premium_paid"))
|
||
sv = _safe_number(row.get("total_surrender_value"))
|
||
if tp > 0 and sv >= tp:
|
||
breakeven_year = int(_safe_number(row.get("policy_year")))
|
||
break
|
||
|
||
# 20年/30年倍数
|
||
multiple_20 = None
|
||
multiple_30 = None
|
||
for row in benefit_rows:
|
||
if not isinstance(row, dict):
|
||
continue
|
||
py = int(_safe_number(row.get("policy_year")))
|
||
tp = _safe_number(row.get("total_premium_paid"))
|
||
sv = _safe_number(row.get("total_surrender_value"))
|
||
if py == 20 and tp > 0:
|
||
multiple_20 = round(sv / tp, 2)
|
||
if py == 30 and tp > 0:
|
||
multiple_30 = round(sv / tp, 2)
|
||
|
||
# 退保开始年度
|
||
withdraw_start_year = None
|
||
withdraw_start_age = None
|
||
for row in data.get("withdrawal_illustration", []):
|
||
if not isinstance(row, dict):
|
||
continue
|
||
if _safe_number(row.get("annual_withdrawal")) > 0:
|
||
withdraw_start_year = int(_safe_number(row.get("policy_year")))
|
||
withdraw_start_age = int(_safe_number(row.get("age")))
|
||
break
|
||
|
||
return {
|
||
"insuredName": insured.get("name") or "",
|
||
"insuredAge": int(_safe_number(insured.get("age"))),
|
||
"insuredGender": insured.get("gender") or "",
|
||
"productName": data.get("product_name") or policy.get("product_name") or "",
|
||
"currency": policy.get("currency") or "",
|
||
"annualPremium": annual_premium,
|
||
"payYears": pay_years,
|
||
"totalPremium": total_premium,
|
||
"breakevenYear": breakeven_year,
|
||
"multiple20": multiple_20,
|
||
"multiple30": multiple_30,
|
||
"withdrawStartYear": withdraw_start_year,
|
||
"withdrawStartAge": withdraw_start_age,
|
||
}
|