wsb1224
007c820715
修正 benefit_illustration JSON 契约，兼容安全的数组返回格式，并把具体缺失字段反馈给 LLM 重试。[llm_client.py (line 184)](D:/work/code/python/coding/baodanagent/api/insurance/ppt/llm_client.py:184) LLM 利益表失败时保留有效正则结果，不再把已有数据全部丢掉。[extraction.py (line 1237)](D:/work/code/python/coding/baodanagent/api/insurance/ppt/extraction.py:1237) Female、Male、男女等误识别结果会用用户选择的产品名纠正。[extraction.py (line 521)](D:/work/code/python/coding/baodanagent/api/insurance/ppt/extraction.py:521) 解析缓存升级至 v7，旧错误缓存自动失效。 新增真正的“退保价值”页签，数据来自利益演示中的年度退保价值，不再错误依赖提领数据。[PptDataReview.vue (line 318)](D:/work/code/python/coding/baodanagent/frontend/src/pages/components/ppt/PptDataReview.vue:318) “提领方案”继续作为独立可选情景；0 行不再显示待校对，也不会影响生成 PPT。[validator.py (line 98)](D:/work/code/python/coding/baodanagent/api/insurance/ppt/validator.py:98) 提领表识别支持跨行标题及“提领/领取”等名称。[regex_extractor.py (line 619)](D:/work/code/python/coding/baodanagent/api/insurance/ppt/regex_extractor.py:619) 补充了对应回归测试。[ppt_poster_optimization_test.py](D:/work/code/python/coding/baodanagent/tests/ppt_poster_optimization_test.py) 验证结果： PPT/解析专项测试：51 passed, 1 skipped 新增问题回归测试：32 passed 前端生产构建：通过 Python 语法检查：通过 git diff --check：通过
959 lines
39 KiB
Python
959 lines
39 KiB
Python
"""纯正则 PDF 数据提取模块 — 不调用 LLM，零延迟。
|
||
|
||
从保险计划书 PDF 的原始文本中，用正则 + 启发式规则提取结构化 JSON。
|
||
提取率目标 > 80%（按保险演示表行数计算）。
|
||
|
||
用法：
|
||
from insurance.ppt.regex_extractor import extract_insurance_regex
|
||
data = extract_insurance_regex(pdf_text)
|
||
"""
|
||
import re
|
||
import logging
|
||
|
||
logger = logging.getLogger(__name__)
|
||
|
||
|
||
# ─── 金额解析 ──────────────────────────────────────────────
|
||
|
||
def _parse_money(s: str) -> float | None:
|
||
"""解析各种格式的金额字符串为 float。
|
||
|
||
支持: "1,234,567.89" "1234567" "1,234,567" "US$1,234" "HK$1234"
|
||
注意: 0 是合法值（如红利为 0 时），仅空字符串和无法解析返回 None。
|
||
"""
|
||
if not s:
|
||
return None
|
||
# 去掉货币符号和空白
|
||
s = re.sub(r'[Uu][Ss]\$|[Hh][Kk]\$|[Cc][Nn][Yy]?¥|¥|\$|\s', '', s.strip())
|
||
# 去掉逗号
|
||
s = s.replace(',', '')
|
||
try:
|
||
return float(s)
|
||
except (ValueError, TypeError):
|
||
return None
|
||
|
||
|
||
# ─── 产品信息提取 ──────────────────────────────────────────
|
||
|
||
def _extract_product_name(text: str) -> str | None:
|
||
"""提取产品名称。
|
||
|
||
支持：带标签格式、计划书标题、繁体、英文缩写。
|
||
"""
|
||
patterns = [
|
||
# 带标签
|
||
r'(?:产品名称|产品名|产品|保单名称)\s*[：:]\s*([^\n,，]+)',
|
||
r'(?:Product(?:\s+Name)?|Plan(?:\s+Name)?)\s*[：:]\s*([^\n,，]+)',
|
||
r'(?:建议书|建議書|計劃書|計劃名稱|保障計劃|保障计划)\s*[：:]\s*([^\n,，]+)',
|
||
r'(?:Proposal)\s*[：:]\s*([^\n,，]+)',
|
||
# 繁体
|
||
r'(?:產品名稱|產品名)\s*[：:]\s*([^\n,，]+)',
|
||
# "XXX 计划" 格式（无冒号标签）
|
||
r'(?:^|\n)\s*([A-Z][\w\s]*(?:Plan|Scheme|Life|Assurance|Insurance|Protection))\s*(?:\n|$)',
|
||
# 中文"XXX计划/保障"格式
|
||
r'(?:^|\n)\s*([一-鿿]{2,10}(?:计划|計劃|保障|寿险|壽險|危疾|储蓄))\s*(?:\n|$)',
|
||
]
|
||
for p in patterns:
|
||
m = re.search(p, text, re.IGNORECASE)
|
||
if m:
|
||
name = m.group(1).strip()
|
||
# 过滤掉太短或纯数字的结果
|
||
if len(name) >= 2 and not name.isdigit():
|
||
return name
|
||
return None
|
||
|
||
|
||
def _extract_currency(text: str) -> str | None:
|
||
"""提取保单货币，支持多币种（优先匹配保费行中的币种符号）。
|
||
|
||
未识别时返回 None（由用户或后续流程确认），不再默认 USD。
|
||
"""
|
||
# 按优先级搜索
|
||
patterns = [
|
||
(r'(?:保单货币|保费货币|Currency)\s*[：:]\s*(USD|USB|HKD|CNY|RMB|SGD|GBP|EUR)', 'direct'),
|
||
(r'\b(USD)\b', 'usd'),
|
||
(r'\b(HKD)\b', 'hkd'),
|
||
(r'\b(CNY|RMB)\b', 'cny'),
|
||
(r'\b(SGD)\b', 'sgd'),
|
||
(r'U\s*S\s*\$', 'us_symbol'),
|
||
(r'H\s*K\s*\$', 'hk_symbol'),
|
||
]
|
||
for pattern, tag in patterns:
|
||
m = re.search(pattern, text, re.IGNORECASE)
|
||
if m:
|
||
if tag == 'us_symbol':
|
||
return 'USD'
|
||
if tag == 'hk_symbol':
|
||
return 'HKD'
|
||
value = m.group(1).upper()
|
||
if value == "USB":
|
||
return "USD"
|
||
if value == "RMB":
|
||
return "CNY"
|
||
return value
|
||
return None
|
||
|
||
|
||
def _extract_labeled_money(text: str, labels: list[str]) -> tuple[float | None, str | None]:
|
||
"""提取明确标签后的金额，同时保留原始命中标签。"""
|
||
label_pattern = "|".join(re.escape(label) for label in labels)
|
||
match = re.search(
|
||
rf"({label_pattern})\s*[：:]\s*(?:[A-Z]{{2,3}}|[￥¥$£€]|US\$|HK\$)?\s*([\d,]+(?:\.\d+)?)",
|
||
text,
|
||
re.IGNORECASE,
|
||
)
|
||
if not match:
|
||
return None, None
|
||
return _parse_money(match.group(2)), match.group(1)
|
||
|
||
|
||
def _extract_smoker(text: str) -> str | None:
|
||
patterns = [
|
||
r"(?:吸烟状态|是否吸烟|吸煙狀態|是否吸煙|Smoking\s*Status|Smoker)\s*[：:]\s*(是|否|吸烟|不吸烟|吸煙|不吸煙|Yes|No|Y|N)",
|
||
r"(?:Non[-\s]?Smoker|Non[-\s]?Smoking)",
|
||
]
|
||
for pattern in patterns:
|
||
match = re.search(pattern, text, re.IGNORECASE)
|
||
if not match:
|
||
continue
|
||
value = match.group(1) if match.lastindex else match.group(0)
|
||
if str(value).strip().lower() in ("是", "吸烟", "吸煙", "yes", "y"):
|
||
return "yes"
|
||
return "no"
|
||
return None
|
||
|
||
|
||
def _extract_annual_premium(text: str) -> float | None:
|
||
"""提取年缴保费。"""
|
||
patterns = [
|
||
r'(?:首期规划保费|首期規劃保費|偿还至形成基金所需保费|償還至形成基金所需保費)\s*[:：]?\s*(?:[A-Z]{2,3})?\$?\s*([\d,]+(?:\.\d+)?)',
|
||
r'(?:每年(?:缴付)?保费|年缴保费|年保费|每年(?:繳付)?保費|年繳保費)\s*[：:]\s*[\$UuSsHhKk]*\s*([\d,]+(?:\.\d+)?)',
|
||
r'(?:Annual\s+Premium|每年(?:缴付)?保费|年缴保费)\s*[:：]?\s*(?:[A-Z]{2,3}\$?\s*)?([\d,]+(?:\.\d+)?)',
|
||
]
|
||
for p in patterns:
|
||
m = re.search(p, text, re.IGNORECASE)
|
||
if m:
|
||
val = _parse_money(m.group(1))
|
||
if val and val >= 100: # 排除太小的数字（可能是年期等）
|
||
return val
|
||
return None
|
||
|
||
|
||
def _extract_insured_info(text: str) -> dict:
|
||
"""提取被保人信息（年龄、性别）。
|
||
|
||
支持：简体/繁体、英文、出生日期推算。
|
||
使用"受保人上下文窗口"避免误取利益表中的年龄数字。
|
||
"""
|
||
info = {'age': None, 'gender': None, 'age_source': None}
|
||
|
||
# ── 年龄：优先匹配带标签的明确年龄 ──
|
||
age_patterns = [
|
||
# 简体
|
||
r'(?:受保人|被保人|投保时|投保人)?\s*年龄\s*[：:]\s*(\d{1,3})\s*岁?',
|
||
r'(?:受保人|被保人)\s*[：:]?\s*(?:[^\n,，]{0,10})?(\d{1,3})\s*岁',
|
||
# 繁体
|
||
r'(?:受保人|被保人|投保時|投保人)?\s*年齡\s*[：:]\s*(\d{1,3})\s*歲?',
|
||
r'(?:受保人|被保人)\s*[：:]?\s*(?:[^\n,，]{0,10})?(\d{1,3})\s*歲',
|
||
# 英文
|
||
r'(?:Issue\s+Age|Age\s+at\s+Entry|Age\s+at\s+Issue|Insured\s+Age)\s*[：:]\s*(\d{1,3})',
|
||
r'(?:Age)\s*[：:]\s*(\d{1,3})',
|
||
]
|
||
for p in age_patterns:
|
||
m = re.search(p, text, re.IGNORECASE)
|
||
if m:
|
||
age = int(m.group(1))
|
||
if 0 <= age <= 120:
|
||
info['age'] = age
|
||
info['age_source'] = 'explicit'
|
||
break
|
||
|
||
# ── 年龄：出生日期推算 ──
|
||
if info['age'] is None:
|
||
dob_patterns = [
|
||
r'(?:出生[日 destinationViewController]?[期日]|Date\s+of\s+Birth|DOB|Birth\s+Date)\s*[：:]\s*(\d{4})[/-](\d{1,2})[/-](\d{1,2})',
|
||
r'(?:出生[日 destinationViewController]?[期日]|Date\s+of\s+Birth|DOB)\s*[：:]\s*(\d{1,2})[/-](\d{1,2})[/-](\d{4})',
|
||
]
|
||
issue_date_patterns = [
|
||
r'(?:保单[日 destinationViewController]?[期日]|Issue\s+Date|Policy\s+Date|投保[日 destinationViewController]?[期日])\s*[：:]\s*(\d{4})[/-](\d{1,2})[/-](\d{1,2})',
|
||
r'(?:保单[日 destinationViewController]?[期日]|Issue\s+Date|Policy\s+Date)\s*[：:]\s*(\d{1,2})[/-](\d{1,2})[/-](\d{4})',
|
||
]
|
||
import datetime
|
||
today = datetime.date.today()
|
||
for dp in dob_patterns:
|
||
dm = re.search(dp, text, re.IGNORECASE)
|
||
if dm:
|
||
try:
|
||
groups = dm.groups()
|
||
if len(groups[0]) == 4:
|
||
dob = datetime.date(int(groups[0]), int(groups[1]), int(groups[2]))
|
||
else:
|
||
dob = datetime.date(int(groups[2]), int(groups[1]), int(groups[0]))
|
||
# 用保单日期或今天推算
|
||
issue_age = None
|
||
for ip in issue_date_patterns:
|
||
im = re.search(ip, text, re.IGNORECASE)
|
||
if im:
|
||
ig = im.groups()
|
||
if len(ig[0]) == 4:
|
||
issue_date = datetime.date(int(ig[0]), int(ig[1]), int(ig[2]))
|
||
else:
|
||
issue_date = datetime.date(int(ig[2]), int(ig[1]), int(ig[0]))
|
||
issue_age = issue_date.year - dob.year - (
|
||
(issue_date.month, issue_date.day) < (dob.month, dob.day)
|
||
)
|
||
break
|
||
if issue_age is None:
|
||
issue_age = today.year - dob.year - (
|
||
(today.month, today.day) < (dob.month, dob.day)
|
||
)
|
||
if 0 <= issue_age <= 120:
|
||
info['age'] = issue_age
|
||
info['age_source'] = 'inferred_from_dob'
|
||
except (ValueError, TypeError):
|
||
pass
|
||
break
|
||
|
||
# ── 性别 ──
|
||
gender_patterns = [
|
||
r'(?:受保人|被保人|投保人)?\s*性[别別]?\s*[：:]\s*(男|女|Male|Female|M|F)',
|
||
r'(?:Gender|Sex)\s*[：:]\s*(Male|Female|M|F)',
|
||
r'(男|女)\s*(?:性|士|仕)',
|
||
]
|
||
for p in gender_patterns:
|
||
m = re.search(p, text, re.IGNORECASE)
|
||
if m:
|
||
raw = m.group(1).strip().upper()
|
||
if raw in ('男', 'M', 'MALE'):
|
||
info['gender'] = 'male'
|
||
elif raw in ('女', 'F', 'FEMALE'):
|
||
info['gender'] = 'female'
|
||
break
|
||
|
||
return info
|
||
|
||
|
||
def _extract_premium_payment_period(text: str) -> int | str | None:
|
||
"""提取保费缴付年期。返回 int（年数）或 '整付'，不再返回 '5年' 等字符串。"""
|
||
patterns = [
|
||
r'(?:保费缴付年期|缴费年期|缴付年期|保費繳付年期|繳費年期|繳付年期)\s*[：:]\s*(\d+)\s*(?:年|years?|yrs?)?',
|
||
r'(?:Premium\s+Payment\s+(?:Period|Term|Years?))\s*[：:]\s*(\d+)\s*(?:years?|yrs?)?',
|
||
r'(?:整付|趸缴|趸繳|Single\s+Premium)',
|
||
r'(?:缴费|缴付|繳費|繳付)\s*(?:期限|年期)\s*[：:]\s*(\d+)\s*年',
|
||
]
|
||
for p in patterns:
|
||
m = re.search(p, text, re.IGNORECASE)
|
||
if m:
|
||
if '整付' in m.group(0) or '趸缴' in m.group(0) or 'Single' in m.group(0).title():
|
||
return '整付'
|
||
year_str = m.group(1)
|
||
try:
|
||
years = int(year_str)
|
||
if 1 <= years <= 100:
|
||
return years
|
||
except ValueError:
|
||
pass
|
||
return None
|
||
|
||
|
||
def _extract_coverage_period(text: str) -> str | None:
|
||
"""提取保障年期。"""
|
||
patterns = [
|
||
r'(?:保障年期|保障期|保障年期|Coverage\s+(?:Period|Term))\s*[：:]\s*([^\n,，]+)',
|
||
r'(?:终身保障|保障至终身|终身|Whole\s+Life|終身)',
|
||
r'(?:保障至|保至)\s*(\d+)\s*岁',
|
||
r'(?:保障至|保至)\s*(\d+)\s*歲',
|
||
]
|
||
for p in patterns:
|
||
m = re.search(p, text, re.IGNORECASE)
|
||
if m:
|
||
if '终身' in m.group(0) or 'Whole' in m.group(0).title():
|
||
return '终身'
|
||
return m.group(1).strip() if m.lastindex else None
|
||
return None
|
||
|
||
|
||
# ─── 演示表提取 ──────────────────────────────────────────
|
||
|
||
# 列名映射：各种 PDF 表头 → 标准字段名
|
||
_COLUMN_ALIASES = {
|
||
# 保单年度
|
||
'保单年度': 'policy_year', '保單年度': 'policy_year',
|
||
'policy year': 'policy_year', 'policyyear': 'policy_year',
|
||
'year': 'policy_year', '年度': 'policy_year',
|
||
'保单': 'policy_year', # 简称
|
||
'受保人年龄': 'age', '受保人年齡': 'age',
|
||
'年龄': 'age', '年齡': 'age',
|
||
'insured age': 'age', 'age': 'age',
|
||
# 已缴总保费
|
||
'缴付保费总额': 'total_premium_paid', '已缴保费': 'total_premium_paid',
|
||
'已付保费总额': 'total_premium_paid', '已缴付保费': 'total_premium_paid',
|
||
'total premium paid': 'total_premium_paid', 'premium paid': 'total_premium_paid',
|
||
'累积已付保费': 'total_premium_paid', '累计已缴': 'total_premium_paid',
|
||
'累计保费': 'total_premium_paid', '累計保費': 'total_premium_paid',
|
||
'已缴保费总额': 'total_premium_paid', '缴付保费': 'total_premium_paid',
|
||
'繳付保費總額': 'total_premium_paid', '已繳保費': 'total_premium_paid',
|
||
'累積已付保費': 'total_premium_paid', '已付保費總額': 'total_premium_paid',
|
||
# 保证现金价值
|
||
'保证现金价值': 'guaranteed_cash_value', '保證現金價值': 'guaranteed_cash_value',
|
||
'guaranteed cash value': 'guaranteed_cash_value', 'gcv': 'guaranteed_cash_value',
|
||
'保证退保价值': 'guaranteed_cash_value', '保证价值': 'guaranteed_cash_value',
|
||
'保证现金': 'guaranteed_cash_value',
|
||
# 归原红利
|
||
'归原红利': 'reversionary_bonus', '歸原紅利': 'reversionary_bonus',
|
||
'reversionary bonus': 'reversionary_bonus', '累积归原红利': 'reversionary_bonus',
|
||
'累计归原红利': 'reversionary_bonus', '周年红利': 'reversionary_bonus',
|
||
'复归红利': 'reversionary_bonus', '歸原紅利/復歸紅利': 'reversionary_bonus',
|
||
# 终期分红
|
||
'终期分红': 'terminal_dividend', '終期分紅': 'terminal_dividend',
|
||
'terminal dividend': 'terminal_dividend', '终期红利': 'terminal_dividend',
|
||
'特别红利': 'terminal_dividend',
|
||
# 退保发还总额
|
||
'退保发还总额': 'total_surrender_value', '退保發還總額': 'total_surrender_value',
|
||
'total surrender value': 'total_surrender_value', 'cash surrender value': 'total_surrender_value',
|
||
'退保价值': 'total_surrender_value', '退保总值': 'total_surrender_value',
|
||
'退保金额': 'total_surrender_value', '退保总额': 'total_surrender_value',
|
||
'保证利益总额': 'total_surrender_value',
|
||
# 身故赔偿
|
||
'身故赔偿': 'death_benefit', '身故賠償': 'death_benefit',
|
||
'death benefit': 'death_benefit', 'death': 'death_benefit',
|
||
'身故保障': 'death_benefit', '身故保险金': 'death_benefit',
|
||
# IUL 特有
|
||
'账户价值': 'account_value', '非保证账户价值': 'non_guaranteed_account_value',
|
||
'保证账户价值': 'guaranteed_account_value',
|
||
'非保证现金价值': 'non_guaranteed_cash_value',
|
||
'非保證現金價值': 'non_guaranteed_cash_value',
|
||
'非保证身故赔偿': 'non_guaranteed_death_benefit',
|
||
'非保證身故賠償': 'non_guaranteed_death_benefit',
|
||
'non-guaranteed account value': 'non_guaranteed_account_value',
|
||
'guaranteed account value': 'guaranteed_account_value',
|
||
'non-guaranteed cash value': 'non_guaranteed_cash_value',
|
||
'non-guaranteed death benefit': 'non_guaranteed_death_benefit',
|
||
'保险成本': 'cost_of_insurance', 'cost of insurance': 'cost_of_insurance', 'coi': 'cost_of_insurance',
|
||
# 繁体中文别名
|
||
'保單': 'policy_year', '保證現金': 'guaranteed_cash_value',
|
||
'歸原紅利': 'reversionary_bonus', '復歸紅利': 'reversionary_bonus',
|
||
'週年紅利': 'reversionary_bonus', '終期分紅': 'terminal_dividend',
|
||
'特別紅利': 'terminal_dividend', '退保發還總額': 'total_surrender_value',
|
||
'退保總值': 'total_surrender_value', '退保價值': 'total_surrender_value',
|
||
'身故賠償': 'death_benefit',
|
||
'已付保費總額': 'total_premium_paid', '已繳保費總額': 'total_premium_paid',
|
||
'累積已付保費': 'total_premium_paid',
|
||
'保證價值': 'guaranteed_cash_value', '保證現金價值': 'guaranteed_cash_value',
|
||
# ─── 提领表列名 ────────────────────────────────────────
|
||
# 年度提取金额
|
||
'提取金额': 'annual_withdrawal', '提取金額': 'annual_withdrawal',
|
||
'提款金额': 'annual_withdrawal', '提款金額': 'annual_withdrawal',
|
||
'现金提取': 'annual_withdrawal', '現金提取': 'annual_withdrawal',
|
||
'款项提取': 'annual_withdrawal', '款項提取': 'annual_withdrawal',
|
||
'每年提取': 'annual_withdrawal', '每年提款': 'annual_withdrawal',
|
||
'annual withdrawal': 'annual_withdrawal', 'withdrawal': 'annual_withdrawal',
|
||
'cash withdrawal': 'annual_withdrawal',
|
||
# 累计提取
|
||
'累计提取': 'total_withdrawn', '累計提取': 'total_withdrawn',
|
||
'累计提款': 'total_withdrawn', '累計提款': 'total_withdrawn',
|
||
'累积提取': 'total_withdrawn', '累積提取': 'total_withdrawn',
|
||
'total withdrawn': 'total_withdrawn', 'cumulative withdrawal': 'total_withdrawn',
|
||
# 提取前退保价值
|
||
'提取前退保金额': 'surrender_value_before', '提取前退保價值': 'surrender_value_before',
|
||
'提款前退保金额': 'surrender_value_before', '提款前退保價值': 'surrender_value_before',
|
||
'surrender value before': 'surrender_value_before',
|
||
# 提取后退保价值
|
||
'提取后退保金额': 'surrender_value_after', '提取後退保金額': 'surrender_value_after',
|
||
'提款后退保金额': 'surrender_value_after', '提款後退保金額': 'surrender_value_after',
|
||
'退保金额（提取后）': 'surrender_value_after',
|
||
'surrender value after': 'surrender_value_after',
|
||
}
|
||
|
||
|
||
def _normalize_column_name(name: str) -> str | None:
|
||
"""将 PDF 表头规范化为标准字段名。"""
|
||
normalized = name.strip().lower()
|
||
# 去掉括号内容和多余空格
|
||
normalized = re.sub(r'\(.*?\)|（.*?）', '', normalized).strip()
|
||
normalized = re.sub(r'\s+', ' ', normalized)
|
||
if normalized in _COLUMN_ALIASES:
|
||
return _COLUMN_ALIASES[normalized]
|
||
# 模糊匹配：检查是否包含关键词
|
||
for alias, field in _COLUMN_ALIASES.items():
|
||
if alias in normalized or normalized in alias:
|
||
return field
|
||
return None
|
||
|
||
|
||
def _detect_columns(header_line: str) -> list[tuple[int, str]]:
|
||
"""检测表头槽位和字段名。返回 [(slot_index, field_name), ...]。"""
|
||
columns = []
|
||
|
||
# 表头和数据行都按槽位解析，避免数据中的“—”被数字正则丢弃后整行左移。
|
||
parts = re.split(r'\t', header_line.strip()) if '\t' in header_line else re.split(r'\s{2,}', header_line.strip())
|
||
for slot_index, part in enumerate(parts):
|
||
field = _normalize_column_name(part)
|
||
if field:
|
||
columns.append((slot_index, field))
|
||
|
||
# 如果没有检测到列，尝试逐个关键词搜索
|
||
if len(columns) < 2:
|
||
matches = []
|
||
for alias, field in _COLUMN_ALIASES.items():
|
||
idx = header_line.lower().find(alias)
|
||
if idx >= 0:
|
||
matches.append((idx, field))
|
||
columns = []
|
||
for _, field in sorted(matches, key=lambda item: item[0]):
|
||
if not any(existing == field for _, existing in columns):
|
||
columns.append((len(columns), field))
|
||
|
||
columns.sort(key=lambda x: x[0])
|
||
return columns
|
||
|
||
|
||
def _extract_benefit_rows(text: str) -> list[dict]:
|
||
"""从 PDF 文本中提取利益演示表数据行。
|
||
|
||
策略：
|
||
1. 找到包含"保单年度"等关键词的表头行
|
||
2. 解析表头列位置
|
||
3. 提取后续数字行
|
||
"""
|
||
rows = []
|
||
lines = text.split('\n')
|
||
in_table = False
|
||
columns: list[tuple[int, str]] = []
|
||
consecutive_non_data = 0
|
||
MAX_NON_DATA_LINES = 5 # 允许的最大连续非数据行数
|
||
|
||
for line in lines:
|
||
stripped = line.strip()
|
||
if not stripped:
|
||
continue
|
||
|
||
# 检测表头行
|
||
if not in_table:
|
||
header_fields = ['保单年度', '保單年度', 'policy year', '年度', '受保人年龄', '受保人年齡']
|
||
has_header = any(f in stripped.lower() for f in header_fields)
|
||
# 需要至少有两个数值相关列名
|
||
value_cols = ['价值', '價值', '保费', '保費', '红利', '紅利', '分红', '分紅',
|
||
'赔偿', '賠償', 'surrender', 'premium', 'cash', 'bonus',
|
||
'benefit', 'value', '退保', '保证', '保證']
|
||
has_value = sum(1 for v in value_cols if v in stripped.lower()) >= 1
|
||
|
||
if has_header and has_value:
|
||
columns = _detect_columns(stripped)
|
||
if len(columns) >= 2:
|
||
in_table = True
|
||
consecutive_non_data = 0
|
||
logger.debug(f"检测到演示表头: {[(c[1], c[0]) for c in columns]}")
|
||
continue
|
||
|
||
# 跳过子表头行（如"保证 非保证 总额"等）
|
||
subheader_patterns = ['保证', '非保证', '总额', 'guaranteed', 'non-guaranteed', 'total']
|
||
if sum(1 for p in subheader_patterns if p in stripped.lower()) >= 2:
|
||
continue
|
||
|
||
# 尝试解析数据行
|
||
numbers = re.findall(r'[\d,]+(?:\.\d+)?', stripped)
|
||
if not numbers:
|
||
consecutive_non_data += 1
|
||
if consecutive_non_data >= MAX_NON_DATA_LINES:
|
||
break # 表格结束
|
||
continue
|
||
|
||
consecutive_non_data = 0
|
||
row = _parse_data_row(stripped, columns, numbers)
|
||
if row and (row.get('policy_year') is not None or row.get('age') is not None):
|
||
rows.append(row)
|
||
|
||
return rows
|
||
|
||
|
||
def _parse_data_row(
|
||
line: str, columns: list[tuple[int, str]], numbers: list[str]
|
||
) -> dict | None:
|
||
"""解析单行数据。"""
|
||
row = {}
|
||
|
||
# 提取第一个数字作为保单年度（通常是行首的小数字）
|
||
has_policy_year_column = any(field == 'policy_year' for _, field in columns)
|
||
year_match = re.match(r'^\s*(\d{1,3})\b', line) if has_policy_year_column else None
|
||
if year_match:
|
||
year_val = int(year_match.group(1))
|
||
if 0 < year_val <= 100:
|
||
row['policy_year'] = year_val
|
||
|
||
# 优先按表格槽位映射。制表符保留空槽；空格表格以两个以上空格分栏。
|
||
if columns:
|
||
cells = line.strip().split('\t') if '\t' in line else re.split(r'\s{2,}', line.strip())
|
||
max_slot = max(slot for slot, _ in columns)
|
||
if len(cells) == 1:
|
||
compact_cells = re.findall(
|
||
r'[\d,]+(?:\.\d+)?|[—–-]+|N/?A',
|
||
line,
|
||
re.IGNORECASE,
|
||
)
|
||
if len(compact_cells) > max_slot:
|
||
cells = compact_cells
|
||
if len(cells) > 1 and len(cells) > max_slot:
|
||
for slot, field in columns:
|
||
val = _parse_money(cells[slot])
|
||
if val is not None:
|
||
row[field] = val
|
||
if row.get('policy_year') is not None:
|
||
try:
|
||
row['policy_year'] = int(row['policy_year'])
|
||
except (TypeError, ValueError):
|
||
pass
|
||
else:
|
||
# 无可靠分栏信息时保留旧的顺序回退，供紧凑纯数字表使用。
|
||
value_columns = [(slot, f) for slot, f in columns if f != 'policy_year']
|
||
|
||
# 确定从哪个数字开始分配（跳过已识别的年度）
|
||
start_idx = 0
|
||
if 'policy_year' in row:
|
||
first_num = _parse_money(numbers[0]) if numbers else None
|
||
if first_num is not None and first_num == row['policy_year']:
|
||
start_idx = 1
|
||
elif first_num is not None and first_num > 100:
|
||
start_idx = 0
|
||
|
||
for i, num_str in enumerate(numbers[start_idx:], start=start_idx):
|
||
val = _parse_money(num_str)
|
||
if val is None:
|
||
continue
|
||
col_idx = i - start_idx
|
||
if col_idx < len(value_columns):
|
||
_, field = value_columns[col_idx]
|
||
row[field] = val
|
||
else:
|
||
# 没有列位置信息，按顺序猜测
|
||
all_fields = ['policy_year', 'total_premium_paid', 'guaranteed_cash_value',
|
||
'reversionary_bonus', 'terminal_dividend', 'total_surrender_value',
|
||
'death_benefit']
|
||
for i, num_str in enumerate(numbers):
|
||
if i >= len(all_fields):
|
||
break
|
||
val = _parse_money(num_str)
|
||
if val is not None:
|
||
row[all_fields[i]] = val
|
||
|
||
if row.get('age') is not None and not 0 <= row['age'] <= 150:
|
||
row['age'] = None
|
||
|
||
# 无明确年龄列时，保留带“岁”的年龄供上层结合投保年龄推算。
|
||
if 'policy_year' not in row and row.get('age') is None:
|
||
age_match = re.search(r'(\d{1,3})\s*岁', line)
|
||
if age_match:
|
||
row['age'] = int(age_match.group(1))
|
||
|
||
return row if row else None
|
||
|
||
|
||
def _fill_missing_premiums(rows: list[dict], annual_premium: float | None) -> list[dict]:
|
||
"""填充缺失的 total_premium_paid。"""
|
||
if not annual_premium or annual_premium <= 0:
|
||
return rows
|
||
for row in rows:
|
||
if row.get('total_premium_paid') is None and row.get('policy_year') is not None:
|
||
row['total_premium_paid'] = annual_premium * row['policy_year']
|
||
return rows
|
||
|
||
|
||
def _derive_policy_years(rows: list[dict], insured_age: int | float | None) -> list[dict]:
|
||
"""仅在表格只有年龄列时，使用明确投保年龄推算保单年度。"""
|
||
if insured_age is None:
|
||
return rows
|
||
try:
|
||
issue_age = int(insured_age)
|
||
except (TypeError, ValueError):
|
||
return rows
|
||
ages = []
|
||
for row in rows:
|
||
try:
|
||
if row.get('policy_year') is None and row.get('age') is not None:
|
||
ages.append(int(row['age']))
|
||
except (TypeError, ValueError):
|
||
continue
|
||
if not ages:
|
||
return rows
|
||
first_table_age = min(ages)
|
||
if first_table_age not in (issue_age, issue_age + 1):
|
||
return rows
|
||
|
||
for row in rows:
|
||
if row.get('policy_year') is not None or row.get('age') is None:
|
||
continue
|
||
try:
|
||
derived = int(row['age']) - first_table_age + 1
|
||
except (TypeError, ValueError):
|
||
continue
|
||
if 1 <= derived <= 100:
|
||
row['policy_year'] = derived
|
||
row['policy_year_source'] = 'derived_from_age'
|
||
return rows
|
||
|
||
|
||
def _ensure_required_fields(row: dict) -> dict:
|
||
"""确保每行有所有必需字段。"""
|
||
required = {
|
||
'policy_year': None,
|
||
'age': None,
|
||
'total_premium_paid': None,
|
||
'guaranteed_cash_value': None,
|
||
'reversionary_bonus': None,
|
||
'terminal_dividend': None,
|
||
'total_surrender_value': None,
|
||
'death_benefit': None,
|
||
'guaranteed_account_value': None,
|
||
'non_guaranteed_account_value': None,
|
||
'non_guaranteed_cash_value': None,
|
||
'non_guaranteed_death_benefit': None,
|
||
'cost_of_insurance': None,
|
||
}
|
||
result = dict(required)
|
||
result.update(row)
|
||
return result
|
||
|
||
|
||
# ─── 提领表提取 ──────────────────────────────────────────
|
||
|
||
_WITHDRAWAL_HEADER_KEYWORDS = [
|
||
'提取', '提款', '提领', '提領', '领取', '領取',
|
||
'现金提取', '現金提取', '款项提取', '款項提取',
|
||
'withdrawal', 'cash withdrawal',
|
||
]
|
||
|
||
_WITHDRAWAL_REQUIRED_COLS = [
|
||
'价值', '價值', 'surrender', 'cash', 'value', '退保',
|
||
]
|
||
|
||
|
||
def _extract_withdrawal_rows(text: str) -> list[dict]:
|
||
"""从 PDF 文本中提取提领/提款演示表。
|
||
|
||
注意：只提取"总额"列，不取"保证"子列。
|
||
"""
|
||
rows = []
|
||
lines = text.split('\n')
|
||
in_table = False
|
||
columns: list[tuple[int, str]] = []
|
||
consecutive_non_data = 0
|
||
pending_header_lines = 0
|
||
MAX_NON_DATA_LINES = 5
|
||
|
||
for line in lines:
|
||
stripped = line.strip()
|
||
if not stripped:
|
||
continue
|
||
|
||
if not in_table:
|
||
has_header = any(kw in stripped for kw in _WITHDRAWAL_HEADER_KEYWORDS)
|
||
has_value = any(kw in stripped.lower() for kw in _WITHDRAWAL_REQUIRED_COLS)
|
||
if has_header:
|
||
pending_header_lines = 2
|
||
if (has_header or pending_header_lines > 0) and has_value:
|
||
candidate_columns = _detect_columns(stripped)
|
||
columns = candidate_columns
|
||
if len(columns) >= 2:
|
||
in_table = True
|
||
consecutive_non_data = 0
|
||
logger.debug(f"检测到提领表头: {[(c[1], c[0]) for c in columns]}")
|
||
pending_header_lines = 0
|
||
continue
|
||
if pending_header_lines > 0 and not has_header:
|
||
pending_header_lines -= 1
|
||
continue
|
||
|
||
# 跳过子表头（"保证 非保证 总额"等）
|
||
subheader_pats = ['保证', '保證', '非保证', '非保證', '总额', '總額',
|
||
'guaranteed', 'non-guaranteed', 'total']
|
||
if sum(1 for p in subheader_pats if p in stripped.lower()) >= 2:
|
||
continue
|
||
|
||
numbers = re.findall(r'[\d,]+(?:\.\d+)?', stripped)
|
||
if not numbers:
|
||
consecutive_non_data += 1
|
||
if consecutive_non_data >= MAX_NON_DATA_LINES:
|
||
break
|
||
continue
|
||
|
||
consecutive_non_data = 0
|
||
row = _parse_withdrawal_row(stripped, columns, numbers)
|
||
if row and row.get('policy_year') is not None:
|
||
rows.append(row)
|
||
|
||
return rows
|
||
|
||
|
||
def _parse_withdrawal_row(
|
||
line: str, columns: list[tuple[int, str]], numbers: list[str]
|
||
) -> dict | None:
|
||
"""解析提领表单行。"""
|
||
row = {}
|
||
year_match = re.match(r'^\s*(\d{1,3})\b', line)
|
||
if year_match:
|
||
year_val = int(year_match.group(1))
|
||
if 0 < year_val <= 100:
|
||
row['policy_year'] = year_val
|
||
|
||
# 提领表标准字段（按优先级）
|
||
withdrawal_fields = [
|
||
'annual_withdrawal', 'total_withdrawn',
|
||
'surrender_value_before', 'surrender_value_after',
|
||
]
|
||
|
||
if columns:
|
||
value_columns = [(pos, f) for pos, f in columns if f != 'policy_year']
|
||
start_idx = 0
|
||
if 'policy_year' in row:
|
||
first_num = _parse_money(numbers[0]) if numbers else None
|
||
if first_num is not None and first_num == row['policy_year']:
|
||
start_idx = 1
|
||
|
||
for i, num_str in enumerate(numbers[start_idx:], start=start_idx):
|
||
val = _parse_money(num_str)
|
||
if val is None:
|
||
continue
|
||
col_idx = i - start_idx
|
||
if col_idx < len(value_columns):
|
||
_, field = value_columns[col_idx]
|
||
row[field] = val
|
||
else:
|
||
for i, num_str in enumerate(numbers):
|
||
if i == 0 and 'policy_year' in row:
|
||
continue
|
||
field_idx = i - 1 if 'policy_year' in row else i
|
||
if field_idx >= len(withdrawal_fields):
|
||
break
|
||
val = _parse_money(num_str)
|
||
if val is not None:
|
||
row[withdrawal_fields[field_idx]] = val
|
||
|
||
return row if row else None
|
||
|
||
|
||
def _ensure_withdrawal_fields(row: dict) -> dict:
|
||
"""确保提领行有所有必需字段。"""
|
||
required = {
|
||
'policy_year': None,
|
||
'annual_withdrawal': None,
|
||
'total_withdrawn': None,
|
||
'surrender_value_before': None,
|
||
'surrender_value_after': None,
|
||
}
|
||
result = dict(required)
|
||
result.update(row)
|
||
return result
|
||
|
||
|
||
# ─── 产品类型识别 ─────────────────────────────────────────
|
||
|
||
def _detect_product_type(text: str, benefit_rows: list[dict]) -> str:
|
||
"""从文本和提取数据推断产品类型：savings / ci / iul。"""
|
||
text_lower = text.lower()
|
||
|
||
# 文本关键词判断
|
||
ci_keywords = ['危疾', '重疾', '重大疾病', 'critical illness', 'ci plan',
|
||
'严重疾病', '危疾保障', 'dread disease']
|
||
iul_keywords = ['iul', 'index universal', 'universal life', '指数型万用',
|
||
'万用寿险', '萬用壽險', 'index account', 'index_accounts']
|
||
|
||
ci_score = sum(1 for kw in ci_keywords if kw in text_lower)
|
||
iul_score = sum(1 for kw in iul_keywords if kw in text_lower)
|
||
|
||
# 数据特征判断
|
||
for row in benefit_rows:
|
||
if row.get('account_value') or row.get('non_guaranteed_account_value'):
|
||
iul_score += 3
|
||
if row.get('death_benefit') and not row.get('guaranteed_cash_value'):
|
||
ci_score += 1
|
||
|
||
# policy 字段判断
|
||
if re.search(r'(?:保额|投保额|sum\s*insured|保額)\s*[：:]\s*[\d,]+', text_lower):
|
||
ci_score += 1
|
||
|
||
if iul_score > ci_score and iul_score > 0:
|
||
return 'iul'
|
||
if ci_score > 0:
|
||
return 'ci'
|
||
return 'savings'
|
||
|
||
|
||
# ─── 主提取函数 ──────────────────────────────────────────
|
||
|
||
def extract_insurance_regex(pdf_text: str) -> dict:
|
||
"""从 PDF 原始文本中用纯正则提取结构化保险数据。
|
||
|
||
Args:
|
||
pdf_text: PDF 提取的原始文本
|
||
|
||
Returns:
|
||
符合 extraction schema 的 dict，字段缺失填 null
|
||
"""
|
||
product_name = _extract_product_name(pdf_text) or 'unknown'
|
||
currency = _extract_currency(pdf_text)
|
||
annual_premium = _extract_annual_premium(pdf_text)
|
||
insured_info = _extract_insured_info(pdf_text)
|
||
smoker = _extract_smoker(pdf_text)
|
||
premium_period = _extract_premium_payment_period(pdf_text)
|
||
coverage_period = _extract_coverage_period(pdf_text)
|
||
|
||
# 提取利益演示表
|
||
benefit_rows = _extract_benefit_rows(pdf_text)
|
||
benefit_rows = _derive_policy_years(benefit_rows, insured_info.get('age'))
|
||
|
||
# 提取提领表
|
||
withdrawal_rows = _extract_withdrawal_rows(pdf_text)
|
||
|
||
# 产品类型由上层 infer_plan_type() 推断（基于实际提取数据），此处不重复判断
|
||
|
||
# 填充缺失保费
|
||
benefit_rows = _fill_missing_premiums(benefit_rows, annual_premium)
|
||
|
||
# 确保字段完整
|
||
benefit_rows = [_ensure_required_fields(r) for r in benefit_rows]
|
||
withdrawal_rows = [_ensure_withdrawal_fields(r) for r in withdrawal_rows]
|
||
|
||
# 按年度排序并去重
|
||
seen_years = set()
|
||
deduped = []
|
||
for row in sorted(benefit_rows, key=lambda r: r.get('policy_year') or 0):
|
||
year = row.get('policy_year')
|
||
if year is not None and year not in seen_years:
|
||
seen_years.add(year)
|
||
deduped.append(row)
|
||
benefit_rows = deduped
|
||
|
||
# 提领表去重
|
||
seen_wd_years = set()
|
||
deduped_wd = []
|
||
for row in sorted(withdrawal_rows, key=lambda r: r.get('policy_year') or 0):
|
||
year = row.get('policy_year')
|
||
if year is not None and year not in seen_wd_years:
|
||
seen_wd_years.add(year)
|
||
deduped_wd.append(row)
|
||
withdrawal_rows = deduped_wd
|
||
|
||
# 提取保额（如果有）
|
||
sum_insured = None
|
||
si_match = re.search(
|
||
r'(?:保额|投保额|保額|sum\s*insured)\s*[：:]\s*[\$UuSsHhKk]*\s*([\d,]+(?:\.\d+)?)',
|
||
pdf_text, re.IGNORECASE,
|
||
)
|
||
if si_match:
|
||
sum_insured = _parse_money(si_match.group(1))
|
||
|
||
basic_plan_annual_premium, basic_premium_label = _extract_labeled_money(
|
||
pdf_text,
|
||
["基本计划年保费", "基本計劃年保費", "Basic Plan Annual Premium"],
|
||
)
|
||
basic_sum_insured, basic_sum_label = _extract_labeled_money(
|
||
pdf_text,
|
||
["基本计划名义金额", "基本計劃名義金額", "Basic Sum Assured", "Basic Notional Amount"],
|
||
)
|
||
first_year_amount_due, first_year_label = _extract_labeled_money(
|
||
pdf_text,
|
||
["首年应缴金额", "首年應繳金額", "First Year Amount Due"],
|
||
)
|
||
|
||
logger.info(
|
||
f"[RegexExtractor] 提取完成: product={product_name}, "
|
||
f"currency={currency}, premium={annual_premium}, "
|
||
f"age={insured_info.get('age')}, benefit_rows={len(benefit_rows)}, "
|
||
f"withdrawal_rows={len(withdrawal_rows)}"
|
||
)
|
||
|
||
data = {
|
||
'product_name': product_name,
|
||
'product_type': None, # 由上层 infer_plan_type() 基于实际数据推断
|
||
'insured': {
|
||
'name': None,
|
||
'age': insured_info.get('age'),
|
||
'gender': insured_info.get('gender'),
|
||
'relation': None,
|
||
'smoker': smoker,
|
||
},
|
||
'policy': {
|
||
'product_name': product_name,
|
||
'currency': currency,
|
||
'sum_insured': sum_insured,
|
||
'basic_sum_insured': basic_sum_insured,
|
||
'basic_sum_insured_source_label': basic_sum_label,
|
||
'annual_premium': annual_premium,
|
||
'basic_plan_annual_premium': basic_plan_annual_premium,
|
||
'basic_plan_annual_premium_source_label': basic_premium_label,
|
||
'first_year_amount_due': first_year_amount_due,
|
||
'first_year_amount_due_source_label': first_year_label,
|
||
'premium_payment_period': premium_period,
|
||
'coverage_period': coverage_period,
|
||
},
|
||
'benefit_illustration': benefit_rows,
|
||
'withdrawal_illustration': withdrawal_rows,
|
||
'sales_insights': None,
|
||
}
|
||
|
||
# CI 产品：提取保障项目（如果文本包含 CI 关键词）
|
||
coverage_items = _extract_coverage_items(pdf_text)
|
||
if coverage_items:
|
||
data['coverage_items'] = coverage_items
|
||
|
||
# IUL 产品：提取指数账户（如果文本包含 IUL 关键词）
|
||
index_accounts = _extract_index_accounts(pdf_text)
|
||
if index_accounts:
|
||
data['index_accounts'] = index_accounts
|
||
|
||
return data
|
||
|
||
|
||
def _extract_coverage_items(text: str) -> list[dict]:
|
||
"""提取危疾保险的保障项目。"""
|
||
items = []
|
||
# 匹配 "保障项目 赔付金额" 格式的表
|
||
lines = text.split('\n')
|
||
in_section = False
|
||
for line in lines:
|
||
stripped = line.strip()
|
||
if not stripped:
|
||
continue
|
||
if '保障项目' in stripped or '保障範圍' in stripped or 'coverage' in stripped.lower():
|
||
in_section = True
|
||
continue
|
||
if in_section:
|
||
# 尝试匹配 "项目名 金额" 格式
|
||
m = re.match(r'^(.{2,20})\s+[\$UuSsHhKk]*\s*([\d,]+(?:\.\d+)?)', stripped)
|
||
if m:
|
||
label = m.group(1).strip()
|
||
amount = _parse_money(m.group(2))
|
||
if amount and label:
|
||
items.append({'label': label, 'amount': amount, 'percentage': None, 'description': None})
|
||
elif not re.search(r'\d', stripped):
|
||
break # 非数字行，可能到了下一节
|
||
return items
|
||
|
||
|
||
def _extract_index_accounts(text: str) -> list[dict]:
|
||
"""提取 IUL 指数账户信息。"""
|
||
accounts = []
|
||
# 匹配 "账户名 配置比例 利率" 格式
|
||
patterns = [
|
||
r'(S&P\s*500|Hang\s*Seng|恒生指数|Global\s*index|指数\d?)\s+([\d.]+)%?\s+([\d.]+)%?\s+([\d.]+)%?',
|
||
r'(S&P\s*500|Hang\s*Seng|恒生指数|Global\s*index)\s+([\d.]+)%',
|
||
]
|
||
for p in patterns:
|
||
for m in re.finditer(p, text, re.IGNORECASE):
|
||
account = {
|
||
'name': m.group(1).strip(),
|
||
'allocation': float(m.group(2)),
|
||
'current_rate': float(m.group(3)) if m.lastindex >= 3 else None,
|
||
'guaranteed_floor': float(m.group(4)) if m.lastindex >= 4 else None,
|
||
}
|
||
accounts.append(account)
|
||
return accounts
|
||
|
||
|
||
def count_benefit_rows(data: dict) -> int:
|
||
"""统计提取到的利益演示行数。"""
|
||
rows = data.get('benefit_illustration', [])
|
||
if not isinstance(rows, list):
|
||
return 0
|
||
return len(rows)
|