748 lines
29 KiB
Python
748 lines
29 KiB
Python
|
"""纯正则 PDF 数据提取模块 — 不调用 LLM，零延迟。
|
|||
|
|
|
|||
|
|
从保险计划书 PDF 的原始文本中，用正则 + 启发式规则提取结构化 JSON。
|
|||
|
|
提取率目标 > 80%（按保险演示表行数计算）。
|
|||
|
|
|
|||
|
|
用法：
|
|||
|
|
from insurance.ppt.regex_extractor import extract_insurance_regex
|
|||
|
|
data = extract_insurance_regex(pdf_text)
|
|||
|
|
"""
|
|||
|
|
import re
|
|||
|
|
import logging
|
|||
|
|
|
|||
|
|
logger = logging.getLogger(__name__)
|
|||
|
|
|
|||
|
|
|
|||
|
|
# ─── 金额解析 ──────────────────────────────────────────────
|
|||
|
|
|
|||
|
|
def _parse_money(s: str) -> float | None:
|
|||
|
|
"""解析各种格式的金额字符串为 float。
|
|||
|
|
|
|||
|
|
支持: "1,234,567.89" "1234567" "1,234,567" "US$1,234" "HK$1234"
|
|||
|
|
"""
|
|||
|
|
if not s:
|
|||
|
|
return None
|
|||
|
|
# 去掉货币符号和空白
|
|||
|
|
s = re.sub(r'[Uu][Ss]\$|[Hh][Kk]\$|[Cc][Nn][Yy]?¥|¥|\$|\s', '', s.strip())
|
|||
|
|
# 去掉逗号
|
|||
|
|
s = s.replace(',', '')
|
|||
|
|
try:
|
|||
|
|
v = float(s)
|
|||
|
|
return v if v > 0 else None
|
|||
|
|
except (ValueError, TypeError):
|
|||
|
|
return None
|
|||
|
|
|
|||
|
|
|
|||
|
|
# ─── 产品信息提取 ──────────────────────────────────────────
|
|||
|
|
|
|||
|
|
def _extract_product_name(text: str) -> str | None:
|
|||
|
|
"""提取产品名称。"""
|
|||
|
|
patterns = [
|
|||
|
|
r'(?:产品名称|产品名|产品|保单名称)\s*[：:]\s*([^\n,，]+)',
|
|||
|
|
r'(?:Product(?:\s+Name)?|Plan(?:\s+Name)?)\s*[：:]\s*([^\n,，]+)',
|
|||
|
|
r'(?:建议书|建議書|計劃書|計劃名稱|保障計劃|保障计划)\s*[：:]\s*([^\n,，]+)',
|
|||
|
|
r'(?:Proposal)\s*[：:]\s*([^\n,，]+)',
|
|||
|
|
]
|
|||
|
|
for p in patterns:
|
|||
|
|
m = re.search(p, text, re.IGNORECASE)
|
|||
|
|
if m:
|
|||
|
|
name = m.group(1).strip()
|
|||
|
|
# 过滤掉太短或纯数字的结果
|
|||
|
|
if len(name) >= 2 and not name.isdigit():
|
|||
|
|
return name
|
|||
|
|
return None
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _extract_currency(text: str) -> str:
|
|||
|
|
"""提取保单货币，支持多币种（优先匹配保费行中的币种符号）。"""
|
|||
|
|
# 按优先级搜索
|
|||
|
|
patterns = [
|
|||
|
|
(r'(?:保单货币|保费货币|Currency)\s*[：:]\s*(USD|HKD|CNY|RMB|SGD|GBP|EUR)', 'direct'),
|
|||
|
|
(r'\b(USD)\b', 'usd'),
|
|||
|
|
(r'\b(HKD)\b', 'hkd'),
|
|||
|
|
(r'\b(CNY|RMB)\b', 'cny'),
|
|||
|
|
(r'\b(SGD)\b', 'sgd'),
|
|||
|
|
(r'U\s*S\s*\$', 'us_symbol'),
|
|||
|
|
(r'H\s*K\s*\$', 'hk_symbol'),
|
|||
|
|
]
|
|||
|
|
for pattern, tag in patterns:
|
|||
|
|
m = re.search(pattern, text, re.IGNORECASE)
|
|||
|
|
if m:
|
|||
|
|
if tag == 'us_symbol':
|
|||
|
|
return 'USD'
|
|||
|
|
if tag == 'hk_symbol':
|
|||
|
|
return 'HKD'
|
|||
|
|
return m.group(1).upper()
|
|||
|
|
return 'USD' # 默认
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _extract_annual_premium(text: str) -> float | None:
|
|||
|
|
"""提取年缴保费。"""
|
|||
|
|
patterns = [
|
|||
|
|
r'(?:每年(?:缴付)?保费|年缴保费|Annual\s+Premium|年保费)\s*[：:]\s*[\$UuSsHhKk]*\s*([\d,]+(?:\.\d+)?)',
|
|||
|
|
r'(?:每年(?:缴付)?保费|年缴保费|Annual\s+Premium)\s*[:：]?\s*(?:[A-Z]{2,3}\$?\s*)?([\d,]+(?:\.\d+)?)',
|
|||
|
|
]
|
|||
|
|
for p in patterns:
|
|||
|
|
m = re.search(p, text, re.IGNORECASE)
|
|||
|
|
if m:
|
|||
|
|
val = _parse_money(m.group(1))
|
|||
|
|
if val and val >= 100: # 排除太小的数字（可能是年期等）
|
|||
|
|
return val
|
|||
|
|
return None
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _extract_insured_info(text: str) -> dict:
|
|||
|
|
"""提取被保人信息（年龄、性别）。"""
|
|||
|
|
info = {'age': None, 'gender': None}
|
|||
|
|
|
|||
|
|
# 年龄
|
|||
|
|
age_patterns = [
|
|||
|
|
r'(?:受保人|被保人|投保时)?年龄\s*[：:]\s*(\d{1,3})\s*岁?',
|
|||
|
|
r'(?:Age)\s*[：:]\s*(\d{1,3})',
|
|||
|
|
r'(?:受保人|被保人)\s*[：:]?\s*(?:[^\n,，]{0,10})?(\d{1,3})\s*岁',
|
|||
|
|
]
|
|||
|
|
for p in age_patterns:
|
|||
|
|
m = re.search(p, text, re.IGNORECASE)
|
|||
|
|
if m:
|
|||
|
|
age = int(m.group(1))
|
|||
|
|
if 0 <= age <= 120:
|
|||
|
|
info['age'] = age
|
|||
|
|
break
|
|||
|
|
|
|||
|
|
# 性别
|
|||
|
|
gender_patterns = [
|
|||
|
|
r'(?:受保人|被保人|投保人)?性[别別]?\s*[：:]\s*(男|女|Male|Female|M|F)',
|
|||
|
|
r'(?:Gender|Sex)\s*[：:]\s*(Male|Female|M|F)',
|
|||
|
|
r'(男|女)\s*(?:性|士|仕)',
|
|||
|
|
]
|
|||
|
|
for p in gender_patterns:
|
|||
|
|
m = re.search(p, text, re.IGNORECASE)
|
|||
|
|
if m:
|
|||
|
|
raw = m.group(1).strip().upper()
|
|||
|
|
if raw in ('男', 'M', 'MALE'):
|
|||
|
|
info['gender'] = '男'
|
|||
|
|
elif raw in ('女', 'F', 'FEMALE'):
|
|||
|
|
info['gender'] = '女'
|
|||
|
|
break
|
|||
|
|
|
|||
|
|
return info
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _extract_premium_payment_period(text: str) -> str | None:
|
|||
|
|
"""提取保费缴付年期。"""
|
|||
|
|
patterns = [
|
|||
|
|
r'(?:保费缴付年期|缴费年期|缴付年期|Premium\s+Payment\s+(?:Period|Term|Years?))\s*[：:]\s*(\d+)\s*(?:年|years?|yrs?)?',
|
|||
|
|
r'(?:整付|趸缴|Single\s+Premium)',
|
|||
|
|
r'(?:缴费|缴付)\s*(?:期限|年期)\s*[：:]\s*(\d+)\s*年',
|
|||
|
|
]
|
|||
|
|
for p in patterns:
|
|||
|
|
m = re.search(p, text, re.IGNORECASE)
|
|||
|
|
if m:
|
|||
|
|
if '整付' in m.group(0) or '趸缴' in m.group(0) or 'Single' in m.group(0).title():
|
|||
|
|
return '整付'
|
|||
|
|
year_str = m.group(1)
|
|||
|
|
try:
|
|||
|
|
years = int(year_str)
|
|||
|
|
if 1 <= years <= 100:
|
|||
|
|
return f'{years}年'
|
|||
|
|
except ValueError:
|
|||
|
|
pass
|
|||
|
|
return None
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _extract_coverage_period(text: str) -> str | None:
|
|||
|
|
"""提取保障年期。"""
|
|||
|
|
patterns = [
|
|||
|
|
r'(?:保障年期|保障期|Coverage\s+(?:Period|Term))\s*[：:]\s*([^\n,，]+)',
|
|||
|
|
r'(?:终身保障|保障至终身|终身|Whole\s+Life)',
|
|||
|
|
r'(?:保障至|保至)\s*(\d+)\s*岁',
|
|||
|
|
]
|
|||
|
|
for p in patterns:
|
|||
|
|
m = re.search(p, text, re.IGNORECASE)
|
|||
|
|
if m:
|
|||
|
|
if '终身' in m.group(0) or 'Whole' in m.group(0).title():
|
|||
|
|
return '终身'
|
|||
|
|
return m.group(1).strip() if m.lastindex else None
|
|||
|
|
return None
|
|||
|
|
|
|||
|
|
|
|||
|
|
# ─── 演示表提取 ──────────────────────────────────────────
|
|||
|
|
|
|||
|
|
# 列名映射：各种 PDF 表头 → 标准字段名
|
|||
|
|
_COLUMN_ALIASES = {
|
|||
|
|
# 保单年度
|
|||
|
|
'保单年度': 'policy_year', '保單年度': 'policy_year',
|
|||
|
|
'policy year': 'policy_year', 'policyyear': 'policy_year',
|
|||
|
|
'year': 'policy_year', '年度': 'policy_year',
|
|||
|
|
'保单': 'policy_year', # 简称
|
|||
|
|
'受保人年龄': 'policy_year', # 有些表用年龄代替年度
|
|||
|
|
# 已缴总保费
|
|||
|
|
'缴付保费总额': 'total_premium_paid', '已缴保费': 'total_premium_paid',
|
|||
|
|
'已付保费总额': 'total_premium_paid', '已缴付保费': 'total_premium_paid',
|
|||
|
|
'total premium paid': 'total_premium_paid', 'premium paid': 'total_premium_paid',
|
|||
|
|
'累积已付保费': 'total_premium_paid', '累计已缴': 'total_premium_paid',
|
|||
|
|
'已缴保费总额': 'total_premium_paid', '缴付保费': 'total_premium_paid',
|
|||
|
|
# 保证现金价值
|
|||
|
|
'保证现金价值': 'guaranteed_cash_value', '保證現金價值': 'guaranteed_cash_value',
|
|||
|
|
'guaranteed cash value': 'guaranteed_cash_value', 'gcv': 'guaranteed_cash_value',
|
|||
|
|
'保证退保价值': 'guaranteed_cash_value', '保证价值': 'guaranteed_cash_value',
|
|||
|
|
'保证现金': 'guaranteed_cash_value',
|
|||
|
|
# 归原红利
|
|||
|
|
'归原红利': 'reversionary_bonus', '歸原紅利': 'reversionary_bonus',
|
|||
|
|
'reversionary bonus': 'reversionary_bonus', '累积归原红利': 'reversionary_bonus',
|
|||
|
|
'累计归原红利': 'reversionary_bonus', '周年红利': 'reversionary_bonus',
|
|||
|
|
'复归红利': 'reversionary_bonus', '歸原紅利/復歸紅利': 'reversionary_bonus',
|
|||
|
|
# 终期分红
|
|||
|
|
'终期分红': 'terminal_dividend', '終期分紅': 'terminal_dividend',
|
|||
|
|
'terminal dividend': 'terminal_dividend', '终期红利': 'terminal_dividend',
|
|||
|
|
'特别红利': 'terminal_dividend',
|
|||
|
|
# 退保发还总额
|
|||
|
|
'退保发还总额': 'total_surrender_value', '退保發還總額': 'total_surrender_value',
|
|||
|
|
'total surrender value': 'total_surrender_value', 'cash surrender value': 'total_surrender_value',
|
|||
|
|
'退保价值': 'total_surrender_value', '退保总值': 'total_surrender_value',
|
|||
|
|
'退保金额': 'total_surrender_value', '退保总额': 'total_surrender_value',
|
|||
|
|
'保证利益总额': 'total_surrender_value',
|
|||
|
|
# 身故赔偿
|
|||
|
|
'身故赔偿': 'death_benefit', '身故賠償': 'death_benefit',
|
|||
|
|
'death benefit': 'death_benefit', 'death': 'death_benefit',
|
|||
|
|
'身故保障': 'death_benefit', '身故保险金': 'death_benefit',
|
|||
|
|
# IUL 特有
|
|||
|
|
'账户价值': 'account_value', '非保证账户价值': 'non_guaranteed_account_value',
|
|||
|
|
'保证账户价值': 'guaranteed_account_value',
|
|||
|
|
'non-guaranteed account value': 'non_guaranteed_account_value',
|
|||
|
|
'guaranteed account value': 'guaranteed_account_value',
|
|||
|
|
'保险成本': 'cost_of_insurance', 'cost of insurance': 'cost_of_insurance', 'coi': 'cost_of_insurance',
|
|||
|
|
# 繁体中文别名
|
|||
|
|
'保單': 'policy_year', '保證現金': 'guaranteed_cash_value',
|
|||
|
|
'歸原紅利': 'reversionary_bonus', '復歸紅利': 'reversionary_bonus',
|
|||
|
|
'週年紅利': 'reversionary_bonus', '終期分紅': 'terminal_dividend',
|
|||
|
|
'特別紅利': 'terminal_dividend', '退保發還總額': 'total_surrender_value',
|
|||
|
|
'退保總值': 'total_surrender_value', '退保價值': 'total_surrender_value',
|
|||
|
|
'身故賠償': 'death_benefit',
|
|||
|
|
'已付保費總額': 'total_premium_paid', '已繳保費總額': 'total_premium_paid',
|
|||
|
|
'累積已付保費': 'total_premium_paid',
|
|||
|
|
'保證價值': 'guaranteed_cash_value', '保證現金價值': 'guaranteed_cash_value',
|
|||
|
|
# ─── 提领表列名 ────────────────────────────────────────
|
|||
|
|
# 年度提取金额
|
|||
|
|
'提取金额': 'annual_withdrawal', '提取金額': 'annual_withdrawal',
|
|||
|
|
'提款金额': 'annual_withdrawal', '提款金額': 'annual_withdrawal',
|
|||
|
|
'现金提取': 'annual_withdrawal', '現金提取': 'annual_withdrawal',
|
|||
|
|
'款项提取': 'annual_withdrawal', '款項提取': 'annual_withdrawal',
|
|||
|
|
'每年提取': 'annual_withdrawal', '每年提款': 'annual_withdrawal',
|
|||
|
|
'annual withdrawal': 'annual_withdrawal', 'withdrawal': 'annual_withdrawal',
|
|||
|
|
'cash withdrawal': 'annual_withdrawal',
|
|||
|
|
# 累计提取
|
|||
|
|
'累计提取': 'total_withdrawn', '累計提取': 'total_withdrawn',
|
|||
|
|
'累计提款': 'total_withdrawn', '累計提款': 'total_withdrawn',
|
|||
|
|
'累积提取': 'total_withdrawn', '累積提取': 'total_withdrawn',
|
|||
|
|
'total withdrawn': 'total_withdrawn', 'cumulative withdrawal': 'total_withdrawn',
|
|||
|
|
# 提取前退保价值
|
|||
|
|
'提取前退保金额': 'surrender_value_before', '提取前退保價值': 'surrender_value_before',
|
|||
|
|
'提款前退保金额': 'surrender_value_before', '提款前退保價值': 'surrender_value_before',
|
|||
|
|
'surrender value before': 'surrender_value_before',
|
|||
|
|
# 提取后退保价值
|
|||
|
|
'提取后退保金额': 'surrender_value_after', '提取後退保金額': 'surrender_value_after',
|
|||
|
|
'提款后退保金额': 'surrender_value_after', '提款後退保金額': 'surrender_value_after',
|
|||
|
|
'退保金额（提取后）': 'surrender_value_after',
|
|||
|
|
'surrender value after': 'surrender_value_after',
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _normalize_column_name(name: str) -> str | None:
|
|||
|
|
"""将 PDF 表头规范化为标准字段名。"""
|
|||
|
|
normalized = name.strip().lower()
|
|||
|
|
# 去掉括号内容和多余空格
|
|||
|
|
normalized = re.sub(r'\(.*?\)|（.*?）', '', normalized).strip()
|
|||
|
|
normalized = re.sub(r'\s+', ' ', normalized)
|
|||
|
|
if normalized in _COLUMN_ALIASES:
|
|||
|
|
return _COLUMN_ALIASES[normalized]
|
|||
|
|
# 模糊匹配：检查是否包含关键词
|
|||
|
|
for alias, field in _COLUMN_ALIASES.items():
|
|||
|
|
if alias in normalized or normalized in alias:
|
|||
|
|
return field
|
|||
|
|
return None
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _detect_columns(header_line: str) -> list[tuple[int, str]]:
|
|||
|
|
"""检测表头行的列位置和字段名。返回 [(start_pos, field_name), ...]。"""
|
|||
|
|
columns = []
|
|||
|
|
|
|||
|
|
# 尝试按多个空格或制表符分割
|
|||
|
|
parts = re.split(r'\s{2,}|\t', header_line)
|
|||
|
|
pos = 0
|
|||
|
|
for part in parts:
|
|||
|
|
field = _normalize_column_name(part)
|
|||
|
|
if field:
|
|||
|
|
columns.append((pos, field))
|
|||
|
|
pos += len(part) + 2 # 近似位置
|
|||
|
|
|
|||
|
|
# 如果没有检测到列，尝试逐个关键词搜索
|
|||
|
|
if len(columns) < 2:
|
|||
|
|
for alias, field in _COLUMN_ALIASES.items():
|
|||
|
|
idx = header_line.lower().find(alias)
|
|||
|
|
if idx >= 0:
|
|||
|
|
# 避免重复
|
|||
|
|
if not any(f == field for _, f in columns):
|
|||
|
|
columns.append((idx, field))
|
|||
|
|
|
|||
|
|
columns.sort(key=lambda x: x[0])
|
|||
|
|
return columns
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _extract_benefit_rows(text: str) -> list[dict]:
|
|||
|
|
"""从 PDF 文本中提取利益演示表数据行。
|
|||
|
|
|
|||
|
|
策略：
|
|||
|
|
1. 找到包含"保单年度"等关键词的表头行
|
|||
|
|
2. 解析表头列位置
|
|||
|
|
3. 提取后续数字行
|
|||
|
|
"""
|
|||
|
|
rows = []
|
|||
|
|
lines = text.split('\n')
|
|||
|
|
in_table = False
|
|||
|
|
columns: list[tuple[int, str]] = []
|
|||
|
|
consecutive_non_data = 0
|
|||
|
|
MAX_NON_DATA_LINES = 5 # 允许的最大连续非数据行数
|
|||
|
|
|
|||
|
|
for line in lines:
|
|||
|
|
stripped = line.strip()
|
|||
|
|
if not stripped:
|
|||
|
|
continue
|
|||
|
|
|
|||
|
|
# 检测表头行
|
|||
|
|
if not in_table:
|
|||
|
|
header_fields = ['保单年度', '保單年度', 'policy year', '年度']
|
|||
|
|
has_header = any(f in stripped.lower() for f in header_fields)
|
|||
|
|
# 需要至少有两个数值相关列名
|
|||
|
|
value_cols = ['价值', '保费', '红利', '分红', '赔偿', 'surrender', 'premium',
|
|||
|
|
'cash', 'bonus', 'benefit', 'value', '退保', '保证']
|
|||
|
|
has_value = sum(1 for v in value_cols if v in stripped.lower()) >= 1
|
|||
|
|
|
|||
|
|
if has_header and has_value:
|
|||
|
|
columns = _detect_columns(stripped)
|
|||
|
|
if len(columns) >= 2:
|
|||
|
|
in_table = True
|
|||
|
|
consecutive_non_data = 0
|
|||
|
|
logger.debug(f"检测到演示表头: {[(c[1], c[0]) for c in columns]}")
|
|||
|
|
continue
|
|||
|
|
|
|||
|
|
# 跳过子表头行（如"保证 非保证 总额"等）
|
|||
|
|
subheader_patterns = ['保证', '非保证', '总额', 'guaranteed', 'non-guaranteed', 'total']
|
|||
|
|
if sum(1 for p in subheader_patterns if p in stripped.lower()) >= 2:
|
|||
|
|
continue
|
|||
|
|
|
|||
|
|
# 尝试解析数据行
|
|||
|
|
numbers = re.findall(r'[\d,]+(?:\.\d+)?', stripped)
|
|||
|
|
if not numbers:
|
|||
|
|
consecutive_non_data += 1
|
|||
|
|
if consecutive_non_data >= MAX_NON_DATA_LINES:
|
|||
|
|
break # 表格结束
|
|||
|
|
continue
|
|||
|
|
|
|||
|
|
consecutive_non_data = 0
|
|||
|
|
row = _parse_data_row(stripped, columns, numbers)
|
|||
|
|
if row and row.get('policy_year') is not None:
|
|||
|
|
rows.append(row)
|
|||
|
|
|
|||
|
|
return rows
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _parse_data_row(
|
|||
|
|
line: str, columns: list[tuple[int, str]], numbers: list[str]
|
|||
|
|
) -> dict | None:
|
|||
|
|
"""解析单行数据。"""
|
|||
|
|
row = {}
|
|||
|
|
|
|||
|
|
# 提取第一个数字作为保单年度（通常是行首的小数字）
|
|||
|
|
year_match = re.match(r'^\s*(\d{1,3})\b', line)
|
|||
|
|
if year_match:
|
|||
|
|
year_val = int(year_match.group(1))
|
|||
|
|
if 0 < year_val <= 100:
|
|||
|
|
row['policy_year'] = year_val
|
|||
|
|
|
|||
|
|
# 用列位置映射数值
|
|||
|
|
if columns:
|
|||
|
|
# 获取非 policy_year 的列（按原始顺序）
|
|||
|
|
value_columns = [(pos, f) for pos, f in columns if f != 'policy_year']
|
|||
|
|
|
|||
|
|
# 确定从哪个数字开始分配（跳过已识别的年度）
|
|||
|
|
start_idx = 0
|
|||
|
|
if 'policy_year' in row:
|
|||
|
|
# 验证第一个数字确实是年度
|
|||
|
|
first_num = _parse_money(numbers[0]) if numbers else None
|
|||
|
|
if first_num is not None and first_num == row['policy_year']:
|
|||
|
|
start_idx = 1 # 跳过年度对应的数字
|
|||
|
|
# 如果第一个数字是大的金额（如50000），说明行首没有年度列
|
|||
|
|
elif first_num is not None and first_num > 100:
|
|||
|
|
start_idx = 0 # 从头开始分配给 value_columns
|
|||
|
|
|
|||
|
|
for i, num_str in enumerate(numbers[start_idx:], start=start_idx):
|
|||
|
|
val = _parse_money(num_str)
|
|||
|
|
if val is None:
|
|||
|
|
continue
|
|||
|
|
col_idx = i - start_idx
|
|||
|
|
if col_idx < len(value_columns):
|
|||
|
|
_, field = value_columns[col_idx]
|
|||
|
|
row[field] = val
|
|||
|
|
else:
|
|||
|
|
# 没有列位置信息，按顺序猜测
|
|||
|
|
all_fields = ['policy_year', 'total_premium_paid', 'guaranteed_cash_value',
|
|||
|
|
'reversionary_bonus', 'terminal_dividend', 'total_surrender_value',
|
|||
|
|
'death_benefit']
|
|||
|
|
for i, num_str in enumerate(numbers):
|
|||
|
|
if i >= len(all_fields):
|
|||
|
|
break
|
|||
|
|
val = _parse_money(num_str)
|
|||
|
|
if val is not None:
|
|||
|
|
row[all_fields[i]] = val
|
|||
|
|
|
|||
|
|
# 如果没有识别到保单年度，检查是否有"年龄"列推算
|
|||
|
|
if 'policy_year' not in row:
|
|||
|
|
age_match = re.search(r'(\d{1,3})\s*岁', line)
|
|||
|
|
if age_match:
|
|||
|
|
age = int(age_match.group(1))
|
|||
|
|
row['_age'] = age # 留给上层推算
|
|||
|
|
|
|||
|
|
return row if row else None
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _fill_missing_premiums(rows: list[dict], annual_premium: float | None) -> list[dict]:
|
|||
|
|
"""填充缺失的 total_premium_paid。"""
|
|||
|
|
if not annual_premium or annual_premium <= 0:
|
|||
|
|
return rows
|
|||
|
|
for row in rows:
|
|||
|
|
if row.get('total_premium_paid') is None and row.get('policy_year') is not None:
|
|||
|
|
row['total_premium_paid'] = annual_premium * row['policy_year']
|
|||
|
|
return rows
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _ensure_required_fields(row: dict) -> dict:
|
|||
|
|
"""确保每行有所有必需字段。"""
|
|||
|
|
required = {
|
|||
|
|
'policy_year': None,
|
|||
|
|
'total_premium_paid': None,
|
|||
|
|
'guaranteed_cash_value': None,
|
|||
|
|
'reversionary_bonus': None,
|
|||
|
|
'terminal_dividend': None,
|
|||
|
|
'total_surrender_value': None,
|
|||
|
|
'death_benefit': None,
|
|||
|
|
}
|
|||
|
|
result = dict(required)
|
|||
|
|
result.update(row)
|
|||
|
|
return result
|
|||
|
|
|
|||
|
|
|
|||
|
|
# ─── 提领表提取 ──────────────────────────────────────────
|
|||
|
|
|
|||
|
|
_WITHDRAWAL_HEADER_KEYWORDS = [
|
|||
|
|
'提取', '提款', '现金提取', '現金提取', '款项提取', '款項提取',
|
|||
|
|
'withdrawal', 'cash withdrawal',
|
|||
|
|
]
|
|||
|
|
|
|||
|
|
_WITHDRAWAL_REQUIRED_COLS = [
|
|||
|
|
'价值', '價值', 'surrender', 'cash', 'value', '退保',
|
|||
|
|
]
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _extract_withdrawal_rows(text: str) -> list[dict]:
|
|||
|
|
"""从 PDF 文本中提取提领/提款演示表。
|
|||
|
|
|
|||
|
|
注意：只提取"总额"列，不取"保证"子列。
|
|||
|
|
"""
|
|||
|
|
rows = []
|
|||
|
|
lines = text.split('\n')
|
|||
|
|
in_table = False
|
|||
|
|
columns: list[tuple[int, str]] = []
|
|||
|
|
consecutive_non_data = 0
|
|||
|
|
MAX_NON_DATA_LINES = 5
|
|||
|
|
|
|||
|
|
for line in lines:
|
|||
|
|
stripped = line.strip()
|
|||
|
|
if not stripped:
|
|||
|
|
continue
|
|||
|
|
|
|||
|
|
if not in_table:
|
|||
|
|
has_header = any(kw in stripped for kw in _WITHDRAWAL_HEADER_KEYWORDS)
|
|||
|
|
has_value = any(kw in stripped.lower() for kw in _WITHDRAWAL_REQUIRED_COLS)
|
|||
|
|
if has_header and has_value:
|
|||
|
|
columns = _detect_columns(stripped)
|
|||
|
|
if len(columns) >= 2:
|
|||
|
|
in_table = True
|
|||
|
|
consecutive_non_data = 0
|
|||
|
|
logger.debug(f"检测到提领表头: {[(c[1], c[0]) for c in columns]}")
|
|||
|
|
continue
|
|||
|
|
|
|||
|
|
# 跳过子表头（"保证 非保证 总额"等）
|
|||
|
|
subheader_pats = ['保证', '保證', '非保证', '非保證', '总额', '總額',
|
|||
|
|
'guaranteed', 'non-guaranteed', 'total']
|
|||
|
|
if sum(1 for p in subheader_pats if p in stripped.lower()) >= 2:
|
|||
|
|
continue
|
|||
|
|
|
|||
|
|
numbers = re.findall(r'[\d,]+(?:\.\d+)?', stripped)
|
|||
|
|
if not numbers:
|
|||
|
|
consecutive_non_data += 1
|
|||
|
|
if consecutive_non_data >= MAX_NON_DATA_LINES:
|
|||
|
|
break
|
|||
|
|
continue
|
|||
|
|
|
|||
|
|
consecutive_non_data = 0
|
|||
|
|
row = _parse_withdrawal_row(stripped, columns, numbers)
|
|||
|
|
if row and row.get('policy_year') is not None:
|
|||
|
|
rows.append(row)
|
|||
|
|
|
|||
|
|
return rows
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _parse_withdrawal_row(
|
|||
|
|
line: str, columns: list[tuple[int, str]], numbers: list[str]
|
|||
|
|
) -> dict | None:
|
|||
|
|
"""解析提领表单行。"""
|
|||
|
|
row = {}
|
|||
|
|
year_match = re.match(r'^\s*(\d{1,3})\b', line)
|
|||
|
|
if year_match:
|
|||
|
|
year_val = int(year_match.group(1))
|
|||
|
|
if 0 < year_val <= 100:
|
|||
|
|
row['policy_year'] = year_val
|
|||
|
|
|
|||
|
|
# 提领表标准字段（按优先级）
|
|||
|
|
withdrawal_fields = [
|
|||
|
|
'annual_withdrawal', 'total_withdrawn',
|
|||
|
|
'surrender_value_before', 'surrender_value_after',
|
|||
|
|
]
|
|||
|
|
|
|||
|
|
if columns:
|
|||
|
|
value_columns = [(pos, f) for pos, f in columns if f != 'policy_year']
|
|||
|
|
start_idx = 0
|
|||
|
|
if 'policy_year' in row:
|
|||
|
|
first_num = _parse_money(numbers[0]) if numbers else None
|
|||
|
|
if first_num is not None and first_num == row['policy_year']:
|
|||
|
|
start_idx = 1
|
|||
|
|
|
|||
|
|
for i, num_str in enumerate(numbers[start_idx:], start=start_idx):
|
|||
|
|
val = _parse_money(num_str)
|
|||
|
|
if val is None:
|
|||
|
|
continue
|
|||
|
|
col_idx = i - start_idx
|
|||
|
|
if col_idx < len(value_columns):
|
|||
|
|
_, field = value_columns[col_idx]
|
|||
|
|
row[field] = val
|
|||
|
|
else:
|
|||
|
|
for i, num_str in enumerate(numbers):
|
|||
|
|
if i == 0 and 'policy_year' in row:
|
|||
|
|
continue
|
|||
|
|
field_idx = i - 1 if 'policy_year' in row else i
|
|||
|
|
if field_idx >= len(withdrawal_fields):
|
|||
|
|
break
|
|||
|
|
val = _parse_money(num_str)
|
|||
|
|
if val is not None:
|
|||
|
|
row[withdrawal_fields[field_idx]] = val
|
|||
|
|
|
|||
|
|
return row if row else None
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _ensure_withdrawal_fields(row: dict) -> dict:
|
|||
|
|
"""确保提领行有所有必需字段。"""
|
|||
|
|
required = {
|
|||
|
|
'policy_year': None,
|
|||
|
|
'annual_withdrawal': None,
|
|||
|
|
'total_withdrawn': None,
|
|||
|
|
'surrender_value_before': None,
|
|||
|
|
'surrender_value_after': None,
|
|||
|
|
}
|
|||
|
|
result = dict(required)
|
|||
|
|
result.update(row)
|
|||
|
|
return result
|
|||
|
|
|
|||
|
|
|
|||
|
|
# ─── 产品类型识别 ─────────────────────────────────────────
|
|||
|
|
|
|||
|
|
def _detect_product_type(text: str, benefit_rows: list[dict]) -> str:
|
|||
|
|
"""从文本和提取数据推断产品类型：savings / ci / iul。"""
|
|||
|
|
text_lower = text.lower()
|
|||
|
|
|
|||
|
|
# 文本关键词判断
|
|||
|
|
ci_keywords = ['危疾', '重疾', '重大疾病', 'critical illness', 'ci plan',
|
|||
|
|
'严重疾病', '危疾保障', 'dread disease']
|
|||
|
|
iul_keywords = ['iul', 'index universal', 'universal life', '指数型万用',
|
|||
|
|
'万用寿险', '萬用壽險', 'index account', 'index_accounts']
|
|||
|
|
|
|||
|
|
ci_score = sum(1 for kw in ci_keywords if kw in text_lower)
|
|||
|
|
iul_score = sum(1 for kw in iul_keywords if kw in text_lower)
|
|||
|
|
|
|||
|
|
# 数据特征判断
|
|||
|
|
for row in benefit_rows:
|
|||
|
|
if row.get('account_value') or row.get('non_guaranteed_account_value'):
|
|||
|
|
iul_score += 3
|
|||
|
|
if row.get('death_benefit') and not row.get('guaranteed_cash_value'):
|
|||
|
|
ci_score += 1
|
|||
|
|
|
|||
|
|
# policy 字段判断
|
|||
|
|
if re.search(r'(?:保额|投保额|sum\s*insured|保額)\s*[：:]\s*[\d,]+', text_lower):
|
|||
|
|
ci_score += 1
|
|||
|
|
|
|||
|
|
if iul_score > ci_score and iul_score > 0:
|
|||
|
|
return 'iul'
|
|||
|
|
if ci_score > 0:
|
|||
|
|
return 'ci'
|
|||
|
|
return 'savings'
|
|||
|
|
|
|||
|
|
|
|||
|
|
# ─── 主提取函数 ──────────────────────────────────────────
|
|||
|
|
|
|||
|
|
def extract_insurance_regex(pdf_text: str) -> dict:
|
|||
|
|
"""从 PDF 原始文本中用纯正则提取结构化保险数据。
|
|||
|
|
|
|||
|
|
Args:
|
|||
|
|
pdf_text: PDF 提取的原始文本
|
|||
|
|
|
|||
|
|
Returns:
|
|||
|
|
符合 extraction schema 的 dict，字段缺失填 null
|
|||
|
|
"""
|
|||
|
|
product_name = _extract_product_name(pdf_text) or 'unknown'
|
|||
|
|
currency = _extract_currency(pdf_text)
|
|||
|
|
annual_premium = _extract_annual_premium(pdf_text)
|
|||
|
|
insured_info = _extract_insured_info(pdf_text)
|
|||
|
|
premium_period = _extract_premium_payment_period(pdf_text)
|
|||
|
|
coverage_period = _extract_coverage_period(pdf_text)
|
|||
|
|
|
|||
|
|
# 提取利益演示表
|
|||
|
|
benefit_rows = _extract_benefit_rows(pdf_text)
|
|||
|
|
|
|||
|
|
# 提取提领表
|
|||
|
|
withdrawal_rows = _extract_withdrawal_rows(pdf_text)
|
|||
|
|
|
|||
|
|
# 产品类型由上层 infer_plan_type() 推断（基于实际提取数据），此处不重复判断
|
|||
|
|
|
|||
|
|
# 填充缺失保费
|
|||
|
|
benefit_rows = _fill_missing_premiums(benefit_rows, annual_premium)
|
|||
|
|
|
|||
|
|
# 确保字段完整
|
|||
|
|
benefit_rows = [_ensure_required_fields(r) for r in benefit_rows]
|
|||
|
|
withdrawal_rows = [_ensure_withdrawal_fields(r) for r in withdrawal_rows]
|
|||
|
|
|
|||
|
|
# 按年度排序并去重
|
|||
|
|
seen_years = set()
|
|||
|
|
deduped = []
|
|||
|
|
for row in sorted(benefit_rows, key=lambda r: r.get('policy_year') or 0):
|
|||
|
|
year = row.get('policy_year')
|
|||
|
|
if year is not None and year not in seen_years:
|
|||
|
|
seen_years.add(year)
|
|||
|
|
deduped.append(row)
|
|||
|
|
benefit_rows = deduped
|
|||
|
|
|
|||
|
|
# 提领表去重
|
|||
|
|
seen_wd_years = set()
|
|||
|
|
deduped_wd = []
|
|||
|
|
for row in sorted(withdrawal_rows, key=lambda r: r.get('policy_year') or 0):
|
|||
|
|
year = row.get('policy_year')
|
|||
|
|
if year is not None and year not in seen_wd_years:
|
|||
|
|
seen_wd_years.add(year)
|
|||
|
|
deduped_wd.append(row)
|
|||
|
|
withdrawal_rows = deduped_wd
|
|||
|
|
|
|||
|
|
# 提取保额（如果有）
|
|||
|
|
sum_insured = None
|
|||
|
|
si_match = re.search(
|
|||
|
|
r'(?:保额|投保额|保額|sum\s*insured)\s*[：:]\s*[\$UuSsHhKk]*\s*([\d,]+(?:\.\d+)?)',
|
|||
|
|
pdf_text, re.IGNORECASE,
|
|||
|
|
)
|
|||
|
|
if si_match:
|
|||
|
|
sum_insured = _parse_money(si_match.group(1))
|
|||
|
|
|
|||
|
|
logger.info(
|
|||
|
|
f"[RegexExtractor] 提取完成: product={product_name}, "
|
|||
|
|
f"currency={currency}, premium={annual_premium}, "
|
|||
|
|
f"age={insured_info.get('age')}, benefit_rows={len(benefit_rows)}, "
|
|||
|
|
f"withdrawal_rows={len(withdrawal_rows)}"
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
data = {
|
|||
|
|
'product_name': product_name,
|
|||
|
|
'product_type': 'savings', # 由上层 infer_plan_type() 覆盖
|
|||
|
|
'insured': {
|
|||
|
|
'name': None,
|
|||
|
|
'age': insured_info.get('age'),
|
|||
|
|
'gender': insured_info.get('gender'),
|
|||
|
|
'relation': None,
|
|||
|
|
'smoker': None,
|
|||
|
|
},
|
|||
|
|
'policy': {
|
|||
|
|
'product_name': product_name,
|
|||
|
|
'currency': currency,
|
|||
|
|
'sum_insured': sum_insured,
|
|||
|
|
'basic_sum_insured': None,
|
|||
|
|
'annual_premium': annual_premium,
|
|||
|
|
'premium_payment_period': premium_period,
|
|||
|
|
'coverage_period': coverage_period,
|
|||
|
|
},
|
|||
|
|
'benefit_illustration': benefit_rows,
|
|||
|
|
'withdrawal_illustration': withdrawal_rows,
|
|||
|
|
'sales_insights': None,
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
# CI 产品：提取保障项目（如果文本包含 CI 关键词）
|
|||
|
|
coverage_items = _extract_coverage_items(pdf_text)
|
|||
|
|
if coverage_items:
|
|||
|
|
data['coverage_items'] = coverage_items
|
|||
|
|
|
|||
|
|
# IUL 产品：提取指数账户（如果文本包含 IUL 关键词）
|
|||
|
|
index_accounts = _extract_index_accounts(pdf_text)
|
|||
|
|
if index_accounts:
|
|||
|
|
data['index_accounts'] = index_accounts
|
|||
|
|
|
|||
|
|
return data
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _extract_coverage_items(text: str) -> list[dict]:
|
|||
|
|
"""提取危疾保险的保障项目。"""
|
|||
|
|
items = []
|
|||
|
|
# 匹配 "保障项目 赔付金额" 格式的表
|
|||
|
|
lines = text.split('\n')
|
|||
|
|
in_section = False
|
|||
|
|
for line in lines:
|
|||
|
|
stripped = line.strip()
|
|||
|
|
if not stripped:
|
|||
|
|
continue
|
|||
|
|
if '保障项目' in stripped or '保障範圍' in stripped or 'coverage' in stripped.lower():
|
|||
|
|
in_section = True
|
|||
|
|
continue
|
|||
|
|
if in_section:
|
|||
|
|
# 尝试匹配 "项目名 金额" 格式
|
|||
|
|
m = re.match(r'^(.{2,20})\s+[\$UuSsHhKk]*\s*([\d,]+(?:\.\d+)?)', stripped)
|
|||
|
|
if m:
|
|||
|
|
label = m.group(1).strip()
|
|||
|
|
amount = _parse_money(m.group(2))
|
|||
|
|
if amount and label:
|
|||
|
|
items.append({'label': label, 'amount': amount, 'percentage': None, 'description': None})
|
|||
|
|
elif not re.search(r'\d', stripped):
|
|||
|
|
break # 非数字行，可能到了下一节
|
|||
|
|
return items
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _extract_index_accounts(text: str) -> list[dict]:
|
|||
|
|
"""提取 IUL 指数账户信息。"""
|
|||
|
|
accounts = []
|
|||
|
|
# 匹配 "账户名 配置比例 利率" 格式
|
|||
|
|
patterns = [
|
|||
|
|
r'(S&P\s*500|Hang\s*Seng|恒生指数|Global\s*index|指数\d?)\s+([\d.]+)%?\s+([\d.]+)%?\s+([\d.]+)%?',
|
|||
|
|
r'(S&P\s*500|Hang\s*Seng|恒生指数|Global\s*index)\s+([\d.]+)%',
|
|||
|
|
]
|
|||
|
|
for p in patterns:
|
|||
|
|
for m in re.finditer(p, text, re.IGNORECASE):
|
|||
|
|
account = {
|
|||
|
|
'name': m.group(1).strip(),
|
|||
|
|
'allocation': float(m.group(2)),
|
|||
|
|
'current_rate': float(m.group(3)) if m.lastindex >= 3 else None,
|
|||
|
|
'guaranteed_floor': float(m.group(4)) if m.lastindex >= 4 else None,
|
|||
|
|
}
|
|||
|
|
accounts.append(account)
|
|||
|
|
return accounts
|
|||
|
|
|
|||
|
|
|
|||
|
|
def count_benefit_rows(data: dict) -> int:
|
|||
|
|
"""统计提取到的利益演示行数。"""
|
|||
|
|
rows = data.get('benefit_illustration', [])
|
|||
|
|
if not isinstance(rows, list):
|
|||
|
|
return 0
|
|||
|
|
return len(rows)
|