"""金标 PDF 回归测试框架。 提供： - 金标 JSON 格式定义 - 提取结果与金标准的对比工具 - 字段准确率、表格行召回率、数字单元格准确率计算 用法： from insurance.ppt.golden_test import compare_extraction, load_gold_standard result = compare_extraction(extracted_data, gold_standard) print(result.summary()) """ import json import os import logging from dataclasses import dataclass, field from typing import Optional logger = logging.getLogger(__name__) @dataclass class FieldMatch: """单个字段的匹配结果。""" path: str expected: any actual: any match: bool # 精确匹配 near_match: bool = False # 近似匹配（数值误差 < 1%） missing: bool = False # 提取结果中缺失 @dataclass class ComparisonResult: """对比结果。""" product_name: FieldMatch = None insured_age: FieldMatch = None insured_gender: FieldMatch = None currency: FieldMatch = None annual_premium: FieldMatch = None premium_payment_period: FieldMatch = None sum_insured: FieldMatch = None benefit_row_matches: list = field(default_factory=list) # list[FieldMatch] field_accuracy: float = 0.0 row_recall: float = 0.0 number_accuracy: float = 0.0 total_fields: int = 0 matched_fields: int = 0 missing_fields: int = 0 def summary(self) -> str: return ( f"字段准确率: {self.field_accuracy:.1%} ({self.matched_fields}/{self.total_fields})\n" f"表格行召回率: {self.row_recall:.1%}\n" f"数字单元格准确率: {self.number_accuracy:.1%}\n" f"缺失字段: {self.missing_fields}" ) def _safe_float(value) -> Optional[float]: """安全转为 float。""" if value is None: return None try: return float(value) except (TypeError, ValueError): return None def _compare_scalar(path: str, expected, actual, tolerance: float = 0.0) -> FieldMatch: """对比标量字段。""" if expected is None: return FieldMatch(path=path, expected=expected, actual=actual, match=True) if actual is None or actual == "" or actual == "unknown": return FieldMatch(path=path, expected=expected, actual=actual, match=False, missing=True) # 字符串比较 if isinstance(expected, str): exp_lower = expected.strip().lower() act_lower = str(actual).strip().lower() exact = exp_lower == act_lower near = exp_lower in act_lower or act_lower in exp_lower return FieldMatch(path=path, expected=expected, actual=actual, match=exact, near_match=near) # 数值比较 exp_num = _safe_float(expected) act_num = _safe_float(actual) if exp_num is not None and act_num is not None: exact = abs(exp_num - act_num) < 0.01 near = abs(exp_num - act_num) / max(abs(exp_num), 1) < tolerance return FieldMatch(path=path, expected=expected, actual=actual, match=exact, near_match=near) return FieldMatch(path=path, expected=expected, actual=actual, match=str(expected) == str(actual)) def _compare_benefit_rows(gold_rows: list, actual_rows: list) -> tuple[list[FieldMatch], float, float]: """对比利益演示表行。 返回 (行匹配列表, 行召回率, 数字准确率)。 """ if not gold_rows: return [], 1.0, 1.0 # 按 policy_year 建索引 gold_by_year = {} for row in gold_rows: year = row.get("policy_year") or row.get("policyYear") if year is not None: gold_by_year[int(year)] = row actual_by_year = {} for row in actual_rows: year = row.get("policy_year") or row.get("policyYear") if year is not None: actual_by_year[int(year)] = row matches = [] total_cells = 0 matched_cells = 0 found_years = 0 numeric_fields = [ "total_premium_paid", "guaranteed_cash_value", "reversionary_bonus", "terminal_dividend", "total_surrender_value", "death_benefit", ] for year, gold_row in gold_by_year.items(): actual_row = actual_by_year.get(year) if actual_row is None: matches.append(FieldMatch( path=f"benefit_illustration[{year}]", expected=f"第 {year} 年整行", actual=None, match=False, missing=True, )) continue found_years += 1 for field_name in numeric_fields: gold_val = gold_row.get(field_name) if gold_val is None: continue total_cells += 1 actual_val = actual_row.get(field_name) fm = _compare_scalar( f"benefit_illustration[{year}].{field_name}", gold_val, actual_val, tolerance=0.01, ) if fm.match or fm.near_match: matched_cells += 1 matches.append(fm) row_recall = found_years / len(gold_by_year) if gold_by_year else 1.0 number_accuracy = matched_cells / total_cells if total_cells > 0 else 1.0 return matches, row_recall, number_accuracy def compare_extraction(extracted: dict, gold: dict) -> ComparisonResult: """对比提取结果与金标准。 Args: extracted: 提取结果 data 字段 gold: 金标准 JSON（见 load_gold_standard 格式） Returns: ComparisonResult 包含各维度准确率 """ result = ComparisonResult() # 标量字段对比 ext_insured = extracted.get("insured") or {} gold_insured = gold.get("insured") or {} ext_policy = extracted.get("policy") or {} gold_policy = gold.get("policy") or {} result.product_name = _compare_scalar("product_name", gold.get("product_name"), extracted.get("product_name")) result.insured_age = _compare_scalar("insured.age", gold_insured.get("age"), ext_insured.get("age")) result.insured_gender = _compare_scalar("insured.gender", gold_insured.get("gender"), ext_insured.get("gender")) result.currency = _compare_scalar("policy.currency", gold_policy.get("currency"), ext_policy.get("currency")) result.annual_premium = _compare_scalar("policy.annual_premium", gold_policy.get("annual_premium"), ext_policy.get("annual_premium"), tolerance=0.01) result.premium_payment_period = _compare_scalar("policy.premium_payment_period", gold_policy.get("premium_payment_period"), ext_policy.get("premium_payment_period")) result.sum_insured = _compare_scalar("policy.sum_insured", gold_policy.get("sum_insured"), ext_policy.get("sum_insured"), tolerance=0.01) scalar_matches = [ result.product_name, result.insured_age, result.insured_gender, result.currency, result.annual_premium, result.premium_payment_period, result.sum_insured, ] scalar_matches = [m for m in scalar_matches if m is not None] # 利益表对比 gold_rows = gold.get("benefit_illustration") or [] actual_rows = extracted.get("benefit_illustration") or [] result.benefit_row_matches, result.row_recall, result.number_accuracy = _compare_benefit_rows(gold_rows, actual_rows) # 汇总 all_matches = scalar_matches + result.benefit_row_matches result.total_fields = len(all_matches) result.matched_fields = sum(1 for m in all_matches if m.match or m.near_match) result.missing_fields = sum(1 for m in all_matches if m.missing) result.field_accuracy = result.matched_fields / result.total_fields if result.total_fields > 0 else 0.0 return result def load_gold_standard(path: str) -> dict: """加载金标准 JSON 文件。 金标准格式： { "product_name": "产品全称", "plan_type": "savings", "insured": {"age": 35, "gender": "male"}, "policy": { "currency": "USD", "annual_premium": 100000, "premium_payment_period": 5, "sum_insured": null }, "benefit_illustration": [ {"policy_year": 1, "total_premium_paid": 100000, ...}, ... ], "metadata": { "source": "公司名", "pdf_type": "digital|scanned|mixed", "language": "zh-CN|zh-TW|en", "notes": "备注" } } """ with open(path, "r", encoding="utf-8") as f: return json.load(f)