"""PDF 提取服务 — 从 PDF 计划书中提取结构化数据。""" import os import json import time import hashlib import logging import re import shutil import subprocess import tempfile from dataclasses import dataclass from typing import Callable, Optional logger = logging.getLogger(__name__) CACHE_VERSION = 4 def _format_exception(exc: Exception) -> str: message = str(exc) or repr(exc) return f"{exc.__class__.__name__}: {message}" @dataclass class ExtractionResult: pdf_path: str product_name: str plan_type: str # savings/ci/iul status: str # success/partial/error data: Optional[dict] = None usage: Optional[dict] = None error: Optional[str] = None duration_ms: float = 0 def infer_plan_type(raw: dict) -> str: """从 LLM 输出推断产品类型。""" t = str(raw.get("product_type", "")).lower() if "ci" in t or "critical" in t: return "ci" if "iul" in t or "universal" in t: return "iul" rows = raw.get("benefit_illustration", []) if not isinstance(rows, list): rows = [] has_savings = any( r.get("total_surrender_value") is not None or r.get("guaranteed_cash_value") is not None or r.get("reversionary_bonus") is not None for r in rows if isinstance(r, dict) ) has_ci = any( r.get("surrender_value_total") is not None or r.get("death_benefit_total") is not None for r in rows if isinstance(r, dict) ) has_iul = any( r.get("cash_value") is not None or r.get("account_value") is not None for r in rows if isinstance(r, dict) ) if has_iul: return "iul" if has_savings: return "savings" if has_ci: return "ci" policy = raw.get("policy", {}) if isinstance(policy, dict): if policy.get("index_account_rate") is not None or policy.get("capital_partition") is not None: return "iul" if policy.get("sum_insured") is not None: return "ci" return "savings" def _hash_file(file_path: str) -> str: """计算文件 SHA-256 哈希。""" h = hashlib.sha256() with open(file_path, "rb") as f: for chunk in iter(lambda: f.read(8192), b""): h.update(chunk) return h.hexdigest() def _get_cache_path(pdf_path: str, cache_dir: str) -> str: """获取缓存文件路径。""" file_hash = _hash_file(pdf_path) return os.path.join(cache_dir, f"{file_hash}.json") def _format_pdf_pages(text_parts: list[str]) -> str: """保留真实页码,供后续按封面、投保信息和利益表筛选关键页面。""" return "\n\n".join( f"[PAGE {page_num}]\n{content.strip()}" for page_num, content in enumerate(text_parts, start=1) if content and content.strip() ) def _extract_pdf_text(pdf_path: str, max_chars: int = 120000) -> str: """提取 PDF 文本,支持多种 PDF 解析库。 优先级:PyMuPDF > PyPDF2 > pdfplumber > pypdf """ text = "" # 尝试 PyMuPDF (fitz 或 pymupdf) try: # PyMuPDF 1.24.0+ 使用 pymupdf,旧版本使用 fitz try: import fitz # PyMuPDF except ImportError: import pymupdf as fitz # 新版本 PyMuPDF doc = fitz.open(pdf_path) text_parts = [] for page in doc: text_parts.append(page.get_text()) doc.close() text = _format_pdf_pages(text_parts) if text.strip() and not _looks_corrupted(text): logger.info(f"使用 PyMuPDF 提取成功: {len(text)} 字符") return text[:max_chars] if len(text) > max_chars else text if text.strip(): logger.warning("PyMuPDF 提取结果疑似乱码,将尝试 OCR") except ImportError: logger.debug("PyMuPDF 未安装,尝试下一个库") except Exception as e: logger.warning(f"PyMuPDF 提取失败: {e}") # 尝试 PyPDF2 try: from PyPDF2 import PdfReader reader = PdfReader(pdf_path) text_parts = [] for page in reader.pages: page_text = page.extract_text() text_parts.append(page_text or "") text = _format_pdf_pages(text_parts) if text.strip() and not _looks_corrupted(text): logger.info(f"使用 PyPDF2 提取成功: {len(text)} 字符") return text[:max_chars] if len(text) > max_chars else text except ImportError: logger.debug("PyPDF2 未安装,尝试下一个库") except Exception as e: logger.warning(f"PyPDF2 提取失败: {e}") # 尝试 pypdf try: from pypdf import PdfReader reader = PdfReader(pdf_path) text_parts = [] for page in reader.pages: page_text = page.extract_text() text_parts.append(page_text or "") text = _format_pdf_pages(text_parts) if text.strip() and not _looks_corrupted(text): logger.info(f"使用 pypdf 提取成功: {len(text)} 字符") return text[:max_chars] if len(text) > max_chars else text except ImportError: logger.debug("pypdf 未安装,尝试下一个库") except Exception as e: logger.warning(f"pypdf 提取失败: {e}") # 尝试 pdfplumber try: import pdfplumber with pdfplumber.open(pdf_path) as pdf: text_parts = [] for page in pdf.pages: page_text = page.extract_text() text_parts.append(page_text or "") text = _format_pdf_pages(text_parts) if text.strip() and not _looks_corrupted(text): logger.info(f"使用 pdfplumber 提取成功: {len(text)} 字符") return text[:max_chars] if len(text) > max_chars else text except ImportError: logger.debug("pdfplumber 未安装") except Exception as e: logger.warning(f"pdfplumber 提取失败: {e}") ocr_text = _extract_pdf_text_ocr(pdf_path, max_chars=max_chars) if ocr_text: return ocr_text # 所有库都失败 logger.error( "无法提取 PDF 文本,请安装以下任一库:\n" " pip install PyMuPDF\n" " pip install PyPDF2\n" " pip install pypdf\n" " pip install pdfplumber" ) return "" def _looks_corrupted(text: str) -> bool: """检测 PDF 文本是否乱码。""" if not text or len(text) < 50: return True visible = [c for c in text if not c.isspace()] if not visible: return True bad_chars = sum( 1 for c in visible if c in ("\ufffd", "\uffff") or ord(c) < 32 or 0x7F <= ord(c) <= 0x9F or 0xE000 <= ord(c) <= 0xF8FF ) readable_chars = sum( 1 for c in visible if ( (c.isascii() and (c.isalnum() or c in ".,:;!?%+-_/()[]{}$¥¥'")) or "\u3400" <= c <= "\u9fff" ) ) return bad_chars / len(visible) > 0.03 or readable_chars / len(visible) < 0.35 def _extract_pdf_text_ocr( pdf_path: str, max_chars: int = 120000, max_pages: int = 40, ) -> str: """对扫描件或字体映射损坏的 PDF 使用 Tesseract OCR。""" tesseract = shutil.which("tesseract") if not tesseract: logger.warning("PDF 文本疑似乱码,但未安装 Tesseract OCR") return "" try: try: import fitz except ImportError: import pymupdf as fitz doc = fitz.open(pdf_path) text_parts = [] page_count = min(len(doc), max_pages) with tempfile.TemporaryDirectory(prefix="insurance-pdf-ocr-") as temp_dir: for index in range(page_count): page = doc[index] pixmap = page.get_pixmap( matrix=fitz.Matrix(2.5, 2.5), colorspace=fitz.csGRAY, ) image_path = os.path.join(temp_dir, f"page-{index + 1}.png") pixmap.save(image_path) completed = subprocess.run( [ tesseract, image_path, "stdout", "-l", "chi_sim+eng", "--psm", "6", ], capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=90, check=False, ) if completed.returncode == 0 and completed.stdout.strip(): text_parts.append(completed.stdout) else: text_parts.append("") logger.warning( "Tesseract OCR 第 %s 页失败: %s", index + 1, completed.stderr.strip()[:300], ) doc.close() text = _format_pdf_pages(text_parts) if text.strip() and not _looks_corrupted(text): logger.info("使用 Tesseract OCR 提取成功: %s 字符", len(text)) text = text[:max_chars] if len(text) > max_chars else text return f"[OCR]\n{text}" logger.warning("Tesseract OCR 未能产生可用文本") except Exception as exc: logger.warning("Tesseract OCR 提取失败: %s", _format_exception(exc)) return "" def _normalized_product_name(data: dict) -> str: product_name = str(data.get("product_name") or "").strip() if not product_name: return "unknown" if product_name.lower() == "unknown": return "unknown" return product_name def _apply_filename_hints(data: dict, pdf_path: str, plan_type: str) -> dict: """用文件名中的明确产品编码纠正 OCR 容易误读的产品名。""" if not isinstance(data, dict): return data filename = os.path.basename(pdf_path) if plan_type == "iul" and re.search(r"(?:^|[_-])SIUL3(?:[_-]|$)", filename, re.IGNORECASE): data["product_name"] = "Manulife SIUL 3" policy = data.get("policy") if isinstance(policy, dict) and "product_name" in policy: policy["product_name"] = "Manulife SIUL 3" return data def assess_extraction_payload(data: Optional[dict], plan_type: str) -> tuple[str, str]: """Return extraction status and a user-facing error when data is incomplete.""" if not isinstance(data, dict) or not data: return "partial", "结构化结果为空,请补充识别数据后再生成 PPT" insured = data.get("insured") or {} policy = data.get("policy") or {} benefit_rows = data.get("benefit_illustration") benefit_rows = benefit_rows if isinstance(benefit_rows, list) else [] product_name = _normalized_product_name(data) insured_age = insured.get("age") def has_positive_number(value) -> bool: try: return float(value) > 0 except (TypeError, ValueError): return False problems = [] if product_name == "unknown": problems.append("产品名称未识别") if not has_positive_number(insured_age): problems.append("被保人年龄缺失") if not benefit_rows: problems.append("利益演示为空") normalized_type = (plan_type or "").lower() if normalized_type == "iul": if not has_positive_number(policy.get("sum_insured")): problems.append("保额缺失") index_accounts = data.get("index_accounts") if not isinstance(index_accounts, list) or not index_accounts: problems.append("指数账户缺失") elif normalized_type == "ci": if not has_positive_number(policy.get("sum_insured")): problems.append("保额缺失") coverage_items = data.get("coverage_items") if not isinstance(coverage_items, list) or not coverage_items: problems.append("保障项目缺失") else: if not has_positive_number(policy.get("annual_premium")): problems.append("年缴保费缺失") if problems: return "partial", ";".join(problems[:3]) return "success", "" def _payload_score(data: Optional[dict]) -> int: """用于比较两次提取结果,优先保留关键字段更完整的一次。""" if not isinstance(data, dict): return 0 insured = data.get("insured") if isinstance(data.get("insured"), dict) else {} policy = data.get("policy") if isinstance(data.get("policy"), dict) else {} rows = data.get("benefit_illustration") row_count = len(rows) if isinstance(rows, list) else 0 return ( row_count * 10 + (5 if _normalized_product_name(data) != "unknown" else 0) + (5 if insured.get("age") else 0) + sum(1 for value in policy.values() if value not in (None, "", [], {})) ) class ExtractionOrchestrator: """PDF 提取编排器。 提取策略(按优先级): 1. 缓存命中 → 直接返回 2. 正则提取(零延迟)→ 成功则仅用 LLM 做轻量分析 3. 正则不足 → 回退到完整 LLM 提取 """ # 正则提取行数阈值:低于此值回退到 LLM REGEX_ROW_THRESHOLD = 3 def __init__(self, use_cache: bool = True, cache_dir: str = ".cache/insurance-ppt"): self.use_cache = use_cache self.cache_dir = cache_dir async def extract_plan( self, pdf_path: str, plan_type: str = "savings", force_reparse: bool = False, progress_callback: Optional[Callable[[int, str], None]] = None, ) -> ExtractionResult: """从 PDF 提取结构化数据。 优先使用正则提取(~50ms),仅在行数不足时回退到 LLM。 """ from insurance.ppt.llm_client import llm_client from insurance.ppt.prompts import ( SAVINGS_PLAN_SYSTEM_PROMPT, CI_PLAN_SYSTEM_PROMPT, IUL_SYSTEM_PROMPT, ANALYSIS_SYSTEM_PROMPT, build_analysis_prompt, select_key_pages, ) from insurance.ppt.regex_extractor import extract_insurance_regex, count_benefit_rows start = time.time() abs_path = os.path.abspath(pdf_path) if not os.path.exists(abs_path): return ExtractionResult( pdf_path=abs_path, product_name="unknown", plan_type=plan_type, status="error", error="文件不存在", duration_ms=(time.time() - start) * 1000, ) # 检查缓存 if self.use_cache and not force_reparse: cached = self._load_from_cache(abs_path) if cached: if progress_callback: progress_callback(100, "已使用历史解析结果") cached.duration_ms = (time.time() - start) * 1000 return cached # 提取 PDF 文本 if progress_callback: progress_callback(10, "正在读取 PDF 文本") pdf_text = _extract_pdf_text(abs_path) if not pdf_text: return ExtractionResult( pdf_path=abs_path, product_name="unknown", plan_type=plan_type, status="error", error="无法提取 PDF 文本", duration_ms=(time.time() - start) * 1000, ) if progress_callback: progress_callback(30, "PDF 文本读取完成,正在识别数据表") # ─── 正则提取(第一阶段,零 LLM 调用)──────────────── regex_start = time.time() used_ocr = pdf_text.startswith("[OCR]") regex_data = extract_insurance_regex(pdf_text) regex_rows = count_benefit_rows(regex_data) regex_ms = (time.time() - regex_start) * 1000 if progress_callback: progress_callback( 50, f"规则识别完成,共识别 {regex_rows} 行利益数据", ) extraction_stats = { "regex_rows": regex_rows, "regex_ms": round(regex_ms, 1), "method": "regex+analysis" if regex_rows >= self.REGEX_ROW_THRESHOLD else "llm_full", } response = None # LLM 响应(可能未调用) if regex_rows >= self.REGEX_ROW_THRESHOLD and not used_ocr: # ─── 正则成功:仅调用 LLM 做轻量分析 ──────────── data = regex_data llm_start = time.time() try: # 选取关键页面(4-5页,~8000字符,远小于原来的 20000+) key_pages = select_key_pages(pdf_text, max_pages=5, max_chars=8000) analysis_prompt = build_analysis_prompt(key_pages, data) if progress_callback: progress_callback(60, "正在补充产品分析") analysis_result, response = await llm_client.structured_output( prompt=analysis_prompt, system_prompt=ANALYSIS_SYSTEM_PROMPT, ) # 将分析结果合并到 data if isinstance(analysis_result, dict): data["sales_insights"] = { "key_points": analysis_result.get("keyPoints", []), "gaps": analysis_result.get("gaps", []), "suggested_questions": analysis_result.get("suggestedQuestions", []), } extraction_stats["llm_tokens"] = { "input": response.tokens.get("input", 0) if response.tokens else 0, "output": response.tokens.get("output", 0) if response.tokens else 0, } extraction_stats["llm_ms"] = round((time.time() - llm_start) * 1000, 1) if progress_callback: progress_callback(90, "产品分析完成,正在校验数据") except Exception as e: # 分析失败不影响已提取的数据,只记日志 logger.warning(f"[ExtractionOrchestrator] LLM 分析失败(不影响数据提取): {_format_exception(e)}") extraction_stats["llm_error"] = _format_exception(e) extraction_stats["llm_ms"] = round((time.time() - llm_start) * 1000, 1) else: # ─── 正则不足:回退到完整 LLM 提取 ──────────────── logger.info( f"[ExtractionOrchestrator] 正则提取行数不足({regex_rows}), " f"回退到完整 LLM 提取" ) prompts = { "savings": SAVINGS_PLAN_SYSTEM_PROMPT, "ci": CI_PLAN_SYSTEM_PROMPT, "iul": IUL_SYSTEM_PROMPT, } system_prompt = prompts.get(plan_type, SAVINGS_PLAN_SYSTEM_PROMPT) llm_start = time.time() try: if progress_callback: progress_callback(55, "规则识别不足,正在等待 AI 结构化数据") extraction_text = select_key_pages( pdf_text, max_pages=10, max_chars=28000, ) data, response = await llm_client.structured_output( prompt=f"请从以下PDF关键页面中提取保险计划书数据:\n\n{extraction_text}", system_prompt=system_prompt, ) initial_status, initial_error = assess_extraction_payload(data, plan_type) if initial_status == "partial": if progress_callback: progress_callback(80, "首次识别不完整,正在再次核对关键字段") corrective_prompt = ( f"上一次提取结果不完整({initial_error})。" "请重新检查 PDF 文本,重点补齐产品名称、被保人年龄、保额、" "指数账户和全部利益演示年度。只输出完整 JSON。\n\n" f"上一次结果:\n{json.dumps(data, ensure_ascii=False)[:5000]}\n\n" f"PDF关键页面:\n{extraction_text}" ) try: corrected_data, corrected_response = await llm_client.structured_output( prompt=corrective_prompt, system_prompt=system_prompt, ) if _payload_score(corrected_data) > _payload_score(data): data = corrected_data response = corrected_response except Exception as correction_error: logger.warning( "[ExtractionOrchestrator] 不完整结果二次核对失败: %s", _format_exception(correction_error), ) extraction_stats["llm_tokens"] = { "input": response.tokens.get("input", 0) if response.tokens else 0, "output": response.tokens.get("output", 0) if response.tokens else 0, } if progress_callback: progress_callback(90, "AI 结构化完成,正在校验数据") except Exception as e: return ExtractionResult( pdf_path=abs_path, product_name="unknown", plan_type=plan_type, status="error", error=f"LLM 调用失败: {_format_exception(e)}", duration_ms=(time.time() - start) * 1000, ) extraction_stats["llm_ms"] = round((time.time() - llm_start) * 1000, 1) data = _apply_filename_hints(data, abs_path, plan_type) # 储蓄险数据修复:确保 total >= gcv if plan_type == "savings" and isinstance(data.get("benefit_illustration"), list): for row in data["benefit_illustration"]: if not isinstance(row, dict): continue gcv = float(row.get("guaranteed_cash_value") or 0) rev = float(row.get("reversionary_bonus") or 0) term = float(row.get("terminal_dividend") or 0) total = float(row.get("total_surrender_value") or 0) if total < gcv: row["total_surrender_value"] = gcv + rev + term # 推断产品类型 detected_type = plan_type if plan_type in ("savings", "ci", "iul") else infer_plan_type(data) product_name = _normalized_product_name(data) status, extraction_error = assess_extraction_payload(data, detected_type) # 只缓存完整结果,避免后续复用错误或缺字段的解析。 if self.use_cache and status == "success": self._save_to_cache(abs_path, data) total_ms = (time.time() - start) * 1000 extraction_stats["total_ms"] = round(total_ms, 1) logger.info(f"[ExtractionOrchestrator] 提取完成: {extraction_stats}") if progress_callback: progress_callback(100, "数据校验完成") return ExtractionResult( pdf_path=abs_path, product_name=product_name, plan_type=detected_type, status=status, data=data, usage={"input": response.tokens.get("input", 0), "output": response.tokens.get("output", 0)} if response and response.tokens else None, error=extraction_error or None, duration_ms=total_ms, ) async def extract_multiple(self, pdf_paths: list[str], plan_type: str = "savings") -> list[ExtractionResult]: """顺序提取多个 PDF。""" results = [] for pdf_path in pdf_paths: logger.info(f" 📄 {os.path.basename(pdf_path)}...") results.append(await self.extract_plan(pdf_path, plan_type)) return results def _load_from_cache(self, pdf_path: str) -> Optional[ExtractionResult]: """从缓存加载。""" try: cache_path = _get_cache_path(pdf_path, self.cache_dir) if not os.path.exists(cache_path): return None with open(cache_path, "r", encoding="utf-8") as f: raw = json.load(f) meta = raw.get("_meta", {}) if meta.get("cacheVersion") != CACHE_VERSION: return None data = raw.get("_data", raw) product_name = _normalized_product_name(data) plan_type = infer_plan_type(data) status, extraction_error = assess_extraction_payload(data, plan_type) return ExtractionResult( pdf_path=pdf_path, product_name=product_name, plan_type=plan_type, status=status, data=data, error=extraction_error or None, ) except Exception: return None def _save_to_cache(self, pdf_path: str, data: dict): """写入缓存。""" try: os.makedirs(self.cache_dir, exist_ok=True) file_hash = _hash_file(pdf_path) cache_data = { "_data": data, "_meta": { "cacheVersion": CACHE_VERSION, "originalFile": os.path.basename(pdf_path), "extractedAt": __import__("datetime").datetime.now().isoformat(), "fileHash": file_hash, }, } cache_path = os.path.join(self.cache_dir, f"{file_hash}.json") with open(cache_path, "w", encoding="utf-8") as f: json.dump(cache_data, f, ensure_ascii=False, indent=2) except Exception as e: logger.warning(f"缓存写入失败: {e}") async def extract_for_poster(self, filepath: str) -> dict: """提取海报所需的关键字段(精简 prompt,降低 LLM 成本)。 返回: {age, gender, currency, sum_assured, premium_term, annual_premium, coverage_period, key_benefits} """ from insurance.ppt.llm_client import llm_client abs_path = os.path.abspath(filepath) text = _extract_pdf_text(abs_path) if not text: raise ValueError("无法提取 PDF 文本") text = text[:6000] system_prompt = ( "你是一位保险计划书解析专家。请从以下计划书内容中提取海报所需的关键字段。\n" "输出 JSON 格式(不要包含 markdown 代码块标记):\n" '{"age": 35, "gender": "男", "currency": "USD", "sum_assured": 500000, ' '"premium_term": 5, "annual_premium": 100000, "coverage_period": "终身", ' '"key_benefits": ["身故赔偿", "全残保障"]}\n' "注意:数值用数字,不要带货币符号。" ) result, _response = await llm_client.structured_output( text, system_prompt, schema={ "type": "object", "properties": { "age": {"type": "number"}, "gender": {"type": "string"}, "currency": {"type": "string"}, "sum_assured": {"type": "number"}, "premium_term": {"type": "number"}, "annual_premium": {"type": "number"}, "coverage_period": {"type": "string"}, "key_benefits": {"type": "array", "items": {"type": "string"}}, }, "required": ["age", "gender", "currency", "sum_assured", "annual_premium"], }, ) return result