"""FieldValue 与提取候选合并。 这里只处理来源、值和冲突，不推断保险业务公式。 """ from __future__ import annotations import math from collections import defaultdict from decimal import Decimal, InvalidOperation from typing import Any SOURCE_WEIGHTS = { "pdf_table": 1.0, "profile_table": 0.98, "regex": 0.86, "ocr": 0.78, "llm": 0.68, "context_hint": 0.2, } def missing_field_value(*, currency: str | None = None, unit: str | None = None) -> dict: return { "status": "missing", "value": None, "currency": currency, "unit": unit, "confidence": None, "evidence": [], } def extracted_field_value( value: Any, *, currency: str | None = None, unit: str | None = None, confidence: float | None = None, evidence: list[dict] | None = None, ) -> dict: if value is None: return missing_field_value(currency=currency, unit=unit) return { "status": "extracted", "value": value, "currency": currency, "unit": unit, "confidence": _confidence(confidence), "evidence": list(evidence or []), } def confirmed_field_value(value: Any, *, previous: dict | None = None, reason: str) -> dict: reason = str(reason or "").strip() if not reason: raise ValueError("人工覆盖必须填写原因") previous = previous or {} return { "status": "confirmed", "value": value, "currency": previous.get("currency"), "unit": previous.get("unit"), "confidence": previous.get("confidence"), "evidence": list(previous.get("evidence") or []), "overrideReason": reason, } def make_candidate( field_path: str, raw_value: Any, *, normalized_value: Any = None, source: str, currency: str | None = None, unit: str | None = None, confidence: float | None = None, evidence: list[dict] | None = None, extractor_version: str = "", context_hint: dict | None = None, ) -> dict: value = raw_value if normalized_value is None else normalized_value return { "fieldPath": str(field_path), "rawValue": raw_value, "normalizedValue": value, "source": str(source), "currency": currency, "unit": unit, "confidence": _confidence(confidence), "evidence": list(evidence or []), "extractorVersion": str(extractor_version or ""), "contextHint": dict(context_hint or {}), } def reconcile_candidates(candidates: list[dict]) -> dict[str, dict]: """按字段合并候选；不同真实值全部保留为 conflict。""" grouped: dict[str, list[dict]] = defaultdict(list) for candidate in candidates: path = str(candidate.get("fieldPath") or "").strip() if path: grouped[path].append(candidate) result: dict[str, dict] = {} for field_path, items in grouped.items(): ranked = sorted(items, key=_candidate_score, reverse=True) values: list[Any] = [] by_value: dict[str, list[dict]] = defaultdict(list) for item in ranked: key = _value_key(item.get("normalizedValue")) by_value[key].append(item) if not any(_same_value(item.get("normalizedValue"), current) for current in values): values.append(item.get("normalizedValue")) best = ranked[0] merged_evidence = _dedupe_evidence([ evidence for item in by_value[_value_key(best.get("normalizedValue"))] for evidence in item.get("evidence") or [] ]) if len(values) > 1: result[field_path] = { "status": "conflict", "value": None, "currency": best.get("currency"), "unit": best.get("unit"), "confidence": best.get("confidence"), "evidence": _dedupe_evidence([ evidence for item in ranked for evidence in item.get("evidence") or [] ]), "conflictCandidates": values, } else: result[field_path] = extracted_field_value( best.get("normalizedValue"), currency=best.get("currency"), unit=best.get("unit"), confidence=best.get("confidence"), evidence=merged_evidence, ) return result def validate_field_value(value: dict, field_path: str = "") -> list[dict]: issues: list[dict] = [] if not isinstance(value, dict): return [_issue("FIELD_VALUE_INVALID", "字段值格式无效", field_path)] status = value.get("status") if status not in {"missing", "extracted", "conflict", "confirmed", "derived"}: issues.append(_issue("FIELD_STATUS_INVALID", "字段状态无效", field_path)) if status == "missing" and value.get("value") is not None: issues.append(_issue("MISSING_VALUE_NOT_NULL", "缺失字段的值必须为 null", field_path)) if status == "conflict" and len(value.get("conflictCandidates") or []) < 2: issues.append(_issue("CONFLICT_CANDIDATES_MISSING", "冲突字段必须保留候选值", field_path)) if status == "derived" and not str(value.get("derivedBy") or "").strip(): issues.append(_issue("DERIVATION_MISSING", "推导字段必须记录公式版本", field_path)) if value.get("overrideReason") and status != "confirmed": issues.append(_issue("OVERRIDE_STATUS_INVALID", "人工覆盖字段必须为 confirmed", field_path)) scalar = value.get("value") if isinstance(scalar, float) and not math.isfinite(scalar): issues.append(_issue("NUMBER_INVALID", "字段数值必须为有限数", field_path)) return issues def _candidate_score(candidate: dict) -> tuple: source = str(candidate.get("source") or "") evidence = candidate.get("evidence") or [] has_bbox = any(item.get("bbox") for item in evidence if isinstance(item, dict)) confidence = candidate.get("confidence") return ( 1 if has_bbox else 0, SOURCE_WEIGHTS.get(source, 0.5), float(confidence) if confidence is not None else 0.0, len(evidence), ) def _confidence(value) -> float | None: if value is None: return None try: parsed = float(value) except (TypeError, ValueError): return None if not math.isfinite(parsed): return None return min(1.0, max(0.0, parsed)) def _same_value(left: Any, right: Any) -> bool: try: return Decimal(str(left).replace(",", "")) == Decimal(str(right).replace(",", "")) except (InvalidOperation, ValueError): return left == right def _value_key(value: Any) -> str: try: return f"number:{Decimal(str(value).replace(',', '')).normalize()}" except (InvalidOperation, ValueError): return f"value:{value!r}" def _dedupe_evidence(items: list[dict]) -> list[dict]: unique: list[dict] = [] seen = set() for item in items: if not isinstance(item, dict): continue key = ( item.get("documentId"), item.get("pageNumber"), tuple(item.get("bbox") or ()), item.get("tableId"), item.get("rowId"), item.get("columnId"), item.get("textHash"), ) if key not in seen: seen.add(key) unique.append(dict(item)) return unique def _issue(code: str, message: str, path: str) -> dict: return {"code": code, "message": message, "path": path, "severity": "error"}