baodan/api/insurance/plan_data/field_values.py

"""FieldValue 与提取候选合并。

这里只处理来源、值和冲突，不推断保险业务公式。
"""
from __future__ import annotations

import math
from collections import defaultdict
from decimal import Decimal, InvalidOperation
from typing import Any


SOURCE_WEIGHTS = {
    "pdf_table": 1.0,
    "profile_table": 0.98,
    "regex": 0.86,
    "ocr": 0.78,
    "llm": 0.68,
    "context_hint": 0.2,
}


def missing_field_value(*, currency: str | None = None, unit: str | None = None) -> dict:
    return {
        "status": "missing",
        "value": None,
        "currency": currency,
        "unit": unit,
        "confidence": None,
        "evidence": [],
    }


def extracted_field_value(
    value: Any,
    *,
    currency: str | None = None,
    unit: str | None = None,
    confidence: float | None = None,
    evidence: list[dict] | None = None,
) -> dict:
    if value is None:
        return missing_field_value(currency=currency, unit=unit)
    return {
        "status": "extracted",
        "value": value,
        "currency": currency,
        "unit": unit,
        "confidence": _confidence(confidence),
        "evidence": list(evidence or []),
    }


def confirmed_field_value(value: Any, *, previous: dict | None = None, reason: str) -> dict:
    reason = str(reason or "").strip()
    if not reason:
        raise ValueError("人工覆盖必须填写原因")
    previous = previous or {}
    return {
        "status": "confirmed",
        "value": value,
        "currency": previous.get("currency"),
        "unit": previous.get("unit"),
        "confidence": previous.get("confidence"),
        "evidence": list(previous.get("evidence") or []),
        "overrideReason": reason,
    }


def make_candidate(
    field_path: str,
    raw_value: Any,
    *,
    normalized_value: Any = None,
    source: str,
    currency: str | None = None,
    unit: str | None = None,
    confidence: float | None = None,
    evidence: list[dict] | None = None,
    extractor_version: str = "",
    context_hint: dict | None = None,
) -> dict:
    value = raw_value if normalized_value is None else normalized_value
    return {
        "fieldPath": str(field_path),
        "rawValue": raw_value,
        "normalizedValue": value,
        "source": str(source),
        "currency": currency,
        "unit": unit,
        "confidence": _confidence(confidence),
        "evidence": list(evidence or []),
        "extractorVersion": str(extractor_version or ""),
        "contextHint": dict(context_hint or {}),
    }


def reconcile_candidates(candidates: list[dict]) -> dict[str, dict]:
    """按字段合并候选；不同真实值全部保留为 conflict。"""
    grouped: dict[str, list[dict]] = defaultdict(list)
    for candidate in candidates:
        path = str(candidate.get("fieldPath") or "").strip()
        if path:
            grouped[path].append(candidate)

    result: dict[str, dict] = {}
    for field_path, items in grouped.items():
        ranked = sorted(items, key=_candidate_score, reverse=True)
        values: list[Any] = []
        by_value: dict[str, list[dict]] = defaultdict(list)
        for item in ranked:
            key = _value_key(item.get("normalizedValue"))
            by_value[key].append(item)
            if not any(_same_value(item.get("normalizedValue"), current) for current in values):
                values.append(item.get("normalizedValue"))

        best = ranked[0]
        merged_evidence = _dedupe_evidence([
            evidence
            for item in by_value[_value_key(best.get("normalizedValue"))]
            for evidence in item.get("evidence") or []
        ])
        if len(values) > 1:
            result[field_path] = {
                "status": "conflict",
                "value": None,
                "currency": best.get("currency"),
                "unit": best.get("unit"),
                "confidence": best.get("confidence"),
                "evidence": _dedupe_evidence([
                    evidence for item in ranked for evidence in item.get("evidence") or []
                ]),
                "conflictCandidates": values,
            }
        else:
            result[field_path] = extracted_field_value(
                best.get("normalizedValue"),
                currency=best.get("currency"),
                unit=best.get("unit"),
                confidence=best.get("confidence"),
                evidence=merged_evidence,
            )
    return result


def validate_field_value(value: dict, field_path: str = "") -> list[dict]:
    issues: list[dict] = []
    if not isinstance(value, dict):
        return [_issue("FIELD_VALUE_INVALID", "字段值格式无效", field_path)]
    status = value.get("status")
    if status not in {"missing", "extracted", "conflict", "confirmed", "derived"}:
        issues.append(_issue("FIELD_STATUS_INVALID", "字段状态无效", field_path))
    if status == "missing" and value.get("value") is not None:
        issues.append(_issue("MISSING_VALUE_NOT_NULL", "缺失字段的值必须为 null", field_path))
    if status == "conflict" and len(value.get("conflictCandidates") or []) < 2:
        issues.append(_issue("CONFLICT_CANDIDATES_MISSING", "冲突字段必须保留候选值", field_path))
    if status == "derived" and not str(value.get("derivedBy") or "").strip():
        issues.append(_issue("DERIVATION_MISSING", "推导字段必须记录公式版本", field_path))
    if value.get("overrideReason") and status != "confirmed":
        issues.append(_issue("OVERRIDE_STATUS_INVALID", "人工覆盖字段必须为 confirmed", field_path))
    scalar = value.get("value")
    if isinstance(scalar, float) and not math.isfinite(scalar):
        issues.append(_issue("NUMBER_INVALID", "字段数值必须为有限数", field_path))
    return issues


def _candidate_score(candidate: dict) -> tuple:
    source = str(candidate.get("source") or "")
    evidence = candidate.get("evidence") or []
    has_bbox = any(item.get("bbox") for item in evidence if isinstance(item, dict))
    confidence = candidate.get("confidence")
    return (
        1 if has_bbox else 0,
        SOURCE_WEIGHTS.get(source, 0.5),
        float(confidence) if confidence is not None else 0.0,
        len(evidence),
    )


def _confidence(value) -> float | None:
    if value is None:
        return None
    try:
        parsed = float(value)
    except (TypeError, ValueError):
        return None
    if not math.isfinite(parsed):
        return None
    return min(1.0, max(0.0, parsed))


def _same_value(left: Any, right: Any) -> bool:
    try:
        return Decimal(str(left).replace(",", "")) == Decimal(str(right).replace(",", ""))
    except (InvalidOperation, ValueError):
        return left == right


def _value_key(value: Any) -> str:
    try:
        return f"number:{Decimal(str(value).replace(',', '')).normalize()}"
    except (InvalidOperation, ValueError):
        return f"value:{value!r}"


def _dedupe_evidence(items: list[dict]) -> list[dict]:
    unique: list[dict] = []
    seen = set()
    for item in items:
        if not isinstance(item, dict):
            continue
        key = (
            item.get("documentId"), item.get("pageNumber"),
            tuple(item.get("bbox") or ()), item.get("tableId"),
            item.get("rowId"), item.get("columnId"), item.get("textHash"),
        )
        if key not in seen:
            seen.add(key)
            unique.append(dict(item))
    return unique


def _issue(code: str, message: str, path: str) -> dict:
    return {"code": code, "message": message, "path": path, "severity": "error"}