#!/usr/bin/env python3
"""FEAT-8b stages 1-2 — extract + dedup the GerbilManager source spreadsheets.

Stage 1: parse the 10 Stammbaum pedigree charts and the Wurfchronik litter
         chronicle into normalised animals.json + litters.json, and extract
         embedded photos (anchor-mapped to animals).
Stage 2: dedup animals (key = normalise(name)+DOB, corroborated by DOB+genotype)
         and emit a German-friendly review-report.md for Julian's wife. No DB load.

Re-runnable per file (later Wurfchronik "Teil2+" / more charts just re-run).
Migration tooling — Python, not product code. Zero third-party deps.

See FEAT-8a-format-spec.md (in Pam's hive workspace) for the format analysis.
"""
import os
import re
import sys
import json
import glob
import shutil
import argparse
import unicodedata
from collections import Counter

import xlsx_util as xu
import genotype as gt

HERE = os.path.dirname(os.path.abspath(__file__))
DEFAULT_STAMMBAEUME = r"C:\Users\gulum\dev\Sttammbäume"
DEFAULT_WURFCHRONIK = r"C:\Users\gulum\dev\Wurfchronik der Kleine Chaoten Teil1.xlsx"
OUT = os.path.join(HERE, "output")

DOB = re.compile(r"\*\s?(\d{1,2}\.\d{1,2}\.(?:\d{4}|\d{2}))")
DEATH = re.compile(r"\+\s?(\d{1,2}\.\d{1,2}\.(?:\d{4}|\d{2})|\d{4})")


# ---------------------------------------------------------------- helpers ----
def gen_of(colnum, offset=0):
    """Map a column number to a generation band (0=proband ... 5=deepest)."""
    effective_col = colnum - offset
    if effective_col <= 3:
        return 0          # Column B (2) -> proband
    if effective_col <= 6:
        return 1          # Column E (5) -> parents
    if effective_col <= 9:
        return 2          # Column H (8) -> grandparents
    if effective_col <= 12:
        return 3          # Column K (11) -> great-grandparents
    if effective_col <= 15:
        return 4          # Column N (14) -> gg-grandparents
    return 5              # Column Q (17) or deeper -> ggg-grandparents


def norm_name(name):
    if not name:
        return ""
    n = name.lower()
    n = re.sub(r"\[.*?\]", " ", n)               # drop [line] tags (Wurfchronik)
    n = re.sub(r"\bgen\.\b", " ", n)             # "gen." nickname marker
    n = re.sub(r"\bv\.\s?d\.\b", " von den ", n)
    n = re.sub(r"\b(von der|von den|von|of)\b", " ", n)  # cattery/line connectors
    n = unicodedata.normalize("NFKD", n)
    n = re.sub(r"[^a-z0-9äöüß]", "", n)
    return n


def norm_dob(d):
    if not d:
        return ""
    p = d.split(".")
    if len(p) == 3 and len(p[2]) == 2:
        p[2] = "20" + p[2]
    return ".".join(x.zfill(2) if i < 2 else x for i, x in enumerate(p))


def slug(name, dob):
    base = norm_name(name) or "unbekannt"
    d = norm_dob(dob).replace(".", "")
    return (base[:40] + ("-" + d if d else "")) or "unbekannt"


def clean_name(raw):
    """Strip detail/markers from a name cell, keep the human name + [line]."""
    n = raw.strip().strip(",").strip()
    return n


# --- (name, Zucht) canonicalisation -------------------------------------------
# Julian (FEAT-8 ruling): the [brackets] in Wurfchronik names ARE the Zucht
# (breeding line) and are equivalent to the Stammbaum "of/von <line>" suffix.
# Both fold into one canonical (call-name, zucht) pair; the Zucht acts as a
# dedup DISCRIMINATOR (same name+DOB but different Zucht = different animal).
_BRACKET_ZUCHT = re.compile(r"^(.*?)\s*\[([^\]]+)\]\s*$")
_SUFFIX_ZUCHT = re.compile(
    r"^(.+?)\s+(?:of|von\s+den|von\s+der|v\.\s?d\.|von)\s+(.+)$", re.IGNORECASE)
# canonical values are post-norm_zucht (word-final 'n' folded: "kleinechaote")
ZUCHT_ALIASES = {
    "zdkc": "kleinechaote",   # "Zucht der kleinen Chaoten" (home cattery shorthand)
}

# A Farbschlag value must NOT contain cattery/line connectors (v.d./von/of/gen.) — when it
# does, a parent's NAME has bled into the Farbschlag cell (cross-cell chart read, PEDIGREE-LINK
# bug: e.g. "Victoria Welby gen. Welby v.d. Kleinen Chaoten" became a Farbschlag variant and
# spawned a phantom conflict). Reject such values so they don't pollute farbschlag/conflicts.
_NAME_MARKER = re.compile(r"\bv\.\s?d\.|\bvon\b|\bof\b|\bgen\.", re.IGNORECASE)


def looks_like_animal_name(text):
    """True if a candidate Farbschlag cell actually looks like an animal name (has a
    cattery/line connector). Real Farbschläge are short colour words without these."""
    return bool(_NAME_MARKER.search(text or ""))


def split_name_zucht(raw):
    """'Luna [ZdkC]' -> ('Luna','ZdkC'); 'Pikachu of Black Forest' ->
    ('Pikachu','Black Forest'); plain names -> (name, '')."""
    n = (raw or "").strip()
    m = _BRACKET_ZUCHT.match(n)
    if m:
        return m.group(1).strip(), m.group(2).strip()
    m = _SUFFIX_ZUCHT.match(n)
    if m:
        return m.group(1).strip(), m.group(2).strip()
    return n, ""


def norm_zucht(z):
    """Canonical Zucht key: drops Zucht/von/der fillers, folds declension
    ('kleinen Chaoten' == 'kleine Chaoten'), resolves known shorthands."""
    if not z:
        return ""
    n = z.lower()
    # TOLERANT KC-MATCHER FIX: trailing \b after '.' never fires when next char is ' '
    # (both are non-word chars), so "v.d. kleinen" was NOT stripped. Drop the trailing \b.
    n = re.sub(r"\bv\.\s?d\.", " ", n)
    n = re.sub(r"\b(zucht|privatzucht|der|die|den|des|dem|von|of)\b", " ", n)
    n = re.sub(r"[^a-z0-9äöüß ]", " ", n)
    words = [w[:-1] if len(w) > 4 and w.endswith("n") else w for w in n.split()]
    key = "".join(words)
    return ZUCHT_ALIASES.get(key, key)


def is_clan_zucht(z):
    """True if the zucht name (raw or canonical) identifies the Kleine Chaoten home cattery.

    Accepts all known spellings: 'Zucht der Kleinen Chaoten', 'kleinen Chaoten',
    'v.d. Kleinen Chaoten', '[ZdkC]', 'kleinechaote', etc.
    """
    return norm_zucht(z) == "kleinechaote"


# Name/breeder markers for an externally-acquired animal whose ancestry is NOT
# in the breeder's own charts: pet shops, "von Privat" private hobbyists and
# foreign catteries ("from … , Croatia"). Such founders have genuinely UNKNOWN
# parents — the chart-position heuristic must not fabricate parents for them.
_EXTERNAL_MARKERS = re.compile(
    r"\b(zooladen|zoohandlung|obi|fressnapf|dehner|von\s+privat|privatkauf|"
    r"vom\s+bauern|aus\s+der\s+zoohandlung)\b", re.IGNORECASE)
# Foreign acquisition: "<name> from <cattery>, <Country>" (a comma + country word).
_FOREIGN_FROM = re.compile(
    r"\bfrom\b.+,\s*(croatia|kroatien|poland|polen|netherlands|niederlande|"
    r"belgium|belgien|france|frankreich|austria|österreich|switzerland|schweiz|"
    r"italy|italien|spain|spanien|czech|tschechien|hungary|ungarn)\b", re.IGNORECASE)


def is_external_origin(name, zucht=None, breeder=None):
    """True if an animal is an externally-acquired founder with genuinely unknown
    ancestry (pet shop, private hobbyist, foreign cattery). For such animals the
    positional chart heuristic must NOT invent parents — their parents are unknown.

    NOT external: own stock ('… von den Kleinen Chaoten') and ordinary German
    catteries that appear as in-chart ancestors (e.g. 'of Black Forest') — those
    can legitimately have charted ancestors elsewhere; this gate is only for the
    leaf/founder markers above.
    """
    blob = " ".join(p for p in (name, zucht, breeder) if p)
    if _EXTERNAL_MARKERS.search(blob):
        return True
    if _FOREIGN_FROM.search(name or ""):
        return True
    return False


def canon_pair(raw):
    """Full raw name -> (normalised call-name, canonical zucht)."""
    name, zucht = split_name_zucht(raw)
    return norm_name(name), norm_zucht(zucht)


# --------------------------------------------------- Stammbaum extraction ----
def parse_detail(text):
    """From a string that contains *DOB and/or genotype, pull (dob, death, geno_str).

    For compact lines ("Name,*DOB[/+death], genotype") the genotype is everything
    after the date — we must NOT scan from the first locus-looking letter, or stray
    name words ("den", "of") get swallowed as genotype tokens.
    """
    dob = DOB.search(text)
    death = DEATH.search(text)
    geno = ""
    if dob:
        tail = text[dob.end():]
        # FIX (ab8fdb0b): broaden the leading death-marker strip BEFORE the
        # looks_like_genotype check, so a death marker right after the DOB never
        # leaks into the genotype string. Covers: bare "/+", "/+<numeric date>",
        # "/+<year>", "/+<Textmonth>'<year>" (e.g. "/+April'2017"), "/+ ," and a
        # leading "+<date>" (e.g. "+09.07.2017" — Fegur). The full-text DEATH regex
        # still captures the actual death date, so no data is lost.
        tail = re.sub(
            r"^\s*/?\+\s*"
            r"(?:\d{1,2}\.\d{1,2}\.(?:\d{4}|\d{2})|\d{4}|[A-Za-zÄÖÜäöü]+'?\s?\d{2,4})?",
            "", tail)
        tail = tail.lstrip(" ,").strip()
        # FIX-4 (Skarlett): strip trailing "/ +YEAR" death-year artifacts leaked from compact
        # chart cells (e.g. "… rere / +2018"). The DEATH regex still captures the year from
        # the full cell text, so it appears as a death-date conflict — not a genotype conflict.
        tail = re.sub(r"\s*/\s*\+\d{4}\s*$", "", tail).strip()
        if gt.looks_like_genotype(tail):
            geno = tail
    return (dob.group(1) if dob else "",
            death.group(1) if death else "",
            geno)


def extract_stammbaum(path):
    """Return list of animal dicts for one chart file."""
    fname = os.path.basename(path)
    z = __import__("zipfile").ZipFile(path)
    ss = xu.shared_strings(z)
    sheets = xu.sheet_paths(z)
    cells = xu.read_cells(z, sheets[0], ss)
    fillsex = xu.cell_fill_sex(z, sheets[0])     # box colour -> sex (blue=male, white=female)

    has_col2 = any(c == 2 for (c, r) in cells)
    col_offset = 0 if has_col2 else 3

    # group cells by column for block reconstruction
    by_col = {}
    for (c, r), t in cells.items():
        by_col.setdefault(c, []).append((r, t))
    for c in by_col:
        by_col[c].sort()

    animals = []
    used = set()

    for (c, r), t in sorted(cells.items()):
        if (c, r) in used:
            continue
        compact = re.match(r"^(.+?),\s*\*", t)        # "Name,*DOB, genotype"
        is_block_dob = bool(re.match(r"^\*\s?\d", t))   # standalone "*DOB"
        if not compact and not is_block_dob:
            continue

        name_row = r
        if compact:
            name = clean_name(compact.group(1))
            dob, death, geno = parse_detail(t)
            farbschlag = ""
            breeder = ""
            used.add((c, r))
        else:
            # full block: name above, farbschlag/genotype/breeder below
            dob, death, geno0 = parse_detail(t)
            name = ""
            for rr in range(r - 3, r):
                if (c, rr) in cells and not re.match(r"^\*?\s?\d", cells[(c, rr)]) \
                        and not gt.looks_like_genotype(cells[(c, rr)]):
                    name = clean_name(cells[(c, rr)])
                    name_row = rr
                    used.add((c, rr))
                    break
            farbschlag = ""
            geno = geno0
            breeder = ""
            # BAND-AWARE (Julian-confirmed): early bands (gen 0-1, cols B/E/H) are 5-cell blocks
            # WITH a Farbschlag cell; deep bands (gen >= 2, cols K/N/Q...) are 3-cell blocks
            # (Name/DOB/Genotype) with NO Farbschlag — colour is derived from the genotype. So in
            # deep bands we must NOT grab the next block's name or a stray health note as Farbschlag.
            deep_band = gen_of(c, col_offset) >= 2
            for rr in range(r + 1, r + 4):
                cell = cells.get((c, rr))
                if not cell:
                    continue
                if gt.looks_like_genotype(cell):
                    geno = cell
                    used.add((c, rr))
                elif re.search(r"\b(Zucht|Privatzucht)\b", cell) or cell.startswith("("):
                    breeder = cell
                    used.add((c, rr))
                elif not deep_band and not farbschlag and not re.match(r"^\*?\s?\d", cell) \
                        and not looks_like_animal_name(cell):
                    farbschlag = cell
                    used.add((c, rr))
            used.add( (c, r) )

        if "verpaarung" in name.lower():
            continue

        g = parse_detail(t) if compact else (dob, death, geno)
        genodict = gt.parse(geno)
        # Zucht: from the name's [tag]/of-von suffix, else from the breeder line
        _, zraw = split_name_zucht(name)
        if not zraw and breeder:
            zraw = breeder
        animals.append({
            "id": None,                       # assigned in dedup
            "name": name,
            "nameVariants": [],
            "dob": dob,
            "death": death,
            "gender": fillsex.get((c, r)),    # box colour: blue=male, white=female
            "farbschlag": farbschlag,
            "genotype": genodict,
            "deaf": genodict.get("deaf"),
            "tags": genodict.get("tags", []),
            "breeder": breeder,
            "zucht": zraw,
            "parentRefs": [],
            "photos": [],
            "sourceFiles": [fname],
            "_gen": gen_of(c, col_offset),
            "_col": c,
            "_row": name_row,
            "_file": fname,
            "_zucht": norm_zucht(zraw),
        })

    # name-pair cells "X & Y" (deepest generation, names only)
    for (c, r), t in cells.items():
        if (c, r) in used:
            continue
        if " & " in t and not DOB.search(t) and len(t) < 90 and gen_of(c, col_offset) >= 4:
            for part in t.split(" & "):
                part = clean_name(part)
                if part:
                    _, zraw = split_name_zucht(part)
                    animals.append({
                        "id": None, "name": part, "nameVariants": [],
                        "dob": "", "death": "", "gender": None, "farbschlag": "",
                        "genotype": gt.parse(""), "deaf": None, "tags": [],
                        "breeder": "", "zucht": zraw,
                        "parentRefs": [], "photos": [], "sourceFiles": [fname],
                        "_gen": gen_of(c, col_offset), "_col": c, "_row": r, "_file": fname,
                        "_zucht": norm_zucht(zraw),
                    })

    _reconstruct_parents(animals)
    _attach_photos(z, sheets, animals, fname)
    return animals


def _reconstruct_parents(animals):
    """Positional: an animal's parents are the bracketing blocks one generation
    deeper (father = nearest block above, mother = nearest below). Role guess is
    by vertical position (German charts: Vater oben) — flagged for review; the
    Wurfchronik is authoritative for matched animals (Stage 3)."""
    by_gen = {}
    for a in animals:
        by_gen.setdefault(a["_gen"], []).append(a)
    for g, group in by_gen.items():
        nxt = sorted(by_gen.get(g + 1, []), key=lambda a: a["_row"])
        if not nxt:
            continue
        for a in group:
            # Externally-acquired founders (pet shop / von Privat / foreign cattery)
            # have genuinely UNKNOWN ancestry — do NOT invent chart-position parents
            # for them (tickets #5 Bill von Privat, #13 Cooky vom Zooladen, #28 Zadar).
            if is_external_origin(a.get("name"), a.get("zucht"), a.get("breeder")):
                continue
            r = a["_row"]
            above = [x for x in nxt if x["_row"] <= r]
            below = [x for x in nxt if x["_row"] > r]
            father = above[-1] if above else None
            mother = below[0] if below else None
            for parent, role in ((father, "father"), (mother, "mother")):
                if parent:
                    p_name = parent["name"] or "unbekannt"
                    a["parentRefs"].append({
                        "name": p_name,
                        "dob": parent["dob"],
                        "roleGuess": role,
                        "method": "chart-position",
                        "confidence": "medium",
                    })


def _attach_photos(z, sheets, animals, fname):
    anchors = [a for a in xu.image_anchors(z)]
    if not anchors:
        return
        
    by_gen = {}
    for a in animals:
        by_gen.setdefault(a["_gen"], []).append(a)
    media_dir = os.path.join(OUT, "photos")
    col_offset = 0 if any(a["_col"] == 2 for a in animals) else 3

    # A photo sits one column either side of (or on) its animal's name cell;
    # the generation is read from the photo's column. Whether photos are LEFT or
    # RIGHT of the name varies, and the old fixed-column heuristic (col 1 / col 4)
    # misclassified sheets that have neither, shifting every photo one generation
    # toward the proband (e.g. „Stammbaum von Kazuya“ / „Picus Son“: Kazuya wore
    # his father's photo, the father his grandfather's). Instead, try BOTH
    # interpretations and keep the one that places photos closest to their
    # assigned animal's name column (minimal horizontal misalignment).
    def get_anchor_gen(colnum, left_style):
        effective_col = colnum - col_offset
        if left_style:
            if effective_col <= 3: return 0
            if effective_col <= 6: return 1
            if effective_col <= 9: return 2
            if effective_col <= 12: return 3
            if effective_col <= 15: return 4
            return 5
        else:
            if effective_col <= 4: return 0
            if effective_col <= 7: return 1
            if effective_col <= 10: return 2
            if effective_col <= 13: return 3
            if effective_col <= 16: return 4
            return 5

    def targets_for(left_style):
        out = []
        for (sp, col, row, media) in anchors:
            cands = by_gen.get(get_anchor_gen(col, left_style), []) or animals
            out.append(min(cands, key=lambda a: abs((a["_row"] - row) - 10)) if cands else None)
        return out

    def mean_col_offset(targets):
        vals = [abs(col - t["_col"]) for (sp, col, row, m), t in zip(anchors, targets) if t]
        return sum(vals) / len(vals) if vals else 0.0

    left_targets = targets_for(True)
    right_targets = targets_for(False)
    targets = (
        left_targets
        if mean_col_offset(left_targets) <= mean_col_offset(right_targets)
        else right_targets
    )

    for (sp, col, row, media), target in zip(anchors, targets):
        if not target:
            continue
        ext = os.path.splitext(media)[1] or ".img"
        sl = slug(target["name"], target["dob"])
        dest_dir = os.path.join(media_dir, sl)
        os.makedirs(dest_dir, exist_ok=True)
        rel = f"photos/{sl}/{os.path.basename(media)}"
        try:
            with z.open(media) as src, open(os.path.join(OUT, rel), "wb") as dst:
                shutil.copyfileobj(src, dst)
            if rel not in target["photos"]:
                target["photos"].append(rel)
        except KeyError:
            pass


# -------------------------------------------------- Wurfchronik extraction ---
def extract_wurfchronik(path):
    """Return list of litter dicts. Parses columns BY HEADER (sheets differ)."""
    fname = os.path.basename(path)
    z = __import__("zipfile").ZipFile(path)
    ss = xu.shared_strings(z)
    litters = []
    for sp in xu.sheet_paths(z):
        cells = xu.read_cells(z, sp, ss)
        if not cells:
            continue
        # build row -> {colnum: text}
        rows = {}
        for (c, r), t in cells.items():
            rows.setdefault(r, {})[c] = t
        hdr = xu.header_row(cells)            # colnum -> header label
        hdr_row = min(r for (_, r) in cells)  # the header row number, to skip it
        # map header label -> colnum (fuzzy by keyword)
        def find(*keys):
            for c, lbl in hdr.items():
                low = lbl.lower()
                if any(k in low for k in keys):
                    return c
            return None
        col_id = find("wurfbuchstabe", "buchstabe")
        col_date = find("geburtsdatum", "datum")
        col_dam = find("mutter")
        col_sire = find("vater")
        col_ws = find("ws", "wurfstärke", "wurfstaerke")
        col_breakdown = find("männchen", "maennchen", "weibchen")
        col_zn = find("zuchtnummer")
        col_note = find("bemerkung", "anmerkung")
        # Julian (authoritative): Tabelle1 has an UNLABELED numeric column between
        # Vater and Wurfstärke = "Überlebende bis zum Abgabedatum" (survivors to
        # go-home). Tabelle2 dropped it. Header-based find() cannot see it, so
        # detect it positionally.
        col_survived = None
        if col_sire and col_ws and col_ws - col_sire > 1:
            mapped = {col_id, col_date, col_dam, col_sire, col_ws,
                      col_breakdown, col_zn, col_note}
            for c in range(col_sire + 1, col_ws):
                if c not in mapped:
                    col_survived = c
                    break
        sheet_name = os.path.basename(sp)
        for r in sorted(rows):
            if r == hdr_row:                  # skip the header row itself
                continue
            row = rows[r]
            # skip empty-id + "Jahr YYYY" section rows
            txt_b = row.get(col_date, "") if col_date else ""
            if not row.get(col_id):
                continue
            if "jahr" in " ".join(row.values()).lower() and not DOB.search(txt_b):
                continue
            dob = DOB.search(txt_b)
            lid = row.get(col_id, "")
            datestr = dob.group(1) if dob else ""
            dam_raw = row.get(col_dam, "") if col_dam else ""
            sire_raw = row.get(col_sire, "") if col_sire else ""
            _, dam_zucht = split_name_zucht(dam_raw)
            # '&' = multiple sires possible (litter with uncertain/dual sire)
            sire_names = [s.strip() for s in sire_raw.split("&") if s.strip()]
            _, sire_zucht = split_name_zucht(sire_names[0] if sire_names else "")

            # Numeric layout is read VALUE-ADAPTIVELY per row: a handful of
            # Tabelle2 rows insert an extra numeric column (Überlebende, T1
            # order E,F) before WS and shift the breakdown right of its header.
            warnings = []
            scan_end = (col_note or (col_sire or 4) + 6) + 1
            bd_col = None
            for c in range((col_sire or 4) + 1, scan_end):
                if re.fullmatch(r"\d+(?:[,;]\d+){1,3}", row.get(c, "")):
                    bd_col = c
                    break
            if bd_col and col_breakdown and bd_col != col_breakdown:
                warnings.append(
                    f"Spaltenschema-Abweichung: Geschlechter-Aufschlüsselung in "
                    f"Spalte {xu.num_to_col(bd_col)} statt "
                    f"{xu.num_to_col(col_breakdown)} gefunden")
            # single numeric cells between Vater and the breakdown: E (Überlebende)
            # and/or F (Wurfstärke), in T1 order
            singles = []
            for c in range((col_sire or 4) + 1, bd_col or scan_end):
                v = row.get(c, "")
                if re.fullmatch(r"\d+", v):
                    singles.append(int(v))
            if len(singles) >= 2:
                survived, total_born = singles[0], singles[1]
                if not col_survived:
                    warnings.append(
                        "Spaltenschema-Abweichung: zusätzliche Zahlenspalte als "
                        "„Überlebende“ interpretiert (bitte prüfen)")
            elif len(singles) == 1:
                total_born = singles[0]
                survived = None
            else:
                total_born = _to_int(row.get(col_ws)) if col_ws else None
                survived = _to_int(row.get(col_survived)) if col_survived else None

            # breakdown "2,0,2,0" = Männchen, Weibchen, Totgeburt, später
            # verstorben ('s' = died after birth, before Abgabe — Julian).
            bd = row.get(bd_col, "") if bd_col else ""
            m = re.findall(r"\d+", bd)
            males = females = stillborn = died_later = None
            if m:
                vals = [int(x) for x in m] + [None] * 4
                males, females, stillborn, died_later = vals[:4]
            # Validation (Julian): Überlebende E should equal F − TG − s.
            # Mismatch = data-quality signal for the review report, NOT a blocker.
            if survived is not None and total_born is not None:
                expected = total_born - (stillborn or 0) - (died_later or 0)
                if survived != expected:
                    warnings.append(
                        f"Überlebende ({survived}) ≠ Wurfstärke ({total_born}) "
                        f"− Totgeburten ({stillborn or 0}) − später verstorben "
                        f"({died_later or 0}) = {expected}")
            litters.append({
                "id": f"{sheet_name.replace('.xml','')}-{lid}-{norm_dob(datestr)}",
                "litterId": lid,
                "date": datestr,
                "damName": dam_raw,
                "damZucht": dam_zucht,
                "sireName": sire_raw,
                "sireNames": sire_names,
                "sireZucht": sire_zucht,
                "totalBorn": total_born,
                "survivedToGoHome": survived,
                "males": males,
                "females": females,
                "stillborn": stillborn,
                "diedLater": died_later,
                "breakdownRaw": bd,
                "zuchtnummer": row.get(col_zn, "") if col_zn else "",
                "note": row.get(col_note, "") if col_note else "",
                "warnings": warnings,
                "sourceFile": fname,
                "sheet": sheet_name,
                "row": r,
            })
    return litters


def _to_int(s):
    if not s:
        return None
    m = re.search(r"\d+", s)
    return int(m.group()) if m else None


# ------------------------------------------------------------- stage 2: dedup
def _geno_key(genodict):
    """Canonical, order-independent key of a genotype's mapped loci — used for conflict
    detection so Uw==G (and allele ordering) no longer count as a conflict."""
    m = genodict.get("mapped8locus", {})
    return "|".join(f"{locus}:{','.join(sorted(m[locus]))}" for locus in sorted(m))


# --- "presence wins" merge rule (Julian) -------------------------------------
# When two source variants of the SAME animal differ ONLY by a token PRESENT in one and
# ABSENT in the other — a whole locus (e.g. spsp recorded in one chart, omitted in another)
# or a modifier on the same base allele (e^f vs e, i.e. the [f] marker) — keep the present
# token; that is NOT a conflict. A genuine VALUE contradiction (different filled alleles:
# E vs e, D vs d, c^h vs c^chm) OR unknown-vs-filled (D- vs DD, the '?' second allele) STILL
# quarantines for human decision. (Markers/flags WP/DP/WFNZ/hörend are already tags/flags,
# never part of the genotype, so they never reach here.)
def _split_allele(a):
    return tuple(a.split("^", 1)) if "^" in a else (a, "")


def _alleles_compatible(a, b):
    if a == b:
        return True
    if a == "?" or b == "?":
        return True                        # specific-wins: unknown allele is compatible with any
                                           # specified value (C- vs CC -> CC; G- vs Gg -> Gg)
    (ba, ma), (bb, mb) = _split_allele(a), _split_allele(b)
    if ba != bb:
        return False                       # different base allele = real value diff (E vs e, D vs d)
    return ma == "" or mb == ""            # same base, modifier present-vs-absent -> presence wins


def _pair_compatible(p, q):
    if len(p) != 2 or len(q) != 2:
        return p == q
    return ((_alleles_compatible(p[0], q[0]) and _alleles_compatible(p[1], q[1])) or
            (_alleles_compatible(p[0], q[1]) and _alleles_compatible(p[1], q[0])))


def _genotype_conflict(mapped_list):
    """True only if two variants GENUINELY contradict at a shared locus. A locus present in
    one variant and absent in another is fine (presence wins); so is a modifier present-vs-
    absent on the same base allele. Replaces the old `len(distinct geno keys) > 1` test."""
    loci = set()
    for m in mapped_list:
        loci.update(m.keys())
    for locus in loci:
        pairs = [m[locus] for m in mapped_list if locus in m]
        for i in range(len(pairs)):
            for j in range(i + 1, len(pairs)):
                if not _pair_compatible(pairs[i], pairs[j]):
                    return True
    return False


def dedup(animals):
    """Merge by normalise(call-name)+DOB, with the canonical Zucht as
    DISCRIMINATOR (Julian: same name+DOB+Zucht = same animal; different Zucht =
    different animal). Returns (merged, conflicts, orphans, zucht_splits)."""
    groups = {}
    orphans = []
    for a in animals:
        call, _ = split_name_zucht(a["name"])
        key = (norm_name(call), norm_dob(a["dob"]))
        if not key[0] or not key[1]:
            orphans.append(a)
            # orphans still get a stable id but are not merged
            key = ("__orphan__", id(a))
        groups.setdefault(key, []).append(a)

    # split groups whose members carry DIFFERENT (non-empty) canonical Zuchten;
    # members without a Zucht merge into the group only if it is unambiguous.
    final_groups = []
    zucht_splits = []
    for key, grp in groups.items():
        by_zucht = {}
        for a in grp:
            by_zucht.setdefault(a.get("_zucht", ""), []).append(a)
        nonempty = sorted(z for z in by_zucht if z)
        if len(nonempty) <= 1:
            final_groups.append(grp)
            continue
        # genuine split: same call-name+DOB, different Zucht
        for z in nonempty:
            final_groups.append(by_zucht[z])
        if "" in by_zucht:
            # Zucht-less mentions cannot be attributed -> own (flagged) entry
            final_groups.append(by_zucht[""])
        zucht_splits.append({
            "name": grp[0]["name"],
            "dob": norm_dob(grp[0]["dob"]),
            "zuechte": sorted(set(a["zucht"] for a in grp if a.get("zucht"))),
            "files": sorted(set(f for a in grp for f in a["sourceFiles"])),
        })

    # 2. Merge groups that have different DOBs but same call-name (and Zucht) and matching parents
    def get_parent_keys(a):
        parent_refs = a.get("parentRefs", [])
        f_name = next((p["name"] for p in parent_refs if p.get("roleGuess") == "father"), "")
        m_name = next((p["name"] for p in parent_refs if p.get("roleGuess") == "mother"), "")
        return norm_name(f_name), norm_name(m_name)

    def groups_parents_match(g1, g2):
        for a1 in g1:
            for a2 in g2:
                f1, m1 = get_parent_keys(a1)
                f2, m2 = get_parent_keys(a2)
                if f1 and f2 and f1 == f2 and m1 and m2 and m1 == m2:
                    return True
        return False

    i = 0
    while i < len(final_groups):
        g1 = final_groups[i]
        call1, _ = split_name_zucht(g1[0]["name"])
        n_call1 = norm_name(call1)
        zucht1 = g1[0].get("_zucht", "")
        
        j = i + 1
        merged_any = False
        while j < len(final_groups):
            g2 = final_groups[j]
            call2, _ = split_name_zucht(g2[0]["name"])
            n_call2 = norm_name(call2)
            zucht2 = g2[0].get("_zucht", "")
            
            if n_call1 == n_call2:
                if zucht1 == zucht2 or not zucht1 or not zucht2:
                    if groups_parents_match(g1, g2):
                        g1.extend(g2)
                        final_groups.pop(j)
                        merged_any = True
                        continue
            j += 1
        if merged_any:
            continue
        i += 1

    merged = []
    conflicts = []
    for grp in final_groups:
        base = dict(grp[0])
        variants = set([base["name"]])
        files = set(base["sourceFiles"])
        photos = list(base["photos"])
        parent_refs = list(base["parentRefs"])
        genos = set()
        geno_keys = set()   # GEN-3b: conflict on NORMALIZED genotype (Uw==G) not raw text
        mapped_variants = []  # mapped8locus per variant — for the 'presence wins' conflict test
        farb = set()
        deaths = set()
        deaf_seen = set()
        tags_set = set()
        genders = []
        for a in grp:
            variants.add(a["name"])
            files.update(a["sourceFiles"])
            photos.extend(a["photos"])
            parent_refs.extend(a["parentRefs"])
            if a.get("gender"):
                genders.append(a["gender"])
            if a["genotype"]["mapped8locus"]:
                genos.add(a["genotype"]["rawGenotype"])
                geno_keys.add(_geno_key(a["genotype"]))
                mapped_variants.append(a["genotype"]["mapped8locus"])
            if a["farbschlag"]:
                farb.add(a["farbschlag"])
            if a["death"]:
                deaths.add(norm_dob(a["death"]))
            if a.get("deaf") is not None:
                deaf_seen.add(a["deaf"])
            tags_set.update(a.get("tags", []))
        # pick the richest genotype: most mapped loci, then fewest unknowns ('?' alleles = specific
        # wins, FIX-2), then longest raw string as final tiebreaker.
        def _specificity(gd):
            return sum(1 for pair in gd["mapped8locus"].values() for a in pair if a != "?")
        best = max((a["genotype"] for a in grp),
                   key=lambda gd: (len(gd["mapped8locus"]), _specificity(gd), len(gd["rawGenotype"])))
        
        # Find best DOB (proband first)
        best_dob = ""
        for a in grp:
            if a.get("_gen") == 0 and a.get("dob"):
                best_dob = a["dob"]
                break
        if not best_dob:
            for a in grp:
                if a.get("dob"):
                    best_dob = a["dob"]
                    break
        chosen_dob = norm_dob(best_dob or base["dob"])

        out = {
            "id": slug(base["name"], chosen_dob),
            "name": base["name"],
            "nameVariants": sorted(v for v in variants if v),
            "dob": chosen_dob,
            "death": sorted(deaths)[0] if deaths else "",
            # box-colour sex (blue=male, white=female): majority across mentions, else None.
            "gender": Counter(genders).most_common(1)[0][0] if genders else None,
            "farbschlag": sorted(farb)[0] if farb else "",
            "farbschlagVariants": sorted(farb),
            "genotype": best,
            "breeder": next((a["breeder"] for a in grp if a["breeder"]), ""),
            "zucht": next((a["zucht"] for a in grp if a.get("zucht")), ""),
            "zuchtCanon": next((a["_zucht"] for a in grp if a.get("_zucht")), ""),
            "parentRefs": _dedup_parentrefs(parent_refs),
            "photos": sorted(set(photos)),
            "sourceFiles": sorted(files),
            "mentions": len(grp),
            # GEN-3b: hearing/deaf phenotype flag (deaf wins if any mention says so) + tags.
            "deaf": (True if True in deaf_seen else (False if False in deaf_seen else None)),
            "tags": sorted(tags_set),
            # FEAT-8c: machine-readable quarantine marker so the API loader can skip
            # conflicting records without parsing the German review report.
            "conflict": False,
            # DOB-Remap (manuelle Entscheidung): original/corrected birthdate so the
            # downstream Datenherkunft can record the discarded original value.
            "dobRemap": next((a["dobRemap"] for a in grp if a.get("dobRemap")), None),
        }
        merged.append(out)
        # conflict: same animal, GENUINELY disagreeing genotype (presence-vs-absence is NOT a
        # conflict — Julian's 'presence wins') or >1 distinct farbschlag or >1 distinct death.
        if _genotype_conflict(mapped_variants) or len(farb) > 1 or len(deaths) > 1:
            out["conflict"] = True
            conflicts.append({
                "id": out["id"], "name": base["name"], "dob": out["dob"],
                "genotypes": sorted(genos), "farbschlaege": sorted(farb),
                "deaths": sorted(deaths), "files": sorted(files),
            })
    merged.sort(key=lambda a: (a["dob"], a["name"]))
    # ids must stay unique even when a Zucht split duplicates name+DOB slugs
    seen_ids = {}
    for a in merged:
        if a["id"] in seen_ids:
            seen_ids[a["id"]] += 1
            a["id"] = f"{a['id']}-{seen_ids[a['id']]}"
        else:
            seen_ids[a["id"]] = 1
    return merged, conflicts, orphans, zucht_splits


# ------------------------------------------- animal <-> litter matching ------
def match_litters(merged, litters):
    """Attach each animal to its Wurfchronik litter (Pam-validated build order:
    litters are canonical, animals match onto them via DOB + (Vater, Mutter)).
    Sets a['litterRef']; returns match statistics for the report."""
    by_date = {}
    for l in litters:
        d = norm_dob(l["date"])
        if d:
            by_date.setdefault(d, []).append(l)

    stats = {"parents": 0, "dateOnly": 0, "ambiguous": 0}
    for a in merged:
        if not a["dob"]:
            continue
        cands = by_date.get(a["dob"])
        if not cands:
            continue
        a_parents = set()
        for ref in a["parentRefs"]:
            cn, _ = canon_pair(ref["name"])
            if cn:
                a_parents.add(cn)

        def score(l):
            s = 0
            for nm in [l["damName"]] + l.get("sireNames", []):
                cn, _ = canon_pair(nm)
                if cn and cn in a_parents:
                    s += 1
            return s

        scored = sorted(((score(l), l["id"]) for l in cands), reverse=True)
        best_score, best_id = scored[0]
        if best_score > 0 and (len(scored) == 1 or scored[1][0] < best_score):
            a["litterRef"] = {"litterId": best_id,
                              "method": "geburtsdatum+eltern", "confidence": "hoch"}
            stats["parents"] += 1
        elif len(cands) == 1:
            a["litterRef"] = {"litterId": cands[0]["id"],
                              "method": "nur-geburtsdatum", "confidence": "niedrig"}
            stats["dateOnly"] += 1
        else:
            a["litterRef"] = {"litterId": None, "method": "mehrdeutig",
                              "candidates": [l["id"] for l in cands]}
            stats["ambiguous"] += 1
    return stats


def _dedup_parentrefs(refs):
    seen = {}
    for r in refs:
        k = (norm_name(r["name"]), r["roleGuess"])
        if k not in seen:
            seen[k] = r
    return list(seen.values())


# ------------------------------------------------------------------ reporting
def write_report(merged, conflicts, orphans, raw_count, litters, photo_count,
                 zucht_splits, match_stats):
    multi = [a for a in merged if a["mentions"] > 1]
    with_dob = [a for a in merged if a["dob"]]
    val_warn = [l for l in litters if any("≠" in w for w in l["warnings"])]
    schema_warn = [l for l in litters if any("Abweichung" in w for w in l["warnings"])]
    matched = match_stats["parents"] + match_stats["dateOnly"]

    L = []
    L.append("# FEAT-8b — Import-Vorschau & Prüfbericht (Stammbäume + Wurfchronik)\n")
    L.append("_Automatisch erzeugt von `tools/import/extract.py` — **noch nichts in die Datenbank geladen.** "
             "Bitte prüfen, bevor importiert wird._\n")
    L.append("## Überblick\n")
    L.append(f"- Rohe Tier-Einträge aus den Stammbäumen: **{raw_count}**")
    L.append(f"- Nach Zusammenführung (eindeutige Tiere): **{len(merged)}**")
    L.append(f"  - davon mit Geburtsdatum: {len(with_dob)}")
    L.append(f"  - in mehreren Dateien gefunden (Dubletten zusammengeführt): {len(multi)}")
    L.append(f"- Konflikte zur Klärung: **{len(conflicts)}**")
    L.append(f"- Mehrdeutige / unvollständige Einträge (ohne Name+Datum): **{len(orphans)}**")
    L.append(f"- Fotos zugeordnet: **{photo_count}**")
    L.append(f"- Würfe aus der Wurfchronik: **{len(litters)}**")
    L.append(f"  - Tiere mit Wurf verknüpft: **{matched}** "
             f"(davon über Geburtsdatum **und** Eltern: {match_stats['parents']}, "
             f"nur über Geburtsdatum: {match_stats['dateOnly']}; "
             f"mehrdeutig: {match_stats['ambiguous']})")
    L.append(f"  - Würfe mit Datenqualitäts-Hinweisen: {len(val_warn)} "
             f"(+ {len(schema_warn)} Zeilen mit abweichendem Spaltenschema)\n")

    L.append("## Zusammenführungs-Schlüssel\n")
    L.append("Tiere wurden zusammengeführt über **normalisierter Rufname + Geburtsdatum**, "
             "mit der **Zucht als Unterscheidungsmerkmal** (Julians Regel: die `[Klammern]` "
             "in der Wurfchronik und das `of/von <Linie>`-Suffix der Stammbäume bezeichnen "
             "beide die Zucht und werden zusammengeführt — z. B. `[ZdkC]` ≙ "
             "`von den Kleinen Chaoten`). Namensvarianten (z. B. `v.d.` ↔ `von den`, "
             "`gen.`-Spitznamen) werden als `nameVariants` erhalten.\n")

    if zucht_splits:
        L.append("### Gleicher Name + Geburtsdatum, aber unterschiedliche Zucht "
                 "(NICHT zusammengeführt — bitte prüfen)\n")
        L.append("| Tier | Geburtsdatum | Zuchten | Dateien |")
        L.append("|---|---|---|---|")
        for s in zucht_splits[:50]:
            L.append("| {} | {} | {} | {} |".format(
                split_name_zucht(s["name"])[0], s["dob"],
                " // ".join(s["zuechte"]),
                ", ".join(os.path.splitext(f)[0] for f in s["files"])))
        L.append("")

    L.append("## ⚠️ Konflikte (bitte prüfen)\n")
    if conflicts:
        L.append("Gleiches Tier (Name+Datum), aber widersprüchliche Angaben in verschiedenen Dateien:\n")
        L.append("| Tier | Geburtsdatum | abweichende Genotypen | abweichende Farbschläge | Sterbedaten | Dateien |")
        L.append("|---|---|---|---|---|---|")
        for c in conflicts[:200]:
            L.append("| {} | {} | {} | {} | {} | {} |".format(
                c["name"], c["dob"],
                " // ".join(c["genotypes"]) or "—",
                " // ".join(c["farbschlaege"]) or "—",
                " // ".join(c["deaths"]) or "—",
                ", ".join(os.path.splitext(f)[0] for f in c["files"])))
    else:
        L.append("_Keine._\n")

    L.append("\n## Mehrdeutige / unvollständige Einträge\n")
    L.append(f"{len(orphans)} Einträge ohne sichere Name+Datum-Kombination "
             "(z. B. `Name1 & Name2`-Paarzellen der tiefsten Generation, oder Zellen ohne Datum). "
             "Diese werden NICHT automatisch zusammengeführt.\n")
    sample = [o for o in orphans if o["name"]][:40]
    for o in sample:
        L.append(f"- {o['name']}  ·  {o.get('_file','')}")

    # orphan -> likely same-named full record (soft hint, not auto-merged)
    from collections import Counter
    name_index = {}
    for a in merged:
        if a["dob"]:
            name_index.setdefault(norm_name(a["name"]), []).append(a)
    matchable = []
    for o in orphans:
        if not o["name"]:
            continue
        cands = name_index.get(norm_name(o["name"]))
        if cands:
            matchable.append((o, cands))
    L.append("\n## Wahrscheinliche Zuordnungen unvollständiger Einträge\n")
    L.append(f"{len(matchable)} namenlose/datenlose Einträge tragen denselben Namen wie ein "
             "vollständiges Tier — vermutlich dasselbe Tier (zur Bestätigung):\n")
    for o, cands in matchable[:60]:
        opts = "; ".join(f"{c['name']} (*{c['dob']})" for c in cands[:3])
        L.append(f"- „{o['name']}“  →  {opts}")

    # unmapped-token summary (for Kevin / GEN-2 + the wife)
    tok = Counter()
    for a in merged:
        for t in a["genotype"]["unmappedTokens"]:
            tok[t] += 1
    L.append("\n## Nicht ins 8-Loci-Modell abgebildete Tokens (verbatim erhalten)\n")
    L.append("Diese Tokens stehen weiter in `rawGenotype`/`unmappedTokens` — Entscheidung "
             "(Modell erweitern vs. als Notiz) liegt bei Julian/Kevin:\n")
    L.append("| Token | Vorkommen | Bedeutung (Vermutung) |")
    L.append("|---|---|---|")
    hint = {"Uwuw[d]": "9. Locus Uw (nicht im Modell)", "UwUw": "9. Locus Uw",
            "[WFNZ]": "Marker", "[DP]": "Marker (Dunkelpigment?)", "DP": "Marker",
            "WP": "Marker", "[WP]": "Marker", "C(C)": "Schreibweise (C trägt c)",
            "chmchm": "Schreibweise (c[chm]c[chm])"}
    for t, n in tok.most_common(25):
        L.append(f"| `{t}` | {n} | {hint.get(t, '?')} |")

    # Wurfchronik data quality (Julian: E sollte = F − TG − s sein)
    L.append("\n## Wurfchronik — Datenqualitäts-Hinweise\n")
    L.append("Julians Spaltenregel: **Überlebende bis Abgabe (E) = Wurfstärke (F) − "
             "Totgeburten (TG) − später verstorben (s)**. Bei diesen Würfen geht die "
             "Rechnung nicht auf — kein Import-Hindernis, aber ein Hinweis auf "
             "Tippfehler oder fehlende Einträge:\n")
    if val_warn:
        L.append("| Wurf | Datum | Mutter × Vater | Hinweis |")
        L.append("|---|---|---|---|")
        for l in val_warn[:120]:
            L.append("| {} | {} | {} × {} | {} |".format(
                l["litterId"], l["date"], l["damName"], l["sireName"],
                "; ".join(w for w in l["warnings"] if "≠" in w)))
        if len(val_warn) > 120:
            L.append(f"\n… und {len(val_warn) - 120} weitere (siehe `litters.json`).")
    else:
        L.append("_Keine — alle Würfe sind rechnerisch konsistent._")

    L.append("\n### Zeilen mit abweichendem Spaltenschema (automatisch interpretiert)\n")
    L.append(f"{len(schema_warn)} Zeilen (überwiegend Tabelle2 ab 2014) tragen eine "
             "zusätzliche Zahlenspalte vor der Wurfstärke bzw. eine verschobene "
             "Geschlechter-Aufschlüsselung. Sie wurden nach dem Muster von Tabelle1 "
             "gelesen (**Überlebende, Wurfstärke, Aufschlüsselung**) — bei "
             f"{sum(1 for l in schema_warn if not any('≠' in w for w in l['warnings']))} "
             "davon geht die Rechnung E = F − TG − s damit exakt auf, was die Lesart "
             "bestätigt. Alle betroffenen Zeilen sind in `litters.json` mit "
             "`warnings` markiert. Beispiele:\n")
    for l in schema_warn[:8]:
        L.append(f"- Wurf {l['litterId']} ({l['date']}): E={l['survivedToGoHome']}, "
                 f"F={l['totalBorn']}, Aufschlüsselung `{l['breakdownRaw']}`")

    L.append("\n## Hinweise für den Import (Stufe 3, später)\n")
    L.append("- **Wurfchronik = Quelle der Würfe** (Datum, Wurfstärke, Eltern, Zuchtnummer); "
             "**Stammbäume = Abstammung + Genotyp + Fotos**. Verknüpfung über Geburtsdatum + Elternnamen.")
    L.append("- Eltern-Verknüpfungen (`parentRefs`) stammen aus der **Position im Stammbaum** "
             "(Vater oben / Mutter unten, mittlere Konfidenz) — die Wurfchronik korrigiert dies maßgeblich.")
    L.append("- Genotyp: `mapped8locus` (A C D E G P Sp Re), `rawGenotype` (wortgetreu), "
             "`unmappedTokens` (z. B. `Uw`, `Sls`, `Dea`, Marker wie `WFNZ/WP/DP`) — **nichts geht verloren**.")
    L.append("- `-` (unbekanntes zweites Allel) → `?` (Platzhalter; Annahme, bitte bestätigen).")
    with open(os.path.join(OUT, "review-report.md"), "w", encoding="utf-8") as f:
        f.write("\n".join(L) + "\n")


# ------------------------------------------------------------------------ main
def apply_dob_remaps(raw_animals, path):
    """PRE-dedup: a conflict-decision carrying `correctDob` marks a record as a DUPLICATE with a
    wrong birthdate — remap that raw record's DOB to correctDob so dedup MERGES it into the
    canonical same-named animal (e.g. Chelsea *15.10.2021 -> *02.04.2021). Match =
    canon_pair(name)+(dob) with same Zucht-aware logic as apply_conflict_decisions (see there).
    Tolerates a missing/garbled file. Returns the remap count.
    Must run BEFORE dedup (it changes the dedup identity). (god/HUMANQUESTION D — Dubletten.)"""
    remaps_full = {}   # (nameCanon, zuchtCanon, dob) -> correctDob  — decision carries Zucht
    remaps_name = {}   # (nameCanon, dob) -> correctDob              — no Zucht in decision
    try:
        with open(path, encoding="utf-8") as fh:
            for r in (json.load(fh).get("resolutions") or []):
                if r.get("correctDob"):
                    nc, zc = canon_pair(r.get("name", ""))
                    dob = norm_dob(r.get("dob", ""))
                    if zc:
                        remaps_full[(nc, zc, dob)] = r["correctDob"]
                    else:
                        remaps_name[(nc, dob)] = r["correctDob"]
    except (OSError, ValueError):
        return 0
    if not remaps_full and not remaps_name:
        return 0
    n = 0
    for a in raw_animals:
        nc, zc = canon_pair(a.get("name", ""))
        dob = norm_dob(a.get("dob", ""))
        new = remaps_full.get((nc, zc, dob)) or remaps_name.get((nc, dob))
        if new and a.get("dob") != new:
            # Keep the discarded original DOB so the provenance/Datenherkunft can
            # show „Geburtsdatum X verworfen, per Entscheidung auf Y geändert.“
            a["dobRemap"] = {"original": a.get("dob"), "corrected": new}
            a["dob"] = new
            n += 1
    return n


def apply_conflict_decisions(merged, conflicts, path):
    """Consume human conflict resolutions (tools/import/conflict-decisions.json) so the wife's
    answers UN-QUARANTINE animals. Schema: {"resolutions":[{name, dob, decision, genotype?,
    farbschlag?, source}]}. Match = canon_pair(name)+(dob):
      - When the decision name CARRIES a Zucht (zuchtCanon != ''), match on the FULL
        (nameCanon, zuchtCanon, dob) triple — preserves the C3 rule that same name+DOB but
        different Zucht = different animal.
      - When the decision has NO Zucht, fall back to (nameCanon, dob) name-only match.
    Both spellings v.d. / von den fold to the same canon. A matching animal: clear its conflict,
    mark resolvedByDecision; an explicit `genotype` (breeder notation) is parsed and becomes
    authoritative, `farbschlag` overrides too. Tolerates a missing/empty/garbled file.
    Returns the number of conflicts resolved. (god/HUMANQUESTION D.)"""
    decisions_full = {}   # (nameCanon, zuchtCanon, dob) -> r  — when decision carries a Zucht
    decisions_name = {}   # (nameCanon, dob) -> r              — fallback, decision has no Zucht
    decisions_ref = {}    # externalRef (merged-animal id) -> r — for NAMELESS animals whose
                          # (name="" + dob) key is shared by several records: the externalRef
                          # (the dedup slug, e.g. „unbekannt-13082025-3") pins exactly one.
    try:
        with open(path, encoding="utf-8") as fh:
            for r in (json.load(fh).get("resolutions") or []):
                ref = r.get("externalRef")
                if ref:
                    decisions_ref[ref] = r
                # An externalRef-only decision (no name) must NOT register a name/dob
                # key — a ("", "") key would match every nameless, dateless animal.
                if not r.get("name"):
                    continue
                nc, zc = canon_pair(r.get("name", ""))
                dob = norm_dob(r.get("dob", ""))
                if zc:
                    decisions_full[(nc, zc, dob)] = r
                else:
                    decisions_name[(nc, dob)] = r
    except (OSError, ValueError):
        return 0
    if not decisions_full and not decisions_name and not decisions_ref:
        return 0

    resolved = 0
    for a in merged:
        nc, zc = canon_pair(a["name"])
        dob = norm_dob(a["dob"])
        # externalRef (the dedup id) wins — it is the most specific key and the only
        # way to address one of several same-(name,dob) nameless animals.
        d = decisions_ref.get(a.get("id")) or decisions_full.get((nc, zc, dob)) \
            or decisions_name.get((nc, dob))
        if not d:
            continue
        a["resolvedByDecision"] = True
        if d.get("genotype"):
            # CR-10: validate the parsed genotype — a typo'd decision string yields empty
            # mapped8locus and would silently blank the animal's genotype while marking it
            # 'resolved'. Only apply if the parse produces non-empty loci.
            parsed = gt.parse(d["genotype"])
            if parsed.get("mapped8locus"):
                a["genotype"] = parsed
            else:
                # Keep the existing genotype; flag as a warning in the report.
                a.setdefault("decisionWarnings", []).append(
                    f"Ungültiger Override-Genotyp '{d['genotype']}' — "
                    "konnte nicht geparst werden (mapped8locus leer). "
                    "Bestehender Genotyp behalten; Konflikt wurde trotzdem aufgelöst."
                )
        if d.get("farbschlag"):
            a["farbschlag"] = d["farbschlag"]
            a["farbschlagVariants"] = [d["farbschlag"]]
        if d.get("dateOfDeath"):                       # D5 death-date resolutions
            a["death"] = norm_dob(d["dateOfDeath"])
        if d.get("gender"):                            # gender override (box colour misread)
            g = d["gender"].strip().lower()
            g = {"m": "male", "männlich": "male", "w": "female",
                 "f": "female", "weiblich": "female"}.get(g, g)
            if g in ("male", "female"):
                a["gender"] = g
        if "father" in d or "mother" in d:
            new_refs = []
            if d.get("father"):
                new_refs.append({
                    "name": d["father"],
                    # Optional fatherDob disambiguates a parent when several
                    # same-named animals exist (the downstream resolver matches
                    # name AND, when given, dob — e.g. two „Ella“).
                    "dob": d.get("fatherDob", ""),
                    "roleGuess": "father",
                    "method": "decision",
                    "confidence": "high"
                })
            if d.get("mother"):
                new_refs.append({
                    "name": d["mother"],
                    "dob": d.get("motherDob", ""),
                    "roleGuess": "mother",
                    "method": "decision",
                    "confidence": "high"
                })
            a["parentRefs"] = new_refs
        if a.get("conflict"):
            a["conflict"] = False
            conflicts[:] = [c for c in conflicts if c.get("id") != a["id"]]
            resolved += 1
    return resolved


def main():
    try:
        sys.stdout.reconfigure(encoding="utf-8", errors="replace")
    except Exception:
        pass
    ap = argparse.ArgumentParser(description="FEAT-8b extractor (stages 1-2)")
    ap.add_argument("--stammbaeume", default=DEFAULT_STAMMBAEUME)
    ap.add_argument("--wurfchronik", default=DEFAULT_WURFCHRONIK)
    args = ap.parse_args()

    if os.path.isdir(OUT):
        # keep the report stable across reruns but refresh data/photos
        for sub in ("photos",):
            p = os.path.join(OUT, sub)
            if os.path.isdir(p):
                shutil.rmtree(p)
    os.makedirs(OUT, exist_ok=True)

    raw_animals = []
    # Skip Excel lock/owner files ("~$...") that appear while a workbook is open.
    files = sorted(f for f in glob.glob(os.path.join(args.stammbaeume, "*.xlsx"))
                   if not os.path.basename(f).startswith("~$"))
    print(f"Stammbaum-Dateien: {len(files)}")
    for path in files:
        got = extract_stammbaum(path)
        print(f"  {len(got):4d}  {os.path.basename(path)}")
        raw_animals.extend(got)

    # Skip pseudo-animal records (like "DD-Tumor bei Geschwister") that are actually notes
    raw_animals = [a for a in raw_animals if "DD-Tumor" not in a["name"]]

    litters = []
    if os.path.isfile(args.wurfchronik):
        litters = extract_wurfchronik(args.wurfchronik)
        print(f"Wurfchronik: {len(litters)} Würfe")

    decisions_path = os.path.join(HERE, "conflict-decisions.json")
    dob_remaps = apply_dob_remaps(raw_animals, decisions_path)   # before dedup (changes identity)
    merged, conflicts, orphans, zucht_splits = dedup(raw_animals)
    resolved_by_decision = apply_conflict_decisions(merged, conflicts, decisions_path)
    match_stats = match_litters(merged, litters)
    photo_count = sum(len(a["photos"]) for a in merged)

    # strip private (_) fields from the JSON output
    def clean(a):
        return {k: v for k, v in a.items() if not k.startswith("_")}

    with open(os.path.join(OUT, "animals.json"), "w", encoding="utf-8") as f:
        json.dump([clean(a) for a in merged], f, ensure_ascii=False, indent=2)
    with open(os.path.join(OUT, "litters.json"), "w", encoding="utf-8") as f:
        json.dump(litters, f, ensure_ascii=False, indent=2)

    write_report(merged, conflicts, orphans, len(raw_animals), litters, photo_count,
                 zucht_splits, match_stats)

    print(f"\nRoh: {len(raw_animals)}  →  eindeutig: {len(merged)}  "
          f"| Konflikte: {len(conflicts)}  | per Entscheidung gelöst: {resolved_by_decision}  "
          f"| DOB-Remaps: {dob_remaps}  | Zucht-Splits: {len(zucht_splits)}  "
          f"| Orphans: {len(orphans)}  | Fotos: {photo_count}")
    print(f"Wurf-Verknüpfung: {match_stats['parents']} (Datum+Eltern), "
          f"{match_stats['dateOnly']} (nur Datum), {match_stats['ambiguous']} mehrdeutig "
          f"| Wurf-Warnungen: {sum(1 for l in litters if l['warnings'])}")
    print(f"Ausgabe in {OUT}")


if __name__ == "__main__":
    main()