GerbilManager/tools/import/merge_and_resolve.py

import os
import json
import re
import uuid
import sys
from datetime import datetime

# Prevent encoding crashes on Windows consoles when printing unicode
if sys.platform.startswith('win'):
    try:
        sys.stdout.reconfigure(encoding='utf-8')
    except Exception:
        pass

# Paths
DIR_PATH = r"C:\Users\gulum\dev\Wurfchronik_Bilder"
SEEDS_PATH = r"C:\Users\gulum\dev\GerbilManager\gerbil-manager-web\src\genetics\colorVarietySeed.backend.json"
OUTPUT_DIR = r"C:\Users\gulum\dev\GerbilManager\tools\import\output"
OUTPUT_FILE = os.path.join(OUTPUT_DIR, "resolved_import.json")

def generate_guid(key_str):
    """Generate a stable UUID string based on a key."""
    return str(uuid.uuid5(uuid.NAMESPACE_DNS, key_str))

def to_valid_guid(val):
    if not val:
        return None
    val_str = str(val).strip()
    try:
        uuid.UUID(val_str)
        return val_str
    except ValueError:
        return generate_guid(val_str)

def normalize_name(name):
    if not name:
        return ""
    return "".join(c for c in name.lower() if c.isalnum())

def get_normalized_contact_name(name):
    if not name:
        return "", False
    
    n = "".join(c for c in name.lower() if c.isalnum())
    
    to_discard = {
        "chevroletcamarooftopolino",
        "cindyvprivatzuchtgießen",
        "cindyvprivatzuchtgiessen",
        "inuschofblackforest",
        "stichvonprivatzuchtgießen",
        "stichvonprivatzuchtgiessen",
        "tarzanofsamsimar",
        "hanserennersposeidon",
        "livingforcesidefix",
        "livingforcesnando"
    }
    if n in to_discard:
        return "", False
        
    norm_map = {
        "alessandrab": "Alessandra Bartoletti",
        "alexsandrab": "Alessandra Bartoletti",
        "andreafey": "Andrea und Stefanie Fey",
        "andreastefaniefey": "Andrea und Stefanie Fey",
        "anettw": "Annett Wernecke",
        "angelachristoph": "Angela und Ekki Christoph",
        "angelaekkichristoph": "Angela und Ekki Christoph",
        "angie": "Angie Reichert-Cambeis",
        "angierc": "Angie Reichert-Cambeis",
        "anjasch": "Anja Schaumburg",
        "ankeb": "Anke Busch",
        "ankeksch": "Anke Koppenhöfer",
        "annaq": "Anna Quark",
        "annikab": "Annika Balser",
        "annkathrind": "Ann-kathrin Dressler",
        "arturaleksandrapolamundrzynski": "Artur, Aleksandra und Pola Mundrzynski",
        "arturaleksandraundpolamundrzynski": "Artur, Aleksandra und Pola Mundrzynski",
        "astridr": "Astrid Rohmann",
        "astridroh": "Astrid Rohmann",
        "babarastehle": "Barbara Stehle",
        "barbarar": "Barbara Riegler",
        "biancab": "Bianca Bernhardt",
        "biancam": "Bianca Mayer + Alex Wachten",
        "biancamayeralexw": "Bianca Mayer + Alex Wachten",
        "birgitk": "Birgit Knopp",
        "birgitkropp": "Birgit Knopp",
        "birgittabüskens": "Birgitta Müller-Büskens",
        "birgittamüllerbüskens": "Brigitta Müller-Büskens",
        "blackforestgv": "Black Forest",
        "brigittast": "Brigitta Struve",
        "buntefellnasen": "bunten Fellnasen",
        "buntenfellnase": "bunten Fellnasen",
        "buntenfellnasen": "bunten Fellnasen",
        "carolal": "Carola und Hannah Lerch",
        "carolalerch": "Carola und Hannah Lerch",
        "chalfontstud": "Chalfont Stud, Freddy Braun",
        "christianefuchs": "Christiane Fuchs + Michael Höhnert",
        "christiek": "Christiane Kiessling",
        "christinam": "Christine Monika Mai",
        "christinek": "Christiane Kiessling",
        "christiner": "Christina Rudloff",
        "claudialöhr": "Claudia Löhr und Michael Koob",
        "claudialöhrmichaelkoob": "Claudia Löhr und Michael Koob",
        "colorfulfurrygerbils": "Colorful Furry Gerbils",
        "colourfulfurrygerbils": "Colorful Furry Gerbils",
        "danielakasberger": "Daniela Käsberger",
        "dörthe": "Dörthe Petzmann",
        "emelyhehlhorn": "Emely Mehlhorn",
        "fabianb": "Fabian Büdel",
        "floriang": "Florian Gries",
        "hannahraths": "Hannah Ruths",
        "hanserenners": "Hanse Renner",
        "haraldmariareitzlennemann": "Harald und Maria Reitz-Lennemann",
        "haraldundmariarl": "Harald und Maria Reitz-Lennemann",
        "heike": "Heike + Heiko Scheurich",
        "heikebarklay": "Heike Barklage",
        "heikoloos": "Heiko Loos und Annette Becker",
        "heikoloosannettebecker": "Heiko Loos und Annette Becker",
        "inahübner": "Ina Hübner + Nicolai Thome",
        "inapeines": "Ina Pleines",
        "janab": "Jana Beikert",
        "jasiminweber": "Jasmin Weber und Ronny Reichelt",
        "jasmin": "Jasmin Weber",
        "jasminweber": "Jasmin Weber und Ronny Reichelt",
        "jasminweberromyreichelt": "Jasmin Weber und Ronny Reichelt",
        "jasminweberronnyreichelt": "Jasmin Weber und Ronny Reichelt",
        "joystaatberg": "Joy Staalberg",
        "juliaa": "Julia Ast",
        "juttam": "Jutta Metz",
        "karina": "Karina Luft",
        "karinborgotti": "Karin Borsotti",
        "katjal": "Katja Leffeck",
        "kimw": "Kim Waldschmidt",
        "kkchaos": "KK-Chaos",
        "kkchaosofkkchaos": "KK-Chaos",
        "kleinechaoten": "Zucht der kleinen Chaoten",
        "kleinenchaoten": "Zucht der kleinen Chaoten",
        "kriegernmitkrallen": "Krieger mit Krallen",
        "lennylengo": "Lenny Lengo",
        "lisa": "Lisa und Marcel Kunz",
        "lisach": "Lisa und Lydia Christ",
        "lisachrist": "Lisa und Lydia Christ",
        "lisalydiachrist": "Lisa und Lydia Christ",
        "littlefellows": "little fellows",
        "littlerunners": "little runners",
        "maikef": "Maike Franz",
        "maintalerpz": "Privatzucht Maintal",
        "marinau": "Marina Unger",
        "martinaandreaswestfeld": "Martina und Andreas Westfeld",
        "martinaw": "Martina und Andreas Westfeld",
        "martinawestfeld": "Martina und Andreas Westfeld",
        "meikesch": "Meike und Heiko Sch.",
        "melanief": "Melanie Fey und Thorben Meier",
        "melaniefey": "Melanie Fey und Thorben Meier",
        "melaniefeythorbenmeier": "Melanie Fey und Thorben Meier",
        "michaeldavidschmitz": "Michael und David Schmitz",
        "michaelschmitz": "Michael und David Schmitz",
        "nataliereitz": "Nathalie Reitz",
        "natascham": "Natascha Marienfeld",
        "nicolefischler": "Nicole Tischler",
        "nicolel": "Nicole Lannert",
        "nicolen": "Nicole Nuzzo",
        "nicolet": "Nicole Tischler",
        "nicolew": "Nicole Webersinn",
        "nicost": "Nico Stamm",
        "nielsh": "Nils H.",
        "noelstrahbach": "Noel Strohbach",
        "nora": "Nora Rudersdorf geb. Holzbach",
        "norah": "Nora Holzbach",
        "norarh": "Nora Rudersdorf",
        "norarudersdorf": "Nora Rudersdorf geb. Holzbach",
        "oflennylengo": "Lenny Lengo",
        "pascale": "Pascale Diefenbach",
        "pascaledießenbach": "Pascale Diefenbach",
        "patriciap": "Patricia Petry",
        "paul": "Paul W.",
        "paula": "Paula Gabler",
        "paulag": "Paula Gabler",
        "privatzuchtmuecke": "Privatzucht Mücke",
        "privatzuchtmücke": "Privatzucht Mücke",
        "pzmaintal": "Privatzucht Maintal",
        "pzmuecke": "Privatzucht Mücke",
        "pzmücke": "Privatzucht Mücke",
        "pzseligenstadt": "Privatzucht Seligenstadt",
        "ramonag": "Ramona Gömpel",
        "ranialößler": "Rania Löffler",
        "rominahubrich": "Romina Milde / Hubrich",
        "rominamilde": "Romina Milde / Hubrich",
        "romonag": "Ramona Gömpel",
        "ronialöffler": "Rania Löffler",
        "sandrak": "Sandra Kubas",
        "sarahlöser": "Sarah Löwer",
        "sarahlöuer": "Sarah Löwer",
        "sarahz": "Sarah Zitzer",
        "saskiaw": "Saskia Wucher",
        "schlossmaus": "Schlossmäuse",
        "schlossmäuse": "Schlossmäuse",
        "schlossmäusen": "Schlossmäuse",
        "sebastiansch": "Sebastian Schmitt",
        "serinaberg": "Selina Berg",
        "silkewolfganghintze": "Silke und Wolfgang Hintze",
        "smilla": "Smilla H.",
        "steffi": "Steffi K.",
        "susanneninat": "Susanne + Nina Thomas",
        "susannet": "Susanne + Torsten Saum",
        "theresavalenca": "Theresia Valenca (Kopp)",
        "theresiavalenca": "Theresia Valenca (Kopp)",
        "timpf": "Tim Pfeiffer und Michelle Mai",
        "timpfeifermichellemai": "Tim Pfeiffer und Michelle Mai",
        "timpfeiffer": "Tim Pfeiffer und Michelle Mai",
        "timpfeiffermichellemai": "Tim Pfeiffer und Michelle Mai",
        "tonoböckenseld": "Tono Böckenfeld",
        "topol": "Topolino",
        "ulrichmüller": "Ulrich + Angela Müller",
        "ulrikec": "Ulrike Cordes",
        "ulriker": "Ulrike Ruppel",
        "ulrikesch": "Ulrike Schulz",
        "ulriket": "Ulrike Treml",
        "vanessab": "Vanessa Becker",
        "vanessag": "Vanessa Groll",
        "veragreywitz": "Vera Geywitz",
        "vonprivatzuchtmaintal": "Privatzucht Maintal",
        "wolfgangfaus": "Wolfgang Faust",
        "yvonne": "Yvonne Obendorfer",
        "zuchtderkleinenchaoten": "Zucht der kleinen Chaoten",
    }
    
    if n in norm_map:
        return norm_map[n], True
        
    return name, True

def get_normalized_gerbil_name(name):
    if not name:
        return ""
    n = name.strip()
    norm_key = "".join(c for c in n.lower() if c.isalnum())
    
    gerbil_norm_map = {
        "samgenshellyvdbuntenfellnasen": "Sammy gen. Shelly von den bunten Fellnasen",
        "sammygenshellyvdbuntenfellnasen": "Sammy gen. Shelly von den bunten Fellnasen",
        "schmidt": "Schmidti",
        "sheila": "Sheila of Ulmer Strolche",
        "shinichi": "Shinichi von PZ Mücke",
        "silenosgenadonis": "Silenos gen. Adonis von den Kleinen Chaoten",
        "silenosgenadonisvdkleinenchaoten": "Silenos gen. Adonis von den Kleinen Chaoten",
        "silver": "Silver von den kleinen Chaoten",
        "snoops": "Snoopsi",
        "sokrates": "Sokrates von den Kleinen Chaoten",
        "splash": "Slash",
        "teiko": "Teiko von den kleinen Chaoten",
        "trixy": "Trixxy von den Kleinen Chaoten",
        "unique": "Unique of Wild Dreams",
    }
    
    if norm_key in gerbil_norm_map:
        return gerbil_norm_map[norm_key]
        
    return n

def get_call_name(name):
    if not name:
        return ""
    n = name.strip()
    n = re.sub(r'\[[^\]]+\]$', '', n).strip()
    n = re.split(r'\s+(?:of|von\s+den|von\s+der|v\.\s?d\.|von)\s+', n, flags=re.IGNORECASE)[0].strip()
    return n

def get_dedup_name_key(name):
    if not name:
        return ""
    n = name.lower().strip()
    # Strip common suffixes/prefixes and parentheticals
    n = re.sub(r'\b(?:von\s+den|v\.?\s*d\.?|v\.?o\.?)\s+(?:kleinen\s+)?chaoten\b', '', n)
    n = re.sub(r'\bvon\s+der\s+schlossm\w+\b', '', n)
    n = re.sub(r'\bvon\s+der\s+bunten\s+fellnasen?\b', '', n)
    n = re.sub(r'\bof\s+black\s+forest\b', '', n)
    n = re.sub(r'\b\(?rv\)?\b', '', n)
    n = re.sub(r'\bgen\.\s+\w+', '', n)
    # Clean up parentheses or brackets
    n = re.sub(r'\(.*?\)', '', n)
    n = re.sub(r'\[.*?\]', '', n)
    return "".join(c for c in n if c.isalnum())

def clean_color_name(c_desc):
    if not c_desc:
        return "", False
    
    # Lowercase and strip
    c = c_desc.lower().strip()
    
    # Check for Schecke
    is_schecke = False
    if re.search(r'\bsp\b|\bsp\d|\bsp[*(²³]|\bspotted|\bschecke|[- ]sp\b|\w+sp\b', c):
        is_schecke = True
        
    # Standardize parentheticals for schimmel
    c = c.replace("(schimmel)", "schimmel")
    c = c.replace("(schimmel-hell)", "schimmel hell")
    c = c.replace("(schimmel hell)", "schimmel hell")
    
    # Strip schecke/sp markers and any trailing text starting from sp
    c = re.sub(r'\([- ]?sp(otted)?\)', '', c) # handles (-sp)
    c = re.sub(r'[- ]?sp(otted)?\b.*', '', c) # handles -sp(k), -sp*(k), -sp, etc.
    c = re.sub(r'[- ]?schecke\b.*', '', c)
    c = re.sub(r'[- ]?spotted\b.*', '', c)
    
    # Strip any other parentheticals, symbols, or trailing stars/numbers
    c = re.sub(r'\s*\(.*?\)\s*', ' ', c)
    c = re.sub(r'[²³*]', '', c)
    c = c.strip()
    
    # Mapping table for abbreviations, typos, and specific combinations
    mapping = {
        "antra": "anthrazit",
        "anthra": "anthrazit",
        "ankazit": "anthrazit",
        "antrazit": "anthrazit",
        "pew": "rew",
        "bew": "hermelin",
        "harder": "marder",
        "kohli": "kohlfuchs",
        "aligerfuchs": "algierfuchs",
        "algiesfuchs": "algierfuchs",
        "schw": "schwarz",
        "sa": "silberagouti",
        "a": "agouti",
        "cp-sa": "cp-silberagouti",
        "cp-a": "cp-agouti",
        "cp-a-hell": "cp-agouti-hell",
        "cp-aisa": "cp-agouti",
        "cp-a / rcp-sa": "cp-agouti",
    }
    
    if c in mapping:
        c = mapping[c]
        
    return c, is_schecke

def resolve_color_and_genotype(color_val, existing_genotype, variety_map, variety_genotypes):
    if not color_val:
        return None, existing_genotype
    color_str = str(color_val).strip()
    clean_name, is_schecke = clean_color_name(color_str)
    # Match color in variety_map
    color_variety_id = None
    if clean_name in variety_map:
        color_variety_id = variety_map[clean_name]
    else:
        for seed_name, seed_id in variety_map.items():
            if seed_name in clean_name or clean_name in seed_name:
                color_variety_id = seed_id
                break
    # Update genotype if it's a Schecke
    genotype = existing_genotype
    if is_schecke:
        if genotype:
            if "spsp" in genotype:
                genotype = genotype.replace("spsp", "Spsp")
            elif "Spsp" not in genotype and "Sp" not in genotype:
                genotype = f"{genotype} Spsp".strip()
        else:
            canonical = variety_genotypes.get(color_variety_id)
            if canonical:
                if "spsp" in canonical:
                    genotype = canonical.replace("spsp", "Spsp")
                else:
                    genotype = f"{canonical} Spsp".strip()
            else:
                genotype = "Spsp"
    return color_variety_id, genotype

def parse_date(d):
    """Convert variations of date formats to YYYY-MM-DD."""
    if not d or d == "0001-01-01":
        return None
    d = str(d).strip()
    # Try YYYY-MM-DD
    if re.match(r"^\d{4}-\d{2}-\d{2}$", d):
        parts = d.split("-")
        year = int(parts[0])
        if 1900 < year < 2000:
            year += 100
            d = f"{year}-{parts[1]}-{parts[2]}"
        elif year == 1900:
            return None
        return d
    # Try DD.MM.YYYY or D.M.YY
    match = re.match(r"^(\d{1,2})\.(\d{1,2})\.(\d{2,4})$", d)
    if match:
        day = int(match.group(1))
        month = int(match.group(2))
        year = int(match.group(3))
        if year < 100:
            year += 2000
        elif 1900 < year < 2000:
            year += 100
        elif year == 1900:
            return None
        try:
            return datetime(year, month, day).strftime("%Y-%m-%d")
        except ValueError:
            pass
    # Try ISO timestamp
    try:
        dt = datetime.fromisoformat(d.replace("Z", "+00:00"))
        year = dt.year
        if 1900 < year < 2000:
            dt = dt.replace(year=year + 100)
            return dt.strftime("%Y-%m-%d")
        elif year == 1900:
            return None
        return dt.strftime("%Y-%m-%d")
    except ValueError:
        pass
    return None

def date_to_days(dt_str):
    if not dt_str:
        return None
    try:
        return (datetime.strptime(dt_str, "%Y-%m-%d") - datetime(2000, 1, 1)).days
    except ValueError:
        return None

def days_to_date(days):
    import datetime as dt
    return (dt.datetime(2000, 1, 1) + dt.timedelta(days=int(days))).strftime("%Y-%m-%d")

def parse_death_info(notes, status, existing_dod, existing_cod):
    if not notes:
        return status, existing_dod, existing_cod
        
    has_death_indicator = '†' in notes or 'verstorben' in notes.lower() or 'gestorben' in notes.lower() or 'todesdatum' in notes.lower() or '/+' in notes
    if '+' in notes:
        if re.search(r'\+\s*(?:LE|AS|Unbekannt|gestorben|verstorben|tod)\b', notes, re.I) or re.search(r'\+\s*\d{1,2}\.\d{1,2}\.\d{2,4}', notes) or '/+' in notes:
            has_death_indicator = True
            
    resolved_status = status
    if has_death_indicator:
        resolved_status = "Deceased"
        
    dod = existing_dod
    cod = existing_cod
    
    # Look for date near death indicator
    found_date = None
    for m in re.finditer(r'([+†]\s*(?:LE|AS|Unbekannt|[a-zA-ZäöüÄÖÜß0-9()/ +,;.:-]{1,100}?)?\s*)(\d{1,2}\.\d{1,2}\.\d{2,4})', notes, re.I):
        marker_text = m.group(0)
        if '†' in marker_text or re.search(r'\+\s*(?:LE|AS|Unbekannt|gestorben|verstorben|tod|\d)', marker_text, re.I):
            found_date = parse_date(m.group(2))
            if found_date:
                break
                
    if not found_date:
        m_death_marker = re.search(r'(†\s*)(\d{1,2}\.\d{1,2}\.\d{2,4})', notes)
        if m_death_marker:
            found_date = parse_date(m_death_marker.group(2))
        
    if found_date and not dod:
        dod = found_date
        
    if not cod:
        for m in re.finditer(r'([+†])\s*([a-zA-ZäöüÄÖÜß0-9()/ +,;.:-]{1,100}?)\s*\d{1,2}\.\d{1,2}\.\d{2,4}', notes, re.I):
            indicator = m.group(1)
            cod_candidate = m.group(2).strip()
            if indicator == '+' and not re.search(r'\b(?:LE|AS|Unbekannt|gestorben|verstorben|tod)\b', cod_candidate, re.I):
                continue
            if cod_candidate:
                cod = cod_candidate
                break
        
        if not cod:
            m_cod2 = re.search(r'[+†]\s*\d{1,2}\.\d{1,2}\.\d{2,4}\s*([a-zA-ZäöüÄÖÜß0-9()/ +,;.:-]{1,100})', notes, re.I)
            if m_cod2:
                cod_candidate = m_cod2.group(1).strip()
                if cod_candidate:
                    cod = cod_candidate
            else:
                m_cod3 = re.search(r'[+†]\s*(LE|AS|Unbekannt)\b', notes, re.I)
                if m_cod3:
                    cod = m_cod3.group(1).strip()

    if cod:
        cod_lower = cod.lower().strip()
        if cod_lower in ("le", "le (lungenentzündung)", "lungenentzündung", "lungenentzündung (lungenentzündung)"):
            cod = "Lungenentzündung"
        elif cod_lower in ("as", "altersschwäche", "altenschwäche", "alter"):
            cod = "Altersschwäche"
        elif cod_lower in ("unbekannt", "unklar"):
            cod = "Unbekannt"
        else:
            # Replace abbreviations with full names (case-insensitive)
            cod = re.sub(r'\bLE\b', 'Lungenentzündung', cod, flags=re.I)
            cod = re.sub(r'\bAS\b', 'Altersschwäche', cod, flags=re.I)
            cod = cod.replace('+', ' + ')
            # Clean up multiple spaces
            cod = re.sub(r'\s+', ' ', cod).strip(' ,;.-')
            
            # Re-check after replacement
            cod_lower = cod.lower().strip()
            if cod_lower in ("le", "le (lungenentzündung)", "lungenentzündung", "lungenentzündung (lungenentzündung)"):
                cod = "Lungenentzündung"
            elif cod_lower in ("as", "altersschwäche", "altenschwäche", "alter"):
                cod = "Altersschwäche"
            elif cod_lower in ("unbekannt", "unklar"):
                cod = "Unbekannt"
            
    return resolved_status, dod, cod


# ── Litter dedup & parent-role helpers (pure, unit-tested in test_merge_resolve.py) ──

def _norm_pname(s):
    return normalize_name(s) if s else ""


def names_no_conflict(l1, l2):
    """Parent names don't contradict (equal per role, or one side empty)."""
    f1, f2 = _norm_pname(l1.get("_father_name")), _norm_pname(l2.get("_father_name"))
    m1, m2 = _norm_pname(l1.get("_mother_name")), _norm_pname(l2.get("_mother_name"))
    f_ok = (not f1) or (not f2) or (f1 == f2)
    m_ok = (not m1) or (not m2) or (m1 == m2)
    return f_ok and m_ok


def names_overlap(l1, l2):
    """At least one role has a non-empty matching name (positive evidence)."""
    f1, f2 = _norm_pname(l1.get("_father_name")), _norm_pname(l2.get("_father_name"))
    m1, m2 = _norm_pname(l1.get("_mother_name")), _norm_pname(l2.get("_mother_name"))
    return bool((f1 and f1 == f2) or (m1 and m1 == m2))


def litter_compatible(l1, l2):
    """Two litter records describe the same litter: same date and compatible parents.

    - Both sides have both parents → must match exactly.
    - Asymmetric (one side resolved, the other not) → merge only if names don't
      contradict; for dateless litters require a POSITIVE name match (a shared
      null date is no evidence), so unrelated nameless stubs stay separate.
    - Neither side has parents → never blind-merge.
    """
    if l1["Date"] != l2["Date"]:
        return False
    f1, m1 = l1.get("FatherId"), l1.get("MotherId")
    f2, m2 = l2.get("FatherId"), l2.get("MotherId")
    if f1 and f2 and m1 and m2:
        return f1 == f2 and m1 == m2
    asymmetric = (bool(f1 or m1) and not (f2 or m2)) or (bool(f2 or m2) and not (f1 or m1))
    if asymmetric:
        if not names_no_conflict(l1, l2):
            return False
        if l1["Date"] is None:
            return names_overlap(l1, l2)
        return True
    return False


def assign_parent_roles(father_id, mother_id, gender_of):
    """Assign two resolved parent IDs to father/mother roles by gender.

    Drops self-pairing duplicates (same animal in both roles) and never returns
    two same-role parents. `gender_of` maps an id to 'male'|'female'|'unknown'|None.
    Returns (father_id, mother_id).
    """
    ids = []
    for gid in (father_id, mother_id):
        if gid and gid not in ids:
            ids.append(gid)
    males = [g for g in ids if gender_of(g) == "male"]
    females = [g for g in ids if gender_of(g) == "female"]
    unknowns = [g for g in ids if gender_of(g) == "unknown"]
    father = males[0] if males else (unknowns.pop(0) if unknowns else None)
    mother = females[0] if females else (unknowns.pop(0) if unknowns else None)
    return father, mother


# A gerbil lives at most ~6 years, so a parent can be at most ~6 years older than
# its offspring (and must be born before it). Links outside this window are
# impossible — e.g. a 2013 animal resolved onto a 2022 litter (Jayjay → Solice).
MAX_PARENT_AGE_DAYS = 6 * 366


def parent_age_plausible(parent_dob, litter_date):
    """Could a parent born `parent_dob` have offspring born on `litter_date`?

    Requires birth strictly before the litter and within the gerbil lifespan.
    Unknown/unparseable dates return True (cannot disprove). Accepts any date
    format parse_date understands.
    """
    pd = date_to_days(parse_date(parent_dob)) if parent_dob else None
    ld = date_to_days(parse_date(litter_date)) if litter_date else None
    if pd is None or ld is None:
        return True
    return 0 < (ld - pd) <= MAX_PARENT_AGE_DAYS


def pick_parent_ref(parent_refs, role, child_dob, avoid_name=None, gender_of=None):
    """Choose the best parent ref for a role from possibly-conflicting chart refs.

    A Stammbaum lists an animal at several positions, so its parentRefs can carry
    contradictory guesses (the first one is not necessarily right). Rank candidates
    (lower = better):
      0  right/unknown gender for the role, age-plausible dated ref
      1  right/unknown gender, no DOB (usable, but a plausible dated ref wins)
      2  right/unknown gender, dated but age-impossible
      3  resolved gender is clearly WRONG for the role (e.g. a female father)
      4  would duplicate the animal chosen for the other role

    Gender is decisive over DOB: a dated female ref must not win the father slot
    over an undated male/unknown one. `gender_of(name)` returns 'male'/'female'
    or None (unknown/ambiguous → not penalised). Returns the chosen ref or None.
    """
    role_refs = [p for p in parent_refs if p.get("roleGuess") == role]
    if not role_refs:
        return None
    avoid = normalize_name(avoid_name) if avoid_name else None
    expected = "male" if role == "father" else "female"

    def rank(p):
        if avoid is not None and normalize_name(p.get("name")) == avoid:
            return 4  # would duplicate the other parent role
        g = gender_of(p.get("name")) if gender_of else None
        if g in ("male", "female") and g != expected:
            return 3  # wrong sex for this role
        dob = p.get("dob")
        if not dob:
            return 1
        return 0 if parent_age_plausible(dob, child_dob) else 2

    order = sorted(range(len(role_refs)), key=lambda i: (rank(role_refs[i]), i))
    return role_refs[order[0]]


def main():
    print("Loading color variety seeds...")
    variety_map = {}
    variety_genotypes = {}
    
    # Load from C# ApplicationContext.cs catalog for stable database GUIDs (index + 1)
    here = os.path.dirname(os.path.abspath(__file__))
    app_context_path = os.path.abspath(os.path.join(here, "../../GerbilManagerWebAPI/ApplicationContext.cs"))
    cs_name_to_id = {}
    if os.path.exists(app_context_path):
        with open(app_context_path, 'r', encoding='utf-8') as f:
            content = f.read()
        catalog_match = re.search(r'catalog\s*=\s*\{(.*?)\};', content, re.DOTALL)
        if catalog_match:
            block = catalog_match.group(1)
            entries = re.findall(r'\(\s*"([^"]+)"\s*,\s*"([^"]+)"\s*,\s*(\d+)\s*\)', block)
            for idx, (name, genotype, sort_order) in enumerate(entries):
                variety_id = f"00000000-0000-0000-0000-{idx + 1:012d}"
                cs_name_to_id[name.strip().lower()] = variety_id
                variety_map[name.strip().lower()] = variety_id
                variety_genotypes[variety_id] = genotype.strip()
    else:
        print(f"Warning: ApplicationContext.cs not found at {app_context_path}")

    if os.path.exists(SEEDS_PATH):
        with open(SEEDS_PATH, "r", encoding="utf-8") as f:
            seeds = json.load(f)
        for v in seeds:
            name_lower = v["name"].strip().lower()
            variety_id = cs_name_to_id.get(name_lower)
            if not variety_id:
                variety_id = f"00000000-0000-0000-0000-{v['sortOrder'] + 1:012d}"
                variety_map[name_lower] = variety_id
            
            # Map English name if present
            if "english" in v and v["english"]:
                variety_map[v["english"].strip().lower()] = variety_id
                
            if variety_id not in variety_genotypes:
                variety_genotypes[variety_id] = v.get("canonicalGenotype")
    else:
        print(f"Warning: Seeds path not found at {SEEDS_PATH}")

    md_files = sorted([f for f in os.listdir(DIR_PATH) if f.lower().endswith('.md')])
    print(f"Found {len(md_files)} markdown files in {DIR_PATH}.")

    raw_contacts = []
    raw_litters = []
    raw_gerbils = []

    # Map to track which files contained explicit dates
    file_explicit_dates = {}

    # 1. Parse JSON blocks from all markdown files and scope IDs by filename
    for filename in md_files:
        filepath = os.path.join(DIR_PATH, filename)
        with open(filepath, 'r', encoding='utf-8') as f:
            content = f.read()

        json_start = content.find("```json")
        if json_start == -1:
            continue
        json_start += len("```json")
        json_end = content.rfind("```")
        if json_end == -1 or json_end <= json_start:
            continue

        json_str = content[json_start:json_end].strip()
        try:
            data = json.loads(json_str)
            
            contacts_key = next((k for k in data if k.lower() == 'contacts'), None)
            litters_key = next((k for k in data if k.lower() == 'litters'), None)
            gerbils_key = next((k for k in data if k.lower() == 'gerbils'), None)

            file_contacts = data.get(contacts_key, []) if contacts_key else []
            file_litters = data.get(litters_key, []) if litters_key else []
            file_gerbils = data.get(gerbils_key, []) if gerbils_key else []

            # Find explicit dates on this page to build chronological ordering
            explicit_dates = []
            for l in file_litters:
                d = parse_date(l.get("Date") or l.get("date") or l.get("DateOfBirth") or l.get("dateOfBirth"))
                if d:
                    explicit_dates.append(d)
            for g in file_gerbils:
                d = parse_date(g.get("DateOfBirth") or g.get("dateOfBirth"))
                if d:
                    explicit_dates.append(d)
            
            if explicit_dates:
                # Convert to days since 2000-01-01
                day_vals = [date_to_days(d) for d in explicit_dates if date_to_days(d) is not None]
                if day_vals:
                    file_explicit_dates[filename] = sum(day_vals) / len(day_vals)

            def scope_id(local_id):
                if not local_id:
                    return None
                local_str = str(local_id).strip()
                if not local_str:
                    return None
                return generate_guid(f"{filename}-{local_str}")

            for c in file_contacts:
                old_id = c.get("Id") or c.get("id")
                c["_filename"] = filename
                c["_scoped_id"] = scope_id(old_id)
                raw_contacts.append(c)

            for l in file_litters:
                old_id = l.get("Id") or l.get("id")
                l["_filename"] = filename
                l["_scoped_id"] = scope_id(old_id)
                l["_scoped_father_id"] = scope_id(l.get("FatherId") or l.get("fatherId"))
                l["_scoped_mother_id"] = scope_id(l.get("MotherId") or l.get("motherId"))
                raw_litters.append(l)

            for g in file_gerbils:
                old_id = g.get("Id") or g.get("id")
                g["_filename"] = filename
                g["_scoped_id"] = scope_id(old_id)
                g["_scoped_litter_id"] = scope_id(g.get("LitterId") or g.get("litterId"))
                g["_scoped_origin_cid"] = scope_id(g.get("OriginContactId") or g.get("originContactId"))
                
                receiver_val = (g.get("ReceiverContactId") or g.get("receiverContactId") or 
                                g.get("BuyerId") or g.get("buyerId") or 
                                g.get("BuyerContactId") or g.get("buyerContactId") or 
                                g.get("givenAwayContactId") or g.get("givenAwayToContactId") or 
                                g.get("ownerContactId") or g.get("ownerId"))
                g["_scoped_receiver_cid"] = scope_id(receiver_val)
                raw_gerbils.append(g)

        except json.JSONDecodeError as e:
            print(f"Failed to parse JSON in {filename}: {e}")

    # Chronological Interpolation: Estimate the date of each page based on neighboring pages with dates
    file_dates = {}
    sorted_files = sorted(md_files)
    
    # Simple linear interpolation / extrapolation
    for i, fn in enumerate(sorted_files):
        if fn in file_explicit_dates:
            file_dates[fn] = file_explicit_dates[fn]
        else:
            # Look left for closest explicit date
            left_val, left_dist = None, None
            for j in range(i - 1, -1, -1):
                if sorted_files[j] in file_explicit_dates:
                    left_val = file_explicit_dates[sorted_files[j]]
                    left_dist = i - j
                    break
            
            # Look right for closest explicit date
            right_val, right_dist = None, None
            for j in range(i + 1, len(sorted_files)):
                if sorted_files[j] in file_explicit_dates:
                    right_val = file_explicit_dates[sorted_files[j]]
                    right_dist = j - i
                    break
            
            if left_val is not None and right_val is not None:
                # Interpolate
                file_dates[fn] = left_val + (right_val - left_val) * (left_dist / (left_dist + right_dist))
            elif left_val is not None:
                # Extrapolate right (assume 30 days per page gap as placeholder)
                file_dates[fn] = left_val + (left_dist * 30)
            elif right_val is not None:
                # Extrapolate left
                file_dates[fn] = right_val - (right_dist * 30)
            else:
                # No dates in entire log? Default to 2011-01-01
                file_dates[fn] = date_to_days("2011-01-01")

    print(f"Parsed {len(raw_contacts)} raw contacts, {len(raw_litters)} raw litters, {len(raw_gerbils)} raw gerbils.")

    # Run extract.py to make sure stammbaum data is up to date
    import subprocess
    print("Running extract.py to extract Stammbäume...")
    try:
        subprocess.run([sys.executable, "extract.py"], check=True)
    except Exception as e:
        print(f"Warning: Failed to run extract.py: {e}")

    # Load Stammbaum data
    stammbaum_only_animals = []
    animals_path = os.path.join(OUTPUT_DIR, "animals.json")
    if os.path.exists(animals_path):
        with open(animals_path, "r", encoding="utf-8") as f:
            stammbaum_animals = json.load(f)
        for a in stammbaum_animals:
            sources = a.get("sourceFiles", [])
            if any("stammbaum" in str(s).lower() for s in sources):
                stammbaum_only_animals.append(a)
        print(f"Loaded {len(stammbaum_only_animals)} Stammbaum animals.")
    else:
        print(f"Warning: Stammbaum animals.json not found at {animals_path}")

    # Run extract_docx.py to make sure docx data is up to date
    print("Running extract_docx.py to extract docx...")
    try:
        subprocess.run([sys.executable, "extract_docx.py"], check=True)
    except Exception as e:
        print(f"Warning: Failed to run extract_docx.py: {e}")

    # Load Docx data
    docx_animals = []
    docx_animals_path = os.path.join(OUTPUT_DIR, "docx_animals.json")
    if os.path.exists(docx_animals_path):
        with open(docx_animals_path, "r", encoding="utf-8") as f:
            docx_animals = json.load(f)
        print(f"Loaded {len(docx_animals)} animals from docx.")
    else:
        print(f"Warning: docx_animals.json not found at {docx_animals_path}")

    docx_litters = []
    docx_litters_path = os.path.join(OUTPUT_DIR, "docx_litters.json")
    if os.path.exists(docx_litters_path):
        with open(docx_litters_path, "r", encoding="utf-8") as f:
            docx_litters = json.load(f)
        print(f"Loaded {len(docx_litters)} litters from docx.")
    else:
        print(f"Warning: docx_litters.json not found at {docx_litters_path}")

    # Extract docx buyer contacts and add to raw_contacts
    for da in docx_animals:
        o_name = (da.get("owner") or "").strip()
        if o_name:
            o_scoped_id = generate_guid(f"docx-contact-{normalize_name(o_name)}")
            raw_contacts.append({
                "Name": o_name,
                "_filename": "Wurfchronik-Detail",
                "_scoped_id": o_scoped_id
            })

    # Map docx litters and append to raw_litters
    docx_litter_id_map = {} # wsCode -> scoped_id
    for dl in docx_litters:
        l_name = dl["litterId"]
        dob_val = parse_date(dl["dob"])
        f_name = get_normalized_gerbil_name(dl["fatherName"])
        m_name = get_normalized_gerbil_name(dl["motherName"])
        ws_code = dl["wsCode"]
        note_val = dl.get("note")
        
        # Parse survived/total born from wsCode (e.g. 4/5)
        total_born = None
        deaths_8w = None
        if "/" in ws_code:
            parts = ws_code.split("/")
            if len(parts) == 2:
                try:
                    survived = int(parts[0])
                    total = int(parts[1])
                    total_born = total
                    deaths_8w = max(0, total - survived)
                except ValueError:
                    pass

        l_scoped_id = generate_guid(f"docx-litter-{normalize_name(ws_code)}-{dob_val or '0001-01-01'}")
        docx_litter_id_map[(ws_code, dob_val)] = l_scoped_id
        
        raw_litters.append({
            "Id": l_scoped_id,
            "Name": l_name,
            "Date": dob_val,
            "TotalBorn": total_born,
            "DeathsWithin8Weeks": deaths_8w,
            "FatherId": generate_guid(f"stammbaum-animal-{normalize_name(f_name)}"), # placeholder
            "MotherId": generate_guid(f"stammbaum-animal-{normalize_name(m_name)}"), # placeholder
            "ExpectedGoHomeDate": None,
            "Notes": note_val if note_val else "Docx imported litter",
            "PairingCode": None,
            "ExternalRef": f"docx-litter-{l_scoped_id}",
            "LitterLetter": l_name[0] if l_name and len(l_name) > 0 else None,
            "_father_name": f_name,
            "_mother_name": m_name,
            "_filename": "Wurfchronik-Detail",
            "_scoped_id": l_scoped_id,
            "_scoped_father_id": generate_guid(f"stammbaum-animal-{normalize_name(f_name)}"),
            "_scoped_mother_id": generate_guid(f"stammbaum-animal-{normalize_name(m_name)}")
        })


    # Extract stammbaum contacts and add to raw_contacts
    for a in stammbaum_only_animals:
        b_name = (a.get("breeder") or "").strip()
        if b_name:
            b_scoped_id = generate_guid(f"stammbaum-contact-{normalize_name(b_name)}")
            raw_contacts.append({
                "Name": b_name,
                "_filename": "Stammbaum",
                "_scoped_id": b_scoped_id
            })
        z_name = (a.get("zucht") or "").strip()
        if z_name:
            z_scoped_id = generate_guid(f"stammbaum-contact-{normalize_name(z_name)}")
            raw_contacts.append({
                "Name": z_name,
                "_filename": "Stammbaum",
                "_scoped_id": z_scoped_id
            })

    # Residency propagation for stammbaum animals
    def is_clan_zucht(z):
        if not z:
            return False
        norm = z.lower()
        return "klein" in norm and "chaot" in norm and "extern" not in norm

    stammbaum_resident_ids = set()
    for a in stammbaum_only_animals:
        if is_clan_zucht(a.get("zucht")) or is_clan_zucht(a.get("zuchtCanon")):
            stammbaum_resident_ids.add(a["id"])

    # Propagate residency to parents of resident offspring
    for _ in range(5):
        for a in stammbaum_only_animals:
            if a["id"] in stammbaum_resident_ids:
                for p_ref in a.get("parentRefs", []):
                    p_key = (normalize_name(p_ref["name"]), parse_date(p_ref["dob"]))
                    for cand in stammbaum_only_animals:
                        if normalize_name(cand["name"]) == p_key[0]:
                            cand_dob = parse_date(cand["dob"])
                            if not p_key[1] or cand_dob == p_key[1]:
                                stammbaum_resident_ids.add(cand["id"])

    # Pre-index Wurfchronik litters from markdown
    md_litters_idx = {}
    for rl in raw_litters:
        f_name = get_normalized_gerbil_name(rl.get("FatherName") or rl.get("fatherName") or rl.get("ParentMaleName") or rl.get("parentMaleName"))
        m_name = get_normalized_gerbil_name(rl.get("MotherName") or rl.get("motherName") or rl.get("ParentFemaleName") or rl.get("parentFemaleName"))
        ldate = parse_date(rl.get("Date") or rl.get("date") or rl.get("DateOfBirth") or rl.get("dateOfBirth"))
        if f_name and m_name and ldate:
            key = (normalize_name(f_name), normalize_name(m_name), ldate)
            md_litters_idx[key] = rl

    # Gender index for parent-ref selection: normalized name → 'male' | 'female'
    # | 'ambiguous'. Drives the gender-aware ranking in pick_parent_ref so a dated
    # but wrong-sex ref (e.g. female „Danielle“) cannot win the father slot over
    # an undated male/unknown one (e.g. „Hagrid Rubeus“).
    gender_idx = {}
    for a in stammbaum_only_animals:
        g = (a.get("gender") or "").lower().strip()
        g = g if g in ("male", "female") else None
        for key in {normalize_name(a.get("name")), normalize_name(get_call_name(a.get("name") or ""))}:
            if not key:
                continue
            if key not in gender_idx:
                gender_idx[key] = g
            elif gender_idx[key] != g:
                gender_idx[key] = "ambiguous"

    def gender_of_name(name):
        v = gender_idx.get(normalize_name(name))
        return v if v in ("male", "female") else None

    # Create virtual litters for stammbaum animals
    created_virtual_litters = {}
    for a in stammbaum_only_animals:
        parent_refs = a.get("parentRefs", [])
        child_dob_raw = a.get("dob")
        father_ref = pick_parent_ref(parent_refs, "father", child_dob_raw, gender_of=gender_of_name)
        mother_ref = pick_parent_ref(parent_refs, "mother", child_dob_raw,
                                     avoid_name=father_ref.get("name") if father_ref else None,
                                     gender_of=gender_of_name)
        
        a["_mapped_litter_scoped_id"] = None
        if father_ref and mother_ref:
            f_name = get_normalized_gerbil_name(father_ref.get("name"))
            m_name = get_normalized_gerbil_name(mother_ref.get("name"))
            dob_val = parse_date(a.get("dob"))
            
            mapped_litter = None
            if dob_val:
                key = (normalize_name(f_name), normalize_name(m_name), dob_val)
                mapped_litter = md_litters_idx.get(key)
                
            if mapped_litter:
                a["_mapped_litter_scoped_id"] = mapped_litter["_scoped_id"]
            else:
                v_key = (normalize_name(f_name), normalize_name(m_name), dob_val or "0001-01-01")
                if v_key in created_virtual_litters:
                    a["_mapped_litter_scoped_id"] = created_virtual_litters[v_key]
                else:
                    l_scoped_id = generate_guid(f"virtual-litter-{v_key[0]}-{v_key[1]}-{v_key[2]}")
                    created_virtual_litters[v_key] = l_scoped_id
                    a["_mapped_litter_scoped_id"] = l_scoped_id
                    
                    # Try to link parents to actual parsed stammbaum animals
                    f_scoped_id = None
                    m_scoped_id = None
                    f_dob = parse_date(father_ref.get("dob"))
                    m_dob = parse_date(mother_ref.get("dob"))
                    
                    for p_cand in stammbaum_only_animals:
                        p_gender = str(p_cand.get("gender") or "").lower().strip()
                        if p_gender in ["w", "f", "female", "weiblich"]:
                            continue
                        p_dob = parse_date(p_cand.get("dob"))
                        if dob_val and p_dob and p_dob >= dob_val:
                            continue
                        cand_call_norm = normalize_name(get_call_name(p_cand["name"])) or "unbekannt"
                        cand_name_norm = normalize_name(p_cand["name"]) or "unbekannt"
                        if cand_call_norm == normalize_name(f_name) or cand_name_norm == normalize_name(f_name):
                            if not f_dob or p_dob == f_dob:
                                f_scoped_id = generate_guid(f"stammbaum-animal-{p_cand['id']}")
                                break
                    for p_cand in stammbaum_only_animals:
                        p_gender = str(p_cand.get("gender") or "").lower().strip()
                        if p_gender in ["m", "male", "männlich"]:
                            continue
                        p_dob = parse_date(p_cand.get("dob"))
                        if dob_val and p_dob and p_dob >= dob_val:
                            continue
                        cand_call_norm = normalize_name(get_call_name(p_cand["name"])) or "unbekannt"
                        cand_name_norm = normalize_name(p_cand["name"]) or "unbekannt"
                        if cand_call_norm == normalize_name(m_name) or cand_name_norm == normalize_name(m_name):
                            if not m_dob or p_dob == m_dob:
                                m_scoped_id = generate_guid(f"stammbaum-animal-{p_cand['id']}")
                                break

                    raw_litters.append({
                        "Id": l_scoped_id,
                        "Name": f"Wurf von {f_name} + {m_name}",
                        "Date": dob_val,
                        "TotalBorn": None,
                        "DeathsWithin8Weeks": None,
                        "FatherId": f_scoped_id or generate_guid(f"stammbaum-animal-{normalize_name(f_name)}"),
                        "MotherId": m_scoped_id or generate_guid(f"stammbaum-animal-{normalize_name(m_name)}"),
                        "ExpectedGoHomeDate": None,
                        "Notes": "Pedigree virtual litter",
                        "PairingCode": None,
                        "ExternalRef": f"virtual-{l_scoped_id}",
                        "LitterLetter": None,
                        "_father_name": f_name,
                        "_mother_name": m_name,
                        "_filename": "Stammbaum",
                        "_scoped_id": l_scoped_id,
                        "_scoped_father_id": f_scoped_id or generate_guid(f"stammbaum-animal-{normalize_name(f_name)}"),
                        "_scoped_mother_id": m_scoped_id or generate_guid(f"stammbaum-animal-{normalize_name(m_name)}")
                    })

    # 2. Resolve Contacts globally (deduplicate by normalized name)

    contact_by_norm_name = {}
    contact_id_map = {} # scoped_old_id -> global_guid

    for rc in raw_contacts:
        name_val = rc.get("Name") or rc.get("name") or rc.get("FullName") or rc.get("fullName")
        if not name_val:
            first = rc.get("FirstName") or rc.get("firstName")
            last = rc.get("LastName") or rc.get("lastName")
            if first or last:
                name_val = f"{first or ''} {last or ''}".strip()
        
        if not name_val:
            continue

        canon_name, should_keep = get_normalized_contact_name(name_val)
        scoped_id = rc["_scoped_id"]
        if not should_keep:
            if scoped_id:
                contact_id_map[scoped_id] = None
            continue

        norm_name = normalize_name(canon_name)

        if norm_name not in contact_by_norm_name:
            global_guid = generate_guid(f"contact-{norm_name}")
            contact_by_norm_name[norm_name] = {
                "Id": global_guid,
                "Name": canon_name,
                "Email": rc.get("Email") or rc.get("email"),
                "Phone": rc.get("Phone") or rc.get("phone"),
                "Address": rc.get("Address") or rc.get("address"),
                "Notes": rc.get("Notes") or rc.get("notes") or rc.get("Note") or rc.get("note")
            }
        else:
            gc = contact_by_norm_name[norm_name]
            if not gc["Email"] and (rc.get("Email") or rc.get("email")):
                gc["Email"] = rc.get("Email") or rc.get("email")
            if not gc["Phone"] and (rc.get("Phone") or rc.get("phone")):
                gc["Phone"] = rc.get("Phone") or rc.get("phone")
            if not gc["Address"] and (rc.get("Address") or rc.get("address")):
                gc["Address"] = rc.get("Address") or rc.get("address")
            if not gc["Notes"] and (rc.get("Notes") or rc.get("notes") or rc.get("Note") or rc.get("note")):
                gc["Notes"] = rc.get("Notes") or rc.get("notes") or rc.get("Note") or rc.get("note")

        if scoped_id:
            contact_id_map[scoped_id] = contact_by_norm_name[norm_name]["Id"]

    resolved_contacts = list(contact_by_norm_name.values())
    print(f"Resolved to {len(resolved_contacts)} unique contacts.")

    # 3. Process Litters (normalize keys)
    resolved_litters = []
    litter_id_map = {} # scoped_old_id -> new_guid
    litter_by_scoped_id = {}
    
    for rl in raw_litters:
        filename = rl.get("_filename")
        name_val = rl.get("Name") or rl.get("name")
        if not name_val:
            name_val = "Wurf"

        dob_val = parse_date(rl.get("Date") or rl.get("date") or rl.get("DateOfBirth") or rl.get("dateOfBirth"))

        new_guid = rl["_scoped_id"]
        if not new_guid:
            new_guid = generate_guid(f"litter-{filename}-{name_val}-{dob_val}")

        scoped_old_id = rl["_scoped_id"]
        if scoped_old_id:
            litter_id_map[scoped_old_id] = new_guid

        total_born = rl.get("TotalBorn") or rl.get("totalBorn") or rl.get("LitterSize") or rl.get("litterSize") or rl.get("size") or rl.get("totalPups")
        if total_born is not None:
            try:
                total_born = int(total_born)
            except ValueError:
                total_born = None

        deaths_8w = rl.get("DeathsWithin8Weeks") or rl.get("deathsWithin8Weeks")
        if deaths_8w is not None:
            try:
                deaths_8w = int(deaths_8w)
            except ValueError:
                deaths_8w = None
        elif total_born is not None and rl.get("survived") is not None:
            try:
                deaths_8w = total_born - int(rl.get("survived"))
            except ValueError:
                pass

        father_name = rl.get("FatherName") or rl.get("fatherName") or rl.get("ParentMaleName") or rl.get("parentMaleName") or rl.get("_father_name")
        mother_name = rl.get("MotherName") or rl.get("motherName") or rl.get("ParentFemaleName") or rl.get("parentFemaleName") or rl.get("_mother_name")

        raw_ext_ref = rl.get("ExternalRef") or rl.get("externalRef") or rl.get("Id") or rl.get("id")
        ext_ref_scoped = f"{filename}-{raw_ext_ref}" if raw_ext_ref else None

        l_record = {
            "Id": new_guid,
            "Name": name_val,
            "Date": dob_val,
            "TotalBorn": total_born,
            "DeathsWithin8Weeks": deaths_8w,
            "FatherId": rl["_scoped_father_id"],
            "MotherId": rl["_scoped_mother_id"],
            "ExpectedGoHomeDate": parse_date(rl.get("ExpectedGoHomeDate") or rl.get("expectedGoHomeDate")),
            "Notes": rl.get("Notes") or rl.get("notes") or rl.get("Note") or rl.get("note"),
            "PairingCode": rl.get("PairingCode") or rl.get("pairingCode"),
            "ExternalRef": ext_ref_scoped,
            "LitterLetter": rl.get("LitterLetter") or rl.get("litterLetter"),
            "_father_name": father_name,
            "_mother_name": mother_name,
            "_filename": filename
        }
        resolved_litters.append(l_record)
        litter_by_scoped_id[new_guid] = l_record

    print(f"Processed {len(resolved_litters)} litters.")

    # 3b. Deduplicate litters: same date + compatible parents → merge
    # This handles the "sibling pairing" case: Stammbaum shows the same parental
    # litter twice (once under the father branch, once under the mother branch),
    # generating two separate litter records with the same date but only one of
    # them has FatherId/MotherId resolved.
    litter_canonical_map = {}  # old_id -> canonical_id (for dedup within this step)

    # Group by date for efficiency
    by_date = {}
    for l in resolved_litters:
        by_date.setdefault(l["Date"], []).append(l)

    litter_dedup_canonical = {}  # old_litter_id -> canonical_litter_id
    deduped_litters = []

    for date_val, group in by_date.items():
        # Partition into compatible subsets
        sub_groups = []
        for l in group:
            placed = False
            for sub in sub_groups:
                if all(litter_compatible(l, member) for member in sub):
                    sub.append(l)
                    placed = True
                    break
            if not placed:
                sub_groups.append([l])

        for sub in sub_groups:
            if len(sub) == 1:
                deduped_litters.append(sub[0])
                litter_dedup_canonical[sub[0]["Id"]] = sub[0]["Id"]
                continue

            # Pick the canonical record: prefer the one with parents set
            canonical = next((l for l in sub if l.get("FatherId") or l.get("MotherId")), sub[0])
            for l in sub:
                litter_dedup_canonical[l["Id"]] = canonical["Id"]
                if l is not canonical:
                    litter_id_map[l["Id"]] = canonical["Id"]

            deduped_litters.append(canonical)
            if len(sub) > 1:
                merged_names = [l["Id"] for l in sub if l is not canonical]
                print(f"Litter-Dedup: merged {len(sub)} same-date litters on {date_val} → {canonical['Name']} (absorbed: {', '.join(merged_names)})")

    n_merged = len(resolved_litters) - len(deduped_litters)
    if n_merged:
        print(f"Litter-Dedup: {n_merged} redundant litter record(s) removed.")
    resolved_litters = deduped_litters
    litter_by_scoped_id = {l["Id"]: l for l in resolved_litters}

    # 4. Normalize and group Gerbils

    # Helper to lookup litter dates for birth date estimation
    def get_litter_date(l_id):
        if l_id in litter_by_scoped_id:
            d = litter_by_scoped_id[l_id]["Date"]
            if d:
                return d
        return None

    all_processed_gerbils = []
    for rg in raw_gerbils:
        filename = rg.get("_filename")
        name_val = get_normalized_gerbil_name(rg.get("Name") or rg.get("name") or rg.get("callName"))
        if not name_val:
            name_val = "Unbekannt"

        dob_val = parse_date(rg.get("DateOfBirth") or rg.get("dateOfBirth"))
        
        new_guid = rg["_scoped_id"]
        if not new_guid:
            new_guid = generate_guid(f"gerbil-{filename}-{name_val}-{dob_val}")

        # Gender normalization
        gender_val = rg.get("Gender") or rg.get("gender") or "unknown"
        gender_val = str(gender_val).lower().strip()
        if gender_val in ["m", "male", "männlich"]:
            gender = "male"
        elif gender_val in ["w", "f", "female", "weiblich"]:
            gender = "female"
        else:
            gender = "unknown"

        dod_val = parse_date(rg.get("DateOfDeath") or rg.get("dateOfDeath") or rg.get("deathDate"))
        gohome_val = parse_date(rg.get("GoHomeDate") or rg.get("goHomeDate") or rg.get("DateGivenAway") or rg.get("givenAwayDate") or rg.get("DateOfHandover") or rg.get("dateOfHandover") or rg.get("HandoverDate") or rg.get("DateOfSale") or rg.get("dateOfSale"))

        status_val = rg.get("Status") or rg.get("status") or ""
        status_val = str(status_val).lower().strip()
        status = "Breeding"
        if dod_val or "deceased" in status_val or "verstorben" in status_val or "tod" in status_val or "dead" in status_val:
            status = "Deceased"
        elif gohome_val or "givenaway" in status_val or "abgegeben" in status_val or "verkauft" in status_val or "sold" in status_val:
            status = "GivenAway"
        elif "forsale" in status_val or "abzugeben" in status_val:
            status = "ForSale"
        elif "pet" in status_val or "liebhaber" in status_val:
            status = "Pet"
        else:
            if dod_val:
                status = "Deceased"
            elif gohome_val:
                status = "GivenAway"

        notes_val = rg.get("Notes") or rg.get("notes") or rg.get("Note") or rg.get("note")
        existing_cod = rg.get("CauseOfDeath") or rg.get("causeOfDeath") or rg.get("DeathCause")
        status, dod_val, cause_of_death_val = parse_death_info(notes_val, status, dod_val, existing_cod)

        # Explicit resolution for Ken'ichi's cause of death
        if name_val == "Ken'ichi" and dob_val == "2015-03-01":
            cause_of_death_val = "Duftdrüsen-Tumor"

        color_val = rg.get("ColorVarietyId") or rg.get("colorVarietyId") or rg.get("Color") or rg.get("color") or rg.get("ColorDescription") or rg.get("colorDescription")
        existing_gt = rg.get("Genotype") or rg.get("genotype")
        color_variety_id, genotype_val = resolve_color_and_genotype(
            color_val, existing_gt, variety_map, variety_genotypes
        )
        if color_val and not color_variety_id:
            try:
                uuid.UUID(str(color_val).strip())
                color_variety_id = str(color_val).strip()
            except ValueError:
                print(f"Warning: Unknown color variety '{color_val}' for gerbil '{name_val}' on page {filename}")

        origin_cid = rg["_scoped_origin_cid"]
        receiver_cid = rg["_scoped_receiver_cid"]

        if origin_cid in contact_id_map:
            origin_cid = contact_id_map[origin_cid]
        if receiver_cid in contact_id_map:
            receiver_cid = contact_id_map[receiver_cid]

        is_resident = rg.get("IsResident") or rg.get("isResident")
        if is_resident is None:
            is_resident = True
        else:
            is_resident = str(is_resident).lower() == "true"

        traits = rg.get("CharacterTraits") or rg.get("characterTraits") or []
        if not isinstance(traits, list):
            traits = [str(traits)]
        char_note = rg.get("CharacterNote") or rg.get("characterNote")

        is_deaf = rg.get("IsDeaf") or rg.get("isDeaf")
        if is_deaf is not None:
            is_deaf = str(is_deaf).lower() == "true"

        old_litter_id = rg["_scoped_litter_id"]

        raw_ext_ref = rg.get("ExternalRef") or rg.get("externalRef") or rg.get("Id") or rg.get("id")
        ext_ref_scoped = f"{filename}-{raw_ext_ref}" if raw_ext_ref else None

        # Estimated effective date of birth for conflict checking
        eff_dob_val = dob_val
        if not eff_dob_val and old_litter_id:
            # Try to get litter date if litter was resolved
            mapped_l_id = litter_id_map.get(old_litter_id)
            if mapped_l_id:
                eff_dob_val = get_litter_date(mapped_l_id)
        
        # Determine explicit or litter-derived birth date (None if unknown/parent)
        birth_date = dob_val
        if not birth_date and old_litter_id:
            mapped_l_id = litter_id_map.get(old_litter_id)
            if mapped_l_id:
                birth_date = get_litter_date(mapped_l_id)

        # If still no effective date, use the estimated page date
        if not eff_dob_val:
            eff_dob_val = days_to_date(file_dates[filename])

        raw_breeder = rg.get("OriginBreeder") or rg.get("originBreeder")
        if raw_breeder:
            norm_b, keep_b = get_normalized_contact_name(raw_breeder)
            raw_breeder = norm_b if keep_b else None

        parent_refs = []
        if old_litter_id:
            rl = next((l for l in raw_litters if l.get("_scoped_id") == old_litter_id), None)
            if rl:
                f_name = rl.get("FatherName") or rl.get("fatherName") or rl.get("ParentMaleName") or rl.get("parentMaleName") or rl.get("_father_name")
                m_name = rl.get("MotherName") or rl.get("motherName") or rl.get("ParentFemaleName") or rl.get("parentFemaleName") or rl.get("_mother_name")
                if f_name:
                    parent_refs.append({"name": f_name, "roleGuess": "father"})
                if m_name:
                    parent_refs.append({"name": m_name, "roleGuess": "mother"})

        all_processed_gerbils.append({
            "Id": new_guid,
            "Name": name_val,
            "Gender": gender,
            "Status": status,
            "LitterId": old_litter_id, # mapped later
            "OriginContactId": origin_cid,
            "ReceiverContactId": receiver_cid,
            "EnclosureId": None,
            "ColorVarietyId": color_variety_id,
            "DateOfBirth": dob_val,
            "DateOfDeath": dod_val,
            "CauseOfDeath": cause_of_death_val,
            "GoHomeDate": gohome_val,
            "Genotype": genotype_val,
            "Notes": rg.get("Notes") or rg.get("notes") or rg.get("Note") or rg.get("note"),
            "ImportSource": rg.get("ImportSource") or rg.get("importSource") or filename,
            "ExternalRef": ext_ref_scoped,
            "RawImportData": rg.get("RawImportData") or rg.get("rawImportData") or json.dumps({"colorDescription": color_val if not color_variety_id else None}),
            "OriginBreeder": raw_breeder or ("Zucht der kleinen Chaoten" if is_resident else None),
            "NameSearch": normalize_name(name_val),
            "CharacterTraits": traits,
            "CharacterNote": char_note,
            "IsDeaf": is_deaf,
            "IsResident": is_resident,
            "parentRefs": parent_refs,
            "_photos": rg.get("photos", []),
            "_old_scoped_litter_id": old_litter_id,
            "_eff_dob": eff_dob_val,
            "_birth_date": birth_date,
            "_filename": filename,
            "_old_id": rg.get("Id") or rg.get("id")
        })

    # Map and append stammbaum animals to all_processed_gerbils
    for a in stammbaum_only_animals:
        a_id = a["id"]
        name_val = get_normalized_gerbil_name(a["name"])
        
        gender_val = str(a.get("gender") or "").lower().strip()
        if gender_val in ["m", "male"]:
            gender = "male"
        elif gender_val in ["w", "f", "female"]:
            gender = "female"
        else:
            gender = "unknown"

        dob_val = parse_date(a.get("dob"))
        dod_val = parse_date(a.get("death"))
        
        status = "Breeding"
        if dod_val:
            status = "Deceased"
        elif a_id not in stammbaum_resident_ids:
            status = "GivenAway"
        else:
            if dob_val:
                try:
                    dt_dob = datetime.strptime(dob_val, "%Y-%m-%d")
                    dt_now = datetime.now()
                    age_years = (dt_now - dt_dob).days / 365.25
                    if age_years >= 6.0:
                        status = "Deceased"
                except Exception:
                    pass

        # Color variety mapping
        existing_gt = a["genotype"]["rawGenotype"] if a["genotype"]["rawGenotype"] else None
        color_variety_id, genotype_val = resolve_color_and_genotype(
            a.get("farbschlag"), existing_gt, variety_map, variety_genotypes
        )
        if not color_variety_id:
            for fbv in a.get("farbschlagVariants", []):
                cv_id, gt_val = resolve_color_and_genotype(fbv, existing_gt, variety_map, variety_genotypes)
                if cv_id:
                    color_variety_id = cv_id
                    genotype_val = gt_val
                    break

        # Contact mapping
        origin_cid = None
        b_name = (a.get("breeder") or "").strip()
        z_name = (a.get("zucht") or "").strip()
        if b_name:
            origin_cid = generate_guid(f"stammbaum-contact-{normalize_name(b_name)}")
        elif z_name:
            origin_cid = generate_guid(f"stammbaum-contact-{normalize_name(z_name)}")

        # Map to resolved global contact GUID
        if origin_cid in contact_id_map:
            origin_cid = contact_id_map[origin_cid]
        else:
            origin_cid = None


        scoped_id = generate_guid(f"stammbaum-animal-{a_id}")
        scoped_litter_id = a.get("_mapped_litter_scoped_id")

        is_deaf = a["genotype"].get("deaf")

        stammbaum_breeder = a.get("breeder") if a.get("breeder") else (a.get("zucht") if a.get("zucht") else None)
        if stammbaum_breeder:
            norm_b, keep_b = get_normalized_contact_name(stammbaum_breeder)
            stammbaum_breeder = norm_b if keep_b else None

        all_processed_gerbils.append({
            "Id": scoped_id,
            "Name": name_val,
            "Gender": gender,
            "Status": status,
            "LitterId": scoped_litter_id, # mapped later in step 5
            "OriginContactId": origin_cid,
            "ReceiverContactId": None,
            "EnclosureId": None,
            "ColorVarietyId": color_variety_id,
            "DateOfBirth": dob_val,
            "DateOfDeath": dod_val,
            "CauseOfDeath": "Duftdrüsen-Tumor" if name_val == "Ken'ichi" and dob_val == "2015-03-01" else None,
            "GoHomeDate": None,
            "Genotype": genotype_val,
            "Notes": None,
            "ImportSource": ", ".join(a.get("sourceFiles", [])),
            "ExternalRef": f"stammbaum-{a_id}",
            "RawImportData": json.dumps({
                "rawGenotype": a["genotype"]["rawGenotype"],
                "unmappedTokens": a["genotype"]["unmappedTokens"],
                "breederText": a.get("breeder", "")
            }, ensure_ascii=False),
            "OriginBreeder": stammbaum_breeder,
            "NameSearch": normalize_name(name_val),
            "CharacterTraits": [],
            "CharacterNote": None,
            "IsDeaf": is_deaf,
            "IsResident": a_id in stammbaum_resident_ids,
            "parentRefs": a.get("parentRefs", []),
            "_photos": a.get("photos", []),
            "_conflict": bool(a.get("conflict")),
            "_resolved_by_decision": bool(a.get("resolvedByDecision")),
            "_old_scoped_litter_id": scoped_litter_id,
            "_eff_dob": dob_val or "2010-01-01",
            "_birth_date": dob_val,
            "_filename": a.get("sourceFiles", ["Stammbaum"])[0],
            "_old_id": a_id
        })

    # Map and append docx animals to all_processed_gerbils
    for idx, da in enumerate(docx_animals):
        name_val = get_normalized_gerbil_name(da["name"])
        gender = da["gender"]
        
        dob_val = parse_date(da.get("litterDob"))
        dod_val = parse_date(da.get("deathDate"))
        gohome_val = parse_date(da.get("abgabeDate"))
        
        # Status precedence
        status = "Breeding"
        if dod_val:
            status = "Deceased"
        elif gohome_val or da.get("owner"):
            status = "GivenAway"

        # Color variety mapping
        color_variety_id, genotype_val = resolve_color_and_genotype(
            da.get("farbschlag"), None, variety_map, variety_genotypes
        )

        # Contact mapping (buyer)
        receiver_cid = None
        o_name = (da.get("owner") or "").strip()
        if o_name:
            receiver_cid = generate_guid(f"docx-contact-{normalize_name(o_name)}")
            if receiver_cid in contact_id_map:
                receiver_cid = contact_id_map[receiver_cid]
            else:
                receiver_cid = None

        # Scoped ID
        scoped_id = generate_guid(f"docx-animal-{idx}-{normalize_name(name_val)}-{dob_val or '0001-01-01'}")
        
        # Litter ID mapping
        ws_code = da.get("wsCode")
        scoped_litter_id = docx_litter_id_map.get((ws_code, dob_val))


        # Raw import details
        raw_import_payload = json.dumps({
            "abgabeWeight": da.get("abgabeWeight", ""),
            "deathCause": da.get("deathCause", ""),
            "partnerName": da.get("partnerName", ""),
            "partnerDob": da.get("partnerDob", "")
        }, ensure_ascii=False)

        # Residents: if sold/given away, it's not a resident
        is_resident = not bool(o_name)

        parent_refs = []
        if scoped_litter_id:
            dl = next((l for l in docx_litters if docx_litter_id_map.get((l["wsCode"], parse_date(l["dob"]))) == scoped_litter_id), None)
            if dl:
                if dl.get("fatherName"):
                    parent_refs.append({"name": dl["fatherName"], "roleGuess": "father"})
                if dl.get("motherName"):
                    parent_refs.append({"name": dl["motherName"], "roleGuess": "mother"})

        all_processed_gerbils.append({
            "Id": scoped_id,
            "Name": name_val,
            "Gender": gender,
            "Status": status,
            "LitterId": scoped_litter_id,
            "OriginContactId": None,
            "ReceiverContactId": receiver_cid, # mapped in step 5
            "EnclosureId": None,
            "ColorVarietyId": color_variety_id,
            "DateOfBirth": dob_val,
            "DateOfDeath": dod_val,
            "CauseOfDeath": da.get("deathCause"),
            "GoHomeDate": gohome_val,
            "Genotype": genotype_val,
            "Notes": None,
            "ImportSource": "Wurfchronik-Detail.docx",
            "ExternalRef": f"docx-{idx}-{normalize_name(name_val)}-{dob_val or '0001-01-01'}",
            "RawImportData": raw_import_payload,
            "OriginBreeder": "Zucht der kleinen Chaoten",
            "NameSearch": normalize_name(name_val),
            "CharacterTraits": [],
            "CharacterNote": None,
            "IsDeaf": None,
            "IsResident": is_resident,
            "parentRefs": parent_refs,
            "_photos": da.get("photos", []),
            "_old_scoped_litter_id": scoped_litter_id,
            "_eff_dob": dob_val or "2020-01-01",
            "_birth_date": dob_val,
            "_filename": "Wurfchronik-Detail.docx",
            "_old_id": name_val
        })

    # Build parenting dates lookup using old scoped IDs
    parent_litter_dates = {}

    for l in resolved_litters:
        ld = l["Date"]
        if ld:
            for pid in [l["FatherId"], l["MotherId"]]:
                if pid:
                    parent_litter_dates.setdefault(pid, []).append(ld)

    def are_compatible(g1, g2):
        # Must have same gender (or one unknown)
        if g1["Gender"] != "unknown" and g2["Gender"] != "unknown" and g1["Gender"] != g2["Gender"]:
            bd1 = g1.get("_birth_date")
            bd2 = g2.get("_birth_date")
            if not (bd1 and bd2 and bd1 == bd2):
                return False
        
        bd1 = g1.get("_birth_date")
        bd2 = g2.get("_birth_date")
        
        # New rule: if name and parents match, they are compatible regardless of DOB!
        p1 = g1.get("parentRefs", [])
        p2 = g2.get("parentRefs", [])
        f1 = next((p["name"] for p in p1 if p.get("roleGuess") == "father"), "")
        m1 = next((p["name"] for p in p1 if p.get("roleGuess") == "mother"), "")
        f2 = next((p["name"] for p in p2 if p.get("roleGuess") == "father"), "")
        m2 = next((p["name"] for p in p2 if p.get("roleGuess") == "mother"), "")
        
        parents_match = False
        if f1 and f2 and m1 and m2:
            if normalize_name(f1) == normalize_name(f2) and normalize_name(m1) == normalize_name(m2):
                parents_match = True
                
        if not parents_match:
            # If both have explicit birth dates, they must match within 30 days
            if bd1 and bd2:
                days1 = date_to_days(bd1)
                days2 = date_to_days(bd2)
                if days1 is not None and days2 is not None:
                    if abs(days1 - days2) > 30:
                        return False
                
        # If g1 has birth date, and g2 has parenting dates, birth date must be before parenting dates
        p_dates2 = parent_litter_dates.get(g2["Id"], [])
        if bd1:
            for pd in p_dates2:
                if pd <= bd1: # Can't have litter before or on birth date
                    return False
                    
        p_dates1 = parent_litter_dates.get(g1["Id"], [])
        if bd2:
            for pd in p_dates1:
                if pd <= bd2:
                    return False
                    
        return True

    def _record_source_files(g):
        """All distinct source files a single raw record drew from.

        ImportSource is either a comma-joined Stammbaum file list, a single
        Wurfchronik filename, or the per-litter filename; _filename is the
        primary file. We union both so nothing is lost."""
        files = set()
        imp = g.get("ImportSource")
        if imp:
            for part in str(imp).split(","):
                part = part.strip()
                if part:
                    files.add(part)
        fn = g.get("_filename")
        if fn:
            files.add(fn)
        return files

    def build_provenance(records, best_g, extra_notes=None):
        """Aggregate data-provenance across every raw record merged into one
        resolved gerbil. Returns a JSON string (stored on the Gerbil entity as a
        nullable text column) so the Rennmausakte can show where the entry came
        from. `records` is the list of raw records that became this gerbil;
        `best_g` is the chosen primary record."""
        source_files = set()
        from_wurfchronik = False
        any_conflict = False
        any_decision = False
        for r in records:
            source_files |= _record_source_files(r)
            for f in _record_source_files(r):
                if "wurfchronik" in f.lower():
                    from_wurfchronik = True
            if r.get("_conflict"):
                any_conflict = True
            if r.get("_resolved_by_decision"):
                any_decision = True

        notes = []
        merged_count = len(records)
        if merged_count > 1:
            notes.append(f"aus {merged_count} Datensätzen zusammengeführt")
        if any_decision:
            notes.append("per manueller Entscheidung zugeordnet")
        if any_conflict:
            notes.append("Konflikt per Entscheidung gelöst")

        # Parent derivation: surface the strongest parentRef method/confidence
        # the primary record carries (chart-position etc.).
        parent_method = None
        parent_confidence = None
        for ref in best_g.get("parentRefs", []) or []:
            if ref.get("method") and not parent_method:
                parent_method = ref.get("method")
            if ref.get("confidence") and not parent_confidence:
                parent_confidence = ref.get("confidence")

        if extra_notes:
            for n in extra_notes:
                if n and n not in notes:
                    notes.append(n)

        prov = {
            "sourceFiles": sorted(source_files),
            "mergedRecordCount": merged_count,
            "fromWurfchronik": from_wurfchronik,
            "notes": notes,
        }
        if parent_method:
            prov["parentMethod"] = parent_method
        if parent_confidence:
            prov["parentConfidence"] = parent_confidence
        return json.dumps(prov, ensure_ascii=False)

    # Group gerbils by name to perform deduplication
    gerbil_groups = {}
    for g in all_processed_gerbils:
        name_key = get_dedup_name_key(g["Name"])
        if not name_key:
            name_key = "unbekannt"
        gerbil_groups.setdefault(name_key, []).append(g)

    resolved_gerbils = []
    gerbil_id_map = {} # old_scoped_id -> final_id

    color_keys = set(variety_map.keys())

    for name_key, group in gerbil_groups.items():
        is_placeholder = (
            name_key in color_keys or 
            any(p in name_key for p in ["unbekannt", "unbenannt", "baby", "welpe", "jungtier", "unknown", "welpen"]) or
            len(name_key) <= 2
        )
        
        if is_placeholder:
            # Placeholders: do NOT merge, keep all separate
            for g in group:
                g["Provenance"] = build_provenance([g], g)
                resolved_gerbils.append(g)
                gerbil_id_map[g["Id"]] = g["Id"]
            continue

        # Partition group into compatible subsets
        sub_groups = []
        for g in group:
            placed = False
            for sub in sub_groups:
                if all(are_compatible(g, member) for member in sub):
                    sub.append(g)
                    placed = True
                    break
            if not placed:
                sub_groups.append([g])

        # Merge each partition sub-group into a single gerbil
        for sub in sub_groups:
            if len(sub) == 1:
                g = sub[0]
                g["Provenance"] = build_provenance([g], g)
                resolved_gerbils.append(g)
                gerbil_id_map[g["Id"]] = g["Id"]
                continue
            
            # Find the best primary record to merge into
            best_g = None
            best_score = -1
            for g in sub:
                score = 0
                if g["_old_scoped_litter_id"]: score += 10
                if g["DateOfBirth"]: score += 5
                if g["Genotype"]: score += 3
                if g["ColorVarietyId"]: score += 2
                if g["ImportSource"] and "stammbaum" in g["ImportSource"].lower(): score += 20
                if g["Notes"] and not any(kw in g["Notes"].lower() for kw in ["parent", "mutter", "vater", "dam", "sire"]): score += 1
                if len(g["Name"]) > len(name_key) + 5: # likely has clan suffix
                    score += 15
                if score > best_score:
                    best_score = score
                    best_g = g
            
            # Merge fields
            merged_notes = []
            if best_g["Notes"]:
                merged_notes.append(best_g["Notes"])
            
            # Merge photos
            merged_photos = list(best_g.get("_photos", []))

            # Track sources for debugging
            sources = [best_g["_filename"]]

            for g in sub:
                if g == best_g:
                    continue
                gerbil_id_map[g["Id"]] = best_g["Id"]
                sources.append(g["_filename"])

                for ph in g.get("_photos", []):
                    if ph not in merged_photos:
                        merged_photos.append(ph)

                if not best_g.get("_old_scoped_litter_id") and g.get("_old_scoped_litter_id"):
                    best_g["_old_scoped_litter_id"] = g["_old_scoped_litter_id"]
                if not best_g["LitterId"] and g["LitterId"]:
                    best_g["LitterId"] = g["LitterId"]
                if not best_g["DateOfBirth"] and g["DateOfBirth"]:
                    best_g["DateOfBirth"] = g["DateOfBirth"]
                if not best_g["DateOfDeath"] and g["DateOfDeath"]:
                    best_g["DateOfDeath"] = g["DateOfDeath"]
                if not best_g["CauseOfDeath"] and g["CauseOfDeath"]:
                    best_g["CauseOfDeath"] = g["CauseOfDeath"]
                if not best_g["GoHomeDate"] and g["GoHomeDate"]:
                    best_g["GoHomeDate"] = g["GoHomeDate"]
                if not best_g["Genotype"] and g["Genotype"]:
                    best_g["Genotype"] = g["Genotype"]
                if not best_g["ColorVarietyId"] and g["ColorVarietyId"]:
                    best_g["ColorVarietyId"] = g["ColorVarietyId"]
                if not best_g["OriginContactId"] and g["OriginContactId"]:
                    best_g["OriginContactId"] = g["OriginContactId"]
                if not best_g["ReceiverContactId"] and g["ReceiverContactId"]:
                    best_g["ReceiverContactId"] = g["ReceiverContactId"]
                if g["IsResident"]:
                    best_g["IsResident"] = True
                
                # Reconcile Gender: prefer a known gender over unknown, and prefer stammbaum over other sources
                if best_g["Gender"] == "unknown" and g["Gender"] != "unknown":
                    best_g["Gender"] = g["Gender"]
                elif best_g["Gender"] != "unknown" and g["Gender"] != "unknown" and best_g["Gender"] != g["Gender"]:
                    if g["ImportSource"] and "stammbaum" in g["ImportSource"].lower():
                        if not best_g["ImportSource"] or "stammbaum" not in best_g["ImportSource"].lower():
                            best_g["Gender"] = g["Gender"]
                
                # Status precedence: Deceased > GivenAway > Breeding/Pet
                if g["Status"] == "Deceased":
                    best_g["Status"] = "Deceased"
                elif g["Status"] == "GivenAway" and best_g["Status"] not in ["Deceased"]:
                    best_g["Status"] = "GivenAway"
                
                if g["Notes"] and g["Notes"] not in merged_notes:
                     # Ignore redundant dummy notes
                     if not any(kw in g["Notes"].lower() for kw in ["parent listed", "mutter von", "vater von", "dam of", "sire of"]):
                         merged_notes.append(g["Notes"])

            # Reconcile fields based on number of source files supporting them
            conflict_notes = []
            for field in ["DateOfBirth", "DateOfDeath", "Gender", "Genotype", "ColorVarietyId"]:
                votes = {}
                for g in sub:
                    val = g.get(field)
                    if val and val != "unknown":
                        # count source files
                        sources_count = len(str(g.get("ImportSource") or "").split(","))
                        votes[val] = votes.get(val, 0) + sources_count
                if votes:
                    best_val = max(votes, key=votes.get)
                    # If the records disagreed on a field, the merge had to pick a
                    # winner — record that as a provenance note.
                    if len(votes) > 1:
                        FIELD_LABEL = {
                            "DateOfBirth": "Geburtsdatum",
                            "DateOfDeath": "Sterbedatum",
                            "Gender": "Geschlecht",
                            "Genotype": "Genotyp",
                            "ColorVarietyId": "Farbschlag",
                        }
                        conflict_notes.append(
                            f"Konflikt bei {FIELD_LABEL[field]} per Mehrheitsentscheidung gelöst"
                        )
                    best_g[field] = best_val
                    # Keep helper fields in sync if we changed DateOfBirth
                    if field == "DateOfBirth":
                        best_g["_birth_date"] = best_val
                        best_g["_eff_dob"] = best_val or "2010-01-01"

            if merged_notes:
                best_g["Notes"] = " | ".join(merged_notes)

            best_g["_photos"] = merged_photos
            best_g["Provenance"] = build_provenance(sub, best_g, extra_notes=conflict_notes)

            # Print merge trace
            print(f"Deduplicated same-animal name '{best_g['Name']}': merged {len(sub)} entries across files: {', '.join(sources)}")

            resolved_gerbils.append(best_g)
            gerbil_id_map[best_g["Id"]] = best_g["Id"]

    print(f"Deduplicated to {len(resolved_gerbils)} unique gerbil records.")

    # Apply age-based death threshold (6.0 years) to all resolved gerbils
    dt_now = datetime.now()
    for g in resolved_gerbils:
        if g.get("Status") != "Deceased" and g.get("Status") != "GivenAway":
            if not g.get("DateOfDeath") and not g.get("ReceiverContactId"):
                dob_str = g.get("DateOfBirth")
                if dob_str:
                    try:
                        dt_dob = datetime.strptime(dob_str, "%Y-%m-%d")
                        age_years = (dt_now - dt_dob).days / 365.25
                        if age_years >= 6.0:
                            g["Status"] = "Deceased"
                    except Exception:
                        pass

    # 5. Map Gerbils to Litters
    for g in resolved_gerbils:
        old_lid = g["_old_scoped_litter_id"]
        l_guid = litter_id_map.get(old_lid)
        g["LitterId"] = l_guid
        
        if l_guid and l_guid in litter_by_scoped_id:
            l = litter_by_scoped_id[l_guid]
            l_date = l.get("Date")
            if l_date:
                if not g.get("DateOfBirth"):
                    g["DateOfBirth"] = l_date
                    g["_birth_date"] = l_date
                    g["_eff_dob"] = l_date

        # Apply age-based death threshold (6.0 years) to all resolved gerbils (including newly backfilled ones)
        if g.get("Status") != "Deceased" and g.get("Status") != "GivenAway":
            if not g.get("DateOfDeath") and not g.get("ReceiverContactId"):
                dob_str = g.get("DateOfBirth")
                if dob_str:
                    try:
                        dt_dob = datetime.strptime(dob_str, "%Y-%m-%d")
                        age_years = (dt_now - dt_dob).days / 365.25
                        if age_years >= 6.0:
                            g["Status"] = "Deceased"
                    except Exception:
                        pass

        # Clean helper fields
        del g["_old_scoped_litter_id"]
        del g["_eff_dob"]
        if "_birth_date" in g:
            del g["_birth_date"]
        del g["_filename"]
        del g["_old_id"]
        g.pop("_conflict", None)
        g.pop("_resolved_by_decision", None)

    # Gather final valid gerbil IDs
    valid_gerbil_ids = {g["Id"] for g in resolved_gerbils}

    # Create name lookup for resolved gerbils
    gerbil_by_norm_name = {}
    for g in resolved_gerbils:
        n_key = normalize_name(g["Name"])
        gerbil_by_norm_name.setdefault(n_key, []).append(g)
        
        # Also index by call-name to resolve parents who are only listed by call-name
        c_key = normalize_name(get_call_name(g["Name"]))
        if c_key != n_key:
            gerbil_by_norm_name.setdefault(c_key, []).append(g)

    # Map raw Guid if present (convert if old_id mapped to new_guid)
    for l in resolved_litters:
        if l["FatherId"] in gerbil_id_map:
            l["FatherId"] = gerbil_id_map[l["FatherId"]]
        if l["MotherId"] in gerbil_id_map:
            l["MotherId"] = gerbil_id_map[l["MotherId"]]

        # Clean foreign keys that do not point to a valid gerbil
        if l["FatherId"] and l["FatherId"] not in valid_gerbil_ids:
            l["FatherId"] = None
        if l["MotherId"] and l["MotherId"] not in valid_gerbil_ids:
            l["MotherId"] = None

    # Parent Resolver (Global Name Matching)
    resolved_fathers = 0
    resolved_mothers = 0

    gerbil_by_id_final = {g["Id"]: g for g in resolved_gerbils}

    def _final_gender(gid):
        g = gerbil_by_id_final.get(gid)
        return g["Gender"] if g else None

    def _resolve_name(name, prefer_gender, litter_date):
        """Resolve a parent name to the best matching final gerbil.

        Gender is a PREFERENCE, not a hard filter: a reversed parent (e.g. a
        female listed in the father position, as the Stammbaum often does) still
        resolves to a gerbil — the role is corrected afterwards by gender. This
        is what previously left FatherId/MotherId null (the candidate was
        filtered out for having the "wrong" gender for its slot).
        """
        if not name:
            return None
        cands = []
        for c in gerbil_by_norm_name.get(normalize_name(name), []):
            final_id = gerbil_id_map.get(c["Id"])
            if not final_id:
                continue
            final_c = gerbil_by_id_final.get(final_id)
            if not final_c:
                continue
            # Parent must be age-plausible: born before the litter and within the
            # gerbil lifespan (skips e.g. a 2013 animal for a 2022 litter).
            if not parent_age_plausible(final_c["DateOfBirth"], litter_date):
                continue
            cands.append(final_c)
        if not cands:
            return None
        # Prefer the gender expected for this role, then unknown, then anything.
        for pool in (
            [c for c in cands if c["Gender"] == prefer_gender],
            [c for c in cands if c["Gender"] == "unknown"],
            cands,
        ):
            if pool:
                return pool[0]
        return None

    for l in resolved_litters:
        # Pre-check: if _father_name points to a known female and _mother_name to a
        # known male → swap names (Stammbaum positions reversed). Helps the name
        # resolver pick the right same-name candidate before role normalization.
        f_name_pre = l.get("_father_name", "")
        m_name_pre = l.get("_mother_name", "")
        if f_name_pre and m_name_pre:
            f_gender = next((g["Gender"] for g in gerbil_by_norm_name.get(normalize_name(f_name_pre), []) if g["Gender"] != "unknown"), None)
            m_gender = next((g["Gender"] for g in gerbil_by_norm_name.get(normalize_name(m_name_pre), []) if g["Gender"] != "unknown"), None)
            if f_gender == "female" and m_gender == "male":
                l["_father_name"], l["_mother_name"] = m_name_pre, f_name_pre

        if l["_father_name"] and not l["FatherId"]:
            cand = _resolve_name(l["_father_name"], "male", l["Date"])
            if cand:
                l["FatherId"] = cand["Id"]
                resolved_fathers += 1

        if l["_mother_name"] and not l["MotherId"]:
            cand = _resolve_name(l["_mother_name"], "female", l["Date"])
            if cand:
                l["MotherId"] = cand["Id"]
                resolved_mothers += 1

        # Cleanup internal keys
        del l["_father_name"]
        del l["_mother_name"]
        del l["_filename"]

    # Role normalization: assign each resolved parent to the role matching its
    # gender, eliminate self-pairings (same animal in both roles), and never let
    # impossible duplicates survive (two males / two females). This corrects
    # reversed Stammbaum positions including the cases the simple swap missed
    # (one parent of "unknown" gender, or a self-paired litter).
    role_fixes = 0
    for l in resolved_litters:
        father, mother = assign_parent_roles(l.get("FatherId"), l.get("MotherId"), _final_gender)
        if (l.get("FatherId"), l.get("MotherId")) != (father, mother):
            role_fixes += 1
        l["FatherId"] = father
        l["MotherId"] = mother
    if role_fixes:
        print(f"Role-normalization: corrected {role_fixes} litter(s) (gender roles / self-pairings).")

    # Parent-age sanity check: drop any resolved parent that cannot belong to the
    # litter — born after the offspring, or more than a gerbil lifespan earlier.
    # Catches mis-resolved links the name matcher still let through (e.g. Jayjay,
    # *2013, wrongly attached to Solice's 2022 litter).
    age_drops = []
    for l in resolved_litters:
        ldate = l.get("Date")
        for role in ("FatherId", "MotherId"):
            pid = l.get(role)
            if not pid:
                continue
            p = gerbil_by_id_final.get(pid)
            if p and not parent_age_plausible(p.get("DateOfBirth"), ldate):
                age_drops.append((l.get("Name"), role, p.get("Name"), p.get("DateOfBirth"), ldate))
                l[role] = None
    if age_drops:
        print(f"Parent-age sanity check: dropped {len(age_drops)} implausible parent link(s):")
        for lname, role, pname, pdob, ldate in age_drops[:20]:
            print(f"    {lname}: {role}={pname} (*{pdob}) vs litter {ldate}")

    print(f"Globally resolved {resolved_fathers} fathers and {resolved_mothers} mothers.")


    # 5b. Second-pass litter dedup: now that FatherId/MotherId are known,
    # merge litters that have the same date AND the same parents.
    # This is the core of the "sibling pairing" fix: Blue Wave and Sunny Sky
    # both come from Wonderman × Unique — their two separate litter records
    # must now become one, so their children share the same LitterId.
    litter_by_id_post = {l["Id"]: l for l in resolved_litters}
    gerbil_by_litter = {}
    for g in resolved_gerbils:
        lid = g.get("LitterId")
        if lid:
            gerbil_by_litter.setdefault(lid, []).append(g)

    def _litter_same_parents(l1, l2):
        """Strict: same date + both parents known and matching."""
        if l1["Date"] != l2["Date"]:
            return False
        f1, m1 = l1.get("FatherId"), l1.get("MotherId")
        f2, m2 = l2.get("FatherId"), l2.get("MotherId")
        if not f1 or not f2 or not m1 or not m2:
            return False
        return f1 == f2 and m1 == m2

    by_date2 = {}
    for l in resolved_litters:
        by_date2.setdefault(l["Date"], []).append(l)

    deduped2 = []
    litter_remap2 = {}  # old_id -> canonical_id

    for date_val, group in by_date2.items():
        sub_groups = []
        for l in group:
            placed = False
            for sub in sub_groups:
                if all(_litter_same_parents(l, m) for m in sub):
                    sub.append(l)
                    placed = True
                    break
            if not placed:
                sub_groups.append([l])

        for sub in sub_groups:
            # Prefer the canonical that has the most children
            canonical = max(sub, key=lambda l: len(gerbil_by_litter.get(l["Id"], [])))
            for l in sub:
                litter_remap2[l["Id"]] = canonical["Id"]
            deduped2.append(canonical)
            if len(sub) > 1:
                siblings = [g["Name"] for l in sub for g in gerbil_by_litter.get(l["Id"], []) if l is not canonical]
                print(f"Sibling-Litter-Merge on {date_val}: {canonical['Name']} absorbed sibling half — children now share LitterId: {[g['Name'] for g in gerbil_by_litter.get(canonical['Id'], [])] + siblings}")

    # Remap LitterId in all gerbils
    n_remapped = 0
    for g in resolved_gerbils:
        old_lid = g.get("LitterId")
        if old_lid and old_lid in litter_remap2 and litter_remap2[old_lid] != old_lid:
            g["LitterId"] = litter_remap2[old_lid]
            n_remapped += 1

    n_merged2 = len(resolved_litters) - len(deduped2)
    if n_merged2:
        print(f"Sibling-Litter-Dedup: {n_merged2} additional litter record(s) merged ({n_remapped} gerbil LitterIds remapped).")
    resolved_litters = deduped2
    litter_by_scoped_id = {l["Id"]: l for l in resolved_litters}

    # Set IsBreeder and IsReceiver flags on contacts
    breeder_ids = {g["OriginContactId"] for g in resolved_gerbils if g.get("OriginContactId")}
    receiver_ids = {g["ReceiverContactId"] for g in resolved_gerbils if g.get("ReceiverContactId")}
    for c in resolved_contacts:
        c_id = c["Id"]
        is_breeder = c_id in breeder_ids
        is_receiver = c_id in receiver_ids
        if not is_breeder and not is_receiver:
            is_receiver = True
        c["IsBreeder"] = is_breeder
        c["IsReceiver"] = is_receiver

    # Set and map gerbilPhotos
    resolved_photos = []
    for g in resolved_gerbils:
        for idx, photo_rel in enumerate(g.get("_photos", [])):
            photo_guid = generate_guid(f"photo-{photo_rel}")
            ext = os.path.splitext(photo_rel)[1] or ".jpeg"
            fn_guid = photo_guid.replace("-", "")
            resolved_photos.append({
                "Id": photo_guid,
                "GerbilId": g["Id"],
                "FileName": f"{fn_guid}{ext}",
                "SortOrder": idx,
                "_source_path": photo_rel
            })

    # 6. Save final output JSON payload
    os.makedirs(OUTPUT_DIR, exist_ok=True)
    payload = {
        "contacts": resolved_contacts,
        "litters": resolved_litters,
        "gerbils": resolved_gerbils,
        "gerbilPhotos": resolved_photos
    }

    with open(OUTPUT_FILE, "w", encoding="utf-8") as f:
        json.dump(payload, f, ensure_ascii=False, indent=2)

    print(f"Successfully wrote database-ready import file to: {OUTPUT_FILE}")
    print(f"  Contacts: {len(payload['contacts'])}")
    print(f"  Litters:  {len(payload['litters'])}")
    print(f"  Gerbils:  {len(payload['gerbils'])}")
    print(f"  Photos:   {len(payload['gerbilPhotos'])}")

if __name__ == "__main__":
    main()