GerbilManager/tools/import/test_merge_resolve.py

"""Zero-dep tests for merge_and_resolve.py litter-dedup & parent-role logic.

Run: python test_merge_resolve.py   (exit 0 = all pass)

Covers the sibling-pairing data fix:
- litter_compatible(): same-date / compatible-parent dedup, incl. the dateless
  guard that stops unrelated nameless stubs from blind-merging.
- assign_parent_roles(): gender-correct role assignment, self-pairing removal,
  no two same-role parents — the fix for the 16 self-pairings / 30 gender-role
  errors that the old simple swap missed.
"""
import sys
import merge_and_resolve as m


def check(name, cond):
    if not cond:
        print(f"FAIL: {name}")
        check.failed += 1
    else:
        print(f"ok:   {name}")
check.failed = 0


def litter(date, fid=None, mid=None, fname=None, mname=None):
    return {
        "Date": date,
        "FatherId": fid,
        "MotherId": mid,
        "_father_name": fname,
        "_mother_name": mname,
    }


# ── litter_compatible: both sides fully parented ──
check(
    "same date + same parents → compatible",
    m.litter_compatible(litter("2018-09-22", "F", "M"), litter("2018-09-22", "F", "M")),
)
check(
    "same date + different parents → NOT compatible",
    not m.litter_compatible(litter("2018-09-22", "F", "M"), litter("2018-09-22", "X", "Y")),
)
check(
    "different date → NOT compatible",
    not m.litter_compatible(litter("2018-09-22", "F", "M"), litter("2019-01-01", "F", "M")),
)

# ── asymmetric (one resolved, one not), real date ──
check(
    "asymmetric same real date, names agree → merge",
    m.litter_compatible(
        litter("2018-09-22", "F", "M", "Wonderman", "Unique"),
        litter("2018-09-22", None, None, "Wonderman", "Unique"),
    ),
)
check(
    "asymmetric same real date, conflicting names → NO merge",
    not m.litter_compatible(
        litter("2018-09-22", "F", "M", "Wonderman", "Unique"),
        litter("2018-09-22", None, None, "Someone", "Else"),
    ),
)
check(
    "asymmetric real date, parented side nameless stub → merge (no conflict)",
    m.litter_compatible(
        litter("2018-09-22", "F", "M", "Wonderman", "Unique"),
        litter("2018-09-22", None, None, None, None),
    ),
)

# ── dateless guard: the bug that wrongly merged unrelated stubs ──
check(
    "dateless asymmetric, NO name evidence → do NOT merge (was the bug)",
    not m.litter_compatible(
        litter(None, "F", "M", "Akina", "Arrow"),
        litter(None, None, None, None, None),
    ),
)
check(
    "dateless asymmetric WITH positive name match → merge",
    m.litter_compatible(
        litter(None, "F", "M", "Akina", "Arrow"),
        litter(None, None, None, "Akina", None),
    ),
)
check(
    "dateless asymmetric, contradicting names → do NOT merge",
    not m.litter_compatible(
        litter(None, "F", "M", "Akina", "Arrow"),
        litter(None, None, None, "Mismatch", None),
    ),
)

# ── neither side parented → never blind-merge ──
check(
    "neither parented, same date → NOT compatible",
    not m.litter_compatible(litter("2018-09-22"), litter("2018-09-22")),
)


# ── assign_parent_roles ──
GENDER = {"bock": "male", "bock2": "male", "maus": "female", "maus2": "female", "u": "unknown", "u2": "unknown"}
gof = lambda gid: GENDER.get(gid)

check("correct roles stay put", m.assign_parent_roles("bock", "maus", gof) == ("bock", "maus"))
check("reversed roles get swapped", m.assign_parent_roles("maus", "bock", gof) == ("bock", "maus"))
check(
    "self-pairing collapses to gender-correct single role (male→father)",
    m.assign_parent_roles("bock", "bock", gof) == ("bock", None),
)
check(
    "self-pairing collapses to gender-correct single role (female→mother)",
    m.assign_parent_roles("maus", "maus", gof) == (None, "maus"),
)
check(
    "female in father slot, empty mother → moved to mother",
    m.assign_parent_roles("maus", None, gof) == (None, "maus"),
)
check(
    "male in mother slot, empty father → moved to father",
    m.assign_parent_roles(None, "bock", gof) == ("bock", None),
)
check(
    "two males → keep one father, drop impossible second",
    m.assign_parent_roles("bock", "bock2", gof) == ("bock", None),
)
check(
    "two females → keep one mother, drop impossible second",
    m.assign_parent_roles("maus", "maus2", gof) == (None, "maus"),
)
check(
    "male + unknown → unknown fills mother",
    m.assign_parent_roles("bock", "u", gof) == ("bock", "u"),
)
check(
    "female + unknown → unknown fills father",
    m.assign_parent_roles("u", "maus", gof) == ("u", "maus"),
)
check("both empty → both None", m.assign_parent_roles(None, None, gof) == (None, None))
check(
    "single unknown parent kept as father",
    m.assign_parent_roles("u", None, gof) == ("u", None),
)


# ── name helpers ──
check("names_no_conflict: one side empty", m.names_no_conflict(litter(None, fname="A"), litter(None)))
check(
    "names_no_conflict: contradiction detected",
    not m.names_no_conflict(litter(None, fname="A"), litter(None, fname="B")),
)
check("names_overlap: matching father name", m.names_overlap(litter(None, fname="A"), litter(None, fname="A")))
check("names_overlap: nothing in common", not m.names_overlap(litter(None, fname="A"), litter(None, mname="B")))


# ── canon_name_key: v.d. ↔ von den abbreviation folds to one key (#30) ──
check("canon_name_key: v.d. and von den fold equal",
      m.canon_name_key("BlackFire v.d. Kleinen Chaoten")
      == m.canon_name_key("BlackFire von den Kleinen Chaoten"))
check("canon_name_key: distinct names stay distinct",
      m.canon_name_key("Theodore von den Kleinen Chaoten")
      != m.canon_name_key("Tony von den Kleinen Chaoten"))

# ── is_external_origin: pet-shop / private / foreign founders (#5/#13/#28) ──
check("merge is_external_origin: 'von Privat'", m.is_external_origin("Bill von Privat"))
check("merge is_external_origin: 'vom Zooladen (OBI)'",
      m.is_external_origin("Cooky vom Zooladen (OBI)"))
check("merge is_external_origin: foreign Croatia",
      m.is_external_origin("Zadar from Zeko i ptica, Croatia"))
check("merge is_external_origin: 'of Black Forest' NOT external",
      not m.is_external_origin("Hagrid Rubeus of Black Forest", "Black Forest"))


# ── parent_age_plausible: born before child, within ~6y lifespan ──
check("age: parent 1y before child → plausible", m.parent_age_plausible("16.04.2021", "27.03.2022"))
check("age: parent born AFTER child → implausible", not m.parent_age_plausible("2023-01-01", "2022-03-27"))
check("age: parent born SAME day → implausible", not m.parent_age_plausible("2022-03-27", "2022-03-27"))
check("age: 9 years older (Jayjay→Solice) → implausible", not m.parent_age_plausible("19.06.2013", "27.03.2022"))
check("age: exactly ~5y older → plausible", m.parent_age_plausible("01.06.2017", "01.05.2022"))
check("age: 7 years older → implausible", not m.parent_age_plausible("25.10.2018", "13.08.2025"))
check("age: unknown parent dob → plausible (can't disprove)", m.parent_age_plausible(None, "2022-03-27"))
check("age: unknown litter date → plausible", m.parent_age_plausible("2021-04-16", None))


# ── pick_parent_ref: prefer age-plausible ref, avoid duplicating the other role ──
def pref(name, role, dob=None):
    return {"name": name, "roleGuess": role, "dob": dob}

# Solice case: first father ref (Jayjay, no DOB) loses to the dated, plausible Lui.
solice_refs = [
    pref("Jayjay", "father"),
    pref("Lui von den Kleinen Chaoten", "mother", "16.04.2021"),
    pref("Lui von den Kleinen Chaoten", "father", "16.04.2021"),
    pref("Molly of Black Forest", "mother", "13.09.2021"),
]
f = m.pick_parent_ref(solice_refs, "father", "27.03.2022")
check("pick: father = plausible-dated Lui, not first-listed Jayjay",
      f and f["name"] == "Lui von den Kleinen Chaoten")
mo = m.pick_parent_ref(solice_refs, "mother", "27.03.2022", avoid_name=f["name"])
check("pick: mother = Molly (Lui avoided as it is the father)",
      mo and mo["name"] == "Molly of Black Forest")

check("pick: a dated-but-impossible ref loses to a plausible one",
      m.pick_parent_ref([pref("Old", "father", "2010-01-01"), pref("Dad", "father", "2021-01-01")],
                        "father", "2022-03-27")["name"] == "Dad")
check("pick: no ref for role → None",
      m.pick_parent_ref([pref("X", "mother", "2021-01-01")], "father", "2022-03-27") is None)
check("pick: single ref is returned",
      m.pick_parent_ref([pref("Solo", "father")], "father", "2022-03-27")["name"] == "Solo")

# Gender-aware: a dated FEMALE ref must not win the father slot over an undated
# male/unknown one (Molly regression: Hagrid (unknown, no DOB) vs Danielle
# (female, dated) → father must be Hagrid, not Danielle).
molly_refs = [
    pref("Hagrid Rubeus of Black Forest", "father"),
    pref("Arya Stark von den Kleinen Chaoten", "mother", "30.06.2020"),
    pref("Danielle von den Kleinen Chaoten", "father", "04.03.2020"),
    pref("Hagrid Rubeus of Black Forest", "mother", "18.07.2019"),
]
gender = {
    m.normalize_name("Hagrid Rubeus of Black Forest"): None,         # unknown
    m.normalize_name("Danielle von den Kleinen Chaoten"): "female",
    m.normalize_name("Arya Stark von den Kleinen Chaoten"): "female",
}
gof = lambda name: gender.get(m.normalize_name(name))
fr = m.pick_parent_ref(molly_refs, "father", "13.09.2021", gender_of=gof)
check("pick(gender): father = unknown-sex Hagrid, not dated female Danielle",
      fr and fr["name"] == "Hagrid Rubeus of Black Forest")
mr = m.pick_parent_ref(molly_refs, "mother", "13.09.2021", avoid_name=fr["name"], gender_of=gof)
check("pick(gender): mother = Arya (female)", mr and mr["name"].startswith("Arya"))


# ── Provenance history: chronological, file-attributed German log ──
import json as _json


def _hist(prov_json):
    return _json.loads(prov_json)["history"]


# build_entity_provenance carries history through verbatim.
_prov = _json.loads(
    m.build_entity_provenance(["A.xlsx"], 1, notes=["x"], history=["line one"])
)
check("build_entity_provenance includes history key", _prov.get("history") == ["line one"])
check("build_entity_provenance defaults history to []",
      _json.loads(m.build_entity_provenance(["A.xlsx"], 1)).get("history") == [])

# Single-record gerbil history: names the file and the per-field facts.
g_single = {
    "Name": "Picus", "DateOfBirth": "2022-03-27", "Gender": "male",
    "Genotype": "aa", "ColorVarietyId": None, "DateOfDeath": None,
    "ImportSource": "Stammbaum von Picus Son.xlsx",
    "_filename": "Stammbaum von Picus Son.xlsx", "parentRefs": [],
}
h = m._build_gerbil_history([g_single], g_single, {})
check("history: first line names the source file",
      h[0] == "In „Stammbaum von Picus Son.xlsx“ gefunden.")
check("history: dob line names file + formatted date",
      "Geburtsdatum (27.03.2022) aus „Stammbaum von Picus Son.xlsx“." in h)
check("history: gender line is German + file-attributed",
      "Geschlecht (männlich) aus „Stammbaum von Picus Son.xlsx“." in h)
check("history: genotype line file-attributed",
      "Genotyp aus „Stammbaum von Picus Son.xlsx“." in h)

# Merged gerbil: a field sourced from a DIFFERENT file is attributed to THAT file.
g_best = {
    "Name": "Solice", "DateOfBirth": "2022-03-27", "Gender": "male",
    "Genotype": None, "ColorVarietyId": None, "DateOfDeath": None,
    "ImportSource": "Stammbaum von Picus Son.xlsx",
    "_filename": "Stammbaum von Picus Son.xlsx", "parentRefs": [],
}
g_other = {
    "Name": "Solice", "DateOfBirth": "2022-03-27", "Gender": "male",
    "Genotype": "aa", "ColorVarietyId": None, "DateOfDeath": None,
    "ImportSource": "Wurfchronik-Detail.docx",
    "_filename": "Wurfchronik-Detail.docx", "parentRefs": [],
}
# Genotype was filled from g_other → its line must name the docx file.
g_best["Genotype"] = "aa"
fs = {"DateOfBirth": g_best, "Gender": g_best, "Genotype": g_other}
h2 = m._build_gerbil_history([g_best, g_other], g_best, fs)
check("history(merge): genotype attributed to the file that supplied it",
      "Genotyp aus „Wurfchronik-Detail.docx“." in h2)
check("history(merge): dob attributed to primary file",
      "Geburtsdatum (27.03.2022) aus „Stammbaum von Picus Son.xlsx“." in h2)
check("history(merge): merge line names the absorbed file",
      "Auch in „Wurfchronik-Detail.docx“ gefunden → Datensätze zusammengeführt." in h2)
check("history(merge): Wurfchronik line present",
      "Angaben aus der Wurfchronik übernommen." in h2)

# Date formatting helper.
check("_de_date: ISO → DD.MM.YYYY", m._de_date("2022-03-27") == "27.03.2022")
check("_de_date: passes through non-ISO", m._de_date("unbekannt") == "unbekannt")


# ── Discard history: a discarded source value records reason + replacement ──
# _format_discard: majority-vote conflict (losing value + file → winner + file).
_d_mehr = m._format_discard({
    "label": "Geburtsdatum", "value": "14.06.2015", "file": "A.xlsx",
    "reason": "abweichend", "replacement": "14.06.2017", "repl_file": "B.xlsx",
    "replacement_note": "Mehrheit",
})
check("discard: starts with warning marker", _d_mehr.startswith(m.DISCARD_MARK))
check("discard(majority): names losing value + its file",
      "Geburtsdatum 14.06.2015 aus „A.xlsx“ verworfen" in _d_mehr)
check("discard(majority): states the reason", "— abweichend" in _d_mehr)
check("discard(majority): names replacement + its file + note",
      "14.06.2017 aus „B.xlsx“ verwendet (Mehrheit)." in _d_mehr)

# _format_discard: a parent dropped with NO replacement.
_d_noerepl = m._format_discard({
    "text": "Vater „Jayjay“ (*19.06.2013) verworfen — unplausibel (9 Jahre älter "
            "als das Kind); kein Ersatz",
})
check("discard(text): verbatim text gets the warning marker",
      _d_noerepl == m.DISCARD_MARK + "Vater „Jayjay“ (*19.06.2013) verworfen — "
      "unplausibel (9 Jahre älter als das Kind); kein Ersatz")

# explain_pick_rejections: a wrong-sex father candidate is explained (Molly case).
_picks = m.explain_pick_rejections(
    molly_refs, "father", "13.09.2021", fr, gender_of=gof,
)
_pick_father = next((d for d in _picks if "Danielle" in (d.get("value") or "")), None)
check("pick-reject: wrong-sex father candidate is recorded",
      _pick_father is not None)
check("pick-reject: reason = wrong sex for the father role",
      _pick_father and "falsches Geschlecht für die Vaterrolle" in _pick_father["reason"])
check("pick-reject: replacement names the chosen Hagrid",
      _pick_father and "Hagrid" in (_pick_father.get("replacement") or ""))

# explain_pick_rejections: an age-impossible candidate is explained.
_age_refs = [pref("Old", "father", "2010-01-01"), pref("Dad", "father", "2021-01-01")]
_chosen = m.pick_parent_ref(_age_refs, "father", "2022-03-27")
_age_picks = m.explain_pick_rejections(_age_refs, "father", "2022-03-27", _chosen)
check("pick-reject(age): age-impossible candidate recorded with reason",
      any("unplausibles Alter" in d["reason"] for d in _age_picks))

# _build_gerbil_history threads field_discards (after merge) and parent_discards
# (after the parent line) into the timeline.
g_disc = {
    "Name": "Solice", "DateOfBirth": "2022-03-27", "Gender": "male",
    "Genotype": None, "ColorVarietyId": None, "DateOfDeath": None,
    "ImportSource": "Stammbaum.xlsx", "_filename": "Stammbaum.xlsx",
    "parentRefs": [],
    "_discarded": [{"text": "Vater „Jayjay“ verworfen — unplausibel; kein Ersatz"}],
}
h_disc = m._build_gerbil_history(
    [g_disc], g_disc, {},
    field_discards=[{
        "label": "Geschlecht", "value": "weiblich", "file": "Wurfchronik.docx",
        "reason": "abweichend", "replacement": "männlich", "repl_file": "Stammbaum.xlsx",
        "replacement_note": "Mehrheit",
    }],
    parent_discards=g_disc["_discarded"],
)
check("history: field-discard line present (majority vote)",
      any("Geschlecht weiblich aus „Wurfchronik.docx“ verworfen" in s for s in h_disc))
check("history: parent-discard line present (dropped parent)",
      any("Vater „Jayjay“ verworfen" in s for s in h_disc))
check("history: discard lines carry the warning marker",
      all(s.startswith(m.DISCARD_MARK) for s in h_disc if "verworfen" in s))


# ── enrich_from_contracts: SaleContract record emission ───────────────────────
# A contract whose buyer resolves to a contact and whose animal call-name matches
# a breeder-owned gerbil must yield a SaleContract record carrying the buyer
# ContactId, a deterministic Id, the parsed dates and the matched gerbil id.
def _balu():
    # A breeder-owned ("Chaoten") gerbil whose call-name is "Balu".
    return {
        "Id": "11111111-1111-1111-1111-111111111111",
        "Name": "Balu von den kleinen Chaoten",
        "Gender": "male", "DateOfBirth": "2022-05-01",
        "OriginBreeder": "Zucht der kleinen Chaoten",
        "Status": "Active", "ColorVarietyId": None,
        "Provenance": None,
    }

_g = _balu()
_resolved = [_g]
_contacts_by_norm = {}
_contracts = [{
    "sourceFile": "Zucht der kleinen Chaoten _ Schwarz (Balu) - Max Muster_.docx",
    "buyer": "Max Muster", "animals": ["Balu"], "color": "schwarz",
    "gender": "Male", "dob": "2022-05-01",
    "handoverDate": "2022-07-01", "contractDate": "2022-07-01", "price": "30,00",
}]
_stats, _sale = m.enrich_from_contracts(_contracts, _resolved, _contacts_by_norm, {})
check("contracts: exactly one SaleContract record emitted", len(_sale) == 1)
_rec = _sale[0] if _sale else {}
check("contracts: record Id is deterministic from filename",
      _rec.get("Id") == m.generate_guid(
          "contract-Zucht der kleinen Chaoten _ Schwarz (Balu) - Max Muster_.docx"))
check("contracts: record ContactId is the resolved buyer contact",
      _rec.get("ContactId") and
      _rec["ContactId"] == _contacts_by_norm.get(m.normalize_name("Max Muster"), {}).get("Id"))
check("contracts: record lists the matched gerbil",
      _rec.get("Animals") == [_g["Id"]])
check("contracts: price parsed as float", _rec.get("Price") == 30.0)
check("contracts: dates carried through",
      _rec.get("HandoverDate") == "2022-07-01" and _rec.get("ContractDate") == "2022-07-01")
check("contracts: stats count the created record", _stats.get("records_created") == 1)

# A dateless contract is skipped from record creation (non-nullable DateOnly) but
# still counted, and the buyer contact is still created.
_c2 = [{
    "sourceFile": "Zucht der kleinen Chaoten _ (Nala) - Erika Muster_.docx",
    "buyer": "Erika Muster", "animals": ["Nala"], "color": "",
    "gender": "", "dob": "", "handoverDate": "", "contractDate": "", "price": "",
}]
_stats2, _sale2 = m.enrich_from_contracts(_c2, [], {}, {})
check("contracts: dateless contract skipped from records", len(_sale2) == 0)
check("contracts: dateless contract counted", _stats2.get("dateless_skipped") == 1)

# Price-only / no-date fallback: contract with only a contractDate gets it copied
# into HandoverDate too (and vice versa), and an animal-less contract still
# becomes a record (better to show it than drop it).
_c3 = [{
    "sourceFile": "Zucht der kleinen Chaoten _ (Unbekannt) - Tom Muster_.docx",
    "buyer": "Tom Muster", "animals": ["Unbekannt"], "color": "",
    "gender": "", "dob": "", "handoverDate": "", "contractDate": "2023-01-15",
    "price": "",
}]
_stats3, _sale3 = m.enrich_from_contracts(_c3, [], {}, {})
check("contracts: animal-less contract still becomes a record", len(_sale3) == 1)
check("contracts: missing handover falls back to contract date",
      _sale3 and _sale3[0]["HandoverDate"] == "2023-01-15"
      and _sale3[0]["ContractDate"] == "2023-01-15")
check("contracts: animal-less record has empty Animals list",
      _sale3 and _sale3[0]["Animals"] == [])


# ── Integration: assert the resolved_import.json output reflects the ticket fixes ──
# (Only when the pipeline has already been run; tolerant if the file is absent.)
import os as _os, json as _json
_resolved = _os.path.join(_os.path.dirname(__file__), "output", "resolved_import.json")
if _os.path.exists(_resolved):
    _d = _json.load(open(_resolved, encoding="utf-8"))
    _G = {g["Id"]: g for g in _d["gerbils"]}
    _L = {l["Id"]: l for l in _d["litters"]}

    def _find(sub, dob=None):
        sub = sub.lower()
        for g in _d["gerbils"]:
            if sub in g["Name"].lower() and (dob is None or g.get("DateOfBirth") == dob):
                return g
        return None

    def _parents(g):
        l = _L.get(g.get("LitterId")) if g else None
        if not l:
            return (None, None)
        f = _G.get(l.get("FatherId"))
        m = _G.get(l.get("MotherId"))
        return (f["Name"] if f else None, m["Name"] if m else None)

    # #5/#13/#28: external founders → no parents
    for tag, nm in [("#5 Bill", "Bill von Privat"),
                    ("#13 Cooky", "Cooky vom Zooladen"),
                    ("#28 Zadar", "Zadar from Zeko")]:
        g = _find(nm)
        check(f"{tag}: external founder has no litter/parents",
              g is not None and not g.get("LitterId"))

    # #18: Hagrid is a SINGLE resolved record (the DOB-less shell merged away)
    _hag = [g for g in _d["gerbils"] if g["Name"].lower() == "hagrid rubeus of black forest"]
    check("#18 Hagrid: exactly one resolved record", len(_hag) == 1)
    if _hag:
        # #17/#20: external ancestor is NOT resident; parents Snickers × Milka
        check("#17/#20 Hagrid: isResident == False", _hag[0].get("IsResident") is False)
        f, mo = _parents(_hag[0])
        check("#18 Hagrid: father Snickers, mother Milka",
              (f or "").startswith("Snickers") and (mo or "").startswith("Milka"))

    # #2 Mozart → female; #16 Arya, #23 Yuki, #36 Gold parent corrections
    _moz = _find("Mozart of Lennylengo")
    check("#2 Mozart: gender female", _moz is not None and _moz.get("Gender") == "female")

    def _check_parents(tag, nm, exp_f, exp_m):
        g = _find(nm)
        f, mo = _parents(g)
        check(f"{tag}: father ~ {exp_f}", (f or "").lower().startswith(exp_f.lower()))
        check(f"{tag}: mother ~ {exp_m}", (mo or "").lower().startswith(exp_m.lower()))

    _check_parents("#16 Arya", "Arya Stark von den Kleinen", "Vance", "Sansa Stark")
    _check_parents("#23 Yuki", "Yuki von den Kleinen", "Chevrolet Camaro", "Izumi")
    _check_parents("#36 Gold", "Gold v.d. Kleinen", "Trogir", "Chelsea")
    _check_parents("#35 Zac", "Zac gen. Action", "Vance", "Dorie")
    _check_parents("#9 Beatrice", "Beatrice von den kleinen", "Dante", "Malina")
    _check_parents("#30 Theodore", "Theodore von den Kleinen", "BlackFire", "Katara")
else:
    print("note: output/resolved_import.json not present — skipped integration assertions")


if check.failed:
    print(f"\n{check.failed} test(s) FAILED")
    sys.exit(1)
print("\nAll merge_and_resolve tests passed.")