From 72ad706ca6f2a6820395480f5fe91f40993a2b5e Mon Sep 17 00:00:00 2001 From: Gulum Date: Tue, 23 Jun 2026 08:25:43 +0200 Subject: [PATCH] =?UTF-8?q?fix(import):=20Eltern-/Herkunft-/Dubletten-Korr?= =?UTF-8?q?ekturen=20=E2=80=94=20Ticket-Triage=20Cluster?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Akane (Roni=Vater male + Mutter Fumi), Sunny (Bill+Melly), Danielle (Mutter Ella, motherDob-Disambiguierung), Cherry Berry (female), Black-Forest-Label vereinheitlicht, namenloser Bock entdoppelt (mergeExternalRefs). Co-Authored-By: Claude Opus 4.8 --- tools/import/conflict-decisions.json | 51 +++++- tools/import/extract.py | 7 +- tools/import/merge_and_resolve.py | 222 ++++++++++++++++++++++++++- tools/import/test_merge_resolve.py | 44 ++++++ 4 files changed, 319 insertions(+), 5 deletions(-) diff --git a/tools/import/conflict-decisions.json b/tools/import/conflict-decisions.json index f74b732..187a71b 100644 --- a/tools/import/conflict-decisions.json +++ b/tools/import/conflict-decisions.json @@ -1,5 +1,5 @@ { - "_doc": "Human conflict resolutions for the import quarantine (HUMANQUESTION section D / C6). The importer consumes this to UN-QUARANTINE an animal: for a matching (name + dob) it accepts the given authoritative field(s) — `genotype`, `farbschlag`, and/or `dateOfDeath` (DD.MM.YYYY) — and skips the conflict. Special field `correctDob` (DD.MM.YYYY): the matched (name + dob) record is a DUPLICATE with a WRONG birthdate — remap its DOB to `correctDob` BEFORE dedup so it merges into the canonical same-named animal. Key match = normalize(name) + dob, same identity as dedup. Maintained by god (Michael) as Julian/his wife answer the D-conflicts; originals (xlsx) stay read-only.", + "_doc": "Human conflict resolutions for the import quarantine (HUMANQUESTION section D / C6). The importer consumes this to UN-QUARANTINE an animal: for a matching (name + dob) it accepts the given authoritative field(s) — `genotype`, `farbschlag`, and/or `dateOfDeath` (DD.MM.YYYY) — and skips the conflict. Special field `correctDob` (DD.MM.YYYY): the matched (name + dob) record is a DUPLICATE with a WRONG birthdate — remap its DOB to `correctDob` BEFORE dedup so it merges into the canonical same-named animal. Other override fields per resolution: `gender` (male|female|m|w) — fix a misread box-colour gender (applies to stammbaum AND Wurfchronik/docx animals via merge_and_resolve.apply_decision_overrides); `father`/`mother` — authoritative parent NAMES; optional `fatherDob`/`motherDob` (DD.MM.YYYY) disambiguate a parent when several same-named animals exist. Special top-level array `addAnimals` [{name, gender, zucht?, dob?}] materialises a non-resident stub gerbil for a KNOWN parent that has no own source record (e.g. a mother named only on a Wurfchronik litter), so the litter's parent link resolves. Key match = normalize(call-name) + dob, same identity as dedup. Maintained by god (Michael) as Julian/his wife answer the D-conflicts; originals (xlsx) stay read-only.", "resolutions": [ { "name": "Firefly von den Kleinen Chaoten", @@ -328,6 +328,55 @@ "father": "Taro von den kleinen Chaoten", "mother": "Beatrice von den kleinen Chaoten", "source": "Züchterin 2026-06-22 — Ticket #11" + }, + { + "name": "Roni", + "dob": "27.01.2022", + "decision": "gender = male. Roni is the FATHER (not mother) of the T21/Z21/22 litters (Wurfchronik names father=Roni, mother=Fumi). Box colour / chart had her typed female, so role normalisation wrongly put Roni in the mother slot and left the father empty. Setting male lets Roni take the father role; Fumi (added via addAnimals) fills the mother slot.", + "gender": "male", + "source": "Züchterin 2026-06-22 — Ticket 88389f8e (Akane: Roni ist Vater)" + }, + { + "name": "Cherry Berry's Quqquluuruu", + "dob": "", + "decision": "gender = female (weisse Box = weiblich, blaue = maennlich). Pure pedigree ancestor without a birthdate, so the box-colour gender was never read — the Stammbaum showed a question mark instead of the female icon. dob MUST be empty so the name-only override key matches.", + "gender": "female", + "source": "Züchterin 2026-06-22 — Ticket 94892100 (Weibchen-Icon fehlt)" + }, + { + "name": "Sunny von PZ Karl", + "dob": "10.04.2014", + "decision": "father is Bill von Privat, mother is Melly von Privat (from Stammbaum von Yurikas und Pintos Sohn.xlsx / Renner-Pro-3). The chart-position resolver had wrongly given Hiro of Golden Lights as father.", + "father": "Bill von Privat", + "mother": "Melly von Privat", + "source": "Züchterin 2026-06-22 — Ticket 0ba551a3 (Sunny falscher Vater)" + }, + { + "name": "Danielle von den Kleinen Chaoten", + "dob": "04.03.2020", + "decision": "mother is Ella (Tochter von Louis+Roswitha, *10.06.2019) — Louis & Roswitha are Danielle's GRANDparents, not her parents. The father is an unnamed brother of Ella (a sibling pairing) with no own record, so he stays unknown. Two 'Ella' exist; motherDob pins the correct one (*10.06.2019), and the second Ella (*13.04.2023, born after Danielle) is age-impossible anyway.", + "mother": "Ella", + "motherDob": "10.06.2019", + "source": "Züchterin 2026-06-22 — Tickets f77aa4a6 / 4692fd5c (Danielle echte Mutter Ella)" + }, + { + "name": "Unbekannt", + "dob": "15.02.2024", + "decision": "Nameless buck *15.02.2024 (son of Inochi gen. Picu) is recorded twice — once via 'Stammbaum von Picus Son.xlsx' (externalRef stammbaum-unbekannt-15022024-2) and once via 'Stammbaum von Alberto Kids.xlsx' (externalRef stammbaum-unbekannt-15022024). Same buck. The two reconstructed litters (Picus Son vs Alberto Kids) differ only by the mother's birth YEAR (a typo): mother 13.11.2023 vs 13.11.2022. Merge the two bucks and the two mothers into one each via mergeExternalRefs.", + "mergeExternalRefs": [ + ["stammbaum-unbekannt-15022024-2", "stammbaum-unbekannt-15022024"], + ["stammbaum-unbekannt-13112023", "stammbaum-unbekannt-13112022"] + ], + "source": "Züchterin 2026-06-22 — Ticket f618dcc3 (doppelter namenloser Bock)" + } + ], + "addAnimals": [ + { + "name": "Fumi von den Kleinen Chaoten", + "gender": "female", + "zucht": "Zucht der kleinen Chaoten", + "decision": "Mother of the T21/Z21/22 litters (Wurfchronik names mother=Fumi, father=Roni) but she has no own source record, so her litters showed an empty mother. Materialised as a non-resident stub so the parent link resolves; her own parents stay unknown.", + "source": "Züchterin 2026-06-22 — Ticket 88389f8e (Akane: Mutter Fumi)" } ] } diff --git a/tools/import/extract.py b/tools/import/extract.py index b7cdd37..2ca792b 100644 --- a/tools/import/extract.py +++ b/tools/import/extract.py @@ -1151,7 +1151,10 @@ def apply_conflict_decisions(merged, conflicts, path): if d.get("father"): new_refs.append({ "name": d["father"], - "dob": "", + # Optional fatherDob disambiguates a parent when several + # same-named animals exist (the downstream resolver matches + # name AND, when given, dob — e.g. two „Ella“). + "dob": d.get("fatherDob", ""), "roleGuess": "father", "method": "decision", "confidence": "high" @@ -1159,7 +1162,7 @@ def apply_conflict_decisions(merged, conflicts, path): if d.get("mother"): new_refs.append({ "name": d["mother"], - "dob": "", + "dob": d.get("motherDob", ""), "roleGuess": "mother", "method": "decision", "confidence": "high" diff --git a/tools/import/merge_and_resolve.py b/tools/import/merge_and_resolve.py index 4a3bbea..7423cd0 100644 --- a/tools/import/merge_and_resolve.py +++ b/tools/import/merge_and_resolve.py @@ -17,6 +17,29 @@ DIR_PATH = r"C:\Users\gulum\dev\Wurfchronik_Bilder" SEEDS_PATH = r"C:\Users\gulum\dev\GerbilManager\gerbil-manager-web\src\genetics\colorVarietySeed.backend.json" OUTPUT_DIR = r"C:\Users\gulum\dev\GerbilManager\tools\import\output" OUTPUT_FILE = os.path.join(OUTPUT_DIR, "resolved_import.json") +CONFLICT_DECISIONS_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), + "conflict-decisions.json") + + +def load_conflict_decisions(): + """Load tools/import/conflict-decisions.json (the breeder's authoritative + answers). Returns (resolutions, addAnimals). Tolerates a missing/garbled file + by returning empty lists — the import must never crash on a bad decision file.""" + try: + with open(CONFLICT_DECISIONS_PATH, encoding="utf-8") as fh: + data = json.load(fh) + return (data.get("resolutions") or [], data.get("addAnimals") or []) + except (OSError, ValueError): + return ([], []) + + +def _norm_decision_gender(g): + """Map a free-text gender override ('m'/'w'/'männlich'/…) to 'male'/'female' + or None when it is neither (so a typo never blanks a real gender).""" + g = (g or "").strip().lower() + return {"m": "male", "männlich": "male", "male": "male", + "w": "female", "f": "female", "weiblich": "female", + "female": "female"}.get(g) def generate_guid(key_str): """Generate a stable UUID string based on a key.""" @@ -201,7 +224,8 @@ def get_normalized_contact_name(name): "birgitkropp": "Birgit Knopp", "birgittabüskens": "Birgitta Müller-Büskens", "birgittamüllerbüskens": "Brigitta Müller-Büskens", - "blackforestgv": "Black Forest", + "blackforest": "Clan of Black Forest", + "blackforestgv": "Clan of Black Forest", "brigittast": "Brigitta Struve", "buntefellnasen": "bunten Fellnasen", "buntenfellnase": "bunten Fellnasen", @@ -1664,12 +1688,20 @@ def main(): a["_mapped_litter_scoped_id"] = None + # A human conflict-decision override is authoritative — never let the + # Wurfchronik-Vorrang heuristic below re-attach the animal to a same-date + # chronicle litter and overrule the breeder's named parents. + _has_decision_parent = ( + (father_ref or {}).get("method") == "decision" or + (mother_ref or {}).get("method") == "decision") + # Wurfchronik-Vorrang: if the chart parents don't yield an exact # Wurfchronik match but EXACTLY ONE Wurfchronik litter (with named # parents) exists for this birthdate, that authoritative litter wins over # a fabricated chart-position litter (#12 Tony, #15 Odelia, #31 Jamie). dob_val_attach = parse_date(a.get("dob")) - if dob_val_attach and not is_external_origin(a.get("name"), a.get("zucht"), a.get("breeder")): + if dob_val_attach and not _has_decision_parent \ + and not is_external_origin(a.get("name"), a.get("zucht"), a.get("breeder")): same_date = md_litters_by_date.get(dob_val_attach, []) exact_hit = None if father_ref and mother_ref: @@ -1763,6 +1795,47 @@ def main(): "_scoped_mother_id": m_scoped_id or generate_guid(f"stammbaum-animal-{normalize_name(m_name)}") }) + elif _has_decision_parent and (father_ref or mother_ref): + # Single KNOWN parent from a human override (the other parent is + # genuinely unknown — e.g. a sibling-pairing where only the mother is + # named: Danielle's mother = Ella, father = unnamed brother). + # Create a one-parent virtual litter; the global name resolver fills + # the known side (gender + age-plausibility pick the right same-named + # animal, e.g. Ella *10.06.2019 over Ella *13.04.2023), the other + # side stays null. + known = father_ref or mother_ref + known_role = "father" if father_ref else "mother" + known_name = get_normalized_gerbil_name(known.get("name")) + dob_val = parse_date(a.get("dob")) + v_key = (normalize_name(known_name), known_role, dob_val or "0001-01-01") + if v_key in created_virtual_litters: + a["_mapped_litter_scoped_id"] = created_virtual_litters[v_key] + else: + l_scoped_id = generate_guid( + f"virtual-litter-1p-{v_key[0]}-{v_key[1]}-{v_key[2]}") + created_virtual_litters[v_key] = l_scoped_id + a["_mapped_litter_scoped_id"] = l_scoped_id + raw_litters.append({ + "Id": l_scoped_id, + "Name": f"Wurf von {known_name}", + "Date": dob_val, + "TotalBorn": None, + "DeathsWithin8Weeks": None, + "FatherId": None, + "MotherId": None, + "ExpectedGoHomeDate": None, + "Notes": "Pedigree virtual litter (ein Elternteil bekannt)", + "PairingCode": None, + "ExternalRef": f"virtual-{l_scoped_id}", + "LitterLetter": None, + "_father_name": known_name if known_role == "father" else "", + "_mother_name": known_name if known_role == "mother" else "", + "_filename": "Stammbaum", + "_scoped_id": l_scoped_id, + "_scoped_father_id": None, + "_scoped_mother_id": None, + }) + # 2. Resolve Contacts globally (deduplicate by normalized name) contact_by_norm_name = {} @@ -2362,6 +2435,144 @@ def main(): "_old_id": name_val }) + # ── Conflict-decision overrides at the gerbil level ─────────────────────── + # extract.apply_conflict_decisions already applies genotype/farbschlag/dod/ + # gender/parent overrides to STAMMBAUM animals (animals.json). Wurfchronik/ + # docx animals (docx_animals.json) never pass through that path, so a gender + # override on a Wurfchronik animal (e.g. „Roni“ — actually the FATHER) would + # otherwise be lost. Apply gender overrides here uniformly to every processed + # gerbil, materialise `addAnimals` stubs for known parents that have no own + # source record (e.g. „Fumi“), and collapse explicitly-paired duplicate + # records via `mergeExternalRefs` (nameless animals never auto-merge). + _decisions, _add_animals = load_conflict_decisions() + + def _gerbil_call_key(g): + return normalize_name(get_call_name(g.get("Name") or "")) + + # 1) gender overrides (match normalize(call-name) + ISO dob; empty decision + # dob = name-only, for ancestors without a birthdate). + _gender_overrides = {} + for d in _decisions: + gg = _norm_decision_gender(d.get("gender")) + if not gg: + continue + ck = normalize_name(get_call_name(d.get("name") or "")) + iso = parse_date(d.get("dob")) if d.get("dob") else "" + _gender_overrides[(ck, iso or "")] = gg + _gender_applied = 0 + for g in all_processed_gerbils: + ck = _gerbil_call_key(g) + iso = g.get("DateOfBirth") or "" + gg = _gender_overrides.get((ck, iso)) or _gender_overrides.get((ck, "")) + if gg and g.get("Gender") != gg: + g["Gender"] = gg + g["_resolved_by_decision"] = True + _gender_applied += 1 + + # 2) addAnimals — materialise a non-resident stub for a KNOWN parent that has + # no own source record, so its litter's parent link can resolve. Skip when + # a record of that name already exists (avoid duplicates). + _existing_call_keys = {_gerbil_call_key(g) for g in all_processed_gerbils} + _stubs_added = 0 + for a in _add_animals: + nm = (a.get("name") or "").strip() + if not nm: + continue + ck = normalize_name(get_call_name(nm)) + if ck in _existing_call_keys: + continue + gender = _norm_decision_gender(a.get("gender")) or "unknown" + dob_iso = parse_date(a.get("dob")) if a.get("dob") else None + zucht = (a.get("zucht") or "").strip() or None + breeder_disp = None + if zucht: + nb, keep = get_normalized_contact_name(zucht) + breeder_disp = nb if keep else None + stub_id = generate_guid(f"decision-animal-{normalize_name(nm)}") + all_processed_gerbils.append({ + "Id": stub_id, + "Name": get_normalized_gerbil_name(nm), + "Gender": gender, + "Status": "Breeding", + "LitterId": None, + "OriginContactId": None, + "ReceiverContactId": None, + "EnclosureId": None, + "ColorVarietyId": None, + "DateOfBirth": dob_iso, + "DateOfDeath": None, + "CauseOfDeath": None, + "GoHomeDate": None, + "Genotype": None, + "Notes": None, + "ImportSource": "conflict-decisions.json", + "ExternalRef": f"decision-{normalize_name(nm)}", + "RawImportData": json.dumps({"addedByDecision": True}, ensure_ascii=False), + "OriginBreeder": breeder_disp, + "NameSearch": normalize_name(get_normalized_gerbil_name(nm)), + "CharacterTraits": [], + "CharacterNote": None, + "IsDeaf": None, + "IsResident": False, + "parentRefs": [], + "_photos": [], + "_old_scoped_litter_id": None, + "_eff_dob": dob_iso or "2015-01-01", + "_birth_date": dob_iso, + "_filename": "conflict-decisions.json", + "_old_id": stub_id, + "_resolved_by_decision": True, + }) + _existing_call_keys.add(ck) + _stubs_added += 1 + + # 3) mergeExternalRefs — collapse explicitly-paired duplicate records (esp. + # nameless animals that the name-based dedup keeps separate). Each pair is + # [keepRefSuffix, dropRefSuffix]; the drop record is removed and its old id + # mapped onto the keep record. Refs are matched as a suffix of ExternalRef + # (extract animals become ExternalRef "stammbaum-"). + _by_ext = {} + for g in all_processed_gerbils: + er = g.get("ExternalRef") + if er: + _by_ext.setdefault(er, []).append(g) + + def _find_by_ref_suffix(suffix): + hits = [g for er, gs in _by_ext.items() if er and er.endswith(suffix) for g in gs] + return hits + + _premerged_ids = {} # dropped old id -> kept id (consumed in step 5 mapping) + _merge_pairs_done = 0 + for d in _decisions: + for pair in (d.get("mergeExternalRefs") or []): + if not isinstance(pair, (list, tuple)) or len(pair) != 2: + continue + keep_hits = _find_by_ref_suffix(pair[0]) + drop_hits = _find_by_ref_suffix(pair[1]) + if not keep_hits or not drop_hits: + continue + keep = keep_hits[0] + for drop in drop_hits: + if drop is keep: + continue + # Fill gaps on the keeper from the dropped twin, then remove it. + for fld in ("DateOfBirth", "DateOfDeath", "Genotype", + "ColorVarietyId", "LitterId", "_old_scoped_litter_id", + "OriginContactId"): + if not keep.get(fld) and drop.get(fld): + keep[fld] = drop[fld] + if keep.get("Gender") in (None, "unknown") and drop.get("Gender") not in (None, "unknown"): + keep["Gender"] = drop["Gender"] + _premerged_ids[drop["Id"]] = keep["Id"] + _merge_pairs_done += 1 + + if _premerged_ids: + all_processed_gerbils = [g for g in all_processed_gerbils + if g["Id"] not in _premerged_ids] + + print(f" Entscheidungs-Overrides: Geschlecht={_gender_applied}, " + f"Stub-Tiere={_stubs_added}, ExternalRef-Merges={_merge_pairs_done}.") + # Build parenting dates lookup using old scoped IDs parent_litter_dates = {} @@ -2810,6 +3021,13 @@ def main(): # Map raw Guid if present (convert if old_id mapped to new_guid) for l in resolved_litters: + # First redirect any parent pointing at a record dropped by an explicit + # mergeExternalRefs pre-merge onto its surviving twin (then through the + # normal id map), so the two collapsed litters share the same parents. + if l["FatherId"] in _premerged_ids: + l["FatherId"] = _premerged_ids[l["FatherId"]] + if l["MotherId"] in _premerged_ids: + l["MotherId"] = _premerged_ids[l["MotherId"]] if l["FatherId"] in gerbil_id_map: l["FatherId"] = gerbil_id_map[l["FatherId"]] if l["MotherId"] in gerbil_id_map: diff --git a/tools/import/test_merge_resolve.py b/tools/import/test_merge_resolve.py index bb89d10..65578fc 100644 --- a/tools/import/test_merge_resolve.py +++ b/tools/import/test_merge_resolve.py @@ -490,6 +490,50 @@ if _os.path.exists(_resolved): _check_parents("#35 Zac", "Zac gen. Action", "Vance", "Dorie") _check_parents("#9 Beatrice", "Beatrice von den kleinen", "Dante", "Malina") _check_parents("#30 Theodore", "Theodore von den Kleinen", "BlackFire", "Katara") + + # ── New ticket-triage fixes (non-genetics import cluster) ───────────────── + # Akane: Roni is the FATHER (gender flipped male), mother = Fumi (stub). + _check_parents("Akane (wrong-parents)", "Akane", "Roni", "Fumi") + _roni = next((g for g in _d["gerbils"] + if g["Name"] == "Roni" and g.get("DateOfBirth") == "2022-01-27"), None) + check("Roni: gender flipped to male", _roni is not None and _roni.get("Gender") == "male") + _fumi = _find("Fumi von den Kleinen") + check("Fumi: materialised as a non-resident stub", + _fumi is not None and _fumi.get("IsResident") is False) + + # Sunny von PZ Karl: father corrected Hiro → Bill von Privat. + _check_parents("Sunny (parents)", "Sunny von PZ Karl", "Bill von Privat", "Melly von Privat") + + # Danielle: mother = Ella *10.06.2019 (the elder one), father unknown. + _dan = _find("Danielle von den Kleinen") + _df, _dm = _parents(_dan) + check("Danielle: mother is Ella", (_dm or "") == "Ella") + check("Danielle: father unknown (sibling pairing)", _df is None) + if _dan and _dan.get("LitterId"): + _dl = _L.get(_dan["LitterId"]) + _dmom = _G.get(_dl.get("MotherId")) if _dl else None + check("Danielle: mother Ella is the *2019-06-10 one (not the *2023 Ella)", + _dmom is not None and _dmom.get("DateOfBirth") == "2019-06-10") + + # Cherry Berry's Quqquluuruu: gender override female (box colour misread). + _cherry = _find("Cherry Berry") + check("Cherry Berry: gender override female", + _cherry is not None and _cherry.get("Gender") == "female") + + # Origin-label: all 'of Black Forest' animals → 'Clan of Black Forest', no + # animal left on the old 'Black Forest' label and no duplicate contact. + _bf_left = [g for g in _d["gerbils"] if (g.get("OriginBreeder") or "") == "Black Forest"] + check("Origin-label: no animal still on 'Black Forest'", len(_bf_left) == 0) + _bf_contacts = [c for c in _d["contacts"] if c["Name"] == "Black Forest"] + check("Origin-label: stray 'Black Forest' contact merged away", len(_bf_contacts) == 0) + + # Duplicate-merge: the nameless buck *15.02.2024 (son of Inochi gen. Picu) + # exists only once after the externalRef merge. + _bucks = [g for g in _d["gerbils"] + if g.get("DateOfBirth") == "2024-02-15" + and "unbekannt" in (g.get("ExternalRef") or "").lower()] + check("Duplicate-merge: nameless buck *15.02.2024 deduped to one record", + len(_bucks) == 1) else: print("note: output/resolved_import.json not present — skipped integration assertions")