fix(import): Eltern-/Herkunft-/Dubletten-Korrekturen — Ticket-Triage Cluster
Akane (Roni=Vater male + Mutter Fumi), Sunny (Bill+Melly), Danielle (Mutter Ella, motherDob-Disambiguierung), Cherry Berry (female), Black-Forest-Label vereinheitlicht, namenloser Bock entdoppelt (mergeExternalRefs). Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -17,6 +17,29 @@ DIR_PATH = r"C:\Users\gulum\dev\Wurfchronik_Bilder"
|
||||
SEEDS_PATH = r"C:\Users\gulum\dev\GerbilManager\gerbil-manager-web\src\genetics\colorVarietySeed.backend.json"
|
||||
OUTPUT_DIR = r"C:\Users\gulum\dev\GerbilManager\tools\import\output"
|
||||
OUTPUT_FILE = os.path.join(OUTPUT_DIR, "resolved_import.json")
|
||||
CONFLICT_DECISIONS_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)),
|
||||
"conflict-decisions.json")
|
||||
|
||||
|
||||
def load_conflict_decisions():
|
||||
"""Load tools/import/conflict-decisions.json (the breeder's authoritative
|
||||
answers). Returns (resolutions, addAnimals). Tolerates a missing/garbled file
|
||||
by returning empty lists — the import must never crash on a bad decision file."""
|
||||
try:
|
||||
with open(CONFLICT_DECISIONS_PATH, encoding="utf-8") as fh:
|
||||
data = json.load(fh)
|
||||
return (data.get("resolutions") or [], data.get("addAnimals") or [])
|
||||
except (OSError, ValueError):
|
||||
return ([], [])
|
||||
|
||||
|
||||
def _norm_decision_gender(g):
|
||||
"""Map a free-text gender override ('m'/'w'/'männlich'/…) to 'male'/'female'
|
||||
or None when it is neither (so a typo never blanks a real gender)."""
|
||||
g = (g or "").strip().lower()
|
||||
return {"m": "male", "männlich": "male", "male": "male",
|
||||
"w": "female", "f": "female", "weiblich": "female",
|
||||
"female": "female"}.get(g)
|
||||
|
||||
def generate_guid(key_str):
|
||||
"""Generate a stable UUID string based on a key."""
|
||||
@@ -201,7 +224,8 @@ def get_normalized_contact_name(name):
|
||||
"birgitkropp": "Birgit Knopp",
|
||||
"birgittabüskens": "Birgitta Müller-Büskens",
|
||||
"birgittamüllerbüskens": "Brigitta Müller-Büskens",
|
||||
"blackforestgv": "Black Forest",
|
||||
"blackforest": "Clan of Black Forest",
|
||||
"blackforestgv": "Clan of Black Forest",
|
||||
"brigittast": "Brigitta Struve",
|
||||
"buntefellnasen": "bunten Fellnasen",
|
||||
"buntenfellnase": "bunten Fellnasen",
|
||||
@@ -1664,12 +1688,20 @@ def main():
|
||||
|
||||
a["_mapped_litter_scoped_id"] = None
|
||||
|
||||
# A human conflict-decision override is authoritative — never let the
|
||||
# Wurfchronik-Vorrang heuristic below re-attach the animal to a same-date
|
||||
# chronicle litter and overrule the breeder's named parents.
|
||||
_has_decision_parent = (
|
||||
(father_ref or {}).get("method") == "decision" or
|
||||
(mother_ref or {}).get("method") == "decision")
|
||||
|
||||
# Wurfchronik-Vorrang: if the chart parents don't yield an exact
|
||||
# Wurfchronik match but EXACTLY ONE Wurfchronik litter (with named
|
||||
# parents) exists for this birthdate, that authoritative litter wins over
|
||||
# a fabricated chart-position litter (#12 Tony, #15 Odelia, #31 Jamie).
|
||||
dob_val_attach = parse_date(a.get("dob"))
|
||||
if dob_val_attach and not is_external_origin(a.get("name"), a.get("zucht"), a.get("breeder")):
|
||||
if dob_val_attach and not _has_decision_parent \
|
||||
and not is_external_origin(a.get("name"), a.get("zucht"), a.get("breeder")):
|
||||
same_date = md_litters_by_date.get(dob_val_attach, [])
|
||||
exact_hit = None
|
||||
if father_ref and mother_ref:
|
||||
@@ -1763,6 +1795,47 @@ def main():
|
||||
"_scoped_mother_id": m_scoped_id or generate_guid(f"stammbaum-animal-{normalize_name(m_name)}")
|
||||
})
|
||||
|
||||
elif _has_decision_parent and (father_ref or mother_ref):
|
||||
# Single KNOWN parent from a human override (the other parent is
|
||||
# genuinely unknown — e.g. a sibling-pairing where only the mother is
|
||||
# named: Danielle's mother = Ella, father = unnamed brother).
|
||||
# Create a one-parent virtual litter; the global name resolver fills
|
||||
# the known side (gender + age-plausibility pick the right same-named
|
||||
# animal, e.g. Ella *10.06.2019 over Ella *13.04.2023), the other
|
||||
# side stays null.
|
||||
known = father_ref or mother_ref
|
||||
known_role = "father" if father_ref else "mother"
|
||||
known_name = get_normalized_gerbil_name(known.get("name"))
|
||||
dob_val = parse_date(a.get("dob"))
|
||||
v_key = (normalize_name(known_name), known_role, dob_val or "0001-01-01")
|
||||
if v_key in created_virtual_litters:
|
||||
a["_mapped_litter_scoped_id"] = created_virtual_litters[v_key]
|
||||
else:
|
||||
l_scoped_id = generate_guid(
|
||||
f"virtual-litter-1p-{v_key[0]}-{v_key[1]}-{v_key[2]}")
|
||||
created_virtual_litters[v_key] = l_scoped_id
|
||||
a["_mapped_litter_scoped_id"] = l_scoped_id
|
||||
raw_litters.append({
|
||||
"Id": l_scoped_id,
|
||||
"Name": f"Wurf von {known_name}",
|
||||
"Date": dob_val,
|
||||
"TotalBorn": None,
|
||||
"DeathsWithin8Weeks": None,
|
||||
"FatherId": None,
|
||||
"MotherId": None,
|
||||
"ExpectedGoHomeDate": None,
|
||||
"Notes": "Pedigree virtual litter (ein Elternteil bekannt)",
|
||||
"PairingCode": None,
|
||||
"ExternalRef": f"virtual-{l_scoped_id}",
|
||||
"LitterLetter": None,
|
||||
"_father_name": known_name if known_role == "father" else "",
|
||||
"_mother_name": known_name if known_role == "mother" else "",
|
||||
"_filename": "Stammbaum",
|
||||
"_scoped_id": l_scoped_id,
|
||||
"_scoped_father_id": None,
|
||||
"_scoped_mother_id": None,
|
||||
})
|
||||
|
||||
# 2. Resolve Contacts globally (deduplicate by normalized name)
|
||||
|
||||
contact_by_norm_name = {}
|
||||
@@ -2362,6 +2435,144 @@ def main():
|
||||
"_old_id": name_val
|
||||
})
|
||||
|
||||
# ── Conflict-decision overrides at the gerbil level ───────────────────────
|
||||
# extract.apply_conflict_decisions already applies genotype/farbschlag/dod/
|
||||
# gender/parent overrides to STAMMBAUM animals (animals.json). Wurfchronik/
|
||||
# docx animals (docx_animals.json) never pass through that path, so a gender
|
||||
# override on a Wurfchronik animal (e.g. „Roni“ — actually the FATHER) would
|
||||
# otherwise be lost. Apply gender overrides here uniformly to every processed
|
||||
# gerbil, materialise `addAnimals` stubs for known parents that have no own
|
||||
# source record (e.g. „Fumi“), and collapse explicitly-paired duplicate
|
||||
# records via `mergeExternalRefs` (nameless animals never auto-merge).
|
||||
_decisions, _add_animals = load_conflict_decisions()
|
||||
|
||||
def _gerbil_call_key(g):
|
||||
return normalize_name(get_call_name(g.get("Name") or ""))
|
||||
|
||||
# 1) gender overrides (match normalize(call-name) + ISO dob; empty decision
|
||||
# dob = name-only, for ancestors without a birthdate).
|
||||
_gender_overrides = {}
|
||||
for d in _decisions:
|
||||
gg = _norm_decision_gender(d.get("gender"))
|
||||
if not gg:
|
||||
continue
|
||||
ck = normalize_name(get_call_name(d.get("name") or ""))
|
||||
iso = parse_date(d.get("dob")) if d.get("dob") else ""
|
||||
_gender_overrides[(ck, iso or "")] = gg
|
||||
_gender_applied = 0
|
||||
for g in all_processed_gerbils:
|
||||
ck = _gerbil_call_key(g)
|
||||
iso = g.get("DateOfBirth") or ""
|
||||
gg = _gender_overrides.get((ck, iso)) or _gender_overrides.get((ck, ""))
|
||||
if gg and g.get("Gender") != gg:
|
||||
g["Gender"] = gg
|
||||
g["_resolved_by_decision"] = True
|
||||
_gender_applied += 1
|
||||
|
||||
# 2) addAnimals — materialise a non-resident stub for a KNOWN parent that has
|
||||
# no own source record, so its litter's parent link can resolve. Skip when
|
||||
# a record of that name already exists (avoid duplicates).
|
||||
_existing_call_keys = {_gerbil_call_key(g) for g in all_processed_gerbils}
|
||||
_stubs_added = 0
|
||||
for a in _add_animals:
|
||||
nm = (a.get("name") or "").strip()
|
||||
if not nm:
|
||||
continue
|
||||
ck = normalize_name(get_call_name(nm))
|
||||
if ck in _existing_call_keys:
|
||||
continue
|
||||
gender = _norm_decision_gender(a.get("gender")) or "unknown"
|
||||
dob_iso = parse_date(a.get("dob")) if a.get("dob") else None
|
||||
zucht = (a.get("zucht") or "").strip() or None
|
||||
breeder_disp = None
|
||||
if zucht:
|
||||
nb, keep = get_normalized_contact_name(zucht)
|
||||
breeder_disp = nb if keep else None
|
||||
stub_id = generate_guid(f"decision-animal-{normalize_name(nm)}")
|
||||
all_processed_gerbils.append({
|
||||
"Id": stub_id,
|
||||
"Name": get_normalized_gerbil_name(nm),
|
||||
"Gender": gender,
|
||||
"Status": "Breeding",
|
||||
"LitterId": None,
|
||||
"OriginContactId": None,
|
||||
"ReceiverContactId": None,
|
||||
"EnclosureId": None,
|
||||
"ColorVarietyId": None,
|
||||
"DateOfBirth": dob_iso,
|
||||
"DateOfDeath": None,
|
||||
"CauseOfDeath": None,
|
||||
"GoHomeDate": None,
|
||||
"Genotype": None,
|
||||
"Notes": None,
|
||||
"ImportSource": "conflict-decisions.json",
|
||||
"ExternalRef": f"decision-{normalize_name(nm)}",
|
||||
"RawImportData": json.dumps({"addedByDecision": True}, ensure_ascii=False),
|
||||
"OriginBreeder": breeder_disp,
|
||||
"NameSearch": normalize_name(get_normalized_gerbil_name(nm)),
|
||||
"CharacterTraits": [],
|
||||
"CharacterNote": None,
|
||||
"IsDeaf": None,
|
||||
"IsResident": False,
|
||||
"parentRefs": [],
|
||||
"_photos": [],
|
||||
"_old_scoped_litter_id": None,
|
||||
"_eff_dob": dob_iso or "2015-01-01",
|
||||
"_birth_date": dob_iso,
|
||||
"_filename": "conflict-decisions.json",
|
||||
"_old_id": stub_id,
|
||||
"_resolved_by_decision": True,
|
||||
})
|
||||
_existing_call_keys.add(ck)
|
||||
_stubs_added += 1
|
||||
|
||||
# 3) mergeExternalRefs — collapse explicitly-paired duplicate records (esp.
|
||||
# nameless animals that the name-based dedup keeps separate). Each pair is
|
||||
# [keepRefSuffix, dropRefSuffix]; the drop record is removed and its old id
|
||||
# mapped onto the keep record. Refs are matched as a suffix of ExternalRef
|
||||
# (extract animals become ExternalRef "stammbaum-<id>").
|
||||
_by_ext = {}
|
||||
for g in all_processed_gerbils:
|
||||
er = g.get("ExternalRef")
|
||||
if er:
|
||||
_by_ext.setdefault(er, []).append(g)
|
||||
|
||||
def _find_by_ref_suffix(suffix):
|
||||
hits = [g for er, gs in _by_ext.items() if er and er.endswith(suffix) for g in gs]
|
||||
return hits
|
||||
|
||||
_premerged_ids = {} # dropped old id -> kept id (consumed in step 5 mapping)
|
||||
_merge_pairs_done = 0
|
||||
for d in _decisions:
|
||||
for pair in (d.get("mergeExternalRefs") or []):
|
||||
if not isinstance(pair, (list, tuple)) or len(pair) != 2:
|
||||
continue
|
||||
keep_hits = _find_by_ref_suffix(pair[0])
|
||||
drop_hits = _find_by_ref_suffix(pair[1])
|
||||
if not keep_hits or not drop_hits:
|
||||
continue
|
||||
keep = keep_hits[0]
|
||||
for drop in drop_hits:
|
||||
if drop is keep:
|
||||
continue
|
||||
# Fill gaps on the keeper from the dropped twin, then remove it.
|
||||
for fld in ("DateOfBirth", "DateOfDeath", "Genotype",
|
||||
"ColorVarietyId", "LitterId", "_old_scoped_litter_id",
|
||||
"OriginContactId"):
|
||||
if not keep.get(fld) and drop.get(fld):
|
||||
keep[fld] = drop[fld]
|
||||
if keep.get("Gender") in (None, "unknown") and drop.get("Gender") not in (None, "unknown"):
|
||||
keep["Gender"] = drop["Gender"]
|
||||
_premerged_ids[drop["Id"]] = keep["Id"]
|
||||
_merge_pairs_done += 1
|
||||
|
||||
if _premerged_ids:
|
||||
all_processed_gerbils = [g for g in all_processed_gerbils
|
||||
if g["Id"] not in _premerged_ids]
|
||||
|
||||
print(f" Entscheidungs-Overrides: Geschlecht={_gender_applied}, "
|
||||
f"Stub-Tiere={_stubs_added}, ExternalRef-Merges={_merge_pairs_done}.")
|
||||
|
||||
# Build parenting dates lookup using old scoped IDs
|
||||
parent_litter_dates = {}
|
||||
|
||||
@@ -2810,6 +3021,13 @@ def main():
|
||||
|
||||
# Map raw Guid if present (convert if old_id mapped to new_guid)
|
||||
for l in resolved_litters:
|
||||
# First redirect any parent pointing at a record dropped by an explicit
|
||||
# mergeExternalRefs pre-merge onto its surviving twin (then through the
|
||||
# normal id map), so the two collapsed litters share the same parents.
|
||||
if l["FatherId"] in _premerged_ids:
|
||||
l["FatherId"] = _premerged_ids[l["FatherId"]]
|
||||
if l["MotherId"] in _premerged_ids:
|
||||
l["MotherId"] = _premerged_ids[l["MotherId"]]
|
||||
if l["FatherId"] in gerbil_id_map:
|
||||
l["FatherId"] = gerbil_id_map[l["FatherId"]]
|
||||
if l["MotherId"] in gerbil_id_map:
|
||||
|
||||
Reference in New Issue
Block a user