Gulum/GerbilManager
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(import): Stammbaum-/Eltern-/isResident-Tickets der Züchterin (18)

Browse Source 
Import-Logik:
- Externe Gründer (Zooladen/„von Privat"/„from …, <Land>") bekommen keine
  erfundenen Chart-Eltern mehr → Bill, Cooky, Zadar haben korrekt unbekannte
  Eltern (is_external_origin). [#5,#13,#28]
- Hagrid: externe Zucht wird nicht mehr als Bestand markiert (isResident=false)
  + Leerhüllen-Dedup → ein Datensatz mit Eltern Snickers × Milka. [#17,#18,#20a]
- Gender-Index: eindeutiges Geschlecht schlägt unbekanntes Duplikat → Rollen-
  Auflösung repariert (Vance→Mutter Enya, Zac→Vater Vance/Mutter Dorie). [#33,#35]
- „Eltern: X + Y"-Wurfnotizen werden geparst (~46 Würfe); v.d.↔von-den-Namens-
  kanon (Theodore→BlackFire). [#9,#11,#30]

Daten-Overrides (conflict-decisions.json, jetzt auch Gender + exakte Eltern):
- Mozart→weiblich [#2], Yuki=Camaro×Izumi [#23], Gold-Mutter=Chelsea [#36],
  Arya=Vance×Sansa (Geschwisterverpaarung) [#16], Tony→Sammy [#12],
  Odelia [#15], Jamie→Danny [#31], Silver=Taro×Beatrice [#11].

Frontend: GerbilDetailPage blendet für isResident=false Würfe + Charakter aus
(Hinweis nonResidentNote). [#17,#20b]

Tests: test_extract/test_merge_resolve erweitert; vitest 135, playwright tiere 34 grün.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Gulum
2026-06-22 21:16:53 +02:00
parent 9c6ca2118a
commit ce1704c3b7
9 changed files with 465 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files

238
tools/import/merge_and_resolve.py
View File

@@ -117,6 +117,40 @@ def normalize_name(name):
return ""
return "".join(c for c in name.lower() if c.isalnum())
_EXTERNAL_MARKERS_RE = re.compile(
r"\b(zooladen|zoohandlung|obi|fressnapf|dehner|von\s+privat|privatkauf|"
r"vom\s+bauern|aus\s+der\s+zoohandlung)\b", re.IGNORECASE)
_FOREIGN_FROM_RE = re.compile(
r"\bfrom\b.+,\s*(croatia|kroatien|poland|polen|netherlands|niederlande|"
r"belgium|belgien|france|frankreich|austria|österreich|switzerland|schweiz|"
r"italy|italien|spain|spanien|czech|tschechien|hungary|ungarn)\b", re.IGNORECASE)
def is_external_origin(name, zucht=None, breeder=None):
"""Externally-acquired founder with genuinely unknown ancestry (pet shop,
private hobbyist, foreign cattery). Mirrors extract.is_external_origin so the
merge stage never attaches chart/Wurfchronik parents to such animals."""
blob = " ".join(p for p in (name, zucht, breeder) if p)
if _EXTERNAL_MARKERS_RE.search(blob):
return True
if _FOREIGN_FROM_RE.search(name or ""):
return True
return False
def canon_name_key(name):
"""Connector-folding name key: collapses the cattery connectors so that
abbreviation variants of the SAME animal match — e.g. „BlackFire v.d.
Kleinen Chaoten“ and „BlackFire von den Kleinen Chaoten“ both fold to the
same key (ticket #30). 'v.d.' / 'v. d.' → 'von den', then alnum-reduced.
Used as a secondary index next to normalize_name (never for GUIDs)."""
if not name:
return ""
n = name.lower()
n = re.sub(r"\bv\.?\s*d\.?\b", "von den", n) # v.d. / v. d. / vd → von den
return "".join(c for c in n if c.isalnum())
def get_normalized_contact_name(name):
if not name:
return "", False
@@ -1518,12 +1552,26 @@ def main():
norm = z.lower()
return "klein" in norm and "chaot" in norm and "extern" not in norm
def is_external_cattery(a):
"""True if an animal belongs to a NAMED, non-clan cattery (e.g. 'Black
Forest', 'LennyLengo', a foreign line) or the externally-acquired
founder markers. Such animals were never in the breeder's own stock, so
residency must NOT be propagated onto them (tickets #17/#20 — Hagrid of
Black Forest). Animals with no cattery at all are name-only lineage
ancestors and stay eligible for propagation."""
for z in (a.get("zucht"), a.get("zuchtCanon"), a.get("breeder")):
z = (z or "").strip()
if z and not is_clan_zucht(z):
return True
return False
stammbaum_resident_ids = set()
for a in stammbaum_only_animals:
if is_clan_zucht(a.get("zucht")) or is_clan_zucht(a.get("zuchtCanon")):
stammbaum_resident_ids.add(a["id"])
# Propagate residency to parents of resident offspring
# Propagate residency to parents of resident offspring — but never onto
# animals from a named external cattery (they were never in this stock).
for _ in range(5):
for a in stammbaum_only_animals:
if a["id"] in stammbaum_resident_ids:
@@ -1533,22 +1581,45 @@ def main():
if normalize_name(cand["name"]) == p_key[0]:
cand_dob = parse_date(cand["dob"])
if not p_key[1] or cand_dob == p_key[1]:
if is_external_cattery(cand):
continue
stammbaum_resident_ids.add(cand["id"])
# Pre-index Wurfchronik litters from markdown
md_litters_idx = {}
# …and a date-only index of Wurfchronik litters that NAME both parents. The
# Wurfchronik is authoritative: when a chart-position reconstruction picks the
# WRONG parents (so the (father,mother,date) key misses) but exactly ONE
# Wurfchronik litter exists for that birthdate, attach the animal to it rather
# than fabricating a virtual litter with bad parents (tickets #12 Tony,
# #15 Odelia, #31 Jamie — Wurfchronik-Vorrang vor chart-position).
md_litters_by_date = {}
def _md_parent_names(rl):
f = rl.get("FatherName") or rl.get("fatherName") or rl.get("ParentMaleName") or rl.get("parentMaleName") or rl.get("_father_name")
m = rl.get("MotherName") or rl.get("motherName") or rl.get("ParentFemaleName") or rl.get("parentFemaleName") or rl.get("_mother_name")
if not f and not m:
_note = rl.get("Notes") or rl.get("notes") or rl.get("Note") or rl.get("note") or ""
_m = re.search(r"Eltern:\s*(.+?)\s*\+\s*(.+?)\s*(?:;|$)", _note)
if _m and "/" not in _m.group(1) and "/" not in _m.group(2):
f, m = _m.group(1).strip(), _m.group(2).strip()
return get_normalized_gerbil_name(f), get_normalized_gerbil_name(m)
for rl in raw_litters:
f_name = get_normalized_gerbil_name(rl.get("FatherName") or rl.get("fatherName") or rl.get("ParentMaleName") or rl.get("parentMaleName"))
m_name = get_normalized_gerbil_name(rl.get("MotherName") or rl.get("motherName") or rl.get("ParentFemaleName") or rl.get("parentFemaleName"))
f_name, m_name = _md_parent_names(rl)
ldate = parse_date(rl.get("Date") or rl.get("date") or rl.get("DateOfBirth") or rl.get("dateOfBirth"))
if f_name and m_name and ldate:
key = (normalize_name(f_name), normalize_name(m_name), ldate)
md_litters_idx[key] = rl
md_litters_by_date.setdefault(ldate, []).append(rl)
# Gender index for parent-ref selection: normalized name → 'male' | 'female'
# | 'ambiguous'. Drives the gender-aware ranking in pick_parent_ref so a dated
# but wrong-sex ref (e.g. female „Danielle“) cannot win the father slot over
# an undated male/unknown one (e.g. „Hagrid Rubeus“).
# A name's gender is "ambiguous" ONLY when BOTH male and female records exist
# for it. A definite gender beats an UNKNOWN (None) duplicate — otherwise a
# bare DOB-less ancestor box (gender=None) would poison a name that another
# record clearly types (e.g. Dorie of Black Forest, Zadar): they would fall
# back to None and lose gender-based role disambiguation (tickets #35, #33).
gender_idx = {}
for a in stammbaum_only_animals:
g = (a.get("gender") or "").lower().strip()
@@ -1556,10 +1627,13 @@ def main():
for key in {normalize_name(a.get("name")), normalize_name(get_call_name(a.get("name") or ""))}:
if not key:
continue
if key not in gender_idx:
gender_idx[key] = g
elif gender_idx[key] != g:
gender_idx[key] = "ambiguous"
cur = gender_idx.get(key, "__unset__")
if cur == "__unset__" or cur is None:
gender_idx[key] = g # first value, or upgrade None → definite
elif g is None or g == cur:
pass # keep existing definite gender
else:
gender_idx[key] = "ambiguous" # genuine male vs female conflict
def gender_of_name(name):
v = gender_idx.get(normalize_name(name))
@@ -1589,16 +1663,35 @@ def main():
a["_pick_discards"] = pick_discards
a["_mapped_litter_scoped_id"] = None
if father_ref and mother_ref:
# Wurfchronik-Vorrang: if the chart parents don't yield an exact
# Wurfchronik match but EXACTLY ONE Wurfchronik litter (with named
# parents) exists for this birthdate, that authoritative litter wins over
# a fabricated chart-position litter (#12 Tony, #15 Odelia, #31 Jamie).
dob_val_attach = parse_date(a.get("dob"))
if dob_val_attach and not is_external_origin(a.get("name"), a.get("zucht"), a.get("breeder")):
same_date = md_litters_by_date.get(dob_val_attach, [])
exact_hit = None
if father_ref and mother_ref:
exact_key = (normalize_name(get_normalized_gerbil_name(father_ref.get("name"))),
normalize_name(get_normalized_gerbil_name(mother_ref.get("name"))),
dob_val_attach)
exact_hit = md_litters_idx.get(exact_key)
if not exact_hit and len(same_date) == 1:
a["_mapped_litter_scoped_id"] = same_date[0]["_scoped_id"]
if a["_mapped_litter_scoped_id"]:
pass
elif father_ref and mother_ref:
f_name = get_normalized_gerbil_name(father_ref.get("name"))
m_name = get_normalized_gerbil_name(mother_ref.get("name"))
dob_val = parse_date(a.get("dob"))
mapped_litter = None
if dob_val:
key = (normalize_name(f_name), normalize_name(m_name), dob_val)
mapped_litter = md_litters_idx.get(key)
if mapped_litter:
a["_mapped_litter_scoped_id"] = mapped_litter["_scoped_id"]
else:
@@ -1610,13 +1703,19 @@ def main():
created_virtual_litters[v_key] = l_scoped_id
a["_mapped_litter_scoped_id"] = l_scoped_id
# Try to link parents to actual parsed stammbaum animals
# Try to link parents to actual parsed stammbaum animals.
# For human override refs (conflict-decisions, method="decision")
# skip this loose call-name preliminary linking and let the
# global name resolver pick the exact named animal instead
# (e.g. resident „Elena“, not „Elena of KK Chaos“ — #15).
f_decision = (father_ref or {}).get("method") == "decision"
m_decision = (mother_ref or {}).get("method") == "decision"
f_scoped_id = None
m_scoped_id = None
f_dob = parse_date(father_ref.get("dob"))
m_dob = parse_date(mother_ref.get("dob"))
for p_cand in stammbaum_only_animals:
for p_cand in ([] if f_decision else stammbaum_only_animals):
p_gender = str(p_cand.get("gender") or "").lower().strip()
if p_gender in ["w", "f", "female", "weiblich"]:
continue
@@ -1629,7 +1728,7 @@ def main():
if not f_dob or p_dob == f_dob:
f_scoped_id = generate_guid(f"stammbaum-animal-{p_cand['id']}")
break
for p_cand in stammbaum_only_animals:
for p_cand in ([] if m_decision else stammbaum_only_animals):
p_gender = str(p_cand.get("gender") or "").lower().strip()
if p_gender in ["m", "male", "männlich"]:
continue
@@ -1766,6 +1865,22 @@ def main():
father_name = rl.get("FatherName") or rl.get("fatherName") or rl.get("ParentMaleName") or rl.get("parentMaleName") or rl.get("_father_name")
mother_name = rl.get("MotherName") or rl.get("motherName") or rl.get("ParentFemaleName") or rl.get("parentFemaleName") or rl.get("_mother_name")
# Parse parents from a free-text Wurf note „Eltern: X + Y“ when the
# structured parent names/ids are missing (ticket #9 Beatrice/Q-Wurf,
# #11 Silver — ~46 Wurfchronik litters carry parents only in the note).
# The order is father + mother (German chart convention); the name
# resolver corrects the role afterwards by gender, so a swap is safe.
if not father_name and not mother_name:
_note = rl.get("Notes") or rl.get("notes") or rl.get("Note") or rl.get("note") or ""
_m = re.search(r"Eltern:\s*(.+?)\s*\+\s*(.+?)\s*(?:;|$)", _note)
if _m:
_p1 = _m.group(1).strip()
_p2 = _m.group(2).strip()
# Skip ambiguous "Lee/Dean" style alternatives (a slash = unsure).
if _p1 and _p2 and "/" not in _p1 and "/" not in _p2:
father_name = father_name or _p1
mother_name = mother_name or _p2
raw_ext_ref = rl.get("ExternalRef") or rl.get("externalRef") or rl.get("Id") or rl.get("id")
ext_ref_scoped = f"{filename}-{raw_ext_ref}" if raw_ext_ref else None
@@ -2257,14 +2372,29 @@ def main():
if pid:
parent_litter_dates.setdefault(pid, []).append(ld)
def _is_empty_shell(g):
"""A same-name record carrying no birthdate AND no own parent refs — a
DOB-less Stammbaum mention (e.g. Hagrid Rubeus appearing as a bare
ancestor box). Such shells must fold into the DOB-/parent-bearing record
of the same name even if a stray chart placed them as a parent of an
age-incompatible litter (ticket #18). Their own parent-attributions are
unreliable, so the parenting-date guard must not keep them separate."""
return not g.get("_birth_date") and not (g.get("parentRefs") or [])
def are_compatible(g1, g2):
# Same-name empty shell ↔ real record: always merge (see _is_empty_shell).
if _is_empty_shell(g1) or _is_empty_shell(g2):
if g1["Gender"] == "unknown" or g2["Gender"] == "unknown" \
or g1["Gender"] == g2["Gender"]:
return True
# Must have same gender (or one unknown)
if g1["Gender"] != "unknown" and g2["Gender"] != "unknown" and g1["Gender"] != g2["Gender"]:
bd1 = g1.get("_birth_date")
bd2 = g2.get("_birth_date")
if not (bd1 and bd2 and bd1 == bd2):
return False
bd1 = g1.get("_birth_date")
bd2 = g2.get("_birth_date")
@@ -2666,13 +2796,17 @@ def main():
# Create name lookup for resolved gerbils
gerbil_by_norm_name = {}
for g in resolved_gerbils:
keys = set()
n_key = normalize_name(g["Name"])
gerbil_by_norm_name.setdefault(n_key, []).append(g)
keys.add(n_key)
# Also index by call-name to resolve parents who are only listed by call-name
c_key = normalize_name(get_call_name(g["Name"]))
if c_key != n_key:
gerbil_by_norm_name.setdefault(c_key, []).append(g)
keys.add(normalize_name(get_call_name(g["Name"])))
# …and by the connector-folding canon key so abbreviation variants match
# (e.g. „BlackFire v.d. Kleinen Chaoten“ vs „… von den …“ — ticket #30).
keys.add(canon_name_key(g["Name"]))
for k in keys:
if k:
gerbil_by_norm_name.setdefault(k, []).append(g)
# Map raw Guid if present (convert if old_id mapped to new_guid)
for l in resolved_litters:
@@ -2708,29 +2842,49 @@ def main():
"""
if not name:
return None
cands = []
for c in gerbil_by_norm_name.get(normalize_name(name), []):
final_id = gerbil_id_map.get(c["Id"])
if not final_id:
# Try the exact normalized key, then the connector-folding canon key and
# the call-name (handles v.d. ↔ von den abbreviation variants — #30).
lookups = []
for k in (normalize_name(name), canon_name_key(name),
normalize_name(get_call_name(name))):
if k and k not in lookups:
lookups.append(k)
# Resolve key-by-key so an EXACT full-name match (lookups[0]) wins over a
# mere call-name/canon fallback — e.g. override mother „Elena“ must pick
# the resident „Elena“, not „Elena of KK Chaos“ whose call-name is also
# „Elena“ (ticket #15).
seen_ids = set()
for k in lookups:
cands = []
for c in gerbil_by_norm_name.get(k, []):
if c["Id"] in seen_ids:
continue
seen_ids.add(c["Id"])
final_id = gerbil_id_map.get(c["Id"])
if not final_id:
continue
final_c = gerbil_by_id_final.get(final_id)
if not final_c:
continue
# Parent must be age-plausible: born before the litter and within
# the gerbil lifespan (skips e.g. a 2013 animal for a 2022 litter).
if not parent_age_plausible(final_c["DateOfBirth"], litter_date):
continue
cands.append(final_c)
if not cands:
continue
final_c = gerbil_by_id_final.get(final_id)
if not final_c:
continue
# Parent must be age-plausible: born before the litter and within the
# gerbil lifespan (skips e.g. a 2013 animal for a 2022 litter).
if not parent_age_plausible(final_c["DateOfBirth"], litter_date):
continue
cands.append(final_c)
if not cands:
return None
# Prefer the gender expected for this role, then unknown, then anything.
for pool in (
[c for c in cands if c["Gender"] == prefer_gender],
[c for c in cands if c["Gender"] == "unknown"],
cands,
):
if pool:
return pool[0]
# Within a key, a candidate whose FULL name equals the lookup beats one
# that only matched via call-name (resident „Elena“ > „Elena of KK
# Chaos“ — #15). Stable sort keeps prior ordering otherwise.
cands.sort(key=lambda c: 0 if normalize_name(c["Name"]) == k else 1)
# Prefer the gender expected for this role, then unknown, then anything.
for pool in (
[c for c in cands if c["Gender"] == prefer_gender],
[c for c in cands if c["Gender"] == "unknown"],
cands,
):
if pool:
return pool[0]
return None
for l in resolved_litters: