feat(provenance): verworfene Daten mit Grund + Ersatz im Verlauf zeigen

Der Herkunfts-Verlauf nennt jetzt auch, wenn ein Wert aus einer Quelle VERWORFEN
wurde — mit Grund und was stattdessen verwendet wurde (⚠-Markierung), z. B.:
- „⚠ Vater-Kandidat ‚Danielle' (*04.03.2020) verworfen — falsches Geschlecht für
  die Vaterrolle; ‚Hagrid Rubeus' verwendet." (Molly)
- „⚠ Geschlecht weiblich aus ‚Wurfchronik.docx' verworfen — abweichend; männlich
  aus ‚Alberto Kids.xlsx' verwendet (Mehrheit)." (Solice)
- „⚠ Vater ‚Jasper Jacob' (*2018) verworfen — unplausibel (7 Jahre älter); kein
  Ersatz." / DOB-Remap: „⚠ Geburtsdatum 22.08.2018 verworfen — per manueller
  Entscheidung korrigiert; 14.07.2019 verwendet." (Kazuya)

Import: generischer _format_discard()-Helfer; Verwerfungen aus Mehrheitsentscheid,
pick_parent_ref-Ablehnungen (explain_pick_rejections), Rollen-Normalisierung,
Eltern-Alter-Sanity-Check und DOB-Remaps werden über _discarded/_prov_args an die
Tiere gehängt; Gerbil-Provenance wird in einem finalen Pass gebaut (nachdem alle
Verwerfungen feststehen). Frontend hebt ⚠-Zeilen ab (+ Legende).

Tests grün (python merge/extract, vitest 129, playwright 8, dotnet 212).

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
2026-06-22 16:34:33 +02:00
parent 2befe53df9
commit aa473b764e
8 changed files with 423 additions and 15 deletions

View File

@@ -774,6 +774,9 @@ def dedup(animals):
# FEAT-8c: machine-readable quarantine marker so the API loader can skip
# conflicting records without parsing the German review report.
"conflict": False,
# DOB-Remap (manuelle Entscheidung): original/corrected birthdate so the
# downstream Datenherkunft can record the discarded original value.
"dobRemap": next((a["dobRemap"] for a in grp if a.get("dobRemap")), None),
}
merged.append(out)
# conflict: same animal, GENUINELY disagreeing genotype (presence-vs-absence is NOT a
@@ -1037,6 +1040,9 @@ def apply_dob_remaps(raw_animals, path):
dob = norm_dob(a.get("dob", ""))
new = remaps_full.get((nc, zc, dob)) or remaps_name.get((nc, dob))
if new and a.get("dob") != new:
# Keep the discarded original DOB so the provenance/Datenherkunft can
# show „Geburtsdatum X verworfen, per Entscheidung auf Y geändert.“
a["dobRemap"] = {"original": a.get("dob"), "corrected": new}
a["dob"] = new
n += 1
return n

View File

@@ -713,8 +713,63 @@ def pick_parent_ref(parent_refs, role, child_dob, avoid_name=None, gender_of=Non
return role_refs[order[0]]
def explain_pick_rejections(parent_refs, role, child_dob, chosen, avoid_name=None,
gender_of=None):
"""Explain why other refs for `role` lost to `chosen` in pick_parent_ref.
Returns a list of discard dicts (for _format_discard) — one per distinct
rejected candidate name that was beaten for a clear reason (wrong sex, age-
impossible, or duplicate of the other parent). Mirrors pick_parent_ref's
ranking so the history can explain the same decision it made.
"""
role_refs = [p for p in parent_refs if p.get("roleGuess") == role]
if not role_refs or chosen is None:
return []
avoid = normalize_name(avoid_name) if avoid_name else None
expected = "male" if role == "father" else "female"
role_de = "Vaterrolle" if role == "father" else "Mutterrolle"
cand_de = "Vater-Kandidat" if role == "father" else "Mutter-Kandidat"
chosen_name = chosen.get("name")
repl_disp = chosen_name
if chosen.get("dob"):
repl_disp = f"{chosen_name} (*{_de_date(parse_date(chosen.get('dob'))) or chosen.get('dob')})"
out = []
seen = set()
for p in role_refs:
name = p.get("name")
if not name or normalize_name(name) == normalize_name(chosen_name or ""):
continue
key = normalize_name(name)
if key in seen:
continue
g = gender_of(name) if gender_of else None
dob = p.get("dob")
reason = None
if avoid is not None and key == avoid:
reason = "bereits als anderer Elternteil gewählt"
elif g in ("male", "female") and g != expected:
reason = f"falsches Geschlecht für die {role_de}"
elif dob and not parent_age_plausible(dob, child_dob):
reason = "unplausibles Alter für diesen Wurf"
if reason is None:
continue
seen.add(key)
disp = name
if dob:
disp = f"{name} (*{_de_date(parse_date(dob)) or dob})"
out.append({
"label": cand_de,
"value": f"{disp}",
"reason": reason,
"replacement": f"{repl_disp}",
})
return out
def _build_gerbil_history(records, best_g, field_source, parent_method=None,
any_decision=False, any_conflict=False, conflict_notes=None):
any_decision=False, any_conflict=False, conflict_notes=None,
field_discards=None, parent_discards=None):
"""Build an ordered, file-attributed German history for a resolved gerbil.
Reads like a chronological log:
@@ -773,6 +828,13 @@ def _build_gerbil_history(records, best_g, field_source, parent_method=None,
else:
history.append("In weiterem Datensatz gefunden → Datensätze zusammengeführt.")
# Discarded field values from majority-vote conflict resolution: the LOSING
# values, their file, and what won instead.
for d in (field_discards or []):
line = _format_discard(d)
if line not in history:
history.append(line)
# Parent derivation.
if parent_method:
method_label = {
@@ -791,6 +853,14 @@ def _build_gerbil_history(records, best_g, field_source, parent_method=None,
else:
history.append(f"Eltern über {method_label} erkannt.")
# Discarded parent candidates / links (pick_parent_ref rejections, role
# normalization drops, parent-age sanity check). Threaded in after the merge
# via the gerbil's _discarded list so they read in chronological order.
for d in (parent_discards or []):
line = _format_discard(d)
if line not in history:
history.append(line)
# Manual decisions / conflicts.
if any_decision:
history.append("Zuordnung per manueller Entscheidung getroffen.")
@@ -821,6 +891,59 @@ def _de_gender(g):
return {"male": "männlich", "female": "weiblich"}.get(g, g)
# Leading marker that visually flags a discard ("data was thrown away") line in
# the history timeline. The frontend keys discard styling off this marker.
DISCARD_MARK = ""
def _format_discard(d):
"""Render one discard record into a German history line.
A discard record is a dict describing a value/candidate the pipeline threw
away. Recognised keys:
• text — a fully pre-formatted line (used verbatim, marker added)
• label — German field label (e.g. „Geburtsdatum“)
• value — the discarded value (already display-formatted)
• file — source file the discarded value came from (attributed)
• reason — why it was dropped (e.g. „abweichend“, „unplausibel …“)
• replacement — what was used instead (already display-formatted)
• repl_file — source file the replacement came from
Produces lines like:
„⚠ Geburtsdatum 14.06.2015 aus A.xlsx verworfen — abweichend;
14.06.2017 aus B.xlsx verwendet (Mehrheit).“
Generic across entity types so litters/contacts can reuse it.
"""
if d.get("text"):
return DISCARD_MARK + d["text"]
parts = []
label = d.get("label")
value = d.get("value")
if label and value is not None:
parts.append(f"{label} {value}")
elif label:
parts.append(str(label))
elif value is not None:
parts.append(str(value))
head = " ".join(parts) if parts else "Wert"
if d.get("file"):
head += f" aus {_quote_file(d['file'])}"
line = f"{head} verworfen"
if d.get("reason"):
line += f"{d['reason']}"
repl = d.get("replacement")
if repl is not None and repl != "":
instead = str(repl)
if d.get("repl_file"):
instead += f" aus {_quote_file(d['repl_file'])}"
suffix = d.get("replacement_note")
line += f"; {instead} verwendet"
if suffix:
line += f" ({suffix})"
elif d.get("no_replacement"):
line += "; kein Ersatz"
return DISCARD_MARK + line + "."
def main():
print("Loading color variety seeds...")
variety_map = {}
@@ -864,6 +987,14 @@ def main():
else:
print(f"Warning: Seeds path not found at {SEEDS_PATH}")
# Reverse map (variety GUID → human name) for discard/replacement history
# lines that need to show a colour value rather than a raw GUID. First wins
# so we keep the canonical lower-case catalog name.
variety_id_to_name = {}
for name_lower, vid in variety_map.items():
if vid not in variety_id_to_name:
variety_id_to_name[vid] = name_lower
md_files = sorted([f for f in os.listdir(DIR_PATH) if f.lower().endswith('.md')])
print(f"Found {len(md_files)} markdown files in {DIR_PATH}.")
@@ -1187,7 +1318,20 @@ def main():
mother_ref = pick_parent_ref(parent_refs, "mother", child_dob_raw,
avoid_name=father_ref.get("name") if father_ref else None,
gender_of=gender_of_name)
# Record rejected parent-ref candidates (wrong sex / age-impossible /
# duplicate of the other role) so this animal's gerbil history can
# explain which Stammbaum positions were discarded and what won instead.
pick_discards = []
pick_discards += explain_pick_rejections(parent_refs, "father", child_dob_raw,
father_ref, gender_of=gender_of_name)
pick_discards += explain_pick_rejections(
parent_refs, "mother", child_dob_raw, mother_ref,
avoid_name=father_ref.get("name") if father_ref else None,
gender_of=gender_of_name)
if pick_discards:
a["_pick_discards"] = pick_discards
a["_mapped_litter_scoped_id"] = None
if father_ref and mother_ref:
f_name = get_normalized_gerbil_name(father_ref.get("name"))
@@ -1623,6 +1767,23 @@ def main():
"_old_id": rg.get("Id") or rg.get("id")
})
# Build the discard list for one stammbaum animal: rejected parent-ref
# candidates (pick_parent_ref) plus a manual DOB-remap decision, if any.
def _stammbaum_discards(a):
discards = list(a.get("_pick_discards", []))
remap = a.get("dobRemap")
if remap and remap.get("original") and remap.get("corrected"):
orig = _de_date(parse_date(remap["original"])) or remap["original"]
corr = _de_date(parse_date(remap["corrected"])) or remap["corrected"]
if orig != corr:
discards.append({
"label": PROV_FIELD_LABELS["DateOfBirth"],
"value": orig,
"reason": "per manueller Entscheidung korrigiert",
"replacement": corr,
})
return discards
# Map and append stammbaum animals to all_processed_gerbils
for a in stammbaum_only_animals:
a_id = a["id"]
@@ -1731,7 +1892,10 @@ def main():
"_eff_dob": dob_val or "2010-01-01",
"_birth_date": dob_val,
"_filename": a.get("sourceFiles", ["Stammbaum"])[0],
"_old_id": a_id
"_old_id": a_id,
# Rejected Stammbaum parent-ref candidates for this animal (filled by
# pick_parent_ref above) — surfaced in this gerbil's discard history.
"_discarded": _stammbaum_discards(a),
})
# Map and append docx animals to all_processed_gerbils
@@ -1885,7 +2049,8 @@ def main():
return True
def build_provenance(records, best_g, extra_notes=None, field_source=None):
def build_provenance(records, best_g, extra_notes=None, field_source=None,
field_discards=None):
"""Aggregate data-provenance across every raw record merged into one
resolved gerbil. Returns a JSON string (stored on the Gerbil entity as a
nullable text column) so the Rennmausakte can show where the entry came
@@ -1937,6 +2102,8 @@ def main():
any_decision=any_decision,
any_conflict=any_conflict,
conflict_notes=extra_notes or [],
field_discards=field_discards,
parent_discards=best_g.get("_discarded"),
)
return build_entity_provenance(
@@ -1971,7 +2138,9 @@ def main():
if is_placeholder:
# Placeholders: do NOT merge, keep all separate
for g in group:
g["Provenance"] = build_provenance([g], g)
# Provenance is built in a final pass (after parent resolution),
# so parent-link discards land in the right gerbil's history.
g["_prov_args"] = ([g], g, None, None, None)
resolved_gerbils.append(g)
gerbil_id_map[g["Id"]] = g["Id"]
continue
@@ -1992,7 +2161,7 @@ def main():
for sub in sub_groups:
if len(sub) == 1:
g = sub[0]
g["Provenance"] = build_provenance([g], g)
g["_prov_args"] = ([g], g, None, None, None)
resolved_gerbils.append(g)
gerbil_id_map[g["Id"]] = g["Id"]
continue
@@ -2019,6 +2188,17 @@ def main():
if best_g["Notes"]:
merged_notes.append(best_g["Notes"])
# Carry over discard records (rejected parent-refs / dob remaps) from
# every merged record so none are lost when the primary changes.
merged_discards = list(best_g.get("_discarded") or [])
for g in sub:
if g is best_g:
continue
for d in (g.get("_discarded") or []):
if d not in merged_discards:
merged_discards.append(d)
best_g["_discarded"] = merged_discards
# Merge photos
merged_photos = list(best_g.get("_photos", []))
@@ -2094,8 +2274,21 @@ def main():
if not any(kw in g["Notes"].lower() for kw in ["parent listed", "mutter von", "vater von", "dam of", "sire of"]):
merged_notes.append(g["Notes"])
# Display formatter per field for discard/replacement lines.
def _disp(field, val):
if val is None or val == "" or val == "unknown":
return None
if field in ("DateOfBirth", "DateOfDeath"):
return _de_date(val)
if field == "Gender":
return _de_gender(val)
if field == "ColorVarietyId":
return variety_id_to_name.get(val, "Farbschlag")
return str(val)
# Reconcile fields based on number of source files supporting them
conflict_notes = []
field_discards = []
for field in ["DateOfBirth", "DateOfDeath", "Gender", "Genotype", "ColorVarietyId"]:
votes = {}
for g in sub:
@@ -2118,6 +2311,25 @@ def main():
winner = next((g for g in sub if g.get(field) == best_val), None)
if winner is not None:
field_source[field] = winner
# Record each LOSING value: which file it came from, that it
# was discarded as differing, and that the majority value (and
# its file) was used instead.
if len(votes) > 1:
repl_disp = _disp(field, best_val)
repl_file = _primary_file_of(winner) if winner is not None else None
for lose_val in votes:
if lose_val == best_val:
continue
loser = next((g for g in sub if g.get(field) == lose_val), None)
field_discards.append({
"label": PROV_FIELD_LABELS.get(field, field),
"value": _disp(field, lose_val),
"file": _primary_file_of(loser) if loser is not None else None,
"reason": "abweichend",
"replacement": repl_disp,
"repl_file": repl_file,
"replacement_note": "Mehrheit",
})
# Keep helper fields in sync if we changed DateOfBirth
if field == "DateOfBirth":
best_g["_birth_date"] = best_val
@@ -2127,9 +2339,9 @@ def main():
best_g["Notes"] = " | ".join(merged_notes)
best_g["_photos"] = merged_photos
best_g["Provenance"] = build_provenance(
sub, best_g, extra_notes=conflict_notes, field_source=field_source
)
# Defer provenance to a final pass so parent-link discards (added
# later by the parent-age / role-normalization passes) are included.
best_g["_prov_args"] = (sub, best_g, conflict_notes, field_source, field_discards)
# Print merge trace
print(f"Deduplicated same-animal name '{best_g['Name']}': merged {len(sub)} entries across files: {', '.join(sources)}")
@@ -2294,6 +2506,25 @@ def main():
del l["_mother_name"]
del l["_filename"]
# Children-of-litter lookup so a dropped parent link can be explained in the
# offspring's Datenherkunft (the discard is most meaningful on the child).
children_by_litter = {}
for g in resolved_gerbils:
lid = g.get("LitterId")
if lid:
children_by_litter.setdefault(lid, []).append(g)
def _add_child_discard(litter, discard):
"""Attach a parent-link discard to every child gerbil of the litter."""
for child in children_by_litter.get(litter["Id"], []):
dl = child.setdefault("_discarded", [])
if discard not in dl:
dl.append(discard)
def _pname(pid):
p = gerbil_by_id_final.get(pid)
return p.get("Name") if p else None
# Role normalization: assign each resolved parent to the role matching its
# gender, eliminate self-pairings (same animal in both roles), and never let
# impossible duplicates survive (two males / two females). This corrects
@@ -2301,9 +2532,23 @@ def main():
# (one parent of "unknown" gender, or a self-paired litter).
role_fixes = 0
for l in resolved_litters:
before = (l.get("FatherId"), l.get("MotherId"))
father, mother = assign_parent_roles(l.get("FatherId"), l.get("MotherId"), _final_gender)
if (l.get("FatherId"), l.get("MotherId")) != (father, mother):
if before != (father, mother):
role_fixes += 1
after = {father, mother}
# A parent id present before but gone after was dropped by role
# normalization (self-pairing or two-of-the-same-sex). Explain it.
for pid in before:
if pid and pid not in after:
pname = _pname(pid)
if before[0] == before[1]:
reason = "Selbstverpaarung — Tier kann nicht beide Elternteile sein"
else:
reason = "ein Wurf hat nur einen Vater und eine Mutter"
_add_child_discard(l, {
"text": f"Elternteil „{pname}“ verworfen — {reason}",
})
l["FatherId"] = father
l["MotherId"] = mother
if role_fixes:
@@ -2323,6 +2568,23 @@ def main():
p = gerbil_by_id_final.get(pid)
if p and not parent_age_plausible(p.get("DateOfBirth"), ldate):
age_drops.append((l.get("Name"), role, p.get("Name"), p.get("DateOfBirth"), ldate))
# Explain the drop in each child's history: which parent, its DOB,
# why (implausible age), and that no replacement was used.
role_de = "Vater" if role == "FatherId" else "Mutter"
pdob_disp = _de_date(p.get("DateOfBirth")) or "unbekannt"
age_reason = "unplausibles Alter für diesen Wurf"
pd = date_to_days(parse_date(p.get("DateOfBirth"))) if p.get("DateOfBirth") else None
ld = date_to_days(parse_date(ldate)) if ldate else None
if pd is not None and ld is not None:
years = abs(ld - pd) / 365.25
if ld - pd <= 0:
age_reason = "unplausibel (nicht vor dem Kind geboren)"
else:
age_reason = f"unplausibel ({years:.0f} Jahre älter als das Kind)"
_add_child_discard(l, {
"text": (f"{role_de}{p.get('Name')}“ (*{pdob_disp}) verworfen "
f"{age_reason}; kein Ersatz"),
})
l[role] = None
if age_drops:
print(f"Parent-age sanity check: dropped {len(age_drops)} implausible parent link(s):")
@@ -2402,6 +2664,27 @@ def main():
resolved_litters = deduped2
litter_by_scoped_id = {l["Id"]: l for l in resolved_litters}
# Final gerbil-provenance pass: now that parent links are fully resolved and
# all discards (majority-vote conflicts during dedup; rejected parent-refs;
# role-normalization drops; parent-age drops) are attached to each gerbil's
# _discarded list, render the provenance JSON with the discard history lines.
n_discards = 0
for g in resolved_gerbils:
args = g.pop("_prov_args", None)
if g.get("_discarded"):
n_discards += len(g["_discarded"])
if args is not None:
records, best_g, conflict_notes, field_source, field_discards = args
g["Provenance"] = build_provenance(
records, best_g, extra_notes=conflict_notes,
field_source=field_source, field_discards=field_discards,
)
else:
g["Provenance"] = build_provenance([g], g)
g.pop("_discarded", None)
if n_discards:
print(f"Datenherkunft: recorded {n_discards} discard line(s) across gerbils.")
# Datenherkunft for litters: which source files contributed, whether this is
# a Wurfchronik litter vs a Stammbaum-reconstructed ("virtual") litter, how
# many raw records merged into it, plus human-readable notes. Accumulators

View File

@@ -277,6 +277,74 @@ check("_de_date: ISO → DD.MM.YYYY", m._de_date("2022-03-27") == "27.03.2022")
check("_de_date: passes through non-ISO", m._de_date("unbekannt") == "unbekannt")
# ── Discard history: a discarded source value records reason + replacement ──
# _format_discard: majority-vote conflict (losing value + file → winner + file).
_d_mehr = m._format_discard({
"label": "Geburtsdatum", "value": "14.06.2015", "file": "A.xlsx",
"reason": "abweichend", "replacement": "14.06.2017", "repl_file": "B.xlsx",
"replacement_note": "Mehrheit",
})
check("discard: starts with warning marker", _d_mehr.startswith(m.DISCARD_MARK))
check("discard(majority): names losing value + its file",
"Geburtsdatum 14.06.2015 aus „A.xlsx“ verworfen" in _d_mehr)
check("discard(majority): states the reason", "— abweichend" in _d_mehr)
check("discard(majority): names replacement + its file + note",
"14.06.2017 aus „B.xlsx“ verwendet (Mehrheit)." in _d_mehr)
# _format_discard: a parent dropped with NO replacement.
_d_noerepl = m._format_discard({
"text": "Vater „Jayjay“ (*19.06.2013) verworfen — unplausibel (9 Jahre älter "
"als das Kind); kein Ersatz",
})
check("discard(text): verbatim text gets the warning marker",
_d_noerepl == m.DISCARD_MARK + "Vater „Jayjay“ (*19.06.2013) verworfen — "
"unplausibel (9 Jahre älter als das Kind); kein Ersatz")
# explain_pick_rejections: a wrong-sex father candidate is explained (Molly case).
_picks = m.explain_pick_rejections(
molly_refs, "father", "13.09.2021", fr, gender_of=gof,
)
_pick_father = next((d for d in _picks if "Danielle" in (d.get("value") or "")), None)
check("pick-reject: wrong-sex father candidate is recorded",
_pick_father is not None)
check("pick-reject: reason = wrong sex for the father role",
_pick_father and "falsches Geschlecht für die Vaterrolle" in _pick_father["reason"])
check("pick-reject: replacement names the chosen Hagrid",
_pick_father and "Hagrid" in (_pick_father.get("replacement") or ""))
# explain_pick_rejections: an age-impossible candidate is explained.
_age_refs = [pref("Old", "father", "2010-01-01"), pref("Dad", "father", "2021-01-01")]
_chosen = m.pick_parent_ref(_age_refs, "father", "2022-03-27")
_age_picks = m.explain_pick_rejections(_age_refs, "father", "2022-03-27", _chosen)
check("pick-reject(age): age-impossible candidate recorded with reason",
any("unplausibles Alter" in d["reason"] for d in _age_picks))
# _build_gerbil_history threads field_discards (after merge) and parent_discards
# (after the parent line) into the timeline.
g_disc = {
"Name": "Solice", "DateOfBirth": "2022-03-27", "Gender": "male",
"Genotype": None, "ColorVarietyId": None, "DateOfDeath": None,
"ImportSource": "Stammbaum.xlsx", "_filename": "Stammbaum.xlsx",
"parentRefs": [],
"_discarded": [{"text": "Vater „Jayjay“ verworfen — unplausibel; kein Ersatz"}],
}
h_disc = m._build_gerbil_history(
[g_disc], g_disc, {},
field_discards=[{
"label": "Geschlecht", "value": "weiblich", "file": "Wurfchronik.docx",
"reason": "abweichend", "replacement": "männlich", "repl_file": "Stammbaum.xlsx",
"replacement_note": "Mehrheit",
}],
parent_discards=g_disc["_discarded"],
)
check("history: field-discard line present (majority vote)",
any("Geschlecht weiblich aus „Wurfchronik.docx“ verworfen" in s for s in h_disc))
check("history: parent-discard line present (dropped parent)",
any("Vater „Jayjay“ verworfen" in s for s in h_disc))
check("history: discard lines carry the warning marker",
all(s.startswith(m.DISCARD_MARK) for s in h_disc if "verworfen" in s))
if check.failed:
print(f"\n{check.failed} test(s) FAILED")
sys.exit(1)