feat(provenance): verworfene Daten mit Grund + Ersatz im Verlauf zeigen
Der Herkunfts-Verlauf nennt jetzt auch, wenn ein Wert aus einer Quelle VERWORFEN wurde — mit Grund und was stattdessen verwendet wurde (⚠-Markierung), z. B.: - „⚠ Vater-Kandidat ‚Danielle' (*04.03.2020) verworfen — falsches Geschlecht für die Vaterrolle; ‚Hagrid Rubeus' verwendet." (Molly) - „⚠ Geschlecht weiblich aus ‚Wurfchronik.docx' verworfen — abweichend; männlich aus ‚Alberto Kids.xlsx' verwendet (Mehrheit)." (Solice) - „⚠ Vater ‚Jasper Jacob' (*2018) verworfen — unplausibel (7 Jahre älter); kein Ersatz." / DOB-Remap: „⚠ Geburtsdatum 22.08.2018 verworfen — per manueller Entscheidung korrigiert; 14.07.2019 verwendet." (Kazuya) Import: generischer _format_discard()-Helfer; Verwerfungen aus Mehrheitsentscheid, pick_parent_ref-Ablehnungen (explain_pick_rejections), Rollen-Normalisierung, Eltern-Alter-Sanity-Check und DOB-Remaps werden über _discarded/_prov_args an die Tiere gehängt; Gerbil-Provenance wird in einem finalen Pass gebaut (nachdem alle Verwerfungen feststehen). Frontend hebt ⚠-Zeilen ab (+ Legende). Tests grün (python merge/extract, vitest 129, playwright 8, dotnet 212). Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -774,6 +774,9 @@ def dedup(animals):
|
||||
# FEAT-8c: machine-readable quarantine marker so the API loader can skip
|
||||
# conflicting records without parsing the German review report.
|
||||
"conflict": False,
|
||||
# DOB-Remap (manuelle Entscheidung): original/corrected birthdate so the
|
||||
# downstream Datenherkunft can record the discarded original value.
|
||||
"dobRemap": next((a["dobRemap"] for a in grp if a.get("dobRemap")), None),
|
||||
}
|
||||
merged.append(out)
|
||||
# conflict: same animal, GENUINELY disagreeing genotype (presence-vs-absence is NOT a
|
||||
@@ -1037,6 +1040,9 @@ def apply_dob_remaps(raw_animals, path):
|
||||
dob = norm_dob(a.get("dob", ""))
|
||||
new = remaps_full.get((nc, zc, dob)) or remaps_name.get((nc, dob))
|
||||
if new and a.get("dob") != new:
|
||||
# Keep the discarded original DOB so the provenance/Datenherkunft can
|
||||
# show „Geburtsdatum X verworfen, per Entscheidung auf Y geändert.“
|
||||
a["dobRemap"] = {"original": a.get("dob"), "corrected": new}
|
||||
a["dob"] = new
|
||||
n += 1
|
||||
return n
|
||||
|
||||
@@ -713,8 +713,63 @@ def pick_parent_ref(parent_refs, role, child_dob, avoid_name=None, gender_of=Non
|
||||
return role_refs[order[0]]
|
||||
|
||||
|
||||
def explain_pick_rejections(parent_refs, role, child_dob, chosen, avoid_name=None,
|
||||
gender_of=None):
|
||||
"""Explain why other refs for `role` lost to `chosen` in pick_parent_ref.
|
||||
|
||||
Returns a list of discard dicts (for _format_discard) — one per distinct
|
||||
rejected candidate name that was beaten for a clear reason (wrong sex, age-
|
||||
impossible, or duplicate of the other parent). Mirrors pick_parent_ref's
|
||||
ranking so the history can explain the same decision it made.
|
||||
"""
|
||||
role_refs = [p for p in parent_refs if p.get("roleGuess") == role]
|
||||
if not role_refs or chosen is None:
|
||||
return []
|
||||
avoid = normalize_name(avoid_name) if avoid_name else None
|
||||
expected = "male" if role == "father" else "female"
|
||||
role_de = "Vaterrolle" if role == "father" else "Mutterrolle"
|
||||
cand_de = "Vater-Kandidat" if role == "father" else "Mutter-Kandidat"
|
||||
chosen_name = chosen.get("name")
|
||||
repl_disp = chosen_name
|
||||
if chosen.get("dob"):
|
||||
repl_disp = f"{chosen_name} (*{_de_date(parse_date(chosen.get('dob'))) or chosen.get('dob')})"
|
||||
|
||||
out = []
|
||||
seen = set()
|
||||
for p in role_refs:
|
||||
name = p.get("name")
|
||||
if not name or normalize_name(name) == normalize_name(chosen_name or ""):
|
||||
continue
|
||||
key = normalize_name(name)
|
||||
if key in seen:
|
||||
continue
|
||||
g = gender_of(name) if gender_of else None
|
||||
dob = p.get("dob")
|
||||
reason = None
|
||||
if avoid is not None and key == avoid:
|
||||
reason = "bereits als anderer Elternteil gewählt"
|
||||
elif g in ("male", "female") and g != expected:
|
||||
reason = f"falsches Geschlecht für die {role_de}"
|
||||
elif dob and not parent_age_plausible(dob, child_dob):
|
||||
reason = "unplausibles Alter für diesen Wurf"
|
||||
if reason is None:
|
||||
continue
|
||||
seen.add(key)
|
||||
disp = name
|
||||
if dob:
|
||||
disp = f"{name} (*{_de_date(parse_date(dob)) or dob})"
|
||||
out.append({
|
||||
"label": cand_de,
|
||||
"value": f"„{disp}“",
|
||||
"reason": reason,
|
||||
"replacement": f"„{repl_disp}“",
|
||||
})
|
||||
return out
|
||||
|
||||
|
||||
def _build_gerbil_history(records, best_g, field_source, parent_method=None,
|
||||
any_decision=False, any_conflict=False, conflict_notes=None):
|
||||
any_decision=False, any_conflict=False, conflict_notes=None,
|
||||
field_discards=None, parent_discards=None):
|
||||
"""Build an ordered, file-attributed German history for a resolved gerbil.
|
||||
|
||||
Reads like a chronological log:
|
||||
@@ -773,6 +828,13 @@ def _build_gerbil_history(records, best_g, field_source, parent_method=None,
|
||||
else:
|
||||
history.append("In weiterem Datensatz gefunden → Datensätze zusammengeführt.")
|
||||
|
||||
# Discarded field values from majority-vote conflict resolution: the LOSING
|
||||
# values, their file, and what won instead.
|
||||
for d in (field_discards or []):
|
||||
line = _format_discard(d)
|
||||
if line not in history:
|
||||
history.append(line)
|
||||
|
||||
# Parent derivation.
|
||||
if parent_method:
|
||||
method_label = {
|
||||
@@ -791,6 +853,14 @@ def _build_gerbil_history(records, best_g, field_source, parent_method=None,
|
||||
else:
|
||||
history.append(f"Eltern über {method_label} erkannt.")
|
||||
|
||||
# Discarded parent candidates / links (pick_parent_ref rejections, role
|
||||
# normalization drops, parent-age sanity check). Threaded in after the merge
|
||||
# via the gerbil's _discarded list so they read in chronological order.
|
||||
for d in (parent_discards or []):
|
||||
line = _format_discard(d)
|
||||
if line not in history:
|
||||
history.append(line)
|
||||
|
||||
# Manual decisions / conflicts.
|
||||
if any_decision:
|
||||
history.append("Zuordnung per manueller Entscheidung getroffen.")
|
||||
@@ -821,6 +891,59 @@ def _de_gender(g):
|
||||
return {"male": "männlich", "female": "weiblich"}.get(g, g)
|
||||
|
||||
|
||||
# Leading marker that visually flags a discard ("data was thrown away") line in
|
||||
# the history timeline. The frontend keys discard styling off this marker.
|
||||
DISCARD_MARK = "⚠ "
|
||||
|
||||
|
||||
def _format_discard(d):
|
||||
"""Render one discard record into a German history line.
|
||||
|
||||
A discard record is a dict describing a value/candidate the pipeline threw
|
||||
away. Recognised keys:
|
||||
• text — a fully pre-formatted line (used verbatim, marker added)
|
||||
• label — German field label (e.g. „Geburtsdatum“)
|
||||
• value — the discarded value (already display-formatted)
|
||||
• file — source file the discarded value came from (attributed)
|
||||
• reason — why it was dropped (e.g. „abweichend“, „unplausibel …“)
|
||||
• replacement — what was used instead (already display-formatted)
|
||||
• repl_file — source file the replacement came from
|
||||
Produces lines like:
|
||||
„⚠ Geburtsdatum 14.06.2015 aus ‚A.xlsx‘ verworfen — abweichend;
|
||||
14.06.2017 aus ‚B.xlsx‘ verwendet (Mehrheit).“
|
||||
Generic across entity types so litters/contacts can reuse it.
|
||||
"""
|
||||
if d.get("text"):
|
||||
return DISCARD_MARK + d["text"]
|
||||
parts = []
|
||||
label = d.get("label")
|
||||
value = d.get("value")
|
||||
if label and value is not None:
|
||||
parts.append(f"{label} {value}")
|
||||
elif label:
|
||||
parts.append(str(label))
|
||||
elif value is not None:
|
||||
parts.append(str(value))
|
||||
head = " ".join(parts) if parts else "Wert"
|
||||
if d.get("file"):
|
||||
head += f" aus {_quote_file(d['file'])}"
|
||||
line = f"{head} verworfen"
|
||||
if d.get("reason"):
|
||||
line += f" — {d['reason']}"
|
||||
repl = d.get("replacement")
|
||||
if repl is not None and repl != "":
|
||||
instead = str(repl)
|
||||
if d.get("repl_file"):
|
||||
instead += f" aus {_quote_file(d['repl_file'])}"
|
||||
suffix = d.get("replacement_note")
|
||||
line += f"; {instead} verwendet"
|
||||
if suffix:
|
||||
line += f" ({suffix})"
|
||||
elif d.get("no_replacement"):
|
||||
line += "; kein Ersatz"
|
||||
return DISCARD_MARK + line + "."
|
||||
|
||||
|
||||
def main():
|
||||
print("Loading color variety seeds...")
|
||||
variety_map = {}
|
||||
@@ -864,6 +987,14 @@ def main():
|
||||
else:
|
||||
print(f"Warning: Seeds path not found at {SEEDS_PATH}")
|
||||
|
||||
# Reverse map (variety GUID → human name) for discard/replacement history
|
||||
# lines that need to show a colour value rather than a raw GUID. First wins
|
||||
# so we keep the canonical lower-case catalog name.
|
||||
variety_id_to_name = {}
|
||||
for name_lower, vid in variety_map.items():
|
||||
if vid not in variety_id_to_name:
|
||||
variety_id_to_name[vid] = name_lower
|
||||
|
||||
md_files = sorted([f for f in os.listdir(DIR_PATH) if f.lower().endswith('.md')])
|
||||
print(f"Found {len(md_files)} markdown files in {DIR_PATH}.")
|
||||
|
||||
@@ -1187,7 +1318,20 @@ def main():
|
||||
mother_ref = pick_parent_ref(parent_refs, "mother", child_dob_raw,
|
||||
avoid_name=father_ref.get("name") if father_ref else None,
|
||||
gender_of=gender_of_name)
|
||||
|
||||
|
||||
# Record rejected parent-ref candidates (wrong sex / age-impossible /
|
||||
# duplicate of the other role) so this animal's gerbil history can
|
||||
# explain which Stammbaum positions were discarded and what won instead.
|
||||
pick_discards = []
|
||||
pick_discards += explain_pick_rejections(parent_refs, "father", child_dob_raw,
|
||||
father_ref, gender_of=gender_of_name)
|
||||
pick_discards += explain_pick_rejections(
|
||||
parent_refs, "mother", child_dob_raw, mother_ref,
|
||||
avoid_name=father_ref.get("name") if father_ref else None,
|
||||
gender_of=gender_of_name)
|
||||
if pick_discards:
|
||||
a["_pick_discards"] = pick_discards
|
||||
|
||||
a["_mapped_litter_scoped_id"] = None
|
||||
if father_ref and mother_ref:
|
||||
f_name = get_normalized_gerbil_name(father_ref.get("name"))
|
||||
@@ -1623,6 +1767,23 @@ def main():
|
||||
"_old_id": rg.get("Id") or rg.get("id")
|
||||
})
|
||||
|
||||
# Build the discard list for one stammbaum animal: rejected parent-ref
|
||||
# candidates (pick_parent_ref) plus a manual DOB-remap decision, if any.
|
||||
def _stammbaum_discards(a):
|
||||
discards = list(a.get("_pick_discards", []))
|
||||
remap = a.get("dobRemap")
|
||||
if remap and remap.get("original") and remap.get("corrected"):
|
||||
orig = _de_date(parse_date(remap["original"])) or remap["original"]
|
||||
corr = _de_date(parse_date(remap["corrected"])) or remap["corrected"]
|
||||
if orig != corr:
|
||||
discards.append({
|
||||
"label": PROV_FIELD_LABELS["DateOfBirth"],
|
||||
"value": orig,
|
||||
"reason": "per manueller Entscheidung korrigiert",
|
||||
"replacement": corr,
|
||||
})
|
||||
return discards
|
||||
|
||||
# Map and append stammbaum animals to all_processed_gerbils
|
||||
for a in stammbaum_only_animals:
|
||||
a_id = a["id"]
|
||||
@@ -1731,7 +1892,10 @@ def main():
|
||||
"_eff_dob": dob_val or "2010-01-01",
|
||||
"_birth_date": dob_val,
|
||||
"_filename": a.get("sourceFiles", ["Stammbaum"])[0],
|
||||
"_old_id": a_id
|
||||
"_old_id": a_id,
|
||||
# Rejected Stammbaum parent-ref candidates for this animal (filled by
|
||||
# pick_parent_ref above) — surfaced in this gerbil's discard history.
|
||||
"_discarded": _stammbaum_discards(a),
|
||||
})
|
||||
|
||||
# Map and append docx animals to all_processed_gerbils
|
||||
@@ -1885,7 +2049,8 @@ def main():
|
||||
|
||||
return True
|
||||
|
||||
def build_provenance(records, best_g, extra_notes=None, field_source=None):
|
||||
def build_provenance(records, best_g, extra_notes=None, field_source=None,
|
||||
field_discards=None):
|
||||
"""Aggregate data-provenance across every raw record merged into one
|
||||
resolved gerbil. Returns a JSON string (stored on the Gerbil entity as a
|
||||
nullable text column) so the Rennmausakte can show where the entry came
|
||||
@@ -1937,6 +2102,8 @@ def main():
|
||||
any_decision=any_decision,
|
||||
any_conflict=any_conflict,
|
||||
conflict_notes=extra_notes or [],
|
||||
field_discards=field_discards,
|
||||
parent_discards=best_g.get("_discarded"),
|
||||
)
|
||||
|
||||
return build_entity_provenance(
|
||||
@@ -1971,7 +2138,9 @@ def main():
|
||||
if is_placeholder:
|
||||
# Placeholders: do NOT merge, keep all separate
|
||||
for g in group:
|
||||
g["Provenance"] = build_provenance([g], g)
|
||||
# Provenance is built in a final pass (after parent resolution),
|
||||
# so parent-link discards land in the right gerbil's history.
|
||||
g["_prov_args"] = ([g], g, None, None, None)
|
||||
resolved_gerbils.append(g)
|
||||
gerbil_id_map[g["Id"]] = g["Id"]
|
||||
continue
|
||||
@@ -1992,7 +2161,7 @@ def main():
|
||||
for sub in sub_groups:
|
||||
if len(sub) == 1:
|
||||
g = sub[0]
|
||||
g["Provenance"] = build_provenance([g], g)
|
||||
g["_prov_args"] = ([g], g, None, None, None)
|
||||
resolved_gerbils.append(g)
|
||||
gerbil_id_map[g["Id"]] = g["Id"]
|
||||
continue
|
||||
@@ -2019,6 +2188,17 @@ def main():
|
||||
if best_g["Notes"]:
|
||||
merged_notes.append(best_g["Notes"])
|
||||
|
||||
# Carry over discard records (rejected parent-refs / dob remaps) from
|
||||
# every merged record so none are lost when the primary changes.
|
||||
merged_discards = list(best_g.get("_discarded") or [])
|
||||
for g in sub:
|
||||
if g is best_g:
|
||||
continue
|
||||
for d in (g.get("_discarded") or []):
|
||||
if d not in merged_discards:
|
||||
merged_discards.append(d)
|
||||
best_g["_discarded"] = merged_discards
|
||||
|
||||
# Merge photos
|
||||
merged_photos = list(best_g.get("_photos", []))
|
||||
|
||||
@@ -2094,8 +2274,21 @@ def main():
|
||||
if not any(kw in g["Notes"].lower() for kw in ["parent listed", "mutter von", "vater von", "dam of", "sire of"]):
|
||||
merged_notes.append(g["Notes"])
|
||||
|
||||
# Display formatter per field for discard/replacement lines.
|
||||
def _disp(field, val):
|
||||
if val is None or val == "" or val == "unknown":
|
||||
return None
|
||||
if field in ("DateOfBirth", "DateOfDeath"):
|
||||
return _de_date(val)
|
||||
if field == "Gender":
|
||||
return _de_gender(val)
|
||||
if field == "ColorVarietyId":
|
||||
return variety_id_to_name.get(val, "Farbschlag")
|
||||
return str(val)
|
||||
|
||||
# Reconcile fields based on number of source files supporting them
|
||||
conflict_notes = []
|
||||
field_discards = []
|
||||
for field in ["DateOfBirth", "DateOfDeath", "Gender", "Genotype", "ColorVarietyId"]:
|
||||
votes = {}
|
||||
for g in sub:
|
||||
@@ -2118,6 +2311,25 @@ def main():
|
||||
winner = next((g for g in sub if g.get(field) == best_val), None)
|
||||
if winner is not None:
|
||||
field_source[field] = winner
|
||||
# Record each LOSING value: which file it came from, that it
|
||||
# was discarded as differing, and that the majority value (and
|
||||
# its file) was used instead.
|
||||
if len(votes) > 1:
|
||||
repl_disp = _disp(field, best_val)
|
||||
repl_file = _primary_file_of(winner) if winner is not None else None
|
||||
for lose_val in votes:
|
||||
if lose_val == best_val:
|
||||
continue
|
||||
loser = next((g for g in sub if g.get(field) == lose_val), None)
|
||||
field_discards.append({
|
||||
"label": PROV_FIELD_LABELS.get(field, field),
|
||||
"value": _disp(field, lose_val),
|
||||
"file": _primary_file_of(loser) if loser is not None else None,
|
||||
"reason": "abweichend",
|
||||
"replacement": repl_disp,
|
||||
"repl_file": repl_file,
|
||||
"replacement_note": "Mehrheit",
|
||||
})
|
||||
# Keep helper fields in sync if we changed DateOfBirth
|
||||
if field == "DateOfBirth":
|
||||
best_g["_birth_date"] = best_val
|
||||
@@ -2127,9 +2339,9 @@ def main():
|
||||
best_g["Notes"] = " | ".join(merged_notes)
|
||||
|
||||
best_g["_photos"] = merged_photos
|
||||
best_g["Provenance"] = build_provenance(
|
||||
sub, best_g, extra_notes=conflict_notes, field_source=field_source
|
||||
)
|
||||
# Defer provenance to a final pass so parent-link discards (added
|
||||
# later by the parent-age / role-normalization passes) are included.
|
||||
best_g["_prov_args"] = (sub, best_g, conflict_notes, field_source, field_discards)
|
||||
|
||||
# Print merge trace
|
||||
print(f"Deduplicated same-animal name '{best_g['Name']}': merged {len(sub)} entries across files: {', '.join(sources)}")
|
||||
@@ -2294,6 +2506,25 @@ def main():
|
||||
del l["_mother_name"]
|
||||
del l["_filename"]
|
||||
|
||||
# Children-of-litter lookup so a dropped parent link can be explained in the
|
||||
# offspring's Datenherkunft (the discard is most meaningful on the child).
|
||||
children_by_litter = {}
|
||||
for g in resolved_gerbils:
|
||||
lid = g.get("LitterId")
|
||||
if lid:
|
||||
children_by_litter.setdefault(lid, []).append(g)
|
||||
|
||||
def _add_child_discard(litter, discard):
|
||||
"""Attach a parent-link discard to every child gerbil of the litter."""
|
||||
for child in children_by_litter.get(litter["Id"], []):
|
||||
dl = child.setdefault("_discarded", [])
|
||||
if discard not in dl:
|
||||
dl.append(discard)
|
||||
|
||||
def _pname(pid):
|
||||
p = gerbil_by_id_final.get(pid)
|
||||
return p.get("Name") if p else None
|
||||
|
||||
# Role normalization: assign each resolved parent to the role matching its
|
||||
# gender, eliminate self-pairings (same animal in both roles), and never let
|
||||
# impossible duplicates survive (two males / two females). This corrects
|
||||
@@ -2301,9 +2532,23 @@ def main():
|
||||
# (one parent of "unknown" gender, or a self-paired litter).
|
||||
role_fixes = 0
|
||||
for l in resolved_litters:
|
||||
before = (l.get("FatherId"), l.get("MotherId"))
|
||||
father, mother = assign_parent_roles(l.get("FatherId"), l.get("MotherId"), _final_gender)
|
||||
if (l.get("FatherId"), l.get("MotherId")) != (father, mother):
|
||||
if before != (father, mother):
|
||||
role_fixes += 1
|
||||
after = {father, mother}
|
||||
# A parent id present before but gone after was dropped by role
|
||||
# normalization (self-pairing or two-of-the-same-sex). Explain it.
|
||||
for pid in before:
|
||||
if pid and pid not in after:
|
||||
pname = _pname(pid)
|
||||
if before[0] == before[1]:
|
||||
reason = "Selbstverpaarung — Tier kann nicht beide Elternteile sein"
|
||||
else:
|
||||
reason = "ein Wurf hat nur einen Vater und eine Mutter"
|
||||
_add_child_discard(l, {
|
||||
"text": f"Elternteil „{pname}“ verworfen — {reason}",
|
||||
})
|
||||
l["FatherId"] = father
|
||||
l["MotherId"] = mother
|
||||
if role_fixes:
|
||||
@@ -2323,6 +2568,23 @@ def main():
|
||||
p = gerbil_by_id_final.get(pid)
|
||||
if p and not parent_age_plausible(p.get("DateOfBirth"), ldate):
|
||||
age_drops.append((l.get("Name"), role, p.get("Name"), p.get("DateOfBirth"), ldate))
|
||||
# Explain the drop in each child's history: which parent, its DOB,
|
||||
# why (implausible age), and that no replacement was used.
|
||||
role_de = "Vater" if role == "FatherId" else "Mutter"
|
||||
pdob_disp = _de_date(p.get("DateOfBirth")) or "unbekannt"
|
||||
age_reason = "unplausibles Alter für diesen Wurf"
|
||||
pd = date_to_days(parse_date(p.get("DateOfBirth"))) if p.get("DateOfBirth") else None
|
||||
ld = date_to_days(parse_date(ldate)) if ldate else None
|
||||
if pd is not None and ld is not None:
|
||||
years = abs(ld - pd) / 365.25
|
||||
if ld - pd <= 0:
|
||||
age_reason = "unplausibel (nicht vor dem Kind geboren)"
|
||||
else:
|
||||
age_reason = f"unplausibel ({years:.0f} Jahre älter als das Kind)"
|
||||
_add_child_discard(l, {
|
||||
"text": (f"{role_de} „{p.get('Name')}“ (*{pdob_disp}) verworfen "
|
||||
f"— {age_reason}; kein Ersatz"),
|
||||
})
|
||||
l[role] = None
|
||||
if age_drops:
|
||||
print(f"Parent-age sanity check: dropped {len(age_drops)} implausible parent link(s):")
|
||||
@@ -2402,6 +2664,27 @@ def main():
|
||||
resolved_litters = deduped2
|
||||
litter_by_scoped_id = {l["Id"]: l for l in resolved_litters}
|
||||
|
||||
# Final gerbil-provenance pass: now that parent links are fully resolved and
|
||||
# all discards (majority-vote conflicts during dedup; rejected parent-refs;
|
||||
# role-normalization drops; parent-age drops) are attached to each gerbil's
|
||||
# _discarded list, render the provenance JSON with the discard history lines.
|
||||
n_discards = 0
|
||||
for g in resolved_gerbils:
|
||||
args = g.pop("_prov_args", None)
|
||||
if g.get("_discarded"):
|
||||
n_discards += len(g["_discarded"])
|
||||
if args is not None:
|
||||
records, best_g, conflict_notes, field_source, field_discards = args
|
||||
g["Provenance"] = build_provenance(
|
||||
records, best_g, extra_notes=conflict_notes,
|
||||
field_source=field_source, field_discards=field_discards,
|
||||
)
|
||||
else:
|
||||
g["Provenance"] = build_provenance([g], g)
|
||||
g.pop("_discarded", None)
|
||||
if n_discards:
|
||||
print(f"Datenherkunft: recorded {n_discards} discard line(s) across gerbils.")
|
||||
|
||||
# Datenherkunft for litters: which source files contributed, whether this is
|
||||
# a Wurfchronik litter vs a Stammbaum-reconstructed ("virtual") litter, how
|
||||
# many raw records merged into it, plus human-readable notes. Accumulators
|
||||
|
||||
@@ -277,6 +277,74 @@ check("_de_date: ISO → DD.MM.YYYY", m._de_date("2022-03-27") == "27.03.2022")
|
||||
check("_de_date: passes through non-ISO", m._de_date("unbekannt") == "unbekannt")
|
||||
|
||||
|
||||
# ── Discard history: a discarded source value records reason + replacement ──
|
||||
# _format_discard: majority-vote conflict (losing value + file → winner + file).
|
||||
_d_mehr = m._format_discard({
|
||||
"label": "Geburtsdatum", "value": "14.06.2015", "file": "A.xlsx",
|
||||
"reason": "abweichend", "replacement": "14.06.2017", "repl_file": "B.xlsx",
|
||||
"replacement_note": "Mehrheit",
|
||||
})
|
||||
check("discard: starts with warning marker", _d_mehr.startswith(m.DISCARD_MARK))
|
||||
check("discard(majority): names losing value + its file",
|
||||
"Geburtsdatum 14.06.2015 aus „A.xlsx“ verworfen" in _d_mehr)
|
||||
check("discard(majority): states the reason", "— abweichend" in _d_mehr)
|
||||
check("discard(majority): names replacement + its file + note",
|
||||
"14.06.2017 aus „B.xlsx“ verwendet (Mehrheit)." in _d_mehr)
|
||||
|
||||
# _format_discard: a parent dropped with NO replacement.
|
||||
_d_noerepl = m._format_discard({
|
||||
"text": "Vater „Jayjay“ (*19.06.2013) verworfen — unplausibel (9 Jahre älter "
|
||||
"als das Kind); kein Ersatz",
|
||||
})
|
||||
check("discard(text): verbatim text gets the warning marker",
|
||||
_d_noerepl == m.DISCARD_MARK + "Vater „Jayjay“ (*19.06.2013) verworfen — "
|
||||
"unplausibel (9 Jahre älter als das Kind); kein Ersatz")
|
||||
|
||||
# explain_pick_rejections: a wrong-sex father candidate is explained (Molly case).
|
||||
_picks = m.explain_pick_rejections(
|
||||
molly_refs, "father", "13.09.2021", fr, gender_of=gof,
|
||||
)
|
||||
_pick_father = next((d for d in _picks if "Danielle" in (d.get("value") or "")), None)
|
||||
check("pick-reject: wrong-sex father candidate is recorded",
|
||||
_pick_father is not None)
|
||||
check("pick-reject: reason = wrong sex for the father role",
|
||||
_pick_father and "falsches Geschlecht für die Vaterrolle" in _pick_father["reason"])
|
||||
check("pick-reject: replacement names the chosen Hagrid",
|
||||
_pick_father and "Hagrid" in (_pick_father.get("replacement") or ""))
|
||||
|
||||
# explain_pick_rejections: an age-impossible candidate is explained.
|
||||
_age_refs = [pref("Old", "father", "2010-01-01"), pref("Dad", "father", "2021-01-01")]
|
||||
_chosen = m.pick_parent_ref(_age_refs, "father", "2022-03-27")
|
||||
_age_picks = m.explain_pick_rejections(_age_refs, "father", "2022-03-27", _chosen)
|
||||
check("pick-reject(age): age-impossible candidate recorded with reason",
|
||||
any("unplausibles Alter" in d["reason"] for d in _age_picks))
|
||||
|
||||
# _build_gerbil_history threads field_discards (after merge) and parent_discards
|
||||
# (after the parent line) into the timeline.
|
||||
g_disc = {
|
||||
"Name": "Solice", "DateOfBirth": "2022-03-27", "Gender": "male",
|
||||
"Genotype": None, "ColorVarietyId": None, "DateOfDeath": None,
|
||||
"ImportSource": "Stammbaum.xlsx", "_filename": "Stammbaum.xlsx",
|
||||
"parentRefs": [],
|
||||
"_discarded": [{"text": "Vater „Jayjay“ verworfen — unplausibel; kein Ersatz"}],
|
||||
}
|
||||
h_disc = m._build_gerbil_history(
|
||||
[g_disc], g_disc, {},
|
||||
field_discards=[{
|
||||
"label": "Geschlecht", "value": "weiblich", "file": "Wurfchronik.docx",
|
||||
"reason": "abweichend", "replacement": "männlich", "repl_file": "Stammbaum.xlsx",
|
||||
"replacement_note": "Mehrheit",
|
||||
}],
|
||||
parent_discards=g_disc["_discarded"],
|
||||
)
|
||||
check("history: field-discard line present (majority vote)",
|
||||
any("Geschlecht weiblich aus „Wurfchronik.docx“ verworfen" in s for s in h_disc))
|
||||
check("history: parent-discard line present (dropped parent)",
|
||||
any("Vater „Jayjay“ verworfen" in s for s in h_disc))
|
||||
check("history: discard lines carry the warning marker",
|
||||
all(s.startswith(m.DISCARD_MARK) for s in h_disc if "verworfen" in s))
|
||||
|
||||
|
||||
if check.failed:
|
||||
print(f"\n{check.failed} test(s) FAILED")
|
||||
sys.exit(1)
|
||||
|
||||
Reference in New Issue
Block a user