feat(provenance): verworfene Daten mit Grund + Ersatz im Verlauf zeigen
Der Herkunfts-Verlauf nennt jetzt auch, wenn ein Wert aus einer Quelle VERWORFEN wurde — mit Grund und was stattdessen verwendet wurde (⚠-Markierung), z. B.: - „⚠ Vater-Kandidat ‚Danielle' (*04.03.2020) verworfen — falsches Geschlecht für die Vaterrolle; ‚Hagrid Rubeus' verwendet." (Molly) - „⚠ Geschlecht weiblich aus ‚Wurfchronik.docx' verworfen — abweichend; männlich aus ‚Alberto Kids.xlsx' verwendet (Mehrheit)." (Solice) - „⚠ Vater ‚Jasper Jacob' (*2018) verworfen — unplausibel (7 Jahre älter); kein Ersatz." / DOB-Remap: „⚠ Geburtsdatum 22.08.2018 verworfen — per manueller Entscheidung korrigiert; 14.07.2019 verwendet." (Kazuya) Import: generischer _format_discard()-Helfer; Verwerfungen aus Mehrheitsentscheid, pick_parent_ref-Ablehnungen (explain_pick_rejections), Rollen-Normalisierung, Eltern-Alter-Sanity-Check und DOB-Remaps werden über _discarded/_prov_args an die Tiere gehängt; Gerbil-Provenance wird in einem finalen Pass gebaut (nachdem alle Verwerfungen feststehen). Frontend hebt ⚠-Zeilen ab (+ Legende). Tests grün (python merge/extract, vitest 129, playwright 8, dotnet 212). Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -135,7 +135,11 @@ export function seedDb(): MockDb {
|
|||||||
'Geburtsdatum (12.03.2025) aus „Stammbaum von Krümel.xlsx“.',
|
'Geburtsdatum (12.03.2025) aus „Stammbaum von Krümel.xlsx“.',
|
||||||
'Genotyp aus „Stammbaum von Krümel.xlsx“.',
|
'Genotyp aus „Stammbaum von Krümel.xlsx“.',
|
||||||
'Auch in „Wurfchronik-Detail.docx“ gefunden → Datensätze zusammengeführt.',
|
'Auch in „Wurfchronik-Detail.docx“ gefunden → Datensätze zusammengeführt.',
|
||||||
|
// VERWORFEN: abweichender Wert aus einer Quelle wurde verworfen (Grund + Ersatz).
|
||||||
|
'⚠ Geburtsdatum 14.06.2015 aus „Wurfchronik-Detail.docx“ verworfen — abweichend; 12.03.2025 aus „Stammbaum von Krümel.xlsx“ verwendet (Mehrheit).',
|
||||||
'Eltern über Position im Stammbaum erkannt (Quelle: „Stammbaum von Krümel.xlsx“).',
|
'Eltern über Position im Stammbaum erkannt (Quelle: „Stammbaum von Krümel.xlsx“).',
|
||||||
|
// VERWORFEN: unplausibler Elternteil verworfen, kein Ersatz verwendet.
|
||||||
|
'⚠ Vater „Jayjay“ (*19.06.2013) verworfen — unplausibel (9 Jahre älter als das Kind); kein Ersatz.',
|
||||||
'Angaben aus der Wurfchronik übernommen.',
|
'Angaben aus der Wurfchronik übernommen.',
|
||||||
],
|
],
|
||||||
}),
|
}),
|
||||||
|
|||||||
@@ -21,6 +21,17 @@ test('Tierakte: "Datenherkunft"-Button öffnet den Nachverfolgungs-Dialog', asyn
|
|||||||
'Auch in „Wurfchronik-Detail.docx“ gefunden → Datensätze zusammengeführt.',
|
'Auch in „Wurfchronik-Detail.docx“ gefunden → Datensätze zusammengeführt.',
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// VERWORFEN: eine Quelle wurde verworfen — mit Grund und verwendetem Ersatz.
|
||||||
|
await expect(dialog).toContainText(p.discardLegend)
|
||||||
|
await expect(dialog).toContainText(
|
||||||
|
'⚠ Geburtsdatum 14.06.2015 aus „Wurfchronik-Detail.docx“ verworfen — abweichend; 12.03.2025 aus „Stammbaum von Krümel.xlsx“ verwendet (Mehrheit).',
|
||||||
|
)
|
||||||
|
await expect(dialog).toContainText(
|
||||||
|
'⚠ Vater „Jayjay“ (*19.06.2013) verworfen — unplausibel (9 Jahre älter als das Kind); kein Ersatz.',
|
||||||
|
)
|
||||||
|
// Die Verwerfen-Zeile ist visuell als solche markiert (eigene CSS-Klasse).
|
||||||
|
await expect(dialog.locator('.provenance__history-step--discard').first()).toBeVisible()
|
||||||
|
|
||||||
// Quelldateien werden angezeigt.
|
// Quelldateien werden angezeigt.
|
||||||
await expect(dialog).toContainText(p.sourceFilesTitle)
|
await expect(dialog).toContainText(p.sourceFilesTitle)
|
||||||
await expect(dialog).toContainText('Stammbaum von Krümel.xlsx')
|
await expect(dialog).toContainText('Stammbaum von Krümel.xlsx')
|
||||||
|
|||||||
@@ -12,6 +12,11 @@ import { de } from '../strings/de'
|
|||||||
import type { EntityProvenance } from '../api/types'
|
import type { EntityProvenance } from '../api/types'
|
||||||
import './provenanceDialog.css'
|
import './provenanceDialog.css'
|
||||||
|
|
||||||
|
/** Leading marker the Python merge prefixes onto discard ("verworfen") history
|
||||||
|
* lines (data thrown away with reason + replacement). Kept in sync with
|
||||||
|
* DISCARD_MARK in tools/import/merge_and_resolve.py. */
|
||||||
|
const DISCARD_PREFIX = '⚠ '
|
||||||
|
|
||||||
interface ProvenanceDialogProps {
|
interface ProvenanceDialogProps {
|
||||||
open: boolean
|
open: boolean
|
||||||
onClose: () => void
|
onClose: () => void
|
||||||
@@ -92,12 +97,27 @@ export default function ProvenanceDialog({ open, onClose, provenance, entityLabe
|
|||||||
{p.history.length > 0 && (
|
{p.history.length > 0 && (
|
||||||
<section className="provenance__section">
|
<section className="provenance__section">
|
||||||
<div className="provenance__section-title">{t.historyTitle}</div>
|
<div className="provenance__section-title">{t.historyTitle}</div>
|
||||||
|
{p.history.some((s) => s.startsWith(DISCARD_PREFIX)) && (
|
||||||
|
<p className="provenance__discard-legend">{t.discardLegend}</p>
|
||||||
|
)}
|
||||||
<ol className="provenance__history">
|
<ol className="provenance__history">
|
||||||
{p.history.map((step, i) => (
|
{p.history.map((step, i) => {
|
||||||
<li key={`${i}-${step}`} className="provenance__history-step">
|
// Discard lines (Python prefixes them with „⚠ “) are styled
|
||||||
{step}
|
// distinctly so a thrown-away value stands out from accepted facts.
|
||||||
</li>
|
const isDiscard = step.startsWith(DISCARD_PREFIX)
|
||||||
))}
|
return (
|
||||||
|
<li
|
||||||
|
key={`${i}-${step}`}
|
||||||
|
className={
|
||||||
|
isDiscard
|
||||||
|
? 'provenance__history-step provenance__history-step--discard'
|
||||||
|
: 'provenance__history-step'
|
||||||
|
}
|
||||||
|
>
|
||||||
|
{step}
|
||||||
|
</li>
|
||||||
|
)
|
||||||
|
})}
|
||||||
</ol>
|
</ol>
|
||||||
</section>
|
</section>
|
||||||
)}
|
)}
|
||||||
|
|||||||
@@ -163,6 +163,19 @@
|
|||||||
word-break: break-word;
|
word-break: break-word;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* Verworfene Werte (Datenherkunft): hervorgehoben, damit klar wird, dass hier
|
||||||
|
Daten aus einer Quelle verworfen wurden — mit Grund und Ersatz. */
|
||||||
|
.provenance__history-step--discard {
|
||||||
|
color: var(--color-danger, #9a3412);
|
||||||
|
font-style: italic;
|
||||||
|
}
|
||||||
|
|
||||||
|
.provenance__discard-legend {
|
||||||
|
margin: 0 0 0.5rem;
|
||||||
|
font-size: 0.78rem;
|
||||||
|
color: var(--color-text-muted, #888);
|
||||||
|
}
|
||||||
|
|
||||||
.provenance__actions {
|
.provenance__actions {
|
||||||
display: flex;
|
display: flex;
|
||||||
justify-content: flex-end;
|
justify-content: flex-end;
|
||||||
|
|||||||
@@ -896,6 +896,9 @@ export const de = {
|
|||||||
},
|
},
|
||||||
/** Überschrift des chronologischen, dateibezogenen Verlaufs (Primärinhalt). */
|
/** Überschrift des chronologischen, dateibezogenen Verlaufs (Primärinhalt). */
|
||||||
historyTitle: 'Verlauf',
|
historyTitle: 'Verlauf',
|
||||||
|
/** Legende für die mit „⚠“ markierten Verlaufszeilen, in denen Daten aus
|
||||||
|
* einer Quelle verworfen wurden (mit Grund und verwendetem Ersatz). */
|
||||||
|
discardLegend: '⚠ markiert verworfene Quelldaten (mit Grund und verwendetem Ersatz).',
|
||||||
/** Überschriften / Feldbeschriftungen. */
|
/** Überschriften / Feldbeschriftungen. */
|
||||||
sourceFilesTitle: 'Quelldateien',
|
sourceFilesTitle: 'Quelldateien',
|
||||||
sourceFilesCount: (n: number) => (n === 1 ? 'aus 1 Quelle' : `aus ${n} Quellen`),
|
sourceFilesCount: (n: number) => (n === 1 ? 'aus 1 Quelle' : `aus ${n} Quellen`),
|
||||||
|
|||||||
@@ -774,6 +774,9 @@ def dedup(animals):
|
|||||||
# FEAT-8c: machine-readable quarantine marker so the API loader can skip
|
# FEAT-8c: machine-readable quarantine marker so the API loader can skip
|
||||||
# conflicting records without parsing the German review report.
|
# conflicting records without parsing the German review report.
|
||||||
"conflict": False,
|
"conflict": False,
|
||||||
|
# DOB-Remap (manuelle Entscheidung): original/corrected birthdate so the
|
||||||
|
# downstream Datenherkunft can record the discarded original value.
|
||||||
|
"dobRemap": next((a["dobRemap"] for a in grp if a.get("dobRemap")), None),
|
||||||
}
|
}
|
||||||
merged.append(out)
|
merged.append(out)
|
||||||
# conflict: same animal, GENUINELY disagreeing genotype (presence-vs-absence is NOT a
|
# conflict: same animal, GENUINELY disagreeing genotype (presence-vs-absence is NOT a
|
||||||
@@ -1037,6 +1040,9 @@ def apply_dob_remaps(raw_animals, path):
|
|||||||
dob = norm_dob(a.get("dob", ""))
|
dob = norm_dob(a.get("dob", ""))
|
||||||
new = remaps_full.get((nc, zc, dob)) or remaps_name.get((nc, dob))
|
new = remaps_full.get((nc, zc, dob)) or remaps_name.get((nc, dob))
|
||||||
if new and a.get("dob") != new:
|
if new and a.get("dob") != new:
|
||||||
|
# Keep the discarded original DOB so the provenance/Datenherkunft can
|
||||||
|
# show „Geburtsdatum X verworfen, per Entscheidung auf Y geändert.“
|
||||||
|
a["dobRemap"] = {"original": a.get("dob"), "corrected": new}
|
||||||
a["dob"] = new
|
a["dob"] = new
|
||||||
n += 1
|
n += 1
|
||||||
return n
|
return n
|
||||||
|
|||||||
@@ -713,8 +713,63 @@ def pick_parent_ref(parent_refs, role, child_dob, avoid_name=None, gender_of=Non
|
|||||||
return role_refs[order[0]]
|
return role_refs[order[0]]
|
||||||
|
|
||||||
|
|
||||||
|
def explain_pick_rejections(parent_refs, role, child_dob, chosen, avoid_name=None,
|
||||||
|
gender_of=None):
|
||||||
|
"""Explain why other refs for `role` lost to `chosen` in pick_parent_ref.
|
||||||
|
|
||||||
|
Returns a list of discard dicts (for _format_discard) — one per distinct
|
||||||
|
rejected candidate name that was beaten for a clear reason (wrong sex, age-
|
||||||
|
impossible, or duplicate of the other parent). Mirrors pick_parent_ref's
|
||||||
|
ranking so the history can explain the same decision it made.
|
||||||
|
"""
|
||||||
|
role_refs = [p for p in parent_refs if p.get("roleGuess") == role]
|
||||||
|
if not role_refs or chosen is None:
|
||||||
|
return []
|
||||||
|
avoid = normalize_name(avoid_name) if avoid_name else None
|
||||||
|
expected = "male" if role == "father" else "female"
|
||||||
|
role_de = "Vaterrolle" if role == "father" else "Mutterrolle"
|
||||||
|
cand_de = "Vater-Kandidat" if role == "father" else "Mutter-Kandidat"
|
||||||
|
chosen_name = chosen.get("name")
|
||||||
|
repl_disp = chosen_name
|
||||||
|
if chosen.get("dob"):
|
||||||
|
repl_disp = f"{chosen_name} (*{_de_date(parse_date(chosen.get('dob'))) or chosen.get('dob')})"
|
||||||
|
|
||||||
|
out = []
|
||||||
|
seen = set()
|
||||||
|
for p in role_refs:
|
||||||
|
name = p.get("name")
|
||||||
|
if not name or normalize_name(name) == normalize_name(chosen_name or ""):
|
||||||
|
continue
|
||||||
|
key = normalize_name(name)
|
||||||
|
if key in seen:
|
||||||
|
continue
|
||||||
|
g = gender_of(name) if gender_of else None
|
||||||
|
dob = p.get("dob")
|
||||||
|
reason = None
|
||||||
|
if avoid is not None and key == avoid:
|
||||||
|
reason = "bereits als anderer Elternteil gewählt"
|
||||||
|
elif g in ("male", "female") and g != expected:
|
||||||
|
reason = f"falsches Geschlecht für die {role_de}"
|
||||||
|
elif dob and not parent_age_plausible(dob, child_dob):
|
||||||
|
reason = "unplausibles Alter für diesen Wurf"
|
||||||
|
if reason is None:
|
||||||
|
continue
|
||||||
|
seen.add(key)
|
||||||
|
disp = name
|
||||||
|
if dob:
|
||||||
|
disp = f"{name} (*{_de_date(parse_date(dob)) or dob})"
|
||||||
|
out.append({
|
||||||
|
"label": cand_de,
|
||||||
|
"value": f"„{disp}“",
|
||||||
|
"reason": reason,
|
||||||
|
"replacement": f"„{repl_disp}“",
|
||||||
|
})
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
def _build_gerbil_history(records, best_g, field_source, parent_method=None,
|
def _build_gerbil_history(records, best_g, field_source, parent_method=None,
|
||||||
any_decision=False, any_conflict=False, conflict_notes=None):
|
any_decision=False, any_conflict=False, conflict_notes=None,
|
||||||
|
field_discards=None, parent_discards=None):
|
||||||
"""Build an ordered, file-attributed German history for a resolved gerbil.
|
"""Build an ordered, file-attributed German history for a resolved gerbil.
|
||||||
|
|
||||||
Reads like a chronological log:
|
Reads like a chronological log:
|
||||||
@@ -773,6 +828,13 @@ def _build_gerbil_history(records, best_g, field_source, parent_method=None,
|
|||||||
else:
|
else:
|
||||||
history.append("In weiterem Datensatz gefunden → Datensätze zusammengeführt.")
|
history.append("In weiterem Datensatz gefunden → Datensätze zusammengeführt.")
|
||||||
|
|
||||||
|
# Discarded field values from majority-vote conflict resolution: the LOSING
|
||||||
|
# values, their file, and what won instead.
|
||||||
|
for d in (field_discards or []):
|
||||||
|
line = _format_discard(d)
|
||||||
|
if line not in history:
|
||||||
|
history.append(line)
|
||||||
|
|
||||||
# Parent derivation.
|
# Parent derivation.
|
||||||
if parent_method:
|
if parent_method:
|
||||||
method_label = {
|
method_label = {
|
||||||
@@ -791,6 +853,14 @@ def _build_gerbil_history(records, best_g, field_source, parent_method=None,
|
|||||||
else:
|
else:
|
||||||
history.append(f"Eltern über {method_label} erkannt.")
|
history.append(f"Eltern über {method_label} erkannt.")
|
||||||
|
|
||||||
|
# Discarded parent candidates / links (pick_parent_ref rejections, role
|
||||||
|
# normalization drops, parent-age sanity check). Threaded in after the merge
|
||||||
|
# via the gerbil's _discarded list so they read in chronological order.
|
||||||
|
for d in (parent_discards or []):
|
||||||
|
line = _format_discard(d)
|
||||||
|
if line not in history:
|
||||||
|
history.append(line)
|
||||||
|
|
||||||
# Manual decisions / conflicts.
|
# Manual decisions / conflicts.
|
||||||
if any_decision:
|
if any_decision:
|
||||||
history.append("Zuordnung per manueller Entscheidung getroffen.")
|
history.append("Zuordnung per manueller Entscheidung getroffen.")
|
||||||
@@ -821,6 +891,59 @@ def _de_gender(g):
|
|||||||
return {"male": "männlich", "female": "weiblich"}.get(g, g)
|
return {"male": "männlich", "female": "weiblich"}.get(g, g)
|
||||||
|
|
||||||
|
|
||||||
|
# Leading marker that visually flags a discard ("data was thrown away") line in
|
||||||
|
# the history timeline. The frontend keys discard styling off this marker.
|
||||||
|
DISCARD_MARK = "⚠ "
|
||||||
|
|
||||||
|
|
||||||
|
def _format_discard(d):
|
||||||
|
"""Render one discard record into a German history line.
|
||||||
|
|
||||||
|
A discard record is a dict describing a value/candidate the pipeline threw
|
||||||
|
away. Recognised keys:
|
||||||
|
• text — a fully pre-formatted line (used verbatim, marker added)
|
||||||
|
• label — German field label (e.g. „Geburtsdatum“)
|
||||||
|
• value — the discarded value (already display-formatted)
|
||||||
|
• file — source file the discarded value came from (attributed)
|
||||||
|
• reason — why it was dropped (e.g. „abweichend“, „unplausibel …“)
|
||||||
|
• replacement — what was used instead (already display-formatted)
|
||||||
|
• repl_file — source file the replacement came from
|
||||||
|
Produces lines like:
|
||||||
|
„⚠ Geburtsdatum 14.06.2015 aus ‚A.xlsx‘ verworfen — abweichend;
|
||||||
|
14.06.2017 aus ‚B.xlsx‘ verwendet (Mehrheit).“
|
||||||
|
Generic across entity types so litters/contacts can reuse it.
|
||||||
|
"""
|
||||||
|
if d.get("text"):
|
||||||
|
return DISCARD_MARK + d["text"]
|
||||||
|
parts = []
|
||||||
|
label = d.get("label")
|
||||||
|
value = d.get("value")
|
||||||
|
if label and value is not None:
|
||||||
|
parts.append(f"{label} {value}")
|
||||||
|
elif label:
|
||||||
|
parts.append(str(label))
|
||||||
|
elif value is not None:
|
||||||
|
parts.append(str(value))
|
||||||
|
head = " ".join(parts) if parts else "Wert"
|
||||||
|
if d.get("file"):
|
||||||
|
head += f" aus {_quote_file(d['file'])}"
|
||||||
|
line = f"{head} verworfen"
|
||||||
|
if d.get("reason"):
|
||||||
|
line += f" — {d['reason']}"
|
||||||
|
repl = d.get("replacement")
|
||||||
|
if repl is not None and repl != "":
|
||||||
|
instead = str(repl)
|
||||||
|
if d.get("repl_file"):
|
||||||
|
instead += f" aus {_quote_file(d['repl_file'])}"
|
||||||
|
suffix = d.get("replacement_note")
|
||||||
|
line += f"; {instead} verwendet"
|
||||||
|
if suffix:
|
||||||
|
line += f" ({suffix})"
|
||||||
|
elif d.get("no_replacement"):
|
||||||
|
line += "; kein Ersatz"
|
||||||
|
return DISCARD_MARK + line + "."
|
||||||
|
|
||||||
|
|
||||||
def main():
|
def main():
|
||||||
print("Loading color variety seeds...")
|
print("Loading color variety seeds...")
|
||||||
variety_map = {}
|
variety_map = {}
|
||||||
@@ -864,6 +987,14 @@ def main():
|
|||||||
else:
|
else:
|
||||||
print(f"Warning: Seeds path not found at {SEEDS_PATH}")
|
print(f"Warning: Seeds path not found at {SEEDS_PATH}")
|
||||||
|
|
||||||
|
# Reverse map (variety GUID → human name) for discard/replacement history
|
||||||
|
# lines that need to show a colour value rather than a raw GUID. First wins
|
||||||
|
# so we keep the canonical lower-case catalog name.
|
||||||
|
variety_id_to_name = {}
|
||||||
|
for name_lower, vid in variety_map.items():
|
||||||
|
if vid not in variety_id_to_name:
|
||||||
|
variety_id_to_name[vid] = name_lower
|
||||||
|
|
||||||
md_files = sorted([f for f in os.listdir(DIR_PATH) if f.lower().endswith('.md')])
|
md_files = sorted([f for f in os.listdir(DIR_PATH) if f.lower().endswith('.md')])
|
||||||
print(f"Found {len(md_files)} markdown files in {DIR_PATH}.")
|
print(f"Found {len(md_files)} markdown files in {DIR_PATH}.")
|
||||||
|
|
||||||
@@ -1187,7 +1318,20 @@ def main():
|
|||||||
mother_ref = pick_parent_ref(parent_refs, "mother", child_dob_raw,
|
mother_ref = pick_parent_ref(parent_refs, "mother", child_dob_raw,
|
||||||
avoid_name=father_ref.get("name") if father_ref else None,
|
avoid_name=father_ref.get("name") if father_ref else None,
|
||||||
gender_of=gender_of_name)
|
gender_of=gender_of_name)
|
||||||
|
|
||||||
|
# Record rejected parent-ref candidates (wrong sex / age-impossible /
|
||||||
|
# duplicate of the other role) so this animal's gerbil history can
|
||||||
|
# explain which Stammbaum positions were discarded and what won instead.
|
||||||
|
pick_discards = []
|
||||||
|
pick_discards += explain_pick_rejections(parent_refs, "father", child_dob_raw,
|
||||||
|
father_ref, gender_of=gender_of_name)
|
||||||
|
pick_discards += explain_pick_rejections(
|
||||||
|
parent_refs, "mother", child_dob_raw, mother_ref,
|
||||||
|
avoid_name=father_ref.get("name") if father_ref else None,
|
||||||
|
gender_of=gender_of_name)
|
||||||
|
if pick_discards:
|
||||||
|
a["_pick_discards"] = pick_discards
|
||||||
|
|
||||||
a["_mapped_litter_scoped_id"] = None
|
a["_mapped_litter_scoped_id"] = None
|
||||||
if father_ref and mother_ref:
|
if father_ref and mother_ref:
|
||||||
f_name = get_normalized_gerbil_name(father_ref.get("name"))
|
f_name = get_normalized_gerbil_name(father_ref.get("name"))
|
||||||
@@ -1623,6 +1767,23 @@ def main():
|
|||||||
"_old_id": rg.get("Id") or rg.get("id")
|
"_old_id": rg.get("Id") or rg.get("id")
|
||||||
})
|
})
|
||||||
|
|
||||||
|
# Build the discard list for one stammbaum animal: rejected parent-ref
|
||||||
|
# candidates (pick_parent_ref) plus a manual DOB-remap decision, if any.
|
||||||
|
def _stammbaum_discards(a):
|
||||||
|
discards = list(a.get("_pick_discards", []))
|
||||||
|
remap = a.get("dobRemap")
|
||||||
|
if remap and remap.get("original") and remap.get("corrected"):
|
||||||
|
orig = _de_date(parse_date(remap["original"])) or remap["original"]
|
||||||
|
corr = _de_date(parse_date(remap["corrected"])) or remap["corrected"]
|
||||||
|
if orig != corr:
|
||||||
|
discards.append({
|
||||||
|
"label": PROV_FIELD_LABELS["DateOfBirth"],
|
||||||
|
"value": orig,
|
||||||
|
"reason": "per manueller Entscheidung korrigiert",
|
||||||
|
"replacement": corr,
|
||||||
|
})
|
||||||
|
return discards
|
||||||
|
|
||||||
# Map and append stammbaum animals to all_processed_gerbils
|
# Map and append stammbaum animals to all_processed_gerbils
|
||||||
for a in stammbaum_only_animals:
|
for a in stammbaum_only_animals:
|
||||||
a_id = a["id"]
|
a_id = a["id"]
|
||||||
@@ -1731,7 +1892,10 @@ def main():
|
|||||||
"_eff_dob": dob_val or "2010-01-01",
|
"_eff_dob": dob_val or "2010-01-01",
|
||||||
"_birth_date": dob_val,
|
"_birth_date": dob_val,
|
||||||
"_filename": a.get("sourceFiles", ["Stammbaum"])[0],
|
"_filename": a.get("sourceFiles", ["Stammbaum"])[0],
|
||||||
"_old_id": a_id
|
"_old_id": a_id,
|
||||||
|
# Rejected Stammbaum parent-ref candidates for this animal (filled by
|
||||||
|
# pick_parent_ref above) — surfaced in this gerbil's discard history.
|
||||||
|
"_discarded": _stammbaum_discards(a),
|
||||||
})
|
})
|
||||||
|
|
||||||
# Map and append docx animals to all_processed_gerbils
|
# Map and append docx animals to all_processed_gerbils
|
||||||
@@ -1885,7 +2049,8 @@ def main():
|
|||||||
|
|
||||||
return True
|
return True
|
||||||
|
|
||||||
def build_provenance(records, best_g, extra_notes=None, field_source=None):
|
def build_provenance(records, best_g, extra_notes=None, field_source=None,
|
||||||
|
field_discards=None):
|
||||||
"""Aggregate data-provenance across every raw record merged into one
|
"""Aggregate data-provenance across every raw record merged into one
|
||||||
resolved gerbil. Returns a JSON string (stored on the Gerbil entity as a
|
resolved gerbil. Returns a JSON string (stored on the Gerbil entity as a
|
||||||
nullable text column) so the Rennmausakte can show where the entry came
|
nullable text column) so the Rennmausakte can show where the entry came
|
||||||
@@ -1937,6 +2102,8 @@ def main():
|
|||||||
any_decision=any_decision,
|
any_decision=any_decision,
|
||||||
any_conflict=any_conflict,
|
any_conflict=any_conflict,
|
||||||
conflict_notes=extra_notes or [],
|
conflict_notes=extra_notes or [],
|
||||||
|
field_discards=field_discards,
|
||||||
|
parent_discards=best_g.get("_discarded"),
|
||||||
)
|
)
|
||||||
|
|
||||||
return build_entity_provenance(
|
return build_entity_provenance(
|
||||||
@@ -1971,7 +2138,9 @@ def main():
|
|||||||
if is_placeholder:
|
if is_placeholder:
|
||||||
# Placeholders: do NOT merge, keep all separate
|
# Placeholders: do NOT merge, keep all separate
|
||||||
for g in group:
|
for g in group:
|
||||||
g["Provenance"] = build_provenance([g], g)
|
# Provenance is built in a final pass (after parent resolution),
|
||||||
|
# so parent-link discards land in the right gerbil's history.
|
||||||
|
g["_prov_args"] = ([g], g, None, None, None)
|
||||||
resolved_gerbils.append(g)
|
resolved_gerbils.append(g)
|
||||||
gerbil_id_map[g["Id"]] = g["Id"]
|
gerbil_id_map[g["Id"]] = g["Id"]
|
||||||
continue
|
continue
|
||||||
@@ -1992,7 +2161,7 @@ def main():
|
|||||||
for sub in sub_groups:
|
for sub in sub_groups:
|
||||||
if len(sub) == 1:
|
if len(sub) == 1:
|
||||||
g = sub[0]
|
g = sub[0]
|
||||||
g["Provenance"] = build_provenance([g], g)
|
g["_prov_args"] = ([g], g, None, None, None)
|
||||||
resolved_gerbils.append(g)
|
resolved_gerbils.append(g)
|
||||||
gerbil_id_map[g["Id"]] = g["Id"]
|
gerbil_id_map[g["Id"]] = g["Id"]
|
||||||
continue
|
continue
|
||||||
@@ -2019,6 +2188,17 @@ def main():
|
|||||||
if best_g["Notes"]:
|
if best_g["Notes"]:
|
||||||
merged_notes.append(best_g["Notes"])
|
merged_notes.append(best_g["Notes"])
|
||||||
|
|
||||||
|
# Carry over discard records (rejected parent-refs / dob remaps) from
|
||||||
|
# every merged record so none are lost when the primary changes.
|
||||||
|
merged_discards = list(best_g.get("_discarded") or [])
|
||||||
|
for g in sub:
|
||||||
|
if g is best_g:
|
||||||
|
continue
|
||||||
|
for d in (g.get("_discarded") or []):
|
||||||
|
if d not in merged_discards:
|
||||||
|
merged_discards.append(d)
|
||||||
|
best_g["_discarded"] = merged_discards
|
||||||
|
|
||||||
# Merge photos
|
# Merge photos
|
||||||
merged_photos = list(best_g.get("_photos", []))
|
merged_photos = list(best_g.get("_photos", []))
|
||||||
|
|
||||||
@@ -2094,8 +2274,21 @@ def main():
|
|||||||
if not any(kw in g["Notes"].lower() for kw in ["parent listed", "mutter von", "vater von", "dam of", "sire of"]):
|
if not any(kw in g["Notes"].lower() for kw in ["parent listed", "mutter von", "vater von", "dam of", "sire of"]):
|
||||||
merged_notes.append(g["Notes"])
|
merged_notes.append(g["Notes"])
|
||||||
|
|
||||||
|
# Display formatter per field for discard/replacement lines.
|
||||||
|
def _disp(field, val):
|
||||||
|
if val is None or val == "" or val == "unknown":
|
||||||
|
return None
|
||||||
|
if field in ("DateOfBirth", "DateOfDeath"):
|
||||||
|
return _de_date(val)
|
||||||
|
if field == "Gender":
|
||||||
|
return _de_gender(val)
|
||||||
|
if field == "ColorVarietyId":
|
||||||
|
return variety_id_to_name.get(val, "Farbschlag")
|
||||||
|
return str(val)
|
||||||
|
|
||||||
# Reconcile fields based on number of source files supporting them
|
# Reconcile fields based on number of source files supporting them
|
||||||
conflict_notes = []
|
conflict_notes = []
|
||||||
|
field_discards = []
|
||||||
for field in ["DateOfBirth", "DateOfDeath", "Gender", "Genotype", "ColorVarietyId"]:
|
for field in ["DateOfBirth", "DateOfDeath", "Gender", "Genotype", "ColorVarietyId"]:
|
||||||
votes = {}
|
votes = {}
|
||||||
for g in sub:
|
for g in sub:
|
||||||
@@ -2118,6 +2311,25 @@ def main():
|
|||||||
winner = next((g for g in sub if g.get(field) == best_val), None)
|
winner = next((g for g in sub if g.get(field) == best_val), None)
|
||||||
if winner is not None:
|
if winner is not None:
|
||||||
field_source[field] = winner
|
field_source[field] = winner
|
||||||
|
# Record each LOSING value: which file it came from, that it
|
||||||
|
# was discarded as differing, and that the majority value (and
|
||||||
|
# its file) was used instead.
|
||||||
|
if len(votes) > 1:
|
||||||
|
repl_disp = _disp(field, best_val)
|
||||||
|
repl_file = _primary_file_of(winner) if winner is not None else None
|
||||||
|
for lose_val in votes:
|
||||||
|
if lose_val == best_val:
|
||||||
|
continue
|
||||||
|
loser = next((g for g in sub if g.get(field) == lose_val), None)
|
||||||
|
field_discards.append({
|
||||||
|
"label": PROV_FIELD_LABELS.get(field, field),
|
||||||
|
"value": _disp(field, lose_val),
|
||||||
|
"file": _primary_file_of(loser) if loser is not None else None,
|
||||||
|
"reason": "abweichend",
|
||||||
|
"replacement": repl_disp,
|
||||||
|
"repl_file": repl_file,
|
||||||
|
"replacement_note": "Mehrheit",
|
||||||
|
})
|
||||||
# Keep helper fields in sync if we changed DateOfBirth
|
# Keep helper fields in sync if we changed DateOfBirth
|
||||||
if field == "DateOfBirth":
|
if field == "DateOfBirth":
|
||||||
best_g["_birth_date"] = best_val
|
best_g["_birth_date"] = best_val
|
||||||
@@ -2127,9 +2339,9 @@ def main():
|
|||||||
best_g["Notes"] = " | ".join(merged_notes)
|
best_g["Notes"] = " | ".join(merged_notes)
|
||||||
|
|
||||||
best_g["_photos"] = merged_photos
|
best_g["_photos"] = merged_photos
|
||||||
best_g["Provenance"] = build_provenance(
|
# Defer provenance to a final pass so parent-link discards (added
|
||||||
sub, best_g, extra_notes=conflict_notes, field_source=field_source
|
# later by the parent-age / role-normalization passes) are included.
|
||||||
)
|
best_g["_prov_args"] = (sub, best_g, conflict_notes, field_source, field_discards)
|
||||||
|
|
||||||
# Print merge trace
|
# Print merge trace
|
||||||
print(f"Deduplicated same-animal name '{best_g['Name']}': merged {len(sub)} entries across files: {', '.join(sources)}")
|
print(f"Deduplicated same-animal name '{best_g['Name']}': merged {len(sub)} entries across files: {', '.join(sources)}")
|
||||||
@@ -2294,6 +2506,25 @@ def main():
|
|||||||
del l["_mother_name"]
|
del l["_mother_name"]
|
||||||
del l["_filename"]
|
del l["_filename"]
|
||||||
|
|
||||||
|
# Children-of-litter lookup so a dropped parent link can be explained in the
|
||||||
|
# offspring's Datenherkunft (the discard is most meaningful on the child).
|
||||||
|
children_by_litter = {}
|
||||||
|
for g in resolved_gerbils:
|
||||||
|
lid = g.get("LitterId")
|
||||||
|
if lid:
|
||||||
|
children_by_litter.setdefault(lid, []).append(g)
|
||||||
|
|
||||||
|
def _add_child_discard(litter, discard):
|
||||||
|
"""Attach a parent-link discard to every child gerbil of the litter."""
|
||||||
|
for child in children_by_litter.get(litter["Id"], []):
|
||||||
|
dl = child.setdefault("_discarded", [])
|
||||||
|
if discard not in dl:
|
||||||
|
dl.append(discard)
|
||||||
|
|
||||||
|
def _pname(pid):
|
||||||
|
p = gerbil_by_id_final.get(pid)
|
||||||
|
return p.get("Name") if p else None
|
||||||
|
|
||||||
# Role normalization: assign each resolved parent to the role matching its
|
# Role normalization: assign each resolved parent to the role matching its
|
||||||
# gender, eliminate self-pairings (same animal in both roles), and never let
|
# gender, eliminate self-pairings (same animal in both roles), and never let
|
||||||
# impossible duplicates survive (two males / two females). This corrects
|
# impossible duplicates survive (two males / two females). This corrects
|
||||||
@@ -2301,9 +2532,23 @@ def main():
|
|||||||
# (one parent of "unknown" gender, or a self-paired litter).
|
# (one parent of "unknown" gender, or a self-paired litter).
|
||||||
role_fixes = 0
|
role_fixes = 0
|
||||||
for l in resolved_litters:
|
for l in resolved_litters:
|
||||||
|
before = (l.get("FatherId"), l.get("MotherId"))
|
||||||
father, mother = assign_parent_roles(l.get("FatherId"), l.get("MotherId"), _final_gender)
|
father, mother = assign_parent_roles(l.get("FatherId"), l.get("MotherId"), _final_gender)
|
||||||
if (l.get("FatherId"), l.get("MotherId")) != (father, mother):
|
if before != (father, mother):
|
||||||
role_fixes += 1
|
role_fixes += 1
|
||||||
|
after = {father, mother}
|
||||||
|
# A parent id present before but gone after was dropped by role
|
||||||
|
# normalization (self-pairing or two-of-the-same-sex). Explain it.
|
||||||
|
for pid in before:
|
||||||
|
if pid and pid not in after:
|
||||||
|
pname = _pname(pid)
|
||||||
|
if before[0] == before[1]:
|
||||||
|
reason = "Selbstverpaarung — Tier kann nicht beide Elternteile sein"
|
||||||
|
else:
|
||||||
|
reason = "ein Wurf hat nur einen Vater und eine Mutter"
|
||||||
|
_add_child_discard(l, {
|
||||||
|
"text": f"Elternteil „{pname}“ verworfen — {reason}",
|
||||||
|
})
|
||||||
l["FatherId"] = father
|
l["FatherId"] = father
|
||||||
l["MotherId"] = mother
|
l["MotherId"] = mother
|
||||||
if role_fixes:
|
if role_fixes:
|
||||||
@@ -2323,6 +2568,23 @@ def main():
|
|||||||
p = gerbil_by_id_final.get(pid)
|
p = gerbil_by_id_final.get(pid)
|
||||||
if p and not parent_age_plausible(p.get("DateOfBirth"), ldate):
|
if p and not parent_age_plausible(p.get("DateOfBirth"), ldate):
|
||||||
age_drops.append((l.get("Name"), role, p.get("Name"), p.get("DateOfBirth"), ldate))
|
age_drops.append((l.get("Name"), role, p.get("Name"), p.get("DateOfBirth"), ldate))
|
||||||
|
# Explain the drop in each child's history: which parent, its DOB,
|
||||||
|
# why (implausible age), and that no replacement was used.
|
||||||
|
role_de = "Vater" if role == "FatherId" else "Mutter"
|
||||||
|
pdob_disp = _de_date(p.get("DateOfBirth")) or "unbekannt"
|
||||||
|
age_reason = "unplausibles Alter für diesen Wurf"
|
||||||
|
pd = date_to_days(parse_date(p.get("DateOfBirth"))) if p.get("DateOfBirth") else None
|
||||||
|
ld = date_to_days(parse_date(ldate)) if ldate else None
|
||||||
|
if pd is not None and ld is not None:
|
||||||
|
years = abs(ld - pd) / 365.25
|
||||||
|
if ld - pd <= 0:
|
||||||
|
age_reason = "unplausibel (nicht vor dem Kind geboren)"
|
||||||
|
else:
|
||||||
|
age_reason = f"unplausibel ({years:.0f} Jahre älter als das Kind)"
|
||||||
|
_add_child_discard(l, {
|
||||||
|
"text": (f"{role_de} „{p.get('Name')}“ (*{pdob_disp}) verworfen "
|
||||||
|
f"— {age_reason}; kein Ersatz"),
|
||||||
|
})
|
||||||
l[role] = None
|
l[role] = None
|
||||||
if age_drops:
|
if age_drops:
|
||||||
print(f"Parent-age sanity check: dropped {len(age_drops)} implausible parent link(s):")
|
print(f"Parent-age sanity check: dropped {len(age_drops)} implausible parent link(s):")
|
||||||
@@ -2402,6 +2664,27 @@ def main():
|
|||||||
resolved_litters = deduped2
|
resolved_litters = deduped2
|
||||||
litter_by_scoped_id = {l["Id"]: l for l in resolved_litters}
|
litter_by_scoped_id = {l["Id"]: l for l in resolved_litters}
|
||||||
|
|
||||||
|
# Final gerbil-provenance pass: now that parent links are fully resolved and
|
||||||
|
# all discards (majority-vote conflicts during dedup; rejected parent-refs;
|
||||||
|
# role-normalization drops; parent-age drops) are attached to each gerbil's
|
||||||
|
# _discarded list, render the provenance JSON with the discard history lines.
|
||||||
|
n_discards = 0
|
||||||
|
for g in resolved_gerbils:
|
||||||
|
args = g.pop("_prov_args", None)
|
||||||
|
if g.get("_discarded"):
|
||||||
|
n_discards += len(g["_discarded"])
|
||||||
|
if args is not None:
|
||||||
|
records, best_g, conflict_notes, field_source, field_discards = args
|
||||||
|
g["Provenance"] = build_provenance(
|
||||||
|
records, best_g, extra_notes=conflict_notes,
|
||||||
|
field_source=field_source, field_discards=field_discards,
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
g["Provenance"] = build_provenance([g], g)
|
||||||
|
g.pop("_discarded", None)
|
||||||
|
if n_discards:
|
||||||
|
print(f"Datenherkunft: recorded {n_discards} discard line(s) across gerbils.")
|
||||||
|
|
||||||
# Datenherkunft for litters: which source files contributed, whether this is
|
# Datenherkunft for litters: which source files contributed, whether this is
|
||||||
# a Wurfchronik litter vs a Stammbaum-reconstructed ("virtual") litter, how
|
# a Wurfchronik litter vs a Stammbaum-reconstructed ("virtual") litter, how
|
||||||
# many raw records merged into it, plus human-readable notes. Accumulators
|
# many raw records merged into it, plus human-readable notes. Accumulators
|
||||||
|
|||||||
@@ -277,6 +277,74 @@ check("_de_date: ISO → DD.MM.YYYY", m._de_date("2022-03-27") == "27.03.2022")
|
|||||||
check("_de_date: passes through non-ISO", m._de_date("unbekannt") == "unbekannt")
|
check("_de_date: passes through non-ISO", m._de_date("unbekannt") == "unbekannt")
|
||||||
|
|
||||||
|
|
||||||
|
# ── Discard history: a discarded source value records reason + replacement ──
|
||||||
|
# _format_discard: majority-vote conflict (losing value + file → winner + file).
|
||||||
|
_d_mehr = m._format_discard({
|
||||||
|
"label": "Geburtsdatum", "value": "14.06.2015", "file": "A.xlsx",
|
||||||
|
"reason": "abweichend", "replacement": "14.06.2017", "repl_file": "B.xlsx",
|
||||||
|
"replacement_note": "Mehrheit",
|
||||||
|
})
|
||||||
|
check("discard: starts with warning marker", _d_mehr.startswith(m.DISCARD_MARK))
|
||||||
|
check("discard(majority): names losing value + its file",
|
||||||
|
"Geburtsdatum 14.06.2015 aus „A.xlsx“ verworfen" in _d_mehr)
|
||||||
|
check("discard(majority): states the reason", "— abweichend" in _d_mehr)
|
||||||
|
check("discard(majority): names replacement + its file + note",
|
||||||
|
"14.06.2017 aus „B.xlsx“ verwendet (Mehrheit)." in _d_mehr)
|
||||||
|
|
||||||
|
# _format_discard: a parent dropped with NO replacement.
|
||||||
|
_d_noerepl = m._format_discard({
|
||||||
|
"text": "Vater „Jayjay“ (*19.06.2013) verworfen — unplausibel (9 Jahre älter "
|
||||||
|
"als das Kind); kein Ersatz",
|
||||||
|
})
|
||||||
|
check("discard(text): verbatim text gets the warning marker",
|
||||||
|
_d_noerepl == m.DISCARD_MARK + "Vater „Jayjay“ (*19.06.2013) verworfen — "
|
||||||
|
"unplausibel (9 Jahre älter als das Kind); kein Ersatz")
|
||||||
|
|
||||||
|
# explain_pick_rejections: a wrong-sex father candidate is explained (Molly case).
|
||||||
|
_picks = m.explain_pick_rejections(
|
||||||
|
molly_refs, "father", "13.09.2021", fr, gender_of=gof,
|
||||||
|
)
|
||||||
|
_pick_father = next((d for d in _picks if "Danielle" in (d.get("value") or "")), None)
|
||||||
|
check("pick-reject: wrong-sex father candidate is recorded",
|
||||||
|
_pick_father is not None)
|
||||||
|
check("pick-reject: reason = wrong sex for the father role",
|
||||||
|
_pick_father and "falsches Geschlecht für die Vaterrolle" in _pick_father["reason"])
|
||||||
|
check("pick-reject: replacement names the chosen Hagrid",
|
||||||
|
_pick_father and "Hagrid" in (_pick_father.get("replacement") or ""))
|
||||||
|
|
||||||
|
# explain_pick_rejections: an age-impossible candidate is explained.
|
||||||
|
_age_refs = [pref("Old", "father", "2010-01-01"), pref("Dad", "father", "2021-01-01")]
|
||||||
|
_chosen = m.pick_parent_ref(_age_refs, "father", "2022-03-27")
|
||||||
|
_age_picks = m.explain_pick_rejections(_age_refs, "father", "2022-03-27", _chosen)
|
||||||
|
check("pick-reject(age): age-impossible candidate recorded with reason",
|
||||||
|
any("unplausibles Alter" in d["reason"] for d in _age_picks))
|
||||||
|
|
||||||
|
# _build_gerbil_history threads field_discards (after merge) and parent_discards
|
||||||
|
# (after the parent line) into the timeline.
|
||||||
|
g_disc = {
|
||||||
|
"Name": "Solice", "DateOfBirth": "2022-03-27", "Gender": "male",
|
||||||
|
"Genotype": None, "ColorVarietyId": None, "DateOfDeath": None,
|
||||||
|
"ImportSource": "Stammbaum.xlsx", "_filename": "Stammbaum.xlsx",
|
||||||
|
"parentRefs": [],
|
||||||
|
"_discarded": [{"text": "Vater „Jayjay“ verworfen — unplausibel; kein Ersatz"}],
|
||||||
|
}
|
||||||
|
h_disc = m._build_gerbil_history(
|
||||||
|
[g_disc], g_disc, {},
|
||||||
|
field_discards=[{
|
||||||
|
"label": "Geschlecht", "value": "weiblich", "file": "Wurfchronik.docx",
|
||||||
|
"reason": "abweichend", "replacement": "männlich", "repl_file": "Stammbaum.xlsx",
|
||||||
|
"replacement_note": "Mehrheit",
|
||||||
|
}],
|
||||||
|
parent_discards=g_disc["_discarded"],
|
||||||
|
)
|
||||||
|
check("history: field-discard line present (majority vote)",
|
||||||
|
any("Geschlecht weiblich aus „Wurfchronik.docx“ verworfen" in s for s in h_disc))
|
||||||
|
check("history: parent-discard line present (dropped parent)",
|
||||||
|
any("Vater „Jayjay“ verworfen" in s for s in h_disc))
|
||||||
|
check("history: discard lines carry the warning marker",
|
||||||
|
all(s.startswith(m.DISCARD_MARK) for s in h_disc if "verworfen" in s))
|
||||||
|
|
||||||
|
|
||||||
if check.failed:
|
if check.failed:
|
||||||
print(f"\n{check.failed} test(s) FAILED")
|
print(f"\n{check.failed} test(s) FAILED")
|
||||||
sys.exit(1)
|
sys.exit(1)
|
||||||
|
|||||||
Reference in New Issue
Block a user