From cf672c09331f68a6fe1894d3eddae384fadab529 Mon Sep 17 00:00:00 2001 From: Gulum Date: Mon, 22 Jun 2026 16:21:24 +0200 Subject: [PATCH] feat(provenance): Datenherkunft als chronologischer, datei-attribuierter Verlauf MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Die Herkunft liest sich jetzt wie eine History: pro Feld wird genannt, aus welcher Quelldatei der Wert kam, plus Merge-/Entscheidungs-/Wurfchronik-Schritte. Beispiel: „In ‚X.xlsx' gefunden." → „Geburtsdatum (…) aus ‚X.xlsx'." → „Auch in ‚Y.docx' gefunden → zusammengeführt." → „Eltern über Position erkannt (Quelle …)." Import: build_entity_provenance() um history[] erweitert; ein field_source- Tracking im Gerbil-Dedup hält fest, welcher Datensatz/welche Datei jedes Feld (DOB/Genotyp/Geschlecht/Farbschlag/Sterbedatum) lieferte — auch über Empty-Fill und Mehrheitsentscheid hinweg. Kontakte und Würfe erhalten analoge, datei- attribuierte Verläufe. Kein DB-Migration nötig (history liegt im Provenance-JSON). Frontend: ProvenanceDialog rendert den Verlauf als nummerierte Timeline. Tests: history wird erzeugt, nennt Dateien, überlebt Ingest (dotnet 212, vitest 129, playwright 17, python merge/extract grün). Co-Authored-By: Claude Opus 4.8 --- .../IngestResolvedServiceTests.cs | 6 +- gerbil-manager-web/e2e/mock-data.ts | 18 ++ gerbil-manager-web/e2e/provenance.spec.ts | 16 + gerbil-manager-web/src/api/types.ts | 7 + .../src/components/ProvenanceDialog.tsx | 14 + .../src/components/provenanceDialog.css | 14 + gerbil-manager-web/src/strings/de.ts | 2 + tools/import/merge_and_resolve.py | 285 +++++++++++++++--- tools/import/test_merge_resolve.py | 64 ++++ 9 files changed, 389 insertions(+), 37 deletions(-) diff --git a/GerbilManager.Tests/IngestResolvedServiceTests.cs b/GerbilManager.Tests/IngestResolvedServiceTests.cs index 732ffb0..06d4606 100644 --- a/GerbilManager.Tests/IngestResolvedServiceTests.cs +++ b/GerbilManager.Tests/IngestResolvedServiceTests.cs @@ -87,7 +87,7 @@ namespace GerbilManager.Tests CharacterNote = (string?)null, IsDeaf = false, IsResident = true, - Provenance = (string?)"{\"sourceFiles\":[\"Stammbaum von Papa.xlsx\",\"Wurfchronik-Detail.docx\"],\"mergedRecordCount\":2,\"fromWurfchronik\":true,\"notes\":[\"aus 2 Datensätzen zusammengeführt\"]}" + Provenance = (string?)"{\"sourceFiles\":[\"Stammbaum von Papa.xlsx\",\"Wurfchronik-Detail.docx\"],\"mergedRecordCount\":2,\"fromWurfchronik\":true,\"notes\":[\"aus 2 Datensätzen zusammengeführt\"],\"history\":[\"In \\u201eStammbaum von Papa.xlsx\\u201c gefunden.\",\"Genotyp aus \\u201eWurfchronik-Detail.docx\\u201c.\"]}" }, new { @@ -185,6 +185,10 @@ namespace GerbilManager.Tests Assert.NotNull(father.Provenance); Assert.Contains("Stammbaum von Papa.xlsx", father.Provenance); Assert.Contains("mergedRecordCount", father.Provenance); + // The chronological, file-attributed history survives ingest verbatim + // (stored as opaque JSON on the text column — no schema for it). + Assert.Contains("history", father.Provenance); + Assert.Contains("Genotyp aus", father.Provenance); Assert.Null(mother.Provenance); // Contact + litter provenance also round-trips through the ingest. diff --git a/gerbil-manager-web/e2e/mock-data.ts b/gerbil-manager-web/e2e/mock-data.ts index 4e99b03..cfef01e 100644 --- a/gerbil-manager-web/e2e/mock-data.ts +++ b/gerbil-manager-web/e2e/mock-data.ts @@ -130,6 +130,14 @@ export function seedDb(): MockDb { parentMethod: 'chart-position', parentConfidence: 'medium', notes: ['aus 2 Datensätzen zusammengeführt'], + history: [ + 'In „Stammbaum von Krümel.xlsx“ gefunden.', + 'Geburtsdatum (12.03.2025) aus „Stammbaum von Krümel.xlsx“.', + 'Genotyp aus „Stammbaum von Krümel.xlsx“.', + 'Auch in „Wurfchronik-Detail.docx“ gefunden → Datensätze zusammengeführt.', + 'Eltern über Position im Stammbaum erkannt (Quelle: „Stammbaum von Krümel.xlsx“).', + 'Angaben aus der Wurfchronik übernommen.', + ], }), }, { ...gerbil('fridolin', 'Fridolin', 'male', '2023-05-01', 'w-fridolin', 'cv-schwarz', 'aa CC DD EE GG PP spsp rere'), enclosureId: 'enc-gross', originBreeder: 'Zoohandlung Meier', profilePhotoUrl: 'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==' }, @@ -203,6 +211,11 @@ export function seedDb(): MockDb { mergedRecordCount: 2, fromWurfchronik: true, notes: ['aus Wurfchronik', 'aus 2 Datensätzen zusammengeführt', 'Geschwister-Würfe zusammengeführt'], + history: [ + 'Wurf aus Wurfchronik „Wurfchronik Teil 1_page_0009.md“.', + 'Auch in „Wurfchronik Teil 1_page_0028.md“ gefunden → Datensätze zusammengeführt.', + 'Geschwister-Würfe zusammengeführt.', + ], }), }, { id: 'w-fridolin', name: 'Wurf F', date: '2023-05-01', totalBorn: 4, expectedGoHomeDate: null, notes: null, fatherId: 'balu', motherId: 'maja' }, @@ -229,6 +242,11 @@ export function seedDb(): MockDb { mergedRecordCount: 2, fromWurfchronik: true, notes: ['aus 2 Datensätzen zusammengeführt', 'als Züchter erkannt', 'als Abnehmer erkannt'], + history: [ + 'In „Stammbaum von Krümel.xlsx“ als Züchter erkannt.', + 'Auch in „Wurfchronik Teil 1_page_0001.md“ gefunden → Datensätze zusammengeführt.', + 'Sowohl als Züchter als auch als Abnehmer geführt.', + ], }), }, { id: 'con-huber', name: 'Familie Huber', email: null, phone: '0151 2345678', address: null, notes: null, isBreeder: false, isReceiver: true }, diff --git a/gerbil-manager-web/e2e/provenance.spec.ts b/gerbil-manager-web/e2e/provenance.spec.ts index a65da58..eeb518e 100644 --- a/gerbil-manager-web/e2e/provenance.spec.ts +++ b/gerbil-manager-web/e2e/provenance.spec.ts @@ -13,6 +13,14 @@ test('Tierakte: "Datenherkunft"-Button öffnet den Nachverfolgungs-Dialog', asyn const dialog = page.getByRole('dialog', { name: p.dialogTitle }) await expect(dialog).toBeVisible() + // VERLAUF: chronologischer, dateibezogener Verlauf als Primärinhalt. + await expect(dialog).toContainText(p.historyTitle) + await expect(dialog).toContainText('In „Stammbaum von Krümel.xlsx“ gefunden.') + await expect(dialog).toContainText('Geburtsdatum (12.03.2025) aus „Stammbaum von Krümel.xlsx“.') + await expect(dialog).toContainText( + 'Auch in „Wurfchronik-Detail.docx“ gefunden → Datensätze zusammengeführt.', + ) + // Quelldateien werden angezeigt. await expect(dialog).toContainText(p.sourceFilesTitle) await expect(dialog).toContainText('Stammbaum von Krümel.xlsx') @@ -53,6 +61,10 @@ test('Kontakt: "Datenherkunft"-Button öffnet den Nachverfolgungs-Dialog', async const dialog = page.getByRole('dialog', { name: p.dialogTitle }) await expect(dialog).toBeVisible() + // VERLAUF: dateibezogene Schritte des Kontakts. + await expect(dialog).toContainText(p.historyTitle) + await expect(dialog).toContainText('In „Stammbaum von Krümel.xlsx“ als Züchter erkannt.') + // Quelldateien + Zusammenführung + Kontakt-Rolle-Hinweise. await expect(dialog).toContainText(p.sourceFilesTitle) await expect(dialog).toContainText('Wurfchronik Teil 1_page_0001.md') @@ -73,6 +85,10 @@ test('Wurf: "Datenherkunft"-Button öffnet den Nachverfolgungs-Dialog', async ({ const dialog = page.getByRole('dialog', { name: p.dialogTitle }) await expect(dialog).toBeVisible() + // VERLAUF: dateibezogene Schritte des Wurfs. + await expect(dialog).toContainText(p.historyTitle) + await expect(dialog).toContainText('Wurf aus Wurfchronik „Wurfchronik Teil 1_page_0009.md“.') + await expect(dialog).toContainText(p.sourceFilesTitle) await expect(dialog).toContainText('Wurfchronik Teil 1_page_0009.md') await expect(dialog).toContainText(p.mergedCount(2)) diff --git a/gerbil-manager-web/src/api/types.ts b/gerbil-manager-web/src/api/types.ts index 53e46bf..725cd89 100644 --- a/gerbil-manager-web/src/api/types.ts +++ b/gerbil-manager-web/src/api/types.ts @@ -88,6 +88,13 @@ export interface GerbilProvenance { parentConfidence?: string /** Menschlich lesbare Herkunftshinweise (deutsch). */ notes: string[] + /** + * Chronologischer, dateibezogener Verlauf (deutsch): liest sich wie ein + * Protokoll, WELCHE Quelldatei WELCHE Angabe beigetragen hat (z. B. + * „Geburtsdatum (27.03.2022) aus ‚Stammbaum von X.xlsx'."). Primärinhalt des + * Datenherkunft-Dialogs. + */ + history: string[] } /** diff --git a/gerbil-manager-web/src/components/ProvenanceDialog.tsx b/gerbil-manager-web/src/components/ProvenanceDialog.tsx index 1e34ea6..8b72519 100644 --- a/gerbil-manager-web/src/components/ProvenanceDialog.tsx +++ b/gerbil-manager-web/src/components/ProvenanceDialog.tsx @@ -33,6 +33,7 @@ function parseProvenance(raw?: string | null): EntityProvenance | null { parentMethod: p.parentMethod, parentConfidence: p.parentConfidence, notes: Array.isArray(p.notes) ? p.notes : [], + history: Array.isArray(p.history) ? p.history : [], } } catch { return null @@ -88,6 +89,19 @@ export default function ProvenanceDialog({ open, onClose, provenance, entityLabe <>

{entityLabel ? t.introFor(entityLabel) : t.intro}

+ {p.history.length > 0 && ( +
+
{t.historyTitle}
+
    + {p.history.map((step, i) => ( +
  1. + {step} +
    2. + ))} +
+
+ )} +
{t.mergedTitle} diff --git a/gerbil-manager-web/src/components/provenanceDialog.css b/gerbil-manager-web/src/components/provenanceDialog.css index a8318ba..dc1a045 100644 --- a/gerbil-manager-web/src/components/provenanceDialog.css +++ b/gerbil-manager-web/src/components/provenanceDialog.css @@ -149,6 +149,20 @@ font-size: 0.85rem; } +/* Chronologischer Verlauf: liest sich wie ein Protokoll der Datenherkunft. */ +.provenance__history { + margin: 0; + padding-left: 1.25rem; + display: grid; + gap: 0.35rem; +} + +.provenance__history-step { + font-size: 0.88rem; + line-height: 1.35; + word-break: break-word; +} + .provenance__actions { display: flex; justify-content: flex-end; diff --git a/gerbil-manager-web/src/strings/de.ts b/gerbil-manager-web/src/strings/de.ts index 5e8f10e..29ba8d8 100644 --- a/gerbil-manager-web/src/strings/de.ts +++ b/gerbil-manager-web/src/strings/de.ts @@ -894,6 +894,8 @@ export const de = { contact: 'dieses Kontakts', litter: 'dieses Wurfs', }, + /** Überschrift des chronologischen, dateibezogenen Verlaufs (Primärinhalt). */ + historyTitle: 'Verlauf', /** Überschriften / Feldbeschriftungen. */ sourceFilesTitle: 'Quelldateien', sourceFilesCount: (n: number) => (n === 1 ? 'aus 1 Quelle' : `aus ${n} Quellen`), diff --git a/tools/import/merge_and_resolve.py b/tools/import/merge_and_resolve.py index 052f23f..18b8258 100644 --- a/tools/import/merge_and_resolve.py +++ b/tools/import/merge_and_resolve.py @@ -23,14 +23,17 @@ def generate_guid(key_str): return str(uuid.uuid5(uuid.NAMESPACE_DNS, key_str)) def build_entity_provenance(source_files, merged_record_count, notes=None, - from_wurfchronik=None, extra=None): + from_wurfchronik=None, extra=None, history=None): """Generic data-provenance JSON builder shared by gerbils, contacts and litters. Mirrors the GerbilProvenance frontend contract: - { sourceFiles, mergedRecordCount, fromWurfchronik, notes, ... } + { sourceFiles, mergedRecordCount, fromWurfchronik, notes, history, ... } `source_files` is any iterable of filenames; `from_wurfchronik` is auto- derived from the filenames when left as None. `extra` may carry entity- - specific keys (e.g. parentMethod/parentConfidence for gerbils). Returns a - JSON string (stored on the nullable Provenance text column).""" + specific keys (e.g. parentMethod/parentConfidence for gerbils). `history` + is an ordered list of human-readable German lines that read like a + chronological log of where each fact came from (the primary content shown + in the Datenherkunft dialog). Returns a JSON string (stored on the nullable + Provenance text column).""" files = sorted({f for f in source_files if f}) if from_wurfchronik is None: from_wurfchronik = any("wurfchronik" in f.lower() for f in files) @@ -39,6 +42,7 @@ def build_entity_provenance(source_files, merged_record_count, notes=None, "mergedRecordCount": merged_record_count, "fromWurfchronik": bool(from_wurfchronik), "notes": list(notes or []), + "history": list(history or []), } if extra: for k, v in extra.items(): @@ -46,6 +50,58 @@ def build_entity_provenance(source_files, merged_record_count, notes=None, prov[k] = v return json.dumps(prov, ensure_ascii=False) + +def _quote_file(fname): + """German typographic quotes around a source filename for history lines.""" + return f"„{fname}“" + + +# Human-readable German labels for the significant fields we attribute to files. +PROV_FIELD_LABELS = { + "DateOfBirth": "Geburtsdatum", + "DateOfDeath": "Sterbedatum", + "Gender": "Geschlecht", + "Genotype": "Genotyp", + "ColorVarietyId": "Farbschlag", + "Name": "Name", +} + + +def _primary_file_of(record): + """The single most representative source file of one raw record. + + Prefers an explicit Stammbaum/Wurfchronik filename from ImportSource, else + the record's _filename. Used to attribute a field value to a concrete file + in the history log.""" + fn = record.get("_filename") + if fn: + return fn + imp = record.get("ImportSource") + if imp: + first = str(imp).split(",")[0].strip() + if first: + return first + return None + + +def _record_source_files(g): + """All distinct source files a single raw gerbil record drew from. + + ImportSource is either a comma-joined Stammbaum file list, a single + Wurfchronik filename, or the per-litter filename; _filename is the primary + file. We union both so nothing is lost.""" + files = set() + imp = g.get("ImportSource") + if imp: + for part in str(imp).split(","): + part = part.strip() + if part: + files.add(part) + fn = g.get("_filename") + if fn: + files.add(fn) + return files + def to_valid_guid(val): if not val: return None @@ -657,6 +713,114 @@ def pick_parent_ref(parent_refs, role, child_dob, avoid_name=None, gender_of=Non return role_refs[order[0]] +def _build_gerbil_history(records, best_g, field_source, parent_method=None, + any_decision=False, any_conflict=False, conflict_notes=None): + """Build an ordered, file-attributed German history for a resolved gerbil. + + Reads like a chronological log: + • „In ‚X.xlsx' gefunden." + • „Geburtsdatum (27.03.2022) aus ‚X.xlsx'." + • „Auch in ‚Y.xlsx' gefunden → Datensätze zusammengeführt." + • „Genotyp aus ‚Z.xlsx'." + • „Eltern über Position im Stammbaum erkannt (Quelle: ‚X.xlsx')." + • „Aus Wurfchronik übernommen." + + `field_source` maps a field name to the raw record that supplied its final + value; when present we name that record's file, otherwise we fall back to + the primary record. The records are visited in a stable order (primary + first, then the rest sorted by file) so the log is deterministic.""" + conflict_notes = conflict_notes or [] + history = [] + + # Order records: best_g first, then others by primary file name (stable). + others = [r for r in records if r is not best_g] + others.sort(key=lambda r: (_primary_file_of(r) or "")) + ordered = [best_g] + others + + best_file = _primary_file_of(best_g) + if best_file: + history.append(f"In {_quote_file(best_file)} gefunden.") + else: + history.append("Im Import gefunden.") + + # Field-by-field attribution: name the file that supplied each fact. + def attr_line(field, formatter): + rec = field_source.get(field) or best_g + val = best_g.get(field) + if not val or val == "unknown": + return + fname = _primary_file_of(rec) + label = PROV_FIELD_LABELS.get(field, field) + text = formatter(label, val) + if fname: + history.append(f"{text} aus {_quote_file(fname)}.") + else: + history.append(f"{text} (Quelle unbekannt).") + + attr_line("DateOfBirth", lambda label, v: f"{label} ({_de_date(v)})") + attr_line("Gender", lambda label, v: f"{label} ({_de_gender(v)})") + attr_line("Genotype", lambda label, v: f"{label}") + attr_line("ColorVarietyId", lambda label, v: f"{label}") + attr_line("DateOfDeath", lambda label, v: f"{label} ({_de_date(v)})") + + # Merge step: every additional record that contributed. + for r in others: + fname = _primary_file_of(r) + if fname: + history.append( + f"Auch in {_quote_file(fname)} gefunden → Datensätze zusammengeführt." + ) + else: + history.append("In weiterem Datensatz gefunden → Datensätze zusammengeführt.") + + # Parent derivation. + if parent_method: + method_label = { + "chart-position": "Position im Stammbaum", + "geburtsdatum+eltern": "Geburtsdatum und Elternnamen", + "nur-geburtsdatum": "Geburtsdatum", + "decision": "manuelle Entscheidung", + }.get(parent_method, parent_method) + # The parent evidence comes from a Stammbaum chart — attribute to the + # primary record's file when it is a Stammbaum. + parent_file = best_file if best_file and "stammbaum" in best_file.lower() else None + if parent_file: + history.append( + f"Eltern über {method_label} erkannt (Quelle: {_quote_file(parent_file)})." + ) + else: + history.append(f"Eltern über {method_label} erkannt.") + + # Manual decisions / conflicts. + if any_decision: + history.append("Zuordnung per manueller Entscheidung getroffen.") + if any_conflict: + history.append("Konflikt per Entscheidung gelöst.") + for cn in conflict_notes: + if cn and cn not in history: + history.append(cn + ".") + + # Wurfchronik provenance line. + if any("wurfchronik" in f.lower() for r in records for f in _record_source_files(r)): + history.append("Angaben aus der Wurfchronik übernommen.") + + return history + + +def _de_date(iso): + """YYYY-MM-DD → DD.MM.YYYY for display; pass through anything else.""" + if not iso: + return iso + m = re.match(r"^(\d{4})-(\d{2})-(\d{2})$", str(iso)) + if m: + return f"{m.group(3)}.{m.group(2)}.{m.group(1)}" + return iso + + +def _de_gender(g): + return {"male": "männlich", "female": "weiblich"}.get(g, g) + + def main(): print("Loading color variety seeds...") variety_map = {} @@ -1721,30 +1885,14 @@ def main(): return True - def _record_source_files(g): - """All distinct source files a single raw record drew from. - - ImportSource is either a comma-joined Stammbaum file list, a single - Wurfchronik filename, or the per-litter filename; _filename is the - primary file. We union both so nothing is lost.""" - files = set() - imp = g.get("ImportSource") - if imp: - for part in str(imp).split(","): - part = part.strip() - if part: - files.add(part) - fn = g.get("_filename") - if fn: - files.add(fn) - return files - - def build_provenance(records, best_g, extra_notes=None): + def build_provenance(records, best_g, extra_notes=None, field_source=None): """Aggregate data-provenance across every raw record merged into one resolved gerbil. Returns a JSON string (stored on the Gerbil entity as a nullable text column) so the Rennmausakte can show where the entry came from. `records` is the list of raw records that became this gerbil; - `best_g` is the chosen primary record.""" + `best_g` is the chosen primary record. `field_source` maps a significant + field name (DateOfBirth/Genotype/…) to the raw record that supplied its + final value, so the history can name the exact file for each fact.""" source_files = set() from_wurfchronik = False any_conflict = False @@ -1783,12 +1931,21 @@ def main(): if n and n not in notes: notes.append(n) + history = _build_gerbil_history( + records, best_g, field_source or {}, + parent_method=parent_method, + any_decision=any_decision, + any_conflict=any_conflict, + conflict_notes=extra_notes or [], + ) + return build_entity_provenance( source_files, merged_count, notes=notes, from_wurfchronik=from_wurfchronik, extra={"parentMethod": parent_method, "parentConfidence": parent_confidence}, + history=history, ) # Group gerbils by name to perform deduplication @@ -1861,13 +2018,24 @@ def main(): merged_notes = [] if best_g["Notes"]: merged_notes.append(best_g["Notes"]) - + # Merge photos merged_photos = list(best_g.get("_photos", [])) # Track sources for debugging sources = [best_g["_filename"]] + # Per-field file attribution: which raw record supplied each final + # field value. Seed with best_g for every field it already carries; + # the fill loop and voting loop update it as winners change. + ATTRIB_FIELDS = ["DateOfBirth", "DateOfDeath", "Gender", "Genotype", + "ColorVarietyId", "Name"] + field_source = {} + for fld in ATTRIB_FIELDS: + v = best_g.get(fld) + if v and v != "unknown": + field_source[fld] = best_g + for g in sub: if g == best_g: continue @@ -1884,16 +2052,20 @@ def main(): best_g["LitterId"] = g["LitterId"] if not best_g["DateOfBirth"] and g["DateOfBirth"]: best_g["DateOfBirth"] = g["DateOfBirth"] + field_source["DateOfBirth"] = g if not best_g["DateOfDeath"] and g["DateOfDeath"]: best_g["DateOfDeath"] = g["DateOfDeath"] + field_source["DateOfDeath"] = g if not best_g["CauseOfDeath"] and g["CauseOfDeath"]: best_g["CauseOfDeath"] = g["CauseOfDeath"] if not best_g["GoHomeDate"] and g["GoHomeDate"]: best_g["GoHomeDate"] = g["GoHomeDate"] if not best_g["Genotype"] and g["Genotype"]: best_g["Genotype"] = g["Genotype"] + field_source["Genotype"] = g if not best_g["ColorVarietyId"] and g["ColorVarietyId"]: best_g["ColorVarietyId"] = g["ColorVarietyId"] + field_source["ColorVarietyId"] = g if not best_g["OriginContactId"] and g["OriginContactId"]: best_g["OriginContactId"] = g["OriginContactId"] if not best_g["ReceiverContactId"] and g["ReceiverContactId"]: @@ -1904,10 +2076,12 @@ def main(): # Reconcile Gender: prefer a known gender over unknown, and prefer stammbaum over other sources if best_g["Gender"] == "unknown" and g["Gender"] != "unknown": best_g["Gender"] = g["Gender"] + field_source["Gender"] = g elif best_g["Gender"] != "unknown" and g["Gender"] != "unknown" and best_g["Gender"] != g["Gender"]: if g["ImportSource"] and "stammbaum" in g["ImportSource"].lower(): if not best_g["ImportSource"] or "stammbaum" not in best_g["ImportSource"].lower(): best_g["Gender"] = g["Gender"] + field_source["Gender"] = g # Status precedence: Deceased > GivenAway > Breeding/Pet if g["Status"] == "Deceased": @@ -1935,17 +2109,15 @@ def main(): # If the records disagreed on a field, the merge had to pick a # winner — record that as a provenance note. if len(votes) > 1: - FIELD_LABEL = { - "DateOfBirth": "Geburtsdatum", - "DateOfDeath": "Sterbedatum", - "Gender": "Geschlecht", - "Genotype": "Genotyp", - "ColorVarietyId": "Farbschlag", - } conflict_notes.append( - f"Konflikt bei {FIELD_LABEL[field]} per Mehrheitsentscheidung gelöst" + f"Konflikt bei {PROV_FIELD_LABELS[field]} per Mehrheitsentscheidung gelöst" ) best_g[field] = best_val + # Attribute the winning value to a record that actually holds + # it, so the history names the right file. + winner = next((g for g in sub if g.get(field) == best_val), None) + if winner is not None: + field_source[field] = winner # Keep helper fields in sync if we changed DateOfBirth if field == "DateOfBirth": best_g["_birth_date"] = best_val @@ -1955,7 +2127,9 @@ def main(): best_g["Notes"] = " | ".join(merged_notes) best_g["_photos"] = merged_photos - best_g["Provenance"] = build_provenance(sub, best_g, extra_notes=conflict_notes) + best_g["Provenance"] = build_provenance( + sub, best_g, extra_notes=conflict_notes, field_source=field_source + ) # Print merge trace print(f"Deduplicated same-animal name '{best_g['Name']}': merged {len(sub)} entries across files: {', '.join(sources)}") @@ -2246,9 +2420,31 @@ def main(): if l_merged_count > 1: l_notes.append(f"aus {l_merged_count} Datensätzen zusammengeführt") l_notes.append("Geschwister-Würfe zusammengeführt") + + # Chronological, file-attributed history for the litter. + l_files_sorted = sorted({f for f in l_source_files if f}) + l_history = [] + first_file = l_files_sorted[0] if l_files_sorted else None + if is_virtual and not l_from_wurfchronik: + if first_file: + l_history.append( + f"Aus Stammbaum-Diagramm rekonstruiert ({_quote_file(first_file)})." + ) + else: + l_history.append("Aus Stammbaum-Diagramm rekonstruiert.") + elif first_file: + l_history.append(f"Wurf aus Wurfchronik {_quote_file(first_file)}.") + else: + l_history.append("Wurf im Import gefunden.") + for f in l_files_sorted[1:]: + l_history.append( + f"Auch in {_quote_file(f)} gefunden → Datensätze zusammengeführt." + ) + if l_merged_count > 1: + l_history.append("Geschwister-Würfe zusammengeführt.") l["Provenance"] = build_entity_provenance( l_source_files, l_merged_count, notes=l_notes, - from_wurfchronik=l_from_wurfchronik, + from_wurfchronik=l_from_wurfchronik, history=l_history, ) # Set IsBreeder and IsReceiver flags on contacts @@ -2275,7 +2471,24 @@ def main(): c_notes.append("als Züchter erkannt") if is_receiver: c_notes.append("als Abnehmer erkannt") - c["Provenance"] = build_entity_provenance(c_source_files, c_merged_count, notes=c_notes) + + # Chronological, file-attributed history for the contact. + c_files_sorted = sorted({f for f in c_source_files if f}) + c_history = [] + role_word = "Züchter" if is_breeder else "Abnehmer" + if c_files_sorted: + c_history.append(f"In {_quote_file(c_files_sorted[0])} als {role_word} erkannt.") + for f in c_files_sorted[1:]: + c_history.append( + f"Auch in {_quote_file(f)} gefunden → Datensätze zusammengeführt." + ) + else: + c_history.append(f"Im Import als {role_word} erkannt.") + if is_breeder and is_receiver: + c_history.append("Sowohl als Züchter als auch als Abnehmer geführt.") + c["Provenance"] = build_entity_provenance( + c_source_files, c_merged_count, notes=c_notes, history=c_history + ) # Set and map gerbilPhotos resolved_photos = [] diff --git a/tools/import/test_merge_resolve.py b/tools/import/test_merge_resolve.py index 668bd54..3fbce82 100644 --- a/tools/import/test_merge_resolve.py +++ b/tools/import/test_merge_resolve.py @@ -213,6 +213,70 @@ mr = m.pick_parent_ref(molly_refs, "mother", "13.09.2021", avoid_name=fr["name"] check("pick(gender): mother = Arya (female)", mr and mr["name"].startswith("Arya")) +# ── Provenance history: chronological, file-attributed German log ── +import json as _json + + +def _hist(prov_json): + return _json.loads(prov_json)["history"] + + +# build_entity_provenance carries history through verbatim. +_prov = _json.loads( + m.build_entity_provenance(["A.xlsx"], 1, notes=["x"], history=["line one"]) +) +check("build_entity_provenance includes history key", _prov.get("history") == ["line one"]) +check("build_entity_provenance defaults history to []", + _json.loads(m.build_entity_provenance(["A.xlsx"], 1)).get("history") == []) + +# Single-record gerbil history: names the file and the per-field facts. +g_single = { + "Name": "Picus", "DateOfBirth": "2022-03-27", "Gender": "male", + "Genotype": "aa", "ColorVarietyId": None, "DateOfDeath": None, + "ImportSource": "Stammbaum von Picus Son.xlsx", + "_filename": "Stammbaum von Picus Son.xlsx", "parentRefs": [], +} +h = m._build_gerbil_history([g_single], g_single, {}) +check("history: first line names the source file", + h[0] == "In „Stammbaum von Picus Son.xlsx“ gefunden.") +check("history: dob line names file + formatted date", + "Geburtsdatum (27.03.2022) aus „Stammbaum von Picus Son.xlsx“." in h) +check("history: gender line is German + file-attributed", + "Geschlecht (männlich) aus „Stammbaum von Picus Son.xlsx“." in h) +check("history: genotype line file-attributed", + "Genotyp aus „Stammbaum von Picus Son.xlsx“." in h) + +# Merged gerbil: a field sourced from a DIFFERENT file is attributed to THAT file. +g_best = { + "Name": "Solice", "DateOfBirth": "2022-03-27", "Gender": "male", + "Genotype": None, "ColorVarietyId": None, "DateOfDeath": None, + "ImportSource": "Stammbaum von Picus Son.xlsx", + "_filename": "Stammbaum von Picus Son.xlsx", "parentRefs": [], +} +g_other = { + "Name": "Solice", "DateOfBirth": "2022-03-27", "Gender": "male", + "Genotype": "aa", "ColorVarietyId": None, "DateOfDeath": None, + "ImportSource": "Wurfchronik-Detail.docx", + "_filename": "Wurfchronik-Detail.docx", "parentRefs": [], +} +# Genotype was filled from g_other → its line must name the docx file. +g_best["Genotype"] = "aa" +fs = {"DateOfBirth": g_best, "Gender": g_best, "Genotype": g_other} +h2 = m._build_gerbil_history([g_best, g_other], g_best, fs) +check("history(merge): genotype attributed to the file that supplied it", + "Genotyp aus „Wurfchronik-Detail.docx“." in h2) +check("history(merge): dob attributed to primary file", + "Geburtsdatum (27.03.2022) aus „Stammbaum von Picus Son.xlsx“." in h2) +check("history(merge): merge line names the absorbed file", + "Auch in „Wurfchronik-Detail.docx“ gefunden → Datensätze zusammengeführt." in h2) +check("history(merge): Wurfchronik line present", + "Angaben aus der Wurfchronik übernommen." in h2) + +# Date formatting helper. +check("_de_date: ISO → DD.MM.YYYY", m._de_date("2022-03-27") == "27.03.2022") +check("_de_date: passes through non-ISO", m._de_date("unbekannt") == "unbekannt") + + if check.failed: print(f"\n{check.failed} test(s) FAILED") sys.exit(1)