Gulum
0b99cfd2bd
Die Züchterin hat 24 neue/aktualisierte Stammbaum-xlsx geliefert (Ordner
"neuestammbäume"); sie liegen jetzt im kanonischen Quellverzeichnis
Sttammbäume (12 neue Charts, 7 aktualisierte, 3 identisch, das inhaltsgleiche
"Picus Son (2)" ausgelassen). Prod ist per Upload-Ingest aktualisiert:
2372 -> 2451 Tiere, 916 -> 965 Würfe, 432 -> 507 Fotos, 2198 -> 2275 Tiere
mit Geburtsdatum. Overrides/verified-Zeilen, manuelle Tiere und Tickets
haben den Ingest unverändert überlebt.
Zwei Datenfehler, die die neuen Charts aufgedeckt haben — datengetrieben und
re-ingest-stabil gefixt statt an der globalen Heuristik zu drehen:
- litterChildren kennt jetzt `add` [Name | {name, dob}] als Gegenstück zu
`keep`: hängt ein Jungtier an DIESEN Wurf und entfernt den alten Wurf, wenn
er dadurch kinderlos UND virtuell ist. Nötig, weil "Pukas Kids" Akanes
Eltern komplett UNTER ihren Block setzt (N80 Roni = Vater, N81 Fumi =
Mutter) — _reconstruct_parents griff eine Zeile zu hoch, paarte Irish
Coffee (Bonapartes Mutter) mit Roni und riss Akane aus dem Z21-Wurf in
einen Phantom-Wurf, der in der Wurfchronik auftauchte (Ticket 88389f8e).
- Merle: durch das neue Geburtsdatum (18.06.2023) mergt der addAnimals-Stub
in den Chart-Datensatz und verliert dabei sein isResident -> expliziter
resolutions-Override (Ticket 36a3fcde/a8f11ac0, Züchterin: Zuchttier).
Außerdem: Excel legt neben Fotos teils EMF/WMF-Vektorvorschauen ab, die
Browser nicht darstellen können (kaputte Bildkachel in der Tier-Akte) ->
extract._attach_photos überspringt .emf/.wmf (5 Fotos betroffen).
Regressionstests für alle drei Punkte; alle Python-Suites grün.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
4484 lines
208 KiB
Python
4484 lines
208 KiB
Python
import os
|
||
import json
|
||
import re
|
||
import uuid
|
||
import sys
|
||
from datetime import datetime, timedelta
|
||
|
||
import genotype as gt
|
||
|
||
# Prevent encoding crashes on Windows consoles when printing unicode
|
||
if sys.platform.startswith('win'):
|
||
try:
|
||
sys.stdout.reconfigure(encoding='utf-8')
|
||
except Exception:
|
||
pass
|
||
|
||
# Paths
|
||
DIR_PATH = r"C:\Users\gulum\dev\Wurfchronik_Bilder"
|
||
SEEDS_PATH = r"C:\Users\gulum\dev\GerbilManager\gerbil-manager-web\src\genetics\colorVarietySeed.backend.json"
|
||
OUTPUT_DIR = r"C:\Users\gulum\dev\GerbilManager\tools\import\output"
|
||
OUTPUT_FILE = os.path.join(OUTPUT_DIR, "resolved_import.json")
|
||
CONFLICT_DECISIONS_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)),
|
||
"conflict-decisions.json")
|
||
|
||
|
||
def load_conflict_decisions():
|
||
"""Load tools/import/conflict-decisions.json (the breeder's authoritative
|
||
answers). Returns (resolutions, addAnimals, addLitters). Tolerates a
|
||
missing/garbled file by returning empty lists — the import must never crash
|
||
on a bad decision file."""
|
||
try:
|
||
with open(CONFLICT_DECISIONS_PATH, encoding="utf-8") as fh:
|
||
data = json.load(fh)
|
||
return (data.get("resolutions") or [], data.get("addAnimals") or [],
|
||
data.get("addLitters") or [])
|
||
except (OSError, ValueError):
|
||
return ([], [], [])
|
||
|
||
|
||
def load_suppress_refs():
|
||
"""Load suppressExternalRefs from conflict-decisions.json — the ExternalRefs of
|
||
gerbils/litters the breeder confirmed are phantoms/OCR-misread duplicates and that
|
||
should be dropped from the final payload (e.g. 'Muga' = misread 'Nduga'). Tolerates
|
||
a missing/garbled file."""
|
||
try:
|
||
with open(CONFLICT_DECISIONS_PATH, encoding="utf-8") as fh:
|
||
data = json.load(fh)
|
||
return set(data.get("suppressExternalRefs") or [])
|
||
except (OSError, ValueError):
|
||
return set()
|
||
|
||
|
||
def _norm_decision_gender(g):
|
||
"""Map a free-text gender override ('m'/'w'/'männlich'/…) to 'male'/'female'
|
||
or None when it is neither (so a typo never blanks a real gender)."""
|
||
g = (g or "").strip().lower()
|
||
return {"m": "male", "männlich": "male", "male": "male",
|
||
"w": "female", "f": "female", "weiblich": "female",
|
||
"female": "female"}.get(g)
|
||
|
||
def generate_guid(key_str):
|
||
"""Generate a stable UUID string based on a key."""
|
||
return str(uuid.uuid5(uuid.NAMESPACE_DNS, key_str))
|
||
|
||
def build_entity_provenance(source_files, merged_record_count, notes=None,
|
||
from_wurfchronik=None, extra=None, history=None):
|
||
"""Generic data-provenance JSON builder shared by gerbils, contacts and
|
||
litters. Mirrors the GerbilProvenance frontend contract:
|
||
{ sourceFiles, mergedRecordCount, fromWurfchronik, notes, history, ... }
|
||
`source_files` is any iterable of filenames; `from_wurfchronik` is auto-
|
||
derived from the filenames when left as None. `extra` may carry entity-
|
||
specific keys (e.g. parentMethod/parentConfidence for gerbils). `history`
|
||
is an ordered list of human-readable German lines that read like a
|
||
chronological log of where each fact came from (the primary content shown
|
||
in the Datenherkunft dialog). Returns a JSON string (stored on the nullable
|
||
Provenance text column)."""
|
||
files = sorted({f for f in source_files if f})
|
||
if from_wurfchronik is None:
|
||
from_wurfchronik = any("wurfchronik" in f.lower() for f in files)
|
||
prov = {
|
||
"sourceFiles": files,
|
||
"mergedRecordCount": merged_record_count,
|
||
"fromWurfchronik": bool(from_wurfchronik),
|
||
"notes": list(notes or []),
|
||
"history": list(history or []),
|
||
}
|
||
if extra:
|
||
for k, v in extra.items():
|
||
if v is not None:
|
||
prov[k] = v
|
||
return json.dumps(prov, ensure_ascii=False)
|
||
|
||
|
||
def _quote_file(fname):
|
||
"""German typographic quotes around a source filename for history lines."""
|
||
return f"„{fname}“"
|
||
|
||
|
||
# Human-readable German labels for the significant fields we attribute to files.
|
||
PROV_FIELD_LABELS = {
|
||
"DateOfBirth": "Geburtsdatum",
|
||
"DateOfDeath": "Sterbedatum",
|
||
"Gender": "Geschlecht",
|
||
"Genotype": "Genotyp",
|
||
"ColorVarietyId": "Farbschlag",
|
||
"Name": "Name",
|
||
}
|
||
|
||
|
||
def _primary_file_of(record):
|
||
"""The single most representative source file of one raw record.
|
||
|
||
Prefers an explicit Stammbaum/Wurfchronik filename from ImportSource, else
|
||
the record's _filename. Used to attribute a field value to a concrete file
|
||
in the history log."""
|
||
fn = record.get("_filename")
|
||
if fn:
|
||
return fn
|
||
imp = record.get("ImportSource")
|
||
if imp:
|
||
first = str(imp).split(",")[0].strip()
|
||
if first:
|
||
return first
|
||
return None
|
||
|
||
|
||
def _record_source_files(g):
|
||
"""All distinct source files a single raw gerbil record drew from.
|
||
|
||
ImportSource is either a comma-joined Stammbaum file list, a single
|
||
Wurfchronik filename, or the per-litter filename; _filename is the primary
|
||
file. We union both so nothing is lost."""
|
||
files = set()
|
||
imp = g.get("ImportSource")
|
||
if imp:
|
||
for part in str(imp).split(","):
|
||
part = part.strip()
|
||
if part:
|
||
files.add(part)
|
||
fn = g.get("_filename")
|
||
if fn:
|
||
files.add(fn)
|
||
return files
|
||
|
||
def to_valid_guid(val):
|
||
if not val:
|
||
return None
|
||
val_str = str(val).strip()
|
||
try:
|
||
uuid.UUID(val_str)
|
||
return val_str
|
||
except ValueError:
|
||
return generate_guid(val_str)
|
||
|
||
def normalize_name(name):
|
||
if not name:
|
||
return ""
|
||
return "".join(c for c in name.lower() if c.isalnum())
|
||
|
||
|
||
_EXTERNAL_MARKERS_RE = re.compile(
|
||
r"\b(zooladen|zoohandlung|obi|fressnapf|dehner|von\s+privat|privatkauf|"
|
||
r"vom\s+bauern|aus\s+der\s+zoohandlung)\b", re.IGNORECASE)
|
||
_FOREIGN_FROM_RE = re.compile(
|
||
r"\bfrom\b.+,\s*(croatia|kroatien|poland|polen|netherlands|niederlande|"
|
||
r"belgium|belgien|france|frankreich|austria|österreich|switzerland|schweiz|"
|
||
r"italy|italien|spain|spanien|czech|tschechien|hungary|ungarn)\b", re.IGNORECASE)
|
||
|
||
|
||
def is_external_origin(name, zucht=None, breeder=None):
|
||
"""Externally-acquired founder with genuinely unknown ancestry (pet shop,
|
||
private hobbyist, foreign cattery). Mirrors extract.is_external_origin so the
|
||
merge stage never attaches chart/Wurfchronik parents to such animals."""
|
||
blob = " ".join(p for p in (name, zucht, breeder) if p)
|
||
if _EXTERNAL_MARKERS_RE.search(blob):
|
||
return True
|
||
if _FOREIGN_FROM_RE.search(name or ""):
|
||
return True
|
||
return False
|
||
|
||
|
||
def canon_name_key(name):
|
||
"""Connector-folding name key: collapses the cattery connectors so that
|
||
abbreviation variants of the SAME animal match — e.g. „BlackFire v.d.
|
||
Kleinen Chaoten“ and „BlackFire von den Kleinen Chaoten“ both fold to the
|
||
same key (ticket #30). 'v.d.' / 'v. d.' → 'von den', then alnum-reduced.
|
||
Used as a secondary index next to normalize_name (never for GUIDs)."""
|
||
if not name:
|
||
return ""
|
||
n = name.lower()
|
||
n = re.sub(r"\bv\.?\s*d\.?\b", "von den", n) # v.d. / v. d. / vd → von den
|
||
return "".join(c for c in n if c.isalnum())
|
||
|
||
def get_normalized_contact_name(name):
|
||
if not name:
|
||
return "", False
|
||
|
||
n = "".join(c for c in name.lower() if c.isalnum())
|
||
|
||
to_discard = {
|
||
"chevroletcamarooftopolino",
|
||
"cindyvprivatzuchtgießen",
|
||
"cindyvprivatzuchtgiessen",
|
||
"inuschofblackforest",
|
||
"stichvonprivatzuchtgießen",
|
||
"stichvonprivatzuchtgiessen",
|
||
"tarzanofsamsimar",
|
||
"hanserennersposeidon",
|
||
"livingforcesidefix",
|
||
"livingforcesnando"
|
||
}
|
||
if n in to_discard:
|
||
return "", False
|
||
|
||
norm_map = {
|
||
"alessandrab": "Alessandra Bartoletti",
|
||
"alexsandrab": "Alessandra Bartoletti",
|
||
"andreafey": "Andrea und Stefanie Fey",
|
||
"andreastefaniefey": "Andrea und Stefanie Fey",
|
||
"anettw": "Annett Wernecke",
|
||
"angelachristoph": "Angela und Ekki Christoph",
|
||
"angelaekkichristoph": "Angela und Ekki Christoph",
|
||
"angie": "Angie Reichert-Cambeis",
|
||
"angierc": "Angie Reichert-Cambeis",
|
||
"anjasch": "Anja Schaumburg",
|
||
"ankeb": "Anke Busch",
|
||
"ankeksch": "Anke Koppenhöfer",
|
||
"annaq": "Anna Quark",
|
||
"annikab": "Annika Balser",
|
||
"annkathrind": "Ann-kathrin Dressler",
|
||
"arturaleksandrapolamundrzynski": "Artur, Aleksandra und Pola Mundrzynski",
|
||
"arturaleksandraundpolamundrzynski": "Artur, Aleksandra und Pola Mundrzynski",
|
||
"astridr": "Astrid Rohmann",
|
||
"astridroh": "Astrid Rohmann",
|
||
"babarastehle": "Barbara Stehle",
|
||
"barbarar": "Barbara Riegler",
|
||
"biancab": "Bianca Bernhardt",
|
||
"biancam": "Bianca Mayer + Alex Wachten",
|
||
"biancamayeralexw": "Bianca Mayer + Alex Wachten",
|
||
"birgitk": "Birgit Knopp",
|
||
"birgitkropp": "Birgit Knopp",
|
||
"birgittabüskens": "Birgitta Müller-Büskens",
|
||
"birgittamüllerbüskens": "Brigitta Müller-Büskens",
|
||
"blackforest": "Clan of Black Forest",
|
||
"blackforestgv": "Clan of Black Forest",
|
||
"brigittast": "Brigitta Struve",
|
||
"buntefellnasen": "Zucht von den bunten Fellnasen",
|
||
"buntenfellnase": "Zucht von den bunten Fellnasen",
|
||
"buntenfellnasen": "Zucht von den bunten Fellnasen",
|
||
"carolal": "Carola und Hannah Lerch",
|
||
"carolalerch": "Carola und Hannah Lerch",
|
||
"chalfontstud": "Chalfont Stud, Freddy Braun",
|
||
"christianefuchs": "Christiane Fuchs + Michael Höhnert",
|
||
"christiek": "Christiane Kiessling",
|
||
"christinam": "Christine Monika Mai",
|
||
"christinek": "Christiane Kiessling",
|
||
"christiner": "Christina Rudloff",
|
||
"claudialöhr": "Claudia Löhr und Michael Koob",
|
||
"claudialöhrmichaelkoob": "Claudia Löhr und Michael Koob",
|
||
"colorfulfurrygerbils": "Colorful Furry Gerbils",
|
||
"colourfulfurrygerbils": "Colorful Furry Gerbils",
|
||
"danielakasberger": "Daniela Käsberger",
|
||
"dörthe": "Dörthe Petzmann",
|
||
"emelyhehlhorn": "Emely Mehlhorn",
|
||
"fabianb": "Fabian Büdel",
|
||
"floriang": "Florian Gries",
|
||
"hannahraths": "Hannah Ruths",
|
||
"hanserenners": "Hanse Renner",
|
||
"haraldmariareitzlennemann": "Harald und Maria Reitz-Lennemann",
|
||
"haraldundmariarl": "Harald und Maria Reitz-Lennemann",
|
||
"heike": "Heike + Heiko Scheurich",
|
||
"heikebarklay": "Heike Barklage",
|
||
"heikoloos": "Heiko Loos und Annette Becker",
|
||
"heikoloosannettebecker": "Heiko Loos und Annette Becker",
|
||
"inahübner": "Ina Hübner + Nicolai Thome",
|
||
"inapeines": "Ina Pleines",
|
||
"janab": "Jana Beikert",
|
||
"jasiminweber": "Jasmin Weber und Ronny Reichelt",
|
||
"jasmin": "Jasmin Weber",
|
||
"jasminweber": "Jasmin Weber und Ronny Reichelt",
|
||
"jasminweberromyreichelt": "Jasmin Weber und Ronny Reichelt",
|
||
"jasminweberronnyreichelt": "Jasmin Weber und Ronny Reichelt",
|
||
"joystaatberg": "Joy Staalberg",
|
||
"juliaa": "Julia Ast",
|
||
"juttam": "Jutta Metz",
|
||
"karina": "Karina Luft",
|
||
"karinborgotti": "Karin Borsotti",
|
||
"katjal": "Katja Leffeck",
|
||
"kimw": "Kim Waldschmidt",
|
||
"kkchaos": "KK-Chaos",
|
||
"kkchaosofkkchaos": "KK-Chaos",
|
||
"kleinechaoten": "Zucht der kleinen Chaoten",
|
||
"kleinenchaoten": "Zucht der kleinen Chaoten",
|
||
"kriegernmitkrallen": "Krieger mit Krallen",
|
||
"lennylengo": "Lenny Lengo",
|
||
"lisa": "Lisa und Marcel Kunz",
|
||
"lisach": "Lisa und Lydia Christ",
|
||
"lisachrist": "Lisa und Lydia Christ",
|
||
"lisalydiachrist": "Lisa und Lydia Christ",
|
||
"littlefellows": "little fellows",
|
||
"littlerunners": "little runners",
|
||
"maikef": "Maike Franz",
|
||
"maintalerpz": "Privatzucht Maintal",
|
||
"marinau": "Marina Unger",
|
||
"martinaandreaswestfeld": "Martina und Andreas Westfeld",
|
||
"martinaw": "Martina und Andreas Westfeld",
|
||
"martinawestfeld": "Martina und Andreas Westfeld",
|
||
"meikesch": "Meike und Heiko Sch.",
|
||
"melanief": "Melanie Fey und Thorben Meier",
|
||
"melaniefey": "Melanie Fey und Thorben Meier",
|
||
"melaniefeythorbenmeier": "Melanie Fey und Thorben Meier",
|
||
"michaeldavidschmitz": "Michael und David Schmitz",
|
||
"michaelschmitz": "Michael und David Schmitz",
|
||
"nataliereitz": "Nathalie Reitz",
|
||
"natascham": "Natascha Marienfeld",
|
||
"nicolefischler": "Nicole Tischler",
|
||
"nicolel": "Nicole Lannert",
|
||
"nicolen": "Nicole Nuzzo",
|
||
"nicolet": "Nicole Tischler",
|
||
"nicolew": "Nicole Webersinn",
|
||
"nicost": "Nico Stamm",
|
||
"nielsh": "Nils H.",
|
||
"noelstrahbach": "Noel Strohbach",
|
||
"nora": "Nora Rudersdorf geb. Holzbach",
|
||
"norah": "Nora Holzbach",
|
||
"norarh": "Nora Rudersdorf",
|
||
"norarudersdorf": "Nora Rudersdorf geb. Holzbach",
|
||
"oflennylengo": "Lenny Lengo",
|
||
"pascale": "Pascale Diefenbach",
|
||
"pascaledießenbach": "Pascale Diefenbach",
|
||
"patriciap": "Patricia Petry",
|
||
"paul": "Paul W.",
|
||
"paula": "Paula Gabler",
|
||
"paulag": "Paula Gabler",
|
||
"privatzuchtmuecke": "Privatzucht Mücke",
|
||
"privatzuchtmücke": "Privatzucht Mücke",
|
||
"pzmaintal": "Privatzucht Maintal",
|
||
"pzmuecke": "Privatzucht Mücke",
|
||
"pzmücke": "Privatzucht Mücke",
|
||
"pzseligenstadt": "Privatzucht Seligenstadt",
|
||
"ramonag": "Ramona Gömpel",
|
||
"ranialößler": "Rania Löffler",
|
||
"rominahubrich": "Romina Milde / Hubrich",
|
||
"rominamilde": "Romina Milde / Hubrich",
|
||
"romonag": "Ramona Gömpel",
|
||
"ronialöffler": "Rania Löffler",
|
||
"sandrak": "Sandra Kubas",
|
||
"sarahlöser": "Sarah Löwer",
|
||
"sarahlöuer": "Sarah Löwer",
|
||
"sarahz": "Sarah Zitzer",
|
||
"saskiaw": "Saskia Wucher",
|
||
"schlossmaus": "Schlossmäuse",
|
||
"schlossmäuse": "Schlossmäuse",
|
||
"schlossmäusen": "Schlossmäuse",
|
||
"sebastiansch": "Sebastian Schmitt",
|
||
"serinaberg": "Selina Berg",
|
||
"silkewolfganghintze": "Silke und Wolfgang Hintze",
|
||
"smilla": "Smilla H.",
|
||
"steffi": "Steffi K.",
|
||
"susanneninat": "Susanne + Nina Thomas",
|
||
"susannet": "Susanne + Torsten Saum",
|
||
"theresavalenca": "Theresia Valenca (Kopp)",
|
||
"theresiavalenca": "Theresia Valenca (Kopp)",
|
||
"timpf": "Tim Pfeiffer und Michelle Mai",
|
||
"timpfeifermichellemai": "Tim Pfeiffer und Michelle Mai",
|
||
"timpfeiffer": "Tim Pfeiffer und Michelle Mai",
|
||
"timpfeiffermichellemai": "Tim Pfeiffer und Michelle Mai",
|
||
"tonoböckenseld": "Tono Böckenfeld",
|
||
"topol": "Topolino",
|
||
"ulrichmüller": "Ulrich + Angela Müller",
|
||
"ulrikec": "Ulrike Cordes",
|
||
"ulriker": "Ulrike Ruppel",
|
||
"ulrikesch": "Ulrike Schulz",
|
||
"ulriket": "Ulrike Treml",
|
||
"vanessab": "Vanessa Becker",
|
||
"vanessag": "Vanessa Groll",
|
||
"veragreywitz": "Vera Geywitz",
|
||
"vonprivatzuchtmaintal": "Privatzucht Maintal",
|
||
"wolfgangfaus": "Wolfgang Faust",
|
||
"yvonne": "Yvonne Obendorfer",
|
||
"zuchtderkleinenchaoten": "Zucht der kleinen Chaoten",
|
||
}
|
||
|
||
if n in norm_map:
|
||
return norm_map[n], True
|
||
|
||
return name, True
|
||
|
||
def get_normalized_gerbil_name(name):
|
||
if not name:
|
||
return ""
|
||
n = name.strip()
|
||
norm_key = "".join(c for c in n.lower() if c.isalnum())
|
||
|
||
gerbil_norm_map = {
|
||
"samgenshellyvdbuntenfellnasen": "Sammy gen. Shelly von den bunten Fellnasen",
|
||
"sammygenshellyvdbuntenfellnasen": "Sammy gen. Shelly von den bunten Fellnasen",
|
||
"schmidt": "Schmidti",
|
||
"sheila": "Sheila of Ulmer Strolche",
|
||
"shinichi": "Shinichi von PZ Mücke",
|
||
"silenosgenadonis": "Silenos gen. Adonis von den Kleinen Chaoten",
|
||
"silenosgenadonisvdkleinenchaoten": "Silenos gen. Adonis von den Kleinen Chaoten",
|
||
"silver": "Silver von den kleinen Chaoten",
|
||
"snoops": "Snoopsi",
|
||
"sokrates": "Sokrates von den Kleinen Chaoten",
|
||
"splash": "Slash",
|
||
"teiko": "Teiko von den kleinen Chaoten",
|
||
"trixy": "Trixxy von den Kleinen Chaoten",
|
||
"unique": "Unique of Wild Dreams",
|
||
}
|
||
|
||
if norm_key in gerbil_norm_map:
|
||
return gerbil_norm_map[norm_key]
|
||
|
||
return n
|
||
|
||
def get_call_name(name):
|
||
if not name:
|
||
return ""
|
||
n = name.strip()
|
||
n = re.sub(r'\[[^\]]+\]$', '', n).strip()
|
||
n = re.split(r'\s+(?:of|von\s+den|von\s+der|v\.\s?d\.|von)\s+', n, flags=re.IGNORECASE)[0].strip()
|
||
return n
|
||
|
||
def get_dedup_name_key(name):
|
||
if not name:
|
||
return ""
|
||
n = name.lower().strip()
|
||
# Strip common suffixes/prefixes and parentheticals
|
||
n = re.sub(r'\b(?:von\s+den|v\.?\s*d\.?|v\.?o\.?)\s+(?:kleinen\s+)?chaoten\b', '', n)
|
||
n = re.sub(r'\bvon\s+der\s+schlossm\w+\b', '', n)
|
||
n = re.sub(r'\bvon\s+der\s+bunten\s+fellnasen?\b', '', n)
|
||
n = re.sub(r'\bof\s+black\s+forest\b', '', n)
|
||
n = re.sub(r'\b\(?rv\)?\b', '', n)
|
||
n = re.sub(r'\bgen\.\s+\w+', '', n)
|
||
# Clean up parentheses or brackets
|
||
n = re.sub(r'\(.*?\)', '', n)
|
||
n = re.sub(r'\[.*?\]', '', n)
|
||
return "".join(c for c in n if c.isalnum())
|
||
|
||
def clean_color_name(c_desc):
|
||
"""Normalise a free-text colour label to a catalog key + Schecke flag.
|
||
|
||
Returns (clean_name, is_schecke). A PARENTHETICAL „(schimmel)" is NOT a
|
||
definitive Schimmel — the breeder writes it to mean „könnte sich später als
|
||
Schimmel entpuppen" (ticket e22764aa). So we STRIP the „(…)" instead of
|
||
folding it into the name (which used to turn „Blaufuchs(schimmel)" into the
|
||
wrong „blaufuchsschimmel"); the still-uncertain Schimmel-modifier is carried
|
||
by the genotype (ee[-] = Fuchs, Schimmel unknown), not the colour label.
|
||
"""
|
||
if not c_desc:
|
||
return "", False
|
||
|
||
# Lowercase and strip
|
||
c = c_desc.lower().strip()
|
||
|
||
# Check for Schecke
|
||
is_schecke = False
|
||
if re.search(r'\bsp\b|\bsp\d|\bsp[*(²³]|\bspotted|\bschecke|[- ]sp\b|\w+sp\b', c):
|
||
is_schecke = True
|
||
|
||
# Strip schecke/sp markers and any trailing text starting from sp
|
||
c = re.sub(r'\([- ]?sp(otted)?\)', '', c) # handles (-sp)
|
||
c = re.sub(r'[- ]?sp(otted)?\b.*', '', c) # handles -sp(k), -sp*(k), -sp, etc.
|
||
c = re.sub(r'[- ]?schecke\b.*', '', c)
|
||
c = re.sub(r'[- ]?spotted\b.*', '', c)
|
||
|
||
# Strip any other parentheticals (incl. „(schimmel)" = „möglich/unbestimmt"),
|
||
# symbols, or trailing stars/numbers. The parenthetical is deliberately NOT
|
||
# promoted to a definitive part of the colour name (ticket e22764aa).
|
||
c = re.sub(r'\s*\(.*?\)\s*', ' ', c)
|
||
c = re.sub(r'[²³*]', '', c)
|
||
c = c.strip()
|
||
|
||
# Mapping table for abbreviations, typos, and specific combinations
|
||
mapping = {
|
||
"antra": "anthrazit",
|
||
"anthra": "anthrazit",
|
||
"ankazit": "anthrazit",
|
||
"antrazit": "anthrazit",
|
||
"pew": "rew",
|
||
"bew": "hermelin",
|
||
"harder": "marder",
|
||
"kohli": "kohlfuchs",
|
||
"aligerfuchs": "algierfuchs",
|
||
"algiesfuchs": "algierfuchs",
|
||
"schw": "schwarz",
|
||
"sa": "silberagouti",
|
||
"a": "agouti",
|
||
"cp-sa": "cp-silberagouti",
|
||
"cp-a": "cp-agouti",
|
||
"cp-a-hell": "cp-agouti-hell",
|
||
"cp-aisa": "cp-agouti",
|
||
"cp-a / rcp-sa": "cp-agouti",
|
||
}
|
||
|
||
if c in mapping:
|
||
c = mapping[c]
|
||
|
||
return c, is_schecke
|
||
|
||
def _match_color_label(clean_name, variety_map):
|
||
"""Map a cleaned colour label to a ColorVariety id (text-only path).
|
||
|
||
Exact name wins; otherwise pick the LONGEST/most-specific substring match
|
||
(ticket 3f5942a2 — the old code broke on the FIRST substring hit, so „Goldfuchs"
|
||
matched the shorter „Gold" first). Among substring candidates the longest seed
|
||
name wins, then the longest clean_name overlap; ties broken deterministically.
|
||
"""
|
||
if not clean_name:
|
||
return None
|
||
if clean_name in variety_map:
|
||
return variety_map[clean_name]
|
||
candidates = []
|
||
for seed_name, seed_id in variety_map.items():
|
||
if not seed_name:
|
||
continue
|
||
if seed_name in clean_name or clean_name in seed_name:
|
||
# Specificity score: prefer the longer seed name (more specific),
|
||
# then the closeness of lengths so „goldfuchs" beats „gold" for the
|
||
# label „goldfuchs".
|
||
candidates.append((len(seed_name), -abs(len(seed_name) - len(clean_name)),
|
||
seed_name, seed_id))
|
||
if not candidates:
|
||
return None
|
||
candidates.sort(reverse=True)
|
||
return candidates[0][3]
|
||
|
||
|
||
def resolve_color_and_genotype(color_val, existing_genotype, variety_map, variety_genotypes):
|
||
"""Resolve a gerbil's stored ColorVariety id + genotype.
|
||
|
||
GENOTYPE WINS (ticket cluster genetics-farbschlag): when a parseable genotype
|
||
is present and the genetics engine (genotype.genotype_to_farbschlag — a faithful
|
||
Python mirror of catalog.ts) computes a KNOWN catalog variety, that variety is
|
||
authoritative for colorVarietyId. The free-text colour label is only a fallback
|
||
(no genotype, or genotype resolves to „Unbekannt"). This fixes the imports where
|
||
the source label ignored a locus (dd → „Agouti" instead of „Dilute Agouti",
|
||
ee → „Gold" instead of „Goldfuchs", parenthetical „(schimmel)", …).
|
||
|
||
Returns (color_variety_id, genotype). `genotype` is the (possibly Schecke-
|
||
annotated) genotype STRING — never silently flips an explicit spsp to Spsp.
|
||
"""
|
||
if not color_val and not existing_genotype:
|
||
return None, existing_genotype
|
||
|
||
clean_name, is_schecke = clean_color_name(str(color_val).strip()) if color_val else ("", False)
|
||
|
||
# 1) Genotype-derived variety (authoritative when it resolves to a known name).
|
||
# GUARD (VORSICHTIG): only trust the genotype when it parsed CLEANLY enough to
|
||
# decide a colour — both the C and E loci must be mapped. The breeder sometimes
|
||
# writes the genotype in the COMPACT catalog notation („cchmcchm", „efef",
|
||
# „chch") which this parser leaves UNMAPPED (it expects the bracketed „c[chm]"
|
||
# form); a dropped C/E locus would silently read as wild-type and mis-recolour
|
||
# an otherwise-correct animal (e.g. Marder→Schwarz, Orangeschimmel→Agouti). When
|
||
# the parse is incomplete we keep the source text label instead.
|
||
color_variety_id = None
|
||
geno_name = None
|
||
if existing_genotype:
|
||
try:
|
||
mapped = gt.parse(existing_genotype).get("mapped8locus") or {}
|
||
except Exception:
|
||
mapped = {}
|
||
if mapped.get("C") and mapped.get("E"):
|
||
fs = gt.genotype_to_farbschlag(mapped)
|
||
if fs and fs != gt.UNKNOWN_FARBSCHLAG:
|
||
geno_name = fs
|
||
color_variety_id = variety_map.get(fs.strip().lower())
|
||
|
||
# 2) Fall back to the text label when the genotype gave nothing usable.
|
||
if not color_variety_id:
|
||
color_variety_id = _match_color_label(clean_name, variety_map)
|
||
|
||
# Update genotype if the LABEL says Schecke — but never override an explicit
|
||
# Sp-locus already present in the source genotype (ticket e09d6f22: a source
|
||
# „spsp" must NOT be flipped to „Spsp" just because the label looked scheckig;
|
||
# the source genotype is authoritative for the Sp-locus). Only ADD Spsp when
|
||
# the genotype carries no Sp token at all.
|
||
genotype = existing_genotype
|
||
if is_schecke:
|
||
if genotype:
|
||
if "Sp" not in genotype and "sp" not in genotype:
|
||
genotype = f"{genotype} Spsp".strip()
|
||
else:
|
||
canonical = variety_genotypes.get(color_variety_id)
|
||
if canonical:
|
||
if "spsp" in canonical:
|
||
genotype = canonical.replace("spsp", "Spsp")
|
||
else:
|
||
genotype = f"{canonical} Spsp".strip()
|
||
else:
|
||
genotype = "Spsp"
|
||
return color_variety_id, genotype
|
||
|
||
def parse_date(d):
|
||
"""Convert variations of date formats to YYYY-MM-DD."""
|
||
if not d or d == "0001-01-01":
|
||
return None
|
||
d = str(d).strip()
|
||
# Try YYYY-MM-DD
|
||
if re.match(r"^\d{4}-\d{2}-\d{2}$", d):
|
||
parts = d.split("-")
|
||
year = int(parts[0])
|
||
if 1900 < year < 2000:
|
||
year += 100
|
||
d = f"{year}-{parts[1]}-{parts[2]}"
|
||
elif year == 1900:
|
||
return None
|
||
return d
|
||
# Try DD.MM.YYYY or D.M.YY
|
||
match = re.match(r"^(\d{1,2})\.(\d{1,2})\.(\d{2,4})$", d)
|
||
if match:
|
||
day = int(match.group(1))
|
||
month = int(match.group(2))
|
||
year = int(match.group(3))
|
||
if year < 100:
|
||
year += 2000
|
||
elif 1900 < year < 2000:
|
||
year += 100
|
||
elif year == 1900:
|
||
return None
|
||
try:
|
||
return datetime(year, month, day).strftime("%Y-%m-%d")
|
||
except ValueError:
|
||
pass
|
||
# Try ISO timestamp
|
||
try:
|
||
dt = datetime.fromisoformat(d.replace("Z", "+00:00"))
|
||
year = dt.year
|
||
if 1900 < year < 2000:
|
||
dt = dt.replace(year=year + 100)
|
||
return dt.strftime("%Y-%m-%d")
|
||
elif year == 1900:
|
||
return None
|
||
return dt.strftime("%Y-%m-%d")
|
||
except ValueError:
|
||
pass
|
||
return None
|
||
|
||
def date_to_days(dt_str):
|
||
if not dt_str:
|
||
return None
|
||
try:
|
||
return (datetime.strptime(dt_str, "%Y-%m-%d") - datetime(2000, 1, 1)).days
|
||
except ValueError:
|
||
return None
|
||
|
||
def days_to_date(days):
|
||
import datetime as dt
|
||
return (dt.datetime(2000, 1, 1) + dt.timedelta(days=int(days))).strftime("%Y-%m-%d")
|
||
|
||
def parse_death_info(notes, status, existing_dod, existing_cod):
|
||
if not notes:
|
||
return status, existing_dod, existing_cod
|
||
|
||
has_death_indicator = '†' in notes or 'verstorben' in notes.lower() or 'gestorben' in notes.lower() or 'todesdatum' in notes.lower() or '/+' in notes
|
||
if '+' in notes:
|
||
if re.search(r'\+\s*(?:LE|AS|Unbekannt|gestorben|verstorben|tod)\b', notes, re.I) or re.search(r'\+\s*\d{1,2}\.\d{1,2}\.\d{2,4}', notes) or '/+' in notes:
|
||
has_death_indicator = True
|
||
|
||
resolved_status = status
|
||
if has_death_indicator:
|
||
resolved_status = "Deceased"
|
||
|
||
dod = existing_dod
|
||
cod = existing_cod
|
||
|
||
# Look for date near death indicator
|
||
found_date = None
|
||
for m in re.finditer(r'([+†]\s*(?:LE|AS|Unbekannt|[a-zA-ZäöüÄÖÜß0-9()/ +,;.:-]{1,100}?)?\s*)(\d{1,2}\.\d{1,2}\.\d{2,4})', notes, re.I):
|
||
marker_text = m.group(0)
|
||
if '†' in marker_text or re.search(r'\+\s*(?:LE|AS|Unbekannt|gestorben|verstorben|tod|\d)', marker_text, re.I):
|
||
found_date = parse_date(m.group(2))
|
||
if found_date:
|
||
break
|
||
|
||
if not found_date:
|
||
m_death_marker = re.search(r'(†\s*)(\d{1,2}\.\d{1,2}\.\d{2,4})', notes)
|
||
if m_death_marker:
|
||
found_date = parse_date(m_death_marker.group(2))
|
||
|
||
# FIX (ec9267b9): markerless death dates. When a death indicator is present but
|
||
# no +/cross marker precedes the date (e.g. "Verstorbener Welpe am 30.03.15 …",
|
||
# "Verstorben am 04.12.18 …"), take the FIRST plausible date as the death date
|
||
# — but only if we have a death keyword and still have no date. Status logic is
|
||
# unchanged; we never overwrite an existing dateOfDeath.
|
||
if not found_date and has_death_indicator and not existing_dod:
|
||
if re.search(r'\b(?:verstorb\w*|gestorb\w*|verstarb\w*|todesdatum)\b', notes, re.I):
|
||
m_plain = re.search(r'\b(\d{1,2}\.\d{1,2}\.\d{2,4})\b', notes)
|
||
if m_plain:
|
||
found_date = parse_date(m_plain.group(1))
|
||
|
||
if found_date and not dod:
|
||
dod = found_date
|
||
|
||
if not cod:
|
||
for m in re.finditer(r'([+†])\s*([a-zA-ZäöüÄÖÜß0-9()/ +,;.:-]{1,100}?)\s*\d{1,2}\.\d{1,2}\.\d{2,4}', notes, re.I):
|
||
indicator = m.group(1)
|
||
cod_candidate = m.group(2).strip()
|
||
if indicator == '+' and not re.search(r'\b(?:LE|AS|Unbekannt|gestorben|verstorben|tod)\b', cod_candidate, re.I):
|
||
continue
|
||
if cod_candidate:
|
||
cod = cod_candidate
|
||
break
|
||
|
||
if not cod:
|
||
m_cod2 = re.search(r'[+†]\s*\d{1,2}\.\d{1,2}\.\d{2,4}\s*([a-zA-ZäöüÄÖÜß0-9()/ +,;.:-]{1,100})', notes, re.I)
|
||
if m_cod2:
|
||
cod_candidate = m_cod2.group(1).strip()
|
||
if cod_candidate:
|
||
cod = cod_candidate
|
||
else:
|
||
m_cod3 = re.search(r'[+†]\s*(LE|AS|Unbekannt)\b', notes, re.I)
|
||
if m_cod3:
|
||
cod = m_cod3.group(1).strip()
|
||
|
||
if cod:
|
||
cod_lower = cod.lower().strip()
|
||
if cod_lower in ("le", "le (lungenentzündung)", "lungenentzündung", "lungenentzündung (lungenentzündung)"):
|
||
cod = "Lungenentzündung"
|
||
elif cod_lower in ("as", "altersschwäche", "altenschwäche", "alter"):
|
||
cod = "Altersschwäche"
|
||
elif cod_lower in ("unbekannt", "unklar"):
|
||
cod = "Unbekannt"
|
||
else:
|
||
# Replace abbreviations with full names (case-insensitive)
|
||
cod = re.sub(r'\bLE\b', 'Lungenentzündung', cod, flags=re.I)
|
||
cod = re.sub(r'\bAS\b', 'Altersschwäche', cod, flags=re.I)
|
||
cod = cod.replace('+', ' + ')
|
||
# Clean up multiple spaces
|
||
cod = re.sub(r'\s+', ' ', cod).strip(' ,;.-')
|
||
|
||
# Re-check after replacement
|
||
cod_lower = cod.lower().strip()
|
||
if cod_lower in ("le", "le (lungenentzündung)", "lungenentzündung", "lungenentzündung (lungenentzündung)"):
|
||
cod = "Lungenentzündung"
|
||
elif cod_lower in ("as", "altersschwäche", "altenschwäche", "alter"):
|
||
cod = "Altersschwäche"
|
||
elif cod_lower in ("unbekannt", "unklar"):
|
||
cod = "Unbekannt"
|
||
|
||
# Collapse redundant "X (X)" (e.g. Abkürzung + ausgeschriebene Form in Klammern,
|
||
# "AS (Altersschwäche)" → nach Expansion "Altersschwäche (Altersschwäche)") auf "X".
|
||
if cod:
|
||
m_dup = re.match(r'^\s*(.+?)\s*\(\s*(.+?)\s*\)\s*$', cod)
|
||
if m_dup and m_dup.group(1).strip().lower() == m_dup.group(2).strip().lower():
|
||
cod = m_dup.group(1).strip()
|
||
|
||
return resolved_status, dod, cod
|
||
|
||
|
||
# ── Litter dedup & parent-role helpers (pure, unit-tested in test_merge_resolve.py) ──
|
||
|
||
def _norm_pname(s):
|
||
return normalize_name(s) if s else ""
|
||
|
||
|
||
def names_no_conflict(l1, l2):
|
||
"""Parent names don't contradict (equal per role, or one side empty)."""
|
||
f1, f2 = _norm_pname(l1.get("_father_name")), _norm_pname(l2.get("_father_name"))
|
||
m1, m2 = _norm_pname(l1.get("_mother_name")), _norm_pname(l2.get("_mother_name"))
|
||
f_ok = (not f1) or (not f2) or (f1 == f2)
|
||
m_ok = (not m1) or (not m2) or (m1 == m2)
|
||
return f_ok and m_ok
|
||
|
||
|
||
def names_overlap(l1, l2):
|
||
"""At least one role has a non-empty matching name (positive evidence)."""
|
||
f1, f2 = _norm_pname(l1.get("_father_name")), _norm_pname(l2.get("_father_name"))
|
||
m1, m2 = _norm_pname(l1.get("_mother_name")), _norm_pname(l2.get("_mother_name"))
|
||
return bool((f1 and f1 == f2) or (m1 and m1 == m2))
|
||
|
||
|
||
def litter_compatible(l1, l2):
|
||
"""Two litter records describe the same litter: same date and compatible parents.
|
||
|
||
- Both sides have both parents → must match exactly.
|
||
- Asymmetric (one side resolved, the other not) → merge only if names don't
|
||
contradict; for dateless litters require a POSITIVE name match (a shared
|
||
null date is no evidence), so unrelated nameless stubs stay separate.
|
||
- Neither side has parents → never blind-merge.
|
||
"""
|
||
if l1["Date"] != l2["Date"]:
|
||
return False
|
||
f1, m1 = l1.get("FatherId"), l1.get("MotherId")
|
||
f2, m2 = l2.get("FatherId"), l2.get("MotherId")
|
||
if f1 and f2 and m1 and m2:
|
||
return f1 == f2 and m1 == m2
|
||
asymmetric = (bool(f1 or m1) and not (f2 or m2)) or (bool(f2 or m2) and not (f1 or m1))
|
||
if asymmetric:
|
||
if not names_no_conflict(l1, l2):
|
||
return False
|
||
if l1["Date"] is None:
|
||
return names_overlap(l1, l2)
|
||
return True
|
||
return False
|
||
|
||
|
||
def assign_parent_roles(father_id, mother_id, gender_of):
|
||
"""Assign two resolved parent IDs to father/mother roles by gender.
|
||
|
||
Drops self-pairing duplicates (same animal in both roles) and never returns
|
||
two same-role parents. `gender_of` maps an id to 'male'|'female'|'unknown'|None.
|
||
Returns (father_id, mother_id).
|
||
"""
|
||
ids = []
|
||
for gid in (father_id, mother_id):
|
||
if gid and gid not in ids:
|
||
ids.append(gid)
|
||
males = [g for g in ids if gender_of(g) == "male"]
|
||
females = [g for g in ids if gender_of(g) == "female"]
|
||
unknowns = [g for g in ids if gender_of(g) == "unknown"]
|
||
father = males[0] if males else (unknowns.pop(0) if unknowns else None)
|
||
mother = females[0] if females else (unknowns.pop(0) if unknowns else None)
|
||
return father, mother
|
||
|
||
|
||
# A gerbil lives at most ~6 years, so a parent can be at most ~6 years older than
|
||
# its offspring (and must be born before it). Links outside this window are
|
||
# impossible — e.g. a 2013 animal resolved onto a 2022 litter (Jayjay → Solice).
|
||
MAX_PARENT_AGE_DAYS = 2379
|
||
|
||
|
||
def parent_age_plausible(parent_dob, litter_date):
|
||
"""Could a parent born `parent_dob` have offspring born on `litter_date`?
|
||
|
||
Requires birth strictly before the litter and within the gerbil lifespan.
|
||
Unknown/unparseable dates return True (cannot disprove). Accepts any date
|
||
format parse_date understands.
|
||
"""
|
||
pd = date_to_days(parse_date(parent_dob)) if parent_dob else None
|
||
ld = date_to_days(parse_date(litter_date)) if litter_date else None
|
||
if pd is None or ld is None:
|
||
return True
|
||
return 0 < (ld - pd) <= MAX_PARENT_AGE_DAYS
|
||
|
||
|
||
def pick_parent_ref(parent_refs, role, child_dob, avoid_name=None, gender_of=None):
|
||
"""Choose the best parent ref for a role from possibly-conflicting chart refs.
|
||
|
||
A Stammbaum lists an animal at several positions, so its parentRefs can carry
|
||
contradictory guesses (the first one is not necessarily right). Rank candidates
|
||
(lower = better):
|
||
0 right/unknown gender for the role, age-plausible dated ref
|
||
1 right/unknown gender, no DOB (usable, but a plausible dated ref wins)
|
||
2 right/unknown gender, dated but age-impossible
|
||
3 resolved gender is clearly WRONG for the role (e.g. a female father)
|
||
4 would duplicate the animal chosen for the other role
|
||
|
||
Gender is decisive over DOB: a dated female ref must not win the father slot
|
||
over an undated male/unknown one. `gender_of(name)` returns 'male'/'female'
|
||
or None (unknown/ambiguous → not penalised). Returns the chosen ref or None.
|
||
"""
|
||
role_refs = [p for p in parent_refs if p.get("roleGuess") == role]
|
||
if not role_refs:
|
||
return None
|
||
avoid = normalize_name(avoid_name) if avoid_name else None
|
||
expected = "male" if role == "father" else "female"
|
||
|
||
def rank(p):
|
||
if avoid is not None and normalize_name(p.get("name")) == avoid:
|
||
return 4 # would duplicate the other parent role
|
||
g = gender_of(p.get("name")) if gender_of else None
|
||
if g in ("male", "female") and g != expected:
|
||
return 3 # wrong sex for this role
|
||
dob = p.get("dob")
|
||
if not dob:
|
||
return 1
|
||
return 0 if parent_age_plausible(dob, child_dob) else 2
|
||
|
||
order = sorted(range(len(role_refs)), key=lambda i: (rank(role_refs[i]), i))
|
||
return role_refs[order[0]]
|
||
|
||
|
||
def explain_pick_rejections(parent_refs, role, child_dob, chosen, avoid_name=None,
|
||
gender_of=None):
|
||
"""Explain why other refs for `role` lost to `chosen` in pick_parent_ref.
|
||
|
||
Returns a list of discard dicts (for _format_discard) — one per distinct
|
||
rejected candidate name that was beaten for a clear reason (wrong sex, age-
|
||
impossible, or duplicate of the other parent). Mirrors pick_parent_ref's
|
||
ranking so the history can explain the same decision it made.
|
||
"""
|
||
role_refs = [p for p in parent_refs if p.get("roleGuess") == role]
|
||
if not role_refs or chosen is None:
|
||
return []
|
||
avoid = normalize_name(avoid_name) if avoid_name else None
|
||
expected = "male" if role == "father" else "female"
|
||
role_de = "Vaterrolle" if role == "father" else "Mutterrolle"
|
||
cand_de = "Vater-Kandidat" if role == "father" else "Mutter-Kandidat"
|
||
chosen_name = chosen.get("name")
|
||
repl_disp = chosen_name
|
||
if chosen.get("dob"):
|
||
repl_disp = f"{chosen_name} (*{_de_date(parse_date(chosen.get('dob'))) or chosen.get('dob')})"
|
||
|
||
out = []
|
||
seen = set()
|
||
for p in role_refs:
|
||
name = p.get("name")
|
||
if not name or normalize_name(name) == normalize_name(chosen_name or ""):
|
||
continue
|
||
key = normalize_name(name)
|
||
if key in seen:
|
||
continue
|
||
g = gender_of(name) if gender_of else None
|
||
dob = p.get("dob")
|
||
reason = None
|
||
if avoid is not None and key == avoid:
|
||
reason = "bereits als anderer Elternteil gewählt"
|
||
elif g in ("male", "female") and g != expected:
|
||
reason = f"falsches Geschlecht für die {role_de}"
|
||
elif dob and not parent_age_plausible(dob, child_dob):
|
||
reason = "unplausibles Alter für diesen Wurf"
|
||
if reason is None:
|
||
continue
|
||
seen.add(key)
|
||
disp = name
|
||
if dob:
|
||
disp = f"{name} (*{_de_date(parse_date(dob)) or dob})"
|
||
out.append({
|
||
"label": cand_de,
|
||
"value": f"„{disp}“",
|
||
"reason": reason,
|
||
"replacement": f"„{repl_disp}“",
|
||
})
|
||
return out
|
||
|
||
|
||
def _build_gerbil_history(records, best_g, field_source, parent_method=None,
|
||
any_decision=False, any_conflict=False, conflict_notes=None,
|
||
field_discards=None, parent_discards=None):
|
||
"""Build an ordered, file-attributed German history for a resolved gerbil.
|
||
|
||
Reads like a chronological log:
|
||
• „In ‚X.xlsx' gefunden."
|
||
• „Geburtsdatum (27.03.2022) aus ‚X.xlsx'."
|
||
• „Auch in ‚Y.xlsx' gefunden → Datensätze zusammengeführt."
|
||
• „Genotyp aus ‚Z.xlsx'."
|
||
• „Eltern über Position im Stammbaum erkannt (Quelle: ‚X.xlsx')."
|
||
• „Aus Wurfchronik übernommen."
|
||
|
||
`field_source` maps a field name to the raw record that supplied its final
|
||
value; when present we name that record's file, otherwise we fall back to
|
||
the primary record. The records are visited in a stable order (primary
|
||
first, then the rest sorted by file) so the log is deterministic."""
|
||
conflict_notes = conflict_notes or []
|
||
history = []
|
||
|
||
# Order records: best_g first, then others by primary file name (stable).
|
||
others = [r for r in records if r is not best_g]
|
||
others.sort(key=lambda r: (_primary_file_of(r) or ""))
|
||
ordered = [best_g] + others
|
||
|
||
best_file = _primary_file_of(best_g)
|
||
if best_file:
|
||
history.append(f"In {_quote_file(best_file)} gefunden.")
|
||
else:
|
||
history.append("Im Import gefunden.")
|
||
|
||
# Field-by-field attribution: name the file that supplied each fact.
|
||
def attr_line(field, formatter):
|
||
rec = field_source.get(field) or best_g
|
||
val = best_g.get(field)
|
||
if not val or val == "unknown":
|
||
return
|
||
fname = _primary_file_of(rec)
|
||
label = PROV_FIELD_LABELS.get(field, field)
|
||
text = formatter(label, val)
|
||
if fname:
|
||
history.append(f"{text} aus {_quote_file(fname)}.")
|
||
else:
|
||
history.append(f"{text} (Quelle unbekannt).")
|
||
|
||
attr_line("DateOfBirth", lambda label, v: f"{label} ({_de_date(v)})")
|
||
attr_line("Gender", lambda label, v: f"{label} ({_de_gender(v)})")
|
||
attr_line("Genotype", lambda label, v: f"{label}")
|
||
attr_line("ColorVarietyId", lambda label, v: f"{label}")
|
||
attr_line("DateOfDeath", lambda label, v: f"{label} ({_de_date(v)})")
|
||
|
||
# Merge step: every additional record that contributed.
|
||
for r in others:
|
||
fname = _primary_file_of(r)
|
||
if fname:
|
||
history.append(
|
||
f"Auch in {_quote_file(fname)} gefunden → Datensätze zusammengeführt."
|
||
)
|
||
else:
|
||
history.append("In weiterem Datensatz gefunden → Datensätze zusammengeführt.")
|
||
|
||
# Discarded field values from majority-vote conflict resolution: the LOSING
|
||
# values, their file, and what won instead.
|
||
for d in (field_discards or []):
|
||
line = _format_discard(d)
|
||
if line not in history:
|
||
history.append(line)
|
||
|
||
# Parent derivation.
|
||
if parent_method:
|
||
method_label = {
|
||
"chart-position": "Position im Stammbaum",
|
||
"geburtsdatum+eltern": "Geburtsdatum und Elternnamen",
|
||
"nur-geburtsdatum": "Geburtsdatum",
|
||
"decision": "manuelle Entscheidung",
|
||
}.get(parent_method, parent_method)
|
||
# The parent evidence comes from a Stammbaum chart — attribute to the
|
||
# primary record's file when it is a Stammbaum.
|
||
parent_file = best_file if best_file and "stammbaum" in best_file.lower() else None
|
||
if parent_file:
|
||
history.append(
|
||
f"Eltern über {method_label} erkannt (Quelle: {_quote_file(parent_file)})."
|
||
)
|
||
else:
|
||
history.append(f"Eltern über {method_label} erkannt.")
|
||
|
||
# Discarded parent candidates / links (pick_parent_ref rejections, role
|
||
# normalization drops, parent-age sanity check). Threaded in after the merge
|
||
# via the gerbil's _discarded list so they read in chronological order.
|
||
for d in (parent_discards or []):
|
||
line = _format_discard(d)
|
||
if line not in history:
|
||
history.append(line)
|
||
|
||
# Manual decisions / conflicts.
|
||
if any_decision:
|
||
history.append("Zuordnung per manueller Entscheidung getroffen.")
|
||
if any_conflict:
|
||
history.append("Konflikt per Entscheidung gelöst.")
|
||
for cn in conflict_notes:
|
||
if cn and cn not in history:
|
||
history.append(cn + ".")
|
||
|
||
# Wurfchronik provenance line.
|
||
if any("wurfchronik" in f.lower() for r in records for f in _record_source_files(r)):
|
||
history.append("Angaben aus der Wurfchronik übernommen.")
|
||
|
||
return history
|
||
|
||
|
||
def _de_date(iso):
|
||
"""YYYY-MM-DD → DD.MM.YYYY for display; pass through anything else."""
|
||
if not iso:
|
||
return iso
|
||
m = re.match(r"^(\d{4})-(\d{2})-(\d{2})$", str(iso))
|
||
if m:
|
||
return f"{m.group(3)}.{m.group(2)}.{m.group(1)}"
|
||
return iso
|
||
|
||
|
||
def _de_gender(g):
|
||
return {"male": "männlich", "female": "weiblich"}.get(g, g)
|
||
|
||
|
||
# Leading marker that visually flags a discard ("data was thrown away") line in
|
||
# the history timeline. The frontend keys discard styling off this marker.
|
||
DISCARD_MARK = "⚠ "
|
||
|
||
|
||
def _format_discard(d):
|
||
"""Render one discard record into a German history line.
|
||
|
||
A discard record is a dict describing a value/candidate the pipeline threw
|
||
away. Recognised keys:
|
||
• text — a fully pre-formatted line (used verbatim, marker added)
|
||
• label — German field label (e.g. „Geburtsdatum“)
|
||
• value — the discarded value (already display-formatted)
|
||
• file — source file the discarded value came from (attributed)
|
||
• reason — why it was dropped (e.g. „abweichend“, „unplausibel …“)
|
||
• replacement — what was used instead (already display-formatted)
|
||
• repl_file — source file the replacement came from
|
||
Produces lines like:
|
||
„⚠ Geburtsdatum 14.06.2015 aus ‚A.xlsx‘ verworfen — abweichend;
|
||
14.06.2017 aus ‚B.xlsx‘ verwendet (Mehrheit).“
|
||
Generic across entity types so litters/contacts can reuse it.
|
||
"""
|
||
if d.get("text"):
|
||
return DISCARD_MARK + d["text"]
|
||
parts = []
|
||
label = d.get("label")
|
||
value = d.get("value")
|
||
if label and value is not None:
|
||
parts.append(f"{label} {value}")
|
||
elif label:
|
||
parts.append(str(label))
|
||
elif value is not None:
|
||
parts.append(str(value))
|
||
head = " ".join(parts) if parts else "Wert"
|
||
if d.get("file"):
|
||
head += f" aus {_quote_file(d['file'])}"
|
||
line = f"{head} verworfen"
|
||
if d.get("reason"):
|
||
line += f" — {d['reason']}"
|
||
repl = d.get("replacement")
|
||
if repl is not None and repl != "":
|
||
instead = str(repl)
|
||
if d.get("repl_file"):
|
||
instead += f" aus {_quote_file(d['repl_file'])}"
|
||
suffix = d.get("replacement_note")
|
||
line += f"; {instead} verwendet"
|
||
if suffix:
|
||
line += f" ({suffix})"
|
||
elif d.get("no_replacement"):
|
||
line += "; kein Ersatz"
|
||
return DISCARD_MARK + line + "."
|
||
|
||
|
||
def _append_history(prov_json, line):
|
||
"""Append one history line to an existing Provenance JSON string and add the
|
||
contract source file. Returns the updated JSON string."""
|
||
try:
|
||
prov = json.loads(prov_json) if prov_json else {}
|
||
except (ValueError, TypeError):
|
||
prov = {}
|
||
prov.setdefault("history", [])
|
||
if line not in prov["history"]:
|
||
prov["history"].append(line)
|
||
return json.dumps(prov, ensure_ascii=False)
|
||
|
||
|
||
def _parse_price(raw):
|
||
"""Parse a contract price string ('27,50' / '30.00' / '') → float (0.0 if empty).
|
||
|
||
extract_contracts.py emits German-formatted numbers ('27,50'); accept both
|
||
comma and dot decimal separators. Unparseable/empty → 0.0 (a price-less
|
||
contract is still a valid contract record)."""
|
||
if raw is None:
|
||
return 0.0
|
||
s = str(raw).strip()
|
||
if not s:
|
||
return 0.0
|
||
s = s.replace(".", "").replace(",", ".") if ("," in s) else s
|
||
try:
|
||
return round(float(s), 2)
|
||
except ValueError:
|
||
return 0.0
|
||
|
||
|
||
def enrich_from_contracts(contracts, resolved_gerbils, contact_by_norm_name,
|
||
contact_id_map, exclude_decisions=None):
|
||
"""Conservatively fold Abgabevertrag data into the resolved gerbils AND emit
|
||
one SaleContract record per contract with a resolvable buyer.
|
||
|
||
For every parsed contract we (a) ensure the buyer exists as a (receiver)
|
||
contact, reusing the existing contact dedup/normalisation, and (b) try to
|
||
match each animal call-name to exactly one resolved gerbil that the breeder
|
||
bred ("…Chaoten"). On a confident match we set ReceiverContactId /
|
||
GoHomeDate / Status=GivenAway — but only where not already set differently —
|
||
and add a provenance history line. Ambiguous or absent matches are logged,
|
||
never guessed.
|
||
|
||
In addition we build a `sale_contracts` list (one record per contract whose
|
||
buyer resolves to a contact). Each record carries a deterministic Id (from
|
||
the source filename, so re-ingest is idempotent), the resolved buyer
|
||
ContactId, the parsed Price, the parsed dates and the gerbil ids that
|
||
matched for that contract. Contracts with NO date at all are skipped from
|
||
record creation (the SaleContract.HandoverDate/ContractDate columns are
|
||
non-nullable DateOnly) and counted in stats["dateless_skipped"]; contracts
|
||
whose buyer cannot be resolved are counted in stats["no_buyer_skipped"].
|
||
|
||
Returns (stats, sale_contracts). Mutates resolved_gerbils +
|
||
contact_by_norm_name in place. New buyer contacts are appended via
|
||
contact_by_norm_name so the later IsReceiver-flag pass picks them up
|
||
automatically.
|
||
"""
|
||
stats = {
|
||
"contracts": len(contracts), "buyers_created": 0, "buyers_existing": 0,
|
||
"matched": 0, "ambiguous_skipped": 0, "no_match_skipped": 0,
|
||
"receiver_set": 0, "gohome_set": 0, "status_givenaway": 0,
|
||
"conflicts": 0,
|
||
"records_created": 0, "no_buyer_skipped": 0, "dateless_skipped": 0,
|
||
"records_with_animal": 0, "records_with_date": 0,
|
||
}
|
||
sale_contracts = []
|
||
# The same contract filename can appear more than once in contracts.json
|
||
# (the .docx is filed in several subfolders of the share). The record Id is
|
||
# derived from the filename, so we must collapse those into ONE record per
|
||
# Id (a duplicate PK would break ingest). Keyed by Id; animal lists are
|
||
# merged and a missing date is back-filled from the duplicate.
|
||
records_by_id = {}
|
||
if not contracts:
|
||
return stats, sale_contracts
|
||
|
||
# excludeContractMatch (MemPalace stolperfalle): some breeding/parent animals
|
||
# (e.g. Makoto, Danako) are wrongly matched to a contract whose filename names
|
||
# them as PARENTS of the sold pup, not as the sold animals. A decision with
|
||
# `excludeContractMatch: true` pins such an animal OUT of contract matching: it
|
||
# never gets ReceiverContactId/GoHomeDate/Status=GivenAway from a contract.
|
||
# The override that runs BEFORE enrich (isResident/notes/receiver) cannot undo
|
||
# this afterwards, so the gate must sit here, inside the match loop.
|
||
_exclude_namedob = set() # {(call-name, iso-dob), (call-name, "")}
|
||
_exclude_extref = set() # externalRef suffixes
|
||
for d in (exclude_decisions or []):
|
||
if not d.get("excludeContractMatch"):
|
||
continue
|
||
er = (d.get("externalRef") or "").strip()
|
||
if er:
|
||
_exclude_extref.add(er)
|
||
nm = d.get("name")
|
||
if nm is not None:
|
||
ck = normalize_name(get_call_name(nm or ""))
|
||
iso = parse_date(d.get("dob")) if d.get("dob") else ""
|
||
_exclude_namedob.add((ck, iso or ""))
|
||
|
||
def _is_excluded(g):
|
||
if _exclude_extref:
|
||
er = g.get("ExternalRef") or ""
|
||
if er and (er in _exclude_extref or
|
||
any(er.endswith(k) for k in _exclude_extref)):
|
||
return True
|
||
if _exclude_namedob:
|
||
ck = normalize_name(get_call_name(g.get("Name") or ""))
|
||
iso = g.get("DateOfBirth") or ""
|
||
if (ck, iso) in _exclude_namedob or (ck, "") in _exclude_namedob:
|
||
return True
|
||
return False
|
||
|
||
# Index breeder-owned gerbils by dedup name key. Contracts only ever sell
|
||
# animals the breeder bred, so restrict candidates to her own stock to avoid
|
||
# colliding with same-named foreign-bred animals.
|
||
def is_own(g):
|
||
ob = (g.get("OriginBreeder") or "").lower()
|
||
return ("chaoten" in ob) or (g.get("OriginBreeder") is None)
|
||
|
||
index = {}
|
||
for g in resolved_gerbils:
|
||
if not is_own(g):
|
||
continue
|
||
key = get_dedup_name_key(get_call_name(g.get("Name", "")))
|
||
if key:
|
||
index.setdefault(key, []).append(g)
|
||
|
||
def year_of(iso):
|
||
return iso[:4] if iso else None
|
||
|
||
# Reject buyer values that are obviously label leakage / non-person noise
|
||
# (defends against any stale contracts.json produced before the parser fix).
|
||
_buyer_junk = re.compile(
|
||
r"^(?:stra\w+e|wohnort|fon|e-?mail|handy|festnetz|telefon|homepage|"
|
||
r"zuchtname|facebook|name)\s*:?\s*$|^[\d\s/]+$", re.IGNORECASE)
|
||
|
||
for c in contracts:
|
||
fname = c.get("sourceFile", "")
|
||
buyer_raw = (c.get("buyer") or "").strip()
|
||
if buyer_raw and _buyer_junk.match(buyer_raw):
|
||
buyer_raw = ""
|
||
|
||
# --- buyer contact (reuse curated normalisation/dedup) ---
|
||
buyer_global_id = None
|
||
if buyer_raw:
|
||
canon, keep = get_normalized_contact_name(buyer_raw)
|
||
if keep and canon:
|
||
norm = normalize_name(canon)
|
||
if norm in contact_by_norm_name:
|
||
gc = contact_by_norm_name[norm]
|
||
gc.setdefault("_source_files", set()).add(fname)
|
||
gc["_merged_count"] = gc.get("_merged_count", 1) + 1
|
||
stats["buyers_existing"] += 1
|
||
else:
|
||
gid = generate_guid(f"contact-{norm}")
|
||
contact_by_norm_name[norm] = {
|
||
"Id": gid, "Name": canon, "Email": None, "Phone": None,
|
||
"Address": None, "Notes": None, "NameSuffix": None,
|
||
"_source_files": {fname}, "_merged_count": 1,
|
||
}
|
||
stats["buyers_created"] += 1
|
||
buyer_global_id = contact_by_norm_name[norm]["Id"]
|
||
|
||
# --- match each animal call-name to a resolved gerbil ---
|
||
handover = parse_date(c.get("handoverDate"))
|
||
contract_date = parse_date(c.get("contractDate"))
|
||
c_year = year_of(parse_date(c.get("dob"))) if c.get("dob") else None
|
||
c_color = (c.get("color") or "").strip().lower()
|
||
matched_gerbil_ids = [] # gerbils this contract resolved to (for the record)
|
||
|
||
for call in (c.get("animals") or []):
|
||
key = get_dedup_name_key(get_call_name(call))
|
||
if not key:
|
||
continue
|
||
cands = index.get(key, [])
|
||
if not cands:
|
||
stats["no_match_skipped"] += 1
|
||
continue
|
||
|
||
# Corroborate when more than one candidate shares the call-name.
|
||
chosen = None
|
||
if len(cands) == 1:
|
||
chosen = cands[0]
|
||
else:
|
||
scored = []
|
||
for g in cands:
|
||
score = 0
|
||
g_year = year_of(g.get("DateOfBirth"))
|
||
if c_year and g_year and c_year == g_year:
|
||
score += 2
|
||
if c_color and g.get("ColorVarietyId"):
|
||
# color match is corroboration; we don't have the name
|
||
# here, so only DOB drives disambiguation strongly.
|
||
pass
|
||
scored.append((score, g))
|
||
scored.sort(key=lambda t: t[0], reverse=True)
|
||
if scored[0][0] >= 2 and (len(scored) == 1 or scored[0][0] > scored[1][0]):
|
||
chosen = scored[0][1]
|
||
else:
|
||
stats["ambiguous_skipped"] += 1
|
||
continue
|
||
|
||
# excludeContractMatch: this animal is pinned out of contract matching
|
||
# (it is a parent named in the filename, not the sold pup). Skip BEFORE
|
||
# any receiver/gohome/status mutation and before counting it as matched.
|
||
if _is_excluded(chosen):
|
||
stats["no_match_skipped"] += 1
|
||
continue
|
||
|
||
stats["matched"] += 1
|
||
if chosen.get("Id") and chosen["Id"] not in matched_gerbil_ids:
|
||
matched_gerbil_ids.append(chosen["Id"])
|
||
|
||
# --- set receiver, only if not already set differently ---
|
||
if buyer_global_id:
|
||
cur = chosen.get("ReceiverContactId")
|
||
if not cur:
|
||
chosen["ReceiverContactId"] = buyer_global_id
|
||
stats["receiver_set"] += 1
|
||
chosen["Provenance"] = _append_history(
|
||
chosen.get("Provenance"),
|
||
f"Abgabe an „{buyer_raw}“ aus Vertrag {_quote_file(fname)} übernommen.")
|
||
elif cur != buyer_global_id:
|
||
stats["conflicts"] += 1
|
||
chosen["Provenance"] = _append_history(
|
||
chosen.get("Provenance"),
|
||
f"Vertrag {_quote_file(fname)} nennt anderen Abnehmer „{buyer_raw}“ "
|
||
f"— bestehende Zuordnung beibehalten.")
|
||
|
||
# --- set go-home date, only if empty ---
|
||
if handover and not chosen.get("GoHomeDate"):
|
||
chosen["GoHomeDate"] = handover
|
||
stats["gohome_set"] += 1
|
||
chosen["Provenance"] = _append_history(
|
||
chosen.get("Provenance"),
|
||
f"Abgabedatum {handover} aus Vertrag {_quote_file(fname)} übernommen.")
|
||
|
||
# --- status: derive GivenAway if we set a receiver and it isn't
|
||
# already a stronger state (Deceased). ---
|
||
if chosen.get("ReceiverContactId") and chosen.get("Status") not in (
|
||
"Deceased", "GivenAway"):
|
||
chosen["Status"] = "GivenAway"
|
||
stats["status_givenaway"] += 1
|
||
|
||
# --- emit a SaleContract record for this contract ---------------------
|
||
# Only contracts with a resolvable buyer become records (the row needs a
|
||
# ContactId). A record with zero matched animals is still kept — better
|
||
# to show the contract than to drop it.
|
||
if not buyer_global_id:
|
||
stats["no_buyer_skipped"] += 1
|
||
continue
|
||
|
||
# HandoverDate/ContractDate are non-nullable DateOnly in the DB. Fall
|
||
# back from one to the other; if BOTH are missing, skip the record
|
||
# (we do not invent dates) and count it.
|
||
h = handover or contract_date
|
||
cd = contract_date or handover
|
||
if not h: # implies cd is also None
|
||
stats["dateless_skipped"] += 1
|
||
continue
|
||
|
||
rec_id = generate_guid(f"contract-{fname}")
|
||
existing = records_by_id.get(rec_id)
|
||
if existing is None:
|
||
records_by_id[rec_id] = {
|
||
"Id": rec_id,
|
||
"ContactId": buyer_global_id,
|
||
"Price": _parse_price(c.get("price")),
|
||
"HandoverDate": h,
|
||
"ContractDate": cd,
|
||
"FileName": fname,
|
||
"Animals": list(matched_gerbil_ids),
|
||
}
|
||
else:
|
||
# Same filename seen again — merge animal matches; back-fill price.
|
||
for gid in matched_gerbil_ids:
|
||
if gid not in existing["Animals"]:
|
||
existing["Animals"].append(gid)
|
||
if not existing["Price"]:
|
||
existing["Price"] = _parse_price(c.get("price"))
|
||
|
||
sale_contracts = list(records_by_id.values())
|
||
stats["records_created"] = len(sale_contracts)
|
||
stats["records_with_animal"] = sum(1 for r in sale_contracts if r["Animals"])
|
||
stats["records_with_date"] = sum(1 for r in sale_contracts if r["HandoverDate"])
|
||
return stats, sale_contracts
|
||
|
||
|
||
def main():
|
||
print("Loading color variety seeds...")
|
||
variety_map = {}
|
||
variety_genotypes = {}
|
||
|
||
# Load from C# ApplicationContext.cs catalog for stable database GUIDs (index + 1)
|
||
here = os.path.dirname(os.path.abspath(__file__))
|
||
app_context_path = os.path.abspath(os.path.join(here, "../../GerbilManagerWebAPI/ApplicationContext.cs"))
|
||
cs_name_to_id = {}
|
||
if os.path.exists(app_context_path):
|
||
with open(app_context_path, 'r', encoding='utf-8') as f:
|
||
content = f.read()
|
||
catalog_match = re.search(r'catalog\s*=\s*\{(.*?)\};', content, re.DOTALL)
|
||
if catalog_match:
|
||
block = catalog_match.group(1)
|
||
entries = re.findall(r'\(\s*"([^"]+)"\s*,\s*"([^"]+)"\s*,\s*(\d+)\s*\)', block)
|
||
for idx, (name, genotype, sort_order) in enumerate(entries):
|
||
variety_id = f"00000000-0000-0000-0000-{idx + 1:012d}"
|
||
cs_name_to_id[name.strip().lower()] = variety_id
|
||
variety_map[name.strip().lower()] = variety_id
|
||
variety_genotypes[variety_id] = genotype.strip()
|
||
else:
|
||
print(f"Warning: ApplicationContext.cs not found at {app_context_path}")
|
||
|
||
if os.path.exists(SEEDS_PATH):
|
||
with open(SEEDS_PATH, "r", encoding="utf-8") as f:
|
||
seeds = json.load(f)
|
||
for v in seeds:
|
||
name_lower = v["name"].strip().lower()
|
||
variety_id = cs_name_to_id.get(name_lower)
|
||
if not variety_id:
|
||
variety_id = f"00000000-0000-0000-0000-{v['sortOrder'] + 1:012d}"
|
||
variety_map[name_lower] = variety_id
|
||
|
||
# Map English name if present
|
||
if "english" in v and v["english"]:
|
||
variety_map[v["english"].strip().lower()] = variety_id
|
||
|
||
if variety_id not in variety_genotypes:
|
||
variety_genotypes[variety_id] = v.get("canonicalGenotype")
|
||
else:
|
||
print(f"Warning: Seeds path not found at {SEEDS_PATH}")
|
||
|
||
# Reverse map (variety GUID → human name) for discard/replacement history
|
||
# lines that need to show a colour value rather than a raw GUID. First wins
|
||
# so we keep the canonical lower-case catalog name.
|
||
variety_id_to_name = {}
|
||
for name_lower, vid in variety_map.items():
|
||
if vid not in variety_id_to_name:
|
||
variety_id_to_name[vid] = name_lower
|
||
|
||
md_files = sorted([f for f in os.listdir(DIR_PATH) if f.lower().endswith('.md')])
|
||
print(f"Found {len(md_files)} markdown files in {DIR_PATH}.")
|
||
|
||
raw_contacts = []
|
||
raw_litters = []
|
||
raw_gerbils = []
|
||
|
||
# Map to track which files contained explicit dates
|
||
file_explicit_dates = {}
|
||
|
||
# 1. Parse JSON blocks from all markdown files and scope IDs by filename
|
||
for filename in md_files:
|
||
filepath = os.path.join(DIR_PATH, filename)
|
||
with open(filepath, 'r', encoding='utf-8') as f:
|
||
content = f.read()
|
||
|
||
json_start = content.find("```json")
|
||
if json_start == -1:
|
||
continue
|
||
json_start += len("```json")
|
||
json_end = content.rfind("```")
|
||
if json_end == -1 or json_end <= json_start:
|
||
continue
|
||
|
||
json_str = content[json_start:json_end].strip()
|
||
try:
|
||
data = json.loads(json_str)
|
||
|
||
contacts_key = next((k for k in data if k.lower() == 'contacts'), None)
|
||
litters_key = next((k for k in data if k.lower() == 'litters'), None)
|
||
gerbils_key = next((k for k in data if k.lower() == 'gerbils'), None)
|
||
|
||
file_contacts = data.get(contacts_key, []) if contacts_key else []
|
||
file_litters = data.get(litters_key, []) if litters_key else []
|
||
file_gerbils = data.get(gerbils_key, []) if gerbils_key else []
|
||
|
||
# Find explicit dates on this page to build chronological ordering
|
||
explicit_dates = []
|
||
for l in file_litters:
|
||
d = parse_date(l.get("Date") or l.get("date") or l.get("DateOfBirth") or l.get("dateOfBirth"))
|
||
if d:
|
||
explicit_dates.append(d)
|
||
for g in file_gerbils:
|
||
d = parse_date(g.get("DateOfBirth") or g.get("dateOfBirth"))
|
||
if d:
|
||
explicit_dates.append(d)
|
||
|
||
if explicit_dates:
|
||
# Convert to days since 2000-01-01
|
||
day_vals = [date_to_days(d) for d in explicit_dates if date_to_days(d) is not None]
|
||
if day_vals:
|
||
file_explicit_dates[filename] = sum(day_vals) / len(day_vals)
|
||
|
||
def scope_id(local_id):
|
||
if not local_id:
|
||
return None
|
||
local_str = str(local_id).strip()
|
||
if not local_str:
|
||
return None
|
||
return generate_guid(f"{filename}-{local_str}")
|
||
|
||
for c in file_contacts:
|
||
old_id = c.get("Id") or c.get("id")
|
||
c["_filename"] = filename
|
||
c["_scoped_id"] = scope_id(old_id)
|
||
raw_contacts.append(c)
|
||
|
||
for l in file_litters:
|
||
old_id = l.get("Id") or l.get("id")
|
||
l["_filename"] = filename
|
||
l["_scoped_id"] = scope_id(old_id)
|
||
l["_scoped_father_id"] = scope_id(l.get("FatherId") or l.get("fatherId"))
|
||
l["_scoped_mother_id"] = scope_id(l.get("MotherId") or l.get("motherId"))
|
||
raw_litters.append(l)
|
||
|
||
for g in file_gerbils:
|
||
old_id = g.get("Id") or g.get("id")
|
||
g["_filename"] = filename
|
||
g["_scoped_id"] = scope_id(old_id)
|
||
g["_scoped_litter_id"] = scope_id(g.get("LitterId") or g.get("litterId"))
|
||
g["_scoped_origin_cid"] = scope_id(g.get("OriginContactId") or g.get("originContactId"))
|
||
|
||
receiver_val = (g.get("ReceiverContactId") or g.get("receiverContactId") or
|
||
g.get("BuyerId") or g.get("buyerId") or
|
||
g.get("BuyerContactId") or g.get("buyerContactId") or
|
||
g.get("givenAwayContactId") or g.get("givenAwayToContactId") or
|
||
g.get("ownerContactId") or g.get("ownerId"))
|
||
g["_scoped_receiver_cid"] = scope_id(receiver_val)
|
||
raw_gerbils.append(g)
|
||
|
||
except json.JSONDecodeError as e:
|
||
print(f"Failed to parse JSON in {filename}: {e}")
|
||
|
||
# Chronological Interpolation: Estimate the date of each page based on neighboring pages with dates
|
||
file_dates = {}
|
||
sorted_files = sorted(md_files)
|
||
|
||
# Simple linear interpolation / extrapolation
|
||
for i, fn in enumerate(sorted_files):
|
||
if fn in file_explicit_dates:
|
||
file_dates[fn] = file_explicit_dates[fn]
|
||
else:
|
||
# Look left for closest explicit date
|
||
left_val, left_dist = None, None
|
||
for j in range(i - 1, -1, -1):
|
||
if sorted_files[j] in file_explicit_dates:
|
||
left_val = file_explicit_dates[sorted_files[j]]
|
||
left_dist = i - j
|
||
break
|
||
|
||
# Look right for closest explicit date
|
||
right_val, right_dist = None, None
|
||
for j in range(i + 1, len(sorted_files)):
|
||
if sorted_files[j] in file_explicit_dates:
|
||
right_val = file_explicit_dates[sorted_files[j]]
|
||
right_dist = j - i
|
||
break
|
||
|
||
if left_val is not None and right_val is not None:
|
||
# Interpolate
|
||
file_dates[fn] = left_val + (right_val - left_val) * (left_dist / (left_dist + right_dist))
|
||
elif left_val is not None:
|
||
# Extrapolate right (assume 30 days per page gap as placeholder)
|
||
file_dates[fn] = left_val + (left_dist * 30)
|
||
elif right_val is not None:
|
||
# Extrapolate left
|
||
file_dates[fn] = right_val - (right_dist * 30)
|
||
else:
|
||
# No dates in entire log? Default to 2011-01-01
|
||
file_dates[fn] = date_to_days("2011-01-01")
|
||
|
||
print(f"Parsed {len(raw_contacts)} raw contacts, {len(raw_litters)} raw litters, {len(raw_gerbils)} raw gerbils.")
|
||
|
||
# Run extract.py to make sure stammbaum data is up to date
|
||
import subprocess
|
||
print("Running extract.py to extract Stammbäume...")
|
||
try:
|
||
subprocess.run([sys.executable, "extract.py"], check=True)
|
||
except Exception as e:
|
||
print(f"Warning: Failed to run extract.py: {e}")
|
||
|
||
# Load Stammbaum data
|
||
stammbaum_only_animals = []
|
||
animals_path = os.path.join(OUTPUT_DIR, "animals.json")
|
||
if os.path.exists(animals_path):
|
||
with open(animals_path, "r", encoding="utf-8") as f:
|
||
stammbaum_animals = json.load(f)
|
||
for a in stammbaum_animals:
|
||
sources = a.get("sourceFiles", [])
|
||
if any("stammbaum" in str(s).lower() for s in sources):
|
||
stammbaum_only_animals.append(a)
|
||
print(f"Loaded {len(stammbaum_only_animals)} Stammbaum animals.")
|
||
else:
|
||
print(f"Warning: Stammbaum animals.json not found at {animals_path}")
|
||
|
||
# Run extract_docx.py to make sure docx data is up to date
|
||
print("Running extract_docx.py to extract docx...")
|
||
try:
|
||
subprocess.run([sys.executable, "extract_docx.py"], check=True)
|
||
except Exception as e:
|
||
print(f"Warning: Failed to run extract_docx.py: {e}")
|
||
|
||
# Load Docx data
|
||
docx_animals = []
|
||
docx_animals_path = os.path.join(OUTPUT_DIR, "docx_animals.json")
|
||
if os.path.exists(docx_animals_path):
|
||
with open(docx_animals_path, "r", encoding="utf-8") as f:
|
||
docx_animals = json.load(f)
|
||
print(f"Loaded {len(docx_animals)} animals from docx.")
|
||
else:
|
||
print(f"Warning: docx_animals.json not found at {docx_animals_path}")
|
||
|
||
docx_litters = []
|
||
docx_litters_path = os.path.join(OUTPUT_DIR, "docx_litters.json")
|
||
if os.path.exists(docx_litters_path):
|
||
with open(docx_litters_path, "r", encoding="utf-8") as f:
|
||
docx_litters = json.load(f)
|
||
print(f"Loaded {len(docx_litters)} litters from docx.")
|
||
else:
|
||
print(f"Warning: docx_litters.json not found at {docx_litters_path}")
|
||
|
||
# Abgabevertrag-Datensätze (aus extract_contracts.py). Optional: wenn die
|
||
# Datei fehlt, läuft der Import ohne Vertrags-Anreicherung normal weiter.
|
||
contracts = []
|
||
contracts_path = os.path.join(OUTPUT_DIR, "contracts.json")
|
||
if os.path.exists(contracts_path):
|
||
with open(contracts_path, "r", encoding="utf-8") as f:
|
||
contracts = json.load(f)
|
||
print(f"Loaded {len(contracts)} Abgabeverträge.")
|
||
else:
|
||
print(f"Info: contracts.json not found at {contracts_path} (Vertrags-Anreicherung übersprungen)")
|
||
|
||
# Extract docx buyer contacts and add to raw_contacts
|
||
for da in docx_animals:
|
||
o_name = (da.get("owner") or "").strip()
|
||
if o_name:
|
||
o_scoped_id = generate_guid(f"docx-contact-{normalize_name(o_name)}")
|
||
raw_contacts.append({
|
||
"Name": o_name,
|
||
"_filename": "Wurfchronik-Detail",
|
||
"_scoped_id": o_scoped_id
|
||
})
|
||
|
||
# Map docx litters and append to raw_litters
|
||
docx_litter_id_map = {} # wsCode -> scoped_id
|
||
for dl in docx_litters:
|
||
l_name = dl["litterId"]
|
||
dob_val = parse_date(dl["dob"])
|
||
f_name = get_normalized_gerbil_name(dl["fatherName"])
|
||
m_name = get_normalized_gerbil_name(dl["motherName"])
|
||
ws_code = dl["wsCode"]
|
||
note_val = dl.get("note")
|
||
|
||
# Parse survived/total born from wsCode (e.g. 4/5)
|
||
total_born = None
|
||
deaths_8w = None
|
||
if "/" in ws_code:
|
||
parts = ws_code.split("/")
|
||
if len(parts) == 2:
|
||
try:
|
||
survived = int(parts[0])
|
||
total = int(parts[1])
|
||
total_born = total
|
||
deaths_8w = max(0, total - survived)
|
||
except ValueError:
|
||
pass
|
||
|
||
l_scoped_id = generate_guid(f"docx-litter-{normalize_name(ws_code)}-{dob_val or '0001-01-01'}")
|
||
docx_litter_id_map[(ws_code, dob_val)] = l_scoped_id
|
||
|
||
raw_litters.append({
|
||
"Id": l_scoped_id,
|
||
"Name": l_name,
|
||
"Date": dob_val,
|
||
"TotalBorn": total_born,
|
||
"DeathsWithin8Weeks": deaths_8w,
|
||
"FatherId": generate_guid(f"stammbaum-animal-{normalize_name(f_name)}"), # placeholder
|
||
"MotherId": generate_guid(f"stammbaum-animal-{normalize_name(m_name)}"), # placeholder
|
||
"ExpectedGoHomeDate": None,
|
||
"Notes": note_val if note_val else "Docx imported litter",
|
||
"PairingCode": None,
|
||
"ExternalRef": f"docx-litter-{l_scoped_id}",
|
||
"LitterLetter": l_name[0] if l_name and len(l_name) > 0 else None,
|
||
"_father_name": f_name,
|
||
"_mother_name": m_name,
|
||
"_filename": "Wurfchronik-Detail",
|
||
"_scoped_id": l_scoped_id,
|
||
"_scoped_father_id": generate_guid(f"stammbaum-animal-{normalize_name(f_name)}"),
|
||
"_scoped_mother_id": generate_guid(f"stammbaum-animal-{normalize_name(m_name)}")
|
||
})
|
||
|
||
|
||
# Extract stammbaum contacts and add to raw_contacts
|
||
for a in stammbaum_only_animals:
|
||
b_name = (a.get("breeder") or "").strip()
|
||
if b_name:
|
||
b_scoped_id = generate_guid(f"stammbaum-contact-{normalize_name(b_name)}")
|
||
raw_contacts.append({
|
||
"Name": b_name,
|
||
"_filename": "Stammbaum",
|
||
"_scoped_id": b_scoped_id
|
||
})
|
||
z_name = (a.get("zucht") or "").strip()
|
||
if z_name:
|
||
z_scoped_id = generate_guid(f"stammbaum-contact-{normalize_name(z_name)}")
|
||
raw_contacts.append({
|
||
"Name": z_name,
|
||
"_filename": "Stammbaum",
|
||
"_scoped_id": z_scoped_id
|
||
})
|
||
|
||
# Residency propagation for stammbaum animals
|
||
def is_clan_zucht(z):
|
||
if not z:
|
||
return False
|
||
norm = z.lower()
|
||
return "klein" in norm and "chaot" in norm and "extern" not in norm
|
||
|
||
def is_external_cattery(a):
|
||
"""True if an animal belongs to a NAMED, non-clan cattery (e.g. 'Black
|
||
Forest', 'LennyLengo', a foreign line) or the externally-acquired
|
||
founder markers. Such animals were never in the breeder's own stock, so
|
||
residency must NOT be propagated onto them (tickets #17/#20 — Hagrid of
|
||
Black Forest). Animals with no cattery at all are name-only lineage
|
||
ancestors and stay eligible for propagation."""
|
||
for z in (a.get("zucht"), a.get("zuchtCanon"), a.get("breeder")):
|
||
z = (z or "").strip()
|
||
if z and not is_clan_zucht(z):
|
||
return True
|
||
name = a.get("name", "")
|
||
if name:
|
||
m = re.search(r"\b(?:of|von\s+den|von\s+der|v\.\s?d\.|von)\s+(.+)$", name, re.IGNORECASE)
|
||
if m:
|
||
cattery = m.group(1).strip()
|
||
if cattery and not is_clan_zucht(cattery):
|
||
return True
|
||
m = re.search(r"^(.+?'s)\s+", name, re.IGNORECASE)
|
||
if m:
|
||
cattery = m.group(1).strip()
|
||
if cattery and not is_clan_zucht(cattery):
|
||
return True
|
||
return False
|
||
|
||
stammbaum_resident_ids = set()
|
||
for a in stammbaum_only_animals:
|
||
if is_clan_zucht(a.get("zucht")) or is_clan_zucht(a.get("zuchtCanon")):
|
||
stammbaum_resident_ids.add(a["id"])
|
||
|
||
# Propagate residency to parents of resident offspring — but never onto
|
||
# animals from a named external cattery (they were never in this stock).
|
||
for _ in range(5):
|
||
for a in stammbaum_only_animals:
|
||
if a["id"] in stammbaum_resident_ids:
|
||
for p_ref in a.get("parentRefs", []):
|
||
p_key = (normalize_name(p_ref["name"]), parse_date(p_ref["dob"]))
|
||
for cand in stammbaum_only_animals:
|
||
if normalize_name(cand["name"]) == p_key[0]:
|
||
cand_dob = parse_date(cand["dob"])
|
||
if not p_key[1] or cand_dob == p_key[1]:
|
||
if is_external_cattery(cand):
|
||
continue
|
||
stammbaum_resident_ids.add(cand["id"])
|
||
|
||
# Pre-index Wurfchronik litters from markdown
|
||
md_litters_idx = {}
|
||
# …and a date-only index of Wurfchronik litters that NAME both parents. The
|
||
# Wurfchronik is authoritative: when a chart-position reconstruction picks the
|
||
# WRONG parents (so the (father,mother,date) key misses) but exactly ONE
|
||
# Wurfchronik litter exists for that birthdate, attach the animal to it rather
|
||
# than fabricating a virtual litter with bad parents (tickets #12 Tony,
|
||
# #15 Odelia, #31 Jamie — Wurfchronik-Vorrang vor chart-position).
|
||
md_litters_by_date = {}
|
||
def _md_parent_names(rl):
|
||
f = rl.get("FatherName") or rl.get("fatherName") or rl.get("ParentMaleName") or rl.get("parentMaleName") or rl.get("_father_name")
|
||
m = rl.get("MotherName") or rl.get("motherName") or rl.get("ParentFemaleName") or rl.get("parentFemaleName") or rl.get("_mother_name")
|
||
if not f and not m:
|
||
_note = rl.get("Notes") or rl.get("notes") or rl.get("Note") or rl.get("note") or ""
|
||
_m = re.search(r"(?:Eltern|Pairing|Paarung|Paar):\s*(.+?)\s*(?:\+|\&)\s*(.+?)\s*(?:;|$)", _note, re.IGNORECASE)
|
||
if _m and "/" not in _m.group(1) and "/" not in _m.group(2):
|
||
f, m = _m.group(1).strip(), _m.group(2).strip()
|
||
return get_normalized_gerbil_name(f), get_normalized_gerbil_name(m)
|
||
for rl in raw_litters:
|
||
f_name, m_name = _md_parent_names(rl)
|
||
ldate = parse_date(rl.get("Date") or rl.get("date") or rl.get("DateOfBirth") or rl.get("dateOfBirth"))
|
||
if f_name and m_name and ldate:
|
||
key = (normalize_name(f_name), normalize_name(m_name), ldate)
|
||
md_litters_idx[key] = rl
|
||
md_litters_by_date.setdefault(ldate, []).append(rl)
|
||
|
||
# Gender index for parent-ref selection: normalized name → 'male' | 'female'
|
||
# | 'ambiguous'. Drives the gender-aware ranking in pick_parent_ref so a dated
|
||
# but wrong-sex ref (e.g. female „Danielle“) cannot win the father slot over
|
||
# an undated male/unknown one (e.g. „Hagrid Rubeus“).
|
||
# A name's gender is "ambiguous" ONLY when BOTH male and female records exist
|
||
# for it. A definite gender beats an UNKNOWN (None) duplicate — otherwise a
|
||
# bare DOB-less ancestor box (gender=None) would poison a name that another
|
||
# record clearly types (e.g. Dorie of Black Forest, Zadar): they would fall
|
||
# back to None and lose gender-based role disambiguation (tickets #35, #33).
|
||
gender_idx = {}
|
||
for a in stammbaum_only_animals:
|
||
g = (a.get("gender") or "").lower().strip()
|
||
g = g if g in ("male", "female") else None
|
||
for key in {normalize_name(a.get("name")), normalize_name(get_call_name(a.get("name") or ""))}:
|
||
if not key:
|
||
continue
|
||
cur = gender_idx.get(key, "__unset__")
|
||
if cur == "__unset__" or cur is None:
|
||
gender_idx[key] = g # first value, or upgrade None → definite
|
||
elif g is None or g == cur:
|
||
pass # keep existing definite gender
|
||
else:
|
||
gender_idx[key] = "ambiguous" # genuine male vs female conflict
|
||
|
||
def gender_of_name(name):
|
||
v = gender_idx.get(normalize_name(name))
|
||
return v if v in ("male", "female") else None
|
||
|
||
# Create virtual litters for stammbaum animals
|
||
created_virtual_litters = {}
|
||
for a in stammbaum_only_animals:
|
||
parent_refs = a.get("parentRefs", [])
|
||
child_dob_raw = a.get("dob")
|
||
father_ref = pick_parent_ref(parent_refs, "father", child_dob_raw, gender_of=gender_of_name)
|
||
mother_ref = pick_parent_ref(parent_refs, "mother", child_dob_raw,
|
||
avoid_name=father_ref.get("name") if father_ref else None,
|
||
gender_of=gender_of_name)
|
||
|
||
# Record rejected parent-ref candidates (wrong sex / age-impossible /
|
||
# duplicate of the other role) so this animal's gerbil history can
|
||
# explain which Stammbaum positions were discarded and what won instead.
|
||
pick_discards = []
|
||
pick_discards += explain_pick_rejections(parent_refs, "father", child_dob_raw,
|
||
father_ref, gender_of=gender_of_name)
|
||
pick_discards += explain_pick_rejections(
|
||
parent_refs, "mother", child_dob_raw, mother_ref,
|
||
avoid_name=father_ref.get("name") if father_ref else None,
|
||
gender_of=gender_of_name)
|
||
if pick_discards:
|
||
a["_pick_discards"] = pick_discards
|
||
|
||
a["_mapped_litter_scoped_id"] = None
|
||
|
||
# A human conflict-decision override is authoritative — never let the
|
||
# Wurfchronik-Vorrang heuristic below re-attach the animal to a same-date
|
||
# chronicle litter and overrule the breeder's named parents.
|
||
_has_decision_parent = (
|
||
(father_ref or {}).get("method") == "decision" or
|
||
(mother_ref or {}).get("method") == "decision")
|
||
|
||
# Wurfchronik-Vorrang: if the chart parents don't yield an exact
|
||
# Wurfchronik match but EXACTLY ONE Wurfchronik litter (with named
|
||
# parents) exists for this birthdate, that authoritative litter wins over
|
||
# a fabricated chart-position litter (#12 Tony, #15 Odelia, #31 Jamie).
|
||
dob_val_attach = parse_date(a.get("dob"))
|
||
if dob_val_attach and not _has_decision_parent \
|
||
and not is_external_origin(a.get("name"), a.get("zucht"), a.get("breeder")) \
|
||
and not is_external_cattery(a):
|
||
same_date = md_litters_by_date.get(dob_val_attach, [])
|
||
exact_hit = None
|
||
if father_ref and mother_ref:
|
||
exact_key = (normalize_name(get_normalized_gerbil_name(father_ref.get("name"))),
|
||
normalize_name(get_normalized_gerbil_name(mother_ref.get("name"))),
|
||
dob_val_attach)
|
||
exact_hit = md_litters_idx.get(exact_key)
|
||
if not exact_hit and len(same_date) == 1:
|
||
a["_mapped_litter_scoped_id"] = same_date[0]["_scoped_id"]
|
||
|
||
if a["_mapped_litter_scoped_id"]:
|
||
pass
|
||
elif father_ref and mother_ref:
|
||
f_name = get_normalized_gerbil_name(father_ref.get("name"))
|
||
m_name = get_normalized_gerbil_name(mother_ref.get("name"))
|
||
dob_val = parse_date(a.get("dob"))
|
||
|
||
mapped_litter = None
|
||
if dob_val:
|
||
key = (normalize_name(f_name), normalize_name(m_name), dob_val)
|
||
mapped_litter = md_litters_idx.get(key)
|
||
|
||
if mapped_litter:
|
||
a["_mapped_litter_scoped_id"] = mapped_litter["_scoped_id"]
|
||
else:
|
||
v_key = (normalize_name(f_name), normalize_name(m_name), dob_val or "0001-01-01")
|
||
if v_key in created_virtual_litters:
|
||
a["_mapped_litter_scoped_id"] = created_virtual_litters[v_key]
|
||
else:
|
||
l_scoped_id = generate_guid(f"virtual-litter-{v_key[0]}-{v_key[1]}-{v_key[2]}")
|
||
created_virtual_litters[v_key] = l_scoped_id
|
||
a["_mapped_litter_scoped_id"] = l_scoped_id
|
||
|
||
# Try to link parents to actual parsed stammbaum animals.
|
||
# For human override refs (conflict-decisions, method="decision")
|
||
# skip this loose call-name preliminary linking and let the
|
||
# global name resolver pick the exact named animal instead
|
||
# (e.g. resident „Elena“, not „Elena of KK Chaos“ — #15).
|
||
f_decision = (father_ref or {}).get("method") == "decision"
|
||
m_decision = (mother_ref or {}).get("method") == "decision"
|
||
f_scoped_id = None
|
||
m_scoped_id = None
|
||
f_dob = parse_date(father_ref.get("dob"))
|
||
m_dob = parse_date(mother_ref.get("dob"))
|
||
|
||
for p_cand in ([] if f_decision else stammbaum_only_animals):
|
||
p_gender = str(p_cand.get("gender") or "").lower().strip()
|
||
if p_gender in ["w", "f", "female", "weiblich"]:
|
||
continue
|
||
p_dob = parse_date(p_cand.get("dob"))
|
||
if dob_val and p_dob and p_dob >= dob_val:
|
||
continue
|
||
cand_call_norm = normalize_name(get_call_name(p_cand["name"])) or "unbekannt"
|
||
cand_name_norm = normalize_name(p_cand["name"]) or "unbekannt"
|
||
if cand_call_norm == normalize_name(f_name) or cand_name_norm == normalize_name(f_name):
|
||
if not f_dob or p_dob == f_dob:
|
||
f_scoped_id = generate_guid(f"stammbaum-animal-{p_cand['id']}")
|
||
break
|
||
for p_cand in ([] if m_decision else stammbaum_only_animals):
|
||
p_gender = str(p_cand.get("gender") or "").lower().strip()
|
||
if p_gender in ["m", "male", "männlich"]:
|
||
continue
|
||
p_dob = parse_date(p_cand.get("dob"))
|
||
if dob_val and p_dob and p_dob >= dob_val:
|
||
continue
|
||
cand_call_norm = normalize_name(get_call_name(p_cand["name"])) or "unbekannt"
|
||
cand_name_norm = normalize_name(p_cand["name"]) or "unbekannt"
|
||
if cand_call_norm == normalize_name(m_name) or cand_name_norm == normalize_name(m_name):
|
||
if not m_dob or p_dob == m_dob:
|
||
m_scoped_id = generate_guid(f"stammbaum-animal-{p_cand['id']}")
|
||
break
|
||
|
||
raw_litters.append({
|
||
"Id": l_scoped_id,
|
||
"Name": f"Wurf von {f_name} + {m_name}",
|
||
"Date": dob_val,
|
||
"TotalBorn": None,
|
||
"DeathsWithin8Weeks": None,
|
||
"FatherId": f_scoped_id or generate_guid(f"stammbaum-animal-{normalize_name(f_name)}"),
|
||
"MotherId": m_scoped_id or generate_guid(f"stammbaum-animal-{normalize_name(m_name)}"),
|
||
"ExpectedGoHomeDate": None,
|
||
"Notes": "Pedigree virtual litter",
|
||
"PairingCode": None,
|
||
"ExternalRef": f"virtual-{l_scoped_id}",
|
||
"LitterLetter": None,
|
||
"_father_name": f_name,
|
||
"_mother_name": m_name,
|
||
"_filename": "Stammbaum",
|
||
"_scoped_id": l_scoped_id,
|
||
"_scoped_father_id": f_scoped_id or generate_guid(f"stammbaum-animal-{normalize_name(f_name)}"),
|
||
"_scoped_mother_id": m_scoped_id or generate_guid(f"stammbaum-animal-{normalize_name(m_name)}")
|
||
})
|
||
|
||
elif _has_decision_parent and (father_ref or mother_ref):
|
||
# Single KNOWN parent from a human override (the other parent is
|
||
# genuinely unknown — e.g. a sibling-pairing where only the mother is
|
||
# named: Danielle's mother = Ella, father = unnamed brother).
|
||
# Create a one-parent virtual litter; the global name resolver fills
|
||
# the known side (gender + age-plausibility pick the right same-named
|
||
# animal, e.g. Ella *10.06.2019 over Ella *13.04.2023), the other
|
||
# side stays null.
|
||
known = father_ref or mother_ref
|
||
known_role = "father" if father_ref else "mother"
|
||
known_name = get_normalized_gerbil_name(known.get("name"))
|
||
dob_val = parse_date(a.get("dob"))
|
||
v_key = (normalize_name(known_name), known_role, dob_val or "0001-01-01")
|
||
if v_key in created_virtual_litters:
|
||
a["_mapped_litter_scoped_id"] = created_virtual_litters[v_key]
|
||
else:
|
||
l_scoped_id = generate_guid(
|
||
f"virtual-litter-1p-{v_key[0]}-{v_key[1]}-{v_key[2]}")
|
||
created_virtual_litters[v_key] = l_scoped_id
|
||
a["_mapped_litter_scoped_id"] = l_scoped_id
|
||
raw_litters.append({
|
||
"Id": l_scoped_id,
|
||
"Name": f"Wurf von {known_name}",
|
||
"Date": dob_val,
|
||
"TotalBorn": None,
|
||
"DeathsWithin8Weeks": None,
|
||
"FatherId": None,
|
||
"MotherId": None,
|
||
"ExpectedGoHomeDate": None,
|
||
"Notes": "Pedigree virtual litter (ein Elternteil bekannt)",
|
||
"PairingCode": None,
|
||
"ExternalRef": f"virtual-{l_scoped_id}",
|
||
"LitterLetter": None,
|
||
"_father_name": known_name if known_role == "father" else "",
|
||
"_mother_name": known_name if known_role == "mother" else "",
|
||
"_filename": "Stammbaum",
|
||
"_scoped_id": l_scoped_id,
|
||
"_scoped_father_id": None,
|
||
"_scoped_mother_id": None,
|
||
})
|
||
|
||
# 2. Resolve Contacts globally (deduplicate by normalized name)
|
||
|
||
contact_by_norm_name = {}
|
||
contact_id_map = {} # scoped_old_id -> global_guid
|
||
|
||
for rc in raw_contacts:
|
||
name_val = rc.get("Name") or rc.get("name") or rc.get("FullName") or rc.get("fullName")
|
||
if not name_val:
|
||
first = rc.get("FirstName") or rc.get("firstName")
|
||
last = rc.get("LastName") or rc.get("lastName")
|
||
if first or last:
|
||
name_val = f"{first or ''} {last or ''}".strip()
|
||
|
||
if not name_val:
|
||
continue
|
||
|
||
canon_name, should_keep = get_normalized_contact_name(name_val)
|
||
scoped_id = rc["_scoped_id"]
|
||
if not should_keep:
|
||
if scoped_id:
|
||
contact_id_map[scoped_id] = None
|
||
continue
|
||
|
||
norm_name = normalize_name(canon_name)
|
||
|
||
rc_file = rc.get("_filename")
|
||
if norm_name not in contact_by_norm_name:
|
||
global_guid = generate_guid(f"contact-{norm_name}")
|
||
contact_by_norm_name[norm_name] = {
|
||
"Id": global_guid,
|
||
"Name": canon_name,
|
||
"Email": rc.get("Email") or rc.get("email"),
|
||
"Phone": rc.get("Phone") or rc.get("phone"),
|
||
"Address": rc.get("Address") or rc.get("address"),
|
||
"Notes": rc.get("Notes") or rc.get("notes") or rc.get("Note") or rc.get("note"),
|
||
# Provenance accumulators (consumed below, stripped from helper keys).
|
||
"_source_files": set([rc_file]) if rc_file else set(),
|
||
"_merged_count": 1,
|
||
}
|
||
else:
|
||
gc = contact_by_norm_name[norm_name]
|
||
if not gc["Email"] and (rc.get("Email") or rc.get("email")):
|
||
gc["Email"] = rc.get("Email") or rc.get("email")
|
||
if not gc["Phone"] and (rc.get("Phone") or rc.get("phone")):
|
||
gc["Phone"] = rc.get("Phone") or rc.get("phone")
|
||
if not gc["Address"] and (rc.get("Address") or rc.get("address")):
|
||
gc["Address"] = rc.get("Address") or rc.get("address")
|
||
if not gc["Notes"] and (rc.get("Notes") or rc.get("notes") or rc.get("Note") or rc.get("note")):
|
||
gc["Notes"] = rc.get("Notes") or rc.get("notes") or rc.get("Note") or rc.get("note")
|
||
if rc_file:
|
||
gc["_source_files"].add(rc_file)
|
||
gc["_merged_count"] += 1
|
||
|
||
if scoped_id:
|
||
contact_id_map[scoped_id] = contact_by_norm_name[norm_name]["Id"]
|
||
|
||
resolved_contacts = list(contact_by_norm_name.values())
|
||
print(f"Resolved to {len(resolved_contacts)} unique contacts.")
|
||
|
||
# 3. Process Litters (normalize keys)
|
||
resolved_litters = []
|
||
litter_id_map = {} # scoped_old_id -> new_guid
|
||
litter_by_scoped_id = {}
|
||
|
||
for rl in raw_litters:
|
||
filename = rl.get("_filename")
|
||
name_val = rl.get("Name") or rl.get("name")
|
||
if not name_val:
|
||
name_val = "Wurf"
|
||
|
||
dob_val = parse_date(rl.get("Date") or rl.get("date") or rl.get("DateOfBirth") or rl.get("dateOfBirth"))
|
||
|
||
new_guid = rl["_scoped_id"]
|
||
if not new_guid:
|
||
new_guid = generate_guid(f"litter-{filename}-{name_val}-{dob_val}")
|
||
|
||
scoped_old_id = rl["_scoped_id"]
|
||
if scoped_old_id:
|
||
litter_id_map[scoped_old_id] = new_guid
|
||
|
||
total_born = rl.get("TotalBorn") or rl.get("totalBorn") or rl.get("LitterSize") or rl.get("litterSize") or rl.get("size") or rl.get("totalPups")
|
||
if total_born is not None:
|
||
try:
|
||
total_born = int(total_born)
|
||
except ValueError:
|
||
total_born = None
|
||
|
||
deaths_8w = rl.get("DeathsWithin8Weeks") or rl.get("deathsWithin8Weeks")
|
||
if deaths_8w is not None:
|
||
try:
|
||
deaths_8w = int(deaths_8w)
|
||
except ValueError:
|
||
deaths_8w = None
|
||
elif total_born is not None and rl.get("survived") is not None:
|
||
try:
|
||
deaths_8w = total_born - int(rl.get("survived"))
|
||
except ValueError:
|
||
pass
|
||
|
||
father_name = rl.get("FatherName") or rl.get("fatherName") or rl.get("ParentMaleName") or rl.get("parentMaleName") or rl.get("_father_name")
|
||
mother_name = rl.get("MotherName") or rl.get("motherName") or rl.get("ParentFemaleName") or rl.get("parentFemaleName") or rl.get("_mother_name")
|
||
|
||
# Parse parents from a free-text Wurf note „Eltern: X + Y“ when the
|
||
# structured parent names/ids are missing (ticket #9 Beatrice/Q-Wurf,
|
||
# #11 Silver — ~46 Wurfchronik litters carry parents only in the note).
|
||
# The order is father + mother (German chart convention); the name
|
||
# resolver corrects the role afterwards by gender, so a swap is safe.
|
||
if not father_name and not mother_name:
|
||
_note = rl.get("Notes") or rl.get("notes") or rl.get("Note") or rl.get("note") or ""
|
||
_m = re.search(r"(?:Eltern|Pairing|Paarung|Paar):\s*(.+?)\s*(?:\+|\&)\s*(.+?)\s*(?:;|$)", _note, re.IGNORECASE)
|
||
if _m:
|
||
_p1 = _m.group(1).strip()
|
||
_p2 = _m.group(2).strip()
|
||
# Skip ambiguous "Lee/Dean" style alternatives (a slash = unsure).
|
||
if _p1 and _p2 and "/" not in _p1 and "/" not in _p2:
|
||
father_name = father_name or _p1
|
||
mother_name = mother_name or _p2
|
||
|
||
raw_ext_ref = rl.get("ExternalRef") or rl.get("externalRef") or rl.get("Id") or rl.get("id")
|
||
ext_ref_scoped = f"{filename}-{raw_ext_ref}" if raw_ext_ref else None
|
||
|
||
l_record = {
|
||
"Id": new_guid,
|
||
"Name": name_val,
|
||
"Date": dob_val,
|
||
"TotalBorn": total_born,
|
||
"DeathsWithin8Weeks": deaths_8w,
|
||
"FatherId": rl["_scoped_father_id"],
|
||
"MotherId": rl["_scoped_mother_id"],
|
||
"ExpectedGoHomeDate": parse_date(rl.get("ExpectedGoHomeDate") or rl.get("expectedGoHomeDate")),
|
||
"Notes": rl.get("Notes") or rl.get("notes") or rl.get("Note") or rl.get("note"),
|
||
"PairingCode": rl.get("PairingCode") or rl.get("pairingCode"),
|
||
"ExternalRef": ext_ref_scoped,
|
||
"LitterLetter": rl.get("LitterLetter") or rl.get("litterLetter"),
|
||
"_father_name": father_name,
|
||
"_mother_name": mother_name,
|
||
"_filename": filename,
|
||
# Provenance accumulators (canonical absorbs these during dedup below).
|
||
"_source_files": set([filename]) if filename else set(),
|
||
"_merged_count": 1,
|
||
# Virtual litters are reconstructed from a Stammbaum chart, not the
|
||
# Wurfchronik — flagged on the raw record's _filename == "Stammbaum".
|
||
"_virtual": filename == "Stammbaum",
|
||
}
|
||
resolved_litters.append(l_record)
|
||
litter_by_scoped_id[new_guid] = l_record
|
||
|
||
print(f"Processed {len(resolved_litters)} litters.")
|
||
|
||
# 3b. Deduplicate litters: same date + compatible parents → merge
|
||
# This handles the "sibling pairing" case: Stammbaum shows the same parental
|
||
# litter twice (once under the father branch, once under the mother branch),
|
||
# generating two separate litter records with the same date but only one of
|
||
# them has FatherId/MotherId resolved.
|
||
litter_canonical_map = {} # old_id -> canonical_id (for dedup within this step)
|
||
|
||
# Group by date for efficiency
|
||
by_date = {}
|
||
for l in resolved_litters:
|
||
by_date.setdefault(l["Date"], []).append(l)
|
||
|
||
litter_dedup_canonical = {} # old_litter_id -> canonical_litter_id
|
||
deduped_litters = []
|
||
|
||
for date_val, group in by_date.items():
|
||
# Partition into compatible subsets
|
||
sub_groups = []
|
||
for l in group:
|
||
placed = False
|
||
for sub in sub_groups:
|
||
if all(litter_compatible(l, member) for member in sub):
|
||
sub.append(l)
|
||
placed = True
|
||
break
|
||
if not placed:
|
||
sub_groups.append([l])
|
||
|
||
for sub in sub_groups:
|
||
if len(sub) == 1:
|
||
deduped_litters.append(sub[0])
|
||
litter_dedup_canonical[sub[0]["Id"]] = sub[0]["Id"]
|
||
continue
|
||
|
||
# Pick the canonical record: prefer the one with parents set
|
||
canonical = next((l for l in sub if l.get("FatherId") or l.get("MotherId")), sub[0])
|
||
for l in sub:
|
||
litter_dedup_canonical[l["Id"]] = canonical["Id"]
|
||
if l is not canonical:
|
||
litter_id_map[l["Id"]] = canonical["Id"]
|
||
canonical["_source_files"] |= l.get("_source_files", set())
|
||
canonical["_merged_count"] += l.get("_merged_count", 1)
|
||
if not l.get("_virtual"):
|
||
canonical["_virtual"] = False
|
||
|
||
deduped_litters.append(canonical)
|
||
if len(sub) > 1:
|
||
merged_names = [l["Id"] for l in sub if l is not canonical]
|
||
print(f"Litter-Dedup: merged {len(sub)} same-date litters on {date_val} → {canonical['Name']} (absorbed: {', '.join(merged_names)})")
|
||
|
||
n_merged = len(resolved_litters) - len(deduped_litters)
|
||
if n_merged:
|
||
print(f"Litter-Dedup: {n_merged} redundant litter record(s) removed.")
|
||
resolved_litters = deduped_litters
|
||
litter_by_scoped_id = {l["Id"]: l for l in resolved_litters}
|
||
|
||
# 4. Normalize and group Gerbils
|
||
|
||
# Helper to lookup litter dates for birth date estimation
|
||
def get_litter_date(l_id):
|
||
if l_id in litter_by_scoped_id:
|
||
d = litter_by_scoped_id[l_id]["Date"]
|
||
if d:
|
||
return d
|
||
return None
|
||
|
||
all_processed_gerbils = []
|
||
for rg in raw_gerbils:
|
||
filename = rg.get("_filename")
|
||
name_val = get_normalized_gerbil_name(rg.get("Name") or rg.get("name") or rg.get("callName"))
|
||
if not name_val:
|
||
name_val = "Unbekannt"
|
||
|
||
dob_val = parse_date(rg.get("DateOfBirth") or rg.get("dateOfBirth"))
|
||
|
||
new_guid = rg["_scoped_id"]
|
||
if not new_guid:
|
||
new_guid = generate_guid(f"gerbil-{filename}-{name_val}-{dob_val}")
|
||
|
||
# Gender normalization
|
||
gender_val = rg.get("Gender") or rg.get("gender") or "unknown"
|
||
gender_val = str(gender_val).lower().strip()
|
||
if gender_val in ["m", "male", "männlich"]:
|
||
gender = "male"
|
||
elif gender_val in ["w", "f", "female", "weiblich"]:
|
||
gender = "female"
|
||
else:
|
||
gender = "unknown"
|
||
|
||
dod_val = parse_date(rg.get("DateOfDeath") or rg.get("dateOfDeath") or rg.get("deathDate"))
|
||
gohome_val = parse_date(rg.get("GoHomeDate") or rg.get("goHomeDate") or rg.get("DateGivenAway") or rg.get("givenAwayDate") or rg.get("DateOfHandover") or rg.get("dateOfHandover") or rg.get("HandoverDate") or rg.get("DateOfSale") or rg.get("dateOfSale"))
|
||
|
||
status_val = rg.get("Status") or rg.get("status") or ""
|
||
status_val = str(status_val).lower().strip()
|
||
status = "Breeding"
|
||
if dod_val or "deceased" in status_val or "verstorben" in status_val or "tod" in status_val or "dead" in status_val:
|
||
status = "Deceased"
|
||
elif gohome_val or "givenaway" in status_val or "abgegeben" in status_val or "verkauft" in status_val or "sold" in status_val:
|
||
status = "GivenAway"
|
||
elif "forsale" in status_val or "abzugeben" in status_val:
|
||
status = "ForSale"
|
||
elif "pet" in status_val or "liebhaber" in status_val:
|
||
status = "Pet"
|
||
else:
|
||
if dod_val:
|
||
status = "Deceased"
|
||
elif gohome_val:
|
||
status = "GivenAway"
|
||
|
||
notes_val = rg.get("Notes") or rg.get("notes") or rg.get("Note") or rg.get("note")
|
||
existing_cod = rg.get("CauseOfDeath") or rg.get("causeOfDeath") or rg.get("DeathCause")
|
||
status, dod_val, cause_of_death_val = parse_death_info(notes_val, status, dod_val, existing_cod)
|
||
|
||
# Explicit resolution for Ken'ichi's cause of death
|
||
if name_val == "Ken'ichi" and dob_val == "2015-03-01":
|
||
cause_of_death_val = "Duftdrüsen-Tumor"
|
||
|
||
color_val = rg.get("ColorVarietyId") or rg.get("colorVarietyId") or rg.get("Color") or rg.get("color") or rg.get("ColorDescription") or rg.get("colorDescription")
|
||
existing_gt = rg.get("Genotype") or rg.get("genotype")
|
||
color_variety_id, genotype_val = resolve_color_and_genotype(
|
||
color_val, existing_gt, variety_map, variety_genotypes
|
||
)
|
||
if color_val and not color_variety_id:
|
||
try:
|
||
uuid.UUID(str(color_val).strip())
|
||
color_variety_id = str(color_val).strip()
|
||
except ValueError:
|
||
print(f"Warning: Unknown color variety '{color_val}' for gerbil '{name_val}' on page {filename}")
|
||
|
||
origin_cid = rg["_scoped_origin_cid"]
|
||
receiver_cid = rg["_scoped_receiver_cid"]
|
||
|
||
if origin_cid in contact_id_map:
|
||
origin_cid = contact_id_map[origin_cid]
|
||
if receiver_cid in contact_id_map:
|
||
receiver_cid = contact_id_map[receiver_cid]
|
||
|
||
# Wurfchronik junglings default to NON-resident (Ticket 381f7e51): being
|
||
# born in the chronicle does NOT by itself make an animal her breeding
|
||
# stock. Residency is recomputed authoritatively by the late isResident
|
||
# sweep (parent-of-own-litter only). Only an explicit truthy IsResident
|
||
# on the raw record is honoured here.
|
||
is_resident = rg.get("IsResident") or rg.get("isResident")
|
||
if is_resident is None:
|
||
is_resident = False
|
||
else:
|
||
is_resident = str(is_resident).lower() == "true"
|
||
|
||
traits = rg.get("CharacterTraits") or rg.get("characterTraits") or []
|
||
if not isinstance(traits, list):
|
||
traits = [str(traits)]
|
||
char_note = rg.get("CharacterNote") or rg.get("characterNote")
|
||
|
||
is_deaf = rg.get("IsDeaf") or rg.get("isDeaf")
|
||
if is_deaf is not None:
|
||
is_deaf = str(is_deaf).lower() == "true"
|
||
|
||
old_litter_id = rg["_scoped_litter_id"]
|
||
|
||
raw_ext_ref = rg.get("ExternalRef") or rg.get("externalRef") or rg.get("Id") or rg.get("id")
|
||
ext_ref_scoped = f"{filename}-{raw_ext_ref}" if raw_ext_ref else None
|
||
|
||
# Estimated effective date of birth for conflict checking
|
||
eff_dob_val = dob_val
|
||
if not eff_dob_val and old_litter_id:
|
||
# Try to get litter date if litter was resolved
|
||
mapped_l_id = litter_id_map.get(old_litter_id)
|
||
if mapped_l_id:
|
||
eff_dob_val = get_litter_date(mapped_l_id)
|
||
|
||
# Determine explicit or litter-derived birth date (None if unknown/parent)
|
||
birth_date = dob_val
|
||
if not birth_date and old_litter_id:
|
||
mapped_l_id = litter_id_map.get(old_litter_id)
|
||
if mapped_l_id:
|
||
birth_date = get_litter_date(mapped_l_id)
|
||
|
||
# If still no effective date, use the estimated page date
|
||
if not eff_dob_val:
|
||
eff_dob_val = days_to_date(file_dates[filename])
|
||
|
||
raw_breeder = rg.get("OriginBreeder") or rg.get("originBreeder")
|
||
if raw_breeder:
|
||
norm_b, keep_b = get_normalized_contact_name(raw_breeder)
|
||
raw_breeder = norm_b if keep_b else None
|
||
|
||
parent_refs = []
|
||
if old_litter_id:
|
||
rl = next((l for l in raw_litters if l.get("_scoped_id") == old_litter_id), None)
|
||
if rl:
|
||
f_name = rl.get("FatherName") or rl.get("fatherName") or rl.get("ParentMaleName") or rl.get("parentMaleName") or rl.get("_father_name")
|
||
m_name = rl.get("MotherName") or rl.get("motherName") or rl.get("ParentFemaleName") or rl.get("parentFemaleName") or rl.get("_mother_name")
|
||
if f_name:
|
||
parent_refs.append({"name": f_name, "roleGuess": "father"})
|
||
if m_name:
|
||
parent_refs.append({"name": m_name, "roleGuess": "mother"})
|
||
|
||
all_processed_gerbils.append({
|
||
"Id": new_guid,
|
||
"Name": name_val,
|
||
"Gender": gender,
|
||
"Status": status,
|
||
"LitterId": old_litter_id, # mapped later
|
||
"OriginContactId": origin_cid,
|
||
"ReceiverContactId": receiver_cid,
|
||
"EnclosureId": None,
|
||
"ColorVarietyId": color_variety_id,
|
||
"DateOfBirth": dob_val,
|
||
"DateOfDeath": dod_val,
|
||
"CauseOfDeath": cause_of_death_val,
|
||
"GoHomeDate": gohome_val,
|
||
"Genotype": genotype_val,
|
||
"Notes": rg.get("Notes") or rg.get("notes") or rg.get("Note") or rg.get("note"),
|
||
"ImportSource": rg.get("ImportSource") or rg.get("importSource") or filename,
|
||
"ExternalRef": ext_ref_scoped,
|
||
"RawImportData": rg.get("RawImportData") or rg.get("rawImportData") or json.dumps({"colorDescription": color_val if not color_variety_id else None}),
|
||
"OriginBreeder": raw_breeder or ("Zucht der kleinen Chaoten" if is_resident else None),
|
||
"NameSearch": normalize_name(name_val),
|
||
"CharacterTraits": traits,
|
||
"CharacterNote": char_note,
|
||
"IsDeaf": is_deaf,
|
||
"IsResident": is_resident,
|
||
"parentRefs": parent_refs,
|
||
"_photos": rg.get("photos", []),
|
||
"_old_scoped_litter_id": old_litter_id,
|
||
"_eff_dob": eff_dob_val,
|
||
"_birth_date": birth_date,
|
||
"_filename": filename,
|
||
"_old_id": rg.get("Id") or rg.get("id")
|
||
})
|
||
|
||
# Build the discard list for one stammbaum animal: rejected parent-ref
|
||
# candidates (pick_parent_ref) plus a manual DOB-remap decision, if any.
|
||
def _stammbaum_discards(a):
|
||
discards = list(a.get("_pick_discards", []))
|
||
remap = a.get("dobRemap")
|
||
if remap and remap.get("original") and remap.get("corrected"):
|
||
orig = _de_date(parse_date(remap["original"])) or remap["original"]
|
||
corr = _de_date(parse_date(remap["corrected"])) or remap["corrected"]
|
||
if orig != corr:
|
||
discards.append({
|
||
"label": PROV_FIELD_LABELS["DateOfBirth"],
|
||
"value": orig,
|
||
"reason": "per manueller Entscheidung korrigiert",
|
||
"replacement": corr,
|
||
})
|
||
return discards
|
||
|
||
# Map and append stammbaum animals to all_processed_gerbils
|
||
for a in stammbaum_only_animals:
|
||
a_id = a["id"]
|
||
name_val = get_normalized_gerbil_name(a["name"])
|
||
|
||
gender_val = str(a.get("gender") or "").lower().strip()
|
||
if gender_val in ["m", "male"]:
|
||
gender = "male"
|
||
elif gender_val in ["w", "f", "female"]:
|
||
gender = "female"
|
||
else:
|
||
gender = "unknown"
|
||
|
||
dob_val = parse_date(a.get("dob"))
|
||
dod_val = parse_date(a.get("death"))
|
||
|
||
status = "Breeding"
|
||
if dod_val:
|
||
status = "Deceased"
|
||
elif a_id not in stammbaum_resident_ids:
|
||
status = "GivenAway"
|
||
else:
|
||
if dob_val:
|
||
try:
|
||
dt_dob = datetime.strptime(dob_val, "%Y-%m-%d")
|
||
dt_now = datetime.now()
|
||
age_years = (dt_now - dt_dob).days / 365.25
|
||
if age_years >= 6.0:
|
||
status = "Deceased"
|
||
except Exception:
|
||
pass
|
||
|
||
# Color variety mapping
|
||
existing_gt = a["genotype"]["rawGenotype"] if a["genotype"]["rawGenotype"] else None
|
||
color_variety_id, genotype_val = resolve_color_and_genotype(
|
||
a.get("farbschlag"), existing_gt, variety_map, variety_genotypes
|
||
)
|
||
if not color_variety_id:
|
||
for fbv in a.get("farbschlagVariants", []):
|
||
cv_id, gt_val = resolve_color_and_genotype(fbv, existing_gt, variety_map, variety_genotypes)
|
||
if cv_id:
|
||
color_variety_id = cv_id
|
||
genotype_val = gt_val
|
||
break
|
||
|
||
# Contact mapping
|
||
origin_cid = None
|
||
b_name = (a.get("breeder") or "").strip()
|
||
z_name = (a.get("zucht") or "").strip()
|
||
if b_name:
|
||
origin_cid = generate_guid(f"stammbaum-contact-{normalize_name(b_name)}")
|
||
elif z_name:
|
||
origin_cid = generate_guid(f"stammbaum-contact-{normalize_name(z_name)}")
|
||
|
||
# Map to resolved global contact GUID
|
||
if origin_cid in contact_id_map:
|
||
origin_cid = contact_id_map[origin_cid]
|
||
else:
|
||
origin_cid = None
|
||
|
||
|
||
scoped_id = generate_guid(f"stammbaum-animal-{a_id}")
|
||
scoped_litter_id = a.get("_mapped_litter_scoped_id")
|
||
|
||
is_deaf = a["genotype"].get("deaf")
|
||
|
||
stammbaum_breeder = a.get("breeder") if a.get("breeder") else (a.get("zucht") if a.get("zucht") else None)
|
||
if stammbaum_breeder:
|
||
norm_b, keep_b = get_normalized_contact_name(stammbaum_breeder)
|
||
stammbaum_breeder = norm_b if keep_b else None
|
||
|
||
all_processed_gerbils.append({
|
||
"Id": scoped_id,
|
||
"Name": name_val,
|
||
"Gender": gender,
|
||
"Status": status,
|
||
"LitterId": scoped_litter_id, # mapped later in step 5
|
||
"OriginContactId": origin_cid,
|
||
"ReceiverContactId": None,
|
||
"EnclosureId": None,
|
||
"ColorVarietyId": color_variety_id,
|
||
"DateOfBirth": dob_val,
|
||
"DateOfDeath": dod_val,
|
||
"CauseOfDeath": "Duftdrüsen-Tumor" if name_val == "Ken'ichi" and dob_val == "2015-03-01" else None,
|
||
"GoHomeDate": None,
|
||
"Genotype": genotype_val,
|
||
"Notes": None,
|
||
"ImportSource": ", ".join(a.get("sourceFiles", [])),
|
||
"ExternalRef": f"stammbaum-{a_id}",
|
||
"RawImportData": json.dumps({
|
||
"rawGenotype": a["genotype"]["rawGenotype"],
|
||
"unmappedTokens": a["genotype"]["unmappedTokens"],
|
||
"breederText": a.get("breeder", "")
|
||
}, ensure_ascii=False),
|
||
"OriginBreeder": stammbaum_breeder,
|
||
"NameSearch": normalize_name(name_val),
|
||
"CharacterTraits": [],
|
||
"CharacterNote": None,
|
||
"IsDeaf": is_deaf,
|
||
"IsResident": a_id in stammbaum_resident_ids,
|
||
"parentRefs": a.get("parentRefs", []),
|
||
"_photos": a.get("photos", []),
|
||
"_conflict": bool(a.get("conflict")),
|
||
"_resolved_by_decision": bool(a.get("resolvedByDecision")),
|
||
"_old_scoped_litter_id": scoped_litter_id,
|
||
"_eff_dob": dob_val or "2010-01-01",
|
||
"_birth_date": dob_val,
|
||
"_filename": a.get("sourceFiles", ["Stammbaum"])[0],
|
||
"_old_id": a_id,
|
||
# Rejected Stammbaum parent-ref candidates for this animal (filled by
|
||
# pick_parent_ref above) — surfaced in this gerbil's discard history.
|
||
"_discarded": _stammbaum_discards(a),
|
||
})
|
||
|
||
# Map and append docx animals to all_processed_gerbils
|
||
for idx, da in enumerate(docx_animals):
|
||
name_val = get_normalized_gerbil_name(da["name"])
|
||
gender = da["gender"]
|
||
|
||
dob_val = parse_date(da.get("litterDob"))
|
||
dod_val = parse_date(da.get("deathDate"))
|
||
gohome_val = parse_date(da.get("abgabeDate"))
|
||
|
||
# Status precedence
|
||
status = "Breeding"
|
||
if dod_val:
|
||
status = "Deceased"
|
||
elif gohome_val or da.get("owner"):
|
||
status = "GivenAway"
|
||
|
||
# Color variety mapping
|
||
color_variety_id, genotype_val = resolve_color_and_genotype(
|
||
da.get("farbschlag"), None, variety_map, variety_genotypes
|
||
)
|
||
|
||
# Contact mapping (buyer)
|
||
receiver_cid = None
|
||
o_name = (da.get("owner") or "").strip()
|
||
if o_name:
|
||
receiver_cid = generate_guid(f"docx-contact-{normalize_name(o_name)}")
|
||
if receiver_cid in contact_id_map:
|
||
receiver_cid = contact_id_map[receiver_cid]
|
||
else:
|
||
receiver_cid = None
|
||
|
||
# Scoped ID
|
||
scoped_id = generate_guid(f"docx-animal-{idx}-{normalize_name(name_val)}-{dob_val or '0001-01-01'}")
|
||
|
||
# Litter ID mapping
|
||
ws_code = da.get("wsCode")
|
||
scoped_litter_id = docx_litter_id_map.get((ws_code, dob_val))
|
||
|
||
|
||
# Raw import details
|
||
raw_import_payload = json.dumps({
|
||
"abgabeWeight": da.get("abgabeWeight", ""),
|
||
"deathCause": da.get("deathCause", ""),
|
||
"partnerName": da.get("partnerName", ""),
|
||
"partnerDob": da.get("partnerDob", "")
|
||
}, ensure_ascii=False)
|
||
|
||
# Residents: if sold/given away, it's not a resident
|
||
is_resident = not bool(o_name)
|
||
|
||
parent_refs = []
|
||
if scoped_litter_id:
|
||
dl = next((l for l in docx_litters if docx_litter_id_map.get((l["wsCode"], parse_date(l["dob"]))) == scoped_litter_id), None)
|
||
if dl:
|
||
if dl.get("fatherName"):
|
||
parent_refs.append({"name": dl["fatherName"], "roleGuess": "father"})
|
||
if dl.get("motherName"):
|
||
parent_refs.append({"name": dl["motherName"], "roleGuess": "mother"})
|
||
|
||
all_processed_gerbils.append({
|
||
"Id": scoped_id,
|
||
"Name": name_val,
|
||
"Gender": gender,
|
||
"Status": status,
|
||
"LitterId": scoped_litter_id,
|
||
"OriginContactId": None,
|
||
"ReceiverContactId": receiver_cid, # mapped in step 5
|
||
"EnclosureId": None,
|
||
"ColorVarietyId": color_variety_id,
|
||
"DateOfBirth": dob_val,
|
||
"DateOfDeath": dod_val,
|
||
"CauseOfDeath": da.get("deathCause"),
|
||
"GoHomeDate": gohome_val,
|
||
"Genotype": genotype_val,
|
||
"Notes": None,
|
||
"ImportSource": "Wurfchronik-Detail.docx",
|
||
"ExternalRef": f"docx-{idx}-{normalize_name(name_val)}-{dob_val or '0001-01-01'}",
|
||
"RawImportData": raw_import_payload,
|
||
"OriginBreeder": "Zucht der kleinen Chaoten",
|
||
"NameSearch": normalize_name(name_val),
|
||
"CharacterTraits": [],
|
||
"CharacterNote": None,
|
||
"IsDeaf": None,
|
||
"IsResident": is_resident,
|
||
"parentRefs": parent_refs,
|
||
"_photos": da.get("photos", []),
|
||
"_old_scoped_litter_id": scoped_litter_id,
|
||
"_eff_dob": dob_val or "2020-01-01",
|
||
"_birth_date": dob_val,
|
||
"_filename": "Wurfchronik-Detail.docx",
|
||
"_old_id": name_val
|
||
})
|
||
|
||
# ── Conflict-decision overrides at the gerbil level ───────────────────────
|
||
# extract.apply_conflict_decisions already applies genotype/farbschlag/dod/
|
||
# gender/parent overrides to STAMMBAUM animals (animals.json). Wurfchronik/
|
||
# docx animals (docx_animals.json) never pass through that path, so a gender
|
||
# override on a Wurfchronik animal (e.g. „Roni“ — actually the FATHER) would
|
||
# otherwise be lost. Apply gender overrides here uniformly to every processed
|
||
# gerbil, materialise `addAnimals` stubs for known parents that have no own
|
||
# source record (e.g. „Fumi“), and collapse explicitly-paired duplicate
|
||
# records via `mergeExternalRefs` (nameless animals never auto-merge).
|
||
_decisions, _add_animals, _add_litters = load_conflict_decisions()
|
||
|
||
def _gerbil_call_key(g):
|
||
return normalize_name(get_call_name(g.get("Name") or ""))
|
||
|
||
# 1) gender overrides (match normalize(call-name) + ISO dob; empty decision
|
||
# dob = name-only, for ancestors without a birthdate).
|
||
_gender_overrides = {}
|
||
for d in _decisions:
|
||
gg = _norm_decision_gender(d.get("gender"))
|
||
if not gg:
|
||
continue
|
||
ck = normalize_name(get_call_name(d.get("name") or ""))
|
||
iso = parse_date(d.get("dob")) if d.get("dob") else ""
|
||
_gender_overrides[(ck, iso or "")] = gg
|
||
_gender_applied = 0
|
||
for g in all_processed_gerbils:
|
||
ck = _gerbil_call_key(g)
|
||
iso = g.get("DateOfBirth") or ""
|
||
gg = _gender_overrides.get((ck, iso)) or _gender_overrides.get((ck, ""))
|
||
if gg and g.get("Gender") != gg:
|
||
g["Gender"] = gg
|
||
g["_resolved_by_decision"] = True
|
||
_gender_applied += 1
|
||
|
||
# Resolve a contact NAME (e.g. an addAnimals `receiver` like „Ulrike Neu“) to
|
||
# its global contact id. Prefers an already-resolved contact; otherwise falls
|
||
# back to the deterministic id (same scheme as contact resolution above) so a
|
||
# receiver named only here still maps to the canonical id.
|
||
def _resolve_contact_id_by_name(name):
|
||
raw = (name or "").strip()
|
||
if not raw:
|
||
return None
|
||
canon, keep = get_normalized_contact_name(raw)
|
||
if not keep:
|
||
return None
|
||
norm = normalize_name(canon)
|
||
existing = contact_by_norm_name.get(norm)
|
||
if existing:
|
||
return existing["Id"]
|
||
cid = generate_guid(f"contact-{norm}")
|
||
new_c = {
|
||
"Id": cid,
|
||
"Name": canon,
|
||
"Phone": None,
|
||
"Email": None,
|
||
"Street": None,
|
||
"City": None,
|
||
"IsBreeder": False,
|
||
"IsReceiver": True,
|
||
"Notes": "Automatisch generiert durch Züchterin-Entscheidung (Receiver)"
|
||
}
|
||
contact_by_norm_name[norm] = new_c
|
||
return cid
|
||
|
||
|
||
# 2) addAnimals — materialise a stub for a KNOWN animal that has no own source
|
||
# record, so its litter's parent link can resolve (e.g. „Fumi“). Optional
|
||
# fields: gender, zucht, dob, isResident (bool), receiver (contact name →
|
||
# ReceiverContactId), notes (→ Notes), farbschlag/color (best-effort → Notes).
|
||
# Skip a bare stub when a record of that call-name already exists (avoid
|
||
# duplicates); but a deliberately-added RICH animal (one carrying isResident/
|
||
# receiver/notes/color, i.e. genuinely new data — not just a parent-link
|
||
# placeholder) is created even when the call-name collides with an unrelated
|
||
# foreign animal (e.g. the new resident „Merle“ vs the existing „Merle of
|
||
# Samsimar“). It gets its own deterministic id, so no real duplicate arises.
|
||
_existing_call_keys = {_gerbil_call_key(g) for g in all_processed_gerbils}
|
||
_RICH_FIELDS = ("isResident", "receiver", "notes", "farbschlag", "color")
|
||
_stubs_added = 0
|
||
for a in _add_animals:
|
||
nm = (a.get("name") or "").strip()
|
||
if not nm:
|
||
continue
|
||
ck = normalize_name(get_call_name(nm))
|
||
is_rich = any(a.get(k) is not None for k in _RICH_FIELDS)
|
||
if ck in _existing_call_keys and not is_rich:
|
||
continue
|
||
gender = _norm_decision_gender(a.get("gender")) or "unknown"
|
||
dob_iso = parse_date(a.get("dob")) if a.get("dob") else None
|
||
zucht = (a.get("zucht") or "").strip() or None
|
||
breeder_disp = None
|
||
if zucht:
|
||
nb, keep = get_normalized_contact_name(zucht)
|
||
breeder_disp = nb if keep else None
|
||
# Optional rich fields.
|
||
is_resident = bool(a.get("isResident")) if a.get("isResident") is not None else False
|
||
receiver_cid = _resolve_contact_id_by_name(a.get("receiver"))
|
||
notes_val = (a.get("notes") or "").strip() or None
|
||
color_txt = (a.get("farbschlag") or a.get("color") or "").strip() or None
|
||
# Best-effort colour: keep it human-readable in Notes (no genotype to map).
|
||
if color_txt:
|
||
color_line = f"Farbschlag: {color_txt}"
|
||
notes_val = f"{notes_val} | {color_line}" if notes_val else color_line
|
||
status = "GivenAway" if receiver_cid else "Breeding"
|
||
stub_id = generate_guid(f"decision-animal-{normalize_name(nm)}")
|
||
all_processed_gerbils.append({
|
||
"Id": stub_id,
|
||
"Name": get_normalized_gerbil_name(nm),
|
||
"Gender": gender,
|
||
"Status": status,
|
||
"LitterId": None,
|
||
"OriginContactId": None,
|
||
"ReceiverContactId": receiver_cid,
|
||
"EnclosureId": None,
|
||
"ColorVarietyId": None,
|
||
"DateOfBirth": dob_iso,
|
||
"DateOfDeath": None,
|
||
"CauseOfDeath": None,
|
||
"GoHomeDate": None,
|
||
"Genotype": None,
|
||
"Notes": notes_val,
|
||
"ImportSource": "conflict-decisions.json",
|
||
"ExternalRef": f"decision-{normalize_name(nm)}",
|
||
"RawImportData": json.dumps({"addedByDecision": True}, ensure_ascii=False),
|
||
"OriginBreeder": breeder_disp or ("Zucht der kleinen Chaoten" if is_resident else None),
|
||
"NameSearch": normalize_name(get_normalized_gerbil_name(nm)),
|
||
"CharacterTraits": [],
|
||
"CharacterNote": None,
|
||
"IsDeaf": None,
|
||
"IsResident": is_resident,
|
||
"parentRefs": [],
|
||
"_photos": [],
|
||
"_old_scoped_litter_id": None,
|
||
"_eff_dob": dob_iso or "2015-01-01",
|
||
"_birth_date": dob_iso,
|
||
"_filename": "conflict-decisions.json",
|
||
"_old_id": stub_id,
|
||
"_resolved_by_decision": True,
|
||
# Ein addAnimals-Stub mit explizitem isResident ist eine Mensch-
|
||
# Entscheidung → vom späteren Residenz-Sweep NICHT anfassen lassen
|
||
# (z. B. die resident-Stubs Merle/Pete aus Akanes Fremd-Wurf).
|
||
"_resident_override": a.get("isResident") is not None,
|
||
})
|
||
_existing_call_keys.add(ck)
|
||
_stubs_added += 1
|
||
|
||
# 3) mergeExternalRefs — collapse explicitly-paired duplicate records (esp.
|
||
# nameless animals that the name-based dedup keeps separate). Each pair is
|
||
# [keepRefSuffix, dropRefSuffix]; the drop record is removed and its old id
|
||
# mapped onto the keep record. Refs are matched as a suffix of ExternalRef
|
||
# (extract animals become ExternalRef "stammbaum-<id>").
|
||
_by_ext = {}
|
||
for g in all_processed_gerbils:
|
||
er = g.get("ExternalRef")
|
||
if er:
|
||
_by_ext.setdefault(er, []).append(g)
|
||
|
||
def _find_by_ref_suffix(suffix):
|
||
hits = [g for er, gs in _by_ext.items() if er and er.endswith(suffix) for g in gs]
|
||
return hits
|
||
|
||
_premerged_ids = {} # dropped old id -> kept id (consumed in step 5 mapping)
|
||
_merge_pairs_done = 0
|
||
for d in _decisions:
|
||
for pair in (d.get("mergeExternalRefs") or []):
|
||
if not isinstance(pair, (list, tuple)) or len(pair) != 2:
|
||
continue
|
||
keep_hits = _find_by_ref_suffix(pair[0])
|
||
drop_hits = _find_by_ref_suffix(pair[1])
|
||
if not keep_hits or not drop_hits:
|
||
continue
|
||
keep = keep_hits[0]
|
||
for drop in drop_hits:
|
||
if drop is keep:
|
||
continue
|
||
# Fill gaps on the keeper from the dropped twin, then remove it.
|
||
for fld in ("DateOfBirth", "DateOfDeath", "Genotype",
|
||
"ColorVarietyId", "LitterId", "_old_scoped_litter_id",
|
||
"OriginContactId"):
|
||
if not keep.get(fld) and drop.get(fld):
|
||
keep[fld] = drop[fld]
|
||
if keep.get("Gender") in (None, "unknown") and drop.get("Gender") not in (None, "unknown"):
|
||
keep["Gender"] = drop["Gender"]
|
||
_premerged_ids[drop["Id"]] = keep["Id"]
|
||
_merge_pairs_done += 1
|
||
|
||
if _premerged_ids:
|
||
all_processed_gerbils = [g for g in all_processed_gerbils
|
||
if g["Id"] not in _premerged_ids]
|
||
|
||
# Apply correctDob remappings from decisions BEFORE deduplication!
|
||
_dob_remap_pre_dedup = 0
|
||
for d in _decisions:
|
||
correct_dob = d.get("correctDob")
|
||
if correct_dob:
|
||
er = (d.get("externalRef") or "").strip()
|
||
nm = d.get("name")
|
||
dob_iso = parse_date(correct_dob)
|
||
if dob_iso:
|
||
# Find matching raw gerbils
|
||
for g in all_processed_gerbils:
|
||
match = False
|
||
g_er = g.get("ExternalRef") or ""
|
||
if er and g_er == er:
|
||
match = True
|
||
elif er and g_er.endswith(er):
|
||
match = True
|
||
elif nm:
|
||
g_ck = normalize_name(get_call_name(g.get("Name") or ""))
|
||
d_ck = normalize_name(get_call_name(nm))
|
||
g_dob = g.get("DateOfBirth") or ""
|
||
d_dob = parse_date(d.get("dob")) if d.get("dob") else ""
|
||
if g_ck == d_ck and (not d_dob or g_dob == d_dob):
|
||
match = True
|
||
|
||
if match:
|
||
g["DateOfBirth"] = dob_iso
|
||
g["_birth_date"] = dob_iso
|
||
g["_eff_dob"] = dob_iso
|
||
_dob_remap_pre_dedup += 1
|
||
if _dob_remap_pre_dedup:
|
||
print(f" Pre-Dedup DOB Remappings: {_dob_remap_pre_dedup}")
|
||
|
||
print(f" Entscheidungs-Overrides: Geschlecht={_gender_applied}, "
|
||
f"Stub-Tiere={_stubs_added}, ExternalRef-Merges={_merge_pairs_done}.")
|
||
|
||
# Build parenting dates lookup using old scoped IDs
|
||
parent_litter_dates = {}
|
||
|
||
for l in resolved_litters:
|
||
ld = l["Date"]
|
||
if ld:
|
||
for pid in [l["FatherId"], l["MotherId"]]:
|
||
if pid:
|
||
parent_litter_dates.setdefault(pid, []).append(ld)
|
||
|
||
def _is_empty_shell(g):
|
||
"""A same-name record carrying no birthdate AND no own parent refs — a
|
||
DOB-less Stammbaum mention (e.g. Hagrid Rubeus appearing as a bare
|
||
ancestor box). Such shells must fold into the DOB-/parent-bearing record
|
||
of the same name even if a stray chart placed them as a parent of an
|
||
age-incompatible litter (ticket #18). Their own parent-attributions are
|
||
unreliable, so the parenting-date guard must not keep them separate."""
|
||
return not g.get("_birth_date") and not (g.get("parentRefs") or [])
|
||
|
||
def are_compatible(g1, g2):
|
||
# Same-name empty shell ↔ real record: always merge (see _is_empty_shell).
|
||
if _is_empty_shell(g1) or _is_empty_shell(g2):
|
||
if g1["Gender"] == "unknown" or g2["Gender"] == "unknown" \
|
||
or g1["Gender"] == g2["Gender"]:
|
||
return True
|
||
|
||
# Must have same gender (or one unknown)
|
||
if g1["Gender"] != "unknown" and g2["Gender"] != "unknown" and g1["Gender"] != g2["Gender"]:
|
||
bd1 = g1.get("_birth_date")
|
||
bd2 = g2.get("_birth_date")
|
||
if not (bd1 and bd2 and bd1 == bd2):
|
||
return False
|
||
|
||
bd1 = g1.get("_birth_date")
|
||
bd2 = g2.get("_birth_date")
|
||
|
||
# New rule: if name and parents match, they are compatible regardless of DOB!
|
||
p1 = g1.get("parentRefs", [])
|
||
p2 = g2.get("parentRefs", [])
|
||
f1 = next((p["name"] for p in p1 if p.get("roleGuess") == "father"), "")
|
||
m1 = next((p["name"] for p in p1 if p.get("roleGuess") == "mother"), "")
|
||
f2 = next((p["name"] for p in p2 if p.get("roleGuess") == "father"), "")
|
||
m2 = next((p["name"] for p in p2 if p.get("roleGuess") == "mother"), "")
|
||
|
||
parents_match = False
|
||
if f1 and f2 and m1 and m2:
|
||
if normalize_name(f1) == normalize_name(f2) and normalize_name(m1) == normalize_name(m2):
|
||
parents_match = True
|
||
|
||
if not parents_match:
|
||
# If both have explicit birth dates, they must match within 30 days
|
||
if bd1 and bd2:
|
||
days1 = date_to_days(bd1)
|
||
days2 = date_to_days(bd2)
|
||
if days1 is not None and days2 is not None:
|
||
if abs(days1 - days2) > 30:
|
||
return False
|
||
|
||
# If g1 has birth date, and g2 has parenting dates, birth date must be before parenting dates
|
||
p_dates2 = parent_litter_dates.get(g2["Id"], [])
|
||
if bd1:
|
||
for pd in p_dates2:
|
||
if pd <= bd1: # Can't have litter before or on birth date
|
||
return False
|
||
|
||
p_dates1 = parent_litter_dates.get(g1["Id"], [])
|
||
if bd2:
|
||
for pd in p_dates1:
|
||
if pd <= bd2:
|
||
return False
|
||
|
||
return True
|
||
|
||
def build_provenance(records, best_g, extra_notes=None, field_source=None,
|
||
field_discards=None):
|
||
"""Aggregate data-provenance across every raw record merged into one
|
||
resolved gerbil. Returns a JSON string (stored on the Gerbil entity as a
|
||
nullable text column) so the Rennmausakte can show where the entry came
|
||
from. `records` is the list of raw records that became this gerbil;
|
||
`best_g` is the chosen primary record. `field_source` maps a significant
|
||
field name (DateOfBirth/Genotype/…) to the raw record that supplied its
|
||
final value, so the history can name the exact file for each fact."""
|
||
source_files = set()
|
||
from_wurfchronik = False
|
||
any_conflict = False
|
||
any_decision = False
|
||
for r in records:
|
||
source_files |= _record_source_files(r)
|
||
for f in _record_source_files(r):
|
||
if "wurfchronik" in f.lower():
|
||
from_wurfchronik = True
|
||
if r.get("_conflict"):
|
||
any_conflict = True
|
||
if r.get("_resolved_by_decision"):
|
||
any_decision = True
|
||
|
||
notes = []
|
||
merged_count = len(records)
|
||
if merged_count > 1:
|
||
notes.append(f"aus {merged_count} Datensätzen zusammengeführt")
|
||
if any_decision:
|
||
notes.append("per manueller Entscheidung zugeordnet")
|
||
if any_conflict:
|
||
notes.append("Konflikt per Entscheidung gelöst")
|
||
|
||
# Parent derivation: surface the strongest parentRef method/confidence
|
||
# the primary record carries (chart-position etc.).
|
||
parent_method = None
|
||
parent_confidence = None
|
||
for ref in best_g.get("parentRefs", []) or []:
|
||
if ref.get("method") and not parent_method:
|
||
parent_method = ref.get("method")
|
||
if ref.get("confidence") and not parent_confidence:
|
||
parent_confidence = ref.get("confidence")
|
||
|
||
if extra_notes:
|
||
for n in extra_notes:
|
||
if n and n not in notes:
|
||
notes.append(n)
|
||
|
||
history = _build_gerbil_history(
|
||
records, best_g, field_source or {},
|
||
parent_method=parent_method,
|
||
any_decision=any_decision,
|
||
any_conflict=any_conflict,
|
||
conflict_notes=extra_notes or [],
|
||
field_discards=field_discards,
|
||
parent_discards=best_g.get("_discarded"),
|
||
)
|
||
|
||
return build_entity_provenance(
|
||
source_files,
|
||
merged_count,
|
||
notes=notes,
|
||
from_wurfchronik=from_wurfchronik,
|
||
extra={"parentMethod": parent_method, "parentConfidence": parent_confidence},
|
||
history=history,
|
||
)
|
||
|
||
# Group gerbils by name to perform deduplication
|
||
gerbil_groups = {}
|
||
for g in all_processed_gerbils:
|
||
name_key = get_dedup_name_key(g["Name"])
|
||
if not name_key:
|
||
name_key = "unbekannt"
|
||
gerbil_groups.setdefault(name_key, []).append(g)
|
||
|
||
resolved_gerbils = []
|
||
gerbil_id_map = {} # old_scoped_id -> final_id
|
||
|
||
color_keys = set(variety_map.keys())
|
||
|
||
for name_key, group in gerbil_groups.items():
|
||
is_placeholder = (
|
||
name_key in color_keys or
|
||
any(p in name_key for p in ["unbekannt", "unbenannt", "baby", "welpe", "jungtier", "unknown", "welpen"]) or
|
||
len(name_key) <= 2
|
||
)
|
||
|
||
if is_placeholder:
|
||
# Placeholders: do NOT merge, keep all separate
|
||
for g in group:
|
||
# Provenance is built in a final pass (after parent resolution),
|
||
# so parent-link discards land in the right gerbil's history.
|
||
g["_prov_args"] = ([g], g, None, None, None)
|
||
resolved_gerbils.append(g)
|
||
gerbil_id_map[g["Id"]] = g["Id"]
|
||
continue
|
||
|
||
# Partition group into compatible subsets
|
||
sub_groups = []
|
||
for g in group:
|
||
placed = False
|
||
for sub in sub_groups:
|
||
if all(are_compatible(g, member) for member in sub):
|
||
sub.append(g)
|
||
placed = True
|
||
break
|
||
if not placed:
|
||
sub_groups.append([g])
|
||
|
||
# Merge each partition sub-group into a single gerbil
|
||
for sub in sub_groups:
|
||
if len(sub) == 1:
|
||
g = sub[0]
|
||
g["_prov_args"] = ([g], g, None, None, None)
|
||
resolved_gerbils.append(g)
|
||
gerbil_id_map[g["Id"]] = g["Id"]
|
||
continue
|
||
|
||
# Find the best primary record to merge into
|
||
best_g = None
|
||
best_score = -1
|
||
for g in sub:
|
||
score = 0
|
||
if g["_old_scoped_litter_id"]: score += 10
|
||
if g["DateOfBirth"]: score += 5
|
||
if g["Genotype"]: score += 3
|
||
if g["ColorVarietyId"]: score += 2
|
||
if g["ImportSource"] and "stammbaum" in g["ImportSource"].lower(): score += 20
|
||
if g["Notes"] and not any(kw in g["Notes"].lower() for kw in ["parent", "mutter", "vater", "dam", "sire"]): score += 1
|
||
if len(g["Name"]) > len(name_key) + 5: # likely has clan suffix
|
||
score += 15
|
||
if score > best_score:
|
||
best_score = score
|
||
best_g = g
|
||
|
||
# Merge fields
|
||
merged_notes = []
|
||
if best_g["Notes"]:
|
||
merged_notes.append(best_g["Notes"])
|
||
|
||
# Carry over discard records (rejected parent-refs / dob remaps) from
|
||
# every merged record so none are lost when the primary changes.
|
||
merged_discards = list(best_g.get("_discarded") or [])
|
||
for g in sub:
|
||
if g is best_g:
|
||
continue
|
||
for d in (g.get("_discarded") or []):
|
||
if d not in merged_discards:
|
||
merged_discards.append(d)
|
||
best_g["_discarded"] = merged_discards
|
||
|
||
# Merge photos
|
||
merged_photos = list(best_g.get("_photos", []))
|
||
|
||
# Track sources for debugging
|
||
sources = [best_g["_filename"]]
|
||
|
||
# Per-field file attribution: which raw record supplied each final
|
||
# field value. Seed with best_g for every field it already carries;
|
||
# the fill loop and voting loop update it as winners change.
|
||
ATTRIB_FIELDS = ["DateOfBirth", "DateOfDeath", "Gender", "Genotype",
|
||
"ColorVarietyId", "Name"]
|
||
field_source = {}
|
||
for fld in ATTRIB_FIELDS:
|
||
v = best_g.get(fld)
|
||
if v and v != "unknown":
|
||
field_source[fld] = best_g
|
||
|
||
for g in sub:
|
||
if g == best_g:
|
||
continue
|
||
gerbil_id_map[g["Id"]] = best_g["Id"]
|
||
sources.append(g["_filename"])
|
||
|
||
for ph in g.get("_photos", []):
|
||
if ph not in merged_photos:
|
||
merged_photos.append(ph)
|
||
|
||
if not best_g.get("_old_scoped_litter_id") and g.get("_old_scoped_litter_id"):
|
||
best_g["_old_scoped_litter_id"] = g["_old_scoped_litter_id"]
|
||
if not best_g["LitterId"] and g["LitterId"]:
|
||
best_g["LitterId"] = g["LitterId"]
|
||
if not best_g["DateOfBirth"] and g["DateOfBirth"]:
|
||
best_g["DateOfBirth"] = g["DateOfBirth"]
|
||
field_source["DateOfBirth"] = g
|
||
if not best_g["DateOfDeath"] and g["DateOfDeath"]:
|
||
best_g["DateOfDeath"] = g["DateOfDeath"]
|
||
field_source["DateOfDeath"] = g
|
||
if not best_g["CauseOfDeath"] and g["CauseOfDeath"]:
|
||
best_g["CauseOfDeath"] = g["CauseOfDeath"]
|
||
if not best_g["GoHomeDate"] and g["GoHomeDate"]:
|
||
best_g["GoHomeDate"] = g["GoHomeDate"]
|
||
if not best_g["Genotype"] and g["Genotype"]:
|
||
best_g["Genotype"] = g["Genotype"]
|
||
field_source["Genotype"] = g
|
||
if not best_g["ColorVarietyId"] and g["ColorVarietyId"]:
|
||
best_g["ColorVarietyId"] = g["ColorVarietyId"]
|
||
field_source["ColorVarietyId"] = g
|
||
if not best_g["OriginContactId"] and g["OriginContactId"]:
|
||
best_g["OriginContactId"] = g["OriginContactId"]
|
||
if not best_g["ReceiverContactId"] and g["ReceiverContactId"]:
|
||
best_g["ReceiverContactId"] = g["ReceiverContactId"]
|
||
if g["IsResident"]:
|
||
best_g["IsResident"] = True
|
||
|
||
# Reconcile Gender: prefer a known gender over unknown, and prefer stammbaum over other sources
|
||
if best_g["Gender"] == "unknown" and g["Gender"] != "unknown":
|
||
best_g["Gender"] = g["Gender"]
|
||
field_source["Gender"] = g
|
||
elif best_g["Gender"] != "unknown" and g["Gender"] != "unknown" and best_g["Gender"] != g["Gender"]:
|
||
if g["ImportSource"] and "stammbaum" in g["ImportSource"].lower():
|
||
if not best_g["ImportSource"] or "stammbaum" not in best_g["ImportSource"].lower():
|
||
best_g["Gender"] = g["Gender"]
|
||
field_source["Gender"] = g
|
||
|
||
# Status precedence: Deceased > GivenAway > Breeding/Pet
|
||
if g["Status"] == "Deceased":
|
||
best_g["Status"] = "Deceased"
|
||
elif g["Status"] == "GivenAway" and best_g["Status"] not in ["Deceased"]:
|
||
best_g["Status"] = "GivenAway"
|
||
|
||
if g["Notes"] and g["Notes"] not in merged_notes:
|
||
# Ignore redundant dummy notes
|
||
if not any(kw in g["Notes"].lower() for kw in ["parent listed", "mutter von", "vater von", "dam of", "sire of"]):
|
||
merged_notes.append(g["Notes"])
|
||
|
||
# Display formatter per field for discard/replacement lines.
|
||
def _disp(field, val):
|
||
if val is None or val == "" or val == "unknown":
|
||
return None
|
||
if field in ("DateOfBirth", "DateOfDeath"):
|
||
return _de_date(val)
|
||
if field == "Gender":
|
||
return _de_gender(val)
|
||
if field == "ColorVarietyId":
|
||
return variety_id_to_name.get(val, "Farbschlag")
|
||
return str(val)
|
||
|
||
# Reconcile fields based on number of source files supporting them
|
||
conflict_notes = []
|
||
field_discards = []
|
||
for field in ["DateOfBirth", "DateOfDeath", "Gender", "Genotype", "ColorVarietyId"]:
|
||
votes = {}
|
||
for g in sub:
|
||
val = g.get(field)
|
||
if val and val != "unknown":
|
||
# count source files
|
||
sources_count = len(str(g.get("ImportSource") or "").split(","))
|
||
votes[val] = votes.get(val, 0) + sources_count
|
||
if votes:
|
||
best_val = max(votes, key=votes.get)
|
||
# If the records disagreed on a field, the merge had to pick a
|
||
# winner — record that as a provenance note.
|
||
if len(votes) > 1:
|
||
conflict_notes.append(
|
||
f"Konflikt bei {PROV_FIELD_LABELS[field]} per Mehrheitsentscheidung gelöst"
|
||
)
|
||
best_g[field] = best_val
|
||
# Attribute the winning value to a record that actually holds
|
||
# it, so the history names the right file.
|
||
winner = next((g for g in sub if g.get(field) == best_val), None)
|
||
if winner is not None:
|
||
field_source[field] = winner
|
||
# Record each LOSING value: which file it came from, that it
|
||
# was discarded as differing, and that the majority value (and
|
||
# its file) was used instead.
|
||
if len(votes) > 1:
|
||
repl_disp = _disp(field, best_val)
|
||
repl_file = _primary_file_of(winner) if winner is not None else None
|
||
for lose_val in votes:
|
||
if lose_val == best_val:
|
||
continue
|
||
loser = next((g for g in sub if g.get(field) == lose_val), None)
|
||
field_discards.append({
|
||
"label": PROV_FIELD_LABELS.get(field, field),
|
||
"value": _disp(field, lose_val),
|
||
"file": _primary_file_of(loser) if loser is not None else None,
|
||
"reason": "abweichend",
|
||
"replacement": repl_disp,
|
||
"repl_file": repl_file,
|
||
"replacement_note": "Mehrheit",
|
||
})
|
||
# Keep helper fields in sync if we changed DateOfBirth
|
||
if field == "DateOfBirth":
|
||
best_g["_birth_date"] = best_val
|
||
best_g["_eff_dob"] = best_val or "2010-01-01"
|
||
|
||
if merged_notes:
|
||
best_g["Notes"] = " | ".join(merged_notes)
|
||
|
||
best_g["_photos"] = merged_photos
|
||
# Defer provenance to a final pass so parent-link discards (added
|
||
# later by the parent-age / role-normalization passes) are included.
|
||
best_g["_prov_args"] = (sub, best_g, conflict_notes, field_source, field_discards)
|
||
|
||
# Print merge trace
|
||
print(f"Deduplicated same-animal name '{best_g['Name']}': merged {len(sub)} entries across files: {', '.join(sources)}")
|
||
|
||
resolved_gerbils.append(best_g)
|
||
gerbil_id_map[best_g["Id"]] = best_g["Id"]
|
||
|
||
print(f"Deduplicated to {len(resolved_gerbils)} unique gerbil records.")
|
||
|
||
# ── isResident/notes/receiver-Overrides (Mensch-Entscheidung) ─────────────
|
||
# NACH dem Dedup anwenden, sonst würde die Dedup-Zusammenführung (IsResident=True
|
||
# falls eine Variante resident ist) sie wieder überschreiben. Auch NACH der
|
||
# Residenz-Propagation (die nur auf Stammbaum-Roh-Tiere wirkt) → diese Overrides
|
||
# sind das letzte Wort. Matcht primär über `externalRef` (präzise, auch für
|
||
# namenlose Tiere) oder sonst über normalize(call-name)+ISO-dob (leere dob =
|
||
# name-only, für Vorfahren ohne Datum). Ein Match kann zusätzlich `notes` (→
|
||
# Notes, autoritativ) und `receiver` (Kontaktname → ReceiverContactId, Status=
|
||
# GivenAway) setzen — so bekommt ein bereits existierendes Tier (z. B. Akane)
|
||
# seinen Lebenslauf + Abnehmer, ohne ein addAnimals-Stub zu sein.
|
||
_ovr_by_extref = {}
|
||
_ovr_by_namedob = {}
|
||
for d in _decisions:
|
||
has_ovr = (("isResident" in d) or d.get("notes") or d.get("receiver")
|
||
or d.get("renameTo") or d.get("deceased") or d.get("dateOfDeath")
|
||
or d.get("correctDob") or d.get("genotype") or d.get("farbschlag")
|
||
or d.get("gender") or d.get("father") or d.get("mother")
|
||
or d.get("originBreeder"))
|
||
if not has_ovr:
|
||
continue
|
||
er = (d.get("externalRef") or "").strip()
|
||
if er:
|
||
_ovr_by_extref[er] = d
|
||
nm = d.get("name")
|
||
if nm is not None:
|
||
ck = normalize_name(get_call_name(nm or ""))
|
||
iso = parse_date(d.get("dob")) if d.get("dob") else ""
|
||
_ovr_by_namedob[(ck, iso or "")] = d
|
||
_isres_applied = 0
|
||
if _ovr_by_extref or _ovr_by_namedob:
|
||
for g in resolved_gerbils:
|
||
er = g.get("ExternalRef") or ""
|
||
ck = normalize_name(get_call_name(g.get("Name") or ""))
|
||
iso = g.get("DateOfBirth") or ""
|
||
if er and er in _ovr_by_extref:
|
||
d = _ovr_by_extref[er]
|
||
elif er and any(er.endswith(k) for k in _ovr_by_extref):
|
||
d = next(v for k, v in _ovr_by_extref.items() if er.endswith(k))
|
||
elif (ck, iso) in _ovr_by_namedob:
|
||
d = _ovr_by_namedob[(ck, iso)]
|
||
elif (ck, "") in _ovr_by_namedob:
|
||
d = _ovr_by_namedob[(ck, "")]
|
||
else:
|
||
continue
|
||
applied = False
|
||
if "originBreeder" in d:
|
||
val = d["originBreeder"]
|
||
if g.get("OriginBreeder") != val:
|
||
g["OriginBreeder"] = val
|
||
applied = True
|
||
if "isResident" in d:
|
||
val = bool(d["isResident"])
|
||
# Ein expliziter Override (true ODER false) ist das letzte Wort und
|
||
# darf vom späteren Residenz-Sweep NICHT überschrieben werden.
|
||
g["_resident_override"] = True
|
||
if g.get("IsResident") != val:
|
||
g["IsResident"] = val
|
||
# Ein Nicht-Bestandstier ist kein eigenes Zuchttier → den eigenen
|
||
# Zucht-Breeder entfernen (nur, wenn er auf die eigene Zucht zeigt).
|
||
if not val and g.get("OriginBreeder") == "Zucht der kleinen Chaoten":
|
||
g["OriginBreeder"] = None
|
||
applied = True
|
||
notes_ovr = (d.get("notes") or "").strip()
|
||
if notes_ovr and g.get("Notes") != notes_ovr:
|
||
g["Notes"] = notes_ovr
|
||
applied = True
|
||
rec_name = d.get("receiver")
|
||
if rec_name:
|
||
rec_cid = _resolve_contact_id_by_name(rec_name)
|
||
if rec_cid and g.get("ReceiverContactId") != rec_cid:
|
||
g["ReceiverContactId"] = rec_cid
|
||
if g.get("Status") not in ("Deceased",):
|
||
g["Status"] = "GivenAway"
|
||
applied = True
|
||
# renameTo: korrigiert den Anzeigenamen eines bestehenden Tieres. Der
|
||
# Match-Key bleibt der ursprüngliche name/externalRef (NICHT umdeuten) —
|
||
# erst HIER wird der Name überschrieben.
|
||
rename_to = (d.get("renameTo") or "").strip()
|
||
if rename_to and g.get("Name") != rename_to:
|
||
# Quell-Namen merken, damit spätere Eltern-Verknüpfungen, die noch den
|
||
# alten Namen referenzieren, weiter auflösen (siehe Namens-Index unten,
|
||
# Ticket e0a0c304 Kruke→Kuke: sonst verlieren Kukes Würfe die Mutter).
|
||
g["_pre_rename_name"] = g.get("Name")
|
||
g["Name"] = rename_to
|
||
applied = True
|
||
# deceased: markiert ein bestehendes Tier als verstorben, falls noch
|
||
# nicht. dateOfDeath wird bewusst NICHT angefasst.
|
||
if d.get("deceased") and g.get("Status") != "Deceased":
|
||
g["Status"] = "Deceased"
|
||
applied = True
|
||
# dateOfDeath: setzt das Todesdatum (falls noch keins) + Status Deceased.
|
||
# rpro3 hat keine Sterbespalte, daher gibt die Züchterin/das Ticket das Datum vor.
|
||
if d.get("dateOfDeath") and not g.get("DateOfDeath"):
|
||
_dod = parse_date(d["dateOfDeath"])
|
||
if _dod:
|
||
g["DateOfDeath"] = _dod
|
||
g["Status"] = "Deceased"
|
||
applied = True
|
||
if d.get("correctDob") and not g.get("DateOfBirth"):
|
||
dob_iso = parse_date(d["correctDob"])
|
||
g["DateOfBirth"] = dob_iso
|
||
g["_birth_date"] = dob_iso
|
||
g["_eff_dob"] = dob_iso
|
||
applied = True
|
||
if d.get("genotype") and not g.get("Genotype"):
|
||
g["Genotype"] = d["genotype"].strip()
|
||
applied = True
|
||
if d.get("farbschlag") and not g.get("ColorVarietyId"):
|
||
cv_id = variety_map.get(d["farbschlag"].strip().lower())
|
||
if cv_id:
|
||
g["ColorVarietyId"] = cv_id
|
||
applied = True
|
||
if d.get("gender") and g.get("Gender") in ("unknown", None):
|
||
gender = d["gender"].strip().lower()
|
||
gender = {"m": "male", "männlich": "male", "w": "female",
|
||
"f": "female", "weiblich": "female"}.get(gender, gender)
|
||
if gender in ("male", "female"):
|
||
g["Gender"] = gender
|
||
applied = True
|
||
if d.get("father"):
|
||
g["_override_father_name"] = d["father"].strip()
|
||
g["_override_father_dob"] = d.get("fatherDob")
|
||
applied = True
|
||
if d.get("mother"):
|
||
g["_override_mother_name"] = d["mother"].strip()
|
||
g["_override_mother_dob"] = d.get("motherDob")
|
||
applied = True
|
||
if applied:
|
||
_isres_applied += 1
|
||
if _isres_applied:
|
||
print(f"isResident/notes/receiver/dob/geno-Overrides angewandt: {_isres_applied}")
|
||
|
||
|
||
|
||
# Apply age-based death threshold (6.0 years) to all resolved gerbils
|
||
dt_now = datetime.now()
|
||
for g in resolved_gerbils:
|
||
if g.get("Status") != "Deceased" and g.get("Status") != "GivenAway":
|
||
if not g.get("DateOfDeath") and not g.get("ReceiverContactId"):
|
||
dob_str = g.get("DateOfBirth")
|
||
if dob_str:
|
||
try:
|
||
dt_dob = datetime.strptime(dob_str, "%Y-%m-%d")
|
||
age_years = (dt_now - dt_dob).days / 365.25
|
||
if age_years >= 6.0:
|
||
g["Status"] = "Deceased"
|
||
except Exception:
|
||
pass
|
||
|
||
# 5. Map Gerbils to Litters
|
||
for g in resolved_gerbils:
|
||
old_lid = g["_old_scoped_litter_id"]
|
||
l_guid = litter_id_map.get(old_lid)
|
||
g["LitterId"] = l_guid
|
||
|
||
if l_guid and l_guid in litter_by_scoped_id:
|
||
l = litter_by_scoped_id[l_guid]
|
||
l_date = l.get("Date")
|
||
if l_date:
|
||
if not g.get("DateOfBirth"):
|
||
g["DateOfBirth"] = l_date
|
||
g["_birth_date"] = l_date
|
||
g["_eff_dob"] = l_date
|
||
|
||
# Apply age-based death threshold (6.0 years) to all resolved gerbils (including newly backfilled ones)
|
||
if g.get("Status") != "Deceased" and g.get("Status") != "GivenAway":
|
||
if not g.get("DateOfDeath") and not g.get("ReceiverContactId"):
|
||
dob_str = g.get("DateOfBirth")
|
||
if dob_str:
|
||
try:
|
||
dt_dob = datetime.strptime(dob_str, "%Y-%m-%d")
|
||
age_years = (dt_now - dt_dob).days / 365.25
|
||
if age_years >= 6.0:
|
||
g["Status"] = "Deceased"
|
||
except Exception:
|
||
pass
|
||
|
||
# Clean helper fields
|
||
del g["_old_scoped_litter_id"]
|
||
del g["_eff_dob"]
|
||
if "_birth_date" in g:
|
||
del g["_birth_date"]
|
||
del g["_filename"]
|
||
del g["_old_id"]
|
||
g.pop("_conflict", None)
|
||
g.pop("_resolved_by_decision", None)
|
||
|
||
# ── CREATE VIRTUAL LITTERS FOR PARENT OVERRIDES ──
|
||
_virtual_overrides_created = 0
|
||
for g in resolved_gerbils:
|
||
f_name = g.get("_override_father_name")
|
||
m_name = g.get("_override_mother_name")
|
||
if (f_name or m_name) and not g.get("LitterId"):
|
||
litter_id = generate_guid(f"override-virtual-litter-{g['Id']}")
|
||
g["LitterId"] = litter_id
|
||
resolved_litters.append({
|
||
"Id": litter_id,
|
||
"Name": f"Wurf von {f_name or '—'} + {m_name or '—'}",
|
||
"Date": g.get("DateOfBirth"),
|
||
"TotalBorn": 1,
|
||
"DeathsWithin8Weeks": 0,
|
||
"FatherId": None,
|
||
"MotherId": None,
|
||
"Notes": f"Virtueller Wurf für {g['Name']} (über Decisions-Override angelegt).",
|
||
"LitterLetter": None,
|
||
"ExternalRef": f"override-virtual-litter-{g['Id']}",
|
||
"_father_name": f_name or "",
|
||
"_mother_name": m_name or "",
|
||
"_filename": "conflict-decisions.json",
|
||
"_source_files": {"conflict-decisions.json"},
|
||
"_merged_count": 1,
|
||
"_virtual": True
|
||
})
|
||
_virtual_overrides_created += 1
|
||
if _virtual_overrides_created:
|
||
print(f"Mensch-Entscheidungs-Eltern-Overrides: {_virtual_overrides_created} virtuelle Würfe erstellt.")
|
||
|
||
# Gather final valid gerbil IDs
|
||
valid_gerbil_ids = {g["Id"] for g in resolved_gerbils}
|
||
|
||
# Create name lookup for resolved gerbils
|
||
gerbil_by_norm_name = {}
|
||
for g in resolved_gerbils:
|
||
keys = set()
|
||
n_key = normalize_name(g["Name"])
|
||
keys.add(n_key)
|
||
# Also index by call-name to resolve parents who are only listed by call-name
|
||
keys.add(normalize_name(get_call_name(g["Name"])))
|
||
# …and by the connector-folding canon key so abbreviation variants match
|
||
# (e.g. „BlackFire v.d. Kleinen Chaoten“ vs „… von den …“ — ticket #30).
|
||
keys.add(canon_name_key(g["Name"]))
|
||
# Umbenannte Tiere (renameTo) auch unter ihrem Quell-Namen indexieren, damit
|
||
# Eltern-Verknüpfungen, die noch den alten Namen referenzieren, weiter auflösen.
|
||
pre = g.get("_pre_rename_name")
|
||
if pre:
|
||
keys.add(normalize_name(pre))
|
||
keys.add(normalize_name(get_call_name(pre)))
|
||
keys.add(canon_name_key(pre))
|
||
for k in keys:
|
||
if k:
|
||
gerbil_by_norm_name.setdefault(k, []).append(g)
|
||
|
||
# Map raw Guid if present (convert if old_id mapped to new_guid)
|
||
for l in resolved_litters:
|
||
# First redirect any parent pointing at a record dropped by an explicit
|
||
# mergeExternalRefs pre-merge onto its surviving twin (then through the
|
||
# normal id map), so the two collapsed litters share the same parents.
|
||
if l["FatherId"] in _premerged_ids:
|
||
l["FatherId"] = _premerged_ids[l["FatherId"]]
|
||
if l["MotherId"] in _premerged_ids:
|
||
l["MotherId"] = _premerged_ids[l["MotherId"]]
|
||
if l["FatherId"] in gerbil_id_map:
|
||
l["FatherId"] = gerbil_id_map[l["FatherId"]]
|
||
if l["MotherId"] in gerbil_id_map:
|
||
l["MotherId"] = gerbil_id_map[l["MotherId"]]
|
||
|
||
# Clean foreign keys that do not point to a valid gerbil
|
||
if l["FatherId"] and l["FatherId"] not in valid_gerbil_ids:
|
||
l["FatherId"] = None
|
||
if l["MotherId"] and l["MotherId"] not in valid_gerbil_ids:
|
||
l["MotherId"] = None
|
||
|
||
# Parent Resolver (Global Name Matching)
|
||
resolved_fathers = 0
|
||
resolved_mothers = 0
|
||
|
||
gerbil_by_id_final = {g["Id"]: g for g in resolved_gerbils}
|
||
|
||
def _final_gender(gid):
|
||
g = gerbil_by_id_final.get(gid)
|
||
return g["Gender"] if g else None
|
||
|
||
def _resolve_name(name, prefer_gender, litter_date):
|
||
"""Resolve a parent name to the best matching final gerbil.
|
||
|
||
Gender is a PREFERENCE, not a hard filter: a reversed parent (e.g. a
|
||
female listed in the father position, as the Stammbaum often does) still
|
||
resolves to a gerbil — the role is corrected afterwards by gender. This
|
||
is what previously left FatherId/MotherId null (the candidate was
|
||
filtered out for having the "wrong" gender for its slot).
|
||
"""
|
||
if not name:
|
||
return None
|
||
# Try the exact normalized key, then the connector-folding canon key and
|
||
# the call-name (handles v.d. ↔ von den abbreviation variants — #30).
|
||
lookups = []
|
||
for k in (normalize_name(name), canon_name_key(name),
|
||
normalize_name(get_call_name(name))):
|
||
if k and k not in lookups:
|
||
lookups.append(k)
|
||
# Resolve key-by-key so an EXACT full-name match (lookups[0]) wins over a
|
||
# mere call-name/canon fallback — e.g. override mother „Elena“ must pick
|
||
# the resident „Elena“, not „Elena of KK Chaos“ whose call-name is also
|
||
# „Elena“ (ticket #15).
|
||
seen_ids = set()
|
||
for k in lookups:
|
||
cands = []
|
||
for c in gerbil_by_norm_name.get(k, []):
|
||
if c["Id"] in seen_ids:
|
||
continue
|
||
seen_ids.add(c["Id"])
|
||
final_id = gerbil_id_map.get(c["Id"])
|
||
if not final_id:
|
||
continue
|
||
final_c = gerbil_by_id_final.get(final_id)
|
||
if not final_c:
|
||
continue
|
||
# Parent must be age-plausible: born before the litter and within
|
||
# the gerbil lifespan (skips e.g. a 2013 animal for a 2022 litter).
|
||
if not parent_age_plausible(final_c["DateOfBirth"], litter_date):
|
||
continue
|
||
# Sterbedatum-Plausibilität: ein vor dem Wurf verstorbenes Tier kann kein
|
||
# Elternteil sein (Namensvetter-Fall: die gleichnamige, noch lebende Maus ist
|
||
# die echte Mutter/der echte Vater). ~40 Tage Toleranz, falls ein Elternteil
|
||
# kurz vor der Geburt starb. Nur filtern, wenn beide Daten bekannt.
|
||
dod = final_c.get("DateOfDeath")
|
||
if dod and litter_date:
|
||
try:
|
||
if datetime.strptime(dod, "%Y-%m-%d") < datetime.strptime(litter_date, "%Y-%m-%d") - timedelta(days=40):
|
||
continue
|
||
except Exception:
|
||
pass
|
||
cands.append(final_c)
|
||
if not cands:
|
||
continue
|
||
# Within a key, a candidate whose FULL name equals the lookup beats one
|
||
# that only matched via call-name (resident „Elena“ > „Elena of KK
|
||
# Chaos“ — #15). Stable sort keeps prior ordering otherwise.
|
||
cands.sort(key=lambda c: 0 if normalize_name(c["Name"]) == k else 1)
|
||
# Prefer the gender expected for this role, then unknown, then anything.
|
||
for pool in (
|
||
[c for c in cands if c["Gender"] == prefer_gender],
|
||
[c for c in cands if c["Gender"] == "unknown"],
|
||
cands,
|
||
):
|
||
if pool:
|
||
return pool[0]
|
||
return None
|
||
|
||
# ── Fehlende Wurf-Eltern aus der xlsx-Wurfchronik (output/litters.json) füllen ──
|
||
# extract/merge konsumiert litters.json bisher NICHT; viele md-only/Wurfchronik-
|
||
# Würfe haben deshalb keinen Eltern-Namen. Hier NUR Lücken füllen (nie überschreiben)
|
||
# und NUR bei eindeutigem Match (eindeutiges Datum, sonst Buchstabe+Datum). Die
|
||
# eigentliche Verknüpfung macht danach _resolve_name (Alters-/Gender-Plausibilität) —
|
||
# nicht auflösbare Namen bleiben unverknüpft (kein Falsch-Link).
|
||
try:
|
||
_lj_path = os.path.join(OUTPUT_DIR, "litters.json")
|
||
_lj = json.load(open(_lj_path, encoding="utf-8")) if os.path.exists(_lj_path) else []
|
||
_lj = _lj if isinstance(_lj, list) else _lj.get("litters", [])
|
||
except Exception:
|
||
_lj = []
|
||
|
||
def _lj_iso(s):
|
||
try:
|
||
dd, mm, yy = (s or "").split("."); return f"{yy}-{mm}-{dd}"
|
||
except Exception:
|
||
return s or ""
|
||
|
||
def _strip_zucht(n):
|
||
return (n or "").split(" [")[0].strip()
|
||
|
||
_lj_by_date = {}
|
||
for _e in _lj:
|
||
_lj_by_date.setdefault(_lj_iso(_e.get("date")), []).append(_e)
|
||
|
||
_lj_filled = 0
|
||
_lj_mort = 0
|
||
for l in resolved_litters:
|
||
cands = _lj_by_date.get(l.get("Date") or "", [])
|
||
if len(cands) > 1 and l.get("LitterLetter"):
|
||
narrowed = [e for e in cands if str(e.get("litterId") or "").upper() == str(l.get("LitterLetter")).upper()]
|
||
if narrowed:
|
||
cands = narrowed
|
||
if len(cands) != 1:
|
||
continue
|
||
e = cands[0]
|
||
# Eltern-Namen — nur Luecken (nie ueberschreiben)
|
||
if not l.get("FatherId") and not l.get("_father_name") and _strip_zucht(e.get("sireName")):
|
||
l["_father_name"] = _strip_zucht(e.get("sireName")); _lj_filled += 1
|
||
if not l.get("MotherId") and not l.get("_mother_name") and _strip_zucht(e.get("damName")):
|
||
l["_mother_name"] = _strip_zucht(e.get("damName")); _lj_filled += 1
|
||
# Totgeburten / Fruehsterblichkeit / Wurfstaerke — nur Luecken
|
||
if l.get("Stillborn") is None and e.get("stillborn") is not None:
|
||
l["Stillborn"] = e.get("stillborn"); _lj_mort += 1
|
||
if l.get("DeathsWithin8Weeks") is None and e.get("diedLater") is not None:
|
||
l["DeathsWithin8Weeks"] = e.get("diedLater")
|
||
if l.get("TotalBorn") is None and e.get("totalBorn") is not None:
|
||
l["TotalBorn"] = e.get("totalBorn")
|
||
if _lj_filled or _lj_mort:
|
||
print(f"litters.json ergaenzt: {_lj_filled} Eltern-Namen, {_lj_mort} Totgeburt-Angaben")
|
||
|
||
for l in resolved_litters:
|
||
# Pre-check: if _father_name points to a known female and _mother_name to a
|
||
# known male → swap names (Stammbaum positions reversed). Helps the name
|
||
# resolver pick the right same-name candidate before role normalization.
|
||
f_name_pre = l.get("_father_name", "")
|
||
m_name_pre = l.get("_mother_name", "")
|
||
if f_name_pre and m_name_pre:
|
||
f_gender = next((g["Gender"] for g in gerbil_by_norm_name.get(normalize_name(f_name_pre), []) if g["Gender"] != "unknown"), None)
|
||
m_gender = next((g["Gender"] for g in gerbil_by_norm_name.get(normalize_name(m_name_pre), []) if g["Gender"] != "unknown"), None)
|
||
if f_gender == "female" and m_gender == "male":
|
||
l["_father_name"], l["_mother_name"] = m_name_pre, f_name_pre
|
||
|
||
if l["_father_name"] and not l["FatherId"]:
|
||
cand = _resolve_name(l["_father_name"], "male", l["Date"])
|
||
if cand:
|
||
l["FatherId"] = cand["Id"]
|
||
resolved_fathers += 1
|
||
|
||
if l["_mother_name"] and not l["MotherId"]:
|
||
cand = _resolve_name(l["_mother_name"], "female", l["Date"])
|
||
if cand:
|
||
l["MotherId"] = cand["Id"]
|
||
resolved_mothers += 1
|
||
|
||
# Cleanup internal keys
|
||
del l["_father_name"]
|
||
del l["_mother_name"]
|
||
del l["_filename"]
|
||
|
||
# Children-of-litter lookup so a dropped parent link can be explained in the
|
||
# offspring's Datenherkunft (the discard is most meaningful on the child).
|
||
children_by_litter = {}
|
||
for g in resolved_gerbils:
|
||
lid = g.get("LitterId")
|
||
if lid:
|
||
children_by_litter.setdefault(lid, []).append(g)
|
||
|
||
def _add_child_discard(litter, discard):
|
||
"""Attach a parent-link discard to every child gerbil of the litter."""
|
||
for child in children_by_litter.get(litter["Id"], []):
|
||
dl = child.setdefault("_discarded", [])
|
||
if discard not in dl:
|
||
dl.append(discard)
|
||
|
||
def _pname(pid):
|
||
p = gerbil_by_id_final.get(pid)
|
||
return p.get("Name") if p else None
|
||
|
||
# Role normalization: assign each resolved parent to the role matching its
|
||
# gender, eliminate self-pairings (same animal in both roles), and never let
|
||
# impossible duplicates survive (two males / two females). This corrects
|
||
# reversed Stammbaum positions including the cases the simple swap missed
|
||
# (one parent of "unknown" gender, or a self-paired litter).
|
||
role_fixes = 0
|
||
for l in resolved_litters:
|
||
before = (l.get("FatherId"), l.get("MotherId"))
|
||
father, mother = assign_parent_roles(l.get("FatherId"), l.get("MotherId"), _final_gender)
|
||
if before != (father, mother):
|
||
role_fixes += 1
|
||
after = {father, mother}
|
||
# A parent id present before but gone after was dropped by role
|
||
# normalization (self-pairing or two-of-the-same-sex). Explain it.
|
||
for pid in before:
|
||
if pid and pid not in after:
|
||
pname = _pname(pid)
|
||
if before[0] == before[1]:
|
||
reason = "Selbstverpaarung — Tier kann nicht beide Elternteile sein"
|
||
else:
|
||
reason = "ein Wurf hat nur einen Vater und eine Mutter"
|
||
_add_child_discard(l, {
|
||
"text": f"Elternteil „{pname}“ verworfen — {reason}",
|
||
})
|
||
l["FatherId"] = father
|
||
l["MotherId"] = mother
|
||
if role_fixes:
|
||
print(f"Role-normalization: corrected {role_fixes} litter(s) (gender roles / self-pairings).")
|
||
|
||
# Parent-age sanity check: drop any resolved parent that cannot belong to the
|
||
# litter — born after the offspring, or more than a gerbil lifespan earlier.
|
||
# Catches mis-resolved links the name matcher still let through (e.g. Jayjay,
|
||
# *2013, wrongly attached to Solice's 2022 litter).
|
||
age_drops = []
|
||
for l in resolved_litters:
|
||
ldate = l.get("Date")
|
||
for role in ("FatherId", "MotherId"):
|
||
pid = l.get(role)
|
||
if not pid:
|
||
continue
|
||
p = gerbil_by_id_final.get(pid)
|
||
if p and not parent_age_plausible(p.get("DateOfBirth"), ldate):
|
||
age_drops.append((l.get("Name"), role, p.get("Name"), p.get("DateOfBirth"), ldate))
|
||
# Explain the drop in each child's history: which parent, its DOB,
|
||
# why (implausible age), and that no replacement was used.
|
||
role_de = "Vater" if role == "FatherId" else "Mutter"
|
||
pdob_disp = _de_date(p.get("DateOfBirth")) or "unbekannt"
|
||
age_reason = "unplausibles Alter für diesen Wurf"
|
||
pd = date_to_days(parse_date(p.get("DateOfBirth"))) if p.get("DateOfBirth") else None
|
||
ld = date_to_days(parse_date(ldate)) if ldate else None
|
||
if pd is not None and ld is not None:
|
||
years = abs(ld - pd) / 365.25
|
||
if ld - pd <= 0:
|
||
age_reason = "unplausibel (nicht vor dem Kind geboren)"
|
||
else:
|
||
age_reason = f"unplausibel ({years:.0f} Jahre älter als das Kind)"
|
||
_add_child_discard(l, {
|
||
"text": (f"{role_de} „{p.get('Name')}“ (*{pdob_disp}) verworfen "
|
||
f"— {age_reason}; kein Ersatz"),
|
||
})
|
||
l[role] = None
|
||
if age_drops:
|
||
print(f"Parent-age sanity check: dropped {len(age_drops)} implausible parent link(s):")
|
||
for lname, role, pname, pdob, ldate in age_drops[:20]:
|
||
print(f" {lname}: {role}={pname} (*{pdob}) vs litter {ldate}")
|
||
|
||
print(f"Globally resolved {resolved_fathers} fathers and {resolved_mothers} mothers.")
|
||
|
||
|
||
# 5b. Second-pass litter dedup: now that FatherId/MotherId are known,
|
||
# merge litters that have the same date AND the same parents.
|
||
# This is the core of the "sibling pairing" fix: Blue Wave and Sunny Sky
|
||
# both come from Wonderman × Unique — their two separate litter records
|
||
# must now become one, so their children share the same LitterId.
|
||
litter_by_id_post = {l["Id"]: l for l in resolved_litters}
|
||
gerbil_by_litter = {}
|
||
for g in resolved_gerbils:
|
||
lid = g.get("LitterId")
|
||
if lid:
|
||
gerbil_by_litter.setdefault(lid, []).append(g)
|
||
|
||
def _litter_same_parents(l1, l2):
|
||
"""Strict: same date + both parents known and matching."""
|
||
if l1["Date"] != l2["Date"]:
|
||
return False
|
||
f1, m1 = l1.get("FatherId"), l1.get("MotherId")
|
||
f2, m2 = l2.get("FatherId"), l2.get("MotherId")
|
||
if not f1 or not f2 or not m1 or not m2:
|
||
return False
|
||
return f1 == f2 and m1 == m2
|
||
|
||
by_date2 = {}
|
||
for l in resolved_litters:
|
||
by_date2.setdefault(l["Date"], []).append(l)
|
||
|
||
deduped2 = []
|
||
litter_remap2 = {} # old_id -> canonical_id
|
||
|
||
for date_val, group in by_date2.items():
|
||
sub_groups = []
|
||
for l in group:
|
||
placed = False
|
||
for sub in sub_groups:
|
||
if all(_litter_same_parents(l, m) for m in sub):
|
||
sub.append(l)
|
||
placed = True
|
||
break
|
||
if not placed:
|
||
sub_groups.append([l])
|
||
|
||
for sub in sub_groups:
|
||
# Prefer the canonical that has the most children
|
||
canonical = max(sub, key=lambda l: len(gerbil_by_litter.get(l["Id"], [])))
|
||
for l in sub:
|
||
litter_remap2[l["Id"]] = canonical["Id"]
|
||
if l is not canonical:
|
||
canonical["_source_files"] |= l.get("_source_files", set())
|
||
canonical["_merged_count"] += l.get("_merged_count", 1)
|
||
if not l.get("_virtual"):
|
||
canonical["_virtual"] = False
|
||
deduped2.append(canonical)
|
||
if len(sub) > 1:
|
||
siblings = [g["Name"] for l in sub for g in gerbil_by_litter.get(l["Id"], []) if l is not canonical]
|
||
print(f"Sibling-Litter-Merge on {date_val}: {canonical['Name']} absorbed sibling half — children now share LitterId: {[g['Name'] for g in gerbil_by_litter.get(canonical['Id'], [])] + siblings}")
|
||
|
||
# Remap LitterId in all gerbils
|
||
n_remapped = 0
|
||
for g in resolved_gerbils:
|
||
old_lid = g.get("LitterId")
|
||
if old_lid and old_lid in litter_remap2 and litter_remap2[old_lid] != old_lid:
|
||
g["LitterId"] = litter_remap2[old_lid]
|
||
n_remapped += 1
|
||
|
||
n_merged2 = len(resolved_litters) - len(deduped2)
|
||
if n_merged2:
|
||
print(f"Sibling-Litter-Dedup: {n_merged2} additional litter record(s) merged ({n_remapped} gerbil LitterIds remapped).")
|
||
resolved_litters = deduped2
|
||
litter_by_scoped_id = {l["Id"]: l for l in resolved_litters}
|
||
|
||
# Final gerbil-provenance pass: now that parent links are fully resolved and
|
||
# all discards (majority-vote conflicts during dedup; rejected parent-refs;
|
||
# role-normalization drops; parent-age drops) are attached to each gerbil's
|
||
# _discarded list, render the provenance JSON with the discard history lines.
|
||
n_discards = 0
|
||
for g in resolved_gerbils:
|
||
args = g.pop("_prov_args", None)
|
||
if g.get("_discarded"):
|
||
n_discards += len(g["_discarded"])
|
||
if args is not None:
|
||
records, best_g, conflict_notes, field_source, field_discards = args
|
||
g["Provenance"] = build_provenance(
|
||
records, best_g, extra_notes=conflict_notes,
|
||
field_source=field_source, field_discards=field_discards,
|
||
)
|
||
else:
|
||
g["Provenance"] = build_provenance([g], g)
|
||
g.pop("_discarded", None)
|
||
if n_discards:
|
||
print(f"Datenherkunft: recorded {n_discards} discard line(s) across gerbils.")
|
||
|
||
# Abgabevertrag-Anreicherung: Käufer als Abnehmer-Kontakte anlegen und —
|
||
# konservativ — auf eindeutig passende Tiere ReceiverContactId/GoHomeDate/
|
||
# Status=GivenAway setzen (nur falls noch nicht gesetzt). Provenance-
|
||
# Historie wird ergänzt. Neue Käuferkontakte landen in contact_by_norm_name
|
||
# und werden danach automatisch als IsReceiver markiert.
|
||
cstats, sale_contracts = enrich_from_contracts(
|
||
contracts, resolved_gerbils, contact_by_norm_name, contact_id_map,
|
||
exclude_decisions=_decisions)
|
||
if contracts:
|
||
print("Abgabeverträge: "
|
||
f"{cstats['contracts']} geladen, {cstats['matched']} Tier-Treffer, "
|
||
f"{cstats['ambiguous_skipped']} mehrdeutig übersprungen, "
|
||
f"{cstats['no_match_skipped']} ohne Treffer.")
|
||
print(" Kontakte: "
|
||
f"{cstats['buyers_created']} neu, {cstats['buyers_existing']} bestehend.")
|
||
print(" Gesetzt: "
|
||
f"ReceiverContactId={cstats['receiver_set']}, "
|
||
f"GoHomeDate={cstats['gohome_set']}, "
|
||
f"Status=GivenAway={cstats['status_givenaway']}, "
|
||
f"Konflikte={cstats['conflicts']}.")
|
||
print(" Vertragszeilen: "
|
||
f"{cstats['records_created']} angelegt "
|
||
f"({cstats['records_with_animal']} mit Tier, "
|
||
f"{cstats['records_with_date']} mit Originaldatum), "
|
||
f"{cstats['no_buyer_skipped']} ohne Käufer übersprungen, "
|
||
f"{cstats['dateless_skipped']} ohne Datum übersprungen.")
|
||
# Re-materialise contacts so freshly created buyer contacts are exported.
|
||
resolved_contacts = list(contact_by_norm_name.values())
|
||
|
||
# Datenherkunft for litters: which source files contributed, whether this is
|
||
# a Wurfchronik litter vs a Stammbaum-reconstructed ("virtual") litter, how
|
||
# many raw records merged into it, plus human-readable notes. Accumulators
|
||
# (_source_files/_merged_count/_virtual) were filled during the two dedup
|
||
# passes above; strip them after use.
|
||
# Id → Gerbil lookup so we can inspect a litter's parents (residency) below.
|
||
_gerbil_by_id = {g["Id"]: g for g in resolved_gerbils if g.get("Id")}
|
||
|
||
# Per-litter flags captured here (before _virtual is popped) for the late
|
||
# residency sweep: was the litter reconstructed from a chart (virtual) and
|
||
# did it come from the Wurfchronik?
|
||
_litter_is_virtual = {}
|
||
_litter_from_wurfchronik = {}
|
||
|
||
for l in resolved_litters:
|
||
l_source_files = l.pop("_source_files", set())
|
||
l_merged_count = l.pop("_merged_count", 1)
|
||
is_virtual = l.pop("_virtual", False)
|
||
l_from_wurfchronik = any("wurfchronik" in str(f).lower() for f in l_source_files)
|
||
_litter_is_virtual[l["Id"]] = is_virtual
|
||
_litter_from_wurfchronik[l["Id"]] = l_from_wurfchronik
|
||
l_notes = []
|
||
if is_virtual and not l_from_wurfchronik:
|
||
l_notes.append("aus Stammbaum-Diagramm rekonstruiert")
|
||
elif l_from_wurfchronik:
|
||
l_notes.append("aus Wurfchronik")
|
||
if l_merged_count > 1:
|
||
l_notes.append(f"aus {l_merged_count} Datensätzen zusammengeführt")
|
||
l_notes.append("Geschwister-Würfe zusammengeführt")
|
||
|
||
# Chronological, file-attributed history for the litter.
|
||
l_files_sorted = sorted({f for f in l_source_files if f})
|
||
l_history = []
|
||
first_file = l_files_sorted[0] if l_files_sorted else None
|
||
if is_virtual and not l_from_wurfchronik:
|
||
if first_file:
|
||
l_history.append(
|
||
f"Aus Stammbaum-Diagramm rekonstruiert ({_quote_file(first_file)})."
|
||
)
|
||
else:
|
||
l_history.append("Aus Stammbaum-Diagramm rekonstruiert.")
|
||
elif first_file:
|
||
l_history.append(f"Wurf aus Wurfchronik {_quote_file(first_file)}.")
|
||
else:
|
||
l_history.append("Wurf im Import gefunden.")
|
||
for f in l_files_sorted[1:]:
|
||
l_history.append(
|
||
f"Auch in {_quote_file(f)} gefunden → Datensätze zusammengeführt."
|
||
)
|
||
if l_merged_count > 1:
|
||
l_history.append("Geschwister-Würfe zusammengeführt.")
|
||
l["Provenance"] = build_entity_provenance(
|
||
l_source_files, l_merged_count, notes=l_notes,
|
||
from_wurfchronik=l_from_wurfchronik, history=l_history,
|
||
)
|
||
# ShowInChronicle wird ERST NACH dem isResident-Sweep gesetzt (siehe unten),
|
||
# weil das Kriterium „mind. ein Bestandstier-Elternteil" die finale Residenz
|
||
# braucht — der Sweep läuft nach addLitters.
|
||
|
||
# ── addLitters: manuell ergänzte Würfe injizieren ─────────────────────────
|
||
# Späte Injektion NACH der finalen Gerbil-/Litter-Assemblierung, damit die
|
||
# finalen Tier-Ids feststehen. Jeder Eintrag verknüpft Mutter/Vater/Kinder
|
||
# über deren Namen(+dob) und setzt bei den Kindern die neue LitterId. Würfe mit
|
||
# showInChronicle=false (z. B. Akanes Wurf bei Clan of Black Forest) erscheinen
|
||
# nur auf der Tier-Seite, nicht in der Wurfchronik.
|
||
def _find_final_gerbil(name, dob=None):
|
||
if not name:
|
||
return None
|
||
target_ck = normalize_name(get_call_name(name))
|
||
target_full = normalize_name(name)
|
||
iso = parse_date(dob) if dob else None
|
||
# Match precedence (most specific first) so a deliberately-added animal
|
||
# (full name „Merle“) is preferred over an unrelated foreign animal that
|
||
# only shares the call-name („Merle of Samsimar“):
|
||
# 1. full-name + dob 2. full-name 3. call-name + dob 4. call-name
|
||
full_dob = full_only = ck_dob = ck_only = None
|
||
for g in resolved_gerbils:
|
||
g_full = normalize_name(g.get("Name") or "")
|
||
g_ck = normalize_name(get_call_name(g.get("Name") or ""))
|
||
same_dob = iso and g.get("DateOfBirth") == iso
|
||
if g_full == target_full:
|
||
if same_dob and full_dob is None:
|
||
full_dob = g
|
||
elif full_only is None:
|
||
full_only = g
|
||
elif g_ck == target_ck:
|
||
if same_dob and ck_dob is None:
|
||
ck_dob = g
|
||
elif ck_only is None:
|
||
ck_only = g
|
||
return full_dob or full_only or ck_dob or ck_only
|
||
|
||
_litters_injected = 0
|
||
for al in (_add_litters or []):
|
||
al_name = (al.get("name") or "").strip()
|
||
if not al_name:
|
||
continue
|
||
mother = _find_final_gerbil(al.get("mother"), al.get("motherDob"))
|
||
father = _find_final_gerbil(al.get("father"), al.get("fatherDob"))
|
||
child_gerbils = []
|
||
for cn in (al.get("children") or []):
|
||
# Kind als String ODER {name, dob} — dob disambiguiert gleichnamige Tiere
|
||
# (z. B. Ahnen-Elliot *2015 vs. Zuchttier-Elliot *2023).
|
||
if isinstance(cn, dict):
|
||
cg = _find_final_gerbil(cn.get("name"), cn.get("dob"))
|
||
else:
|
||
cg = _find_final_gerbil(cn)
|
||
if cg:
|
||
child_gerbils.append(cg)
|
||
slug = normalize_name(al_name)
|
||
litter_id = generate_guid(f"decision-litter-{slug}")
|
||
al_date = parse_date(al["date"]) if al.get("date") else None
|
||
al_notes = (al.get("notes") or "").strip() or None
|
||
new_litter = {
|
||
"Id": litter_id,
|
||
"Name": al_name,
|
||
"Date": al_date,
|
||
"TotalBorn": al.get("totalBorn"),
|
||
"DeathsWithin8Weeks": None,
|
||
"FatherId": father["Id"] if father else None,
|
||
"MotherId": mother["Id"] if mother else None,
|
||
"ExpectedGoHomeDate": None,
|
||
"Notes": al_notes,
|
||
"PairingCode": None,
|
||
"ExternalRef": f"decision-litter-{slug}",
|
||
"LitterLetter": None,
|
||
"ShowInChronicle": bool(al.get("showInChronicle", True)),
|
||
"Provenance": build_entity_provenance(
|
||
["conflict-decisions.json"], 1,
|
||
notes=["Manuell aus Ticket ergänzt"],
|
||
from_wurfchronik=False,
|
||
history=["Manuell aus dem Ticket der Züchterin ergänzt "
|
||
"(conflict-decisions.json)."],
|
||
),
|
||
}
|
||
for cg in child_gerbils:
|
||
cg["LitterId"] = litter_id
|
||
resolved_litters.append(new_litter)
|
||
_litters_injected += 1
|
||
print(f"addLitters injiziert: '{al_name}' "
|
||
f"(Mutter={mother['Name'] if mother else '—'}, "
|
||
f"Vater={father['Name'] if father else '—'}, "
|
||
f"Kinder={len(child_gerbils)}, ShowInChronicle={new_litter['ShowInChronicle']})")
|
||
if _litters_injected:
|
||
print(f"addLitters: {_litters_injected} Wurf/Würfe injiziert.")
|
||
|
||
# ── rpro3Pedigrees: KOMPLETTE Ahnentafel eines Tiers aus _rpro3.db ziehen ──────
|
||
# Für „bitte den ganzen Stammbaum aus RennerPro" (nicht nur eine Generation). Für
|
||
# jeden Root {name, dob} werden ALLE Vorfahren über die rpro3-Eltern-Zeiger
|
||
# (pid_raw/mid_raw) rekursiv materialisiert: bereits vorhandene Tiere (Name+DOB)
|
||
# werden WIEDERVERWENDET, fehlende als externe Ahnen neu angelegt (deterministische
|
||
# Id `rpro3-anc-<rid>`, Genotyp aus rpro3-Fcode → korrekter Farbschlag, Herkunft aus
|
||
# origin). Fehlende Eltern-Verknüpfungen werden als (versteckte) Ahnen-Würfe ergänzt;
|
||
# Tiere mit bereits vorhandenem Geburtswurf bleiben unangetastet.
|
||
try:
|
||
import compare_rpro3 as _Crp
|
||
_peds = []
|
||
try:
|
||
with open(CONFLICT_DECISIONS_PATH, encoding="utf-8") as _pfh:
|
||
_peds = (json.load(_pfh).get("rpro3Pedigrees") or [])
|
||
except Exception:
|
||
_peds = []
|
||
_dbp = os.path.join(os.path.dirname(os.path.abspath(__file__)), "_rpro3.db")
|
||
if _peds and os.path.exists(_dbp):
|
||
_R = _Crp.load_rpro3(_dbp)
|
||
_A = _R["animals"]
|
||
_byrid = {a["rid"]: a for a in _A}
|
||
def _iso(x):
|
||
return x.isoformat() if hasattr(x, "isoformat") else (x or None)
|
||
def _rn(s):
|
||
return normalize_name(get_call_name(s or ""))
|
||
# Strenger Match für Ahnen: bei bekanntem DOB nur exakt (Name+DOB); bei
|
||
# DOB-losem rpro3-Ahnen nur wiederverwenden, wenn genau EIN ebenfalls DOB-loses
|
||
# App-Tier gleichen Namens existiert — sonst NEU anlegen (verhindert, dass ein
|
||
# DOB-loser Ahn fälschlich an einen dat. Namensvetter gehängt wird, z. B.
|
||
# Kilians Mutter „Lila" ≠ Zuchttier „Lila *2014").
|
||
_res_by_name = {}
|
||
for _gg in resolved_gerbils:
|
||
_res_by_name.setdefault(_rn(_gg.get("Name")), []).append(_gg)
|
||
def _strict_anc_match(name, dob):
|
||
cs = _res_by_name.get(_rn(name), [])
|
||
if dob:
|
||
return next((x for x in cs if x.get("DateOfBirth") == dob), None)
|
||
dobless = [x for x in cs if not x.get("DateOfBirth")]
|
||
return dobless[0] if len(dobless) == 1 else None
|
||
def _find_rpro3(name, dob):
|
||
cs = [a for a in _A if _rn(a["name"]) == _rn(name)]
|
||
if dob:
|
||
mm = next((a for a in cs if _iso(a["dob"]) == dob), None)
|
||
if mm:
|
||
return mm
|
||
return cs[0] if len(cs) == 1 else None
|
||
_pg = _pl = 0
|
||
for ped in _peds:
|
||
pdob = parse_date(ped.get("dob")) if ped.get("dob") else None
|
||
root = _find_rpro3(ped.get("name"), pdob)
|
||
if not root:
|
||
print(f"rpro3Pedigrees: Root '{ped.get('name')}' nicht in rpro3 gefunden")
|
||
continue
|
||
# Vorfahren sammeln
|
||
nodes, links, seen, stack = {}, [], set(), [root]
|
||
while stack:
|
||
a = stack.pop()
|
||
if not a or a["rid"] in seen:
|
||
continue
|
||
seen.add(a["rid"]); nodes[a["rid"]] = a
|
||
f = _byrid.get(a.get("pid_raw")); m = _byrid.get(a.get("mid_raw"))
|
||
if f or m:
|
||
links.append((a["rid"], f["rid"] if f else None, m["rid"] if m else None))
|
||
for pp in (f, m):
|
||
if pp and pp["rid"] not in seen:
|
||
stack.append(pp)
|
||
# rid → App-Gerbil (vorhandene wiederverwenden, sonst anlegen)
|
||
rid2g = {}
|
||
for rid, a in nodes.items():
|
||
ex = _strict_anc_match(a["name"], _iso(a.get("dob")))
|
||
if ex:
|
||
rid2g[rid] = ex; continue
|
||
gid = generate_guid(f"rpro3-anc-{rid}")
|
||
gender = {"männlich": "male", "weiblich": "female"}.get(
|
||
(a.get("sex") or "").lower(), "unknown")
|
||
fcode = (a.get("fcode") or "").strip() or None
|
||
stub = {
|
||
"Id": gid, "Name": get_normalized_gerbil_name(a["name"]),
|
||
"Gender": gender, "Status": "Deceased", "LitterId": None,
|
||
"OriginContactId": None, "ReceiverContactId": None, "EnclosureId": None,
|
||
"ColorVarietyId": None, "DateOfBirth": _iso(a.get("dob")), "DateOfDeath": None,
|
||
"CauseOfDeath": None, "GoHomeDate": None, "Genotype": fcode,
|
||
"Notes": None, "ImportSource": "rpro3 (_rpro3.db)",
|
||
"ExternalRef": f"rpro3-anc-{rid}",
|
||
"RawImportData": json.dumps({"rpro3Rid": rid}, ensure_ascii=False),
|
||
"OriginBreeder": (a.get("origin") or "").strip() or None,
|
||
"NameSearch": normalize_name(get_normalized_gerbil_name(a["name"])),
|
||
"CharacterTraits": [], "CharacterNote": None, "IsDeaf": None,
|
||
"IsResident": False, "parentRefs": [], "_photos": [],
|
||
"_old_scoped_litter_id": None, "_eff_dob": _iso(a.get("dob")) or "2008-01-01",
|
||
"_birth_date": _iso(a.get("dob")), "_filename": "_rpro3.db", "_old_id": gid,
|
||
"Provenance": build_entity_provenance(
|
||
["_rpro3.db"], 1, notes=["Vorfahre aus RennerPro (RennmausPro III)"],
|
||
from_wurfchronik=False,
|
||
history=[f"Als Vorfahre aus RennerPro übernommen (rid {rid})."]),
|
||
}
|
||
resolved_gerbils.append(stub); rid2g[rid] = stub; _pg += 1
|
||
# Eltern-Verknüpfungen als Ahnen-Würfe (nur wo Kind noch keinen Wurf hat)
|
||
for c, f, m in links:
|
||
cg = rid2g.get(c)
|
||
if not cg or cg.get("LitterId"):
|
||
continue
|
||
fg = rid2g.get(f); mg = rid2g.get(m)
|
||
if not fg and not mg:
|
||
continue
|
||
lid = generate_guid(f"rpro3-litter-{c}")
|
||
lit = {
|
||
"Id": lid,
|
||
"Name": f"Wurf von {fg['Name'] if fg else '?'} + {mg['Name'] if mg else '?'}",
|
||
"Date": cg.get("DateOfBirth"), "TotalBorn": None, "DeathsWithin8Weeks": None,
|
||
"FatherId": fg["Id"] if fg else None, "MotherId": mg["Id"] if mg else None,
|
||
"ExpectedGoHomeDate": None, "Notes": None, "PairingCode": None,
|
||
"ExternalRef": f"rpro3-litter-{c}", "LitterLetter": None,
|
||
"ShowInChronicle": False,
|
||
"Provenance": build_entity_provenance(
|
||
["_rpro3.db"], 1, notes=["Ahnen-Wurf aus RennerPro"],
|
||
from_wurfchronik=False,
|
||
history=["Eltern-Verknüpfung aus RennerPro (RennmausPro III)."]),
|
||
}
|
||
cg["LitterId"] = lid
|
||
resolved_litters.append(lit); _pl += 1
|
||
if _pg or _pl:
|
||
print(f"rpro3Pedigrees injiziert: {_pg} Ahnen, {_pl} Ahnen-Würfe")
|
||
except Exception as _e:
|
||
print(f"rpro3Pedigrees übersprungen: {_e}")
|
||
|
||
# ── litterParents: Eltern eines BESTEHENDEN Wurfs erzwingen (Mensch-Entscheidung) ──
|
||
# Für Fälle, in denen der Resolver einen Namensvetter oder ein verstorbenes Tier
|
||
# verknüpft hat und die echte Mutter/der echte Vater eindeutig benannt ist. Match per
|
||
# litterName+date (oder nur date); Vater/Mutter via _find_final_gerbil(name, dob).
|
||
try:
|
||
with open(CONFLICT_DECISIONS_PATH, encoding="utf-8") as _lpfh:
|
||
_litter_parents = (json.load(_lpfh).get("litterParents") or [])
|
||
except Exception:
|
||
_litter_parents = []
|
||
_lp_applied = 0
|
||
for lp in _litter_parents:
|
||
lp_name = (lp.get("litterName") or "").strip().lower()
|
||
lp_date = parse_date(lp["date"]) if lp.get("date") else None
|
||
if not lp_name and not lp_date:
|
||
continue
|
||
for l in resolved_litters:
|
||
if lp_name and (l.get("Name") or "").strip().lower() != lp_name:
|
||
continue
|
||
if lp_date and l.get("Date") != lp_date:
|
||
continue
|
||
if lp.get("father"):
|
||
fa = _find_final_gerbil(lp["father"], lp.get("fatherDob"))
|
||
if fa:
|
||
l["FatherId"] = fa["Id"]
|
||
if lp.get("mother"):
|
||
mo = _find_final_gerbil(lp["mother"], lp.get("motherDob"))
|
||
if mo:
|
||
l["MotherId"] = mo["Id"]
|
||
_lp_applied += 1
|
||
if _lp_applied:
|
||
print(f"litterParents-Overrides angewandt: {_lp_applied}")
|
||
|
||
# ── litterOrder: Reihenfolge der Jungtiere im Wurf (Wurfchronik = Geburtsgewicht) ──
|
||
# Die Züchterin gibt die korrekte Reihenfolge vor (nicht alphabetisch). Match per
|
||
# litterName+date; Kinder per Wurf (LitterId) + Namensabgleich; setzt BirthOrder=1..N.
|
||
try:
|
||
with open(CONFLICT_DECISIONS_PATH, encoding="utf-8") as _lofh:
|
||
_litter_orders = (json.load(_lofh).get("litterOrder") or [])
|
||
except Exception:
|
||
_litter_orders = []
|
||
_lo_applied = 0
|
||
for lo in _litter_orders:
|
||
lo_name = (lo.get("litterName") or "").strip().lower()
|
||
lo_date = parse_date(lo["date"]) if lo.get("date") else None
|
||
order = lo.get("order") or []
|
||
if not order or (not lo_name and not lo_date):
|
||
continue
|
||
for l in resolved_litters:
|
||
if lo_name and (l.get("Name") or "").strip().lower() != lo_name:
|
||
continue
|
||
if lo_date and l.get("Date") != lo_date:
|
||
continue
|
||
kids = [g for g in resolved_gerbils if g.get("LitterId") == l["Id"]]
|
||
for idx, cname in enumerate(order, start=1):
|
||
ck = normalize_name(get_call_name(cname))
|
||
for g in kids:
|
||
if normalize_name(get_call_name(g.get("Name") or "")) == ck:
|
||
g["BirthOrder"] = idx
|
||
_lo_applied += 1
|
||
break
|
||
break
|
||
if _lo_applied:
|
||
print(f"litterOrder: {_lo_applied} Jungtier-Reihenfolgen gesetzt")
|
||
|
||
# ── litterChildren: autoritative Kinder-Whitelist eines Wurfs (Mensch-Entscheidung) ──
|
||
# Für Fälle, in denen der Wurf-Merge fremde Tiere (z. B. Abnehmer-Sammelseiten der
|
||
# Wurfchronik) in einen datierten Wurf gezogen hat (Tickets 4b9f49fb M-Wurf, 8d259edf Gale).
|
||
# Match per date + `matchChild` (ein aktuell zugeordnetes Kind, disambiguiert gleich-datierte
|
||
# Würfe). Alle Kinder, deren Ruf-Name NICHT in `keep` steht, werden aus dem Wurf gelöst
|
||
# (LitterId=None) — sie verschwinden aus dem Wurf, bleiben aber als Tier erhalten. Optional
|
||
# `renameTo` setzt den kanonischen Wurf-Namen (z. B. 'Ungeklärt' -> 'M-Wurf').
|
||
# Optional `add` [Name | {name, dob}] hängt ein Jungtier an DIESEN Wurf (Gegenstück zu `keep`):
|
||
# nötig, wenn die Chart-Positions-Heuristik ein Jungtier in einen falschen (virtuellen)
|
||
# Stammbaum-Wurf gezogen hat, obwohl die Wurfchronik es einem echten Wurf zuordnet
|
||
# (Ticket 36a3fcde/88389f8e: Akane gehört in den Z21-Wurf Roni × Fumi). Der vorherige Wurf
|
||
# wird entfernt, wenn er dadurch ein kinderloser VIRTUELLER Wurf wird — sonst blieben
|
||
# Phantom-Würfe in der Wurfchronik stehen.
|
||
try:
|
||
with open(CONFLICT_DECISIONS_PATH, encoding="utf-8") as _lcfh:
|
||
_litter_children = (json.load(_lcfh).get("litterChildren") or [])
|
||
except Exception:
|
||
_litter_children = []
|
||
|
||
def _drop_if_empty_virtual(litter_id):
|
||
"""Entfernt einen VIRTUELLEN Wurf, der kein Jungtier mehr hat (Phantom-Wurf aus der
|
||
Chart-Positions-Heuristik). Echte Wurfchronik-Würfe bleiben immer erhalten — die haben
|
||
legitim keine namentlich erfassten Jungtiere."""
|
||
if not litter_id:
|
||
return 0
|
||
if any(g.get("LitterId") == litter_id for g in resolved_gerbils):
|
||
return 0
|
||
for i, cand in enumerate(resolved_litters):
|
||
if cand["Id"] != litter_id:
|
||
continue
|
||
ext = cand.get("ExternalRef") or ""
|
||
if cand.get("_virtual") or "-virtual-" in ext or ext.startswith("override-virtual-litter"):
|
||
del resolved_litters[i]
|
||
return 1
|
||
return 0
|
||
return 0
|
||
|
||
_lc_removed = 0
|
||
_lc_added = 0
|
||
_lc_dropped = 0
|
||
for lc in _litter_children:
|
||
lc_date = parse_date(lc["date"]) if lc.get("date") else None
|
||
match_child = normalize_name(get_call_name(lc.get("matchChild") or ""))
|
||
keep = {normalize_name(get_call_name(n)) for n in (lc.get("keep") or [])}
|
||
add = lc.get("add") or []
|
||
if not keep and not add:
|
||
continue
|
||
for l in resolved_litters:
|
||
if lc_date and l.get("Date") != lc_date:
|
||
continue
|
||
kids = [g for g in resolved_gerbils if g.get("LitterId") == l["Id"]]
|
||
names = {normalize_name(get_call_name(g.get("Name") or "")) for g in kids}
|
||
if match_child and match_child not in names:
|
||
continue # falscher, nur zufällig gleich-datierter Wurf
|
||
if keep:
|
||
for g in kids:
|
||
if normalize_name(get_call_name(g.get("Name") or "")) not in keep:
|
||
g["LitterId"] = None
|
||
_lc_removed += 1
|
||
for entry in add:
|
||
a_name = entry.get("name") if isinstance(entry, dict) else entry
|
||
a_dob = entry.get("dob") if isinstance(entry, dict) else None
|
||
child = _find_final_gerbil(a_name, a_dob)
|
||
if not child or child.get("LitterId") == l["Id"]:
|
||
continue
|
||
prev_litter_id = child.get("LitterId")
|
||
child["LitterId"] = l["Id"]
|
||
_lc_added += 1
|
||
_lc_dropped += _drop_if_empty_virtual(prev_litter_id)
|
||
if lc.get("renameTo"):
|
||
l["Name"] = lc["renameTo"]
|
||
break
|
||
if _lc_removed:
|
||
print(f"litterChildren: {_lc_removed} fälschlich zugeordnete Jungtiere aus Würfen gelöst")
|
||
if _lc_added:
|
||
print(f"litterChildren: {_lc_added} Jungtier(e) dem korrekten Wurf zugeordnet "
|
||
f"({_lc_dropped} leerer Phantom-Wurf/Würfe entfernt)")
|
||
|
||
# ── isResident-Sweep (Ticket 381f7e51, freigegeben) ───────────────────────
|
||
# Entscheidung der Züchterin: „Nimm wirklich ALLE Tiere aus dem Bestand heraus,
|
||
# MIT AUSNAHME der, die bereits ein ELTERNTIER sind." Datengetrieben + re-ingest-
|
||
# stabil neu berechnet, GANZ AM ENDE (nach finaler Eltern-/Wurf-Verknüpfung +
|
||
# addLitters), damit ein Tier erst dann als Elternteil erkennbar ist.
|
||
#
|
||
# Regel — IsResident=true GENAU DANN, wenn EINES gilt:
|
||
# (1) ein EXPLIZITER conflict-decisions-Override (_resident_override) hat
|
||
# isResident gesetzt (true ODER false) → gewinnt IMMER, der Sweep fasst
|
||
# solche Tiere NICHT an (z. B. Echo/Elia=true, Akane/Naémi/Bentley=false).
|
||
# (2) das Tier ist Mutter/Vater EINES IHRER EIGENEN Würfe. Ein „eigener Wurf"
|
||
# ist ein Wurf, der KEINE reine rekonstruierte Ahnen-Verpaarung ist:
|
||
# (a) nicht virtuell (echter Wurfchronik-/docx-Wurf) ODER
|
||
# (b) aus der Wurfchronik ODER
|
||
# (c) hat clan-benannte Nachzucht („von den Kleinen Chaoten") ODER
|
||
# (d) hat einen per Override residenten Elternteil ODER
|
||
# (e) ist ein addLitters-Wurf mit ShowInChronicle=true.
|
||
# Reine Fremd-Ahnen-Paarungen (z. B. Antares of Ulmer Strolche × …) zählen
|
||
# NICHT → externe Ahnen werden NICHT fälschlich resident.
|
||
# SONST: IsResident=false.
|
||
_children_by_litter = {}
|
||
for g in resolved_gerbils:
|
||
lid = g.get("LitterId")
|
||
if lid:
|
||
_children_by_litter.setdefault(lid, []).append(g)
|
||
|
||
# Explizit per Override resident gesetzte Tiere (für „eigener Wurf"-Kriterium d).
|
||
_override_resident_ids = {
|
||
g["Id"] for g in resolved_gerbils
|
||
if g.get("_resident_override") and g.get("IsResident")
|
||
}
|
||
|
||
def _is_own_litter(l):
|
||
lid = l.get("Id")
|
||
if not _litter_is_virtual.get(lid, True):
|
||
return True # (a) echter Wurf
|
||
if _litter_from_wurfchronik.get(lid, False):
|
||
return True # (b) Wurfchronik
|
||
for ch in _children_by_litter.get(lid, []): # (c) Clan-Nachzucht
|
||
if is_clan_zucht(ch.get("Name")):
|
||
return True
|
||
for pid in (l.get("FatherId"), l.get("MotherId")): # (d) resid. Elternteil
|
||
if pid in _override_resident_ids:
|
||
return True
|
||
if l.get("ExternalRef", "").startswith("decision-litter-"): # (e) addLitters
|
||
return bool(l.get("ShowInChronicle"))
|
||
return False
|
||
|
||
_own_parent_ids = set()
|
||
for l in resolved_litters:
|
||
if _is_own_litter(l):
|
||
for pid in (l.get("FatherId"), l.get("MotherId")):
|
||
if pid:
|
||
_own_parent_ids.add(pid)
|
||
|
||
def is_placeholder_name(name):
|
||
if not name:
|
||
return True
|
||
n = name.lower().strip()
|
||
n = re.sub(r'^(?:rv\s+|gv\s+|v\.\s*privat\s+|privat\s+)', '', n)
|
||
n = re.sub(r'\s+privat.*', '', n)
|
||
norm = "".join(c for c in n if c.isalnum())
|
||
if not norm:
|
||
return True
|
||
if norm in ["unbekannt", "unknown", "name", "jungtier", "jungtiere"]:
|
||
return True
|
||
# Check if name is a color variety name
|
||
color_keys = set(variety_map.keys())
|
||
for ck in color_keys:
|
||
ck_norm = "".join(c for c in ck if c.isalnum())
|
||
if norm == ck_norm:
|
||
return True
|
||
return False
|
||
|
||
_sweep_to_false = 0
|
||
_sweep_to_true = 0
|
||
for g in resolved_gerbils:
|
||
if g.get("_resident_override"):
|
||
continue # (1) Override gewinnt
|
||
should = g["Id"] in _own_parent_ids # (2) Elternteil eigenen Wurfs
|
||
if should and is_placeholder_name(g.get("Name")):
|
||
should = False # Platzhalter werden nie automatisch resident
|
||
if g.get("IsResident") != should:
|
||
if should:
|
||
_sweep_to_true += 1
|
||
else:
|
||
_sweep_to_false += 1
|
||
if g.get("OriginBreeder") == "Zucht der kleinen Chaoten":
|
||
g["OriginBreeder"] = None
|
||
g["IsResident"] = should
|
||
print(f"isResident-Sweep: {_sweep_to_true} → resident, {_sweep_to_false} → nicht-resident "
|
||
f"(Elternteile eigener Würfe: {len(_own_parent_ids)}).")
|
||
|
||
# ShowInChronicle (Ticket ea41257a) — JETZT, mit der FINALEN Residenz. Die
|
||
# Wurfchronik zeigt nur die eigenen, dokumentierten Würfe. ZIELGENAU ausblenden:
|
||
# ShowInChronicle=False NUR wenn der Wurf (a) virtuell rekonstruiert ist UND (b)
|
||
# NICHT aus der Wurfchronik stammt UND (c) keiner der beiden bekannten Eltern ein
|
||
# Bestandstier ist. So bleiben ihre echten (evtl. nur im Stammbaum stehenden)
|
||
# Würfe mit mind. einem Bestandstier-Elternteil sichtbar; nur reine Fremd-Ahnen-
|
||
# Paarungen verschwinden. addLitters-Würfe behalten ihren expliziten Wert.
|
||
_gerbil_by_id_final2 = {g["Id"]: g for g in resolved_gerbils if g.get("Id")}
|
||
for l in resolved_litters:
|
||
if l.get("ExternalRef", "").startswith("decision-litter-"):
|
||
continue # addLitters: expliziter ShowInChronicle bleibt
|
||
lid = l.get("Id")
|
||
is_virtual = _litter_is_virtual.get(lid, False)
|
||
l_from_wurfchronik = _litter_from_wurfchronik.get(lid, False)
|
||
_parents = [_gerbil_by_id_final2.get(l.get("FatherId")),
|
||
_gerbil_by_id_final2.get(l.get("MotherId"))]
|
||
_any_resident_parent = any(bool(p.get("IsResident")) for p in _parents if p)
|
||
|
||
_mother = _gerbil_by_id_final2.get(l.get("MotherId"))
|
||
_mother_is_external = False
|
||
if _mother:
|
||
_mother_is_external = is_external_cattery(_mother) and not _mother.get("IsResident")
|
||
|
||
_pure_ancestor_pairing = (
|
||
is_virtual and not l_from_wurfchronik and (not _any_resident_parent or _mother_is_external)
|
||
)
|
||
l["ShowInChronicle"] = not _pure_ancestor_pairing
|
||
|
||
# Hilfsfelder entfernen (nicht in den Payload exportieren).
|
||
for g in resolved_gerbils:
|
||
g.pop("_resident_override", None)
|
||
|
||
# Set IsBreeder and IsReceiver flags on contacts
|
||
breeder_ids = {g["OriginContactId"] for g in resolved_gerbils if g.get("OriginContactId")}
|
||
receiver_ids = {g["ReceiverContactId"] for g in resolved_gerbils if g.get("ReceiverContactId")}
|
||
for c in resolved_contacts:
|
||
c_id = c["Id"]
|
||
is_breeder = c_id in breeder_ids
|
||
is_receiver = c_id in receiver_ids
|
||
if not is_breeder and not is_receiver:
|
||
is_receiver = True
|
||
c["IsBreeder"] = is_breeder
|
||
c["IsReceiver"] = is_receiver
|
||
|
||
# Datenherkunft: where this (deduplicated) contact came from, plus the
|
||
# role we inferred. Accumulator keys (_source_files/_merged_count) were
|
||
# filled during the contact dedup above; strip them after use.
|
||
c_source_files = c.pop("_source_files", set())
|
||
c_merged_count = c.pop("_merged_count", 1)
|
||
c_notes = []
|
||
if c_merged_count > 1:
|
||
c_notes.append(f"aus {c_merged_count} Datensätzen zusammengeführt")
|
||
if is_breeder:
|
||
c_notes.append("als Züchter erkannt")
|
||
if is_receiver:
|
||
c_notes.append("als Abnehmer erkannt")
|
||
|
||
# Chronological, file-attributed history for the contact.
|
||
c_files_sorted = sorted({f for f in c_source_files if f})
|
||
c_history = []
|
||
role_word = "Züchter" if is_breeder else "Abnehmer"
|
||
if c_files_sorted:
|
||
c_history.append(f"In {_quote_file(c_files_sorted[0])} als {role_word} erkannt.")
|
||
for f in c_files_sorted[1:]:
|
||
c_history.append(
|
||
f"Auch in {_quote_file(f)} gefunden → Datensätze zusammengeführt."
|
||
)
|
||
else:
|
||
c_history.append(f"Im Import als {role_word} erkannt.")
|
||
if is_breeder and is_receiver:
|
||
c_history.append("Sowohl als Züchter als auch als Abnehmer geführt.")
|
||
c["Provenance"] = build_entity_provenance(
|
||
c_source_files, c_merged_count, notes=c_notes, history=c_history
|
||
)
|
||
|
||
# Set and map gerbilPhotos
|
||
resolved_photos = []
|
||
for g in resolved_gerbils:
|
||
for idx, photo_rel in enumerate(g.get("_photos", [])):
|
||
photo_guid = generate_guid(f"photo-{photo_rel}")
|
||
ext = os.path.splitext(photo_rel)[1] or ".jpeg"
|
||
fn_guid = photo_guid.replace("-", "")
|
||
resolved_photos.append({
|
||
"Id": photo_guid,
|
||
"GerbilId": g["Id"],
|
||
"FileName": f"{fn_guid}{ext}",
|
||
"SortOrder": idx,
|
||
"_source_path": photo_rel
|
||
})
|
||
|
||
# ── suppressExternalRefs: bestätigte Phantom-/Dubletten-Records entfernen ──
|
||
# Vom Owner bestätigte OCR-Fehllesungen/Dubletten (z. B. 'Muga' = 'Nduga', Ticket
|
||
# a9c0d0f9): Gerbil + ggf. zugehöriger (kinderloser) Phantom-Wurf werden per ExternalRef
|
||
# final entfernt; verwaiste Referenzen werden defensiv genullt.
|
||
_suppress = load_suppress_refs()
|
||
if _suppress:
|
||
removed_g = {g["Id"] for g in resolved_gerbils if g.get("ExternalRef") in _suppress}
|
||
removed_l = {l["Id"] for l in resolved_litters if l.get("ExternalRef") in _suppress}
|
||
resolved_gerbils = [g for g in resolved_gerbils if g.get("ExternalRef") not in _suppress]
|
||
resolved_litters = [l for l in resolved_litters if l.get("ExternalRef") not in _suppress]
|
||
for l in resolved_litters:
|
||
if l.get("MotherId") in removed_g:
|
||
l["MotherId"] = None
|
||
if l.get("FatherId") in removed_g:
|
||
l["FatherId"] = None
|
||
for g in resolved_gerbils:
|
||
if g.get("LitterId") in removed_l:
|
||
g["LitterId"] = None
|
||
resolved_photos = [p for p in resolved_photos if p.get("GerbilId") not in removed_g]
|
||
print(f"suppressExternalRefs: {len(removed_g)} Gerbil(s) + {len(removed_l)} Wurf/Würfe entfernt.")
|
||
|
||
# Herkunft-Normalisierung (Ticket Bodo aaee068b): Die eigene Zucht wurde in
|
||
# Quelldaten teils als „Clan (der) kleine(n) Chaoten" geführt. Die Züchterin will
|
||
# überall „Zucht der Kleinen Chaoten" (Groß-K). Finaler Anzeige-Sweep NACH aller
|
||
# Logik (die interne Sentinel-Form „Zucht der kleinen Chaoten" wird oben verglichen,
|
||
# daher erst hier vereinheitlichen).
|
||
_origin_fixed = 0
|
||
for g in resolved_gerbils:
|
||
ob = g.get("OriginBreeder")
|
||
if not ob:
|
||
continue
|
||
low = ob.lower()
|
||
if ("chaoten" in low and ("clan" in low or "zucht" in low)):
|
||
if ob != "Zucht der Kleinen Chaoten":
|
||
g["OriginBreeder"] = "Zucht der Kleinen Chaoten"
|
||
_origin_fixed += 1
|
||
if _origin_fixed:
|
||
print(f"Herkunft normalisiert auf 'Zucht der Kleinen Chaoten': {_origin_fixed} Tier(e)")
|
||
|
||
# 6. Save final output JSON payload
|
||
os.makedirs(OUTPUT_DIR, exist_ok=True)
|
||
payload = {
|
||
"contacts": resolved_contacts,
|
||
"litters": resolved_litters,
|
||
"gerbils": resolved_gerbils,
|
||
"gerbilPhotos": resolved_photos,
|
||
"saleContracts": sale_contracts,
|
||
}
|
||
|
||
with open(OUTPUT_FILE, "w", encoding="utf-8") as f:
|
||
json.dump(payload, f, ensure_ascii=False, indent=2)
|
||
|
||
print(f"Successfully wrote database-ready import file to: {OUTPUT_FILE}")
|
||
print(f" Contacts: {len(payload['contacts'])}")
|
||
print(f" Litters: {len(payload['litters'])}")
|
||
print(f" Gerbils: {len(payload['gerbils'])}")
|
||
print(f" Photos: {len(payload['gerbilPhotos'])}")
|
||
print(f" SaleContracts: {len(payload['saleContracts'])}")
|
||
|
||
if __name__ == "__main__":
|
||
main()
|