Gulum
2e7911074f
Daten-Fixes (conflict-decisions.json, re-ingest-stabil) für ~30 Tickets:
Merges (Jamie/Hiro/Mino/Jana/Blacky/Sakura/Malou/Socke→Marty), Eltern-Korrekturen
(Jacky/Idefix/Ichika/Roni/Ethan), Kruke→Kuke (+ Todesdatum), Targa-Wurf R14 + Druna,
Stacy/Merle/Domi/Eliza; Joghurt-Phantomwurf entfernt.
Code-Fixes:
- Gaida & alle Verstorbenen: Status wird aus Todesdatum/Abgabe abgeleitet
(Program.cs Startup-Sweep heilt Altfälle; IngestResolved re-derived nach Freeze).
- CoCo: Scheckungsart wird bei jeder Schecke angezeigt (Platzhalter wenn leer).
- M-Wurf/Gale: über-gemergte Fremdtiere via neuem litterChildren-Override entfernt.
- renameTo eltern-verknüpfungssicher (Quell-Name im Index); dateOfDeath als Override.
Prod-fähige Triage (API):
- GET /feedback/{id} + GET /feedback?status= (kein 2-MB-Dump).
- POST /import/ingest-resolved/upload (multipart) → Ingest gegen Prod ohne SSH.
Tests: 280 Backend, 149 Frontend, alle Python, betroffene Playwright grün.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
4442 lines
206 KiB
Python
4442 lines
206 KiB
Python
import os
|
||
import json
|
||
import re
|
||
import uuid
|
||
import sys
|
||
from datetime import datetime, timedelta
|
||
|
||
import genotype as gt
|
||
|
||
# Prevent encoding crashes on Windows consoles when printing unicode
|
||
if sys.platform.startswith('win'):
|
||
try:
|
||
sys.stdout.reconfigure(encoding='utf-8')
|
||
except Exception:
|
||
pass
|
||
|
||
# Paths
|
||
DIR_PATH = r"C:\Users\gulum\dev\Wurfchronik_Bilder"
|
||
SEEDS_PATH = r"C:\Users\gulum\dev\GerbilManager\gerbil-manager-web\src\genetics\colorVarietySeed.backend.json"
|
||
OUTPUT_DIR = r"C:\Users\gulum\dev\GerbilManager\tools\import\output"
|
||
OUTPUT_FILE = os.path.join(OUTPUT_DIR, "resolved_import.json")
|
||
CONFLICT_DECISIONS_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)),
|
||
"conflict-decisions.json")
|
||
|
||
|
||
def load_conflict_decisions():
|
||
"""Load tools/import/conflict-decisions.json (the breeder's authoritative
|
||
answers). Returns (resolutions, addAnimals, addLitters). Tolerates a
|
||
missing/garbled file by returning empty lists — the import must never crash
|
||
on a bad decision file."""
|
||
try:
|
||
with open(CONFLICT_DECISIONS_PATH, encoding="utf-8") as fh:
|
||
data = json.load(fh)
|
||
return (data.get("resolutions") or [], data.get("addAnimals") or [],
|
||
data.get("addLitters") or [])
|
||
except (OSError, ValueError):
|
||
return ([], [], [])
|
||
|
||
|
||
def load_suppress_refs():
|
||
"""Load suppressExternalRefs from conflict-decisions.json — the ExternalRefs of
|
||
gerbils/litters the breeder confirmed are phantoms/OCR-misread duplicates and that
|
||
should be dropped from the final payload (e.g. 'Muga' = misread 'Nduga'). Tolerates
|
||
a missing/garbled file."""
|
||
try:
|
||
with open(CONFLICT_DECISIONS_PATH, encoding="utf-8") as fh:
|
||
data = json.load(fh)
|
||
return set(data.get("suppressExternalRefs") or [])
|
||
except (OSError, ValueError):
|
||
return set()
|
||
|
||
|
||
def _norm_decision_gender(g):
|
||
"""Map a free-text gender override ('m'/'w'/'männlich'/…) to 'male'/'female'
|
||
or None when it is neither (so a typo never blanks a real gender)."""
|
||
g = (g or "").strip().lower()
|
||
return {"m": "male", "männlich": "male", "male": "male",
|
||
"w": "female", "f": "female", "weiblich": "female",
|
||
"female": "female"}.get(g)
|
||
|
||
def generate_guid(key_str):
|
||
"""Generate a stable UUID string based on a key."""
|
||
return str(uuid.uuid5(uuid.NAMESPACE_DNS, key_str))
|
||
|
||
def build_entity_provenance(source_files, merged_record_count, notes=None,
|
||
from_wurfchronik=None, extra=None, history=None):
|
||
"""Generic data-provenance JSON builder shared by gerbils, contacts and
|
||
litters. Mirrors the GerbilProvenance frontend contract:
|
||
{ sourceFiles, mergedRecordCount, fromWurfchronik, notes, history, ... }
|
||
`source_files` is any iterable of filenames; `from_wurfchronik` is auto-
|
||
derived from the filenames when left as None. `extra` may carry entity-
|
||
specific keys (e.g. parentMethod/parentConfidence for gerbils). `history`
|
||
is an ordered list of human-readable German lines that read like a
|
||
chronological log of where each fact came from (the primary content shown
|
||
in the Datenherkunft dialog). Returns a JSON string (stored on the nullable
|
||
Provenance text column)."""
|
||
files = sorted({f for f in source_files if f})
|
||
if from_wurfchronik is None:
|
||
from_wurfchronik = any("wurfchronik" in f.lower() for f in files)
|
||
prov = {
|
||
"sourceFiles": files,
|
||
"mergedRecordCount": merged_record_count,
|
||
"fromWurfchronik": bool(from_wurfchronik),
|
||
"notes": list(notes or []),
|
||
"history": list(history or []),
|
||
}
|
||
if extra:
|
||
for k, v in extra.items():
|
||
if v is not None:
|
||
prov[k] = v
|
||
return json.dumps(prov, ensure_ascii=False)
|
||
|
||
|
||
def _quote_file(fname):
|
||
"""German typographic quotes around a source filename for history lines."""
|
||
return f"„{fname}“"
|
||
|
||
|
||
# Human-readable German labels for the significant fields we attribute to files.
|
||
PROV_FIELD_LABELS = {
|
||
"DateOfBirth": "Geburtsdatum",
|
||
"DateOfDeath": "Sterbedatum",
|
||
"Gender": "Geschlecht",
|
||
"Genotype": "Genotyp",
|
||
"ColorVarietyId": "Farbschlag",
|
||
"Name": "Name",
|
||
}
|
||
|
||
|
||
def _primary_file_of(record):
|
||
"""The single most representative source file of one raw record.
|
||
|
||
Prefers an explicit Stammbaum/Wurfchronik filename from ImportSource, else
|
||
the record's _filename. Used to attribute a field value to a concrete file
|
||
in the history log."""
|
||
fn = record.get("_filename")
|
||
if fn:
|
||
return fn
|
||
imp = record.get("ImportSource")
|
||
if imp:
|
||
first = str(imp).split(",")[0].strip()
|
||
if first:
|
||
return first
|
||
return None
|
||
|
||
|
||
def _record_source_files(g):
|
||
"""All distinct source files a single raw gerbil record drew from.
|
||
|
||
ImportSource is either a comma-joined Stammbaum file list, a single
|
||
Wurfchronik filename, or the per-litter filename; _filename is the primary
|
||
file. We union both so nothing is lost."""
|
||
files = set()
|
||
imp = g.get("ImportSource")
|
||
if imp:
|
||
for part in str(imp).split(","):
|
||
part = part.strip()
|
||
if part:
|
||
files.add(part)
|
||
fn = g.get("_filename")
|
||
if fn:
|
||
files.add(fn)
|
||
return files
|
||
|
||
def to_valid_guid(val):
|
||
if not val:
|
||
return None
|
||
val_str = str(val).strip()
|
||
try:
|
||
uuid.UUID(val_str)
|
||
return val_str
|
||
except ValueError:
|
||
return generate_guid(val_str)
|
||
|
||
def normalize_name(name):
|
||
if not name:
|
||
return ""
|
||
return "".join(c for c in name.lower() if c.isalnum())
|
||
|
||
|
||
_EXTERNAL_MARKERS_RE = re.compile(
|
||
r"\b(zooladen|zoohandlung|obi|fressnapf|dehner|von\s+privat|privatkauf|"
|
||
r"vom\s+bauern|aus\s+der\s+zoohandlung)\b", re.IGNORECASE)
|
||
_FOREIGN_FROM_RE = re.compile(
|
||
r"\bfrom\b.+,\s*(croatia|kroatien|poland|polen|netherlands|niederlande|"
|
||
r"belgium|belgien|france|frankreich|austria|österreich|switzerland|schweiz|"
|
||
r"italy|italien|spain|spanien|czech|tschechien|hungary|ungarn)\b", re.IGNORECASE)
|
||
|
||
|
||
def is_external_origin(name, zucht=None, breeder=None):
|
||
"""Externally-acquired founder with genuinely unknown ancestry (pet shop,
|
||
private hobbyist, foreign cattery). Mirrors extract.is_external_origin so the
|
||
merge stage never attaches chart/Wurfchronik parents to such animals."""
|
||
blob = " ".join(p for p in (name, zucht, breeder) if p)
|
||
if _EXTERNAL_MARKERS_RE.search(blob):
|
||
return True
|
||
if _FOREIGN_FROM_RE.search(name or ""):
|
||
return True
|
||
return False
|
||
|
||
|
||
def canon_name_key(name):
|
||
"""Connector-folding name key: collapses the cattery connectors so that
|
||
abbreviation variants of the SAME animal match — e.g. „BlackFire v.d.
|
||
Kleinen Chaoten“ and „BlackFire von den Kleinen Chaoten“ both fold to the
|
||
same key (ticket #30). 'v.d.' / 'v. d.' → 'von den', then alnum-reduced.
|
||
Used as a secondary index next to normalize_name (never for GUIDs)."""
|
||
if not name:
|
||
return ""
|
||
n = name.lower()
|
||
n = re.sub(r"\bv\.?\s*d\.?\b", "von den", n) # v.d. / v. d. / vd → von den
|
||
return "".join(c for c in n if c.isalnum())
|
||
|
||
def get_normalized_contact_name(name):
|
||
if not name:
|
||
return "", False
|
||
|
||
n = "".join(c for c in name.lower() if c.isalnum())
|
||
|
||
to_discard = {
|
||
"chevroletcamarooftopolino",
|
||
"cindyvprivatzuchtgießen",
|
||
"cindyvprivatzuchtgiessen",
|
||
"inuschofblackforest",
|
||
"stichvonprivatzuchtgießen",
|
||
"stichvonprivatzuchtgiessen",
|
||
"tarzanofsamsimar",
|
||
"hanserennersposeidon",
|
||
"livingforcesidefix",
|
||
"livingforcesnando"
|
||
}
|
||
if n in to_discard:
|
||
return "", False
|
||
|
||
norm_map = {
|
||
"alessandrab": "Alessandra Bartoletti",
|
||
"alexsandrab": "Alessandra Bartoletti",
|
||
"andreafey": "Andrea und Stefanie Fey",
|
||
"andreastefaniefey": "Andrea und Stefanie Fey",
|
||
"anettw": "Annett Wernecke",
|
||
"angelachristoph": "Angela und Ekki Christoph",
|
||
"angelaekkichristoph": "Angela und Ekki Christoph",
|
||
"angie": "Angie Reichert-Cambeis",
|
||
"angierc": "Angie Reichert-Cambeis",
|
||
"anjasch": "Anja Schaumburg",
|
||
"ankeb": "Anke Busch",
|
||
"ankeksch": "Anke Koppenhöfer",
|
||
"annaq": "Anna Quark",
|
||
"annikab": "Annika Balser",
|
||
"annkathrind": "Ann-kathrin Dressler",
|
||
"arturaleksandrapolamundrzynski": "Artur, Aleksandra und Pola Mundrzynski",
|
||
"arturaleksandraundpolamundrzynski": "Artur, Aleksandra und Pola Mundrzynski",
|
||
"astridr": "Astrid Rohmann",
|
||
"astridroh": "Astrid Rohmann",
|
||
"babarastehle": "Barbara Stehle",
|
||
"barbarar": "Barbara Riegler",
|
||
"biancab": "Bianca Bernhardt",
|
||
"biancam": "Bianca Mayer + Alex Wachten",
|
||
"biancamayeralexw": "Bianca Mayer + Alex Wachten",
|
||
"birgitk": "Birgit Knopp",
|
||
"birgitkropp": "Birgit Knopp",
|
||
"birgittabüskens": "Birgitta Müller-Büskens",
|
||
"birgittamüllerbüskens": "Brigitta Müller-Büskens",
|
||
"blackforest": "Clan of Black Forest",
|
||
"blackforestgv": "Clan of Black Forest",
|
||
"brigittast": "Brigitta Struve",
|
||
"buntefellnasen": "Zucht von den bunten Fellnasen",
|
||
"buntenfellnase": "Zucht von den bunten Fellnasen",
|
||
"buntenfellnasen": "Zucht von den bunten Fellnasen",
|
||
"carolal": "Carola und Hannah Lerch",
|
||
"carolalerch": "Carola und Hannah Lerch",
|
||
"chalfontstud": "Chalfont Stud, Freddy Braun",
|
||
"christianefuchs": "Christiane Fuchs + Michael Höhnert",
|
||
"christiek": "Christiane Kiessling",
|
||
"christinam": "Christine Monika Mai",
|
||
"christinek": "Christiane Kiessling",
|
||
"christiner": "Christina Rudloff",
|
||
"claudialöhr": "Claudia Löhr und Michael Koob",
|
||
"claudialöhrmichaelkoob": "Claudia Löhr und Michael Koob",
|
||
"colorfulfurrygerbils": "Colorful Furry Gerbils",
|
||
"colourfulfurrygerbils": "Colorful Furry Gerbils",
|
||
"danielakasberger": "Daniela Käsberger",
|
||
"dörthe": "Dörthe Petzmann",
|
||
"emelyhehlhorn": "Emely Mehlhorn",
|
||
"fabianb": "Fabian Büdel",
|
||
"floriang": "Florian Gries",
|
||
"hannahraths": "Hannah Ruths",
|
||
"hanserenners": "Hanse Renner",
|
||
"haraldmariareitzlennemann": "Harald und Maria Reitz-Lennemann",
|
||
"haraldundmariarl": "Harald und Maria Reitz-Lennemann",
|
||
"heike": "Heike + Heiko Scheurich",
|
||
"heikebarklay": "Heike Barklage",
|
||
"heikoloos": "Heiko Loos und Annette Becker",
|
||
"heikoloosannettebecker": "Heiko Loos und Annette Becker",
|
||
"inahübner": "Ina Hübner + Nicolai Thome",
|
||
"inapeines": "Ina Pleines",
|
||
"janab": "Jana Beikert",
|
||
"jasiminweber": "Jasmin Weber und Ronny Reichelt",
|
||
"jasmin": "Jasmin Weber",
|
||
"jasminweber": "Jasmin Weber und Ronny Reichelt",
|
||
"jasminweberromyreichelt": "Jasmin Weber und Ronny Reichelt",
|
||
"jasminweberronnyreichelt": "Jasmin Weber und Ronny Reichelt",
|
||
"joystaatberg": "Joy Staalberg",
|
||
"juliaa": "Julia Ast",
|
||
"juttam": "Jutta Metz",
|
||
"karina": "Karina Luft",
|
||
"karinborgotti": "Karin Borsotti",
|
||
"katjal": "Katja Leffeck",
|
||
"kimw": "Kim Waldschmidt",
|
||
"kkchaos": "KK-Chaos",
|
||
"kkchaosofkkchaos": "KK-Chaos",
|
||
"kleinechaoten": "Zucht der kleinen Chaoten",
|
||
"kleinenchaoten": "Zucht der kleinen Chaoten",
|
||
"kriegernmitkrallen": "Krieger mit Krallen",
|
||
"lennylengo": "Lenny Lengo",
|
||
"lisa": "Lisa und Marcel Kunz",
|
||
"lisach": "Lisa und Lydia Christ",
|
||
"lisachrist": "Lisa und Lydia Christ",
|
||
"lisalydiachrist": "Lisa und Lydia Christ",
|
||
"littlefellows": "little fellows",
|
||
"littlerunners": "little runners",
|
||
"maikef": "Maike Franz",
|
||
"maintalerpz": "Privatzucht Maintal",
|
||
"marinau": "Marina Unger",
|
||
"martinaandreaswestfeld": "Martina und Andreas Westfeld",
|
||
"martinaw": "Martina und Andreas Westfeld",
|
||
"martinawestfeld": "Martina und Andreas Westfeld",
|
||
"meikesch": "Meike und Heiko Sch.",
|
||
"melanief": "Melanie Fey und Thorben Meier",
|
||
"melaniefey": "Melanie Fey und Thorben Meier",
|
||
"melaniefeythorbenmeier": "Melanie Fey und Thorben Meier",
|
||
"michaeldavidschmitz": "Michael und David Schmitz",
|
||
"michaelschmitz": "Michael und David Schmitz",
|
||
"nataliereitz": "Nathalie Reitz",
|
||
"natascham": "Natascha Marienfeld",
|
||
"nicolefischler": "Nicole Tischler",
|
||
"nicolel": "Nicole Lannert",
|
||
"nicolen": "Nicole Nuzzo",
|
||
"nicolet": "Nicole Tischler",
|
||
"nicolew": "Nicole Webersinn",
|
||
"nicost": "Nico Stamm",
|
||
"nielsh": "Nils H.",
|
||
"noelstrahbach": "Noel Strohbach",
|
||
"nora": "Nora Rudersdorf geb. Holzbach",
|
||
"norah": "Nora Holzbach",
|
||
"norarh": "Nora Rudersdorf",
|
||
"norarudersdorf": "Nora Rudersdorf geb. Holzbach",
|
||
"oflennylengo": "Lenny Lengo",
|
||
"pascale": "Pascale Diefenbach",
|
||
"pascaledießenbach": "Pascale Diefenbach",
|
||
"patriciap": "Patricia Petry",
|
||
"paul": "Paul W.",
|
||
"paula": "Paula Gabler",
|
||
"paulag": "Paula Gabler",
|
||
"privatzuchtmuecke": "Privatzucht Mücke",
|
||
"privatzuchtmücke": "Privatzucht Mücke",
|
||
"pzmaintal": "Privatzucht Maintal",
|
||
"pzmuecke": "Privatzucht Mücke",
|
||
"pzmücke": "Privatzucht Mücke",
|
||
"pzseligenstadt": "Privatzucht Seligenstadt",
|
||
"ramonag": "Ramona Gömpel",
|
||
"ranialößler": "Rania Löffler",
|
||
"rominahubrich": "Romina Milde / Hubrich",
|
||
"rominamilde": "Romina Milde / Hubrich",
|
||
"romonag": "Ramona Gömpel",
|
||
"ronialöffler": "Rania Löffler",
|
||
"sandrak": "Sandra Kubas",
|
||
"sarahlöser": "Sarah Löwer",
|
||
"sarahlöuer": "Sarah Löwer",
|
||
"sarahz": "Sarah Zitzer",
|
||
"saskiaw": "Saskia Wucher",
|
||
"schlossmaus": "Schlossmäuse",
|
||
"schlossmäuse": "Schlossmäuse",
|
||
"schlossmäusen": "Schlossmäuse",
|
||
"sebastiansch": "Sebastian Schmitt",
|
||
"serinaberg": "Selina Berg",
|
||
"silkewolfganghintze": "Silke und Wolfgang Hintze",
|
||
"smilla": "Smilla H.",
|
||
"steffi": "Steffi K.",
|
||
"susanneninat": "Susanne + Nina Thomas",
|
||
"susannet": "Susanne + Torsten Saum",
|
||
"theresavalenca": "Theresia Valenca (Kopp)",
|
||
"theresiavalenca": "Theresia Valenca (Kopp)",
|
||
"timpf": "Tim Pfeiffer und Michelle Mai",
|
||
"timpfeifermichellemai": "Tim Pfeiffer und Michelle Mai",
|
||
"timpfeiffer": "Tim Pfeiffer und Michelle Mai",
|
||
"timpfeiffermichellemai": "Tim Pfeiffer und Michelle Mai",
|
||
"tonoböckenseld": "Tono Böckenfeld",
|
||
"topol": "Topolino",
|
||
"ulrichmüller": "Ulrich + Angela Müller",
|
||
"ulrikec": "Ulrike Cordes",
|
||
"ulriker": "Ulrike Ruppel",
|
||
"ulrikesch": "Ulrike Schulz",
|
||
"ulriket": "Ulrike Treml",
|
||
"vanessab": "Vanessa Becker",
|
||
"vanessag": "Vanessa Groll",
|
||
"veragreywitz": "Vera Geywitz",
|
||
"vonprivatzuchtmaintal": "Privatzucht Maintal",
|
||
"wolfgangfaus": "Wolfgang Faust",
|
||
"yvonne": "Yvonne Obendorfer",
|
||
"zuchtderkleinenchaoten": "Zucht der kleinen Chaoten",
|
||
}
|
||
|
||
if n in norm_map:
|
||
return norm_map[n], True
|
||
|
||
return name, True
|
||
|
||
def get_normalized_gerbil_name(name):
|
||
if not name:
|
||
return ""
|
||
n = name.strip()
|
||
norm_key = "".join(c for c in n.lower() if c.isalnum())
|
||
|
||
gerbil_norm_map = {
|
||
"samgenshellyvdbuntenfellnasen": "Sammy gen. Shelly von den bunten Fellnasen",
|
||
"sammygenshellyvdbuntenfellnasen": "Sammy gen. Shelly von den bunten Fellnasen",
|
||
"schmidt": "Schmidti",
|
||
"sheila": "Sheila of Ulmer Strolche",
|
||
"shinichi": "Shinichi von PZ Mücke",
|
||
"silenosgenadonis": "Silenos gen. Adonis von den Kleinen Chaoten",
|
||
"silenosgenadonisvdkleinenchaoten": "Silenos gen. Adonis von den Kleinen Chaoten",
|
||
"silver": "Silver von den kleinen Chaoten",
|
||
"snoops": "Snoopsi",
|
||
"sokrates": "Sokrates von den Kleinen Chaoten",
|
||
"splash": "Slash",
|
||
"teiko": "Teiko von den kleinen Chaoten",
|
||
"trixy": "Trixxy von den Kleinen Chaoten",
|
||
"unique": "Unique of Wild Dreams",
|
||
}
|
||
|
||
if norm_key in gerbil_norm_map:
|
||
return gerbil_norm_map[norm_key]
|
||
|
||
return n
|
||
|
||
def get_call_name(name):
|
||
if not name:
|
||
return ""
|
||
n = name.strip()
|
||
n = re.sub(r'\[[^\]]+\]$', '', n).strip()
|
||
n = re.split(r'\s+(?:of|von\s+den|von\s+der|v\.\s?d\.|von)\s+', n, flags=re.IGNORECASE)[0].strip()
|
||
return n
|
||
|
||
def get_dedup_name_key(name):
|
||
if not name:
|
||
return ""
|
||
n = name.lower().strip()
|
||
# Strip common suffixes/prefixes and parentheticals
|
||
n = re.sub(r'\b(?:von\s+den|v\.?\s*d\.?|v\.?o\.?)\s+(?:kleinen\s+)?chaoten\b', '', n)
|
||
n = re.sub(r'\bvon\s+der\s+schlossm\w+\b', '', n)
|
||
n = re.sub(r'\bvon\s+der\s+bunten\s+fellnasen?\b', '', n)
|
||
n = re.sub(r'\bof\s+black\s+forest\b', '', n)
|
||
n = re.sub(r'\b\(?rv\)?\b', '', n)
|
||
n = re.sub(r'\bgen\.\s+\w+', '', n)
|
||
# Clean up parentheses or brackets
|
||
n = re.sub(r'\(.*?\)', '', n)
|
||
n = re.sub(r'\[.*?\]', '', n)
|
||
return "".join(c for c in n if c.isalnum())
|
||
|
||
def clean_color_name(c_desc):
|
||
"""Normalise a free-text colour label to a catalog key + Schecke flag.
|
||
|
||
Returns (clean_name, is_schecke). A PARENTHETICAL „(schimmel)" is NOT a
|
||
definitive Schimmel — the breeder writes it to mean „könnte sich später als
|
||
Schimmel entpuppen" (ticket e22764aa). So we STRIP the „(…)" instead of
|
||
folding it into the name (which used to turn „Blaufuchs(schimmel)" into the
|
||
wrong „blaufuchsschimmel"); the still-uncertain Schimmel-modifier is carried
|
||
by the genotype (ee[-] = Fuchs, Schimmel unknown), not the colour label.
|
||
"""
|
||
if not c_desc:
|
||
return "", False
|
||
|
||
# Lowercase and strip
|
||
c = c_desc.lower().strip()
|
||
|
||
# Check for Schecke
|
||
is_schecke = False
|
||
if re.search(r'\bsp\b|\bsp\d|\bsp[*(²³]|\bspotted|\bschecke|[- ]sp\b|\w+sp\b', c):
|
||
is_schecke = True
|
||
|
||
# Strip schecke/sp markers and any trailing text starting from sp
|
||
c = re.sub(r'\([- ]?sp(otted)?\)', '', c) # handles (-sp)
|
||
c = re.sub(r'[- ]?sp(otted)?\b.*', '', c) # handles -sp(k), -sp*(k), -sp, etc.
|
||
c = re.sub(r'[- ]?schecke\b.*', '', c)
|
||
c = re.sub(r'[- ]?spotted\b.*', '', c)
|
||
|
||
# Strip any other parentheticals (incl. „(schimmel)" = „möglich/unbestimmt"),
|
||
# symbols, or trailing stars/numbers. The parenthetical is deliberately NOT
|
||
# promoted to a definitive part of the colour name (ticket e22764aa).
|
||
c = re.sub(r'\s*\(.*?\)\s*', ' ', c)
|
||
c = re.sub(r'[²³*]', '', c)
|
||
c = c.strip()
|
||
|
||
# Mapping table for abbreviations, typos, and specific combinations
|
||
mapping = {
|
||
"antra": "anthrazit",
|
||
"anthra": "anthrazit",
|
||
"ankazit": "anthrazit",
|
||
"antrazit": "anthrazit",
|
||
"pew": "rew",
|
||
"bew": "hermelin",
|
||
"harder": "marder",
|
||
"kohli": "kohlfuchs",
|
||
"aligerfuchs": "algierfuchs",
|
||
"algiesfuchs": "algierfuchs",
|
||
"schw": "schwarz",
|
||
"sa": "silberagouti",
|
||
"a": "agouti",
|
||
"cp-sa": "cp-silberagouti",
|
||
"cp-a": "cp-agouti",
|
||
"cp-a-hell": "cp-agouti-hell",
|
||
"cp-aisa": "cp-agouti",
|
||
"cp-a / rcp-sa": "cp-agouti",
|
||
}
|
||
|
||
if c in mapping:
|
||
c = mapping[c]
|
||
|
||
return c, is_schecke
|
||
|
||
def _match_color_label(clean_name, variety_map):
|
||
"""Map a cleaned colour label to a ColorVariety id (text-only path).
|
||
|
||
Exact name wins; otherwise pick the LONGEST/most-specific substring match
|
||
(ticket 3f5942a2 — the old code broke on the FIRST substring hit, so „Goldfuchs"
|
||
matched the shorter „Gold" first). Among substring candidates the longest seed
|
||
name wins, then the longest clean_name overlap; ties broken deterministically.
|
||
"""
|
||
if not clean_name:
|
||
return None
|
||
if clean_name in variety_map:
|
||
return variety_map[clean_name]
|
||
candidates = []
|
||
for seed_name, seed_id in variety_map.items():
|
||
if not seed_name:
|
||
continue
|
||
if seed_name in clean_name or clean_name in seed_name:
|
||
# Specificity score: prefer the longer seed name (more specific),
|
||
# then the closeness of lengths so „goldfuchs" beats „gold" for the
|
||
# label „goldfuchs".
|
||
candidates.append((len(seed_name), -abs(len(seed_name) - len(clean_name)),
|
||
seed_name, seed_id))
|
||
if not candidates:
|
||
return None
|
||
candidates.sort(reverse=True)
|
||
return candidates[0][3]
|
||
|
||
|
||
def resolve_color_and_genotype(color_val, existing_genotype, variety_map, variety_genotypes):
|
||
"""Resolve a gerbil's stored ColorVariety id + genotype.
|
||
|
||
GENOTYPE WINS (ticket cluster genetics-farbschlag): when a parseable genotype
|
||
is present and the genetics engine (genotype.genotype_to_farbschlag — a faithful
|
||
Python mirror of catalog.ts) computes a KNOWN catalog variety, that variety is
|
||
authoritative for colorVarietyId. The free-text colour label is only a fallback
|
||
(no genotype, or genotype resolves to „Unbekannt"). This fixes the imports where
|
||
the source label ignored a locus (dd → „Agouti" instead of „Dilute Agouti",
|
||
ee → „Gold" instead of „Goldfuchs", parenthetical „(schimmel)", …).
|
||
|
||
Returns (color_variety_id, genotype). `genotype` is the (possibly Schecke-
|
||
annotated) genotype STRING — never silently flips an explicit spsp to Spsp.
|
||
"""
|
||
if not color_val and not existing_genotype:
|
||
return None, existing_genotype
|
||
|
||
clean_name, is_schecke = clean_color_name(str(color_val).strip()) if color_val else ("", False)
|
||
|
||
# 1) Genotype-derived variety (authoritative when it resolves to a known name).
|
||
# GUARD (VORSICHTIG): only trust the genotype when it parsed CLEANLY enough to
|
||
# decide a colour — both the C and E loci must be mapped. The breeder sometimes
|
||
# writes the genotype in the COMPACT catalog notation („cchmcchm", „efef",
|
||
# „chch") which this parser leaves UNMAPPED (it expects the bracketed „c[chm]"
|
||
# form); a dropped C/E locus would silently read as wild-type and mis-recolour
|
||
# an otherwise-correct animal (e.g. Marder→Schwarz, Orangeschimmel→Agouti). When
|
||
# the parse is incomplete we keep the source text label instead.
|
||
color_variety_id = None
|
||
geno_name = None
|
||
if existing_genotype:
|
||
try:
|
||
mapped = gt.parse(existing_genotype).get("mapped8locus") or {}
|
||
except Exception:
|
||
mapped = {}
|
||
if mapped.get("C") and mapped.get("E"):
|
||
fs = gt.genotype_to_farbschlag(mapped)
|
||
if fs and fs != gt.UNKNOWN_FARBSCHLAG:
|
||
geno_name = fs
|
||
color_variety_id = variety_map.get(fs.strip().lower())
|
||
|
||
# 2) Fall back to the text label when the genotype gave nothing usable.
|
||
if not color_variety_id:
|
||
color_variety_id = _match_color_label(clean_name, variety_map)
|
||
|
||
# Update genotype if the LABEL says Schecke — but never override an explicit
|
||
# Sp-locus already present in the source genotype (ticket e09d6f22: a source
|
||
# „spsp" must NOT be flipped to „Spsp" just because the label looked scheckig;
|
||
# the source genotype is authoritative for the Sp-locus). Only ADD Spsp when
|
||
# the genotype carries no Sp token at all.
|
||
genotype = existing_genotype
|
||
if is_schecke:
|
||
if genotype:
|
||
if "Sp" not in genotype and "sp" not in genotype:
|
||
genotype = f"{genotype} Spsp".strip()
|
||
else:
|
||
canonical = variety_genotypes.get(color_variety_id)
|
||
if canonical:
|
||
if "spsp" in canonical:
|
||
genotype = canonical.replace("spsp", "Spsp")
|
||
else:
|
||
genotype = f"{canonical} Spsp".strip()
|
||
else:
|
||
genotype = "Spsp"
|
||
return color_variety_id, genotype
|
||
|
||
def parse_date(d):
|
||
"""Convert variations of date formats to YYYY-MM-DD."""
|
||
if not d or d == "0001-01-01":
|
||
return None
|
||
d = str(d).strip()
|
||
# Try YYYY-MM-DD
|
||
if re.match(r"^\d{4}-\d{2}-\d{2}$", d):
|
||
parts = d.split("-")
|
||
year = int(parts[0])
|
||
if 1900 < year < 2000:
|
||
year += 100
|
||
d = f"{year}-{parts[1]}-{parts[2]}"
|
||
elif year == 1900:
|
||
return None
|
||
return d
|
||
# Try DD.MM.YYYY or D.M.YY
|
||
match = re.match(r"^(\d{1,2})\.(\d{1,2})\.(\d{2,4})$", d)
|
||
if match:
|
||
day = int(match.group(1))
|
||
month = int(match.group(2))
|
||
year = int(match.group(3))
|
||
if year < 100:
|
||
year += 2000
|
||
elif 1900 < year < 2000:
|
||
year += 100
|
||
elif year == 1900:
|
||
return None
|
||
try:
|
||
return datetime(year, month, day).strftime("%Y-%m-%d")
|
||
except ValueError:
|
||
pass
|
||
# Try ISO timestamp
|
||
try:
|
||
dt = datetime.fromisoformat(d.replace("Z", "+00:00"))
|
||
year = dt.year
|
||
if 1900 < year < 2000:
|
||
dt = dt.replace(year=year + 100)
|
||
return dt.strftime("%Y-%m-%d")
|
||
elif year == 1900:
|
||
return None
|
||
return dt.strftime("%Y-%m-%d")
|
||
except ValueError:
|
||
pass
|
||
return None
|
||
|
||
def date_to_days(dt_str):
|
||
if not dt_str:
|
||
return None
|
||
try:
|
||
return (datetime.strptime(dt_str, "%Y-%m-%d") - datetime(2000, 1, 1)).days
|
||
except ValueError:
|
||
return None
|
||
|
||
def days_to_date(days):
|
||
import datetime as dt
|
||
return (dt.datetime(2000, 1, 1) + dt.timedelta(days=int(days))).strftime("%Y-%m-%d")
|
||
|
||
def parse_death_info(notes, status, existing_dod, existing_cod):
|
||
if not notes:
|
||
return status, existing_dod, existing_cod
|
||
|
||
has_death_indicator = '†' in notes or 'verstorben' in notes.lower() or 'gestorben' in notes.lower() or 'todesdatum' in notes.lower() or '/+' in notes
|
||
if '+' in notes:
|
||
if re.search(r'\+\s*(?:LE|AS|Unbekannt|gestorben|verstorben|tod)\b', notes, re.I) or re.search(r'\+\s*\d{1,2}\.\d{1,2}\.\d{2,4}', notes) or '/+' in notes:
|
||
has_death_indicator = True
|
||
|
||
resolved_status = status
|
||
if has_death_indicator:
|
||
resolved_status = "Deceased"
|
||
|
||
dod = existing_dod
|
||
cod = existing_cod
|
||
|
||
# Look for date near death indicator
|
||
found_date = None
|
||
for m in re.finditer(r'([+†]\s*(?:LE|AS|Unbekannt|[a-zA-ZäöüÄÖÜß0-9()/ +,;.:-]{1,100}?)?\s*)(\d{1,2}\.\d{1,2}\.\d{2,4})', notes, re.I):
|
||
marker_text = m.group(0)
|
||
if '†' in marker_text or re.search(r'\+\s*(?:LE|AS|Unbekannt|gestorben|verstorben|tod|\d)', marker_text, re.I):
|
||
found_date = parse_date(m.group(2))
|
||
if found_date:
|
||
break
|
||
|
||
if not found_date:
|
||
m_death_marker = re.search(r'(†\s*)(\d{1,2}\.\d{1,2}\.\d{2,4})', notes)
|
||
if m_death_marker:
|
||
found_date = parse_date(m_death_marker.group(2))
|
||
|
||
# FIX (ec9267b9): markerless death dates. When a death indicator is present but
|
||
# no +/cross marker precedes the date (e.g. "Verstorbener Welpe am 30.03.15 …",
|
||
# "Verstorben am 04.12.18 …"), take the FIRST plausible date as the death date
|
||
# — but only if we have a death keyword and still have no date. Status logic is
|
||
# unchanged; we never overwrite an existing dateOfDeath.
|
||
if not found_date and has_death_indicator and not existing_dod:
|
||
if re.search(r'\b(?:verstorb\w*|gestorb\w*|verstarb\w*|todesdatum)\b', notes, re.I):
|
||
m_plain = re.search(r'\b(\d{1,2}\.\d{1,2}\.\d{2,4})\b', notes)
|
||
if m_plain:
|
||
found_date = parse_date(m_plain.group(1))
|
||
|
||
if found_date and not dod:
|
||
dod = found_date
|
||
|
||
if not cod:
|
||
for m in re.finditer(r'([+†])\s*([a-zA-ZäöüÄÖÜß0-9()/ +,;.:-]{1,100}?)\s*\d{1,2}\.\d{1,2}\.\d{2,4}', notes, re.I):
|
||
indicator = m.group(1)
|
||
cod_candidate = m.group(2).strip()
|
||
if indicator == '+' and not re.search(r'\b(?:LE|AS|Unbekannt|gestorben|verstorben|tod)\b', cod_candidate, re.I):
|
||
continue
|
||
if cod_candidate:
|
||
cod = cod_candidate
|
||
break
|
||
|
||
if not cod:
|
||
m_cod2 = re.search(r'[+†]\s*\d{1,2}\.\d{1,2}\.\d{2,4}\s*([a-zA-ZäöüÄÖÜß0-9()/ +,;.:-]{1,100})', notes, re.I)
|
||
if m_cod2:
|
||
cod_candidate = m_cod2.group(1).strip()
|
||
if cod_candidate:
|
||
cod = cod_candidate
|
||
else:
|
||
m_cod3 = re.search(r'[+†]\s*(LE|AS|Unbekannt)\b', notes, re.I)
|
||
if m_cod3:
|
||
cod = m_cod3.group(1).strip()
|
||
|
||
if cod:
|
||
cod_lower = cod.lower().strip()
|
||
if cod_lower in ("le", "le (lungenentzündung)", "lungenentzündung", "lungenentzündung (lungenentzündung)"):
|
||
cod = "Lungenentzündung"
|
||
elif cod_lower in ("as", "altersschwäche", "altenschwäche", "alter"):
|
||
cod = "Altersschwäche"
|
||
elif cod_lower in ("unbekannt", "unklar"):
|
||
cod = "Unbekannt"
|
||
else:
|
||
# Replace abbreviations with full names (case-insensitive)
|
||
cod = re.sub(r'\bLE\b', 'Lungenentzündung', cod, flags=re.I)
|
||
cod = re.sub(r'\bAS\b', 'Altersschwäche', cod, flags=re.I)
|
||
cod = cod.replace('+', ' + ')
|
||
# Clean up multiple spaces
|
||
cod = re.sub(r'\s+', ' ', cod).strip(' ,;.-')
|
||
|
||
# Re-check after replacement
|
||
cod_lower = cod.lower().strip()
|
||
if cod_lower in ("le", "le (lungenentzündung)", "lungenentzündung", "lungenentzündung (lungenentzündung)"):
|
||
cod = "Lungenentzündung"
|
||
elif cod_lower in ("as", "altersschwäche", "altenschwäche", "alter"):
|
||
cod = "Altersschwäche"
|
||
elif cod_lower in ("unbekannt", "unklar"):
|
||
cod = "Unbekannt"
|
||
|
||
# Collapse redundant "X (X)" (e.g. Abkürzung + ausgeschriebene Form in Klammern,
|
||
# "AS (Altersschwäche)" → nach Expansion "Altersschwäche (Altersschwäche)") auf "X".
|
||
if cod:
|
||
m_dup = re.match(r'^\s*(.+?)\s*\(\s*(.+?)\s*\)\s*$', cod)
|
||
if m_dup and m_dup.group(1).strip().lower() == m_dup.group(2).strip().lower():
|
||
cod = m_dup.group(1).strip()
|
||
|
||
return resolved_status, dod, cod
|
||
|
||
|
||
# ── Litter dedup & parent-role helpers (pure, unit-tested in test_merge_resolve.py) ──
|
||
|
||
def _norm_pname(s):
|
||
return normalize_name(s) if s else ""
|
||
|
||
|
||
def names_no_conflict(l1, l2):
|
||
"""Parent names don't contradict (equal per role, or one side empty)."""
|
||
f1, f2 = _norm_pname(l1.get("_father_name")), _norm_pname(l2.get("_father_name"))
|
||
m1, m2 = _norm_pname(l1.get("_mother_name")), _norm_pname(l2.get("_mother_name"))
|
||
f_ok = (not f1) or (not f2) or (f1 == f2)
|
||
m_ok = (not m1) or (not m2) or (m1 == m2)
|
||
return f_ok and m_ok
|
||
|
||
|
||
def names_overlap(l1, l2):
|
||
"""At least one role has a non-empty matching name (positive evidence)."""
|
||
f1, f2 = _norm_pname(l1.get("_father_name")), _norm_pname(l2.get("_father_name"))
|
||
m1, m2 = _norm_pname(l1.get("_mother_name")), _norm_pname(l2.get("_mother_name"))
|
||
return bool((f1 and f1 == f2) or (m1 and m1 == m2))
|
||
|
||
|
||
def litter_compatible(l1, l2):
|
||
"""Two litter records describe the same litter: same date and compatible parents.
|
||
|
||
- Both sides have both parents → must match exactly.
|
||
- Asymmetric (one side resolved, the other not) → merge only if names don't
|
||
contradict; for dateless litters require a POSITIVE name match (a shared
|
||
null date is no evidence), so unrelated nameless stubs stay separate.
|
||
- Neither side has parents → never blind-merge.
|
||
"""
|
||
if l1["Date"] != l2["Date"]:
|
||
return False
|
||
f1, m1 = l1.get("FatherId"), l1.get("MotherId")
|
||
f2, m2 = l2.get("FatherId"), l2.get("MotherId")
|
||
if f1 and f2 and m1 and m2:
|
||
return f1 == f2 and m1 == m2
|
||
asymmetric = (bool(f1 or m1) and not (f2 or m2)) or (bool(f2 or m2) and not (f1 or m1))
|
||
if asymmetric:
|
||
if not names_no_conflict(l1, l2):
|
||
return False
|
||
if l1["Date"] is None:
|
||
return names_overlap(l1, l2)
|
||
return True
|
||
return False
|
||
|
||
|
||
def assign_parent_roles(father_id, mother_id, gender_of):
|
||
"""Assign two resolved parent IDs to father/mother roles by gender.
|
||
|
||
Drops self-pairing duplicates (same animal in both roles) and never returns
|
||
two same-role parents. `gender_of` maps an id to 'male'|'female'|'unknown'|None.
|
||
Returns (father_id, mother_id).
|
||
"""
|
||
ids = []
|
||
for gid in (father_id, mother_id):
|
||
if gid and gid not in ids:
|
||
ids.append(gid)
|
||
males = [g for g in ids if gender_of(g) == "male"]
|
||
females = [g for g in ids if gender_of(g) == "female"]
|
||
unknowns = [g for g in ids if gender_of(g) == "unknown"]
|
||
father = males[0] if males else (unknowns.pop(0) if unknowns else None)
|
||
mother = females[0] if females else (unknowns.pop(0) if unknowns else None)
|
||
return father, mother
|
||
|
||
|
||
# A gerbil lives at most ~6 years, so a parent can be at most ~6 years older than
|
||
# its offspring (and must be born before it). Links outside this window are
|
||
# impossible — e.g. a 2013 animal resolved onto a 2022 litter (Jayjay → Solice).
|
||
MAX_PARENT_AGE_DAYS = 2379
|
||
|
||
|
||
def parent_age_plausible(parent_dob, litter_date):
|
||
"""Could a parent born `parent_dob` have offspring born on `litter_date`?
|
||
|
||
Requires birth strictly before the litter and within the gerbil lifespan.
|
||
Unknown/unparseable dates return True (cannot disprove). Accepts any date
|
||
format parse_date understands.
|
||
"""
|
||
pd = date_to_days(parse_date(parent_dob)) if parent_dob else None
|
||
ld = date_to_days(parse_date(litter_date)) if litter_date else None
|
||
if pd is None or ld is None:
|
||
return True
|
||
return 0 < (ld - pd) <= MAX_PARENT_AGE_DAYS
|
||
|
||
|
||
def pick_parent_ref(parent_refs, role, child_dob, avoid_name=None, gender_of=None):
|
||
"""Choose the best parent ref for a role from possibly-conflicting chart refs.
|
||
|
||
A Stammbaum lists an animal at several positions, so its parentRefs can carry
|
||
contradictory guesses (the first one is not necessarily right). Rank candidates
|
||
(lower = better):
|
||
0 right/unknown gender for the role, age-plausible dated ref
|
||
1 right/unknown gender, no DOB (usable, but a plausible dated ref wins)
|
||
2 right/unknown gender, dated but age-impossible
|
||
3 resolved gender is clearly WRONG for the role (e.g. a female father)
|
||
4 would duplicate the animal chosen for the other role
|
||
|
||
Gender is decisive over DOB: a dated female ref must not win the father slot
|
||
over an undated male/unknown one. `gender_of(name)` returns 'male'/'female'
|
||
or None (unknown/ambiguous → not penalised). Returns the chosen ref or None.
|
||
"""
|
||
role_refs = [p for p in parent_refs if p.get("roleGuess") == role]
|
||
if not role_refs:
|
||
return None
|
||
avoid = normalize_name(avoid_name) if avoid_name else None
|
||
expected = "male" if role == "father" else "female"
|
||
|
||
def rank(p):
|
||
if avoid is not None and normalize_name(p.get("name")) == avoid:
|
||
return 4 # would duplicate the other parent role
|
||
g = gender_of(p.get("name")) if gender_of else None
|
||
if g in ("male", "female") and g != expected:
|
||
return 3 # wrong sex for this role
|
||
dob = p.get("dob")
|
||
if not dob:
|
||
return 1
|
||
return 0 if parent_age_plausible(dob, child_dob) else 2
|
||
|
||
order = sorted(range(len(role_refs)), key=lambda i: (rank(role_refs[i]), i))
|
||
return role_refs[order[0]]
|
||
|
||
|
||
def explain_pick_rejections(parent_refs, role, child_dob, chosen, avoid_name=None,
|
||
gender_of=None):
|
||
"""Explain why other refs for `role` lost to `chosen` in pick_parent_ref.
|
||
|
||
Returns a list of discard dicts (for _format_discard) — one per distinct
|
||
rejected candidate name that was beaten for a clear reason (wrong sex, age-
|
||
impossible, or duplicate of the other parent). Mirrors pick_parent_ref's
|
||
ranking so the history can explain the same decision it made.
|
||
"""
|
||
role_refs = [p for p in parent_refs if p.get("roleGuess") == role]
|
||
if not role_refs or chosen is None:
|
||
return []
|
||
avoid = normalize_name(avoid_name) if avoid_name else None
|
||
expected = "male" if role == "father" else "female"
|
||
role_de = "Vaterrolle" if role == "father" else "Mutterrolle"
|
||
cand_de = "Vater-Kandidat" if role == "father" else "Mutter-Kandidat"
|
||
chosen_name = chosen.get("name")
|
||
repl_disp = chosen_name
|
||
if chosen.get("dob"):
|
||
repl_disp = f"{chosen_name} (*{_de_date(parse_date(chosen.get('dob'))) or chosen.get('dob')})"
|
||
|
||
out = []
|
||
seen = set()
|
||
for p in role_refs:
|
||
name = p.get("name")
|
||
if not name or normalize_name(name) == normalize_name(chosen_name or ""):
|
||
continue
|
||
key = normalize_name(name)
|
||
if key in seen:
|
||
continue
|
||
g = gender_of(name) if gender_of else None
|
||
dob = p.get("dob")
|
||
reason = None
|
||
if avoid is not None and key == avoid:
|
||
reason = "bereits als anderer Elternteil gewählt"
|
||
elif g in ("male", "female") and g != expected:
|
||
reason = f"falsches Geschlecht für die {role_de}"
|
||
elif dob and not parent_age_plausible(dob, child_dob):
|
||
reason = "unplausibles Alter für diesen Wurf"
|
||
if reason is None:
|
||
continue
|
||
seen.add(key)
|
||
disp = name
|
||
if dob:
|
||
disp = f"{name} (*{_de_date(parse_date(dob)) or dob})"
|
||
out.append({
|
||
"label": cand_de,
|
||
"value": f"„{disp}“",
|
||
"reason": reason,
|
||
"replacement": f"„{repl_disp}“",
|
||
})
|
||
return out
|
||
|
||
|
||
def _build_gerbil_history(records, best_g, field_source, parent_method=None,
|
||
any_decision=False, any_conflict=False, conflict_notes=None,
|
||
field_discards=None, parent_discards=None):
|
||
"""Build an ordered, file-attributed German history for a resolved gerbil.
|
||
|
||
Reads like a chronological log:
|
||
• „In ‚X.xlsx' gefunden."
|
||
• „Geburtsdatum (27.03.2022) aus ‚X.xlsx'."
|
||
• „Auch in ‚Y.xlsx' gefunden → Datensätze zusammengeführt."
|
||
• „Genotyp aus ‚Z.xlsx'."
|
||
• „Eltern über Position im Stammbaum erkannt (Quelle: ‚X.xlsx')."
|
||
• „Aus Wurfchronik übernommen."
|
||
|
||
`field_source` maps a field name to the raw record that supplied its final
|
||
value; when present we name that record's file, otherwise we fall back to
|
||
the primary record. The records are visited in a stable order (primary
|
||
first, then the rest sorted by file) so the log is deterministic."""
|
||
conflict_notes = conflict_notes or []
|
||
history = []
|
||
|
||
# Order records: best_g first, then others by primary file name (stable).
|
||
others = [r for r in records if r is not best_g]
|
||
others.sort(key=lambda r: (_primary_file_of(r) or ""))
|
||
ordered = [best_g] + others
|
||
|
||
best_file = _primary_file_of(best_g)
|
||
if best_file:
|
||
history.append(f"In {_quote_file(best_file)} gefunden.")
|
||
else:
|
||
history.append("Im Import gefunden.")
|
||
|
||
# Field-by-field attribution: name the file that supplied each fact.
|
||
def attr_line(field, formatter):
|
||
rec = field_source.get(field) or best_g
|
||
val = best_g.get(field)
|
||
if not val or val == "unknown":
|
||
return
|
||
fname = _primary_file_of(rec)
|
||
label = PROV_FIELD_LABELS.get(field, field)
|
||
text = formatter(label, val)
|
||
if fname:
|
||
history.append(f"{text} aus {_quote_file(fname)}.")
|
||
else:
|
||
history.append(f"{text} (Quelle unbekannt).")
|
||
|
||
attr_line("DateOfBirth", lambda label, v: f"{label} ({_de_date(v)})")
|
||
attr_line("Gender", lambda label, v: f"{label} ({_de_gender(v)})")
|
||
attr_line("Genotype", lambda label, v: f"{label}")
|
||
attr_line("ColorVarietyId", lambda label, v: f"{label}")
|
||
attr_line("DateOfDeath", lambda label, v: f"{label} ({_de_date(v)})")
|
||
|
||
# Merge step: every additional record that contributed.
|
||
for r in others:
|
||
fname = _primary_file_of(r)
|
||
if fname:
|
||
history.append(
|
||
f"Auch in {_quote_file(fname)} gefunden → Datensätze zusammengeführt."
|
||
)
|
||
else:
|
||
history.append("In weiterem Datensatz gefunden → Datensätze zusammengeführt.")
|
||
|
||
# Discarded field values from majority-vote conflict resolution: the LOSING
|
||
# values, their file, and what won instead.
|
||
for d in (field_discards or []):
|
||
line = _format_discard(d)
|
||
if line not in history:
|
||
history.append(line)
|
||
|
||
# Parent derivation.
|
||
if parent_method:
|
||
method_label = {
|
||
"chart-position": "Position im Stammbaum",
|
||
"geburtsdatum+eltern": "Geburtsdatum und Elternnamen",
|
||
"nur-geburtsdatum": "Geburtsdatum",
|
||
"decision": "manuelle Entscheidung",
|
||
}.get(parent_method, parent_method)
|
||
# The parent evidence comes from a Stammbaum chart — attribute to the
|
||
# primary record's file when it is a Stammbaum.
|
||
parent_file = best_file if best_file and "stammbaum" in best_file.lower() else None
|
||
if parent_file:
|
||
history.append(
|
||
f"Eltern über {method_label} erkannt (Quelle: {_quote_file(parent_file)})."
|
||
)
|
||
else:
|
||
history.append(f"Eltern über {method_label} erkannt.")
|
||
|
||
# Discarded parent candidates / links (pick_parent_ref rejections, role
|
||
# normalization drops, parent-age sanity check). Threaded in after the merge
|
||
# via the gerbil's _discarded list so they read in chronological order.
|
||
for d in (parent_discards or []):
|
||
line = _format_discard(d)
|
||
if line not in history:
|
||
history.append(line)
|
||
|
||
# Manual decisions / conflicts.
|
||
if any_decision:
|
||
history.append("Zuordnung per manueller Entscheidung getroffen.")
|
||
if any_conflict:
|
||
history.append("Konflikt per Entscheidung gelöst.")
|
||
for cn in conflict_notes:
|
||
if cn and cn not in history:
|
||
history.append(cn + ".")
|
||
|
||
# Wurfchronik provenance line.
|
||
if any("wurfchronik" in f.lower() for r in records for f in _record_source_files(r)):
|
||
history.append("Angaben aus der Wurfchronik übernommen.")
|
||
|
||
return history
|
||
|
||
|
||
def _de_date(iso):
|
||
"""YYYY-MM-DD → DD.MM.YYYY for display; pass through anything else."""
|
||
if not iso:
|
||
return iso
|
||
m = re.match(r"^(\d{4})-(\d{2})-(\d{2})$", str(iso))
|
||
if m:
|
||
return f"{m.group(3)}.{m.group(2)}.{m.group(1)}"
|
||
return iso
|
||
|
||
|
||
def _de_gender(g):
|
||
return {"male": "männlich", "female": "weiblich"}.get(g, g)
|
||
|
||
|
||
# Leading marker that visually flags a discard ("data was thrown away") line in
|
||
# the history timeline. The frontend keys discard styling off this marker.
|
||
DISCARD_MARK = "⚠ "
|
||
|
||
|
||
def _format_discard(d):
|
||
"""Render one discard record into a German history line.
|
||
|
||
A discard record is a dict describing a value/candidate the pipeline threw
|
||
away. Recognised keys:
|
||
• text — a fully pre-formatted line (used verbatim, marker added)
|
||
• label — German field label (e.g. „Geburtsdatum“)
|
||
• value — the discarded value (already display-formatted)
|
||
• file — source file the discarded value came from (attributed)
|
||
• reason — why it was dropped (e.g. „abweichend“, „unplausibel …“)
|
||
• replacement — what was used instead (already display-formatted)
|
||
• repl_file — source file the replacement came from
|
||
Produces lines like:
|
||
„⚠ Geburtsdatum 14.06.2015 aus ‚A.xlsx‘ verworfen — abweichend;
|
||
14.06.2017 aus ‚B.xlsx‘ verwendet (Mehrheit).“
|
||
Generic across entity types so litters/contacts can reuse it.
|
||
"""
|
||
if d.get("text"):
|
||
return DISCARD_MARK + d["text"]
|
||
parts = []
|
||
label = d.get("label")
|
||
value = d.get("value")
|
||
if label and value is not None:
|
||
parts.append(f"{label} {value}")
|
||
elif label:
|
||
parts.append(str(label))
|
||
elif value is not None:
|
||
parts.append(str(value))
|
||
head = " ".join(parts) if parts else "Wert"
|
||
if d.get("file"):
|
||
head += f" aus {_quote_file(d['file'])}"
|
||
line = f"{head} verworfen"
|
||
if d.get("reason"):
|
||
line += f" — {d['reason']}"
|
||
repl = d.get("replacement")
|
||
if repl is not None and repl != "":
|
||
instead = str(repl)
|
||
if d.get("repl_file"):
|
||
instead += f" aus {_quote_file(d['repl_file'])}"
|
||
suffix = d.get("replacement_note")
|
||
line += f"; {instead} verwendet"
|
||
if suffix:
|
||
line += f" ({suffix})"
|
||
elif d.get("no_replacement"):
|
||
line += "; kein Ersatz"
|
||
return DISCARD_MARK + line + "."
|
||
|
||
|
||
def _append_history(prov_json, line):
|
||
"""Append one history line to an existing Provenance JSON string and add the
|
||
contract source file. Returns the updated JSON string."""
|
||
try:
|
||
prov = json.loads(prov_json) if prov_json else {}
|
||
except (ValueError, TypeError):
|
||
prov = {}
|
||
prov.setdefault("history", [])
|
||
if line not in prov["history"]:
|
||
prov["history"].append(line)
|
||
return json.dumps(prov, ensure_ascii=False)
|
||
|
||
|
||
def _parse_price(raw):
|
||
"""Parse a contract price string ('27,50' / '30.00' / '') → float (0.0 if empty).
|
||
|
||
extract_contracts.py emits German-formatted numbers ('27,50'); accept both
|
||
comma and dot decimal separators. Unparseable/empty → 0.0 (a price-less
|
||
contract is still a valid contract record)."""
|
||
if raw is None:
|
||
return 0.0
|
||
s = str(raw).strip()
|
||
if not s:
|
||
return 0.0
|
||
s = s.replace(".", "").replace(",", ".") if ("," in s) else s
|
||
try:
|
||
return round(float(s), 2)
|
||
except ValueError:
|
||
return 0.0
|
||
|
||
|
||
def enrich_from_contracts(contracts, resolved_gerbils, contact_by_norm_name,
|
||
contact_id_map, exclude_decisions=None):
|
||
"""Conservatively fold Abgabevertrag data into the resolved gerbils AND emit
|
||
one SaleContract record per contract with a resolvable buyer.
|
||
|
||
For every parsed contract we (a) ensure the buyer exists as a (receiver)
|
||
contact, reusing the existing contact dedup/normalisation, and (b) try to
|
||
match each animal call-name to exactly one resolved gerbil that the breeder
|
||
bred ("…Chaoten"). On a confident match we set ReceiverContactId /
|
||
GoHomeDate / Status=GivenAway — but only where not already set differently —
|
||
and add a provenance history line. Ambiguous or absent matches are logged,
|
||
never guessed.
|
||
|
||
In addition we build a `sale_contracts` list (one record per contract whose
|
||
buyer resolves to a contact). Each record carries a deterministic Id (from
|
||
the source filename, so re-ingest is idempotent), the resolved buyer
|
||
ContactId, the parsed Price, the parsed dates and the gerbil ids that
|
||
matched for that contract. Contracts with NO date at all are skipped from
|
||
record creation (the SaleContract.HandoverDate/ContractDate columns are
|
||
non-nullable DateOnly) and counted in stats["dateless_skipped"]; contracts
|
||
whose buyer cannot be resolved are counted in stats["no_buyer_skipped"].
|
||
|
||
Returns (stats, sale_contracts). Mutates resolved_gerbils +
|
||
contact_by_norm_name in place. New buyer contacts are appended via
|
||
contact_by_norm_name so the later IsReceiver-flag pass picks them up
|
||
automatically.
|
||
"""
|
||
stats = {
|
||
"contracts": len(contracts), "buyers_created": 0, "buyers_existing": 0,
|
||
"matched": 0, "ambiguous_skipped": 0, "no_match_skipped": 0,
|
||
"receiver_set": 0, "gohome_set": 0, "status_givenaway": 0,
|
||
"conflicts": 0,
|
||
"records_created": 0, "no_buyer_skipped": 0, "dateless_skipped": 0,
|
||
"records_with_animal": 0, "records_with_date": 0,
|
||
}
|
||
sale_contracts = []
|
||
# The same contract filename can appear more than once in contracts.json
|
||
# (the .docx is filed in several subfolders of the share). The record Id is
|
||
# derived from the filename, so we must collapse those into ONE record per
|
||
# Id (a duplicate PK would break ingest). Keyed by Id; animal lists are
|
||
# merged and a missing date is back-filled from the duplicate.
|
||
records_by_id = {}
|
||
if not contracts:
|
||
return stats, sale_contracts
|
||
|
||
# excludeContractMatch (MemPalace stolperfalle): some breeding/parent animals
|
||
# (e.g. Makoto, Danako) are wrongly matched to a contract whose filename names
|
||
# them as PARENTS of the sold pup, not as the sold animals. A decision with
|
||
# `excludeContractMatch: true` pins such an animal OUT of contract matching: it
|
||
# never gets ReceiverContactId/GoHomeDate/Status=GivenAway from a contract.
|
||
# The override that runs BEFORE enrich (isResident/notes/receiver) cannot undo
|
||
# this afterwards, so the gate must sit here, inside the match loop.
|
||
_exclude_namedob = set() # {(call-name, iso-dob), (call-name, "")}
|
||
_exclude_extref = set() # externalRef suffixes
|
||
for d in (exclude_decisions or []):
|
||
if not d.get("excludeContractMatch"):
|
||
continue
|
||
er = (d.get("externalRef") or "").strip()
|
||
if er:
|
||
_exclude_extref.add(er)
|
||
nm = d.get("name")
|
||
if nm is not None:
|
||
ck = normalize_name(get_call_name(nm or ""))
|
||
iso = parse_date(d.get("dob")) if d.get("dob") else ""
|
||
_exclude_namedob.add((ck, iso or ""))
|
||
|
||
def _is_excluded(g):
|
||
if _exclude_extref:
|
||
er = g.get("ExternalRef") or ""
|
||
if er and (er in _exclude_extref or
|
||
any(er.endswith(k) for k in _exclude_extref)):
|
||
return True
|
||
if _exclude_namedob:
|
||
ck = normalize_name(get_call_name(g.get("Name") or ""))
|
||
iso = g.get("DateOfBirth") or ""
|
||
if (ck, iso) in _exclude_namedob or (ck, "") in _exclude_namedob:
|
||
return True
|
||
return False
|
||
|
||
# Index breeder-owned gerbils by dedup name key. Contracts only ever sell
|
||
# animals the breeder bred, so restrict candidates to her own stock to avoid
|
||
# colliding with same-named foreign-bred animals.
|
||
def is_own(g):
|
||
ob = (g.get("OriginBreeder") or "").lower()
|
||
return ("chaoten" in ob) or (g.get("OriginBreeder") is None)
|
||
|
||
index = {}
|
||
for g in resolved_gerbils:
|
||
if not is_own(g):
|
||
continue
|
||
key = get_dedup_name_key(get_call_name(g.get("Name", "")))
|
||
if key:
|
||
index.setdefault(key, []).append(g)
|
||
|
||
def year_of(iso):
|
||
return iso[:4] if iso else None
|
||
|
||
# Reject buyer values that are obviously label leakage / non-person noise
|
||
# (defends against any stale contracts.json produced before the parser fix).
|
||
_buyer_junk = re.compile(
|
||
r"^(?:stra\w+e|wohnort|fon|e-?mail|handy|festnetz|telefon|homepage|"
|
||
r"zuchtname|facebook|name)\s*:?\s*$|^[\d\s/]+$", re.IGNORECASE)
|
||
|
||
for c in contracts:
|
||
fname = c.get("sourceFile", "")
|
||
buyer_raw = (c.get("buyer") or "").strip()
|
||
if buyer_raw and _buyer_junk.match(buyer_raw):
|
||
buyer_raw = ""
|
||
|
||
# --- buyer contact (reuse curated normalisation/dedup) ---
|
||
buyer_global_id = None
|
||
if buyer_raw:
|
||
canon, keep = get_normalized_contact_name(buyer_raw)
|
||
if keep and canon:
|
||
norm = normalize_name(canon)
|
||
if norm in contact_by_norm_name:
|
||
gc = contact_by_norm_name[norm]
|
||
gc.setdefault("_source_files", set()).add(fname)
|
||
gc["_merged_count"] = gc.get("_merged_count", 1) + 1
|
||
stats["buyers_existing"] += 1
|
||
else:
|
||
gid = generate_guid(f"contact-{norm}")
|
||
contact_by_norm_name[norm] = {
|
||
"Id": gid, "Name": canon, "Email": None, "Phone": None,
|
||
"Address": None, "Notes": None, "NameSuffix": None,
|
||
"_source_files": {fname}, "_merged_count": 1,
|
||
}
|
||
stats["buyers_created"] += 1
|
||
buyer_global_id = contact_by_norm_name[norm]["Id"]
|
||
|
||
# --- match each animal call-name to a resolved gerbil ---
|
||
handover = parse_date(c.get("handoverDate"))
|
||
contract_date = parse_date(c.get("contractDate"))
|
||
c_year = year_of(parse_date(c.get("dob"))) if c.get("dob") else None
|
||
c_color = (c.get("color") or "").strip().lower()
|
||
matched_gerbil_ids = [] # gerbils this contract resolved to (for the record)
|
||
|
||
for call in (c.get("animals") or []):
|
||
key = get_dedup_name_key(get_call_name(call))
|
||
if not key:
|
||
continue
|
||
cands = index.get(key, [])
|
||
if not cands:
|
||
stats["no_match_skipped"] += 1
|
||
continue
|
||
|
||
# Corroborate when more than one candidate shares the call-name.
|
||
chosen = None
|
||
if len(cands) == 1:
|
||
chosen = cands[0]
|
||
else:
|
||
scored = []
|
||
for g in cands:
|
||
score = 0
|
||
g_year = year_of(g.get("DateOfBirth"))
|
||
if c_year and g_year and c_year == g_year:
|
||
score += 2
|
||
if c_color and g.get("ColorVarietyId"):
|
||
# color match is corroboration; we don't have the name
|
||
# here, so only DOB drives disambiguation strongly.
|
||
pass
|
||
scored.append((score, g))
|
||
scored.sort(key=lambda t: t[0], reverse=True)
|
||
if scored[0][0] >= 2 and (len(scored) == 1 or scored[0][0] > scored[1][0]):
|
||
chosen = scored[0][1]
|
||
else:
|
||
stats["ambiguous_skipped"] += 1
|
||
continue
|
||
|
||
# excludeContractMatch: this animal is pinned out of contract matching
|
||
# (it is a parent named in the filename, not the sold pup). Skip BEFORE
|
||
# any receiver/gohome/status mutation and before counting it as matched.
|
||
if _is_excluded(chosen):
|
||
stats["no_match_skipped"] += 1
|
||
continue
|
||
|
||
stats["matched"] += 1
|
||
if chosen.get("Id") and chosen["Id"] not in matched_gerbil_ids:
|
||
matched_gerbil_ids.append(chosen["Id"])
|
||
|
||
# --- set receiver, only if not already set differently ---
|
||
if buyer_global_id:
|
||
cur = chosen.get("ReceiverContactId")
|
||
if not cur:
|
||
chosen["ReceiverContactId"] = buyer_global_id
|
||
stats["receiver_set"] += 1
|
||
chosen["Provenance"] = _append_history(
|
||
chosen.get("Provenance"),
|
||
f"Abgabe an „{buyer_raw}“ aus Vertrag {_quote_file(fname)} übernommen.")
|
||
elif cur != buyer_global_id:
|
||
stats["conflicts"] += 1
|
||
chosen["Provenance"] = _append_history(
|
||
chosen.get("Provenance"),
|
||
f"Vertrag {_quote_file(fname)} nennt anderen Abnehmer „{buyer_raw}“ "
|
||
f"— bestehende Zuordnung beibehalten.")
|
||
|
||
# --- set go-home date, only if empty ---
|
||
if handover and not chosen.get("GoHomeDate"):
|
||
chosen["GoHomeDate"] = handover
|
||
stats["gohome_set"] += 1
|
||
chosen["Provenance"] = _append_history(
|
||
chosen.get("Provenance"),
|
||
f"Abgabedatum {handover} aus Vertrag {_quote_file(fname)} übernommen.")
|
||
|
||
# --- status: derive GivenAway if we set a receiver and it isn't
|
||
# already a stronger state (Deceased). ---
|
||
if chosen.get("ReceiverContactId") and chosen.get("Status") not in (
|
||
"Deceased", "GivenAway"):
|
||
chosen["Status"] = "GivenAway"
|
||
stats["status_givenaway"] += 1
|
||
|
||
# --- emit a SaleContract record for this contract ---------------------
|
||
# Only contracts with a resolvable buyer become records (the row needs a
|
||
# ContactId). A record with zero matched animals is still kept — better
|
||
# to show the contract than to drop it.
|
||
if not buyer_global_id:
|
||
stats["no_buyer_skipped"] += 1
|
||
continue
|
||
|
||
# HandoverDate/ContractDate are non-nullable DateOnly in the DB. Fall
|
||
# back from one to the other; if BOTH are missing, skip the record
|
||
# (we do not invent dates) and count it.
|
||
h = handover or contract_date
|
||
cd = contract_date or handover
|
||
if not h: # implies cd is also None
|
||
stats["dateless_skipped"] += 1
|
||
continue
|
||
|
||
rec_id = generate_guid(f"contract-{fname}")
|
||
existing = records_by_id.get(rec_id)
|
||
if existing is None:
|
||
records_by_id[rec_id] = {
|
||
"Id": rec_id,
|
||
"ContactId": buyer_global_id,
|
||
"Price": _parse_price(c.get("price")),
|
||
"HandoverDate": h,
|
||
"ContractDate": cd,
|
||
"FileName": fname,
|
||
"Animals": list(matched_gerbil_ids),
|
||
}
|
||
else:
|
||
# Same filename seen again — merge animal matches; back-fill price.
|
||
for gid in matched_gerbil_ids:
|
||
if gid not in existing["Animals"]:
|
||
existing["Animals"].append(gid)
|
||
if not existing["Price"]:
|
||
existing["Price"] = _parse_price(c.get("price"))
|
||
|
||
sale_contracts = list(records_by_id.values())
|
||
stats["records_created"] = len(sale_contracts)
|
||
stats["records_with_animal"] = sum(1 for r in sale_contracts if r["Animals"])
|
||
stats["records_with_date"] = sum(1 for r in sale_contracts if r["HandoverDate"])
|
||
return stats, sale_contracts
|
||
|
||
|
||
def main():
|
||
print("Loading color variety seeds...")
|
||
variety_map = {}
|
||
variety_genotypes = {}
|
||
|
||
# Load from C# ApplicationContext.cs catalog for stable database GUIDs (index + 1)
|
||
here = os.path.dirname(os.path.abspath(__file__))
|
||
app_context_path = os.path.abspath(os.path.join(here, "../../GerbilManagerWebAPI/ApplicationContext.cs"))
|
||
cs_name_to_id = {}
|
||
if os.path.exists(app_context_path):
|
||
with open(app_context_path, 'r', encoding='utf-8') as f:
|
||
content = f.read()
|
||
catalog_match = re.search(r'catalog\s*=\s*\{(.*?)\};', content, re.DOTALL)
|
||
if catalog_match:
|
||
block = catalog_match.group(1)
|
||
entries = re.findall(r'\(\s*"([^"]+)"\s*,\s*"([^"]+)"\s*,\s*(\d+)\s*\)', block)
|
||
for idx, (name, genotype, sort_order) in enumerate(entries):
|
||
variety_id = f"00000000-0000-0000-0000-{idx + 1:012d}"
|
||
cs_name_to_id[name.strip().lower()] = variety_id
|
||
variety_map[name.strip().lower()] = variety_id
|
||
variety_genotypes[variety_id] = genotype.strip()
|
||
else:
|
||
print(f"Warning: ApplicationContext.cs not found at {app_context_path}")
|
||
|
||
if os.path.exists(SEEDS_PATH):
|
||
with open(SEEDS_PATH, "r", encoding="utf-8") as f:
|
||
seeds = json.load(f)
|
||
for v in seeds:
|
||
name_lower = v["name"].strip().lower()
|
||
variety_id = cs_name_to_id.get(name_lower)
|
||
if not variety_id:
|
||
variety_id = f"00000000-0000-0000-0000-{v['sortOrder'] + 1:012d}"
|
||
variety_map[name_lower] = variety_id
|
||
|
||
# Map English name if present
|
||
if "english" in v and v["english"]:
|
||
variety_map[v["english"].strip().lower()] = variety_id
|
||
|
||
if variety_id not in variety_genotypes:
|
||
variety_genotypes[variety_id] = v.get("canonicalGenotype")
|
||
else:
|
||
print(f"Warning: Seeds path not found at {SEEDS_PATH}")
|
||
|
||
# Reverse map (variety GUID → human name) for discard/replacement history
|
||
# lines that need to show a colour value rather than a raw GUID. First wins
|
||
# so we keep the canonical lower-case catalog name.
|
||
variety_id_to_name = {}
|
||
for name_lower, vid in variety_map.items():
|
||
if vid not in variety_id_to_name:
|
||
variety_id_to_name[vid] = name_lower
|
||
|
||
md_files = sorted([f for f in os.listdir(DIR_PATH) if f.lower().endswith('.md')])
|
||
print(f"Found {len(md_files)} markdown files in {DIR_PATH}.")
|
||
|
||
raw_contacts = []
|
||
raw_litters = []
|
||
raw_gerbils = []
|
||
|
||
# Map to track which files contained explicit dates
|
||
file_explicit_dates = {}
|
||
|
||
# 1. Parse JSON blocks from all markdown files and scope IDs by filename
|
||
for filename in md_files:
|
||
filepath = os.path.join(DIR_PATH, filename)
|
||
with open(filepath, 'r', encoding='utf-8') as f:
|
||
content = f.read()
|
||
|
||
json_start = content.find("```json")
|
||
if json_start == -1:
|
||
continue
|
||
json_start += len("```json")
|
||
json_end = content.rfind("```")
|
||
if json_end == -1 or json_end <= json_start:
|
||
continue
|
||
|
||
json_str = content[json_start:json_end].strip()
|
||
try:
|
||
data = json.loads(json_str)
|
||
|
||
contacts_key = next((k for k in data if k.lower() == 'contacts'), None)
|
||
litters_key = next((k for k in data if k.lower() == 'litters'), None)
|
||
gerbils_key = next((k for k in data if k.lower() == 'gerbils'), None)
|
||
|
||
file_contacts = data.get(contacts_key, []) if contacts_key else []
|
||
file_litters = data.get(litters_key, []) if litters_key else []
|
||
file_gerbils = data.get(gerbils_key, []) if gerbils_key else []
|
||
|
||
# Find explicit dates on this page to build chronological ordering
|
||
explicit_dates = []
|
||
for l in file_litters:
|
||
d = parse_date(l.get("Date") or l.get("date") or l.get("DateOfBirth") or l.get("dateOfBirth"))
|
||
if d:
|
||
explicit_dates.append(d)
|
||
for g in file_gerbils:
|
||
d = parse_date(g.get("DateOfBirth") or g.get("dateOfBirth"))
|
||
if d:
|
||
explicit_dates.append(d)
|
||
|
||
if explicit_dates:
|
||
# Convert to days since 2000-01-01
|
||
day_vals = [date_to_days(d) for d in explicit_dates if date_to_days(d) is not None]
|
||
if day_vals:
|
||
file_explicit_dates[filename] = sum(day_vals) / len(day_vals)
|
||
|
||
def scope_id(local_id):
|
||
if not local_id:
|
||
return None
|
||
local_str = str(local_id).strip()
|
||
if not local_str:
|
||
return None
|
||
return generate_guid(f"{filename}-{local_str}")
|
||
|
||
for c in file_contacts:
|
||
old_id = c.get("Id") or c.get("id")
|
||
c["_filename"] = filename
|
||
c["_scoped_id"] = scope_id(old_id)
|
||
raw_contacts.append(c)
|
||
|
||
for l in file_litters:
|
||
old_id = l.get("Id") or l.get("id")
|
||
l["_filename"] = filename
|
||
l["_scoped_id"] = scope_id(old_id)
|
||
l["_scoped_father_id"] = scope_id(l.get("FatherId") or l.get("fatherId"))
|
||
l["_scoped_mother_id"] = scope_id(l.get("MotherId") or l.get("motherId"))
|
||
raw_litters.append(l)
|
||
|
||
for g in file_gerbils:
|
||
old_id = g.get("Id") or g.get("id")
|
||
g["_filename"] = filename
|
||
g["_scoped_id"] = scope_id(old_id)
|
||
g["_scoped_litter_id"] = scope_id(g.get("LitterId") or g.get("litterId"))
|
||
g["_scoped_origin_cid"] = scope_id(g.get("OriginContactId") or g.get("originContactId"))
|
||
|
||
receiver_val = (g.get("ReceiverContactId") or g.get("receiverContactId") or
|
||
g.get("BuyerId") or g.get("buyerId") or
|
||
g.get("BuyerContactId") or g.get("buyerContactId") or
|
||
g.get("givenAwayContactId") or g.get("givenAwayToContactId") or
|
||
g.get("ownerContactId") or g.get("ownerId"))
|
||
g["_scoped_receiver_cid"] = scope_id(receiver_val)
|
||
raw_gerbils.append(g)
|
||
|
||
except json.JSONDecodeError as e:
|
||
print(f"Failed to parse JSON in {filename}: {e}")
|
||
|
||
# Chronological Interpolation: Estimate the date of each page based on neighboring pages with dates
|
||
file_dates = {}
|
||
sorted_files = sorted(md_files)
|
||
|
||
# Simple linear interpolation / extrapolation
|
||
for i, fn in enumerate(sorted_files):
|
||
if fn in file_explicit_dates:
|
||
file_dates[fn] = file_explicit_dates[fn]
|
||
else:
|
||
# Look left for closest explicit date
|
||
left_val, left_dist = None, None
|
||
for j in range(i - 1, -1, -1):
|
||
if sorted_files[j] in file_explicit_dates:
|
||
left_val = file_explicit_dates[sorted_files[j]]
|
||
left_dist = i - j
|
||
break
|
||
|
||
# Look right for closest explicit date
|
||
right_val, right_dist = None, None
|
||
for j in range(i + 1, len(sorted_files)):
|
||
if sorted_files[j] in file_explicit_dates:
|
||
right_val = file_explicit_dates[sorted_files[j]]
|
||
right_dist = j - i
|
||
break
|
||
|
||
if left_val is not None and right_val is not None:
|
||
# Interpolate
|
||
file_dates[fn] = left_val + (right_val - left_val) * (left_dist / (left_dist + right_dist))
|
||
elif left_val is not None:
|
||
# Extrapolate right (assume 30 days per page gap as placeholder)
|
||
file_dates[fn] = left_val + (left_dist * 30)
|
||
elif right_val is not None:
|
||
# Extrapolate left
|
||
file_dates[fn] = right_val - (right_dist * 30)
|
||
else:
|
||
# No dates in entire log? Default to 2011-01-01
|
||
file_dates[fn] = date_to_days("2011-01-01")
|
||
|
||
print(f"Parsed {len(raw_contacts)} raw contacts, {len(raw_litters)} raw litters, {len(raw_gerbils)} raw gerbils.")
|
||
|
||
# Run extract.py to make sure stammbaum data is up to date
|
||
import subprocess
|
||
print("Running extract.py to extract Stammbäume...")
|
||
try:
|
||
subprocess.run([sys.executable, "extract.py"], check=True)
|
||
except Exception as e:
|
||
print(f"Warning: Failed to run extract.py: {e}")
|
||
|
||
# Load Stammbaum data
|
||
stammbaum_only_animals = []
|
||
animals_path = os.path.join(OUTPUT_DIR, "animals.json")
|
||
if os.path.exists(animals_path):
|
||
with open(animals_path, "r", encoding="utf-8") as f:
|
||
stammbaum_animals = json.load(f)
|
||
for a in stammbaum_animals:
|
||
sources = a.get("sourceFiles", [])
|
||
if any("stammbaum" in str(s).lower() for s in sources):
|
||
stammbaum_only_animals.append(a)
|
||
print(f"Loaded {len(stammbaum_only_animals)} Stammbaum animals.")
|
||
else:
|
||
print(f"Warning: Stammbaum animals.json not found at {animals_path}")
|
||
|
||
# Run extract_docx.py to make sure docx data is up to date
|
||
print("Running extract_docx.py to extract docx...")
|
||
try:
|
||
subprocess.run([sys.executable, "extract_docx.py"], check=True)
|
||
except Exception as e:
|
||
print(f"Warning: Failed to run extract_docx.py: {e}")
|
||
|
||
# Load Docx data
|
||
docx_animals = []
|
||
docx_animals_path = os.path.join(OUTPUT_DIR, "docx_animals.json")
|
||
if os.path.exists(docx_animals_path):
|
||
with open(docx_animals_path, "r", encoding="utf-8") as f:
|
||
docx_animals = json.load(f)
|
||
print(f"Loaded {len(docx_animals)} animals from docx.")
|
||
else:
|
||
print(f"Warning: docx_animals.json not found at {docx_animals_path}")
|
||
|
||
docx_litters = []
|
||
docx_litters_path = os.path.join(OUTPUT_DIR, "docx_litters.json")
|
||
if os.path.exists(docx_litters_path):
|
||
with open(docx_litters_path, "r", encoding="utf-8") as f:
|
||
docx_litters = json.load(f)
|
||
print(f"Loaded {len(docx_litters)} litters from docx.")
|
||
else:
|
||
print(f"Warning: docx_litters.json not found at {docx_litters_path}")
|
||
|
||
# Abgabevertrag-Datensätze (aus extract_contracts.py). Optional: wenn die
|
||
# Datei fehlt, läuft der Import ohne Vertrags-Anreicherung normal weiter.
|
||
contracts = []
|
||
contracts_path = os.path.join(OUTPUT_DIR, "contracts.json")
|
||
if os.path.exists(contracts_path):
|
||
with open(contracts_path, "r", encoding="utf-8") as f:
|
||
contracts = json.load(f)
|
||
print(f"Loaded {len(contracts)} Abgabeverträge.")
|
||
else:
|
||
print(f"Info: contracts.json not found at {contracts_path} (Vertrags-Anreicherung übersprungen)")
|
||
|
||
# Extract docx buyer contacts and add to raw_contacts
|
||
for da in docx_animals:
|
||
o_name = (da.get("owner") or "").strip()
|
||
if o_name:
|
||
o_scoped_id = generate_guid(f"docx-contact-{normalize_name(o_name)}")
|
||
raw_contacts.append({
|
||
"Name": o_name,
|
||
"_filename": "Wurfchronik-Detail",
|
||
"_scoped_id": o_scoped_id
|
||
})
|
||
|
||
# Map docx litters and append to raw_litters
|
||
docx_litter_id_map = {} # wsCode -> scoped_id
|
||
for dl in docx_litters:
|
||
l_name = dl["litterId"]
|
||
dob_val = parse_date(dl["dob"])
|
||
f_name = get_normalized_gerbil_name(dl["fatherName"])
|
||
m_name = get_normalized_gerbil_name(dl["motherName"])
|
||
ws_code = dl["wsCode"]
|
||
note_val = dl.get("note")
|
||
|
||
# Parse survived/total born from wsCode (e.g. 4/5)
|
||
total_born = None
|
||
deaths_8w = None
|
||
if "/" in ws_code:
|
||
parts = ws_code.split("/")
|
||
if len(parts) == 2:
|
||
try:
|
||
survived = int(parts[0])
|
||
total = int(parts[1])
|
||
total_born = total
|
||
deaths_8w = max(0, total - survived)
|
||
except ValueError:
|
||
pass
|
||
|
||
l_scoped_id = generate_guid(f"docx-litter-{normalize_name(ws_code)}-{dob_val or '0001-01-01'}")
|
||
docx_litter_id_map[(ws_code, dob_val)] = l_scoped_id
|
||
|
||
raw_litters.append({
|
||
"Id": l_scoped_id,
|
||
"Name": l_name,
|
||
"Date": dob_val,
|
||
"TotalBorn": total_born,
|
||
"DeathsWithin8Weeks": deaths_8w,
|
||
"FatherId": generate_guid(f"stammbaum-animal-{normalize_name(f_name)}"), # placeholder
|
||
"MotherId": generate_guid(f"stammbaum-animal-{normalize_name(m_name)}"), # placeholder
|
||
"ExpectedGoHomeDate": None,
|
||
"Notes": note_val if note_val else "Docx imported litter",
|
||
"PairingCode": None,
|
||
"ExternalRef": f"docx-litter-{l_scoped_id}",
|
||
"LitterLetter": l_name[0] if l_name and len(l_name) > 0 else None,
|
||
"_father_name": f_name,
|
||
"_mother_name": m_name,
|
||
"_filename": "Wurfchronik-Detail",
|
||
"_scoped_id": l_scoped_id,
|
||
"_scoped_father_id": generate_guid(f"stammbaum-animal-{normalize_name(f_name)}"),
|
||
"_scoped_mother_id": generate_guid(f"stammbaum-animal-{normalize_name(m_name)}")
|
||
})
|
||
|
||
|
||
# Extract stammbaum contacts and add to raw_contacts
|
||
for a in stammbaum_only_animals:
|
||
b_name = (a.get("breeder") or "").strip()
|
||
if b_name:
|
||
b_scoped_id = generate_guid(f"stammbaum-contact-{normalize_name(b_name)}")
|
||
raw_contacts.append({
|
||
"Name": b_name,
|
||
"_filename": "Stammbaum",
|
||
"_scoped_id": b_scoped_id
|
||
})
|
||
z_name = (a.get("zucht") or "").strip()
|
||
if z_name:
|
||
z_scoped_id = generate_guid(f"stammbaum-contact-{normalize_name(z_name)}")
|
||
raw_contacts.append({
|
||
"Name": z_name,
|
||
"_filename": "Stammbaum",
|
||
"_scoped_id": z_scoped_id
|
||
})
|
||
|
||
# Residency propagation for stammbaum animals
|
||
def is_clan_zucht(z):
|
||
if not z:
|
||
return False
|
||
norm = z.lower()
|
||
return "klein" in norm and "chaot" in norm and "extern" not in norm
|
||
|
||
def is_external_cattery(a):
|
||
"""True if an animal belongs to a NAMED, non-clan cattery (e.g. 'Black
|
||
Forest', 'LennyLengo', a foreign line) or the externally-acquired
|
||
founder markers. Such animals were never in the breeder's own stock, so
|
||
residency must NOT be propagated onto them (tickets #17/#20 — Hagrid of
|
||
Black Forest). Animals with no cattery at all are name-only lineage
|
||
ancestors and stay eligible for propagation."""
|
||
for z in (a.get("zucht"), a.get("zuchtCanon"), a.get("breeder")):
|
||
z = (z or "").strip()
|
||
if z and not is_clan_zucht(z):
|
||
return True
|
||
name = a.get("name", "")
|
||
if name:
|
||
m = re.search(r"\b(?:of|von\s+den|von\s+der|v\.\s?d\.|von)\s+(.+)$", name, re.IGNORECASE)
|
||
if m:
|
||
cattery = m.group(1).strip()
|
||
if cattery and not is_clan_zucht(cattery):
|
||
return True
|
||
m = re.search(r"^(.+?'s)\s+", name, re.IGNORECASE)
|
||
if m:
|
||
cattery = m.group(1).strip()
|
||
if cattery and not is_clan_zucht(cattery):
|
||
return True
|
||
return False
|
||
|
||
stammbaum_resident_ids = set()
|
||
for a in stammbaum_only_animals:
|
||
if is_clan_zucht(a.get("zucht")) or is_clan_zucht(a.get("zuchtCanon")):
|
||
stammbaum_resident_ids.add(a["id"])
|
||
|
||
# Propagate residency to parents of resident offspring — but never onto
|
||
# animals from a named external cattery (they were never in this stock).
|
||
for _ in range(5):
|
||
for a in stammbaum_only_animals:
|
||
if a["id"] in stammbaum_resident_ids:
|
||
for p_ref in a.get("parentRefs", []):
|
||
p_key = (normalize_name(p_ref["name"]), parse_date(p_ref["dob"]))
|
||
for cand in stammbaum_only_animals:
|
||
if normalize_name(cand["name"]) == p_key[0]:
|
||
cand_dob = parse_date(cand["dob"])
|
||
if not p_key[1] or cand_dob == p_key[1]:
|
||
if is_external_cattery(cand):
|
||
continue
|
||
stammbaum_resident_ids.add(cand["id"])
|
||
|
||
# Pre-index Wurfchronik litters from markdown
|
||
md_litters_idx = {}
|
||
# …and a date-only index of Wurfchronik litters that NAME both parents. The
|
||
# Wurfchronik is authoritative: when a chart-position reconstruction picks the
|
||
# WRONG parents (so the (father,mother,date) key misses) but exactly ONE
|
||
# Wurfchronik litter exists for that birthdate, attach the animal to it rather
|
||
# than fabricating a virtual litter with bad parents (tickets #12 Tony,
|
||
# #15 Odelia, #31 Jamie — Wurfchronik-Vorrang vor chart-position).
|
||
md_litters_by_date = {}
|
||
def _md_parent_names(rl):
|
||
f = rl.get("FatherName") or rl.get("fatherName") or rl.get("ParentMaleName") or rl.get("parentMaleName") or rl.get("_father_name")
|
||
m = rl.get("MotherName") or rl.get("motherName") or rl.get("ParentFemaleName") or rl.get("parentFemaleName") or rl.get("_mother_name")
|
||
if not f and not m:
|
||
_note = rl.get("Notes") or rl.get("notes") or rl.get("Note") or rl.get("note") or ""
|
||
_m = re.search(r"(?:Eltern|Pairing|Paarung|Paar):\s*(.+?)\s*(?:\+|\&)\s*(.+?)\s*(?:;|$)", _note, re.IGNORECASE)
|
||
if _m and "/" not in _m.group(1) and "/" not in _m.group(2):
|
||
f, m = _m.group(1).strip(), _m.group(2).strip()
|
||
return get_normalized_gerbil_name(f), get_normalized_gerbil_name(m)
|
||
for rl in raw_litters:
|
||
f_name, m_name = _md_parent_names(rl)
|
||
ldate = parse_date(rl.get("Date") or rl.get("date") or rl.get("DateOfBirth") or rl.get("dateOfBirth"))
|
||
if f_name and m_name and ldate:
|
||
key = (normalize_name(f_name), normalize_name(m_name), ldate)
|
||
md_litters_idx[key] = rl
|
||
md_litters_by_date.setdefault(ldate, []).append(rl)
|
||
|
||
# Gender index for parent-ref selection: normalized name → 'male' | 'female'
|
||
# | 'ambiguous'. Drives the gender-aware ranking in pick_parent_ref so a dated
|
||
# but wrong-sex ref (e.g. female „Danielle“) cannot win the father slot over
|
||
# an undated male/unknown one (e.g. „Hagrid Rubeus“).
|
||
# A name's gender is "ambiguous" ONLY when BOTH male and female records exist
|
||
# for it. A definite gender beats an UNKNOWN (None) duplicate — otherwise a
|
||
# bare DOB-less ancestor box (gender=None) would poison a name that another
|
||
# record clearly types (e.g. Dorie of Black Forest, Zadar): they would fall
|
||
# back to None and lose gender-based role disambiguation (tickets #35, #33).
|
||
gender_idx = {}
|
||
for a in stammbaum_only_animals:
|
||
g = (a.get("gender") or "").lower().strip()
|
||
g = g if g in ("male", "female") else None
|
||
for key in {normalize_name(a.get("name")), normalize_name(get_call_name(a.get("name") or ""))}:
|
||
if not key:
|
||
continue
|
||
cur = gender_idx.get(key, "__unset__")
|
||
if cur == "__unset__" or cur is None:
|
||
gender_idx[key] = g # first value, or upgrade None → definite
|
||
elif g is None or g == cur:
|
||
pass # keep existing definite gender
|
||
else:
|
||
gender_idx[key] = "ambiguous" # genuine male vs female conflict
|
||
|
||
def gender_of_name(name):
|
||
v = gender_idx.get(normalize_name(name))
|
||
return v if v in ("male", "female") else None
|
||
|
||
# Create virtual litters for stammbaum animals
|
||
created_virtual_litters = {}
|
||
for a in stammbaum_only_animals:
|
||
parent_refs = a.get("parentRefs", [])
|
||
child_dob_raw = a.get("dob")
|
||
father_ref = pick_parent_ref(parent_refs, "father", child_dob_raw, gender_of=gender_of_name)
|
||
mother_ref = pick_parent_ref(parent_refs, "mother", child_dob_raw,
|
||
avoid_name=father_ref.get("name") if father_ref else None,
|
||
gender_of=gender_of_name)
|
||
|
||
# Record rejected parent-ref candidates (wrong sex / age-impossible /
|
||
# duplicate of the other role) so this animal's gerbil history can
|
||
# explain which Stammbaum positions were discarded and what won instead.
|
||
pick_discards = []
|
||
pick_discards += explain_pick_rejections(parent_refs, "father", child_dob_raw,
|
||
father_ref, gender_of=gender_of_name)
|
||
pick_discards += explain_pick_rejections(
|
||
parent_refs, "mother", child_dob_raw, mother_ref,
|
||
avoid_name=father_ref.get("name") if father_ref else None,
|
||
gender_of=gender_of_name)
|
||
if pick_discards:
|
||
a["_pick_discards"] = pick_discards
|
||
|
||
a["_mapped_litter_scoped_id"] = None
|
||
|
||
# A human conflict-decision override is authoritative — never let the
|
||
# Wurfchronik-Vorrang heuristic below re-attach the animal to a same-date
|
||
# chronicle litter and overrule the breeder's named parents.
|
||
_has_decision_parent = (
|
||
(father_ref or {}).get("method") == "decision" or
|
||
(mother_ref or {}).get("method") == "decision")
|
||
|
||
# Wurfchronik-Vorrang: if the chart parents don't yield an exact
|
||
# Wurfchronik match but EXACTLY ONE Wurfchronik litter (with named
|
||
# parents) exists for this birthdate, that authoritative litter wins over
|
||
# a fabricated chart-position litter (#12 Tony, #15 Odelia, #31 Jamie).
|
||
dob_val_attach = parse_date(a.get("dob"))
|
||
if dob_val_attach and not _has_decision_parent \
|
||
and not is_external_origin(a.get("name"), a.get("zucht"), a.get("breeder")) \
|
||
and not is_external_cattery(a):
|
||
same_date = md_litters_by_date.get(dob_val_attach, [])
|
||
exact_hit = None
|
||
if father_ref and mother_ref:
|
||
exact_key = (normalize_name(get_normalized_gerbil_name(father_ref.get("name"))),
|
||
normalize_name(get_normalized_gerbil_name(mother_ref.get("name"))),
|
||
dob_val_attach)
|
||
exact_hit = md_litters_idx.get(exact_key)
|
||
if not exact_hit and len(same_date) == 1:
|
||
a["_mapped_litter_scoped_id"] = same_date[0]["_scoped_id"]
|
||
|
||
if a["_mapped_litter_scoped_id"]:
|
||
pass
|
||
elif father_ref and mother_ref:
|
||
f_name = get_normalized_gerbil_name(father_ref.get("name"))
|
||
m_name = get_normalized_gerbil_name(mother_ref.get("name"))
|
||
dob_val = parse_date(a.get("dob"))
|
||
|
||
mapped_litter = None
|
||
if dob_val:
|
||
key = (normalize_name(f_name), normalize_name(m_name), dob_val)
|
||
mapped_litter = md_litters_idx.get(key)
|
||
|
||
if mapped_litter:
|
||
a["_mapped_litter_scoped_id"] = mapped_litter["_scoped_id"]
|
||
else:
|
||
v_key = (normalize_name(f_name), normalize_name(m_name), dob_val or "0001-01-01")
|
||
if v_key in created_virtual_litters:
|
||
a["_mapped_litter_scoped_id"] = created_virtual_litters[v_key]
|
||
else:
|
||
l_scoped_id = generate_guid(f"virtual-litter-{v_key[0]}-{v_key[1]}-{v_key[2]}")
|
||
created_virtual_litters[v_key] = l_scoped_id
|
||
a["_mapped_litter_scoped_id"] = l_scoped_id
|
||
|
||
# Try to link parents to actual parsed stammbaum animals.
|
||
# For human override refs (conflict-decisions, method="decision")
|
||
# skip this loose call-name preliminary linking and let the
|
||
# global name resolver pick the exact named animal instead
|
||
# (e.g. resident „Elena“, not „Elena of KK Chaos“ — #15).
|
||
f_decision = (father_ref or {}).get("method") == "decision"
|
||
m_decision = (mother_ref or {}).get("method") == "decision"
|
||
f_scoped_id = None
|
||
m_scoped_id = None
|
||
f_dob = parse_date(father_ref.get("dob"))
|
||
m_dob = parse_date(mother_ref.get("dob"))
|
||
|
||
for p_cand in ([] if f_decision else stammbaum_only_animals):
|
||
p_gender = str(p_cand.get("gender") or "").lower().strip()
|
||
if p_gender in ["w", "f", "female", "weiblich"]:
|
||
continue
|
||
p_dob = parse_date(p_cand.get("dob"))
|
||
if dob_val and p_dob and p_dob >= dob_val:
|
||
continue
|
||
cand_call_norm = normalize_name(get_call_name(p_cand["name"])) or "unbekannt"
|
||
cand_name_norm = normalize_name(p_cand["name"]) or "unbekannt"
|
||
if cand_call_norm == normalize_name(f_name) or cand_name_norm == normalize_name(f_name):
|
||
if not f_dob or p_dob == f_dob:
|
||
f_scoped_id = generate_guid(f"stammbaum-animal-{p_cand['id']}")
|
||
break
|
||
for p_cand in ([] if m_decision else stammbaum_only_animals):
|
||
p_gender = str(p_cand.get("gender") or "").lower().strip()
|
||
if p_gender in ["m", "male", "männlich"]:
|
||
continue
|
||
p_dob = parse_date(p_cand.get("dob"))
|
||
if dob_val and p_dob and p_dob >= dob_val:
|
||
continue
|
||
cand_call_norm = normalize_name(get_call_name(p_cand["name"])) or "unbekannt"
|
||
cand_name_norm = normalize_name(p_cand["name"]) or "unbekannt"
|
||
if cand_call_norm == normalize_name(m_name) or cand_name_norm == normalize_name(m_name):
|
||
if not m_dob or p_dob == m_dob:
|
||
m_scoped_id = generate_guid(f"stammbaum-animal-{p_cand['id']}")
|
||
break
|
||
|
||
raw_litters.append({
|
||
"Id": l_scoped_id,
|
||
"Name": f"Wurf von {f_name} + {m_name}",
|
||
"Date": dob_val,
|
||
"TotalBorn": None,
|
||
"DeathsWithin8Weeks": None,
|
||
"FatherId": f_scoped_id or generate_guid(f"stammbaum-animal-{normalize_name(f_name)}"),
|
||
"MotherId": m_scoped_id or generate_guid(f"stammbaum-animal-{normalize_name(m_name)}"),
|
||
"ExpectedGoHomeDate": None,
|
||
"Notes": "Pedigree virtual litter",
|
||
"PairingCode": None,
|
||
"ExternalRef": f"virtual-{l_scoped_id}",
|
||
"LitterLetter": None,
|
||
"_father_name": f_name,
|
||
"_mother_name": m_name,
|
||
"_filename": "Stammbaum",
|
||
"_scoped_id": l_scoped_id,
|
||
"_scoped_father_id": f_scoped_id or generate_guid(f"stammbaum-animal-{normalize_name(f_name)}"),
|
||
"_scoped_mother_id": m_scoped_id or generate_guid(f"stammbaum-animal-{normalize_name(m_name)}")
|
||
})
|
||
|
||
elif _has_decision_parent and (father_ref or mother_ref):
|
||
# Single KNOWN parent from a human override (the other parent is
|
||
# genuinely unknown — e.g. a sibling-pairing where only the mother is
|
||
# named: Danielle's mother = Ella, father = unnamed brother).
|
||
# Create a one-parent virtual litter; the global name resolver fills
|
||
# the known side (gender + age-plausibility pick the right same-named
|
||
# animal, e.g. Ella *10.06.2019 over Ella *13.04.2023), the other
|
||
# side stays null.
|
||
known = father_ref or mother_ref
|
||
known_role = "father" if father_ref else "mother"
|
||
known_name = get_normalized_gerbil_name(known.get("name"))
|
||
dob_val = parse_date(a.get("dob"))
|
||
v_key = (normalize_name(known_name), known_role, dob_val or "0001-01-01")
|
||
if v_key in created_virtual_litters:
|
||
a["_mapped_litter_scoped_id"] = created_virtual_litters[v_key]
|
||
else:
|
||
l_scoped_id = generate_guid(
|
||
f"virtual-litter-1p-{v_key[0]}-{v_key[1]}-{v_key[2]}")
|
||
created_virtual_litters[v_key] = l_scoped_id
|
||
a["_mapped_litter_scoped_id"] = l_scoped_id
|
||
raw_litters.append({
|
||
"Id": l_scoped_id,
|
||
"Name": f"Wurf von {known_name}",
|
||
"Date": dob_val,
|
||
"TotalBorn": None,
|
||
"DeathsWithin8Weeks": None,
|
||
"FatherId": None,
|
||
"MotherId": None,
|
||
"ExpectedGoHomeDate": None,
|
||
"Notes": "Pedigree virtual litter (ein Elternteil bekannt)",
|
||
"PairingCode": None,
|
||
"ExternalRef": f"virtual-{l_scoped_id}",
|
||
"LitterLetter": None,
|
||
"_father_name": known_name if known_role == "father" else "",
|
||
"_mother_name": known_name if known_role == "mother" else "",
|
||
"_filename": "Stammbaum",
|
||
"_scoped_id": l_scoped_id,
|
||
"_scoped_father_id": None,
|
||
"_scoped_mother_id": None,
|
||
})
|
||
|
||
# 2. Resolve Contacts globally (deduplicate by normalized name)
|
||
|
||
contact_by_norm_name = {}
|
||
contact_id_map = {} # scoped_old_id -> global_guid
|
||
|
||
for rc in raw_contacts:
|
||
name_val = rc.get("Name") or rc.get("name") or rc.get("FullName") or rc.get("fullName")
|
||
if not name_val:
|
||
first = rc.get("FirstName") or rc.get("firstName")
|
||
last = rc.get("LastName") or rc.get("lastName")
|
||
if first or last:
|
||
name_val = f"{first or ''} {last or ''}".strip()
|
||
|
||
if not name_val:
|
||
continue
|
||
|
||
canon_name, should_keep = get_normalized_contact_name(name_val)
|
||
scoped_id = rc["_scoped_id"]
|
||
if not should_keep:
|
||
if scoped_id:
|
||
contact_id_map[scoped_id] = None
|
||
continue
|
||
|
||
norm_name = normalize_name(canon_name)
|
||
|
||
rc_file = rc.get("_filename")
|
||
if norm_name not in contact_by_norm_name:
|
||
global_guid = generate_guid(f"contact-{norm_name}")
|
||
contact_by_norm_name[norm_name] = {
|
||
"Id": global_guid,
|
||
"Name": canon_name,
|
||
"Email": rc.get("Email") or rc.get("email"),
|
||
"Phone": rc.get("Phone") or rc.get("phone"),
|
||
"Address": rc.get("Address") or rc.get("address"),
|
||
"Notes": rc.get("Notes") or rc.get("notes") or rc.get("Note") or rc.get("note"),
|
||
# Provenance accumulators (consumed below, stripped from helper keys).
|
||
"_source_files": set([rc_file]) if rc_file else set(),
|
||
"_merged_count": 1,
|
||
}
|
||
else:
|
||
gc = contact_by_norm_name[norm_name]
|
||
if not gc["Email"] and (rc.get("Email") or rc.get("email")):
|
||
gc["Email"] = rc.get("Email") or rc.get("email")
|
||
if not gc["Phone"] and (rc.get("Phone") or rc.get("phone")):
|
||
gc["Phone"] = rc.get("Phone") or rc.get("phone")
|
||
if not gc["Address"] and (rc.get("Address") or rc.get("address")):
|
||
gc["Address"] = rc.get("Address") or rc.get("address")
|
||
if not gc["Notes"] and (rc.get("Notes") or rc.get("notes") or rc.get("Note") or rc.get("note")):
|
||
gc["Notes"] = rc.get("Notes") or rc.get("notes") or rc.get("Note") or rc.get("note")
|
||
if rc_file:
|
||
gc["_source_files"].add(rc_file)
|
||
gc["_merged_count"] += 1
|
||
|
||
if scoped_id:
|
||
contact_id_map[scoped_id] = contact_by_norm_name[norm_name]["Id"]
|
||
|
||
resolved_contacts = list(contact_by_norm_name.values())
|
||
print(f"Resolved to {len(resolved_contacts)} unique contacts.")
|
||
|
||
# 3. Process Litters (normalize keys)
|
||
resolved_litters = []
|
||
litter_id_map = {} # scoped_old_id -> new_guid
|
||
litter_by_scoped_id = {}
|
||
|
||
for rl in raw_litters:
|
||
filename = rl.get("_filename")
|
||
name_val = rl.get("Name") or rl.get("name")
|
||
if not name_val:
|
||
name_val = "Wurf"
|
||
|
||
dob_val = parse_date(rl.get("Date") or rl.get("date") or rl.get("DateOfBirth") or rl.get("dateOfBirth"))
|
||
|
||
new_guid = rl["_scoped_id"]
|
||
if not new_guid:
|
||
new_guid = generate_guid(f"litter-{filename}-{name_val}-{dob_val}")
|
||
|
||
scoped_old_id = rl["_scoped_id"]
|
||
if scoped_old_id:
|
||
litter_id_map[scoped_old_id] = new_guid
|
||
|
||
total_born = rl.get("TotalBorn") or rl.get("totalBorn") or rl.get("LitterSize") or rl.get("litterSize") or rl.get("size") or rl.get("totalPups")
|
||
if total_born is not None:
|
||
try:
|
||
total_born = int(total_born)
|
||
except ValueError:
|
||
total_born = None
|
||
|
||
deaths_8w = rl.get("DeathsWithin8Weeks") or rl.get("deathsWithin8Weeks")
|
||
if deaths_8w is not None:
|
||
try:
|
||
deaths_8w = int(deaths_8w)
|
||
except ValueError:
|
||
deaths_8w = None
|
||
elif total_born is not None and rl.get("survived") is not None:
|
||
try:
|
||
deaths_8w = total_born - int(rl.get("survived"))
|
||
except ValueError:
|
||
pass
|
||
|
||
father_name = rl.get("FatherName") or rl.get("fatherName") or rl.get("ParentMaleName") or rl.get("parentMaleName") or rl.get("_father_name")
|
||
mother_name = rl.get("MotherName") or rl.get("motherName") or rl.get("ParentFemaleName") or rl.get("parentFemaleName") or rl.get("_mother_name")
|
||
|
||
# Parse parents from a free-text Wurf note „Eltern: X + Y“ when the
|
||
# structured parent names/ids are missing (ticket #9 Beatrice/Q-Wurf,
|
||
# #11 Silver — ~46 Wurfchronik litters carry parents only in the note).
|
||
# The order is father + mother (German chart convention); the name
|
||
# resolver corrects the role afterwards by gender, so a swap is safe.
|
||
if not father_name and not mother_name:
|
||
_note = rl.get("Notes") or rl.get("notes") or rl.get("Note") or rl.get("note") or ""
|
||
_m = re.search(r"(?:Eltern|Pairing|Paarung|Paar):\s*(.+?)\s*(?:\+|\&)\s*(.+?)\s*(?:;|$)", _note, re.IGNORECASE)
|
||
if _m:
|
||
_p1 = _m.group(1).strip()
|
||
_p2 = _m.group(2).strip()
|
||
# Skip ambiguous "Lee/Dean" style alternatives (a slash = unsure).
|
||
if _p1 and _p2 and "/" not in _p1 and "/" not in _p2:
|
||
father_name = father_name or _p1
|
||
mother_name = mother_name or _p2
|
||
|
||
raw_ext_ref = rl.get("ExternalRef") or rl.get("externalRef") or rl.get("Id") or rl.get("id")
|
||
ext_ref_scoped = f"{filename}-{raw_ext_ref}" if raw_ext_ref else None
|
||
|
||
l_record = {
|
||
"Id": new_guid,
|
||
"Name": name_val,
|
||
"Date": dob_val,
|
||
"TotalBorn": total_born,
|
||
"DeathsWithin8Weeks": deaths_8w,
|
||
"FatherId": rl["_scoped_father_id"],
|
||
"MotherId": rl["_scoped_mother_id"],
|
||
"ExpectedGoHomeDate": parse_date(rl.get("ExpectedGoHomeDate") or rl.get("expectedGoHomeDate")),
|
||
"Notes": rl.get("Notes") or rl.get("notes") or rl.get("Note") or rl.get("note"),
|
||
"PairingCode": rl.get("PairingCode") or rl.get("pairingCode"),
|
||
"ExternalRef": ext_ref_scoped,
|
||
"LitterLetter": rl.get("LitterLetter") or rl.get("litterLetter"),
|
||
"_father_name": father_name,
|
||
"_mother_name": mother_name,
|
||
"_filename": filename,
|
||
# Provenance accumulators (canonical absorbs these during dedup below).
|
||
"_source_files": set([filename]) if filename else set(),
|
||
"_merged_count": 1,
|
||
# Virtual litters are reconstructed from a Stammbaum chart, not the
|
||
# Wurfchronik — flagged on the raw record's _filename == "Stammbaum".
|
||
"_virtual": filename == "Stammbaum",
|
||
}
|
||
resolved_litters.append(l_record)
|
||
litter_by_scoped_id[new_guid] = l_record
|
||
|
||
print(f"Processed {len(resolved_litters)} litters.")
|
||
|
||
# 3b. Deduplicate litters: same date + compatible parents → merge
|
||
# This handles the "sibling pairing" case: Stammbaum shows the same parental
|
||
# litter twice (once under the father branch, once under the mother branch),
|
||
# generating two separate litter records with the same date but only one of
|
||
# them has FatherId/MotherId resolved.
|
||
litter_canonical_map = {} # old_id -> canonical_id (for dedup within this step)
|
||
|
||
# Group by date for efficiency
|
||
by_date = {}
|
||
for l in resolved_litters:
|
||
by_date.setdefault(l["Date"], []).append(l)
|
||
|
||
litter_dedup_canonical = {} # old_litter_id -> canonical_litter_id
|
||
deduped_litters = []
|
||
|
||
for date_val, group in by_date.items():
|
||
# Partition into compatible subsets
|
||
sub_groups = []
|
||
for l in group:
|
||
placed = False
|
||
for sub in sub_groups:
|
||
if all(litter_compatible(l, member) for member in sub):
|
||
sub.append(l)
|
||
placed = True
|
||
break
|
||
if not placed:
|
||
sub_groups.append([l])
|
||
|
||
for sub in sub_groups:
|
||
if len(sub) == 1:
|
||
deduped_litters.append(sub[0])
|
||
litter_dedup_canonical[sub[0]["Id"]] = sub[0]["Id"]
|
||
continue
|
||
|
||
# Pick the canonical record: prefer the one with parents set
|
||
canonical = next((l for l in sub if l.get("FatherId") or l.get("MotherId")), sub[0])
|
||
for l in sub:
|
||
litter_dedup_canonical[l["Id"]] = canonical["Id"]
|
||
if l is not canonical:
|
||
litter_id_map[l["Id"]] = canonical["Id"]
|
||
canonical["_source_files"] |= l.get("_source_files", set())
|
||
canonical["_merged_count"] += l.get("_merged_count", 1)
|
||
if not l.get("_virtual"):
|
||
canonical["_virtual"] = False
|
||
|
||
deduped_litters.append(canonical)
|
||
if len(sub) > 1:
|
||
merged_names = [l["Id"] for l in sub if l is not canonical]
|
||
print(f"Litter-Dedup: merged {len(sub)} same-date litters on {date_val} → {canonical['Name']} (absorbed: {', '.join(merged_names)})")
|
||
|
||
n_merged = len(resolved_litters) - len(deduped_litters)
|
||
if n_merged:
|
||
print(f"Litter-Dedup: {n_merged} redundant litter record(s) removed.")
|
||
resolved_litters = deduped_litters
|
||
litter_by_scoped_id = {l["Id"]: l for l in resolved_litters}
|
||
|
||
# 4. Normalize and group Gerbils
|
||
|
||
# Helper to lookup litter dates for birth date estimation
|
||
def get_litter_date(l_id):
|
||
if l_id in litter_by_scoped_id:
|
||
d = litter_by_scoped_id[l_id]["Date"]
|
||
if d:
|
||
return d
|
||
return None
|
||
|
||
all_processed_gerbils = []
|
||
for rg in raw_gerbils:
|
||
filename = rg.get("_filename")
|
||
name_val = get_normalized_gerbil_name(rg.get("Name") or rg.get("name") or rg.get("callName"))
|
||
if not name_val:
|
||
name_val = "Unbekannt"
|
||
|
||
dob_val = parse_date(rg.get("DateOfBirth") or rg.get("dateOfBirth"))
|
||
|
||
new_guid = rg["_scoped_id"]
|
||
if not new_guid:
|
||
new_guid = generate_guid(f"gerbil-{filename}-{name_val}-{dob_val}")
|
||
|
||
# Gender normalization
|
||
gender_val = rg.get("Gender") or rg.get("gender") or "unknown"
|
||
gender_val = str(gender_val).lower().strip()
|
||
if gender_val in ["m", "male", "männlich"]:
|
||
gender = "male"
|
||
elif gender_val in ["w", "f", "female", "weiblich"]:
|
||
gender = "female"
|
||
else:
|
||
gender = "unknown"
|
||
|
||
dod_val = parse_date(rg.get("DateOfDeath") or rg.get("dateOfDeath") or rg.get("deathDate"))
|
||
gohome_val = parse_date(rg.get("GoHomeDate") or rg.get("goHomeDate") or rg.get("DateGivenAway") or rg.get("givenAwayDate") or rg.get("DateOfHandover") or rg.get("dateOfHandover") or rg.get("HandoverDate") or rg.get("DateOfSale") or rg.get("dateOfSale"))
|
||
|
||
status_val = rg.get("Status") or rg.get("status") or ""
|
||
status_val = str(status_val).lower().strip()
|
||
status = "Breeding"
|
||
if dod_val or "deceased" in status_val or "verstorben" in status_val or "tod" in status_val or "dead" in status_val:
|
||
status = "Deceased"
|
||
elif gohome_val or "givenaway" in status_val or "abgegeben" in status_val or "verkauft" in status_val or "sold" in status_val:
|
||
status = "GivenAway"
|
||
elif "forsale" in status_val or "abzugeben" in status_val:
|
||
status = "ForSale"
|
||
elif "pet" in status_val or "liebhaber" in status_val:
|
||
status = "Pet"
|
||
else:
|
||
if dod_val:
|
||
status = "Deceased"
|
||
elif gohome_val:
|
||
status = "GivenAway"
|
||
|
||
notes_val = rg.get("Notes") or rg.get("notes") or rg.get("Note") or rg.get("note")
|
||
existing_cod = rg.get("CauseOfDeath") or rg.get("causeOfDeath") or rg.get("DeathCause")
|
||
status, dod_val, cause_of_death_val = parse_death_info(notes_val, status, dod_val, existing_cod)
|
||
|
||
# Explicit resolution for Ken'ichi's cause of death
|
||
if name_val == "Ken'ichi" and dob_val == "2015-03-01":
|
||
cause_of_death_val = "Duftdrüsen-Tumor"
|
||
|
||
color_val = rg.get("ColorVarietyId") or rg.get("colorVarietyId") or rg.get("Color") or rg.get("color") or rg.get("ColorDescription") or rg.get("colorDescription")
|
||
existing_gt = rg.get("Genotype") or rg.get("genotype")
|
||
color_variety_id, genotype_val = resolve_color_and_genotype(
|
||
color_val, existing_gt, variety_map, variety_genotypes
|
||
)
|
||
if color_val and not color_variety_id:
|
||
try:
|
||
uuid.UUID(str(color_val).strip())
|
||
color_variety_id = str(color_val).strip()
|
||
except ValueError:
|
||
print(f"Warning: Unknown color variety '{color_val}' for gerbil '{name_val}' on page {filename}")
|
||
|
||
origin_cid = rg["_scoped_origin_cid"]
|
||
receiver_cid = rg["_scoped_receiver_cid"]
|
||
|
||
if origin_cid in contact_id_map:
|
||
origin_cid = contact_id_map[origin_cid]
|
||
if receiver_cid in contact_id_map:
|
||
receiver_cid = contact_id_map[receiver_cid]
|
||
|
||
# Wurfchronik junglings default to NON-resident (Ticket 381f7e51): being
|
||
# born in the chronicle does NOT by itself make an animal her breeding
|
||
# stock. Residency is recomputed authoritatively by the late isResident
|
||
# sweep (parent-of-own-litter only). Only an explicit truthy IsResident
|
||
# on the raw record is honoured here.
|
||
is_resident = rg.get("IsResident") or rg.get("isResident")
|
||
if is_resident is None:
|
||
is_resident = False
|
||
else:
|
||
is_resident = str(is_resident).lower() == "true"
|
||
|
||
traits = rg.get("CharacterTraits") or rg.get("characterTraits") or []
|
||
if not isinstance(traits, list):
|
||
traits = [str(traits)]
|
||
char_note = rg.get("CharacterNote") or rg.get("characterNote")
|
||
|
||
is_deaf = rg.get("IsDeaf") or rg.get("isDeaf")
|
||
if is_deaf is not None:
|
||
is_deaf = str(is_deaf).lower() == "true"
|
||
|
||
old_litter_id = rg["_scoped_litter_id"]
|
||
|
||
raw_ext_ref = rg.get("ExternalRef") or rg.get("externalRef") or rg.get("Id") or rg.get("id")
|
||
ext_ref_scoped = f"{filename}-{raw_ext_ref}" if raw_ext_ref else None
|
||
|
||
# Estimated effective date of birth for conflict checking
|
||
eff_dob_val = dob_val
|
||
if not eff_dob_val and old_litter_id:
|
||
# Try to get litter date if litter was resolved
|
||
mapped_l_id = litter_id_map.get(old_litter_id)
|
||
if mapped_l_id:
|
||
eff_dob_val = get_litter_date(mapped_l_id)
|
||
|
||
# Determine explicit or litter-derived birth date (None if unknown/parent)
|
||
birth_date = dob_val
|
||
if not birth_date and old_litter_id:
|
||
mapped_l_id = litter_id_map.get(old_litter_id)
|
||
if mapped_l_id:
|
||
birth_date = get_litter_date(mapped_l_id)
|
||
|
||
# If still no effective date, use the estimated page date
|
||
if not eff_dob_val:
|
||
eff_dob_val = days_to_date(file_dates[filename])
|
||
|
||
raw_breeder = rg.get("OriginBreeder") or rg.get("originBreeder")
|
||
if raw_breeder:
|
||
norm_b, keep_b = get_normalized_contact_name(raw_breeder)
|
||
raw_breeder = norm_b if keep_b else None
|
||
|
||
parent_refs = []
|
||
if old_litter_id:
|
||
rl = next((l for l in raw_litters if l.get("_scoped_id") == old_litter_id), None)
|
||
if rl:
|
||
f_name = rl.get("FatherName") or rl.get("fatherName") or rl.get("ParentMaleName") or rl.get("parentMaleName") or rl.get("_father_name")
|
||
m_name = rl.get("MotherName") or rl.get("motherName") or rl.get("ParentFemaleName") or rl.get("parentFemaleName") or rl.get("_mother_name")
|
||
if f_name:
|
||
parent_refs.append({"name": f_name, "roleGuess": "father"})
|
||
if m_name:
|
||
parent_refs.append({"name": m_name, "roleGuess": "mother"})
|
||
|
||
all_processed_gerbils.append({
|
||
"Id": new_guid,
|
||
"Name": name_val,
|
||
"Gender": gender,
|
||
"Status": status,
|
||
"LitterId": old_litter_id, # mapped later
|
||
"OriginContactId": origin_cid,
|
||
"ReceiverContactId": receiver_cid,
|
||
"EnclosureId": None,
|
||
"ColorVarietyId": color_variety_id,
|
||
"DateOfBirth": dob_val,
|
||
"DateOfDeath": dod_val,
|
||
"CauseOfDeath": cause_of_death_val,
|
||
"GoHomeDate": gohome_val,
|
||
"Genotype": genotype_val,
|
||
"Notes": rg.get("Notes") or rg.get("notes") or rg.get("Note") or rg.get("note"),
|
||
"ImportSource": rg.get("ImportSource") or rg.get("importSource") or filename,
|
||
"ExternalRef": ext_ref_scoped,
|
||
"RawImportData": rg.get("RawImportData") or rg.get("rawImportData") or json.dumps({"colorDescription": color_val if not color_variety_id else None}),
|
||
"OriginBreeder": raw_breeder or ("Zucht der kleinen Chaoten" if is_resident else None),
|
||
"NameSearch": normalize_name(name_val),
|
||
"CharacterTraits": traits,
|
||
"CharacterNote": char_note,
|
||
"IsDeaf": is_deaf,
|
||
"IsResident": is_resident,
|
||
"parentRefs": parent_refs,
|
||
"_photos": rg.get("photos", []),
|
||
"_old_scoped_litter_id": old_litter_id,
|
||
"_eff_dob": eff_dob_val,
|
||
"_birth_date": birth_date,
|
||
"_filename": filename,
|
||
"_old_id": rg.get("Id") or rg.get("id")
|
||
})
|
||
|
||
# Build the discard list for one stammbaum animal: rejected parent-ref
|
||
# candidates (pick_parent_ref) plus a manual DOB-remap decision, if any.
|
||
def _stammbaum_discards(a):
|
||
discards = list(a.get("_pick_discards", []))
|
||
remap = a.get("dobRemap")
|
||
if remap and remap.get("original") and remap.get("corrected"):
|
||
orig = _de_date(parse_date(remap["original"])) or remap["original"]
|
||
corr = _de_date(parse_date(remap["corrected"])) or remap["corrected"]
|
||
if orig != corr:
|
||
discards.append({
|
||
"label": PROV_FIELD_LABELS["DateOfBirth"],
|
||
"value": orig,
|
||
"reason": "per manueller Entscheidung korrigiert",
|
||
"replacement": corr,
|
||
})
|
||
return discards
|
||
|
||
# Map and append stammbaum animals to all_processed_gerbils
|
||
for a in stammbaum_only_animals:
|
||
a_id = a["id"]
|
||
name_val = get_normalized_gerbil_name(a["name"])
|
||
|
||
gender_val = str(a.get("gender") or "").lower().strip()
|
||
if gender_val in ["m", "male"]:
|
||
gender = "male"
|
||
elif gender_val in ["w", "f", "female"]:
|
||
gender = "female"
|
||
else:
|
||
gender = "unknown"
|
||
|
||
dob_val = parse_date(a.get("dob"))
|
||
dod_val = parse_date(a.get("death"))
|
||
|
||
status = "Breeding"
|
||
if dod_val:
|
||
status = "Deceased"
|
||
elif a_id not in stammbaum_resident_ids:
|
||
status = "GivenAway"
|
||
else:
|
||
if dob_val:
|
||
try:
|
||
dt_dob = datetime.strptime(dob_val, "%Y-%m-%d")
|
||
dt_now = datetime.now()
|
||
age_years = (dt_now - dt_dob).days / 365.25
|
||
if age_years >= 6.0:
|
||
status = "Deceased"
|
||
except Exception:
|
||
pass
|
||
|
||
# Color variety mapping
|
||
existing_gt = a["genotype"]["rawGenotype"] if a["genotype"]["rawGenotype"] else None
|
||
color_variety_id, genotype_val = resolve_color_and_genotype(
|
||
a.get("farbschlag"), existing_gt, variety_map, variety_genotypes
|
||
)
|
||
if not color_variety_id:
|
||
for fbv in a.get("farbschlagVariants", []):
|
||
cv_id, gt_val = resolve_color_and_genotype(fbv, existing_gt, variety_map, variety_genotypes)
|
||
if cv_id:
|
||
color_variety_id = cv_id
|
||
genotype_val = gt_val
|
||
break
|
||
|
||
# Contact mapping
|
||
origin_cid = None
|
||
b_name = (a.get("breeder") or "").strip()
|
||
z_name = (a.get("zucht") or "").strip()
|
||
if b_name:
|
||
origin_cid = generate_guid(f"stammbaum-contact-{normalize_name(b_name)}")
|
||
elif z_name:
|
||
origin_cid = generate_guid(f"stammbaum-contact-{normalize_name(z_name)}")
|
||
|
||
# Map to resolved global contact GUID
|
||
if origin_cid in contact_id_map:
|
||
origin_cid = contact_id_map[origin_cid]
|
||
else:
|
||
origin_cid = None
|
||
|
||
|
||
scoped_id = generate_guid(f"stammbaum-animal-{a_id}")
|
||
scoped_litter_id = a.get("_mapped_litter_scoped_id")
|
||
|
||
is_deaf = a["genotype"].get("deaf")
|
||
|
||
stammbaum_breeder = a.get("breeder") if a.get("breeder") else (a.get("zucht") if a.get("zucht") else None)
|
||
if stammbaum_breeder:
|
||
norm_b, keep_b = get_normalized_contact_name(stammbaum_breeder)
|
||
stammbaum_breeder = norm_b if keep_b else None
|
||
|
||
all_processed_gerbils.append({
|
||
"Id": scoped_id,
|
||
"Name": name_val,
|
||
"Gender": gender,
|
||
"Status": status,
|
||
"LitterId": scoped_litter_id, # mapped later in step 5
|
||
"OriginContactId": origin_cid,
|
||
"ReceiverContactId": None,
|
||
"EnclosureId": None,
|
||
"ColorVarietyId": color_variety_id,
|
||
"DateOfBirth": dob_val,
|
||
"DateOfDeath": dod_val,
|
||
"CauseOfDeath": "Duftdrüsen-Tumor" if name_val == "Ken'ichi" and dob_val == "2015-03-01" else None,
|
||
"GoHomeDate": None,
|
||
"Genotype": genotype_val,
|
||
"Notes": None,
|
||
"ImportSource": ", ".join(a.get("sourceFiles", [])),
|
||
"ExternalRef": f"stammbaum-{a_id}",
|
||
"RawImportData": json.dumps({
|
||
"rawGenotype": a["genotype"]["rawGenotype"],
|
||
"unmappedTokens": a["genotype"]["unmappedTokens"],
|
||
"breederText": a.get("breeder", "")
|
||
}, ensure_ascii=False),
|
||
"OriginBreeder": stammbaum_breeder,
|
||
"NameSearch": normalize_name(name_val),
|
||
"CharacterTraits": [],
|
||
"CharacterNote": None,
|
||
"IsDeaf": is_deaf,
|
||
"IsResident": a_id in stammbaum_resident_ids,
|
||
"parentRefs": a.get("parentRefs", []),
|
||
"_photos": a.get("photos", []),
|
||
"_conflict": bool(a.get("conflict")),
|
||
"_resolved_by_decision": bool(a.get("resolvedByDecision")),
|
||
"_old_scoped_litter_id": scoped_litter_id,
|
||
"_eff_dob": dob_val or "2010-01-01",
|
||
"_birth_date": dob_val,
|
||
"_filename": a.get("sourceFiles", ["Stammbaum"])[0],
|
||
"_old_id": a_id,
|
||
# Rejected Stammbaum parent-ref candidates for this animal (filled by
|
||
# pick_parent_ref above) — surfaced in this gerbil's discard history.
|
||
"_discarded": _stammbaum_discards(a),
|
||
})
|
||
|
||
# Map and append docx animals to all_processed_gerbils
|
||
for idx, da in enumerate(docx_animals):
|
||
name_val = get_normalized_gerbil_name(da["name"])
|
||
gender = da["gender"]
|
||
|
||
dob_val = parse_date(da.get("litterDob"))
|
||
dod_val = parse_date(da.get("deathDate"))
|
||
gohome_val = parse_date(da.get("abgabeDate"))
|
||
|
||
# Status precedence
|
||
status = "Breeding"
|
||
if dod_val:
|
||
status = "Deceased"
|
||
elif gohome_val or da.get("owner"):
|
||
status = "GivenAway"
|
||
|
||
# Color variety mapping
|
||
color_variety_id, genotype_val = resolve_color_and_genotype(
|
||
da.get("farbschlag"), None, variety_map, variety_genotypes
|
||
)
|
||
|
||
# Contact mapping (buyer)
|
||
receiver_cid = None
|
||
o_name = (da.get("owner") or "").strip()
|
||
if o_name:
|
||
receiver_cid = generate_guid(f"docx-contact-{normalize_name(o_name)}")
|
||
if receiver_cid in contact_id_map:
|
||
receiver_cid = contact_id_map[receiver_cid]
|
||
else:
|
||
receiver_cid = None
|
||
|
||
# Scoped ID
|
||
scoped_id = generate_guid(f"docx-animal-{idx}-{normalize_name(name_val)}-{dob_val or '0001-01-01'}")
|
||
|
||
# Litter ID mapping
|
||
ws_code = da.get("wsCode")
|
||
scoped_litter_id = docx_litter_id_map.get((ws_code, dob_val))
|
||
|
||
|
||
# Raw import details
|
||
raw_import_payload = json.dumps({
|
||
"abgabeWeight": da.get("abgabeWeight", ""),
|
||
"deathCause": da.get("deathCause", ""),
|
||
"partnerName": da.get("partnerName", ""),
|
||
"partnerDob": da.get("partnerDob", "")
|
||
}, ensure_ascii=False)
|
||
|
||
# Residents: if sold/given away, it's not a resident
|
||
is_resident = not bool(o_name)
|
||
|
||
parent_refs = []
|
||
if scoped_litter_id:
|
||
dl = next((l for l in docx_litters if docx_litter_id_map.get((l["wsCode"], parse_date(l["dob"]))) == scoped_litter_id), None)
|
||
if dl:
|
||
if dl.get("fatherName"):
|
||
parent_refs.append({"name": dl["fatherName"], "roleGuess": "father"})
|
||
if dl.get("motherName"):
|
||
parent_refs.append({"name": dl["motherName"], "roleGuess": "mother"})
|
||
|
||
all_processed_gerbils.append({
|
||
"Id": scoped_id,
|
||
"Name": name_val,
|
||
"Gender": gender,
|
||
"Status": status,
|
||
"LitterId": scoped_litter_id,
|
||
"OriginContactId": None,
|
||
"ReceiverContactId": receiver_cid, # mapped in step 5
|
||
"EnclosureId": None,
|
||
"ColorVarietyId": color_variety_id,
|
||
"DateOfBirth": dob_val,
|
||
"DateOfDeath": dod_val,
|
||
"CauseOfDeath": da.get("deathCause"),
|
||
"GoHomeDate": gohome_val,
|
||
"Genotype": genotype_val,
|
||
"Notes": None,
|
||
"ImportSource": "Wurfchronik-Detail.docx",
|
||
"ExternalRef": f"docx-{idx}-{normalize_name(name_val)}-{dob_val or '0001-01-01'}",
|
||
"RawImportData": raw_import_payload,
|
||
"OriginBreeder": "Zucht der kleinen Chaoten",
|
||
"NameSearch": normalize_name(name_val),
|
||
"CharacterTraits": [],
|
||
"CharacterNote": None,
|
||
"IsDeaf": None,
|
||
"IsResident": is_resident,
|
||
"parentRefs": parent_refs,
|
||
"_photos": da.get("photos", []),
|
||
"_old_scoped_litter_id": scoped_litter_id,
|
||
"_eff_dob": dob_val or "2020-01-01",
|
||
"_birth_date": dob_val,
|
||
"_filename": "Wurfchronik-Detail.docx",
|
||
"_old_id": name_val
|
||
})
|
||
|
||
# ── Conflict-decision overrides at the gerbil level ───────────────────────
|
||
# extract.apply_conflict_decisions already applies genotype/farbschlag/dod/
|
||
# gender/parent overrides to STAMMBAUM animals (animals.json). Wurfchronik/
|
||
# docx animals (docx_animals.json) never pass through that path, so a gender
|
||
# override on a Wurfchronik animal (e.g. „Roni“ — actually the FATHER) would
|
||
# otherwise be lost. Apply gender overrides here uniformly to every processed
|
||
# gerbil, materialise `addAnimals` stubs for known parents that have no own
|
||
# source record (e.g. „Fumi“), and collapse explicitly-paired duplicate
|
||
# records via `mergeExternalRefs` (nameless animals never auto-merge).
|
||
_decisions, _add_animals, _add_litters = load_conflict_decisions()
|
||
|
||
def _gerbil_call_key(g):
|
||
return normalize_name(get_call_name(g.get("Name") or ""))
|
||
|
||
# 1) gender overrides (match normalize(call-name) + ISO dob; empty decision
|
||
# dob = name-only, for ancestors without a birthdate).
|
||
_gender_overrides = {}
|
||
for d in _decisions:
|
||
gg = _norm_decision_gender(d.get("gender"))
|
||
if not gg:
|
||
continue
|
||
ck = normalize_name(get_call_name(d.get("name") or ""))
|
||
iso = parse_date(d.get("dob")) if d.get("dob") else ""
|
||
_gender_overrides[(ck, iso or "")] = gg
|
||
_gender_applied = 0
|
||
for g in all_processed_gerbils:
|
||
ck = _gerbil_call_key(g)
|
||
iso = g.get("DateOfBirth") or ""
|
||
gg = _gender_overrides.get((ck, iso)) or _gender_overrides.get((ck, ""))
|
||
if gg and g.get("Gender") != gg:
|
||
g["Gender"] = gg
|
||
g["_resolved_by_decision"] = True
|
||
_gender_applied += 1
|
||
|
||
# Resolve a contact NAME (e.g. an addAnimals `receiver` like „Ulrike Neu“) to
|
||
# its global contact id. Prefers an already-resolved contact; otherwise falls
|
||
# back to the deterministic id (same scheme as contact resolution above) so a
|
||
# receiver named only here still maps to the canonical id.
|
||
def _resolve_contact_id_by_name(name):
|
||
raw = (name or "").strip()
|
||
if not raw:
|
||
return None
|
||
canon, keep = get_normalized_contact_name(raw)
|
||
if not keep:
|
||
return None
|
||
norm = normalize_name(canon)
|
||
existing = contact_by_norm_name.get(norm)
|
||
if existing:
|
||
return existing["Id"]
|
||
cid = generate_guid(f"contact-{norm}")
|
||
new_c = {
|
||
"Id": cid,
|
||
"Name": canon,
|
||
"Phone": None,
|
||
"Email": None,
|
||
"Street": None,
|
||
"City": None,
|
||
"IsBreeder": False,
|
||
"IsReceiver": True,
|
||
"Notes": "Automatisch generiert durch Züchterin-Entscheidung (Receiver)"
|
||
}
|
||
contact_by_norm_name[norm] = new_c
|
||
return cid
|
||
|
||
|
||
# 2) addAnimals — materialise a stub for a KNOWN animal that has no own source
|
||
# record, so its litter's parent link can resolve (e.g. „Fumi“). Optional
|
||
# fields: gender, zucht, dob, isResident (bool), receiver (contact name →
|
||
# ReceiverContactId), notes (→ Notes), farbschlag/color (best-effort → Notes).
|
||
# Skip a bare stub when a record of that call-name already exists (avoid
|
||
# duplicates); but a deliberately-added RICH animal (one carrying isResident/
|
||
# receiver/notes/color, i.e. genuinely new data — not just a parent-link
|
||
# placeholder) is created even when the call-name collides with an unrelated
|
||
# foreign animal (e.g. the new resident „Merle“ vs the existing „Merle of
|
||
# Samsimar“). It gets its own deterministic id, so no real duplicate arises.
|
||
_existing_call_keys = {_gerbil_call_key(g) for g in all_processed_gerbils}
|
||
_RICH_FIELDS = ("isResident", "receiver", "notes", "farbschlag", "color")
|
||
_stubs_added = 0
|
||
for a in _add_animals:
|
||
nm = (a.get("name") or "").strip()
|
||
if not nm:
|
||
continue
|
||
ck = normalize_name(get_call_name(nm))
|
||
is_rich = any(a.get(k) is not None for k in _RICH_FIELDS)
|
||
if ck in _existing_call_keys and not is_rich:
|
||
continue
|
||
gender = _norm_decision_gender(a.get("gender")) or "unknown"
|
||
dob_iso = parse_date(a.get("dob")) if a.get("dob") else None
|
||
zucht = (a.get("zucht") or "").strip() or None
|
||
breeder_disp = None
|
||
if zucht:
|
||
nb, keep = get_normalized_contact_name(zucht)
|
||
breeder_disp = nb if keep else None
|
||
# Optional rich fields.
|
||
is_resident = bool(a.get("isResident")) if a.get("isResident") is not None else False
|
||
receiver_cid = _resolve_contact_id_by_name(a.get("receiver"))
|
||
notes_val = (a.get("notes") or "").strip() or None
|
||
color_txt = (a.get("farbschlag") or a.get("color") or "").strip() or None
|
||
# Best-effort colour: keep it human-readable in Notes (no genotype to map).
|
||
if color_txt:
|
||
color_line = f"Farbschlag: {color_txt}"
|
||
notes_val = f"{notes_val} | {color_line}" if notes_val else color_line
|
||
status = "GivenAway" if receiver_cid else "Breeding"
|
||
stub_id = generate_guid(f"decision-animal-{normalize_name(nm)}")
|
||
all_processed_gerbils.append({
|
||
"Id": stub_id,
|
||
"Name": get_normalized_gerbil_name(nm),
|
||
"Gender": gender,
|
||
"Status": status,
|
||
"LitterId": None,
|
||
"OriginContactId": None,
|
||
"ReceiverContactId": receiver_cid,
|
||
"EnclosureId": None,
|
||
"ColorVarietyId": None,
|
||
"DateOfBirth": dob_iso,
|
||
"DateOfDeath": None,
|
||
"CauseOfDeath": None,
|
||
"GoHomeDate": None,
|
||
"Genotype": None,
|
||
"Notes": notes_val,
|
||
"ImportSource": "conflict-decisions.json",
|
||
"ExternalRef": f"decision-{normalize_name(nm)}",
|
||
"RawImportData": json.dumps({"addedByDecision": True}, ensure_ascii=False),
|
||
"OriginBreeder": breeder_disp or ("Zucht der kleinen Chaoten" if is_resident else None),
|
||
"NameSearch": normalize_name(get_normalized_gerbil_name(nm)),
|
||
"CharacterTraits": [],
|
||
"CharacterNote": None,
|
||
"IsDeaf": None,
|
||
"IsResident": is_resident,
|
||
"parentRefs": [],
|
||
"_photos": [],
|
||
"_old_scoped_litter_id": None,
|
||
"_eff_dob": dob_iso or "2015-01-01",
|
||
"_birth_date": dob_iso,
|
||
"_filename": "conflict-decisions.json",
|
||
"_old_id": stub_id,
|
||
"_resolved_by_decision": True,
|
||
# Ein addAnimals-Stub mit explizitem isResident ist eine Mensch-
|
||
# Entscheidung → vom späteren Residenz-Sweep NICHT anfassen lassen
|
||
# (z. B. die resident-Stubs Merle/Pete aus Akanes Fremd-Wurf).
|
||
"_resident_override": a.get("isResident") is not None,
|
||
})
|
||
_existing_call_keys.add(ck)
|
||
_stubs_added += 1
|
||
|
||
# 3) mergeExternalRefs — collapse explicitly-paired duplicate records (esp.
|
||
# nameless animals that the name-based dedup keeps separate). Each pair is
|
||
# [keepRefSuffix, dropRefSuffix]; the drop record is removed and its old id
|
||
# mapped onto the keep record. Refs are matched as a suffix of ExternalRef
|
||
# (extract animals become ExternalRef "stammbaum-<id>").
|
||
_by_ext = {}
|
||
for g in all_processed_gerbils:
|
||
er = g.get("ExternalRef")
|
||
if er:
|
||
_by_ext.setdefault(er, []).append(g)
|
||
|
||
def _find_by_ref_suffix(suffix):
|
||
hits = [g for er, gs in _by_ext.items() if er and er.endswith(suffix) for g in gs]
|
||
return hits
|
||
|
||
_premerged_ids = {} # dropped old id -> kept id (consumed in step 5 mapping)
|
||
_merge_pairs_done = 0
|
||
for d in _decisions:
|
||
for pair in (d.get("mergeExternalRefs") or []):
|
||
if not isinstance(pair, (list, tuple)) or len(pair) != 2:
|
||
continue
|
||
keep_hits = _find_by_ref_suffix(pair[0])
|
||
drop_hits = _find_by_ref_suffix(pair[1])
|
||
if not keep_hits or not drop_hits:
|
||
continue
|
||
keep = keep_hits[0]
|
||
for drop in drop_hits:
|
||
if drop is keep:
|
||
continue
|
||
# Fill gaps on the keeper from the dropped twin, then remove it.
|
||
for fld in ("DateOfBirth", "DateOfDeath", "Genotype",
|
||
"ColorVarietyId", "LitterId", "_old_scoped_litter_id",
|
||
"OriginContactId"):
|
||
if not keep.get(fld) and drop.get(fld):
|
||
keep[fld] = drop[fld]
|
||
if keep.get("Gender") in (None, "unknown") and drop.get("Gender") not in (None, "unknown"):
|
||
keep["Gender"] = drop["Gender"]
|
||
_premerged_ids[drop["Id"]] = keep["Id"]
|
||
_merge_pairs_done += 1
|
||
|
||
if _premerged_ids:
|
||
all_processed_gerbils = [g for g in all_processed_gerbils
|
||
if g["Id"] not in _premerged_ids]
|
||
|
||
# Apply correctDob remappings from decisions BEFORE deduplication!
|
||
_dob_remap_pre_dedup = 0
|
||
for d in _decisions:
|
||
correct_dob = d.get("correctDob")
|
||
if correct_dob:
|
||
er = (d.get("externalRef") or "").strip()
|
||
nm = d.get("name")
|
||
dob_iso = parse_date(correct_dob)
|
||
if dob_iso:
|
||
# Find matching raw gerbils
|
||
for g in all_processed_gerbils:
|
||
match = False
|
||
g_er = g.get("ExternalRef") or ""
|
||
if er and g_er == er:
|
||
match = True
|
||
elif er and g_er.endswith(er):
|
||
match = True
|
||
elif nm:
|
||
g_ck = normalize_name(get_call_name(g.get("Name") or ""))
|
||
d_ck = normalize_name(get_call_name(nm))
|
||
g_dob = g.get("DateOfBirth") or ""
|
||
d_dob = parse_date(d.get("dob")) if d.get("dob") else ""
|
||
if g_ck == d_ck and (not d_dob or g_dob == d_dob):
|
||
match = True
|
||
|
||
if match:
|
||
g["DateOfBirth"] = dob_iso
|
||
g["_birth_date"] = dob_iso
|
||
g["_eff_dob"] = dob_iso
|
||
_dob_remap_pre_dedup += 1
|
||
if _dob_remap_pre_dedup:
|
||
print(f" Pre-Dedup DOB Remappings: {_dob_remap_pre_dedup}")
|
||
|
||
print(f" Entscheidungs-Overrides: Geschlecht={_gender_applied}, "
|
||
f"Stub-Tiere={_stubs_added}, ExternalRef-Merges={_merge_pairs_done}.")
|
||
|
||
# Build parenting dates lookup using old scoped IDs
|
||
parent_litter_dates = {}
|
||
|
||
for l in resolved_litters:
|
||
ld = l["Date"]
|
||
if ld:
|
||
for pid in [l["FatherId"], l["MotherId"]]:
|
||
if pid:
|
||
parent_litter_dates.setdefault(pid, []).append(ld)
|
||
|
||
def _is_empty_shell(g):
|
||
"""A same-name record carrying no birthdate AND no own parent refs — a
|
||
DOB-less Stammbaum mention (e.g. Hagrid Rubeus appearing as a bare
|
||
ancestor box). Such shells must fold into the DOB-/parent-bearing record
|
||
of the same name even if a stray chart placed them as a parent of an
|
||
age-incompatible litter (ticket #18). Their own parent-attributions are
|
||
unreliable, so the parenting-date guard must not keep them separate."""
|
||
return not g.get("_birth_date") and not (g.get("parentRefs") or [])
|
||
|
||
def are_compatible(g1, g2):
|
||
# Same-name empty shell ↔ real record: always merge (see _is_empty_shell).
|
||
if _is_empty_shell(g1) or _is_empty_shell(g2):
|
||
if g1["Gender"] == "unknown" or g2["Gender"] == "unknown" \
|
||
or g1["Gender"] == g2["Gender"]:
|
||
return True
|
||
|
||
# Must have same gender (or one unknown)
|
||
if g1["Gender"] != "unknown" and g2["Gender"] != "unknown" and g1["Gender"] != g2["Gender"]:
|
||
bd1 = g1.get("_birth_date")
|
||
bd2 = g2.get("_birth_date")
|
||
if not (bd1 and bd2 and bd1 == bd2):
|
||
return False
|
||
|
||
bd1 = g1.get("_birth_date")
|
||
bd2 = g2.get("_birth_date")
|
||
|
||
# New rule: if name and parents match, they are compatible regardless of DOB!
|
||
p1 = g1.get("parentRefs", [])
|
||
p2 = g2.get("parentRefs", [])
|
||
f1 = next((p["name"] for p in p1 if p.get("roleGuess") == "father"), "")
|
||
m1 = next((p["name"] for p in p1 if p.get("roleGuess") == "mother"), "")
|
||
f2 = next((p["name"] for p in p2 if p.get("roleGuess") == "father"), "")
|
||
m2 = next((p["name"] for p in p2 if p.get("roleGuess") == "mother"), "")
|
||
|
||
parents_match = False
|
||
if f1 and f2 and m1 and m2:
|
||
if normalize_name(f1) == normalize_name(f2) and normalize_name(m1) == normalize_name(m2):
|
||
parents_match = True
|
||
|
||
if not parents_match:
|
||
# If both have explicit birth dates, they must match within 30 days
|
||
if bd1 and bd2:
|
||
days1 = date_to_days(bd1)
|
||
days2 = date_to_days(bd2)
|
||
if days1 is not None and days2 is not None:
|
||
if abs(days1 - days2) > 30:
|
||
return False
|
||
|
||
# If g1 has birth date, and g2 has parenting dates, birth date must be before parenting dates
|
||
p_dates2 = parent_litter_dates.get(g2["Id"], [])
|
||
if bd1:
|
||
for pd in p_dates2:
|
||
if pd <= bd1: # Can't have litter before or on birth date
|
||
return False
|
||
|
||
p_dates1 = parent_litter_dates.get(g1["Id"], [])
|
||
if bd2:
|
||
for pd in p_dates1:
|
||
if pd <= bd2:
|
||
return False
|
||
|
||
return True
|
||
|
||
def build_provenance(records, best_g, extra_notes=None, field_source=None,
|
||
field_discards=None):
|
||
"""Aggregate data-provenance across every raw record merged into one
|
||
resolved gerbil. Returns a JSON string (stored on the Gerbil entity as a
|
||
nullable text column) so the Rennmausakte can show where the entry came
|
||
from. `records` is the list of raw records that became this gerbil;
|
||
`best_g` is the chosen primary record. `field_source` maps a significant
|
||
field name (DateOfBirth/Genotype/…) to the raw record that supplied its
|
||
final value, so the history can name the exact file for each fact."""
|
||
source_files = set()
|
||
from_wurfchronik = False
|
||
any_conflict = False
|
||
any_decision = False
|
||
for r in records:
|
||
source_files |= _record_source_files(r)
|
||
for f in _record_source_files(r):
|
||
if "wurfchronik" in f.lower():
|
||
from_wurfchronik = True
|
||
if r.get("_conflict"):
|
||
any_conflict = True
|
||
if r.get("_resolved_by_decision"):
|
||
any_decision = True
|
||
|
||
notes = []
|
||
merged_count = len(records)
|
||
if merged_count > 1:
|
||
notes.append(f"aus {merged_count} Datensätzen zusammengeführt")
|
||
if any_decision:
|
||
notes.append("per manueller Entscheidung zugeordnet")
|
||
if any_conflict:
|
||
notes.append("Konflikt per Entscheidung gelöst")
|
||
|
||
# Parent derivation: surface the strongest parentRef method/confidence
|
||
# the primary record carries (chart-position etc.).
|
||
parent_method = None
|
||
parent_confidence = None
|
||
for ref in best_g.get("parentRefs", []) or []:
|
||
if ref.get("method") and not parent_method:
|
||
parent_method = ref.get("method")
|
||
if ref.get("confidence") and not parent_confidence:
|
||
parent_confidence = ref.get("confidence")
|
||
|
||
if extra_notes:
|
||
for n in extra_notes:
|
||
if n and n not in notes:
|
||
notes.append(n)
|
||
|
||
history = _build_gerbil_history(
|
||
records, best_g, field_source or {},
|
||
parent_method=parent_method,
|
||
any_decision=any_decision,
|
||
any_conflict=any_conflict,
|
||
conflict_notes=extra_notes or [],
|
||
field_discards=field_discards,
|
||
parent_discards=best_g.get("_discarded"),
|
||
)
|
||
|
||
return build_entity_provenance(
|
||
source_files,
|
||
merged_count,
|
||
notes=notes,
|
||
from_wurfchronik=from_wurfchronik,
|
||
extra={"parentMethod": parent_method, "parentConfidence": parent_confidence},
|
||
history=history,
|
||
)
|
||
|
||
# Group gerbils by name to perform deduplication
|
||
gerbil_groups = {}
|
||
for g in all_processed_gerbils:
|
||
name_key = get_dedup_name_key(g["Name"])
|
||
if not name_key:
|
||
name_key = "unbekannt"
|
||
gerbil_groups.setdefault(name_key, []).append(g)
|
||
|
||
resolved_gerbils = []
|
||
gerbil_id_map = {} # old_scoped_id -> final_id
|
||
|
||
color_keys = set(variety_map.keys())
|
||
|
||
for name_key, group in gerbil_groups.items():
|
||
is_placeholder = (
|
||
name_key in color_keys or
|
||
any(p in name_key for p in ["unbekannt", "unbenannt", "baby", "welpe", "jungtier", "unknown", "welpen"]) or
|
||
len(name_key) <= 2
|
||
)
|
||
|
||
if is_placeholder:
|
||
# Placeholders: do NOT merge, keep all separate
|
||
for g in group:
|
||
# Provenance is built in a final pass (after parent resolution),
|
||
# so parent-link discards land in the right gerbil's history.
|
||
g["_prov_args"] = ([g], g, None, None, None)
|
||
resolved_gerbils.append(g)
|
||
gerbil_id_map[g["Id"]] = g["Id"]
|
||
continue
|
||
|
||
# Partition group into compatible subsets
|
||
sub_groups = []
|
||
for g in group:
|
||
placed = False
|
||
for sub in sub_groups:
|
||
if all(are_compatible(g, member) for member in sub):
|
||
sub.append(g)
|
||
placed = True
|
||
break
|
||
if not placed:
|
||
sub_groups.append([g])
|
||
|
||
# Merge each partition sub-group into a single gerbil
|
||
for sub in sub_groups:
|
||
if len(sub) == 1:
|
||
g = sub[0]
|
||
g["_prov_args"] = ([g], g, None, None, None)
|
||
resolved_gerbils.append(g)
|
||
gerbil_id_map[g["Id"]] = g["Id"]
|
||
continue
|
||
|
||
# Find the best primary record to merge into
|
||
best_g = None
|
||
best_score = -1
|
||
for g in sub:
|
||
score = 0
|
||
if g["_old_scoped_litter_id"]: score += 10
|
||
if g["DateOfBirth"]: score += 5
|
||
if g["Genotype"]: score += 3
|
||
if g["ColorVarietyId"]: score += 2
|
||
if g["ImportSource"] and "stammbaum" in g["ImportSource"].lower(): score += 20
|
||
if g["Notes"] and not any(kw in g["Notes"].lower() for kw in ["parent", "mutter", "vater", "dam", "sire"]): score += 1
|
||
if len(g["Name"]) > len(name_key) + 5: # likely has clan suffix
|
||
score += 15
|
||
if score > best_score:
|
||
best_score = score
|
||
best_g = g
|
||
|
||
# Merge fields
|
||
merged_notes = []
|
||
if best_g["Notes"]:
|
||
merged_notes.append(best_g["Notes"])
|
||
|
||
# Carry over discard records (rejected parent-refs / dob remaps) from
|
||
# every merged record so none are lost when the primary changes.
|
||
merged_discards = list(best_g.get("_discarded") or [])
|
||
for g in sub:
|
||
if g is best_g:
|
||
continue
|
||
for d in (g.get("_discarded") or []):
|
||
if d not in merged_discards:
|
||
merged_discards.append(d)
|
||
best_g["_discarded"] = merged_discards
|
||
|
||
# Merge photos
|
||
merged_photos = list(best_g.get("_photos", []))
|
||
|
||
# Track sources for debugging
|
||
sources = [best_g["_filename"]]
|
||
|
||
# Per-field file attribution: which raw record supplied each final
|
||
# field value. Seed with best_g for every field it already carries;
|
||
# the fill loop and voting loop update it as winners change.
|
||
ATTRIB_FIELDS = ["DateOfBirth", "DateOfDeath", "Gender", "Genotype",
|
||
"ColorVarietyId", "Name"]
|
||
field_source = {}
|
||
for fld in ATTRIB_FIELDS:
|
||
v = best_g.get(fld)
|
||
if v and v != "unknown":
|
||
field_source[fld] = best_g
|
||
|
||
for g in sub:
|
||
if g == best_g:
|
||
continue
|
||
gerbil_id_map[g["Id"]] = best_g["Id"]
|
||
sources.append(g["_filename"])
|
||
|
||
for ph in g.get("_photos", []):
|
||
if ph not in merged_photos:
|
||
merged_photos.append(ph)
|
||
|
||
if not best_g.get("_old_scoped_litter_id") and g.get("_old_scoped_litter_id"):
|
||
best_g["_old_scoped_litter_id"] = g["_old_scoped_litter_id"]
|
||
if not best_g["LitterId"] and g["LitterId"]:
|
||
best_g["LitterId"] = g["LitterId"]
|
||
if not best_g["DateOfBirth"] and g["DateOfBirth"]:
|
||
best_g["DateOfBirth"] = g["DateOfBirth"]
|
||
field_source["DateOfBirth"] = g
|
||
if not best_g["DateOfDeath"] and g["DateOfDeath"]:
|
||
best_g["DateOfDeath"] = g["DateOfDeath"]
|
||
field_source["DateOfDeath"] = g
|
||
if not best_g["CauseOfDeath"] and g["CauseOfDeath"]:
|
||
best_g["CauseOfDeath"] = g["CauseOfDeath"]
|
||
if not best_g["GoHomeDate"] and g["GoHomeDate"]:
|
||
best_g["GoHomeDate"] = g["GoHomeDate"]
|
||
if not best_g["Genotype"] and g["Genotype"]:
|
||
best_g["Genotype"] = g["Genotype"]
|
||
field_source["Genotype"] = g
|
||
if not best_g["ColorVarietyId"] and g["ColorVarietyId"]:
|
||
best_g["ColorVarietyId"] = g["ColorVarietyId"]
|
||
field_source["ColorVarietyId"] = g
|
||
if not best_g["OriginContactId"] and g["OriginContactId"]:
|
||
best_g["OriginContactId"] = g["OriginContactId"]
|
||
if not best_g["ReceiverContactId"] and g["ReceiverContactId"]:
|
||
best_g["ReceiverContactId"] = g["ReceiverContactId"]
|
||
if g["IsResident"]:
|
||
best_g["IsResident"] = True
|
||
|
||
# Reconcile Gender: prefer a known gender over unknown, and prefer stammbaum over other sources
|
||
if best_g["Gender"] == "unknown" and g["Gender"] != "unknown":
|
||
best_g["Gender"] = g["Gender"]
|
||
field_source["Gender"] = g
|
||
elif best_g["Gender"] != "unknown" and g["Gender"] != "unknown" and best_g["Gender"] != g["Gender"]:
|
||
if g["ImportSource"] and "stammbaum" in g["ImportSource"].lower():
|
||
if not best_g["ImportSource"] or "stammbaum" not in best_g["ImportSource"].lower():
|
||
best_g["Gender"] = g["Gender"]
|
||
field_source["Gender"] = g
|
||
|
||
# Status precedence: Deceased > GivenAway > Breeding/Pet
|
||
if g["Status"] == "Deceased":
|
||
best_g["Status"] = "Deceased"
|
||
elif g["Status"] == "GivenAway" and best_g["Status"] not in ["Deceased"]:
|
||
best_g["Status"] = "GivenAway"
|
||
|
||
if g["Notes"] and g["Notes"] not in merged_notes:
|
||
# Ignore redundant dummy notes
|
||
if not any(kw in g["Notes"].lower() for kw in ["parent listed", "mutter von", "vater von", "dam of", "sire of"]):
|
||
merged_notes.append(g["Notes"])
|
||
|
||
# Display formatter per field for discard/replacement lines.
|
||
def _disp(field, val):
|
||
if val is None or val == "" or val == "unknown":
|
||
return None
|
||
if field in ("DateOfBirth", "DateOfDeath"):
|
||
return _de_date(val)
|
||
if field == "Gender":
|
||
return _de_gender(val)
|
||
if field == "ColorVarietyId":
|
||
return variety_id_to_name.get(val, "Farbschlag")
|
||
return str(val)
|
||
|
||
# Reconcile fields based on number of source files supporting them
|
||
conflict_notes = []
|
||
field_discards = []
|
||
for field in ["DateOfBirth", "DateOfDeath", "Gender", "Genotype", "ColorVarietyId"]:
|
||
votes = {}
|
||
for g in sub:
|
||
val = g.get(field)
|
||
if val and val != "unknown":
|
||
# count source files
|
||
sources_count = len(str(g.get("ImportSource") or "").split(","))
|
||
votes[val] = votes.get(val, 0) + sources_count
|
||
if votes:
|
||
best_val = max(votes, key=votes.get)
|
||
# If the records disagreed on a field, the merge had to pick a
|
||
# winner — record that as a provenance note.
|
||
if len(votes) > 1:
|
||
conflict_notes.append(
|
||
f"Konflikt bei {PROV_FIELD_LABELS[field]} per Mehrheitsentscheidung gelöst"
|
||
)
|
||
best_g[field] = best_val
|
||
# Attribute the winning value to a record that actually holds
|
||
# it, so the history names the right file.
|
||
winner = next((g for g in sub if g.get(field) == best_val), None)
|
||
if winner is not None:
|
||
field_source[field] = winner
|
||
# Record each LOSING value: which file it came from, that it
|
||
# was discarded as differing, and that the majority value (and
|
||
# its file) was used instead.
|
||
if len(votes) > 1:
|
||
repl_disp = _disp(field, best_val)
|
||
repl_file = _primary_file_of(winner) if winner is not None else None
|
||
for lose_val in votes:
|
||
if lose_val == best_val:
|
||
continue
|
||
loser = next((g for g in sub if g.get(field) == lose_val), None)
|
||
field_discards.append({
|
||
"label": PROV_FIELD_LABELS.get(field, field),
|
||
"value": _disp(field, lose_val),
|
||
"file": _primary_file_of(loser) if loser is not None else None,
|
||
"reason": "abweichend",
|
||
"replacement": repl_disp,
|
||
"repl_file": repl_file,
|
||
"replacement_note": "Mehrheit",
|
||
})
|
||
# Keep helper fields in sync if we changed DateOfBirth
|
||
if field == "DateOfBirth":
|
||
best_g["_birth_date"] = best_val
|
||
best_g["_eff_dob"] = best_val or "2010-01-01"
|
||
|
||
if merged_notes:
|
||
best_g["Notes"] = " | ".join(merged_notes)
|
||
|
||
best_g["_photos"] = merged_photos
|
||
# Defer provenance to a final pass so parent-link discards (added
|
||
# later by the parent-age / role-normalization passes) are included.
|
||
best_g["_prov_args"] = (sub, best_g, conflict_notes, field_source, field_discards)
|
||
|
||
# Print merge trace
|
||
print(f"Deduplicated same-animal name '{best_g['Name']}': merged {len(sub)} entries across files: {', '.join(sources)}")
|
||
|
||
resolved_gerbils.append(best_g)
|
||
gerbil_id_map[best_g["Id"]] = best_g["Id"]
|
||
|
||
print(f"Deduplicated to {len(resolved_gerbils)} unique gerbil records.")
|
||
|
||
# ── isResident/notes/receiver-Overrides (Mensch-Entscheidung) ─────────────
|
||
# NACH dem Dedup anwenden, sonst würde die Dedup-Zusammenführung (IsResident=True
|
||
# falls eine Variante resident ist) sie wieder überschreiben. Auch NACH der
|
||
# Residenz-Propagation (die nur auf Stammbaum-Roh-Tiere wirkt) → diese Overrides
|
||
# sind das letzte Wort. Matcht primär über `externalRef` (präzise, auch für
|
||
# namenlose Tiere) oder sonst über normalize(call-name)+ISO-dob (leere dob =
|
||
# name-only, für Vorfahren ohne Datum). Ein Match kann zusätzlich `notes` (→
|
||
# Notes, autoritativ) und `receiver` (Kontaktname → ReceiverContactId, Status=
|
||
# GivenAway) setzen — so bekommt ein bereits existierendes Tier (z. B. Akane)
|
||
# seinen Lebenslauf + Abnehmer, ohne ein addAnimals-Stub zu sein.
|
||
_ovr_by_extref = {}
|
||
_ovr_by_namedob = {}
|
||
for d in _decisions:
|
||
has_ovr = (("isResident" in d) or d.get("notes") or d.get("receiver")
|
||
or d.get("renameTo") or d.get("deceased") or d.get("dateOfDeath")
|
||
or d.get("correctDob") or d.get("genotype") or d.get("farbschlag")
|
||
or d.get("gender") or d.get("father") or d.get("mother")
|
||
or d.get("originBreeder"))
|
||
if not has_ovr:
|
||
continue
|
||
er = (d.get("externalRef") or "").strip()
|
||
if er:
|
||
_ovr_by_extref[er] = d
|
||
nm = d.get("name")
|
||
if nm is not None:
|
||
ck = normalize_name(get_call_name(nm or ""))
|
||
iso = parse_date(d.get("dob")) if d.get("dob") else ""
|
||
_ovr_by_namedob[(ck, iso or "")] = d
|
||
_isres_applied = 0
|
||
if _ovr_by_extref or _ovr_by_namedob:
|
||
for g in resolved_gerbils:
|
||
er = g.get("ExternalRef") or ""
|
||
ck = normalize_name(get_call_name(g.get("Name") or ""))
|
||
iso = g.get("DateOfBirth") or ""
|
||
if er and er in _ovr_by_extref:
|
||
d = _ovr_by_extref[er]
|
||
elif er and any(er.endswith(k) for k in _ovr_by_extref):
|
||
d = next(v for k, v in _ovr_by_extref.items() if er.endswith(k))
|
||
elif (ck, iso) in _ovr_by_namedob:
|
||
d = _ovr_by_namedob[(ck, iso)]
|
||
elif (ck, "") in _ovr_by_namedob:
|
||
d = _ovr_by_namedob[(ck, "")]
|
||
else:
|
||
continue
|
||
applied = False
|
||
if "originBreeder" in d:
|
||
val = d["originBreeder"]
|
||
if g.get("OriginBreeder") != val:
|
||
g["OriginBreeder"] = val
|
||
applied = True
|
||
if "isResident" in d:
|
||
val = bool(d["isResident"])
|
||
# Ein expliziter Override (true ODER false) ist das letzte Wort und
|
||
# darf vom späteren Residenz-Sweep NICHT überschrieben werden.
|
||
g["_resident_override"] = True
|
||
if g.get("IsResident") != val:
|
||
g["IsResident"] = val
|
||
# Ein Nicht-Bestandstier ist kein eigenes Zuchttier → den eigenen
|
||
# Zucht-Breeder entfernen (nur, wenn er auf die eigene Zucht zeigt).
|
||
if not val and g.get("OriginBreeder") == "Zucht der kleinen Chaoten":
|
||
g["OriginBreeder"] = None
|
||
applied = True
|
||
notes_ovr = (d.get("notes") or "").strip()
|
||
if notes_ovr and g.get("Notes") != notes_ovr:
|
||
g["Notes"] = notes_ovr
|
||
applied = True
|
||
rec_name = d.get("receiver")
|
||
if rec_name:
|
||
rec_cid = _resolve_contact_id_by_name(rec_name)
|
||
if rec_cid and g.get("ReceiverContactId") != rec_cid:
|
||
g["ReceiverContactId"] = rec_cid
|
||
if g.get("Status") not in ("Deceased",):
|
||
g["Status"] = "GivenAway"
|
||
applied = True
|
||
# renameTo: korrigiert den Anzeigenamen eines bestehenden Tieres. Der
|
||
# Match-Key bleibt der ursprüngliche name/externalRef (NICHT umdeuten) —
|
||
# erst HIER wird der Name überschrieben.
|
||
rename_to = (d.get("renameTo") or "").strip()
|
||
if rename_to and g.get("Name") != rename_to:
|
||
# Quell-Namen merken, damit spätere Eltern-Verknüpfungen, die noch den
|
||
# alten Namen referenzieren, weiter auflösen (siehe Namens-Index unten,
|
||
# Ticket e0a0c304 Kruke→Kuke: sonst verlieren Kukes Würfe die Mutter).
|
||
g["_pre_rename_name"] = g.get("Name")
|
||
g["Name"] = rename_to
|
||
applied = True
|
||
# deceased: markiert ein bestehendes Tier als verstorben, falls noch
|
||
# nicht. dateOfDeath wird bewusst NICHT angefasst.
|
||
if d.get("deceased") and g.get("Status") != "Deceased":
|
||
g["Status"] = "Deceased"
|
||
applied = True
|
||
# dateOfDeath: setzt das Todesdatum (falls noch keins) + Status Deceased.
|
||
# rpro3 hat keine Sterbespalte, daher gibt die Züchterin/das Ticket das Datum vor.
|
||
if d.get("dateOfDeath") and not g.get("DateOfDeath"):
|
||
_dod = parse_date(d["dateOfDeath"])
|
||
if _dod:
|
||
g["DateOfDeath"] = _dod
|
||
g["Status"] = "Deceased"
|
||
applied = True
|
||
if d.get("correctDob") and not g.get("DateOfBirth"):
|
||
dob_iso = parse_date(d["correctDob"])
|
||
g["DateOfBirth"] = dob_iso
|
||
g["_birth_date"] = dob_iso
|
||
g["_eff_dob"] = dob_iso
|
||
applied = True
|
||
if d.get("genotype") and not g.get("Genotype"):
|
||
g["Genotype"] = d["genotype"].strip()
|
||
applied = True
|
||
if d.get("farbschlag") and not g.get("ColorVarietyId"):
|
||
cv_id = variety_map.get(d["farbschlag"].strip().lower())
|
||
if cv_id:
|
||
g["ColorVarietyId"] = cv_id
|
||
applied = True
|
||
if d.get("gender") and g.get("Gender") in ("unknown", None):
|
||
gender = d["gender"].strip().lower()
|
||
gender = {"m": "male", "männlich": "male", "w": "female",
|
||
"f": "female", "weiblich": "female"}.get(gender, gender)
|
||
if gender in ("male", "female"):
|
||
g["Gender"] = gender
|
||
applied = True
|
||
if d.get("father"):
|
||
g["_override_father_name"] = d["father"].strip()
|
||
g["_override_father_dob"] = d.get("fatherDob")
|
||
applied = True
|
||
if d.get("mother"):
|
||
g["_override_mother_name"] = d["mother"].strip()
|
||
g["_override_mother_dob"] = d.get("motherDob")
|
||
applied = True
|
||
if applied:
|
||
_isres_applied += 1
|
||
if _isres_applied:
|
||
print(f"isResident/notes/receiver/dob/geno-Overrides angewandt: {_isres_applied}")
|
||
|
||
|
||
|
||
# Apply age-based death threshold (6.0 years) to all resolved gerbils
|
||
dt_now = datetime.now()
|
||
for g in resolved_gerbils:
|
||
if g.get("Status") != "Deceased" and g.get("Status") != "GivenAway":
|
||
if not g.get("DateOfDeath") and not g.get("ReceiverContactId"):
|
||
dob_str = g.get("DateOfBirth")
|
||
if dob_str:
|
||
try:
|
||
dt_dob = datetime.strptime(dob_str, "%Y-%m-%d")
|
||
age_years = (dt_now - dt_dob).days / 365.25
|
||
if age_years >= 6.0:
|
||
g["Status"] = "Deceased"
|
||
except Exception:
|
||
pass
|
||
|
||
# 5. Map Gerbils to Litters
|
||
for g in resolved_gerbils:
|
||
old_lid = g["_old_scoped_litter_id"]
|
||
l_guid = litter_id_map.get(old_lid)
|
||
g["LitterId"] = l_guid
|
||
|
||
if l_guid and l_guid in litter_by_scoped_id:
|
||
l = litter_by_scoped_id[l_guid]
|
||
l_date = l.get("Date")
|
||
if l_date:
|
||
if not g.get("DateOfBirth"):
|
||
g["DateOfBirth"] = l_date
|
||
g["_birth_date"] = l_date
|
||
g["_eff_dob"] = l_date
|
||
|
||
# Apply age-based death threshold (6.0 years) to all resolved gerbils (including newly backfilled ones)
|
||
if g.get("Status") != "Deceased" and g.get("Status") != "GivenAway":
|
||
if not g.get("DateOfDeath") and not g.get("ReceiverContactId"):
|
||
dob_str = g.get("DateOfBirth")
|
||
if dob_str:
|
||
try:
|
||
dt_dob = datetime.strptime(dob_str, "%Y-%m-%d")
|
||
age_years = (dt_now - dt_dob).days / 365.25
|
||
if age_years >= 6.0:
|
||
g["Status"] = "Deceased"
|
||
except Exception:
|
||
pass
|
||
|
||
# Clean helper fields
|
||
del g["_old_scoped_litter_id"]
|
||
del g["_eff_dob"]
|
||
if "_birth_date" in g:
|
||
del g["_birth_date"]
|
||
del g["_filename"]
|
||
del g["_old_id"]
|
||
g.pop("_conflict", None)
|
||
g.pop("_resolved_by_decision", None)
|
||
|
||
# ── CREATE VIRTUAL LITTERS FOR PARENT OVERRIDES ──
|
||
_virtual_overrides_created = 0
|
||
for g in resolved_gerbils:
|
||
f_name = g.get("_override_father_name")
|
||
m_name = g.get("_override_mother_name")
|
||
if (f_name or m_name) and not g.get("LitterId"):
|
||
litter_id = generate_guid(f"override-virtual-litter-{g['Id']}")
|
||
g["LitterId"] = litter_id
|
||
resolved_litters.append({
|
||
"Id": litter_id,
|
||
"Name": f"Wurf von {f_name or '—'} + {m_name or '—'}",
|
||
"Date": g.get("DateOfBirth"),
|
||
"TotalBorn": 1,
|
||
"DeathsWithin8Weeks": 0,
|
||
"FatherId": None,
|
||
"MotherId": None,
|
||
"Notes": f"Virtueller Wurf für {g['Name']} (über Decisions-Override angelegt).",
|
||
"LitterLetter": None,
|
||
"ExternalRef": f"override-virtual-litter-{g['Id']}",
|
||
"_father_name": f_name or "",
|
||
"_mother_name": m_name or "",
|
||
"_filename": "conflict-decisions.json",
|
||
"_source_files": {"conflict-decisions.json"},
|
||
"_merged_count": 1,
|
||
"_virtual": True
|
||
})
|
||
_virtual_overrides_created += 1
|
||
if _virtual_overrides_created:
|
||
print(f"Mensch-Entscheidungs-Eltern-Overrides: {_virtual_overrides_created} virtuelle Würfe erstellt.")
|
||
|
||
# Gather final valid gerbil IDs
|
||
valid_gerbil_ids = {g["Id"] for g in resolved_gerbils}
|
||
|
||
# Create name lookup for resolved gerbils
|
||
gerbil_by_norm_name = {}
|
||
for g in resolved_gerbils:
|
||
keys = set()
|
||
n_key = normalize_name(g["Name"])
|
||
keys.add(n_key)
|
||
# Also index by call-name to resolve parents who are only listed by call-name
|
||
keys.add(normalize_name(get_call_name(g["Name"])))
|
||
# …and by the connector-folding canon key so abbreviation variants match
|
||
# (e.g. „BlackFire v.d. Kleinen Chaoten“ vs „… von den …“ — ticket #30).
|
||
keys.add(canon_name_key(g["Name"]))
|
||
# Umbenannte Tiere (renameTo) auch unter ihrem Quell-Namen indexieren, damit
|
||
# Eltern-Verknüpfungen, die noch den alten Namen referenzieren, weiter auflösen.
|
||
pre = g.get("_pre_rename_name")
|
||
if pre:
|
||
keys.add(normalize_name(pre))
|
||
keys.add(normalize_name(get_call_name(pre)))
|
||
keys.add(canon_name_key(pre))
|
||
for k in keys:
|
||
if k:
|
||
gerbil_by_norm_name.setdefault(k, []).append(g)
|
||
|
||
# Map raw Guid if present (convert if old_id mapped to new_guid)
|
||
for l in resolved_litters:
|
||
# First redirect any parent pointing at a record dropped by an explicit
|
||
# mergeExternalRefs pre-merge onto its surviving twin (then through the
|
||
# normal id map), so the two collapsed litters share the same parents.
|
||
if l["FatherId"] in _premerged_ids:
|
||
l["FatherId"] = _premerged_ids[l["FatherId"]]
|
||
if l["MotherId"] in _premerged_ids:
|
||
l["MotherId"] = _premerged_ids[l["MotherId"]]
|
||
if l["FatherId"] in gerbil_id_map:
|
||
l["FatherId"] = gerbil_id_map[l["FatherId"]]
|
||
if l["MotherId"] in gerbil_id_map:
|
||
l["MotherId"] = gerbil_id_map[l["MotherId"]]
|
||
|
||
# Clean foreign keys that do not point to a valid gerbil
|
||
if l["FatherId"] and l["FatherId"] not in valid_gerbil_ids:
|
||
l["FatherId"] = None
|
||
if l["MotherId"] and l["MotherId"] not in valid_gerbil_ids:
|
||
l["MotherId"] = None
|
||
|
||
# Parent Resolver (Global Name Matching)
|
||
resolved_fathers = 0
|
||
resolved_mothers = 0
|
||
|
||
gerbil_by_id_final = {g["Id"]: g for g in resolved_gerbils}
|
||
|
||
def _final_gender(gid):
|
||
g = gerbil_by_id_final.get(gid)
|
||
return g["Gender"] if g else None
|
||
|
||
def _resolve_name(name, prefer_gender, litter_date):
|
||
"""Resolve a parent name to the best matching final gerbil.
|
||
|
||
Gender is a PREFERENCE, not a hard filter: a reversed parent (e.g. a
|
||
female listed in the father position, as the Stammbaum often does) still
|
||
resolves to a gerbil — the role is corrected afterwards by gender. This
|
||
is what previously left FatherId/MotherId null (the candidate was
|
||
filtered out for having the "wrong" gender for its slot).
|
||
"""
|
||
if not name:
|
||
return None
|
||
# Try the exact normalized key, then the connector-folding canon key and
|
||
# the call-name (handles v.d. ↔ von den abbreviation variants — #30).
|
||
lookups = []
|
||
for k in (normalize_name(name), canon_name_key(name),
|
||
normalize_name(get_call_name(name))):
|
||
if k and k not in lookups:
|
||
lookups.append(k)
|
||
# Resolve key-by-key so an EXACT full-name match (lookups[0]) wins over a
|
||
# mere call-name/canon fallback — e.g. override mother „Elena“ must pick
|
||
# the resident „Elena“, not „Elena of KK Chaos“ whose call-name is also
|
||
# „Elena“ (ticket #15).
|
||
seen_ids = set()
|
||
for k in lookups:
|
||
cands = []
|
||
for c in gerbil_by_norm_name.get(k, []):
|
||
if c["Id"] in seen_ids:
|
||
continue
|
||
seen_ids.add(c["Id"])
|
||
final_id = gerbil_id_map.get(c["Id"])
|
||
if not final_id:
|
||
continue
|
||
final_c = gerbil_by_id_final.get(final_id)
|
||
if not final_c:
|
||
continue
|
||
# Parent must be age-plausible: born before the litter and within
|
||
# the gerbil lifespan (skips e.g. a 2013 animal for a 2022 litter).
|
||
if not parent_age_plausible(final_c["DateOfBirth"], litter_date):
|
||
continue
|
||
# Sterbedatum-Plausibilität: ein vor dem Wurf verstorbenes Tier kann kein
|
||
# Elternteil sein (Namensvetter-Fall: die gleichnamige, noch lebende Maus ist
|
||
# die echte Mutter/der echte Vater). ~40 Tage Toleranz, falls ein Elternteil
|
||
# kurz vor der Geburt starb. Nur filtern, wenn beide Daten bekannt.
|
||
dod = final_c.get("DateOfDeath")
|
||
if dod and litter_date:
|
||
try:
|
||
if datetime.strptime(dod, "%Y-%m-%d") < datetime.strptime(litter_date, "%Y-%m-%d") - timedelta(days=40):
|
||
continue
|
||
except Exception:
|
||
pass
|
||
cands.append(final_c)
|
||
if not cands:
|
||
continue
|
||
# Within a key, a candidate whose FULL name equals the lookup beats one
|
||
# that only matched via call-name (resident „Elena“ > „Elena of KK
|
||
# Chaos“ — #15). Stable sort keeps prior ordering otherwise.
|
||
cands.sort(key=lambda c: 0 if normalize_name(c["Name"]) == k else 1)
|
||
# Prefer the gender expected for this role, then unknown, then anything.
|
||
for pool in (
|
||
[c for c in cands if c["Gender"] == prefer_gender],
|
||
[c for c in cands if c["Gender"] == "unknown"],
|
||
cands,
|
||
):
|
||
if pool:
|
||
return pool[0]
|
||
return None
|
||
|
||
# ── Fehlende Wurf-Eltern aus der xlsx-Wurfchronik (output/litters.json) füllen ──
|
||
# extract/merge konsumiert litters.json bisher NICHT; viele md-only/Wurfchronik-
|
||
# Würfe haben deshalb keinen Eltern-Namen. Hier NUR Lücken füllen (nie überschreiben)
|
||
# und NUR bei eindeutigem Match (eindeutiges Datum, sonst Buchstabe+Datum). Die
|
||
# eigentliche Verknüpfung macht danach _resolve_name (Alters-/Gender-Plausibilität) —
|
||
# nicht auflösbare Namen bleiben unverknüpft (kein Falsch-Link).
|
||
try:
|
||
_lj_path = os.path.join(OUTPUT_DIR, "litters.json")
|
||
_lj = json.load(open(_lj_path, encoding="utf-8")) if os.path.exists(_lj_path) else []
|
||
_lj = _lj if isinstance(_lj, list) else _lj.get("litters", [])
|
||
except Exception:
|
||
_lj = []
|
||
|
||
def _lj_iso(s):
|
||
try:
|
||
dd, mm, yy = (s or "").split("."); return f"{yy}-{mm}-{dd}"
|
||
except Exception:
|
||
return s or ""
|
||
|
||
def _strip_zucht(n):
|
||
return (n or "").split(" [")[0].strip()
|
||
|
||
_lj_by_date = {}
|
||
for _e in _lj:
|
||
_lj_by_date.setdefault(_lj_iso(_e.get("date")), []).append(_e)
|
||
|
||
_lj_filled = 0
|
||
_lj_mort = 0
|
||
for l in resolved_litters:
|
||
cands = _lj_by_date.get(l.get("Date") or "", [])
|
||
if len(cands) > 1 and l.get("LitterLetter"):
|
||
narrowed = [e for e in cands if str(e.get("litterId") or "").upper() == str(l.get("LitterLetter")).upper()]
|
||
if narrowed:
|
||
cands = narrowed
|
||
if len(cands) != 1:
|
||
continue
|
||
e = cands[0]
|
||
# Eltern-Namen — nur Luecken (nie ueberschreiben)
|
||
if not l.get("FatherId") and not l.get("_father_name") and _strip_zucht(e.get("sireName")):
|
||
l["_father_name"] = _strip_zucht(e.get("sireName")); _lj_filled += 1
|
||
if not l.get("MotherId") and not l.get("_mother_name") and _strip_zucht(e.get("damName")):
|
||
l["_mother_name"] = _strip_zucht(e.get("damName")); _lj_filled += 1
|
||
# Totgeburten / Fruehsterblichkeit / Wurfstaerke — nur Luecken
|
||
if l.get("Stillborn") is None and e.get("stillborn") is not None:
|
||
l["Stillborn"] = e.get("stillborn"); _lj_mort += 1
|
||
if l.get("DeathsWithin8Weeks") is None and e.get("diedLater") is not None:
|
||
l["DeathsWithin8Weeks"] = e.get("diedLater")
|
||
if l.get("TotalBorn") is None and e.get("totalBorn") is not None:
|
||
l["TotalBorn"] = e.get("totalBorn")
|
||
if _lj_filled or _lj_mort:
|
||
print(f"litters.json ergaenzt: {_lj_filled} Eltern-Namen, {_lj_mort} Totgeburt-Angaben")
|
||
|
||
for l in resolved_litters:
|
||
# Pre-check: if _father_name points to a known female and _mother_name to a
|
||
# known male → swap names (Stammbaum positions reversed). Helps the name
|
||
# resolver pick the right same-name candidate before role normalization.
|
||
f_name_pre = l.get("_father_name", "")
|
||
m_name_pre = l.get("_mother_name", "")
|
||
if f_name_pre and m_name_pre:
|
||
f_gender = next((g["Gender"] for g in gerbil_by_norm_name.get(normalize_name(f_name_pre), []) if g["Gender"] != "unknown"), None)
|
||
m_gender = next((g["Gender"] for g in gerbil_by_norm_name.get(normalize_name(m_name_pre), []) if g["Gender"] != "unknown"), None)
|
||
if f_gender == "female" and m_gender == "male":
|
||
l["_father_name"], l["_mother_name"] = m_name_pre, f_name_pre
|
||
|
||
if l["_father_name"] and not l["FatherId"]:
|
||
cand = _resolve_name(l["_father_name"], "male", l["Date"])
|
||
if cand:
|
||
l["FatherId"] = cand["Id"]
|
||
resolved_fathers += 1
|
||
|
||
if l["_mother_name"] and not l["MotherId"]:
|
||
cand = _resolve_name(l["_mother_name"], "female", l["Date"])
|
||
if cand:
|
||
l["MotherId"] = cand["Id"]
|
||
resolved_mothers += 1
|
||
|
||
# Cleanup internal keys
|
||
del l["_father_name"]
|
||
del l["_mother_name"]
|
||
del l["_filename"]
|
||
|
||
# Children-of-litter lookup so a dropped parent link can be explained in the
|
||
# offspring's Datenherkunft (the discard is most meaningful on the child).
|
||
children_by_litter = {}
|
||
for g in resolved_gerbils:
|
||
lid = g.get("LitterId")
|
||
if lid:
|
||
children_by_litter.setdefault(lid, []).append(g)
|
||
|
||
def _add_child_discard(litter, discard):
|
||
"""Attach a parent-link discard to every child gerbil of the litter."""
|
||
for child in children_by_litter.get(litter["Id"], []):
|
||
dl = child.setdefault("_discarded", [])
|
||
if discard not in dl:
|
||
dl.append(discard)
|
||
|
||
def _pname(pid):
|
||
p = gerbil_by_id_final.get(pid)
|
||
return p.get("Name") if p else None
|
||
|
||
# Role normalization: assign each resolved parent to the role matching its
|
||
# gender, eliminate self-pairings (same animal in both roles), and never let
|
||
# impossible duplicates survive (two males / two females). This corrects
|
||
# reversed Stammbaum positions including the cases the simple swap missed
|
||
# (one parent of "unknown" gender, or a self-paired litter).
|
||
role_fixes = 0
|
||
for l in resolved_litters:
|
||
before = (l.get("FatherId"), l.get("MotherId"))
|
||
father, mother = assign_parent_roles(l.get("FatherId"), l.get("MotherId"), _final_gender)
|
||
if before != (father, mother):
|
||
role_fixes += 1
|
||
after = {father, mother}
|
||
# A parent id present before but gone after was dropped by role
|
||
# normalization (self-pairing or two-of-the-same-sex). Explain it.
|
||
for pid in before:
|
||
if pid and pid not in after:
|
||
pname = _pname(pid)
|
||
if before[0] == before[1]:
|
||
reason = "Selbstverpaarung — Tier kann nicht beide Elternteile sein"
|
||
else:
|
||
reason = "ein Wurf hat nur einen Vater und eine Mutter"
|
||
_add_child_discard(l, {
|
||
"text": f"Elternteil „{pname}“ verworfen — {reason}",
|
||
})
|
||
l["FatherId"] = father
|
||
l["MotherId"] = mother
|
||
if role_fixes:
|
||
print(f"Role-normalization: corrected {role_fixes} litter(s) (gender roles / self-pairings).")
|
||
|
||
# Parent-age sanity check: drop any resolved parent that cannot belong to the
|
||
# litter — born after the offspring, or more than a gerbil lifespan earlier.
|
||
# Catches mis-resolved links the name matcher still let through (e.g. Jayjay,
|
||
# *2013, wrongly attached to Solice's 2022 litter).
|
||
age_drops = []
|
||
for l in resolved_litters:
|
||
ldate = l.get("Date")
|
||
for role in ("FatherId", "MotherId"):
|
||
pid = l.get(role)
|
||
if not pid:
|
||
continue
|
||
p = gerbil_by_id_final.get(pid)
|
||
if p and not parent_age_plausible(p.get("DateOfBirth"), ldate):
|
||
age_drops.append((l.get("Name"), role, p.get("Name"), p.get("DateOfBirth"), ldate))
|
||
# Explain the drop in each child's history: which parent, its DOB,
|
||
# why (implausible age), and that no replacement was used.
|
||
role_de = "Vater" if role == "FatherId" else "Mutter"
|
||
pdob_disp = _de_date(p.get("DateOfBirth")) or "unbekannt"
|
||
age_reason = "unplausibles Alter für diesen Wurf"
|
||
pd = date_to_days(parse_date(p.get("DateOfBirth"))) if p.get("DateOfBirth") else None
|
||
ld = date_to_days(parse_date(ldate)) if ldate else None
|
||
if pd is not None and ld is not None:
|
||
years = abs(ld - pd) / 365.25
|
||
if ld - pd <= 0:
|
||
age_reason = "unplausibel (nicht vor dem Kind geboren)"
|
||
else:
|
||
age_reason = f"unplausibel ({years:.0f} Jahre älter als das Kind)"
|
||
_add_child_discard(l, {
|
||
"text": (f"{role_de} „{p.get('Name')}“ (*{pdob_disp}) verworfen "
|
||
f"— {age_reason}; kein Ersatz"),
|
||
})
|
||
l[role] = None
|
||
if age_drops:
|
||
print(f"Parent-age sanity check: dropped {len(age_drops)} implausible parent link(s):")
|
||
for lname, role, pname, pdob, ldate in age_drops[:20]:
|
||
print(f" {lname}: {role}={pname} (*{pdob}) vs litter {ldate}")
|
||
|
||
print(f"Globally resolved {resolved_fathers} fathers and {resolved_mothers} mothers.")
|
||
|
||
|
||
# 5b. Second-pass litter dedup: now that FatherId/MotherId are known,
|
||
# merge litters that have the same date AND the same parents.
|
||
# This is the core of the "sibling pairing" fix: Blue Wave and Sunny Sky
|
||
# both come from Wonderman × Unique — their two separate litter records
|
||
# must now become one, so their children share the same LitterId.
|
||
litter_by_id_post = {l["Id"]: l for l in resolved_litters}
|
||
gerbil_by_litter = {}
|
||
for g in resolved_gerbils:
|
||
lid = g.get("LitterId")
|
||
if lid:
|
||
gerbil_by_litter.setdefault(lid, []).append(g)
|
||
|
||
def _litter_same_parents(l1, l2):
|
||
"""Strict: same date + both parents known and matching."""
|
||
if l1["Date"] != l2["Date"]:
|
||
return False
|
||
f1, m1 = l1.get("FatherId"), l1.get("MotherId")
|
||
f2, m2 = l2.get("FatherId"), l2.get("MotherId")
|
||
if not f1 or not f2 or not m1 or not m2:
|
||
return False
|
||
return f1 == f2 and m1 == m2
|
||
|
||
by_date2 = {}
|
||
for l in resolved_litters:
|
||
by_date2.setdefault(l["Date"], []).append(l)
|
||
|
||
deduped2 = []
|
||
litter_remap2 = {} # old_id -> canonical_id
|
||
|
||
for date_val, group in by_date2.items():
|
||
sub_groups = []
|
||
for l in group:
|
||
placed = False
|
||
for sub in sub_groups:
|
||
if all(_litter_same_parents(l, m) for m in sub):
|
||
sub.append(l)
|
||
placed = True
|
||
break
|
||
if not placed:
|
||
sub_groups.append([l])
|
||
|
||
for sub in sub_groups:
|
||
# Prefer the canonical that has the most children
|
||
canonical = max(sub, key=lambda l: len(gerbil_by_litter.get(l["Id"], [])))
|
||
for l in sub:
|
||
litter_remap2[l["Id"]] = canonical["Id"]
|
||
if l is not canonical:
|
||
canonical["_source_files"] |= l.get("_source_files", set())
|
||
canonical["_merged_count"] += l.get("_merged_count", 1)
|
||
if not l.get("_virtual"):
|
||
canonical["_virtual"] = False
|
||
deduped2.append(canonical)
|
||
if len(sub) > 1:
|
||
siblings = [g["Name"] for l in sub for g in gerbil_by_litter.get(l["Id"], []) if l is not canonical]
|
||
print(f"Sibling-Litter-Merge on {date_val}: {canonical['Name']} absorbed sibling half — children now share LitterId: {[g['Name'] for g in gerbil_by_litter.get(canonical['Id'], [])] + siblings}")
|
||
|
||
# Remap LitterId in all gerbils
|
||
n_remapped = 0
|
||
for g in resolved_gerbils:
|
||
old_lid = g.get("LitterId")
|
||
if old_lid and old_lid in litter_remap2 and litter_remap2[old_lid] != old_lid:
|
||
g["LitterId"] = litter_remap2[old_lid]
|
||
n_remapped += 1
|
||
|
||
n_merged2 = len(resolved_litters) - len(deduped2)
|
||
if n_merged2:
|
||
print(f"Sibling-Litter-Dedup: {n_merged2} additional litter record(s) merged ({n_remapped} gerbil LitterIds remapped).")
|
||
resolved_litters = deduped2
|
||
litter_by_scoped_id = {l["Id"]: l for l in resolved_litters}
|
||
|
||
# Final gerbil-provenance pass: now that parent links are fully resolved and
|
||
# all discards (majority-vote conflicts during dedup; rejected parent-refs;
|
||
# role-normalization drops; parent-age drops) are attached to each gerbil's
|
||
# _discarded list, render the provenance JSON with the discard history lines.
|
||
n_discards = 0
|
||
for g in resolved_gerbils:
|
||
args = g.pop("_prov_args", None)
|
||
if g.get("_discarded"):
|
||
n_discards += len(g["_discarded"])
|
||
if args is not None:
|
||
records, best_g, conflict_notes, field_source, field_discards = args
|
||
g["Provenance"] = build_provenance(
|
||
records, best_g, extra_notes=conflict_notes,
|
||
field_source=field_source, field_discards=field_discards,
|
||
)
|
||
else:
|
||
g["Provenance"] = build_provenance([g], g)
|
||
g.pop("_discarded", None)
|
||
if n_discards:
|
||
print(f"Datenherkunft: recorded {n_discards} discard line(s) across gerbils.")
|
||
|
||
# Abgabevertrag-Anreicherung: Käufer als Abnehmer-Kontakte anlegen und —
|
||
# konservativ — auf eindeutig passende Tiere ReceiverContactId/GoHomeDate/
|
||
# Status=GivenAway setzen (nur falls noch nicht gesetzt). Provenance-
|
||
# Historie wird ergänzt. Neue Käuferkontakte landen in contact_by_norm_name
|
||
# und werden danach automatisch als IsReceiver markiert.
|
||
cstats, sale_contracts = enrich_from_contracts(
|
||
contracts, resolved_gerbils, contact_by_norm_name, contact_id_map,
|
||
exclude_decisions=_decisions)
|
||
if contracts:
|
||
print("Abgabeverträge: "
|
||
f"{cstats['contracts']} geladen, {cstats['matched']} Tier-Treffer, "
|
||
f"{cstats['ambiguous_skipped']} mehrdeutig übersprungen, "
|
||
f"{cstats['no_match_skipped']} ohne Treffer.")
|
||
print(" Kontakte: "
|
||
f"{cstats['buyers_created']} neu, {cstats['buyers_existing']} bestehend.")
|
||
print(" Gesetzt: "
|
||
f"ReceiverContactId={cstats['receiver_set']}, "
|
||
f"GoHomeDate={cstats['gohome_set']}, "
|
||
f"Status=GivenAway={cstats['status_givenaway']}, "
|
||
f"Konflikte={cstats['conflicts']}.")
|
||
print(" Vertragszeilen: "
|
||
f"{cstats['records_created']} angelegt "
|
||
f"({cstats['records_with_animal']} mit Tier, "
|
||
f"{cstats['records_with_date']} mit Originaldatum), "
|
||
f"{cstats['no_buyer_skipped']} ohne Käufer übersprungen, "
|
||
f"{cstats['dateless_skipped']} ohne Datum übersprungen.")
|
||
# Re-materialise contacts so freshly created buyer contacts are exported.
|
||
resolved_contacts = list(contact_by_norm_name.values())
|
||
|
||
# Datenherkunft for litters: which source files contributed, whether this is
|
||
# a Wurfchronik litter vs a Stammbaum-reconstructed ("virtual") litter, how
|
||
# many raw records merged into it, plus human-readable notes. Accumulators
|
||
# (_source_files/_merged_count/_virtual) were filled during the two dedup
|
||
# passes above; strip them after use.
|
||
# Id → Gerbil lookup so we can inspect a litter's parents (residency) below.
|
||
_gerbil_by_id = {g["Id"]: g for g in resolved_gerbils if g.get("Id")}
|
||
|
||
# Per-litter flags captured here (before _virtual is popped) for the late
|
||
# residency sweep: was the litter reconstructed from a chart (virtual) and
|
||
# did it come from the Wurfchronik?
|
||
_litter_is_virtual = {}
|
||
_litter_from_wurfchronik = {}
|
||
|
||
for l in resolved_litters:
|
||
l_source_files = l.pop("_source_files", set())
|
||
l_merged_count = l.pop("_merged_count", 1)
|
||
is_virtual = l.pop("_virtual", False)
|
||
l_from_wurfchronik = any("wurfchronik" in str(f).lower() for f in l_source_files)
|
||
_litter_is_virtual[l["Id"]] = is_virtual
|
||
_litter_from_wurfchronik[l["Id"]] = l_from_wurfchronik
|
||
l_notes = []
|
||
if is_virtual and not l_from_wurfchronik:
|
||
l_notes.append("aus Stammbaum-Diagramm rekonstruiert")
|
||
elif l_from_wurfchronik:
|
||
l_notes.append("aus Wurfchronik")
|
||
if l_merged_count > 1:
|
||
l_notes.append(f"aus {l_merged_count} Datensätzen zusammengeführt")
|
||
l_notes.append("Geschwister-Würfe zusammengeführt")
|
||
|
||
# Chronological, file-attributed history for the litter.
|
||
l_files_sorted = sorted({f for f in l_source_files if f})
|
||
l_history = []
|
||
first_file = l_files_sorted[0] if l_files_sorted else None
|
||
if is_virtual and not l_from_wurfchronik:
|
||
if first_file:
|
||
l_history.append(
|
||
f"Aus Stammbaum-Diagramm rekonstruiert ({_quote_file(first_file)})."
|
||
)
|
||
else:
|
||
l_history.append("Aus Stammbaum-Diagramm rekonstruiert.")
|
||
elif first_file:
|
||
l_history.append(f"Wurf aus Wurfchronik {_quote_file(first_file)}.")
|
||
else:
|
||
l_history.append("Wurf im Import gefunden.")
|
||
for f in l_files_sorted[1:]:
|
||
l_history.append(
|
||
f"Auch in {_quote_file(f)} gefunden → Datensätze zusammengeführt."
|
||
)
|
||
if l_merged_count > 1:
|
||
l_history.append("Geschwister-Würfe zusammengeführt.")
|
||
l["Provenance"] = build_entity_provenance(
|
||
l_source_files, l_merged_count, notes=l_notes,
|
||
from_wurfchronik=l_from_wurfchronik, history=l_history,
|
||
)
|
||
# ShowInChronicle wird ERST NACH dem isResident-Sweep gesetzt (siehe unten),
|
||
# weil das Kriterium „mind. ein Bestandstier-Elternteil" die finale Residenz
|
||
# braucht — der Sweep läuft nach addLitters.
|
||
|
||
# ── addLitters: manuell ergänzte Würfe injizieren ─────────────────────────
|
||
# Späte Injektion NACH der finalen Gerbil-/Litter-Assemblierung, damit die
|
||
# finalen Tier-Ids feststehen. Jeder Eintrag verknüpft Mutter/Vater/Kinder
|
||
# über deren Namen(+dob) und setzt bei den Kindern die neue LitterId. Würfe mit
|
||
# showInChronicle=false (z. B. Akanes Wurf bei Clan of Black Forest) erscheinen
|
||
# nur auf der Tier-Seite, nicht in der Wurfchronik.
|
||
def _find_final_gerbil(name, dob=None):
|
||
if not name:
|
||
return None
|
||
target_ck = normalize_name(get_call_name(name))
|
||
target_full = normalize_name(name)
|
||
iso = parse_date(dob) if dob else None
|
||
# Match precedence (most specific first) so a deliberately-added animal
|
||
# (full name „Merle“) is preferred over an unrelated foreign animal that
|
||
# only shares the call-name („Merle of Samsimar“):
|
||
# 1. full-name + dob 2. full-name 3. call-name + dob 4. call-name
|
||
full_dob = full_only = ck_dob = ck_only = None
|
||
for g in resolved_gerbils:
|
||
g_full = normalize_name(g.get("Name") or "")
|
||
g_ck = normalize_name(get_call_name(g.get("Name") or ""))
|
||
same_dob = iso and g.get("DateOfBirth") == iso
|
||
if g_full == target_full:
|
||
if same_dob and full_dob is None:
|
||
full_dob = g
|
||
elif full_only is None:
|
||
full_only = g
|
||
elif g_ck == target_ck:
|
||
if same_dob and ck_dob is None:
|
||
ck_dob = g
|
||
elif ck_only is None:
|
||
ck_only = g
|
||
return full_dob or full_only or ck_dob or ck_only
|
||
|
||
_litters_injected = 0
|
||
for al in (_add_litters or []):
|
||
al_name = (al.get("name") or "").strip()
|
||
if not al_name:
|
||
continue
|
||
mother = _find_final_gerbil(al.get("mother"), al.get("motherDob"))
|
||
father = _find_final_gerbil(al.get("father"), al.get("fatherDob"))
|
||
child_gerbils = []
|
||
for cn in (al.get("children") or []):
|
||
# Kind als String ODER {name, dob} — dob disambiguiert gleichnamige Tiere
|
||
# (z. B. Ahnen-Elliot *2015 vs. Zuchttier-Elliot *2023).
|
||
if isinstance(cn, dict):
|
||
cg = _find_final_gerbil(cn.get("name"), cn.get("dob"))
|
||
else:
|
||
cg = _find_final_gerbil(cn)
|
||
if cg:
|
||
child_gerbils.append(cg)
|
||
slug = normalize_name(al_name)
|
||
litter_id = generate_guid(f"decision-litter-{slug}")
|
||
al_date = parse_date(al["date"]) if al.get("date") else None
|
||
al_notes = (al.get("notes") or "").strip() or None
|
||
new_litter = {
|
||
"Id": litter_id,
|
||
"Name": al_name,
|
||
"Date": al_date,
|
||
"TotalBorn": al.get("totalBorn"),
|
||
"DeathsWithin8Weeks": None,
|
||
"FatherId": father["Id"] if father else None,
|
||
"MotherId": mother["Id"] if mother else None,
|
||
"ExpectedGoHomeDate": None,
|
||
"Notes": al_notes,
|
||
"PairingCode": None,
|
||
"ExternalRef": f"decision-litter-{slug}",
|
||
"LitterLetter": None,
|
||
"ShowInChronicle": bool(al.get("showInChronicle", True)),
|
||
"Provenance": build_entity_provenance(
|
||
["conflict-decisions.json"], 1,
|
||
notes=["Manuell aus Ticket ergänzt"],
|
||
from_wurfchronik=False,
|
||
history=["Manuell aus dem Ticket der Züchterin ergänzt "
|
||
"(conflict-decisions.json)."],
|
||
),
|
||
}
|
||
for cg in child_gerbils:
|
||
cg["LitterId"] = litter_id
|
||
resolved_litters.append(new_litter)
|
||
_litters_injected += 1
|
||
print(f"addLitters injiziert: '{al_name}' "
|
||
f"(Mutter={mother['Name'] if mother else '—'}, "
|
||
f"Vater={father['Name'] if father else '—'}, "
|
||
f"Kinder={len(child_gerbils)}, ShowInChronicle={new_litter['ShowInChronicle']})")
|
||
if _litters_injected:
|
||
print(f"addLitters: {_litters_injected} Wurf/Würfe injiziert.")
|
||
|
||
# ── rpro3Pedigrees: KOMPLETTE Ahnentafel eines Tiers aus _rpro3.db ziehen ──────
|
||
# Für „bitte den ganzen Stammbaum aus RennerPro" (nicht nur eine Generation). Für
|
||
# jeden Root {name, dob} werden ALLE Vorfahren über die rpro3-Eltern-Zeiger
|
||
# (pid_raw/mid_raw) rekursiv materialisiert: bereits vorhandene Tiere (Name+DOB)
|
||
# werden WIEDERVERWENDET, fehlende als externe Ahnen neu angelegt (deterministische
|
||
# Id `rpro3-anc-<rid>`, Genotyp aus rpro3-Fcode → korrekter Farbschlag, Herkunft aus
|
||
# origin). Fehlende Eltern-Verknüpfungen werden als (versteckte) Ahnen-Würfe ergänzt;
|
||
# Tiere mit bereits vorhandenem Geburtswurf bleiben unangetastet.
|
||
try:
|
||
import compare_rpro3 as _Crp
|
||
_peds = []
|
||
try:
|
||
with open(CONFLICT_DECISIONS_PATH, encoding="utf-8") as _pfh:
|
||
_peds = (json.load(_pfh).get("rpro3Pedigrees") or [])
|
||
except Exception:
|
||
_peds = []
|
||
_dbp = os.path.join(os.path.dirname(os.path.abspath(__file__)), "_rpro3.db")
|
||
if _peds and os.path.exists(_dbp):
|
||
_R = _Crp.load_rpro3(_dbp)
|
||
_A = _R["animals"]
|
||
_byrid = {a["rid"]: a for a in _A}
|
||
def _iso(x):
|
||
return x.isoformat() if hasattr(x, "isoformat") else (x or None)
|
||
def _rn(s):
|
||
return normalize_name(get_call_name(s or ""))
|
||
# Strenger Match für Ahnen: bei bekanntem DOB nur exakt (Name+DOB); bei
|
||
# DOB-losem rpro3-Ahnen nur wiederverwenden, wenn genau EIN ebenfalls DOB-loses
|
||
# App-Tier gleichen Namens existiert — sonst NEU anlegen (verhindert, dass ein
|
||
# DOB-loser Ahn fälschlich an einen dat. Namensvetter gehängt wird, z. B.
|
||
# Kilians Mutter „Lila" ≠ Zuchttier „Lila *2014").
|
||
_res_by_name = {}
|
||
for _gg in resolved_gerbils:
|
||
_res_by_name.setdefault(_rn(_gg.get("Name")), []).append(_gg)
|
||
def _strict_anc_match(name, dob):
|
||
cs = _res_by_name.get(_rn(name), [])
|
||
if dob:
|
||
return next((x for x in cs if x.get("DateOfBirth") == dob), None)
|
||
dobless = [x for x in cs if not x.get("DateOfBirth")]
|
||
return dobless[0] if len(dobless) == 1 else None
|
||
def _find_rpro3(name, dob):
|
||
cs = [a for a in _A if _rn(a["name"]) == _rn(name)]
|
||
if dob:
|
||
mm = next((a for a in cs if _iso(a["dob"]) == dob), None)
|
||
if mm:
|
||
return mm
|
||
return cs[0] if len(cs) == 1 else None
|
||
_pg = _pl = 0
|
||
for ped in _peds:
|
||
pdob = parse_date(ped.get("dob")) if ped.get("dob") else None
|
||
root = _find_rpro3(ped.get("name"), pdob)
|
||
if not root:
|
||
print(f"rpro3Pedigrees: Root '{ped.get('name')}' nicht in rpro3 gefunden")
|
||
continue
|
||
# Vorfahren sammeln
|
||
nodes, links, seen, stack = {}, [], set(), [root]
|
||
while stack:
|
||
a = stack.pop()
|
||
if not a or a["rid"] in seen:
|
||
continue
|
||
seen.add(a["rid"]); nodes[a["rid"]] = a
|
||
f = _byrid.get(a.get("pid_raw")); m = _byrid.get(a.get("mid_raw"))
|
||
if f or m:
|
||
links.append((a["rid"], f["rid"] if f else None, m["rid"] if m else None))
|
||
for pp in (f, m):
|
||
if pp and pp["rid"] not in seen:
|
||
stack.append(pp)
|
||
# rid → App-Gerbil (vorhandene wiederverwenden, sonst anlegen)
|
||
rid2g = {}
|
||
for rid, a in nodes.items():
|
||
ex = _strict_anc_match(a["name"], _iso(a.get("dob")))
|
||
if ex:
|
||
rid2g[rid] = ex; continue
|
||
gid = generate_guid(f"rpro3-anc-{rid}")
|
||
gender = {"männlich": "male", "weiblich": "female"}.get(
|
||
(a.get("sex") or "").lower(), "unknown")
|
||
fcode = (a.get("fcode") or "").strip() or None
|
||
stub = {
|
||
"Id": gid, "Name": get_normalized_gerbil_name(a["name"]),
|
||
"Gender": gender, "Status": "Deceased", "LitterId": None,
|
||
"OriginContactId": None, "ReceiverContactId": None, "EnclosureId": None,
|
||
"ColorVarietyId": None, "DateOfBirth": _iso(a.get("dob")), "DateOfDeath": None,
|
||
"CauseOfDeath": None, "GoHomeDate": None, "Genotype": fcode,
|
||
"Notes": None, "ImportSource": "rpro3 (_rpro3.db)",
|
||
"ExternalRef": f"rpro3-anc-{rid}",
|
||
"RawImportData": json.dumps({"rpro3Rid": rid}, ensure_ascii=False),
|
||
"OriginBreeder": (a.get("origin") or "").strip() or None,
|
||
"NameSearch": normalize_name(get_normalized_gerbil_name(a["name"])),
|
||
"CharacterTraits": [], "CharacterNote": None, "IsDeaf": None,
|
||
"IsResident": False, "parentRefs": [], "_photos": [],
|
||
"_old_scoped_litter_id": None, "_eff_dob": _iso(a.get("dob")) or "2008-01-01",
|
||
"_birth_date": _iso(a.get("dob")), "_filename": "_rpro3.db", "_old_id": gid,
|
||
"Provenance": build_entity_provenance(
|
||
["_rpro3.db"], 1, notes=["Vorfahre aus RennerPro (RennmausPro III)"],
|
||
from_wurfchronik=False,
|
||
history=[f"Als Vorfahre aus RennerPro übernommen (rid {rid})."]),
|
||
}
|
||
resolved_gerbils.append(stub); rid2g[rid] = stub; _pg += 1
|
||
# Eltern-Verknüpfungen als Ahnen-Würfe (nur wo Kind noch keinen Wurf hat)
|
||
for c, f, m in links:
|
||
cg = rid2g.get(c)
|
||
if not cg or cg.get("LitterId"):
|
||
continue
|
||
fg = rid2g.get(f); mg = rid2g.get(m)
|
||
if not fg and not mg:
|
||
continue
|
||
lid = generate_guid(f"rpro3-litter-{c}")
|
||
lit = {
|
||
"Id": lid,
|
||
"Name": f"Wurf von {fg['Name'] if fg else '?'} + {mg['Name'] if mg else '?'}",
|
||
"Date": cg.get("DateOfBirth"), "TotalBorn": None, "DeathsWithin8Weeks": None,
|
||
"FatherId": fg["Id"] if fg else None, "MotherId": mg["Id"] if mg else None,
|
||
"ExpectedGoHomeDate": None, "Notes": None, "PairingCode": None,
|
||
"ExternalRef": f"rpro3-litter-{c}", "LitterLetter": None,
|
||
"ShowInChronicle": False,
|
||
"Provenance": build_entity_provenance(
|
||
["_rpro3.db"], 1, notes=["Ahnen-Wurf aus RennerPro"],
|
||
from_wurfchronik=False,
|
||
history=["Eltern-Verknüpfung aus RennerPro (RennmausPro III)."]),
|
||
}
|
||
cg["LitterId"] = lid
|
||
resolved_litters.append(lit); _pl += 1
|
||
if _pg or _pl:
|
||
print(f"rpro3Pedigrees injiziert: {_pg} Ahnen, {_pl} Ahnen-Würfe")
|
||
except Exception as _e:
|
||
print(f"rpro3Pedigrees übersprungen: {_e}")
|
||
|
||
# ── litterParents: Eltern eines BESTEHENDEN Wurfs erzwingen (Mensch-Entscheidung) ──
|
||
# Für Fälle, in denen der Resolver einen Namensvetter oder ein verstorbenes Tier
|
||
# verknüpft hat und die echte Mutter/der echte Vater eindeutig benannt ist. Match per
|
||
# litterName+date (oder nur date); Vater/Mutter via _find_final_gerbil(name, dob).
|
||
try:
|
||
with open(CONFLICT_DECISIONS_PATH, encoding="utf-8") as _lpfh:
|
||
_litter_parents = (json.load(_lpfh).get("litterParents") or [])
|
||
except Exception:
|
||
_litter_parents = []
|
||
_lp_applied = 0
|
||
for lp in _litter_parents:
|
||
lp_name = (lp.get("litterName") or "").strip().lower()
|
||
lp_date = parse_date(lp["date"]) if lp.get("date") else None
|
||
if not lp_name and not lp_date:
|
||
continue
|
||
for l in resolved_litters:
|
||
if lp_name and (l.get("Name") or "").strip().lower() != lp_name:
|
||
continue
|
||
if lp_date and l.get("Date") != lp_date:
|
||
continue
|
||
if lp.get("father"):
|
||
fa = _find_final_gerbil(lp["father"], lp.get("fatherDob"))
|
||
if fa:
|
||
l["FatherId"] = fa["Id"]
|
||
if lp.get("mother"):
|
||
mo = _find_final_gerbil(lp["mother"], lp.get("motherDob"))
|
||
if mo:
|
||
l["MotherId"] = mo["Id"]
|
||
_lp_applied += 1
|
||
if _lp_applied:
|
||
print(f"litterParents-Overrides angewandt: {_lp_applied}")
|
||
|
||
# ── litterOrder: Reihenfolge der Jungtiere im Wurf (Wurfchronik = Geburtsgewicht) ──
|
||
# Die Züchterin gibt die korrekte Reihenfolge vor (nicht alphabetisch). Match per
|
||
# litterName+date; Kinder per Wurf (LitterId) + Namensabgleich; setzt BirthOrder=1..N.
|
||
try:
|
||
with open(CONFLICT_DECISIONS_PATH, encoding="utf-8") as _lofh:
|
||
_litter_orders = (json.load(_lofh).get("litterOrder") or [])
|
||
except Exception:
|
||
_litter_orders = []
|
||
_lo_applied = 0
|
||
for lo in _litter_orders:
|
||
lo_name = (lo.get("litterName") or "").strip().lower()
|
||
lo_date = parse_date(lo["date"]) if lo.get("date") else None
|
||
order = lo.get("order") or []
|
||
if not order or (not lo_name and not lo_date):
|
||
continue
|
||
for l in resolved_litters:
|
||
if lo_name and (l.get("Name") or "").strip().lower() != lo_name:
|
||
continue
|
||
if lo_date and l.get("Date") != lo_date:
|
||
continue
|
||
kids = [g for g in resolved_gerbils if g.get("LitterId") == l["Id"]]
|
||
for idx, cname in enumerate(order, start=1):
|
||
ck = normalize_name(get_call_name(cname))
|
||
for g in kids:
|
||
if normalize_name(get_call_name(g.get("Name") or "")) == ck:
|
||
g["BirthOrder"] = idx
|
||
_lo_applied += 1
|
||
break
|
||
break
|
||
if _lo_applied:
|
||
print(f"litterOrder: {_lo_applied} Jungtier-Reihenfolgen gesetzt")
|
||
|
||
# ── litterChildren: autoritative Kinder-Whitelist eines Wurfs (Mensch-Entscheidung) ──
|
||
# Für Fälle, in denen der Wurf-Merge fremde Tiere (z. B. Abnehmer-Sammelseiten der
|
||
# Wurfchronik) in einen datierten Wurf gezogen hat (Tickets 4b9f49fb M-Wurf, 8d259edf Gale).
|
||
# Match per date + `matchChild` (ein aktuell zugeordnetes Kind, disambiguiert gleich-datierte
|
||
# Würfe). Alle Kinder, deren Ruf-Name NICHT in `keep` steht, werden aus dem Wurf gelöst
|
||
# (LitterId=None) — sie verschwinden aus dem Wurf, bleiben aber als Tier erhalten. Optional
|
||
# `renameTo` setzt den kanonischen Wurf-Namen (z. B. 'Ungeklärt' -> 'M-Wurf').
|
||
try:
|
||
with open(CONFLICT_DECISIONS_PATH, encoding="utf-8") as _lcfh:
|
||
_litter_children = (json.load(_lcfh).get("litterChildren") or [])
|
||
except Exception:
|
||
_litter_children = []
|
||
_lc_removed = 0
|
||
for lc in _litter_children:
|
||
lc_date = parse_date(lc["date"]) if lc.get("date") else None
|
||
match_child = normalize_name(get_call_name(lc.get("matchChild") or ""))
|
||
keep = {normalize_name(get_call_name(n)) for n in (lc.get("keep") or [])}
|
||
if not keep:
|
||
continue
|
||
for l in resolved_litters:
|
||
if lc_date and l.get("Date") != lc_date:
|
||
continue
|
||
kids = [g for g in resolved_gerbils if g.get("LitterId") == l["Id"]]
|
||
names = {normalize_name(get_call_name(g.get("Name") or "")) for g in kids}
|
||
if match_child and match_child not in names:
|
||
continue # falscher, nur zufällig gleich-datierter Wurf
|
||
for g in kids:
|
||
if normalize_name(get_call_name(g.get("Name") or "")) not in keep:
|
||
g["LitterId"] = None
|
||
_lc_removed += 1
|
||
if lc.get("renameTo"):
|
||
l["Name"] = lc["renameTo"]
|
||
break
|
||
if _lc_removed:
|
||
print(f"litterChildren: {_lc_removed} fälschlich zugeordnete Jungtiere aus Würfen gelöst")
|
||
|
||
# ── isResident-Sweep (Ticket 381f7e51, freigegeben) ───────────────────────
|
||
# Entscheidung der Züchterin: „Nimm wirklich ALLE Tiere aus dem Bestand heraus,
|
||
# MIT AUSNAHME der, die bereits ein ELTERNTIER sind." Datengetrieben + re-ingest-
|
||
# stabil neu berechnet, GANZ AM ENDE (nach finaler Eltern-/Wurf-Verknüpfung +
|
||
# addLitters), damit ein Tier erst dann als Elternteil erkennbar ist.
|
||
#
|
||
# Regel — IsResident=true GENAU DANN, wenn EINES gilt:
|
||
# (1) ein EXPLIZITER conflict-decisions-Override (_resident_override) hat
|
||
# isResident gesetzt (true ODER false) → gewinnt IMMER, der Sweep fasst
|
||
# solche Tiere NICHT an (z. B. Echo/Elia=true, Akane/Naémi/Bentley=false).
|
||
# (2) das Tier ist Mutter/Vater EINES IHRER EIGENEN Würfe. Ein „eigener Wurf"
|
||
# ist ein Wurf, der KEINE reine rekonstruierte Ahnen-Verpaarung ist:
|
||
# (a) nicht virtuell (echter Wurfchronik-/docx-Wurf) ODER
|
||
# (b) aus der Wurfchronik ODER
|
||
# (c) hat clan-benannte Nachzucht („von den Kleinen Chaoten") ODER
|
||
# (d) hat einen per Override residenten Elternteil ODER
|
||
# (e) ist ein addLitters-Wurf mit ShowInChronicle=true.
|
||
# Reine Fremd-Ahnen-Paarungen (z. B. Antares of Ulmer Strolche × …) zählen
|
||
# NICHT → externe Ahnen werden NICHT fälschlich resident.
|
||
# SONST: IsResident=false.
|
||
_children_by_litter = {}
|
||
for g in resolved_gerbils:
|
||
lid = g.get("LitterId")
|
||
if lid:
|
||
_children_by_litter.setdefault(lid, []).append(g)
|
||
|
||
# Explizit per Override resident gesetzte Tiere (für „eigener Wurf"-Kriterium d).
|
||
_override_resident_ids = {
|
||
g["Id"] for g in resolved_gerbils
|
||
if g.get("_resident_override") and g.get("IsResident")
|
||
}
|
||
|
||
def _is_own_litter(l):
|
||
lid = l.get("Id")
|
||
if not _litter_is_virtual.get(lid, True):
|
||
return True # (a) echter Wurf
|
||
if _litter_from_wurfchronik.get(lid, False):
|
||
return True # (b) Wurfchronik
|
||
for ch in _children_by_litter.get(lid, []): # (c) Clan-Nachzucht
|
||
if is_clan_zucht(ch.get("Name")):
|
||
return True
|
||
for pid in (l.get("FatherId"), l.get("MotherId")): # (d) resid. Elternteil
|
||
if pid in _override_resident_ids:
|
||
return True
|
||
if l.get("ExternalRef", "").startswith("decision-litter-"): # (e) addLitters
|
||
return bool(l.get("ShowInChronicle"))
|
||
return False
|
||
|
||
_own_parent_ids = set()
|
||
for l in resolved_litters:
|
||
if _is_own_litter(l):
|
||
for pid in (l.get("FatherId"), l.get("MotherId")):
|
||
if pid:
|
||
_own_parent_ids.add(pid)
|
||
|
||
def is_placeholder_name(name):
|
||
if not name:
|
||
return True
|
||
n = name.lower().strip()
|
||
n = re.sub(r'^(?:rv\s+|gv\s+|v\.\s*privat\s+|privat\s+)', '', n)
|
||
n = re.sub(r'\s+privat.*', '', n)
|
||
norm = "".join(c for c in n if c.isalnum())
|
||
if not norm:
|
||
return True
|
||
if norm in ["unbekannt", "unknown", "name", "jungtier", "jungtiere"]:
|
||
return True
|
||
# Check if name is a color variety name
|
||
color_keys = set(variety_map.keys())
|
||
for ck in color_keys:
|
||
ck_norm = "".join(c for c in ck if c.isalnum())
|
||
if norm == ck_norm:
|
||
return True
|
||
return False
|
||
|
||
_sweep_to_false = 0
|
||
_sweep_to_true = 0
|
||
for g in resolved_gerbils:
|
||
if g.get("_resident_override"):
|
||
continue # (1) Override gewinnt
|
||
should = g["Id"] in _own_parent_ids # (2) Elternteil eigenen Wurfs
|
||
if should and is_placeholder_name(g.get("Name")):
|
||
should = False # Platzhalter werden nie automatisch resident
|
||
if g.get("IsResident") != should:
|
||
if should:
|
||
_sweep_to_true += 1
|
||
else:
|
||
_sweep_to_false += 1
|
||
if g.get("OriginBreeder") == "Zucht der kleinen Chaoten":
|
||
g["OriginBreeder"] = None
|
||
g["IsResident"] = should
|
||
print(f"isResident-Sweep: {_sweep_to_true} → resident, {_sweep_to_false} → nicht-resident "
|
||
f"(Elternteile eigener Würfe: {len(_own_parent_ids)}).")
|
||
|
||
# ShowInChronicle (Ticket ea41257a) — JETZT, mit der FINALEN Residenz. Die
|
||
# Wurfchronik zeigt nur die eigenen, dokumentierten Würfe. ZIELGENAU ausblenden:
|
||
# ShowInChronicle=False NUR wenn der Wurf (a) virtuell rekonstruiert ist UND (b)
|
||
# NICHT aus der Wurfchronik stammt UND (c) keiner der beiden bekannten Eltern ein
|
||
# Bestandstier ist. So bleiben ihre echten (evtl. nur im Stammbaum stehenden)
|
||
# Würfe mit mind. einem Bestandstier-Elternteil sichtbar; nur reine Fremd-Ahnen-
|
||
# Paarungen verschwinden. addLitters-Würfe behalten ihren expliziten Wert.
|
||
_gerbil_by_id_final2 = {g["Id"]: g for g in resolved_gerbils if g.get("Id")}
|
||
for l in resolved_litters:
|
||
if l.get("ExternalRef", "").startswith("decision-litter-"):
|
||
continue # addLitters: expliziter ShowInChronicle bleibt
|
||
lid = l.get("Id")
|
||
is_virtual = _litter_is_virtual.get(lid, False)
|
||
l_from_wurfchronik = _litter_from_wurfchronik.get(lid, False)
|
||
_parents = [_gerbil_by_id_final2.get(l.get("FatherId")),
|
||
_gerbil_by_id_final2.get(l.get("MotherId"))]
|
||
_any_resident_parent = any(bool(p.get("IsResident")) for p in _parents if p)
|
||
|
||
_mother = _gerbil_by_id_final2.get(l.get("MotherId"))
|
||
_mother_is_external = False
|
||
if _mother:
|
||
_mother_is_external = is_external_cattery(_mother) and not _mother.get("IsResident")
|
||
|
||
_pure_ancestor_pairing = (
|
||
is_virtual and not l_from_wurfchronik and (not _any_resident_parent or _mother_is_external)
|
||
)
|
||
l["ShowInChronicle"] = not _pure_ancestor_pairing
|
||
|
||
# Hilfsfelder entfernen (nicht in den Payload exportieren).
|
||
for g in resolved_gerbils:
|
||
g.pop("_resident_override", None)
|
||
|
||
# Set IsBreeder and IsReceiver flags on contacts
|
||
breeder_ids = {g["OriginContactId"] for g in resolved_gerbils if g.get("OriginContactId")}
|
||
receiver_ids = {g["ReceiverContactId"] for g in resolved_gerbils if g.get("ReceiverContactId")}
|
||
for c in resolved_contacts:
|
||
c_id = c["Id"]
|
||
is_breeder = c_id in breeder_ids
|
||
is_receiver = c_id in receiver_ids
|
||
if not is_breeder and not is_receiver:
|
||
is_receiver = True
|
||
c["IsBreeder"] = is_breeder
|
||
c["IsReceiver"] = is_receiver
|
||
|
||
# Datenherkunft: where this (deduplicated) contact came from, plus the
|
||
# role we inferred. Accumulator keys (_source_files/_merged_count) were
|
||
# filled during the contact dedup above; strip them after use.
|
||
c_source_files = c.pop("_source_files", set())
|
||
c_merged_count = c.pop("_merged_count", 1)
|
||
c_notes = []
|
||
if c_merged_count > 1:
|
||
c_notes.append(f"aus {c_merged_count} Datensätzen zusammengeführt")
|
||
if is_breeder:
|
||
c_notes.append("als Züchter erkannt")
|
||
if is_receiver:
|
||
c_notes.append("als Abnehmer erkannt")
|
||
|
||
# Chronological, file-attributed history for the contact.
|
||
c_files_sorted = sorted({f for f in c_source_files if f})
|
||
c_history = []
|
||
role_word = "Züchter" if is_breeder else "Abnehmer"
|
||
if c_files_sorted:
|
||
c_history.append(f"In {_quote_file(c_files_sorted[0])} als {role_word} erkannt.")
|
||
for f in c_files_sorted[1:]:
|
||
c_history.append(
|
||
f"Auch in {_quote_file(f)} gefunden → Datensätze zusammengeführt."
|
||
)
|
||
else:
|
||
c_history.append(f"Im Import als {role_word} erkannt.")
|
||
if is_breeder and is_receiver:
|
||
c_history.append("Sowohl als Züchter als auch als Abnehmer geführt.")
|
||
c["Provenance"] = build_entity_provenance(
|
||
c_source_files, c_merged_count, notes=c_notes, history=c_history
|
||
)
|
||
|
||
# Set and map gerbilPhotos
|
||
resolved_photos = []
|
||
for g in resolved_gerbils:
|
||
for idx, photo_rel in enumerate(g.get("_photos", [])):
|
||
photo_guid = generate_guid(f"photo-{photo_rel}")
|
||
ext = os.path.splitext(photo_rel)[1] or ".jpeg"
|
||
fn_guid = photo_guid.replace("-", "")
|
||
resolved_photos.append({
|
||
"Id": photo_guid,
|
||
"GerbilId": g["Id"],
|
||
"FileName": f"{fn_guid}{ext}",
|
||
"SortOrder": idx,
|
||
"_source_path": photo_rel
|
||
})
|
||
|
||
# ── suppressExternalRefs: bestätigte Phantom-/Dubletten-Records entfernen ──
|
||
# Vom Owner bestätigte OCR-Fehllesungen/Dubletten (z. B. 'Muga' = 'Nduga', Ticket
|
||
# a9c0d0f9): Gerbil + ggf. zugehöriger (kinderloser) Phantom-Wurf werden per ExternalRef
|
||
# final entfernt; verwaiste Referenzen werden defensiv genullt.
|
||
_suppress = load_suppress_refs()
|
||
if _suppress:
|
||
removed_g = {g["Id"] for g in resolved_gerbils if g.get("ExternalRef") in _suppress}
|
||
removed_l = {l["Id"] for l in resolved_litters if l.get("ExternalRef") in _suppress}
|
||
resolved_gerbils = [g for g in resolved_gerbils if g.get("ExternalRef") not in _suppress]
|
||
resolved_litters = [l for l in resolved_litters if l.get("ExternalRef") not in _suppress]
|
||
for l in resolved_litters:
|
||
if l.get("MotherId") in removed_g:
|
||
l["MotherId"] = None
|
||
if l.get("FatherId") in removed_g:
|
||
l["FatherId"] = None
|
||
for g in resolved_gerbils:
|
||
if g.get("LitterId") in removed_l:
|
||
g["LitterId"] = None
|
||
resolved_photos = [p for p in resolved_photos if p.get("GerbilId") not in removed_g]
|
||
print(f"suppressExternalRefs: {len(removed_g)} Gerbil(s) + {len(removed_l)} Wurf/Würfe entfernt.")
|
||
|
||
# Herkunft-Normalisierung (Ticket Bodo aaee068b): Die eigene Zucht wurde in
|
||
# Quelldaten teils als „Clan (der) kleine(n) Chaoten" geführt. Die Züchterin will
|
||
# überall „Zucht der Kleinen Chaoten" (Groß-K). Finaler Anzeige-Sweep NACH aller
|
||
# Logik (die interne Sentinel-Form „Zucht der kleinen Chaoten" wird oben verglichen,
|
||
# daher erst hier vereinheitlichen).
|
||
_origin_fixed = 0
|
||
for g in resolved_gerbils:
|
||
ob = g.get("OriginBreeder")
|
||
if not ob:
|
||
continue
|
||
low = ob.lower()
|
||
if ("chaoten" in low and ("clan" in low or "zucht" in low)):
|
||
if ob != "Zucht der Kleinen Chaoten":
|
||
g["OriginBreeder"] = "Zucht der Kleinen Chaoten"
|
||
_origin_fixed += 1
|
||
if _origin_fixed:
|
||
print(f"Herkunft normalisiert auf 'Zucht der Kleinen Chaoten': {_origin_fixed} Tier(e)")
|
||
|
||
# 6. Save final output JSON payload
|
||
os.makedirs(OUTPUT_DIR, exist_ok=True)
|
||
payload = {
|
||
"contacts": resolved_contacts,
|
||
"litters": resolved_litters,
|
||
"gerbils": resolved_gerbils,
|
||
"gerbilPhotos": resolved_photos,
|
||
"saleContracts": sale_contracts,
|
||
}
|
||
|
||
with open(OUTPUT_FILE, "w", encoding="utf-8") as f:
|
||
json.dump(payload, f, ensure_ascii=False, indent=2)
|
||
|
||
print(f"Successfully wrote database-ready import file to: {OUTPUT_FILE}")
|
||
print(f" Contacts: {len(payload['contacts'])}")
|
||
print(f" Litters: {len(payload['litters'])}")
|
||
print(f" Gerbils: {len(payload['gerbils'])}")
|
||
print(f" Photos: {len(payload['gerbilPhotos'])}")
|
||
print(f" SaleContracts: {len(payload['saleContracts'])}")
|
||
|
||
if __name__ == "__main__":
|
||
main()
|