import os import json import re import uuid import sys from datetime import datetime, timedelta import genotype as gt # Prevent encoding crashes on Windows consoles when printing unicode if sys.platform.startswith('win'): try: sys.stdout.reconfigure(encoding='utf-8') except Exception: pass # Paths DIR_PATH = r"C:\Users\gulum\dev\Wurfchronik_Bilder" SEEDS_PATH = r"C:\Users\gulum\dev\GerbilManager\gerbil-manager-web\src\genetics\colorVarietySeed.backend.json" OUTPUT_DIR = r"C:\Users\gulum\dev\GerbilManager\tools\import\output" OUTPUT_FILE = os.path.join(OUTPUT_DIR, "resolved_import.json") CONFLICT_DECISIONS_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), "conflict-decisions.json") def load_conflict_decisions(): """Load tools/import/conflict-decisions.json (the breeder's authoritative answers). Returns (resolutions, addAnimals, addLitters). Tolerates a missing/garbled file by returning empty lists — the import must never crash on a bad decision file.""" try: with open(CONFLICT_DECISIONS_PATH, encoding="utf-8") as fh: data = json.load(fh) return (data.get("resolutions") or [], data.get("addAnimals") or [], data.get("addLitters") or []) except (OSError, ValueError): return ([], [], []) def load_suppress_refs(): """Load suppressExternalRefs from conflict-decisions.json — the ExternalRefs of gerbils/litters the breeder confirmed are phantoms/OCR-misread duplicates and that should be dropped from the final payload (e.g. 'Muga' = misread 'Nduga'). Tolerates a missing/garbled file.""" try: with open(CONFLICT_DECISIONS_PATH, encoding="utf-8") as fh: data = json.load(fh) return set(data.get("suppressExternalRefs") or []) except (OSError, ValueError): return set() def _norm_decision_gender(g): """Map a free-text gender override ('m'/'w'/'männlich'/…) to 'male'/'female' or None when it is neither (so a typo never blanks a real gender).""" g = (g or "").strip().lower() return {"m": "male", "männlich": "male", "male": "male", "w": "female", "f": "female", "weiblich": "female", "female": "female"}.get(g) def generate_guid(key_str): """Generate a stable UUID string based on a key.""" return str(uuid.uuid5(uuid.NAMESPACE_DNS, key_str)) def build_entity_provenance(source_files, merged_record_count, notes=None, from_wurfchronik=None, extra=None, history=None): """Generic data-provenance JSON builder shared by gerbils, contacts and litters. Mirrors the GerbilProvenance frontend contract: { sourceFiles, mergedRecordCount, fromWurfchronik, notes, history, ... } `source_files` is any iterable of filenames; `from_wurfchronik` is auto- derived from the filenames when left as None. `extra` may carry entity- specific keys (e.g. parentMethod/parentConfidence for gerbils). `history` is an ordered list of human-readable German lines that read like a chronological log of where each fact came from (the primary content shown in the Datenherkunft dialog). Returns a JSON string (stored on the nullable Provenance text column).""" files = sorted({f for f in source_files if f}) if from_wurfchronik is None: from_wurfchronik = any("wurfchronik" in f.lower() for f in files) prov = { "sourceFiles": files, "mergedRecordCount": merged_record_count, "fromWurfchronik": bool(from_wurfchronik), "notes": list(notes or []), "history": list(history or []), } if extra: for k, v in extra.items(): if v is not None: prov[k] = v return json.dumps(prov, ensure_ascii=False) def _quote_file(fname): """German typographic quotes around a source filename for history lines.""" return f"„{fname}“" # Human-readable German labels for the significant fields we attribute to files. PROV_FIELD_LABELS = { "DateOfBirth": "Geburtsdatum", "DateOfDeath": "Sterbedatum", "Gender": "Geschlecht", "Genotype": "Genotyp", "ColorVarietyId": "Farbschlag", "Name": "Name", } def _primary_file_of(record): """The single most representative source file of one raw record. Prefers an explicit Stammbaum/Wurfchronik filename from ImportSource, else the record's _filename. Used to attribute a field value to a concrete file in the history log.""" fn = record.get("_filename") if fn: return fn imp = record.get("ImportSource") if imp: first = str(imp).split(",")[0].strip() if first: return first return None def _record_source_files(g): """All distinct source files a single raw gerbil record drew from. ImportSource is either a comma-joined Stammbaum file list, a single Wurfchronik filename, or the per-litter filename; _filename is the primary file. We union both so nothing is lost.""" files = set() imp = g.get("ImportSource") if imp: for part in str(imp).split(","): part = part.strip() if part: files.add(part) fn = g.get("_filename") if fn: files.add(fn) return files def to_valid_guid(val): if not val: return None val_str = str(val).strip() try: uuid.UUID(val_str) return val_str except ValueError: return generate_guid(val_str) def normalize_name(name): if not name: return "" return "".join(c for c in name.lower() if c.isalnum()) _EXTERNAL_MARKERS_RE = re.compile( r"\b(zooladen|zoohandlung|obi|fressnapf|dehner|von\s+privat|privatkauf|" r"vom\s+bauern|aus\s+der\s+zoohandlung)\b", re.IGNORECASE) _FOREIGN_FROM_RE = re.compile( r"\bfrom\b.+,\s*(croatia|kroatien|poland|polen|netherlands|niederlande|" r"belgium|belgien|france|frankreich|austria|österreich|switzerland|schweiz|" r"italy|italien|spain|spanien|czech|tschechien|hungary|ungarn)\b", re.IGNORECASE) def is_external_origin(name, zucht=None, breeder=None): """Externally-acquired founder with genuinely unknown ancestry (pet shop, private hobbyist, foreign cattery). Mirrors extract.is_external_origin so the merge stage never attaches chart/Wurfchronik parents to such animals.""" blob = " ".join(p for p in (name, zucht, breeder) if p) if _EXTERNAL_MARKERS_RE.search(blob): return True if _FOREIGN_FROM_RE.search(name or ""): return True return False def canon_name_key(name): """Connector-folding name key: collapses the cattery connectors so that abbreviation variants of the SAME animal match — e.g. „BlackFire v.d. Kleinen Chaoten“ and „BlackFire von den Kleinen Chaoten“ both fold to the same key (ticket #30). 'v.d.' / 'v. d.' → 'von den', then alnum-reduced. Used as a secondary index next to normalize_name (never for GUIDs).""" if not name: return "" n = name.lower() n = re.sub(r"\bv\.?\s*d\.?\b", "von den", n) # v.d. / v. d. / vd → von den return "".join(c for c in n if c.isalnum()) def get_normalized_contact_name(name): if not name: return "", False n = "".join(c for c in name.lower() if c.isalnum()) to_discard = { "chevroletcamarooftopolino", "cindyvprivatzuchtgießen", "cindyvprivatzuchtgiessen", "inuschofblackforest", "stichvonprivatzuchtgießen", "stichvonprivatzuchtgiessen", "tarzanofsamsimar", "hanserennersposeidon", "livingforcesidefix", "livingforcesnando" } if n in to_discard: return "", False norm_map = { "alessandrab": "Alessandra Bartoletti", "alexsandrab": "Alessandra Bartoletti", "andreafey": "Andrea und Stefanie Fey", "andreastefaniefey": "Andrea und Stefanie Fey", "anettw": "Annett Wernecke", "angelachristoph": "Angela und Ekki Christoph", "angelaekkichristoph": "Angela und Ekki Christoph", "angie": "Angie Reichert-Cambeis", "angierc": "Angie Reichert-Cambeis", "anjasch": "Anja Schaumburg", "ankeb": "Anke Busch", "ankeksch": "Anke Koppenhöfer", "annaq": "Anna Quark", "annikab": "Annika Balser", "annkathrind": "Ann-kathrin Dressler", "arturaleksandrapolamundrzynski": "Artur, Aleksandra und Pola Mundrzynski", "arturaleksandraundpolamundrzynski": "Artur, Aleksandra und Pola Mundrzynski", "astridr": "Astrid Rohmann", "astridroh": "Astrid Rohmann", "babarastehle": "Barbara Stehle", "barbarar": "Barbara Riegler", "biancab": "Bianca Bernhardt", "biancam": "Bianca Mayer + Alex Wachten", "biancamayeralexw": "Bianca Mayer + Alex Wachten", "birgitk": "Birgit Knopp", "birgitkropp": "Birgit Knopp", "birgittabüskens": "Birgitta Müller-Büskens", "birgittamüllerbüskens": "Brigitta Müller-Büskens", "blackforest": "Clan of Black Forest", "blackforestgv": "Clan of Black Forest", "brigittast": "Brigitta Struve", "buntefellnasen": "Zucht von den bunten Fellnasen", "buntenfellnase": "Zucht von den bunten Fellnasen", "buntenfellnasen": "Zucht von den bunten Fellnasen", "carolal": "Carola und Hannah Lerch", "carolalerch": "Carola und Hannah Lerch", "chalfontstud": "Chalfont Stud, Freddy Braun", "christianefuchs": "Christiane Fuchs + Michael Höhnert", "christiek": "Christiane Kiessling", "christinam": "Christine Monika Mai", "christinek": "Christiane Kiessling", "christiner": "Christina Rudloff", "claudialöhr": "Claudia Löhr und Michael Koob", "claudialöhrmichaelkoob": "Claudia Löhr und Michael Koob", "colorfulfurrygerbils": "Colorful Furry Gerbils", "colourfulfurrygerbils": "Colorful Furry Gerbils", "danielakasberger": "Daniela Käsberger", "dörthe": "Dörthe Petzmann", "emelyhehlhorn": "Emely Mehlhorn", "fabianb": "Fabian Büdel", "floriang": "Florian Gries", "hannahraths": "Hannah Ruths", "hanserenners": "Hanse Renner", "haraldmariareitzlennemann": "Harald und Maria Reitz-Lennemann", "haraldundmariarl": "Harald und Maria Reitz-Lennemann", "heike": "Heike + Heiko Scheurich", "heikebarklay": "Heike Barklage", "heikoloos": "Heiko Loos und Annette Becker", "heikoloosannettebecker": "Heiko Loos und Annette Becker", "inahübner": "Ina Hübner + Nicolai Thome", "inapeines": "Ina Pleines", "janab": "Jana Beikert", "jasiminweber": "Jasmin Weber und Ronny Reichelt", "jasmin": "Jasmin Weber", "jasminweber": "Jasmin Weber und Ronny Reichelt", "jasminweberromyreichelt": "Jasmin Weber und Ronny Reichelt", "jasminweberronnyreichelt": "Jasmin Weber und Ronny Reichelt", "joystaatberg": "Joy Staalberg", "juliaa": "Julia Ast", "juttam": "Jutta Metz", "karina": "Karina Luft", "karinborgotti": "Karin Borsotti", "katjal": "Katja Leffeck", "kimw": "Kim Waldschmidt", "kkchaos": "KK-Chaos", "kkchaosofkkchaos": "KK-Chaos", "kleinechaoten": "Zucht der kleinen Chaoten", "kleinenchaoten": "Zucht der kleinen Chaoten", "kriegernmitkrallen": "Krieger mit Krallen", "lennylengo": "Lenny Lengo", "lisa": "Lisa und Marcel Kunz", "lisach": "Lisa und Lydia Christ", "lisachrist": "Lisa und Lydia Christ", "lisalydiachrist": "Lisa und Lydia Christ", "littlefellows": "little fellows", "littlerunners": "little runners", "maikef": "Maike Franz", "maintalerpz": "Privatzucht Maintal", "marinau": "Marina Unger", "martinaandreaswestfeld": "Martina und Andreas Westfeld", "martinaw": "Martina und Andreas Westfeld", "martinawestfeld": "Martina und Andreas Westfeld", "meikesch": "Meike und Heiko Sch.", "melanief": "Melanie Fey und Thorben Meier", "melaniefey": "Melanie Fey und Thorben Meier", "melaniefeythorbenmeier": "Melanie Fey und Thorben Meier", "michaeldavidschmitz": "Michael und David Schmitz", "michaelschmitz": "Michael und David Schmitz", "nataliereitz": "Nathalie Reitz", "natascham": "Natascha Marienfeld", "nicolefischler": "Nicole Tischler", "nicolel": "Nicole Lannert", "nicolen": "Nicole Nuzzo", "nicolet": "Nicole Tischler", "nicolew": "Nicole Webersinn", "nicost": "Nico Stamm", "nielsh": "Nils H.", "noelstrahbach": "Noel Strohbach", "nora": "Nora Rudersdorf geb. Holzbach", "norah": "Nora Holzbach", "norarh": "Nora Rudersdorf", "norarudersdorf": "Nora Rudersdorf geb. Holzbach", "oflennylengo": "Lenny Lengo", "pascale": "Pascale Diefenbach", "pascaledießenbach": "Pascale Diefenbach", "patriciap": "Patricia Petry", "paul": "Paul W.", "paula": "Paula Gabler", "paulag": "Paula Gabler", "privatzuchtmuecke": "Privatzucht Mücke", "privatzuchtmücke": "Privatzucht Mücke", "pzmaintal": "Privatzucht Maintal", "pzmuecke": "Privatzucht Mücke", "pzmücke": "Privatzucht Mücke", "pzseligenstadt": "Privatzucht Seligenstadt", "ramonag": "Ramona Gömpel", "ranialößler": "Rania Löffler", "rominahubrich": "Romina Milde / Hubrich", "rominamilde": "Romina Milde / Hubrich", "romonag": "Ramona Gömpel", "ronialöffler": "Rania Löffler", "sandrak": "Sandra Kubas", "sarahlöser": "Sarah Löwer", "sarahlöuer": "Sarah Löwer", "sarahz": "Sarah Zitzer", "saskiaw": "Saskia Wucher", "schlossmaus": "Schlossmäuse", "schlossmäuse": "Schlossmäuse", "schlossmäusen": "Schlossmäuse", "sebastiansch": "Sebastian Schmitt", "serinaberg": "Selina Berg", "silkewolfganghintze": "Silke und Wolfgang Hintze", "smilla": "Smilla H.", "steffi": "Steffi K.", "susanneninat": "Susanne + Nina Thomas", "susannet": "Susanne + Torsten Saum", "theresavalenca": "Theresia Valenca (Kopp)", "theresiavalenca": "Theresia Valenca (Kopp)", "timpf": "Tim Pfeiffer und Michelle Mai", "timpfeifermichellemai": "Tim Pfeiffer und Michelle Mai", "timpfeiffer": "Tim Pfeiffer und Michelle Mai", "timpfeiffermichellemai": "Tim Pfeiffer und Michelle Mai", "tonoböckenseld": "Tono Böckenfeld", "topol": "Topolino", "ulrichmüller": "Ulrich + Angela Müller", "ulrikec": "Ulrike Cordes", "ulriker": "Ulrike Ruppel", "ulrikesch": "Ulrike Schulz", "ulriket": "Ulrike Treml", "vanessab": "Vanessa Becker", "vanessag": "Vanessa Groll", "veragreywitz": "Vera Geywitz", "vonprivatzuchtmaintal": "Privatzucht Maintal", "wolfgangfaus": "Wolfgang Faust", "yvonne": "Yvonne Obendorfer", "zuchtderkleinenchaoten": "Zucht der kleinen Chaoten", } if n in norm_map: return norm_map[n], True return name, True def get_normalized_gerbil_name(name): if not name: return "" n = name.strip() norm_key = "".join(c for c in n.lower() if c.isalnum()) gerbil_norm_map = { "samgenshellyvdbuntenfellnasen": "Sammy gen. Shelly von den bunten Fellnasen", "sammygenshellyvdbuntenfellnasen": "Sammy gen. Shelly von den bunten Fellnasen", "schmidt": "Schmidti", "sheila": "Sheila of Ulmer Strolche", "shinichi": "Shinichi von PZ Mücke", "silenosgenadonis": "Silenos gen. Adonis von den Kleinen Chaoten", "silenosgenadonisvdkleinenchaoten": "Silenos gen. Adonis von den Kleinen Chaoten", "silver": "Silver von den kleinen Chaoten", "snoops": "Snoopsi", "sokrates": "Sokrates von den Kleinen Chaoten", "splash": "Slash", "teiko": "Teiko von den kleinen Chaoten", "trixy": "Trixxy von den Kleinen Chaoten", "unique": "Unique of Wild Dreams", } if norm_key in gerbil_norm_map: return gerbil_norm_map[norm_key] return n def get_call_name(name): if not name: return "" n = name.strip() n = re.sub(r'\[[^\]]+\]$', '', n).strip() n = re.split(r'\s+(?:of|von\s+den|von\s+der|v\.\s?d\.|von)\s+', n, flags=re.IGNORECASE)[0].strip() return n def get_dedup_name_key(name): if not name: return "" n = name.lower().strip() # Strip common suffixes/prefixes and parentheticals n = re.sub(r'\b(?:von\s+den|v\.?\s*d\.?|v\.?o\.?)\s+(?:kleinen\s+)?chaoten\b', '', n) n = re.sub(r'\bvon\s+der\s+schlossm\w+\b', '', n) n = re.sub(r'\bvon\s+der\s+bunten\s+fellnasen?\b', '', n) n = re.sub(r'\bof\s+black\s+forest\b', '', n) n = re.sub(r'\b\(?rv\)?\b', '', n) n = re.sub(r'\bgen\.\s+\w+', '', n) # Clean up parentheses or brackets n = re.sub(r'\(.*?\)', '', n) n = re.sub(r'\[.*?\]', '', n) return "".join(c for c in n if c.isalnum()) def clean_color_name(c_desc): """Normalise a free-text colour label to a catalog key + Schecke flag. Returns (clean_name, is_schecke). A PARENTHETICAL „(schimmel)" is NOT a definitive Schimmel — the breeder writes it to mean „könnte sich später als Schimmel entpuppen" (ticket e22764aa). So we STRIP the „(…)" instead of folding it into the name (which used to turn „Blaufuchs(schimmel)" into the wrong „blaufuchsschimmel"); the still-uncertain Schimmel-modifier is carried by the genotype (ee[-] = Fuchs, Schimmel unknown), not the colour label. """ if not c_desc: return "", False # Lowercase and strip c = c_desc.lower().strip() # Check for Schecke is_schecke = False if re.search(r'\bsp\b|\bsp\d|\bsp[*(²³]|\bspotted|\bschecke|[- ]sp\b|\w+sp\b', c): is_schecke = True # Strip schecke/sp markers and any trailing text starting from sp c = re.sub(r'\([- ]?sp(otted)?\)', '', c) # handles (-sp) c = re.sub(r'[- ]?sp(otted)?\b.*', '', c) # handles -sp(k), -sp*(k), -sp, etc. c = re.sub(r'[- ]?schecke\b.*', '', c) c = re.sub(r'[- ]?spotted\b.*', '', c) # Strip any other parentheticals (incl. „(schimmel)" = „möglich/unbestimmt"), # symbols, or trailing stars/numbers. The parenthetical is deliberately NOT # promoted to a definitive part of the colour name (ticket e22764aa). c = re.sub(r'\s*\(.*?\)\s*', ' ', c) c = re.sub(r'[²³*]', '', c) c = c.strip() # Mapping table for abbreviations, typos, and specific combinations mapping = { "antra": "anthrazit", "anthra": "anthrazit", "ankazit": "anthrazit", "antrazit": "anthrazit", "pew": "rew", "bew": "hermelin", "harder": "marder", "kohli": "kohlfuchs", "aligerfuchs": "algierfuchs", "algiesfuchs": "algierfuchs", "schw": "schwarz", "sa": "silberagouti", "a": "agouti", "cp-sa": "cp-silberagouti", "cp-a": "cp-agouti", "cp-a-hell": "cp-agouti-hell", "cp-aisa": "cp-agouti", "cp-a / rcp-sa": "cp-agouti", } if c in mapping: c = mapping[c] return c, is_schecke def _match_color_label(clean_name, variety_map): """Map a cleaned colour label to a ColorVariety id (text-only path). Exact name wins; otherwise pick the LONGEST/most-specific substring match (ticket 3f5942a2 — the old code broke on the FIRST substring hit, so „Goldfuchs" matched the shorter „Gold" first). Among substring candidates the longest seed name wins, then the longest clean_name overlap; ties broken deterministically. """ if not clean_name: return None if clean_name in variety_map: return variety_map[clean_name] candidates = [] for seed_name, seed_id in variety_map.items(): if not seed_name: continue if seed_name in clean_name or clean_name in seed_name: # Specificity score: prefer the longer seed name (more specific), # then the closeness of lengths so „goldfuchs" beats „gold" for the # label „goldfuchs". candidates.append((len(seed_name), -abs(len(seed_name) - len(clean_name)), seed_name, seed_id)) if not candidates: return None candidates.sort(reverse=True) return candidates[0][3] def resolve_color_and_genotype(color_val, existing_genotype, variety_map, variety_genotypes): """Resolve a gerbil's stored ColorVariety id + genotype. GENOTYPE WINS (ticket cluster genetics-farbschlag): when a parseable genotype is present and the genetics engine (genotype.genotype_to_farbschlag — a faithful Python mirror of catalog.ts) computes a KNOWN catalog variety, that variety is authoritative for colorVarietyId. The free-text colour label is only a fallback (no genotype, or genotype resolves to „Unbekannt"). This fixes the imports where the source label ignored a locus (dd → „Agouti" instead of „Dilute Agouti", ee → „Gold" instead of „Goldfuchs", parenthetical „(schimmel)", …). Returns (color_variety_id, genotype). `genotype` is the (possibly Schecke- annotated) genotype STRING — never silently flips an explicit spsp to Spsp. """ if not color_val and not existing_genotype: return None, existing_genotype clean_name, is_schecke = clean_color_name(str(color_val).strip()) if color_val else ("", False) # 1) Genotype-derived variety (authoritative when it resolves to a known name). # GUARD (VORSICHTIG): only trust the genotype when it parsed CLEANLY enough to # decide a colour — both the C and E loci must be mapped. The breeder sometimes # writes the genotype in the COMPACT catalog notation („cchmcchm", „efef", # „chch") which this parser leaves UNMAPPED (it expects the bracketed „c[chm]" # form); a dropped C/E locus would silently read as wild-type and mis-recolour # an otherwise-correct animal (e.g. Marder→Schwarz, Orangeschimmel→Agouti). When # the parse is incomplete we keep the source text label instead. color_variety_id = None geno_name = None if existing_genotype: try: mapped = gt.parse(existing_genotype).get("mapped8locus") or {} except Exception: mapped = {} if mapped.get("C") and mapped.get("E"): fs = gt.genotype_to_farbschlag(mapped) if fs and fs != gt.UNKNOWN_FARBSCHLAG: geno_name = fs color_variety_id = variety_map.get(fs.strip().lower()) # 2) Fall back to the text label when the genotype gave nothing usable. if not color_variety_id: color_variety_id = _match_color_label(clean_name, variety_map) # Update genotype if the LABEL says Schecke — but never override an explicit # Sp-locus already present in the source genotype (ticket e09d6f22: a source # „spsp" must NOT be flipped to „Spsp" just because the label looked scheckig; # the source genotype is authoritative for the Sp-locus). Only ADD Spsp when # the genotype carries no Sp token at all. genotype = existing_genotype if is_schecke: if genotype: if "Sp" not in genotype and "sp" not in genotype: genotype = f"{genotype} Spsp".strip() else: canonical = variety_genotypes.get(color_variety_id) if canonical: if "spsp" in canonical: genotype = canonical.replace("spsp", "Spsp") else: genotype = f"{canonical} Spsp".strip() else: genotype = "Spsp" return color_variety_id, genotype def parse_date(d): """Convert variations of date formats to YYYY-MM-DD.""" if not d or d == "0001-01-01": return None d = str(d).strip() # Try YYYY-MM-DD if re.match(r"^\d{4}-\d{2}-\d{2}$", d): parts = d.split("-") year = int(parts[0]) if 1900 < year < 2000: year += 100 d = f"{year}-{parts[1]}-{parts[2]}" elif year == 1900: return None return d # Try DD.MM.YYYY or D.M.YY match = re.match(r"^(\d{1,2})\.(\d{1,2})\.(\d{2,4})$", d) if match: day = int(match.group(1)) month = int(match.group(2)) year = int(match.group(3)) if year < 100: year += 2000 elif 1900 < year < 2000: year += 100 elif year == 1900: return None try: return datetime(year, month, day).strftime("%Y-%m-%d") except ValueError: pass # Try ISO timestamp try: dt = datetime.fromisoformat(d.replace("Z", "+00:00")) year = dt.year if 1900 < year < 2000: dt = dt.replace(year=year + 100) return dt.strftime("%Y-%m-%d") elif year == 1900: return None return dt.strftime("%Y-%m-%d") except ValueError: pass return None def date_to_days(dt_str): if not dt_str: return None try: return (datetime.strptime(dt_str, "%Y-%m-%d") - datetime(2000, 1, 1)).days except ValueError: return None def days_to_date(days): import datetime as dt return (dt.datetime(2000, 1, 1) + dt.timedelta(days=int(days))).strftime("%Y-%m-%d") def parse_death_info(notes, status, existing_dod, existing_cod): if not notes: return status, existing_dod, existing_cod has_death_indicator = '†' in notes or 'verstorben' in notes.lower() or 'gestorben' in notes.lower() or 'todesdatum' in notes.lower() or '/+' in notes if '+' in notes: if re.search(r'\+\s*(?:LE|AS|Unbekannt|gestorben|verstorben|tod)\b', notes, re.I) or re.search(r'\+\s*\d{1,2}\.\d{1,2}\.\d{2,4}', notes) or '/+' in notes: has_death_indicator = True resolved_status = status if has_death_indicator: resolved_status = "Deceased" dod = existing_dod cod = existing_cod # Look for date near death indicator found_date = None for m in re.finditer(r'([+†]\s*(?:LE|AS|Unbekannt|[a-zA-ZäöüÄÖÜß0-9()/ +,;.:-]{1,100}?)?\s*)(\d{1,2}\.\d{1,2}\.\d{2,4})', notes, re.I): marker_text = m.group(0) if '†' in marker_text or re.search(r'\+\s*(?:LE|AS|Unbekannt|gestorben|verstorben|tod|\d)', marker_text, re.I): found_date = parse_date(m.group(2)) if found_date: break if not found_date: m_death_marker = re.search(r'(†\s*)(\d{1,2}\.\d{1,2}\.\d{2,4})', notes) if m_death_marker: found_date = parse_date(m_death_marker.group(2)) # FIX (ec9267b9): markerless death dates. When a death indicator is present but # no +/cross marker precedes the date (e.g. "Verstorbener Welpe am 30.03.15 …", # "Verstorben am 04.12.18 …"), take the FIRST plausible date as the death date # — but only if we have a death keyword and still have no date. Status logic is # unchanged; we never overwrite an existing dateOfDeath. if not found_date and has_death_indicator and not existing_dod: if re.search(r'\b(?:verstorb\w*|gestorb\w*|verstarb\w*|todesdatum)\b', notes, re.I): m_plain = re.search(r'\b(\d{1,2}\.\d{1,2}\.\d{2,4})\b', notes) if m_plain: found_date = parse_date(m_plain.group(1)) if found_date and not dod: dod = found_date if not cod: for m in re.finditer(r'([+†])\s*([a-zA-ZäöüÄÖÜß0-9()/ +,;.:-]{1,100}?)\s*\d{1,2}\.\d{1,2}\.\d{2,4}', notes, re.I): indicator = m.group(1) cod_candidate = m.group(2).strip() if indicator == '+' and not re.search(r'\b(?:LE|AS|Unbekannt|gestorben|verstorben|tod)\b', cod_candidate, re.I): continue if cod_candidate: cod = cod_candidate break if not cod: m_cod2 = re.search(r'[+†]\s*\d{1,2}\.\d{1,2}\.\d{2,4}\s*([a-zA-ZäöüÄÖÜß0-9()/ +,;.:-]{1,100})', notes, re.I) if m_cod2: cod_candidate = m_cod2.group(1).strip() if cod_candidate: cod = cod_candidate else: m_cod3 = re.search(r'[+†]\s*(LE|AS|Unbekannt)\b', notes, re.I) if m_cod3: cod = m_cod3.group(1).strip() if cod: cod_lower = cod.lower().strip() if cod_lower in ("le", "le (lungenentzündung)", "lungenentzündung", "lungenentzündung (lungenentzündung)"): cod = "Lungenentzündung" elif cod_lower in ("as", "altersschwäche", "altenschwäche", "alter"): cod = "Altersschwäche" elif cod_lower in ("unbekannt", "unklar"): cod = "Unbekannt" else: # Replace abbreviations with full names (case-insensitive) cod = re.sub(r'\bLE\b', 'Lungenentzündung', cod, flags=re.I) cod = re.sub(r'\bAS\b', 'Altersschwäche', cod, flags=re.I) cod = cod.replace('+', ' + ') # Clean up multiple spaces cod = re.sub(r'\s+', ' ', cod).strip(' ,;.-') # Re-check after replacement cod_lower = cod.lower().strip() if cod_lower in ("le", "le (lungenentzündung)", "lungenentzündung", "lungenentzündung (lungenentzündung)"): cod = "Lungenentzündung" elif cod_lower in ("as", "altersschwäche", "altenschwäche", "alter"): cod = "Altersschwäche" elif cod_lower in ("unbekannt", "unklar"): cod = "Unbekannt" # Collapse redundant "X (X)" (e.g. Abkürzung + ausgeschriebene Form in Klammern, # "AS (Altersschwäche)" → nach Expansion "Altersschwäche (Altersschwäche)") auf "X". if cod: m_dup = re.match(r'^\s*(.+?)\s*\(\s*(.+?)\s*\)\s*$', cod) if m_dup and m_dup.group(1).strip().lower() == m_dup.group(2).strip().lower(): cod = m_dup.group(1).strip() return resolved_status, dod, cod # ── Litter dedup & parent-role helpers (pure, unit-tested in test_merge_resolve.py) ── def _norm_pname(s): return normalize_name(s) if s else "" def names_no_conflict(l1, l2): """Parent names don't contradict (equal per role, or one side empty).""" f1, f2 = _norm_pname(l1.get("_father_name")), _norm_pname(l2.get("_father_name")) m1, m2 = _norm_pname(l1.get("_mother_name")), _norm_pname(l2.get("_mother_name")) f_ok = (not f1) or (not f2) or (f1 == f2) m_ok = (not m1) or (not m2) or (m1 == m2) return f_ok and m_ok def names_overlap(l1, l2): """At least one role has a non-empty matching name (positive evidence).""" f1, f2 = _norm_pname(l1.get("_father_name")), _norm_pname(l2.get("_father_name")) m1, m2 = _norm_pname(l1.get("_mother_name")), _norm_pname(l2.get("_mother_name")) return bool((f1 and f1 == f2) or (m1 and m1 == m2)) def litter_compatible(l1, l2): """Two litter records describe the same litter: same date and compatible parents. - Both sides have both parents → must match exactly. - Asymmetric (one side resolved, the other not) → merge only if names don't contradict; for dateless litters require a POSITIVE name match (a shared null date is no evidence), so unrelated nameless stubs stay separate. - Neither side has parents → never blind-merge. """ if l1["Date"] != l2["Date"]: return False f1, m1 = l1.get("FatherId"), l1.get("MotherId") f2, m2 = l2.get("FatherId"), l2.get("MotherId") if f1 and f2 and m1 and m2: return f1 == f2 and m1 == m2 asymmetric = (bool(f1 or m1) and not (f2 or m2)) or (bool(f2 or m2) and not (f1 or m1)) if asymmetric: if not names_no_conflict(l1, l2): return False if l1["Date"] is None: return names_overlap(l1, l2) return True return False def assign_parent_roles(father_id, mother_id, gender_of): """Assign two resolved parent IDs to father/mother roles by gender. Drops self-pairing duplicates (same animal in both roles) and never returns two same-role parents. `gender_of` maps an id to 'male'|'female'|'unknown'|None. Returns (father_id, mother_id). """ ids = [] for gid in (father_id, mother_id): if gid and gid not in ids: ids.append(gid) males = [g for g in ids if gender_of(g) == "male"] females = [g for g in ids if gender_of(g) == "female"] unknowns = [g for g in ids if gender_of(g) == "unknown"] father = males[0] if males else (unknowns.pop(0) if unknowns else None) mother = females[0] if females else (unknowns.pop(0) if unknowns else None) return father, mother # A gerbil lives at most ~6 years, so a parent can be at most ~6 years older than # its offspring (and must be born before it). Links outside this window are # impossible — e.g. a 2013 animal resolved onto a 2022 litter (Jayjay → Solice). MAX_PARENT_AGE_DAYS = 2379 def parent_age_plausible(parent_dob, litter_date): """Could a parent born `parent_dob` have offspring born on `litter_date`? Requires birth strictly before the litter and within the gerbil lifespan. Unknown/unparseable dates return True (cannot disprove). Accepts any date format parse_date understands. """ pd = date_to_days(parse_date(parent_dob)) if parent_dob else None ld = date_to_days(parse_date(litter_date)) if litter_date else None if pd is None or ld is None: return True return 0 < (ld - pd) <= MAX_PARENT_AGE_DAYS def pick_parent_ref(parent_refs, role, child_dob, avoid_name=None, gender_of=None): """Choose the best parent ref for a role from possibly-conflicting chart refs. A Stammbaum lists an animal at several positions, so its parentRefs can carry contradictory guesses (the first one is not necessarily right). Rank candidates (lower = better): 0 right/unknown gender for the role, age-plausible dated ref 1 right/unknown gender, no DOB (usable, but a plausible dated ref wins) 2 right/unknown gender, dated but age-impossible 3 resolved gender is clearly WRONG for the role (e.g. a female father) 4 would duplicate the animal chosen for the other role Gender is decisive over DOB: a dated female ref must not win the father slot over an undated male/unknown one. `gender_of(name)` returns 'male'/'female' or None (unknown/ambiguous → not penalised). Returns the chosen ref or None. """ role_refs = [p for p in parent_refs if p.get("roleGuess") == role] if not role_refs: return None avoid = normalize_name(avoid_name) if avoid_name else None expected = "male" if role == "father" else "female" def rank(p): if avoid is not None and normalize_name(p.get("name")) == avoid: return 4 # would duplicate the other parent role g = gender_of(p.get("name")) if gender_of else None if g in ("male", "female") and g != expected: return 3 # wrong sex for this role dob = p.get("dob") if not dob: return 1 return 0 if parent_age_plausible(dob, child_dob) else 2 order = sorted(range(len(role_refs)), key=lambda i: (rank(role_refs[i]), i)) return role_refs[order[0]] def explain_pick_rejections(parent_refs, role, child_dob, chosen, avoid_name=None, gender_of=None): """Explain why other refs for `role` lost to `chosen` in pick_parent_ref. Returns a list of discard dicts (for _format_discard) — one per distinct rejected candidate name that was beaten for a clear reason (wrong sex, age- impossible, or duplicate of the other parent). Mirrors pick_parent_ref's ranking so the history can explain the same decision it made. """ role_refs = [p for p in parent_refs if p.get("roleGuess") == role] if not role_refs or chosen is None: return [] avoid = normalize_name(avoid_name) if avoid_name else None expected = "male" if role == "father" else "female" role_de = "Vaterrolle" if role == "father" else "Mutterrolle" cand_de = "Vater-Kandidat" if role == "father" else "Mutter-Kandidat" chosen_name = chosen.get("name") repl_disp = chosen_name if chosen.get("dob"): repl_disp = f"{chosen_name} (*{_de_date(parse_date(chosen.get('dob'))) or chosen.get('dob')})" out = [] seen = set() for p in role_refs: name = p.get("name") if not name or normalize_name(name) == normalize_name(chosen_name or ""): continue key = normalize_name(name) if key in seen: continue g = gender_of(name) if gender_of else None dob = p.get("dob") reason = None if avoid is not None and key == avoid: reason = "bereits als anderer Elternteil gewählt" elif g in ("male", "female") and g != expected: reason = f"falsches Geschlecht für die {role_de}" elif dob and not parent_age_plausible(dob, child_dob): reason = "unplausibles Alter für diesen Wurf" if reason is None: continue seen.add(key) disp = name if dob: disp = f"{name} (*{_de_date(parse_date(dob)) or dob})" out.append({ "label": cand_de, "value": f"„{disp}“", "reason": reason, "replacement": f"„{repl_disp}“", }) return out def _build_gerbil_history(records, best_g, field_source, parent_method=None, any_decision=False, any_conflict=False, conflict_notes=None, field_discards=None, parent_discards=None): """Build an ordered, file-attributed German history for a resolved gerbil. Reads like a chronological log: • „In ‚X.xlsx' gefunden." • „Geburtsdatum (27.03.2022) aus ‚X.xlsx'." • „Auch in ‚Y.xlsx' gefunden → Datensätze zusammengeführt." • „Genotyp aus ‚Z.xlsx'." • „Eltern über Position im Stammbaum erkannt (Quelle: ‚X.xlsx')." • „Aus Wurfchronik übernommen." `field_source` maps a field name to the raw record that supplied its final value; when present we name that record's file, otherwise we fall back to the primary record. The records are visited in a stable order (primary first, then the rest sorted by file) so the log is deterministic.""" conflict_notes = conflict_notes or [] history = [] # Order records: best_g first, then others by primary file name (stable). others = [r for r in records if r is not best_g] others.sort(key=lambda r: (_primary_file_of(r) or "")) ordered = [best_g] + others best_file = _primary_file_of(best_g) if best_file: history.append(f"In {_quote_file(best_file)} gefunden.") else: history.append("Im Import gefunden.") # Field-by-field attribution: name the file that supplied each fact. def attr_line(field, formatter): rec = field_source.get(field) or best_g val = best_g.get(field) if not val or val == "unknown": return fname = _primary_file_of(rec) label = PROV_FIELD_LABELS.get(field, field) text = formatter(label, val) if fname: history.append(f"{text} aus {_quote_file(fname)}.") else: history.append(f"{text} (Quelle unbekannt).") attr_line("DateOfBirth", lambda label, v: f"{label} ({_de_date(v)})") attr_line("Gender", lambda label, v: f"{label} ({_de_gender(v)})") attr_line("Genotype", lambda label, v: f"{label}") attr_line("ColorVarietyId", lambda label, v: f"{label}") attr_line("DateOfDeath", lambda label, v: f"{label} ({_de_date(v)})") # Merge step: every additional record that contributed. for r in others: fname = _primary_file_of(r) if fname: history.append( f"Auch in {_quote_file(fname)} gefunden → Datensätze zusammengeführt." ) else: history.append("In weiterem Datensatz gefunden → Datensätze zusammengeführt.") # Discarded field values from majority-vote conflict resolution: the LOSING # values, their file, and what won instead. for d in (field_discards or []): line = _format_discard(d) if line not in history: history.append(line) # Parent derivation. if parent_method: method_label = { "chart-position": "Position im Stammbaum", "geburtsdatum+eltern": "Geburtsdatum und Elternnamen", "nur-geburtsdatum": "Geburtsdatum", "decision": "manuelle Entscheidung", }.get(parent_method, parent_method) # The parent evidence comes from a Stammbaum chart — attribute to the # primary record's file when it is a Stammbaum. parent_file = best_file if best_file and "stammbaum" in best_file.lower() else None if parent_file: history.append( f"Eltern über {method_label} erkannt (Quelle: {_quote_file(parent_file)})." ) else: history.append(f"Eltern über {method_label} erkannt.") # Discarded parent candidates / links (pick_parent_ref rejections, role # normalization drops, parent-age sanity check). Threaded in after the merge # via the gerbil's _discarded list so they read in chronological order. for d in (parent_discards or []): line = _format_discard(d) if line not in history: history.append(line) # Manual decisions / conflicts. if any_decision: history.append("Zuordnung per manueller Entscheidung getroffen.") if any_conflict: history.append("Konflikt per Entscheidung gelöst.") for cn in conflict_notes: if cn and cn not in history: history.append(cn + ".") # Wurfchronik provenance line. if any("wurfchronik" in f.lower() for r in records for f in _record_source_files(r)): history.append("Angaben aus der Wurfchronik übernommen.") return history def _de_date(iso): """YYYY-MM-DD → DD.MM.YYYY for display; pass through anything else.""" if not iso: return iso m = re.match(r"^(\d{4})-(\d{2})-(\d{2})$", str(iso)) if m: return f"{m.group(3)}.{m.group(2)}.{m.group(1)}" return iso def _de_gender(g): return {"male": "männlich", "female": "weiblich"}.get(g, g) # Leading marker that visually flags a discard ("data was thrown away") line in # the history timeline. The frontend keys discard styling off this marker. DISCARD_MARK = "⚠ " def _format_discard(d): """Render one discard record into a German history line. A discard record is a dict describing a value/candidate the pipeline threw away. Recognised keys: • text — a fully pre-formatted line (used verbatim, marker added) • label — German field label (e.g. „Geburtsdatum“) • value — the discarded value (already display-formatted) • file — source file the discarded value came from (attributed) • reason — why it was dropped (e.g. „abweichend“, „unplausibel …“) • replacement — what was used instead (already display-formatted) • repl_file — source file the replacement came from Produces lines like: „⚠ Geburtsdatum 14.06.2015 aus ‚A.xlsx‘ verworfen — abweichend; 14.06.2017 aus ‚B.xlsx‘ verwendet (Mehrheit).“ Generic across entity types so litters/contacts can reuse it. """ if d.get("text"): return DISCARD_MARK + d["text"] parts = [] label = d.get("label") value = d.get("value") if label and value is not None: parts.append(f"{label} {value}") elif label: parts.append(str(label)) elif value is not None: parts.append(str(value)) head = " ".join(parts) if parts else "Wert" if d.get("file"): head += f" aus {_quote_file(d['file'])}" line = f"{head} verworfen" if d.get("reason"): line += f" — {d['reason']}" repl = d.get("replacement") if repl is not None and repl != "": instead = str(repl) if d.get("repl_file"): instead += f" aus {_quote_file(d['repl_file'])}" suffix = d.get("replacement_note") line += f"; {instead} verwendet" if suffix: line += f" ({suffix})" elif d.get("no_replacement"): line += "; kein Ersatz" return DISCARD_MARK + line + "." def _append_history(prov_json, line): """Append one history line to an existing Provenance JSON string and add the contract source file. Returns the updated JSON string.""" try: prov = json.loads(prov_json) if prov_json else {} except (ValueError, TypeError): prov = {} prov.setdefault("history", []) if line not in prov["history"]: prov["history"].append(line) return json.dumps(prov, ensure_ascii=False) def _parse_price(raw): """Parse a contract price string ('27,50' / '30.00' / '') → float (0.0 if empty). extract_contracts.py emits German-formatted numbers ('27,50'); accept both comma and dot decimal separators. Unparseable/empty → 0.0 (a price-less contract is still a valid contract record).""" if raw is None: return 0.0 s = str(raw).strip() if not s: return 0.0 s = s.replace(".", "").replace(",", ".") if ("," in s) else s try: return round(float(s), 2) except ValueError: return 0.0 def enrich_from_contracts(contracts, resolved_gerbils, contact_by_norm_name, contact_id_map, exclude_decisions=None): """Conservatively fold Abgabevertrag data into the resolved gerbils AND emit one SaleContract record per contract with a resolvable buyer. For every parsed contract we (a) ensure the buyer exists as a (receiver) contact, reusing the existing contact dedup/normalisation, and (b) try to match each animal call-name to exactly one resolved gerbil that the breeder bred ("…Chaoten"). On a confident match we set ReceiverContactId / GoHomeDate / Status=GivenAway — but only where not already set differently — and add a provenance history line. Ambiguous or absent matches are logged, never guessed. In addition we build a `sale_contracts` list (one record per contract whose buyer resolves to a contact). Each record carries a deterministic Id (from the source filename, so re-ingest is idempotent), the resolved buyer ContactId, the parsed Price, the parsed dates and the gerbil ids that matched for that contract. Contracts with NO date at all are skipped from record creation (the SaleContract.HandoverDate/ContractDate columns are non-nullable DateOnly) and counted in stats["dateless_skipped"]; contracts whose buyer cannot be resolved are counted in stats["no_buyer_skipped"]. Returns (stats, sale_contracts). Mutates resolved_gerbils + contact_by_norm_name in place. New buyer contacts are appended via contact_by_norm_name so the later IsReceiver-flag pass picks them up automatically. """ stats = { "contracts": len(contracts), "buyers_created": 0, "buyers_existing": 0, "matched": 0, "ambiguous_skipped": 0, "no_match_skipped": 0, "receiver_set": 0, "gohome_set": 0, "status_givenaway": 0, "conflicts": 0, "records_created": 0, "no_buyer_skipped": 0, "dateless_skipped": 0, "records_with_animal": 0, "records_with_date": 0, } sale_contracts = [] # The same contract filename can appear more than once in contracts.json # (the .docx is filed in several subfolders of the share). The record Id is # derived from the filename, so we must collapse those into ONE record per # Id (a duplicate PK would break ingest). Keyed by Id; animal lists are # merged and a missing date is back-filled from the duplicate. records_by_id = {} if not contracts: return stats, sale_contracts # excludeContractMatch (MemPalace stolperfalle): some breeding/parent animals # (e.g. Makoto, Danako) are wrongly matched to a contract whose filename names # them as PARENTS of the sold pup, not as the sold animals. A decision with # `excludeContractMatch: true` pins such an animal OUT of contract matching: it # never gets ReceiverContactId/GoHomeDate/Status=GivenAway from a contract. # The override that runs BEFORE enrich (isResident/notes/receiver) cannot undo # this afterwards, so the gate must sit here, inside the match loop. _exclude_namedob = set() # {(call-name, iso-dob), (call-name, "")} _exclude_extref = set() # externalRef suffixes for d in (exclude_decisions or []): if not d.get("excludeContractMatch"): continue er = (d.get("externalRef") or "").strip() if er: _exclude_extref.add(er) nm = d.get("name") if nm is not None: ck = normalize_name(get_call_name(nm or "")) iso = parse_date(d.get("dob")) if d.get("dob") else "" _exclude_namedob.add((ck, iso or "")) def _is_excluded(g): if _exclude_extref: er = g.get("ExternalRef") or "" if er and (er in _exclude_extref or any(er.endswith(k) for k in _exclude_extref)): return True if _exclude_namedob: ck = normalize_name(get_call_name(g.get("Name") or "")) iso = g.get("DateOfBirth") or "" if (ck, iso) in _exclude_namedob or (ck, "") in _exclude_namedob: return True return False # Index breeder-owned gerbils by dedup name key. Contracts only ever sell # animals the breeder bred, so restrict candidates to her own stock to avoid # colliding with same-named foreign-bred animals. def is_own(g): ob = (g.get("OriginBreeder") or "").lower() return ("chaoten" in ob) or (g.get("OriginBreeder") is None) index = {} for g in resolved_gerbils: if not is_own(g): continue key = get_dedup_name_key(get_call_name(g.get("Name", ""))) if key: index.setdefault(key, []).append(g) def year_of(iso): return iso[:4] if iso else None # Reject buyer values that are obviously label leakage / non-person noise # (defends against any stale contracts.json produced before the parser fix). _buyer_junk = re.compile( r"^(?:stra\w+e|wohnort|fon|e-?mail|handy|festnetz|telefon|homepage|" r"zuchtname|facebook|name)\s*:?\s*$|^[\d\s/]+$", re.IGNORECASE) for c in contracts: fname = c.get("sourceFile", "") buyer_raw = (c.get("buyer") or "").strip() if buyer_raw and _buyer_junk.match(buyer_raw): buyer_raw = "" # --- buyer contact (reuse curated normalisation/dedup) --- buyer_global_id = None if buyer_raw: canon, keep = get_normalized_contact_name(buyer_raw) if keep and canon: norm = normalize_name(canon) if norm in contact_by_norm_name: gc = contact_by_norm_name[norm] gc.setdefault("_source_files", set()).add(fname) gc["_merged_count"] = gc.get("_merged_count", 1) + 1 stats["buyers_existing"] += 1 else: gid = generate_guid(f"contact-{norm}") contact_by_norm_name[norm] = { "Id": gid, "Name": canon, "Email": None, "Phone": None, "Address": None, "Notes": None, "NameSuffix": None, "_source_files": {fname}, "_merged_count": 1, } stats["buyers_created"] += 1 buyer_global_id = contact_by_norm_name[norm]["Id"] # --- match each animal call-name to a resolved gerbil --- handover = parse_date(c.get("handoverDate")) contract_date = parse_date(c.get("contractDate")) c_year = year_of(parse_date(c.get("dob"))) if c.get("dob") else None c_color = (c.get("color") or "").strip().lower() matched_gerbil_ids = [] # gerbils this contract resolved to (for the record) for call in (c.get("animals") or []): key = get_dedup_name_key(get_call_name(call)) if not key: continue cands = index.get(key, []) if not cands: stats["no_match_skipped"] += 1 continue # Corroborate when more than one candidate shares the call-name. chosen = None if len(cands) == 1: chosen = cands[0] else: scored = [] for g in cands: score = 0 g_year = year_of(g.get("DateOfBirth")) if c_year and g_year and c_year == g_year: score += 2 if c_color and g.get("ColorVarietyId"): # color match is corroboration; we don't have the name # here, so only DOB drives disambiguation strongly. pass scored.append((score, g)) scored.sort(key=lambda t: t[0], reverse=True) if scored[0][0] >= 2 and (len(scored) == 1 or scored[0][0] > scored[1][0]): chosen = scored[0][1] else: stats["ambiguous_skipped"] += 1 continue # excludeContractMatch: this animal is pinned out of contract matching # (it is a parent named in the filename, not the sold pup). Skip BEFORE # any receiver/gohome/status mutation and before counting it as matched. if _is_excluded(chosen): stats["no_match_skipped"] += 1 continue stats["matched"] += 1 if chosen.get("Id") and chosen["Id"] not in matched_gerbil_ids: matched_gerbil_ids.append(chosen["Id"]) # --- set receiver, only if not already set differently --- if buyer_global_id: cur = chosen.get("ReceiverContactId") if not cur: chosen["ReceiverContactId"] = buyer_global_id stats["receiver_set"] += 1 chosen["Provenance"] = _append_history( chosen.get("Provenance"), f"Abgabe an „{buyer_raw}“ aus Vertrag {_quote_file(fname)} übernommen.") elif cur != buyer_global_id: stats["conflicts"] += 1 chosen["Provenance"] = _append_history( chosen.get("Provenance"), f"Vertrag {_quote_file(fname)} nennt anderen Abnehmer „{buyer_raw}“ " f"— bestehende Zuordnung beibehalten.") # --- set go-home date, only if empty --- if handover and not chosen.get("GoHomeDate"): chosen["GoHomeDate"] = handover stats["gohome_set"] += 1 chosen["Provenance"] = _append_history( chosen.get("Provenance"), f"Abgabedatum {handover} aus Vertrag {_quote_file(fname)} übernommen.") # --- status: derive GivenAway if we set a receiver and it isn't # already a stronger state (Deceased). --- if chosen.get("ReceiverContactId") and chosen.get("Status") not in ( "Deceased", "GivenAway"): chosen["Status"] = "GivenAway" stats["status_givenaway"] += 1 # --- emit a SaleContract record for this contract --------------------- # Only contracts with a resolvable buyer become records (the row needs a # ContactId). A record with zero matched animals is still kept — better # to show the contract than to drop it. if not buyer_global_id: stats["no_buyer_skipped"] += 1 continue # HandoverDate/ContractDate are non-nullable DateOnly in the DB. Fall # back from one to the other; if BOTH are missing, skip the record # (we do not invent dates) and count it. h = handover or contract_date cd = contract_date or handover if not h: # implies cd is also None stats["dateless_skipped"] += 1 continue rec_id = generate_guid(f"contract-{fname}") existing = records_by_id.get(rec_id) if existing is None: records_by_id[rec_id] = { "Id": rec_id, "ContactId": buyer_global_id, "Price": _parse_price(c.get("price")), "HandoverDate": h, "ContractDate": cd, "FileName": fname, "Animals": list(matched_gerbil_ids), } else: # Same filename seen again — merge animal matches; back-fill price. for gid in matched_gerbil_ids: if gid not in existing["Animals"]: existing["Animals"].append(gid) if not existing["Price"]: existing["Price"] = _parse_price(c.get("price")) sale_contracts = list(records_by_id.values()) stats["records_created"] = len(sale_contracts) stats["records_with_animal"] = sum(1 for r in sale_contracts if r["Animals"]) stats["records_with_date"] = sum(1 for r in sale_contracts if r["HandoverDate"]) return stats, sale_contracts def main(): print("Loading color variety seeds...") variety_map = {} variety_genotypes = {} # Load from C# ApplicationContext.cs catalog for stable database GUIDs (index + 1) here = os.path.dirname(os.path.abspath(__file__)) app_context_path = os.path.abspath(os.path.join(here, "../../GerbilManagerWebAPI/ApplicationContext.cs")) cs_name_to_id = {} if os.path.exists(app_context_path): with open(app_context_path, 'r', encoding='utf-8') as f: content = f.read() catalog_match = re.search(r'catalog\s*=\s*\{(.*?)\};', content, re.DOTALL) if catalog_match: block = catalog_match.group(1) entries = re.findall(r'\(\s*"([^"]+)"\s*,\s*"([^"]+)"\s*,\s*(\d+)\s*\)', block) for idx, (name, genotype, sort_order) in enumerate(entries): variety_id = f"00000000-0000-0000-0000-{idx + 1:012d}" cs_name_to_id[name.strip().lower()] = variety_id variety_map[name.strip().lower()] = variety_id variety_genotypes[variety_id] = genotype.strip() else: print(f"Warning: ApplicationContext.cs not found at {app_context_path}") if os.path.exists(SEEDS_PATH): with open(SEEDS_PATH, "r", encoding="utf-8") as f: seeds = json.load(f) for v in seeds: name_lower = v["name"].strip().lower() variety_id = cs_name_to_id.get(name_lower) if not variety_id: variety_id = f"00000000-0000-0000-0000-{v['sortOrder'] + 1:012d}" variety_map[name_lower] = variety_id # Map English name if present if "english" in v and v["english"]: variety_map[v["english"].strip().lower()] = variety_id if variety_id not in variety_genotypes: variety_genotypes[variety_id] = v.get("canonicalGenotype") else: print(f"Warning: Seeds path not found at {SEEDS_PATH}") # Reverse map (variety GUID → human name) for discard/replacement history # lines that need to show a colour value rather than a raw GUID. First wins # so we keep the canonical lower-case catalog name. variety_id_to_name = {} for name_lower, vid in variety_map.items(): if vid not in variety_id_to_name: variety_id_to_name[vid] = name_lower md_files = sorted([f for f in os.listdir(DIR_PATH) if f.lower().endswith('.md')]) print(f"Found {len(md_files)} markdown files in {DIR_PATH}.") raw_contacts = [] raw_litters = [] raw_gerbils = [] # Map to track which files contained explicit dates file_explicit_dates = {} # 1. Parse JSON blocks from all markdown files and scope IDs by filename for filename in md_files: filepath = os.path.join(DIR_PATH, filename) with open(filepath, 'r', encoding='utf-8') as f: content = f.read() json_start = content.find("```json") if json_start == -1: continue json_start += len("```json") json_end = content.rfind("```") if json_end == -1 or json_end <= json_start: continue json_str = content[json_start:json_end].strip() try: data = json.loads(json_str) contacts_key = next((k for k in data if k.lower() == 'contacts'), None) litters_key = next((k for k in data if k.lower() == 'litters'), None) gerbils_key = next((k for k in data if k.lower() == 'gerbils'), None) file_contacts = data.get(contacts_key, []) if contacts_key else [] file_litters = data.get(litters_key, []) if litters_key else [] file_gerbils = data.get(gerbils_key, []) if gerbils_key else [] # Find explicit dates on this page to build chronological ordering explicit_dates = [] for l in file_litters: d = parse_date(l.get("Date") or l.get("date") or l.get("DateOfBirth") or l.get("dateOfBirth")) if d: explicit_dates.append(d) for g in file_gerbils: d = parse_date(g.get("DateOfBirth") or g.get("dateOfBirth")) if d: explicit_dates.append(d) if explicit_dates: # Convert to days since 2000-01-01 day_vals = [date_to_days(d) for d in explicit_dates if date_to_days(d) is not None] if day_vals: file_explicit_dates[filename] = sum(day_vals) / len(day_vals) def scope_id(local_id): if not local_id: return None local_str = str(local_id).strip() if not local_str: return None return generate_guid(f"{filename}-{local_str}") for c in file_contacts: old_id = c.get("Id") or c.get("id") c["_filename"] = filename c["_scoped_id"] = scope_id(old_id) raw_contacts.append(c) for l in file_litters: old_id = l.get("Id") or l.get("id") l["_filename"] = filename l["_scoped_id"] = scope_id(old_id) l["_scoped_father_id"] = scope_id(l.get("FatherId") or l.get("fatherId")) l["_scoped_mother_id"] = scope_id(l.get("MotherId") or l.get("motherId")) raw_litters.append(l) for g in file_gerbils: old_id = g.get("Id") or g.get("id") g["_filename"] = filename g["_scoped_id"] = scope_id(old_id) g["_scoped_litter_id"] = scope_id(g.get("LitterId") or g.get("litterId")) g["_scoped_origin_cid"] = scope_id(g.get("OriginContactId") or g.get("originContactId")) receiver_val = (g.get("ReceiverContactId") or g.get("receiverContactId") or g.get("BuyerId") or g.get("buyerId") or g.get("BuyerContactId") or g.get("buyerContactId") or g.get("givenAwayContactId") or g.get("givenAwayToContactId") or g.get("ownerContactId") or g.get("ownerId")) g["_scoped_receiver_cid"] = scope_id(receiver_val) raw_gerbils.append(g) except json.JSONDecodeError as e: print(f"Failed to parse JSON in {filename}: {e}") # Chronological Interpolation: Estimate the date of each page based on neighboring pages with dates file_dates = {} sorted_files = sorted(md_files) # Simple linear interpolation / extrapolation for i, fn in enumerate(sorted_files): if fn in file_explicit_dates: file_dates[fn] = file_explicit_dates[fn] else: # Look left for closest explicit date left_val, left_dist = None, None for j in range(i - 1, -1, -1): if sorted_files[j] in file_explicit_dates: left_val = file_explicit_dates[sorted_files[j]] left_dist = i - j break # Look right for closest explicit date right_val, right_dist = None, None for j in range(i + 1, len(sorted_files)): if sorted_files[j] in file_explicit_dates: right_val = file_explicit_dates[sorted_files[j]] right_dist = j - i break if left_val is not None and right_val is not None: # Interpolate file_dates[fn] = left_val + (right_val - left_val) * (left_dist / (left_dist + right_dist)) elif left_val is not None: # Extrapolate right (assume 30 days per page gap as placeholder) file_dates[fn] = left_val + (left_dist * 30) elif right_val is not None: # Extrapolate left file_dates[fn] = right_val - (right_dist * 30) else: # No dates in entire log? Default to 2011-01-01 file_dates[fn] = date_to_days("2011-01-01") print(f"Parsed {len(raw_contacts)} raw contacts, {len(raw_litters)} raw litters, {len(raw_gerbils)} raw gerbils.") # Run extract.py to make sure stammbaum data is up to date import subprocess print("Running extract.py to extract Stammbäume...") try: subprocess.run([sys.executable, "extract.py"], check=True) except Exception as e: print(f"Warning: Failed to run extract.py: {e}") # Load Stammbaum data stammbaum_only_animals = [] animals_path = os.path.join(OUTPUT_DIR, "animals.json") if os.path.exists(animals_path): with open(animals_path, "r", encoding="utf-8") as f: stammbaum_animals = json.load(f) for a in stammbaum_animals: sources = a.get("sourceFiles", []) if any("stammbaum" in str(s).lower() for s in sources): stammbaum_only_animals.append(a) print(f"Loaded {len(stammbaum_only_animals)} Stammbaum animals.") else: print(f"Warning: Stammbaum animals.json not found at {animals_path}") # Run extract_docx.py to make sure docx data is up to date print("Running extract_docx.py to extract docx...") try: subprocess.run([sys.executable, "extract_docx.py"], check=True) except Exception as e: print(f"Warning: Failed to run extract_docx.py: {e}") # Load Docx data docx_animals = [] docx_animals_path = os.path.join(OUTPUT_DIR, "docx_animals.json") if os.path.exists(docx_animals_path): with open(docx_animals_path, "r", encoding="utf-8") as f: docx_animals = json.load(f) print(f"Loaded {len(docx_animals)} animals from docx.") else: print(f"Warning: docx_animals.json not found at {docx_animals_path}") docx_litters = [] docx_litters_path = os.path.join(OUTPUT_DIR, "docx_litters.json") if os.path.exists(docx_litters_path): with open(docx_litters_path, "r", encoding="utf-8") as f: docx_litters = json.load(f) print(f"Loaded {len(docx_litters)} litters from docx.") else: print(f"Warning: docx_litters.json not found at {docx_litters_path}") # Abgabevertrag-Datensätze (aus extract_contracts.py). Optional: wenn die # Datei fehlt, läuft der Import ohne Vertrags-Anreicherung normal weiter. contracts = [] contracts_path = os.path.join(OUTPUT_DIR, "contracts.json") if os.path.exists(contracts_path): with open(contracts_path, "r", encoding="utf-8") as f: contracts = json.load(f) print(f"Loaded {len(contracts)} Abgabeverträge.") else: print(f"Info: contracts.json not found at {contracts_path} (Vertrags-Anreicherung übersprungen)") # Extract docx buyer contacts and add to raw_contacts for da in docx_animals: o_name = (da.get("owner") or "").strip() if o_name: o_scoped_id = generate_guid(f"docx-contact-{normalize_name(o_name)}") raw_contacts.append({ "Name": o_name, "_filename": "Wurfchronik-Detail", "_scoped_id": o_scoped_id }) # Map docx litters and append to raw_litters docx_litter_id_map = {} # wsCode -> scoped_id for dl in docx_litters: l_name = dl["litterId"] dob_val = parse_date(dl["dob"]) f_name = get_normalized_gerbil_name(dl["fatherName"]) m_name = get_normalized_gerbil_name(dl["motherName"]) ws_code = dl["wsCode"] note_val = dl.get("note") # Parse survived/total born from wsCode (e.g. 4/5) total_born = None deaths_8w = None if "/" in ws_code: parts = ws_code.split("/") if len(parts) == 2: try: survived = int(parts[0]) total = int(parts[1]) total_born = total deaths_8w = max(0, total - survived) except ValueError: pass l_scoped_id = generate_guid(f"docx-litter-{normalize_name(ws_code)}-{dob_val or '0001-01-01'}") docx_litter_id_map[(ws_code, dob_val)] = l_scoped_id raw_litters.append({ "Id": l_scoped_id, "Name": l_name, "Date": dob_val, "TotalBorn": total_born, "DeathsWithin8Weeks": deaths_8w, "FatherId": generate_guid(f"stammbaum-animal-{normalize_name(f_name)}"), # placeholder "MotherId": generate_guid(f"stammbaum-animal-{normalize_name(m_name)}"), # placeholder "ExpectedGoHomeDate": None, "Notes": note_val if note_val else "Docx imported litter", "PairingCode": None, "ExternalRef": f"docx-litter-{l_scoped_id}", "LitterLetter": l_name[0] if l_name and len(l_name) > 0 else None, "_father_name": f_name, "_mother_name": m_name, "_filename": "Wurfchronik-Detail", "_scoped_id": l_scoped_id, "_scoped_father_id": generate_guid(f"stammbaum-animal-{normalize_name(f_name)}"), "_scoped_mother_id": generate_guid(f"stammbaum-animal-{normalize_name(m_name)}") }) # Extract stammbaum contacts and add to raw_contacts for a in stammbaum_only_animals: b_name = (a.get("breeder") or "").strip() if b_name: b_scoped_id = generate_guid(f"stammbaum-contact-{normalize_name(b_name)}") raw_contacts.append({ "Name": b_name, "_filename": "Stammbaum", "_scoped_id": b_scoped_id }) z_name = (a.get("zucht") or "").strip() if z_name: z_scoped_id = generate_guid(f"stammbaum-contact-{normalize_name(z_name)}") raw_contacts.append({ "Name": z_name, "_filename": "Stammbaum", "_scoped_id": z_scoped_id }) # Residency propagation for stammbaum animals def is_clan_zucht(z): if not z: return False norm = z.lower() return "klein" in norm and "chaot" in norm and "extern" not in norm def is_external_cattery(a): """True if an animal belongs to a NAMED, non-clan cattery (e.g. 'Black Forest', 'LennyLengo', a foreign line) or the externally-acquired founder markers. Such animals were never in the breeder's own stock, so residency must NOT be propagated onto them (tickets #17/#20 — Hagrid of Black Forest). Animals with no cattery at all are name-only lineage ancestors and stay eligible for propagation.""" for z in (a.get("zucht"), a.get("zuchtCanon"), a.get("breeder")): z = (z or "").strip() if z and not is_clan_zucht(z): return True name = a.get("name", "") if name: m = re.search(r"\b(?:of|von\s+den|von\s+der|v\.\s?d\.|von)\s+(.+)$", name, re.IGNORECASE) if m: cattery = m.group(1).strip() if cattery and not is_clan_zucht(cattery): return True m = re.search(r"^(.+?'s)\s+", name, re.IGNORECASE) if m: cattery = m.group(1).strip() if cattery and not is_clan_zucht(cattery): return True return False stammbaum_resident_ids = set() for a in stammbaum_only_animals: if is_clan_zucht(a.get("zucht")) or is_clan_zucht(a.get("zuchtCanon")): stammbaum_resident_ids.add(a["id"]) # Propagate residency to parents of resident offspring — but never onto # animals from a named external cattery (they were never in this stock). for _ in range(5): for a in stammbaum_only_animals: if a["id"] in stammbaum_resident_ids: for p_ref in a.get("parentRefs", []): p_key = (normalize_name(p_ref["name"]), parse_date(p_ref["dob"])) for cand in stammbaum_only_animals: if normalize_name(cand["name"]) == p_key[0]: cand_dob = parse_date(cand["dob"]) if not p_key[1] or cand_dob == p_key[1]: if is_external_cattery(cand): continue stammbaum_resident_ids.add(cand["id"]) # Pre-index Wurfchronik litters from markdown md_litters_idx = {} # …and a date-only index of Wurfchronik litters that NAME both parents. The # Wurfchronik is authoritative: when a chart-position reconstruction picks the # WRONG parents (so the (father,mother,date) key misses) but exactly ONE # Wurfchronik litter exists for that birthdate, attach the animal to it rather # than fabricating a virtual litter with bad parents (tickets #12 Tony, # #15 Odelia, #31 Jamie — Wurfchronik-Vorrang vor chart-position). md_litters_by_date = {} def _md_parent_names(rl): f = rl.get("FatherName") or rl.get("fatherName") or rl.get("ParentMaleName") or rl.get("parentMaleName") or rl.get("_father_name") m = rl.get("MotherName") or rl.get("motherName") or rl.get("ParentFemaleName") or rl.get("parentFemaleName") or rl.get("_mother_name") if not f and not m: _note = rl.get("Notes") or rl.get("notes") or rl.get("Note") or rl.get("note") or "" _m = re.search(r"(?:Eltern|Pairing|Paarung|Paar):\s*(.+?)\s*(?:\+|\&)\s*(.+?)\s*(?:;|$)", _note, re.IGNORECASE) if _m and "/" not in _m.group(1) and "/" not in _m.group(2): f, m = _m.group(1).strip(), _m.group(2).strip() return get_normalized_gerbil_name(f), get_normalized_gerbil_name(m) for rl in raw_litters: f_name, m_name = _md_parent_names(rl) ldate = parse_date(rl.get("Date") or rl.get("date") or rl.get("DateOfBirth") or rl.get("dateOfBirth")) if f_name and m_name and ldate: key = (normalize_name(f_name), normalize_name(m_name), ldate) md_litters_idx[key] = rl md_litters_by_date.setdefault(ldate, []).append(rl) # Gender index for parent-ref selection: normalized name → 'male' | 'female' # | 'ambiguous'. Drives the gender-aware ranking in pick_parent_ref so a dated # but wrong-sex ref (e.g. female „Danielle“) cannot win the father slot over # an undated male/unknown one (e.g. „Hagrid Rubeus“). # A name's gender is "ambiguous" ONLY when BOTH male and female records exist # for it. A definite gender beats an UNKNOWN (None) duplicate — otherwise a # bare DOB-less ancestor box (gender=None) would poison a name that another # record clearly types (e.g. Dorie of Black Forest, Zadar): they would fall # back to None and lose gender-based role disambiguation (tickets #35, #33). gender_idx = {} for a in stammbaum_only_animals: g = (a.get("gender") or "").lower().strip() g = g if g in ("male", "female") else None for key in {normalize_name(a.get("name")), normalize_name(get_call_name(a.get("name") or ""))}: if not key: continue cur = gender_idx.get(key, "__unset__") if cur == "__unset__" or cur is None: gender_idx[key] = g # first value, or upgrade None → definite elif g is None or g == cur: pass # keep existing definite gender else: gender_idx[key] = "ambiguous" # genuine male vs female conflict def gender_of_name(name): v = gender_idx.get(normalize_name(name)) return v if v in ("male", "female") else None # Create virtual litters for stammbaum animals created_virtual_litters = {} for a in stammbaum_only_animals: parent_refs = a.get("parentRefs", []) child_dob_raw = a.get("dob") father_ref = pick_parent_ref(parent_refs, "father", child_dob_raw, gender_of=gender_of_name) mother_ref = pick_parent_ref(parent_refs, "mother", child_dob_raw, avoid_name=father_ref.get("name") if father_ref else None, gender_of=gender_of_name) # Record rejected parent-ref candidates (wrong sex / age-impossible / # duplicate of the other role) so this animal's gerbil history can # explain which Stammbaum positions were discarded and what won instead. pick_discards = [] pick_discards += explain_pick_rejections(parent_refs, "father", child_dob_raw, father_ref, gender_of=gender_of_name) pick_discards += explain_pick_rejections( parent_refs, "mother", child_dob_raw, mother_ref, avoid_name=father_ref.get("name") if father_ref else None, gender_of=gender_of_name) if pick_discards: a["_pick_discards"] = pick_discards a["_mapped_litter_scoped_id"] = None # A human conflict-decision override is authoritative — never let the # Wurfchronik-Vorrang heuristic below re-attach the animal to a same-date # chronicle litter and overrule the breeder's named parents. _has_decision_parent = ( (father_ref or {}).get("method") == "decision" or (mother_ref or {}).get("method") == "decision") # Wurfchronik-Vorrang: if the chart parents don't yield an exact # Wurfchronik match but EXACTLY ONE Wurfchronik litter (with named # parents) exists for this birthdate, that authoritative litter wins over # a fabricated chart-position litter (#12 Tony, #15 Odelia, #31 Jamie). dob_val_attach = parse_date(a.get("dob")) if dob_val_attach and not _has_decision_parent \ and not is_external_origin(a.get("name"), a.get("zucht"), a.get("breeder")) \ and not is_external_cattery(a): same_date = md_litters_by_date.get(dob_val_attach, []) exact_hit = None if father_ref and mother_ref: exact_key = (normalize_name(get_normalized_gerbil_name(father_ref.get("name"))), normalize_name(get_normalized_gerbil_name(mother_ref.get("name"))), dob_val_attach) exact_hit = md_litters_idx.get(exact_key) if not exact_hit and len(same_date) == 1: a["_mapped_litter_scoped_id"] = same_date[0]["_scoped_id"] if a["_mapped_litter_scoped_id"]: pass elif father_ref and mother_ref: f_name = get_normalized_gerbil_name(father_ref.get("name")) m_name = get_normalized_gerbil_name(mother_ref.get("name")) dob_val = parse_date(a.get("dob")) mapped_litter = None if dob_val: key = (normalize_name(f_name), normalize_name(m_name), dob_val) mapped_litter = md_litters_idx.get(key) if mapped_litter: a["_mapped_litter_scoped_id"] = mapped_litter["_scoped_id"] else: v_key = (normalize_name(f_name), normalize_name(m_name), dob_val or "0001-01-01") if v_key in created_virtual_litters: a["_mapped_litter_scoped_id"] = created_virtual_litters[v_key] else: l_scoped_id = generate_guid(f"virtual-litter-{v_key[0]}-{v_key[1]}-{v_key[2]}") created_virtual_litters[v_key] = l_scoped_id a["_mapped_litter_scoped_id"] = l_scoped_id # Try to link parents to actual parsed stammbaum animals. # For human override refs (conflict-decisions, method="decision") # skip this loose call-name preliminary linking and let the # global name resolver pick the exact named animal instead # (e.g. resident „Elena“, not „Elena of KK Chaos“ — #15). f_decision = (father_ref or {}).get("method") == "decision" m_decision = (mother_ref or {}).get("method") == "decision" f_scoped_id = None m_scoped_id = None f_dob = parse_date(father_ref.get("dob")) m_dob = parse_date(mother_ref.get("dob")) for p_cand in ([] if f_decision else stammbaum_only_animals): p_gender = str(p_cand.get("gender") or "").lower().strip() if p_gender in ["w", "f", "female", "weiblich"]: continue p_dob = parse_date(p_cand.get("dob")) if dob_val and p_dob and p_dob >= dob_val: continue cand_call_norm = normalize_name(get_call_name(p_cand["name"])) or "unbekannt" cand_name_norm = normalize_name(p_cand["name"]) or "unbekannt" if cand_call_norm == normalize_name(f_name) or cand_name_norm == normalize_name(f_name): if not f_dob or p_dob == f_dob: f_scoped_id = generate_guid(f"stammbaum-animal-{p_cand['id']}") break for p_cand in ([] if m_decision else stammbaum_only_animals): p_gender = str(p_cand.get("gender") or "").lower().strip() if p_gender in ["m", "male", "männlich"]: continue p_dob = parse_date(p_cand.get("dob")) if dob_val and p_dob and p_dob >= dob_val: continue cand_call_norm = normalize_name(get_call_name(p_cand["name"])) or "unbekannt" cand_name_norm = normalize_name(p_cand["name"]) or "unbekannt" if cand_call_norm == normalize_name(m_name) or cand_name_norm == normalize_name(m_name): if not m_dob or p_dob == m_dob: m_scoped_id = generate_guid(f"stammbaum-animal-{p_cand['id']}") break raw_litters.append({ "Id": l_scoped_id, "Name": f"Wurf von {f_name} + {m_name}", "Date": dob_val, "TotalBorn": None, "DeathsWithin8Weeks": None, "FatherId": f_scoped_id or generate_guid(f"stammbaum-animal-{normalize_name(f_name)}"), "MotherId": m_scoped_id or generate_guid(f"stammbaum-animal-{normalize_name(m_name)}"), "ExpectedGoHomeDate": None, "Notes": "Pedigree virtual litter", "PairingCode": None, "ExternalRef": f"virtual-{l_scoped_id}", "LitterLetter": None, "_father_name": f_name, "_mother_name": m_name, "_filename": "Stammbaum", "_scoped_id": l_scoped_id, "_scoped_father_id": f_scoped_id or generate_guid(f"stammbaum-animal-{normalize_name(f_name)}"), "_scoped_mother_id": m_scoped_id or generate_guid(f"stammbaum-animal-{normalize_name(m_name)}") }) elif _has_decision_parent and (father_ref or mother_ref): # Single KNOWN parent from a human override (the other parent is # genuinely unknown — e.g. a sibling-pairing where only the mother is # named: Danielle's mother = Ella, father = unnamed brother). # Create a one-parent virtual litter; the global name resolver fills # the known side (gender + age-plausibility pick the right same-named # animal, e.g. Ella *10.06.2019 over Ella *13.04.2023), the other # side stays null. known = father_ref or mother_ref known_role = "father" if father_ref else "mother" known_name = get_normalized_gerbil_name(known.get("name")) dob_val = parse_date(a.get("dob")) v_key = (normalize_name(known_name), known_role, dob_val or "0001-01-01") if v_key in created_virtual_litters: a["_mapped_litter_scoped_id"] = created_virtual_litters[v_key] else: l_scoped_id = generate_guid( f"virtual-litter-1p-{v_key[0]}-{v_key[1]}-{v_key[2]}") created_virtual_litters[v_key] = l_scoped_id a["_mapped_litter_scoped_id"] = l_scoped_id raw_litters.append({ "Id": l_scoped_id, "Name": f"Wurf von {known_name}", "Date": dob_val, "TotalBorn": None, "DeathsWithin8Weeks": None, "FatherId": None, "MotherId": None, "ExpectedGoHomeDate": None, "Notes": "Pedigree virtual litter (ein Elternteil bekannt)", "PairingCode": None, "ExternalRef": f"virtual-{l_scoped_id}", "LitterLetter": None, "_father_name": known_name if known_role == "father" else "", "_mother_name": known_name if known_role == "mother" else "", "_filename": "Stammbaum", "_scoped_id": l_scoped_id, "_scoped_father_id": None, "_scoped_mother_id": None, }) # 2. Resolve Contacts globally (deduplicate by normalized name) contact_by_norm_name = {} contact_id_map = {} # scoped_old_id -> global_guid for rc in raw_contacts: name_val = rc.get("Name") or rc.get("name") or rc.get("FullName") or rc.get("fullName") if not name_val: first = rc.get("FirstName") or rc.get("firstName") last = rc.get("LastName") or rc.get("lastName") if first or last: name_val = f"{first or ''} {last or ''}".strip() if not name_val: continue canon_name, should_keep = get_normalized_contact_name(name_val) scoped_id = rc["_scoped_id"] if not should_keep: if scoped_id: contact_id_map[scoped_id] = None continue norm_name = normalize_name(canon_name) rc_file = rc.get("_filename") if norm_name not in contact_by_norm_name: global_guid = generate_guid(f"contact-{norm_name}") contact_by_norm_name[norm_name] = { "Id": global_guid, "Name": canon_name, "Email": rc.get("Email") or rc.get("email"), "Phone": rc.get("Phone") or rc.get("phone"), "Address": rc.get("Address") or rc.get("address"), "Notes": rc.get("Notes") or rc.get("notes") or rc.get("Note") or rc.get("note"), # Provenance accumulators (consumed below, stripped from helper keys). "_source_files": set([rc_file]) if rc_file else set(), "_merged_count": 1, } else: gc = contact_by_norm_name[norm_name] if not gc["Email"] and (rc.get("Email") or rc.get("email")): gc["Email"] = rc.get("Email") or rc.get("email") if not gc["Phone"] and (rc.get("Phone") or rc.get("phone")): gc["Phone"] = rc.get("Phone") or rc.get("phone") if not gc["Address"] and (rc.get("Address") or rc.get("address")): gc["Address"] = rc.get("Address") or rc.get("address") if not gc["Notes"] and (rc.get("Notes") or rc.get("notes") or rc.get("Note") or rc.get("note")): gc["Notes"] = rc.get("Notes") or rc.get("notes") or rc.get("Note") or rc.get("note") if rc_file: gc["_source_files"].add(rc_file) gc["_merged_count"] += 1 if scoped_id: contact_id_map[scoped_id] = contact_by_norm_name[norm_name]["Id"] resolved_contacts = list(contact_by_norm_name.values()) print(f"Resolved to {len(resolved_contacts)} unique contacts.") # 3. Process Litters (normalize keys) resolved_litters = [] litter_id_map = {} # scoped_old_id -> new_guid litter_by_scoped_id = {} for rl in raw_litters: filename = rl.get("_filename") name_val = rl.get("Name") or rl.get("name") if not name_val: name_val = "Wurf" dob_val = parse_date(rl.get("Date") or rl.get("date") or rl.get("DateOfBirth") or rl.get("dateOfBirth")) new_guid = rl["_scoped_id"] if not new_guid: new_guid = generate_guid(f"litter-{filename}-{name_val}-{dob_val}") scoped_old_id = rl["_scoped_id"] if scoped_old_id: litter_id_map[scoped_old_id] = new_guid total_born = rl.get("TotalBorn") or rl.get("totalBorn") or rl.get("LitterSize") or rl.get("litterSize") or rl.get("size") or rl.get("totalPups") if total_born is not None: try: total_born = int(total_born) except ValueError: total_born = None deaths_8w = rl.get("DeathsWithin8Weeks") or rl.get("deathsWithin8Weeks") if deaths_8w is not None: try: deaths_8w = int(deaths_8w) except ValueError: deaths_8w = None elif total_born is not None and rl.get("survived") is not None: try: deaths_8w = total_born - int(rl.get("survived")) except ValueError: pass father_name = rl.get("FatherName") or rl.get("fatherName") or rl.get("ParentMaleName") or rl.get("parentMaleName") or rl.get("_father_name") mother_name = rl.get("MotherName") or rl.get("motherName") or rl.get("ParentFemaleName") or rl.get("parentFemaleName") or rl.get("_mother_name") # Parse parents from a free-text Wurf note „Eltern: X + Y“ when the # structured parent names/ids are missing (ticket #9 Beatrice/Q-Wurf, # #11 Silver — ~46 Wurfchronik litters carry parents only in the note). # The order is father + mother (German chart convention); the name # resolver corrects the role afterwards by gender, so a swap is safe. if not father_name and not mother_name: _note = rl.get("Notes") or rl.get("notes") or rl.get("Note") or rl.get("note") or "" _m = re.search(r"(?:Eltern|Pairing|Paarung|Paar):\s*(.+?)\s*(?:\+|\&)\s*(.+?)\s*(?:;|$)", _note, re.IGNORECASE) if _m: _p1 = _m.group(1).strip() _p2 = _m.group(2).strip() # Skip ambiguous "Lee/Dean" style alternatives (a slash = unsure). if _p1 and _p2 and "/" not in _p1 and "/" not in _p2: father_name = father_name or _p1 mother_name = mother_name or _p2 raw_ext_ref = rl.get("ExternalRef") or rl.get("externalRef") or rl.get("Id") or rl.get("id") ext_ref_scoped = f"{filename}-{raw_ext_ref}" if raw_ext_ref else None l_record = { "Id": new_guid, "Name": name_val, "Date": dob_val, "TotalBorn": total_born, "DeathsWithin8Weeks": deaths_8w, "FatherId": rl["_scoped_father_id"], "MotherId": rl["_scoped_mother_id"], "ExpectedGoHomeDate": parse_date(rl.get("ExpectedGoHomeDate") or rl.get("expectedGoHomeDate")), "Notes": rl.get("Notes") or rl.get("notes") or rl.get("Note") or rl.get("note"), "PairingCode": rl.get("PairingCode") or rl.get("pairingCode"), "ExternalRef": ext_ref_scoped, "LitterLetter": rl.get("LitterLetter") or rl.get("litterLetter"), "_father_name": father_name, "_mother_name": mother_name, "_filename": filename, # Provenance accumulators (canonical absorbs these during dedup below). "_source_files": set([filename]) if filename else set(), "_merged_count": 1, # Virtual litters are reconstructed from a Stammbaum chart, not the # Wurfchronik — flagged on the raw record's _filename == "Stammbaum". "_virtual": filename == "Stammbaum", } resolved_litters.append(l_record) litter_by_scoped_id[new_guid] = l_record print(f"Processed {len(resolved_litters)} litters.") # 3b. Deduplicate litters: same date + compatible parents → merge # This handles the "sibling pairing" case: Stammbaum shows the same parental # litter twice (once under the father branch, once under the mother branch), # generating two separate litter records with the same date but only one of # them has FatherId/MotherId resolved. litter_canonical_map = {} # old_id -> canonical_id (for dedup within this step) # Group by date for efficiency by_date = {} for l in resolved_litters: by_date.setdefault(l["Date"], []).append(l) litter_dedup_canonical = {} # old_litter_id -> canonical_litter_id deduped_litters = [] for date_val, group in by_date.items(): # Partition into compatible subsets sub_groups = [] for l in group: placed = False for sub in sub_groups: if all(litter_compatible(l, member) for member in sub): sub.append(l) placed = True break if not placed: sub_groups.append([l]) for sub in sub_groups: if len(sub) == 1: deduped_litters.append(sub[0]) litter_dedup_canonical[sub[0]["Id"]] = sub[0]["Id"] continue # Pick the canonical record: prefer the one with parents set canonical = next((l for l in sub if l.get("FatherId") or l.get("MotherId")), sub[0]) for l in sub: litter_dedup_canonical[l["Id"]] = canonical["Id"] if l is not canonical: litter_id_map[l["Id"]] = canonical["Id"] canonical["_source_files"] |= l.get("_source_files", set()) canonical["_merged_count"] += l.get("_merged_count", 1) if not l.get("_virtual"): canonical["_virtual"] = False deduped_litters.append(canonical) if len(sub) > 1: merged_names = [l["Id"] for l in sub if l is not canonical] print(f"Litter-Dedup: merged {len(sub)} same-date litters on {date_val} → {canonical['Name']} (absorbed: {', '.join(merged_names)})") n_merged = len(resolved_litters) - len(deduped_litters) if n_merged: print(f"Litter-Dedup: {n_merged} redundant litter record(s) removed.") resolved_litters = deduped_litters litter_by_scoped_id = {l["Id"]: l for l in resolved_litters} # 4. Normalize and group Gerbils # Helper to lookup litter dates for birth date estimation def get_litter_date(l_id): if l_id in litter_by_scoped_id: d = litter_by_scoped_id[l_id]["Date"] if d: return d return None all_processed_gerbils = [] for rg in raw_gerbils: filename = rg.get("_filename") name_val = get_normalized_gerbil_name(rg.get("Name") or rg.get("name") or rg.get("callName")) if not name_val: name_val = "Unbekannt" dob_val = parse_date(rg.get("DateOfBirth") or rg.get("dateOfBirth")) new_guid = rg["_scoped_id"] if not new_guid: new_guid = generate_guid(f"gerbil-{filename}-{name_val}-{dob_val}") # Gender normalization gender_val = rg.get("Gender") or rg.get("gender") or "unknown" gender_val = str(gender_val).lower().strip() if gender_val in ["m", "male", "männlich"]: gender = "male" elif gender_val in ["w", "f", "female", "weiblich"]: gender = "female" else: gender = "unknown" dod_val = parse_date(rg.get("DateOfDeath") or rg.get("dateOfDeath") or rg.get("deathDate")) gohome_val = parse_date(rg.get("GoHomeDate") or rg.get("goHomeDate") or rg.get("DateGivenAway") or rg.get("givenAwayDate") or rg.get("DateOfHandover") or rg.get("dateOfHandover") or rg.get("HandoverDate") or rg.get("DateOfSale") or rg.get("dateOfSale")) status_val = rg.get("Status") or rg.get("status") or "" status_val = str(status_val).lower().strip() status = "Breeding" if dod_val or "deceased" in status_val or "verstorben" in status_val or "tod" in status_val or "dead" in status_val: status = "Deceased" elif gohome_val or "givenaway" in status_val or "abgegeben" in status_val or "verkauft" in status_val or "sold" in status_val: status = "GivenAway" elif "forsale" in status_val or "abzugeben" in status_val: status = "ForSale" elif "pet" in status_val or "liebhaber" in status_val: status = "Pet" else: if dod_val: status = "Deceased" elif gohome_val: status = "GivenAway" notes_val = rg.get("Notes") or rg.get("notes") or rg.get("Note") or rg.get("note") existing_cod = rg.get("CauseOfDeath") or rg.get("causeOfDeath") or rg.get("DeathCause") status, dod_val, cause_of_death_val = parse_death_info(notes_val, status, dod_val, existing_cod) # Explicit resolution for Ken'ichi's cause of death if name_val == "Ken'ichi" and dob_val == "2015-03-01": cause_of_death_val = "Duftdrüsen-Tumor" color_val = rg.get("ColorVarietyId") or rg.get("colorVarietyId") or rg.get("Color") or rg.get("color") or rg.get("ColorDescription") or rg.get("colorDescription") existing_gt = rg.get("Genotype") or rg.get("genotype") color_variety_id, genotype_val = resolve_color_and_genotype( color_val, existing_gt, variety_map, variety_genotypes ) if color_val and not color_variety_id: try: uuid.UUID(str(color_val).strip()) color_variety_id = str(color_val).strip() except ValueError: print(f"Warning: Unknown color variety '{color_val}' for gerbil '{name_val}' on page {filename}") origin_cid = rg["_scoped_origin_cid"] receiver_cid = rg["_scoped_receiver_cid"] if origin_cid in contact_id_map: origin_cid = contact_id_map[origin_cid] if receiver_cid in contact_id_map: receiver_cid = contact_id_map[receiver_cid] # Wurfchronik junglings default to NON-resident (Ticket 381f7e51): being # born in the chronicle does NOT by itself make an animal her breeding # stock. Residency is recomputed authoritatively by the late isResident # sweep (parent-of-own-litter only). Only an explicit truthy IsResident # on the raw record is honoured here. is_resident = rg.get("IsResident") or rg.get("isResident") if is_resident is None: is_resident = False else: is_resident = str(is_resident).lower() == "true" traits = rg.get("CharacterTraits") or rg.get("characterTraits") or [] if not isinstance(traits, list): traits = [str(traits)] char_note = rg.get("CharacterNote") or rg.get("characterNote") is_deaf = rg.get("IsDeaf") or rg.get("isDeaf") if is_deaf is not None: is_deaf = str(is_deaf).lower() == "true" old_litter_id = rg["_scoped_litter_id"] raw_ext_ref = rg.get("ExternalRef") or rg.get("externalRef") or rg.get("Id") or rg.get("id") ext_ref_scoped = f"{filename}-{raw_ext_ref}" if raw_ext_ref else None # Estimated effective date of birth for conflict checking eff_dob_val = dob_val if not eff_dob_val and old_litter_id: # Try to get litter date if litter was resolved mapped_l_id = litter_id_map.get(old_litter_id) if mapped_l_id: eff_dob_val = get_litter_date(mapped_l_id) # Determine explicit or litter-derived birth date (None if unknown/parent) birth_date = dob_val if not birth_date and old_litter_id: mapped_l_id = litter_id_map.get(old_litter_id) if mapped_l_id: birth_date = get_litter_date(mapped_l_id) # If still no effective date, use the estimated page date if not eff_dob_val: eff_dob_val = days_to_date(file_dates[filename]) raw_breeder = rg.get("OriginBreeder") or rg.get("originBreeder") if raw_breeder: norm_b, keep_b = get_normalized_contact_name(raw_breeder) raw_breeder = norm_b if keep_b else None parent_refs = [] if old_litter_id: rl = next((l for l in raw_litters if l.get("_scoped_id") == old_litter_id), None) if rl: f_name = rl.get("FatherName") or rl.get("fatherName") or rl.get("ParentMaleName") or rl.get("parentMaleName") or rl.get("_father_name") m_name = rl.get("MotherName") or rl.get("motherName") or rl.get("ParentFemaleName") or rl.get("parentFemaleName") or rl.get("_mother_name") if f_name: parent_refs.append({"name": f_name, "roleGuess": "father"}) if m_name: parent_refs.append({"name": m_name, "roleGuess": "mother"}) all_processed_gerbils.append({ "Id": new_guid, "Name": name_val, "Gender": gender, "Status": status, "LitterId": old_litter_id, # mapped later "OriginContactId": origin_cid, "ReceiverContactId": receiver_cid, "EnclosureId": None, "ColorVarietyId": color_variety_id, "DateOfBirth": dob_val, "DateOfDeath": dod_val, "CauseOfDeath": cause_of_death_val, "GoHomeDate": gohome_val, "Genotype": genotype_val, "Notes": rg.get("Notes") or rg.get("notes") or rg.get("Note") or rg.get("note"), "ImportSource": rg.get("ImportSource") or rg.get("importSource") or filename, "ExternalRef": ext_ref_scoped, "RawImportData": rg.get("RawImportData") or rg.get("rawImportData") or json.dumps({"colorDescription": color_val if not color_variety_id else None}), "OriginBreeder": raw_breeder or ("Zucht der kleinen Chaoten" if is_resident else None), "NameSearch": normalize_name(name_val), "CharacterTraits": traits, "CharacterNote": char_note, "IsDeaf": is_deaf, "IsResident": is_resident, "parentRefs": parent_refs, "_photos": rg.get("photos", []), "_old_scoped_litter_id": old_litter_id, "_eff_dob": eff_dob_val, "_birth_date": birth_date, "_filename": filename, "_old_id": rg.get("Id") or rg.get("id") }) # Build the discard list for one stammbaum animal: rejected parent-ref # candidates (pick_parent_ref) plus a manual DOB-remap decision, if any. def _stammbaum_discards(a): discards = list(a.get("_pick_discards", [])) remap = a.get("dobRemap") if remap and remap.get("original") and remap.get("corrected"): orig = _de_date(parse_date(remap["original"])) or remap["original"] corr = _de_date(parse_date(remap["corrected"])) or remap["corrected"] if orig != corr: discards.append({ "label": PROV_FIELD_LABELS["DateOfBirth"], "value": orig, "reason": "per manueller Entscheidung korrigiert", "replacement": corr, }) return discards # Map and append stammbaum animals to all_processed_gerbils for a in stammbaum_only_animals: a_id = a["id"] name_val = get_normalized_gerbil_name(a["name"]) gender_val = str(a.get("gender") or "").lower().strip() if gender_val in ["m", "male"]: gender = "male" elif gender_val in ["w", "f", "female"]: gender = "female" else: gender = "unknown" dob_val = parse_date(a.get("dob")) dod_val = parse_date(a.get("death")) status = "Breeding" if dod_val: status = "Deceased" elif a_id not in stammbaum_resident_ids: status = "GivenAway" else: if dob_val: try: dt_dob = datetime.strptime(dob_val, "%Y-%m-%d") dt_now = datetime.now() age_years = (dt_now - dt_dob).days / 365.25 if age_years >= 6.0: status = "Deceased" except Exception: pass # Color variety mapping existing_gt = a["genotype"]["rawGenotype"] if a["genotype"]["rawGenotype"] else None color_variety_id, genotype_val = resolve_color_and_genotype( a.get("farbschlag"), existing_gt, variety_map, variety_genotypes ) if not color_variety_id: for fbv in a.get("farbschlagVariants", []): cv_id, gt_val = resolve_color_and_genotype(fbv, existing_gt, variety_map, variety_genotypes) if cv_id: color_variety_id = cv_id genotype_val = gt_val break # Contact mapping origin_cid = None b_name = (a.get("breeder") or "").strip() z_name = (a.get("zucht") or "").strip() if b_name: origin_cid = generate_guid(f"stammbaum-contact-{normalize_name(b_name)}") elif z_name: origin_cid = generate_guid(f"stammbaum-contact-{normalize_name(z_name)}") # Map to resolved global contact GUID if origin_cid in contact_id_map: origin_cid = contact_id_map[origin_cid] else: origin_cid = None scoped_id = generate_guid(f"stammbaum-animal-{a_id}") scoped_litter_id = a.get("_mapped_litter_scoped_id") is_deaf = a["genotype"].get("deaf") stammbaum_breeder = a.get("breeder") if a.get("breeder") else (a.get("zucht") if a.get("zucht") else None) if stammbaum_breeder: norm_b, keep_b = get_normalized_contact_name(stammbaum_breeder) stammbaum_breeder = norm_b if keep_b else None all_processed_gerbils.append({ "Id": scoped_id, "Name": name_val, "Gender": gender, "Status": status, "LitterId": scoped_litter_id, # mapped later in step 5 "OriginContactId": origin_cid, "ReceiverContactId": None, "EnclosureId": None, "ColorVarietyId": color_variety_id, "DateOfBirth": dob_val, "DateOfDeath": dod_val, "CauseOfDeath": "Duftdrüsen-Tumor" if name_val == "Ken'ichi" and dob_val == "2015-03-01" else None, "GoHomeDate": None, "Genotype": genotype_val, "Notes": None, "ImportSource": ", ".join(a.get("sourceFiles", [])), "ExternalRef": f"stammbaum-{a_id}", "RawImportData": json.dumps({ "rawGenotype": a["genotype"]["rawGenotype"], "unmappedTokens": a["genotype"]["unmappedTokens"], "breederText": a.get("breeder", "") }, ensure_ascii=False), "OriginBreeder": stammbaum_breeder, "NameSearch": normalize_name(name_val), "CharacterTraits": [], "CharacterNote": None, "IsDeaf": is_deaf, "IsResident": a_id in stammbaum_resident_ids, "parentRefs": a.get("parentRefs", []), "_photos": a.get("photos", []), "_conflict": bool(a.get("conflict")), "_resolved_by_decision": bool(a.get("resolvedByDecision")), "_old_scoped_litter_id": scoped_litter_id, "_eff_dob": dob_val or "2010-01-01", "_birth_date": dob_val, "_filename": a.get("sourceFiles", ["Stammbaum"])[0], "_old_id": a_id, # Rejected Stammbaum parent-ref candidates for this animal (filled by # pick_parent_ref above) — surfaced in this gerbil's discard history. "_discarded": _stammbaum_discards(a), }) # Map and append docx animals to all_processed_gerbils for idx, da in enumerate(docx_animals): name_val = get_normalized_gerbil_name(da["name"]) gender = da["gender"] dob_val = parse_date(da.get("litterDob")) dod_val = parse_date(da.get("deathDate")) gohome_val = parse_date(da.get("abgabeDate")) # Status precedence status = "Breeding" if dod_val: status = "Deceased" elif gohome_val or da.get("owner"): status = "GivenAway" # Color variety mapping color_variety_id, genotype_val = resolve_color_and_genotype( da.get("farbschlag"), None, variety_map, variety_genotypes ) # Contact mapping (buyer) receiver_cid = None o_name = (da.get("owner") or "").strip() if o_name: receiver_cid = generate_guid(f"docx-contact-{normalize_name(o_name)}") if receiver_cid in contact_id_map: receiver_cid = contact_id_map[receiver_cid] else: receiver_cid = None # Scoped ID scoped_id = generate_guid(f"docx-animal-{idx}-{normalize_name(name_val)}-{dob_val or '0001-01-01'}") # Litter ID mapping ws_code = da.get("wsCode") scoped_litter_id = docx_litter_id_map.get((ws_code, dob_val)) # Raw import details raw_import_payload = json.dumps({ "abgabeWeight": da.get("abgabeWeight", ""), "deathCause": da.get("deathCause", ""), "partnerName": da.get("partnerName", ""), "partnerDob": da.get("partnerDob", "") }, ensure_ascii=False) # Residents: if sold/given away, it's not a resident is_resident = not bool(o_name) parent_refs = [] if scoped_litter_id: dl = next((l for l in docx_litters if docx_litter_id_map.get((l["wsCode"], parse_date(l["dob"]))) == scoped_litter_id), None) if dl: if dl.get("fatherName"): parent_refs.append({"name": dl["fatherName"], "roleGuess": "father"}) if dl.get("motherName"): parent_refs.append({"name": dl["motherName"], "roleGuess": "mother"}) all_processed_gerbils.append({ "Id": scoped_id, "Name": name_val, "Gender": gender, "Status": status, "LitterId": scoped_litter_id, "OriginContactId": None, "ReceiverContactId": receiver_cid, # mapped in step 5 "EnclosureId": None, "ColorVarietyId": color_variety_id, "DateOfBirth": dob_val, "DateOfDeath": dod_val, "CauseOfDeath": da.get("deathCause"), "GoHomeDate": gohome_val, "Genotype": genotype_val, "Notes": None, "ImportSource": "Wurfchronik-Detail.docx", "ExternalRef": f"docx-{idx}-{normalize_name(name_val)}-{dob_val or '0001-01-01'}", "RawImportData": raw_import_payload, "OriginBreeder": "Zucht der kleinen Chaoten", "NameSearch": normalize_name(name_val), "CharacterTraits": [], "CharacterNote": None, "IsDeaf": None, "IsResident": is_resident, "parentRefs": parent_refs, "_photos": da.get("photos", []), "_old_scoped_litter_id": scoped_litter_id, "_eff_dob": dob_val or "2020-01-01", "_birth_date": dob_val, "_filename": "Wurfchronik-Detail.docx", "_old_id": name_val }) # ── Conflict-decision overrides at the gerbil level ─────────────────────── # extract.apply_conflict_decisions already applies genotype/farbschlag/dod/ # gender/parent overrides to STAMMBAUM animals (animals.json). Wurfchronik/ # docx animals (docx_animals.json) never pass through that path, so a gender # override on a Wurfchronik animal (e.g. „Roni“ — actually the FATHER) would # otherwise be lost. Apply gender overrides here uniformly to every processed # gerbil, materialise `addAnimals` stubs for known parents that have no own # source record (e.g. „Fumi“), and collapse explicitly-paired duplicate # records via `mergeExternalRefs` (nameless animals never auto-merge). _decisions, _add_animals, _add_litters = load_conflict_decisions() def _gerbil_call_key(g): return normalize_name(get_call_name(g.get("Name") or "")) # 1) gender overrides (match normalize(call-name) + ISO dob; empty decision # dob = name-only, for ancestors without a birthdate). _gender_overrides = {} for d in _decisions: gg = _norm_decision_gender(d.get("gender")) if not gg: continue ck = normalize_name(get_call_name(d.get("name") or "")) iso = parse_date(d.get("dob")) if d.get("dob") else "" _gender_overrides[(ck, iso or "")] = gg _gender_applied = 0 for g in all_processed_gerbils: ck = _gerbil_call_key(g) iso = g.get("DateOfBirth") or "" gg = _gender_overrides.get((ck, iso)) or _gender_overrides.get((ck, "")) if gg and g.get("Gender") != gg: g["Gender"] = gg g["_resolved_by_decision"] = True _gender_applied += 1 # Resolve a contact NAME (e.g. an addAnimals `receiver` like „Ulrike Neu“) to # its global contact id. Prefers an already-resolved contact; otherwise falls # back to the deterministic id (same scheme as contact resolution above) so a # receiver named only here still maps to the canonical id. def _resolve_contact_id_by_name(name): raw = (name or "").strip() if not raw: return None canon, keep = get_normalized_contact_name(raw) if not keep: return None norm = normalize_name(canon) existing = contact_by_norm_name.get(norm) if existing: return existing["Id"] cid = generate_guid(f"contact-{norm}") new_c = { "Id": cid, "Name": canon, "Phone": None, "Email": None, "Street": None, "City": None, "IsBreeder": False, "IsReceiver": True, "Notes": "Automatisch generiert durch Züchterin-Entscheidung (Receiver)" } contact_by_norm_name[norm] = new_c return cid # 2) addAnimals — materialise a stub for a KNOWN animal that has no own source # record, so its litter's parent link can resolve (e.g. „Fumi“). Optional # fields: gender, zucht, dob, isResident (bool), receiver (contact name → # ReceiverContactId), notes (→ Notes), farbschlag/color (best-effort → Notes). # Skip a bare stub when a record of that call-name already exists (avoid # duplicates); but a deliberately-added RICH animal (one carrying isResident/ # receiver/notes/color, i.e. genuinely new data — not just a parent-link # placeholder) is created even when the call-name collides with an unrelated # foreign animal (e.g. the new resident „Merle“ vs the existing „Merle of # Samsimar“). It gets its own deterministic id, so no real duplicate arises. _existing_call_keys = {_gerbil_call_key(g) for g in all_processed_gerbils} _RICH_FIELDS = ("isResident", "receiver", "notes", "farbschlag", "color") _stubs_added = 0 for a in _add_animals: nm = (a.get("name") or "").strip() if not nm: continue ck = normalize_name(get_call_name(nm)) is_rich = any(a.get(k) is not None for k in _RICH_FIELDS) if ck in _existing_call_keys and not is_rich: continue gender = _norm_decision_gender(a.get("gender")) or "unknown" dob_iso = parse_date(a.get("dob")) if a.get("dob") else None zucht = (a.get("zucht") or "").strip() or None breeder_disp = None if zucht: nb, keep = get_normalized_contact_name(zucht) breeder_disp = nb if keep else None # Optional rich fields. is_resident = bool(a.get("isResident")) if a.get("isResident") is not None else False receiver_cid = _resolve_contact_id_by_name(a.get("receiver")) notes_val = (a.get("notes") or "").strip() or None color_txt = (a.get("farbschlag") or a.get("color") or "").strip() or None # Best-effort colour: keep it human-readable in Notes (no genotype to map). if color_txt: color_line = f"Farbschlag: {color_txt}" notes_val = f"{notes_val} | {color_line}" if notes_val else color_line status = "GivenAway" if receiver_cid else "Breeding" stub_id = generate_guid(f"decision-animal-{normalize_name(nm)}") all_processed_gerbils.append({ "Id": stub_id, "Name": get_normalized_gerbil_name(nm), "Gender": gender, "Status": status, "LitterId": None, "OriginContactId": None, "ReceiverContactId": receiver_cid, "EnclosureId": None, "ColorVarietyId": None, "DateOfBirth": dob_iso, "DateOfDeath": None, "CauseOfDeath": None, "GoHomeDate": None, "Genotype": None, "Notes": notes_val, "ImportSource": "conflict-decisions.json", "ExternalRef": f"decision-{normalize_name(nm)}", "RawImportData": json.dumps({"addedByDecision": True}, ensure_ascii=False), "OriginBreeder": breeder_disp or ("Zucht der kleinen Chaoten" if is_resident else None), "NameSearch": normalize_name(get_normalized_gerbil_name(nm)), "CharacterTraits": [], "CharacterNote": None, "IsDeaf": None, "IsResident": is_resident, "parentRefs": [], "_photos": [], "_old_scoped_litter_id": None, "_eff_dob": dob_iso or "2015-01-01", "_birth_date": dob_iso, "_filename": "conflict-decisions.json", "_old_id": stub_id, "_resolved_by_decision": True, # Ein addAnimals-Stub mit explizitem isResident ist eine Mensch- # Entscheidung → vom späteren Residenz-Sweep NICHT anfassen lassen # (z. B. die resident-Stubs Merle/Pete aus Akanes Fremd-Wurf). "_resident_override": a.get("isResident") is not None, }) _existing_call_keys.add(ck) _stubs_added += 1 # 3) mergeExternalRefs — collapse explicitly-paired duplicate records (esp. # nameless animals that the name-based dedup keeps separate). Each pair is # [keepRefSuffix, dropRefSuffix]; the drop record is removed and its old id # mapped onto the keep record. Refs are matched as a suffix of ExternalRef # (extract animals become ExternalRef "stammbaum-"). _by_ext = {} for g in all_processed_gerbils: er = g.get("ExternalRef") if er: _by_ext.setdefault(er, []).append(g) def _find_by_ref_suffix(suffix): hits = [g for er, gs in _by_ext.items() if er and er.endswith(suffix) for g in gs] return hits _premerged_ids = {} # dropped old id -> kept id (consumed in step 5 mapping) _merge_pairs_done = 0 for d in _decisions: for pair in (d.get("mergeExternalRefs") or []): if not isinstance(pair, (list, tuple)) or len(pair) != 2: continue keep_hits = _find_by_ref_suffix(pair[0]) drop_hits = _find_by_ref_suffix(pair[1]) if not keep_hits or not drop_hits: continue keep = keep_hits[0] for drop in drop_hits: if drop is keep: continue # Fill gaps on the keeper from the dropped twin, then remove it. for fld in ("DateOfBirth", "DateOfDeath", "Genotype", "ColorVarietyId", "LitterId", "_old_scoped_litter_id", "OriginContactId"): if not keep.get(fld) and drop.get(fld): keep[fld] = drop[fld] if keep.get("Gender") in (None, "unknown") and drop.get("Gender") not in (None, "unknown"): keep["Gender"] = drop["Gender"] _premerged_ids[drop["Id"]] = keep["Id"] _merge_pairs_done += 1 if _premerged_ids: all_processed_gerbils = [g for g in all_processed_gerbils if g["Id"] not in _premerged_ids] # Apply correctDob remappings from decisions BEFORE deduplication! _dob_remap_pre_dedup = 0 for d in _decisions: correct_dob = d.get("correctDob") if correct_dob: er = (d.get("externalRef") or "").strip() nm = d.get("name") dob_iso = parse_date(correct_dob) if dob_iso: # Find matching raw gerbils for g in all_processed_gerbils: match = False g_er = g.get("ExternalRef") or "" if er and g_er == er: match = True elif er and g_er.endswith(er): match = True elif nm: g_ck = normalize_name(get_call_name(g.get("Name") or "")) d_ck = normalize_name(get_call_name(nm)) g_dob = g.get("DateOfBirth") or "" d_dob = parse_date(d.get("dob")) if d.get("dob") else "" if g_ck == d_ck and (not d_dob or g_dob == d_dob): match = True if match: g["DateOfBirth"] = dob_iso g["_birth_date"] = dob_iso g["_eff_dob"] = dob_iso _dob_remap_pre_dedup += 1 if _dob_remap_pre_dedup: print(f" Pre-Dedup DOB Remappings: {_dob_remap_pre_dedup}") print(f" Entscheidungs-Overrides: Geschlecht={_gender_applied}, " f"Stub-Tiere={_stubs_added}, ExternalRef-Merges={_merge_pairs_done}.") # Build parenting dates lookup using old scoped IDs parent_litter_dates = {} for l in resolved_litters: ld = l["Date"] if ld: for pid in [l["FatherId"], l["MotherId"]]: if pid: parent_litter_dates.setdefault(pid, []).append(ld) def _is_empty_shell(g): """A same-name record carrying no birthdate AND no own parent refs — a DOB-less Stammbaum mention (e.g. Hagrid Rubeus appearing as a bare ancestor box). Such shells must fold into the DOB-/parent-bearing record of the same name even if a stray chart placed them as a parent of an age-incompatible litter (ticket #18). Their own parent-attributions are unreliable, so the parenting-date guard must not keep them separate.""" return not g.get("_birth_date") and not (g.get("parentRefs") or []) def are_compatible(g1, g2): # Same-name empty shell ↔ real record: always merge (see _is_empty_shell). if _is_empty_shell(g1) or _is_empty_shell(g2): if g1["Gender"] == "unknown" or g2["Gender"] == "unknown" \ or g1["Gender"] == g2["Gender"]: return True # Must have same gender (or one unknown) if g1["Gender"] != "unknown" and g2["Gender"] != "unknown" and g1["Gender"] != g2["Gender"]: bd1 = g1.get("_birth_date") bd2 = g2.get("_birth_date") if not (bd1 and bd2 and bd1 == bd2): return False bd1 = g1.get("_birth_date") bd2 = g2.get("_birth_date") # New rule: if name and parents match, they are compatible regardless of DOB! p1 = g1.get("parentRefs", []) p2 = g2.get("parentRefs", []) f1 = next((p["name"] for p in p1 if p.get("roleGuess") == "father"), "") m1 = next((p["name"] for p in p1 if p.get("roleGuess") == "mother"), "") f2 = next((p["name"] for p in p2 if p.get("roleGuess") == "father"), "") m2 = next((p["name"] for p in p2 if p.get("roleGuess") == "mother"), "") parents_match = False if f1 and f2 and m1 and m2: if normalize_name(f1) == normalize_name(f2) and normalize_name(m1) == normalize_name(m2): parents_match = True if not parents_match: # If both have explicit birth dates, they must match within 30 days if bd1 and bd2: days1 = date_to_days(bd1) days2 = date_to_days(bd2) if days1 is not None and days2 is not None: if abs(days1 - days2) > 30: return False # If g1 has birth date, and g2 has parenting dates, birth date must be before parenting dates p_dates2 = parent_litter_dates.get(g2["Id"], []) if bd1: for pd in p_dates2: if pd <= bd1: # Can't have litter before or on birth date return False p_dates1 = parent_litter_dates.get(g1["Id"], []) if bd2: for pd in p_dates1: if pd <= bd2: return False return True def build_provenance(records, best_g, extra_notes=None, field_source=None, field_discards=None): """Aggregate data-provenance across every raw record merged into one resolved gerbil. Returns a JSON string (stored on the Gerbil entity as a nullable text column) so the Rennmausakte can show where the entry came from. `records` is the list of raw records that became this gerbil; `best_g` is the chosen primary record. `field_source` maps a significant field name (DateOfBirth/Genotype/…) to the raw record that supplied its final value, so the history can name the exact file for each fact.""" source_files = set() from_wurfchronik = False any_conflict = False any_decision = False for r in records: source_files |= _record_source_files(r) for f in _record_source_files(r): if "wurfchronik" in f.lower(): from_wurfchronik = True if r.get("_conflict"): any_conflict = True if r.get("_resolved_by_decision"): any_decision = True notes = [] merged_count = len(records) if merged_count > 1: notes.append(f"aus {merged_count} Datensätzen zusammengeführt") if any_decision: notes.append("per manueller Entscheidung zugeordnet") if any_conflict: notes.append("Konflikt per Entscheidung gelöst") # Parent derivation: surface the strongest parentRef method/confidence # the primary record carries (chart-position etc.). parent_method = None parent_confidence = None for ref in best_g.get("parentRefs", []) or []: if ref.get("method") and not parent_method: parent_method = ref.get("method") if ref.get("confidence") and not parent_confidence: parent_confidence = ref.get("confidence") if extra_notes: for n in extra_notes: if n and n not in notes: notes.append(n) history = _build_gerbil_history( records, best_g, field_source or {}, parent_method=parent_method, any_decision=any_decision, any_conflict=any_conflict, conflict_notes=extra_notes or [], field_discards=field_discards, parent_discards=best_g.get("_discarded"), ) return build_entity_provenance( source_files, merged_count, notes=notes, from_wurfchronik=from_wurfchronik, extra={"parentMethod": parent_method, "parentConfidence": parent_confidence}, history=history, ) # Group gerbils by name to perform deduplication gerbil_groups = {} for g in all_processed_gerbils: name_key = get_dedup_name_key(g["Name"]) if not name_key: name_key = "unbekannt" gerbil_groups.setdefault(name_key, []).append(g) resolved_gerbils = [] gerbil_id_map = {} # old_scoped_id -> final_id color_keys = set(variety_map.keys()) for name_key, group in gerbil_groups.items(): is_placeholder = ( name_key in color_keys or any(p in name_key for p in ["unbekannt", "unbenannt", "baby", "welpe", "jungtier", "unknown", "welpen"]) or len(name_key) <= 2 ) if is_placeholder: # Placeholders: do NOT merge, keep all separate for g in group: # Provenance is built in a final pass (after parent resolution), # so parent-link discards land in the right gerbil's history. g["_prov_args"] = ([g], g, None, None, None) resolved_gerbils.append(g) gerbil_id_map[g["Id"]] = g["Id"] continue # Partition group into compatible subsets sub_groups = [] for g in group: placed = False for sub in sub_groups: if all(are_compatible(g, member) for member in sub): sub.append(g) placed = True break if not placed: sub_groups.append([g]) # Merge each partition sub-group into a single gerbil for sub in sub_groups: if len(sub) == 1: g = sub[0] g["_prov_args"] = ([g], g, None, None, None) resolved_gerbils.append(g) gerbil_id_map[g["Id"]] = g["Id"] continue # Find the best primary record to merge into best_g = None best_score = -1 for g in sub: score = 0 if g["_old_scoped_litter_id"]: score += 10 if g["DateOfBirth"]: score += 5 if g["Genotype"]: score += 3 if g["ColorVarietyId"]: score += 2 if g["ImportSource"] and "stammbaum" in g["ImportSource"].lower(): score += 20 if g["Notes"] and not any(kw in g["Notes"].lower() for kw in ["parent", "mutter", "vater", "dam", "sire"]): score += 1 if len(g["Name"]) > len(name_key) + 5: # likely has clan suffix score += 15 if score > best_score: best_score = score best_g = g # Merge fields merged_notes = [] if best_g["Notes"]: merged_notes.append(best_g["Notes"]) # Carry over discard records (rejected parent-refs / dob remaps) from # every merged record so none are lost when the primary changes. merged_discards = list(best_g.get("_discarded") or []) for g in sub: if g is best_g: continue for d in (g.get("_discarded") or []): if d not in merged_discards: merged_discards.append(d) best_g["_discarded"] = merged_discards # Merge photos merged_photos = list(best_g.get("_photos", [])) # Track sources for debugging sources = [best_g["_filename"]] # Per-field file attribution: which raw record supplied each final # field value. Seed with best_g for every field it already carries; # the fill loop and voting loop update it as winners change. ATTRIB_FIELDS = ["DateOfBirth", "DateOfDeath", "Gender", "Genotype", "ColorVarietyId", "Name"] field_source = {} for fld in ATTRIB_FIELDS: v = best_g.get(fld) if v and v != "unknown": field_source[fld] = best_g for g in sub: if g == best_g: continue gerbil_id_map[g["Id"]] = best_g["Id"] sources.append(g["_filename"]) for ph in g.get("_photos", []): if ph not in merged_photos: merged_photos.append(ph) if not best_g.get("_old_scoped_litter_id") and g.get("_old_scoped_litter_id"): best_g["_old_scoped_litter_id"] = g["_old_scoped_litter_id"] if not best_g["LitterId"] and g["LitterId"]: best_g["LitterId"] = g["LitterId"] if not best_g["DateOfBirth"] and g["DateOfBirth"]: best_g["DateOfBirth"] = g["DateOfBirth"] field_source["DateOfBirth"] = g if not best_g["DateOfDeath"] and g["DateOfDeath"]: best_g["DateOfDeath"] = g["DateOfDeath"] field_source["DateOfDeath"] = g if not best_g["CauseOfDeath"] and g["CauseOfDeath"]: best_g["CauseOfDeath"] = g["CauseOfDeath"] if not best_g["GoHomeDate"] and g["GoHomeDate"]: best_g["GoHomeDate"] = g["GoHomeDate"] if not best_g["Genotype"] and g["Genotype"]: best_g["Genotype"] = g["Genotype"] field_source["Genotype"] = g if not best_g["ColorVarietyId"] and g["ColorVarietyId"]: best_g["ColorVarietyId"] = g["ColorVarietyId"] field_source["ColorVarietyId"] = g if not best_g["OriginContactId"] and g["OriginContactId"]: best_g["OriginContactId"] = g["OriginContactId"] if not best_g["ReceiverContactId"] and g["ReceiverContactId"]: best_g["ReceiverContactId"] = g["ReceiverContactId"] if g["IsResident"]: best_g["IsResident"] = True # Reconcile Gender: prefer a known gender over unknown, and prefer stammbaum over other sources if best_g["Gender"] == "unknown" and g["Gender"] != "unknown": best_g["Gender"] = g["Gender"] field_source["Gender"] = g elif best_g["Gender"] != "unknown" and g["Gender"] != "unknown" and best_g["Gender"] != g["Gender"]: if g["ImportSource"] and "stammbaum" in g["ImportSource"].lower(): if not best_g["ImportSource"] or "stammbaum" not in best_g["ImportSource"].lower(): best_g["Gender"] = g["Gender"] field_source["Gender"] = g # Status precedence: Deceased > GivenAway > Breeding/Pet if g["Status"] == "Deceased": best_g["Status"] = "Deceased" elif g["Status"] == "GivenAway" and best_g["Status"] not in ["Deceased"]: best_g["Status"] = "GivenAway" if g["Notes"] and g["Notes"] not in merged_notes: # Ignore redundant dummy notes if not any(kw in g["Notes"].lower() for kw in ["parent listed", "mutter von", "vater von", "dam of", "sire of"]): merged_notes.append(g["Notes"]) # Display formatter per field for discard/replacement lines. def _disp(field, val): if val is None or val == "" or val == "unknown": return None if field in ("DateOfBirth", "DateOfDeath"): return _de_date(val) if field == "Gender": return _de_gender(val) if field == "ColorVarietyId": return variety_id_to_name.get(val, "Farbschlag") return str(val) # Reconcile fields based on number of source files supporting them conflict_notes = [] field_discards = [] for field in ["DateOfBirth", "DateOfDeath", "Gender", "Genotype", "ColorVarietyId"]: votes = {} for g in sub: val = g.get(field) if val and val != "unknown": # count source files sources_count = len(str(g.get("ImportSource") or "").split(",")) votes[val] = votes.get(val, 0) + sources_count if votes: best_val = max(votes, key=votes.get) # If the records disagreed on a field, the merge had to pick a # winner — record that as a provenance note. if len(votes) > 1: conflict_notes.append( f"Konflikt bei {PROV_FIELD_LABELS[field]} per Mehrheitsentscheidung gelöst" ) best_g[field] = best_val # Attribute the winning value to a record that actually holds # it, so the history names the right file. winner = next((g for g in sub if g.get(field) == best_val), None) if winner is not None: field_source[field] = winner # Record each LOSING value: which file it came from, that it # was discarded as differing, and that the majority value (and # its file) was used instead. if len(votes) > 1: repl_disp = _disp(field, best_val) repl_file = _primary_file_of(winner) if winner is not None else None for lose_val in votes: if lose_val == best_val: continue loser = next((g for g in sub if g.get(field) == lose_val), None) field_discards.append({ "label": PROV_FIELD_LABELS.get(field, field), "value": _disp(field, lose_val), "file": _primary_file_of(loser) if loser is not None else None, "reason": "abweichend", "replacement": repl_disp, "repl_file": repl_file, "replacement_note": "Mehrheit", }) # Keep helper fields in sync if we changed DateOfBirth if field == "DateOfBirth": best_g["_birth_date"] = best_val best_g["_eff_dob"] = best_val or "2010-01-01" if merged_notes: best_g["Notes"] = " | ".join(merged_notes) best_g["_photos"] = merged_photos # Defer provenance to a final pass so parent-link discards (added # later by the parent-age / role-normalization passes) are included. best_g["_prov_args"] = (sub, best_g, conflict_notes, field_source, field_discards) # Print merge trace print(f"Deduplicated same-animal name '{best_g['Name']}': merged {len(sub)} entries across files: {', '.join(sources)}") resolved_gerbils.append(best_g) gerbil_id_map[best_g["Id"]] = best_g["Id"] print(f"Deduplicated to {len(resolved_gerbils)} unique gerbil records.") # ── isResident/notes/receiver-Overrides (Mensch-Entscheidung) ───────────── # NACH dem Dedup anwenden, sonst würde die Dedup-Zusammenführung (IsResident=True # falls eine Variante resident ist) sie wieder überschreiben. Auch NACH der # Residenz-Propagation (die nur auf Stammbaum-Roh-Tiere wirkt) → diese Overrides # sind das letzte Wort. Matcht primär über `externalRef` (präzise, auch für # namenlose Tiere) oder sonst über normalize(call-name)+ISO-dob (leere dob = # name-only, für Vorfahren ohne Datum). Ein Match kann zusätzlich `notes` (→ # Notes, autoritativ) und `receiver` (Kontaktname → ReceiverContactId, Status= # GivenAway) setzen — so bekommt ein bereits existierendes Tier (z. B. Akane) # seinen Lebenslauf + Abnehmer, ohne ein addAnimals-Stub zu sein. _ovr_by_extref = {} _ovr_by_namedob = {} for d in _decisions: has_ovr = (("isResident" in d) or d.get("notes") or d.get("receiver") or d.get("renameTo") or d.get("deceased") or d.get("dateOfDeath") or d.get("correctDob") or d.get("genotype") or d.get("farbschlag") or d.get("gender") or d.get("father") or d.get("mother") or d.get("originBreeder")) if not has_ovr: continue er = (d.get("externalRef") or "").strip() if er: _ovr_by_extref[er] = d nm = d.get("name") if nm is not None: ck = normalize_name(get_call_name(nm or "")) iso = parse_date(d.get("dob")) if d.get("dob") else "" _ovr_by_namedob[(ck, iso or "")] = d _isres_applied = 0 if _ovr_by_extref or _ovr_by_namedob: for g in resolved_gerbils: er = g.get("ExternalRef") or "" ck = normalize_name(get_call_name(g.get("Name") or "")) iso = g.get("DateOfBirth") or "" if er and er in _ovr_by_extref: d = _ovr_by_extref[er] elif er and any(er.endswith(k) for k in _ovr_by_extref): d = next(v for k, v in _ovr_by_extref.items() if er.endswith(k)) elif (ck, iso) in _ovr_by_namedob: d = _ovr_by_namedob[(ck, iso)] elif (ck, "") in _ovr_by_namedob: d = _ovr_by_namedob[(ck, "")] else: continue applied = False if "originBreeder" in d: val = d["originBreeder"] if g.get("OriginBreeder") != val: g["OriginBreeder"] = val applied = True if "isResident" in d: val = bool(d["isResident"]) # Ein expliziter Override (true ODER false) ist das letzte Wort und # darf vom späteren Residenz-Sweep NICHT überschrieben werden. g["_resident_override"] = True if g.get("IsResident") != val: g["IsResident"] = val # Ein Nicht-Bestandstier ist kein eigenes Zuchttier → den eigenen # Zucht-Breeder entfernen (nur, wenn er auf die eigene Zucht zeigt). if not val and g.get("OriginBreeder") == "Zucht der kleinen Chaoten": g["OriginBreeder"] = None applied = True notes_ovr = (d.get("notes") or "").strip() if notes_ovr and g.get("Notes") != notes_ovr: g["Notes"] = notes_ovr applied = True rec_name = d.get("receiver") if rec_name: rec_cid = _resolve_contact_id_by_name(rec_name) if rec_cid and g.get("ReceiverContactId") != rec_cid: g["ReceiverContactId"] = rec_cid if g.get("Status") not in ("Deceased",): g["Status"] = "GivenAway" applied = True # renameTo: korrigiert den Anzeigenamen eines bestehenden Tieres. Der # Match-Key bleibt der ursprüngliche name/externalRef (NICHT umdeuten) — # erst HIER wird der Name überschrieben. rename_to = (d.get("renameTo") or "").strip() if rename_to and g.get("Name") != rename_to: # Quell-Namen merken, damit spätere Eltern-Verknüpfungen, die noch den # alten Namen referenzieren, weiter auflösen (siehe Namens-Index unten, # Ticket e0a0c304 Kruke→Kuke: sonst verlieren Kukes Würfe die Mutter). g["_pre_rename_name"] = g.get("Name") g["Name"] = rename_to applied = True # deceased: markiert ein bestehendes Tier als verstorben, falls noch # nicht. dateOfDeath wird bewusst NICHT angefasst. if d.get("deceased") and g.get("Status") != "Deceased": g["Status"] = "Deceased" applied = True # dateOfDeath: setzt das Todesdatum (falls noch keins) + Status Deceased. # rpro3 hat keine Sterbespalte, daher gibt die Züchterin/das Ticket das Datum vor. if d.get("dateOfDeath") and not g.get("DateOfDeath"): _dod = parse_date(d["dateOfDeath"]) if _dod: g["DateOfDeath"] = _dod g["Status"] = "Deceased" applied = True if d.get("correctDob") and not g.get("DateOfBirth"): dob_iso = parse_date(d["correctDob"]) g["DateOfBirth"] = dob_iso g["_birth_date"] = dob_iso g["_eff_dob"] = dob_iso applied = True if d.get("genotype") and not g.get("Genotype"): g["Genotype"] = d["genotype"].strip() applied = True if d.get("farbschlag") and not g.get("ColorVarietyId"): cv_id = variety_map.get(d["farbschlag"].strip().lower()) if cv_id: g["ColorVarietyId"] = cv_id applied = True if d.get("gender") and g.get("Gender") in ("unknown", None): gender = d["gender"].strip().lower() gender = {"m": "male", "männlich": "male", "w": "female", "f": "female", "weiblich": "female"}.get(gender, gender) if gender in ("male", "female"): g["Gender"] = gender applied = True if d.get("father"): g["_override_father_name"] = d["father"].strip() g["_override_father_dob"] = d.get("fatherDob") applied = True if d.get("mother"): g["_override_mother_name"] = d["mother"].strip() g["_override_mother_dob"] = d.get("motherDob") applied = True if applied: _isres_applied += 1 if _isres_applied: print(f"isResident/notes/receiver/dob/geno-Overrides angewandt: {_isres_applied}") # Apply age-based death threshold (6.0 years) to all resolved gerbils dt_now = datetime.now() for g in resolved_gerbils: if g.get("Status") != "Deceased" and g.get("Status") != "GivenAway": if not g.get("DateOfDeath") and not g.get("ReceiverContactId"): dob_str = g.get("DateOfBirth") if dob_str: try: dt_dob = datetime.strptime(dob_str, "%Y-%m-%d") age_years = (dt_now - dt_dob).days / 365.25 if age_years >= 6.0: g["Status"] = "Deceased" except Exception: pass # 5. Map Gerbils to Litters for g in resolved_gerbils: old_lid = g["_old_scoped_litter_id"] l_guid = litter_id_map.get(old_lid) g["LitterId"] = l_guid if l_guid and l_guid in litter_by_scoped_id: l = litter_by_scoped_id[l_guid] l_date = l.get("Date") if l_date: if not g.get("DateOfBirth"): g["DateOfBirth"] = l_date g["_birth_date"] = l_date g["_eff_dob"] = l_date # Apply age-based death threshold (6.0 years) to all resolved gerbils (including newly backfilled ones) if g.get("Status") != "Deceased" and g.get("Status") != "GivenAway": if not g.get("DateOfDeath") and not g.get("ReceiverContactId"): dob_str = g.get("DateOfBirth") if dob_str: try: dt_dob = datetime.strptime(dob_str, "%Y-%m-%d") age_years = (dt_now - dt_dob).days / 365.25 if age_years >= 6.0: g["Status"] = "Deceased" except Exception: pass # Clean helper fields del g["_old_scoped_litter_id"] del g["_eff_dob"] if "_birth_date" in g: del g["_birth_date"] del g["_filename"] del g["_old_id"] g.pop("_conflict", None) g.pop("_resolved_by_decision", None) # ── CREATE VIRTUAL LITTERS FOR PARENT OVERRIDES ── _virtual_overrides_created = 0 for g in resolved_gerbils: f_name = g.get("_override_father_name") m_name = g.get("_override_mother_name") if (f_name or m_name) and not g.get("LitterId"): litter_id = generate_guid(f"override-virtual-litter-{g['Id']}") g["LitterId"] = litter_id resolved_litters.append({ "Id": litter_id, "Name": f"Wurf von {f_name or '—'} + {m_name or '—'}", "Date": g.get("DateOfBirth"), "TotalBorn": 1, "DeathsWithin8Weeks": 0, "FatherId": None, "MotherId": None, "Notes": f"Virtueller Wurf für {g['Name']} (über Decisions-Override angelegt).", "LitterLetter": None, "ExternalRef": f"override-virtual-litter-{g['Id']}", "_father_name": f_name or "", "_mother_name": m_name or "", "_filename": "conflict-decisions.json", "_source_files": {"conflict-decisions.json"}, "_merged_count": 1, "_virtual": True }) _virtual_overrides_created += 1 if _virtual_overrides_created: print(f"Mensch-Entscheidungs-Eltern-Overrides: {_virtual_overrides_created} virtuelle Würfe erstellt.") # Gather final valid gerbil IDs valid_gerbil_ids = {g["Id"] for g in resolved_gerbils} # Create name lookup for resolved gerbils gerbil_by_norm_name = {} for g in resolved_gerbils: keys = set() n_key = normalize_name(g["Name"]) keys.add(n_key) # Also index by call-name to resolve parents who are only listed by call-name keys.add(normalize_name(get_call_name(g["Name"]))) # …and by the connector-folding canon key so abbreviation variants match # (e.g. „BlackFire v.d. Kleinen Chaoten“ vs „… von den …“ — ticket #30). keys.add(canon_name_key(g["Name"])) # Umbenannte Tiere (renameTo) auch unter ihrem Quell-Namen indexieren, damit # Eltern-Verknüpfungen, die noch den alten Namen referenzieren, weiter auflösen. pre = g.get("_pre_rename_name") if pre: keys.add(normalize_name(pre)) keys.add(normalize_name(get_call_name(pre))) keys.add(canon_name_key(pre)) for k in keys: if k: gerbil_by_norm_name.setdefault(k, []).append(g) # Map raw Guid if present (convert if old_id mapped to new_guid) for l in resolved_litters: # First redirect any parent pointing at a record dropped by an explicit # mergeExternalRefs pre-merge onto its surviving twin (then through the # normal id map), so the two collapsed litters share the same parents. if l["FatherId"] in _premerged_ids: l["FatherId"] = _premerged_ids[l["FatherId"]] if l["MotherId"] in _premerged_ids: l["MotherId"] = _premerged_ids[l["MotherId"]] if l["FatherId"] in gerbil_id_map: l["FatherId"] = gerbil_id_map[l["FatherId"]] if l["MotherId"] in gerbil_id_map: l["MotherId"] = gerbil_id_map[l["MotherId"]] # Clean foreign keys that do not point to a valid gerbil if l["FatherId"] and l["FatherId"] not in valid_gerbil_ids: l["FatherId"] = None if l["MotherId"] and l["MotherId"] not in valid_gerbil_ids: l["MotherId"] = None # Parent Resolver (Global Name Matching) resolved_fathers = 0 resolved_mothers = 0 gerbil_by_id_final = {g["Id"]: g for g in resolved_gerbils} def _final_gender(gid): g = gerbil_by_id_final.get(gid) return g["Gender"] if g else None def _resolve_name(name, prefer_gender, litter_date): """Resolve a parent name to the best matching final gerbil. Gender is a PREFERENCE, not a hard filter: a reversed parent (e.g. a female listed in the father position, as the Stammbaum often does) still resolves to a gerbil — the role is corrected afterwards by gender. This is what previously left FatherId/MotherId null (the candidate was filtered out for having the "wrong" gender for its slot). """ if not name: return None # Try the exact normalized key, then the connector-folding canon key and # the call-name (handles v.d. ↔ von den abbreviation variants — #30). lookups = [] for k in (normalize_name(name), canon_name_key(name), normalize_name(get_call_name(name))): if k and k not in lookups: lookups.append(k) # Resolve key-by-key so an EXACT full-name match (lookups[0]) wins over a # mere call-name/canon fallback — e.g. override mother „Elena“ must pick # the resident „Elena“, not „Elena of KK Chaos“ whose call-name is also # „Elena“ (ticket #15). seen_ids = set() for k in lookups: cands = [] for c in gerbil_by_norm_name.get(k, []): if c["Id"] in seen_ids: continue seen_ids.add(c["Id"]) final_id = gerbil_id_map.get(c["Id"]) if not final_id: continue final_c = gerbil_by_id_final.get(final_id) if not final_c: continue # Parent must be age-plausible: born before the litter and within # the gerbil lifespan (skips e.g. a 2013 animal for a 2022 litter). if not parent_age_plausible(final_c["DateOfBirth"], litter_date): continue # Sterbedatum-Plausibilität: ein vor dem Wurf verstorbenes Tier kann kein # Elternteil sein (Namensvetter-Fall: die gleichnamige, noch lebende Maus ist # die echte Mutter/der echte Vater). ~40 Tage Toleranz, falls ein Elternteil # kurz vor der Geburt starb. Nur filtern, wenn beide Daten bekannt. dod = final_c.get("DateOfDeath") if dod and litter_date: try: if datetime.strptime(dod, "%Y-%m-%d") < datetime.strptime(litter_date, "%Y-%m-%d") - timedelta(days=40): continue except Exception: pass cands.append(final_c) if not cands: continue # Within a key, a candidate whose FULL name equals the lookup beats one # that only matched via call-name (resident „Elena“ > „Elena of KK # Chaos“ — #15). Stable sort keeps prior ordering otherwise. cands.sort(key=lambda c: 0 if normalize_name(c["Name"]) == k else 1) # Prefer the gender expected for this role, then unknown, then anything. for pool in ( [c for c in cands if c["Gender"] == prefer_gender], [c for c in cands if c["Gender"] == "unknown"], cands, ): if pool: return pool[0] return None # ── Fehlende Wurf-Eltern aus der xlsx-Wurfchronik (output/litters.json) füllen ── # extract/merge konsumiert litters.json bisher NICHT; viele md-only/Wurfchronik- # Würfe haben deshalb keinen Eltern-Namen. Hier NUR Lücken füllen (nie überschreiben) # und NUR bei eindeutigem Match (eindeutiges Datum, sonst Buchstabe+Datum). Die # eigentliche Verknüpfung macht danach _resolve_name (Alters-/Gender-Plausibilität) — # nicht auflösbare Namen bleiben unverknüpft (kein Falsch-Link). try: _lj_path = os.path.join(OUTPUT_DIR, "litters.json") _lj = json.load(open(_lj_path, encoding="utf-8")) if os.path.exists(_lj_path) else [] _lj = _lj if isinstance(_lj, list) else _lj.get("litters", []) except Exception: _lj = [] def _lj_iso(s): try: dd, mm, yy = (s or "").split("."); return f"{yy}-{mm}-{dd}" except Exception: return s or "" def _strip_zucht(n): return (n or "").split(" [")[0].strip() _lj_by_date = {} for _e in _lj: _lj_by_date.setdefault(_lj_iso(_e.get("date")), []).append(_e) _lj_filled = 0 _lj_mort = 0 for l in resolved_litters: cands = _lj_by_date.get(l.get("Date") or "", []) if len(cands) > 1 and l.get("LitterLetter"): narrowed = [e for e in cands if str(e.get("litterId") or "").upper() == str(l.get("LitterLetter")).upper()] if narrowed: cands = narrowed if len(cands) != 1: continue e = cands[0] # Eltern-Namen — nur Luecken (nie ueberschreiben) if not l.get("FatherId") and not l.get("_father_name") and _strip_zucht(e.get("sireName")): l["_father_name"] = _strip_zucht(e.get("sireName")); _lj_filled += 1 if not l.get("MotherId") and not l.get("_mother_name") and _strip_zucht(e.get("damName")): l["_mother_name"] = _strip_zucht(e.get("damName")); _lj_filled += 1 # Totgeburten / Fruehsterblichkeit / Wurfstaerke — nur Luecken if l.get("Stillborn") is None and e.get("stillborn") is not None: l["Stillborn"] = e.get("stillborn"); _lj_mort += 1 if l.get("DeathsWithin8Weeks") is None and e.get("diedLater") is not None: l["DeathsWithin8Weeks"] = e.get("diedLater") if l.get("TotalBorn") is None and e.get("totalBorn") is not None: l["TotalBorn"] = e.get("totalBorn") if _lj_filled or _lj_mort: print(f"litters.json ergaenzt: {_lj_filled} Eltern-Namen, {_lj_mort} Totgeburt-Angaben") for l in resolved_litters: # Pre-check: if _father_name points to a known female and _mother_name to a # known male → swap names (Stammbaum positions reversed). Helps the name # resolver pick the right same-name candidate before role normalization. f_name_pre = l.get("_father_name", "") m_name_pre = l.get("_mother_name", "") if f_name_pre and m_name_pre: f_gender = next((g["Gender"] for g in gerbil_by_norm_name.get(normalize_name(f_name_pre), []) if g["Gender"] != "unknown"), None) m_gender = next((g["Gender"] for g in gerbil_by_norm_name.get(normalize_name(m_name_pre), []) if g["Gender"] != "unknown"), None) if f_gender == "female" and m_gender == "male": l["_father_name"], l["_mother_name"] = m_name_pre, f_name_pre if l["_father_name"] and not l["FatherId"]: cand = _resolve_name(l["_father_name"], "male", l["Date"]) if cand: l["FatherId"] = cand["Id"] resolved_fathers += 1 if l["_mother_name"] and not l["MotherId"]: cand = _resolve_name(l["_mother_name"], "female", l["Date"]) if cand: l["MotherId"] = cand["Id"] resolved_mothers += 1 # Cleanup internal keys del l["_father_name"] del l["_mother_name"] del l["_filename"] # Children-of-litter lookup so a dropped parent link can be explained in the # offspring's Datenherkunft (the discard is most meaningful on the child). children_by_litter = {} for g in resolved_gerbils: lid = g.get("LitterId") if lid: children_by_litter.setdefault(lid, []).append(g) def _add_child_discard(litter, discard): """Attach a parent-link discard to every child gerbil of the litter.""" for child in children_by_litter.get(litter["Id"], []): dl = child.setdefault("_discarded", []) if discard not in dl: dl.append(discard) def _pname(pid): p = gerbil_by_id_final.get(pid) return p.get("Name") if p else None # Role normalization: assign each resolved parent to the role matching its # gender, eliminate self-pairings (same animal in both roles), and never let # impossible duplicates survive (two males / two females). This corrects # reversed Stammbaum positions including the cases the simple swap missed # (one parent of "unknown" gender, or a self-paired litter). role_fixes = 0 for l in resolved_litters: before = (l.get("FatherId"), l.get("MotherId")) father, mother = assign_parent_roles(l.get("FatherId"), l.get("MotherId"), _final_gender) if before != (father, mother): role_fixes += 1 after = {father, mother} # A parent id present before but gone after was dropped by role # normalization (self-pairing or two-of-the-same-sex). Explain it. for pid in before: if pid and pid not in after: pname = _pname(pid) if before[0] == before[1]: reason = "Selbstverpaarung — Tier kann nicht beide Elternteile sein" else: reason = "ein Wurf hat nur einen Vater und eine Mutter" _add_child_discard(l, { "text": f"Elternteil „{pname}“ verworfen — {reason}", }) l["FatherId"] = father l["MotherId"] = mother if role_fixes: print(f"Role-normalization: corrected {role_fixes} litter(s) (gender roles / self-pairings).") # Parent-age sanity check: drop any resolved parent that cannot belong to the # litter — born after the offspring, or more than a gerbil lifespan earlier. # Catches mis-resolved links the name matcher still let through (e.g. Jayjay, # *2013, wrongly attached to Solice's 2022 litter). age_drops = [] for l in resolved_litters: ldate = l.get("Date") for role in ("FatherId", "MotherId"): pid = l.get(role) if not pid: continue p = gerbil_by_id_final.get(pid) if p and not parent_age_plausible(p.get("DateOfBirth"), ldate): age_drops.append((l.get("Name"), role, p.get("Name"), p.get("DateOfBirth"), ldate)) # Explain the drop in each child's history: which parent, its DOB, # why (implausible age), and that no replacement was used. role_de = "Vater" if role == "FatherId" else "Mutter" pdob_disp = _de_date(p.get("DateOfBirth")) or "unbekannt" age_reason = "unplausibles Alter für diesen Wurf" pd = date_to_days(parse_date(p.get("DateOfBirth"))) if p.get("DateOfBirth") else None ld = date_to_days(parse_date(ldate)) if ldate else None if pd is not None and ld is not None: years = abs(ld - pd) / 365.25 if ld - pd <= 0: age_reason = "unplausibel (nicht vor dem Kind geboren)" else: age_reason = f"unplausibel ({years:.0f} Jahre älter als das Kind)" _add_child_discard(l, { "text": (f"{role_de} „{p.get('Name')}“ (*{pdob_disp}) verworfen " f"— {age_reason}; kein Ersatz"), }) l[role] = None if age_drops: print(f"Parent-age sanity check: dropped {len(age_drops)} implausible parent link(s):") for lname, role, pname, pdob, ldate in age_drops[:20]: print(f" {lname}: {role}={pname} (*{pdob}) vs litter {ldate}") print(f"Globally resolved {resolved_fathers} fathers and {resolved_mothers} mothers.") # 5b. Second-pass litter dedup: now that FatherId/MotherId are known, # merge litters that have the same date AND the same parents. # This is the core of the "sibling pairing" fix: Blue Wave and Sunny Sky # both come from Wonderman × Unique — their two separate litter records # must now become one, so their children share the same LitterId. litter_by_id_post = {l["Id"]: l for l in resolved_litters} gerbil_by_litter = {} for g in resolved_gerbils: lid = g.get("LitterId") if lid: gerbil_by_litter.setdefault(lid, []).append(g) def _litter_same_parents(l1, l2): """Strict: same date + both parents known and matching.""" if l1["Date"] != l2["Date"]: return False f1, m1 = l1.get("FatherId"), l1.get("MotherId") f2, m2 = l2.get("FatherId"), l2.get("MotherId") if not f1 or not f2 or not m1 or not m2: return False return f1 == f2 and m1 == m2 by_date2 = {} for l in resolved_litters: by_date2.setdefault(l["Date"], []).append(l) deduped2 = [] litter_remap2 = {} # old_id -> canonical_id for date_val, group in by_date2.items(): sub_groups = [] for l in group: placed = False for sub in sub_groups: if all(_litter_same_parents(l, m) for m in sub): sub.append(l) placed = True break if not placed: sub_groups.append([l]) for sub in sub_groups: # Prefer the canonical that has the most children canonical = max(sub, key=lambda l: len(gerbil_by_litter.get(l["Id"], []))) for l in sub: litter_remap2[l["Id"]] = canonical["Id"] if l is not canonical: canonical["_source_files"] |= l.get("_source_files", set()) canonical["_merged_count"] += l.get("_merged_count", 1) if not l.get("_virtual"): canonical["_virtual"] = False deduped2.append(canonical) if len(sub) > 1: siblings = [g["Name"] for l in sub for g in gerbil_by_litter.get(l["Id"], []) if l is not canonical] print(f"Sibling-Litter-Merge on {date_val}: {canonical['Name']} absorbed sibling half — children now share LitterId: {[g['Name'] for g in gerbil_by_litter.get(canonical['Id'], [])] + siblings}") # Remap LitterId in all gerbils n_remapped = 0 for g in resolved_gerbils: old_lid = g.get("LitterId") if old_lid and old_lid in litter_remap2 and litter_remap2[old_lid] != old_lid: g["LitterId"] = litter_remap2[old_lid] n_remapped += 1 n_merged2 = len(resolved_litters) - len(deduped2) if n_merged2: print(f"Sibling-Litter-Dedup: {n_merged2} additional litter record(s) merged ({n_remapped} gerbil LitterIds remapped).") resolved_litters = deduped2 litter_by_scoped_id = {l["Id"]: l for l in resolved_litters} # Final gerbil-provenance pass: now that parent links are fully resolved and # all discards (majority-vote conflicts during dedup; rejected parent-refs; # role-normalization drops; parent-age drops) are attached to each gerbil's # _discarded list, render the provenance JSON with the discard history lines. n_discards = 0 for g in resolved_gerbils: args = g.pop("_prov_args", None) if g.get("_discarded"): n_discards += len(g["_discarded"]) if args is not None: records, best_g, conflict_notes, field_source, field_discards = args g["Provenance"] = build_provenance( records, best_g, extra_notes=conflict_notes, field_source=field_source, field_discards=field_discards, ) else: g["Provenance"] = build_provenance([g], g) g.pop("_discarded", None) if n_discards: print(f"Datenherkunft: recorded {n_discards} discard line(s) across gerbils.") # Abgabevertrag-Anreicherung: Käufer als Abnehmer-Kontakte anlegen und — # konservativ — auf eindeutig passende Tiere ReceiverContactId/GoHomeDate/ # Status=GivenAway setzen (nur falls noch nicht gesetzt). Provenance- # Historie wird ergänzt. Neue Käuferkontakte landen in contact_by_norm_name # und werden danach automatisch als IsReceiver markiert. cstats, sale_contracts = enrich_from_contracts( contracts, resolved_gerbils, contact_by_norm_name, contact_id_map, exclude_decisions=_decisions) if contracts: print("Abgabeverträge: " f"{cstats['contracts']} geladen, {cstats['matched']} Tier-Treffer, " f"{cstats['ambiguous_skipped']} mehrdeutig übersprungen, " f"{cstats['no_match_skipped']} ohne Treffer.") print(" Kontakte: " f"{cstats['buyers_created']} neu, {cstats['buyers_existing']} bestehend.") print(" Gesetzt: " f"ReceiverContactId={cstats['receiver_set']}, " f"GoHomeDate={cstats['gohome_set']}, " f"Status=GivenAway={cstats['status_givenaway']}, " f"Konflikte={cstats['conflicts']}.") print(" Vertragszeilen: " f"{cstats['records_created']} angelegt " f"({cstats['records_with_animal']} mit Tier, " f"{cstats['records_with_date']} mit Originaldatum), " f"{cstats['no_buyer_skipped']} ohne Käufer übersprungen, " f"{cstats['dateless_skipped']} ohne Datum übersprungen.") # Re-materialise contacts so freshly created buyer contacts are exported. resolved_contacts = list(contact_by_norm_name.values()) # Datenherkunft for litters: which source files contributed, whether this is # a Wurfchronik litter vs a Stammbaum-reconstructed ("virtual") litter, how # many raw records merged into it, plus human-readable notes. Accumulators # (_source_files/_merged_count/_virtual) were filled during the two dedup # passes above; strip them after use. # Id → Gerbil lookup so we can inspect a litter's parents (residency) below. _gerbil_by_id = {g["Id"]: g for g in resolved_gerbils if g.get("Id")} # Per-litter flags captured here (before _virtual is popped) for the late # residency sweep: was the litter reconstructed from a chart (virtual) and # did it come from the Wurfchronik? _litter_is_virtual = {} _litter_from_wurfchronik = {} for l in resolved_litters: l_source_files = l.pop("_source_files", set()) l_merged_count = l.pop("_merged_count", 1) is_virtual = l.pop("_virtual", False) l_from_wurfchronik = any("wurfchronik" in str(f).lower() for f in l_source_files) _litter_is_virtual[l["Id"]] = is_virtual _litter_from_wurfchronik[l["Id"]] = l_from_wurfchronik l_notes = [] if is_virtual and not l_from_wurfchronik: l_notes.append("aus Stammbaum-Diagramm rekonstruiert") elif l_from_wurfchronik: l_notes.append("aus Wurfchronik") if l_merged_count > 1: l_notes.append(f"aus {l_merged_count} Datensätzen zusammengeführt") l_notes.append("Geschwister-Würfe zusammengeführt") # Chronological, file-attributed history for the litter. l_files_sorted = sorted({f for f in l_source_files if f}) l_history = [] first_file = l_files_sorted[0] if l_files_sorted else None if is_virtual and not l_from_wurfchronik: if first_file: l_history.append( f"Aus Stammbaum-Diagramm rekonstruiert ({_quote_file(first_file)})." ) else: l_history.append("Aus Stammbaum-Diagramm rekonstruiert.") elif first_file: l_history.append(f"Wurf aus Wurfchronik {_quote_file(first_file)}.") else: l_history.append("Wurf im Import gefunden.") for f in l_files_sorted[1:]: l_history.append( f"Auch in {_quote_file(f)} gefunden → Datensätze zusammengeführt." ) if l_merged_count > 1: l_history.append("Geschwister-Würfe zusammengeführt.") l["Provenance"] = build_entity_provenance( l_source_files, l_merged_count, notes=l_notes, from_wurfchronik=l_from_wurfchronik, history=l_history, ) # ShowInChronicle wird ERST NACH dem isResident-Sweep gesetzt (siehe unten), # weil das Kriterium „mind. ein Bestandstier-Elternteil" die finale Residenz # braucht — der Sweep läuft nach addLitters. # ── addLitters: manuell ergänzte Würfe injizieren ───────────────────────── # Späte Injektion NACH der finalen Gerbil-/Litter-Assemblierung, damit die # finalen Tier-Ids feststehen. Jeder Eintrag verknüpft Mutter/Vater/Kinder # über deren Namen(+dob) und setzt bei den Kindern die neue LitterId. Würfe mit # showInChronicle=false (z. B. Akanes Wurf bei Clan of Black Forest) erscheinen # nur auf der Tier-Seite, nicht in der Wurfchronik. def _find_final_gerbil(name, dob=None): if not name: return None target_ck = normalize_name(get_call_name(name)) target_full = normalize_name(name) iso = parse_date(dob) if dob else None # Match precedence (most specific first) so a deliberately-added animal # (full name „Merle“) is preferred over an unrelated foreign animal that # only shares the call-name („Merle of Samsimar“): # 1. full-name + dob 2. full-name 3. call-name + dob 4. call-name full_dob = full_only = ck_dob = ck_only = None for g in resolved_gerbils: g_full = normalize_name(g.get("Name") or "") g_ck = normalize_name(get_call_name(g.get("Name") or "")) same_dob = iso and g.get("DateOfBirth") == iso if g_full == target_full: if same_dob and full_dob is None: full_dob = g elif full_only is None: full_only = g elif g_ck == target_ck: if same_dob and ck_dob is None: ck_dob = g elif ck_only is None: ck_only = g return full_dob or full_only or ck_dob or ck_only _litters_injected = 0 for al in (_add_litters or []): al_name = (al.get("name") or "").strip() if not al_name: continue mother = _find_final_gerbil(al.get("mother"), al.get("motherDob")) father = _find_final_gerbil(al.get("father"), al.get("fatherDob")) child_gerbils = [] for cn in (al.get("children") or []): # Kind als String ODER {name, dob} — dob disambiguiert gleichnamige Tiere # (z. B. Ahnen-Elliot *2015 vs. Zuchttier-Elliot *2023). if isinstance(cn, dict): cg = _find_final_gerbil(cn.get("name"), cn.get("dob")) else: cg = _find_final_gerbil(cn) if cg: child_gerbils.append(cg) slug = normalize_name(al_name) litter_id = generate_guid(f"decision-litter-{slug}") al_date = parse_date(al["date"]) if al.get("date") else None al_notes = (al.get("notes") or "").strip() or None new_litter = { "Id": litter_id, "Name": al_name, "Date": al_date, "TotalBorn": al.get("totalBorn"), "DeathsWithin8Weeks": None, "FatherId": father["Id"] if father else None, "MotherId": mother["Id"] if mother else None, "ExpectedGoHomeDate": None, "Notes": al_notes, "PairingCode": None, "ExternalRef": f"decision-litter-{slug}", "LitterLetter": None, "ShowInChronicle": bool(al.get("showInChronicle", True)), "Provenance": build_entity_provenance( ["conflict-decisions.json"], 1, notes=["Manuell aus Ticket ergänzt"], from_wurfchronik=False, history=["Manuell aus dem Ticket der Züchterin ergänzt " "(conflict-decisions.json)."], ), } for cg in child_gerbils: cg["LitterId"] = litter_id resolved_litters.append(new_litter) _litters_injected += 1 print(f"addLitters injiziert: '{al_name}' " f"(Mutter={mother['Name'] if mother else '—'}, " f"Vater={father['Name'] if father else '—'}, " f"Kinder={len(child_gerbils)}, ShowInChronicle={new_litter['ShowInChronicle']})") if _litters_injected: print(f"addLitters: {_litters_injected} Wurf/Würfe injiziert.") # ── rpro3Pedigrees: KOMPLETTE Ahnentafel eines Tiers aus _rpro3.db ziehen ────── # Für „bitte den ganzen Stammbaum aus RennerPro" (nicht nur eine Generation). Für # jeden Root {name, dob} werden ALLE Vorfahren über die rpro3-Eltern-Zeiger # (pid_raw/mid_raw) rekursiv materialisiert: bereits vorhandene Tiere (Name+DOB) # werden WIEDERVERWENDET, fehlende als externe Ahnen neu angelegt (deterministische # Id `rpro3-anc-`, Genotyp aus rpro3-Fcode → korrekter Farbschlag, Herkunft aus # origin). Fehlende Eltern-Verknüpfungen werden als (versteckte) Ahnen-Würfe ergänzt; # Tiere mit bereits vorhandenem Geburtswurf bleiben unangetastet. try: import compare_rpro3 as _Crp _peds = [] try: with open(CONFLICT_DECISIONS_PATH, encoding="utf-8") as _pfh: _peds = (json.load(_pfh).get("rpro3Pedigrees") or []) except Exception: _peds = [] _dbp = os.path.join(os.path.dirname(os.path.abspath(__file__)), "_rpro3.db") if _peds and os.path.exists(_dbp): _R = _Crp.load_rpro3(_dbp) _A = _R["animals"] _byrid = {a["rid"]: a for a in _A} def _iso(x): return x.isoformat() if hasattr(x, "isoformat") else (x or None) def _rn(s): return normalize_name(get_call_name(s or "")) # Strenger Match für Ahnen: bei bekanntem DOB nur exakt (Name+DOB); bei # DOB-losem rpro3-Ahnen nur wiederverwenden, wenn genau EIN ebenfalls DOB-loses # App-Tier gleichen Namens existiert — sonst NEU anlegen (verhindert, dass ein # DOB-loser Ahn fälschlich an einen dat. Namensvetter gehängt wird, z. B. # Kilians Mutter „Lila" ≠ Zuchttier „Lila *2014"). _res_by_name = {} for _gg in resolved_gerbils: _res_by_name.setdefault(_rn(_gg.get("Name")), []).append(_gg) def _strict_anc_match(name, dob): cs = _res_by_name.get(_rn(name), []) if dob: return next((x for x in cs if x.get("DateOfBirth") == dob), None) dobless = [x for x in cs if not x.get("DateOfBirth")] return dobless[0] if len(dobless) == 1 else None def _find_rpro3(name, dob): cs = [a for a in _A if _rn(a["name"]) == _rn(name)] if dob: mm = next((a for a in cs if _iso(a["dob"]) == dob), None) if mm: return mm return cs[0] if len(cs) == 1 else None _pg = _pl = 0 for ped in _peds: pdob = parse_date(ped.get("dob")) if ped.get("dob") else None root = _find_rpro3(ped.get("name"), pdob) if not root: print(f"rpro3Pedigrees: Root '{ped.get('name')}' nicht in rpro3 gefunden") continue # Vorfahren sammeln nodes, links, seen, stack = {}, [], set(), [root] while stack: a = stack.pop() if not a or a["rid"] in seen: continue seen.add(a["rid"]); nodes[a["rid"]] = a f = _byrid.get(a.get("pid_raw")); m = _byrid.get(a.get("mid_raw")) if f or m: links.append((a["rid"], f["rid"] if f else None, m["rid"] if m else None)) for pp in (f, m): if pp and pp["rid"] not in seen: stack.append(pp) # rid → App-Gerbil (vorhandene wiederverwenden, sonst anlegen) rid2g = {} for rid, a in nodes.items(): ex = _strict_anc_match(a["name"], _iso(a.get("dob"))) if ex: rid2g[rid] = ex; continue gid = generate_guid(f"rpro3-anc-{rid}") gender = {"männlich": "male", "weiblich": "female"}.get( (a.get("sex") or "").lower(), "unknown") fcode = (a.get("fcode") or "").strip() or None stub = { "Id": gid, "Name": get_normalized_gerbil_name(a["name"]), "Gender": gender, "Status": "Deceased", "LitterId": None, "OriginContactId": None, "ReceiverContactId": None, "EnclosureId": None, "ColorVarietyId": None, "DateOfBirth": _iso(a.get("dob")), "DateOfDeath": None, "CauseOfDeath": None, "GoHomeDate": None, "Genotype": fcode, "Notes": None, "ImportSource": "rpro3 (_rpro3.db)", "ExternalRef": f"rpro3-anc-{rid}", "RawImportData": json.dumps({"rpro3Rid": rid}, ensure_ascii=False), "OriginBreeder": (a.get("origin") or "").strip() or None, "NameSearch": normalize_name(get_normalized_gerbil_name(a["name"])), "CharacterTraits": [], "CharacterNote": None, "IsDeaf": None, "IsResident": False, "parentRefs": [], "_photos": [], "_old_scoped_litter_id": None, "_eff_dob": _iso(a.get("dob")) or "2008-01-01", "_birth_date": _iso(a.get("dob")), "_filename": "_rpro3.db", "_old_id": gid, "Provenance": build_entity_provenance( ["_rpro3.db"], 1, notes=["Vorfahre aus RennerPro (RennmausPro III)"], from_wurfchronik=False, history=[f"Als Vorfahre aus RennerPro übernommen (rid {rid})."]), } resolved_gerbils.append(stub); rid2g[rid] = stub; _pg += 1 # Eltern-Verknüpfungen als Ahnen-Würfe (nur wo Kind noch keinen Wurf hat) for c, f, m in links: cg = rid2g.get(c) if not cg or cg.get("LitterId"): continue fg = rid2g.get(f); mg = rid2g.get(m) if not fg and not mg: continue lid = generate_guid(f"rpro3-litter-{c}") lit = { "Id": lid, "Name": f"Wurf von {fg['Name'] if fg else '?'} + {mg['Name'] if mg else '?'}", "Date": cg.get("DateOfBirth"), "TotalBorn": None, "DeathsWithin8Weeks": None, "FatherId": fg["Id"] if fg else None, "MotherId": mg["Id"] if mg else None, "ExpectedGoHomeDate": None, "Notes": None, "PairingCode": None, "ExternalRef": f"rpro3-litter-{c}", "LitterLetter": None, "ShowInChronicle": False, "Provenance": build_entity_provenance( ["_rpro3.db"], 1, notes=["Ahnen-Wurf aus RennerPro"], from_wurfchronik=False, history=["Eltern-Verknüpfung aus RennerPro (RennmausPro III)."]), } cg["LitterId"] = lid resolved_litters.append(lit); _pl += 1 if _pg or _pl: print(f"rpro3Pedigrees injiziert: {_pg} Ahnen, {_pl} Ahnen-Würfe") except Exception as _e: print(f"rpro3Pedigrees übersprungen: {_e}") # ── litterParents: Eltern eines BESTEHENDEN Wurfs erzwingen (Mensch-Entscheidung) ── # Für Fälle, in denen der Resolver einen Namensvetter oder ein verstorbenes Tier # verknüpft hat und die echte Mutter/der echte Vater eindeutig benannt ist. Match per # litterName+date (oder nur date); Vater/Mutter via _find_final_gerbil(name, dob). try: with open(CONFLICT_DECISIONS_PATH, encoding="utf-8") as _lpfh: _litter_parents = (json.load(_lpfh).get("litterParents") or []) except Exception: _litter_parents = [] _lp_applied = 0 for lp in _litter_parents: lp_name = (lp.get("litterName") or "").strip().lower() lp_date = parse_date(lp["date"]) if lp.get("date") else None if not lp_name and not lp_date: continue for l in resolved_litters: if lp_name and (l.get("Name") or "").strip().lower() != lp_name: continue if lp_date and l.get("Date") != lp_date: continue if lp.get("father"): fa = _find_final_gerbil(lp["father"], lp.get("fatherDob")) if fa: l["FatherId"] = fa["Id"] if lp.get("mother"): mo = _find_final_gerbil(lp["mother"], lp.get("motherDob")) if mo: l["MotherId"] = mo["Id"] _lp_applied += 1 if _lp_applied: print(f"litterParents-Overrides angewandt: {_lp_applied}") # ── litterOrder: Reihenfolge der Jungtiere im Wurf (Wurfchronik = Geburtsgewicht) ── # Die Züchterin gibt die korrekte Reihenfolge vor (nicht alphabetisch). Match per # litterName+date; Kinder per Wurf (LitterId) + Namensabgleich; setzt BirthOrder=1..N. try: with open(CONFLICT_DECISIONS_PATH, encoding="utf-8") as _lofh: _litter_orders = (json.load(_lofh).get("litterOrder") or []) except Exception: _litter_orders = [] _lo_applied = 0 for lo in _litter_orders: lo_name = (lo.get("litterName") or "").strip().lower() lo_date = parse_date(lo["date"]) if lo.get("date") else None order = lo.get("order") or [] if not order or (not lo_name and not lo_date): continue for l in resolved_litters: if lo_name and (l.get("Name") or "").strip().lower() != lo_name: continue if lo_date and l.get("Date") != lo_date: continue kids = [g for g in resolved_gerbils if g.get("LitterId") == l["Id"]] for idx, cname in enumerate(order, start=1): ck = normalize_name(get_call_name(cname)) for g in kids: if normalize_name(get_call_name(g.get("Name") or "")) == ck: g["BirthOrder"] = idx _lo_applied += 1 break break if _lo_applied: print(f"litterOrder: {_lo_applied} Jungtier-Reihenfolgen gesetzt") # ── litterChildren: autoritative Kinder-Whitelist eines Wurfs (Mensch-Entscheidung) ── # Für Fälle, in denen der Wurf-Merge fremde Tiere (z. B. Abnehmer-Sammelseiten der # Wurfchronik) in einen datierten Wurf gezogen hat (Tickets 4b9f49fb M-Wurf, 8d259edf Gale). # Match per date + `matchChild` (ein aktuell zugeordnetes Kind, disambiguiert gleich-datierte # Würfe). Alle Kinder, deren Ruf-Name NICHT in `keep` steht, werden aus dem Wurf gelöst # (LitterId=None) — sie verschwinden aus dem Wurf, bleiben aber als Tier erhalten. Optional # `renameTo` setzt den kanonischen Wurf-Namen (z. B. 'Ungeklärt' -> 'M-Wurf'). # Optional `add` [Name | {name, dob}] hängt ein Jungtier an DIESEN Wurf (Gegenstück zu `keep`): # nötig, wenn die Chart-Positions-Heuristik ein Jungtier in einen falschen (virtuellen) # Stammbaum-Wurf gezogen hat, obwohl die Wurfchronik es einem echten Wurf zuordnet # (Ticket 36a3fcde/88389f8e: Akane gehört in den Z21-Wurf Roni × Fumi). Der vorherige Wurf # wird entfernt, wenn er dadurch ein kinderloser VIRTUELLER Wurf wird — sonst blieben # Phantom-Würfe in der Wurfchronik stehen. try: with open(CONFLICT_DECISIONS_PATH, encoding="utf-8") as _lcfh: _litter_children = (json.load(_lcfh).get("litterChildren") or []) except Exception: _litter_children = [] def _drop_if_empty_virtual(litter_id): """Entfernt einen VIRTUELLEN Wurf, der kein Jungtier mehr hat (Phantom-Wurf aus der Chart-Positions-Heuristik). Echte Wurfchronik-Würfe bleiben immer erhalten — die haben legitim keine namentlich erfassten Jungtiere.""" if not litter_id: return 0 if any(g.get("LitterId") == litter_id for g in resolved_gerbils): return 0 for i, cand in enumerate(resolved_litters): if cand["Id"] != litter_id: continue ext = cand.get("ExternalRef") or "" if cand.get("_virtual") or "-virtual-" in ext or ext.startswith("override-virtual-litter"): del resolved_litters[i] return 1 return 0 return 0 _lc_removed = 0 _lc_added = 0 _lc_dropped = 0 for lc in _litter_children: lc_date = parse_date(lc["date"]) if lc.get("date") else None match_child = normalize_name(get_call_name(lc.get("matchChild") or "")) keep = {normalize_name(get_call_name(n)) for n in (lc.get("keep") or [])} add = lc.get("add") or [] if not keep and not add: continue for l in resolved_litters: if lc_date and l.get("Date") != lc_date: continue kids = [g for g in resolved_gerbils if g.get("LitterId") == l["Id"]] names = {normalize_name(get_call_name(g.get("Name") or "")) for g in kids} if match_child and match_child not in names: continue # falscher, nur zufällig gleich-datierter Wurf if keep: for g in kids: if normalize_name(get_call_name(g.get("Name") or "")) not in keep: g["LitterId"] = None _lc_removed += 1 for entry in add: a_name = entry.get("name") if isinstance(entry, dict) else entry a_dob = entry.get("dob") if isinstance(entry, dict) else None child = _find_final_gerbil(a_name, a_dob) if not child or child.get("LitterId") == l["Id"]: continue prev_litter_id = child.get("LitterId") child["LitterId"] = l["Id"] _lc_added += 1 _lc_dropped += _drop_if_empty_virtual(prev_litter_id) if lc.get("renameTo"): l["Name"] = lc["renameTo"] break if _lc_removed: print(f"litterChildren: {_lc_removed} fälschlich zugeordnete Jungtiere aus Würfen gelöst") if _lc_added: print(f"litterChildren: {_lc_added} Jungtier(e) dem korrekten Wurf zugeordnet " f"({_lc_dropped} leerer Phantom-Wurf/Würfe entfernt)") # ── isResident-Sweep (Ticket 381f7e51, freigegeben) ─────────────────────── # Entscheidung der Züchterin: „Nimm wirklich ALLE Tiere aus dem Bestand heraus, # MIT AUSNAHME der, die bereits ein ELTERNTIER sind." Datengetrieben + re-ingest- # stabil neu berechnet, GANZ AM ENDE (nach finaler Eltern-/Wurf-Verknüpfung + # addLitters), damit ein Tier erst dann als Elternteil erkennbar ist. # # Regel — IsResident=true GENAU DANN, wenn EINES gilt: # (1) ein EXPLIZITER conflict-decisions-Override (_resident_override) hat # isResident gesetzt (true ODER false) → gewinnt IMMER, der Sweep fasst # solche Tiere NICHT an (z. B. Echo/Elia=true, Akane/Naémi/Bentley=false). # (2) das Tier ist Mutter/Vater EINES IHRER EIGENEN Würfe. Ein „eigener Wurf" # ist ein Wurf, der KEINE reine rekonstruierte Ahnen-Verpaarung ist: # (a) nicht virtuell (echter Wurfchronik-/docx-Wurf) ODER # (b) aus der Wurfchronik ODER # (c) hat clan-benannte Nachzucht („von den Kleinen Chaoten") ODER # (d) hat einen per Override residenten Elternteil ODER # (e) ist ein addLitters-Wurf mit ShowInChronicle=true. # Reine Fremd-Ahnen-Paarungen (z. B. Antares of Ulmer Strolche × …) zählen # NICHT → externe Ahnen werden NICHT fälschlich resident. # SONST: IsResident=false. _children_by_litter = {} for g in resolved_gerbils: lid = g.get("LitterId") if lid: _children_by_litter.setdefault(lid, []).append(g) # Explizit per Override resident gesetzte Tiere (für „eigener Wurf"-Kriterium d). _override_resident_ids = { g["Id"] for g in resolved_gerbils if g.get("_resident_override") and g.get("IsResident") } def _is_own_litter(l): lid = l.get("Id") if not _litter_is_virtual.get(lid, True): return True # (a) echter Wurf if _litter_from_wurfchronik.get(lid, False): return True # (b) Wurfchronik for ch in _children_by_litter.get(lid, []): # (c) Clan-Nachzucht if is_clan_zucht(ch.get("Name")): return True for pid in (l.get("FatherId"), l.get("MotherId")): # (d) resid. Elternteil if pid in _override_resident_ids: return True if l.get("ExternalRef", "").startswith("decision-litter-"): # (e) addLitters return bool(l.get("ShowInChronicle")) return False _own_parent_ids = set() for l in resolved_litters: if _is_own_litter(l): for pid in (l.get("FatherId"), l.get("MotherId")): if pid: _own_parent_ids.add(pid) def is_placeholder_name(name): if not name: return True n = name.lower().strip() n = re.sub(r'^(?:rv\s+|gv\s+|v\.\s*privat\s+|privat\s+)', '', n) n = re.sub(r'\s+privat.*', '', n) norm = "".join(c for c in n if c.isalnum()) if not norm: return True if norm in ["unbekannt", "unknown", "name", "jungtier", "jungtiere"]: return True # Check if name is a color variety name color_keys = set(variety_map.keys()) for ck in color_keys: ck_norm = "".join(c for c in ck if c.isalnum()) if norm == ck_norm: return True return False _sweep_to_false = 0 _sweep_to_true = 0 for g in resolved_gerbils: if g.get("_resident_override"): continue # (1) Override gewinnt should = g["Id"] in _own_parent_ids # (2) Elternteil eigenen Wurfs if should and is_placeholder_name(g.get("Name")): should = False # Platzhalter werden nie automatisch resident if g.get("IsResident") != should: if should: _sweep_to_true += 1 else: _sweep_to_false += 1 if g.get("OriginBreeder") == "Zucht der kleinen Chaoten": g["OriginBreeder"] = None g["IsResident"] = should print(f"isResident-Sweep: {_sweep_to_true} → resident, {_sweep_to_false} → nicht-resident " f"(Elternteile eigener Würfe: {len(_own_parent_ids)}).") # ShowInChronicle (Ticket ea41257a) — JETZT, mit der FINALEN Residenz. Die # Wurfchronik zeigt nur die eigenen, dokumentierten Würfe. ZIELGENAU ausblenden: # ShowInChronicle=False NUR wenn der Wurf (a) virtuell rekonstruiert ist UND (b) # NICHT aus der Wurfchronik stammt UND (c) keiner der beiden bekannten Eltern ein # Bestandstier ist. So bleiben ihre echten (evtl. nur im Stammbaum stehenden) # Würfe mit mind. einem Bestandstier-Elternteil sichtbar; nur reine Fremd-Ahnen- # Paarungen verschwinden. addLitters-Würfe behalten ihren expliziten Wert. _gerbil_by_id_final2 = {g["Id"]: g for g in resolved_gerbils if g.get("Id")} for l in resolved_litters: if l.get("ExternalRef", "").startswith("decision-litter-"): continue # addLitters: expliziter ShowInChronicle bleibt lid = l.get("Id") is_virtual = _litter_is_virtual.get(lid, False) l_from_wurfchronik = _litter_from_wurfchronik.get(lid, False) _parents = [_gerbil_by_id_final2.get(l.get("FatherId")), _gerbil_by_id_final2.get(l.get("MotherId"))] _any_resident_parent = any(bool(p.get("IsResident")) for p in _parents if p) _mother = _gerbil_by_id_final2.get(l.get("MotherId")) _mother_is_external = False if _mother: _mother_is_external = is_external_cattery(_mother) and not _mother.get("IsResident") _pure_ancestor_pairing = ( is_virtual and not l_from_wurfchronik and (not _any_resident_parent or _mother_is_external) ) l["ShowInChronicle"] = not _pure_ancestor_pairing # Hilfsfelder entfernen (nicht in den Payload exportieren). for g in resolved_gerbils: g.pop("_resident_override", None) # Set IsBreeder and IsReceiver flags on contacts breeder_ids = {g["OriginContactId"] for g in resolved_gerbils if g.get("OriginContactId")} receiver_ids = {g["ReceiverContactId"] for g in resolved_gerbils if g.get("ReceiverContactId")} for c in resolved_contacts: c_id = c["Id"] is_breeder = c_id in breeder_ids is_receiver = c_id in receiver_ids if not is_breeder and not is_receiver: is_receiver = True c["IsBreeder"] = is_breeder c["IsReceiver"] = is_receiver # Datenherkunft: where this (deduplicated) contact came from, plus the # role we inferred. Accumulator keys (_source_files/_merged_count) were # filled during the contact dedup above; strip them after use. c_source_files = c.pop("_source_files", set()) c_merged_count = c.pop("_merged_count", 1) c_notes = [] if c_merged_count > 1: c_notes.append(f"aus {c_merged_count} Datensätzen zusammengeführt") if is_breeder: c_notes.append("als Züchter erkannt") if is_receiver: c_notes.append("als Abnehmer erkannt") # Chronological, file-attributed history for the contact. c_files_sorted = sorted({f for f in c_source_files if f}) c_history = [] role_word = "Züchter" if is_breeder else "Abnehmer" if c_files_sorted: c_history.append(f"In {_quote_file(c_files_sorted[0])} als {role_word} erkannt.") for f in c_files_sorted[1:]: c_history.append( f"Auch in {_quote_file(f)} gefunden → Datensätze zusammengeführt." ) else: c_history.append(f"Im Import als {role_word} erkannt.") if is_breeder and is_receiver: c_history.append("Sowohl als Züchter als auch als Abnehmer geführt.") c["Provenance"] = build_entity_provenance( c_source_files, c_merged_count, notes=c_notes, history=c_history ) # Set and map gerbilPhotos resolved_photos = [] for g in resolved_gerbils: for idx, photo_rel in enumerate(g.get("_photos", [])): photo_guid = generate_guid(f"photo-{photo_rel}") ext = os.path.splitext(photo_rel)[1] or ".jpeg" fn_guid = photo_guid.replace("-", "") resolved_photos.append({ "Id": photo_guid, "GerbilId": g["Id"], "FileName": f"{fn_guid}{ext}", "SortOrder": idx, "_source_path": photo_rel }) # ── suppressExternalRefs: bestätigte Phantom-/Dubletten-Records entfernen ── # Vom Owner bestätigte OCR-Fehllesungen/Dubletten (z. B. 'Muga' = 'Nduga', Ticket # a9c0d0f9): Gerbil + ggf. zugehöriger (kinderloser) Phantom-Wurf werden per ExternalRef # final entfernt; verwaiste Referenzen werden defensiv genullt. _suppress = load_suppress_refs() if _suppress: removed_g = {g["Id"] for g in resolved_gerbils if g.get("ExternalRef") in _suppress} removed_l = {l["Id"] for l in resolved_litters if l.get("ExternalRef") in _suppress} resolved_gerbils = [g for g in resolved_gerbils if g.get("ExternalRef") not in _suppress] resolved_litters = [l for l in resolved_litters if l.get("ExternalRef") not in _suppress] for l in resolved_litters: if l.get("MotherId") in removed_g: l["MotherId"] = None if l.get("FatherId") in removed_g: l["FatherId"] = None for g in resolved_gerbils: if g.get("LitterId") in removed_l: g["LitterId"] = None resolved_photos = [p for p in resolved_photos if p.get("GerbilId") not in removed_g] print(f"suppressExternalRefs: {len(removed_g)} Gerbil(s) + {len(removed_l)} Wurf/Würfe entfernt.") # Herkunft-Normalisierung (Ticket Bodo aaee068b): Die eigene Zucht wurde in # Quelldaten teils als „Clan (der) kleine(n) Chaoten" geführt. Die Züchterin will # überall „Zucht der Kleinen Chaoten" (Groß-K). Finaler Anzeige-Sweep NACH aller # Logik (die interne Sentinel-Form „Zucht der kleinen Chaoten" wird oben verglichen, # daher erst hier vereinheitlichen). _origin_fixed = 0 for g in resolved_gerbils: ob = g.get("OriginBreeder") if not ob: continue low = ob.lower() if ("chaoten" in low and ("clan" in low or "zucht" in low)): if ob != "Zucht der Kleinen Chaoten": g["OriginBreeder"] = "Zucht der Kleinen Chaoten" _origin_fixed += 1 if _origin_fixed: print(f"Herkunft normalisiert auf 'Zucht der Kleinen Chaoten': {_origin_fixed} Tier(e)") # 6. Save final output JSON payload os.makedirs(OUTPUT_DIR, exist_ok=True) payload = { "contacts": resolved_contacts, "litters": resolved_litters, "gerbils": resolved_gerbils, "gerbilPhotos": resolved_photos, "saleContracts": sale_contracts, } with open(OUTPUT_FILE, "w", encoding="utf-8") as f: json.dump(payload, f, ensure_ascii=False, indent=2) print(f"Successfully wrote database-ready import file to: {OUTPUT_FILE}") print(f" Contacts: {len(payload['contacts'])}") print(f" Litters: {len(payload['litters'])}") print(f" Gerbils: {len(payload['gerbils'])}") print(f" Photos: {len(payload['gerbilPhotos'])}") print(f" SaleContracts: {len(payload['saleContracts'])}") if __name__ == "__main__": main()