import os import json import re import uuid import sys from datetime import datetime # Prevent encoding crashes on Windows consoles when printing unicode if sys.platform.startswith('win'): try: sys.stdout.reconfigure(encoding='utf-8') except Exception: pass # Paths DIR_PATH = r"C:\Users\gulum\dev\Wurfchronik_Bilder" SEEDS_PATH = r"C:\Users\gulum\dev\GerbilManager\gerbil-manager-web\src\genetics\colorVarietySeed.backend.json" OUTPUT_DIR = r"C:\Users\gulum\dev\GerbilManager\tools\import\output" OUTPUT_FILE = os.path.join(OUTPUT_DIR, "resolved_import.json") def generate_guid(key_str): """Generate a stable UUID string based on a key.""" return str(uuid.uuid5(uuid.NAMESPACE_DNS, key_str)) def build_entity_provenance(source_files, merged_record_count, notes=None, from_wurfchronik=None, extra=None, history=None): """Generic data-provenance JSON builder shared by gerbils, contacts and litters. Mirrors the GerbilProvenance frontend contract: { sourceFiles, mergedRecordCount, fromWurfchronik, notes, history, ... } `source_files` is any iterable of filenames; `from_wurfchronik` is auto- derived from the filenames when left as None. `extra` may carry entity- specific keys (e.g. parentMethod/parentConfidence for gerbils). `history` is an ordered list of human-readable German lines that read like a chronological log of where each fact came from (the primary content shown in the Datenherkunft dialog). Returns a JSON string (stored on the nullable Provenance text column).""" files = sorted({f for f in source_files if f}) if from_wurfchronik is None: from_wurfchronik = any("wurfchronik" in f.lower() for f in files) prov = { "sourceFiles": files, "mergedRecordCount": merged_record_count, "fromWurfchronik": bool(from_wurfchronik), "notes": list(notes or []), "history": list(history or []), } if extra: for k, v in extra.items(): if v is not None: prov[k] = v return json.dumps(prov, ensure_ascii=False) def _quote_file(fname): """German typographic quotes around a source filename for history lines.""" return f"„{fname}“" # Human-readable German labels for the significant fields we attribute to files. PROV_FIELD_LABELS = { "DateOfBirth": "Geburtsdatum", "DateOfDeath": "Sterbedatum", "Gender": "Geschlecht", "Genotype": "Genotyp", "ColorVarietyId": "Farbschlag", "Name": "Name", } def _primary_file_of(record): """The single most representative source file of one raw record. Prefers an explicit Stammbaum/Wurfchronik filename from ImportSource, else the record's _filename. Used to attribute a field value to a concrete file in the history log.""" fn = record.get("_filename") if fn: return fn imp = record.get("ImportSource") if imp: first = str(imp).split(",")[0].strip() if first: return first return None def _record_source_files(g): """All distinct source files a single raw gerbil record drew from. ImportSource is either a comma-joined Stammbaum file list, a single Wurfchronik filename, or the per-litter filename; _filename is the primary file. We union both so nothing is lost.""" files = set() imp = g.get("ImportSource") if imp: for part in str(imp).split(","): part = part.strip() if part: files.add(part) fn = g.get("_filename") if fn: files.add(fn) return files def to_valid_guid(val): if not val: return None val_str = str(val).strip() try: uuid.UUID(val_str) return val_str except ValueError: return generate_guid(val_str) def normalize_name(name): if not name: return "" return "".join(c for c in name.lower() if c.isalnum()) def get_normalized_contact_name(name): if not name: return "", False n = "".join(c for c in name.lower() if c.isalnum()) to_discard = { "chevroletcamarooftopolino", "cindyvprivatzuchtgießen", "cindyvprivatzuchtgiessen", "inuschofblackforest", "stichvonprivatzuchtgießen", "stichvonprivatzuchtgiessen", "tarzanofsamsimar", "hanserennersposeidon", "livingforcesidefix", "livingforcesnando" } if n in to_discard: return "", False norm_map = { "alessandrab": "Alessandra Bartoletti", "alexsandrab": "Alessandra Bartoletti", "andreafey": "Andrea und Stefanie Fey", "andreastefaniefey": "Andrea und Stefanie Fey", "anettw": "Annett Wernecke", "angelachristoph": "Angela und Ekki Christoph", "angelaekkichristoph": "Angela und Ekki Christoph", "angie": "Angie Reichert-Cambeis", "angierc": "Angie Reichert-Cambeis", "anjasch": "Anja Schaumburg", "ankeb": "Anke Busch", "ankeksch": "Anke Koppenhöfer", "annaq": "Anna Quark", "annikab": "Annika Balser", "annkathrind": "Ann-kathrin Dressler", "arturaleksandrapolamundrzynski": "Artur, Aleksandra und Pola Mundrzynski", "arturaleksandraundpolamundrzynski": "Artur, Aleksandra und Pola Mundrzynski", "astridr": "Astrid Rohmann", "astridroh": "Astrid Rohmann", "babarastehle": "Barbara Stehle", "barbarar": "Barbara Riegler", "biancab": "Bianca Bernhardt", "biancam": "Bianca Mayer + Alex Wachten", "biancamayeralexw": "Bianca Mayer + Alex Wachten", "birgitk": "Birgit Knopp", "birgitkropp": "Birgit Knopp", "birgittabüskens": "Birgitta Müller-Büskens", "birgittamüllerbüskens": "Brigitta Müller-Büskens", "blackforestgv": "Black Forest", "brigittast": "Brigitta Struve", "buntefellnasen": "bunten Fellnasen", "buntenfellnase": "bunten Fellnasen", "buntenfellnasen": "bunten Fellnasen", "carolal": "Carola und Hannah Lerch", "carolalerch": "Carola und Hannah Lerch", "chalfontstud": "Chalfont Stud, Freddy Braun", "christianefuchs": "Christiane Fuchs + Michael Höhnert", "christiek": "Christiane Kiessling", "christinam": "Christine Monika Mai", "christinek": "Christiane Kiessling", "christiner": "Christina Rudloff", "claudialöhr": "Claudia Löhr und Michael Koob", "claudialöhrmichaelkoob": "Claudia Löhr und Michael Koob", "colorfulfurrygerbils": "Colorful Furry Gerbils", "colourfulfurrygerbils": "Colorful Furry Gerbils", "danielakasberger": "Daniela Käsberger", "dörthe": "Dörthe Petzmann", "emelyhehlhorn": "Emely Mehlhorn", "fabianb": "Fabian Büdel", "floriang": "Florian Gries", "hannahraths": "Hannah Ruths", "hanserenners": "Hanse Renner", "haraldmariareitzlennemann": "Harald und Maria Reitz-Lennemann", "haraldundmariarl": "Harald und Maria Reitz-Lennemann", "heike": "Heike + Heiko Scheurich", "heikebarklay": "Heike Barklage", "heikoloos": "Heiko Loos und Annette Becker", "heikoloosannettebecker": "Heiko Loos und Annette Becker", "inahübner": "Ina Hübner + Nicolai Thome", "inapeines": "Ina Pleines", "janab": "Jana Beikert", "jasiminweber": "Jasmin Weber und Ronny Reichelt", "jasmin": "Jasmin Weber", "jasminweber": "Jasmin Weber und Ronny Reichelt", "jasminweberromyreichelt": "Jasmin Weber und Ronny Reichelt", "jasminweberronnyreichelt": "Jasmin Weber und Ronny Reichelt", "joystaatberg": "Joy Staalberg", "juliaa": "Julia Ast", "juttam": "Jutta Metz", "karina": "Karina Luft", "karinborgotti": "Karin Borsotti", "katjal": "Katja Leffeck", "kimw": "Kim Waldschmidt", "kkchaos": "KK-Chaos", "kkchaosofkkchaos": "KK-Chaos", "kleinechaoten": "Zucht der kleinen Chaoten", "kleinenchaoten": "Zucht der kleinen Chaoten", "kriegernmitkrallen": "Krieger mit Krallen", "lennylengo": "Lenny Lengo", "lisa": "Lisa und Marcel Kunz", "lisach": "Lisa und Lydia Christ", "lisachrist": "Lisa und Lydia Christ", "lisalydiachrist": "Lisa und Lydia Christ", "littlefellows": "little fellows", "littlerunners": "little runners", "maikef": "Maike Franz", "maintalerpz": "Privatzucht Maintal", "marinau": "Marina Unger", "martinaandreaswestfeld": "Martina und Andreas Westfeld", "martinaw": "Martina und Andreas Westfeld", "martinawestfeld": "Martina und Andreas Westfeld", "meikesch": "Meike und Heiko Sch.", "melanief": "Melanie Fey und Thorben Meier", "melaniefey": "Melanie Fey und Thorben Meier", "melaniefeythorbenmeier": "Melanie Fey und Thorben Meier", "michaeldavidschmitz": "Michael und David Schmitz", "michaelschmitz": "Michael und David Schmitz", "nataliereitz": "Nathalie Reitz", "natascham": "Natascha Marienfeld", "nicolefischler": "Nicole Tischler", "nicolel": "Nicole Lannert", "nicolen": "Nicole Nuzzo", "nicolet": "Nicole Tischler", "nicolew": "Nicole Webersinn", "nicost": "Nico Stamm", "nielsh": "Nils H.", "noelstrahbach": "Noel Strohbach", "nora": "Nora Rudersdorf geb. Holzbach", "norah": "Nora Holzbach", "norarh": "Nora Rudersdorf", "norarudersdorf": "Nora Rudersdorf geb. Holzbach", "oflennylengo": "Lenny Lengo", "pascale": "Pascale Diefenbach", "pascaledießenbach": "Pascale Diefenbach", "patriciap": "Patricia Petry", "paul": "Paul W.", "paula": "Paula Gabler", "paulag": "Paula Gabler", "privatzuchtmuecke": "Privatzucht Mücke", "privatzuchtmücke": "Privatzucht Mücke", "pzmaintal": "Privatzucht Maintal", "pzmuecke": "Privatzucht Mücke", "pzmücke": "Privatzucht Mücke", "pzseligenstadt": "Privatzucht Seligenstadt", "ramonag": "Ramona Gömpel", "ranialößler": "Rania Löffler", "rominahubrich": "Romina Milde / Hubrich", "rominamilde": "Romina Milde / Hubrich", "romonag": "Ramona Gömpel", "ronialöffler": "Rania Löffler", "sandrak": "Sandra Kubas", "sarahlöser": "Sarah Löwer", "sarahlöuer": "Sarah Löwer", "sarahz": "Sarah Zitzer", "saskiaw": "Saskia Wucher", "schlossmaus": "Schlossmäuse", "schlossmäuse": "Schlossmäuse", "schlossmäusen": "Schlossmäuse", "sebastiansch": "Sebastian Schmitt", "serinaberg": "Selina Berg", "silkewolfganghintze": "Silke und Wolfgang Hintze", "smilla": "Smilla H.", "steffi": "Steffi K.", "susanneninat": "Susanne + Nina Thomas", "susannet": "Susanne + Torsten Saum", "theresavalenca": "Theresia Valenca (Kopp)", "theresiavalenca": "Theresia Valenca (Kopp)", "timpf": "Tim Pfeiffer und Michelle Mai", "timpfeifermichellemai": "Tim Pfeiffer und Michelle Mai", "timpfeiffer": "Tim Pfeiffer und Michelle Mai", "timpfeiffermichellemai": "Tim Pfeiffer und Michelle Mai", "tonoböckenseld": "Tono Böckenfeld", "topol": "Topolino", "ulrichmüller": "Ulrich + Angela Müller", "ulrikec": "Ulrike Cordes", "ulriker": "Ulrike Ruppel", "ulrikesch": "Ulrike Schulz", "ulriket": "Ulrike Treml", "vanessab": "Vanessa Becker", "vanessag": "Vanessa Groll", "veragreywitz": "Vera Geywitz", "vonprivatzuchtmaintal": "Privatzucht Maintal", "wolfgangfaus": "Wolfgang Faust", "yvonne": "Yvonne Obendorfer", "zuchtderkleinenchaoten": "Zucht der kleinen Chaoten", } if n in norm_map: return norm_map[n], True return name, True def get_normalized_gerbil_name(name): if not name: return "" n = name.strip() norm_key = "".join(c for c in n.lower() if c.isalnum()) gerbil_norm_map = { "samgenshellyvdbuntenfellnasen": "Sammy gen. Shelly von den bunten Fellnasen", "sammygenshellyvdbuntenfellnasen": "Sammy gen. Shelly von den bunten Fellnasen", "schmidt": "Schmidti", "sheila": "Sheila of Ulmer Strolche", "shinichi": "Shinichi von PZ Mücke", "silenosgenadonis": "Silenos gen. Adonis von den Kleinen Chaoten", "silenosgenadonisvdkleinenchaoten": "Silenos gen. Adonis von den Kleinen Chaoten", "silver": "Silver von den kleinen Chaoten", "snoops": "Snoopsi", "sokrates": "Sokrates von den Kleinen Chaoten", "splash": "Slash", "teiko": "Teiko von den kleinen Chaoten", "trixy": "Trixxy von den Kleinen Chaoten", "unique": "Unique of Wild Dreams", } if norm_key in gerbil_norm_map: return gerbil_norm_map[norm_key] return n def get_call_name(name): if not name: return "" n = name.strip() n = re.sub(r'\[[^\]]+\]$', '', n).strip() n = re.split(r'\s+(?:of|von\s+den|von\s+der|v\.\s?d\.|von)\s+', n, flags=re.IGNORECASE)[0].strip() return n def get_dedup_name_key(name): if not name: return "" n = name.lower().strip() # Strip common suffixes/prefixes and parentheticals n = re.sub(r'\b(?:von\s+den|v\.?\s*d\.?|v\.?o\.?)\s+(?:kleinen\s+)?chaoten\b', '', n) n = re.sub(r'\bvon\s+der\s+schlossm\w+\b', '', n) n = re.sub(r'\bvon\s+der\s+bunten\s+fellnasen?\b', '', n) n = re.sub(r'\bof\s+black\s+forest\b', '', n) n = re.sub(r'\b\(?rv\)?\b', '', n) n = re.sub(r'\bgen\.\s+\w+', '', n) # Clean up parentheses or brackets n = re.sub(r'\(.*?\)', '', n) n = re.sub(r'\[.*?\]', '', n) return "".join(c for c in n if c.isalnum()) def clean_color_name(c_desc): if not c_desc: return "", False # Lowercase and strip c = c_desc.lower().strip() # Check for Schecke is_schecke = False if re.search(r'\bsp\b|\bsp\d|\bsp[*(²³]|\bspotted|\bschecke|[- ]sp\b|\w+sp\b', c): is_schecke = True # Standardize parentheticals for schimmel c = c.replace("(schimmel)", "schimmel") c = c.replace("(schimmel-hell)", "schimmel hell") c = c.replace("(schimmel hell)", "schimmel hell") # Strip schecke/sp markers and any trailing text starting from sp c = re.sub(r'\([- ]?sp(otted)?\)', '', c) # handles (-sp) c = re.sub(r'[- ]?sp(otted)?\b.*', '', c) # handles -sp(k), -sp*(k), -sp, etc. c = re.sub(r'[- ]?schecke\b.*', '', c) c = re.sub(r'[- ]?spotted\b.*', '', c) # Strip any other parentheticals, symbols, or trailing stars/numbers c = re.sub(r'\s*\(.*?\)\s*', ' ', c) c = re.sub(r'[²³*]', '', c) c = c.strip() # Mapping table for abbreviations, typos, and specific combinations mapping = { "antra": "anthrazit", "anthra": "anthrazit", "ankazit": "anthrazit", "antrazit": "anthrazit", "pew": "rew", "bew": "hermelin", "harder": "marder", "kohli": "kohlfuchs", "aligerfuchs": "algierfuchs", "algiesfuchs": "algierfuchs", "schw": "schwarz", "sa": "silberagouti", "a": "agouti", "cp-sa": "cp-silberagouti", "cp-a": "cp-agouti", "cp-a-hell": "cp-agouti-hell", "cp-aisa": "cp-agouti", "cp-a / rcp-sa": "cp-agouti", } if c in mapping: c = mapping[c] return c, is_schecke def resolve_color_and_genotype(color_val, existing_genotype, variety_map, variety_genotypes): if not color_val: return None, existing_genotype color_str = str(color_val).strip() clean_name, is_schecke = clean_color_name(color_str) # Match color in variety_map color_variety_id = None if clean_name in variety_map: color_variety_id = variety_map[clean_name] else: for seed_name, seed_id in variety_map.items(): if seed_name in clean_name or clean_name in seed_name: color_variety_id = seed_id break # Update genotype if it's a Schecke genotype = existing_genotype if is_schecke: if genotype: if "spsp" in genotype: genotype = genotype.replace("spsp", "Spsp") elif "Spsp" not in genotype and "Sp" not in genotype: genotype = f"{genotype} Spsp".strip() else: canonical = variety_genotypes.get(color_variety_id) if canonical: if "spsp" in canonical: genotype = canonical.replace("spsp", "Spsp") else: genotype = f"{canonical} Spsp".strip() else: genotype = "Spsp" return color_variety_id, genotype def parse_date(d): """Convert variations of date formats to YYYY-MM-DD.""" if not d or d == "0001-01-01": return None d = str(d).strip() # Try YYYY-MM-DD if re.match(r"^\d{4}-\d{2}-\d{2}$", d): parts = d.split("-") year = int(parts[0]) if 1900 < year < 2000: year += 100 d = f"{year}-{parts[1]}-{parts[2]}" elif year == 1900: return None return d # Try DD.MM.YYYY or D.M.YY match = re.match(r"^(\d{1,2})\.(\d{1,2})\.(\d{2,4})$", d) if match: day = int(match.group(1)) month = int(match.group(2)) year = int(match.group(3)) if year < 100: year += 2000 elif 1900 < year < 2000: year += 100 elif year == 1900: return None try: return datetime(year, month, day).strftime("%Y-%m-%d") except ValueError: pass # Try ISO timestamp try: dt = datetime.fromisoformat(d.replace("Z", "+00:00")) year = dt.year if 1900 < year < 2000: dt = dt.replace(year=year + 100) return dt.strftime("%Y-%m-%d") elif year == 1900: return None return dt.strftime("%Y-%m-%d") except ValueError: pass return None def date_to_days(dt_str): if not dt_str: return None try: return (datetime.strptime(dt_str, "%Y-%m-%d") - datetime(2000, 1, 1)).days except ValueError: return None def days_to_date(days): import datetime as dt return (dt.datetime(2000, 1, 1) + dt.timedelta(days=int(days))).strftime("%Y-%m-%d") def parse_death_info(notes, status, existing_dod, existing_cod): if not notes: return status, existing_dod, existing_cod has_death_indicator = '†' in notes or 'verstorben' in notes.lower() or 'gestorben' in notes.lower() or 'todesdatum' in notes.lower() or '/+' in notes if '+' in notes: if re.search(r'\+\s*(?:LE|AS|Unbekannt|gestorben|verstorben|tod)\b', notes, re.I) or re.search(r'\+\s*\d{1,2}\.\d{1,2}\.\d{2,4}', notes) or '/+' in notes: has_death_indicator = True resolved_status = status if has_death_indicator: resolved_status = "Deceased" dod = existing_dod cod = existing_cod # Look for date near death indicator found_date = None for m in re.finditer(r'([+†]\s*(?:LE|AS|Unbekannt|[a-zA-ZäöüÄÖÜß0-9()/ +,;.:-]{1,100}?)?\s*)(\d{1,2}\.\d{1,2}\.\d{2,4})', notes, re.I): marker_text = m.group(0) if '†' in marker_text or re.search(r'\+\s*(?:LE|AS|Unbekannt|gestorben|verstorben|tod|\d)', marker_text, re.I): found_date = parse_date(m.group(2)) if found_date: break if not found_date: m_death_marker = re.search(r'(†\s*)(\d{1,2}\.\d{1,2}\.\d{2,4})', notes) if m_death_marker: found_date = parse_date(m_death_marker.group(2)) if found_date and not dod: dod = found_date if not cod: for m in re.finditer(r'([+†])\s*([a-zA-ZäöüÄÖÜß0-9()/ +,;.:-]{1,100}?)\s*\d{1,2}\.\d{1,2}\.\d{2,4}', notes, re.I): indicator = m.group(1) cod_candidate = m.group(2).strip() if indicator == '+' and not re.search(r'\b(?:LE|AS|Unbekannt|gestorben|verstorben|tod)\b', cod_candidate, re.I): continue if cod_candidate: cod = cod_candidate break if not cod: m_cod2 = re.search(r'[+†]\s*\d{1,2}\.\d{1,2}\.\d{2,4}\s*([a-zA-ZäöüÄÖÜß0-9()/ +,;.:-]{1,100})', notes, re.I) if m_cod2: cod_candidate = m_cod2.group(1).strip() if cod_candidate: cod = cod_candidate else: m_cod3 = re.search(r'[+†]\s*(LE|AS|Unbekannt)\b', notes, re.I) if m_cod3: cod = m_cod3.group(1).strip() if cod: cod_lower = cod.lower().strip() if cod_lower in ("le", "le (lungenentzündung)", "lungenentzündung", "lungenentzündung (lungenentzündung)"): cod = "Lungenentzündung" elif cod_lower in ("as", "altersschwäche", "altenschwäche", "alter"): cod = "Altersschwäche" elif cod_lower in ("unbekannt", "unklar"): cod = "Unbekannt" else: # Replace abbreviations with full names (case-insensitive) cod = re.sub(r'\bLE\b', 'Lungenentzündung', cod, flags=re.I) cod = re.sub(r'\bAS\b', 'Altersschwäche', cod, flags=re.I) cod = cod.replace('+', ' + ') # Clean up multiple spaces cod = re.sub(r'\s+', ' ', cod).strip(' ,;.-') # Re-check after replacement cod_lower = cod.lower().strip() if cod_lower in ("le", "le (lungenentzündung)", "lungenentzündung", "lungenentzündung (lungenentzündung)"): cod = "Lungenentzündung" elif cod_lower in ("as", "altersschwäche", "altenschwäche", "alter"): cod = "Altersschwäche" elif cod_lower in ("unbekannt", "unklar"): cod = "Unbekannt" return resolved_status, dod, cod # ── Litter dedup & parent-role helpers (pure, unit-tested in test_merge_resolve.py) ── def _norm_pname(s): return normalize_name(s) if s else "" def names_no_conflict(l1, l2): """Parent names don't contradict (equal per role, or one side empty).""" f1, f2 = _norm_pname(l1.get("_father_name")), _norm_pname(l2.get("_father_name")) m1, m2 = _norm_pname(l1.get("_mother_name")), _norm_pname(l2.get("_mother_name")) f_ok = (not f1) or (not f2) or (f1 == f2) m_ok = (not m1) or (not m2) or (m1 == m2) return f_ok and m_ok def names_overlap(l1, l2): """At least one role has a non-empty matching name (positive evidence).""" f1, f2 = _norm_pname(l1.get("_father_name")), _norm_pname(l2.get("_father_name")) m1, m2 = _norm_pname(l1.get("_mother_name")), _norm_pname(l2.get("_mother_name")) return bool((f1 and f1 == f2) or (m1 and m1 == m2)) def litter_compatible(l1, l2): """Two litter records describe the same litter: same date and compatible parents. - Both sides have both parents → must match exactly. - Asymmetric (one side resolved, the other not) → merge only if names don't contradict; for dateless litters require a POSITIVE name match (a shared null date is no evidence), so unrelated nameless stubs stay separate. - Neither side has parents → never blind-merge. """ if l1["Date"] != l2["Date"]: return False f1, m1 = l1.get("FatherId"), l1.get("MotherId") f2, m2 = l2.get("FatherId"), l2.get("MotherId") if f1 and f2 and m1 and m2: return f1 == f2 and m1 == m2 asymmetric = (bool(f1 or m1) and not (f2 or m2)) or (bool(f2 or m2) and not (f1 or m1)) if asymmetric: if not names_no_conflict(l1, l2): return False if l1["Date"] is None: return names_overlap(l1, l2) return True return False def assign_parent_roles(father_id, mother_id, gender_of): """Assign two resolved parent IDs to father/mother roles by gender. Drops self-pairing duplicates (same animal in both roles) and never returns two same-role parents. `gender_of` maps an id to 'male'|'female'|'unknown'|None. Returns (father_id, mother_id). """ ids = [] for gid in (father_id, mother_id): if gid and gid not in ids: ids.append(gid) males = [g for g in ids if gender_of(g) == "male"] females = [g for g in ids if gender_of(g) == "female"] unknowns = [g for g in ids if gender_of(g) == "unknown"] father = males[0] if males else (unknowns.pop(0) if unknowns else None) mother = females[0] if females else (unknowns.pop(0) if unknowns else None) return father, mother # A gerbil lives at most ~6 years, so a parent can be at most ~6 years older than # its offspring (and must be born before it). Links outside this window are # impossible — e.g. a 2013 animal resolved onto a 2022 litter (Jayjay → Solice). MAX_PARENT_AGE_DAYS = 6 * 366 def parent_age_plausible(parent_dob, litter_date): """Could a parent born `parent_dob` have offspring born on `litter_date`? Requires birth strictly before the litter and within the gerbil lifespan. Unknown/unparseable dates return True (cannot disprove). Accepts any date format parse_date understands. """ pd = date_to_days(parse_date(parent_dob)) if parent_dob else None ld = date_to_days(parse_date(litter_date)) if litter_date else None if pd is None or ld is None: return True return 0 < (ld - pd) <= MAX_PARENT_AGE_DAYS def pick_parent_ref(parent_refs, role, child_dob, avoid_name=None, gender_of=None): """Choose the best parent ref for a role from possibly-conflicting chart refs. A Stammbaum lists an animal at several positions, so its parentRefs can carry contradictory guesses (the first one is not necessarily right). Rank candidates (lower = better): 0 right/unknown gender for the role, age-plausible dated ref 1 right/unknown gender, no DOB (usable, but a plausible dated ref wins) 2 right/unknown gender, dated but age-impossible 3 resolved gender is clearly WRONG for the role (e.g. a female father) 4 would duplicate the animal chosen for the other role Gender is decisive over DOB: a dated female ref must not win the father slot over an undated male/unknown one. `gender_of(name)` returns 'male'/'female' or None (unknown/ambiguous → not penalised). Returns the chosen ref or None. """ role_refs = [p for p in parent_refs if p.get("roleGuess") == role] if not role_refs: return None avoid = normalize_name(avoid_name) if avoid_name else None expected = "male" if role == "father" else "female" def rank(p): if avoid is not None and normalize_name(p.get("name")) == avoid: return 4 # would duplicate the other parent role g = gender_of(p.get("name")) if gender_of else None if g in ("male", "female") and g != expected: return 3 # wrong sex for this role dob = p.get("dob") if not dob: return 1 return 0 if parent_age_plausible(dob, child_dob) else 2 order = sorted(range(len(role_refs)), key=lambda i: (rank(role_refs[i]), i)) return role_refs[order[0]] def explain_pick_rejections(parent_refs, role, child_dob, chosen, avoid_name=None, gender_of=None): """Explain why other refs for `role` lost to `chosen` in pick_parent_ref. Returns a list of discard dicts (for _format_discard) — one per distinct rejected candidate name that was beaten for a clear reason (wrong sex, age- impossible, or duplicate of the other parent). Mirrors pick_parent_ref's ranking so the history can explain the same decision it made. """ role_refs = [p for p in parent_refs if p.get("roleGuess") == role] if not role_refs or chosen is None: return [] avoid = normalize_name(avoid_name) if avoid_name else None expected = "male" if role == "father" else "female" role_de = "Vaterrolle" if role == "father" else "Mutterrolle" cand_de = "Vater-Kandidat" if role == "father" else "Mutter-Kandidat" chosen_name = chosen.get("name") repl_disp = chosen_name if chosen.get("dob"): repl_disp = f"{chosen_name} (*{_de_date(parse_date(chosen.get('dob'))) or chosen.get('dob')})" out = [] seen = set() for p in role_refs: name = p.get("name") if not name or normalize_name(name) == normalize_name(chosen_name or ""): continue key = normalize_name(name) if key in seen: continue g = gender_of(name) if gender_of else None dob = p.get("dob") reason = None if avoid is not None and key == avoid: reason = "bereits als anderer Elternteil gewählt" elif g in ("male", "female") and g != expected: reason = f"falsches Geschlecht für die {role_de}" elif dob and not parent_age_plausible(dob, child_dob): reason = "unplausibles Alter für diesen Wurf" if reason is None: continue seen.add(key) disp = name if dob: disp = f"{name} (*{_de_date(parse_date(dob)) or dob})" out.append({ "label": cand_de, "value": f"„{disp}“", "reason": reason, "replacement": f"„{repl_disp}“", }) return out def _build_gerbil_history(records, best_g, field_source, parent_method=None, any_decision=False, any_conflict=False, conflict_notes=None, field_discards=None, parent_discards=None): """Build an ordered, file-attributed German history for a resolved gerbil. Reads like a chronological log: • „In ‚X.xlsx' gefunden." • „Geburtsdatum (27.03.2022) aus ‚X.xlsx'." • „Auch in ‚Y.xlsx' gefunden → Datensätze zusammengeführt." • „Genotyp aus ‚Z.xlsx'." • „Eltern über Position im Stammbaum erkannt (Quelle: ‚X.xlsx')." • „Aus Wurfchronik übernommen." `field_source` maps a field name to the raw record that supplied its final value; when present we name that record's file, otherwise we fall back to the primary record. The records are visited in a stable order (primary first, then the rest sorted by file) so the log is deterministic.""" conflict_notes = conflict_notes or [] history = [] # Order records: best_g first, then others by primary file name (stable). others = [r for r in records if r is not best_g] others.sort(key=lambda r: (_primary_file_of(r) or "")) ordered = [best_g] + others best_file = _primary_file_of(best_g) if best_file: history.append(f"In {_quote_file(best_file)} gefunden.") else: history.append("Im Import gefunden.") # Field-by-field attribution: name the file that supplied each fact. def attr_line(field, formatter): rec = field_source.get(field) or best_g val = best_g.get(field) if not val or val == "unknown": return fname = _primary_file_of(rec) label = PROV_FIELD_LABELS.get(field, field) text = formatter(label, val) if fname: history.append(f"{text} aus {_quote_file(fname)}.") else: history.append(f"{text} (Quelle unbekannt).") attr_line("DateOfBirth", lambda label, v: f"{label} ({_de_date(v)})") attr_line("Gender", lambda label, v: f"{label} ({_de_gender(v)})") attr_line("Genotype", lambda label, v: f"{label}") attr_line("ColorVarietyId", lambda label, v: f"{label}") attr_line("DateOfDeath", lambda label, v: f"{label} ({_de_date(v)})") # Merge step: every additional record that contributed. for r in others: fname = _primary_file_of(r) if fname: history.append( f"Auch in {_quote_file(fname)} gefunden → Datensätze zusammengeführt." ) else: history.append("In weiterem Datensatz gefunden → Datensätze zusammengeführt.") # Discarded field values from majority-vote conflict resolution: the LOSING # values, their file, and what won instead. for d in (field_discards or []): line = _format_discard(d) if line not in history: history.append(line) # Parent derivation. if parent_method: method_label = { "chart-position": "Position im Stammbaum", "geburtsdatum+eltern": "Geburtsdatum und Elternnamen", "nur-geburtsdatum": "Geburtsdatum", "decision": "manuelle Entscheidung", }.get(parent_method, parent_method) # The parent evidence comes from a Stammbaum chart — attribute to the # primary record's file when it is a Stammbaum. parent_file = best_file if best_file and "stammbaum" in best_file.lower() else None if parent_file: history.append( f"Eltern über {method_label} erkannt (Quelle: {_quote_file(parent_file)})." ) else: history.append(f"Eltern über {method_label} erkannt.") # Discarded parent candidates / links (pick_parent_ref rejections, role # normalization drops, parent-age sanity check). Threaded in after the merge # via the gerbil's _discarded list so they read in chronological order. for d in (parent_discards or []): line = _format_discard(d) if line not in history: history.append(line) # Manual decisions / conflicts. if any_decision: history.append("Zuordnung per manueller Entscheidung getroffen.") if any_conflict: history.append("Konflikt per Entscheidung gelöst.") for cn in conflict_notes: if cn and cn not in history: history.append(cn + ".") # Wurfchronik provenance line. if any("wurfchronik" in f.lower() for r in records for f in _record_source_files(r)): history.append("Angaben aus der Wurfchronik übernommen.") return history def _de_date(iso): """YYYY-MM-DD → DD.MM.YYYY for display; pass through anything else.""" if not iso: return iso m = re.match(r"^(\d{4})-(\d{2})-(\d{2})$", str(iso)) if m: return f"{m.group(3)}.{m.group(2)}.{m.group(1)}" return iso def _de_gender(g): return {"male": "männlich", "female": "weiblich"}.get(g, g) # Leading marker that visually flags a discard ("data was thrown away") line in # the history timeline. The frontend keys discard styling off this marker. DISCARD_MARK = "⚠ " def _format_discard(d): """Render one discard record into a German history line. A discard record is a dict describing a value/candidate the pipeline threw away. Recognised keys: • text — a fully pre-formatted line (used verbatim, marker added) • label — German field label (e.g. „Geburtsdatum“) • value — the discarded value (already display-formatted) • file — source file the discarded value came from (attributed) • reason — why it was dropped (e.g. „abweichend“, „unplausibel …“) • replacement — what was used instead (already display-formatted) • repl_file — source file the replacement came from Produces lines like: „⚠ Geburtsdatum 14.06.2015 aus ‚A.xlsx‘ verworfen — abweichend; 14.06.2017 aus ‚B.xlsx‘ verwendet (Mehrheit).“ Generic across entity types so litters/contacts can reuse it. """ if d.get("text"): return DISCARD_MARK + d["text"] parts = [] label = d.get("label") value = d.get("value") if label and value is not None: parts.append(f"{label} {value}") elif label: parts.append(str(label)) elif value is not None: parts.append(str(value)) head = " ".join(parts) if parts else "Wert" if d.get("file"): head += f" aus {_quote_file(d['file'])}" line = f"{head} verworfen" if d.get("reason"): line += f" — {d['reason']}" repl = d.get("replacement") if repl is not None and repl != "": instead = str(repl) if d.get("repl_file"): instead += f" aus {_quote_file(d['repl_file'])}" suffix = d.get("replacement_note") line += f"; {instead} verwendet" if suffix: line += f" ({suffix})" elif d.get("no_replacement"): line += "; kein Ersatz" return DISCARD_MARK + line + "." def _append_history(prov_json, line): """Append one history line to an existing Provenance JSON string and add the contract source file. Returns the updated JSON string.""" try: prov = json.loads(prov_json) if prov_json else {} except (ValueError, TypeError): prov = {} prov.setdefault("history", []) if line not in prov["history"]: prov["history"].append(line) return json.dumps(prov, ensure_ascii=False) def _parse_price(raw): """Parse a contract price string ('27,50' / '30.00' / '') → float (0.0 if empty). extract_contracts.py emits German-formatted numbers ('27,50'); accept both comma and dot decimal separators. Unparseable/empty → 0.0 (a price-less contract is still a valid contract record).""" if raw is None: return 0.0 s = str(raw).strip() if not s: return 0.0 s = s.replace(".", "").replace(",", ".") if ("," in s) else s try: return round(float(s), 2) except ValueError: return 0.0 def enrich_from_contracts(contracts, resolved_gerbils, contact_by_norm_name, contact_id_map): """Conservatively fold Abgabevertrag data into the resolved gerbils AND emit one SaleContract record per contract with a resolvable buyer. For every parsed contract we (a) ensure the buyer exists as a (receiver) contact, reusing the existing contact dedup/normalisation, and (b) try to match each animal call-name to exactly one resolved gerbil that the breeder bred ("…Chaoten"). On a confident match we set ReceiverContactId / GoHomeDate / Status=GivenAway — but only where not already set differently — and add a provenance history line. Ambiguous or absent matches are logged, never guessed. In addition we build a `sale_contracts` list (one record per contract whose buyer resolves to a contact). Each record carries a deterministic Id (from the source filename, so re-ingest is idempotent), the resolved buyer ContactId, the parsed Price, the parsed dates and the gerbil ids that matched for that contract. Contracts with NO date at all are skipped from record creation (the SaleContract.HandoverDate/ContractDate columns are non-nullable DateOnly) and counted in stats["dateless_skipped"]; contracts whose buyer cannot be resolved are counted in stats["no_buyer_skipped"]. Returns (stats, sale_contracts). Mutates resolved_gerbils + contact_by_norm_name in place. New buyer contacts are appended via contact_by_norm_name so the later IsReceiver-flag pass picks them up automatically. """ stats = { "contracts": len(contracts), "buyers_created": 0, "buyers_existing": 0, "matched": 0, "ambiguous_skipped": 0, "no_match_skipped": 0, "receiver_set": 0, "gohome_set": 0, "status_givenaway": 0, "conflicts": 0, "records_created": 0, "no_buyer_skipped": 0, "dateless_skipped": 0, "records_with_animal": 0, "records_with_date": 0, } sale_contracts = [] # The same contract filename can appear more than once in contracts.json # (the .docx is filed in several subfolders of the share). The record Id is # derived from the filename, so we must collapse those into ONE record per # Id (a duplicate PK would break ingest). Keyed by Id; animal lists are # merged and a missing date is back-filled from the duplicate. records_by_id = {} if not contracts: return stats, sale_contracts # Index breeder-owned gerbils by dedup name key. Contracts only ever sell # animals the breeder bred, so restrict candidates to her own stock to avoid # colliding with same-named foreign-bred animals. def is_own(g): ob = (g.get("OriginBreeder") or "").lower() return ("chaoten" in ob) or (g.get("OriginBreeder") is None) index = {} for g in resolved_gerbils: if not is_own(g): continue key = get_dedup_name_key(get_call_name(g.get("Name", ""))) if key: index.setdefault(key, []).append(g) def year_of(iso): return iso[:4] if iso else None # Reject buyer values that are obviously label leakage / non-person noise # (defends against any stale contracts.json produced before the parser fix). _buyer_junk = re.compile( r"^(?:stra\w+e|wohnort|fon|e-?mail|handy|festnetz|telefon|homepage|" r"zuchtname|facebook|name)\s*:?\s*$|^[\d\s/]+$", re.IGNORECASE) for c in contracts: fname = c.get("sourceFile", "") buyer_raw = (c.get("buyer") or "").strip() if buyer_raw and _buyer_junk.match(buyer_raw): buyer_raw = "" # --- buyer contact (reuse curated normalisation/dedup) --- buyer_global_id = None if buyer_raw: canon, keep = get_normalized_contact_name(buyer_raw) if keep and canon: norm = normalize_name(canon) if norm in contact_by_norm_name: gc = contact_by_norm_name[norm] gc.setdefault("_source_files", set()).add(fname) gc["_merged_count"] = gc.get("_merged_count", 1) + 1 stats["buyers_existing"] += 1 else: gid = generate_guid(f"contact-{norm}") contact_by_norm_name[norm] = { "Id": gid, "Name": canon, "Email": None, "Phone": None, "Address": None, "Notes": None, "NameSuffix": None, "_source_files": {fname}, "_merged_count": 1, } stats["buyers_created"] += 1 buyer_global_id = contact_by_norm_name[norm]["Id"] # --- match each animal call-name to a resolved gerbil --- handover = parse_date(c.get("handoverDate")) contract_date = parse_date(c.get("contractDate")) c_year = year_of(parse_date(c.get("dob"))) if c.get("dob") else None c_color = (c.get("color") or "").strip().lower() matched_gerbil_ids = [] # gerbils this contract resolved to (for the record) for call in (c.get("animals") or []): key = get_dedup_name_key(get_call_name(call)) if not key: continue cands = index.get(key, []) if not cands: stats["no_match_skipped"] += 1 continue # Corroborate when more than one candidate shares the call-name. chosen = None if len(cands) == 1: chosen = cands[0] else: scored = [] for g in cands: score = 0 g_year = year_of(g.get("DateOfBirth")) if c_year and g_year and c_year == g_year: score += 2 if c_color and g.get("ColorVarietyId"): # color match is corroboration; we don't have the name # here, so only DOB drives disambiguation strongly. pass scored.append((score, g)) scored.sort(key=lambda t: t[0], reverse=True) if scored[0][0] >= 2 and (len(scored) == 1 or scored[0][0] > scored[1][0]): chosen = scored[0][1] else: stats["ambiguous_skipped"] += 1 continue stats["matched"] += 1 if chosen.get("Id") and chosen["Id"] not in matched_gerbil_ids: matched_gerbil_ids.append(chosen["Id"]) # --- set receiver, only if not already set differently --- if buyer_global_id: cur = chosen.get("ReceiverContactId") if not cur: chosen["ReceiverContactId"] = buyer_global_id stats["receiver_set"] += 1 chosen["Provenance"] = _append_history( chosen.get("Provenance"), f"Abgabe an „{buyer_raw}“ aus Vertrag {_quote_file(fname)} übernommen.") elif cur != buyer_global_id: stats["conflicts"] += 1 chosen["Provenance"] = _append_history( chosen.get("Provenance"), f"Vertrag {_quote_file(fname)} nennt anderen Abnehmer „{buyer_raw}“ " f"— bestehende Zuordnung beibehalten.") # --- set go-home date, only if empty --- if handover and not chosen.get("GoHomeDate"): chosen["GoHomeDate"] = handover stats["gohome_set"] += 1 chosen["Provenance"] = _append_history( chosen.get("Provenance"), f"Abgabedatum {handover} aus Vertrag {_quote_file(fname)} übernommen.") # --- status: derive GivenAway if we set a receiver and it isn't # already a stronger state (Deceased). --- if chosen.get("ReceiverContactId") and chosen.get("Status") not in ( "Deceased", "GivenAway"): chosen["Status"] = "GivenAway" stats["status_givenaway"] += 1 # --- emit a SaleContract record for this contract --------------------- # Only contracts with a resolvable buyer become records (the row needs a # ContactId). A record with zero matched animals is still kept — better # to show the contract than to drop it. if not buyer_global_id: stats["no_buyer_skipped"] += 1 continue # HandoverDate/ContractDate are non-nullable DateOnly in the DB. Fall # back from one to the other; if BOTH are missing, skip the record # (we do not invent dates) and count it. h = handover or contract_date cd = contract_date or handover if not h: # implies cd is also None stats["dateless_skipped"] += 1 continue rec_id = generate_guid(f"contract-{fname}") existing = records_by_id.get(rec_id) if existing is None: records_by_id[rec_id] = { "Id": rec_id, "ContactId": buyer_global_id, "Price": _parse_price(c.get("price")), "HandoverDate": h, "ContractDate": cd, "FileName": fname, "Animals": list(matched_gerbil_ids), } else: # Same filename seen again — merge animal matches; back-fill price. for gid in matched_gerbil_ids: if gid not in existing["Animals"]: existing["Animals"].append(gid) if not existing["Price"]: existing["Price"] = _parse_price(c.get("price")) sale_contracts = list(records_by_id.values()) stats["records_created"] = len(sale_contracts) stats["records_with_animal"] = sum(1 for r in sale_contracts if r["Animals"]) stats["records_with_date"] = sum(1 for r in sale_contracts if r["HandoverDate"]) return stats, sale_contracts def main(): print("Loading color variety seeds...") variety_map = {} variety_genotypes = {} # Load from C# ApplicationContext.cs catalog for stable database GUIDs (index + 1) here = os.path.dirname(os.path.abspath(__file__)) app_context_path = os.path.abspath(os.path.join(here, "../../GerbilManagerWebAPI/ApplicationContext.cs")) cs_name_to_id = {} if os.path.exists(app_context_path): with open(app_context_path, 'r', encoding='utf-8') as f: content = f.read() catalog_match = re.search(r'catalog\s*=\s*\{(.*?)\};', content, re.DOTALL) if catalog_match: block = catalog_match.group(1) entries = re.findall(r'\(\s*"([^"]+)"\s*,\s*"([^"]+)"\s*,\s*(\d+)\s*\)', block) for idx, (name, genotype, sort_order) in enumerate(entries): variety_id = f"00000000-0000-0000-0000-{idx + 1:012d}" cs_name_to_id[name.strip().lower()] = variety_id variety_map[name.strip().lower()] = variety_id variety_genotypes[variety_id] = genotype.strip() else: print(f"Warning: ApplicationContext.cs not found at {app_context_path}") if os.path.exists(SEEDS_PATH): with open(SEEDS_PATH, "r", encoding="utf-8") as f: seeds = json.load(f) for v in seeds: name_lower = v["name"].strip().lower() variety_id = cs_name_to_id.get(name_lower) if not variety_id: variety_id = f"00000000-0000-0000-0000-{v['sortOrder'] + 1:012d}" variety_map[name_lower] = variety_id # Map English name if present if "english" in v and v["english"]: variety_map[v["english"].strip().lower()] = variety_id if variety_id not in variety_genotypes: variety_genotypes[variety_id] = v.get("canonicalGenotype") else: print(f"Warning: Seeds path not found at {SEEDS_PATH}") # Reverse map (variety GUID → human name) for discard/replacement history # lines that need to show a colour value rather than a raw GUID. First wins # so we keep the canonical lower-case catalog name. variety_id_to_name = {} for name_lower, vid in variety_map.items(): if vid not in variety_id_to_name: variety_id_to_name[vid] = name_lower md_files = sorted([f for f in os.listdir(DIR_PATH) if f.lower().endswith('.md')]) print(f"Found {len(md_files)} markdown files in {DIR_PATH}.") raw_contacts = [] raw_litters = [] raw_gerbils = [] # Map to track which files contained explicit dates file_explicit_dates = {} # 1. Parse JSON blocks from all markdown files and scope IDs by filename for filename in md_files: filepath = os.path.join(DIR_PATH, filename) with open(filepath, 'r', encoding='utf-8') as f: content = f.read() json_start = content.find("```json") if json_start == -1: continue json_start += len("```json") json_end = content.rfind("```") if json_end == -1 or json_end <= json_start: continue json_str = content[json_start:json_end].strip() try: data = json.loads(json_str) contacts_key = next((k for k in data if k.lower() == 'contacts'), None) litters_key = next((k for k in data if k.lower() == 'litters'), None) gerbils_key = next((k for k in data if k.lower() == 'gerbils'), None) file_contacts = data.get(contacts_key, []) if contacts_key else [] file_litters = data.get(litters_key, []) if litters_key else [] file_gerbils = data.get(gerbils_key, []) if gerbils_key else [] # Find explicit dates on this page to build chronological ordering explicit_dates = [] for l in file_litters: d = parse_date(l.get("Date") or l.get("date") or l.get("DateOfBirth") or l.get("dateOfBirth")) if d: explicit_dates.append(d) for g in file_gerbils: d = parse_date(g.get("DateOfBirth") or g.get("dateOfBirth")) if d: explicit_dates.append(d) if explicit_dates: # Convert to days since 2000-01-01 day_vals = [date_to_days(d) for d in explicit_dates if date_to_days(d) is not None] if day_vals: file_explicit_dates[filename] = sum(day_vals) / len(day_vals) def scope_id(local_id): if not local_id: return None local_str = str(local_id).strip() if not local_str: return None return generate_guid(f"{filename}-{local_str}") for c in file_contacts: old_id = c.get("Id") or c.get("id") c["_filename"] = filename c["_scoped_id"] = scope_id(old_id) raw_contacts.append(c) for l in file_litters: old_id = l.get("Id") or l.get("id") l["_filename"] = filename l["_scoped_id"] = scope_id(old_id) l["_scoped_father_id"] = scope_id(l.get("FatherId") or l.get("fatherId")) l["_scoped_mother_id"] = scope_id(l.get("MotherId") or l.get("motherId")) raw_litters.append(l) for g in file_gerbils: old_id = g.get("Id") or g.get("id") g["_filename"] = filename g["_scoped_id"] = scope_id(old_id) g["_scoped_litter_id"] = scope_id(g.get("LitterId") or g.get("litterId")) g["_scoped_origin_cid"] = scope_id(g.get("OriginContactId") or g.get("originContactId")) receiver_val = (g.get("ReceiverContactId") or g.get("receiverContactId") or g.get("BuyerId") or g.get("buyerId") or g.get("BuyerContactId") or g.get("buyerContactId") or g.get("givenAwayContactId") or g.get("givenAwayToContactId") or g.get("ownerContactId") or g.get("ownerId")) g["_scoped_receiver_cid"] = scope_id(receiver_val) raw_gerbils.append(g) except json.JSONDecodeError as e: print(f"Failed to parse JSON in {filename}: {e}") # Chronological Interpolation: Estimate the date of each page based on neighboring pages with dates file_dates = {} sorted_files = sorted(md_files) # Simple linear interpolation / extrapolation for i, fn in enumerate(sorted_files): if fn in file_explicit_dates: file_dates[fn] = file_explicit_dates[fn] else: # Look left for closest explicit date left_val, left_dist = None, None for j in range(i - 1, -1, -1): if sorted_files[j] in file_explicit_dates: left_val = file_explicit_dates[sorted_files[j]] left_dist = i - j break # Look right for closest explicit date right_val, right_dist = None, None for j in range(i + 1, len(sorted_files)): if sorted_files[j] in file_explicit_dates: right_val = file_explicit_dates[sorted_files[j]] right_dist = j - i break if left_val is not None and right_val is not None: # Interpolate file_dates[fn] = left_val + (right_val - left_val) * (left_dist / (left_dist + right_dist)) elif left_val is not None: # Extrapolate right (assume 30 days per page gap as placeholder) file_dates[fn] = left_val + (left_dist * 30) elif right_val is not None: # Extrapolate left file_dates[fn] = right_val - (right_dist * 30) else: # No dates in entire log? Default to 2011-01-01 file_dates[fn] = date_to_days("2011-01-01") print(f"Parsed {len(raw_contacts)} raw contacts, {len(raw_litters)} raw litters, {len(raw_gerbils)} raw gerbils.") # Run extract.py to make sure stammbaum data is up to date import subprocess print("Running extract.py to extract Stammbäume...") try: subprocess.run([sys.executable, "extract.py"], check=True) except Exception as e: print(f"Warning: Failed to run extract.py: {e}") # Load Stammbaum data stammbaum_only_animals = [] animals_path = os.path.join(OUTPUT_DIR, "animals.json") if os.path.exists(animals_path): with open(animals_path, "r", encoding="utf-8") as f: stammbaum_animals = json.load(f) for a in stammbaum_animals: sources = a.get("sourceFiles", []) if any("stammbaum" in str(s).lower() for s in sources): stammbaum_only_animals.append(a) print(f"Loaded {len(stammbaum_only_animals)} Stammbaum animals.") else: print(f"Warning: Stammbaum animals.json not found at {animals_path}") # Run extract_docx.py to make sure docx data is up to date print("Running extract_docx.py to extract docx...") try: subprocess.run([sys.executable, "extract_docx.py"], check=True) except Exception as e: print(f"Warning: Failed to run extract_docx.py: {e}") # Load Docx data docx_animals = [] docx_animals_path = os.path.join(OUTPUT_DIR, "docx_animals.json") if os.path.exists(docx_animals_path): with open(docx_animals_path, "r", encoding="utf-8") as f: docx_animals = json.load(f) print(f"Loaded {len(docx_animals)} animals from docx.") else: print(f"Warning: docx_animals.json not found at {docx_animals_path}") docx_litters = [] docx_litters_path = os.path.join(OUTPUT_DIR, "docx_litters.json") if os.path.exists(docx_litters_path): with open(docx_litters_path, "r", encoding="utf-8") as f: docx_litters = json.load(f) print(f"Loaded {len(docx_litters)} litters from docx.") else: print(f"Warning: docx_litters.json not found at {docx_litters_path}") # Abgabevertrag-Datensätze (aus extract_contracts.py). Optional: wenn die # Datei fehlt, läuft der Import ohne Vertrags-Anreicherung normal weiter. contracts = [] contracts_path = os.path.join(OUTPUT_DIR, "contracts.json") if os.path.exists(contracts_path): with open(contracts_path, "r", encoding="utf-8") as f: contracts = json.load(f) print(f"Loaded {len(contracts)} Abgabeverträge.") else: print(f"Info: contracts.json not found at {contracts_path} (Vertrags-Anreicherung übersprungen)") # Extract docx buyer contacts and add to raw_contacts for da in docx_animals: o_name = (da.get("owner") or "").strip() if o_name: o_scoped_id = generate_guid(f"docx-contact-{normalize_name(o_name)}") raw_contacts.append({ "Name": o_name, "_filename": "Wurfchronik-Detail", "_scoped_id": o_scoped_id }) # Map docx litters and append to raw_litters docx_litter_id_map = {} # wsCode -> scoped_id for dl in docx_litters: l_name = dl["litterId"] dob_val = parse_date(dl["dob"]) f_name = get_normalized_gerbil_name(dl["fatherName"]) m_name = get_normalized_gerbil_name(dl["motherName"]) ws_code = dl["wsCode"] note_val = dl.get("note") # Parse survived/total born from wsCode (e.g. 4/5) total_born = None deaths_8w = None if "/" in ws_code: parts = ws_code.split("/") if len(parts) == 2: try: survived = int(parts[0]) total = int(parts[1]) total_born = total deaths_8w = max(0, total - survived) except ValueError: pass l_scoped_id = generate_guid(f"docx-litter-{normalize_name(ws_code)}-{dob_val or '0001-01-01'}") docx_litter_id_map[(ws_code, dob_val)] = l_scoped_id raw_litters.append({ "Id": l_scoped_id, "Name": l_name, "Date": dob_val, "TotalBorn": total_born, "DeathsWithin8Weeks": deaths_8w, "FatherId": generate_guid(f"stammbaum-animal-{normalize_name(f_name)}"), # placeholder "MotherId": generate_guid(f"stammbaum-animal-{normalize_name(m_name)}"), # placeholder "ExpectedGoHomeDate": None, "Notes": note_val if note_val else "Docx imported litter", "PairingCode": None, "ExternalRef": f"docx-litter-{l_scoped_id}", "LitterLetter": l_name[0] if l_name and len(l_name) > 0 else None, "_father_name": f_name, "_mother_name": m_name, "_filename": "Wurfchronik-Detail", "_scoped_id": l_scoped_id, "_scoped_father_id": generate_guid(f"stammbaum-animal-{normalize_name(f_name)}"), "_scoped_mother_id": generate_guid(f"stammbaum-animal-{normalize_name(m_name)}") }) # Extract stammbaum contacts and add to raw_contacts for a in stammbaum_only_animals: b_name = (a.get("breeder") or "").strip() if b_name: b_scoped_id = generate_guid(f"stammbaum-contact-{normalize_name(b_name)}") raw_contacts.append({ "Name": b_name, "_filename": "Stammbaum", "_scoped_id": b_scoped_id }) z_name = (a.get("zucht") or "").strip() if z_name: z_scoped_id = generate_guid(f"stammbaum-contact-{normalize_name(z_name)}") raw_contacts.append({ "Name": z_name, "_filename": "Stammbaum", "_scoped_id": z_scoped_id }) # Residency propagation for stammbaum animals def is_clan_zucht(z): if not z: return False norm = z.lower() return "klein" in norm and "chaot" in norm and "extern" not in norm stammbaum_resident_ids = set() for a in stammbaum_only_animals: if is_clan_zucht(a.get("zucht")) or is_clan_zucht(a.get("zuchtCanon")): stammbaum_resident_ids.add(a["id"]) # Propagate residency to parents of resident offspring for _ in range(5): for a in stammbaum_only_animals: if a["id"] in stammbaum_resident_ids: for p_ref in a.get("parentRefs", []): p_key = (normalize_name(p_ref["name"]), parse_date(p_ref["dob"])) for cand in stammbaum_only_animals: if normalize_name(cand["name"]) == p_key[0]: cand_dob = parse_date(cand["dob"]) if not p_key[1] or cand_dob == p_key[1]: stammbaum_resident_ids.add(cand["id"]) # Pre-index Wurfchronik litters from markdown md_litters_idx = {} for rl in raw_litters: f_name = get_normalized_gerbil_name(rl.get("FatherName") or rl.get("fatherName") or rl.get("ParentMaleName") or rl.get("parentMaleName")) m_name = get_normalized_gerbil_name(rl.get("MotherName") or rl.get("motherName") or rl.get("ParentFemaleName") or rl.get("parentFemaleName")) ldate = parse_date(rl.get("Date") or rl.get("date") or rl.get("DateOfBirth") or rl.get("dateOfBirth")) if f_name and m_name and ldate: key = (normalize_name(f_name), normalize_name(m_name), ldate) md_litters_idx[key] = rl # Gender index for parent-ref selection: normalized name → 'male' | 'female' # | 'ambiguous'. Drives the gender-aware ranking in pick_parent_ref so a dated # but wrong-sex ref (e.g. female „Danielle“) cannot win the father slot over # an undated male/unknown one (e.g. „Hagrid Rubeus“). gender_idx = {} for a in stammbaum_only_animals: g = (a.get("gender") or "").lower().strip() g = g if g in ("male", "female") else None for key in {normalize_name(a.get("name")), normalize_name(get_call_name(a.get("name") or ""))}: if not key: continue if key not in gender_idx: gender_idx[key] = g elif gender_idx[key] != g: gender_idx[key] = "ambiguous" def gender_of_name(name): v = gender_idx.get(normalize_name(name)) return v if v in ("male", "female") else None # Create virtual litters for stammbaum animals created_virtual_litters = {} for a in stammbaum_only_animals: parent_refs = a.get("parentRefs", []) child_dob_raw = a.get("dob") father_ref = pick_parent_ref(parent_refs, "father", child_dob_raw, gender_of=gender_of_name) mother_ref = pick_parent_ref(parent_refs, "mother", child_dob_raw, avoid_name=father_ref.get("name") if father_ref else None, gender_of=gender_of_name) # Record rejected parent-ref candidates (wrong sex / age-impossible / # duplicate of the other role) so this animal's gerbil history can # explain which Stammbaum positions were discarded and what won instead. pick_discards = [] pick_discards += explain_pick_rejections(parent_refs, "father", child_dob_raw, father_ref, gender_of=gender_of_name) pick_discards += explain_pick_rejections( parent_refs, "mother", child_dob_raw, mother_ref, avoid_name=father_ref.get("name") if father_ref else None, gender_of=gender_of_name) if pick_discards: a["_pick_discards"] = pick_discards a["_mapped_litter_scoped_id"] = None if father_ref and mother_ref: f_name = get_normalized_gerbil_name(father_ref.get("name")) m_name = get_normalized_gerbil_name(mother_ref.get("name")) dob_val = parse_date(a.get("dob")) mapped_litter = None if dob_val: key = (normalize_name(f_name), normalize_name(m_name), dob_val) mapped_litter = md_litters_idx.get(key) if mapped_litter: a["_mapped_litter_scoped_id"] = mapped_litter["_scoped_id"] else: v_key = (normalize_name(f_name), normalize_name(m_name), dob_val or "0001-01-01") if v_key in created_virtual_litters: a["_mapped_litter_scoped_id"] = created_virtual_litters[v_key] else: l_scoped_id = generate_guid(f"virtual-litter-{v_key[0]}-{v_key[1]}-{v_key[2]}") created_virtual_litters[v_key] = l_scoped_id a["_mapped_litter_scoped_id"] = l_scoped_id # Try to link parents to actual parsed stammbaum animals f_scoped_id = None m_scoped_id = None f_dob = parse_date(father_ref.get("dob")) m_dob = parse_date(mother_ref.get("dob")) for p_cand in stammbaum_only_animals: p_gender = str(p_cand.get("gender") or "").lower().strip() if p_gender in ["w", "f", "female", "weiblich"]: continue p_dob = parse_date(p_cand.get("dob")) if dob_val and p_dob and p_dob >= dob_val: continue cand_call_norm = normalize_name(get_call_name(p_cand["name"])) or "unbekannt" cand_name_norm = normalize_name(p_cand["name"]) or "unbekannt" if cand_call_norm == normalize_name(f_name) or cand_name_norm == normalize_name(f_name): if not f_dob or p_dob == f_dob: f_scoped_id = generate_guid(f"stammbaum-animal-{p_cand['id']}") break for p_cand in stammbaum_only_animals: p_gender = str(p_cand.get("gender") or "").lower().strip() if p_gender in ["m", "male", "männlich"]: continue p_dob = parse_date(p_cand.get("dob")) if dob_val and p_dob and p_dob >= dob_val: continue cand_call_norm = normalize_name(get_call_name(p_cand["name"])) or "unbekannt" cand_name_norm = normalize_name(p_cand["name"]) or "unbekannt" if cand_call_norm == normalize_name(m_name) or cand_name_norm == normalize_name(m_name): if not m_dob or p_dob == m_dob: m_scoped_id = generate_guid(f"stammbaum-animal-{p_cand['id']}") break raw_litters.append({ "Id": l_scoped_id, "Name": f"Wurf von {f_name} + {m_name}", "Date": dob_val, "TotalBorn": None, "DeathsWithin8Weeks": None, "FatherId": f_scoped_id or generate_guid(f"stammbaum-animal-{normalize_name(f_name)}"), "MotherId": m_scoped_id or generate_guid(f"stammbaum-animal-{normalize_name(m_name)}"), "ExpectedGoHomeDate": None, "Notes": "Pedigree virtual litter", "PairingCode": None, "ExternalRef": f"virtual-{l_scoped_id}", "LitterLetter": None, "_father_name": f_name, "_mother_name": m_name, "_filename": "Stammbaum", "_scoped_id": l_scoped_id, "_scoped_father_id": f_scoped_id or generate_guid(f"stammbaum-animal-{normalize_name(f_name)}"), "_scoped_mother_id": m_scoped_id or generate_guid(f"stammbaum-animal-{normalize_name(m_name)}") }) # 2. Resolve Contacts globally (deduplicate by normalized name) contact_by_norm_name = {} contact_id_map = {} # scoped_old_id -> global_guid for rc in raw_contacts: name_val = rc.get("Name") or rc.get("name") or rc.get("FullName") or rc.get("fullName") if not name_val: first = rc.get("FirstName") or rc.get("firstName") last = rc.get("LastName") or rc.get("lastName") if first or last: name_val = f"{first or ''} {last or ''}".strip() if not name_val: continue canon_name, should_keep = get_normalized_contact_name(name_val) scoped_id = rc["_scoped_id"] if not should_keep: if scoped_id: contact_id_map[scoped_id] = None continue norm_name = normalize_name(canon_name) rc_file = rc.get("_filename") if norm_name not in contact_by_norm_name: global_guid = generate_guid(f"contact-{norm_name}") contact_by_norm_name[norm_name] = { "Id": global_guid, "Name": canon_name, "Email": rc.get("Email") or rc.get("email"), "Phone": rc.get("Phone") or rc.get("phone"), "Address": rc.get("Address") or rc.get("address"), "Notes": rc.get("Notes") or rc.get("notes") or rc.get("Note") or rc.get("note"), # Provenance accumulators (consumed below, stripped from helper keys). "_source_files": set([rc_file]) if rc_file else set(), "_merged_count": 1, } else: gc = contact_by_norm_name[norm_name] if not gc["Email"] and (rc.get("Email") or rc.get("email")): gc["Email"] = rc.get("Email") or rc.get("email") if not gc["Phone"] and (rc.get("Phone") or rc.get("phone")): gc["Phone"] = rc.get("Phone") or rc.get("phone") if not gc["Address"] and (rc.get("Address") or rc.get("address")): gc["Address"] = rc.get("Address") or rc.get("address") if not gc["Notes"] and (rc.get("Notes") or rc.get("notes") or rc.get("Note") or rc.get("note")): gc["Notes"] = rc.get("Notes") or rc.get("notes") or rc.get("Note") or rc.get("note") if rc_file: gc["_source_files"].add(rc_file) gc["_merged_count"] += 1 if scoped_id: contact_id_map[scoped_id] = contact_by_norm_name[norm_name]["Id"] resolved_contacts = list(contact_by_norm_name.values()) print(f"Resolved to {len(resolved_contacts)} unique contacts.") # 3. Process Litters (normalize keys) resolved_litters = [] litter_id_map = {} # scoped_old_id -> new_guid litter_by_scoped_id = {} for rl in raw_litters: filename = rl.get("_filename") name_val = rl.get("Name") or rl.get("name") if not name_val: name_val = "Wurf" dob_val = parse_date(rl.get("Date") or rl.get("date") or rl.get("DateOfBirth") or rl.get("dateOfBirth")) new_guid = rl["_scoped_id"] if not new_guid: new_guid = generate_guid(f"litter-{filename}-{name_val}-{dob_val}") scoped_old_id = rl["_scoped_id"] if scoped_old_id: litter_id_map[scoped_old_id] = new_guid total_born = rl.get("TotalBorn") or rl.get("totalBorn") or rl.get("LitterSize") or rl.get("litterSize") or rl.get("size") or rl.get("totalPups") if total_born is not None: try: total_born = int(total_born) except ValueError: total_born = None deaths_8w = rl.get("DeathsWithin8Weeks") or rl.get("deathsWithin8Weeks") if deaths_8w is not None: try: deaths_8w = int(deaths_8w) except ValueError: deaths_8w = None elif total_born is not None and rl.get("survived") is not None: try: deaths_8w = total_born - int(rl.get("survived")) except ValueError: pass father_name = rl.get("FatherName") or rl.get("fatherName") or rl.get("ParentMaleName") or rl.get("parentMaleName") or rl.get("_father_name") mother_name = rl.get("MotherName") or rl.get("motherName") or rl.get("ParentFemaleName") or rl.get("parentFemaleName") or rl.get("_mother_name") raw_ext_ref = rl.get("ExternalRef") or rl.get("externalRef") or rl.get("Id") or rl.get("id") ext_ref_scoped = f"{filename}-{raw_ext_ref}" if raw_ext_ref else None l_record = { "Id": new_guid, "Name": name_val, "Date": dob_val, "TotalBorn": total_born, "DeathsWithin8Weeks": deaths_8w, "FatherId": rl["_scoped_father_id"], "MotherId": rl["_scoped_mother_id"], "ExpectedGoHomeDate": parse_date(rl.get("ExpectedGoHomeDate") or rl.get("expectedGoHomeDate")), "Notes": rl.get("Notes") or rl.get("notes") or rl.get("Note") or rl.get("note"), "PairingCode": rl.get("PairingCode") or rl.get("pairingCode"), "ExternalRef": ext_ref_scoped, "LitterLetter": rl.get("LitterLetter") or rl.get("litterLetter"), "_father_name": father_name, "_mother_name": mother_name, "_filename": filename, # Provenance accumulators (canonical absorbs these during dedup below). "_source_files": set([filename]) if filename else set(), "_merged_count": 1, # Virtual litters are reconstructed from a Stammbaum chart, not the # Wurfchronik — flagged on the raw record's _filename == "Stammbaum". "_virtual": filename == "Stammbaum", } resolved_litters.append(l_record) litter_by_scoped_id[new_guid] = l_record print(f"Processed {len(resolved_litters)} litters.") # 3b. Deduplicate litters: same date + compatible parents → merge # This handles the "sibling pairing" case: Stammbaum shows the same parental # litter twice (once under the father branch, once under the mother branch), # generating two separate litter records with the same date but only one of # them has FatherId/MotherId resolved. litter_canonical_map = {} # old_id -> canonical_id (for dedup within this step) # Group by date for efficiency by_date = {} for l in resolved_litters: by_date.setdefault(l["Date"], []).append(l) litter_dedup_canonical = {} # old_litter_id -> canonical_litter_id deduped_litters = [] for date_val, group in by_date.items(): # Partition into compatible subsets sub_groups = [] for l in group: placed = False for sub in sub_groups: if all(litter_compatible(l, member) for member in sub): sub.append(l) placed = True break if not placed: sub_groups.append([l]) for sub in sub_groups: if len(sub) == 1: deduped_litters.append(sub[0]) litter_dedup_canonical[sub[0]["Id"]] = sub[0]["Id"] continue # Pick the canonical record: prefer the one with parents set canonical = next((l for l in sub if l.get("FatherId") or l.get("MotherId")), sub[0]) for l in sub: litter_dedup_canonical[l["Id"]] = canonical["Id"] if l is not canonical: litter_id_map[l["Id"]] = canonical["Id"] canonical["_source_files"] |= l.get("_source_files", set()) canonical["_merged_count"] += l.get("_merged_count", 1) if not l.get("_virtual"): canonical["_virtual"] = False deduped_litters.append(canonical) if len(sub) > 1: merged_names = [l["Id"] for l in sub if l is not canonical] print(f"Litter-Dedup: merged {len(sub)} same-date litters on {date_val} → {canonical['Name']} (absorbed: {', '.join(merged_names)})") n_merged = len(resolved_litters) - len(deduped_litters) if n_merged: print(f"Litter-Dedup: {n_merged} redundant litter record(s) removed.") resolved_litters = deduped_litters litter_by_scoped_id = {l["Id"]: l for l in resolved_litters} # 4. Normalize and group Gerbils # Helper to lookup litter dates for birth date estimation def get_litter_date(l_id): if l_id in litter_by_scoped_id: d = litter_by_scoped_id[l_id]["Date"] if d: return d return None all_processed_gerbils = [] for rg in raw_gerbils: filename = rg.get("_filename") name_val = get_normalized_gerbil_name(rg.get("Name") or rg.get("name") or rg.get("callName")) if not name_val: name_val = "Unbekannt" dob_val = parse_date(rg.get("DateOfBirth") or rg.get("dateOfBirth")) new_guid = rg["_scoped_id"] if not new_guid: new_guid = generate_guid(f"gerbil-{filename}-{name_val}-{dob_val}") # Gender normalization gender_val = rg.get("Gender") or rg.get("gender") or "unknown" gender_val = str(gender_val).lower().strip() if gender_val in ["m", "male", "männlich"]: gender = "male" elif gender_val in ["w", "f", "female", "weiblich"]: gender = "female" else: gender = "unknown" dod_val = parse_date(rg.get("DateOfDeath") or rg.get("dateOfDeath") or rg.get("deathDate")) gohome_val = parse_date(rg.get("GoHomeDate") or rg.get("goHomeDate") or rg.get("DateGivenAway") or rg.get("givenAwayDate") or rg.get("DateOfHandover") or rg.get("dateOfHandover") or rg.get("HandoverDate") or rg.get("DateOfSale") or rg.get("dateOfSale")) status_val = rg.get("Status") or rg.get("status") or "" status_val = str(status_val).lower().strip() status = "Breeding" if dod_val or "deceased" in status_val or "verstorben" in status_val or "tod" in status_val or "dead" in status_val: status = "Deceased" elif gohome_val or "givenaway" in status_val or "abgegeben" in status_val or "verkauft" in status_val or "sold" in status_val: status = "GivenAway" elif "forsale" in status_val or "abzugeben" in status_val: status = "ForSale" elif "pet" in status_val or "liebhaber" in status_val: status = "Pet" else: if dod_val: status = "Deceased" elif gohome_val: status = "GivenAway" notes_val = rg.get("Notes") or rg.get("notes") or rg.get("Note") or rg.get("note") existing_cod = rg.get("CauseOfDeath") or rg.get("causeOfDeath") or rg.get("DeathCause") status, dod_val, cause_of_death_val = parse_death_info(notes_val, status, dod_val, existing_cod) # Explicit resolution for Ken'ichi's cause of death if name_val == "Ken'ichi" and dob_val == "2015-03-01": cause_of_death_val = "Duftdrüsen-Tumor" color_val = rg.get("ColorVarietyId") or rg.get("colorVarietyId") or rg.get("Color") or rg.get("color") or rg.get("ColorDescription") or rg.get("colorDescription") existing_gt = rg.get("Genotype") or rg.get("genotype") color_variety_id, genotype_val = resolve_color_and_genotype( color_val, existing_gt, variety_map, variety_genotypes ) if color_val and not color_variety_id: try: uuid.UUID(str(color_val).strip()) color_variety_id = str(color_val).strip() except ValueError: print(f"Warning: Unknown color variety '{color_val}' for gerbil '{name_val}' on page {filename}") origin_cid = rg["_scoped_origin_cid"] receiver_cid = rg["_scoped_receiver_cid"] if origin_cid in contact_id_map: origin_cid = contact_id_map[origin_cid] if receiver_cid in contact_id_map: receiver_cid = contact_id_map[receiver_cid] is_resident = rg.get("IsResident") or rg.get("isResident") if is_resident is None: is_resident = True else: is_resident = str(is_resident).lower() == "true" traits = rg.get("CharacterTraits") or rg.get("characterTraits") or [] if not isinstance(traits, list): traits = [str(traits)] char_note = rg.get("CharacterNote") or rg.get("characterNote") is_deaf = rg.get("IsDeaf") or rg.get("isDeaf") if is_deaf is not None: is_deaf = str(is_deaf).lower() == "true" old_litter_id = rg["_scoped_litter_id"] raw_ext_ref = rg.get("ExternalRef") or rg.get("externalRef") or rg.get("Id") or rg.get("id") ext_ref_scoped = f"{filename}-{raw_ext_ref}" if raw_ext_ref else None # Estimated effective date of birth for conflict checking eff_dob_val = dob_val if not eff_dob_val and old_litter_id: # Try to get litter date if litter was resolved mapped_l_id = litter_id_map.get(old_litter_id) if mapped_l_id: eff_dob_val = get_litter_date(mapped_l_id) # Determine explicit or litter-derived birth date (None if unknown/parent) birth_date = dob_val if not birth_date and old_litter_id: mapped_l_id = litter_id_map.get(old_litter_id) if mapped_l_id: birth_date = get_litter_date(mapped_l_id) # If still no effective date, use the estimated page date if not eff_dob_val: eff_dob_val = days_to_date(file_dates[filename]) raw_breeder = rg.get("OriginBreeder") or rg.get("originBreeder") if raw_breeder: norm_b, keep_b = get_normalized_contact_name(raw_breeder) raw_breeder = norm_b if keep_b else None parent_refs = [] if old_litter_id: rl = next((l for l in raw_litters if l.get("_scoped_id") == old_litter_id), None) if rl: f_name = rl.get("FatherName") or rl.get("fatherName") or rl.get("ParentMaleName") or rl.get("parentMaleName") or rl.get("_father_name") m_name = rl.get("MotherName") or rl.get("motherName") or rl.get("ParentFemaleName") or rl.get("parentFemaleName") or rl.get("_mother_name") if f_name: parent_refs.append({"name": f_name, "roleGuess": "father"}) if m_name: parent_refs.append({"name": m_name, "roleGuess": "mother"}) all_processed_gerbils.append({ "Id": new_guid, "Name": name_val, "Gender": gender, "Status": status, "LitterId": old_litter_id, # mapped later "OriginContactId": origin_cid, "ReceiverContactId": receiver_cid, "EnclosureId": None, "ColorVarietyId": color_variety_id, "DateOfBirth": dob_val, "DateOfDeath": dod_val, "CauseOfDeath": cause_of_death_val, "GoHomeDate": gohome_val, "Genotype": genotype_val, "Notes": rg.get("Notes") or rg.get("notes") or rg.get("Note") or rg.get("note"), "ImportSource": rg.get("ImportSource") or rg.get("importSource") or filename, "ExternalRef": ext_ref_scoped, "RawImportData": rg.get("RawImportData") or rg.get("rawImportData") or json.dumps({"colorDescription": color_val if not color_variety_id else None}), "OriginBreeder": raw_breeder or ("Zucht der kleinen Chaoten" if is_resident else None), "NameSearch": normalize_name(name_val), "CharacterTraits": traits, "CharacterNote": char_note, "IsDeaf": is_deaf, "IsResident": is_resident, "parentRefs": parent_refs, "_photos": rg.get("photos", []), "_old_scoped_litter_id": old_litter_id, "_eff_dob": eff_dob_val, "_birth_date": birth_date, "_filename": filename, "_old_id": rg.get("Id") or rg.get("id") }) # Build the discard list for one stammbaum animal: rejected parent-ref # candidates (pick_parent_ref) plus a manual DOB-remap decision, if any. def _stammbaum_discards(a): discards = list(a.get("_pick_discards", [])) remap = a.get("dobRemap") if remap and remap.get("original") and remap.get("corrected"): orig = _de_date(parse_date(remap["original"])) or remap["original"] corr = _de_date(parse_date(remap["corrected"])) or remap["corrected"] if orig != corr: discards.append({ "label": PROV_FIELD_LABELS["DateOfBirth"], "value": orig, "reason": "per manueller Entscheidung korrigiert", "replacement": corr, }) return discards # Map and append stammbaum animals to all_processed_gerbils for a in stammbaum_only_animals: a_id = a["id"] name_val = get_normalized_gerbil_name(a["name"]) gender_val = str(a.get("gender") or "").lower().strip() if gender_val in ["m", "male"]: gender = "male" elif gender_val in ["w", "f", "female"]: gender = "female" else: gender = "unknown" dob_val = parse_date(a.get("dob")) dod_val = parse_date(a.get("death")) status = "Breeding" if dod_val: status = "Deceased" elif a_id not in stammbaum_resident_ids: status = "GivenAway" else: if dob_val: try: dt_dob = datetime.strptime(dob_val, "%Y-%m-%d") dt_now = datetime.now() age_years = (dt_now - dt_dob).days / 365.25 if age_years >= 6.0: status = "Deceased" except Exception: pass # Color variety mapping existing_gt = a["genotype"]["rawGenotype"] if a["genotype"]["rawGenotype"] else None color_variety_id, genotype_val = resolve_color_and_genotype( a.get("farbschlag"), existing_gt, variety_map, variety_genotypes ) if not color_variety_id: for fbv in a.get("farbschlagVariants", []): cv_id, gt_val = resolve_color_and_genotype(fbv, existing_gt, variety_map, variety_genotypes) if cv_id: color_variety_id = cv_id genotype_val = gt_val break # Contact mapping origin_cid = None b_name = (a.get("breeder") or "").strip() z_name = (a.get("zucht") or "").strip() if b_name: origin_cid = generate_guid(f"stammbaum-contact-{normalize_name(b_name)}") elif z_name: origin_cid = generate_guid(f"stammbaum-contact-{normalize_name(z_name)}") # Map to resolved global contact GUID if origin_cid in contact_id_map: origin_cid = contact_id_map[origin_cid] else: origin_cid = None scoped_id = generate_guid(f"stammbaum-animal-{a_id}") scoped_litter_id = a.get("_mapped_litter_scoped_id") is_deaf = a["genotype"].get("deaf") stammbaum_breeder = a.get("breeder") if a.get("breeder") else (a.get("zucht") if a.get("zucht") else None) if stammbaum_breeder: norm_b, keep_b = get_normalized_contact_name(stammbaum_breeder) stammbaum_breeder = norm_b if keep_b else None all_processed_gerbils.append({ "Id": scoped_id, "Name": name_val, "Gender": gender, "Status": status, "LitterId": scoped_litter_id, # mapped later in step 5 "OriginContactId": origin_cid, "ReceiverContactId": None, "EnclosureId": None, "ColorVarietyId": color_variety_id, "DateOfBirth": dob_val, "DateOfDeath": dod_val, "CauseOfDeath": "Duftdrüsen-Tumor" if name_val == "Ken'ichi" and dob_val == "2015-03-01" else None, "GoHomeDate": None, "Genotype": genotype_val, "Notes": None, "ImportSource": ", ".join(a.get("sourceFiles", [])), "ExternalRef": f"stammbaum-{a_id}", "RawImportData": json.dumps({ "rawGenotype": a["genotype"]["rawGenotype"], "unmappedTokens": a["genotype"]["unmappedTokens"], "breederText": a.get("breeder", "") }, ensure_ascii=False), "OriginBreeder": stammbaum_breeder, "NameSearch": normalize_name(name_val), "CharacterTraits": [], "CharacterNote": None, "IsDeaf": is_deaf, "IsResident": a_id in stammbaum_resident_ids, "parentRefs": a.get("parentRefs", []), "_photos": a.get("photos", []), "_conflict": bool(a.get("conflict")), "_resolved_by_decision": bool(a.get("resolvedByDecision")), "_old_scoped_litter_id": scoped_litter_id, "_eff_dob": dob_val or "2010-01-01", "_birth_date": dob_val, "_filename": a.get("sourceFiles", ["Stammbaum"])[0], "_old_id": a_id, # Rejected Stammbaum parent-ref candidates for this animal (filled by # pick_parent_ref above) — surfaced in this gerbil's discard history. "_discarded": _stammbaum_discards(a), }) # Map and append docx animals to all_processed_gerbils for idx, da in enumerate(docx_animals): name_val = get_normalized_gerbil_name(da["name"]) gender = da["gender"] dob_val = parse_date(da.get("litterDob")) dod_val = parse_date(da.get("deathDate")) gohome_val = parse_date(da.get("abgabeDate")) # Status precedence status = "Breeding" if dod_val: status = "Deceased" elif gohome_val or da.get("owner"): status = "GivenAway" # Color variety mapping color_variety_id, genotype_val = resolve_color_and_genotype( da.get("farbschlag"), None, variety_map, variety_genotypes ) # Contact mapping (buyer) receiver_cid = None o_name = (da.get("owner") or "").strip() if o_name: receiver_cid = generate_guid(f"docx-contact-{normalize_name(o_name)}") if receiver_cid in contact_id_map: receiver_cid = contact_id_map[receiver_cid] else: receiver_cid = None # Scoped ID scoped_id = generate_guid(f"docx-animal-{idx}-{normalize_name(name_val)}-{dob_val or '0001-01-01'}") # Litter ID mapping ws_code = da.get("wsCode") scoped_litter_id = docx_litter_id_map.get((ws_code, dob_val)) # Raw import details raw_import_payload = json.dumps({ "abgabeWeight": da.get("abgabeWeight", ""), "deathCause": da.get("deathCause", ""), "partnerName": da.get("partnerName", ""), "partnerDob": da.get("partnerDob", "") }, ensure_ascii=False) # Residents: if sold/given away, it's not a resident is_resident = not bool(o_name) parent_refs = [] if scoped_litter_id: dl = next((l for l in docx_litters if docx_litter_id_map.get((l["wsCode"], parse_date(l["dob"]))) == scoped_litter_id), None) if dl: if dl.get("fatherName"): parent_refs.append({"name": dl["fatherName"], "roleGuess": "father"}) if dl.get("motherName"): parent_refs.append({"name": dl["motherName"], "roleGuess": "mother"}) all_processed_gerbils.append({ "Id": scoped_id, "Name": name_val, "Gender": gender, "Status": status, "LitterId": scoped_litter_id, "OriginContactId": None, "ReceiverContactId": receiver_cid, # mapped in step 5 "EnclosureId": None, "ColorVarietyId": color_variety_id, "DateOfBirth": dob_val, "DateOfDeath": dod_val, "CauseOfDeath": da.get("deathCause"), "GoHomeDate": gohome_val, "Genotype": genotype_val, "Notes": None, "ImportSource": "Wurfchronik-Detail.docx", "ExternalRef": f"docx-{idx}-{normalize_name(name_val)}-{dob_val or '0001-01-01'}", "RawImportData": raw_import_payload, "OriginBreeder": "Zucht der kleinen Chaoten", "NameSearch": normalize_name(name_val), "CharacterTraits": [], "CharacterNote": None, "IsDeaf": None, "IsResident": is_resident, "parentRefs": parent_refs, "_photos": da.get("photos", []), "_old_scoped_litter_id": scoped_litter_id, "_eff_dob": dob_val or "2020-01-01", "_birth_date": dob_val, "_filename": "Wurfchronik-Detail.docx", "_old_id": name_val }) # Build parenting dates lookup using old scoped IDs parent_litter_dates = {} for l in resolved_litters: ld = l["Date"] if ld: for pid in [l["FatherId"], l["MotherId"]]: if pid: parent_litter_dates.setdefault(pid, []).append(ld) def are_compatible(g1, g2): # Must have same gender (or one unknown) if g1["Gender"] != "unknown" and g2["Gender"] != "unknown" and g1["Gender"] != g2["Gender"]: bd1 = g1.get("_birth_date") bd2 = g2.get("_birth_date") if not (bd1 and bd2 and bd1 == bd2): return False bd1 = g1.get("_birth_date") bd2 = g2.get("_birth_date") # New rule: if name and parents match, they are compatible regardless of DOB! p1 = g1.get("parentRefs", []) p2 = g2.get("parentRefs", []) f1 = next((p["name"] for p in p1 if p.get("roleGuess") == "father"), "") m1 = next((p["name"] for p in p1 if p.get("roleGuess") == "mother"), "") f2 = next((p["name"] for p in p2 if p.get("roleGuess") == "father"), "") m2 = next((p["name"] for p in p2 if p.get("roleGuess") == "mother"), "") parents_match = False if f1 and f2 and m1 and m2: if normalize_name(f1) == normalize_name(f2) and normalize_name(m1) == normalize_name(m2): parents_match = True if not parents_match: # If both have explicit birth dates, they must match within 30 days if bd1 and bd2: days1 = date_to_days(bd1) days2 = date_to_days(bd2) if days1 is not None and days2 is not None: if abs(days1 - days2) > 30: return False # If g1 has birth date, and g2 has parenting dates, birth date must be before parenting dates p_dates2 = parent_litter_dates.get(g2["Id"], []) if bd1: for pd in p_dates2: if pd <= bd1: # Can't have litter before or on birth date return False p_dates1 = parent_litter_dates.get(g1["Id"], []) if bd2: for pd in p_dates1: if pd <= bd2: return False return True def build_provenance(records, best_g, extra_notes=None, field_source=None, field_discards=None): """Aggregate data-provenance across every raw record merged into one resolved gerbil. Returns a JSON string (stored on the Gerbil entity as a nullable text column) so the Rennmausakte can show where the entry came from. `records` is the list of raw records that became this gerbil; `best_g` is the chosen primary record. `field_source` maps a significant field name (DateOfBirth/Genotype/…) to the raw record that supplied its final value, so the history can name the exact file for each fact.""" source_files = set() from_wurfchronik = False any_conflict = False any_decision = False for r in records: source_files |= _record_source_files(r) for f in _record_source_files(r): if "wurfchronik" in f.lower(): from_wurfchronik = True if r.get("_conflict"): any_conflict = True if r.get("_resolved_by_decision"): any_decision = True notes = [] merged_count = len(records) if merged_count > 1: notes.append(f"aus {merged_count} Datensätzen zusammengeführt") if any_decision: notes.append("per manueller Entscheidung zugeordnet") if any_conflict: notes.append("Konflikt per Entscheidung gelöst") # Parent derivation: surface the strongest parentRef method/confidence # the primary record carries (chart-position etc.). parent_method = None parent_confidence = None for ref in best_g.get("parentRefs", []) or []: if ref.get("method") and not parent_method: parent_method = ref.get("method") if ref.get("confidence") and not parent_confidence: parent_confidence = ref.get("confidence") if extra_notes: for n in extra_notes: if n and n not in notes: notes.append(n) history = _build_gerbil_history( records, best_g, field_source or {}, parent_method=parent_method, any_decision=any_decision, any_conflict=any_conflict, conflict_notes=extra_notes or [], field_discards=field_discards, parent_discards=best_g.get("_discarded"), ) return build_entity_provenance( source_files, merged_count, notes=notes, from_wurfchronik=from_wurfchronik, extra={"parentMethod": parent_method, "parentConfidence": parent_confidence}, history=history, ) # Group gerbils by name to perform deduplication gerbil_groups = {} for g in all_processed_gerbils: name_key = get_dedup_name_key(g["Name"]) if not name_key: name_key = "unbekannt" gerbil_groups.setdefault(name_key, []).append(g) resolved_gerbils = [] gerbil_id_map = {} # old_scoped_id -> final_id color_keys = set(variety_map.keys()) for name_key, group in gerbil_groups.items(): is_placeholder = ( name_key in color_keys or any(p in name_key for p in ["unbekannt", "unbenannt", "baby", "welpe", "jungtier", "unknown", "welpen"]) or len(name_key) <= 2 ) if is_placeholder: # Placeholders: do NOT merge, keep all separate for g in group: # Provenance is built in a final pass (after parent resolution), # so parent-link discards land in the right gerbil's history. g["_prov_args"] = ([g], g, None, None, None) resolved_gerbils.append(g) gerbil_id_map[g["Id"]] = g["Id"] continue # Partition group into compatible subsets sub_groups = [] for g in group: placed = False for sub in sub_groups: if all(are_compatible(g, member) for member in sub): sub.append(g) placed = True break if not placed: sub_groups.append([g]) # Merge each partition sub-group into a single gerbil for sub in sub_groups: if len(sub) == 1: g = sub[0] g["_prov_args"] = ([g], g, None, None, None) resolved_gerbils.append(g) gerbil_id_map[g["Id"]] = g["Id"] continue # Find the best primary record to merge into best_g = None best_score = -1 for g in sub: score = 0 if g["_old_scoped_litter_id"]: score += 10 if g["DateOfBirth"]: score += 5 if g["Genotype"]: score += 3 if g["ColorVarietyId"]: score += 2 if g["ImportSource"] and "stammbaum" in g["ImportSource"].lower(): score += 20 if g["Notes"] and not any(kw in g["Notes"].lower() for kw in ["parent", "mutter", "vater", "dam", "sire"]): score += 1 if len(g["Name"]) > len(name_key) + 5: # likely has clan suffix score += 15 if score > best_score: best_score = score best_g = g # Merge fields merged_notes = [] if best_g["Notes"]: merged_notes.append(best_g["Notes"]) # Carry over discard records (rejected parent-refs / dob remaps) from # every merged record so none are lost when the primary changes. merged_discards = list(best_g.get("_discarded") or []) for g in sub: if g is best_g: continue for d in (g.get("_discarded") or []): if d not in merged_discards: merged_discards.append(d) best_g["_discarded"] = merged_discards # Merge photos merged_photos = list(best_g.get("_photos", [])) # Track sources for debugging sources = [best_g["_filename"]] # Per-field file attribution: which raw record supplied each final # field value. Seed with best_g for every field it already carries; # the fill loop and voting loop update it as winners change. ATTRIB_FIELDS = ["DateOfBirth", "DateOfDeath", "Gender", "Genotype", "ColorVarietyId", "Name"] field_source = {} for fld in ATTRIB_FIELDS: v = best_g.get(fld) if v and v != "unknown": field_source[fld] = best_g for g in sub: if g == best_g: continue gerbil_id_map[g["Id"]] = best_g["Id"] sources.append(g["_filename"]) for ph in g.get("_photos", []): if ph not in merged_photos: merged_photos.append(ph) if not best_g.get("_old_scoped_litter_id") and g.get("_old_scoped_litter_id"): best_g["_old_scoped_litter_id"] = g["_old_scoped_litter_id"] if not best_g["LitterId"] and g["LitterId"]: best_g["LitterId"] = g["LitterId"] if not best_g["DateOfBirth"] and g["DateOfBirth"]: best_g["DateOfBirth"] = g["DateOfBirth"] field_source["DateOfBirth"] = g if not best_g["DateOfDeath"] and g["DateOfDeath"]: best_g["DateOfDeath"] = g["DateOfDeath"] field_source["DateOfDeath"] = g if not best_g["CauseOfDeath"] and g["CauseOfDeath"]: best_g["CauseOfDeath"] = g["CauseOfDeath"] if not best_g["GoHomeDate"] and g["GoHomeDate"]: best_g["GoHomeDate"] = g["GoHomeDate"] if not best_g["Genotype"] and g["Genotype"]: best_g["Genotype"] = g["Genotype"] field_source["Genotype"] = g if not best_g["ColorVarietyId"] and g["ColorVarietyId"]: best_g["ColorVarietyId"] = g["ColorVarietyId"] field_source["ColorVarietyId"] = g if not best_g["OriginContactId"] and g["OriginContactId"]: best_g["OriginContactId"] = g["OriginContactId"] if not best_g["ReceiverContactId"] and g["ReceiverContactId"]: best_g["ReceiverContactId"] = g["ReceiverContactId"] if g["IsResident"]: best_g["IsResident"] = True # Reconcile Gender: prefer a known gender over unknown, and prefer stammbaum over other sources if best_g["Gender"] == "unknown" and g["Gender"] != "unknown": best_g["Gender"] = g["Gender"] field_source["Gender"] = g elif best_g["Gender"] != "unknown" and g["Gender"] != "unknown" and best_g["Gender"] != g["Gender"]: if g["ImportSource"] and "stammbaum" in g["ImportSource"].lower(): if not best_g["ImportSource"] or "stammbaum" not in best_g["ImportSource"].lower(): best_g["Gender"] = g["Gender"] field_source["Gender"] = g # Status precedence: Deceased > GivenAway > Breeding/Pet if g["Status"] == "Deceased": best_g["Status"] = "Deceased" elif g["Status"] == "GivenAway" and best_g["Status"] not in ["Deceased"]: best_g["Status"] = "GivenAway" if g["Notes"] and g["Notes"] not in merged_notes: # Ignore redundant dummy notes if not any(kw in g["Notes"].lower() for kw in ["parent listed", "mutter von", "vater von", "dam of", "sire of"]): merged_notes.append(g["Notes"]) # Display formatter per field for discard/replacement lines. def _disp(field, val): if val is None or val == "" or val == "unknown": return None if field in ("DateOfBirth", "DateOfDeath"): return _de_date(val) if field == "Gender": return _de_gender(val) if field == "ColorVarietyId": return variety_id_to_name.get(val, "Farbschlag") return str(val) # Reconcile fields based on number of source files supporting them conflict_notes = [] field_discards = [] for field in ["DateOfBirth", "DateOfDeath", "Gender", "Genotype", "ColorVarietyId"]: votes = {} for g in sub: val = g.get(field) if val and val != "unknown": # count source files sources_count = len(str(g.get("ImportSource") or "").split(",")) votes[val] = votes.get(val, 0) + sources_count if votes: best_val = max(votes, key=votes.get) # If the records disagreed on a field, the merge had to pick a # winner — record that as a provenance note. if len(votes) > 1: conflict_notes.append( f"Konflikt bei {PROV_FIELD_LABELS[field]} per Mehrheitsentscheidung gelöst" ) best_g[field] = best_val # Attribute the winning value to a record that actually holds # it, so the history names the right file. winner = next((g for g in sub if g.get(field) == best_val), None) if winner is not None: field_source[field] = winner # Record each LOSING value: which file it came from, that it # was discarded as differing, and that the majority value (and # its file) was used instead. if len(votes) > 1: repl_disp = _disp(field, best_val) repl_file = _primary_file_of(winner) if winner is not None else None for lose_val in votes: if lose_val == best_val: continue loser = next((g for g in sub if g.get(field) == lose_val), None) field_discards.append({ "label": PROV_FIELD_LABELS.get(field, field), "value": _disp(field, lose_val), "file": _primary_file_of(loser) if loser is not None else None, "reason": "abweichend", "replacement": repl_disp, "repl_file": repl_file, "replacement_note": "Mehrheit", }) # Keep helper fields in sync if we changed DateOfBirth if field == "DateOfBirth": best_g["_birth_date"] = best_val best_g["_eff_dob"] = best_val or "2010-01-01" if merged_notes: best_g["Notes"] = " | ".join(merged_notes) best_g["_photos"] = merged_photos # Defer provenance to a final pass so parent-link discards (added # later by the parent-age / role-normalization passes) are included. best_g["_prov_args"] = (sub, best_g, conflict_notes, field_source, field_discards) # Print merge trace print(f"Deduplicated same-animal name '{best_g['Name']}': merged {len(sub)} entries across files: {', '.join(sources)}") resolved_gerbils.append(best_g) gerbil_id_map[best_g["Id"]] = best_g["Id"] print(f"Deduplicated to {len(resolved_gerbils)} unique gerbil records.") # Apply age-based death threshold (6.0 years) to all resolved gerbils dt_now = datetime.now() for g in resolved_gerbils: if g.get("Status") != "Deceased" and g.get("Status") != "GivenAway": if not g.get("DateOfDeath") and not g.get("ReceiverContactId"): dob_str = g.get("DateOfBirth") if dob_str: try: dt_dob = datetime.strptime(dob_str, "%Y-%m-%d") age_years = (dt_now - dt_dob).days / 365.25 if age_years >= 6.0: g["Status"] = "Deceased" except Exception: pass # 5. Map Gerbils to Litters for g in resolved_gerbils: old_lid = g["_old_scoped_litter_id"] l_guid = litter_id_map.get(old_lid) g["LitterId"] = l_guid if l_guid and l_guid in litter_by_scoped_id: l = litter_by_scoped_id[l_guid] l_date = l.get("Date") if l_date: if not g.get("DateOfBirth"): g["DateOfBirth"] = l_date g["_birth_date"] = l_date g["_eff_dob"] = l_date # Apply age-based death threshold (6.0 years) to all resolved gerbils (including newly backfilled ones) if g.get("Status") != "Deceased" and g.get("Status") != "GivenAway": if not g.get("DateOfDeath") and not g.get("ReceiverContactId"): dob_str = g.get("DateOfBirth") if dob_str: try: dt_dob = datetime.strptime(dob_str, "%Y-%m-%d") age_years = (dt_now - dt_dob).days / 365.25 if age_years >= 6.0: g["Status"] = "Deceased" except Exception: pass # Clean helper fields del g["_old_scoped_litter_id"] del g["_eff_dob"] if "_birth_date" in g: del g["_birth_date"] del g["_filename"] del g["_old_id"] g.pop("_conflict", None) g.pop("_resolved_by_decision", None) # Gather final valid gerbil IDs valid_gerbil_ids = {g["Id"] for g in resolved_gerbils} # Create name lookup for resolved gerbils gerbil_by_norm_name = {} for g in resolved_gerbils: n_key = normalize_name(g["Name"]) gerbil_by_norm_name.setdefault(n_key, []).append(g) # Also index by call-name to resolve parents who are only listed by call-name c_key = normalize_name(get_call_name(g["Name"])) if c_key != n_key: gerbil_by_norm_name.setdefault(c_key, []).append(g) # Map raw Guid if present (convert if old_id mapped to new_guid) for l in resolved_litters: if l["FatherId"] in gerbil_id_map: l["FatherId"] = gerbil_id_map[l["FatherId"]] if l["MotherId"] in gerbil_id_map: l["MotherId"] = gerbil_id_map[l["MotherId"]] # Clean foreign keys that do not point to a valid gerbil if l["FatherId"] and l["FatherId"] not in valid_gerbil_ids: l["FatherId"] = None if l["MotherId"] and l["MotherId"] not in valid_gerbil_ids: l["MotherId"] = None # Parent Resolver (Global Name Matching) resolved_fathers = 0 resolved_mothers = 0 gerbil_by_id_final = {g["Id"]: g for g in resolved_gerbils} def _final_gender(gid): g = gerbil_by_id_final.get(gid) return g["Gender"] if g else None def _resolve_name(name, prefer_gender, litter_date): """Resolve a parent name to the best matching final gerbil. Gender is a PREFERENCE, not a hard filter: a reversed parent (e.g. a female listed in the father position, as the Stammbaum often does) still resolves to a gerbil — the role is corrected afterwards by gender. This is what previously left FatherId/MotherId null (the candidate was filtered out for having the "wrong" gender for its slot). """ if not name: return None cands = [] for c in gerbil_by_norm_name.get(normalize_name(name), []): final_id = gerbil_id_map.get(c["Id"]) if not final_id: continue final_c = gerbil_by_id_final.get(final_id) if not final_c: continue # Parent must be age-plausible: born before the litter and within the # gerbil lifespan (skips e.g. a 2013 animal for a 2022 litter). if not parent_age_plausible(final_c["DateOfBirth"], litter_date): continue cands.append(final_c) if not cands: return None # Prefer the gender expected for this role, then unknown, then anything. for pool in ( [c for c in cands if c["Gender"] == prefer_gender], [c for c in cands if c["Gender"] == "unknown"], cands, ): if pool: return pool[0] return None for l in resolved_litters: # Pre-check: if _father_name points to a known female and _mother_name to a # known male → swap names (Stammbaum positions reversed). Helps the name # resolver pick the right same-name candidate before role normalization. f_name_pre = l.get("_father_name", "") m_name_pre = l.get("_mother_name", "") if f_name_pre and m_name_pre: f_gender = next((g["Gender"] for g in gerbil_by_norm_name.get(normalize_name(f_name_pre), []) if g["Gender"] != "unknown"), None) m_gender = next((g["Gender"] for g in gerbil_by_norm_name.get(normalize_name(m_name_pre), []) if g["Gender"] != "unknown"), None) if f_gender == "female" and m_gender == "male": l["_father_name"], l["_mother_name"] = m_name_pre, f_name_pre if l["_father_name"] and not l["FatherId"]: cand = _resolve_name(l["_father_name"], "male", l["Date"]) if cand: l["FatherId"] = cand["Id"] resolved_fathers += 1 if l["_mother_name"] and not l["MotherId"]: cand = _resolve_name(l["_mother_name"], "female", l["Date"]) if cand: l["MotherId"] = cand["Id"] resolved_mothers += 1 # Cleanup internal keys del l["_father_name"] del l["_mother_name"] del l["_filename"] # Children-of-litter lookup so a dropped parent link can be explained in the # offspring's Datenherkunft (the discard is most meaningful on the child). children_by_litter = {} for g in resolved_gerbils: lid = g.get("LitterId") if lid: children_by_litter.setdefault(lid, []).append(g) def _add_child_discard(litter, discard): """Attach a parent-link discard to every child gerbil of the litter.""" for child in children_by_litter.get(litter["Id"], []): dl = child.setdefault("_discarded", []) if discard not in dl: dl.append(discard) def _pname(pid): p = gerbil_by_id_final.get(pid) return p.get("Name") if p else None # Role normalization: assign each resolved parent to the role matching its # gender, eliminate self-pairings (same animal in both roles), and never let # impossible duplicates survive (two males / two females). This corrects # reversed Stammbaum positions including the cases the simple swap missed # (one parent of "unknown" gender, or a self-paired litter). role_fixes = 0 for l in resolved_litters: before = (l.get("FatherId"), l.get("MotherId")) father, mother = assign_parent_roles(l.get("FatherId"), l.get("MotherId"), _final_gender) if before != (father, mother): role_fixes += 1 after = {father, mother} # A parent id present before but gone after was dropped by role # normalization (self-pairing or two-of-the-same-sex). Explain it. for pid in before: if pid and pid not in after: pname = _pname(pid) if before[0] == before[1]: reason = "Selbstverpaarung — Tier kann nicht beide Elternteile sein" else: reason = "ein Wurf hat nur einen Vater und eine Mutter" _add_child_discard(l, { "text": f"Elternteil „{pname}“ verworfen — {reason}", }) l["FatherId"] = father l["MotherId"] = mother if role_fixes: print(f"Role-normalization: corrected {role_fixes} litter(s) (gender roles / self-pairings).") # Parent-age sanity check: drop any resolved parent that cannot belong to the # litter — born after the offspring, or more than a gerbil lifespan earlier. # Catches mis-resolved links the name matcher still let through (e.g. Jayjay, # *2013, wrongly attached to Solice's 2022 litter). age_drops = [] for l in resolved_litters: ldate = l.get("Date") for role in ("FatherId", "MotherId"): pid = l.get(role) if not pid: continue p = gerbil_by_id_final.get(pid) if p and not parent_age_plausible(p.get("DateOfBirth"), ldate): age_drops.append((l.get("Name"), role, p.get("Name"), p.get("DateOfBirth"), ldate)) # Explain the drop in each child's history: which parent, its DOB, # why (implausible age), and that no replacement was used. role_de = "Vater" if role == "FatherId" else "Mutter" pdob_disp = _de_date(p.get("DateOfBirth")) or "unbekannt" age_reason = "unplausibles Alter für diesen Wurf" pd = date_to_days(parse_date(p.get("DateOfBirth"))) if p.get("DateOfBirth") else None ld = date_to_days(parse_date(ldate)) if ldate else None if pd is not None and ld is not None: years = abs(ld - pd) / 365.25 if ld - pd <= 0: age_reason = "unplausibel (nicht vor dem Kind geboren)" else: age_reason = f"unplausibel ({years:.0f} Jahre älter als das Kind)" _add_child_discard(l, { "text": (f"{role_de} „{p.get('Name')}“ (*{pdob_disp}) verworfen " f"— {age_reason}; kein Ersatz"), }) l[role] = None if age_drops: print(f"Parent-age sanity check: dropped {len(age_drops)} implausible parent link(s):") for lname, role, pname, pdob, ldate in age_drops[:20]: print(f" {lname}: {role}={pname} (*{pdob}) vs litter {ldate}") print(f"Globally resolved {resolved_fathers} fathers and {resolved_mothers} mothers.") # 5b. Second-pass litter dedup: now that FatherId/MotherId are known, # merge litters that have the same date AND the same parents. # This is the core of the "sibling pairing" fix: Blue Wave and Sunny Sky # both come from Wonderman × Unique — their two separate litter records # must now become one, so their children share the same LitterId. litter_by_id_post = {l["Id"]: l for l in resolved_litters} gerbil_by_litter = {} for g in resolved_gerbils: lid = g.get("LitterId") if lid: gerbil_by_litter.setdefault(lid, []).append(g) def _litter_same_parents(l1, l2): """Strict: same date + both parents known and matching.""" if l1["Date"] != l2["Date"]: return False f1, m1 = l1.get("FatherId"), l1.get("MotherId") f2, m2 = l2.get("FatherId"), l2.get("MotherId") if not f1 or not f2 or not m1 or not m2: return False return f1 == f2 and m1 == m2 by_date2 = {} for l in resolved_litters: by_date2.setdefault(l["Date"], []).append(l) deduped2 = [] litter_remap2 = {} # old_id -> canonical_id for date_val, group in by_date2.items(): sub_groups = [] for l in group: placed = False for sub in sub_groups: if all(_litter_same_parents(l, m) for m in sub): sub.append(l) placed = True break if not placed: sub_groups.append([l]) for sub in sub_groups: # Prefer the canonical that has the most children canonical = max(sub, key=lambda l: len(gerbil_by_litter.get(l["Id"], []))) for l in sub: litter_remap2[l["Id"]] = canonical["Id"] if l is not canonical: canonical["_source_files"] |= l.get("_source_files", set()) canonical["_merged_count"] += l.get("_merged_count", 1) if not l.get("_virtual"): canonical["_virtual"] = False deduped2.append(canonical) if len(sub) > 1: siblings = [g["Name"] for l in sub for g in gerbil_by_litter.get(l["Id"], []) if l is not canonical] print(f"Sibling-Litter-Merge on {date_val}: {canonical['Name']} absorbed sibling half — children now share LitterId: {[g['Name'] for g in gerbil_by_litter.get(canonical['Id'], [])] + siblings}") # Remap LitterId in all gerbils n_remapped = 0 for g in resolved_gerbils: old_lid = g.get("LitterId") if old_lid and old_lid in litter_remap2 and litter_remap2[old_lid] != old_lid: g["LitterId"] = litter_remap2[old_lid] n_remapped += 1 n_merged2 = len(resolved_litters) - len(deduped2) if n_merged2: print(f"Sibling-Litter-Dedup: {n_merged2} additional litter record(s) merged ({n_remapped} gerbil LitterIds remapped).") resolved_litters = deduped2 litter_by_scoped_id = {l["Id"]: l for l in resolved_litters} # Final gerbil-provenance pass: now that parent links are fully resolved and # all discards (majority-vote conflicts during dedup; rejected parent-refs; # role-normalization drops; parent-age drops) are attached to each gerbil's # _discarded list, render the provenance JSON with the discard history lines. n_discards = 0 for g in resolved_gerbils: args = g.pop("_prov_args", None) if g.get("_discarded"): n_discards += len(g["_discarded"]) if args is not None: records, best_g, conflict_notes, field_source, field_discards = args g["Provenance"] = build_provenance( records, best_g, extra_notes=conflict_notes, field_source=field_source, field_discards=field_discards, ) else: g["Provenance"] = build_provenance([g], g) g.pop("_discarded", None) if n_discards: print(f"Datenherkunft: recorded {n_discards} discard line(s) across gerbils.") # Abgabevertrag-Anreicherung: Käufer als Abnehmer-Kontakte anlegen und — # konservativ — auf eindeutig passende Tiere ReceiverContactId/GoHomeDate/ # Status=GivenAway setzen (nur falls noch nicht gesetzt). Provenance- # Historie wird ergänzt. Neue Käuferkontakte landen in contact_by_norm_name # und werden danach automatisch als IsReceiver markiert. cstats, sale_contracts = enrich_from_contracts( contracts, resolved_gerbils, contact_by_norm_name, contact_id_map) if contracts: print("Abgabeverträge: " f"{cstats['contracts']} geladen, {cstats['matched']} Tier-Treffer, " f"{cstats['ambiguous_skipped']} mehrdeutig übersprungen, " f"{cstats['no_match_skipped']} ohne Treffer.") print(" Kontakte: " f"{cstats['buyers_created']} neu, {cstats['buyers_existing']} bestehend.") print(" Gesetzt: " f"ReceiverContactId={cstats['receiver_set']}, " f"GoHomeDate={cstats['gohome_set']}, " f"Status=GivenAway={cstats['status_givenaway']}, " f"Konflikte={cstats['conflicts']}.") print(" Vertragszeilen: " f"{cstats['records_created']} angelegt " f"({cstats['records_with_animal']} mit Tier, " f"{cstats['records_with_date']} mit Originaldatum), " f"{cstats['no_buyer_skipped']} ohne Käufer übersprungen, " f"{cstats['dateless_skipped']} ohne Datum übersprungen.") # Re-materialise contacts so freshly created buyer contacts are exported. resolved_contacts = list(contact_by_norm_name.values()) # Datenherkunft for litters: which source files contributed, whether this is # a Wurfchronik litter vs a Stammbaum-reconstructed ("virtual") litter, how # many raw records merged into it, plus human-readable notes. Accumulators # (_source_files/_merged_count/_virtual) were filled during the two dedup # passes above; strip them after use. for l in resolved_litters: l_source_files = l.pop("_source_files", set()) l_merged_count = l.pop("_merged_count", 1) is_virtual = l.pop("_virtual", False) l_from_wurfchronik = any("wurfchronik" in str(f).lower() for f in l_source_files) l_notes = [] if is_virtual and not l_from_wurfchronik: l_notes.append("aus Stammbaum-Diagramm rekonstruiert") elif l_from_wurfchronik: l_notes.append("aus Wurfchronik") if l_merged_count > 1: l_notes.append(f"aus {l_merged_count} Datensätzen zusammengeführt") l_notes.append("Geschwister-Würfe zusammengeführt") # Chronological, file-attributed history for the litter. l_files_sorted = sorted({f for f in l_source_files if f}) l_history = [] first_file = l_files_sorted[0] if l_files_sorted else None if is_virtual and not l_from_wurfchronik: if first_file: l_history.append( f"Aus Stammbaum-Diagramm rekonstruiert ({_quote_file(first_file)})." ) else: l_history.append("Aus Stammbaum-Diagramm rekonstruiert.") elif first_file: l_history.append(f"Wurf aus Wurfchronik {_quote_file(first_file)}.") else: l_history.append("Wurf im Import gefunden.") for f in l_files_sorted[1:]: l_history.append( f"Auch in {_quote_file(f)} gefunden → Datensätze zusammengeführt." ) if l_merged_count > 1: l_history.append("Geschwister-Würfe zusammengeführt.") l["Provenance"] = build_entity_provenance( l_source_files, l_merged_count, notes=l_notes, from_wurfchronik=l_from_wurfchronik, history=l_history, ) # Set IsBreeder and IsReceiver flags on contacts breeder_ids = {g["OriginContactId"] for g in resolved_gerbils if g.get("OriginContactId")} receiver_ids = {g["ReceiverContactId"] for g in resolved_gerbils if g.get("ReceiverContactId")} for c in resolved_contacts: c_id = c["Id"] is_breeder = c_id in breeder_ids is_receiver = c_id in receiver_ids if not is_breeder and not is_receiver: is_receiver = True c["IsBreeder"] = is_breeder c["IsReceiver"] = is_receiver # Datenherkunft: where this (deduplicated) contact came from, plus the # role we inferred. Accumulator keys (_source_files/_merged_count) were # filled during the contact dedup above; strip them after use. c_source_files = c.pop("_source_files", set()) c_merged_count = c.pop("_merged_count", 1) c_notes = [] if c_merged_count > 1: c_notes.append(f"aus {c_merged_count} Datensätzen zusammengeführt") if is_breeder: c_notes.append("als Züchter erkannt") if is_receiver: c_notes.append("als Abnehmer erkannt") # Chronological, file-attributed history for the contact. c_files_sorted = sorted({f for f in c_source_files if f}) c_history = [] role_word = "Züchter" if is_breeder else "Abnehmer" if c_files_sorted: c_history.append(f"In {_quote_file(c_files_sorted[0])} als {role_word} erkannt.") for f in c_files_sorted[1:]: c_history.append( f"Auch in {_quote_file(f)} gefunden → Datensätze zusammengeführt." ) else: c_history.append(f"Im Import als {role_word} erkannt.") if is_breeder and is_receiver: c_history.append("Sowohl als Züchter als auch als Abnehmer geführt.") c["Provenance"] = build_entity_provenance( c_source_files, c_merged_count, notes=c_notes, history=c_history ) # Set and map gerbilPhotos resolved_photos = [] for g in resolved_gerbils: for idx, photo_rel in enumerate(g.get("_photos", [])): photo_guid = generate_guid(f"photo-{photo_rel}") ext = os.path.splitext(photo_rel)[1] or ".jpeg" fn_guid = photo_guid.replace("-", "") resolved_photos.append({ "Id": photo_guid, "GerbilId": g["Id"], "FileName": f"{fn_guid}{ext}", "SortOrder": idx, "_source_path": photo_rel }) # 6. Save final output JSON payload os.makedirs(OUTPUT_DIR, exist_ok=True) payload = { "contacts": resolved_contacts, "litters": resolved_litters, "gerbils": resolved_gerbils, "gerbilPhotos": resolved_photos, "saleContracts": sale_contracts, } with open(OUTPUT_FILE, "w", encoding="utf-8") as f: json.dump(payload, f, ensure_ascii=False, indent=2) print(f"Successfully wrote database-ready import file to: {OUTPUT_FILE}") print(f" Contacts: {len(payload['contacts'])}") print(f" Litters: {len(payload['litters'])}") print(f" Gerbils: {len(payload['gerbils'])}") print(f" Photos: {len(payload['gerbilPhotos'])}") print(f" SaleContracts: {len(payload['saleContracts'])}") if __name__ == "__main__": main()