import os import json import re import uuid import sys from datetime import datetime # Prevent encoding crashes on Windows consoles when printing unicode if sys.platform.startswith('win'): try: sys.stdout.reconfigure(encoding='utf-8') except Exception: pass # Paths DIR_PATH = r"C:\Users\gulum\dev\Wurfchronik_Bilder" SEEDS_PATH = r"C:\Users\gulum\dev\GerbilManager\gerbil-manager-web\src\genetics\colorVarietySeed.backend.json" OUTPUT_DIR = r"C:\Users\gulum\dev\GerbilManager\tools\import\output" OUTPUT_FILE = os.path.join(OUTPUT_DIR, "resolved_import.json") def generate_guid(key_str): """Generate a stable UUID string based on a key.""" return str(uuid.uuid5(uuid.NAMESPACE_DNS, key_str)) def build_entity_provenance(source_files, merged_record_count, notes=None, from_wurfchronik=None, extra=None): """Generic data-provenance JSON builder shared by gerbils, contacts and litters. Mirrors the GerbilProvenance frontend contract: { sourceFiles, mergedRecordCount, fromWurfchronik, notes, ... } `source_files` is any iterable of filenames; `from_wurfchronik` is auto- derived from the filenames when left as None. `extra` may carry entity- specific keys (e.g. parentMethod/parentConfidence for gerbils). Returns a JSON string (stored on the nullable Provenance text column).""" files = sorted({f for f in source_files if f}) if from_wurfchronik is None: from_wurfchronik = any("wurfchronik" in f.lower() for f in files) prov = { "sourceFiles": files, "mergedRecordCount": merged_record_count, "fromWurfchronik": bool(from_wurfchronik), "notes": list(notes or []), } if extra: for k, v in extra.items(): if v is not None: prov[k] = v return json.dumps(prov, ensure_ascii=False) def to_valid_guid(val): if not val: return None val_str = str(val).strip() try: uuid.UUID(val_str) return val_str except ValueError: return generate_guid(val_str) def normalize_name(name): if not name: return "" return "".join(c for c in name.lower() if c.isalnum()) def get_normalized_contact_name(name): if not name: return "", False n = "".join(c for c in name.lower() if c.isalnum()) to_discard = { "chevroletcamarooftopolino", "cindyvprivatzuchtgießen", "cindyvprivatzuchtgiessen", "inuschofblackforest", "stichvonprivatzuchtgießen", "stichvonprivatzuchtgiessen", "tarzanofsamsimar", "hanserennersposeidon", "livingforcesidefix", "livingforcesnando" } if n in to_discard: return "", False norm_map = { "alessandrab": "Alessandra Bartoletti", "alexsandrab": "Alessandra Bartoletti", "andreafey": "Andrea und Stefanie Fey", "andreastefaniefey": "Andrea und Stefanie Fey", "anettw": "Annett Wernecke", "angelachristoph": "Angela und Ekki Christoph", "angelaekkichristoph": "Angela und Ekki Christoph", "angie": "Angie Reichert-Cambeis", "angierc": "Angie Reichert-Cambeis", "anjasch": "Anja Schaumburg", "ankeb": "Anke Busch", "ankeksch": "Anke Koppenhöfer", "annaq": "Anna Quark", "annikab": "Annika Balser", "annkathrind": "Ann-kathrin Dressler", "arturaleksandrapolamundrzynski": "Artur, Aleksandra und Pola Mundrzynski", "arturaleksandraundpolamundrzynski": "Artur, Aleksandra und Pola Mundrzynski", "astridr": "Astrid Rohmann", "astridroh": "Astrid Rohmann", "babarastehle": "Barbara Stehle", "barbarar": "Barbara Riegler", "biancab": "Bianca Bernhardt", "biancam": "Bianca Mayer + Alex Wachten", "biancamayeralexw": "Bianca Mayer + Alex Wachten", "birgitk": "Birgit Knopp", "birgitkropp": "Birgit Knopp", "birgittabüskens": "Birgitta Müller-Büskens", "birgittamüllerbüskens": "Brigitta Müller-Büskens", "blackforestgv": "Black Forest", "brigittast": "Brigitta Struve", "buntefellnasen": "bunten Fellnasen", "buntenfellnase": "bunten Fellnasen", "buntenfellnasen": "bunten Fellnasen", "carolal": "Carola und Hannah Lerch", "carolalerch": "Carola und Hannah Lerch", "chalfontstud": "Chalfont Stud, Freddy Braun", "christianefuchs": "Christiane Fuchs + Michael Höhnert", "christiek": "Christiane Kiessling", "christinam": "Christine Monika Mai", "christinek": "Christiane Kiessling", "christiner": "Christina Rudloff", "claudialöhr": "Claudia Löhr und Michael Koob", "claudialöhrmichaelkoob": "Claudia Löhr und Michael Koob", "colorfulfurrygerbils": "Colorful Furry Gerbils", "colourfulfurrygerbils": "Colorful Furry Gerbils", "danielakasberger": "Daniela Käsberger", "dörthe": "Dörthe Petzmann", "emelyhehlhorn": "Emely Mehlhorn", "fabianb": "Fabian Büdel", "floriang": "Florian Gries", "hannahraths": "Hannah Ruths", "hanserenners": "Hanse Renner", "haraldmariareitzlennemann": "Harald und Maria Reitz-Lennemann", "haraldundmariarl": "Harald und Maria Reitz-Lennemann", "heike": "Heike + Heiko Scheurich", "heikebarklay": "Heike Barklage", "heikoloos": "Heiko Loos und Annette Becker", "heikoloosannettebecker": "Heiko Loos und Annette Becker", "inahübner": "Ina Hübner + Nicolai Thome", "inapeines": "Ina Pleines", "janab": "Jana Beikert", "jasiminweber": "Jasmin Weber und Ronny Reichelt", "jasmin": "Jasmin Weber", "jasminweber": "Jasmin Weber und Ronny Reichelt", "jasminweberromyreichelt": "Jasmin Weber und Ronny Reichelt", "jasminweberronnyreichelt": "Jasmin Weber und Ronny Reichelt", "joystaatberg": "Joy Staalberg", "juliaa": "Julia Ast", "juttam": "Jutta Metz", "karina": "Karina Luft", "karinborgotti": "Karin Borsotti", "katjal": "Katja Leffeck", "kimw": "Kim Waldschmidt", "kkchaos": "KK-Chaos", "kkchaosofkkchaos": "KK-Chaos", "kleinechaoten": "Zucht der kleinen Chaoten", "kleinenchaoten": "Zucht der kleinen Chaoten", "kriegernmitkrallen": "Krieger mit Krallen", "lennylengo": "Lenny Lengo", "lisa": "Lisa und Marcel Kunz", "lisach": "Lisa und Lydia Christ", "lisachrist": "Lisa und Lydia Christ", "lisalydiachrist": "Lisa und Lydia Christ", "littlefellows": "little fellows", "littlerunners": "little runners", "maikef": "Maike Franz", "maintalerpz": "Privatzucht Maintal", "marinau": "Marina Unger", "martinaandreaswestfeld": "Martina und Andreas Westfeld", "martinaw": "Martina und Andreas Westfeld", "martinawestfeld": "Martina und Andreas Westfeld", "meikesch": "Meike und Heiko Sch.", "melanief": "Melanie Fey und Thorben Meier", "melaniefey": "Melanie Fey und Thorben Meier", "melaniefeythorbenmeier": "Melanie Fey und Thorben Meier", "michaeldavidschmitz": "Michael und David Schmitz", "michaelschmitz": "Michael und David Schmitz", "nataliereitz": "Nathalie Reitz", "natascham": "Natascha Marienfeld", "nicolefischler": "Nicole Tischler", "nicolel": "Nicole Lannert", "nicolen": "Nicole Nuzzo", "nicolet": "Nicole Tischler", "nicolew": "Nicole Webersinn", "nicost": "Nico Stamm", "nielsh": "Nils H.", "noelstrahbach": "Noel Strohbach", "nora": "Nora Rudersdorf geb. Holzbach", "norah": "Nora Holzbach", "norarh": "Nora Rudersdorf", "norarudersdorf": "Nora Rudersdorf geb. Holzbach", "oflennylengo": "Lenny Lengo", "pascale": "Pascale Diefenbach", "pascaledießenbach": "Pascale Diefenbach", "patriciap": "Patricia Petry", "paul": "Paul W.", "paula": "Paula Gabler", "paulag": "Paula Gabler", "privatzuchtmuecke": "Privatzucht Mücke", "privatzuchtmücke": "Privatzucht Mücke", "pzmaintal": "Privatzucht Maintal", "pzmuecke": "Privatzucht Mücke", "pzmücke": "Privatzucht Mücke", "pzseligenstadt": "Privatzucht Seligenstadt", "ramonag": "Ramona Gömpel", "ranialößler": "Rania Löffler", "rominahubrich": "Romina Milde / Hubrich", "rominamilde": "Romina Milde / Hubrich", "romonag": "Ramona Gömpel", "ronialöffler": "Rania Löffler", "sandrak": "Sandra Kubas", "sarahlöser": "Sarah Löwer", "sarahlöuer": "Sarah Löwer", "sarahz": "Sarah Zitzer", "saskiaw": "Saskia Wucher", "schlossmaus": "Schlossmäuse", "schlossmäuse": "Schlossmäuse", "schlossmäusen": "Schlossmäuse", "sebastiansch": "Sebastian Schmitt", "serinaberg": "Selina Berg", "silkewolfganghintze": "Silke und Wolfgang Hintze", "smilla": "Smilla H.", "steffi": "Steffi K.", "susanneninat": "Susanne + Nina Thomas", "susannet": "Susanne + Torsten Saum", "theresavalenca": "Theresia Valenca (Kopp)", "theresiavalenca": "Theresia Valenca (Kopp)", "timpf": "Tim Pfeiffer und Michelle Mai", "timpfeifermichellemai": "Tim Pfeiffer und Michelle Mai", "timpfeiffer": "Tim Pfeiffer und Michelle Mai", "timpfeiffermichellemai": "Tim Pfeiffer und Michelle Mai", "tonoböckenseld": "Tono Böckenfeld", "topol": "Topolino", "ulrichmüller": "Ulrich + Angela Müller", "ulrikec": "Ulrike Cordes", "ulriker": "Ulrike Ruppel", "ulrikesch": "Ulrike Schulz", "ulriket": "Ulrike Treml", "vanessab": "Vanessa Becker", "vanessag": "Vanessa Groll", "veragreywitz": "Vera Geywitz", "vonprivatzuchtmaintal": "Privatzucht Maintal", "wolfgangfaus": "Wolfgang Faust", "yvonne": "Yvonne Obendorfer", "zuchtderkleinenchaoten": "Zucht der kleinen Chaoten", } if n in norm_map: return norm_map[n], True return name, True def get_normalized_gerbil_name(name): if not name: return "" n = name.strip() norm_key = "".join(c for c in n.lower() if c.isalnum()) gerbil_norm_map = { "samgenshellyvdbuntenfellnasen": "Sammy gen. Shelly von den bunten Fellnasen", "sammygenshellyvdbuntenfellnasen": "Sammy gen. Shelly von den bunten Fellnasen", "schmidt": "Schmidti", "sheila": "Sheila of Ulmer Strolche", "shinichi": "Shinichi von PZ Mücke", "silenosgenadonis": "Silenos gen. Adonis von den Kleinen Chaoten", "silenosgenadonisvdkleinenchaoten": "Silenos gen. Adonis von den Kleinen Chaoten", "silver": "Silver von den kleinen Chaoten", "snoops": "Snoopsi", "sokrates": "Sokrates von den Kleinen Chaoten", "splash": "Slash", "teiko": "Teiko von den kleinen Chaoten", "trixy": "Trixxy von den Kleinen Chaoten", "unique": "Unique of Wild Dreams", } if norm_key in gerbil_norm_map: return gerbil_norm_map[norm_key] return n def get_call_name(name): if not name: return "" n = name.strip() n = re.sub(r'\[[^\]]+\]$', '', n).strip() n = re.split(r'\s+(?:of|von\s+den|von\s+der|v\.\s?d\.|von)\s+', n, flags=re.IGNORECASE)[0].strip() return n def get_dedup_name_key(name): if not name: return "" n = name.lower().strip() # Strip common suffixes/prefixes and parentheticals n = re.sub(r'\b(?:von\s+den|v\.?\s*d\.?|v\.?o\.?)\s+(?:kleinen\s+)?chaoten\b', '', n) n = re.sub(r'\bvon\s+der\s+schlossm\w+\b', '', n) n = re.sub(r'\bvon\s+der\s+bunten\s+fellnasen?\b', '', n) n = re.sub(r'\bof\s+black\s+forest\b', '', n) n = re.sub(r'\b\(?rv\)?\b', '', n) n = re.sub(r'\bgen\.\s+\w+', '', n) # Clean up parentheses or brackets n = re.sub(r'\(.*?\)', '', n) n = re.sub(r'\[.*?\]', '', n) return "".join(c for c in n if c.isalnum()) def clean_color_name(c_desc): if not c_desc: return "", False # Lowercase and strip c = c_desc.lower().strip() # Check for Schecke is_schecke = False if re.search(r'\bsp\b|\bsp\d|\bsp[*(²³]|\bspotted|\bschecke|[- ]sp\b|\w+sp\b', c): is_schecke = True # Standardize parentheticals for schimmel c = c.replace("(schimmel)", "schimmel") c = c.replace("(schimmel-hell)", "schimmel hell") c = c.replace("(schimmel hell)", "schimmel hell") # Strip schecke/sp markers and any trailing text starting from sp c = re.sub(r'\([- ]?sp(otted)?\)', '', c) # handles (-sp) c = re.sub(r'[- ]?sp(otted)?\b.*', '', c) # handles -sp(k), -sp*(k), -sp, etc. c = re.sub(r'[- ]?schecke\b.*', '', c) c = re.sub(r'[- ]?spotted\b.*', '', c) # Strip any other parentheticals, symbols, or trailing stars/numbers c = re.sub(r'\s*\(.*?\)\s*', ' ', c) c = re.sub(r'[²³*]', '', c) c = c.strip() # Mapping table for abbreviations, typos, and specific combinations mapping = { "antra": "anthrazit", "anthra": "anthrazit", "ankazit": "anthrazit", "antrazit": "anthrazit", "pew": "rew", "bew": "hermelin", "harder": "marder", "kohli": "kohlfuchs", "aligerfuchs": "algierfuchs", "algiesfuchs": "algierfuchs", "schw": "schwarz", "sa": "silberagouti", "a": "agouti", "cp-sa": "cp-silberagouti", "cp-a": "cp-agouti", "cp-a-hell": "cp-agouti-hell", "cp-aisa": "cp-agouti", "cp-a / rcp-sa": "cp-agouti", } if c in mapping: c = mapping[c] return c, is_schecke def resolve_color_and_genotype(color_val, existing_genotype, variety_map, variety_genotypes): if not color_val: return None, existing_genotype color_str = str(color_val).strip() clean_name, is_schecke = clean_color_name(color_str) # Match color in variety_map color_variety_id = None if clean_name in variety_map: color_variety_id = variety_map[clean_name] else: for seed_name, seed_id in variety_map.items(): if seed_name in clean_name or clean_name in seed_name: color_variety_id = seed_id break # Update genotype if it's a Schecke genotype = existing_genotype if is_schecke: if genotype: if "spsp" in genotype: genotype = genotype.replace("spsp", "Spsp") elif "Spsp" not in genotype and "Sp" not in genotype: genotype = f"{genotype} Spsp".strip() else: canonical = variety_genotypes.get(color_variety_id) if canonical: if "spsp" in canonical: genotype = canonical.replace("spsp", "Spsp") else: genotype = f"{canonical} Spsp".strip() else: genotype = "Spsp" return color_variety_id, genotype def parse_date(d): """Convert variations of date formats to YYYY-MM-DD.""" if not d or d == "0001-01-01": return None d = str(d).strip() # Try YYYY-MM-DD if re.match(r"^\d{4}-\d{2}-\d{2}$", d): parts = d.split("-") year = int(parts[0]) if 1900 < year < 2000: year += 100 d = f"{year}-{parts[1]}-{parts[2]}" elif year == 1900: return None return d # Try DD.MM.YYYY or D.M.YY match = re.match(r"^(\d{1,2})\.(\d{1,2})\.(\d{2,4})$", d) if match: day = int(match.group(1)) month = int(match.group(2)) year = int(match.group(3)) if year < 100: year += 2000 elif 1900 < year < 2000: year += 100 elif year == 1900: return None try: return datetime(year, month, day).strftime("%Y-%m-%d") except ValueError: pass # Try ISO timestamp try: dt = datetime.fromisoformat(d.replace("Z", "+00:00")) year = dt.year if 1900 < year < 2000: dt = dt.replace(year=year + 100) return dt.strftime("%Y-%m-%d") elif year == 1900: return None return dt.strftime("%Y-%m-%d") except ValueError: pass return None def date_to_days(dt_str): if not dt_str: return None try: return (datetime.strptime(dt_str, "%Y-%m-%d") - datetime(2000, 1, 1)).days except ValueError: return None def days_to_date(days): import datetime as dt return (dt.datetime(2000, 1, 1) + dt.timedelta(days=int(days))).strftime("%Y-%m-%d") def parse_death_info(notes, status, existing_dod, existing_cod): if not notes: return status, existing_dod, existing_cod has_death_indicator = '†' in notes or 'verstorben' in notes.lower() or 'gestorben' in notes.lower() or 'todesdatum' in notes.lower() or '/+' in notes if '+' in notes: if re.search(r'\+\s*(?:LE|AS|Unbekannt|gestorben|verstorben|tod)\b', notes, re.I) or re.search(r'\+\s*\d{1,2}\.\d{1,2}\.\d{2,4}', notes) or '/+' in notes: has_death_indicator = True resolved_status = status if has_death_indicator: resolved_status = "Deceased" dod = existing_dod cod = existing_cod # Look for date near death indicator found_date = None for m in re.finditer(r'([+†]\s*(?:LE|AS|Unbekannt|[a-zA-ZäöüÄÖÜß0-9()/ +,;.:-]{1,100}?)?\s*)(\d{1,2}\.\d{1,2}\.\d{2,4})', notes, re.I): marker_text = m.group(0) if '†' in marker_text or re.search(r'\+\s*(?:LE|AS|Unbekannt|gestorben|verstorben|tod|\d)', marker_text, re.I): found_date = parse_date(m.group(2)) if found_date: break if not found_date: m_death_marker = re.search(r'(†\s*)(\d{1,2}\.\d{1,2}\.\d{2,4})', notes) if m_death_marker: found_date = parse_date(m_death_marker.group(2)) if found_date and not dod: dod = found_date if not cod: for m in re.finditer(r'([+†])\s*([a-zA-ZäöüÄÖÜß0-9()/ +,;.:-]{1,100}?)\s*\d{1,2}\.\d{1,2}\.\d{2,4}', notes, re.I): indicator = m.group(1) cod_candidate = m.group(2).strip() if indicator == '+' and not re.search(r'\b(?:LE|AS|Unbekannt|gestorben|verstorben|tod)\b', cod_candidate, re.I): continue if cod_candidate: cod = cod_candidate break if not cod: m_cod2 = re.search(r'[+†]\s*\d{1,2}\.\d{1,2}\.\d{2,4}\s*([a-zA-ZäöüÄÖÜß0-9()/ +,;.:-]{1,100})', notes, re.I) if m_cod2: cod_candidate = m_cod2.group(1).strip() if cod_candidate: cod = cod_candidate else: m_cod3 = re.search(r'[+†]\s*(LE|AS|Unbekannt)\b', notes, re.I) if m_cod3: cod = m_cod3.group(1).strip() if cod: cod_lower = cod.lower().strip() if cod_lower in ("le", "le (lungenentzündung)", "lungenentzündung", "lungenentzündung (lungenentzündung)"): cod = "Lungenentzündung" elif cod_lower in ("as", "altersschwäche", "altenschwäche", "alter"): cod = "Altersschwäche" elif cod_lower in ("unbekannt", "unklar"): cod = "Unbekannt" else: # Replace abbreviations with full names (case-insensitive) cod = re.sub(r'\bLE\b', 'Lungenentzündung', cod, flags=re.I) cod = re.sub(r'\bAS\b', 'Altersschwäche', cod, flags=re.I) cod = cod.replace('+', ' + ') # Clean up multiple spaces cod = re.sub(r'\s+', ' ', cod).strip(' ,;.-') # Re-check after replacement cod_lower = cod.lower().strip() if cod_lower in ("le", "le (lungenentzündung)", "lungenentzündung", "lungenentzündung (lungenentzündung)"): cod = "Lungenentzündung" elif cod_lower in ("as", "altersschwäche", "altenschwäche", "alter"): cod = "Altersschwäche" elif cod_lower in ("unbekannt", "unklar"): cod = "Unbekannt" return resolved_status, dod, cod # ── Litter dedup & parent-role helpers (pure, unit-tested in test_merge_resolve.py) ── def _norm_pname(s): return normalize_name(s) if s else "" def names_no_conflict(l1, l2): """Parent names don't contradict (equal per role, or one side empty).""" f1, f2 = _norm_pname(l1.get("_father_name")), _norm_pname(l2.get("_father_name")) m1, m2 = _norm_pname(l1.get("_mother_name")), _norm_pname(l2.get("_mother_name")) f_ok = (not f1) or (not f2) or (f1 == f2) m_ok = (not m1) or (not m2) or (m1 == m2) return f_ok and m_ok def names_overlap(l1, l2): """At least one role has a non-empty matching name (positive evidence).""" f1, f2 = _norm_pname(l1.get("_father_name")), _norm_pname(l2.get("_father_name")) m1, m2 = _norm_pname(l1.get("_mother_name")), _norm_pname(l2.get("_mother_name")) return bool((f1 and f1 == f2) or (m1 and m1 == m2)) def litter_compatible(l1, l2): """Two litter records describe the same litter: same date and compatible parents. - Both sides have both parents → must match exactly. - Asymmetric (one side resolved, the other not) → merge only if names don't contradict; for dateless litters require a POSITIVE name match (a shared null date is no evidence), so unrelated nameless stubs stay separate. - Neither side has parents → never blind-merge. """ if l1["Date"] != l2["Date"]: return False f1, m1 = l1.get("FatherId"), l1.get("MotherId") f2, m2 = l2.get("FatherId"), l2.get("MotherId") if f1 and f2 and m1 and m2: return f1 == f2 and m1 == m2 asymmetric = (bool(f1 or m1) and not (f2 or m2)) or (bool(f2 or m2) and not (f1 or m1)) if asymmetric: if not names_no_conflict(l1, l2): return False if l1["Date"] is None: return names_overlap(l1, l2) return True return False def assign_parent_roles(father_id, mother_id, gender_of): """Assign two resolved parent IDs to father/mother roles by gender. Drops self-pairing duplicates (same animal in both roles) and never returns two same-role parents. `gender_of` maps an id to 'male'|'female'|'unknown'|None. Returns (father_id, mother_id). """ ids = [] for gid in (father_id, mother_id): if gid and gid not in ids: ids.append(gid) males = [g for g in ids if gender_of(g) == "male"] females = [g for g in ids if gender_of(g) == "female"] unknowns = [g for g in ids if gender_of(g) == "unknown"] father = males[0] if males else (unknowns.pop(0) if unknowns else None) mother = females[0] if females else (unknowns.pop(0) if unknowns else None) return father, mother # A gerbil lives at most ~6 years, so a parent can be at most ~6 years older than # its offspring (and must be born before it). Links outside this window are # impossible — e.g. a 2013 animal resolved onto a 2022 litter (Jayjay → Solice). MAX_PARENT_AGE_DAYS = 6 * 366 def parent_age_plausible(parent_dob, litter_date): """Could a parent born `parent_dob` have offspring born on `litter_date`? Requires birth strictly before the litter and within the gerbil lifespan. Unknown/unparseable dates return True (cannot disprove). Accepts any date format parse_date understands. """ pd = date_to_days(parse_date(parent_dob)) if parent_dob else None ld = date_to_days(parse_date(litter_date)) if litter_date else None if pd is None or ld is None: return True return 0 < (ld - pd) <= MAX_PARENT_AGE_DAYS def pick_parent_ref(parent_refs, role, child_dob, avoid_name=None, gender_of=None): """Choose the best parent ref for a role from possibly-conflicting chart refs. A Stammbaum lists an animal at several positions, so its parentRefs can carry contradictory guesses (the first one is not necessarily right). Rank candidates (lower = better): 0 right/unknown gender for the role, age-plausible dated ref 1 right/unknown gender, no DOB (usable, but a plausible dated ref wins) 2 right/unknown gender, dated but age-impossible 3 resolved gender is clearly WRONG for the role (e.g. a female father) 4 would duplicate the animal chosen for the other role Gender is decisive over DOB: a dated female ref must not win the father slot over an undated male/unknown one. `gender_of(name)` returns 'male'/'female' or None (unknown/ambiguous → not penalised). Returns the chosen ref or None. """ role_refs = [p for p in parent_refs if p.get("roleGuess") == role] if not role_refs: return None avoid = normalize_name(avoid_name) if avoid_name else None expected = "male" if role == "father" else "female" def rank(p): if avoid is not None and normalize_name(p.get("name")) == avoid: return 4 # would duplicate the other parent role g = gender_of(p.get("name")) if gender_of else None if g in ("male", "female") and g != expected: return 3 # wrong sex for this role dob = p.get("dob") if not dob: return 1 return 0 if parent_age_plausible(dob, child_dob) else 2 order = sorted(range(len(role_refs)), key=lambda i: (rank(role_refs[i]), i)) return role_refs[order[0]] def main(): print("Loading color variety seeds...") variety_map = {} variety_genotypes = {} # Load from C# ApplicationContext.cs catalog for stable database GUIDs (index + 1) here = os.path.dirname(os.path.abspath(__file__)) app_context_path = os.path.abspath(os.path.join(here, "../../GerbilManagerWebAPI/ApplicationContext.cs")) cs_name_to_id = {} if os.path.exists(app_context_path): with open(app_context_path, 'r', encoding='utf-8') as f: content = f.read() catalog_match = re.search(r'catalog\s*=\s*\{(.*?)\};', content, re.DOTALL) if catalog_match: block = catalog_match.group(1) entries = re.findall(r'\(\s*"([^"]+)"\s*,\s*"([^"]+)"\s*,\s*(\d+)\s*\)', block) for idx, (name, genotype, sort_order) in enumerate(entries): variety_id = f"00000000-0000-0000-0000-{idx + 1:012d}" cs_name_to_id[name.strip().lower()] = variety_id variety_map[name.strip().lower()] = variety_id variety_genotypes[variety_id] = genotype.strip() else: print(f"Warning: ApplicationContext.cs not found at {app_context_path}") if os.path.exists(SEEDS_PATH): with open(SEEDS_PATH, "r", encoding="utf-8") as f: seeds = json.load(f) for v in seeds: name_lower = v["name"].strip().lower() variety_id = cs_name_to_id.get(name_lower) if not variety_id: variety_id = f"00000000-0000-0000-0000-{v['sortOrder'] + 1:012d}" variety_map[name_lower] = variety_id # Map English name if present if "english" in v and v["english"]: variety_map[v["english"].strip().lower()] = variety_id if variety_id not in variety_genotypes: variety_genotypes[variety_id] = v.get("canonicalGenotype") else: print(f"Warning: Seeds path not found at {SEEDS_PATH}") md_files = sorted([f for f in os.listdir(DIR_PATH) if f.lower().endswith('.md')]) print(f"Found {len(md_files)} markdown files in {DIR_PATH}.") raw_contacts = [] raw_litters = [] raw_gerbils = [] # Map to track which files contained explicit dates file_explicit_dates = {} # 1. Parse JSON blocks from all markdown files and scope IDs by filename for filename in md_files: filepath = os.path.join(DIR_PATH, filename) with open(filepath, 'r', encoding='utf-8') as f: content = f.read() json_start = content.find("```json") if json_start == -1: continue json_start += len("```json") json_end = content.rfind("```") if json_end == -1 or json_end <= json_start: continue json_str = content[json_start:json_end].strip() try: data = json.loads(json_str) contacts_key = next((k for k in data if k.lower() == 'contacts'), None) litters_key = next((k for k in data if k.lower() == 'litters'), None) gerbils_key = next((k for k in data if k.lower() == 'gerbils'), None) file_contacts = data.get(contacts_key, []) if contacts_key else [] file_litters = data.get(litters_key, []) if litters_key else [] file_gerbils = data.get(gerbils_key, []) if gerbils_key else [] # Find explicit dates on this page to build chronological ordering explicit_dates = [] for l in file_litters: d = parse_date(l.get("Date") or l.get("date") or l.get("DateOfBirth") or l.get("dateOfBirth")) if d: explicit_dates.append(d) for g in file_gerbils: d = parse_date(g.get("DateOfBirth") or g.get("dateOfBirth")) if d: explicit_dates.append(d) if explicit_dates: # Convert to days since 2000-01-01 day_vals = [date_to_days(d) for d in explicit_dates if date_to_days(d) is not None] if day_vals: file_explicit_dates[filename] = sum(day_vals) / len(day_vals) def scope_id(local_id): if not local_id: return None local_str = str(local_id).strip() if not local_str: return None return generate_guid(f"{filename}-{local_str}") for c in file_contacts: old_id = c.get("Id") or c.get("id") c["_filename"] = filename c["_scoped_id"] = scope_id(old_id) raw_contacts.append(c) for l in file_litters: old_id = l.get("Id") or l.get("id") l["_filename"] = filename l["_scoped_id"] = scope_id(old_id) l["_scoped_father_id"] = scope_id(l.get("FatherId") or l.get("fatherId")) l["_scoped_mother_id"] = scope_id(l.get("MotherId") or l.get("motherId")) raw_litters.append(l) for g in file_gerbils: old_id = g.get("Id") or g.get("id") g["_filename"] = filename g["_scoped_id"] = scope_id(old_id) g["_scoped_litter_id"] = scope_id(g.get("LitterId") or g.get("litterId")) g["_scoped_origin_cid"] = scope_id(g.get("OriginContactId") or g.get("originContactId")) receiver_val = (g.get("ReceiverContactId") or g.get("receiverContactId") or g.get("BuyerId") or g.get("buyerId") or g.get("BuyerContactId") or g.get("buyerContactId") or g.get("givenAwayContactId") or g.get("givenAwayToContactId") or g.get("ownerContactId") or g.get("ownerId")) g["_scoped_receiver_cid"] = scope_id(receiver_val) raw_gerbils.append(g) except json.JSONDecodeError as e: print(f"Failed to parse JSON in {filename}: {e}") # Chronological Interpolation: Estimate the date of each page based on neighboring pages with dates file_dates = {} sorted_files = sorted(md_files) # Simple linear interpolation / extrapolation for i, fn in enumerate(sorted_files): if fn in file_explicit_dates: file_dates[fn] = file_explicit_dates[fn] else: # Look left for closest explicit date left_val, left_dist = None, None for j in range(i - 1, -1, -1): if sorted_files[j] in file_explicit_dates: left_val = file_explicit_dates[sorted_files[j]] left_dist = i - j break # Look right for closest explicit date right_val, right_dist = None, None for j in range(i + 1, len(sorted_files)): if sorted_files[j] in file_explicit_dates: right_val = file_explicit_dates[sorted_files[j]] right_dist = j - i break if left_val is not None and right_val is not None: # Interpolate file_dates[fn] = left_val + (right_val - left_val) * (left_dist / (left_dist + right_dist)) elif left_val is not None: # Extrapolate right (assume 30 days per page gap as placeholder) file_dates[fn] = left_val + (left_dist * 30) elif right_val is not None: # Extrapolate left file_dates[fn] = right_val - (right_dist * 30) else: # No dates in entire log? Default to 2011-01-01 file_dates[fn] = date_to_days("2011-01-01") print(f"Parsed {len(raw_contacts)} raw contacts, {len(raw_litters)} raw litters, {len(raw_gerbils)} raw gerbils.") # Run extract.py to make sure stammbaum data is up to date import subprocess print("Running extract.py to extract Stammbäume...") try: subprocess.run([sys.executable, "extract.py"], check=True) except Exception as e: print(f"Warning: Failed to run extract.py: {e}") # Load Stammbaum data stammbaum_only_animals = [] animals_path = os.path.join(OUTPUT_DIR, "animals.json") if os.path.exists(animals_path): with open(animals_path, "r", encoding="utf-8") as f: stammbaum_animals = json.load(f) for a in stammbaum_animals: sources = a.get("sourceFiles", []) if any("stammbaum" in str(s).lower() for s in sources): stammbaum_only_animals.append(a) print(f"Loaded {len(stammbaum_only_animals)} Stammbaum animals.") else: print(f"Warning: Stammbaum animals.json not found at {animals_path}") # Run extract_docx.py to make sure docx data is up to date print("Running extract_docx.py to extract docx...") try: subprocess.run([sys.executable, "extract_docx.py"], check=True) except Exception as e: print(f"Warning: Failed to run extract_docx.py: {e}") # Load Docx data docx_animals = [] docx_animals_path = os.path.join(OUTPUT_DIR, "docx_animals.json") if os.path.exists(docx_animals_path): with open(docx_animals_path, "r", encoding="utf-8") as f: docx_animals = json.load(f) print(f"Loaded {len(docx_animals)} animals from docx.") else: print(f"Warning: docx_animals.json not found at {docx_animals_path}") docx_litters = [] docx_litters_path = os.path.join(OUTPUT_DIR, "docx_litters.json") if os.path.exists(docx_litters_path): with open(docx_litters_path, "r", encoding="utf-8") as f: docx_litters = json.load(f) print(f"Loaded {len(docx_litters)} litters from docx.") else: print(f"Warning: docx_litters.json not found at {docx_litters_path}") # Extract docx buyer contacts and add to raw_contacts for da in docx_animals: o_name = (da.get("owner") or "").strip() if o_name: o_scoped_id = generate_guid(f"docx-contact-{normalize_name(o_name)}") raw_contacts.append({ "Name": o_name, "_filename": "Wurfchronik-Detail", "_scoped_id": o_scoped_id }) # Map docx litters and append to raw_litters docx_litter_id_map = {} # wsCode -> scoped_id for dl in docx_litters: l_name = dl["litterId"] dob_val = parse_date(dl["dob"]) f_name = get_normalized_gerbil_name(dl["fatherName"]) m_name = get_normalized_gerbil_name(dl["motherName"]) ws_code = dl["wsCode"] note_val = dl.get("note") # Parse survived/total born from wsCode (e.g. 4/5) total_born = None deaths_8w = None if "/" in ws_code: parts = ws_code.split("/") if len(parts) == 2: try: survived = int(parts[0]) total = int(parts[1]) total_born = total deaths_8w = max(0, total - survived) except ValueError: pass l_scoped_id = generate_guid(f"docx-litter-{normalize_name(ws_code)}-{dob_val or '0001-01-01'}") docx_litter_id_map[(ws_code, dob_val)] = l_scoped_id raw_litters.append({ "Id": l_scoped_id, "Name": l_name, "Date": dob_val, "TotalBorn": total_born, "DeathsWithin8Weeks": deaths_8w, "FatherId": generate_guid(f"stammbaum-animal-{normalize_name(f_name)}"), # placeholder "MotherId": generate_guid(f"stammbaum-animal-{normalize_name(m_name)}"), # placeholder "ExpectedGoHomeDate": None, "Notes": note_val if note_val else "Docx imported litter", "PairingCode": None, "ExternalRef": f"docx-litter-{l_scoped_id}", "LitterLetter": l_name[0] if l_name and len(l_name) > 0 else None, "_father_name": f_name, "_mother_name": m_name, "_filename": "Wurfchronik-Detail", "_scoped_id": l_scoped_id, "_scoped_father_id": generate_guid(f"stammbaum-animal-{normalize_name(f_name)}"), "_scoped_mother_id": generate_guid(f"stammbaum-animal-{normalize_name(m_name)}") }) # Extract stammbaum contacts and add to raw_contacts for a in stammbaum_only_animals: b_name = (a.get("breeder") or "").strip() if b_name: b_scoped_id = generate_guid(f"stammbaum-contact-{normalize_name(b_name)}") raw_contacts.append({ "Name": b_name, "_filename": "Stammbaum", "_scoped_id": b_scoped_id }) z_name = (a.get("zucht") or "").strip() if z_name: z_scoped_id = generate_guid(f"stammbaum-contact-{normalize_name(z_name)}") raw_contacts.append({ "Name": z_name, "_filename": "Stammbaum", "_scoped_id": z_scoped_id }) # Residency propagation for stammbaum animals def is_clan_zucht(z): if not z: return False norm = z.lower() return "klein" in norm and "chaot" in norm and "extern" not in norm stammbaum_resident_ids = set() for a in stammbaum_only_animals: if is_clan_zucht(a.get("zucht")) or is_clan_zucht(a.get("zuchtCanon")): stammbaum_resident_ids.add(a["id"]) # Propagate residency to parents of resident offspring for _ in range(5): for a in stammbaum_only_animals: if a["id"] in stammbaum_resident_ids: for p_ref in a.get("parentRefs", []): p_key = (normalize_name(p_ref["name"]), parse_date(p_ref["dob"])) for cand in stammbaum_only_animals: if normalize_name(cand["name"]) == p_key[0]: cand_dob = parse_date(cand["dob"]) if not p_key[1] or cand_dob == p_key[1]: stammbaum_resident_ids.add(cand["id"]) # Pre-index Wurfchronik litters from markdown md_litters_idx = {} for rl in raw_litters: f_name = get_normalized_gerbil_name(rl.get("FatherName") or rl.get("fatherName") or rl.get("ParentMaleName") or rl.get("parentMaleName")) m_name = get_normalized_gerbil_name(rl.get("MotherName") or rl.get("motherName") or rl.get("ParentFemaleName") or rl.get("parentFemaleName")) ldate = parse_date(rl.get("Date") or rl.get("date") or rl.get("DateOfBirth") or rl.get("dateOfBirth")) if f_name and m_name and ldate: key = (normalize_name(f_name), normalize_name(m_name), ldate) md_litters_idx[key] = rl # Gender index for parent-ref selection: normalized name → 'male' | 'female' # | 'ambiguous'. Drives the gender-aware ranking in pick_parent_ref so a dated # but wrong-sex ref (e.g. female „Danielle“) cannot win the father slot over # an undated male/unknown one (e.g. „Hagrid Rubeus“). gender_idx = {} for a in stammbaum_only_animals: g = (a.get("gender") or "").lower().strip() g = g if g in ("male", "female") else None for key in {normalize_name(a.get("name")), normalize_name(get_call_name(a.get("name") or ""))}: if not key: continue if key not in gender_idx: gender_idx[key] = g elif gender_idx[key] != g: gender_idx[key] = "ambiguous" def gender_of_name(name): v = gender_idx.get(normalize_name(name)) return v if v in ("male", "female") else None # Create virtual litters for stammbaum animals created_virtual_litters = {} for a in stammbaum_only_animals: parent_refs = a.get("parentRefs", []) child_dob_raw = a.get("dob") father_ref = pick_parent_ref(parent_refs, "father", child_dob_raw, gender_of=gender_of_name) mother_ref = pick_parent_ref(parent_refs, "mother", child_dob_raw, avoid_name=father_ref.get("name") if father_ref else None, gender_of=gender_of_name) a["_mapped_litter_scoped_id"] = None if father_ref and mother_ref: f_name = get_normalized_gerbil_name(father_ref.get("name")) m_name = get_normalized_gerbil_name(mother_ref.get("name")) dob_val = parse_date(a.get("dob")) mapped_litter = None if dob_val: key = (normalize_name(f_name), normalize_name(m_name), dob_val) mapped_litter = md_litters_idx.get(key) if mapped_litter: a["_mapped_litter_scoped_id"] = mapped_litter["_scoped_id"] else: v_key = (normalize_name(f_name), normalize_name(m_name), dob_val or "0001-01-01") if v_key in created_virtual_litters: a["_mapped_litter_scoped_id"] = created_virtual_litters[v_key] else: l_scoped_id = generate_guid(f"virtual-litter-{v_key[0]}-{v_key[1]}-{v_key[2]}") created_virtual_litters[v_key] = l_scoped_id a["_mapped_litter_scoped_id"] = l_scoped_id # Try to link parents to actual parsed stammbaum animals f_scoped_id = None m_scoped_id = None f_dob = parse_date(father_ref.get("dob")) m_dob = parse_date(mother_ref.get("dob")) for p_cand in stammbaum_only_animals: p_gender = str(p_cand.get("gender") or "").lower().strip() if p_gender in ["w", "f", "female", "weiblich"]: continue p_dob = parse_date(p_cand.get("dob")) if dob_val and p_dob and p_dob >= dob_val: continue cand_call_norm = normalize_name(get_call_name(p_cand["name"])) or "unbekannt" cand_name_norm = normalize_name(p_cand["name"]) or "unbekannt" if cand_call_norm == normalize_name(f_name) or cand_name_norm == normalize_name(f_name): if not f_dob or p_dob == f_dob: f_scoped_id = generate_guid(f"stammbaum-animal-{p_cand['id']}") break for p_cand in stammbaum_only_animals: p_gender = str(p_cand.get("gender") or "").lower().strip() if p_gender in ["m", "male", "männlich"]: continue p_dob = parse_date(p_cand.get("dob")) if dob_val and p_dob and p_dob >= dob_val: continue cand_call_norm = normalize_name(get_call_name(p_cand["name"])) or "unbekannt" cand_name_norm = normalize_name(p_cand["name"]) or "unbekannt" if cand_call_norm == normalize_name(m_name) or cand_name_norm == normalize_name(m_name): if not m_dob or p_dob == m_dob: m_scoped_id = generate_guid(f"stammbaum-animal-{p_cand['id']}") break raw_litters.append({ "Id": l_scoped_id, "Name": f"Wurf von {f_name} + {m_name}", "Date": dob_val, "TotalBorn": None, "DeathsWithin8Weeks": None, "FatherId": f_scoped_id or generate_guid(f"stammbaum-animal-{normalize_name(f_name)}"), "MotherId": m_scoped_id or generate_guid(f"stammbaum-animal-{normalize_name(m_name)}"), "ExpectedGoHomeDate": None, "Notes": "Pedigree virtual litter", "PairingCode": None, "ExternalRef": f"virtual-{l_scoped_id}", "LitterLetter": None, "_father_name": f_name, "_mother_name": m_name, "_filename": "Stammbaum", "_scoped_id": l_scoped_id, "_scoped_father_id": f_scoped_id or generate_guid(f"stammbaum-animal-{normalize_name(f_name)}"), "_scoped_mother_id": m_scoped_id or generate_guid(f"stammbaum-animal-{normalize_name(m_name)}") }) # 2. Resolve Contacts globally (deduplicate by normalized name) contact_by_norm_name = {} contact_id_map = {} # scoped_old_id -> global_guid for rc in raw_contacts: name_val = rc.get("Name") or rc.get("name") or rc.get("FullName") or rc.get("fullName") if not name_val: first = rc.get("FirstName") or rc.get("firstName") last = rc.get("LastName") or rc.get("lastName") if first or last: name_val = f"{first or ''} {last or ''}".strip() if not name_val: continue canon_name, should_keep = get_normalized_contact_name(name_val) scoped_id = rc["_scoped_id"] if not should_keep: if scoped_id: contact_id_map[scoped_id] = None continue norm_name = normalize_name(canon_name) rc_file = rc.get("_filename") if norm_name not in contact_by_norm_name: global_guid = generate_guid(f"contact-{norm_name}") contact_by_norm_name[norm_name] = { "Id": global_guid, "Name": canon_name, "Email": rc.get("Email") or rc.get("email"), "Phone": rc.get("Phone") or rc.get("phone"), "Address": rc.get("Address") or rc.get("address"), "Notes": rc.get("Notes") or rc.get("notes") or rc.get("Note") or rc.get("note"), # Provenance accumulators (consumed below, stripped from helper keys). "_source_files": set([rc_file]) if rc_file else set(), "_merged_count": 1, } else: gc = contact_by_norm_name[norm_name] if not gc["Email"] and (rc.get("Email") or rc.get("email")): gc["Email"] = rc.get("Email") or rc.get("email") if not gc["Phone"] and (rc.get("Phone") or rc.get("phone")): gc["Phone"] = rc.get("Phone") or rc.get("phone") if not gc["Address"] and (rc.get("Address") or rc.get("address")): gc["Address"] = rc.get("Address") or rc.get("address") if not gc["Notes"] and (rc.get("Notes") or rc.get("notes") or rc.get("Note") or rc.get("note")): gc["Notes"] = rc.get("Notes") or rc.get("notes") or rc.get("Note") or rc.get("note") if rc_file: gc["_source_files"].add(rc_file) gc["_merged_count"] += 1 if scoped_id: contact_id_map[scoped_id] = contact_by_norm_name[norm_name]["Id"] resolved_contacts = list(contact_by_norm_name.values()) print(f"Resolved to {len(resolved_contacts)} unique contacts.") # 3. Process Litters (normalize keys) resolved_litters = [] litter_id_map = {} # scoped_old_id -> new_guid litter_by_scoped_id = {} for rl in raw_litters: filename = rl.get("_filename") name_val = rl.get("Name") or rl.get("name") if not name_val: name_val = "Wurf" dob_val = parse_date(rl.get("Date") or rl.get("date") or rl.get("DateOfBirth") or rl.get("dateOfBirth")) new_guid = rl["_scoped_id"] if not new_guid: new_guid = generate_guid(f"litter-{filename}-{name_val}-{dob_val}") scoped_old_id = rl["_scoped_id"] if scoped_old_id: litter_id_map[scoped_old_id] = new_guid total_born = rl.get("TotalBorn") or rl.get("totalBorn") or rl.get("LitterSize") or rl.get("litterSize") or rl.get("size") or rl.get("totalPups") if total_born is not None: try: total_born = int(total_born) except ValueError: total_born = None deaths_8w = rl.get("DeathsWithin8Weeks") or rl.get("deathsWithin8Weeks") if deaths_8w is not None: try: deaths_8w = int(deaths_8w) except ValueError: deaths_8w = None elif total_born is not None and rl.get("survived") is not None: try: deaths_8w = total_born - int(rl.get("survived")) except ValueError: pass father_name = rl.get("FatherName") or rl.get("fatherName") or rl.get("ParentMaleName") or rl.get("parentMaleName") or rl.get("_father_name") mother_name = rl.get("MotherName") or rl.get("motherName") or rl.get("ParentFemaleName") or rl.get("parentFemaleName") or rl.get("_mother_name") raw_ext_ref = rl.get("ExternalRef") or rl.get("externalRef") or rl.get("Id") or rl.get("id") ext_ref_scoped = f"{filename}-{raw_ext_ref}" if raw_ext_ref else None l_record = { "Id": new_guid, "Name": name_val, "Date": dob_val, "TotalBorn": total_born, "DeathsWithin8Weeks": deaths_8w, "FatherId": rl["_scoped_father_id"], "MotherId": rl["_scoped_mother_id"], "ExpectedGoHomeDate": parse_date(rl.get("ExpectedGoHomeDate") or rl.get("expectedGoHomeDate")), "Notes": rl.get("Notes") or rl.get("notes") or rl.get("Note") or rl.get("note"), "PairingCode": rl.get("PairingCode") or rl.get("pairingCode"), "ExternalRef": ext_ref_scoped, "LitterLetter": rl.get("LitterLetter") or rl.get("litterLetter"), "_father_name": father_name, "_mother_name": mother_name, "_filename": filename, # Provenance accumulators (canonical absorbs these during dedup below). "_source_files": set([filename]) if filename else set(), "_merged_count": 1, # Virtual litters are reconstructed from a Stammbaum chart, not the # Wurfchronik — flagged on the raw record's _filename == "Stammbaum". "_virtual": filename == "Stammbaum", } resolved_litters.append(l_record) litter_by_scoped_id[new_guid] = l_record print(f"Processed {len(resolved_litters)} litters.") # 3b. Deduplicate litters: same date + compatible parents → merge # This handles the "sibling pairing" case: Stammbaum shows the same parental # litter twice (once under the father branch, once under the mother branch), # generating two separate litter records with the same date but only one of # them has FatherId/MotherId resolved. litter_canonical_map = {} # old_id -> canonical_id (for dedup within this step) # Group by date for efficiency by_date = {} for l in resolved_litters: by_date.setdefault(l["Date"], []).append(l) litter_dedup_canonical = {} # old_litter_id -> canonical_litter_id deduped_litters = [] for date_val, group in by_date.items(): # Partition into compatible subsets sub_groups = [] for l in group: placed = False for sub in sub_groups: if all(litter_compatible(l, member) for member in sub): sub.append(l) placed = True break if not placed: sub_groups.append([l]) for sub in sub_groups: if len(sub) == 1: deduped_litters.append(sub[0]) litter_dedup_canonical[sub[0]["Id"]] = sub[0]["Id"] continue # Pick the canonical record: prefer the one with parents set canonical = next((l for l in sub if l.get("FatherId") or l.get("MotherId")), sub[0]) for l in sub: litter_dedup_canonical[l["Id"]] = canonical["Id"] if l is not canonical: litter_id_map[l["Id"]] = canonical["Id"] canonical["_source_files"] |= l.get("_source_files", set()) canonical["_merged_count"] += l.get("_merged_count", 1) if not l.get("_virtual"): canonical["_virtual"] = False deduped_litters.append(canonical) if len(sub) > 1: merged_names = [l["Id"] for l in sub if l is not canonical] print(f"Litter-Dedup: merged {len(sub)} same-date litters on {date_val} → {canonical['Name']} (absorbed: {', '.join(merged_names)})") n_merged = len(resolved_litters) - len(deduped_litters) if n_merged: print(f"Litter-Dedup: {n_merged} redundant litter record(s) removed.") resolved_litters = deduped_litters litter_by_scoped_id = {l["Id"]: l for l in resolved_litters} # 4. Normalize and group Gerbils # Helper to lookup litter dates for birth date estimation def get_litter_date(l_id): if l_id in litter_by_scoped_id: d = litter_by_scoped_id[l_id]["Date"] if d: return d return None all_processed_gerbils = [] for rg in raw_gerbils: filename = rg.get("_filename") name_val = get_normalized_gerbil_name(rg.get("Name") or rg.get("name") or rg.get("callName")) if not name_val: name_val = "Unbekannt" dob_val = parse_date(rg.get("DateOfBirth") or rg.get("dateOfBirth")) new_guid = rg["_scoped_id"] if not new_guid: new_guid = generate_guid(f"gerbil-{filename}-{name_val}-{dob_val}") # Gender normalization gender_val = rg.get("Gender") or rg.get("gender") or "unknown" gender_val = str(gender_val).lower().strip() if gender_val in ["m", "male", "männlich"]: gender = "male" elif gender_val in ["w", "f", "female", "weiblich"]: gender = "female" else: gender = "unknown" dod_val = parse_date(rg.get("DateOfDeath") or rg.get("dateOfDeath") or rg.get("deathDate")) gohome_val = parse_date(rg.get("GoHomeDate") or rg.get("goHomeDate") or rg.get("DateGivenAway") or rg.get("givenAwayDate") or rg.get("DateOfHandover") or rg.get("dateOfHandover") or rg.get("HandoverDate") or rg.get("DateOfSale") or rg.get("dateOfSale")) status_val = rg.get("Status") or rg.get("status") or "" status_val = str(status_val).lower().strip() status = "Breeding" if dod_val or "deceased" in status_val or "verstorben" in status_val or "tod" in status_val or "dead" in status_val: status = "Deceased" elif gohome_val or "givenaway" in status_val or "abgegeben" in status_val or "verkauft" in status_val or "sold" in status_val: status = "GivenAway" elif "forsale" in status_val or "abzugeben" in status_val: status = "ForSale" elif "pet" in status_val or "liebhaber" in status_val: status = "Pet" else: if dod_val: status = "Deceased" elif gohome_val: status = "GivenAway" notes_val = rg.get("Notes") or rg.get("notes") or rg.get("Note") or rg.get("note") existing_cod = rg.get("CauseOfDeath") or rg.get("causeOfDeath") or rg.get("DeathCause") status, dod_val, cause_of_death_val = parse_death_info(notes_val, status, dod_val, existing_cod) # Explicit resolution for Ken'ichi's cause of death if name_val == "Ken'ichi" and dob_val == "2015-03-01": cause_of_death_val = "Duftdrüsen-Tumor" color_val = rg.get("ColorVarietyId") or rg.get("colorVarietyId") or rg.get("Color") or rg.get("color") or rg.get("ColorDescription") or rg.get("colorDescription") existing_gt = rg.get("Genotype") or rg.get("genotype") color_variety_id, genotype_val = resolve_color_and_genotype( color_val, existing_gt, variety_map, variety_genotypes ) if color_val and not color_variety_id: try: uuid.UUID(str(color_val).strip()) color_variety_id = str(color_val).strip() except ValueError: print(f"Warning: Unknown color variety '{color_val}' for gerbil '{name_val}' on page {filename}") origin_cid = rg["_scoped_origin_cid"] receiver_cid = rg["_scoped_receiver_cid"] if origin_cid in contact_id_map: origin_cid = contact_id_map[origin_cid] if receiver_cid in contact_id_map: receiver_cid = contact_id_map[receiver_cid] is_resident = rg.get("IsResident") or rg.get("isResident") if is_resident is None: is_resident = True else: is_resident = str(is_resident).lower() == "true" traits = rg.get("CharacterTraits") or rg.get("characterTraits") or [] if not isinstance(traits, list): traits = [str(traits)] char_note = rg.get("CharacterNote") or rg.get("characterNote") is_deaf = rg.get("IsDeaf") or rg.get("isDeaf") if is_deaf is not None: is_deaf = str(is_deaf).lower() == "true" old_litter_id = rg["_scoped_litter_id"] raw_ext_ref = rg.get("ExternalRef") or rg.get("externalRef") or rg.get("Id") or rg.get("id") ext_ref_scoped = f"{filename}-{raw_ext_ref}" if raw_ext_ref else None # Estimated effective date of birth for conflict checking eff_dob_val = dob_val if not eff_dob_val and old_litter_id: # Try to get litter date if litter was resolved mapped_l_id = litter_id_map.get(old_litter_id) if mapped_l_id: eff_dob_val = get_litter_date(mapped_l_id) # Determine explicit or litter-derived birth date (None if unknown/parent) birth_date = dob_val if not birth_date and old_litter_id: mapped_l_id = litter_id_map.get(old_litter_id) if mapped_l_id: birth_date = get_litter_date(mapped_l_id) # If still no effective date, use the estimated page date if not eff_dob_val: eff_dob_val = days_to_date(file_dates[filename]) raw_breeder = rg.get("OriginBreeder") or rg.get("originBreeder") if raw_breeder: norm_b, keep_b = get_normalized_contact_name(raw_breeder) raw_breeder = norm_b if keep_b else None parent_refs = [] if old_litter_id: rl = next((l for l in raw_litters if l.get("_scoped_id") == old_litter_id), None) if rl: f_name = rl.get("FatherName") or rl.get("fatherName") or rl.get("ParentMaleName") or rl.get("parentMaleName") or rl.get("_father_name") m_name = rl.get("MotherName") or rl.get("motherName") or rl.get("ParentFemaleName") or rl.get("parentFemaleName") or rl.get("_mother_name") if f_name: parent_refs.append({"name": f_name, "roleGuess": "father"}) if m_name: parent_refs.append({"name": m_name, "roleGuess": "mother"}) all_processed_gerbils.append({ "Id": new_guid, "Name": name_val, "Gender": gender, "Status": status, "LitterId": old_litter_id, # mapped later "OriginContactId": origin_cid, "ReceiverContactId": receiver_cid, "EnclosureId": None, "ColorVarietyId": color_variety_id, "DateOfBirth": dob_val, "DateOfDeath": dod_val, "CauseOfDeath": cause_of_death_val, "GoHomeDate": gohome_val, "Genotype": genotype_val, "Notes": rg.get("Notes") or rg.get("notes") or rg.get("Note") or rg.get("note"), "ImportSource": rg.get("ImportSource") or rg.get("importSource") or filename, "ExternalRef": ext_ref_scoped, "RawImportData": rg.get("RawImportData") or rg.get("rawImportData") or json.dumps({"colorDescription": color_val if not color_variety_id else None}), "OriginBreeder": raw_breeder or ("Zucht der kleinen Chaoten" if is_resident else None), "NameSearch": normalize_name(name_val), "CharacterTraits": traits, "CharacterNote": char_note, "IsDeaf": is_deaf, "IsResident": is_resident, "parentRefs": parent_refs, "_photos": rg.get("photos", []), "_old_scoped_litter_id": old_litter_id, "_eff_dob": eff_dob_val, "_birth_date": birth_date, "_filename": filename, "_old_id": rg.get("Id") or rg.get("id") }) # Map and append stammbaum animals to all_processed_gerbils for a in stammbaum_only_animals: a_id = a["id"] name_val = get_normalized_gerbil_name(a["name"]) gender_val = str(a.get("gender") or "").lower().strip() if gender_val in ["m", "male"]: gender = "male" elif gender_val in ["w", "f", "female"]: gender = "female" else: gender = "unknown" dob_val = parse_date(a.get("dob")) dod_val = parse_date(a.get("death")) status = "Breeding" if dod_val: status = "Deceased" elif a_id not in stammbaum_resident_ids: status = "GivenAway" else: if dob_val: try: dt_dob = datetime.strptime(dob_val, "%Y-%m-%d") dt_now = datetime.now() age_years = (dt_now - dt_dob).days / 365.25 if age_years >= 6.0: status = "Deceased" except Exception: pass # Color variety mapping existing_gt = a["genotype"]["rawGenotype"] if a["genotype"]["rawGenotype"] else None color_variety_id, genotype_val = resolve_color_and_genotype( a.get("farbschlag"), existing_gt, variety_map, variety_genotypes ) if not color_variety_id: for fbv in a.get("farbschlagVariants", []): cv_id, gt_val = resolve_color_and_genotype(fbv, existing_gt, variety_map, variety_genotypes) if cv_id: color_variety_id = cv_id genotype_val = gt_val break # Contact mapping origin_cid = None b_name = (a.get("breeder") or "").strip() z_name = (a.get("zucht") or "").strip() if b_name: origin_cid = generate_guid(f"stammbaum-contact-{normalize_name(b_name)}") elif z_name: origin_cid = generate_guid(f"stammbaum-contact-{normalize_name(z_name)}") # Map to resolved global contact GUID if origin_cid in contact_id_map: origin_cid = contact_id_map[origin_cid] else: origin_cid = None scoped_id = generate_guid(f"stammbaum-animal-{a_id}") scoped_litter_id = a.get("_mapped_litter_scoped_id") is_deaf = a["genotype"].get("deaf") stammbaum_breeder = a.get("breeder") if a.get("breeder") else (a.get("zucht") if a.get("zucht") else None) if stammbaum_breeder: norm_b, keep_b = get_normalized_contact_name(stammbaum_breeder) stammbaum_breeder = norm_b if keep_b else None all_processed_gerbils.append({ "Id": scoped_id, "Name": name_val, "Gender": gender, "Status": status, "LitterId": scoped_litter_id, # mapped later in step 5 "OriginContactId": origin_cid, "ReceiverContactId": None, "EnclosureId": None, "ColorVarietyId": color_variety_id, "DateOfBirth": dob_val, "DateOfDeath": dod_val, "CauseOfDeath": "Duftdrüsen-Tumor" if name_val == "Ken'ichi" and dob_val == "2015-03-01" else None, "GoHomeDate": None, "Genotype": genotype_val, "Notes": None, "ImportSource": ", ".join(a.get("sourceFiles", [])), "ExternalRef": f"stammbaum-{a_id}", "RawImportData": json.dumps({ "rawGenotype": a["genotype"]["rawGenotype"], "unmappedTokens": a["genotype"]["unmappedTokens"], "breederText": a.get("breeder", "") }, ensure_ascii=False), "OriginBreeder": stammbaum_breeder, "NameSearch": normalize_name(name_val), "CharacterTraits": [], "CharacterNote": None, "IsDeaf": is_deaf, "IsResident": a_id in stammbaum_resident_ids, "parentRefs": a.get("parentRefs", []), "_photos": a.get("photos", []), "_conflict": bool(a.get("conflict")), "_resolved_by_decision": bool(a.get("resolvedByDecision")), "_old_scoped_litter_id": scoped_litter_id, "_eff_dob": dob_val or "2010-01-01", "_birth_date": dob_val, "_filename": a.get("sourceFiles", ["Stammbaum"])[0], "_old_id": a_id }) # Map and append docx animals to all_processed_gerbils for idx, da in enumerate(docx_animals): name_val = get_normalized_gerbil_name(da["name"]) gender = da["gender"] dob_val = parse_date(da.get("litterDob")) dod_val = parse_date(da.get("deathDate")) gohome_val = parse_date(da.get("abgabeDate")) # Status precedence status = "Breeding" if dod_val: status = "Deceased" elif gohome_val or da.get("owner"): status = "GivenAway" # Color variety mapping color_variety_id, genotype_val = resolve_color_and_genotype( da.get("farbschlag"), None, variety_map, variety_genotypes ) # Contact mapping (buyer) receiver_cid = None o_name = (da.get("owner") or "").strip() if o_name: receiver_cid = generate_guid(f"docx-contact-{normalize_name(o_name)}") if receiver_cid in contact_id_map: receiver_cid = contact_id_map[receiver_cid] else: receiver_cid = None # Scoped ID scoped_id = generate_guid(f"docx-animal-{idx}-{normalize_name(name_val)}-{dob_val or '0001-01-01'}") # Litter ID mapping ws_code = da.get("wsCode") scoped_litter_id = docx_litter_id_map.get((ws_code, dob_val)) # Raw import details raw_import_payload = json.dumps({ "abgabeWeight": da.get("abgabeWeight", ""), "deathCause": da.get("deathCause", ""), "partnerName": da.get("partnerName", ""), "partnerDob": da.get("partnerDob", "") }, ensure_ascii=False) # Residents: if sold/given away, it's not a resident is_resident = not bool(o_name) parent_refs = [] if scoped_litter_id: dl = next((l for l in docx_litters if docx_litter_id_map.get((l["wsCode"], parse_date(l["dob"]))) == scoped_litter_id), None) if dl: if dl.get("fatherName"): parent_refs.append({"name": dl["fatherName"], "roleGuess": "father"}) if dl.get("motherName"): parent_refs.append({"name": dl["motherName"], "roleGuess": "mother"}) all_processed_gerbils.append({ "Id": scoped_id, "Name": name_val, "Gender": gender, "Status": status, "LitterId": scoped_litter_id, "OriginContactId": None, "ReceiverContactId": receiver_cid, # mapped in step 5 "EnclosureId": None, "ColorVarietyId": color_variety_id, "DateOfBirth": dob_val, "DateOfDeath": dod_val, "CauseOfDeath": da.get("deathCause"), "GoHomeDate": gohome_val, "Genotype": genotype_val, "Notes": None, "ImportSource": "Wurfchronik-Detail.docx", "ExternalRef": f"docx-{idx}-{normalize_name(name_val)}-{dob_val or '0001-01-01'}", "RawImportData": raw_import_payload, "OriginBreeder": "Zucht der kleinen Chaoten", "NameSearch": normalize_name(name_val), "CharacterTraits": [], "CharacterNote": None, "IsDeaf": None, "IsResident": is_resident, "parentRefs": parent_refs, "_photos": da.get("photos", []), "_old_scoped_litter_id": scoped_litter_id, "_eff_dob": dob_val or "2020-01-01", "_birth_date": dob_val, "_filename": "Wurfchronik-Detail.docx", "_old_id": name_val }) # Build parenting dates lookup using old scoped IDs parent_litter_dates = {} for l in resolved_litters: ld = l["Date"] if ld: for pid in [l["FatherId"], l["MotherId"]]: if pid: parent_litter_dates.setdefault(pid, []).append(ld) def are_compatible(g1, g2): # Must have same gender (or one unknown) if g1["Gender"] != "unknown" and g2["Gender"] != "unknown" and g1["Gender"] != g2["Gender"]: bd1 = g1.get("_birth_date") bd2 = g2.get("_birth_date") if not (bd1 and bd2 and bd1 == bd2): return False bd1 = g1.get("_birth_date") bd2 = g2.get("_birth_date") # New rule: if name and parents match, they are compatible regardless of DOB! p1 = g1.get("parentRefs", []) p2 = g2.get("parentRefs", []) f1 = next((p["name"] for p in p1 if p.get("roleGuess") == "father"), "") m1 = next((p["name"] for p in p1 if p.get("roleGuess") == "mother"), "") f2 = next((p["name"] for p in p2 if p.get("roleGuess") == "father"), "") m2 = next((p["name"] for p in p2 if p.get("roleGuess") == "mother"), "") parents_match = False if f1 and f2 and m1 and m2: if normalize_name(f1) == normalize_name(f2) and normalize_name(m1) == normalize_name(m2): parents_match = True if not parents_match: # If both have explicit birth dates, they must match within 30 days if bd1 and bd2: days1 = date_to_days(bd1) days2 = date_to_days(bd2) if days1 is not None and days2 is not None: if abs(days1 - days2) > 30: return False # If g1 has birth date, and g2 has parenting dates, birth date must be before parenting dates p_dates2 = parent_litter_dates.get(g2["Id"], []) if bd1: for pd in p_dates2: if pd <= bd1: # Can't have litter before or on birth date return False p_dates1 = parent_litter_dates.get(g1["Id"], []) if bd2: for pd in p_dates1: if pd <= bd2: return False return True def _record_source_files(g): """All distinct source files a single raw record drew from. ImportSource is either a comma-joined Stammbaum file list, a single Wurfchronik filename, or the per-litter filename; _filename is the primary file. We union both so nothing is lost.""" files = set() imp = g.get("ImportSource") if imp: for part in str(imp).split(","): part = part.strip() if part: files.add(part) fn = g.get("_filename") if fn: files.add(fn) return files def build_provenance(records, best_g, extra_notes=None): """Aggregate data-provenance across every raw record merged into one resolved gerbil. Returns a JSON string (stored on the Gerbil entity as a nullable text column) so the Rennmausakte can show where the entry came from. `records` is the list of raw records that became this gerbil; `best_g` is the chosen primary record.""" source_files = set() from_wurfchronik = False any_conflict = False any_decision = False for r in records: source_files |= _record_source_files(r) for f in _record_source_files(r): if "wurfchronik" in f.lower(): from_wurfchronik = True if r.get("_conflict"): any_conflict = True if r.get("_resolved_by_decision"): any_decision = True notes = [] merged_count = len(records) if merged_count > 1: notes.append(f"aus {merged_count} Datensätzen zusammengeführt") if any_decision: notes.append("per manueller Entscheidung zugeordnet") if any_conflict: notes.append("Konflikt per Entscheidung gelöst") # Parent derivation: surface the strongest parentRef method/confidence # the primary record carries (chart-position etc.). parent_method = None parent_confidence = None for ref in best_g.get("parentRefs", []) or []: if ref.get("method") and not parent_method: parent_method = ref.get("method") if ref.get("confidence") and not parent_confidence: parent_confidence = ref.get("confidence") if extra_notes: for n in extra_notes: if n and n not in notes: notes.append(n) return build_entity_provenance( source_files, merged_count, notes=notes, from_wurfchronik=from_wurfchronik, extra={"parentMethod": parent_method, "parentConfidence": parent_confidence}, ) # Group gerbils by name to perform deduplication gerbil_groups = {} for g in all_processed_gerbils: name_key = get_dedup_name_key(g["Name"]) if not name_key: name_key = "unbekannt" gerbil_groups.setdefault(name_key, []).append(g) resolved_gerbils = [] gerbil_id_map = {} # old_scoped_id -> final_id color_keys = set(variety_map.keys()) for name_key, group in gerbil_groups.items(): is_placeholder = ( name_key in color_keys or any(p in name_key for p in ["unbekannt", "unbenannt", "baby", "welpe", "jungtier", "unknown", "welpen"]) or len(name_key) <= 2 ) if is_placeholder: # Placeholders: do NOT merge, keep all separate for g in group: g["Provenance"] = build_provenance([g], g) resolved_gerbils.append(g) gerbil_id_map[g["Id"]] = g["Id"] continue # Partition group into compatible subsets sub_groups = [] for g in group: placed = False for sub in sub_groups: if all(are_compatible(g, member) for member in sub): sub.append(g) placed = True break if not placed: sub_groups.append([g]) # Merge each partition sub-group into a single gerbil for sub in sub_groups: if len(sub) == 1: g = sub[0] g["Provenance"] = build_provenance([g], g) resolved_gerbils.append(g) gerbil_id_map[g["Id"]] = g["Id"] continue # Find the best primary record to merge into best_g = None best_score = -1 for g in sub: score = 0 if g["_old_scoped_litter_id"]: score += 10 if g["DateOfBirth"]: score += 5 if g["Genotype"]: score += 3 if g["ColorVarietyId"]: score += 2 if g["ImportSource"] and "stammbaum" in g["ImportSource"].lower(): score += 20 if g["Notes"] and not any(kw in g["Notes"].lower() for kw in ["parent", "mutter", "vater", "dam", "sire"]): score += 1 if len(g["Name"]) > len(name_key) + 5: # likely has clan suffix score += 15 if score > best_score: best_score = score best_g = g # Merge fields merged_notes = [] if best_g["Notes"]: merged_notes.append(best_g["Notes"]) # Merge photos merged_photos = list(best_g.get("_photos", [])) # Track sources for debugging sources = [best_g["_filename"]] for g in sub: if g == best_g: continue gerbil_id_map[g["Id"]] = best_g["Id"] sources.append(g["_filename"]) for ph in g.get("_photos", []): if ph not in merged_photos: merged_photos.append(ph) if not best_g.get("_old_scoped_litter_id") and g.get("_old_scoped_litter_id"): best_g["_old_scoped_litter_id"] = g["_old_scoped_litter_id"] if not best_g["LitterId"] and g["LitterId"]: best_g["LitterId"] = g["LitterId"] if not best_g["DateOfBirth"] and g["DateOfBirth"]: best_g["DateOfBirth"] = g["DateOfBirth"] if not best_g["DateOfDeath"] and g["DateOfDeath"]: best_g["DateOfDeath"] = g["DateOfDeath"] if not best_g["CauseOfDeath"] and g["CauseOfDeath"]: best_g["CauseOfDeath"] = g["CauseOfDeath"] if not best_g["GoHomeDate"] and g["GoHomeDate"]: best_g["GoHomeDate"] = g["GoHomeDate"] if not best_g["Genotype"] and g["Genotype"]: best_g["Genotype"] = g["Genotype"] if not best_g["ColorVarietyId"] and g["ColorVarietyId"]: best_g["ColorVarietyId"] = g["ColorVarietyId"] if not best_g["OriginContactId"] and g["OriginContactId"]: best_g["OriginContactId"] = g["OriginContactId"] if not best_g["ReceiverContactId"] and g["ReceiverContactId"]: best_g["ReceiverContactId"] = g["ReceiverContactId"] if g["IsResident"]: best_g["IsResident"] = True # Reconcile Gender: prefer a known gender over unknown, and prefer stammbaum over other sources if best_g["Gender"] == "unknown" and g["Gender"] != "unknown": best_g["Gender"] = g["Gender"] elif best_g["Gender"] != "unknown" and g["Gender"] != "unknown" and best_g["Gender"] != g["Gender"]: if g["ImportSource"] and "stammbaum" in g["ImportSource"].lower(): if not best_g["ImportSource"] or "stammbaum" not in best_g["ImportSource"].lower(): best_g["Gender"] = g["Gender"] # Status precedence: Deceased > GivenAway > Breeding/Pet if g["Status"] == "Deceased": best_g["Status"] = "Deceased" elif g["Status"] == "GivenAway" and best_g["Status"] not in ["Deceased"]: best_g["Status"] = "GivenAway" if g["Notes"] and g["Notes"] not in merged_notes: # Ignore redundant dummy notes if not any(kw in g["Notes"].lower() for kw in ["parent listed", "mutter von", "vater von", "dam of", "sire of"]): merged_notes.append(g["Notes"]) # Reconcile fields based on number of source files supporting them conflict_notes = [] for field in ["DateOfBirth", "DateOfDeath", "Gender", "Genotype", "ColorVarietyId"]: votes = {} for g in sub: val = g.get(field) if val and val != "unknown": # count source files sources_count = len(str(g.get("ImportSource") or "").split(",")) votes[val] = votes.get(val, 0) + sources_count if votes: best_val = max(votes, key=votes.get) # If the records disagreed on a field, the merge had to pick a # winner — record that as a provenance note. if len(votes) > 1: FIELD_LABEL = { "DateOfBirth": "Geburtsdatum", "DateOfDeath": "Sterbedatum", "Gender": "Geschlecht", "Genotype": "Genotyp", "ColorVarietyId": "Farbschlag", } conflict_notes.append( f"Konflikt bei {FIELD_LABEL[field]} per Mehrheitsentscheidung gelöst" ) best_g[field] = best_val # Keep helper fields in sync if we changed DateOfBirth if field == "DateOfBirth": best_g["_birth_date"] = best_val best_g["_eff_dob"] = best_val or "2010-01-01" if merged_notes: best_g["Notes"] = " | ".join(merged_notes) best_g["_photos"] = merged_photos best_g["Provenance"] = build_provenance(sub, best_g, extra_notes=conflict_notes) # Print merge trace print(f"Deduplicated same-animal name '{best_g['Name']}': merged {len(sub)} entries across files: {', '.join(sources)}") resolved_gerbils.append(best_g) gerbil_id_map[best_g["Id"]] = best_g["Id"] print(f"Deduplicated to {len(resolved_gerbils)} unique gerbil records.") # Apply age-based death threshold (6.0 years) to all resolved gerbils dt_now = datetime.now() for g in resolved_gerbils: if g.get("Status") != "Deceased" and g.get("Status") != "GivenAway": if not g.get("DateOfDeath") and not g.get("ReceiverContactId"): dob_str = g.get("DateOfBirth") if dob_str: try: dt_dob = datetime.strptime(dob_str, "%Y-%m-%d") age_years = (dt_now - dt_dob).days / 365.25 if age_years >= 6.0: g["Status"] = "Deceased" except Exception: pass # 5. Map Gerbils to Litters for g in resolved_gerbils: old_lid = g["_old_scoped_litter_id"] l_guid = litter_id_map.get(old_lid) g["LitterId"] = l_guid if l_guid and l_guid in litter_by_scoped_id: l = litter_by_scoped_id[l_guid] l_date = l.get("Date") if l_date: if not g.get("DateOfBirth"): g["DateOfBirth"] = l_date g["_birth_date"] = l_date g["_eff_dob"] = l_date # Apply age-based death threshold (6.0 years) to all resolved gerbils (including newly backfilled ones) if g.get("Status") != "Deceased" and g.get("Status") != "GivenAway": if not g.get("DateOfDeath") and not g.get("ReceiverContactId"): dob_str = g.get("DateOfBirth") if dob_str: try: dt_dob = datetime.strptime(dob_str, "%Y-%m-%d") age_years = (dt_now - dt_dob).days / 365.25 if age_years >= 6.0: g["Status"] = "Deceased" except Exception: pass # Clean helper fields del g["_old_scoped_litter_id"] del g["_eff_dob"] if "_birth_date" in g: del g["_birth_date"] del g["_filename"] del g["_old_id"] g.pop("_conflict", None) g.pop("_resolved_by_decision", None) # Gather final valid gerbil IDs valid_gerbil_ids = {g["Id"] for g in resolved_gerbils} # Create name lookup for resolved gerbils gerbil_by_norm_name = {} for g in resolved_gerbils: n_key = normalize_name(g["Name"]) gerbil_by_norm_name.setdefault(n_key, []).append(g) # Also index by call-name to resolve parents who are only listed by call-name c_key = normalize_name(get_call_name(g["Name"])) if c_key != n_key: gerbil_by_norm_name.setdefault(c_key, []).append(g) # Map raw Guid if present (convert if old_id mapped to new_guid) for l in resolved_litters: if l["FatherId"] in gerbil_id_map: l["FatherId"] = gerbil_id_map[l["FatherId"]] if l["MotherId"] in gerbil_id_map: l["MotherId"] = gerbil_id_map[l["MotherId"]] # Clean foreign keys that do not point to a valid gerbil if l["FatherId"] and l["FatherId"] not in valid_gerbil_ids: l["FatherId"] = None if l["MotherId"] and l["MotherId"] not in valid_gerbil_ids: l["MotherId"] = None # Parent Resolver (Global Name Matching) resolved_fathers = 0 resolved_mothers = 0 gerbil_by_id_final = {g["Id"]: g for g in resolved_gerbils} def _final_gender(gid): g = gerbil_by_id_final.get(gid) return g["Gender"] if g else None def _resolve_name(name, prefer_gender, litter_date): """Resolve a parent name to the best matching final gerbil. Gender is a PREFERENCE, not a hard filter: a reversed parent (e.g. a female listed in the father position, as the Stammbaum often does) still resolves to a gerbil — the role is corrected afterwards by gender. This is what previously left FatherId/MotherId null (the candidate was filtered out for having the "wrong" gender for its slot). """ if not name: return None cands = [] for c in gerbil_by_norm_name.get(normalize_name(name), []): final_id = gerbil_id_map.get(c["Id"]) if not final_id: continue final_c = gerbil_by_id_final.get(final_id) if not final_c: continue # Parent must be age-plausible: born before the litter and within the # gerbil lifespan (skips e.g. a 2013 animal for a 2022 litter). if not parent_age_plausible(final_c["DateOfBirth"], litter_date): continue cands.append(final_c) if not cands: return None # Prefer the gender expected for this role, then unknown, then anything. for pool in ( [c for c in cands if c["Gender"] == prefer_gender], [c for c in cands if c["Gender"] == "unknown"], cands, ): if pool: return pool[0] return None for l in resolved_litters: # Pre-check: if _father_name points to a known female and _mother_name to a # known male → swap names (Stammbaum positions reversed). Helps the name # resolver pick the right same-name candidate before role normalization. f_name_pre = l.get("_father_name", "") m_name_pre = l.get("_mother_name", "") if f_name_pre and m_name_pre: f_gender = next((g["Gender"] for g in gerbil_by_norm_name.get(normalize_name(f_name_pre), []) if g["Gender"] != "unknown"), None) m_gender = next((g["Gender"] for g in gerbil_by_norm_name.get(normalize_name(m_name_pre), []) if g["Gender"] != "unknown"), None) if f_gender == "female" and m_gender == "male": l["_father_name"], l["_mother_name"] = m_name_pre, f_name_pre if l["_father_name"] and not l["FatherId"]: cand = _resolve_name(l["_father_name"], "male", l["Date"]) if cand: l["FatherId"] = cand["Id"] resolved_fathers += 1 if l["_mother_name"] and not l["MotherId"]: cand = _resolve_name(l["_mother_name"], "female", l["Date"]) if cand: l["MotherId"] = cand["Id"] resolved_mothers += 1 # Cleanup internal keys del l["_father_name"] del l["_mother_name"] del l["_filename"] # Role normalization: assign each resolved parent to the role matching its # gender, eliminate self-pairings (same animal in both roles), and never let # impossible duplicates survive (two males / two females). This corrects # reversed Stammbaum positions including the cases the simple swap missed # (one parent of "unknown" gender, or a self-paired litter). role_fixes = 0 for l in resolved_litters: father, mother = assign_parent_roles(l.get("FatherId"), l.get("MotherId"), _final_gender) if (l.get("FatherId"), l.get("MotherId")) != (father, mother): role_fixes += 1 l["FatherId"] = father l["MotherId"] = mother if role_fixes: print(f"Role-normalization: corrected {role_fixes} litter(s) (gender roles / self-pairings).") # Parent-age sanity check: drop any resolved parent that cannot belong to the # litter — born after the offspring, or more than a gerbil lifespan earlier. # Catches mis-resolved links the name matcher still let through (e.g. Jayjay, # *2013, wrongly attached to Solice's 2022 litter). age_drops = [] for l in resolved_litters: ldate = l.get("Date") for role in ("FatherId", "MotherId"): pid = l.get(role) if not pid: continue p = gerbil_by_id_final.get(pid) if p and not parent_age_plausible(p.get("DateOfBirth"), ldate): age_drops.append((l.get("Name"), role, p.get("Name"), p.get("DateOfBirth"), ldate)) l[role] = None if age_drops: print(f"Parent-age sanity check: dropped {len(age_drops)} implausible parent link(s):") for lname, role, pname, pdob, ldate in age_drops[:20]: print(f" {lname}: {role}={pname} (*{pdob}) vs litter {ldate}") print(f"Globally resolved {resolved_fathers} fathers and {resolved_mothers} mothers.") # 5b. Second-pass litter dedup: now that FatherId/MotherId are known, # merge litters that have the same date AND the same parents. # This is the core of the "sibling pairing" fix: Blue Wave and Sunny Sky # both come from Wonderman × Unique — their two separate litter records # must now become one, so their children share the same LitterId. litter_by_id_post = {l["Id"]: l for l in resolved_litters} gerbil_by_litter = {} for g in resolved_gerbils: lid = g.get("LitterId") if lid: gerbil_by_litter.setdefault(lid, []).append(g) def _litter_same_parents(l1, l2): """Strict: same date + both parents known and matching.""" if l1["Date"] != l2["Date"]: return False f1, m1 = l1.get("FatherId"), l1.get("MotherId") f2, m2 = l2.get("FatherId"), l2.get("MotherId") if not f1 or not f2 or not m1 or not m2: return False return f1 == f2 and m1 == m2 by_date2 = {} for l in resolved_litters: by_date2.setdefault(l["Date"], []).append(l) deduped2 = [] litter_remap2 = {} # old_id -> canonical_id for date_val, group in by_date2.items(): sub_groups = [] for l in group: placed = False for sub in sub_groups: if all(_litter_same_parents(l, m) for m in sub): sub.append(l) placed = True break if not placed: sub_groups.append([l]) for sub in sub_groups: # Prefer the canonical that has the most children canonical = max(sub, key=lambda l: len(gerbil_by_litter.get(l["Id"], []))) for l in sub: litter_remap2[l["Id"]] = canonical["Id"] if l is not canonical: canonical["_source_files"] |= l.get("_source_files", set()) canonical["_merged_count"] += l.get("_merged_count", 1) if not l.get("_virtual"): canonical["_virtual"] = False deduped2.append(canonical) if len(sub) > 1: siblings = [g["Name"] for l in sub for g in gerbil_by_litter.get(l["Id"], []) if l is not canonical] print(f"Sibling-Litter-Merge on {date_val}: {canonical['Name']} absorbed sibling half — children now share LitterId: {[g['Name'] for g in gerbil_by_litter.get(canonical['Id'], [])] + siblings}") # Remap LitterId in all gerbils n_remapped = 0 for g in resolved_gerbils: old_lid = g.get("LitterId") if old_lid and old_lid in litter_remap2 and litter_remap2[old_lid] != old_lid: g["LitterId"] = litter_remap2[old_lid] n_remapped += 1 n_merged2 = len(resolved_litters) - len(deduped2) if n_merged2: print(f"Sibling-Litter-Dedup: {n_merged2} additional litter record(s) merged ({n_remapped} gerbil LitterIds remapped).") resolved_litters = deduped2 litter_by_scoped_id = {l["Id"]: l for l in resolved_litters} # Datenherkunft for litters: which source files contributed, whether this is # a Wurfchronik litter vs a Stammbaum-reconstructed ("virtual") litter, how # many raw records merged into it, plus human-readable notes. Accumulators # (_source_files/_merged_count/_virtual) were filled during the two dedup # passes above; strip them after use. for l in resolved_litters: l_source_files = l.pop("_source_files", set()) l_merged_count = l.pop("_merged_count", 1) is_virtual = l.pop("_virtual", False) l_from_wurfchronik = any("wurfchronik" in str(f).lower() for f in l_source_files) l_notes = [] if is_virtual and not l_from_wurfchronik: l_notes.append("aus Stammbaum-Diagramm rekonstruiert") elif l_from_wurfchronik: l_notes.append("aus Wurfchronik") if l_merged_count > 1: l_notes.append(f"aus {l_merged_count} Datensätzen zusammengeführt") l_notes.append("Geschwister-Würfe zusammengeführt") l["Provenance"] = build_entity_provenance( l_source_files, l_merged_count, notes=l_notes, from_wurfchronik=l_from_wurfchronik, ) # Set IsBreeder and IsReceiver flags on contacts breeder_ids = {g["OriginContactId"] for g in resolved_gerbils if g.get("OriginContactId")} receiver_ids = {g["ReceiverContactId"] for g in resolved_gerbils if g.get("ReceiverContactId")} for c in resolved_contacts: c_id = c["Id"] is_breeder = c_id in breeder_ids is_receiver = c_id in receiver_ids if not is_breeder and not is_receiver: is_receiver = True c["IsBreeder"] = is_breeder c["IsReceiver"] = is_receiver # Datenherkunft: where this (deduplicated) contact came from, plus the # role we inferred. Accumulator keys (_source_files/_merged_count) were # filled during the contact dedup above; strip them after use. c_source_files = c.pop("_source_files", set()) c_merged_count = c.pop("_merged_count", 1) c_notes = [] if c_merged_count > 1: c_notes.append(f"aus {c_merged_count} Datensätzen zusammengeführt") if is_breeder: c_notes.append("als Züchter erkannt") if is_receiver: c_notes.append("als Abnehmer erkannt") c["Provenance"] = build_entity_provenance(c_source_files, c_merged_count, notes=c_notes) # Set and map gerbilPhotos resolved_photos = [] for g in resolved_gerbils: for idx, photo_rel in enumerate(g.get("_photos", [])): photo_guid = generate_guid(f"photo-{photo_rel}") ext = os.path.splitext(photo_rel)[1] or ".jpeg" fn_guid = photo_guid.replace("-", "") resolved_photos.append({ "Id": photo_guid, "GerbilId": g["Id"], "FileName": f"{fn_guid}{ext}", "SortOrder": idx, "_source_path": photo_rel }) # 6. Save final output JSON payload os.makedirs(OUTPUT_DIR, exist_ok=True) payload = { "contacts": resolved_contacts, "litters": resolved_litters, "gerbils": resolved_gerbils, "gerbilPhotos": resolved_photos } with open(OUTPUT_FILE, "w", encoding="utf-8") as f: json.dump(payload, f, ensure_ascii=False, indent=2) print(f"Successfully wrote database-ready import file to: {OUTPUT_FILE}") print(f" Contacts: {len(payload['contacts'])}") print(f" Litters: {len(payload['litters'])}") print(f" Gerbils: {len(payload['gerbils'])}") print(f" Photos: {len(payload['gerbilPhotos'])}") if __name__ == "__main__": main()