From 3021149ddd9805518f726104a7f915c699d617b7 Mon Sep 17 00:00:00 2001 From: Gulum Date: Sat, 13 Jun 2026 09:22:04 +0200 Subject: [PATCH] chore(import): apply user contact deduplication decisions --- tools/import/merge_and_resolve.py | 3195 +++++++++++++++-------------- 1 file changed, 1656 insertions(+), 1539 deletions(-) diff --git a/tools/import/merge_and_resolve.py b/tools/import/merge_and_resolve.py index 286001c..05f29e8 100644 --- a/tools/import/merge_and_resolve.py +++ b/tools/import/merge_and_resolve.py @@ -1,1539 +1,1656 @@ -import os -import json -import re -import uuid -import sys -from datetime import datetime - -# Prevent encoding crashes on Windows consoles when printing unicode -if sys.platform.startswith('win'): - try: - sys.stdout.reconfigure(encoding='utf-8') - except Exception: - pass - -# Paths -DIR_PATH = r"C:\Users\gulum\dev\Wurfchronik_Bilder" -SEEDS_PATH = r"C:\Users\gulum\dev\GerbilManager\gerbil-manager-web\src\genetics\colorVarietySeed.backend.json" -OUTPUT_DIR = r"C:\Users\gulum\dev\GerbilManager\tools\import\output" -OUTPUT_FILE = os.path.join(OUTPUT_DIR, "resolved_import.json") - -def generate_guid(key_str): - """Generate a stable UUID string based on a key.""" - return str(uuid.uuid5(uuid.NAMESPACE_DNS, key_str)) - -def to_valid_guid(val): - if not val: - return None - val_str = str(val).strip() - try: - uuid.UUID(val_str) - return val_str - except ValueError: - return generate_guid(val_str) - -def normalize_name(name): - if not name: - return "" - return "".join(c for c in name.lower() if c.isalnum()) - -def get_normalized_contact_name(name): - if not name: - return "", False - - n = "".join(c for c in name.lower() if c.isalnum()) - - to_discard = { - "chevroletcamarooftopolino", - "cindyvprivatzuchtgießen", - "cindyvprivatzuchtgiessen", - "inuschofblackforest", - "stichvonprivatzuchtgießen", - "stichvonprivatzuchtgiessen", - "tarzanofsamsimar", - "hanserennersposeidon", - "livingforcesidefix", - "livingforcesnando" - } - if n in to_discard: - return "", False - - norm_map = { - "kleinechaoten": "Zucht der kleinen Chaoten", - "kleinenchaoten": "Zucht der kleinen Chaoten", - "zuchtderkleinenchaoten": "Zucht der kleinen Chaoten", - "schlossmaus": "Schlossmäuse", - "schlossmäuse": "Schlossmäuse", - "schlossmäusen": "Schlossmäuse", - "buntefellnasen": "bunten Fellnasen", - "buntenfellnase": "bunten Fellnasen", - "buntenfellnasen": "bunten Fellnasen", - "kkchaos": "KK-Chaos", - "kkchaosofkkchaos": "KK-Chaos", - "oflennylengo": "Lenny Lengo", - "lennylengo": "Lenny Lengo", - "pzmaintal": "Privatzucht Maintal", - "maintalerpz": "Privatzucht Maintal", - "vonprivatzuchtmaintal": "Privatzucht Maintal", - "pzmücke": "Privatzucht Mücke", - "pzmuecke": "Privatzucht Mücke", - "privatzuchtmücke": "Privatzucht Mücke", - "privatzuchtmuecke": "Privatzucht Mücke", - "pzseligenstadt": "Privatzucht Seligenstadt", - "kriegernmitkrallen": "Krieger mit Krallen", - "littlefellows": "little fellows", - "littlerunners": "little runners", - "colourfulfurrygerbils": "Colorful Furry Gerbils", - "colorfulfurrygerbils": "Colorful Furry Gerbils", - "hanserenners": "Hanse Renner", - "blackforestgv": "Black Forest", - "topol": "Topolino", - "ankeb": "Anke Busch", - "anjasch": "Anja Schaumburg", - "arturaleksandraundpolamundrzynski": "Artur, Aleksandra und Pola Mundrzynski", - "arturaleksandrapolamundrzynski": "Artur, Aleksandra und Pola Mundrzynski", - "barbarar": "Barbara Riegler", - "birgittabüskens": "Birgitta Müller-Büskens", - } - - if n in norm_map: - return norm_map[n], True - - return name, True - -def get_call_name(name): - if not name: - return "" - n = name.strip() - n = re.sub(r'\[[^\]]+\]$', '', n).strip() - n = re.split(r'\s+(?:of|von\s+den|von\s+der|v\.\s?d\.|von)\s+', n, flags=re.IGNORECASE)[0].strip() - return n - -def get_dedup_name_key(name): - if not name: - return "" - n = name.lower().strip() - # Strip common suffixes/prefixes and parentheticals - n = re.sub(r'\b(?:von\s+den|v\.?\s*d\.?|v\.?o\.?)\s+(?:kleinen\s+)?chaoten\b', '', n) - n = re.sub(r'\bvon\s+der\s+schlossm\w+\b', '', n) - n = re.sub(r'\bvon\s+der\s+bunten\s+fellnasen?\b', '', n) - n = re.sub(r'\bof\s+black\s+forest\b', '', n) - n = re.sub(r'\b\(?rv\)?\b', '', n) - n = re.sub(r'\bgen\.\s+\w+', '', n) - # Clean up parentheses or brackets - n = re.sub(r'\(.*?\)', '', n) - n = re.sub(r'\[.*?\]', '', n) - return "".join(c for c in n if c.isalnum()) - -def clean_color_name(c_desc): - if not c_desc: - return "", False - - # Lowercase and strip - c = c_desc.lower().strip() - - # Check for Schecke - is_schecke = False - if re.search(r'\bsp\b|\bsp\d|\bsp[*(²³]|\bspotted|\bschecke|[- ]sp\b|\w+sp\b', c): - is_schecke = True - - # Standardize parentheticals for schimmel - c = c.replace("(schimmel)", "schimmel") - c = c.replace("(schimmel-hell)", "schimmel hell") - c = c.replace("(schimmel hell)", "schimmel hell") - - # Strip schecke/sp markers and any trailing text starting from sp - c = re.sub(r'\([- ]?sp(otted)?\)', '', c) # handles (-sp) - c = re.sub(r'[- ]?sp(otted)?\b.*', '', c) # handles -sp(k), -sp*(k), -sp, etc. - c = re.sub(r'[- ]?schecke\b.*', '', c) - c = re.sub(r'[- ]?spotted\b.*', '', c) - - # Strip any other parentheticals, symbols, or trailing stars/numbers - c = re.sub(r'\s*\(.*?\)\s*', ' ', c) - c = re.sub(r'[²³*]', '', c) - c = c.strip() - - # Mapping table for abbreviations, typos, and specific combinations - mapping = { - "antra": "anthrazit", - "anthra": "anthrazit", - "ankazit": "anthrazit", - "antrazit": "anthrazit", - "pew": "rew", - "bew": "hermelin", - "harder": "marder", - "kohli": "kohlfuchs", - "aligerfuchs": "algierfuchs", - "algiesfuchs": "algierfuchs", - "schw": "schwarz", - "sa": "silberagouti", - "a": "agouti", - "cp-sa": "cp-silberagouti", - "cp-a": "cp-agouti", - "cp-a-hell": "cp-agouti-hell", - "cp-aisa": "cp-agouti", - "cp-a / rcp-sa": "cp-agouti", - } - - if c in mapping: - c = mapping[c] - - return c, is_schecke - -def resolve_color_and_genotype(color_val, existing_genotype, variety_map, variety_genotypes): - if not color_val: - return None, existing_genotype - color_str = str(color_val).strip() - clean_name, is_schecke = clean_color_name(color_str) - # Match color in variety_map - color_variety_id = None - if clean_name in variety_map: - color_variety_id = variety_map[clean_name] - else: - for seed_name, seed_id in variety_map.items(): - if seed_name in clean_name or clean_name in seed_name: - color_variety_id = seed_id - break - # Update genotype if it's a Schecke - genotype = existing_genotype - if is_schecke: - if genotype: - if "spsp" in genotype: - genotype = genotype.replace("spsp", "Spsp") - elif "Spsp" not in genotype and "Sp" not in genotype: - genotype = f"{genotype} Spsp".strip() - else: - canonical = variety_genotypes.get(color_variety_id) - if canonical: - if "spsp" in canonical: - genotype = canonical.replace("spsp", "Spsp") - else: - genotype = f"{canonical} Spsp".strip() - else: - genotype = "Spsp" - return color_variety_id, genotype - -def parse_date(d): - """Convert variations of date formats to YYYY-MM-DD.""" - if not d or d == "0001-01-01": - return None - d = str(d).strip() - # Try YYYY-MM-DD - if re.match(r"^\d{4}-\d{2}-\d{2}$", d): - parts = d.split("-") - year = int(parts[0]) - if 1900 < year < 2000: - year += 100 - d = f"{year}-{parts[1]}-{parts[2]}" - elif year == 1900: - return None - return d - # Try DD.MM.YYYY or D.M.YY - match = re.match(r"^(\d{1,2})\.(\d{1,2})\.(\d{2,4})$", d) - if match: - day = int(match.group(1)) - month = int(match.group(2)) - year = int(match.group(3)) - if year < 100: - year += 2000 - elif 1900 < year < 2000: - year += 100 - elif year == 1900: - return None - try: - return datetime(year, month, day).strftime("%Y-%m-%d") - except ValueError: - pass - # Try ISO timestamp - try: - dt = datetime.fromisoformat(d.replace("Z", "+00:00")) - year = dt.year - if 1900 < year < 2000: - dt = dt.replace(year=year + 100) - return dt.strftime("%Y-%m-%d") - elif year == 1900: - return None - return dt.strftime("%Y-%m-%d") - except ValueError: - pass - return None - -def date_to_days(dt_str): - if not dt_str: - return None - try: - return (datetime.strptime(dt_str, "%Y-%m-%d") - datetime(2000, 1, 1)).days - except ValueError: - return None - -def days_to_date(days): - import datetime as dt - return (dt.datetime(2000, 1, 1) + dt.timedelta(days=int(days))).strftime("%Y-%m-%d") - -def parse_death_info(notes, status, existing_dod, existing_cod): - if not notes: - return status, existing_dod, existing_cod - - has_death_indicator = '†' in notes or 'verstorben' in notes.lower() or 'gestorben' in notes.lower() or 'todesdatum' in notes.lower() or '/+' in notes - if '+' in notes: - if re.search(r'\+\s*(?:LE|AS|Unbekannt|gestorben|verstorben|tod)\b', notes, re.I) or re.search(r'\+\s*\d{1,2}\.\d{1,2}\.\d{2,4}', notes) or '/+' in notes: - has_death_indicator = True - - resolved_status = status - if has_death_indicator: - resolved_status = "Deceased" - - dod = existing_dod - cod = existing_cod - - # Look for date near death indicator - found_date = None - for m in re.finditer(r'([+†]\s*(?:LE|AS|Unbekannt|[a-zA-ZäöüÄÖÜß0-9()/ +,;.:-]{1,100}?)?\s*)(\d{1,2}\.\d{1,2}\.\d{2,4})', notes, re.I): - marker_text = m.group(0) - if '†' in marker_text or re.search(r'\+\s*(?:LE|AS|Unbekannt|gestorben|verstorben|tod|\d)', marker_text, re.I): - found_date = parse_date(m.group(2)) - if found_date: - break - - if not found_date: - m_death_marker = re.search(r'(†\s*)(\d{1,2}\.\d{1,2}\.\d{2,4})', notes) - if m_death_marker: - found_date = parse_date(m_death_marker.group(2)) - - if found_date and not dod: - dod = found_date - - if not cod: - for m in re.finditer(r'([+†])\s*([a-zA-ZäöüÄÖÜß0-9()/ +,;.:-]{1,100}?)\s*\d{1,2}\.\d{1,2}\.\d{2,4}', notes, re.I): - indicator = m.group(1) - cod_candidate = m.group(2).strip() - if indicator == '+' and not re.search(r'\b(?:LE|AS|Unbekannt|gestorben|verstorben|tod)\b', cod_candidate, re.I): - continue - if cod_candidate: - cod = cod_candidate - break - - if not cod: - m_cod2 = re.search(r'[+†]\s*\d{1,2}\.\d{1,2}\.\d{2,4}\s*([a-zA-ZäöüÄÖÜß0-9()/ +,;.:-]{1,100})', notes, re.I) - if m_cod2: - cod_candidate = m_cod2.group(1).strip() - if cod_candidate: - cod = cod_candidate - else: - m_cod3 = re.search(r'[+†]\s*(LE|AS|Unbekannt)\b', notes, re.I) - if m_cod3: - cod = m_cod3.group(1).strip() - - if cod: - cod_lower = cod.lower().strip() - if cod_lower in ("le", "le (lungenentzündung)", "lungenentzündung", "lungenentzündung (lungenentzündung)"): - cod = "Lungenentzündung" - elif cod_lower in ("as", "altersschwäche", "altenschwäche", "alter"): - cod = "Altersschwäche" - elif cod_lower in ("unbekannt", "unklar"): - cod = "Unbekannt" - else: - # Replace abbreviations with full names (case-insensitive) - cod = re.sub(r'\bLE\b', 'Lungenentzündung', cod, flags=re.I) - cod = re.sub(r'\bAS\b', 'Altersschwäche', cod, flags=re.I) - cod = cod.replace('+', ' + ') - # Clean up multiple spaces - cod = re.sub(r'\s+', ' ', cod).strip(' ,;.-') - - # Re-check after replacement - cod_lower = cod.lower().strip() - if cod_lower in ("le", "le (lungenentzündung)", "lungenentzündung", "lungenentzündung (lungenentzündung)"): - cod = "Lungenentzündung" - elif cod_lower in ("as", "altersschwäche", "altenschwäche", "alter"): - cod = "Altersschwäche" - elif cod_lower in ("unbekannt", "unklar"): - cod = "Unbekannt" - - return resolved_status, dod, cod - -def main(): - print("Loading color variety seeds...") - variety_map = {} - variety_genotypes = {} - if os.path.exists(SEEDS_PATH): - with open(SEEDS_PATH, "r", encoding="utf-8") as f: - seeds = json.load(f) - for v in seeds: - variety_id = f"00000000-0000-0000-0000-{v['sortOrder'] + 1:012d}" - variety_map[v["name"].strip().lower()] = variety_id - if "english" in v and v["english"]: - variety_map[v["english"].strip().lower()] = variety_id - variety_genotypes[variety_id] = v.get("canonicalGenotype") - else: - print(f"Warning: Seeds path not found at {SEEDS_PATH}") - - md_files = sorted([f for f in os.listdir(DIR_PATH) if f.lower().endswith('.md')]) - print(f"Found {len(md_files)} markdown files in {DIR_PATH}.") - - raw_contacts = [] - raw_litters = [] - raw_gerbils = [] - - # Map to track which files contained explicit dates - file_explicit_dates = {} - - # 1. Parse JSON blocks from all markdown files and scope IDs by filename - for filename in md_files: - filepath = os.path.join(DIR_PATH, filename) - with open(filepath, 'r', encoding='utf-8') as f: - content = f.read() - - json_start = content.find("```json") - if json_start == -1: - continue - json_start += len("```json") - json_end = content.rfind("```") - if json_end == -1 or json_end <= json_start: - continue - - json_str = content[json_start:json_end].strip() - try: - data = json.loads(json_str) - - contacts_key = next((k for k in data if k.lower() == 'contacts'), None) - litters_key = next((k for k in data if k.lower() == 'litters'), None) - gerbils_key = next((k for k in data if k.lower() == 'gerbils'), None) - - file_contacts = data.get(contacts_key, []) if contacts_key else [] - file_litters = data.get(litters_key, []) if litters_key else [] - file_gerbils = data.get(gerbils_key, []) if gerbils_key else [] - - # Find explicit dates on this page to build chronological ordering - explicit_dates = [] - for l in file_litters: - d = parse_date(l.get("Date") or l.get("date") or l.get("DateOfBirth") or l.get("dateOfBirth")) - if d: - explicit_dates.append(d) - for g in file_gerbils: - d = parse_date(g.get("DateOfBirth") or g.get("dateOfBirth")) - if d: - explicit_dates.append(d) - - if explicit_dates: - # Convert to days since 2000-01-01 - day_vals = [date_to_days(d) for d in explicit_dates if date_to_days(d) is not None] - if day_vals: - file_explicit_dates[filename] = sum(day_vals) / len(day_vals) - - def scope_id(local_id): - if not local_id: - return None - local_str = str(local_id).strip() - if not local_str: - return None - return generate_guid(f"{filename}-{local_str}") - - for c in file_contacts: - old_id = c.get("Id") or c.get("id") - c["_filename"] = filename - c["_scoped_id"] = scope_id(old_id) - raw_contacts.append(c) - - for l in file_litters: - old_id = l.get("Id") or l.get("id") - l["_filename"] = filename - l["_scoped_id"] = scope_id(old_id) - l["_scoped_father_id"] = scope_id(l.get("FatherId") or l.get("fatherId")) - l["_scoped_mother_id"] = scope_id(l.get("MotherId") or l.get("motherId")) - raw_litters.append(l) - - for g in file_gerbils: - old_id = g.get("Id") or g.get("id") - g["_filename"] = filename - g["_scoped_id"] = scope_id(old_id) - g["_scoped_litter_id"] = scope_id(g.get("LitterId") or g.get("litterId")) - g["_scoped_origin_cid"] = scope_id(g.get("OriginContactId") or g.get("originContactId")) - - receiver_val = (g.get("ReceiverContactId") or g.get("receiverContactId") or - g.get("BuyerId") or g.get("buyerId") or - g.get("BuyerContactId") or g.get("buyerContactId") or - g.get("givenAwayContactId") or g.get("givenAwayToContactId") or - g.get("ownerContactId") or g.get("ownerId")) - g["_scoped_receiver_cid"] = scope_id(receiver_val) - raw_gerbils.append(g) - - except json.JSONDecodeError as e: - print(f"Failed to parse JSON in {filename}: {e}") - - # Chronological Interpolation: Estimate the date of each page based on neighboring pages with dates - file_dates = {} - sorted_files = sorted(md_files) - - # Simple linear interpolation / extrapolation - for i, fn in enumerate(sorted_files): - if fn in file_explicit_dates: - file_dates[fn] = file_explicit_dates[fn] - else: - # Look left for closest explicit date - left_val, left_dist = None, None - for j in range(i - 1, -1, -1): - if sorted_files[j] in file_explicit_dates: - left_val = file_explicit_dates[sorted_files[j]] - left_dist = i - j - break - - # Look right for closest explicit date - right_val, right_dist = None, None - for j in range(i + 1, len(sorted_files)): - if sorted_files[j] in file_explicit_dates: - right_val = file_explicit_dates[sorted_files[j]] - right_dist = j - i - break - - if left_val is not None and right_val is not None: - # Interpolate - file_dates[fn] = left_val + (right_val - left_val) * (left_dist / (left_dist + right_dist)) - elif left_val is not None: - # Extrapolate right (assume 30 days per page gap as placeholder) - file_dates[fn] = left_val + (left_dist * 30) - elif right_val is not None: - # Extrapolate left - file_dates[fn] = right_val - (right_dist * 30) - else: - # No dates in entire log? Default to 2011-01-01 - file_dates[fn] = date_to_days("2011-01-01") - - print(f"Parsed {len(raw_contacts)} raw contacts, {len(raw_litters)} raw litters, {len(raw_gerbils)} raw gerbils.") - - # Run extract.py to make sure stammbaum data is up to date - import subprocess - print("Running extract.py to extract Stammbäume...") - try: - subprocess.run([sys.executable, "extract.py"], check=True) - except Exception as e: - print(f"Warning: Failed to run extract.py: {e}") - - # Load Stammbaum data - stammbaum_only_animals = [] - animals_path = os.path.join(OUTPUT_DIR, "animals.json") - if os.path.exists(animals_path): - with open(animals_path, "r", encoding="utf-8") as f: - stammbaum_animals = json.load(f) - for a in stammbaum_animals: - sources = a.get("sourceFiles", []) - if any("stammbaum" in str(s).lower() for s in sources): - stammbaum_only_animals.append(a) - print(f"Loaded {len(stammbaum_only_animals)} Stammbaum animals.") - else: - print(f"Warning: Stammbaum animals.json not found at {animals_path}") - - # Run extract_docx.py to make sure docx data is up to date - print("Running extract_docx.py to extract docx...") - try: - subprocess.run([sys.executable, "extract_docx.py"], check=True) - except Exception as e: - print(f"Warning: Failed to run extract_docx.py: {e}") - - # Load Docx data - docx_animals = [] - docx_animals_path = os.path.join(OUTPUT_DIR, "docx_animals.json") - if os.path.exists(docx_animals_path): - with open(docx_animals_path, "r", encoding="utf-8") as f: - docx_animals = json.load(f) - print(f"Loaded {len(docx_animals)} animals from docx.") - else: - print(f"Warning: docx_animals.json not found at {docx_animals_path}") - - docx_litters = [] - docx_litters_path = os.path.join(OUTPUT_DIR, "docx_litters.json") - if os.path.exists(docx_litters_path): - with open(docx_litters_path, "r", encoding="utf-8") as f: - docx_litters = json.load(f) - print(f"Loaded {len(docx_litters)} litters from docx.") - else: - print(f"Warning: docx_litters.json not found at {docx_litters_path}") - - # Extract docx buyer contacts and add to raw_contacts - for da in docx_animals: - o_name = (da.get("owner") or "").strip() - if o_name: - o_scoped_id = generate_guid(f"docx-contact-{normalize_name(o_name)}") - raw_contacts.append({ - "Name": o_name, - "_filename": "Wurfchronik-Detail", - "_scoped_id": o_scoped_id - }) - - # Map docx litters and append to raw_litters - docx_litter_id_map = {} # wsCode -> scoped_id - for dl in docx_litters: - l_name = dl["litterId"] - dob_val = parse_date(dl["dob"]) - f_name = dl["fatherName"] - m_name = dl["motherName"] - ws_code = dl["wsCode"] - note_val = dl.get("note") - - # Parse survived/total born from wsCode (e.g. 4/5) - total_born = None - deaths_8w = None - if "/" in ws_code: - parts = ws_code.split("/") - if len(parts) == 2: - try: - survived = int(parts[0]) - total = int(parts[1]) - total_born = total - deaths_8w = max(0, total - survived) - except ValueError: - pass - - l_scoped_id = generate_guid(f"docx-litter-{normalize_name(ws_code)}-{dob_val or '0001-01-01'}") - docx_litter_id_map[(ws_code, dob_val)] = l_scoped_id - - raw_litters.append({ - "Id": l_scoped_id, - "Name": l_name, - "Date": dob_val or "0001-01-01", - "TotalBorn": total_born, - "DeathsWithin8Weeks": deaths_8w, - "FatherId": generate_guid(f"stammbaum-animal-{normalize_name(f_name)}"), # placeholder - "MotherId": generate_guid(f"stammbaum-animal-{normalize_name(m_name)}"), # placeholder - "ExpectedGoHomeDate": None, - "Notes": note_val if note_val else "Docx imported litter", - "PairingCode": None, - "ExternalRef": f"docx-litter-{l_scoped_id}", - "LitterLetter": l_name[0] if l_name and len(l_name) > 0 else None, - "_father_name": f_name, - "_mother_name": m_name, - "_filename": "Wurfchronik-Detail", - "_scoped_id": l_scoped_id, - "_scoped_father_id": generate_guid(f"stammbaum-animal-{normalize_name(f_name)}"), - "_scoped_mother_id": generate_guid(f"stammbaum-animal-{normalize_name(m_name)}") - }) - - - # Extract stammbaum contacts and add to raw_contacts - for a in stammbaum_only_animals: - b_name = (a.get("breeder") or "").strip() - if b_name: - b_scoped_id = generate_guid(f"stammbaum-contact-{normalize_name(b_name)}") - raw_contacts.append({ - "Name": b_name, - "_filename": "Stammbaum", - "_scoped_id": b_scoped_id - }) - z_name = (a.get("zucht") or "").strip() - if z_name: - z_scoped_id = generate_guid(f"stammbaum-contact-{normalize_name(z_name)}") - raw_contacts.append({ - "Name": z_name, - "_filename": "Stammbaum", - "_scoped_id": z_scoped_id - }) - - # Residency propagation for stammbaum animals - def is_clan_zucht(z): - if not z: - return False - norm = z.lower() - return "klein" in norm and "chaot" in norm and "extern" not in norm - - stammbaum_resident_ids = set() - for a in stammbaum_only_animals: - if is_clan_zucht(a.get("zucht")) or is_clan_zucht(a.get("zuchtCanon")): - stammbaum_resident_ids.add(a["id"]) - - # Propagate residency to parents of resident offspring - for _ in range(5): - for a in stammbaum_only_animals: - if a["id"] in stammbaum_resident_ids: - for p_ref in a.get("parentRefs", []): - p_key = (normalize_name(p_ref["name"]), parse_date(p_ref["dob"])) - for cand in stammbaum_only_animals: - if normalize_name(cand["name"]) == p_key[0]: - cand_dob = parse_date(cand["dob"]) - if not p_key[1] or cand_dob == p_key[1]: - stammbaum_resident_ids.add(cand["id"]) - - # Pre-index Wurfchronik litters from markdown - md_litters_idx = {} - for rl in raw_litters: - f_name = rl.get("FatherName") or rl.get("fatherName") or rl.get("ParentMaleName") or rl.get("parentMaleName") - m_name = rl.get("MotherName") or rl.get("motherName") or rl.get("ParentFemaleName") or rl.get("parentFemaleName") - ldate = parse_date(rl.get("Date") or rl.get("date") or rl.get("DateOfBirth") or rl.get("dateOfBirth")) - if f_name and m_name and ldate: - key = (normalize_name(f_name), normalize_name(m_name), ldate) - md_litters_idx[key] = rl - - # Create virtual litters for stammbaum animals - created_virtual_litters = {} - for a in stammbaum_only_animals: - parent_refs = a.get("parentRefs", []) - father_ref = next((p for p in parent_refs if p.get("roleGuess") == "father"), None) - mother_ref = next((p for p in parent_refs if p.get("roleGuess") == "mother"), None) - - a["_mapped_litter_scoped_id"] = None - if father_ref and mother_ref: - f_name = father_ref.get("name") - m_name = mother_ref.get("name") - dob_val = parse_date(a.get("dob")) - - mapped_litter = None - if dob_val: - key = (normalize_name(f_name), normalize_name(m_name), dob_val) - mapped_litter = md_litters_idx.get(key) - - if mapped_litter: - a["_mapped_litter_scoped_id"] = mapped_litter["_scoped_id"] - else: - v_key = (normalize_name(f_name), normalize_name(m_name), dob_val or "0001-01-01") - if v_key in created_virtual_litters: - a["_mapped_litter_scoped_id"] = created_virtual_litters[v_key] - else: - l_scoped_id = generate_guid(f"virtual-litter-{v_key[0]}-{v_key[1]}-{v_key[2]}") - created_virtual_litters[v_key] = l_scoped_id - a["_mapped_litter_scoped_id"] = l_scoped_id - - # Try to link parents to actual parsed stammbaum animals - f_scoped_id = None - m_scoped_id = None - f_dob = parse_date(father_ref.get("dob")) - m_dob = parse_date(mother_ref.get("dob")) - - for p_cand in stammbaum_only_animals: - cand_call_norm = normalize_name(get_call_name(p_cand["name"])) - if cand_call_norm == normalize_name(f_name) or normalize_name(p_cand["name"]) == normalize_name(f_name): - if not f_dob or parse_date(p_cand.get("dob")) == f_dob: - f_scoped_id = generate_guid(f"stammbaum-animal-{p_cand['id']}") - break - for p_cand in stammbaum_only_animals: - cand_call_norm = normalize_name(get_call_name(p_cand["name"])) - if cand_call_norm == normalize_name(m_name) or normalize_name(p_cand["name"]) == normalize_name(m_name): - if not m_dob or parse_date(p_cand.get("dob")) == m_dob: - m_scoped_id = generate_guid(f"stammbaum-animal-{p_cand['id']}") - break - - raw_litters.append({ - "Id": l_scoped_id, - "Name": f"Wurf von {f_name} + {m_name}", - "Date": dob_val or "0001-01-01", - "TotalBorn": None, - "DeathsWithin8Weeks": None, - "FatherId": f_scoped_id or generate_guid(f"stammbaum-animal-{normalize_name(f_name)}"), - "MotherId": m_scoped_id or generate_guid(f"stammbaum-animal-{normalize_name(m_name)}"), - "ExpectedGoHomeDate": None, - "Notes": "Pedigree virtual litter", - "PairingCode": None, - "ExternalRef": f"virtual-{l_scoped_id}", - "LitterLetter": None, - "_father_name": f_name, - "_mother_name": m_name, - "_filename": "Stammbaum", - "_scoped_id": l_scoped_id, - "_scoped_father_id": f_scoped_id or generate_guid(f"stammbaum-animal-{normalize_name(f_name)}"), - "_scoped_mother_id": m_scoped_id or generate_guid(f"stammbaum-animal-{normalize_name(m_name)}") - }) - - # 2. Resolve Contacts globally (deduplicate by normalized name) - - contact_by_norm_name = {} - contact_id_map = {} # scoped_old_id -> global_guid - - for rc in raw_contacts: - name_val = rc.get("Name") or rc.get("name") or rc.get("FullName") or rc.get("fullName") - if not name_val: - first = rc.get("FirstName") or rc.get("firstName") - last = rc.get("LastName") or rc.get("lastName") - if first or last: - name_val = f"{first or ''} {last or ''}".strip() - - if not name_val: - continue - - canon_name, should_keep = get_normalized_contact_name(name_val) - scoped_id = rc["_scoped_id"] - if not should_keep: - if scoped_id: - contact_id_map[scoped_id] = None - continue - - norm_name = normalize_name(canon_name) - - if norm_name not in contact_by_norm_name: - global_guid = generate_guid(f"contact-{norm_name}") - contact_by_norm_name[norm_name] = { - "Id": global_guid, - "Name": canon_name, - "Email": rc.get("Email") or rc.get("email"), - "Phone": rc.get("Phone") or rc.get("phone"), - "Address": rc.get("Address") or rc.get("address"), - "Notes": rc.get("Notes") or rc.get("notes") or rc.get("Note") or rc.get("note") - } - else: - gc = contact_by_norm_name[norm_name] - if not gc["Email"] and (rc.get("Email") or rc.get("email")): - gc["Email"] = rc.get("Email") or rc.get("email") - if not gc["Phone"] and (rc.get("Phone") or rc.get("phone")): - gc["Phone"] = rc.get("Phone") or rc.get("phone") - if not gc["Address"] and (rc.get("Address") or rc.get("address")): - gc["Address"] = rc.get("Address") or rc.get("address") - if not gc["Notes"] and (rc.get("Notes") or rc.get("notes") or rc.get("Note") or rc.get("note")): - gc["Notes"] = rc.get("Notes") or rc.get("notes") or rc.get("Note") or rc.get("note") - - if scoped_id: - contact_id_map[scoped_id] = contact_by_norm_name[norm_name]["Id"] - - resolved_contacts = list(contact_by_norm_name.values()) - print(f"Resolved to {len(resolved_contacts)} unique contacts.") - - # 3. Process Litters (normalize keys) - resolved_litters = [] - litter_id_map = {} # scoped_old_id -> new_guid - litter_by_scoped_id = {} - - for rl in raw_litters: - filename = rl.get("_filename") - name_val = rl.get("Name") or rl.get("name") - if not name_val: - name_val = "Wurf" - - dob_val = parse_date(rl.get("Date") or rl.get("date") or rl.get("DateOfBirth") or rl.get("dateOfBirth")) - if not dob_val: - dob_val = "0001-01-01" - - new_guid = rl["_scoped_id"] - if not new_guid: - new_guid = generate_guid(f"litter-{filename}-{name_val}-{dob_val}") - - scoped_old_id = rl["_scoped_id"] - if scoped_old_id: - litter_id_map[scoped_old_id] = new_guid - - total_born = rl.get("TotalBorn") or rl.get("totalBorn") or rl.get("LitterSize") or rl.get("litterSize") or rl.get("size") or rl.get("totalPups") - if total_born is not None: - try: - total_born = int(total_born) - except ValueError: - total_born = None - - deaths_8w = rl.get("DeathsWithin8Weeks") or rl.get("deathsWithin8Weeks") - if deaths_8w is not None: - try: - deaths_8w = int(deaths_8w) - except ValueError: - deaths_8w = None - elif total_born is not None and rl.get("survived") is not None: - try: - deaths_8w = total_born - int(rl.get("survived")) - except ValueError: - pass - - father_name = rl.get("FatherName") or rl.get("fatherName") or rl.get("ParentMaleName") or rl.get("parentMaleName") or rl.get("_father_name") - mother_name = rl.get("MotherName") or rl.get("motherName") or rl.get("ParentFemaleName") or rl.get("parentFemaleName") or rl.get("_mother_name") - - raw_ext_ref = rl.get("ExternalRef") or rl.get("externalRef") or rl.get("Id") or rl.get("id") - ext_ref_scoped = f"{filename}-{raw_ext_ref}" if raw_ext_ref else None - - l_record = { - "Id": new_guid, - "Name": name_val, - "Date": dob_val, - "TotalBorn": total_born, - "DeathsWithin8Weeks": deaths_8w, - "FatherId": rl["_scoped_father_id"], - "MotherId": rl["_scoped_mother_id"], - "ExpectedGoHomeDate": parse_date(rl.get("ExpectedGoHomeDate") or rl.get("expectedGoHomeDate")), - "Notes": rl.get("Notes") or rl.get("notes") or rl.get("Note") or rl.get("note"), - "PairingCode": rl.get("PairingCode") or rl.get("pairingCode"), - "ExternalRef": ext_ref_scoped, - "LitterLetter": rl.get("LitterLetter") or rl.get("litterLetter"), - "_father_name": father_name, - "_mother_name": mother_name, - "_filename": filename - } - resolved_litters.append(l_record) - litter_by_scoped_id[new_guid] = l_record - - print(f"Processed {len(resolved_litters)} litters.") - - # Helper to lookup litter dates for birth date estimation - def get_litter_date(l_id): - if l_id in litter_by_scoped_id: - d = litter_by_scoped_id[l_id]["Date"] - if d and d != "0001-01-01": - return d - return None - - # 4. Normalize and group Gerbils - all_processed_gerbils = [] - for rg in raw_gerbils: - filename = rg.get("_filename") - name_val = rg.get("Name") or rg.get("name") or rg.get("callName") - if not name_val: - name_val = "Unbekannt" - - dob_val = parse_date(rg.get("DateOfBirth") or rg.get("dateOfBirth")) - - new_guid = rg["_scoped_id"] - if not new_guid: - new_guid = generate_guid(f"gerbil-{filename}-{name_val}-{dob_val}") - - # Gender normalization - gender_val = rg.get("Gender") or rg.get("gender") or "unknown" - gender_val = str(gender_val).lower().strip() - if gender_val in ["m", "male", "männlich"]: - gender = "male" - elif gender_val in ["w", "f", "female", "weiblich"]: - gender = "female" - else: - gender = "unknown" - - dod_val = parse_date(rg.get("DateOfDeath") or rg.get("dateOfDeath") or rg.get("deathDate")) - gohome_val = parse_date(rg.get("GoHomeDate") or rg.get("goHomeDate") or rg.get("DateGivenAway") or rg.get("givenAwayDate") or rg.get("DateOfHandover") or rg.get("dateOfHandover") or rg.get("HandoverDate") or rg.get("DateOfSale") or rg.get("dateOfSale")) - - status_val = rg.get("Status") or rg.get("status") or "" - status_val = str(status_val).lower().strip() - status = "Breeding" - if dod_val or "deceased" in status_val or "verstorben" in status_val or "tod" in status_val or "dead" in status_val: - status = "Deceased" - elif gohome_val or "givenaway" in status_val or "abgegeben" in status_val or "verkauft" in status_val or "sold" in status_val: - status = "GivenAway" - elif "forsale" in status_val or "abzugeben" in status_val: - status = "ForSale" - elif "pet" in status_val or "liebhaber" in status_val: - status = "Pet" - else: - if dod_val: - status = "Deceased" - elif gohome_val: - status = "GivenAway" - - notes_val = rg.get("Notes") or rg.get("notes") or rg.get("Note") or rg.get("note") - existing_cod = rg.get("CauseOfDeath") or rg.get("causeOfDeath") or rg.get("DeathCause") - status, dod_val, cause_of_death_val = parse_death_info(notes_val, status, dod_val, existing_cod) - - # Explicit resolution for Ken'ichi's cause of death - if name_val == "Ken'ichi" and dob_val == "2015-03-01": - cause_of_death_val = "Duftdrüsen-Tumor" - - color_val = rg.get("ColorVarietyId") or rg.get("colorVarietyId") or rg.get("Color") or rg.get("color") or rg.get("ColorDescription") or rg.get("colorDescription") - existing_gt = rg.get("Genotype") or rg.get("genotype") - color_variety_id, genotype_val = resolve_color_and_genotype( - color_val, existing_gt, variety_map, variety_genotypes - ) - if color_val and not color_variety_id: - try: - uuid.UUID(str(color_val).strip()) - color_variety_id = str(color_val).strip() - except ValueError: - print(f"Warning: Unknown color variety '{color_val}' for gerbil '{name_val}' on page {filename}") - - origin_cid = rg["_scoped_origin_cid"] - receiver_cid = rg["_scoped_receiver_cid"] - - if origin_cid in contact_id_map: - origin_cid = contact_id_map[origin_cid] - if receiver_cid in contact_id_map: - receiver_cid = contact_id_map[receiver_cid] - - is_resident = rg.get("IsResident") or rg.get("isResident") - if is_resident is None: - is_resident = True - else: - is_resident = str(is_resident).lower() == "true" - - traits = rg.get("CharacterTraits") or rg.get("characterTraits") or [] - if not isinstance(traits, list): - traits = [str(traits)] - char_note = rg.get("CharacterNote") or rg.get("characterNote") - - is_deaf = rg.get("IsDeaf") or rg.get("isDeaf") - if is_deaf is not None: - is_deaf = str(is_deaf).lower() == "true" - - old_litter_id = rg["_scoped_litter_id"] - - raw_ext_ref = rg.get("ExternalRef") or rg.get("externalRef") or rg.get("Id") or rg.get("id") - ext_ref_scoped = f"{filename}-{raw_ext_ref}" if raw_ext_ref else None - - # Estimated effective date of birth for conflict checking - eff_dob_val = dob_val - if not eff_dob_val and old_litter_id: - # Try to get litter date if litter was resolved - mapped_l_id = litter_id_map.get(old_litter_id) - if mapped_l_id: - eff_dob_val = get_litter_date(mapped_l_id) - - # Determine explicit or litter-derived birth date (None if unknown/parent) - birth_date = dob_val - if not birth_date and old_litter_id: - mapped_l_id = litter_id_map.get(old_litter_id) - if mapped_l_id: - birth_date = get_litter_date(mapped_l_id) - - # If still no effective date, use the estimated page date - if not eff_dob_val: - eff_dob_val = days_to_date(file_dates[filename]) - - raw_breeder = rg.get("OriginBreeder") or rg.get("originBreeder") - if raw_breeder: - norm_b, keep_b = get_normalized_contact_name(raw_breeder) - raw_breeder = norm_b if keep_b else None - - all_processed_gerbils.append({ - "Id": new_guid, - "Name": name_val, - "Gender": gender, - "Status": status, - "LitterId": old_litter_id, # mapped later - "OriginContactId": origin_cid, - "ReceiverContactId": receiver_cid, - "EnclosureId": None, - "ColorVarietyId": color_variety_id, - "DateOfBirth": dob_val, - "DateOfDeath": dod_val, - "CauseOfDeath": cause_of_death_val, - "GoHomeDate": gohome_val, - "Genotype": genotype_val, - "Notes": rg.get("Notes") or rg.get("notes") or rg.get("Note") or rg.get("note"), - "ImportSource": rg.get("ImportSource") or rg.get("importSource") or filename, - "ExternalRef": ext_ref_scoped, - "RawImportData": rg.get("RawImportData") or rg.get("rawImportData") or json.dumps({"colorDescription": color_val if not color_variety_id else None}), - "OriginBreeder": raw_breeder or ("Zucht der kleinen Chaoten" if is_resident else None), - "NameSearch": normalize_name(name_val), - "CharacterTraits": traits, - "CharacterNote": char_note, - "IsDeaf": is_deaf, - "IsResident": is_resident, - "_photos": rg.get("photos", []), - "_old_scoped_litter_id": old_litter_id, - "_eff_dob": eff_dob_val, - "_birth_date": birth_date, - "_filename": filename, - "_old_id": rg.get("Id") or rg.get("id") - }) - - # Map and append stammbaum animals to all_processed_gerbils - for a in stammbaum_only_animals: - a_id = a["id"] - name_val = a["name"] - - gender_val = str(a.get("gender") or "").lower().strip() - if gender_val in ["m", "male"]: - gender = "male" - elif gender_val in ["w", "f", "female"]: - gender = "female" - else: - gender = "unknown" - - dob_val = parse_date(a.get("dob")) - dod_val = parse_date(a.get("death")) - - status = "Breeding" - if dod_val: - status = "Deceased" - elif a_id not in stammbaum_resident_ids: - status = "GivenAway" - else: - if dob_val: - try: - dt_dob = datetime.strptime(dob_val, "%Y-%m-%d") - dt_now = datetime.now() - age_years = (dt_now - dt_dob).days / 365.25 - if age_years >= 7.0: - status = "Deceased" - except Exception: - pass - - # Color variety mapping - existing_gt = a["genotype"]["rawGenotype"] if a["genotype"]["rawGenotype"] else None - color_variety_id, genotype_val = resolve_color_and_genotype( - a.get("farbschlag"), existing_gt, variety_map, variety_genotypes - ) - if not color_variety_id: - for fbv in a.get("farbschlagVariants", []): - cv_id, gt_val = resolve_color_and_genotype(fbv, existing_gt, variety_map, variety_genotypes) - if cv_id: - color_variety_id = cv_id - genotype_val = gt_val - break - - # Contact mapping - origin_cid = None - b_name = (a.get("breeder") or "").strip() - z_name = (a.get("zucht") or "").strip() - if b_name: - origin_cid = generate_guid(f"stammbaum-contact-{normalize_name(b_name)}") - elif z_name: - origin_cid = generate_guid(f"stammbaum-contact-{normalize_name(z_name)}") - - # Map to resolved global contact GUID - if origin_cid in contact_id_map: - origin_cid = contact_id_map[origin_cid] - else: - origin_cid = None - - - scoped_id = generate_guid(f"stammbaum-animal-{a_id}") - scoped_litter_id = a.get("_mapped_litter_scoped_id") - - is_deaf = a["genotype"].get("deaf") - - stammbaum_breeder = a.get("breeder") if a.get("breeder") else (a.get("zucht") if a.get("zucht") else None) - if stammbaum_breeder: - norm_b, keep_b = get_normalized_contact_name(stammbaum_breeder) - stammbaum_breeder = norm_b if keep_b else None - - all_processed_gerbils.append({ - "Id": scoped_id, - "Name": name_val, - "Gender": gender, - "Status": status, - "LitterId": scoped_litter_id, # mapped later in step 5 - "OriginContactId": origin_cid, - "ReceiverContactId": None, - "EnclosureId": None, - "ColorVarietyId": color_variety_id, - "DateOfBirth": dob_val, - "DateOfDeath": dod_val, - "CauseOfDeath": "Duftdrüsen-Tumor" if name_val == "Ken'ichi" and dob_val == "2015-03-01" else None, - "GoHomeDate": None, - "Genotype": genotype_val, - "Notes": None, - "ImportSource": ", ".join(a.get("sourceFiles", [])), - "ExternalRef": f"stammbaum-{a_id}", - "RawImportData": json.dumps({ - "rawGenotype": a["genotype"]["rawGenotype"], - "unmappedTokens": a["genotype"]["unmappedTokens"], - "breederText": a.get("breeder", "") - }, ensure_ascii=False), - "OriginBreeder": stammbaum_breeder, - "NameSearch": normalize_name(name_val), - "CharacterTraits": [], - "CharacterNote": None, - "IsDeaf": is_deaf, - "IsResident": a_id in stammbaum_resident_ids, - "_photos": a.get("photos", []), - "_old_scoped_litter_id": scoped_litter_id, - "_eff_dob": dob_val or "2010-01-01", - "_birth_date": dob_val, - "_filename": a.get("sourceFiles", ["Stammbaum"])[0], - "_old_id": a_id - }) - - # Map and append docx animals to all_processed_gerbils - for idx, da in enumerate(docx_animals): - name_val = da["name"] - gender = da["gender"] - - dob_val = parse_date(da.get("litterDob")) - dod_val = parse_date(da.get("deathDate")) - gohome_val = parse_date(da.get("abgabeDate")) - - # Status precedence - status = "Breeding" - if dod_val: - status = "Deceased" - elif gohome_val or da.get("owner"): - status = "GivenAway" - - # Color variety mapping - color_variety_id, genotype_val = resolve_color_and_genotype( - da.get("farbschlag"), None, variety_map, variety_genotypes - ) - - # Contact mapping (buyer) - receiver_cid = None - o_name = (da.get("owner") or "").strip() - if o_name: - receiver_cid = generate_guid(f"docx-contact-{normalize_name(o_name)}") - if receiver_cid in contact_id_map: - receiver_cid = contact_id_map[receiver_cid] - else: - receiver_cid = None - - # Scoped ID - scoped_id = generate_guid(f"docx-animal-{idx}-{normalize_name(name_val)}-{dob_val or '0001-01-01'}") - - # Litter ID mapping - ws_code = da.get("wsCode") - scoped_litter_id = docx_litter_id_map.get((ws_code, dob_val)) - - - # Raw import details - raw_import_payload = json.dumps({ - "abgabeWeight": da.get("abgabeWeight", ""), - "deathCause": da.get("deathCause", ""), - "partnerName": da.get("partnerName", ""), - "partnerDob": da.get("partnerDob", "") - }, ensure_ascii=False) - - # Residents: if sold/given away, it's not a resident - is_resident = not bool(o_name) - - all_processed_gerbils.append({ - "Id": scoped_id, - "Name": name_val, - "Gender": gender, - "Status": status, - "LitterId": scoped_litter_id, - "OriginContactId": None, - "ReceiverContactId": receiver_cid, # mapped in step 5 - "EnclosureId": None, - "ColorVarietyId": color_variety_id, - "DateOfBirth": dob_val, - "DateOfDeath": dod_val, - "CauseOfDeath": da.get("deathCause"), - "GoHomeDate": gohome_val, - "Genotype": genotype_val, - "Notes": None, - "ImportSource": "Wurfchronik-Detail.docx", - "ExternalRef": f"docx-{idx}-{normalize_name(name_val)}-{dob_val or '0001-01-01'}", - "RawImportData": raw_import_payload, - "OriginBreeder": "Zucht der kleinen Chaoten", - "NameSearch": normalize_name(name_val), - "CharacterTraits": [], - "CharacterNote": None, - "IsDeaf": None, - "IsResident": is_resident, - "_photos": da.get("photos", []), - "_old_scoped_litter_id": scoped_litter_id, - "_eff_dob": dob_val or "2020-01-01", - "_birth_date": dob_val, - "_filename": "Wurfchronik-Detail.docx", - "_old_id": name_val - }) - - # Build parenting dates lookup using old scoped IDs - parent_litter_dates = {} - - for l in resolved_litters: - ld = l["Date"] - if ld and ld != "0001-01-01": - for pid in [l["FatherId"], l["MotherId"]]: - if pid: - parent_litter_dates.setdefault(pid, []).append(ld) - - def are_compatible(g1, g2): - # Must have same gender (or one unknown) - if g1["Gender"] != "unknown" and g2["Gender"] != "unknown" and g1["Gender"] != g2["Gender"]: - bd1 = g1.get("_birth_date") - bd2 = g2.get("_birth_date") - if not (bd1 and bd2 and bd1 == bd2): - return False - - bd1 = g1.get("_birth_date") - bd2 = g2.get("_birth_date") - - # If both have explicit birth dates, they must match within 30 days - if bd1 and bd2: - days1 = date_to_days(bd1) - days2 = date_to_days(bd2) - if days1 is not None and days2 is not None: - if abs(days1 - days2) > 30: - return False - - # If g1 has birth date, and g2 has parenting dates, birth date must be before parenting dates - p_dates2 = parent_litter_dates.get(g2["Id"], []) - if bd1: - for pd in p_dates2: - if pd <= bd1: # Can't have litter before or on birth date - return False - - p_dates1 = parent_litter_dates.get(g1["Id"], []) - if bd2: - for pd in p_dates1: - if pd <= bd2: - return False - - return True - - # Group gerbils by name to perform deduplication - gerbil_groups = {} - for g in all_processed_gerbils: - name_key = get_dedup_name_key(g["Name"]) - if not name_key: - name_key = "unbekannt" - gerbil_groups.setdefault(name_key, []).append(g) - - resolved_gerbils = [] - gerbil_id_map = {} # old_scoped_id -> final_id - - color_keys = set(variety_map.keys()) - - for name_key, group in gerbil_groups.items(): - is_placeholder = ( - name_key in color_keys or - any(p in name_key for p in ["unbekannt", "unbenannt", "baby", "welpe", "jungtier", "unknown", "welpen"]) or - len(name_key) <= 2 - ) - - if is_placeholder: - # Placeholders: do NOT merge, keep all separate - for g in group: - resolved_gerbils.append(g) - gerbil_id_map[g["Id"]] = g["Id"] - continue - - # Partition group into compatible subsets - sub_groups = [] - for g in group: - placed = False - for sub in sub_groups: - if all(are_compatible(g, member) for member in sub): - sub.append(g) - placed = True - break - if not placed: - sub_groups.append([g]) - - # Merge each partition sub-group into a single gerbil - for sub in sub_groups: - if len(sub) == 1: - g = sub[0] - resolved_gerbils.append(g) - gerbil_id_map[g["Id"]] = g["Id"] - continue - - # Find the best primary record to merge into - best_g = None - best_score = -1 - for g in sub: - score = 0 - if g["_old_scoped_litter_id"]: score += 10 - if g["DateOfBirth"]: score += 5 - if g["Genotype"]: score += 3 - if g["ColorVarietyId"]: score += 2 - if g["ImportSource"] and "stammbaum" in g["ImportSource"].lower(): score += 20 - if g["Notes"] and not any(kw in g["Notes"].lower() for kw in ["parent", "mutter", "vater", "dam", "sire"]): score += 1 - if len(g["Name"]) > len(name_key) + 5: # likely has clan suffix - score += 15 - if score > best_score: - best_score = score - best_g = g - - # Merge fields - merged_notes = [] - if best_g["Notes"]: - merged_notes.append(best_g["Notes"]) - - # Merge photos - merged_photos = list(best_g.get("_photos", [])) - - # Track sources for debugging - sources = [best_g["_filename"]] - - for g in sub: - if g == best_g: - continue - gerbil_id_map[g["Id"]] = best_g["Id"] - sources.append(g["_filename"]) - - for ph in g.get("_photos", []): - if ph not in merged_photos: - merged_photos.append(ph) - - if not best_g["LitterId"] and g["LitterId"]: - best_g["LitterId"] = g["LitterId"] - if not best_g["DateOfBirth"] and g["DateOfBirth"]: - best_g["DateOfBirth"] = g["DateOfBirth"] - if not best_g["DateOfDeath"] and g["DateOfDeath"]: - best_g["DateOfDeath"] = g["DateOfDeath"] - if not best_g["CauseOfDeath"] and g["CauseOfDeath"]: - best_g["CauseOfDeath"] = g["CauseOfDeath"] - if not best_g["GoHomeDate"] and g["GoHomeDate"]: - best_g["GoHomeDate"] = g["GoHomeDate"] - if not best_g["Genotype"] and g["Genotype"]: - best_g["Genotype"] = g["Genotype"] - if not best_g["ColorVarietyId"] and g["ColorVarietyId"]: - best_g["ColorVarietyId"] = g["ColorVarietyId"] - if not best_g["OriginContactId"] and g["OriginContactId"]: - best_g["OriginContactId"] = g["OriginContactId"] - if not best_g["ReceiverContactId"] and g["ReceiverContactId"]: - best_g["ReceiverContactId"] = g["ReceiverContactId"] - if g["IsResident"]: - best_g["IsResident"] = True - - # Reconcile Gender: prefer a known gender over unknown, and prefer stammbaum over other sources - if best_g["Gender"] == "unknown" and g["Gender"] != "unknown": - best_g["Gender"] = g["Gender"] - elif best_g["Gender"] != "unknown" and g["Gender"] != "unknown" and best_g["Gender"] != g["Gender"]: - if g["ImportSource"] and "stammbaum" in g["ImportSource"].lower(): - if not best_g["ImportSource"] or "stammbaum" not in best_g["ImportSource"].lower(): - best_g["Gender"] = g["Gender"] - - # Status precedence: Deceased > GivenAway > Breeding/Pet - if g["Status"] == "Deceased": - best_g["Status"] = "Deceased" - elif g["Status"] == "GivenAway" and best_g["Status"] not in ["Deceased"]: - best_g["Status"] = "GivenAway" - - if g["Notes"] and g["Notes"] not in merged_notes: - # Ignore redundant dummy notes - if not any(kw in g["Notes"].lower() for kw in ["parent listed", "mutter von", "vater von", "dam of", "sire of"]): - merged_notes.append(g["Notes"]) - - - if merged_notes: - best_g["Notes"] = " | ".join(merged_notes) - - best_g["_photos"] = merged_photos - - # Print merge trace - print(f"Deduplicated same-animal name '{best_g['Name']}': merged {len(sub)} entries across files: {', '.join(sources)}") - - resolved_gerbils.append(best_g) - gerbil_id_map[best_g["Id"]] = best_g["Id"] - - print(f"Deduplicated to {len(resolved_gerbils)} unique gerbil records.") - - # 5. Map Gerbils to Litters - for g in resolved_gerbils: - old_lid = g["_old_scoped_litter_id"] - if old_lid in litter_id_map: - g["LitterId"] = litter_id_map[old_lid] - else: - g["LitterId"] = None - # Clean helper fields - del g["_old_scoped_litter_id"] - del g["_eff_dob"] - if "_birth_date" in g: - del g["_birth_date"] - del g["_filename"] - del g["_old_id"] - - # Gather final valid gerbil IDs - valid_gerbil_ids = {g["Id"] for g in resolved_gerbils} - - # Create name lookup for resolved gerbils - gerbil_by_norm_name = {} - for g in resolved_gerbils: - n_key = normalize_name(g["Name"]) - gerbil_by_norm_name.setdefault(n_key, []).append(g) - - # Also index by call-name to resolve parents who are only listed by call-name - c_key = normalize_name(get_call_name(g["Name"])) - if c_key != n_key: - gerbil_by_norm_name.setdefault(c_key, []).append(g) - - # Map raw Guid if present (convert if old_id mapped to new_guid) - for l in resolved_litters: - if l["FatherId"] in gerbil_id_map: - l["FatherId"] = gerbil_id_map[l["FatherId"]] - if l["MotherId"] in gerbil_id_map: - l["MotherId"] = gerbil_id_map[l["MotherId"]] - - # Clean foreign keys that do not point to a valid gerbil - if l["FatherId"] and l["FatherId"] not in valid_gerbil_ids: - l["FatherId"] = None - if l["MotherId"] and l["MotherId"] not in valid_gerbil_ids: - l["MotherId"] = None - - # Parent Resolver (Global Name Matching) - resolved_fathers = 0 - resolved_mothers = 0 - - for l in resolved_litters: - # Match Father by Name - f_name = l["_father_name"] - if f_name and not l["FatherId"]: - f_norm = normalize_name(f_name) - candidates = gerbil_by_norm_name.get(f_norm, []) - valid_candidates = [] - for c in candidates: - # Map to final deduplicated ID - final_id = gerbil_id_map.get(c["Id"]) - if not final_id: - continue - # Retrieve final record - final_c = next((rg for rg in resolved_gerbils if rg["Id"] == final_id), None) - if final_c and final_c["Gender"] in ["male", "unknown"]: - # Ensure parent is born before litter if birth date is known - if l["Date"] != "0001-01-01" and final_c["DateOfBirth"]: - if final_c["DateOfBirth"] < l["Date"]: - valid_candidates.append(final_c) - else: - valid_candidates.append(final_c) - - if len(valid_candidates) == 1: - l["FatherId"] = valid_candidates[0]["Id"] - resolved_fathers += 1 - elif len(valid_candidates) > 1: - l["FatherId"] = valid_candidates[0]["Id"] - resolved_fathers += 1 - - # Match Mother by Name - m_name = l["_mother_name"] - if m_name and not l["MotherId"]: - m_norm = normalize_name(m_name) - candidates = gerbil_by_norm_name.get(m_norm, []) - valid_candidates = [] - for c in candidates: - final_id = gerbil_id_map.get(c["Id"]) - if not final_id: - continue - final_c = next((rg for rg in resolved_gerbils if rg["Id"] == final_id), None) - if final_c and final_c["Gender"] in ["female", "unknown"]: - if l["Date"] != "0001-01-01" and final_c["DateOfBirth"]: - if final_c["DateOfBirth"] < l["Date"]: - valid_candidates.append(final_c) - else: - valid_candidates.append(final_c) - - if len(valid_candidates) == 1: - l["MotherId"] = valid_candidates[0]["Id"] - resolved_mothers += 1 - elif len(valid_candidates) > 1: - l["MotherId"] = valid_candidates[0]["Id"] - resolved_mothers += 1 - - # Cleanup internal keys - del l["_father_name"] - del l["_mother_name"] - del l["_filename"] - - print(f"Globally resolved {resolved_fathers} fathers and {resolved_mothers} mothers.") - - # Set IsBreeder and IsReceiver flags on contacts - breeder_ids = {g["OriginContactId"] for g in resolved_gerbils if g.get("OriginContactId")} - receiver_ids = {g["ReceiverContactId"] for g in resolved_gerbils if g.get("ReceiverContactId")} - for c in resolved_contacts: - c_id = c["Id"] - is_breeder = c_id in breeder_ids - is_receiver = c_id in receiver_ids - if not is_breeder and not is_receiver: - is_receiver = True - c["IsBreeder"] = is_breeder - c["IsReceiver"] = is_receiver - - # Set and map gerbilPhotos - resolved_photos = [] - for g in resolved_gerbils: - for idx, photo_rel in enumerate(g.get("_photos", [])): - photo_guid = generate_guid(f"photo-{photo_rel}") - ext = os.path.splitext(photo_rel)[1] or ".jpeg" - fn_guid = photo_guid.replace("-", "") - resolved_photos.append({ - "Id": photo_guid, - "GerbilId": g["Id"], - "FileName": f"{fn_guid}{ext}", - "SortOrder": idx, - "_source_path": photo_rel - }) - - # 6. Save final output JSON payload - os.makedirs(OUTPUT_DIR, exist_ok=True) - payload = { - "contacts": resolved_contacts, - "litters": resolved_litters, - "gerbils": resolved_gerbils, - "gerbilPhotos": resolved_photos - } - - with open(OUTPUT_FILE, "w", encoding="utf-8") as f: - json.dump(payload, f, ensure_ascii=False, indent=2) - - print(f"Successfully wrote database-ready import file to: {OUTPUT_FILE}") - print(f" Contacts: {len(payload['contacts'])}") - print(f" Litters: {len(payload['litters'])}") - print(f" Gerbils: {len(payload['gerbils'])}") - print(f" Photos: {len(payload['gerbilPhotos'])}") - -if __name__ == "__main__": - main() +import os +import json +import re +import uuid +import sys +from datetime import datetime + +# Prevent encoding crashes on Windows consoles when printing unicode +if sys.platform.startswith('win'): + try: + sys.stdout.reconfigure(encoding='utf-8') + except Exception: + pass + +# Paths +DIR_PATH = r"C:\Users\gulum\dev\Wurfchronik_Bilder" +SEEDS_PATH = r"C:\Users\gulum\dev\GerbilManager\gerbil-manager-web\src\genetics\colorVarietySeed.backend.json" +OUTPUT_DIR = r"C:\Users\gulum\dev\GerbilManager\tools\import\output" +OUTPUT_FILE = os.path.join(OUTPUT_DIR, "resolved_import.json") + +def generate_guid(key_str): + """Generate a stable UUID string based on a key.""" + return str(uuid.uuid5(uuid.NAMESPACE_DNS, key_str)) + +def to_valid_guid(val): + if not val: + return None + val_str = str(val).strip() + try: + uuid.UUID(val_str) + return val_str + except ValueError: + return generate_guid(val_str) + +def normalize_name(name): + if not name: + return "" + return "".join(c for c in name.lower() if c.isalnum()) + +def get_normalized_contact_name(name): + if not name: + return "", False + + n = "".join(c for c in name.lower() if c.isalnum()) + + to_discard = { + "chevroletcamarooftopolino", + "cindyvprivatzuchtgießen", + "cindyvprivatzuchtgiessen", + "inuschofblackforest", + "stichvonprivatzuchtgießen", + "stichvonprivatzuchtgiessen", + "tarzanofsamsimar", + "hanserennersposeidon", + "livingforcesidefix", + "livingforcesnando" + } + if n in to_discard: + return "", False + + norm_map = { + "alessandrab": "Alessandra Bartoletti", + "andreafey": "Andrea und Stefanie Fey", + "andreastefaniefey": "Andrea und Stefanie Fey", + "angelachristoph": "Angela und Ekki Christoph", + "angelaekkichristoph": "Angela und Ekki Christoph", + "angie": "Angie Reichert-Cambeis", + "angierc": "Angie Reichert-Cambeis", + "anjasch": "Anja Schaumburg", + "ankeb": "Anke Busch", + "ankeksch": "Anke Koppenhöfer", + "annaq": "Anna Quark", + "annikab": "Annika Balser", + "annkathrind": "Ann-kathrin Dressler", + "arturaleksandrapolamundrzynski": "Artur, Aleksandra und Pola Mundrzynski", + "arturaleksandraundpolamundrzynski": "Artur, Aleksandra und Pola Mundrzynski", + "astridr": "Astrid Rohmann", + "astridroh": "Astrid Rohmann", + "barbarar": "Barbara Riegler", + "biancab": "Bianca Bernhardt", + "biancam": "Bianca Mayer + Alex Wachten", + "biancamayeralexw": "Bianca Mayer + Alex Wachten", + "birgitk": "Birgit Knopp", + "birgitkropp": "Birgit Knopp", + "birgittabüskens": "Birgitta Müller-Büskens", + "blackforestgv": "Black Forest", + "brigittast": "Brigitta Struve", + "buntefellnasen": "bunten Fellnasen", + "buntenfellnase": "bunten Fellnasen", + "buntenfellnasen": "bunten Fellnasen", + "carolal": "Carola und Hannah Lerch", + "carolalerch": "Carola und Hannah Lerch", + "chalfontstud": "Chalfont Stud, Freddy Braun", + "christianefuchs": "Christiane Fuchs + Michael Höhnert", + "christiek": "Christiane Kiessling", + "christinam": "Christine Monika Mai", + "christinek": "Christiane Kiessling", + "christiner": "Christina Rudloff", + "claudialöhr": "Claudia Löhr und Michael Koob", + "claudialöhrmichaelkoob": "Claudia Löhr und Michael Koob", + "colorfulfurrygerbils": "Colorful Furry Gerbils", + "colourfulfurrygerbils": "Colorful Furry Gerbils", + "danielakasberger": "Daniela Käsberger", + "dörthe": "Dörthe Petzmann", + "emelyhehlhorn": "Emely Mehlhorn", + "fabianb": "Fabian Büdel", + "floriang": "Florian Gries", + "hannahraths": "Hannah Ruths", + "hanserenners": "Hanse Renner", + "haraldmariareitzlennemann": "Harald und Maria Reitz-Lennemann", + "haraldundmariarl": "Harald und Maria Reitz-Lennemann", + "heike": "Heike + Heiko Scheurich", + "heikebarklay": "Heike Barklage", + "heikoloos": "Heiko Loos und Annette Becker", + "heikoloosannettebecker": "Heiko Loos und Annette Becker", + "inahübner": "Ina Hübner + Nicolai Thome", + "inapeines": "Ina Pleines", + "janab": "Jana Beikert", + "jasmin": "Jasmin Weber", + "jasminweber": "Jasmin Weber und Ronny Reichelt", + "jasminweberromyreichelt": "Jasmin Weber und Ronny Reichelt", + "jasminweberronnyreichelt": "Jasmin Weber und Ronny Reichelt", + "joystaatberg": "Joy Staalberg", + "juliaa": "Julia Ast", + "juttam": "Jutta Metz", + "karina": "Karina Luft", + "karinborgotti": "Karin Borsotti", + "katjal": "Katja Leffeck", + "kimw": "Kim Waldschmidt", + "kkchaos": "KK-Chaos", + "kkchaosofkkchaos": "KK-Chaos", + "kleinechaoten": "Zucht der kleinen Chaoten", + "kleinenchaoten": "Zucht der kleinen Chaoten", + "kriegernmitkrallen": "Krieger mit Krallen", + "lennylengo": "Lenny Lengo", + "lisa": "Lisa und Marcel Kunz", + "lisach": "Lisa und Lydia Christ", + "lisachrist": "Lisa und Lydia Christ", + "lisalydiachrist": "Lisa und Lydia Christ", + "littlefellows": "little fellows", + "littlerunners": "little runners", + "maikef": "Maike Franz", + "maintalerpz": "Privatzucht Maintal", + "marinau": "Marina Unger", + "martinaandreaswestfeld": "Martina und Andreas Westfeld", + "martinaw": "Martina und Andreas Westfeld", + "martinawestfeld": "Martina und Andreas Westfeld", + "meikesch": "Meike und Heiko Sch.", + "melanief": "Melanie Fey und Thorben Meier", + "melaniefey": "Melanie Fey und Thorben Meier", + "melaniefeythorbenmeier": "Melanie Fey und Thorben Meier", + "michaeldavidschmitz": "Michael und David Schmitz", + "michaelschmitz": "Michael und David Schmitz", + "natascham": "Natascha Marienfeld", + "nicolel": "Nicole Lannert", + "nicolen": "Nicole Nuzzo", + "nicolet": "Nicole Tischler", + "nicolew": "Nicole Webersinn", + "nicost": "Nico Stamm", + "noelstrahbach": "Noel Strohbach", + "nora": "Nora Rudersdorf geb. Holzbach", + "norah": "Nora Holzbach", + "norarh": "Nora Rudersdorf", + "oflennylengo": "Lenny Lengo", + "pascale": "Pascale Diefenbach", + "pascaledießenbach": "Pascale Diefenbach", + "patriciap": "Patricia Petry", + "paul": "Paul W.", + "paula": "Paula Gabler", + "paulag": "Paula Gabler", + "privatzuchtmuecke": "Privatzucht Mücke", + "privatzuchtmücke": "Privatzucht Mücke", + "pzmaintal": "Privatzucht Maintal", + "pzmuecke": "Privatzucht Mücke", + "pzmücke": "Privatzucht Mücke", + "pzseligenstadt": "Privatzucht Seligenstadt", + "ramonag": "Ramona Gömpel", + "ranialößler": "Rania Löffler", + "rominahubrich": "Romina Milde / Hubrich", + "rominamilde": "Romina Milde / Hubrich", + "sandrak": "Sandra Kubas", + "sarahlöuer": "Sarah Löwer", + "sarahz": "Sarah Zitzer", + "saskiaw": "Saskia Wucher", + "schlossmaus": "Schlossmäuse", + "schlossmäuse": "Schlossmäuse", + "schlossmäusen": "Schlossmäuse", + "sebastiansch": "Sebastian Schmitt", + "silkewolfganghintze": "Silke und Wolfgang Hintze", + "smilla": "Smilla H.", + "steffi": "Steffi K.", + "susanneninat": "Susanne + Nina Thomas", + "susannet": "Susanne + Torsten Saum", + "theresavalenca": "Theresia Valenca (Kopp)", + "theresiavalenca": "Theresia Valenca (Kopp)", + "timpf": "Tim Pfeiffer und Michelle Mai", + "timpfeifermichellemai": "Tim Pfeiffer und Michelle Mai", + "timpfeiffer": "Tim Pfeiffer und Michelle Mai", + "timpfeiffermichellemai": "Tim Pfeiffer und Michelle Mai", + "tonoböckenseld": "Tono Böckenfeld", + "topol": "Topolino", + "ulrichmüller": "Ulrich + Angela Müller", + "ulrikec": "Ulrike Cordes", + "ulriker": "Ulrike Ruppel", + "ulrikesch": "Ulrike Schulz", + "ulriket": "Ulrike Treml", + "vanessab": "Vanessa Becker", + "vanessag": "Vanessa Groll", + "veragreywitz": "Vera Geywitz", + "vonprivatzuchtmaintal": "Privatzucht Maintal", + "wolfgangfaus": "Wolfgang Faust", + "yvonne": "Yvonne Obendorfer", + "zuchtderkleinenchaoten": "Zucht der kleinen Chaoten", + } + + if n in norm_map: + return norm_map[n], True + + return name, True + +def get_call_name(name): + if not name: + return "" + n = name.strip() + n = re.sub(r'\[[^\]]+\]$', '', n).strip() + n = re.split(r'\s+(?:of|von\s+den|von\s+der|v\.\s?d\.|von)\s+', n, flags=re.IGNORECASE)[0].strip() + return n + +def get_dedup_name_key(name): + if not name: + return "" + n = name.lower().strip() + # Strip common suffixes/prefixes and parentheticals + n = re.sub(r'\b(?:von\s+den|v\.?\s*d\.?|v\.?o\.?)\s+(?:kleinen\s+)?chaoten\b', '', n) + n = re.sub(r'\bvon\s+der\s+schlossm\w+\b', '', n) + n = re.sub(r'\bvon\s+der\s+bunten\s+fellnasen?\b', '', n) + n = re.sub(r'\bof\s+black\s+forest\b', '', n) + n = re.sub(r'\b\(?rv\)?\b', '', n) + n = re.sub(r'\bgen\.\s+\w+', '', n) + # Clean up parentheses or brackets + n = re.sub(r'\(.*?\)', '', n) + n = re.sub(r'\[.*?\]', '', n) + return "".join(c for c in n if c.isalnum()) + +def clean_color_name(c_desc): + if not c_desc: + return "", False + + # Lowercase and strip + c = c_desc.lower().strip() + + # Check for Schecke + is_schecke = False + if re.search(r'\bsp\b|\bsp\d|\bsp[*(²³]|\bspotted|\bschecke|[- ]sp\b|\w+sp\b', c): + is_schecke = True + + # Standardize parentheticals for schimmel + c = c.replace("(schimmel)", "schimmel") + c = c.replace("(schimmel-hell)", "schimmel hell") + c = c.replace("(schimmel hell)", "schimmel hell") + + # Strip schecke/sp markers and any trailing text starting from sp + c = re.sub(r'\([- ]?sp(otted)?\)', '', c) # handles (-sp) + c = re.sub(r'[- ]?sp(otted)?\b.*', '', c) # handles -sp(k), -sp*(k), -sp, etc. + c = re.sub(r'[- ]?schecke\b.*', '', c) + c = re.sub(r'[- ]?spotted\b.*', '', c) + + # Strip any other parentheticals, symbols, or trailing stars/numbers + c = re.sub(r'\s*\(.*?\)\s*', ' ', c) + c = re.sub(r'[²³*]', '', c) + c = c.strip() + + # Mapping table for abbreviations, typos, and specific combinations + mapping = { + "antra": "anthrazit", + "anthra": "anthrazit", + "ankazit": "anthrazit", + "antrazit": "anthrazit", + "pew": "rew", + "bew": "hermelin", + "harder": "marder", + "kohli": "kohlfuchs", + "aligerfuchs": "algierfuchs", + "algiesfuchs": "algierfuchs", + "schw": "schwarz", + "sa": "silberagouti", + "a": "agouti", + "cp-sa": "cp-silberagouti", + "cp-a": "cp-agouti", + "cp-a-hell": "cp-agouti-hell", + "cp-aisa": "cp-agouti", + "cp-a / rcp-sa": "cp-agouti", + } + + if c in mapping: + c = mapping[c] + + return c, is_schecke + +def resolve_color_and_genotype(color_val, existing_genotype, variety_map, variety_genotypes): + if not color_val: + return None, existing_genotype + color_str = str(color_val).strip() + clean_name, is_schecke = clean_color_name(color_str) + # Match color in variety_map + color_variety_id = None + if clean_name in variety_map: + color_variety_id = variety_map[clean_name] + else: + for seed_name, seed_id in variety_map.items(): + if seed_name in clean_name or clean_name in seed_name: + color_variety_id = seed_id + break + # Update genotype if it's a Schecke + genotype = existing_genotype + if is_schecke: + if genotype: + if "spsp" in genotype: + genotype = genotype.replace("spsp", "Spsp") + elif "Spsp" not in genotype and "Sp" not in genotype: + genotype = f"{genotype} Spsp".strip() + else: + canonical = variety_genotypes.get(color_variety_id) + if canonical: + if "spsp" in canonical: + genotype = canonical.replace("spsp", "Spsp") + else: + genotype = f"{canonical} Spsp".strip() + else: + genotype = "Spsp" + return color_variety_id, genotype + +def parse_date(d): + """Convert variations of date formats to YYYY-MM-DD.""" + if not d or d == "0001-01-01": + return None + d = str(d).strip() + # Try YYYY-MM-DD + if re.match(r"^\d{4}-\d{2}-\d{2}$", d): + parts = d.split("-") + year = int(parts[0]) + if 1900 < year < 2000: + year += 100 + d = f"{year}-{parts[1]}-{parts[2]}" + elif year == 1900: + return None + return d + # Try DD.MM.YYYY or D.M.YY + match = re.match(r"^(\d{1,2})\.(\d{1,2})\.(\d{2,4})$", d) + if match: + day = int(match.group(1)) + month = int(match.group(2)) + year = int(match.group(3)) + if year < 100: + year += 2000 + elif 1900 < year < 2000: + year += 100 + elif year == 1900: + return None + try: + return datetime(year, month, day).strftime("%Y-%m-%d") + except ValueError: + pass + # Try ISO timestamp + try: + dt = datetime.fromisoformat(d.replace("Z", "+00:00")) + year = dt.year + if 1900 < year < 2000: + dt = dt.replace(year=year + 100) + return dt.strftime("%Y-%m-%d") + elif year == 1900: + return None + return dt.strftime("%Y-%m-%d") + except ValueError: + pass + return None + +def date_to_days(dt_str): + if not dt_str: + return None + try: + return (datetime.strptime(dt_str, "%Y-%m-%d") - datetime(2000, 1, 1)).days + except ValueError: + return None + +def days_to_date(days): + import datetime as dt + return (dt.datetime(2000, 1, 1) + dt.timedelta(days=int(days))).strftime("%Y-%m-%d") + +def parse_death_info(notes, status, existing_dod, existing_cod): + if not notes: + return status, existing_dod, existing_cod + + has_death_indicator = '†' in notes or 'verstorben' in notes.lower() or 'gestorben' in notes.lower() or 'todesdatum' in notes.lower() or '/+' in notes + if '+' in notes: + if re.search(r'\+\s*(?:LE|AS|Unbekannt|gestorben|verstorben|tod)\b', notes, re.I) or re.search(r'\+\s*\d{1,2}\.\d{1,2}\.\d{2,4}', notes) or '/+' in notes: + has_death_indicator = True + + resolved_status = status + if has_death_indicator: + resolved_status = "Deceased" + + dod = existing_dod + cod = existing_cod + + # Look for date near death indicator + found_date = None + for m in re.finditer(r'([+†]\s*(?:LE|AS|Unbekannt|[a-zA-ZäöüÄÖÜß0-9()/ +,;.:-]{1,100}?)?\s*)(\d{1,2}\.\d{1,2}\.\d{2,4})', notes, re.I): + marker_text = m.group(0) + if '†' in marker_text or re.search(r'\+\s*(?:LE|AS|Unbekannt|gestorben|verstorben|tod|\d)', marker_text, re.I): + found_date = parse_date(m.group(2)) + if found_date: + break + + if not found_date: + m_death_marker = re.search(r'(†\s*)(\d{1,2}\.\d{1,2}\.\d{2,4})', notes) + if m_death_marker: + found_date = parse_date(m_death_marker.group(2)) + + if found_date and not dod: + dod = found_date + + if not cod: + for m in re.finditer(r'([+†])\s*([a-zA-ZäöüÄÖÜß0-9()/ +,;.:-]{1,100}?)\s*\d{1,2}\.\d{1,2}\.\d{2,4}', notes, re.I): + indicator = m.group(1) + cod_candidate = m.group(2).strip() + if indicator == '+' and not re.search(r'\b(?:LE|AS|Unbekannt|gestorben|verstorben|tod)\b', cod_candidate, re.I): + continue + if cod_candidate: + cod = cod_candidate + break + + if not cod: + m_cod2 = re.search(r'[+†]\s*\d{1,2}\.\d{1,2}\.\d{2,4}\s*([a-zA-ZäöüÄÖÜß0-9()/ +,;.:-]{1,100})', notes, re.I) + if m_cod2: + cod_candidate = m_cod2.group(1).strip() + if cod_candidate: + cod = cod_candidate + else: + m_cod3 = re.search(r'[+†]\s*(LE|AS|Unbekannt)\b', notes, re.I) + if m_cod3: + cod = m_cod3.group(1).strip() + + if cod: + cod_lower = cod.lower().strip() + if cod_lower in ("le", "le (lungenentzündung)", "lungenentzündung", "lungenentzündung (lungenentzündung)"): + cod = "Lungenentzündung" + elif cod_lower in ("as", "altersschwäche", "altenschwäche", "alter"): + cod = "Altersschwäche" + elif cod_lower in ("unbekannt", "unklar"): + cod = "Unbekannt" + else: + # Replace abbreviations with full names (case-insensitive) + cod = re.sub(r'\bLE\b', 'Lungenentzündung', cod, flags=re.I) + cod = re.sub(r'\bAS\b', 'Altersschwäche', cod, flags=re.I) + cod = cod.replace('+', ' + ') + # Clean up multiple spaces + cod = re.sub(r'\s+', ' ', cod).strip(' ,;.-') + + # Re-check after replacement + cod_lower = cod.lower().strip() + if cod_lower in ("le", "le (lungenentzündung)", "lungenentzündung", "lungenentzündung (lungenentzündung)"): + cod = "Lungenentzündung" + elif cod_lower in ("as", "altersschwäche", "altenschwäche", "alter"): + cod = "Altersschwäche" + elif cod_lower in ("unbekannt", "unklar"): + cod = "Unbekannt" + + return resolved_status, dod, cod + +def main(): + print("Loading color variety seeds...") + variety_map = {} + variety_genotypes = {} + if os.path.exists(SEEDS_PATH): + with open(SEEDS_PATH, "r", encoding="utf-8") as f: + seeds = json.load(f) + for v in seeds: + variety_id = f"00000000-0000-0000-0000-{v['sortOrder'] + 1:012d}" + variety_map[v["name"].strip().lower()] = variety_id + if "english" in v and v["english"]: + variety_map[v["english"].strip().lower()] = variety_id + variety_genotypes[variety_id] = v.get("canonicalGenotype") + else: + print(f"Warning: Seeds path not found at {SEEDS_PATH}") + + md_files = sorted([f for f in os.listdir(DIR_PATH) if f.lower().endswith('.md')]) + print(f"Found {len(md_files)} markdown files in {DIR_PATH}.") + + raw_contacts = [] + raw_litters = [] + raw_gerbils = [] + + # Map to track which files contained explicit dates + file_explicit_dates = {} + + # 1. Parse JSON blocks from all markdown files and scope IDs by filename + for filename in md_files: + filepath = os.path.join(DIR_PATH, filename) + with open(filepath, 'r', encoding='utf-8') as f: + content = f.read() + + json_start = content.find("```json") + if json_start == -1: + continue + json_start += len("```json") + json_end = content.rfind("```") + if json_end == -1 or json_end <= json_start: + continue + + json_str = content[json_start:json_end].strip() + try: + data = json.loads(json_str) + + contacts_key = next((k for k in data if k.lower() == 'contacts'), None) + litters_key = next((k for k in data if k.lower() == 'litters'), None) + gerbils_key = next((k for k in data if k.lower() == 'gerbils'), None) + + file_contacts = data.get(contacts_key, []) if contacts_key else [] + file_litters = data.get(litters_key, []) if litters_key else [] + file_gerbils = data.get(gerbils_key, []) if gerbils_key else [] + + # Find explicit dates on this page to build chronological ordering + explicit_dates = [] + for l in file_litters: + d = parse_date(l.get("Date") or l.get("date") or l.get("DateOfBirth") or l.get("dateOfBirth")) + if d: + explicit_dates.append(d) + for g in file_gerbils: + d = parse_date(g.get("DateOfBirth") or g.get("dateOfBirth")) + if d: + explicit_dates.append(d) + + if explicit_dates: + # Convert to days since 2000-01-01 + day_vals = [date_to_days(d) for d in explicit_dates if date_to_days(d) is not None] + if day_vals: + file_explicit_dates[filename] = sum(day_vals) / len(day_vals) + + def scope_id(local_id): + if not local_id: + return None + local_str = str(local_id).strip() + if not local_str: + return None + return generate_guid(f"{filename}-{local_str}") + + for c in file_contacts: + old_id = c.get("Id") or c.get("id") + c["_filename"] = filename + c["_scoped_id"] = scope_id(old_id) + raw_contacts.append(c) + + for l in file_litters: + old_id = l.get("Id") or l.get("id") + l["_filename"] = filename + l["_scoped_id"] = scope_id(old_id) + l["_scoped_father_id"] = scope_id(l.get("FatherId") or l.get("fatherId")) + l["_scoped_mother_id"] = scope_id(l.get("MotherId") or l.get("motherId")) + raw_litters.append(l) + + for g in file_gerbils: + old_id = g.get("Id") or g.get("id") + g["_filename"] = filename + g["_scoped_id"] = scope_id(old_id) + g["_scoped_litter_id"] = scope_id(g.get("LitterId") or g.get("litterId")) + g["_scoped_origin_cid"] = scope_id(g.get("OriginContactId") or g.get("originContactId")) + + receiver_val = (g.get("ReceiverContactId") or g.get("receiverContactId") or + g.get("BuyerId") or g.get("buyerId") or + g.get("BuyerContactId") or g.get("buyerContactId") or + g.get("givenAwayContactId") or g.get("givenAwayToContactId") or + g.get("ownerContactId") or g.get("ownerId")) + g["_scoped_receiver_cid"] = scope_id(receiver_val) + raw_gerbils.append(g) + + except json.JSONDecodeError as e: + print(f"Failed to parse JSON in {filename}: {e}") + + # Chronological Interpolation: Estimate the date of each page based on neighboring pages with dates + file_dates = {} + sorted_files = sorted(md_files) + + # Simple linear interpolation / extrapolation + for i, fn in enumerate(sorted_files): + if fn in file_explicit_dates: + file_dates[fn] = file_explicit_dates[fn] + else: + # Look left for closest explicit date + left_val, left_dist = None, None + for j in range(i - 1, -1, -1): + if sorted_files[j] in file_explicit_dates: + left_val = file_explicit_dates[sorted_files[j]] + left_dist = i - j + break + + # Look right for closest explicit date + right_val, right_dist = None, None + for j in range(i + 1, len(sorted_files)): + if sorted_files[j] in file_explicit_dates: + right_val = file_explicit_dates[sorted_files[j]] + right_dist = j - i + break + + if left_val is not None and right_val is not None: + # Interpolate + file_dates[fn] = left_val + (right_val - left_val) * (left_dist / (left_dist + right_dist)) + elif left_val is not None: + # Extrapolate right (assume 30 days per page gap as placeholder) + file_dates[fn] = left_val + (left_dist * 30) + elif right_val is not None: + # Extrapolate left + file_dates[fn] = right_val - (right_dist * 30) + else: + # No dates in entire log? Default to 2011-01-01 + file_dates[fn] = date_to_days("2011-01-01") + + print(f"Parsed {len(raw_contacts)} raw contacts, {len(raw_litters)} raw litters, {len(raw_gerbils)} raw gerbils.") + + # Run extract.py to make sure stammbaum data is up to date + import subprocess + print("Running extract.py to extract Stammbäume...") + try: + subprocess.run([sys.executable, "extract.py"], check=True) + except Exception as e: + print(f"Warning: Failed to run extract.py: {e}") + + # Load Stammbaum data + stammbaum_only_animals = [] + animals_path = os.path.join(OUTPUT_DIR, "animals.json") + if os.path.exists(animals_path): + with open(animals_path, "r", encoding="utf-8") as f: + stammbaum_animals = json.load(f) + for a in stammbaum_animals: + sources = a.get("sourceFiles", []) + if any("stammbaum" in str(s).lower() for s in sources): + stammbaum_only_animals.append(a) + print(f"Loaded {len(stammbaum_only_animals)} Stammbaum animals.") + else: + print(f"Warning: Stammbaum animals.json not found at {animals_path}") + + # Run extract_docx.py to make sure docx data is up to date + print("Running extract_docx.py to extract docx...") + try: + subprocess.run([sys.executable, "extract_docx.py"], check=True) + except Exception as e: + print(f"Warning: Failed to run extract_docx.py: {e}") + + # Load Docx data + docx_animals = [] + docx_animals_path = os.path.join(OUTPUT_DIR, "docx_animals.json") + if os.path.exists(docx_animals_path): + with open(docx_animals_path, "r", encoding="utf-8") as f: + docx_animals = json.load(f) + print(f"Loaded {len(docx_animals)} animals from docx.") + else: + print(f"Warning: docx_animals.json not found at {docx_animals_path}") + + docx_litters = [] + docx_litters_path = os.path.join(OUTPUT_DIR, "docx_litters.json") + if os.path.exists(docx_litters_path): + with open(docx_litters_path, "r", encoding="utf-8") as f: + docx_litters = json.load(f) + print(f"Loaded {len(docx_litters)} litters from docx.") + else: + print(f"Warning: docx_litters.json not found at {docx_litters_path}") + + # Extract docx buyer contacts and add to raw_contacts + for da in docx_animals: + o_name = (da.get("owner") or "").strip() + if o_name: + o_scoped_id = generate_guid(f"docx-contact-{normalize_name(o_name)}") + raw_contacts.append({ + "Name": o_name, + "_filename": "Wurfchronik-Detail", + "_scoped_id": o_scoped_id + }) + + # Map docx litters and append to raw_litters + docx_litter_id_map = {} # wsCode -> scoped_id + for dl in docx_litters: + l_name = dl["litterId"] + dob_val = parse_date(dl["dob"]) + f_name = dl["fatherName"] + m_name = dl["motherName"] + ws_code = dl["wsCode"] + note_val = dl.get("note") + + # Parse survived/total born from wsCode (e.g. 4/5) + total_born = None + deaths_8w = None + if "/" in ws_code: + parts = ws_code.split("/") + if len(parts) == 2: + try: + survived = int(parts[0]) + total = int(parts[1]) + total_born = total + deaths_8w = max(0, total - survived) + except ValueError: + pass + + l_scoped_id = generate_guid(f"docx-litter-{normalize_name(ws_code)}-{dob_val or '0001-01-01'}") + docx_litter_id_map[(ws_code, dob_val)] = l_scoped_id + + raw_litters.append({ + "Id": l_scoped_id, + "Name": l_name, + "Date": dob_val or "0001-01-01", + "TotalBorn": total_born, + "DeathsWithin8Weeks": deaths_8w, + "FatherId": generate_guid(f"stammbaum-animal-{normalize_name(f_name)}"), # placeholder + "MotherId": generate_guid(f"stammbaum-animal-{normalize_name(m_name)}"), # placeholder + "ExpectedGoHomeDate": None, + "Notes": note_val if note_val else "Docx imported litter", + "PairingCode": None, + "ExternalRef": f"docx-litter-{l_scoped_id}", + "LitterLetter": l_name[0] if l_name and len(l_name) > 0 else None, + "_father_name": f_name, + "_mother_name": m_name, + "_filename": "Wurfchronik-Detail", + "_scoped_id": l_scoped_id, + "_scoped_father_id": generate_guid(f"stammbaum-animal-{normalize_name(f_name)}"), + "_scoped_mother_id": generate_guid(f"stammbaum-animal-{normalize_name(m_name)}") + }) + + + # Extract stammbaum contacts and add to raw_contacts + for a in stammbaum_only_animals: + b_name = (a.get("breeder") or "").strip() + if b_name: + b_scoped_id = generate_guid(f"stammbaum-contact-{normalize_name(b_name)}") + raw_contacts.append({ + "Name": b_name, + "_filename": "Stammbaum", + "_scoped_id": b_scoped_id + }) + z_name = (a.get("zucht") or "").strip() + if z_name: + z_scoped_id = generate_guid(f"stammbaum-contact-{normalize_name(z_name)}") + raw_contacts.append({ + "Name": z_name, + "_filename": "Stammbaum", + "_scoped_id": z_scoped_id + }) + + # Residency propagation for stammbaum animals + def is_clan_zucht(z): + if not z: + return False + norm = z.lower() + return "klein" in norm and "chaot" in norm and "extern" not in norm + + stammbaum_resident_ids = set() + for a in stammbaum_only_animals: + if is_clan_zucht(a.get("zucht")) or is_clan_zucht(a.get("zuchtCanon")): + stammbaum_resident_ids.add(a["id"]) + + # Propagate residency to parents of resident offspring + for _ in range(5): + for a in stammbaum_only_animals: + if a["id"] in stammbaum_resident_ids: + for p_ref in a.get("parentRefs", []): + p_key = (normalize_name(p_ref["name"]), parse_date(p_ref["dob"])) + for cand in stammbaum_only_animals: + if normalize_name(cand["name"]) == p_key[0]: + cand_dob = parse_date(cand["dob"]) + if not p_key[1] or cand_dob == p_key[1]: + stammbaum_resident_ids.add(cand["id"]) + + # Pre-index Wurfchronik litters from markdown + md_litters_idx = {} + for rl in raw_litters: + f_name = rl.get("FatherName") or rl.get("fatherName") or rl.get("ParentMaleName") or rl.get("parentMaleName") + m_name = rl.get("MotherName") or rl.get("motherName") or rl.get("ParentFemaleName") or rl.get("parentFemaleName") + ldate = parse_date(rl.get("Date") or rl.get("date") or rl.get("DateOfBirth") or rl.get("dateOfBirth")) + if f_name and m_name and ldate: + key = (normalize_name(f_name), normalize_name(m_name), ldate) + md_litters_idx[key] = rl + + # Create virtual litters for stammbaum animals + created_virtual_litters = {} + for a in stammbaum_only_animals: + parent_refs = a.get("parentRefs", []) + father_ref = next((p for p in parent_refs if p.get("roleGuess") == "father"), None) + mother_ref = next((p for p in parent_refs if p.get("roleGuess") == "mother"), None) + + a["_mapped_litter_scoped_id"] = None + if father_ref and mother_ref: + f_name = father_ref.get("name") + m_name = mother_ref.get("name") + dob_val = parse_date(a.get("dob")) + + mapped_litter = None + if dob_val: + key = (normalize_name(f_name), normalize_name(m_name), dob_val) + mapped_litter = md_litters_idx.get(key) + + if mapped_litter: + a["_mapped_litter_scoped_id"] = mapped_litter["_scoped_id"] + else: + v_key = (normalize_name(f_name), normalize_name(m_name), dob_val or "0001-01-01") + if v_key in created_virtual_litters: + a["_mapped_litter_scoped_id"] = created_virtual_litters[v_key] + else: + l_scoped_id = generate_guid(f"virtual-litter-{v_key[0]}-{v_key[1]}-{v_key[2]}") + created_virtual_litters[v_key] = l_scoped_id + a["_mapped_litter_scoped_id"] = l_scoped_id + + # Try to link parents to actual parsed stammbaum animals + f_scoped_id = None + m_scoped_id = None + f_dob = parse_date(father_ref.get("dob")) + m_dob = parse_date(mother_ref.get("dob")) + + for p_cand in stammbaum_only_animals: + cand_call_norm = normalize_name(get_call_name(p_cand["name"])) + if cand_call_norm == normalize_name(f_name) or normalize_name(p_cand["name"]) == normalize_name(f_name): + if not f_dob or parse_date(p_cand.get("dob")) == f_dob: + f_scoped_id = generate_guid(f"stammbaum-animal-{p_cand['id']}") + break + for p_cand in stammbaum_only_animals: + cand_call_norm = normalize_name(get_call_name(p_cand["name"])) + if cand_call_norm == normalize_name(m_name) or normalize_name(p_cand["name"]) == normalize_name(m_name): + if not m_dob or parse_date(p_cand.get("dob")) == m_dob: + m_scoped_id = generate_guid(f"stammbaum-animal-{p_cand['id']}") + break + + raw_litters.append({ + "Id": l_scoped_id, + "Name": f"Wurf von {f_name} + {m_name}", + "Date": dob_val or "0001-01-01", + "TotalBorn": None, + "DeathsWithin8Weeks": None, + "FatherId": f_scoped_id or generate_guid(f"stammbaum-animal-{normalize_name(f_name)}"), + "MotherId": m_scoped_id or generate_guid(f"stammbaum-animal-{normalize_name(m_name)}"), + "ExpectedGoHomeDate": None, + "Notes": "Pedigree virtual litter", + "PairingCode": None, + "ExternalRef": f"virtual-{l_scoped_id}", + "LitterLetter": None, + "_father_name": f_name, + "_mother_name": m_name, + "_filename": "Stammbaum", + "_scoped_id": l_scoped_id, + "_scoped_father_id": f_scoped_id or generate_guid(f"stammbaum-animal-{normalize_name(f_name)}"), + "_scoped_mother_id": m_scoped_id or generate_guid(f"stammbaum-animal-{normalize_name(m_name)}") + }) + + # 2. Resolve Contacts globally (deduplicate by normalized name) + + contact_by_norm_name = {} + contact_id_map = {} # scoped_old_id -> global_guid + + for rc in raw_contacts: + name_val = rc.get("Name") or rc.get("name") or rc.get("FullName") or rc.get("fullName") + if not name_val: + first = rc.get("FirstName") or rc.get("firstName") + last = rc.get("LastName") or rc.get("lastName") + if first or last: + name_val = f"{first or ''} {last or ''}".strip() + + if not name_val: + continue + + canon_name, should_keep = get_normalized_contact_name(name_val) + scoped_id = rc["_scoped_id"] + if not should_keep: + if scoped_id: + contact_id_map[scoped_id] = None + continue + + norm_name = normalize_name(canon_name) + + if norm_name not in contact_by_norm_name: + global_guid = generate_guid(f"contact-{norm_name}") + contact_by_norm_name[norm_name] = { + "Id": global_guid, + "Name": canon_name, + "Email": rc.get("Email") or rc.get("email"), + "Phone": rc.get("Phone") or rc.get("phone"), + "Address": rc.get("Address") or rc.get("address"), + "Notes": rc.get("Notes") or rc.get("notes") or rc.get("Note") or rc.get("note") + } + else: + gc = contact_by_norm_name[norm_name] + if not gc["Email"] and (rc.get("Email") or rc.get("email")): + gc["Email"] = rc.get("Email") or rc.get("email") + if not gc["Phone"] and (rc.get("Phone") or rc.get("phone")): + gc["Phone"] = rc.get("Phone") or rc.get("phone") + if not gc["Address"] and (rc.get("Address") or rc.get("address")): + gc["Address"] = rc.get("Address") or rc.get("address") + if not gc["Notes"] and (rc.get("Notes") or rc.get("notes") or rc.get("Note") or rc.get("note")): + gc["Notes"] = rc.get("Notes") or rc.get("notes") or rc.get("Note") or rc.get("note") + + if scoped_id: + contact_id_map[scoped_id] = contact_by_norm_name[norm_name]["Id"] + + resolved_contacts = list(contact_by_norm_name.values()) + print(f"Resolved to {len(resolved_contacts)} unique contacts.") + + # 3. Process Litters (normalize keys) + resolved_litters = [] + litter_id_map = {} # scoped_old_id -> new_guid + litter_by_scoped_id = {} + + for rl in raw_litters: + filename = rl.get("_filename") + name_val = rl.get("Name") or rl.get("name") + if not name_val: + name_val = "Wurf" + + dob_val = parse_date(rl.get("Date") or rl.get("date") or rl.get("DateOfBirth") or rl.get("dateOfBirth")) + if not dob_val: + dob_val = "0001-01-01" + + new_guid = rl["_scoped_id"] + if not new_guid: + new_guid = generate_guid(f"litter-{filename}-{name_val}-{dob_val}") + + scoped_old_id = rl["_scoped_id"] + if scoped_old_id: + litter_id_map[scoped_old_id] = new_guid + + total_born = rl.get("TotalBorn") or rl.get("totalBorn") or rl.get("LitterSize") or rl.get("litterSize") or rl.get("size") or rl.get("totalPups") + if total_born is not None: + try: + total_born = int(total_born) + except ValueError: + total_born = None + + deaths_8w = rl.get("DeathsWithin8Weeks") or rl.get("deathsWithin8Weeks") + if deaths_8w is not None: + try: + deaths_8w = int(deaths_8w) + except ValueError: + deaths_8w = None + elif total_born is not None and rl.get("survived") is not None: + try: + deaths_8w = total_born - int(rl.get("survived")) + except ValueError: + pass + + father_name = rl.get("FatherName") or rl.get("fatherName") or rl.get("ParentMaleName") or rl.get("parentMaleName") or rl.get("_father_name") + mother_name = rl.get("MotherName") or rl.get("motherName") or rl.get("ParentFemaleName") or rl.get("parentFemaleName") or rl.get("_mother_name") + + raw_ext_ref = rl.get("ExternalRef") or rl.get("externalRef") or rl.get("Id") or rl.get("id") + ext_ref_scoped = f"{filename}-{raw_ext_ref}" if raw_ext_ref else None + + l_record = { + "Id": new_guid, + "Name": name_val, + "Date": dob_val, + "TotalBorn": total_born, + "DeathsWithin8Weeks": deaths_8w, + "FatherId": rl["_scoped_father_id"], + "MotherId": rl["_scoped_mother_id"], + "ExpectedGoHomeDate": parse_date(rl.get("ExpectedGoHomeDate") or rl.get("expectedGoHomeDate")), + "Notes": rl.get("Notes") or rl.get("notes") or rl.get("Note") or rl.get("note"), + "PairingCode": rl.get("PairingCode") or rl.get("pairingCode"), + "ExternalRef": ext_ref_scoped, + "LitterLetter": rl.get("LitterLetter") or rl.get("litterLetter"), + "_father_name": father_name, + "_mother_name": mother_name, + "_filename": filename + } + resolved_litters.append(l_record) + litter_by_scoped_id[new_guid] = l_record + + print(f"Processed {len(resolved_litters)} litters.") + + # Helper to lookup litter dates for birth date estimation + def get_litter_date(l_id): + if l_id in litter_by_scoped_id: + d = litter_by_scoped_id[l_id]["Date"] + if d and d != "0001-01-01": + return d + return None + + # 4. Normalize and group Gerbils + all_processed_gerbils = [] + for rg in raw_gerbils: + filename = rg.get("_filename") + name_val = rg.get("Name") or rg.get("name") or rg.get("callName") + if not name_val: + name_val = "Unbekannt" + + dob_val = parse_date(rg.get("DateOfBirth") or rg.get("dateOfBirth")) + + new_guid = rg["_scoped_id"] + if not new_guid: + new_guid = generate_guid(f"gerbil-{filename}-{name_val}-{dob_val}") + + # Gender normalization + gender_val = rg.get("Gender") or rg.get("gender") or "unknown" + gender_val = str(gender_val).lower().strip() + if gender_val in ["m", "male", "männlich"]: + gender = "male" + elif gender_val in ["w", "f", "female", "weiblich"]: + gender = "female" + else: + gender = "unknown" + + dod_val = parse_date(rg.get("DateOfDeath") or rg.get("dateOfDeath") or rg.get("deathDate")) + gohome_val = parse_date(rg.get("GoHomeDate") or rg.get("goHomeDate") or rg.get("DateGivenAway") or rg.get("givenAwayDate") or rg.get("DateOfHandover") or rg.get("dateOfHandover") or rg.get("HandoverDate") or rg.get("DateOfSale") or rg.get("dateOfSale")) + + status_val = rg.get("Status") or rg.get("status") or "" + status_val = str(status_val).lower().strip() + status = "Breeding" + if dod_val or "deceased" in status_val or "verstorben" in status_val or "tod" in status_val or "dead" in status_val: + status = "Deceased" + elif gohome_val or "givenaway" in status_val or "abgegeben" in status_val or "verkauft" in status_val or "sold" in status_val: + status = "GivenAway" + elif "forsale" in status_val or "abzugeben" in status_val: + status = "ForSale" + elif "pet" in status_val or "liebhaber" in status_val: + status = "Pet" + else: + if dod_val: + status = "Deceased" + elif gohome_val: + status = "GivenAway" + + notes_val = rg.get("Notes") or rg.get("notes") or rg.get("Note") or rg.get("note") + existing_cod = rg.get("CauseOfDeath") or rg.get("causeOfDeath") or rg.get("DeathCause") + status, dod_val, cause_of_death_val = parse_death_info(notes_val, status, dod_val, existing_cod) + + # Explicit resolution for Ken'ichi's cause of death + if name_val == "Ken'ichi" and dob_val == "2015-03-01": + cause_of_death_val = "Duftdrüsen-Tumor" + + color_val = rg.get("ColorVarietyId") or rg.get("colorVarietyId") or rg.get("Color") or rg.get("color") or rg.get("ColorDescription") or rg.get("colorDescription") + existing_gt = rg.get("Genotype") or rg.get("genotype") + color_variety_id, genotype_val = resolve_color_and_genotype( + color_val, existing_gt, variety_map, variety_genotypes + ) + if color_val and not color_variety_id: + try: + uuid.UUID(str(color_val).strip()) + color_variety_id = str(color_val).strip() + except ValueError: + print(f"Warning: Unknown color variety '{color_val}' for gerbil '{name_val}' on page {filename}") + + origin_cid = rg["_scoped_origin_cid"] + receiver_cid = rg["_scoped_receiver_cid"] + + if origin_cid in contact_id_map: + origin_cid = contact_id_map[origin_cid] + if receiver_cid in contact_id_map: + receiver_cid = contact_id_map[receiver_cid] + + is_resident = rg.get("IsResident") or rg.get("isResident") + if is_resident is None: + is_resident = True + else: + is_resident = str(is_resident).lower() == "true" + + traits = rg.get("CharacterTraits") or rg.get("characterTraits") or [] + if not isinstance(traits, list): + traits = [str(traits)] + char_note = rg.get("CharacterNote") or rg.get("characterNote") + + is_deaf = rg.get("IsDeaf") or rg.get("isDeaf") + if is_deaf is not None: + is_deaf = str(is_deaf).lower() == "true" + + old_litter_id = rg["_scoped_litter_id"] + + raw_ext_ref = rg.get("ExternalRef") or rg.get("externalRef") or rg.get("Id") or rg.get("id") + ext_ref_scoped = f"{filename}-{raw_ext_ref}" if raw_ext_ref else None + + # Estimated effective date of birth for conflict checking + eff_dob_val = dob_val + if not eff_dob_val and old_litter_id: + # Try to get litter date if litter was resolved + mapped_l_id = litter_id_map.get(old_litter_id) + if mapped_l_id: + eff_dob_val = get_litter_date(mapped_l_id) + + # Determine explicit or litter-derived birth date (None if unknown/parent) + birth_date = dob_val + if not birth_date and old_litter_id: + mapped_l_id = litter_id_map.get(old_litter_id) + if mapped_l_id: + birth_date = get_litter_date(mapped_l_id) + + # If still no effective date, use the estimated page date + if not eff_dob_val: + eff_dob_val = days_to_date(file_dates[filename]) + + raw_breeder = rg.get("OriginBreeder") or rg.get("originBreeder") + if raw_breeder: + norm_b, keep_b = get_normalized_contact_name(raw_breeder) + raw_breeder = norm_b if keep_b else None + + all_processed_gerbils.append({ + "Id": new_guid, + "Name": name_val, + "Gender": gender, + "Status": status, + "LitterId": old_litter_id, # mapped later + "OriginContactId": origin_cid, + "ReceiverContactId": receiver_cid, + "EnclosureId": None, + "ColorVarietyId": color_variety_id, + "DateOfBirth": dob_val, + "DateOfDeath": dod_val, + "CauseOfDeath": cause_of_death_val, + "GoHomeDate": gohome_val, + "Genotype": genotype_val, + "Notes": rg.get("Notes") or rg.get("notes") or rg.get("Note") or rg.get("note"), + "ImportSource": rg.get("ImportSource") or rg.get("importSource") or filename, + "ExternalRef": ext_ref_scoped, + "RawImportData": rg.get("RawImportData") or rg.get("rawImportData") or json.dumps({"colorDescription": color_val if not color_variety_id else None}), + "OriginBreeder": raw_breeder or ("Zucht der kleinen Chaoten" if is_resident else None), + "NameSearch": normalize_name(name_val), + "CharacterTraits": traits, + "CharacterNote": char_note, + "IsDeaf": is_deaf, + "IsResident": is_resident, + "_photos": rg.get("photos", []), + "_old_scoped_litter_id": old_litter_id, + "_eff_dob": eff_dob_val, + "_birth_date": birth_date, + "_filename": filename, + "_old_id": rg.get("Id") or rg.get("id") + }) + + # Map and append stammbaum animals to all_processed_gerbils + for a in stammbaum_only_animals: + a_id = a["id"] + name_val = a["name"] + + gender_val = str(a.get("gender") or "").lower().strip() + if gender_val in ["m", "male"]: + gender = "male" + elif gender_val in ["w", "f", "female"]: + gender = "female" + else: + gender = "unknown" + + dob_val = parse_date(a.get("dob")) + dod_val = parse_date(a.get("death")) + + status = "Breeding" + if dod_val: + status = "Deceased" + elif a_id not in stammbaum_resident_ids: + status = "GivenAway" + else: + if dob_val: + try: + dt_dob = datetime.strptime(dob_val, "%Y-%m-%d") + dt_now = datetime.now() + age_years = (dt_now - dt_dob).days / 365.25 + if age_years >= 7.0: + status = "Deceased" + except Exception: + pass + + # Color variety mapping + existing_gt = a["genotype"]["rawGenotype"] if a["genotype"]["rawGenotype"] else None + color_variety_id, genotype_val = resolve_color_and_genotype( + a.get("farbschlag"), existing_gt, variety_map, variety_genotypes + ) + if not color_variety_id: + for fbv in a.get("farbschlagVariants", []): + cv_id, gt_val = resolve_color_and_genotype(fbv, existing_gt, variety_map, variety_genotypes) + if cv_id: + color_variety_id = cv_id + genotype_val = gt_val + break + + # Contact mapping + origin_cid = None + b_name = (a.get("breeder") or "").strip() + z_name = (a.get("zucht") or "").strip() + if b_name: + origin_cid = generate_guid(f"stammbaum-contact-{normalize_name(b_name)}") + elif z_name: + origin_cid = generate_guid(f"stammbaum-contact-{normalize_name(z_name)}") + + # Map to resolved global contact GUID + if origin_cid in contact_id_map: + origin_cid = contact_id_map[origin_cid] + else: + origin_cid = None + + + scoped_id = generate_guid(f"stammbaum-animal-{a_id}") + scoped_litter_id = a.get("_mapped_litter_scoped_id") + + is_deaf = a["genotype"].get("deaf") + + stammbaum_breeder = a.get("breeder") if a.get("breeder") else (a.get("zucht") if a.get("zucht") else None) + if stammbaum_breeder: + norm_b, keep_b = get_normalized_contact_name(stammbaum_breeder) + stammbaum_breeder = norm_b if keep_b else None + + all_processed_gerbils.append({ + "Id": scoped_id, + "Name": name_val, + "Gender": gender, + "Status": status, + "LitterId": scoped_litter_id, # mapped later in step 5 + "OriginContactId": origin_cid, + "ReceiverContactId": None, + "EnclosureId": None, + "ColorVarietyId": color_variety_id, + "DateOfBirth": dob_val, + "DateOfDeath": dod_val, + "CauseOfDeath": "Duftdrüsen-Tumor" if name_val == "Ken'ichi" and dob_val == "2015-03-01" else None, + "GoHomeDate": None, + "Genotype": genotype_val, + "Notes": None, + "ImportSource": ", ".join(a.get("sourceFiles", [])), + "ExternalRef": f"stammbaum-{a_id}", + "RawImportData": json.dumps({ + "rawGenotype": a["genotype"]["rawGenotype"], + "unmappedTokens": a["genotype"]["unmappedTokens"], + "breederText": a.get("breeder", "") + }, ensure_ascii=False), + "OriginBreeder": stammbaum_breeder, + "NameSearch": normalize_name(name_val), + "CharacterTraits": [], + "CharacterNote": None, + "IsDeaf": is_deaf, + "IsResident": a_id in stammbaum_resident_ids, + "_photos": a.get("photos", []), + "_old_scoped_litter_id": scoped_litter_id, + "_eff_dob": dob_val or "2010-01-01", + "_birth_date": dob_val, + "_filename": a.get("sourceFiles", ["Stammbaum"])[0], + "_old_id": a_id + }) + + # Map and append docx animals to all_processed_gerbils + for idx, da in enumerate(docx_animals): + name_val = da["name"] + gender = da["gender"] + + dob_val = parse_date(da.get("litterDob")) + dod_val = parse_date(da.get("deathDate")) + gohome_val = parse_date(da.get("abgabeDate")) + + # Status precedence + status = "Breeding" + if dod_val: + status = "Deceased" + elif gohome_val or da.get("owner"): + status = "GivenAway" + + # Color variety mapping + color_variety_id, genotype_val = resolve_color_and_genotype( + da.get("farbschlag"), None, variety_map, variety_genotypes + ) + + # Contact mapping (buyer) + receiver_cid = None + o_name = (da.get("owner") or "").strip() + if o_name: + receiver_cid = generate_guid(f"docx-contact-{normalize_name(o_name)}") + if receiver_cid in contact_id_map: + receiver_cid = contact_id_map[receiver_cid] + else: + receiver_cid = None + + # Scoped ID + scoped_id = generate_guid(f"docx-animal-{idx}-{normalize_name(name_val)}-{dob_val or '0001-01-01'}") + + # Litter ID mapping + ws_code = da.get("wsCode") + scoped_litter_id = docx_litter_id_map.get((ws_code, dob_val)) + + + # Raw import details + raw_import_payload = json.dumps({ + "abgabeWeight": da.get("abgabeWeight", ""), + "deathCause": da.get("deathCause", ""), + "partnerName": da.get("partnerName", ""), + "partnerDob": da.get("partnerDob", "") + }, ensure_ascii=False) + + # Residents: if sold/given away, it's not a resident + is_resident = not bool(o_name) + + all_processed_gerbils.append({ + "Id": scoped_id, + "Name": name_val, + "Gender": gender, + "Status": status, + "LitterId": scoped_litter_id, + "OriginContactId": None, + "ReceiverContactId": receiver_cid, # mapped in step 5 + "EnclosureId": None, + "ColorVarietyId": color_variety_id, + "DateOfBirth": dob_val, + "DateOfDeath": dod_val, + "CauseOfDeath": da.get("deathCause"), + "GoHomeDate": gohome_val, + "Genotype": genotype_val, + "Notes": None, + "ImportSource": "Wurfchronik-Detail.docx", + "ExternalRef": f"docx-{idx}-{normalize_name(name_val)}-{dob_val or '0001-01-01'}", + "RawImportData": raw_import_payload, + "OriginBreeder": "Zucht der kleinen Chaoten", + "NameSearch": normalize_name(name_val), + "CharacterTraits": [], + "CharacterNote": None, + "IsDeaf": None, + "IsResident": is_resident, + "_photos": da.get("photos", []), + "_old_scoped_litter_id": scoped_litter_id, + "_eff_dob": dob_val or "2020-01-01", + "_birth_date": dob_val, + "_filename": "Wurfchronik-Detail.docx", + "_old_id": name_val + }) + + # Build parenting dates lookup using old scoped IDs + parent_litter_dates = {} + + for l in resolved_litters: + ld = l["Date"] + if ld and ld != "0001-01-01": + for pid in [l["FatherId"], l["MotherId"]]: + if pid: + parent_litter_dates.setdefault(pid, []).append(ld) + + def are_compatible(g1, g2): + # Must have same gender (or one unknown) + if g1["Gender"] != "unknown" and g2["Gender"] != "unknown" and g1["Gender"] != g2["Gender"]: + bd1 = g1.get("_birth_date") + bd2 = g2.get("_birth_date") + if not (bd1 and bd2 and bd1 == bd2): + return False + + bd1 = g1.get("_birth_date") + bd2 = g2.get("_birth_date") + + # If both have explicit birth dates, they must match within 30 days + if bd1 and bd2: + days1 = date_to_days(bd1) + days2 = date_to_days(bd2) + if days1 is not None and days2 is not None: + if abs(days1 - days2) > 30: + return False + + # If g1 has birth date, and g2 has parenting dates, birth date must be before parenting dates + p_dates2 = parent_litter_dates.get(g2["Id"], []) + if bd1: + for pd in p_dates2: + if pd <= bd1: # Can't have litter before or on birth date + return False + + p_dates1 = parent_litter_dates.get(g1["Id"], []) + if bd2: + for pd in p_dates1: + if pd <= bd2: + return False + + return True + + # Group gerbils by name to perform deduplication + gerbil_groups = {} + for g in all_processed_gerbils: + name_key = get_dedup_name_key(g["Name"]) + if not name_key: + name_key = "unbekannt" + gerbil_groups.setdefault(name_key, []).append(g) + + resolved_gerbils = [] + gerbil_id_map = {} # old_scoped_id -> final_id + + color_keys = set(variety_map.keys()) + + for name_key, group in gerbil_groups.items(): + is_placeholder = ( + name_key in color_keys or + any(p in name_key for p in ["unbekannt", "unbenannt", "baby", "welpe", "jungtier", "unknown", "welpen"]) or + len(name_key) <= 2 + ) + + if is_placeholder: + # Placeholders: do NOT merge, keep all separate + for g in group: + resolved_gerbils.append(g) + gerbil_id_map[g["Id"]] = g["Id"] + continue + + # Partition group into compatible subsets + sub_groups = [] + for g in group: + placed = False + for sub in sub_groups: + if all(are_compatible(g, member) for member in sub): + sub.append(g) + placed = True + break + if not placed: + sub_groups.append([g]) + + # Merge each partition sub-group into a single gerbil + for sub in sub_groups: + if len(sub) == 1: + g = sub[0] + resolved_gerbils.append(g) + gerbil_id_map[g["Id"]] = g["Id"] + continue + + # Find the best primary record to merge into + best_g = None + best_score = -1 + for g in sub: + score = 0 + if g["_old_scoped_litter_id"]: score += 10 + if g["DateOfBirth"]: score += 5 + if g["Genotype"]: score += 3 + if g["ColorVarietyId"]: score += 2 + if g["ImportSource"] and "stammbaum" in g["ImportSource"].lower(): score += 20 + if g["Notes"] and not any(kw in g["Notes"].lower() for kw in ["parent", "mutter", "vater", "dam", "sire"]): score += 1 + if len(g["Name"]) > len(name_key) + 5: # likely has clan suffix + score += 15 + if score > best_score: + best_score = score + best_g = g + + # Merge fields + merged_notes = [] + if best_g["Notes"]: + merged_notes.append(best_g["Notes"]) + + # Merge photos + merged_photos = list(best_g.get("_photos", [])) + + # Track sources for debugging + sources = [best_g["_filename"]] + + for g in sub: + if g == best_g: + continue + gerbil_id_map[g["Id"]] = best_g["Id"] + sources.append(g["_filename"]) + + for ph in g.get("_photos", []): + if ph not in merged_photos: + merged_photos.append(ph) + + if not best_g["LitterId"] and g["LitterId"]: + best_g["LitterId"] = g["LitterId"] + if not best_g["DateOfBirth"] and g["DateOfBirth"]: + best_g["DateOfBirth"] = g["DateOfBirth"] + if not best_g["DateOfDeath"] and g["DateOfDeath"]: + best_g["DateOfDeath"] = g["DateOfDeath"] + if not best_g["CauseOfDeath"] and g["CauseOfDeath"]: + best_g["CauseOfDeath"] = g["CauseOfDeath"] + if not best_g["GoHomeDate"] and g["GoHomeDate"]: + best_g["GoHomeDate"] = g["GoHomeDate"] + if not best_g["Genotype"] and g["Genotype"]: + best_g["Genotype"] = g["Genotype"] + if not best_g["ColorVarietyId"] and g["ColorVarietyId"]: + best_g["ColorVarietyId"] = g["ColorVarietyId"] + if not best_g["OriginContactId"] and g["OriginContactId"]: + best_g["OriginContactId"] = g["OriginContactId"] + if not best_g["ReceiverContactId"] and g["ReceiverContactId"]: + best_g["ReceiverContactId"] = g["ReceiverContactId"] + if g["IsResident"]: + best_g["IsResident"] = True + + # Reconcile Gender: prefer a known gender over unknown, and prefer stammbaum over other sources + if best_g["Gender"] == "unknown" and g["Gender"] != "unknown": + best_g["Gender"] = g["Gender"] + elif best_g["Gender"] != "unknown" and g["Gender"] != "unknown" and best_g["Gender"] != g["Gender"]: + if g["ImportSource"] and "stammbaum" in g["ImportSource"].lower(): + if not best_g["ImportSource"] or "stammbaum" not in best_g["ImportSource"].lower(): + best_g["Gender"] = g["Gender"] + + # Status precedence: Deceased > GivenAway > Breeding/Pet + if g["Status"] == "Deceased": + best_g["Status"] = "Deceased" + elif g["Status"] == "GivenAway" and best_g["Status"] not in ["Deceased"]: + best_g["Status"] = "GivenAway" + + if g["Notes"] and g["Notes"] not in merged_notes: + # Ignore redundant dummy notes + if not any(kw in g["Notes"].lower() for kw in ["parent listed", "mutter von", "vater von", "dam of", "sire of"]): + merged_notes.append(g["Notes"]) + + + if merged_notes: + best_g["Notes"] = " | ".join(merged_notes) + + best_g["_photos"] = merged_photos + + # Print merge trace + print(f"Deduplicated same-animal name '{best_g['Name']}': merged {len(sub)} entries across files: {', '.join(sources)}") + + resolved_gerbils.append(best_g) + gerbil_id_map[best_g["Id"]] = best_g["Id"] + + print(f"Deduplicated to {len(resolved_gerbils)} unique gerbil records.") + + # 5. Map Gerbils to Litters + for g in resolved_gerbils: + old_lid = g["_old_scoped_litter_id"] + if old_lid in litter_id_map: + g["LitterId"] = litter_id_map[old_lid] + else: + g["LitterId"] = None + # Clean helper fields + del g["_old_scoped_litter_id"] + del g["_eff_dob"] + if "_birth_date" in g: + del g["_birth_date"] + del g["_filename"] + del g["_old_id"] + + # Gather final valid gerbil IDs + valid_gerbil_ids = {g["Id"] for g in resolved_gerbils} + + # Create name lookup for resolved gerbils + gerbil_by_norm_name = {} + for g in resolved_gerbils: + n_key = normalize_name(g["Name"]) + gerbil_by_norm_name.setdefault(n_key, []).append(g) + + # Also index by call-name to resolve parents who are only listed by call-name + c_key = normalize_name(get_call_name(g["Name"])) + if c_key != n_key: + gerbil_by_norm_name.setdefault(c_key, []).append(g) + + # Map raw Guid if present (convert if old_id mapped to new_guid) + for l in resolved_litters: + if l["FatherId"] in gerbil_id_map: + l["FatherId"] = gerbil_id_map[l["FatherId"]] + if l["MotherId"] in gerbil_id_map: + l["MotherId"] = gerbil_id_map[l["MotherId"]] + + # Clean foreign keys that do not point to a valid gerbil + if l["FatherId"] and l["FatherId"] not in valid_gerbil_ids: + l["FatherId"] = None + if l["MotherId"] and l["MotherId"] not in valid_gerbil_ids: + l["MotherId"] = None + + # Parent Resolver (Global Name Matching) + resolved_fathers = 0 + resolved_mothers = 0 + + for l in resolved_litters: + # Match Father by Name + f_name = l["_father_name"] + if f_name and not l["FatherId"]: + f_norm = normalize_name(f_name) + candidates = gerbil_by_norm_name.get(f_norm, []) + valid_candidates = [] + for c in candidates: + # Map to final deduplicated ID + final_id = gerbil_id_map.get(c["Id"]) + if not final_id: + continue + # Retrieve final record + final_c = next((rg for rg in resolved_gerbils if rg["Id"] == final_id), None) + if final_c and final_c["Gender"] in ["male", "unknown"]: + # Ensure parent is born before litter if birth date is known + if l["Date"] != "0001-01-01" and final_c["DateOfBirth"]: + if final_c["DateOfBirth"] < l["Date"]: + valid_candidates.append(final_c) + else: + valid_candidates.append(final_c) + + if len(valid_candidates) == 1: + l["FatherId"] = valid_candidates[0]["Id"] + resolved_fathers += 1 + elif len(valid_candidates) > 1: + l["FatherId"] = valid_candidates[0]["Id"] + resolved_fathers += 1 + + # Match Mother by Name + m_name = l["_mother_name"] + if m_name and not l["MotherId"]: + m_norm = normalize_name(m_name) + candidates = gerbil_by_norm_name.get(m_norm, []) + valid_candidates = [] + for c in candidates: + final_id = gerbil_id_map.get(c["Id"]) + if not final_id: + continue + final_c = next((rg for rg in resolved_gerbils if rg["Id"] == final_id), None) + if final_c and final_c["Gender"] in ["female", "unknown"]: + if l["Date"] != "0001-01-01" and final_c["DateOfBirth"]: + if final_c["DateOfBirth"] < l["Date"]: + valid_candidates.append(final_c) + else: + valid_candidates.append(final_c) + + if len(valid_candidates) == 1: + l["MotherId"] = valid_candidates[0]["Id"] + resolved_mothers += 1 + elif len(valid_candidates) > 1: + l["MotherId"] = valid_candidates[0]["Id"] + resolved_mothers += 1 + + # Cleanup internal keys + del l["_father_name"] + del l["_mother_name"] + del l["_filename"] + + print(f"Globally resolved {resolved_fathers} fathers and {resolved_mothers} mothers.") + + # Set IsBreeder and IsReceiver flags on contacts + breeder_ids = {g["OriginContactId"] for g in resolved_gerbils if g.get("OriginContactId")} + receiver_ids = {g["ReceiverContactId"] for g in resolved_gerbils if g.get("ReceiverContactId")} + for c in resolved_contacts: + c_id = c["Id"] + is_breeder = c_id in breeder_ids + is_receiver = c_id in receiver_ids + if not is_breeder and not is_receiver: + is_receiver = True + c["IsBreeder"] = is_breeder + c["IsReceiver"] = is_receiver + + # Set and map gerbilPhotos + resolved_photos = [] + for g in resolved_gerbils: + for idx, photo_rel in enumerate(g.get("_photos", [])): + photo_guid = generate_guid(f"photo-{photo_rel}") + ext = os.path.splitext(photo_rel)[1] or ".jpeg" + fn_guid = photo_guid.replace("-", "") + resolved_photos.append({ + "Id": photo_guid, + "GerbilId": g["Id"], + "FileName": f"{fn_guid}{ext}", + "SortOrder": idx, + "_source_path": photo_rel + }) + + # 6. Save final output JSON payload + os.makedirs(OUTPUT_DIR, exist_ok=True) + payload = { + "contacts": resolved_contacts, + "litters": resolved_litters, + "gerbils": resolved_gerbils, + "gerbilPhotos": resolved_photos + } + + with open(OUTPUT_FILE, "w", encoding="utf-8") as f: + json.dump(payload, f, ensure_ascii=False, indent=2) + + print(f"Successfully wrote database-ready import file to: {OUTPUT_FILE}") + print(f" Contacts: {len(payload['contacts'])}") + print(f" Litters: {len(payload['litters'])}") + print(f" Gerbils: {len(payload['gerbils'])}") + print(f" Photos: {len(payload['gerbilPhotos'])}") + +if __name__ == "__main__": + main()