1670 lines
70 KiB
Python
1670 lines
70 KiB
Python
import os
|
|
import json
|
|
import re
|
|
import uuid
|
|
import sys
|
|
from datetime import datetime
|
|
|
|
# Prevent encoding crashes on Windows consoles when printing unicode
|
|
if sys.platform.startswith('win'):
|
|
try:
|
|
sys.stdout.reconfigure(encoding='utf-8')
|
|
except Exception:
|
|
pass
|
|
|
|
# Paths
|
|
DIR_PATH = r"C:\Users\gulum\dev\Wurfchronik_Bilder"
|
|
SEEDS_PATH = r"C:\Users\gulum\dev\GerbilManager\gerbil-manager-web\src\genetics\colorVarietySeed.backend.json"
|
|
OUTPUT_DIR = r"C:\Users\gulum\dev\GerbilManager\tools\import\output"
|
|
OUTPUT_FILE = os.path.join(OUTPUT_DIR, "resolved_import.json")
|
|
|
|
def generate_guid(key_str):
|
|
"""Generate a stable UUID string based on a key."""
|
|
return str(uuid.uuid5(uuid.NAMESPACE_DNS, key_str))
|
|
|
|
def to_valid_guid(val):
|
|
if not val:
|
|
return None
|
|
val_str = str(val).strip()
|
|
try:
|
|
uuid.UUID(val_str)
|
|
return val_str
|
|
except ValueError:
|
|
return generate_guid(val_str)
|
|
|
|
def normalize_name(name):
|
|
if not name:
|
|
return ""
|
|
return "".join(c for c in name.lower() if c.isalnum())
|
|
|
|
def get_normalized_contact_name(name):
|
|
if not name:
|
|
return "", False
|
|
|
|
n = "".join(c for c in name.lower() if c.isalnum())
|
|
|
|
to_discard = {
|
|
"chevroletcamarooftopolino",
|
|
"cindyvprivatzuchtgießen",
|
|
"cindyvprivatzuchtgiessen",
|
|
"inuschofblackforest",
|
|
"stichvonprivatzuchtgießen",
|
|
"stichvonprivatzuchtgiessen",
|
|
"tarzanofsamsimar",
|
|
"hanserennersposeidon",
|
|
"livingforcesidefix",
|
|
"livingforcesnando"
|
|
}
|
|
if n in to_discard:
|
|
return "", False
|
|
|
|
norm_map = {
|
|
"alessandrab": "Alessandra Bartoletti",
|
|
"alexsandrab": "Alessandra Bartoletti",
|
|
"andreafey": "Andrea und Stefanie Fey",
|
|
"andreastefaniefey": "Andrea und Stefanie Fey",
|
|
"anettw": "Annett Wernecke",
|
|
"angelachristoph": "Angela und Ekki Christoph",
|
|
"angelaekkichristoph": "Angela und Ekki Christoph",
|
|
"angie": "Angie Reichert-Cambeis",
|
|
"angierc": "Angie Reichert-Cambeis",
|
|
"anjasch": "Anja Schaumburg",
|
|
"ankeb": "Anke Busch",
|
|
"ankeksch": "Anke Koppenhöfer",
|
|
"annaq": "Anna Quark",
|
|
"annikab": "Annika Balser",
|
|
"annkathrind": "Ann-kathrin Dressler",
|
|
"arturaleksandrapolamundrzynski": "Artur, Aleksandra und Pola Mundrzynski",
|
|
"arturaleksandraundpolamundrzynski": "Artur, Aleksandra und Pola Mundrzynski",
|
|
"astridr": "Astrid Rohmann",
|
|
"astridroh": "Astrid Rohmann",
|
|
"babarastehle": "Barbara Stehle",
|
|
"barbarar": "Barbara Riegler",
|
|
"biancab": "Bianca Bernhardt",
|
|
"biancam": "Bianca Mayer + Alex Wachten",
|
|
"biancamayeralexw": "Bianca Mayer + Alex Wachten",
|
|
"birgitk": "Birgit Knopp",
|
|
"birgitkropp": "Birgit Knopp",
|
|
"birgittabüskens": "Birgitta Müller-Büskens",
|
|
"birgittamüllerbüskens": "Brigitta Müller-Büskens",
|
|
"blackforestgv": "Black Forest",
|
|
"brigittast": "Brigitta Struve",
|
|
"buntefellnasen": "bunten Fellnasen",
|
|
"buntenfellnase": "bunten Fellnasen",
|
|
"buntenfellnasen": "bunten Fellnasen",
|
|
"carolal": "Carola und Hannah Lerch",
|
|
"carolalerch": "Carola und Hannah Lerch",
|
|
"chalfontstud": "Chalfont Stud, Freddy Braun",
|
|
"christianefuchs": "Christiane Fuchs + Michael Höhnert",
|
|
"christiek": "Christiane Kiessling",
|
|
"christinam": "Christine Monika Mai",
|
|
"christinek": "Christiane Kiessling",
|
|
"christiner": "Christina Rudloff",
|
|
"claudialöhr": "Claudia Löhr und Michael Koob",
|
|
"claudialöhrmichaelkoob": "Claudia Löhr und Michael Koob",
|
|
"colorfulfurrygerbils": "Colorful Furry Gerbils",
|
|
"colourfulfurrygerbils": "Colorful Furry Gerbils",
|
|
"danielakasberger": "Daniela Käsberger",
|
|
"dörthe": "Dörthe Petzmann",
|
|
"emelyhehlhorn": "Emely Mehlhorn",
|
|
"fabianb": "Fabian Büdel",
|
|
"floriang": "Florian Gries",
|
|
"hannahraths": "Hannah Ruths",
|
|
"hanserenners": "Hanse Renner",
|
|
"haraldmariareitzlennemann": "Harald und Maria Reitz-Lennemann",
|
|
"haraldundmariarl": "Harald und Maria Reitz-Lennemann",
|
|
"heike": "Heike + Heiko Scheurich",
|
|
"heikebarklay": "Heike Barklage",
|
|
"heikoloos": "Heiko Loos und Annette Becker",
|
|
"heikoloosannettebecker": "Heiko Loos und Annette Becker",
|
|
"inahübner": "Ina Hübner + Nicolai Thome",
|
|
"inapeines": "Ina Pleines",
|
|
"janab": "Jana Beikert",
|
|
"jasiminweber": "Jasmin Weber und Ronny Reichelt",
|
|
"jasmin": "Jasmin Weber",
|
|
"jasminweber": "Jasmin Weber und Ronny Reichelt",
|
|
"jasminweberromyreichelt": "Jasmin Weber und Ronny Reichelt",
|
|
"jasminweberronnyreichelt": "Jasmin Weber und Ronny Reichelt",
|
|
"joystaatberg": "Joy Staalberg",
|
|
"juliaa": "Julia Ast",
|
|
"juttam": "Jutta Metz",
|
|
"karina": "Karina Luft",
|
|
"karinborgotti": "Karin Borsotti",
|
|
"katjal": "Katja Leffeck",
|
|
"kimw": "Kim Waldschmidt",
|
|
"kkchaos": "KK-Chaos",
|
|
"kkchaosofkkchaos": "KK-Chaos",
|
|
"kleinechaoten": "Zucht der kleinen Chaoten",
|
|
"kleinenchaoten": "Zucht der kleinen Chaoten",
|
|
"kriegernmitkrallen": "Krieger mit Krallen",
|
|
"lennylengo": "Lenny Lengo",
|
|
"lisa": "Lisa und Marcel Kunz",
|
|
"lisach": "Lisa und Lydia Christ",
|
|
"lisachrist": "Lisa und Lydia Christ",
|
|
"lisalydiachrist": "Lisa und Lydia Christ",
|
|
"littlefellows": "little fellows",
|
|
"littlerunners": "little runners",
|
|
"maikef": "Maike Franz",
|
|
"maintalerpz": "Privatzucht Maintal",
|
|
"marinau": "Marina Unger",
|
|
"martinaandreaswestfeld": "Martina und Andreas Westfeld",
|
|
"martinaw": "Martina und Andreas Westfeld",
|
|
"martinawestfeld": "Martina und Andreas Westfeld",
|
|
"meikesch": "Meike und Heiko Sch.",
|
|
"melanief": "Melanie Fey und Thorben Meier",
|
|
"melaniefey": "Melanie Fey und Thorben Meier",
|
|
"melaniefeythorbenmeier": "Melanie Fey und Thorben Meier",
|
|
"michaeldavidschmitz": "Michael und David Schmitz",
|
|
"michaelschmitz": "Michael und David Schmitz",
|
|
"nataliereitz": "Nathalie Reitz",
|
|
"natascham": "Natascha Marienfeld",
|
|
"nicolefischler": "Nicole Tischler",
|
|
"nicolel": "Nicole Lannert",
|
|
"nicolen": "Nicole Nuzzo",
|
|
"nicolet": "Nicole Tischler",
|
|
"nicolew": "Nicole Webersinn",
|
|
"nicost": "Nico Stamm",
|
|
"nielsh": "Nils H.",
|
|
"noelstrahbach": "Noel Strohbach",
|
|
"nora": "Nora Rudersdorf geb. Holzbach",
|
|
"norah": "Nora Holzbach",
|
|
"norarh": "Nora Rudersdorf",
|
|
"norarudersdorf": "Nora Rudersdorf geb. Holzbach",
|
|
"oflennylengo": "Lenny Lengo",
|
|
"pascale": "Pascale Diefenbach",
|
|
"pascaledießenbach": "Pascale Diefenbach",
|
|
"patriciap": "Patricia Petry",
|
|
"paul": "Paul W.",
|
|
"paula": "Paula Gabler",
|
|
"paulag": "Paula Gabler",
|
|
"privatzuchtmuecke": "Privatzucht Mücke",
|
|
"privatzuchtmücke": "Privatzucht Mücke",
|
|
"pzmaintal": "Privatzucht Maintal",
|
|
"pzmuecke": "Privatzucht Mücke",
|
|
"pzmücke": "Privatzucht Mücke",
|
|
"pzseligenstadt": "Privatzucht Seligenstadt",
|
|
"ramonag": "Ramona Gömpel",
|
|
"ranialößler": "Rania Löffler",
|
|
"rominahubrich": "Romina Milde / Hubrich",
|
|
"rominamilde": "Romina Milde / Hubrich",
|
|
"romonag": "Ramona Gömpel",
|
|
"ronialöffler": "Rania Löffler",
|
|
"sandrak": "Sandra Kubas",
|
|
"sarahlöser": "Sarah Löwer",
|
|
"sarahlöuer": "Sarah Löwer",
|
|
"sarahz": "Sarah Zitzer",
|
|
"saskiaw": "Saskia Wucher",
|
|
"schlossmaus": "Schlossmäuse",
|
|
"schlossmäuse": "Schlossmäuse",
|
|
"schlossmäusen": "Schlossmäuse",
|
|
"sebastiansch": "Sebastian Schmitt",
|
|
"serinaberg": "Selina Berg",
|
|
"silkewolfganghintze": "Silke und Wolfgang Hintze",
|
|
"smilla": "Smilla H.",
|
|
"steffi": "Steffi K.",
|
|
"susanneninat": "Susanne + Nina Thomas",
|
|
"susannet": "Susanne + Torsten Saum",
|
|
"theresavalenca": "Theresia Valenca (Kopp)",
|
|
"theresiavalenca": "Theresia Valenca (Kopp)",
|
|
"timpf": "Tim Pfeiffer und Michelle Mai",
|
|
"timpfeifermichellemai": "Tim Pfeiffer und Michelle Mai",
|
|
"timpfeiffer": "Tim Pfeiffer und Michelle Mai",
|
|
"timpfeiffermichellemai": "Tim Pfeiffer und Michelle Mai",
|
|
"tonoböckenseld": "Tono Böckenfeld",
|
|
"topol": "Topolino",
|
|
"ulrichmüller": "Ulrich + Angela Müller",
|
|
"ulrikec": "Ulrike Cordes",
|
|
"ulriker": "Ulrike Ruppel",
|
|
"ulrikesch": "Ulrike Schulz",
|
|
"ulriket": "Ulrike Treml",
|
|
"vanessab": "Vanessa Becker",
|
|
"vanessag": "Vanessa Groll",
|
|
"veragreywitz": "Vera Geywitz",
|
|
"vonprivatzuchtmaintal": "Privatzucht Maintal",
|
|
"wolfgangfaus": "Wolfgang Faust",
|
|
"yvonne": "Yvonne Obendorfer",
|
|
"zuchtderkleinenchaoten": "Zucht der kleinen Chaoten",
|
|
}
|
|
|
|
if n in norm_map:
|
|
return norm_map[n], True
|
|
|
|
return name, True
|
|
|
|
def get_call_name(name):
|
|
if not name:
|
|
return ""
|
|
n = name.strip()
|
|
n = re.sub(r'\[[^\]]+\]$', '', n).strip()
|
|
n = re.split(r'\s+(?:of|von\s+den|von\s+der|v\.\s?d\.|von)\s+', n, flags=re.IGNORECASE)[0].strip()
|
|
return n
|
|
|
|
def get_dedup_name_key(name):
|
|
if not name:
|
|
return ""
|
|
n = name.lower().strip()
|
|
# Strip common suffixes/prefixes and parentheticals
|
|
n = re.sub(r'\b(?:von\s+den|v\.?\s*d\.?|v\.?o\.?)\s+(?:kleinen\s+)?chaoten\b', '', n)
|
|
n = re.sub(r'\bvon\s+der\s+schlossm\w+\b', '', n)
|
|
n = re.sub(r'\bvon\s+der\s+bunten\s+fellnasen?\b', '', n)
|
|
n = re.sub(r'\bof\s+black\s+forest\b', '', n)
|
|
n = re.sub(r'\b\(?rv\)?\b', '', n)
|
|
n = re.sub(r'\bgen\.\s+\w+', '', n)
|
|
# Clean up parentheses or brackets
|
|
n = re.sub(r'\(.*?\)', '', n)
|
|
n = re.sub(r'\[.*?\]', '', n)
|
|
return "".join(c for c in n if c.isalnum())
|
|
|
|
def clean_color_name(c_desc):
|
|
if not c_desc:
|
|
return "", False
|
|
|
|
# Lowercase and strip
|
|
c = c_desc.lower().strip()
|
|
|
|
# Check for Schecke
|
|
is_schecke = False
|
|
if re.search(r'\bsp\b|\bsp\d|\bsp[*(²³]|\bspotted|\bschecke|[- ]sp\b|\w+sp\b', c):
|
|
is_schecke = True
|
|
|
|
# Standardize parentheticals for schimmel
|
|
c = c.replace("(schimmel)", "schimmel")
|
|
c = c.replace("(schimmel-hell)", "schimmel hell")
|
|
c = c.replace("(schimmel hell)", "schimmel hell")
|
|
|
|
# Strip schecke/sp markers and any trailing text starting from sp
|
|
c = re.sub(r'\([- ]?sp(otted)?\)', '', c) # handles (-sp)
|
|
c = re.sub(r'[- ]?sp(otted)?\b.*', '', c) # handles -sp(k), -sp*(k), -sp, etc.
|
|
c = re.sub(r'[- ]?schecke\b.*', '', c)
|
|
c = re.sub(r'[- ]?spotted\b.*', '', c)
|
|
|
|
# Strip any other parentheticals, symbols, or trailing stars/numbers
|
|
c = re.sub(r'\s*\(.*?\)\s*', ' ', c)
|
|
c = re.sub(r'[²³*]', '', c)
|
|
c = c.strip()
|
|
|
|
# Mapping table for abbreviations, typos, and specific combinations
|
|
mapping = {
|
|
"antra": "anthrazit",
|
|
"anthra": "anthrazit",
|
|
"ankazit": "anthrazit",
|
|
"antrazit": "anthrazit",
|
|
"pew": "rew",
|
|
"bew": "hermelin",
|
|
"harder": "marder",
|
|
"kohli": "kohlfuchs",
|
|
"aligerfuchs": "algierfuchs",
|
|
"algiesfuchs": "algierfuchs",
|
|
"schw": "schwarz",
|
|
"sa": "silberagouti",
|
|
"a": "agouti",
|
|
"cp-sa": "cp-silberagouti",
|
|
"cp-a": "cp-agouti",
|
|
"cp-a-hell": "cp-agouti-hell",
|
|
"cp-aisa": "cp-agouti",
|
|
"cp-a / rcp-sa": "cp-agouti",
|
|
}
|
|
|
|
if c in mapping:
|
|
c = mapping[c]
|
|
|
|
return c, is_schecke
|
|
|
|
def resolve_color_and_genotype(color_val, existing_genotype, variety_map, variety_genotypes):
|
|
if not color_val:
|
|
return None, existing_genotype
|
|
color_str = str(color_val).strip()
|
|
clean_name, is_schecke = clean_color_name(color_str)
|
|
# Match color in variety_map
|
|
color_variety_id = None
|
|
if clean_name in variety_map:
|
|
color_variety_id = variety_map[clean_name]
|
|
else:
|
|
for seed_name, seed_id in variety_map.items():
|
|
if seed_name in clean_name or clean_name in seed_name:
|
|
color_variety_id = seed_id
|
|
break
|
|
# Update genotype if it's a Schecke
|
|
genotype = existing_genotype
|
|
if is_schecke:
|
|
if genotype:
|
|
if "spsp" in genotype:
|
|
genotype = genotype.replace("spsp", "Spsp")
|
|
elif "Spsp" not in genotype and "Sp" not in genotype:
|
|
genotype = f"{genotype} Spsp".strip()
|
|
else:
|
|
canonical = variety_genotypes.get(color_variety_id)
|
|
if canonical:
|
|
if "spsp" in canonical:
|
|
genotype = canonical.replace("spsp", "Spsp")
|
|
else:
|
|
genotype = f"{canonical} Spsp".strip()
|
|
else:
|
|
genotype = "Spsp"
|
|
return color_variety_id, genotype
|
|
|
|
def parse_date(d):
|
|
"""Convert variations of date formats to YYYY-MM-DD."""
|
|
if not d or d == "0001-01-01":
|
|
return None
|
|
d = str(d).strip()
|
|
# Try YYYY-MM-DD
|
|
if re.match(r"^\d{4}-\d{2}-\d{2}$", d):
|
|
parts = d.split("-")
|
|
year = int(parts[0])
|
|
if 1900 < year < 2000:
|
|
year += 100
|
|
d = f"{year}-{parts[1]}-{parts[2]}"
|
|
elif year == 1900:
|
|
return None
|
|
return d
|
|
# Try DD.MM.YYYY or D.M.YY
|
|
match = re.match(r"^(\d{1,2})\.(\d{1,2})\.(\d{2,4})$", d)
|
|
if match:
|
|
day = int(match.group(1))
|
|
month = int(match.group(2))
|
|
year = int(match.group(3))
|
|
if year < 100:
|
|
year += 2000
|
|
elif 1900 < year < 2000:
|
|
year += 100
|
|
elif year == 1900:
|
|
return None
|
|
try:
|
|
return datetime(year, month, day).strftime("%Y-%m-%d")
|
|
except ValueError:
|
|
pass
|
|
# Try ISO timestamp
|
|
try:
|
|
dt = datetime.fromisoformat(d.replace("Z", "+00:00"))
|
|
year = dt.year
|
|
if 1900 < year < 2000:
|
|
dt = dt.replace(year=year + 100)
|
|
return dt.strftime("%Y-%m-%d")
|
|
elif year == 1900:
|
|
return None
|
|
return dt.strftime("%Y-%m-%d")
|
|
except ValueError:
|
|
pass
|
|
return None
|
|
|
|
def date_to_days(dt_str):
|
|
if not dt_str:
|
|
return None
|
|
try:
|
|
return (datetime.strptime(dt_str, "%Y-%m-%d") - datetime(2000, 1, 1)).days
|
|
except ValueError:
|
|
return None
|
|
|
|
def days_to_date(days):
|
|
import datetime as dt
|
|
return (dt.datetime(2000, 1, 1) + dt.timedelta(days=int(days))).strftime("%Y-%m-%d")
|
|
|
|
def parse_death_info(notes, status, existing_dod, existing_cod):
|
|
if not notes:
|
|
return status, existing_dod, existing_cod
|
|
|
|
has_death_indicator = '†' in notes or 'verstorben' in notes.lower() or 'gestorben' in notes.lower() or 'todesdatum' in notes.lower() or '/+' in notes
|
|
if '+' in notes:
|
|
if re.search(r'\+\s*(?:LE|AS|Unbekannt|gestorben|verstorben|tod)\b', notes, re.I) or re.search(r'\+\s*\d{1,2}\.\d{1,2}\.\d{2,4}', notes) or '/+' in notes:
|
|
has_death_indicator = True
|
|
|
|
resolved_status = status
|
|
if has_death_indicator:
|
|
resolved_status = "Deceased"
|
|
|
|
dod = existing_dod
|
|
cod = existing_cod
|
|
|
|
# Look for date near death indicator
|
|
found_date = None
|
|
for m in re.finditer(r'([+†]\s*(?:LE|AS|Unbekannt|[a-zA-ZäöüÄÖÜß0-9()/ +,;.:-]{1,100}?)?\s*)(\d{1,2}\.\d{1,2}\.\d{2,4})', notes, re.I):
|
|
marker_text = m.group(0)
|
|
if '†' in marker_text or re.search(r'\+\s*(?:LE|AS|Unbekannt|gestorben|verstorben|tod|\d)', marker_text, re.I):
|
|
found_date = parse_date(m.group(2))
|
|
if found_date:
|
|
break
|
|
|
|
if not found_date:
|
|
m_death_marker = re.search(r'(†\s*)(\d{1,2}\.\d{1,2}\.\d{2,4})', notes)
|
|
if m_death_marker:
|
|
found_date = parse_date(m_death_marker.group(2))
|
|
|
|
if found_date and not dod:
|
|
dod = found_date
|
|
|
|
if not cod:
|
|
for m in re.finditer(r'([+†])\s*([a-zA-ZäöüÄÖÜß0-9()/ +,;.:-]{1,100}?)\s*\d{1,2}\.\d{1,2}\.\d{2,4}', notes, re.I):
|
|
indicator = m.group(1)
|
|
cod_candidate = m.group(2).strip()
|
|
if indicator == '+' and not re.search(r'\b(?:LE|AS|Unbekannt|gestorben|verstorben|tod)\b', cod_candidate, re.I):
|
|
continue
|
|
if cod_candidate:
|
|
cod = cod_candidate
|
|
break
|
|
|
|
if not cod:
|
|
m_cod2 = re.search(r'[+†]\s*\d{1,2}\.\d{1,2}\.\d{2,4}\s*([a-zA-ZäöüÄÖÜß0-9()/ +,;.:-]{1,100})', notes, re.I)
|
|
if m_cod2:
|
|
cod_candidate = m_cod2.group(1).strip()
|
|
if cod_candidate:
|
|
cod = cod_candidate
|
|
else:
|
|
m_cod3 = re.search(r'[+†]\s*(LE|AS|Unbekannt)\b', notes, re.I)
|
|
if m_cod3:
|
|
cod = m_cod3.group(1).strip()
|
|
|
|
if cod:
|
|
cod_lower = cod.lower().strip()
|
|
if cod_lower in ("le", "le (lungenentzündung)", "lungenentzündung", "lungenentzündung (lungenentzündung)"):
|
|
cod = "Lungenentzündung"
|
|
elif cod_lower in ("as", "altersschwäche", "altenschwäche", "alter"):
|
|
cod = "Altersschwäche"
|
|
elif cod_lower in ("unbekannt", "unklar"):
|
|
cod = "Unbekannt"
|
|
else:
|
|
# Replace abbreviations with full names (case-insensitive)
|
|
cod = re.sub(r'\bLE\b', 'Lungenentzündung', cod, flags=re.I)
|
|
cod = re.sub(r'\bAS\b', 'Altersschwäche', cod, flags=re.I)
|
|
cod = cod.replace('+', ' + ')
|
|
# Clean up multiple spaces
|
|
cod = re.sub(r'\s+', ' ', cod).strip(' ,;.-')
|
|
|
|
# Re-check after replacement
|
|
cod_lower = cod.lower().strip()
|
|
if cod_lower in ("le", "le (lungenentzündung)", "lungenentzündung", "lungenentzündung (lungenentzündung)"):
|
|
cod = "Lungenentzündung"
|
|
elif cod_lower in ("as", "altersschwäche", "altenschwäche", "alter"):
|
|
cod = "Altersschwäche"
|
|
elif cod_lower in ("unbekannt", "unklar"):
|
|
cod = "Unbekannt"
|
|
|
|
return resolved_status, dod, cod
|
|
|
|
def main():
|
|
print("Loading color variety seeds...")
|
|
variety_map = {}
|
|
variety_genotypes = {}
|
|
if os.path.exists(SEEDS_PATH):
|
|
with open(SEEDS_PATH, "r", encoding="utf-8") as f:
|
|
seeds = json.load(f)
|
|
for v in seeds:
|
|
variety_id = f"00000000-0000-0000-0000-{v['sortOrder'] + 1:012d}"
|
|
variety_map[v["name"].strip().lower()] = variety_id
|
|
if "english" in v and v["english"]:
|
|
variety_map[v["english"].strip().lower()] = variety_id
|
|
variety_genotypes[variety_id] = v.get("canonicalGenotype")
|
|
else:
|
|
print(f"Warning: Seeds path not found at {SEEDS_PATH}")
|
|
|
|
md_files = sorted([f for f in os.listdir(DIR_PATH) if f.lower().endswith('.md')])
|
|
print(f"Found {len(md_files)} markdown files in {DIR_PATH}.")
|
|
|
|
raw_contacts = []
|
|
raw_litters = []
|
|
raw_gerbils = []
|
|
|
|
# Map to track which files contained explicit dates
|
|
file_explicit_dates = {}
|
|
|
|
# 1. Parse JSON blocks from all markdown files and scope IDs by filename
|
|
for filename in md_files:
|
|
filepath = os.path.join(DIR_PATH, filename)
|
|
with open(filepath, 'r', encoding='utf-8') as f:
|
|
content = f.read()
|
|
|
|
json_start = content.find("```json")
|
|
if json_start == -1:
|
|
continue
|
|
json_start += len("```json")
|
|
json_end = content.rfind("```")
|
|
if json_end == -1 or json_end <= json_start:
|
|
continue
|
|
|
|
json_str = content[json_start:json_end].strip()
|
|
try:
|
|
data = json.loads(json_str)
|
|
|
|
contacts_key = next((k for k in data if k.lower() == 'contacts'), None)
|
|
litters_key = next((k for k in data if k.lower() == 'litters'), None)
|
|
gerbils_key = next((k for k in data if k.lower() == 'gerbils'), None)
|
|
|
|
file_contacts = data.get(contacts_key, []) if contacts_key else []
|
|
file_litters = data.get(litters_key, []) if litters_key else []
|
|
file_gerbils = data.get(gerbils_key, []) if gerbils_key else []
|
|
|
|
# Find explicit dates on this page to build chronological ordering
|
|
explicit_dates = []
|
|
for l in file_litters:
|
|
d = parse_date(l.get("Date") or l.get("date") or l.get("DateOfBirth") or l.get("dateOfBirth"))
|
|
if d:
|
|
explicit_dates.append(d)
|
|
for g in file_gerbils:
|
|
d = parse_date(g.get("DateOfBirth") or g.get("dateOfBirth"))
|
|
if d:
|
|
explicit_dates.append(d)
|
|
|
|
if explicit_dates:
|
|
# Convert to days since 2000-01-01
|
|
day_vals = [date_to_days(d) for d in explicit_dates if date_to_days(d) is not None]
|
|
if day_vals:
|
|
file_explicit_dates[filename] = sum(day_vals) / len(day_vals)
|
|
|
|
def scope_id(local_id):
|
|
if not local_id:
|
|
return None
|
|
local_str = str(local_id).strip()
|
|
if not local_str:
|
|
return None
|
|
return generate_guid(f"{filename}-{local_str}")
|
|
|
|
for c in file_contacts:
|
|
old_id = c.get("Id") or c.get("id")
|
|
c["_filename"] = filename
|
|
c["_scoped_id"] = scope_id(old_id)
|
|
raw_contacts.append(c)
|
|
|
|
for l in file_litters:
|
|
old_id = l.get("Id") or l.get("id")
|
|
l["_filename"] = filename
|
|
l["_scoped_id"] = scope_id(old_id)
|
|
l["_scoped_father_id"] = scope_id(l.get("FatherId") or l.get("fatherId"))
|
|
l["_scoped_mother_id"] = scope_id(l.get("MotherId") or l.get("motherId"))
|
|
raw_litters.append(l)
|
|
|
|
for g in file_gerbils:
|
|
old_id = g.get("Id") or g.get("id")
|
|
g["_filename"] = filename
|
|
g["_scoped_id"] = scope_id(old_id)
|
|
g["_scoped_litter_id"] = scope_id(g.get("LitterId") or g.get("litterId"))
|
|
g["_scoped_origin_cid"] = scope_id(g.get("OriginContactId") or g.get("originContactId"))
|
|
|
|
receiver_val = (g.get("ReceiverContactId") or g.get("receiverContactId") or
|
|
g.get("BuyerId") or g.get("buyerId") or
|
|
g.get("BuyerContactId") or g.get("buyerContactId") or
|
|
g.get("givenAwayContactId") or g.get("givenAwayToContactId") or
|
|
g.get("ownerContactId") or g.get("ownerId"))
|
|
g["_scoped_receiver_cid"] = scope_id(receiver_val)
|
|
raw_gerbils.append(g)
|
|
|
|
except json.JSONDecodeError as e:
|
|
print(f"Failed to parse JSON in {filename}: {e}")
|
|
|
|
# Chronological Interpolation: Estimate the date of each page based on neighboring pages with dates
|
|
file_dates = {}
|
|
sorted_files = sorted(md_files)
|
|
|
|
# Simple linear interpolation / extrapolation
|
|
for i, fn in enumerate(sorted_files):
|
|
if fn in file_explicit_dates:
|
|
file_dates[fn] = file_explicit_dates[fn]
|
|
else:
|
|
# Look left for closest explicit date
|
|
left_val, left_dist = None, None
|
|
for j in range(i - 1, -1, -1):
|
|
if sorted_files[j] in file_explicit_dates:
|
|
left_val = file_explicit_dates[sorted_files[j]]
|
|
left_dist = i - j
|
|
break
|
|
|
|
# Look right for closest explicit date
|
|
right_val, right_dist = None, None
|
|
for j in range(i + 1, len(sorted_files)):
|
|
if sorted_files[j] in file_explicit_dates:
|
|
right_val = file_explicit_dates[sorted_files[j]]
|
|
right_dist = j - i
|
|
break
|
|
|
|
if left_val is not None and right_val is not None:
|
|
# Interpolate
|
|
file_dates[fn] = left_val + (right_val - left_val) * (left_dist / (left_dist + right_dist))
|
|
elif left_val is not None:
|
|
# Extrapolate right (assume 30 days per page gap as placeholder)
|
|
file_dates[fn] = left_val + (left_dist * 30)
|
|
elif right_val is not None:
|
|
# Extrapolate left
|
|
file_dates[fn] = right_val - (right_dist * 30)
|
|
else:
|
|
# No dates in entire log? Default to 2011-01-01
|
|
file_dates[fn] = date_to_days("2011-01-01")
|
|
|
|
print(f"Parsed {len(raw_contacts)} raw contacts, {len(raw_litters)} raw litters, {len(raw_gerbils)} raw gerbils.")
|
|
|
|
# Run extract.py to make sure stammbaum data is up to date
|
|
import subprocess
|
|
print("Running extract.py to extract Stammbäume...")
|
|
try:
|
|
subprocess.run([sys.executable, "extract.py"], check=True)
|
|
except Exception as e:
|
|
print(f"Warning: Failed to run extract.py: {e}")
|
|
|
|
# Load Stammbaum data
|
|
stammbaum_only_animals = []
|
|
animals_path = os.path.join(OUTPUT_DIR, "animals.json")
|
|
if os.path.exists(animals_path):
|
|
with open(animals_path, "r", encoding="utf-8") as f:
|
|
stammbaum_animals = json.load(f)
|
|
for a in stammbaum_animals:
|
|
sources = a.get("sourceFiles", [])
|
|
if any("stammbaum" in str(s).lower() for s in sources):
|
|
stammbaum_only_animals.append(a)
|
|
print(f"Loaded {len(stammbaum_only_animals)} Stammbaum animals.")
|
|
else:
|
|
print(f"Warning: Stammbaum animals.json not found at {animals_path}")
|
|
|
|
# Run extract_docx.py to make sure docx data is up to date
|
|
print("Running extract_docx.py to extract docx...")
|
|
try:
|
|
subprocess.run([sys.executable, "extract_docx.py"], check=True)
|
|
except Exception as e:
|
|
print(f"Warning: Failed to run extract_docx.py: {e}")
|
|
|
|
# Load Docx data
|
|
docx_animals = []
|
|
docx_animals_path = os.path.join(OUTPUT_DIR, "docx_animals.json")
|
|
if os.path.exists(docx_animals_path):
|
|
with open(docx_animals_path, "r", encoding="utf-8") as f:
|
|
docx_animals = json.load(f)
|
|
print(f"Loaded {len(docx_animals)} animals from docx.")
|
|
else:
|
|
print(f"Warning: docx_animals.json not found at {docx_animals_path}")
|
|
|
|
docx_litters = []
|
|
docx_litters_path = os.path.join(OUTPUT_DIR, "docx_litters.json")
|
|
if os.path.exists(docx_litters_path):
|
|
with open(docx_litters_path, "r", encoding="utf-8") as f:
|
|
docx_litters = json.load(f)
|
|
print(f"Loaded {len(docx_litters)} litters from docx.")
|
|
else:
|
|
print(f"Warning: docx_litters.json not found at {docx_litters_path}")
|
|
|
|
# Extract docx buyer contacts and add to raw_contacts
|
|
for da in docx_animals:
|
|
o_name = (da.get("owner") or "").strip()
|
|
if o_name:
|
|
o_scoped_id = generate_guid(f"docx-contact-{normalize_name(o_name)}")
|
|
raw_contacts.append({
|
|
"Name": o_name,
|
|
"_filename": "Wurfchronik-Detail",
|
|
"_scoped_id": o_scoped_id
|
|
})
|
|
|
|
# Map docx litters and append to raw_litters
|
|
docx_litter_id_map = {} # wsCode -> scoped_id
|
|
for dl in docx_litters:
|
|
l_name = dl["litterId"]
|
|
dob_val = parse_date(dl["dob"])
|
|
f_name = dl["fatherName"]
|
|
m_name = dl["motherName"]
|
|
ws_code = dl["wsCode"]
|
|
note_val = dl.get("note")
|
|
|
|
# Parse survived/total born from wsCode (e.g. 4/5)
|
|
total_born = None
|
|
deaths_8w = None
|
|
if "/" in ws_code:
|
|
parts = ws_code.split("/")
|
|
if len(parts) == 2:
|
|
try:
|
|
survived = int(parts[0])
|
|
total = int(parts[1])
|
|
total_born = total
|
|
deaths_8w = max(0, total - survived)
|
|
except ValueError:
|
|
pass
|
|
|
|
l_scoped_id = generate_guid(f"docx-litter-{normalize_name(ws_code)}-{dob_val or '0001-01-01'}")
|
|
docx_litter_id_map[(ws_code, dob_val)] = l_scoped_id
|
|
|
|
raw_litters.append({
|
|
"Id": l_scoped_id,
|
|
"Name": l_name,
|
|
"Date": dob_val or "0001-01-01",
|
|
"TotalBorn": total_born,
|
|
"DeathsWithin8Weeks": deaths_8w,
|
|
"FatherId": generate_guid(f"stammbaum-animal-{normalize_name(f_name)}"), # placeholder
|
|
"MotherId": generate_guid(f"stammbaum-animal-{normalize_name(m_name)}"), # placeholder
|
|
"ExpectedGoHomeDate": None,
|
|
"Notes": note_val if note_val else "Docx imported litter",
|
|
"PairingCode": None,
|
|
"ExternalRef": f"docx-litter-{l_scoped_id}",
|
|
"LitterLetter": l_name[0] if l_name and len(l_name) > 0 else None,
|
|
"_father_name": f_name,
|
|
"_mother_name": m_name,
|
|
"_filename": "Wurfchronik-Detail",
|
|
"_scoped_id": l_scoped_id,
|
|
"_scoped_father_id": generate_guid(f"stammbaum-animal-{normalize_name(f_name)}"),
|
|
"_scoped_mother_id": generate_guid(f"stammbaum-animal-{normalize_name(m_name)}")
|
|
})
|
|
|
|
|
|
# Extract stammbaum contacts and add to raw_contacts
|
|
for a in stammbaum_only_animals:
|
|
b_name = (a.get("breeder") or "").strip()
|
|
if b_name:
|
|
b_scoped_id = generate_guid(f"stammbaum-contact-{normalize_name(b_name)}")
|
|
raw_contacts.append({
|
|
"Name": b_name,
|
|
"_filename": "Stammbaum",
|
|
"_scoped_id": b_scoped_id
|
|
})
|
|
z_name = (a.get("zucht") or "").strip()
|
|
if z_name:
|
|
z_scoped_id = generate_guid(f"stammbaum-contact-{normalize_name(z_name)}")
|
|
raw_contacts.append({
|
|
"Name": z_name,
|
|
"_filename": "Stammbaum",
|
|
"_scoped_id": z_scoped_id
|
|
})
|
|
|
|
# Residency propagation for stammbaum animals
|
|
def is_clan_zucht(z):
|
|
if not z:
|
|
return False
|
|
norm = z.lower()
|
|
return "klein" in norm and "chaot" in norm and "extern" not in norm
|
|
|
|
stammbaum_resident_ids = set()
|
|
for a in stammbaum_only_animals:
|
|
if is_clan_zucht(a.get("zucht")) or is_clan_zucht(a.get("zuchtCanon")):
|
|
stammbaum_resident_ids.add(a["id"])
|
|
|
|
# Propagate residency to parents of resident offspring
|
|
for _ in range(5):
|
|
for a in stammbaum_only_animals:
|
|
if a["id"] in stammbaum_resident_ids:
|
|
for p_ref in a.get("parentRefs", []):
|
|
p_key = (normalize_name(p_ref["name"]), parse_date(p_ref["dob"]))
|
|
for cand in stammbaum_only_animals:
|
|
if normalize_name(cand["name"]) == p_key[0]:
|
|
cand_dob = parse_date(cand["dob"])
|
|
if not p_key[1] or cand_dob == p_key[1]:
|
|
stammbaum_resident_ids.add(cand["id"])
|
|
|
|
# Pre-index Wurfchronik litters from markdown
|
|
md_litters_idx = {}
|
|
for rl in raw_litters:
|
|
f_name = rl.get("FatherName") or rl.get("fatherName") or rl.get("ParentMaleName") or rl.get("parentMaleName")
|
|
m_name = rl.get("MotherName") or rl.get("motherName") or rl.get("ParentFemaleName") or rl.get("parentFemaleName")
|
|
ldate = parse_date(rl.get("Date") or rl.get("date") or rl.get("DateOfBirth") or rl.get("dateOfBirth"))
|
|
if f_name and m_name and ldate:
|
|
key = (normalize_name(f_name), normalize_name(m_name), ldate)
|
|
md_litters_idx[key] = rl
|
|
|
|
# Create virtual litters for stammbaum animals
|
|
created_virtual_litters = {}
|
|
for a in stammbaum_only_animals:
|
|
parent_refs = a.get("parentRefs", [])
|
|
father_ref = next((p for p in parent_refs if p.get("roleGuess") == "father"), None)
|
|
mother_ref = next((p for p in parent_refs if p.get("roleGuess") == "mother"), None)
|
|
|
|
a["_mapped_litter_scoped_id"] = None
|
|
if father_ref and mother_ref:
|
|
f_name = father_ref.get("name")
|
|
m_name = mother_ref.get("name")
|
|
dob_val = parse_date(a.get("dob"))
|
|
|
|
mapped_litter = None
|
|
if dob_val:
|
|
key = (normalize_name(f_name), normalize_name(m_name), dob_val)
|
|
mapped_litter = md_litters_idx.get(key)
|
|
|
|
if mapped_litter:
|
|
a["_mapped_litter_scoped_id"] = mapped_litter["_scoped_id"]
|
|
else:
|
|
v_key = (normalize_name(f_name), normalize_name(m_name), dob_val or "0001-01-01")
|
|
if v_key in created_virtual_litters:
|
|
a["_mapped_litter_scoped_id"] = created_virtual_litters[v_key]
|
|
else:
|
|
l_scoped_id = generate_guid(f"virtual-litter-{v_key[0]}-{v_key[1]}-{v_key[2]}")
|
|
created_virtual_litters[v_key] = l_scoped_id
|
|
a["_mapped_litter_scoped_id"] = l_scoped_id
|
|
|
|
# Try to link parents to actual parsed stammbaum animals
|
|
f_scoped_id = None
|
|
m_scoped_id = None
|
|
f_dob = parse_date(father_ref.get("dob"))
|
|
m_dob = parse_date(mother_ref.get("dob"))
|
|
|
|
for p_cand in stammbaum_only_animals:
|
|
cand_call_norm = normalize_name(get_call_name(p_cand["name"]))
|
|
if cand_call_norm == normalize_name(f_name) or normalize_name(p_cand["name"]) == normalize_name(f_name):
|
|
if not f_dob or parse_date(p_cand.get("dob")) == f_dob:
|
|
f_scoped_id = generate_guid(f"stammbaum-animal-{p_cand['id']}")
|
|
break
|
|
for p_cand in stammbaum_only_animals:
|
|
cand_call_norm = normalize_name(get_call_name(p_cand["name"]))
|
|
if cand_call_norm == normalize_name(m_name) or normalize_name(p_cand["name"]) == normalize_name(m_name):
|
|
if not m_dob or parse_date(p_cand.get("dob")) == m_dob:
|
|
m_scoped_id = generate_guid(f"stammbaum-animal-{p_cand['id']}")
|
|
break
|
|
|
|
raw_litters.append({
|
|
"Id": l_scoped_id,
|
|
"Name": f"Wurf von {f_name} + {m_name}",
|
|
"Date": dob_val or "0001-01-01",
|
|
"TotalBorn": None,
|
|
"DeathsWithin8Weeks": None,
|
|
"FatherId": f_scoped_id or generate_guid(f"stammbaum-animal-{normalize_name(f_name)}"),
|
|
"MotherId": m_scoped_id or generate_guid(f"stammbaum-animal-{normalize_name(m_name)}"),
|
|
"ExpectedGoHomeDate": None,
|
|
"Notes": "Pedigree virtual litter",
|
|
"PairingCode": None,
|
|
"ExternalRef": f"virtual-{l_scoped_id}",
|
|
"LitterLetter": None,
|
|
"_father_name": f_name,
|
|
"_mother_name": m_name,
|
|
"_filename": "Stammbaum",
|
|
"_scoped_id": l_scoped_id,
|
|
"_scoped_father_id": f_scoped_id or generate_guid(f"stammbaum-animal-{normalize_name(f_name)}"),
|
|
"_scoped_mother_id": m_scoped_id or generate_guid(f"stammbaum-animal-{normalize_name(m_name)}")
|
|
})
|
|
|
|
# 2. Resolve Contacts globally (deduplicate by normalized name)
|
|
|
|
contact_by_norm_name = {}
|
|
contact_id_map = {} # scoped_old_id -> global_guid
|
|
|
|
for rc in raw_contacts:
|
|
name_val = rc.get("Name") or rc.get("name") or rc.get("FullName") or rc.get("fullName")
|
|
if not name_val:
|
|
first = rc.get("FirstName") or rc.get("firstName")
|
|
last = rc.get("LastName") or rc.get("lastName")
|
|
if first or last:
|
|
name_val = f"{first or ''} {last or ''}".strip()
|
|
|
|
if not name_val:
|
|
continue
|
|
|
|
canon_name, should_keep = get_normalized_contact_name(name_val)
|
|
scoped_id = rc["_scoped_id"]
|
|
if not should_keep:
|
|
if scoped_id:
|
|
contact_id_map[scoped_id] = None
|
|
continue
|
|
|
|
norm_name = normalize_name(canon_name)
|
|
|
|
if norm_name not in contact_by_norm_name:
|
|
global_guid = generate_guid(f"contact-{norm_name}")
|
|
contact_by_norm_name[norm_name] = {
|
|
"Id": global_guid,
|
|
"Name": canon_name,
|
|
"Email": rc.get("Email") or rc.get("email"),
|
|
"Phone": rc.get("Phone") or rc.get("phone"),
|
|
"Address": rc.get("Address") or rc.get("address"),
|
|
"Notes": rc.get("Notes") or rc.get("notes") or rc.get("Note") or rc.get("note")
|
|
}
|
|
else:
|
|
gc = contact_by_norm_name[norm_name]
|
|
if not gc["Email"] and (rc.get("Email") or rc.get("email")):
|
|
gc["Email"] = rc.get("Email") or rc.get("email")
|
|
if not gc["Phone"] and (rc.get("Phone") or rc.get("phone")):
|
|
gc["Phone"] = rc.get("Phone") or rc.get("phone")
|
|
if not gc["Address"] and (rc.get("Address") or rc.get("address")):
|
|
gc["Address"] = rc.get("Address") or rc.get("address")
|
|
if not gc["Notes"] and (rc.get("Notes") or rc.get("notes") or rc.get("Note") or rc.get("note")):
|
|
gc["Notes"] = rc.get("Notes") or rc.get("notes") or rc.get("Note") or rc.get("note")
|
|
|
|
if scoped_id:
|
|
contact_id_map[scoped_id] = contact_by_norm_name[norm_name]["Id"]
|
|
|
|
resolved_contacts = list(contact_by_norm_name.values())
|
|
print(f"Resolved to {len(resolved_contacts)} unique contacts.")
|
|
|
|
# 3. Process Litters (normalize keys)
|
|
resolved_litters = []
|
|
litter_id_map = {} # scoped_old_id -> new_guid
|
|
litter_by_scoped_id = {}
|
|
|
|
for rl in raw_litters:
|
|
filename = rl.get("_filename")
|
|
name_val = rl.get("Name") or rl.get("name")
|
|
if not name_val:
|
|
name_val = "Wurf"
|
|
|
|
dob_val = parse_date(rl.get("Date") or rl.get("date") or rl.get("DateOfBirth") or rl.get("dateOfBirth"))
|
|
if not dob_val:
|
|
dob_val = "0001-01-01"
|
|
|
|
new_guid = rl["_scoped_id"]
|
|
if not new_guid:
|
|
new_guid = generate_guid(f"litter-{filename}-{name_val}-{dob_val}")
|
|
|
|
scoped_old_id = rl["_scoped_id"]
|
|
if scoped_old_id:
|
|
litter_id_map[scoped_old_id] = new_guid
|
|
|
|
total_born = rl.get("TotalBorn") or rl.get("totalBorn") or rl.get("LitterSize") or rl.get("litterSize") or rl.get("size") or rl.get("totalPups")
|
|
if total_born is not None:
|
|
try:
|
|
total_born = int(total_born)
|
|
except ValueError:
|
|
total_born = None
|
|
|
|
deaths_8w = rl.get("DeathsWithin8Weeks") or rl.get("deathsWithin8Weeks")
|
|
if deaths_8w is not None:
|
|
try:
|
|
deaths_8w = int(deaths_8w)
|
|
except ValueError:
|
|
deaths_8w = None
|
|
elif total_born is not None and rl.get("survived") is not None:
|
|
try:
|
|
deaths_8w = total_born - int(rl.get("survived"))
|
|
except ValueError:
|
|
pass
|
|
|
|
father_name = rl.get("FatherName") or rl.get("fatherName") or rl.get("ParentMaleName") or rl.get("parentMaleName") or rl.get("_father_name")
|
|
mother_name = rl.get("MotherName") or rl.get("motherName") or rl.get("ParentFemaleName") or rl.get("parentFemaleName") or rl.get("_mother_name")
|
|
|
|
raw_ext_ref = rl.get("ExternalRef") or rl.get("externalRef") or rl.get("Id") or rl.get("id")
|
|
ext_ref_scoped = f"{filename}-{raw_ext_ref}" if raw_ext_ref else None
|
|
|
|
l_record = {
|
|
"Id": new_guid,
|
|
"Name": name_val,
|
|
"Date": dob_val,
|
|
"TotalBorn": total_born,
|
|
"DeathsWithin8Weeks": deaths_8w,
|
|
"FatherId": rl["_scoped_father_id"],
|
|
"MotherId": rl["_scoped_mother_id"],
|
|
"ExpectedGoHomeDate": parse_date(rl.get("ExpectedGoHomeDate") or rl.get("expectedGoHomeDate")),
|
|
"Notes": rl.get("Notes") or rl.get("notes") or rl.get("Note") or rl.get("note"),
|
|
"PairingCode": rl.get("PairingCode") or rl.get("pairingCode"),
|
|
"ExternalRef": ext_ref_scoped,
|
|
"LitterLetter": rl.get("LitterLetter") or rl.get("litterLetter"),
|
|
"_father_name": father_name,
|
|
"_mother_name": mother_name,
|
|
"_filename": filename
|
|
}
|
|
resolved_litters.append(l_record)
|
|
litter_by_scoped_id[new_guid] = l_record
|
|
|
|
print(f"Processed {len(resolved_litters)} litters.")
|
|
|
|
# Helper to lookup litter dates for birth date estimation
|
|
def get_litter_date(l_id):
|
|
if l_id in litter_by_scoped_id:
|
|
d = litter_by_scoped_id[l_id]["Date"]
|
|
if d and d != "0001-01-01":
|
|
return d
|
|
return None
|
|
|
|
# 4. Normalize and group Gerbils
|
|
all_processed_gerbils = []
|
|
for rg in raw_gerbils:
|
|
filename = rg.get("_filename")
|
|
name_val = rg.get("Name") or rg.get("name") or rg.get("callName")
|
|
if not name_val:
|
|
name_val = "Unbekannt"
|
|
|
|
dob_val = parse_date(rg.get("DateOfBirth") or rg.get("dateOfBirth"))
|
|
|
|
new_guid = rg["_scoped_id"]
|
|
if not new_guid:
|
|
new_guid = generate_guid(f"gerbil-{filename}-{name_val}-{dob_val}")
|
|
|
|
# Gender normalization
|
|
gender_val = rg.get("Gender") or rg.get("gender") or "unknown"
|
|
gender_val = str(gender_val).lower().strip()
|
|
if gender_val in ["m", "male", "männlich"]:
|
|
gender = "male"
|
|
elif gender_val in ["w", "f", "female", "weiblich"]:
|
|
gender = "female"
|
|
else:
|
|
gender = "unknown"
|
|
|
|
dod_val = parse_date(rg.get("DateOfDeath") or rg.get("dateOfDeath") or rg.get("deathDate"))
|
|
gohome_val = parse_date(rg.get("GoHomeDate") or rg.get("goHomeDate") or rg.get("DateGivenAway") or rg.get("givenAwayDate") or rg.get("DateOfHandover") or rg.get("dateOfHandover") or rg.get("HandoverDate") or rg.get("DateOfSale") or rg.get("dateOfSale"))
|
|
|
|
status_val = rg.get("Status") or rg.get("status") or ""
|
|
status_val = str(status_val).lower().strip()
|
|
status = "Breeding"
|
|
if dod_val or "deceased" in status_val or "verstorben" in status_val or "tod" in status_val or "dead" in status_val:
|
|
status = "Deceased"
|
|
elif gohome_val or "givenaway" in status_val or "abgegeben" in status_val or "verkauft" in status_val or "sold" in status_val:
|
|
status = "GivenAway"
|
|
elif "forsale" in status_val or "abzugeben" in status_val:
|
|
status = "ForSale"
|
|
elif "pet" in status_val or "liebhaber" in status_val:
|
|
status = "Pet"
|
|
else:
|
|
if dod_val:
|
|
status = "Deceased"
|
|
elif gohome_val:
|
|
status = "GivenAway"
|
|
|
|
notes_val = rg.get("Notes") or rg.get("notes") or rg.get("Note") or rg.get("note")
|
|
existing_cod = rg.get("CauseOfDeath") or rg.get("causeOfDeath") or rg.get("DeathCause")
|
|
status, dod_val, cause_of_death_val = parse_death_info(notes_val, status, dod_val, existing_cod)
|
|
|
|
# Explicit resolution for Ken'ichi's cause of death
|
|
if name_val == "Ken'ichi" and dob_val == "2015-03-01":
|
|
cause_of_death_val = "Duftdrüsen-Tumor"
|
|
|
|
color_val = rg.get("ColorVarietyId") or rg.get("colorVarietyId") or rg.get("Color") or rg.get("color") or rg.get("ColorDescription") or rg.get("colorDescription")
|
|
existing_gt = rg.get("Genotype") or rg.get("genotype")
|
|
color_variety_id, genotype_val = resolve_color_and_genotype(
|
|
color_val, existing_gt, variety_map, variety_genotypes
|
|
)
|
|
if color_val and not color_variety_id:
|
|
try:
|
|
uuid.UUID(str(color_val).strip())
|
|
color_variety_id = str(color_val).strip()
|
|
except ValueError:
|
|
print(f"Warning: Unknown color variety '{color_val}' for gerbil '{name_val}' on page {filename}")
|
|
|
|
origin_cid = rg["_scoped_origin_cid"]
|
|
receiver_cid = rg["_scoped_receiver_cid"]
|
|
|
|
if origin_cid in contact_id_map:
|
|
origin_cid = contact_id_map[origin_cid]
|
|
if receiver_cid in contact_id_map:
|
|
receiver_cid = contact_id_map[receiver_cid]
|
|
|
|
is_resident = rg.get("IsResident") or rg.get("isResident")
|
|
if is_resident is None:
|
|
is_resident = True
|
|
else:
|
|
is_resident = str(is_resident).lower() == "true"
|
|
|
|
traits = rg.get("CharacterTraits") or rg.get("characterTraits") or []
|
|
if not isinstance(traits, list):
|
|
traits = [str(traits)]
|
|
char_note = rg.get("CharacterNote") or rg.get("characterNote")
|
|
|
|
is_deaf = rg.get("IsDeaf") or rg.get("isDeaf")
|
|
if is_deaf is not None:
|
|
is_deaf = str(is_deaf).lower() == "true"
|
|
|
|
old_litter_id = rg["_scoped_litter_id"]
|
|
|
|
raw_ext_ref = rg.get("ExternalRef") or rg.get("externalRef") or rg.get("Id") or rg.get("id")
|
|
ext_ref_scoped = f"{filename}-{raw_ext_ref}" if raw_ext_ref else None
|
|
|
|
# Estimated effective date of birth for conflict checking
|
|
eff_dob_val = dob_val
|
|
if not eff_dob_val and old_litter_id:
|
|
# Try to get litter date if litter was resolved
|
|
mapped_l_id = litter_id_map.get(old_litter_id)
|
|
if mapped_l_id:
|
|
eff_dob_val = get_litter_date(mapped_l_id)
|
|
|
|
# Determine explicit or litter-derived birth date (None if unknown/parent)
|
|
birth_date = dob_val
|
|
if not birth_date and old_litter_id:
|
|
mapped_l_id = litter_id_map.get(old_litter_id)
|
|
if mapped_l_id:
|
|
birth_date = get_litter_date(mapped_l_id)
|
|
|
|
# If still no effective date, use the estimated page date
|
|
if not eff_dob_val:
|
|
eff_dob_val = days_to_date(file_dates[filename])
|
|
|
|
raw_breeder = rg.get("OriginBreeder") or rg.get("originBreeder")
|
|
if raw_breeder:
|
|
norm_b, keep_b = get_normalized_contact_name(raw_breeder)
|
|
raw_breeder = norm_b if keep_b else None
|
|
|
|
all_processed_gerbils.append({
|
|
"Id": new_guid,
|
|
"Name": name_val,
|
|
"Gender": gender,
|
|
"Status": status,
|
|
"LitterId": old_litter_id, # mapped later
|
|
"OriginContactId": origin_cid,
|
|
"ReceiverContactId": receiver_cid,
|
|
"EnclosureId": None,
|
|
"ColorVarietyId": color_variety_id,
|
|
"DateOfBirth": dob_val,
|
|
"DateOfDeath": dod_val,
|
|
"CauseOfDeath": cause_of_death_val,
|
|
"GoHomeDate": gohome_val,
|
|
"Genotype": genotype_val,
|
|
"Notes": rg.get("Notes") or rg.get("notes") or rg.get("Note") or rg.get("note"),
|
|
"ImportSource": rg.get("ImportSource") or rg.get("importSource") or filename,
|
|
"ExternalRef": ext_ref_scoped,
|
|
"RawImportData": rg.get("RawImportData") or rg.get("rawImportData") or json.dumps({"colorDescription": color_val if not color_variety_id else None}),
|
|
"OriginBreeder": raw_breeder or ("Zucht der kleinen Chaoten" if is_resident else None),
|
|
"NameSearch": normalize_name(name_val),
|
|
"CharacterTraits": traits,
|
|
"CharacterNote": char_note,
|
|
"IsDeaf": is_deaf,
|
|
"IsResident": is_resident,
|
|
"_photos": rg.get("photos", []),
|
|
"_old_scoped_litter_id": old_litter_id,
|
|
"_eff_dob": eff_dob_val,
|
|
"_birth_date": birth_date,
|
|
"_filename": filename,
|
|
"_old_id": rg.get("Id") or rg.get("id")
|
|
})
|
|
|
|
# Map and append stammbaum animals to all_processed_gerbils
|
|
for a in stammbaum_only_animals:
|
|
a_id = a["id"]
|
|
name_val = a["name"]
|
|
|
|
gender_val = str(a.get("gender") or "").lower().strip()
|
|
if gender_val in ["m", "male"]:
|
|
gender = "male"
|
|
elif gender_val in ["w", "f", "female"]:
|
|
gender = "female"
|
|
else:
|
|
gender = "unknown"
|
|
|
|
dob_val = parse_date(a.get("dob"))
|
|
dod_val = parse_date(a.get("death"))
|
|
|
|
status = "Breeding"
|
|
if dod_val:
|
|
status = "Deceased"
|
|
elif a_id not in stammbaum_resident_ids:
|
|
status = "GivenAway"
|
|
else:
|
|
if dob_val:
|
|
try:
|
|
dt_dob = datetime.strptime(dob_val, "%Y-%m-%d")
|
|
dt_now = datetime.now()
|
|
age_years = (dt_now - dt_dob).days / 365.25
|
|
if age_years >= 7.0:
|
|
status = "Deceased"
|
|
except Exception:
|
|
pass
|
|
|
|
# Color variety mapping
|
|
existing_gt = a["genotype"]["rawGenotype"] if a["genotype"]["rawGenotype"] else None
|
|
color_variety_id, genotype_val = resolve_color_and_genotype(
|
|
a.get("farbschlag"), existing_gt, variety_map, variety_genotypes
|
|
)
|
|
if not color_variety_id:
|
|
for fbv in a.get("farbschlagVariants", []):
|
|
cv_id, gt_val = resolve_color_and_genotype(fbv, existing_gt, variety_map, variety_genotypes)
|
|
if cv_id:
|
|
color_variety_id = cv_id
|
|
genotype_val = gt_val
|
|
break
|
|
|
|
# Contact mapping
|
|
origin_cid = None
|
|
b_name = (a.get("breeder") or "").strip()
|
|
z_name = (a.get("zucht") or "").strip()
|
|
if b_name:
|
|
origin_cid = generate_guid(f"stammbaum-contact-{normalize_name(b_name)}")
|
|
elif z_name:
|
|
origin_cid = generate_guid(f"stammbaum-contact-{normalize_name(z_name)}")
|
|
|
|
# Map to resolved global contact GUID
|
|
if origin_cid in contact_id_map:
|
|
origin_cid = contact_id_map[origin_cid]
|
|
else:
|
|
origin_cid = None
|
|
|
|
|
|
scoped_id = generate_guid(f"stammbaum-animal-{a_id}")
|
|
scoped_litter_id = a.get("_mapped_litter_scoped_id")
|
|
|
|
is_deaf = a["genotype"].get("deaf")
|
|
|
|
stammbaum_breeder = a.get("breeder") if a.get("breeder") else (a.get("zucht") if a.get("zucht") else None)
|
|
if stammbaum_breeder:
|
|
norm_b, keep_b = get_normalized_contact_name(stammbaum_breeder)
|
|
stammbaum_breeder = norm_b if keep_b else None
|
|
|
|
all_processed_gerbils.append({
|
|
"Id": scoped_id,
|
|
"Name": name_val,
|
|
"Gender": gender,
|
|
"Status": status,
|
|
"LitterId": scoped_litter_id, # mapped later in step 5
|
|
"OriginContactId": origin_cid,
|
|
"ReceiverContactId": None,
|
|
"EnclosureId": None,
|
|
"ColorVarietyId": color_variety_id,
|
|
"DateOfBirth": dob_val,
|
|
"DateOfDeath": dod_val,
|
|
"CauseOfDeath": "Duftdrüsen-Tumor" if name_val == "Ken'ichi" and dob_val == "2015-03-01" else None,
|
|
"GoHomeDate": None,
|
|
"Genotype": genotype_val,
|
|
"Notes": None,
|
|
"ImportSource": ", ".join(a.get("sourceFiles", [])),
|
|
"ExternalRef": f"stammbaum-{a_id}",
|
|
"RawImportData": json.dumps({
|
|
"rawGenotype": a["genotype"]["rawGenotype"],
|
|
"unmappedTokens": a["genotype"]["unmappedTokens"],
|
|
"breederText": a.get("breeder", "")
|
|
}, ensure_ascii=False),
|
|
"OriginBreeder": stammbaum_breeder,
|
|
"NameSearch": normalize_name(name_val),
|
|
"CharacterTraits": [],
|
|
"CharacterNote": None,
|
|
"IsDeaf": is_deaf,
|
|
"IsResident": a_id in stammbaum_resident_ids,
|
|
"_photos": a.get("photos", []),
|
|
"_old_scoped_litter_id": scoped_litter_id,
|
|
"_eff_dob": dob_val or "2010-01-01",
|
|
"_birth_date": dob_val,
|
|
"_filename": a.get("sourceFiles", ["Stammbaum"])[0],
|
|
"_old_id": a_id
|
|
})
|
|
|
|
# Map and append docx animals to all_processed_gerbils
|
|
for idx, da in enumerate(docx_animals):
|
|
name_val = da["name"]
|
|
gender = da["gender"]
|
|
|
|
dob_val = parse_date(da.get("litterDob"))
|
|
dod_val = parse_date(da.get("deathDate"))
|
|
gohome_val = parse_date(da.get("abgabeDate"))
|
|
|
|
# Status precedence
|
|
status = "Breeding"
|
|
if dod_val:
|
|
status = "Deceased"
|
|
elif gohome_val or da.get("owner"):
|
|
status = "GivenAway"
|
|
|
|
# Color variety mapping
|
|
color_variety_id, genotype_val = resolve_color_and_genotype(
|
|
da.get("farbschlag"), None, variety_map, variety_genotypes
|
|
)
|
|
|
|
# Contact mapping (buyer)
|
|
receiver_cid = None
|
|
o_name = (da.get("owner") or "").strip()
|
|
if o_name:
|
|
receiver_cid = generate_guid(f"docx-contact-{normalize_name(o_name)}")
|
|
if receiver_cid in contact_id_map:
|
|
receiver_cid = contact_id_map[receiver_cid]
|
|
else:
|
|
receiver_cid = None
|
|
|
|
# Scoped ID
|
|
scoped_id = generate_guid(f"docx-animal-{idx}-{normalize_name(name_val)}-{dob_val or '0001-01-01'}")
|
|
|
|
# Litter ID mapping
|
|
ws_code = da.get("wsCode")
|
|
scoped_litter_id = docx_litter_id_map.get((ws_code, dob_val))
|
|
|
|
|
|
# Raw import details
|
|
raw_import_payload = json.dumps({
|
|
"abgabeWeight": da.get("abgabeWeight", ""),
|
|
"deathCause": da.get("deathCause", ""),
|
|
"partnerName": da.get("partnerName", ""),
|
|
"partnerDob": da.get("partnerDob", "")
|
|
}, ensure_ascii=False)
|
|
|
|
# Residents: if sold/given away, it's not a resident
|
|
is_resident = not bool(o_name)
|
|
|
|
all_processed_gerbils.append({
|
|
"Id": scoped_id,
|
|
"Name": name_val,
|
|
"Gender": gender,
|
|
"Status": status,
|
|
"LitterId": scoped_litter_id,
|
|
"OriginContactId": None,
|
|
"ReceiverContactId": receiver_cid, # mapped in step 5
|
|
"EnclosureId": None,
|
|
"ColorVarietyId": color_variety_id,
|
|
"DateOfBirth": dob_val,
|
|
"DateOfDeath": dod_val,
|
|
"CauseOfDeath": da.get("deathCause"),
|
|
"GoHomeDate": gohome_val,
|
|
"Genotype": genotype_val,
|
|
"Notes": None,
|
|
"ImportSource": "Wurfchronik-Detail.docx",
|
|
"ExternalRef": f"docx-{idx}-{normalize_name(name_val)}-{dob_val or '0001-01-01'}",
|
|
"RawImportData": raw_import_payload,
|
|
"OriginBreeder": "Zucht der kleinen Chaoten",
|
|
"NameSearch": normalize_name(name_val),
|
|
"CharacterTraits": [],
|
|
"CharacterNote": None,
|
|
"IsDeaf": None,
|
|
"IsResident": is_resident,
|
|
"_photos": da.get("photos", []),
|
|
"_old_scoped_litter_id": scoped_litter_id,
|
|
"_eff_dob": dob_val or "2020-01-01",
|
|
"_birth_date": dob_val,
|
|
"_filename": "Wurfchronik-Detail.docx",
|
|
"_old_id": name_val
|
|
})
|
|
|
|
# Build parenting dates lookup using old scoped IDs
|
|
parent_litter_dates = {}
|
|
|
|
for l in resolved_litters:
|
|
ld = l["Date"]
|
|
if ld and ld != "0001-01-01":
|
|
for pid in [l["FatherId"], l["MotherId"]]:
|
|
if pid:
|
|
parent_litter_dates.setdefault(pid, []).append(ld)
|
|
|
|
def are_compatible(g1, g2):
|
|
# Must have same gender (or one unknown)
|
|
if g1["Gender"] != "unknown" and g2["Gender"] != "unknown" and g1["Gender"] != g2["Gender"]:
|
|
bd1 = g1.get("_birth_date")
|
|
bd2 = g2.get("_birth_date")
|
|
if not (bd1 and bd2 and bd1 == bd2):
|
|
return False
|
|
|
|
bd1 = g1.get("_birth_date")
|
|
bd2 = g2.get("_birth_date")
|
|
|
|
# If both have explicit birth dates, they must match within 30 days
|
|
if bd1 and bd2:
|
|
days1 = date_to_days(bd1)
|
|
days2 = date_to_days(bd2)
|
|
if days1 is not None and days2 is not None:
|
|
if abs(days1 - days2) > 30:
|
|
return False
|
|
|
|
# If g1 has birth date, and g2 has parenting dates, birth date must be before parenting dates
|
|
p_dates2 = parent_litter_dates.get(g2["Id"], [])
|
|
if bd1:
|
|
for pd in p_dates2:
|
|
if pd <= bd1: # Can't have litter before or on birth date
|
|
return False
|
|
|
|
p_dates1 = parent_litter_dates.get(g1["Id"], [])
|
|
if bd2:
|
|
for pd in p_dates1:
|
|
if pd <= bd2:
|
|
return False
|
|
|
|
return True
|
|
|
|
# Group gerbils by name to perform deduplication
|
|
gerbil_groups = {}
|
|
for g in all_processed_gerbils:
|
|
name_key = get_dedup_name_key(g["Name"])
|
|
if not name_key:
|
|
name_key = "unbekannt"
|
|
gerbil_groups.setdefault(name_key, []).append(g)
|
|
|
|
resolved_gerbils = []
|
|
gerbil_id_map = {} # old_scoped_id -> final_id
|
|
|
|
color_keys = set(variety_map.keys())
|
|
|
|
for name_key, group in gerbil_groups.items():
|
|
is_placeholder = (
|
|
name_key in color_keys or
|
|
any(p in name_key for p in ["unbekannt", "unbenannt", "baby", "welpe", "jungtier", "unknown", "welpen"]) or
|
|
len(name_key) <= 2
|
|
)
|
|
|
|
if is_placeholder:
|
|
# Placeholders: do NOT merge, keep all separate
|
|
for g in group:
|
|
resolved_gerbils.append(g)
|
|
gerbil_id_map[g["Id"]] = g["Id"]
|
|
continue
|
|
|
|
# Partition group into compatible subsets
|
|
sub_groups = []
|
|
for g in group:
|
|
placed = False
|
|
for sub in sub_groups:
|
|
if all(are_compatible(g, member) for member in sub):
|
|
sub.append(g)
|
|
placed = True
|
|
break
|
|
if not placed:
|
|
sub_groups.append([g])
|
|
|
|
# Merge each partition sub-group into a single gerbil
|
|
for sub in sub_groups:
|
|
if len(sub) == 1:
|
|
g = sub[0]
|
|
resolved_gerbils.append(g)
|
|
gerbil_id_map[g["Id"]] = g["Id"]
|
|
continue
|
|
|
|
# Find the best primary record to merge into
|
|
best_g = None
|
|
best_score = -1
|
|
for g in sub:
|
|
score = 0
|
|
if g["_old_scoped_litter_id"]: score += 10
|
|
if g["DateOfBirth"]: score += 5
|
|
if g["Genotype"]: score += 3
|
|
if g["ColorVarietyId"]: score += 2
|
|
if g["ImportSource"] and "stammbaum" in g["ImportSource"].lower(): score += 20
|
|
if g["Notes"] and not any(kw in g["Notes"].lower() for kw in ["parent", "mutter", "vater", "dam", "sire"]): score += 1
|
|
if len(g["Name"]) > len(name_key) + 5: # likely has clan suffix
|
|
score += 15
|
|
if score > best_score:
|
|
best_score = score
|
|
best_g = g
|
|
|
|
# Merge fields
|
|
merged_notes = []
|
|
if best_g["Notes"]:
|
|
merged_notes.append(best_g["Notes"])
|
|
|
|
# Merge photos
|
|
merged_photos = list(best_g.get("_photos", []))
|
|
|
|
# Track sources for debugging
|
|
sources = [best_g["_filename"]]
|
|
|
|
for g in sub:
|
|
if g == best_g:
|
|
continue
|
|
gerbil_id_map[g["Id"]] = best_g["Id"]
|
|
sources.append(g["_filename"])
|
|
|
|
for ph in g.get("_photos", []):
|
|
if ph not in merged_photos:
|
|
merged_photos.append(ph)
|
|
|
|
if not best_g["LitterId"] and g["LitterId"]:
|
|
best_g["LitterId"] = g["LitterId"]
|
|
if not best_g["DateOfBirth"] and g["DateOfBirth"]:
|
|
best_g["DateOfBirth"] = g["DateOfBirth"]
|
|
if not best_g["DateOfDeath"] and g["DateOfDeath"]:
|
|
best_g["DateOfDeath"] = g["DateOfDeath"]
|
|
if not best_g["CauseOfDeath"] and g["CauseOfDeath"]:
|
|
best_g["CauseOfDeath"] = g["CauseOfDeath"]
|
|
if not best_g["GoHomeDate"] and g["GoHomeDate"]:
|
|
best_g["GoHomeDate"] = g["GoHomeDate"]
|
|
if not best_g["Genotype"] and g["Genotype"]:
|
|
best_g["Genotype"] = g["Genotype"]
|
|
if not best_g["ColorVarietyId"] and g["ColorVarietyId"]:
|
|
best_g["ColorVarietyId"] = g["ColorVarietyId"]
|
|
if not best_g["OriginContactId"] and g["OriginContactId"]:
|
|
best_g["OriginContactId"] = g["OriginContactId"]
|
|
if not best_g["ReceiverContactId"] and g["ReceiverContactId"]:
|
|
best_g["ReceiverContactId"] = g["ReceiverContactId"]
|
|
if g["IsResident"]:
|
|
best_g["IsResident"] = True
|
|
|
|
# Reconcile Gender: prefer a known gender over unknown, and prefer stammbaum over other sources
|
|
if best_g["Gender"] == "unknown" and g["Gender"] != "unknown":
|
|
best_g["Gender"] = g["Gender"]
|
|
elif best_g["Gender"] != "unknown" and g["Gender"] != "unknown" and best_g["Gender"] != g["Gender"]:
|
|
if g["ImportSource"] and "stammbaum" in g["ImportSource"].lower():
|
|
if not best_g["ImportSource"] or "stammbaum" not in best_g["ImportSource"].lower():
|
|
best_g["Gender"] = g["Gender"]
|
|
|
|
# Status precedence: Deceased > GivenAway > Breeding/Pet
|
|
if g["Status"] == "Deceased":
|
|
best_g["Status"] = "Deceased"
|
|
elif g["Status"] == "GivenAway" and best_g["Status"] not in ["Deceased"]:
|
|
best_g["Status"] = "GivenAway"
|
|
|
|
if g["Notes"] and g["Notes"] not in merged_notes:
|
|
# Ignore redundant dummy notes
|
|
if not any(kw in g["Notes"].lower() for kw in ["parent listed", "mutter von", "vater von", "dam of", "sire of"]):
|
|
merged_notes.append(g["Notes"])
|
|
|
|
|
|
if merged_notes:
|
|
best_g["Notes"] = " | ".join(merged_notes)
|
|
|
|
best_g["_photos"] = merged_photos
|
|
|
|
# Print merge trace
|
|
print(f"Deduplicated same-animal name '{best_g['Name']}': merged {len(sub)} entries across files: {', '.join(sources)}")
|
|
|
|
resolved_gerbils.append(best_g)
|
|
gerbil_id_map[best_g["Id"]] = best_g["Id"]
|
|
|
|
print(f"Deduplicated to {len(resolved_gerbils)} unique gerbil records.")
|
|
|
|
# 5. Map Gerbils to Litters
|
|
for g in resolved_gerbils:
|
|
old_lid = g["_old_scoped_litter_id"]
|
|
if old_lid in litter_id_map:
|
|
g["LitterId"] = litter_id_map[old_lid]
|
|
else:
|
|
g["LitterId"] = None
|
|
# Clean helper fields
|
|
del g["_old_scoped_litter_id"]
|
|
del g["_eff_dob"]
|
|
if "_birth_date" in g:
|
|
del g["_birth_date"]
|
|
del g["_filename"]
|
|
del g["_old_id"]
|
|
|
|
# Gather final valid gerbil IDs
|
|
valid_gerbil_ids = {g["Id"] for g in resolved_gerbils}
|
|
|
|
# Create name lookup for resolved gerbils
|
|
gerbil_by_norm_name = {}
|
|
for g in resolved_gerbils:
|
|
n_key = normalize_name(g["Name"])
|
|
gerbil_by_norm_name.setdefault(n_key, []).append(g)
|
|
|
|
# Also index by call-name to resolve parents who are only listed by call-name
|
|
c_key = normalize_name(get_call_name(g["Name"]))
|
|
if c_key != n_key:
|
|
gerbil_by_norm_name.setdefault(c_key, []).append(g)
|
|
|
|
# Map raw Guid if present (convert if old_id mapped to new_guid)
|
|
for l in resolved_litters:
|
|
if l["FatherId"] in gerbil_id_map:
|
|
l["FatherId"] = gerbil_id_map[l["FatherId"]]
|
|
if l["MotherId"] in gerbil_id_map:
|
|
l["MotherId"] = gerbil_id_map[l["MotherId"]]
|
|
|
|
# Clean foreign keys that do not point to a valid gerbil
|
|
if l["FatherId"] and l["FatherId"] not in valid_gerbil_ids:
|
|
l["FatherId"] = None
|
|
if l["MotherId"] and l["MotherId"] not in valid_gerbil_ids:
|
|
l["MotherId"] = None
|
|
|
|
# Parent Resolver (Global Name Matching)
|
|
resolved_fathers = 0
|
|
resolved_mothers = 0
|
|
|
|
for l in resolved_litters:
|
|
# Match Father by Name
|
|
f_name = l["_father_name"]
|
|
if f_name and not l["FatherId"]:
|
|
f_norm = normalize_name(f_name)
|
|
candidates = gerbil_by_norm_name.get(f_norm, [])
|
|
valid_candidates = []
|
|
for c in candidates:
|
|
# Map to final deduplicated ID
|
|
final_id = gerbil_id_map.get(c["Id"])
|
|
if not final_id:
|
|
continue
|
|
# Retrieve final record
|
|
final_c = next((rg for rg in resolved_gerbils if rg["Id"] == final_id), None)
|
|
if final_c and final_c["Gender"] in ["male", "unknown"]:
|
|
# Ensure parent is born before litter if birth date is known
|
|
if l["Date"] != "0001-01-01" and final_c["DateOfBirth"]:
|
|
if final_c["DateOfBirth"] < l["Date"]:
|
|
valid_candidates.append(final_c)
|
|
else:
|
|
valid_candidates.append(final_c)
|
|
|
|
if len(valid_candidates) == 1:
|
|
l["FatherId"] = valid_candidates[0]["Id"]
|
|
resolved_fathers += 1
|
|
elif len(valid_candidates) > 1:
|
|
l["FatherId"] = valid_candidates[0]["Id"]
|
|
resolved_fathers += 1
|
|
|
|
# Match Mother by Name
|
|
m_name = l["_mother_name"]
|
|
if m_name and not l["MotherId"]:
|
|
m_norm = normalize_name(m_name)
|
|
candidates = gerbil_by_norm_name.get(m_norm, [])
|
|
valid_candidates = []
|
|
for c in candidates:
|
|
final_id = gerbil_id_map.get(c["Id"])
|
|
if not final_id:
|
|
continue
|
|
final_c = next((rg for rg in resolved_gerbils if rg["Id"] == final_id), None)
|
|
if final_c and final_c["Gender"] in ["female", "unknown"]:
|
|
if l["Date"] != "0001-01-01" and final_c["DateOfBirth"]:
|
|
if final_c["DateOfBirth"] < l["Date"]:
|
|
valid_candidates.append(final_c)
|
|
else:
|
|
valid_candidates.append(final_c)
|
|
|
|
if len(valid_candidates) == 1:
|
|
l["MotherId"] = valid_candidates[0]["Id"]
|
|
resolved_mothers += 1
|
|
elif len(valid_candidates) > 1:
|
|
l["MotherId"] = valid_candidates[0]["Id"]
|
|
resolved_mothers += 1
|
|
|
|
# Cleanup internal keys
|
|
del l["_father_name"]
|
|
del l["_mother_name"]
|
|
del l["_filename"]
|
|
|
|
print(f"Globally resolved {resolved_fathers} fathers and {resolved_mothers} mothers.")
|
|
|
|
# Set IsBreeder and IsReceiver flags on contacts
|
|
breeder_ids = {g["OriginContactId"] for g in resolved_gerbils if g.get("OriginContactId")}
|
|
receiver_ids = {g["ReceiverContactId"] for g in resolved_gerbils if g.get("ReceiverContactId")}
|
|
for c in resolved_contacts:
|
|
c_id = c["Id"]
|
|
is_breeder = c_id in breeder_ids
|
|
is_receiver = c_id in receiver_ids
|
|
if not is_breeder and not is_receiver:
|
|
is_receiver = True
|
|
c["IsBreeder"] = is_breeder
|
|
c["IsReceiver"] = is_receiver
|
|
|
|
# Set and map gerbilPhotos
|
|
resolved_photos = []
|
|
for g in resolved_gerbils:
|
|
for idx, photo_rel in enumerate(g.get("_photos", [])):
|
|
photo_guid = generate_guid(f"photo-{photo_rel}")
|
|
ext = os.path.splitext(photo_rel)[1] or ".jpeg"
|
|
fn_guid = photo_guid.replace("-", "")
|
|
resolved_photos.append({
|
|
"Id": photo_guid,
|
|
"GerbilId": g["Id"],
|
|
"FileName": f"{fn_guid}{ext}",
|
|
"SortOrder": idx,
|
|
"_source_path": photo_rel
|
|
})
|
|
|
|
# 6. Save final output JSON payload
|
|
os.makedirs(OUTPUT_DIR, exist_ok=True)
|
|
payload = {
|
|
"contacts": resolved_contacts,
|
|
"litters": resolved_litters,
|
|
"gerbils": resolved_gerbils,
|
|
"gerbilPhotos": resolved_photos
|
|
}
|
|
|
|
with open(OUTPUT_FILE, "w", encoding="utf-8") as f:
|
|
json.dump(payload, f, ensure_ascii=False, indent=2)
|
|
|
|
print(f"Successfully wrote database-ready import file to: {OUTPUT_FILE}")
|
|
print(f" Contacts: {len(payload['contacts'])}")
|
|
print(f" Litters: {len(payload['litters'])}")
|
|
print(f" Gerbils: {len(payload['gerbils'])}")
|
|
print(f" Photos: {len(payload['gerbilPhotos'])}")
|
|
|
|
if __name__ == "__main__":
|
|
main()
|