Gulum
5e14124322
Neuer Parser extract_contracts.py liest die ~1,4k Abgabevertrags-DOCX (\truenas\…\Verträge): er extrahiert aus dem Dokument-Body (zuverlässiger als die Dateinamen) Käufer, Tier(e), Farbschlag, Abgabedatum und Preis — robust gegen Word-Run-Splits (z. B. „F r au"/„3 0,00"); überspringt Vorlage, Abstammungsnachweise und als .docx getarnte .doc. enrich_from_contracts() in merge_and_resolve.py: Käufer werden als Kontakte (IsReceiver) angelegt/zusammengeführt; Tiere werden KONSERVATIV per Rufname (+ DOB-Jahr bei Mehrdeutigkeit) auf eigene Bestandstiere gematcht und erhalten ReceiverContactId, GoHomeDate und Status „abgegeben" — nur wo nicht bereits gesetzt; Konflikte werden geloggt, nicht überschrieben. Jede Übernahme bekommt eine Herkunfts-Zeile („Abgabe an … aus Vertrag … übernommen."). Ergebnis: 1095 Verträge → 783 Tier-Treffer (400 mehrdeutige übersprungen), 274 neue Abnehmer-Kontakte, 153 Tiere mit Abnehmer, 49 mit Abgabedatum, 23 neu „abgegeben". Keine Backend-/Frontend-Änderung nötig (Akte zeigt Abnehmer/ Abgabedatum/Herkunft bereits). SaleContract-Records bewusst nicht erzeugt (bräuchte Migration + ingest-sichere Id — späterer Schritt). Tests: test_extract_contracts.py (Dateiname/Body/Run-Split/Skip-Regeln) + alle bestehenden grün; dotnet 212. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
3006 lines
131 KiB
Python
3006 lines
131 KiB
Python
import os
|
||
import json
|
||
import re
|
||
import uuid
|
||
import sys
|
||
from datetime import datetime
|
||
|
||
# Prevent encoding crashes on Windows consoles when printing unicode
|
||
if sys.platform.startswith('win'):
|
||
try:
|
||
sys.stdout.reconfigure(encoding='utf-8')
|
||
except Exception:
|
||
pass
|
||
|
||
# Paths
|
||
DIR_PATH = r"C:\Users\gulum\dev\Wurfchronik_Bilder"
|
||
SEEDS_PATH = r"C:\Users\gulum\dev\GerbilManager\gerbil-manager-web\src\genetics\colorVarietySeed.backend.json"
|
||
OUTPUT_DIR = r"C:\Users\gulum\dev\GerbilManager\tools\import\output"
|
||
OUTPUT_FILE = os.path.join(OUTPUT_DIR, "resolved_import.json")
|
||
|
||
def generate_guid(key_str):
|
||
"""Generate a stable UUID string based on a key."""
|
||
return str(uuid.uuid5(uuid.NAMESPACE_DNS, key_str))
|
||
|
||
def build_entity_provenance(source_files, merged_record_count, notes=None,
|
||
from_wurfchronik=None, extra=None, history=None):
|
||
"""Generic data-provenance JSON builder shared by gerbils, contacts and
|
||
litters. Mirrors the GerbilProvenance frontend contract:
|
||
{ sourceFiles, mergedRecordCount, fromWurfchronik, notes, history, ... }
|
||
`source_files` is any iterable of filenames; `from_wurfchronik` is auto-
|
||
derived from the filenames when left as None. `extra` may carry entity-
|
||
specific keys (e.g. parentMethod/parentConfidence for gerbils). `history`
|
||
is an ordered list of human-readable German lines that read like a
|
||
chronological log of where each fact came from (the primary content shown
|
||
in the Datenherkunft dialog). Returns a JSON string (stored on the nullable
|
||
Provenance text column)."""
|
||
files = sorted({f for f in source_files if f})
|
||
if from_wurfchronik is None:
|
||
from_wurfchronik = any("wurfchronik" in f.lower() for f in files)
|
||
prov = {
|
||
"sourceFiles": files,
|
||
"mergedRecordCount": merged_record_count,
|
||
"fromWurfchronik": bool(from_wurfchronik),
|
||
"notes": list(notes or []),
|
||
"history": list(history or []),
|
||
}
|
||
if extra:
|
||
for k, v in extra.items():
|
||
if v is not None:
|
||
prov[k] = v
|
||
return json.dumps(prov, ensure_ascii=False)
|
||
|
||
|
||
def _quote_file(fname):
|
||
"""German typographic quotes around a source filename for history lines."""
|
||
return f"„{fname}“"
|
||
|
||
|
||
# Human-readable German labels for the significant fields we attribute to files.
|
||
PROV_FIELD_LABELS = {
|
||
"DateOfBirth": "Geburtsdatum",
|
||
"DateOfDeath": "Sterbedatum",
|
||
"Gender": "Geschlecht",
|
||
"Genotype": "Genotyp",
|
||
"ColorVarietyId": "Farbschlag",
|
||
"Name": "Name",
|
||
}
|
||
|
||
|
||
def _primary_file_of(record):
|
||
"""The single most representative source file of one raw record.
|
||
|
||
Prefers an explicit Stammbaum/Wurfchronik filename from ImportSource, else
|
||
the record's _filename. Used to attribute a field value to a concrete file
|
||
in the history log."""
|
||
fn = record.get("_filename")
|
||
if fn:
|
||
return fn
|
||
imp = record.get("ImportSource")
|
||
if imp:
|
||
first = str(imp).split(",")[0].strip()
|
||
if first:
|
||
return first
|
||
return None
|
||
|
||
|
||
def _record_source_files(g):
|
||
"""All distinct source files a single raw gerbil record drew from.
|
||
|
||
ImportSource is either a comma-joined Stammbaum file list, a single
|
||
Wurfchronik filename, or the per-litter filename; _filename is the primary
|
||
file. We union both so nothing is lost."""
|
||
files = set()
|
||
imp = g.get("ImportSource")
|
||
if imp:
|
||
for part in str(imp).split(","):
|
||
part = part.strip()
|
||
if part:
|
||
files.add(part)
|
||
fn = g.get("_filename")
|
||
if fn:
|
||
files.add(fn)
|
||
return files
|
||
|
||
def to_valid_guid(val):
|
||
if not val:
|
||
return None
|
||
val_str = str(val).strip()
|
||
try:
|
||
uuid.UUID(val_str)
|
||
return val_str
|
||
except ValueError:
|
||
return generate_guid(val_str)
|
||
|
||
def normalize_name(name):
|
||
if not name:
|
||
return ""
|
||
return "".join(c for c in name.lower() if c.isalnum())
|
||
|
||
def get_normalized_contact_name(name):
|
||
if not name:
|
||
return "", False
|
||
|
||
n = "".join(c for c in name.lower() if c.isalnum())
|
||
|
||
to_discard = {
|
||
"chevroletcamarooftopolino",
|
||
"cindyvprivatzuchtgießen",
|
||
"cindyvprivatzuchtgiessen",
|
||
"inuschofblackforest",
|
||
"stichvonprivatzuchtgießen",
|
||
"stichvonprivatzuchtgiessen",
|
||
"tarzanofsamsimar",
|
||
"hanserennersposeidon",
|
||
"livingforcesidefix",
|
||
"livingforcesnando"
|
||
}
|
||
if n in to_discard:
|
||
return "", False
|
||
|
||
norm_map = {
|
||
"alessandrab": "Alessandra Bartoletti",
|
||
"alexsandrab": "Alessandra Bartoletti",
|
||
"andreafey": "Andrea und Stefanie Fey",
|
||
"andreastefaniefey": "Andrea und Stefanie Fey",
|
||
"anettw": "Annett Wernecke",
|
||
"angelachristoph": "Angela und Ekki Christoph",
|
||
"angelaekkichristoph": "Angela und Ekki Christoph",
|
||
"angie": "Angie Reichert-Cambeis",
|
||
"angierc": "Angie Reichert-Cambeis",
|
||
"anjasch": "Anja Schaumburg",
|
||
"ankeb": "Anke Busch",
|
||
"ankeksch": "Anke Koppenhöfer",
|
||
"annaq": "Anna Quark",
|
||
"annikab": "Annika Balser",
|
||
"annkathrind": "Ann-kathrin Dressler",
|
||
"arturaleksandrapolamundrzynski": "Artur, Aleksandra und Pola Mundrzynski",
|
||
"arturaleksandraundpolamundrzynski": "Artur, Aleksandra und Pola Mundrzynski",
|
||
"astridr": "Astrid Rohmann",
|
||
"astridroh": "Astrid Rohmann",
|
||
"babarastehle": "Barbara Stehle",
|
||
"barbarar": "Barbara Riegler",
|
||
"biancab": "Bianca Bernhardt",
|
||
"biancam": "Bianca Mayer + Alex Wachten",
|
||
"biancamayeralexw": "Bianca Mayer + Alex Wachten",
|
||
"birgitk": "Birgit Knopp",
|
||
"birgitkropp": "Birgit Knopp",
|
||
"birgittabüskens": "Birgitta Müller-Büskens",
|
||
"birgittamüllerbüskens": "Brigitta Müller-Büskens",
|
||
"blackforestgv": "Black Forest",
|
||
"brigittast": "Brigitta Struve",
|
||
"buntefellnasen": "bunten Fellnasen",
|
||
"buntenfellnase": "bunten Fellnasen",
|
||
"buntenfellnasen": "bunten Fellnasen",
|
||
"carolal": "Carola und Hannah Lerch",
|
||
"carolalerch": "Carola und Hannah Lerch",
|
||
"chalfontstud": "Chalfont Stud, Freddy Braun",
|
||
"christianefuchs": "Christiane Fuchs + Michael Höhnert",
|
||
"christiek": "Christiane Kiessling",
|
||
"christinam": "Christine Monika Mai",
|
||
"christinek": "Christiane Kiessling",
|
||
"christiner": "Christina Rudloff",
|
||
"claudialöhr": "Claudia Löhr und Michael Koob",
|
||
"claudialöhrmichaelkoob": "Claudia Löhr und Michael Koob",
|
||
"colorfulfurrygerbils": "Colorful Furry Gerbils",
|
||
"colourfulfurrygerbils": "Colorful Furry Gerbils",
|
||
"danielakasberger": "Daniela Käsberger",
|
||
"dörthe": "Dörthe Petzmann",
|
||
"emelyhehlhorn": "Emely Mehlhorn",
|
||
"fabianb": "Fabian Büdel",
|
||
"floriang": "Florian Gries",
|
||
"hannahraths": "Hannah Ruths",
|
||
"hanserenners": "Hanse Renner",
|
||
"haraldmariareitzlennemann": "Harald und Maria Reitz-Lennemann",
|
||
"haraldundmariarl": "Harald und Maria Reitz-Lennemann",
|
||
"heike": "Heike + Heiko Scheurich",
|
||
"heikebarklay": "Heike Barklage",
|
||
"heikoloos": "Heiko Loos und Annette Becker",
|
||
"heikoloosannettebecker": "Heiko Loos und Annette Becker",
|
||
"inahübner": "Ina Hübner + Nicolai Thome",
|
||
"inapeines": "Ina Pleines",
|
||
"janab": "Jana Beikert",
|
||
"jasiminweber": "Jasmin Weber und Ronny Reichelt",
|
||
"jasmin": "Jasmin Weber",
|
||
"jasminweber": "Jasmin Weber und Ronny Reichelt",
|
||
"jasminweberromyreichelt": "Jasmin Weber und Ronny Reichelt",
|
||
"jasminweberronnyreichelt": "Jasmin Weber und Ronny Reichelt",
|
||
"joystaatberg": "Joy Staalberg",
|
||
"juliaa": "Julia Ast",
|
||
"juttam": "Jutta Metz",
|
||
"karina": "Karina Luft",
|
||
"karinborgotti": "Karin Borsotti",
|
||
"katjal": "Katja Leffeck",
|
||
"kimw": "Kim Waldschmidt",
|
||
"kkchaos": "KK-Chaos",
|
||
"kkchaosofkkchaos": "KK-Chaos",
|
||
"kleinechaoten": "Zucht der kleinen Chaoten",
|
||
"kleinenchaoten": "Zucht der kleinen Chaoten",
|
||
"kriegernmitkrallen": "Krieger mit Krallen",
|
||
"lennylengo": "Lenny Lengo",
|
||
"lisa": "Lisa und Marcel Kunz",
|
||
"lisach": "Lisa und Lydia Christ",
|
||
"lisachrist": "Lisa und Lydia Christ",
|
||
"lisalydiachrist": "Lisa und Lydia Christ",
|
||
"littlefellows": "little fellows",
|
||
"littlerunners": "little runners",
|
||
"maikef": "Maike Franz",
|
||
"maintalerpz": "Privatzucht Maintal",
|
||
"marinau": "Marina Unger",
|
||
"martinaandreaswestfeld": "Martina und Andreas Westfeld",
|
||
"martinaw": "Martina und Andreas Westfeld",
|
||
"martinawestfeld": "Martina und Andreas Westfeld",
|
||
"meikesch": "Meike und Heiko Sch.",
|
||
"melanief": "Melanie Fey und Thorben Meier",
|
||
"melaniefey": "Melanie Fey und Thorben Meier",
|
||
"melaniefeythorbenmeier": "Melanie Fey und Thorben Meier",
|
||
"michaeldavidschmitz": "Michael und David Schmitz",
|
||
"michaelschmitz": "Michael und David Schmitz",
|
||
"nataliereitz": "Nathalie Reitz",
|
||
"natascham": "Natascha Marienfeld",
|
||
"nicolefischler": "Nicole Tischler",
|
||
"nicolel": "Nicole Lannert",
|
||
"nicolen": "Nicole Nuzzo",
|
||
"nicolet": "Nicole Tischler",
|
||
"nicolew": "Nicole Webersinn",
|
||
"nicost": "Nico Stamm",
|
||
"nielsh": "Nils H.",
|
||
"noelstrahbach": "Noel Strohbach",
|
||
"nora": "Nora Rudersdorf geb. Holzbach",
|
||
"norah": "Nora Holzbach",
|
||
"norarh": "Nora Rudersdorf",
|
||
"norarudersdorf": "Nora Rudersdorf geb. Holzbach",
|
||
"oflennylengo": "Lenny Lengo",
|
||
"pascale": "Pascale Diefenbach",
|
||
"pascaledießenbach": "Pascale Diefenbach",
|
||
"patriciap": "Patricia Petry",
|
||
"paul": "Paul W.",
|
||
"paula": "Paula Gabler",
|
||
"paulag": "Paula Gabler",
|
||
"privatzuchtmuecke": "Privatzucht Mücke",
|
||
"privatzuchtmücke": "Privatzucht Mücke",
|
||
"pzmaintal": "Privatzucht Maintal",
|
||
"pzmuecke": "Privatzucht Mücke",
|
||
"pzmücke": "Privatzucht Mücke",
|
||
"pzseligenstadt": "Privatzucht Seligenstadt",
|
||
"ramonag": "Ramona Gömpel",
|
||
"ranialößler": "Rania Löffler",
|
||
"rominahubrich": "Romina Milde / Hubrich",
|
||
"rominamilde": "Romina Milde / Hubrich",
|
||
"romonag": "Ramona Gömpel",
|
||
"ronialöffler": "Rania Löffler",
|
||
"sandrak": "Sandra Kubas",
|
||
"sarahlöser": "Sarah Löwer",
|
||
"sarahlöuer": "Sarah Löwer",
|
||
"sarahz": "Sarah Zitzer",
|
||
"saskiaw": "Saskia Wucher",
|
||
"schlossmaus": "Schlossmäuse",
|
||
"schlossmäuse": "Schlossmäuse",
|
||
"schlossmäusen": "Schlossmäuse",
|
||
"sebastiansch": "Sebastian Schmitt",
|
||
"serinaberg": "Selina Berg",
|
||
"silkewolfganghintze": "Silke und Wolfgang Hintze",
|
||
"smilla": "Smilla H.",
|
||
"steffi": "Steffi K.",
|
||
"susanneninat": "Susanne + Nina Thomas",
|
||
"susannet": "Susanne + Torsten Saum",
|
||
"theresavalenca": "Theresia Valenca (Kopp)",
|
||
"theresiavalenca": "Theresia Valenca (Kopp)",
|
||
"timpf": "Tim Pfeiffer und Michelle Mai",
|
||
"timpfeifermichellemai": "Tim Pfeiffer und Michelle Mai",
|
||
"timpfeiffer": "Tim Pfeiffer und Michelle Mai",
|
||
"timpfeiffermichellemai": "Tim Pfeiffer und Michelle Mai",
|
||
"tonoböckenseld": "Tono Böckenfeld",
|
||
"topol": "Topolino",
|
||
"ulrichmüller": "Ulrich + Angela Müller",
|
||
"ulrikec": "Ulrike Cordes",
|
||
"ulriker": "Ulrike Ruppel",
|
||
"ulrikesch": "Ulrike Schulz",
|
||
"ulriket": "Ulrike Treml",
|
||
"vanessab": "Vanessa Becker",
|
||
"vanessag": "Vanessa Groll",
|
||
"veragreywitz": "Vera Geywitz",
|
||
"vonprivatzuchtmaintal": "Privatzucht Maintal",
|
||
"wolfgangfaus": "Wolfgang Faust",
|
||
"yvonne": "Yvonne Obendorfer",
|
||
"zuchtderkleinenchaoten": "Zucht der kleinen Chaoten",
|
||
}
|
||
|
||
if n in norm_map:
|
||
return norm_map[n], True
|
||
|
||
return name, True
|
||
|
||
def get_normalized_gerbil_name(name):
|
||
if not name:
|
||
return ""
|
||
n = name.strip()
|
||
norm_key = "".join(c for c in n.lower() if c.isalnum())
|
||
|
||
gerbil_norm_map = {
|
||
"samgenshellyvdbuntenfellnasen": "Sammy gen. Shelly von den bunten Fellnasen",
|
||
"sammygenshellyvdbuntenfellnasen": "Sammy gen. Shelly von den bunten Fellnasen",
|
||
"schmidt": "Schmidti",
|
||
"sheila": "Sheila of Ulmer Strolche",
|
||
"shinichi": "Shinichi von PZ Mücke",
|
||
"silenosgenadonis": "Silenos gen. Adonis von den Kleinen Chaoten",
|
||
"silenosgenadonisvdkleinenchaoten": "Silenos gen. Adonis von den Kleinen Chaoten",
|
||
"silver": "Silver von den kleinen Chaoten",
|
||
"snoops": "Snoopsi",
|
||
"sokrates": "Sokrates von den Kleinen Chaoten",
|
||
"splash": "Slash",
|
||
"teiko": "Teiko von den kleinen Chaoten",
|
||
"trixy": "Trixxy von den Kleinen Chaoten",
|
||
"unique": "Unique of Wild Dreams",
|
||
}
|
||
|
||
if norm_key in gerbil_norm_map:
|
||
return gerbil_norm_map[norm_key]
|
||
|
||
return n
|
||
|
||
def get_call_name(name):
|
||
if not name:
|
||
return ""
|
||
n = name.strip()
|
||
n = re.sub(r'\[[^\]]+\]$', '', n).strip()
|
||
n = re.split(r'\s+(?:of|von\s+den|von\s+der|v\.\s?d\.|von)\s+', n, flags=re.IGNORECASE)[0].strip()
|
||
return n
|
||
|
||
def get_dedup_name_key(name):
|
||
if not name:
|
||
return ""
|
||
n = name.lower().strip()
|
||
# Strip common suffixes/prefixes and parentheticals
|
||
n = re.sub(r'\b(?:von\s+den|v\.?\s*d\.?|v\.?o\.?)\s+(?:kleinen\s+)?chaoten\b', '', n)
|
||
n = re.sub(r'\bvon\s+der\s+schlossm\w+\b', '', n)
|
||
n = re.sub(r'\bvon\s+der\s+bunten\s+fellnasen?\b', '', n)
|
||
n = re.sub(r'\bof\s+black\s+forest\b', '', n)
|
||
n = re.sub(r'\b\(?rv\)?\b', '', n)
|
||
n = re.sub(r'\bgen\.\s+\w+', '', n)
|
||
# Clean up parentheses or brackets
|
||
n = re.sub(r'\(.*?\)', '', n)
|
||
n = re.sub(r'\[.*?\]', '', n)
|
||
return "".join(c for c in n if c.isalnum())
|
||
|
||
def clean_color_name(c_desc):
|
||
if not c_desc:
|
||
return "", False
|
||
|
||
# Lowercase and strip
|
||
c = c_desc.lower().strip()
|
||
|
||
# Check for Schecke
|
||
is_schecke = False
|
||
if re.search(r'\bsp\b|\bsp\d|\bsp[*(²³]|\bspotted|\bschecke|[- ]sp\b|\w+sp\b', c):
|
||
is_schecke = True
|
||
|
||
# Standardize parentheticals for schimmel
|
||
c = c.replace("(schimmel)", "schimmel")
|
||
c = c.replace("(schimmel-hell)", "schimmel hell")
|
||
c = c.replace("(schimmel hell)", "schimmel hell")
|
||
|
||
# Strip schecke/sp markers and any trailing text starting from sp
|
||
c = re.sub(r'\([- ]?sp(otted)?\)', '', c) # handles (-sp)
|
||
c = re.sub(r'[- ]?sp(otted)?\b.*', '', c) # handles -sp(k), -sp*(k), -sp, etc.
|
||
c = re.sub(r'[- ]?schecke\b.*', '', c)
|
||
c = re.sub(r'[- ]?spotted\b.*', '', c)
|
||
|
||
# Strip any other parentheticals, symbols, or trailing stars/numbers
|
||
c = re.sub(r'\s*\(.*?\)\s*', ' ', c)
|
||
c = re.sub(r'[²³*]', '', c)
|
||
c = c.strip()
|
||
|
||
# Mapping table for abbreviations, typos, and specific combinations
|
||
mapping = {
|
||
"antra": "anthrazit",
|
||
"anthra": "anthrazit",
|
||
"ankazit": "anthrazit",
|
||
"antrazit": "anthrazit",
|
||
"pew": "rew",
|
||
"bew": "hermelin",
|
||
"harder": "marder",
|
||
"kohli": "kohlfuchs",
|
||
"aligerfuchs": "algierfuchs",
|
||
"algiesfuchs": "algierfuchs",
|
||
"schw": "schwarz",
|
||
"sa": "silberagouti",
|
||
"a": "agouti",
|
||
"cp-sa": "cp-silberagouti",
|
||
"cp-a": "cp-agouti",
|
||
"cp-a-hell": "cp-agouti-hell",
|
||
"cp-aisa": "cp-agouti",
|
||
"cp-a / rcp-sa": "cp-agouti",
|
||
}
|
||
|
||
if c in mapping:
|
||
c = mapping[c]
|
||
|
||
return c, is_schecke
|
||
|
||
def resolve_color_and_genotype(color_val, existing_genotype, variety_map, variety_genotypes):
|
||
if not color_val:
|
||
return None, existing_genotype
|
||
color_str = str(color_val).strip()
|
||
clean_name, is_schecke = clean_color_name(color_str)
|
||
# Match color in variety_map
|
||
color_variety_id = None
|
||
if clean_name in variety_map:
|
||
color_variety_id = variety_map[clean_name]
|
||
else:
|
||
for seed_name, seed_id in variety_map.items():
|
||
if seed_name in clean_name or clean_name in seed_name:
|
||
color_variety_id = seed_id
|
||
break
|
||
# Update genotype if it's a Schecke
|
||
genotype = existing_genotype
|
||
if is_schecke:
|
||
if genotype:
|
||
if "spsp" in genotype:
|
||
genotype = genotype.replace("spsp", "Spsp")
|
||
elif "Spsp" not in genotype and "Sp" not in genotype:
|
||
genotype = f"{genotype} Spsp".strip()
|
||
else:
|
||
canonical = variety_genotypes.get(color_variety_id)
|
||
if canonical:
|
||
if "spsp" in canonical:
|
||
genotype = canonical.replace("spsp", "Spsp")
|
||
else:
|
||
genotype = f"{canonical} Spsp".strip()
|
||
else:
|
||
genotype = "Spsp"
|
||
return color_variety_id, genotype
|
||
|
||
def parse_date(d):
|
||
"""Convert variations of date formats to YYYY-MM-DD."""
|
||
if not d or d == "0001-01-01":
|
||
return None
|
||
d = str(d).strip()
|
||
# Try YYYY-MM-DD
|
||
if re.match(r"^\d{4}-\d{2}-\d{2}$", d):
|
||
parts = d.split("-")
|
||
year = int(parts[0])
|
||
if 1900 < year < 2000:
|
||
year += 100
|
||
d = f"{year}-{parts[1]}-{parts[2]}"
|
||
elif year == 1900:
|
||
return None
|
||
return d
|
||
# Try DD.MM.YYYY or D.M.YY
|
||
match = re.match(r"^(\d{1,2})\.(\d{1,2})\.(\d{2,4})$", d)
|
||
if match:
|
||
day = int(match.group(1))
|
||
month = int(match.group(2))
|
||
year = int(match.group(3))
|
||
if year < 100:
|
||
year += 2000
|
||
elif 1900 < year < 2000:
|
||
year += 100
|
||
elif year == 1900:
|
||
return None
|
||
try:
|
||
return datetime(year, month, day).strftime("%Y-%m-%d")
|
||
except ValueError:
|
||
pass
|
||
# Try ISO timestamp
|
||
try:
|
||
dt = datetime.fromisoformat(d.replace("Z", "+00:00"))
|
||
year = dt.year
|
||
if 1900 < year < 2000:
|
||
dt = dt.replace(year=year + 100)
|
||
return dt.strftime("%Y-%m-%d")
|
||
elif year == 1900:
|
||
return None
|
||
return dt.strftime("%Y-%m-%d")
|
||
except ValueError:
|
||
pass
|
||
return None
|
||
|
||
def date_to_days(dt_str):
|
||
if not dt_str:
|
||
return None
|
||
try:
|
||
return (datetime.strptime(dt_str, "%Y-%m-%d") - datetime(2000, 1, 1)).days
|
||
except ValueError:
|
||
return None
|
||
|
||
def days_to_date(days):
|
||
import datetime as dt
|
||
return (dt.datetime(2000, 1, 1) + dt.timedelta(days=int(days))).strftime("%Y-%m-%d")
|
||
|
||
def parse_death_info(notes, status, existing_dod, existing_cod):
|
||
if not notes:
|
||
return status, existing_dod, existing_cod
|
||
|
||
has_death_indicator = '†' in notes or 'verstorben' in notes.lower() or 'gestorben' in notes.lower() or 'todesdatum' in notes.lower() or '/+' in notes
|
||
if '+' in notes:
|
||
if re.search(r'\+\s*(?:LE|AS|Unbekannt|gestorben|verstorben|tod)\b', notes, re.I) or re.search(r'\+\s*\d{1,2}\.\d{1,2}\.\d{2,4}', notes) or '/+' in notes:
|
||
has_death_indicator = True
|
||
|
||
resolved_status = status
|
||
if has_death_indicator:
|
||
resolved_status = "Deceased"
|
||
|
||
dod = existing_dod
|
||
cod = existing_cod
|
||
|
||
# Look for date near death indicator
|
||
found_date = None
|
||
for m in re.finditer(r'([+†]\s*(?:LE|AS|Unbekannt|[a-zA-ZäöüÄÖÜß0-9()/ +,;.:-]{1,100}?)?\s*)(\d{1,2}\.\d{1,2}\.\d{2,4})', notes, re.I):
|
||
marker_text = m.group(0)
|
||
if '†' in marker_text or re.search(r'\+\s*(?:LE|AS|Unbekannt|gestorben|verstorben|tod|\d)', marker_text, re.I):
|
||
found_date = parse_date(m.group(2))
|
||
if found_date:
|
||
break
|
||
|
||
if not found_date:
|
||
m_death_marker = re.search(r'(†\s*)(\d{1,2}\.\d{1,2}\.\d{2,4})', notes)
|
||
if m_death_marker:
|
||
found_date = parse_date(m_death_marker.group(2))
|
||
|
||
if found_date and not dod:
|
||
dod = found_date
|
||
|
||
if not cod:
|
||
for m in re.finditer(r'([+†])\s*([a-zA-ZäöüÄÖÜß0-9()/ +,;.:-]{1,100}?)\s*\d{1,2}\.\d{1,2}\.\d{2,4}', notes, re.I):
|
||
indicator = m.group(1)
|
||
cod_candidate = m.group(2).strip()
|
||
if indicator == '+' and not re.search(r'\b(?:LE|AS|Unbekannt|gestorben|verstorben|tod)\b', cod_candidate, re.I):
|
||
continue
|
||
if cod_candidate:
|
||
cod = cod_candidate
|
||
break
|
||
|
||
if not cod:
|
||
m_cod2 = re.search(r'[+†]\s*\d{1,2}\.\d{1,2}\.\d{2,4}\s*([a-zA-ZäöüÄÖÜß0-9()/ +,;.:-]{1,100})', notes, re.I)
|
||
if m_cod2:
|
||
cod_candidate = m_cod2.group(1).strip()
|
||
if cod_candidate:
|
||
cod = cod_candidate
|
||
else:
|
||
m_cod3 = re.search(r'[+†]\s*(LE|AS|Unbekannt)\b', notes, re.I)
|
||
if m_cod3:
|
||
cod = m_cod3.group(1).strip()
|
||
|
||
if cod:
|
||
cod_lower = cod.lower().strip()
|
||
if cod_lower in ("le", "le (lungenentzündung)", "lungenentzündung", "lungenentzündung (lungenentzündung)"):
|
||
cod = "Lungenentzündung"
|
||
elif cod_lower in ("as", "altersschwäche", "altenschwäche", "alter"):
|
||
cod = "Altersschwäche"
|
||
elif cod_lower in ("unbekannt", "unklar"):
|
||
cod = "Unbekannt"
|
||
else:
|
||
# Replace abbreviations with full names (case-insensitive)
|
||
cod = re.sub(r'\bLE\b', 'Lungenentzündung', cod, flags=re.I)
|
||
cod = re.sub(r'\bAS\b', 'Altersschwäche', cod, flags=re.I)
|
||
cod = cod.replace('+', ' + ')
|
||
# Clean up multiple spaces
|
||
cod = re.sub(r'\s+', ' ', cod).strip(' ,;.-')
|
||
|
||
# Re-check after replacement
|
||
cod_lower = cod.lower().strip()
|
||
if cod_lower in ("le", "le (lungenentzündung)", "lungenentzündung", "lungenentzündung (lungenentzündung)"):
|
||
cod = "Lungenentzündung"
|
||
elif cod_lower in ("as", "altersschwäche", "altenschwäche", "alter"):
|
||
cod = "Altersschwäche"
|
||
elif cod_lower in ("unbekannt", "unklar"):
|
||
cod = "Unbekannt"
|
||
|
||
return resolved_status, dod, cod
|
||
|
||
|
||
# ── Litter dedup & parent-role helpers (pure, unit-tested in test_merge_resolve.py) ──
|
||
|
||
def _norm_pname(s):
|
||
return normalize_name(s) if s else ""
|
||
|
||
|
||
def names_no_conflict(l1, l2):
|
||
"""Parent names don't contradict (equal per role, or one side empty)."""
|
||
f1, f2 = _norm_pname(l1.get("_father_name")), _norm_pname(l2.get("_father_name"))
|
||
m1, m2 = _norm_pname(l1.get("_mother_name")), _norm_pname(l2.get("_mother_name"))
|
||
f_ok = (not f1) or (not f2) or (f1 == f2)
|
||
m_ok = (not m1) or (not m2) or (m1 == m2)
|
||
return f_ok and m_ok
|
||
|
||
|
||
def names_overlap(l1, l2):
|
||
"""At least one role has a non-empty matching name (positive evidence)."""
|
||
f1, f2 = _norm_pname(l1.get("_father_name")), _norm_pname(l2.get("_father_name"))
|
||
m1, m2 = _norm_pname(l1.get("_mother_name")), _norm_pname(l2.get("_mother_name"))
|
||
return bool((f1 and f1 == f2) or (m1 and m1 == m2))
|
||
|
||
|
||
def litter_compatible(l1, l2):
|
||
"""Two litter records describe the same litter: same date and compatible parents.
|
||
|
||
- Both sides have both parents → must match exactly.
|
||
- Asymmetric (one side resolved, the other not) → merge only if names don't
|
||
contradict; for dateless litters require a POSITIVE name match (a shared
|
||
null date is no evidence), so unrelated nameless stubs stay separate.
|
||
- Neither side has parents → never blind-merge.
|
||
"""
|
||
if l1["Date"] != l2["Date"]:
|
||
return False
|
||
f1, m1 = l1.get("FatherId"), l1.get("MotherId")
|
||
f2, m2 = l2.get("FatherId"), l2.get("MotherId")
|
||
if f1 and f2 and m1 and m2:
|
||
return f1 == f2 and m1 == m2
|
||
asymmetric = (bool(f1 or m1) and not (f2 or m2)) or (bool(f2 or m2) and not (f1 or m1))
|
||
if asymmetric:
|
||
if not names_no_conflict(l1, l2):
|
||
return False
|
||
if l1["Date"] is None:
|
||
return names_overlap(l1, l2)
|
||
return True
|
||
return False
|
||
|
||
|
||
def assign_parent_roles(father_id, mother_id, gender_of):
|
||
"""Assign two resolved parent IDs to father/mother roles by gender.
|
||
|
||
Drops self-pairing duplicates (same animal in both roles) and never returns
|
||
two same-role parents. `gender_of` maps an id to 'male'|'female'|'unknown'|None.
|
||
Returns (father_id, mother_id).
|
||
"""
|
||
ids = []
|
||
for gid in (father_id, mother_id):
|
||
if gid and gid not in ids:
|
||
ids.append(gid)
|
||
males = [g for g in ids if gender_of(g) == "male"]
|
||
females = [g for g in ids if gender_of(g) == "female"]
|
||
unknowns = [g for g in ids if gender_of(g) == "unknown"]
|
||
father = males[0] if males else (unknowns.pop(0) if unknowns else None)
|
||
mother = females[0] if females else (unknowns.pop(0) if unknowns else None)
|
||
return father, mother
|
||
|
||
|
||
# A gerbil lives at most ~6 years, so a parent can be at most ~6 years older than
|
||
# its offspring (and must be born before it). Links outside this window are
|
||
# impossible — e.g. a 2013 animal resolved onto a 2022 litter (Jayjay → Solice).
|
||
MAX_PARENT_AGE_DAYS = 6 * 366
|
||
|
||
|
||
def parent_age_plausible(parent_dob, litter_date):
|
||
"""Could a parent born `parent_dob` have offspring born on `litter_date`?
|
||
|
||
Requires birth strictly before the litter and within the gerbil lifespan.
|
||
Unknown/unparseable dates return True (cannot disprove). Accepts any date
|
||
format parse_date understands.
|
||
"""
|
||
pd = date_to_days(parse_date(parent_dob)) if parent_dob else None
|
||
ld = date_to_days(parse_date(litter_date)) if litter_date else None
|
||
if pd is None or ld is None:
|
||
return True
|
||
return 0 < (ld - pd) <= MAX_PARENT_AGE_DAYS
|
||
|
||
|
||
def pick_parent_ref(parent_refs, role, child_dob, avoid_name=None, gender_of=None):
|
||
"""Choose the best parent ref for a role from possibly-conflicting chart refs.
|
||
|
||
A Stammbaum lists an animal at several positions, so its parentRefs can carry
|
||
contradictory guesses (the first one is not necessarily right). Rank candidates
|
||
(lower = better):
|
||
0 right/unknown gender for the role, age-plausible dated ref
|
||
1 right/unknown gender, no DOB (usable, but a plausible dated ref wins)
|
||
2 right/unknown gender, dated but age-impossible
|
||
3 resolved gender is clearly WRONG for the role (e.g. a female father)
|
||
4 would duplicate the animal chosen for the other role
|
||
|
||
Gender is decisive over DOB: a dated female ref must not win the father slot
|
||
over an undated male/unknown one. `gender_of(name)` returns 'male'/'female'
|
||
or None (unknown/ambiguous → not penalised). Returns the chosen ref or None.
|
||
"""
|
||
role_refs = [p for p in parent_refs if p.get("roleGuess") == role]
|
||
if not role_refs:
|
||
return None
|
||
avoid = normalize_name(avoid_name) if avoid_name else None
|
||
expected = "male" if role == "father" else "female"
|
||
|
||
def rank(p):
|
||
if avoid is not None and normalize_name(p.get("name")) == avoid:
|
||
return 4 # would duplicate the other parent role
|
||
g = gender_of(p.get("name")) if gender_of else None
|
||
if g in ("male", "female") and g != expected:
|
||
return 3 # wrong sex for this role
|
||
dob = p.get("dob")
|
||
if not dob:
|
||
return 1
|
||
return 0 if parent_age_plausible(dob, child_dob) else 2
|
||
|
||
order = sorted(range(len(role_refs)), key=lambda i: (rank(role_refs[i]), i))
|
||
return role_refs[order[0]]
|
||
|
||
|
||
def explain_pick_rejections(parent_refs, role, child_dob, chosen, avoid_name=None,
|
||
gender_of=None):
|
||
"""Explain why other refs for `role` lost to `chosen` in pick_parent_ref.
|
||
|
||
Returns a list of discard dicts (for _format_discard) — one per distinct
|
||
rejected candidate name that was beaten for a clear reason (wrong sex, age-
|
||
impossible, or duplicate of the other parent). Mirrors pick_parent_ref's
|
||
ranking so the history can explain the same decision it made.
|
||
"""
|
||
role_refs = [p for p in parent_refs if p.get("roleGuess") == role]
|
||
if not role_refs or chosen is None:
|
||
return []
|
||
avoid = normalize_name(avoid_name) if avoid_name else None
|
||
expected = "male" if role == "father" else "female"
|
||
role_de = "Vaterrolle" if role == "father" else "Mutterrolle"
|
||
cand_de = "Vater-Kandidat" if role == "father" else "Mutter-Kandidat"
|
||
chosen_name = chosen.get("name")
|
||
repl_disp = chosen_name
|
||
if chosen.get("dob"):
|
||
repl_disp = f"{chosen_name} (*{_de_date(parse_date(chosen.get('dob'))) or chosen.get('dob')})"
|
||
|
||
out = []
|
||
seen = set()
|
||
for p in role_refs:
|
||
name = p.get("name")
|
||
if not name or normalize_name(name) == normalize_name(chosen_name or ""):
|
||
continue
|
||
key = normalize_name(name)
|
||
if key in seen:
|
||
continue
|
||
g = gender_of(name) if gender_of else None
|
||
dob = p.get("dob")
|
||
reason = None
|
||
if avoid is not None and key == avoid:
|
||
reason = "bereits als anderer Elternteil gewählt"
|
||
elif g in ("male", "female") and g != expected:
|
||
reason = f"falsches Geschlecht für die {role_de}"
|
||
elif dob and not parent_age_plausible(dob, child_dob):
|
||
reason = "unplausibles Alter für diesen Wurf"
|
||
if reason is None:
|
||
continue
|
||
seen.add(key)
|
||
disp = name
|
||
if dob:
|
||
disp = f"{name} (*{_de_date(parse_date(dob)) or dob})"
|
||
out.append({
|
||
"label": cand_de,
|
||
"value": f"„{disp}“",
|
||
"reason": reason,
|
||
"replacement": f"„{repl_disp}“",
|
||
})
|
||
return out
|
||
|
||
|
||
def _build_gerbil_history(records, best_g, field_source, parent_method=None,
|
||
any_decision=False, any_conflict=False, conflict_notes=None,
|
||
field_discards=None, parent_discards=None):
|
||
"""Build an ordered, file-attributed German history for a resolved gerbil.
|
||
|
||
Reads like a chronological log:
|
||
• „In ‚X.xlsx' gefunden."
|
||
• „Geburtsdatum (27.03.2022) aus ‚X.xlsx'."
|
||
• „Auch in ‚Y.xlsx' gefunden → Datensätze zusammengeführt."
|
||
• „Genotyp aus ‚Z.xlsx'."
|
||
• „Eltern über Position im Stammbaum erkannt (Quelle: ‚X.xlsx')."
|
||
• „Aus Wurfchronik übernommen."
|
||
|
||
`field_source` maps a field name to the raw record that supplied its final
|
||
value; when present we name that record's file, otherwise we fall back to
|
||
the primary record. The records are visited in a stable order (primary
|
||
first, then the rest sorted by file) so the log is deterministic."""
|
||
conflict_notes = conflict_notes or []
|
||
history = []
|
||
|
||
# Order records: best_g first, then others by primary file name (stable).
|
||
others = [r for r in records if r is not best_g]
|
||
others.sort(key=lambda r: (_primary_file_of(r) or ""))
|
||
ordered = [best_g] + others
|
||
|
||
best_file = _primary_file_of(best_g)
|
||
if best_file:
|
||
history.append(f"In {_quote_file(best_file)} gefunden.")
|
||
else:
|
||
history.append("Im Import gefunden.")
|
||
|
||
# Field-by-field attribution: name the file that supplied each fact.
|
||
def attr_line(field, formatter):
|
||
rec = field_source.get(field) or best_g
|
||
val = best_g.get(field)
|
||
if not val or val == "unknown":
|
||
return
|
||
fname = _primary_file_of(rec)
|
||
label = PROV_FIELD_LABELS.get(field, field)
|
||
text = formatter(label, val)
|
||
if fname:
|
||
history.append(f"{text} aus {_quote_file(fname)}.")
|
||
else:
|
||
history.append(f"{text} (Quelle unbekannt).")
|
||
|
||
attr_line("DateOfBirth", lambda label, v: f"{label} ({_de_date(v)})")
|
||
attr_line("Gender", lambda label, v: f"{label} ({_de_gender(v)})")
|
||
attr_line("Genotype", lambda label, v: f"{label}")
|
||
attr_line("ColorVarietyId", lambda label, v: f"{label}")
|
||
attr_line("DateOfDeath", lambda label, v: f"{label} ({_de_date(v)})")
|
||
|
||
# Merge step: every additional record that contributed.
|
||
for r in others:
|
||
fname = _primary_file_of(r)
|
||
if fname:
|
||
history.append(
|
||
f"Auch in {_quote_file(fname)} gefunden → Datensätze zusammengeführt."
|
||
)
|
||
else:
|
||
history.append("In weiterem Datensatz gefunden → Datensätze zusammengeführt.")
|
||
|
||
# Discarded field values from majority-vote conflict resolution: the LOSING
|
||
# values, their file, and what won instead.
|
||
for d in (field_discards or []):
|
||
line = _format_discard(d)
|
||
if line not in history:
|
||
history.append(line)
|
||
|
||
# Parent derivation.
|
||
if parent_method:
|
||
method_label = {
|
||
"chart-position": "Position im Stammbaum",
|
||
"geburtsdatum+eltern": "Geburtsdatum und Elternnamen",
|
||
"nur-geburtsdatum": "Geburtsdatum",
|
||
"decision": "manuelle Entscheidung",
|
||
}.get(parent_method, parent_method)
|
||
# The parent evidence comes from a Stammbaum chart — attribute to the
|
||
# primary record's file when it is a Stammbaum.
|
||
parent_file = best_file if best_file and "stammbaum" in best_file.lower() else None
|
||
if parent_file:
|
||
history.append(
|
||
f"Eltern über {method_label} erkannt (Quelle: {_quote_file(parent_file)})."
|
||
)
|
||
else:
|
||
history.append(f"Eltern über {method_label} erkannt.")
|
||
|
||
# Discarded parent candidates / links (pick_parent_ref rejections, role
|
||
# normalization drops, parent-age sanity check). Threaded in after the merge
|
||
# via the gerbil's _discarded list so they read in chronological order.
|
||
for d in (parent_discards or []):
|
||
line = _format_discard(d)
|
||
if line not in history:
|
||
history.append(line)
|
||
|
||
# Manual decisions / conflicts.
|
||
if any_decision:
|
||
history.append("Zuordnung per manueller Entscheidung getroffen.")
|
||
if any_conflict:
|
||
history.append("Konflikt per Entscheidung gelöst.")
|
||
for cn in conflict_notes:
|
||
if cn and cn not in history:
|
||
history.append(cn + ".")
|
||
|
||
# Wurfchronik provenance line.
|
||
if any("wurfchronik" in f.lower() for r in records for f in _record_source_files(r)):
|
||
history.append("Angaben aus der Wurfchronik übernommen.")
|
||
|
||
return history
|
||
|
||
|
||
def _de_date(iso):
|
||
"""YYYY-MM-DD → DD.MM.YYYY for display; pass through anything else."""
|
||
if not iso:
|
||
return iso
|
||
m = re.match(r"^(\d{4})-(\d{2})-(\d{2})$", str(iso))
|
||
if m:
|
||
return f"{m.group(3)}.{m.group(2)}.{m.group(1)}"
|
||
return iso
|
||
|
||
|
||
def _de_gender(g):
|
||
return {"male": "männlich", "female": "weiblich"}.get(g, g)
|
||
|
||
|
||
# Leading marker that visually flags a discard ("data was thrown away") line in
|
||
# the history timeline. The frontend keys discard styling off this marker.
|
||
DISCARD_MARK = "⚠ "
|
||
|
||
|
||
def _format_discard(d):
|
||
"""Render one discard record into a German history line.
|
||
|
||
A discard record is a dict describing a value/candidate the pipeline threw
|
||
away. Recognised keys:
|
||
• text — a fully pre-formatted line (used verbatim, marker added)
|
||
• label — German field label (e.g. „Geburtsdatum“)
|
||
• value — the discarded value (already display-formatted)
|
||
• file — source file the discarded value came from (attributed)
|
||
• reason — why it was dropped (e.g. „abweichend“, „unplausibel …“)
|
||
• replacement — what was used instead (already display-formatted)
|
||
• repl_file — source file the replacement came from
|
||
Produces lines like:
|
||
„⚠ Geburtsdatum 14.06.2015 aus ‚A.xlsx‘ verworfen — abweichend;
|
||
14.06.2017 aus ‚B.xlsx‘ verwendet (Mehrheit).“
|
||
Generic across entity types so litters/contacts can reuse it.
|
||
"""
|
||
if d.get("text"):
|
||
return DISCARD_MARK + d["text"]
|
||
parts = []
|
||
label = d.get("label")
|
||
value = d.get("value")
|
||
if label and value is not None:
|
||
parts.append(f"{label} {value}")
|
||
elif label:
|
||
parts.append(str(label))
|
||
elif value is not None:
|
||
parts.append(str(value))
|
||
head = " ".join(parts) if parts else "Wert"
|
||
if d.get("file"):
|
||
head += f" aus {_quote_file(d['file'])}"
|
||
line = f"{head} verworfen"
|
||
if d.get("reason"):
|
||
line += f" — {d['reason']}"
|
||
repl = d.get("replacement")
|
||
if repl is not None and repl != "":
|
||
instead = str(repl)
|
||
if d.get("repl_file"):
|
||
instead += f" aus {_quote_file(d['repl_file'])}"
|
||
suffix = d.get("replacement_note")
|
||
line += f"; {instead} verwendet"
|
||
if suffix:
|
||
line += f" ({suffix})"
|
||
elif d.get("no_replacement"):
|
||
line += "; kein Ersatz"
|
||
return DISCARD_MARK + line + "."
|
||
|
||
|
||
def _append_history(prov_json, line):
|
||
"""Append one history line to an existing Provenance JSON string and add the
|
||
contract source file. Returns the updated JSON string."""
|
||
try:
|
||
prov = json.loads(prov_json) if prov_json else {}
|
||
except (ValueError, TypeError):
|
||
prov = {}
|
||
prov.setdefault("history", [])
|
||
if line not in prov["history"]:
|
||
prov["history"].append(line)
|
||
return json.dumps(prov, ensure_ascii=False)
|
||
|
||
|
||
def enrich_from_contracts(contracts, resolved_gerbils, contact_by_norm_name,
|
||
contact_id_map):
|
||
"""Conservatively fold Abgabevertrag data into the resolved gerbils.
|
||
|
||
For every parsed contract we (a) ensure the buyer exists as a (receiver)
|
||
contact, reusing the existing contact dedup/normalisation, and (b) try to
|
||
match each animal call-name to exactly one resolved gerbil that the breeder
|
||
bred ("…Chaoten"). On a confident match we set ReceiverContactId /
|
||
GoHomeDate / Status=GivenAway — but only where not already set differently —
|
||
and add a provenance history line. Ambiguous or absent matches are logged,
|
||
never guessed.
|
||
|
||
Returns a stats dict. Mutates resolved_gerbils + contact_by_norm_name in
|
||
place. New buyer contacts are appended via contact_by_norm_name so the later
|
||
IsReceiver-flag pass picks them up automatically.
|
||
"""
|
||
stats = {
|
||
"contracts": len(contracts), "buyers_created": 0, "buyers_existing": 0,
|
||
"matched": 0, "ambiguous_skipped": 0, "no_match_skipped": 0,
|
||
"receiver_set": 0, "gohome_set": 0, "status_givenaway": 0,
|
||
"conflicts": 0,
|
||
}
|
||
if not contracts:
|
||
return stats
|
||
|
||
# Index breeder-owned gerbils by dedup name key. Contracts only ever sell
|
||
# animals the breeder bred, so restrict candidates to her own stock to avoid
|
||
# colliding with same-named foreign-bred animals.
|
||
def is_own(g):
|
||
ob = (g.get("OriginBreeder") or "").lower()
|
||
return ("chaoten" in ob) or (g.get("OriginBreeder") is None)
|
||
|
||
index = {}
|
||
for g in resolved_gerbils:
|
||
if not is_own(g):
|
||
continue
|
||
key = get_dedup_name_key(get_call_name(g.get("Name", "")))
|
||
if key:
|
||
index.setdefault(key, []).append(g)
|
||
|
||
def year_of(iso):
|
||
return iso[:4] if iso else None
|
||
|
||
# Reject buyer values that are obviously label leakage / non-person noise
|
||
# (defends against any stale contracts.json produced before the parser fix).
|
||
_buyer_junk = re.compile(
|
||
r"^(?:stra\w+e|wohnort|fon|e-?mail|handy|festnetz|telefon|homepage|"
|
||
r"zuchtname|facebook|name)\s*:?\s*$|^[\d\s/]+$", re.IGNORECASE)
|
||
|
||
for c in contracts:
|
||
fname = c.get("sourceFile", "")
|
||
buyer_raw = (c.get("buyer") or "").strip()
|
||
if buyer_raw and _buyer_junk.match(buyer_raw):
|
||
buyer_raw = ""
|
||
|
||
# --- buyer contact (reuse curated normalisation/dedup) ---
|
||
buyer_global_id = None
|
||
if buyer_raw:
|
||
canon, keep = get_normalized_contact_name(buyer_raw)
|
||
if keep and canon:
|
||
norm = normalize_name(canon)
|
||
if norm in contact_by_norm_name:
|
||
gc = contact_by_norm_name[norm]
|
||
gc.setdefault("_source_files", set()).add(fname)
|
||
gc["_merged_count"] = gc.get("_merged_count", 1) + 1
|
||
stats["buyers_existing"] += 1
|
||
else:
|
||
gid = generate_guid(f"contact-{norm}")
|
||
contact_by_norm_name[norm] = {
|
||
"Id": gid, "Name": canon, "Email": None, "Phone": None,
|
||
"Address": None, "Notes": None, "NameSuffix": None,
|
||
"_source_files": {fname}, "_merged_count": 1,
|
||
}
|
||
stats["buyers_created"] += 1
|
||
buyer_global_id = contact_by_norm_name[norm]["Id"]
|
||
|
||
# --- match each animal call-name to a resolved gerbil ---
|
||
handover = parse_date(c.get("handoverDate"))
|
||
c_year = year_of(parse_date(c.get("dob"))) if c.get("dob") else None
|
||
c_color = (c.get("color") or "").strip().lower()
|
||
|
||
for call in (c.get("animals") or []):
|
||
key = get_dedup_name_key(get_call_name(call))
|
||
if not key:
|
||
continue
|
||
cands = index.get(key, [])
|
||
if not cands:
|
||
stats["no_match_skipped"] += 1
|
||
continue
|
||
|
||
# Corroborate when more than one candidate shares the call-name.
|
||
chosen = None
|
||
if len(cands) == 1:
|
||
chosen = cands[0]
|
||
else:
|
||
scored = []
|
||
for g in cands:
|
||
score = 0
|
||
g_year = year_of(g.get("DateOfBirth"))
|
||
if c_year and g_year and c_year == g_year:
|
||
score += 2
|
||
if c_color and g.get("ColorVarietyId"):
|
||
# color match is corroboration; we don't have the name
|
||
# here, so only DOB drives disambiguation strongly.
|
||
pass
|
||
scored.append((score, g))
|
||
scored.sort(key=lambda t: t[0], reverse=True)
|
||
if scored[0][0] >= 2 and (len(scored) == 1 or scored[0][0] > scored[1][0]):
|
||
chosen = scored[0][1]
|
||
else:
|
||
stats["ambiguous_skipped"] += 1
|
||
continue
|
||
|
||
stats["matched"] += 1
|
||
|
||
# --- set receiver, only if not already set differently ---
|
||
if buyer_global_id:
|
||
cur = chosen.get("ReceiverContactId")
|
||
if not cur:
|
||
chosen["ReceiverContactId"] = buyer_global_id
|
||
stats["receiver_set"] += 1
|
||
chosen["Provenance"] = _append_history(
|
||
chosen.get("Provenance"),
|
||
f"Abgabe an „{buyer_raw}“ aus Vertrag {_quote_file(fname)} übernommen.")
|
||
elif cur != buyer_global_id:
|
||
stats["conflicts"] += 1
|
||
chosen["Provenance"] = _append_history(
|
||
chosen.get("Provenance"),
|
||
f"Vertrag {_quote_file(fname)} nennt anderen Abnehmer „{buyer_raw}“ "
|
||
f"— bestehende Zuordnung beibehalten.")
|
||
|
||
# --- set go-home date, only if empty ---
|
||
if handover and not chosen.get("GoHomeDate"):
|
||
chosen["GoHomeDate"] = handover
|
||
stats["gohome_set"] += 1
|
||
chosen["Provenance"] = _append_history(
|
||
chosen.get("Provenance"),
|
||
f"Abgabedatum {handover} aus Vertrag {_quote_file(fname)} übernommen.")
|
||
|
||
# --- status: derive GivenAway if we set a receiver and it isn't
|
||
# already a stronger state (Deceased). ---
|
||
if chosen.get("ReceiverContactId") and chosen.get("Status") not in (
|
||
"Deceased", "GivenAway"):
|
||
chosen["Status"] = "GivenAway"
|
||
stats["status_givenaway"] += 1
|
||
|
||
return stats
|
||
|
||
|
||
def main():
|
||
print("Loading color variety seeds...")
|
||
variety_map = {}
|
||
variety_genotypes = {}
|
||
|
||
# Load from C# ApplicationContext.cs catalog for stable database GUIDs (index + 1)
|
||
here = os.path.dirname(os.path.abspath(__file__))
|
||
app_context_path = os.path.abspath(os.path.join(here, "../../GerbilManagerWebAPI/ApplicationContext.cs"))
|
||
cs_name_to_id = {}
|
||
if os.path.exists(app_context_path):
|
||
with open(app_context_path, 'r', encoding='utf-8') as f:
|
||
content = f.read()
|
||
catalog_match = re.search(r'catalog\s*=\s*\{(.*?)\};', content, re.DOTALL)
|
||
if catalog_match:
|
||
block = catalog_match.group(1)
|
||
entries = re.findall(r'\(\s*"([^"]+)"\s*,\s*"([^"]+)"\s*,\s*(\d+)\s*\)', block)
|
||
for idx, (name, genotype, sort_order) in enumerate(entries):
|
||
variety_id = f"00000000-0000-0000-0000-{idx + 1:012d}"
|
||
cs_name_to_id[name.strip().lower()] = variety_id
|
||
variety_map[name.strip().lower()] = variety_id
|
||
variety_genotypes[variety_id] = genotype.strip()
|
||
else:
|
||
print(f"Warning: ApplicationContext.cs not found at {app_context_path}")
|
||
|
||
if os.path.exists(SEEDS_PATH):
|
||
with open(SEEDS_PATH, "r", encoding="utf-8") as f:
|
||
seeds = json.load(f)
|
||
for v in seeds:
|
||
name_lower = v["name"].strip().lower()
|
||
variety_id = cs_name_to_id.get(name_lower)
|
||
if not variety_id:
|
||
variety_id = f"00000000-0000-0000-0000-{v['sortOrder'] + 1:012d}"
|
||
variety_map[name_lower] = variety_id
|
||
|
||
# Map English name if present
|
||
if "english" in v and v["english"]:
|
||
variety_map[v["english"].strip().lower()] = variety_id
|
||
|
||
if variety_id not in variety_genotypes:
|
||
variety_genotypes[variety_id] = v.get("canonicalGenotype")
|
||
else:
|
||
print(f"Warning: Seeds path not found at {SEEDS_PATH}")
|
||
|
||
# Reverse map (variety GUID → human name) for discard/replacement history
|
||
# lines that need to show a colour value rather than a raw GUID. First wins
|
||
# so we keep the canonical lower-case catalog name.
|
||
variety_id_to_name = {}
|
||
for name_lower, vid in variety_map.items():
|
||
if vid not in variety_id_to_name:
|
||
variety_id_to_name[vid] = name_lower
|
||
|
||
md_files = sorted([f for f in os.listdir(DIR_PATH) if f.lower().endswith('.md')])
|
||
print(f"Found {len(md_files)} markdown files in {DIR_PATH}.")
|
||
|
||
raw_contacts = []
|
||
raw_litters = []
|
||
raw_gerbils = []
|
||
|
||
# Map to track which files contained explicit dates
|
||
file_explicit_dates = {}
|
||
|
||
# 1. Parse JSON blocks from all markdown files and scope IDs by filename
|
||
for filename in md_files:
|
||
filepath = os.path.join(DIR_PATH, filename)
|
||
with open(filepath, 'r', encoding='utf-8') as f:
|
||
content = f.read()
|
||
|
||
json_start = content.find("```json")
|
||
if json_start == -1:
|
||
continue
|
||
json_start += len("```json")
|
||
json_end = content.rfind("```")
|
||
if json_end == -1 or json_end <= json_start:
|
||
continue
|
||
|
||
json_str = content[json_start:json_end].strip()
|
||
try:
|
||
data = json.loads(json_str)
|
||
|
||
contacts_key = next((k for k in data if k.lower() == 'contacts'), None)
|
||
litters_key = next((k for k in data if k.lower() == 'litters'), None)
|
||
gerbils_key = next((k for k in data if k.lower() == 'gerbils'), None)
|
||
|
||
file_contacts = data.get(contacts_key, []) if contacts_key else []
|
||
file_litters = data.get(litters_key, []) if litters_key else []
|
||
file_gerbils = data.get(gerbils_key, []) if gerbils_key else []
|
||
|
||
# Find explicit dates on this page to build chronological ordering
|
||
explicit_dates = []
|
||
for l in file_litters:
|
||
d = parse_date(l.get("Date") or l.get("date") or l.get("DateOfBirth") or l.get("dateOfBirth"))
|
||
if d:
|
||
explicit_dates.append(d)
|
||
for g in file_gerbils:
|
||
d = parse_date(g.get("DateOfBirth") or g.get("dateOfBirth"))
|
||
if d:
|
||
explicit_dates.append(d)
|
||
|
||
if explicit_dates:
|
||
# Convert to days since 2000-01-01
|
||
day_vals = [date_to_days(d) for d in explicit_dates if date_to_days(d) is not None]
|
||
if day_vals:
|
||
file_explicit_dates[filename] = sum(day_vals) / len(day_vals)
|
||
|
||
def scope_id(local_id):
|
||
if not local_id:
|
||
return None
|
||
local_str = str(local_id).strip()
|
||
if not local_str:
|
||
return None
|
||
return generate_guid(f"{filename}-{local_str}")
|
||
|
||
for c in file_contacts:
|
||
old_id = c.get("Id") or c.get("id")
|
||
c["_filename"] = filename
|
||
c["_scoped_id"] = scope_id(old_id)
|
||
raw_contacts.append(c)
|
||
|
||
for l in file_litters:
|
||
old_id = l.get("Id") or l.get("id")
|
||
l["_filename"] = filename
|
||
l["_scoped_id"] = scope_id(old_id)
|
||
l["_scoped_father_id"] = scope_id(l.get("FatherId") or l.get("fatherId"))
|
||
l["_scoped_mother_id"] = scope_id(l.get("MotherId") or l.get("motherId"))
|
||
raw_litters.append(l)
|
||
|
||
for g in file_gerbils:
|
||
old_id = g.get("Id") or g.get("id")
|
||
g["_filename"] = filename
|
||
g["_scoped_id"] = scope_id(old_id)
|
||
g["_scoped_litter_id"] = scope_id(g.get("LitterId") or g.get("litterId"))
|
||
g["_scoped_origin_cid"] = scope_id(g.get("OriginContactId") or g.get("originContactId"))
|
||
|
||
receiver_val = (g.get("ReceiverContactId") or g.get("receiverContactId") or
|
||
g.get("BuyerId") or g.get("buyerId") or
|
||
g.get("BuyerContactId") or g.get("buyerContactId") or
|
||
g.get("givenAwayContactId") or g.get("givenAwayToContactId") or
|
||
g.get("ownerContactId") or g.get("ownerId"))
|
||
g["_scoped_receiver_cid"] = scope_id(receiver_val)
|
||
raw_gerbils.append(g)
|
||
|
||
except json.JSONDecodeError as e:
|
||
print(f"Failed to parse JSON in {filename}: {e}")
|
||
|
||
# Chronological Interpolation: Estimate the date of each page based on neighboring pages with dates
|
||
file_dates = {}
|
||
sorted_files = sorted(md_files)
|
||
|
||
# Simple linear interpolation / extrapolation
|
||
for i, fn in enumerate(sorted_files):
|
||
if fn in file_explicit_dates:
|
||
file_dates[fn] = file_explicit_dates[fn]
|
||
else:
|
||
# Look left for closest explicit date
|
||
left_val, left_dist = None, None
|
||
for j in range(i - 1, -1, -1):
|
||
if sorted_files[j] in file_explicit_dates:
|
||
left_val = file_explicit_dates[sorted_files[j]]
|
||
left_dist = i - j
|
||
break
|
||
|
||
# Look right for closest explicit date
|
||
right_val, right_dist = None, None
|
||
for j in range(i + 1, len(sorted_files)):
|
||
if sorted_files[j] in file_explicit_dates:
|
||
right_val = file_explicit_dates[sorted_files[j]]
|
||
right_dist = j - i
|
||
break
|
||
|
||
if left_val is not None and right_val is not None:
|
||
# Interpolate
|
||
file_dates[fn] = left_val + (right_val - left_val) * (left_dist / (left_dist + right_dist))
|
||
elif left_val is not None:
|
||
# Extrapolate right (assume 30 days per page gap as placeholder)
|
||
file_dates[fn] = left_val + (left_dist * 30)
|
||
elif right_val is not None:
|
||
# Extrapolate left
|
||
file_dates[fn] = right_val - (right_dist * 30)
|
||
else:
|
||
# No dates in entire log? Default to 2011-01-01
|
||
file_dates[fn] = date_to_days("2011-01-01")
|
||
|
||
print(f"Parsed {len(raw_contacts)} raw contacts, {len(raw_litters)} raw litters, {len(raw_gerbils)} raw gerbils.")
|
||
|
||
# Run extract.py to make sure stammbaum data is up to date
|
||
import subprocess
|
||
print("Running extract.py to extract Stammbäume...")
|
||
try:
|
||
subprocess.run([sys.executable, "extract.py"], check=True)
|
||
except Exception as e:
|
||
print(f"Warning: Failed to run extract.py: {e}")
|
||
|
||
# Load Stammbaum data
|
||
stammbaum_only_animals = []
|
||
animals_path = os.path.join(OUTPUT_DIR, "animals.json")
|
||
if os.path.exists(animals_path):
|
||
with open(animals_path, "r", encoding="utf-8") as f:
|
||
stammbaum_animals = json.load(f)
|
||
for a in stammbaum_animals:
|
||
sources = a.get("sourceFiles", [])
|
||
if any("stammbaum" in str(s).lower() for s in sources):
|
||
stammbaum_only_animals.append(a)
|
||
print(f"Loaded {len(stammbaum_only_animals)} Stammbaum animals.")
|
||
else:
|
||
print(f"Warning: Stammbaum animals.json not found at {animals_path}")
|
||
|
||
# Run extract_docx.py to make sure docx data is up to date
|
||
print("Running extract_docx.py to extract docx...")
|
||
try:
|
||
subprocess.run([sys.executable, "extract_docx.py"], check=True)
|
||
except Exception as e:
|
||
print(f"Warning: Failed to run extract_docx.py: {e}")
|
||
|
||
# Load Docx data
|
||
docx_animals = []
|
||
docx_animals_path = os.path.join(OUTPUT_DIR, "docx_animals.json")
|
||
if os.path.exists(docx_animals_path):
|
||
with open(docx_animals_path, "r", encoding="utf-8") as f:
|
||
docx_animals = json.load(f)
|
||
print(f"Loaded {len(docx_animals)} animals from docx.")
|
||
else:
|
||
print(f"Warning: docx_animals.json not found at {docx_animals_path}")
|
||
|
||
docx_litters = []
|
||
docx_litters_path = os.path.join(OUTPUT_DIR, "docx_litters.json")
|
||
if os.path.exists(docx_litters_path):
|
||
with open(docx_litters_path, "r", encoding="utf-8") as f:
|
||
docx_litters = json.load(f)
|
||
print(f"Loaded {len(docx_litters)} litters from docx.")
|
||
else:
|
||
print(f"Warning: docx_litters.json not found at {docx_litters_path}")
|
||
|
||
# Abgabevertrag-Datensätze (aus extract_contracts.py). Optional: wenn die
|
||
# Datei fehlt, läuft der Import ohne Vertrags-Anreicherung normal weiter.
|
||
contracts = []
|
||
contracts_path = os.path.join(OUTPUT_DIR, "contracts.json")
|
||
if os.path.exists(contracts_path):
|
||
with open(contracts_path, "r", encoding="utf-8") as f:
|
||
contracts = json.load(f)
|
||
print(f"Loaded {len(contracts)} Abgabeverträge.")
|
||
else:
|
||
print(f"Info: contracts.json not found at {contracts_path} (Vertrags-Anreicherung übersprungen)")
|
||
|
||
# Extract docx buyer contacts and add to raw_contacts
|
||
for da in docx_animals:
|
||
o_name = (da.get("owner") or "").strip()
|
||
if o_name:
|
||
o_scoped_id = generate_guid(f"docx-contact-{normalize_name(o_name)}")
|
||
raw_contacts.append({
|
||
"Name": o_name,
|
||
"_filename": "Wurfchronik-Detail",
|
||
"_scoped_id": o_scoped_id
|
||
})
|
||
|
||
# Map docx litters and append to raw_litters
|
||
docx_litter_id_map = {} # wsCode -> scoped_id
|
||
for dl in docx_litters:
|
||
l_name = dl["litterId"]
|
||
dob_val = parse_date(dl["dob"])
|
||
f_name = get_normalized_gerbil_name(dl["fatherName"])
|
||
m_name = get_normalized_gerbil_name(dl["motherName"])
|
||
ws_code = dl["wsCode"]
|
||
note_val = dl.get("note")
|
||
|
||
# Parse survived/total born from wsCode (e.g. 4/5)
|
||
total_born = None
|
||
deaths_8w = None
|
||
if "/" in ws_code:
|
||
parts = ws_code.split("/")
|
||
if len(parts) == 2:
|
||
try:
|
||
survived = int(parts[0])
|
||
total = int(parts[1])
|
||
total_born = total
|
||
deaths_8w = max(0, total - survived)
|
||
except ValueError:
|
||
pass
|
||
|
||
l_scoped_id = generate_guid(f"docx-litter-{normalize_name(ws_code)}-{dob_val or '0001-01-01'}")
|
||
docx_litter_id_map[(ws_code, dob_val)] = l_scoped_id
|
||
|
||
raw_litters.append({
|
||
"Id": l_scoped_id,
|
||
"Name": l_name,
|
||
"Date": dob_val,
|
||
"TotalBorn": total_born,
|
||
"DeathsWithin8Weeks": deaths_8w,
|
||
"FatherId": generate_guid(f"stammbaum-animal-{normalize_name(f_name)}"), # placeholder
|
||
"MotherId": generate_guid(f"stammbaum-animal-{normalize_name(m_name)}"), # placeholder
|
||
"ExpectedGoHomeDate": None,
|
||
"Notes": note_val if note_val else "Docx imported litter",
|
||
"PairingCode": None,
|
||
"ExternalRef": f"docx-litter-{l_scoped_id}",
|
||
"LitterLetter": l_name[0] if l_name and len(l_name) > 0 else None,
|
||
"_father_name": f_name,
|
||
"_mother_name": m_name,
|
||
"_filename": "Wurfchronik-Detail",
|
||
"_scoped_id": l_scoped_id,
|
||
"_scoped_father_id": generate_guid(f"stammbaum-animal-{normalize_name(f_name)}"),
|
||
"_scoped_mother_id": generate_guid(f"stammbaum-animal-{normalize_name(m_name)}")
|
||
})
|
||
|
||
|
||
# Extract stammbaum contacts and add to raw_contacts
|
||
for a in stammbaum_only_animals:
|
||
b_name = (a.get("breeder") or "").strip()
|
||
if b_name:
|
||
b_scoped_id = generate_guid(f"stammbaum-contact-{normalize_name(b_name)}")
|
||
raw_contacts.append({
|
||
"Name": b_name,
|
||
"_filename": "Stammbaum",
|
||
"_scoped_id": b_scoped_id
|
||
})
|
||
z_name = (a.get("zucht") or "").strip()
|
||
if z_name:
|
||
z_scoped_id = generate_guid(f"stammbaum-contact-{normalize_name(z_name)}")
|
||
raw_contacts.append({
|
||
"Name": z_name,
|
||
"_filename": "Stammbaum",
|
||
"_scoped_id": z_scoped_id
|
||
})
|
||
|
||
# Residency propagation for stammbaum animals
|
||
def is_clan_zucht(z):
|
||
if not z:
|
||
return False
|
||
norm = z.lower()
|
||
return "klein" in norm and "chaot" in norm and "extern" not in norm
|
||
|
||
stammbaum_resident_ids = set()
|
||
for a in stammbaum_only_animals:
|
||
if is_clan_zucht(a.get("zucht")) or is_clan_zucht(a.get("zuchtCanon")):
|
||
stammbaum_resident_ids.add(a["id"])
|
||
|
||
# Propagate residency to parents of resident offspring
|
||
for _ in range(5):
|
||
for a in stammbaum_only_animals:
|
||
if a["id"] in stammbaum_resident_ids:
|
||
for p_ref in a.get("parentRefs", []):
|
||
p_key = (normalize_name(p_ref["name"]), parse_date(p_ref["dob"]))
|
||
for cand in stammbaum_only_animals:
|
||
if normalize_name(cand["name"]) == p_key[0]:
|
||
cand_dob = parse_date(cand["dob"])
|
||
if not p_key[1] or cand_dob == p_key[1]:
|
||
stammbaum_resident_ids.add(cand["id"])
|
||
|
||
# Pre-index Wurfchronik litters from markdown
|
||
md_litters_idx = {}
|
||
for rl in raw_litters:
|
||
f_name = get_normalized_gerbil_name(rl.get("FatherName") or rl.get("fatherName") or rl.get("ParentMaleName") or rl.get("parentMaleName"))
|
||
m_name = get_normalized_gerbil_name(rl.get("MotherName") or rl.get("motherName") or rl.get("ParentFemaleName") or rl.get("parentFemaleName"))
|
||
ldate = parse_date(rl.get("Date") or rl.get("date") or rl.get("DateOfBirth") or rl.get("dateOfBirth"))
|
||
if f_name and m_name and ldate:
|
||
key = (normalize_name(f_name), normalize_name(m_name), ldate)
|
||
md_litters_idx[key] = rl
|
||
|
||
# Gender index for parent-ref selection: normalized name → 'male' | 'female'
|
||
# | 'ambiguous'. Drives the gender-aware ranking in pick_parent_ref so a dated
|
||
# but wrong-sex ref (e.g. female „Danielle“) cannot win the father slot over
|
||
# an undated male/unknown one (e.g. „Hagrid Rubeus“).
|
||
gender_idx = {}
|
||
for a in stammbaum_only_animals:
|
||
g = (a.get("gender") or "").lower().strip()
|
||
g = g if g in ("male", "female") else None
|
||
for key in {normalize_name(a.get("name")), normalize_name(get_call_name(a.get("name") or ""))}:
|
||
if not key:
|
||
continue
|
||
if key not in gender_idx:
|
||
gender_idx[key] = g
|
||
elif gender_idx[key] != g:
|
||
gender_idx[key] = "ambiguous"
|
||
|
||
def gender_of_name(name):
|
||
v = gender_idx.get(normalize_name(name))
|
||
return v if v in ("male", "female") else None
|
||
|
||
# Create virtual litters for stammbaum animals
|
||
created_virtual_litters = {}
|
||
for a in stammbaum_only_animals:
|
||
parent_refs = a.get("parentRefs", [])
|
||
child_dob_raw = a.get("dob")
|
||
father_ref = pick_parent_ref(parent_refs, "father", child_dob_raw, gender_of=gender_of_name)
|
||
mother_ref = pick_parent_ref(parent_refs, "mother", child_dob_raw,
|
||
avoid_name=father_ref.get("name") if father_ref else None,
|
||
gender_of=gender_of_name)
|
||
|
||
# Record rejected parent-ref candidates (wrong sex / age-impossible /
|
||
# duplicate of the other role) so this animal's gerbil history can
|
||
# explain which Stammbaum positions were discarded and what won instead.
|
||
pick_discards = []
|
||
pick_discards += explain_pick_rejections(parent_refs, "father", child_dob_raw,
|
||
father_ref, gender_of=gender_of_name)
|
||
pick_discards += explain_pick_rejections(
|
||
parent_refs, "mother", child_dob_raw, mother_ref,
|
||
avoid_name=father_ref.get("name") if father_ref else None,
|
||
gender_of=gender_of_name)
|
||
if pick_discards:
|
||
a["_pick_discards"] = pick_discards
|
||
|
||
a["_mapped_litter_scoped_id"] = None
|
||
if father_ref and mother_ref:
|
||
f_name = get_normalized_gerbil_name(father_ref.get("name"))
|
||
m_name = get_normalized_gerbil_name(mother_ref.get("name"))
|
||
dob_val = parse_date(a.get("dob"))
|
||
|
||
mapped_litter = None
|
||
if dob_val:
|
||
key = (normalize_name(f_name), normalize_name(m_name), dob_val)
|
||
mapped_litter = md_litters_idx.get(key)
|
||
|
||
if mapped_litter:
|
||
a["_mapped_litter_scoped_id"] = mapped_litter["_scoped_id"]
|
||
else:
|
||
v_key = (normalize_name(f_name), normalize_name(m_name), dob_val or "0001-01-01")
|
||
if v_key in created_virtual_litters:
|
||
a["_mapped_litter_scoped_id"] = created_virtual_litters[v_key]
|
||
else:
|
||
l_scoped_id = generate_guid(f"virtual-litter-{v_key[0]}-{v_key[1]}-{v_key[2]}")
|
||
created_virtual_litters[v_key] = l_scoped_id
|
||
a["_mapped_litter_scoped_id"] = l_scoped_id
|
||
|
||
# Try to link parents to actual parsed stammbaum animals
|
||
f_scoped_id = None
|
||
m_scoped_id = None
|
||
f_dob = parse_date(father_ref.get("dob"))
|
||
m_dob = parse_date(mother_ref.get("dob"))
|
||
|
||
for p_cand in stammbaum_only_animals:
|
||
p_gender = str(p_cand.get("gender") or "").lower().strip()
|
||
if p_gender in ["w", "f", "female", "weiblich"]:
|
||
continue
|
||
p_dob = parse_date(p_cand.get("dob"))
|
||
if dob_val and p_dob and p_dob >= dob_val:
|
||
continue
|
||
cand_call_norm = normalize_name(get_call_name(p_cand["name"])) or "unbekannt"
|
||
cand_name_norm = normalize_name(p_cand["name"]) or "unbekannt"
|
||
if cand_call_norm == normalize_name(f_name) or cand_name_norm == normalize_name(f_name):
|
||
if not f_dob or p_dob == f_dob:
|
||
f_scoped_id = generate_guid(f"stammbaum-animal-{p_cand['id']}")
|
||
break
|
||
for p_cand in stammbaum_only_animals:
|
||
p_gender = str(p_cand.get("gender") or "").lower().strip()
|
||
if p_gender in ["m", "male", "männlich"]:
|
||
continue
|
||
p_dob = parse_date(p_cand.get("dob"))
|
||
if dob_val and p_dob and p_dob >= dob_val:
|
||
continue
|
||
cand_call_norm = normalize_name(get_call_name(p_cand["name"])) or "unbekannt"
|
||
cand_name_norm = normalize_name(p_cand["name"]) or "unbekannt"
|
||
if cand_call_norm == normalize_name(m_name) or cand_name_norm == normalize_name(m_name):
|
||
if not m_dob or p_dob == m_dob:
|
||
m_scoped_id = generate_guid(f"stammbaum-animal-{p_cand['id']}")
|
||
break
|
||
|
||
raw_litters.append({
|
||
"Id": l_scoped_id,
|
||
"Name": f"Wurf von {f_name} + {m_name}",
|
||
"Date": dob_val,
|
||
"TotalBorn": None,
|
||
"DeathsWithin8Weeks": None,
|
||
"FatherId": f_scoped_id or generate_guid(f"stammbaum-animal-{normalize_name(f_name)}"),
|
||
"MotherId": m_scoped_id or generate_guid(f"stammbaum-animal-{normalize_name(m_name)}"),
|
||
"ExpectedGoHomeDate": None,
|
||
"Notes": "Pedigree virtual litter",
|
||
"PairingCode": None,
|
||
"ExternalRef": f"virtual-{l_scoped_id}",
|
||
"LitterLetter": None,
|
||
"_father_name": f_name,
|
||
"_mother_name": m_name,
|
||
"_filename": "Stammbaum",
|
||
"_scoped_id": l_scoped_id,
|
||
"_scoped_father_id": f_scoped_id or generate_guid(f"stammbaum-animal-{normalize_name(f_name)}"),
|
||
"_scoped_mother_id": m_scoped_id or generate_guid(f"stammbaum-animal-{normalize_name(m_name)}")
|
||
})
|
||
|
||
# 2. Resolve Contacts globally (deduplicate by normalized name)
|
||
|
||
contact_by_norm_name = {}
|
||
contact_id_map = {} # scoped_old_id -> global_guid
|
||
|
||
for rc in raw_contacts:
|
||
name_val = rc.get("Name") or rc.get("name") or rc.get("FullName") or rc.get("fullName")
|
||
if not name_val:
|
||
first = rc.get("FirstName") or rc.get("firstName")
|
||
last = rc.get("LastName") or rc.get("lastName")
|
||
if first or last:
|
||
name_val = f"{first or ''} {last or ''}".strip()
|
||
|
||
if not name_val:
|
||
continue
|
||
|
||
canon_name, should_keep = get_normalized_contact_name(name_val)
|
||
scoped_id = rc["_scoped_id"]
|
||
if not should_keep:
|
||
if scoped_id:
|
||
contact_id_map[scoped_id] = None
|
||
continue
|
||
|
||
norm_name = normalize_name(canon_name)
|
||
|
||
rc_file = rc.get("_filename")
|
||
if norm_name not in contact_by_norm_name:
|
||
global_guid = generate_guid(f"contact-{norm_name}")
|
||
contact_by_norm_name[norm_name] = {
|
||
"Id": global_guid,
|
||
"Name": canon_name,
|
||
"Email": rc.get("Email") or rc.get("email"),
|
||
"Phone": rc.get("Phone") or rc.get("phone"),
|
||
"Address": rc.get("Address") or rc.get("address"),
|
||
"Notes": rc.get("Notes") or rc.get("notes") or rc.get("Note") or rc.get("note"),
|
||
# Provenance accumulators (consumed below, stripped from helper keys).
|
||
"_source_files": set([rc_file]) if rc_file else set(),
|
||
"_merged_count": 1,
|
||
}
|
||
else:
|
||
gc = contact_by_norm_name[norm_name]
|
||
if not gc["Email"] and (rc.get("Email") or rc.get("email")):
|
||
gc["Email"] = rc.get("Email") or rc.get("email")
|
||
if not gc["Phone"] and (rc.get("Phone") or rc.get("phone")):
|
||
gc["Phone"] = rc.get("Phone") or rc.get("phone")
|
||
if not gc["Address"] and (rc.get("Address") or rc.get("address")):
|
||
gc["Address"] = rc.get("Address") or rc.get("address")
|
||
if not gc["Notes"] and (rc.get("Notes") or rc.get("notes") or rc.get("Note") or rc.get("note")):
|
||
gc["Notes"] = rc.get("Notes") or rc.get("notes") or rc.get("Note") or rc.get("note")
|
||
if rc_file:
|
||
gc["_source_files"].add(rc_file)
|
||
gc["_merged_count"] += 1
|
||
|
||
if scoped_id:
|
||
contact_id_map[scoped_id] = contact_by_norm_name[norm_name]["Id"]
|
||
|
||
resolved_contacts = list(contact_by_norm_name.values())
|
||
print(f"Resolved to {len(resolved_contacts)} unique contacts.")
|
||
|
||
# 3. Process Litters (normalize keys)
|
||
resolved_litters = []
|
||
litter_id_map = {} # scoped_old_id -> new_guid
|
||
litter_by_scoped_id = {}
|
||
|
||
for rl in raw_litters:
|
||
filename = rl.get("_filename")
|
||
name_val = rl.get("Name") or rl.get("name")
|
||
if not name_val:
|
||
name_val = "Wurf"
|
||
|
||
dob_val = parse_date(rl.get("Date") or rl.get("date") or rl.get("DateOfBirth") or rl.get("dateOfBirth"))
|
||
|
||
new_guid = rl["_scoped_id"]
|
||
if not new_guid:
|
||
new_guid = generate_guid(f"litter-{filename}-{name_val}-{dob_val}")
|
||
|
||
scoped_old_id = rl["_scoped_id"]
|
||
if scoped_old_id:
|
||
litter_id_map[scoped_old_id] = new_guid
|
||
|
||
total_born = rl.get("TotalBorn") or rl.get("totalBorn") or rl.get("LitterSize") or rl.get("litterSize") or rl.get("size") or rl.get("totalPups")
|
||
if total_born is not None:
|
||
try:
|
||
total_born = int(total_born)
|
||
except ValueError:
|
||
total_born = None
|
||
|
||
deaths_8w = rl.get("DeathsWithin8Weeks") or rl.get("deathsWithin8Weeks")
|
||
if deaths_8w is not None:
|
||
try:
|
||
deaths_8w = int(deaths_8w)
|
||
except ValueError:
|
||
deaths_8w = None
|
||
elif total_born is not None and rl.get("survived") is not None:
|
||
try:
|
||
deaths_8w = total_born - int(rl.get("survived"))
|
||
except ValueError:
|
||
pass
|
||
|
||
father_name = rl.get("FatherName") or rl.get("fatherName") or rl.get("ParentMaleName") or rl.get("parentMaleName") or rl.get("_father_name")
|
||
mother_name = rl.get("MotherName") or rl.get("motherName") or rl.get("ParentFemaleName") or rl.get("parentFemaleName") or rl.get("_mother_name")
|
||
|
||
raw_ext_ref = rl.get("ExternalRef") or rl.get("externalRef") or rl.get("Id") or rl.get("id")
|
||
ext_ref_scoped = f"{filename}-{raw_ext_ref}" if raw_ext_ref else None
|
||
|
||
l_record = {
|
||
"Id": new_guid,
|
||
"Name": name_val,
|
||
"Date": dob_val,
|
||
"TotalBorn": total_born,
|
||
"DeathsWithin8Weeks": deaths_8w,
|
||
"FatherId": rl["_scoped_father_id"],
|
||
"MotherId": rl["_scoped_mother_id"],
|
||
"ExpectedGoHomeDate": parse_date(rl.get("ExpectedGoHomeDate") or rl.get("expectedGoHomeDate")),
|
||
"Notes": rl.get("Notes") or rl.get("notes") or rl.get("Note") or rl.get("note"),
|
||
"PairingCode": rl.get("PairingCode") or rl.get("pairingCode"),
|
||
"ExternalRef": ext_ref_scoped,
|
||
"LitterLetter": rl.get("LitterLetter") or rl.get("litterLetter"),
|
||
"_father_name": father_name,
|
||
"_mother_name": mother_name,
|
||
"_filename": filename,
|
||
# Provenance accumulators (canonical absorbs these during dedup below).
|
||
"_source_files": set([filename]) if filename else set(),
|
||
"_merged_count": 1,
|
||
# Virtual litters are reconstructed from a Stammbaum chart, not the
|
||
# Wurfchronik — flagged on the raw record's _filename == "Stammbaum".
|
||
"_virtual": filename == "Stammbaum",
|
||
}
|
||
resolved_litters.append(l_record)
|
||
litter_by_scoped_id[new_guid] = l_record
|
||
|
||
print(f"Processed {len(resolved_litters)} litters.")
|
||
|
||
# 3b. Deduplicate litters: same date + compatible parents → merge
|
||
# This handles the "sibling pairing" case: Stammbaum shows the same parental
|
||
# litter twice (once under the father branch, once under the mother branch),
|
||
# generating two separate litter records with the same date but only one of
|
||
# them has FatherId/MotherId resolved.
|
||
litter_canonical_map = {} # old_id -> canonical_id (for dedup within this step)
|
||
|
||
# Group by date for efficiency
|
||
by_date = {}
|
||
for l in resolved_litters:
|
||
by_date.setdefault(l["Date"], []).append(l)
|
||
|
||
litter_dedup_canonical = {} # old_litter_id -> canonical_litter_id
|
||
deduped_litters = []
|
||
|
||
for date_val, group in by_date.items():
|
||
# Partition into compatible subsets
|
||
sub_groups = []
|
||
for l in group:
|
||
placed = False
|
||
for sub in sub_groups:
|
||
if all(litter_compatible(l, member) for member in sub):
|
||
sub.append(l)
|
||
placed = True
|
||
break
|
||
if not placed:
|
||
sub_groups.append([l])
|
||
|
||
for sub in sub_groups:
|
||
if len(sub) == 1:
|
||
deduped_litters.append(sub[0])
|
||
litter_dedup_canonical[sub[0]["Id"]] = sub[0]["Id"]
|
||
continue
|
||
|
||
# Pick the canonical record: prefer the one with parents set
|
||
canonical = next((l for l in sub if l.get("FatherId") or l.get("MotherId")), sub[0])
|
||
for l in sub:
|
||
litter_dedup_canonical[l["Id"]] = canonical["Id"]
|
||
if l is not canonical:
|
||
litter_id_map[l["Id"]] = canonical["Id"]
|
||
canonical["_source_files"] |= l.get("_source_files", set())
|
||
canonical["_merged_count"] += l.get("_merged_count", 1)
|
||
if not l.get("_virtual"):
|
||
canonical["_virtual"] = False
|
||
|
||
deduped_litters.append(canonical)
|
||
if len(sub) > 1:
|
||
merged_names = [l["Id"] for l in sub if l is not canonical]
|
||
print(f"Litter-Dedup: merged {len(sub)} same-date litters on {date_val} → {canonical['Name']} (absorbed: {', '.join(merged_names)})")
|
||
|
||
n_merged = len(resolved_litters) - len(deduped_litters)
|
||
if n_merged:
|
||
print(f"Litter-Dedup: {n_merged} redundant litter record(s) removed.")
|
||
resolved_litters = deduped_litters
|
||
litter_by_scoped_id = {l["Id"]: l for l in resolved_litters}
|
||
|
||
# 4. Normalize and group Gerbils
|
||
|
||
# Helper to lookup litter dates for birth date estimation
|
||
def get_litter_date(l_id):
|
||
if l_id in litter_by_scoped_id:
|
||
d = litter_by_scoped_id[l_id]["Date"]
|
||
if d:
|
||
return d
|
||
return None
|
||
|
||
all_processed_gerbils = []
|
||
for rg in raw_gerbils:
|
||
filename = rg.get("_filename")
|
||
name_val = get_normalized_gerbil_name(rg.get("Name") or rg.get("name") or rg.get("callName"))
|
||
if not name_val:
|
||
name_val = "Unbekannt"
|
||
|
||
dob_val = parse_date(rg.get("DateOfBirth") or rg.get("dateOfBirth"))
|
||
|
||
new_guid = rg["_scoped_id"]
|
||
if not new_guid:
|
||
new_guid = generate_guid(f"gerbil-{filename}-{name_val}-{dob_val}")
|
||
|
||
# Gender normalization
|
||
gender_val = rg.get("Gender") or rg.get("gender") or "unknown"
|
||
gender_val = str(gender_val).lower().strip()
|
||
if gender_val in ["m", "male", "männlich"]:
|
||
gender = "male"
|
||
elif gender_val in ["w", "f", "female", "weiblich"]:
|
||
gender = "female"
|
||
else:
|
||
gender = "unknown"
|
||
|
||
dod_val = parse_date(rg.get("DateOfDeath") or rg.get("dateOfDeath") or rg.get("deathDate"))
|
||
gohome_val = parse_date(rg.get("GoHomeDate") or rg.get("goHomeDate") or rg.get("DateGivenAway") or rg.get("givenAwayDate") or rg.get("DateOfHandover") or rg.get("dateOfHandover") or rg.get("HandoverDate") or rg.get("DateOfSale") or rg.get("dateOfSale"))
|
||
|
||
status_val = rg.get("Status") or rg.get("status") or ""
|
||
status_val = str(status_val).lower().strip()
|
||
status = "Breeding"
|
||
if dod_val or "deceased" in status_val or "verstorben" in status_val or "tod" in status_val or "dead" in status_val:
|
||
status = "Deceased"
|
||
elif gohome_val or "givenaway" in status_val or "abgegeben" in status_val or "verkauft" in status_val or "sold" in status_val:
|
||
status = "GivenAway"
|
||
elif "forsale" in status_val or "abzugeben" in status_val:
|
||
status = "ForSale"
|
||
elif "pet" in status_val or "liebhaber" in status_val:
|
||
status = "Pet"
|
||
else:
|
||
if dod_val:
|
||
status = "Deceased"
|
||
elif gohome_val:
|
||
status = "GivenAway"
|
||
|
||
notes_val = rg.get("Notes") or rg.get("notes") or rg.get("Note") or rg.get("note")
|
||
existing_cod = rg.get("CauseOfDeath") or rg.get("causeOfDeath") or rg.get("DeathCause")
|
||
status, dod_val, cause_of_death_val = parse_death_info(notes_val, status, dod_val, existing_cod)
|
||
|
||
# Explicit resolution for Ken'ichi's cause of death
|
||
if name_val == "Ken'ichi" and dob_val == "2015-03-01":
|
||
cause_of_death_val = "Duftdrüsen-Tumor"
|
||
|
||
color_val = rg.get("ColorVarietyId") or rg.get("colorVarietyId") or rg.get("Color") or rg.get("color") or rg.get("ColorDescription") or rg.get("colorDescription")
|
||
existing_gt = rg.get("Genotype") or rg.get("genotype")
|
||
color_variety_id, genotype_val = resolve_color_and_genotype(
|
||
color_val, existing_gt, variety_map, variety_genotypes
|
||
)
|
||
if color_val and not color_variety_id:
|
||
try:
|
||
uuid.UUID(str(color_val).strip())
|
||
color_variety_id = str(color_val).strip()
|
||
except ValueError:
|
||
print(f"Warning: Unknown color variety '{color_val}' for gerbil '{name_val}' on page {filename}")
|
||
|
||
origin_cid = rg["_scoped_origin_cid"]
|
||
receiver_cid = rg["_scoped_receiver_cid"]
|
||
|
||
if origin_cid in contact_id_map:
|
||
origin_cid = contact_id_map[origin_cid]
|
||
if receiver_cid in contact_id_map:
|
||
receiver_cid = contact_id_map[receiver_cid]
|
||
|
||
is_resident = rg.get("IsResident") or rg.get("isResident")
|
||
if is_resident is None:
|
||
is_resident = True
|
||
else:
|
||
is_resident = str(is_resident).lower() == "true"
|
||
|
||
traits = rg.get("CharacterTraits") or rg.get("characterTraits") or []
|
||
if not isinstance(traits, list):
|
||
traits = [str(traits)]
|
||
char_note = rg.get("CharacterNote") or rg.get("characterNote")
|
||
|
||
is_deaf = rg.get("IsDeaf") or rg.get("isDeaf")
|
||
if is_deaf is not None:
|
||
is_deaf = str(is_deaf).lower() == "true"
|
||
|
||
old_litter_id = rg["_scoped_litter_id"]
|
||
|
||
raw_ext_ref = rg.get("ExternalRef") or rg.get("externalRef") or rg.get("Id") or rg.get("id")
|
||
ext_ref_scoped = f"{filename}-{raw_ext_ref}" if raw_ext_ref else None
|
||
|
||
# Estimated effective date of birth for conflict checking
|
||
eff_dob_val = dob_val
|
||
if not eff_dob_val and old_litter_id:
|
||
# Try to get litter date if litter was resolved
|
||
mapped_l_id = litter_id_map.get(old_litter_id)
|
||
if mapped_l_id:
|
||
eff_dob_val = get_litter_date(mapped_l_id)
|
||
|
||
# Determine explicit or litter-derived birth date (None if unknown/parent)
|
||
birth_date = dob_val
|
||
if not birth_date and old_litter_id:
|
||
mapped_l_id = litter_id_map.get(old_litter_id)
|
||
if mapped_l_id:
|
||
birth_date = get_litter_date(mapped_l_id)
|
||
|
||
# If still no effective date, use the estimated page date
|
||
if not eff_dob_val:
|
||
eff_dob_val = days_to_date(file_dates[filename])
|
||
|
||
raw_breeder = rg.get("OriginBreeder") or rg.get("originBreeder")
|
||
if raw_breeder:
|
||
norm_b, keep_b = get_normalized_contact_name(raw_breeder)
|
||
raw_breeder = norm_b if keep_b else None
|
||
|
||
parent_refs = []
|
||
if old_litter_id:
|
||
rl = next((l for l in raw_litters if l.get("_scoped_id") == old_litter_id), None)
|
||
if rl:
|
||
f_name = rl.get("FatherName") or rl.get("fatherName") or rl.get("ParentMaleName") or rl.get("parentMaleName") or rl.get("_father_name")
|
||
m_name = rl.get("MotherName") or rl.get("motherName") or rl.get("ParentFemaleName") or rl.get("parentFemaleName") or rl.get("_mother_name")
|
||
if f_name:
|
||
parent_refs.append({"name": f_name, "roleGuess": "father"})
|
||
if m_name:
|
||
parent_refs.append({"name": m_name, "roleGuess": "mother"})
|
||
|
||
all_processed_gerbils.append({
|
||
"Id": new_guid,
|
||
"Name": name_val,
|
||
"Gender": gender,
|
||
"Status": status,
|
||
"LitterId": old_litter_id, # mapped later
|
||
"OriginContactId": origin_cid,
|
||
"ReceiverContactId": receiver_cid,
|
||
"EnclosureId": None,
|
||
"ColorVarietyId": color_variety_id,
|
||
"DateOfBirth": dob_val,
|
||
"DateOfDeath": dod_val,
|
||
"CauseOfDeath": cause_of_death_val,
|
||
"GoHomeDate": gohome_val,
|
||
"Genotype": genotype_val,
|
||
"Notes": rg.get("Notes") or rg.get("notes") or rg.get("Note") or rg.get("note"),
|
||
"ImportSource": rg.get("ImportSource") or rg.get("importSource") or filename,
|
||
"ExternalRef": ext_ref_scoped,
|
||
"RawImportData": rg.get("RawImportData") or rg.get("rawImportData") or json.dumps({"colorDescription": color_val if not color_variety_id else None}),
|
||
"OriginBreeder": raw_breeder or ("Zucht der kleinen Chaoten" if is_resident else None),
|
||
"NameSearch": normalize_name(name_val),
|
||
"CharacterTraits": traits,
|
||
"CharacterNote": char_note,
|
||
"IsDeaf": is_deaf,
|
||
"IsResident": is_resident,
|
||
"parentRefs": parent_refs,
|
||
"_photos": rg.get("photos", []),
|
||
"_old_scoped_litter_id": old_litter_id,
|
||
"_eff_dob": eff_dob_val,
|
||
"_birth_date": birth_date,
|
||
"_filename": filename,
|
||
"_old_id": rg.get("Id") or rg.get("id")
|
||
})
|
||
|
||
# Build the discard list for one stammbaum animal: rejected parent-ref
|
||
# candidates (pick_parent_ref) plus a manual DOB-remap decision, if any.
|
||
def _stammbaum_discards(a):
|
||
discards = list(a.get("_pick_discards", []))
|
||
remap = a.get("dobRemap")
|
||
if remap and remap.get("original") and remap.get("corrected"):
|
||
orig = _de_date(parse_date(remap["original"])) or remap["original"]
|
||
corr = _de_date(parse_date(remap["corrected"])) or remap["corrected"]
|
||
if orig != corr:
|
||
discards.append({
|
||
"label": PROV_FIELD_LABELS["DateOfBirth"],
|
||
"value": orig,
|
||
"reason": "per manueller Entscheidung korrigiert",
|
||
"replacement": corr,
|
||
})
|
||
return discards
|
||
|
||
# Map and append stammbaum animals to all_processed_gerbils
|
||
for a in stammbaum_only_animals:
|
||
a_id = a["id"]
|
||
name_val = get_normalized_gerbil_name(a["name"])
|
||
|
||
gender_val = str(a.get("gender") or "").lower().strip()
|
||
if gender_val in ["m", "male"]:
|
||
gender = "male"
|
||
elif gender_val in ["w", "f", "female"]:
|
||
gender = "female"
|
||
else:
|
||
gender = "unknown"
|
||
|
||
dob_val = parse_date(a.get("dob"))
|
||
dod_val = parse_date(a.get("death"))
|
||
|
||
status = "Breeding"
|
||
if dod_val:
|
||
status = "Deceased"
|
||
elif a_id not in stammbaum_resident_ids:
|
||
status = "GivenAway"
|
||
else:
|
||
if dob_val:
|
||
try:
|
||
dt_dob = datetime.strptime(dob_val, "%Y-%m-%d")
|
||
dt_now = datetime.now()
|
||
age_years = (dt_now - dt_dob).days / 365.25
|
||
if age_years >= 6.0:
|
||
status = "Deceased"
|
||
except Exception:
|
||
pass
|
||
|
||
# Color variety mapping
|
||
existing_gt = a["genotype"]["rawGenotype"] if a["genotype"]["rawGenotype"] else None
|
||
color_variety_id, genotype_val = resolve_color_and_genotype(
|
||
a.get("farbschlag"), existing_gt, variety_map, variety_genotypes
|
||
)
|
||
if not color_variety_id:
|
||
for fbv in a.get("farbschlagVariants", []):
|
||
cv_id, gt_val = resolve_color_and_genotype(fbv, existing_gt, variety_map, variety_genotypes)
|
||
if cv_id:
|
||
color_variety_id = cv_id
|
||
genotype_val = gt_val
|
||
break
|
||
|
||
# Contact mapping
|
||
origin_cid = None
|
||
b_name = (a.get("breeder") or "").strip()
|
||
z_name = (a.get("zucht") or "").strip()
|
||
if b_name:
|
||
origin_cid = generate_guid(f"stammbaum-contact-{normalize_name(b_name)}")
|
||
elif z_name:
|
||
origin_cid = generate_guid(f"stammbaum-contact-{normalize_name(z_name)}")
|
||
|
||
# Map to resolved global contact GUID
|
||
if origin_cid in contact_id_map:
|
||
origin_cid = contact_id_map[origin_cid]
|
||
else:
|
||
origin_cid = None
|
||
|
||
|
||
scoped_id = generate_guid(f"stammbaum-animal-{a_id}")
|
||
scoped_litter_id = a.get("_mapped_litter_scoped_id")
|
||
|
||
is_deaf = a["genotype"].get("deaf")
|
||
|
||
stammbaum_breeder = a.get("breeder") if a.get("breeder") else (a.get("zucht") if a.get("zucht") else None)
|
||
if stammbaum_breeder:
|
||
norm_b, keep_b = get_normalized_contact_name(stammbaum_breeder)
|
||
stammbaum_breeder = norm_b if keep_b else None
|
||
|
||
all_processed_gerbils.append({
|
||
"Id": scoped_id,
|
||
"Name": name_val,
|
||
"Gender": gender,
|
||
"Status": status,
|
||
"LitterId": scoped_litter_id, # mapped later in step 5
|
||
"OriginContactId": origin_cid,
|
||
"ReceiverContactId": None,
|
||
"EnclosureId": None,
|
||
"ColorVarietyId": color_variety_id,
|
||
"DateOfBirth": dob_val,
|
||
"DateOfDeath": dod_val,
|
||
"CauseOfDeath": "Duftdrüsen-Tumor" if name_val == "Ken'ichi" and dob_val == "2015-03-01" else None,
|
||
"GoHomeDate": None,
|
||
"Genotype": genotype_val,
|
||
"Notes": None,
|
||
"ImportSource": ", ".join(a.get("sourceFiles", [])),
|
||
"ExternalRef": f"stammbaum-{a_id}",
|
||
"RawImportData": json.dumps({
|
||
"rawGenotype": a["genotype"]["rawGenotype"],
|
||
"unmappedTokens": a["genotype"]["unmappedTokens"],
|
||
"breederText": a.get("breeder", "")
|
||
}, ensure_ascii=False),
|
||
"OriginBreeder": stammbaum_breeder,
|
||
"NameSearch": normalize_name(name_val),
|
||
"CharacterTraits": [],
|
||
"CharacterNote": None,
|
||
"IsDeaf": is_deaf,
|
||
"IsResident": a_id in stammbaum_resident_ids,
|
||
"parentRefs": a.get("parentRefs", []),
|
||
"_photos": a.get("photos", []),
|
||
"_conflict": bool(a.get("conflict")),
|
||
"_resolved_by_decision": bool(a.get("resolvedByDecision")),
|
||
"_old_scoped_litter_id": scoped_litter_id,
|
||
"_eff_dob": dob_val or "2010-01-01",
|
||
"_birth_date": dob_val,
|
||
"_filename": a.get("sourceFiles", ["Stammbaum"])[0],
|
||
"_old_id": a_id,
|
||
# Rejected Stammbaum parent-ref candidates for this animal (filled by
|
||
# pick_parent_ref above) — surfaced in this gerbil's discard history.
|
||
"_discarded": _stammbaum_discards(a),
|
||
})
|
||
|
||
# Map and append docx animals to all_processed_gerbils
|
||
for idx, da in enumerate(docx_animals):
|
||
name_val = get_normalized_gerbil_name(da["name"])
|
||
gender = da["gender"]
|
||
|
||
dob_val = parse_date(da.get("litterDob"))
|
||
dod_val = parse_date(da.get("deathDate"))
|
||
gohome_val = parse_date(da.get("abgabeDate"))
|
||
|
||
# Status precedence
|
||
status = "Breeding"
|
||
if dod_val:
|
||
status = "Deceased"
|
||
elif gohome_val or da.get("owner"):
|
||
status = "GivenAway"
|
||
|
||
# Color variety mapping
|
||
color_variety_id, genotype_val = resolve_color_and_genotype(
|
||
da.get("farbschlag"), None, variety_map, variety_genotypes
|
||
)
|
||
|
||
# Contact mapping (buyer)
|
||
receiver_cid = None
|
||
o_name = (da.get("owner") or "").strip()
|
||
if o_name:
|
||
receiver_cid = generate_guid(f"docx-contact-{normalize_name(o_name)}")
|
||
if receiver_cid in contact_id_map:
|
||
receiver_cid = contact_id_map[receiver_cid]
|
||
else:
|
||
receiver_cid = None
|
||
|
||
# Scoped ID
|
||
scoped_id = generate_guid(f"docx-animal-{idx}-{normalize_name(name_val)}-{dob_val or '0001-01-01'}")
|
||
|
||
# Litter ID mapping
|
||
ws_code = da.get("wsCode")
|
||
scoped_litter_id = docx_litter_id_map.get((ws_code, dob_val))
|
||
|
||
|
||
# Raw import details
|
||
raw_import_payload = json.dumps({
|
||
"abgabeWeight": da.get("abgabeWeight", ""),
|
||
"deathCause": da.get("deathCause", ""),
|
||
"partnerName": da.get("partnerName", ""),
|
||
"partnerDob": da.get("partnerDob", "")
|
||
}, ensure_ascii=False)
|
||
|
||
# Residents: if sold/given away, it's not a resident
|
||
is_resident = not bool(o_name)
|
||
|
||
parent_refs = []
|
||
if scoped_litter_id:
|
||
dl = next((l for l in docx_litters if docx_litter_id_map.get((l["wsCode"], parse_date(l["dob"]))) == scoped_litter_id), None)
|
||
if dl:
|
||
if dl.get("fatherName"):
|
||
parent_refs.append({"name": dl["fatherName"], "roleGuess": "father"})
|
||
if dl.get("motherName"):
|
||
parent_refs.append({"name": dl["motherName"], "roleGuess": "mother"})
|
||
|
||
all_processed_gerbils.append({
|
||
"Id": scoped_id,
|
||
"Name": name_val,
|
||
"Gender": gender,
|
||
"Status": status,
|
||
"LitterId": scoped_litter_id,
|
||
"OriginContactId": None,
|
||
"ReceiverContactId": receiver_cid, # mapped in step 5
|
||
"EnclosureId": None,
|
||
"ColorVarietyId": color_variety_id,
|
||
"DateOfBirth": dob_val,
|
||
"DateOfDeath": dod_val,
|
||
"CauseOfDeath": da.get("deathCause"),
|
||
"GoHomeDate": gohome_val,
|
||
"Genotype": genotype_val,
|
||
"Notes": None,
|
||
"ImportSource": "Wurfchronik-Detail.docx",
|
||
"ExternalRef": f"docx-{idx}-{normalize_name(name_val)}-{dob_val or '0001-01-01'}",
|
||
"RawImportData": raw_import_payload,
|
||
"OriginBreeder": "Zucht der kleinen Chaoten",
|
||
"NameSearch": normalize_name(name_val),
|
||
"CharacterTraits": [],
|
||
"CharacterNote": None,
|
||
"IsDeaf": None,
|
||
"IsResident": is_resident,
|
||
"parentRefs": parent_refs,
|
||
"_photos": da.get("photos", []),
|
||
"_old_scoped_litter_id": scoped_litter_id,
|
||
"_eff_dob": dob_val or "2020-01-01",
|
||
"_birth_date": dob_val,
|
||
"_filename": "Wurfchronik-Detail.docx",
|
||
"_old_id": name_val
|
||
})
|
||
|
||
# Build parenting dates lookup using old scoped IDs
|
||
parent_litter_dates = {}
|
||
|
||
for l in resolved_litters:
|
||
ld = l["Date"]
|
||
if ld:
|
||
for pid in [l["FatherId"], l["MotherId"]]:
|
||
if pid:
|
||
parent_litter_dates.setdefault(pid, []).append(ld)
|
||
|
||
def are_compatible(g1, g2):
|
||
# Must have same gender (or one unknown)
|
||
if g1["Gender"] != "unknown" and g2["Gender"] != "unknown" and g1["Gender"] != g2["Gender"]:
|
||
bd1 = g1.get("_birth_date")
|
||
bd2 = g2.get("_birth_date")
|
||
if not (bd1 and bd2 and bd1 == bd2):
|
||
return False
|
||
|
||
bd1 = g1.get("_birth_date")
|
||
bd2 = g2.get("_birth_date")
|
||
|
||
# New rule: if name and parents match, they are compatible regardless of DOB!
|
||
p1 = g1.get("parentRefs", [])
|
||
p2 = g2.get("parentRefs", [])
|
||
f1 = next((p["name"] for p in p1 if p.get("roleGuess") == "father"), "")
|
||
m1 = next((p["name"] for p in p1 if p.get("roleGuess") == "mother"), "")
|
||
f2 = next((p["name"] for p in p2 if p.get("roleGuess") == "father"), "")
|
||
m2 = next((p["name"] for p in p2 if p.get("roleGuess") == "mother"), "")
|
||
|
||
parents_match = False
|
||
if f1 and f2 and m1 and m2:
|
||
if normalize_name(f1) == normalize_name(f2) and normalize_name(m1) == normalize_name(m2):
|
||
parents_match = True
|
||
|
||
if not parents_match:
|
||
# If both have explicit birth dates, they must match within 30 days
|
||
if bd1 and bd2:
|
||
days1 = date_to_days(bd1)
|
||
days2 = date_to_days(bd2)
|
||
if days1 is not None and days2 is not None:
|
||
if abs(days1 - days2) > 30:
|
||
return False
|
||
|
||
# If g1 has birth date, and g2 has parenting dates, birth date must be before parenting dates
|
||
p_dates2 = parent_litter_dates.get(g2["Id"], [])
|
||
if bd1:
|
||
for pd in p_dates2:
|
||
if pd <= bd1: # Can't have litter before or on birth date
|
||
return False
|
||
|
||
p_dates1 = parent_litter_dates.get(g1["Id"], [])
|
||
if bd2:
|
||
for pd in p_dates1:
|
||
if pd <= bd2:
|
||
return False
|
||
|
||
return True
|
||
|
||
def build_provenance(records, best_g, extra_notes=None, field_source=None,
|
||
field_discards=None):
|
||
"""Aggregate data-provenance across every raw record merged into one
|
||
resolved gerbil. Returns a JSON string (stored on the Gerbil entity as a
|
||
nullable text column) so the Rennmausakte can show where the entry came
|
||
from. `records` is the list of raw records that became this gerbil;
|
||
`best_g` is the chosen primary record. `field_source` maps a significant
|
||
field name (DateOfBirth/Genotype/…) to the raw record that supplied its
|
||
final value, so the history can name the exact file for each fact."""
|
||
source_files = set()
|
||
from_wurfchronik = False
|
||
any_conflict = False
|
||
any_decision = False
|
||
for r in records:
|
||
source_files |= _record_source_files(r)
|
||
for f in _record_source_files(r):
|
||
if "wurfchronik" in f.lower():
|
||
from_wurfchronik = True
|
||
if r.get("_conflict"):
|
||
any_conflict = True
|
||
if r.get("_resolved_by_decision"):
|
||
any_decision = True
|
||
|
||
notes = []
|
||
merged_count = len(records)
|
||
if merged_count > 1:
|
||
notes.append(f"aus {merged_count} Datensätzen zusammengeführt")
|
||
if any_decision:
|
||
notes.append("per manueller Entscheidung zugeordnet")
|
||
if any_conflict:
|
||
notes.append("Konflikt per Entscheidung gelöst")
|
||
|
||
# Parent derivation: surface the strongest parentRef method/confidence
|
||
# the primary record carries (chart-position etc.).
|
||
parent_method = None
|
||
parent_confidence = None
|
||
for ref in best_g.get("parentRefs", []) or []:
|
||
if ref.get("method") and not parent_method:
|
||
parent_method = ref.get("method")
|
||
if ref.get("confidence") and not parent_confidence:
|
||
parent_confidence = ref.get("confidence")
|
||
|
||
if extra_notes:
|
||
for n in extra_notes:
|
||
if n and n not in notes:
|
||
notes.append(n)
|
||
|
||
history = _build_gerbil_history(
|
||
records, best_g, field_source or {},
|
||
parent_method=parent_method,
|
||
any_decision=any_decision,
|
||
any_conflict=any_conflict,
|
||
conflict_notes=extra_notes or [],
|
||
field_discards=field_discards,
|
||
parent_discards=best_g.get("_discarded"),
|
||
)
|
||
|
||
return build_entity_provenance(
|
||
source_files,
|
||
merged_count,
|
||
notes=notes,
|
||
from_wurfchronik=from_wurfchronik,
|
||
extra={"parentMethod": parent_method, "parentConfidence": parent_confidence},
|
||
history=history,
|
||
)
|
||
|
||
# Group gerbils by name to perform deduplication
|
||
gerbil_groups = {}
|
||
for g in all_processed_gerbils:
|
||
name_key = get_dedup_name_key(g["Name"])
|
||
if not name_key:
|
||
name_key = "unbekannt"
|
||
gerbil_groups.setdefault(name_key, []).append(g)
|
||
|
||
resolved_gerbils = []
|
||
gerbil_id_map = {} # old_scoped_id -> final_id
|
||
|
||
color_keys = set(variety_map.keys())
|
||
|
||
for name_key, group in gerbil_groups.items():
|
||
is_placeholder = (
|
||
name_key in color_keys or
|
||
any(p in name_key for p in ["unbekannt", "unbenannt", "baby", "welpe", "jungtier", "unknown", "welpen"]) or
|
||
len(name_key) <= 2
|
||
)
|
||
|
||
if is_placeholder:
|
||
# Placeholders: do NOT merge, keep all separate
|
||
for g in group:
|
||
# Provenance is built in a final pass (after parent resolution),
|
||
# so parent-link discards land in the right gerbil's history.
|
||
g["_prov_args"] = ([g], g, None, None, None)
|
||
resolved_gerbils.append(g)
|
||
gerbil_id_map[g["Id"]] = g["Id"]
|
||
continue
|
||
|
||
# Partition group into compatible subsets
|
||
sub_groups = []
|
||
for g in group:
|
||
placed = False
|
||
for sub in sub_groups:
|
||
if all(are_compatible(g, member) for member in sub):
|
||
sub.append(g)
|
||
placed = True
|
||
break
|
||
if not placed:
|
||
sub_groups.append([g])
|
||
|
||
# Merge each partition sub-group into a single gerbil
|
||
for sub in sub_groups:
|
||
if len(sub) == 1:
|
||
g = sub[0]
|
||
g["_prov_args"] = ([g], g, None, None, None)
|
||
resolved_gerbils.append(g)
|
||
gerbil_id_map[g["Id"]] = g["Id"]
|
||
continue
|
||
|
||
# Find the best primary record to merge into
|
||
best_g = None
|
||
best_score = -1
|
||
for g in sub:
|
||
score = 0
|
||
if g["_old_scoped_litter_id"]: score += 10
|
||
if g["DateOfBirth"]: score += 5
|
||
if g["Genotype"]: score += 3
|
||
if g["ColorVarietyId"]: score += 2
|
||
if g["ImportSource"] and "stammbaum" in g["ImportSource"].lower(): score += 20
|
||
if g["Notes"] and not any(kw in g["Notes"].lower() for kw in ["parent", "mutter", "vater", "dam", "sire"]): score += 1
|
||
if len(g["Name"]) > len(name_key) + 5: # likely has clan suffix
|
||
score += 15
|
||
if score > best_score:
|
||
best_score = score
|
||
best_g = g
|
||
|
||
# Merge fields
|
||
merged_notes = []
|
||
if best_g["Notes"]:
|
||
merged_notes.append(best_g["Notes"])
|
||
|
||
# Carry over discard records (rejected parent-refs / dob remaps) from
|
||
# every merged record so none are lost when the primary changes.
|
||
merged_discards = list(best_g.get("_discarded") or [])
|
||
for g in sub:
|
||
if g is best_g:
|
||
continue
|
||
for d in (g.get("_discarded") or []):
|
||
if d not in merged_discards:
|
||
merged_discards.append(d)
|
||
best_g["_discarded"] = merged_discards
|
||
|
||
# Merge photos
|
||
merged_photos = list(best_g.get("_photos", []))
|
||
|
||
# Track sources for debugging
|
||
sources = [best_g["_filename"]]
|
||
|
||
# Per-field file attribution: which raw record supplied each final
|
||
# field value. Seed with best_g for every field it already carries;
|
||
# the fill loop and voting loop update it as winners change.
|
||
ATTRIB_FIELDS = ["DateOfBirth", "DateOfDeath", "Gender", "Genotype",
|
||
"ColorVarietyId", "Name"]
|
||
field_source = {}
|
||
for fld in ATTRIB_FIELDS:
|
||
v = best_g.get(fld)
|
||
if v and v != "unknown":
|
||
field_source[fld] = best_g
|
||
|
||
for g in sub:
|
||
if g == best_g:
|
||
continue
|
||
gerbil_id_map[g["Id"]] = best_g["Id"]
|
||
sources.append(g["_filename"])
|
||
|
||
for ph in g.get("_photos", []):
|
||
if ph not in merged_photos:
|
||
merged_photos.append(ph)
|
||
|
||
if not best_g.get("_old_scoped_litter_id") and g.get("_old_scoped_litter_id"):
|
||
best_g["_old_scoped_litter_id"] = g["_old_scoped_litter_id"]
|
||
if not best_g["LitterId"] and g["LitterId"]:
|
||
best_g["LitterId"] = g["LitterId"]
|
||
if not best_g["DateOfBirth"] and g["DateOfBirth"]:
|
||
best_g["DateOfBirth"] = g["DateOfBirth"]
|
||
field_source["DateOfBirth"] = g
|
||
if not best_g["DateOfDeath"] and g["DateOfDeath"]:
|
||
best_g["DateOfDeath"] = g["DateOfDeath"]
|
||
field_source["DateOfDeath"] = g
|
||
if not best_g["CauseOfDeath"] and g["CauseOfDeath"]:
|
||
best_g["CauseOfDeath"] = g["CauseOfDeath"]
|
||
if not best_g["GoHomeDate"] and g["GoHomeDate"]:
|
||
best_g["GoHomeDate"] = g["GoHomeDate"]
|
||
if not best_g["Genotype"] and g["Genotype"]:
|
||
best_g["Genotype"] = g["Genotype"]
|
||
field_source["Genotype"] = g
|
||
if not best_g["ColorVarietyId"] and g["ColorVarietyId"]:
|
||
best_g["ColorVarietyId"] = g["ColorVarietyId"]
|
||
field_source["ColorVarietyId"] = g
|
||
if not best_g["OriginContactId"] and g["OriginContactId"]:
|
||
best_g["OriginContactId"] = g["OriginContactId"]
|
||
if not best_g["ReceiverContactId"] and g["ReceiverContactId"]:
|
||
best_g["ReceiverContactId"] = g["ReceiverContactId"]
|
||
if g["IsResident"]:
|
||
best_g["IsResident"] = True
|
||
|
||
# Reconcile Gender: prefer a known gender over unknown, and prefer stammbaum over other sources
|
||
if best_g["Gender"] == "unknown" and g["Gender"] != "unknown":
|
||
best_g["Gender"] = g["Gender"]
|
||
field_source["Gender"] = g
|
||
elif best_g["Gender"] != "unknown" and g["Gender"] != "unknown" and best_g["Gender"] != g["Gender"]:
|
||
if g["ImportSource"] and "stammbaum" in g["ImportSource"].lower():
|
||
if not best_g["ImportSource"] or "stammbaum" not in best_g["ImportSource"].lower():
|
||
best_g["Gender"] = g["Gender"]
|
||
field_source["Gender"] = g
|
||
|
||
# Status precedence: Deceased > GivenAway > Breeding/Pet
|
||
if g["Status"] == "Deceased":
|
||
best_g["Status"] = "Deceased"
|
||
elif g["Status"] == "GivenAway" and best_g["Status"] not in ["Deceased"]:
|
||
best_g["Status"] = "GivenAway"
|
||
|
||
if g["Notes"] and g["Notes"] not in merged_notes:
|
||
# Ignore redundant dummy notes
|
||
if not any(kw in g["Notes"].lower() for kw in ["parent listed", "mutter von", "vater von", "dam of", "sire of"]):
|
||
merged_notes.append(g["Notes"])
|
||
|
||
# Display formatter per field for discard/replacement lines.
|
||
def _disp(field, val):
|
||
if val is None or val == "" or val == "unknown":
|
||
return None
|
||
if field in ("DateOfBirth", "DateOfDeath"):
|
||
return _de_date(val)
|
||
if field == "Gender":
|
||
return _de_gender(val)
|
||
if field == "ColorVarietyId":
|
||
return variety_id_to_name.get(val, "Farbschlag")
|
||
return str(val)
|
||
|
||
# Reconcile fields based on number of source files supporting them
|
||
conflict_notes = []
|
||
field_discards = []
|
||
for field in ["DateOfBirth", "DateOfDeath", "Gender", "Genotype", "ColorVarietyId"]:
|
||
votes = {}
|
||
for g in sub:
|
||
val = g.get(field)
|
||
if val and val != "unknown":
|
||
# count source files
|
||
sources_count = len(str(g.get("ImportSource") or "").split(","))
|
||
votes[val] = votes.get(val, 0) + sources_count
|
||
if votes:
|
||
best_val = max(votes, key=votes.get)
|
||
# If the records disagreed on a field, the merge had to pick a
|
||
# winner — record that as a provenance note.
|
||
if len(votes) > 1:
|
||
conflict_notes.append(
|
||
f"Konflikt bei {PROV_FIELD_LABELS[field]} per Mehrheitsentscheidung gelöst"
|
||
)
|
||
best_g[field] = best_val
|
||
# Attribute the winning value to a record that actually holds
|
||
# it, so the history names the right file.
|
||
winner = next((g for g in sub if g.get(field) == best_val), None)
|
||
if winner is not None:
|
||
field_source[field] = winner
|
||
# Record each LOSING value: which file it came from, that it
|
||
# was discarded as differing, and that the majority value (and
|
||
# its file) was used instead.
|
||
if len(votes) > 1:
|
||
repl_disp = _disp(field, best_val)
|
||
repl_file = _primary_file_of(winner) if winner is not None else None
|
||
for lose_val in votes:
|
||
if lose_val == best_val:
|
||
continue
|
||
loser = next((g for g in sub if g.get(field) == lose_val), None)
|
||
field_discards.append({
|
||
"label": PROV_FIELD_LABELS.get(field, field),
|
||
"value": _disp(field, lose_val),
|
||
"file": _primary_file_of(loser) if loser is not None else None,
|
||
"reason": "abweichend",
|
||
"replacement": repl_disp,
|
||
"repl_file": repl_file,
|
||
"replacement_note": "Mehrheit",
|
||
})
|
||
# Keep helper fields in sync if we changed DateOfBirth
|
||
if field == "DateOfBirth":
|
||
best_g["_birth_date"] = best_val
|
||
best_g["_eff_dob"] = best_val or "2010-01-01"
|
||
|
||
if merged_notes:
|
||
best_g["Notes"] = " | ".join(merged_notes)
|
||
|
||
best_g["_photos"] = merged_photos
|
||
# Defer provenance to a final pass so parent-link discards (added
|
||
# later by the parent-age / role-normalization passes) are included.
|
||
best_g["_prov_args"] = (sub, best_g, conflict_notes, field_source, field_discards)
|
||
|
||
# Print merge trace
|
||
print(f"Deduplicated same-animal name '{best_g['Name']}': merged {len(sub)} entries across files: {', '.join(sources)}")
|
||
|
||
resolved_gerbils.append(best_g)
|
||
gerbil_id_map[best_g["Id"]] = best_g["Id"]
|
||
|
||
print(f"Deduplicated to {len(resolved_gerbils)} unique gerbil records.")
|
||
|
||
# Apply age-based death threshold (6.0 years) to all resolved gerbils
|
||
dt_now = datetime.now()
|
||
for g in resolved_gerbils:
|
||
if g.get("Status") != "Deceased" and g.get("Status") != "GivenAway":
|
||
if not g.get("DateOfDeath") and not g.get("ReceiverContactId"):
|
||
dob_str = g.get("DateOfBirth")
|
||
if dob_str:
|
||
try:
|
||
dt_dob = datetime.strptime(dob_str, "%Y-%m-%d")
|
||
age_years = (dt_now - dt_dob).days / 365.25
|
||
if age_years >= 6.0:
|
||
g["Status"] = "Deceased"
|
||
except Exception:
|
||
pass
|
||
|
||
# 5. Map Gerbils to Litters
|
||
for g in resolved_gerbils:
|
||
old_lid = g["_old_scoped_litter_id"]
|
||
l_guid = litter_id_map.get(old_lid)
|
||
g["LitterId"] = l_guid
|
||
|
||
if l_guid and l_guid in litter_by_scoped_id:
|
||
l = litter_by_scoped_id[l_guid]
|
||
l_date = l.get("Date")
|
||
if l_date:
|
||
if not g.get("DateOfBirth"):
|
||
g["DateOfBirth"] = l_date
|
||
g["_birth_date"] = l_date
|
||
g["_eff_dob"] = l_date
|
||
|
||
# Apply age-based death threshold (6.0 years) to all resolved gerbils (including newly backfilled ones)
|
||
if g.get("Status") != "Deceased" and g.get("Status") != "GivenAway":
|
||
if not g.get("DateOfDeath") and not g.get("ReceiverContactId"):
|
||
dob_str = g.get("DateOfBirth")
|
||
if dob_str:
|
||
try:
|
||
dt_dob = datetime.strptime(dob_str, "%Y-%m-%d")
|
||
age_years = (dt_now - dt_dob).days / 365.25
|
||
if age_years >= 6.0:
|
||
g["Status"] = "Deceased"
|
||
except Exception:
|
||
pass
|
||
|
||
# Clean helper fields
|
||
del g["_old_scoped_litter_id"]
|
||
del g["_eff_dob"]
|
||
if "_birth_date" in g:
|
||
del g["_birth_date"]
|
||
del g["_filename"]
|
||
del g["_old_id"]
|
||
g.pop("_conflict", None)
|
||
g.pop("_resolved_by_decision", None)
|
||
|
||
# Gather final valid gerbil IDs
|
||
valid_gerbil_ids = {g["Id"] for g in resolved_gerbils}
|
||
|
||
# Create name lookup for resolved gerbils
|
||
gerbil_by_norm_name = {}
|
||
for g in resolved_gerbils:
|
||
n_key = normalize_name(g["Name"])
|
||
gerbil_by_norm_name.setdefault(n_key, []).append(g)
|
||
|
||
# Also index by call-name to resolve parents who are only listed by call-name
|
||
c_key = normalize_name(get_call_name(g["Name"]))
|
||
if c_key != n_key:
|
||
gerbil_by_norm_name.setdefault(c_key, []).append(g)
|
||
|
||
# Map raw Guid if present (convert if old_id mapped to new_guid)
|
||
for l in resolved_litters:
|
||
if l["FatherId"] in gerbil_id_map:
|
||
l["FatherId"] = gerbil_id_map[l["FatherId"]]
|
||
if l["MotherId"] in gerbil_id_map:
|
||
l["MotherId"] = gerbil_id_map[l["MotherId"]]
|
||
|
||
# Clean foreign keys that do not point to a valid gerbil
|
||
if l["FatherId"] and l["FatherId"] not in valid_gerbil_ids:
|
||
l["FatherId"] = None
|
||
if l["MotherId"] and l["MotherId"] not in valid_gerbil_ids:
|
||
l["MotherId"] = None
|
||
|
||
# Parent Resolver (Global Name Matching)
|
||
resolved_fathers = 0
|
||
resolved_mothers = 0
|
||
|
||
gerbil_by_id_final = {g["Id"]: g for g in resolved_gerbils}
|
||
|
||
def _final_gender(gid):
|
||
g = gerbil_by_id_final.get(gid)
|
||
return g["Gender"] if g else None
|
||
|
||
def _resolve_name(name, prefer_gender, litter_date):
|
||
"""Resolve a parent name to the best matching final gerbil.
|
||
|
||
Gender is a PREFERENCE, not a hard filter: a reversed parent (e.g. a
|
||
female listed in the father position, as the Stammbaum often does) still
|
||
resolves to a gerbil — the role is corrected afterwards by gender. This
|
||
is what previously left FatherId/MotherId null (the candidate was
|
||
filtered out for having the "wrong" gender for its slot).
|
||
"""
|
||
if not name:
|
||
return None
|
||
cands = []
|
||
for c in gerbil_by_norm_name.get(normalize_name(name), []):
|
||
final_id = gerbil_id_map.get(c["Id"])
|
||
if not final_id:
|
||
continue
|
||
final_c = gerbil_by_id_final.get(final_id)
|
||
if not final_c:
|
||
continue
|
||
# Parent must be age-plausible: born before the litter and within the
|
||
# gerbil lifespan (skips e.g. a 2013 animal for a 2022 litter).
|
||
if not parent_age_plausible(final_c["DateOfBirth"], litter_date):
|
||
continue
|
||
cands.append(final_c)
|
||
if not cands:
|
||
return None
|
||
# Prefer the gender expected for this role, then unknown, then anything.
|
||
for pool in (
|
||
[c for c in cands if c["Gender"] == prefer_gender],
|
||
[c for c in cands if c["Gender"] == "unknown"],
|
||
cands,
|
||
):
|
||
if pool:
|
||
return pool[0]
|
||
return None
|
||
|
||
for l in resolved_litters:
|
||
# Pre-check: if _father_name points to a known female and _mother_name to a
|
||
# known male → swap names (Stammbaum positions reversed). Helps the name
|
||
# resolver pick the right same-name candidate before role normalization.
|
||
f_name_pre = l.get("_father_name", "")
|
||
m_name_pre = l.get("_mother_name", "")
|
||
if f_name_pre and m_name_pre:
|
||
f_gender = next((g["Gender"] for g in gerbil_by_norm_name.get(normalize_name(f_name_pre), []) if g["Gender"] != "unknown"), None)
|
||
m_gender = next((g["Gender"] for g in gerbil_by_norm_name.get(normalize_name(m_name_pre), []) if g["Gender"] != "unknown"), None)
|
||
if f_gender == "female" and m_gender == "male":
|
||
l["_father_name"], l["_mother_name"] = m_name_pre, f_name_pre
|
||
|
||
if l["_father_name"] and not l["FatherId"]:
|
||
cand = _resolve_name(l["_father_name"], "male", l["Date"])
|
||
if cand:
|
||
l["FatherId"] = cand["Id"]
|
||
resolved_fathers += 1
|
||
|
||
if l["_mother_name"] and not l["MotherId"]:
|
||
cand = _resolve_name(l["_mother_name"], "female", l["Date"])
|
||
if cand:
|
||
l["MotherId"] = cand["Id"]
|
||
resolved_mothers += 1
|
||
|
||
# Cleanup internal keys
|
||
del l["_father_name"]
|
||
del l["_mother_name"]
|
||
del l["_filename"]
|
||
|
||
# Children-of-litter lookup so a dropped parent link can be explained in the
|
||
# offspring's Datenherkunft (the discard is most meaningful on the child).
|
||
children_by_litter = {}
|
||
for g in resolved_gerbils:
|
||
lid = g.get("LitterId")
|
||
if lid:
|
||
children_by_litter.setdefault(lid, []).append(g)
|
||
|
||
def _add_child_discard(litter, discard):
|
||
"""Attach a parent-link discard to every child gerbil of the litter."""
|
||
for child in children_by_litter.get(litter["Id"], []):
|
||
dl = child.setdefault("_discarded", [])
|
||
if discard not in dl:
|
||
dl.append(discard)
|
||
|
||
def _pname(pid):
|
||
p = gerbil_by_id_final.get(pid)
|
||
return p.get("Name") if p else None
|
||
|
||
# Role normalization: assign each resolved parent to the role matching its
|
||
# gender, eliminate self-pairings (same animal in both roles), and never let
|
||
# impossible duplicates survive (two males / two females). This corrects
|
||
# reversed Stammbaum positions including the cases the simple swap missed
|
||
# (one parent of "unknown" gender, or a self-paired litter).
|
||
role_fixes = 0
|
||
for l in resolved_litters:
|
||
before = (l.get("FatherId"), l.get("MotherId"))
|
||
father, mother = assign_parent_roles(l.get("FatherId"), l.get("MotherId"), _final_gender)
|
||
if before != (father, mother):
|
||
role_fixes += 1
|
||
after = {father, mother}
|
||
# A parent id present before but gone after was dropped by role
|
||
# normalization (self-pairing or two-of-the-same-sex). Explain it.
|
||
for pid in before:
|
||
if pid and pid not in after:
|
||
pname = _pname(pid)
|
||
if before[0] == before[1]:
|
||
reason = "Selbstverpaarung — Tier kann nicht beide Elternteile sein"
|
||
else:
|
||
reason = "ein Wurf hat nur einen Vater und eine Mutter"
|
||
_add_child_discard(l, {
|
||
"text": f"Elternteil „{pname}“ verworfen — {reason}",
|
||
})
|
||
l["FatherId"] = father
|
||
l["MotherId"] = mother
|
||
if role_fixes:
|
||
print(f"Role-normalization: corrected {role_fixes} litter(s) (gender roles / self-pairings).")
|
||
|
||
# Parent-age sanity check: drop any resolved parent that cannot belong to the
|
||
# litter — born after the offspring, or more than a gerbil lifespan earlier.
|
||
# Catches mis-resolved links the name matcher still let through (e.g. Jayjay,
|
||
# *2013, wrongly attached to Solice's 2022 litter).
|
||
age_drops = []
|
||
for l in resolved_litters:
|
||
ldate = l.get("Date")
|
||
for role in ("FatherId", "MotherId"):
|
||
pid = l.get(role)
|
||
if not pid:
|
||
continue
|
||
p = gerbil_by_id_final.get(pid)
|
||
if p and not parent_age_plausible(p.get("DateOfBirth"), ldate):
|
||
age_drops.append((l.get("Name"), role, p.get("Name"), p.get("DateOfBirth"), ldate))
|
||
# Explain the drop in each child's history: which parent, its DOB,
|
||
# why (implausible age), and that no replacement was used.
|
||
role_de = "Vater" if role == "FatherId" else "Mutter"
|
||
pdob_disp = _de_date(p.get("DateOfBirth")) or "unbekannt"
|
||
age_reason = "unplausibles Alter für diesen Wurf"
|
||
pd = date_to_days(parse_date(p.get("DateOfBirth"))) if p.get("DateOfBirth") else None
|
||
ld = date_to_days(parse_date(ldate)) if ldate else None
|
||
if pd is not None and ld is not None:
|
||
years = abs(ld - pd) / 365.25
|
||
if ld - pd <= 0:
|
||
age_reason = "unplausibel (nicht vor dem Kind geboren)"
|
||
else:
|
||
age_reason = f"unplausibel ({years:.0f} Jahre älter als das Kind)"
|
||
_add_child_discard(l, {
|
||
"text": (f"{role_de} „{p.get('Name')}“ (*{pdob_disp}) verworfen "
|
||
f"— {age_reason}; kein Ersatz"),
|
||
})
|
||
l[role] = None
|
||
if age_drops:
|
||
print(f"Parent-age sanity check: dropped {len(age_drops)} implausible parent link(s):")
|
||
for lname, role, pname, pdob, ldate in age_drops[:20]:
|
||
print(f" {lname}: {role}={pname} (*{pdob}) vs litter {ldate}")
|
||
|
||
print(f"Globally resolved {resolved_fathers} fathers and {resolved_mothers} mothers.")
|
||
|
||
|
||
# 5b. Second-pass litter dedup: now that FatherId/MotherId are known,
|
||
# merge litters that have the same date AND the same parents.
|
||
# This is the core of the "sibling pairing" fix: Blue Wave and Sunny Sky
|
||
# both come from Wonderman × Unique — their two separate litter records
|
||
# must now become one, so their children share the same LitterId.
|
||
litter_by_id_post = {l["Id"]: l for l in resolved_litters}
|
||
gerbil_by_litter = {}
|
||
for g in resolved_gerbils:
|
||
lid = g.get("LitterId")
|
||
if lid:
|
||
gerbil_by_litter.setdefault(lid, []).append(g)
|
||
|
||
def _litter_same_parents(l1, l2):
|
||
"""Strict: same date + both parents known and matching."""
|
||
if l1["Date"] != l2["Date"]:
|
||
return False
|
||
f1, m1 = l1.get("FatherId"), l1.get("MotherId")
|
||
f2, m2 = l2.get("FatherId"), l2.get("MotherId")
|
||
if not f1 or not f2 or not m1 or not m2:
|
||
return False
|
||
return f1 == f2 and m1 == m2
|
||
|
||
by_date2 = {}
|
||
for l in resolved_litters:
|
||
by_date2.setdefault(l["Date"], []).append(l)
|
||
|
||
deduped2 = []
|
||
litter_remap2 = {} # old_id -> canonical_id
|
||
|
||
for date_val, group in by_date2.items():
|
||
sub_groups = []
|
||
for l in group:
|
||
placed = False
|
||
for sub in sub_groups:
|
||
if all(_litter_same_parents(l, m) for m in sub):
|
||
sub.append(l)
|
||
placed = True
|
||
break
|
||
if not placed:
|
||
sub_groups.append([l])
|
||
|
||
for sub in sub_groups:
|
||
# Prefer the canonical that has the most children
|
||
canonical = max(sub, key=lambda l: len(gerbil_by_litter.get(l["Id"], [])))
|
||
for l in sub:
|
||
litter_remap2[l["Id"]] = canonical["Id"]
|
||
if l is not canonical:
|
||
canonical["_source_files"] |= l.get("_source_files", set())
|
||
canonical["_merged_count"] += l.get("_merged_count", 1)
|
||
if not l.get("_virtual"):
|
||
canonical["_virtual"] = False
|
||
deduped2.append(canonical)
|
||
if len(sub) > 1:
|
||
siblings = [g["Name"] for l in sub for g in gerbil_by_litter.get(l["Id"], []) if l is not canonical]
|
||
print(f"Sibling-Litter-Merge on {date_val}: {canonical['Name']} absorbed sibling half — children now share LitterId: {[g['Name'] for g in gerbil_by_litter.get(canonical['Id'], [])] + siblings}")
|
||
|
||
# Remap LitterId in all gerbils
|
||
n_remapped = 0
|
||
for g in resolved_gerbils:
|
||
old_lid = g.get("LitterId")
|
||
if old_lid and old_lid in litter_remap2 and litter_remap2[old_lid] != old_lid:
|
||
g["LitterId"] = litter_remap2[old_lid]
|
||
n_remapped += 1
|
||
|
||
n_merged2 = len(resolved_litters) - len(deduped2)
|
||
if n_merged2:
|
||
print(f"Sibling-Litter-Dedup: {n_merged2} additional litter record(s) merged ({n_remapped} gerbil LitterIds remapped).")
|
||
resolved_litters = deduped2
|
||
litter_by_scoped_id = {l["Id"]: l for l in resolved_litters}
|
||
|
||
# Final gerbil-provenance pass: now that parent links are fully resolved and
|
||
# all discards (majority-vote conflicts during dedup; rejected parent-refs;
|
||
# role-normalization drops; parent-age drops) are attached to each gerbil's
|
||
# _discarded list, render the provenance JSON with the discard history lines.
|
||
n_discards = 0
|
||
for g in resolved_gerbils:
|
||
args = g.pop("_prov_args", None)
|
||
if g.get("_discarded"):
|
||
n_discards += len(g["_discarded"])
|
||
if args is not None:
|
||
records, best_g, conflict_notes, field_source, field_discards = args
|
||
g["Provenance"] = build_provenance(
|
||
records, best_g, extra_notes=conflict_notes,
|
||
field_source=field_source, field_discards=field_discards,
|
||
)
|
||
else:
|
||
g["Provenance"] = build_provenance([g], g)
|
||
g.pop("_discarded", None)
|
||
if n_discards:
|
||
print(f"Datenherkunft: recorded {n_discards} discard line(s) across gerbils.")
|
||
|
||
# Abgabevertrag-Anreicherung: Käufer als Abnehmer-Kontakte anlegen und —
|
||
# konservativ — auf eindeutig passende Tiere ReceiverContactId/GoHomeDate/
|
||
# Status=GivenAway setzen (nur falls noch nicht gesetzt). Provenance-
|
||
# Historie wird ergänzt. Neue Käuferkontakte landen in contact_by_norm_name
|
||
# und werden danach automatisch als IsReceiver markiert.
|
||
cstats = enrich_from_contracts(
|
||
contracts, resolved_gerbils, contact_by_norm_name, contact_id_map)
|
||
if contracts:
|
||
print("Abgabeverträge: "
|
||
f"{cstats['contracts']} geladen, {cstats['matched']} Tier-Treffer, "
|
||
f"{cstats['ambiguous_skipped']} mehrdeutig übersprungen, "
|
||
f"{cstats['no_match_skipped']} ohne Treffer.")
|
||
print(" Kontakte: "
|
||
f"{cstats['buyers_created']} neu, {cstats['buyers_existing']} bestehend.")
|
||
print(" Gesetzt: "
|
||
f"ReceiverContactId={cstats['receiver_set']}, "
|
||
f"GoHomeDate={cstats['gohome_set']}, "
|
||
f"Status=GivenAway={cstats['status_givenaway']}, "
|
||
f"Konflikte={cstats['conflicts']}.")
|
||
# Re-materialise contacts so freshly created buyer contacts are exported.
|
||
resolved_contacts = list(contact_by_norm_name.values())
|
||
|
||
# Datenherkunft for litters: which source files contributed, whether this is
|
||
# a Wurfchronik litter vs a Stammbaum-reconstructed ("virtual") litter, how
|
||
# many raw records merged into it, plus human-readable notes. Accumulators
|
||
# (_source_files/_merged_count/_virtual) were filled during the two dedup
|
||
# passes above; strip them after use.
|
||
for l in resolved_litters:
|
||
l_source_files = l.pop("_source_files", set())
|
||
l_merged_count = l.pop("_merged_count", 1)
|
||
is_virtual = l.pop("_virtual", False)
|
||
l_from_wurfchronik = any("wurfchronik" in str(f).lower() for f in l_source_files)
|
||
l_notes = []
|
||
if is_virtual and not l_from_wurfchronik:
|
||
l_notes.append("aus Stammbaum-Diagramm rekonstruiert")
|
||
elif l_from_wurfchronik:
|
||
l_notes.append("aus Wurfchronik")
|
||
if l_merged_count > 1:
|
||
l_notes.append(f"aus {l_merged_count} Datensätzen zusammengeführt")
|
||
l_notes.append("Geschwister-Würfe zusammengeführt")
|
||
|
||
# Chronological, file-attributed history for the litter.
|
||
l_files_sorted = sorted({f for f in l_source_files if f})
|
||
l_history = []
|
||
first_file = l_files_sorted[0] if l_files_sorted else None
|
||
if is_virtual and not l_from_wurfchronik:
|
||
if first_file:
|
||
l_history.append(
|
||
f"Aus Stammbaum-Diagramm rekonstruiert ({_quote_file(first_file)})."
|
||
)
|
||
else:
|
||
l_history.append("Aus Stammbaum-Diagramm rekonstruiert.")
|
||
elif first_file:
|
||
l_history.append(f"Wurf aus Wurfchronik {_quote_file(first_file)}.")
|
||
else:
|
||
l_history.append("Wurf im Import gefunden.")
|
||
for f in l_files_sorted[1:]:
|
||
l_history.append(
|
||
f"Auch in {_quote_file(f)} gefunden → Datensätze zusammengeführt."
|
||
)
|
||
if l_merged_count > 1:
|
||
l_history.append("Geschwister-Würfe zusammengeführt.")
|
||
l["Provenance"] = build_entity_provenance(
|
||
l_source_files, l_merged_count, notes=l_notes,
|
||
from_wurfchronik=l_from_wurfchronik, history=l_history,
|
||
)
|
||
|
||
# Set IsBreeder and IsReceiver flags on contacts
|
||
breeder_ids = {g["OriginContactId"] for g in resolved_gerbils if g.get("OriginContactId")}
|
||
receiver_ids = {g["ReceiverContactId"] for g in resolved_gerbils if g.get("ReceiverContactId")}
|
||
for c in resolved_contacts:
|
||
c_id = c["Id"]
|
||
is_breeder = c_id in breeder_ids
|
||
is_receiver = c_id in receiver_ids
|
||
if not is_breeder and not is_receiver:
|
||
is_receiver = True
|
||
c["IsBreeder"] = is_breeder
|
||
c["IsReceiver"] = is_receiver
|
||
|
||
# Datenherkunft: where this (deduplicated) contact came from, plus the
|
||
# role we inferred. Accumulator keys (_source_files/_merged_count) were
|
||
# filled during the contact dedup above; strip them after use.
|
||
c_source_files = c.pop("_source_files", set())
|
||
c_merged_count = c.pop("_merged_count", 1)
|
||
c_notes = []
|
||
if c_merged_count > 1:
|
||
c_notes.append(f"aus {c_merged_count} Datensätzen zusammengeführt")
|
||
if is_breeder:
|
||
c_notes.append("als Züchter erkannt")
|
||
if is_receiver:
|
||
c_notes.append("als Abnehmer erkannt")
|
||
|
||
# Chronological, file-attributed history for the contact.
|
||
c_files_sorted = sorted({f for f in c_source_files if f})
|
||
c_history = []
|
||
role_word = "Züchter" if is_breeder else "Abnehmer"
|
||
if c_files_sorted:
|
||
c_history.append(f"In {_quote_file(c_files_sorted[0])} als {role_word} erkannt.")
|
||
for f in c_files_sorted[1:]:
|
||
c_history.append(
|
||
f"Auch in {_quote_file(f)} gefunden → Datensätze zusammengeführt."
|
||
)
|
||
else:
|
||
c_history.append(f"Im Import als {role_word} erkannt.")
|
||
if is_breeder and is_receiver:
|
||
c_history.append("Sowohl als Züchter als auch als Abnehmer geführt.")
|
||
c["Provenance"] = build_entity_provenance(
|
||
c_source_files, c_merged_count, notes=c_notes, history=c_history
|
||
)
|
||
|
||
# Set and map gerbilPhotos
|
||
resolved_photos = []
|
||
for g in resolved_gerbils:
|
||
for idx, photo_rel in enumerate(g.get("_photos", [])):
|
||
photo_guid = generate_guid(f"photo-{photo_rel}")
|
||
ext = os.path.splitext(photo_rel)[1] or ".jpeg"
|
||
fn_guid = photo_guid.replace("-", "")
|
||
resolved_photos.append({
|
||
"Id": photo_guid,
|
||
"GerbilId": g["Id"],
|
||
"FileName": f"{fn_guid}{ext}",
|
||
"SortOrder": idx,
|
||
"_source_path": photo_rel
|
||
})
|
||
|
||
# 6. Save final output JSON payload
|
||
os.makedirs(OUTPUT_DIR, exist_ok=True)
|
||
payload = {
|
||
"contacts": resolved_contacts,
|
||
"litters": resolved_litters,
|
||
"gerbils": resolved_gerbils,
|
||
"gerbilPhotos": resolved_photos
|
||
}
|
||
|
||
with open(OUTPUT_FILE, "w", encoding="utf-8") as f:
|
||
json.dump(payload, f, ensure_ascii=False, indent=2)
|
||
|
||
print(f"Successfully wrote database-ready import file to: {OUTPUT_FILE}")
|
||
print(f" Contacts: {len(payload['contacts'])}")
|
||
print(f" Litters: {len(payload['litters'])}")
|
||
print(f" Gerbils: {len(payload['gerbils'])}")
|
||
print(f" Photos: {len(payload['gerbilPhotos'])}")
|
||
|
||
if __name__ == "__main__":
|
||
main()
|